From 3a52b65795f7a81f6f0ae48d96e235388fc86b87 Mon Sep 17 00:00:00 2001 From: Lorenzo Ampil Date: Mon, 21 Oct 2019 12:55:51 +0800 Subject: [PATCH 001/505] Add special tokens to documentation for bert examples to resolve issue: #1561 --- transformers/modeling_bert.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py index 8c92241fa2..9a9cd31b4b 100644 --- a/transformers/modeling_bert.py +++ b/transformers/modeling_bert.py @@ -557,7 +557,7 @@ class BertModel(BertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = BertModel.from_pretrained('bert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -667,7 +667,7 @@ class BertForPreTraining(BertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = BertForPreTraining.from_pretrained('bert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids) prediction_scores, seq_relationship_scores = outputs[:2] @@ -739,7 +739,7 @@ class BertForMaskedLM(BertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = BertForMaskedLM.from_pretrained('bert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids, masked_lm_labels=input_ids) loss, prediction_scores = outputs[:2] @@ -808,7 +808,7 @@ class BertForNextSentencePrediction(BertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = BertForNextSentencePrediction.from_pretrained('bert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids) seq_relationship_scores = outputs[0] @@ -871,7 +871,7 @@ class BertForSequenceClassification(BertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = BertForSequenceClassification.from_pretrained('bert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 outputs = model(input_ids, labels=labels) loss, logits = outputs[:2] @@ -945,7 +945,7 @@ class BertForMultipleChoice(BertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = BertForMultipleChoice.from_pretrained('bert-base-uncased') choices = ["Hello, my dog is cute", "Hello, my cat is amazing"] - input_ids = torch.tensor([tokenizer.encode(s) for s in choices]).unsqueeze(0) # Batch size 1, 2 choices + input_ids = torch.tensor([tokenizer.encode(s, add_special_tokens=True) for s in choices]).unsqueeze(0) # Batch size 1, 2 choices labels = torch.tensor(1).unsqueeze(0) # Batch size 1 outputs = model(input_ids, labels=labels) loss, classification_scores = outputs[:2] @@ -1017,7 +1017,7 @@ class BertForTokenClassification(BertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = BertForTokenClassification.from_pretrained('bert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 labels = torch.tensor([1] * input_ids.size(1)).unsqueeze(0) # Batch size 1 outputs = model(input_ids, labels=labels) loss, scores = outputs[:2] From 6e011690a980c3c3f69fdfc3af8705859250cc6b Mon Sep 17 00:00:00 2001 From: Lorenzo Ampil Date: Sun, 27 Oct 2019 13:59:14 +0800 Subject: [PATCH 002/505] Add special tokens to documentation for the rest of pytorch model examples #1561 --- transformers/modeling_ctrl.py | 4 ++-- transformers/modeling_distilbert.py | 8 ++++---- transformers/modeling_gpt2.py | 4 ++-- transformers/modeling_openai.py | 4 ++-- transformers/modeling_roberta.py | 6 +++--- transformers/modeling_transfo_xl.py | 4 ++-- transformers/modeling_xlm.py | 10 +++++----- transformers/modeling_xlnet.py | 10 +++++----- 8 files changed, 25 insertions(+), 25 deletions(-) diff --git a/transformers/modeling_ctrl.py b/transformers/modeling_ctrl.py index 55e64d318b..b31755e13d 100644 --- a/transformers/modeling_ctrl.py +++ b/transformers/modeling_ctrl.py @@ -261,7 +261,7 @@ class CTRLModel(CTRLPreTrainedModel): tokenizer = CTRLTokenizer.from_pretrained('ctrl') model = CTRLModel.from_pretrained('ctrl') - input_ids = torch.tensor(tokenizer.encode("Links Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Links Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -438,7 +438,7 @@ class CTRLLMHeadModel(CTRLPreTrainedModel): tokenizer = CTRLTokenizer.from_pretrained('ctrl') model = CTRLLMHeadModel.from_pretrained('ctrl') - input_ids = torch.tensor(tokenizer.encode("Links Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Links Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids, labels=input_ids) loss, logits = outputs[:2] diff --git a/transformers/modeling_distilbert.py b/transformers/modeling_distilbert.py index d3b4ccff5d..990d76e378 100644 --- a/transformers/modeling_distilbert.py +++ b/transformers/modeling_distilbert.py @@ -411,7 +411,7 @@ class DistilBertModel(DistilBertPreTrainedModel): tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') model = DistilBertModel.from_pretrained('distilbert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -495,7 +495,7 @@ class DistilBertForMaskedLM(DistilBertPreTrainedModel): tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') model = DistilBertForMaskedLM.from_pretrained('distilbert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids, masked_lm_labels=input_ids) loss, prediction_scores = outputs[:2] @@ -569,7 +569,7 @@ class DistilBertForSequenceClassification(DistilBertPreTrainedModel): tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') model = DistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 outputs = model(input_ids, labels=labels) loss, logits = outputs[:2] @@ -643,7 +643,7 @@ class DistilBertForQuestionAnswering(DistilBertPreTrainedModel): tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') model = DistilBertForQuestionAnswering.from_pretrained('distilbert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 start_positions = torch.tensor([1]) end_positions = torch.tensor([3]) outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions) diff --git a/transformers/modeling_gpt2.py b/transformers/modeling_gpt2.py index 0b5b83aa75..87878abb4e 100644 --- a/transformers/modeling_gpt2.py +++ b/transformers/modeling_gpt2.py @@ -338,7 +338,7 @@ class GPT2Model(GPT2PreTrainedModel): tokenizer = GPT2Tokenizer.from_pretrained('gpt2') model = GPT2Model.from_pretrained('gpt2') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -503,7 +503,7 @@ class GPT2LMHeadModel(GPT2PreTrainedModel): tokenizer = GPT2Tokenizer.from_pretrained('gpt2') model = GPT2LMHeadModel.from_pretrained('gpt2') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids, labels=input_ids) loss, logits = outputs[:2] diff --git a/transformers/modeling_openai.py b/transformers/modeling_openai.py index 52f3b7db72..c6b13dee4e 100644 --- a/transformers/modeling_openai.py +++ b/transformers/modeling_openai.py @@ -343,7 +343,7 @@ class OpenAIGPTModel(OpenAIGPTPreTrainedModel): tokenizer = OpenAIGPTTokenizer.from_pretrained('openai-gpt') model = OpenAIGPTModel.from_pretrained('openai-gpt') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -478,7 +478,7 @@ class OpenAIGPTLMHeadModel(OpenAIGPTPreTrainedModel): tokenizer = OpenAIGPTTokenizer.from_pretrained('openai-gpt') model = OpenAIGPTLMHeadModel.from_pretrained('openai-gpt') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids, labels=input_ids) loss, logits = outputs[:2] diff --git a/transformers/modeling_roberta.py b/transformers/modeling_roberta.py index eb340dc7fb..cbd2e0106d 100644 --- a/transformers/modeling_roberta.py +++ b/transformers/modeling_roberta.py @@ -154,7 +154,7 @@ class RobertaModel(BertModel): tokenizer = RobertaTokenizer.from_pretrained('roberta-base') model = RobertaModel.from_pretrained('roberta-base') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -209,7 +209,7 @@ class RobertaForMaskedLM(BertPreTrainedModel): tokenizer = RobertaTokenizer.from_pretrained('roberta-base') model = RobertaForMaskedLM.from_pretrained('roberta-base') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids, masked_lm_labels=input_ids) loss, prediction_scores = outputs[:2] @@ -303,7 +303,7 @@ class RobertaForSequenceClassification(BertPreTrainedModel): tokenizer = RobertaTokenizer.from_pretrained('roberta-base') model = RobertaForSequenceClassification.from_pretrained('roberta-base') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 outputs = model(input_ids, labels=labels) loss, logits = outputs[:2] diff --git a/transformers/modeling_transfo_xl.py b/transformers/modeling_transfo_xl.py index 6d430e1804..ad1c7bdea4 100644 --- a/transformers/modeling_transfo_xl.py +++ b/transformers/modeling_transfo_xl.py @@ -578,7 +578,7 @@ class TransfoXLModel(TransfoXLPreTrainedModel): tokenizer = TransfoXLTokenizer.from_pretrained('transfo-xl-wt103') model = TransfoXLModel.from_pretrained('transfo-xl-wt103') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids) last_hidden_states, mems = outputs[:2] @@ -808,7 +808,7 @@ class TransfoXLLMHeadModel(TransfoXLPreTrainedModel): tokenizer = TransfoXLTokenizer.from_pretrained('transfo-xl-wt103') model = TransfoXLLMHeadModel.from_pretrained('transfo-xl-wt103') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids) prediction_scores, mems = outputs[:2] diff --git a/transformers/modeling_xlm.py b/transformers/modeling_xlm.py index b29e721556..a7c8f4e941 100644 --- a/transformers/modeling_xlm.py +++ b/transformers/modeling_xlm.py @@ -332,7 +332,7 @@ class XLMModel(XLMPreTrainedModel): tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') model = XLMModel.from_pretrained('xlm-mlm-en-2048') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -607,7 +607,7 @@ class XLMWithLMHeadModel(XLMPreTrainedModel): tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') model = XLMWithLMHeadModel.from_pretrained('xlm-mlm-en-2048') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -671,7 +671,7 @@ class XLMForSequenceClassification(XLMPreTrainedModel): tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') model = XLMForSequenceClassification.from_pretrained('xlm-mlm-en-2048') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 outputs = model(input_ids, labels=labels) loss, logits = outputs[:2] @@ -754,7 +754,7 @@ class XLMForQuestionAnsweringSimple(XLMPreTrainedModel): tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') model = XLMForQuestionAnsweringSimple.from_pretrained('xlm-mlm-en-2048') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 start_positions = torch.tensor([1]) end_positions = torch.tensor([3]) outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions) @@ -849,7 +849,7 @@ class XLMForQuestionAnswering(XLMPreTrainedModel): tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') model = XLMForQuestionAnswering.from_pretrained('xlm-mlm-en-2048') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 start_positions = torch.tensor([1]) end_positions = torch.tensor([3]) outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions) diff --git a/transformers/modeling_xlnet.py b/transformers/modeling_xlnet.py index e191ebadd0..fab405fd2b 100644 --- a/transformers/modeling_xlnet.py +++ b/transformers/modeling_xlnet.py @@ -584,7 +584,7 @@ class XLNetModel(XLNetPreTrainedModel): tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased') model = XLNetModel.from_pretrained('xlnet-large-cased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -900,7 +900,7 @@ class XLNetLMHeadModel(XLNetPreTrainedModel): tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased') model = XLNetLMHeadModel.from_pretrained('xlnet-large-cased') # We show how to setup inputs to predict a next token using a bi-directional context. - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is very ")).unsqueeze(0) # We will predict the masked token + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is very ", add_special_tokens=True)).unsqueeze(0) # We will predict the masked token perm_mask = torch.zeros((1, input_ids.shape[1], input_ids.shape[1]), dtype=torch.float) perm_mask[:, :, -1] = 1.0 # Previous tokens don't see last token target_mapping = torch.zeros((1, 1, input_ids.shape[1]), dtype=torch.float) # Shape [1, 1, seq_length] => let's predict one token @@ -983,7 +983,7 @@ class XLNetForSequenceClassification(XLNetPreTrainedModel): tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased') model = XLNetForSequenceClassification.from_pretrained('xlnet-large-cased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 outputs = model(input_ids, labels=labels) loss, logits = outputs[:2] @@ -1163,7 +1163,7 @@ class XLNetForQuestionAnsweringSimple(XLNetPreTrainedModel): tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') model = XLMForQuestionAnswering.from_pretrained('xlnet-large-cased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 start_positions = torch.tensor([1]) end_positions = torch.tensor([3]) outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions) @@ -1276,7 +1276,7 @@ class XLNetForQuestionAnswering(XLNetPreTrainedModel): tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased') model = XLMForQuestionAnswering.from_pretrained('xlnet-large-cased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 start_positions = torch.tensor([1]) end_positions = torch.tensor([3]) outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions) From ec276d6abad7eae800f1a1a039ddc78fde406009 Mon Sep 17 00:00:00 2001 From: Lorenzo Ampil Date: Sun, 27 Oct 2019 14:00:40 +0800 Subject: [PATCH 003/505] Add special tokens to documentation for the tensorflow model examples #1561 --- transformers/modeling_tf_bert.py | 14 +++++++------- transformers/modeling_tf_ctrl.py | 4 ++-- transformers/modeling_tf_distilbert.py | 8 ++++---- transformers/modeling_tf_gpt2.py | 4 ++-- transformers/modeling_tf_openai.py | 4 ++-- transformers/modeling_tf_roberta.py | 6 +++--- transformers/modeling_tf_transfo_xl.py | 4 ++-- transformers/modeling_tf_xlm.py | 8 ++++---- transformers/modeling_tf_xlnet.py | 10 +++++----- 9 files changed, 31 insertions(+), 31 deletions(-) diff --git a/transformers/modeling_tf_bert.py b/transformers/modeling_tf_bert.py index afe9b2946b..d2d3c7be37 100644 --- a/transformers/modeling_tf_bert.py +++ b/transformers/modeling_tf_bert.py @@ -647,7 +647,7 @@ class TFBertModel(TFBertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = TFBertModel.from_pretrained('bert-base-uncased') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -686,7 +686,7 @@ class TFBertForPreTraining(TFBertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = TFBertForPreTraining.from_pretrained('bert-base-uncased') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) prediction_scores, seq_relationship_scores = outputs[:2] @@ -732,7 +732,7 @@ class TFBertForMaskedLM(TFBertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = TFBertForMaskedLM.from_pretrained('bert-base-uncased') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) prediction_scores = outputs[0] @@ -776,7 +776,7 @@ class TFBertForNextSentencePrediction(TFBertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = TFBertForNextSentencePrediction.from_pretrained('bert-base-uncased') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) seq_relationship_scores = outputs[0] @@ -821,7 +821,7 @@ class TFBertForSequenceClassification(TFBertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = TFBertForSequenceClassification.from_pretrained('bert-base-uncased') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) logits = outputs[0] @@ -952,7 +952,7 @@ class TFBertForTokenClassification(TFBertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = TFBertForTokenClassification.from_pretrained('bert-base-uncased') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) scores = outputs[0] @@ -1005,7 +1005,7 @@ class TFBertForQuestionAnswering(TFBertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = TFBertForQuestionAnswering.from_pretrained('bert-base-uncased') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) start_scores, end_scores = outputs[:2] diff --git a/transformers/modeling_tf_ctrl.py b/transformers/modeling_tf_ctrl.py index c8d181548b..66766a066e 100644 --- a/transformers/modeling_tf_ctrl.py +++ b/transformers/modeling_tf_ctrl.py @@ -402,7 +402,7 @@ class TFCTRLModel(TFCTRLPreTrainedModel): tokenizer = CTRLTokenizer.from_pretrained('ctrl') model = TFCTRLModel.from_pretrained('ctrl') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -465,7 +465,7 @@ class TFCTRLLMHeadModel(TFCTRLPreTrainedModel): tokenizer = CTRLTokenizer.from_pretrained('ctrl') model = TFCTRLLMHeadModel.from_pretrained('ctrl') - input_ids = torch.tensor(tokenizer.encode("Links Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Links Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids, labels=input_ids) loss, logits = outputs[:2] diff --git a/transformers/modeling_tf_distilbert.py b/transformers/modeling_tf_distilbert.py index 188394816e..c2d0f73999 100644 --- a/transformers/modeling_tf_distilbert.py +++ b/transformers/modeling_tf_distilbert.py @@ -532,7 +532,7 @@ class TFDistilBertModel(TFDistilBertPreTrainedModel): tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') model = TFDistilBertModel.from_pretrained('distilbert-base-uncased') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -590,7 +590,7 @@ class TFDistilBertForMaskedLM(TFDistilBertPreTrainedModel): tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') model = TFDistilBertForMaskedLM.from_pretrained('distilbert-base-uncased') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) prediction_scores = outputs[0] @@ -645,7 +645,7 @@ class TFDistilBertForSequenceClassification(TFDistilBertPreTrainedModel): tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') model = TFDistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) logits = outputs[0] @@ -702,7 +702,7 @@ class TFDistilBertForQuestionAnswering(TFDistilBertPreTrainedModel): tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') model = TFDistilBertForQuestionAnswering.from_pretrained('distilbert-base-uncased') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) start_scores, end_scores = outputs[:2] diff --git a/transformers/modeling_tf_gpt2.py b/transformers/modeling_tf_gpt2.py index 4188b273ba..24f2857b80 100644 --- a/transformers/modeling_tf_gpt2.py +++ b/transformers/modeling_tf_gpt2.py @@ -436,7 +436,7 @@ class TFGPT2Model(TFGPT2PreTrainedModel): tokenizer = GPT2Tokenizer.from_pretrained('gpt2') model = TFGPT2Model.from_pretrained('gpt2') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -477,7 +477,7 @@ class TFGPT2LMHeadModel(TFGPT2PreTrainedModel): tokenizer = GPT2Tokenizer.from_pretrained('gpt2') model = TFGPT2LMHeadModel.from_pretrained('gpt2') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) logits = outputs[0] diff --git a/transformers/modeling_tf_openai.py b/transformers/modeling_tf_openai.py index 747c5171fd..08034b2d2e 100644 --- a/transformers/modeling_tf_openai.py +++ b/transformers/modeling_tf_openai.py @@ -413,7 +413,7 @@ class TFOpenAIGPTModel(TFOpenAIGPTPreTrainedModel): tokenizer = OpenAIGPTTokenizer.from_pretrained('openai-gpt') model = TFOpenAIGPTModel.from_pretrained('openai-gpt') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -449,7 +449,7 @@ class TFOpenAIGPTLMHeadModel(TFOpenAIGPTPreTrainedModel): tokenizer = OpenAIGPTTokenizer.from_pretrained('openai-gpt') model = TFOpenAIGPTLMHeadModel.from_pretrained('openai-gpt') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) logits = outputs[0] diff --git a/transformers/modeling_tf_roberta.py b/transformers/modeling_tf_roberta.py index 244c83f2b3..dcf00c3add 100644 --- a/transformers/modeling_tf_roberta.py +++ b/transformers/modeling_tf_roberta.py @@ -204,7 +204,7 @@ class TFRobertaModel(TFRobertaPreTrainedModel): tokenizer = RobertaTokenizer.from_pretrained('roberta-base') model = TFRobertaModel.from_pretrained('roberta-base') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -281,7 +281,7 @@ class TFRobertaForMaskedLM(TFRobertaPreTrainedModel): tokenizer = RobertaTokenizer.from_pretrained('roberta-base') model = TFRobertaForMaskedLM.from_pretrained('roberta-base') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids, masked_lm_labels=input_ids) prediction_scores = outputs[0] @@ -349,7 +349,7 @@ class TFRobertaForSequenceClassification(TFRobertaPreTrainedModel): tokenizer = RoertaTokenizer.from_pretrained('roberta-base') model = TFRobertaForSequenceClassification.from_pretrained('roberta-base') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 labels = tf.constant([1])[None, :] # Batch size 1 outputs = model(input_ids) logits = outputs[0] diff --git a/transformers/modeling_tf_transfo_xl.py b/transformers/modeling_tf_transfo_xl.py index a3e403ce06..87863163f0 100644 --- a/transformers/modeling_tf_transfo_xl.py +++ b/transformers/modeling_tf_transfo_xl.py @@ -654,7 +654,7 @@ class TFTransfoXLModel(TFTransfoXLPreTrainedModel): tokenizer = TransfoXLTokenizer.from_pretrained('transfo-xl-wt103') model = TFTransfoXLModel.from_pretrained('transfo-xl-wt103') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) last_hidden_states, mems = outputs[:2] @@ -696,7 +696,7 @@ class TFTransfoXLLMHeadModel(TFTransfoXLPreTrainedModel): tokenizer = TransfoXLTokenizer.from_pretrained('transfo-xl-wt103') model = TFTransfoXLLMHeadModel.from_pretrained('transfo-xl-wt103') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) prediction_scores, mems = outputs[:2] diff --git a/transformers/modeling_tf_xlm.py b/transformers/modeling_tf_xlm.py index 84de1517ee..7b7305f22e 100644 --- a/transformers/modeling_tf_xlm.py +++ b/transformers/modeling_tf_xlm.py @@ -550,7 +550,7 @@ class TFXLMModel(TFXLMPreTrainedModel): tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') model = TFXLMModel.from_pretrained('xlm-mlm-en-2048') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -623,7 +623,7 @@ class TFXLMWithLMHeadModel(TFXLMPreTrainedModel): tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') model = TFXLMWithLMHeadModel.from_pretrained('xlm-mlm-en-2048') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -667,7 +667,7 @@ class TFXLMForSequenceClassification(TFXLMPreTrainedModel): tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') model = TFXLMForSequenceClassification.from_pretrained('xlm-mlm-en-2048') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 labels = tf.constant([1])[None, :] # Batch size 1 outputs = model(input_ids) logits = outputs[0] @@ -715,7 +715,7 @@ class TFXLMForQuestionAnsweringSimple(TFXLMPreTrainedModel): tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') model = TFXLMForQuestionAnsweringSimple.from_pretrained('xlm-mlm-en-2048') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) start_scores, end_scores = outputs[:2] diff --git a/transformers/modeling_tf_xlnet.py b/transformers/modeling_tf_xlnet.py index 8a25be78c1..d2029db485 100644 --- a/transformers/modeling_tf_xlnet.py +++ b/transformers/modeling_tf_xlnet.py @@ -791,7 +791,7 @@ class TFXLNetModel(TFXLNetPreTrainedModel): tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased') model = TFXLNetModel.from_pretrained('xlnet-large-cased') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -835,7 +835,7 @@ class TFXLNetLMHeadModel(TFXLNetPreTrainedModel): model = TFXLNetLMHeadModel.from_pretrained('xlnet-large-cased') # We show how to setup inputs to predict a next token using a bi-directional context. - input_ids = tf.constant(tokenizer.encode("Hello, my dog is very "))[None, :] # We will predict the masked token + input_ids = tf.constant(tokenizer.encode("Hello, my dog is very ", add_special_tokens=True))[None, :] # We will predict the masked token perm_mask = tf.zeros((1, input_ids.shape[1], input_ids.shape[1])) perm_mask[:, :, -1] = 1.0 # Previous tokens don't see last token target_mapping = tf.zeros((1, 1, input_ids.shape[1])) # Shape [1, 1, seq_length] => let's predict one token @@ -888,7 +888,7 @@ class TFXLNetForSequenceClassification(TFXLNetPreTrainedModel): tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased') model = TFXLNetForSequenceClassification.from_pretrained('xlnet-large-cased') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) logits = outputs[0] @@ -946,7 +946,7 @@ class TFXLNetForQuestionAnsweringSimple(TFXLNetPreTrainedModel): tokenizer = XLNetTokenizer.from_pretrained('xlnet-base-cased') model = TFXLNetForQuestionAnsweringSimple.from_pretrained('xlnet-base-cased') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 outputs = model(input_ids) start_scores, end_scores = outputs[:2] @@ -1010,7 +1010,7 @@ class TFXLNetForQuestionAnsweringSimple(TFXLNetPreTrainedModel): # tokenizer = XLMTokenizer.from_pretrained('xlm-mlm-en-2048') # model = XLMForQuestionAnswering.from_pretrained('xlnet-large-cased') -# input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 +# input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 # start_positions = tf.constant([1]) # end_positions = tf.constant([3]) # outputs = model(input_ids, start_positions=start_positions, end_positions=end_positions) From d36680df546a9d4a20c58c0eab2b14ee054436ca Mon Sep 17 00:00:00 2001 From: Lorenzo Ampil Date: Sun, 27 Oct 2019 14:51:36 +0800 Subject: [PATCH 004/505] Rever changes to TF distilbert due to failed test: TFDistilBertModelTest.test_pt_tf_model_equivalence --- transformers/modeling_tf_distilbert.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/transformers/modeling_tf_distilbert.py b/transformers/modeling_tf_distilbert.py index c2d0f73999..188394816e 100644 --- a/transformers/modeling_tf_distilbert.py +++ b/transformers/modeling_tf_distilbert.py @@ -532,7 +532,7 @@ class TFDistilBertModel(TFDistilBertPreTrainedModel): tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') model = TFDistilBertModel.from_pretrained('distilbert-base-uncased') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -590,7 +590,7 @@ class TFDistilBertForMaskedLM(TFDistilBertPreTrainedModel): tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') model = TFDistilBertForMaskedLM.from_pretrained('distilbert-base-uncased') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 outputs = model(input_ids) prediction_scores = outputs[0] @@ -645,7 +645,7 @@ class TFDistilBertForSequenceClassification(TFDistilBertPreTrainedModel): tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') model = TFDistilBertForSequenceClassification.from_pretrained('distilbert-base-uncased') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 outputs = model(input_ids) logits = outputs[0] @@ -702,7 +702,7 @@ class TFDistilBertForQuestionAnswering(TFDistilBertPreTrainedModel): tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased') model = TFDistilBertForQuestionAnswering.from_pretrained('distilbert-base-uncased') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True))[None, :] # Batch size 1 + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 outputs = model(input_ids) start_scores, end_scores = outputs[:2] From 0e4cc050d63058969c53cce295dea99e90483c01 Mon Sep 17 00:00:00 2001 From: Sergey Mironov Date: Thu, 24 Oct 2019 18:15:55 +0300 Subject: [PATCH 005/505] Add support for resumable downloads for HTTP protocol. --- transformers/configuration_auto.py | 3 ++ transformers/configuration_utils.py | 7 ++++- transformers/file_utils.py | 47 ++++++++++++++++++++++------- transformers/modeling_auto.py | 8 +++++ transformers/modeling_tf_auto.py | 12 ++++++++ transformers/modeling_tf_utils.py | 8 ++++- transformers/modeling_utils.py | 8 ++++- transformers/tokenization_auto.py | 3 ++ transformers/tokenization_utils.py | 6 +++- 9 files changed, 87 insertions(+), 15 deletions(-) diff --git a/transformers/configuration_auto.py b/transformers/configuration_auto.py index edd21a670c..0e8eb03d03 100644 --- a/transformers/configuration_auto.py +++ b/transformers/configuration_auto.py @@ -92,6 +92,9 @@ class AutoConfig(object): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. diff --git a/transformers/configuration_utils.py b/transformers/configuration_utils.py index cfa6502bcd..0f9d609b20 100644 --- a/transformers/configuration_utils.py +++ b/transformers/configuration_utils.py @@ -93,6 +93,9 @@ class PretrainedConfig(object): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. @@ -119,6 +122,7 @@ class PretrainedConfig(object): """ cache_dir = kwargs.pop('cache_dir', None) force_download = kwargs.pop('force_download', False) + resume_download = kwargs.pop('resume_download', False) proxies = kwargs.pop('proxies', None) return_unused_kwargs = kwargs.pop('return_unused_kwargs', False) @@ -130,7 +134,8 @@ class PretrainedConfig(object): config_file = pretrained_model_name_or_path # redirect to the cache, if necessary try: - resolved_config_file = cached_path(config_file, cache_dir=cache_dir, force_download=force_download, proxies=proxies) + resolved_config_file = cached_path(config_file, cache_dir=cache_dir, force_download=force_download, + proxies=proxies, resume_download=resume_download) except EnvironmentError: if pretrained_model_name_or_path in cls.pretrained_config_archive_map: msg = "Couldn't reach server at '{}' to download pretrained model configuration file.".format( diff --git a/transformers/file_utils.py b/transformers/file_utils.py index 27875212ff..24abd60781 100644 --- a/transformers/file_utils.py +++ b/transformers/file_utils.py @@ -22,6 +22,7 @@ from botocore.config import Config from botocore.exceptions import ClientError import requests from tqdm import tqdm +from contextlib import contextmanager logger = logging.getLogger(__name__) # pylint: disable=invalid-name @@ -152,7 +153,7 @@ def filename_to_url(filename, cache_dir=None): return url, etag -def cached_path(url_or_filename, cache_dir=None, force_download=False, proxies=None): +def cached_path(url_or_filename, cache_dir=None, force_download=False, proxies=None, resume_download=False): """ Given something that might be a URL (or might be a local path), determine which. If it's a URL, download the file and cache it, and @@ -161,6 +162,7 @@ def cached_path(url_or_filename, cache_dir=None, force_download=False, proxies=N Args: cache_dir: specify a cache directory to save the file to (overwrite the default cache dir). force_download: if True, re-dowload the file even if it's already cached in the cache dir. + resume_download: if True, resume the download if incompletly recieved file is found. """ if cache_dir is None: cache_dir = TRANSFORMERS_CACHE @@ -173,7 +175,9 @@ def cached_path(url_or_filename, cache_dir=None, force_download=False, proxies=N if parsed.scheme in ('http', 'https', 's3'): # URL, so get it from the cache (downloading if necessary) - return get_from_cache(url_or_filename, cache_dir=cache_dir, force_download=force_download, proxies=proxies) + return get_from_cache(url_or_filename, cache_dir=cache_dir, + force_download=force_download, proxies=proxies, + resume_download=resume_download) elif os.path.exists(url_or_filename): # File, and it exists. return url_or_filename @@ -234,19 +238,22 @@ def s3_get(url, temp_file, proxies=None): s3_resource.Bucket(bucket_name).download_fileobj(s3_path, temp_file) -def http_get(url, temp_file, proxies=None): - req = requests.get(url, stream=True, proxies=proxies) - content_length = req.headers.get('Content-Length') - total = int(content_length) if content_length is not None else None - progress = tqdm(unit="B", total=total) - for chunk in req.iter_content(chunk_size=1024): +def http_get(url, temp_file, proxies=None, resume_size=0): + headers={'Range':'bytes=%d-'%(resume_size,)} if resume_size > 0 else None + response = requests.get(url, stream=True, proxies=proxies, headers=headers) + if response.status_code == 416: # Range not satisfiable + return + content_length = response.headers.get('Content-Length') + total = resume_size + int(content_length) if content_length is not None else None + progress = tqdm(unit="B", total=total, initial=resume_size) + for chunk in response.iter_content(chunk_size=1024): if chunk: # filter out keep-alive new chunks progress.update(len(chunk)) temp_file.write(chunk) progress.close() -def get_from_cache(url, cache_dir=None, force_download=False, proxies=None, etag_timeout=10): +def get_from_cache(url, cache_dir=None, force_download=False, proxies=None, etag_timeout=10, resume_download=False): """ Given a URL, look for the corresponding dataset in the local cache. If it's not there, download it. Then return the path to the cached file. @@ -289,17 +296,35 @@ def get_from_cache(url, cache_dir=None, force_download=False, proxies=None, etag if matching_files: cache_path = os.path.join(cache_dir, matching_files[-1]) + if resume_download: + incomplete_path = cache_path + '.incomplete' + @contextmanager + def _resumable_file_manager(): + with open(incomplete_path,'a+b') as f: + yield f + os.remove(incomplete_path) + temp_file_manager = _resumable_file_manager + if os.path.exists(incomplete_path): + resume_size = os.stat(incomplete_path).st_size + else: + resume_size = 0 + else: + temp_file_manager = tempfile.NamedTemporaryFile + resume_size = 0 + if not os.path.exists(cache_path) or force_download: # Download to temporary file, then copy to cache dir once finished. # Otherwise you get corrupt cache entries if the download gets interrupted. - with tempfile.NamedTemporaryFile() as temp_file: + with temp_file_manager() as temp_file: logger.info("%s not found in cache or force_download set to True, downloading to %s", url, temp_file.name) # GET file object if url.startswith("s3://"): + if resume_download: + logger.warn('Warning: resumable downloads are not implemented for "s3://" urls') s3_get(url, temp_file, proxies=proxies) else: - http_get(url, temp_file, proxies=proxies) + http_get(url, temp_file, proxies=proxies, resume_size=resume_size) # we are copying the file before closing it, so flush to avoid truncation temp_file.flush() diff --git a/transformers/modeling_auto.py b/transformers/modeling_auto.py index d98110d4bd..ea5a539c8c 100644 --- a/transformers/modeling_auto.py +++ b/transformers/modeling_auto.py @@ -112,6 +112,9 @@ class AutoModel(object): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. @@ -237,6 +240,8 @@ class AutoModelWithLMHead(object): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. @@ -357,6 +362,9 @@ class AutoModelForSequenceClassification(object): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. diff --git a/transformers/modeling_tf_auto.py b/transformers/modeling_tf_auto.py index df0ad6e401..cfe19ead2a 100644 --- a/transformers/modeling_tf_auto.py +++ b/transformers/modeling_tf_auto.py @@ -109,6 +109,9 @@ class TFAutoModel(object): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. @@ -237,6 +240,9 @@ class TFAutoModelWithLMHead(object): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. @@ -360,6 +366,9 @@ class TFAutoModelForSequenceClassification(object): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. @@ -472,6 +481,9 @@ class TFAutoModelForQuestionAnswering(object): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. diff --git a/transformers/modeling_tf_utils.py b/transformers/modeling_tf_utils.py index a96e2765fd..aed2a25643 100644 --- a/transformers/modeling_tf_utils.py +++ b/transformers/modeling_tf_utils.py @@ -176,6 +176,9 @@ class TFPreTrainedModel(tf.keras.Model): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. @@ -201,6 +204,7 @@ class TFPreTrainedModel(tf.keras.Model): cache_dir = kwargs.pop('cache_dir', None) from_pt = kwargs.pop('from_pt', False) force_download = kwargs.pop('force_download', False) + resume_download = kwargs.pop('resume_download', False) proxies = kwargs.pop('proxies', None) # Load config @@ -209,6 +213,7 @@ class TFPreTrainedModel(tf.keras.Model): pretrained_model_name_or_path, *model_args, cache_dir=cache_dir, return_unused_kwargs=True, force_download=force_download, + resume_download=resume_download, **kwargs ) else: @@ -236,7 +241,8 @@ class TFPreTrainedModel(tf.keras.Model): # redirect to the cache, if necessary try: - resolved_archive_file = cached_path(archive_file, cache_dir=cache_dir, force_download=force_download, proxies=proxies) + resolved_archive_file = cached_path(archive_file, cache_dir=cache_dir, force_download=force_download, + resume_download=resume_download, proxies=proxies) except EnvironmentError as e: if pretrained_model_name_or_path in cls.pretrained_model_archive_map: logger.error( diff --git a/transformers/modeling_utils.py b/transformers/modeling_utils.py index d082137d5d..db3ab93f8f 100644 --- a/transformers/modeling_utils.py +++ b/transformers/modeling_utils.py @@ -246,6 +246,9 @@ class PreTrainedModel(nn.Module): force_download: (`optional`) boolean, default False: Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. @@ -275,6 +278,7 @@ class PreTrainedModel(nn.Module): cache_dir = kwargs.pop('cache_dir', None) from_tf = kwargs.pop('from_tf', False) force_download = kwargs.pop('force_download', False) + resume_download = kwargs.pop('resume_download', False) proxies = kwargs.pop('proxies', None) output_loading_info = kwargs.pop('output_loading_info', False) @@ -284,6 +288,7 @@ class PreTrainedModel(nn.Module): pretrained_model_name_or_path, *model_args, cache_dir=cache_dir, return_unused_kwargs=True, force_download=force_download, + resume_download=resume_download, **kwargs ) else: @@ -315,7 +320,8 @@ class PreTrainedModel(nn.Module): # redirect to the cache, if necessary try: - resolved_archive_file = cached_path(archive_file, cache_dir=cache_dir, force_download=force_download, proxies=proxies) + resolved_archive_file = cached_path(archive_file, cache_dir=cache_dir, force_download=force_download, + proxies=proxies, resume_download=resume_download) except EnvironmentError: if pretrained_model_name_or_path in cls.pretrained_model_archive_map: msg = "Couldn't reach server at '{}' to download pretrained weights.".format( diff --git a/transformers/tokenization_auto.py b/transformers/tokenization_auto.py index ec056de17f..455f503a9d 100644 --- a/transformers/tokenization_auto.py +++ b/transformers/tokenization_auto.py @@ -87,6 +87,9 @@ class AutoTokenizer(object): force_download: (`optional`) boolean, default False: Force to (re-)download the vocabulary files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index 5e5be872ef..b602ebf20a 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -251,6 +251,9 @@ class PreTrainedTokenizer(object): force_download: (`optional`) boolean, default False: Force to (re-)download the vocabulary files and override the cached versions if they exists. + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. @@ -286,6 +289,7 @@ class PreTrainedTokenizer(object): def _from_pretrained(cls, pretrained_model_name_or_path, *init_inputs, **kwargs): cache_dir = kwargs.pop('cache_dir', None) force_download = kwargs.pop('force_download', False) + resume_download = kwargs.pop('resume_download', False) proxies = kwargs.pop('proxies', None) s3_models = list(cls.max_model_input_sizes.keys()) @@ -352,7 +356,7 @@ class PreTrainedTokenizer(object): if file_path is None: resolved_vocab_files[file_id] = None else: - resolved_vocab_files[file_id] = cached_path(file_path, cache_dir=cache_dir, force_download=force_download, proxies=proxies) + resolved_vocab_files[file_id] = cached_path(file_path, cache_dir=cache_dir, force_download=force_download, proxies=proxies, resume_download=resume_download) except EnvironmentError: if pretrained_model_name_or_path in s3_models: msg = "Couldn't reach server at '{}' to download vocabulary files." From 8d6b9d717c676560d291a7d2913b44e1cf226cd5 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 4 Nov 2019 17:07:51 +0100 Subject: [PATCH 006/505] fix #1532 and encode_plus --- .../tests/tokenization_tests_commons.py | 21 ++++-- transformers/tokenization_utils.py | 74 ++++++++++++++++--- 2 files changed, 77 insertions(+), 18 deletions(-) diff --git a/transformers/tests/tokenization_tests_commons.py b/transformers/tests/tokenization_tests_commons.py index a921696b77..cb137e8395 100644 --- a/transformers/tests/tokenization_tests_commons.py +++ b/transformers/tests/tokenization_tests_commons.py @@ -223,7 +223,11 @@ class CommonTestCases: sequence = tokenizer.encode(seq_0, add_special_tokens=False) num_added_tokens = tokenizer.num_added_tokens() total_length = len(sequence) + num_added_tokens - information = tokenizer.encode_plus(seq_0, max_length=total_length - 2, add_special_tokens=True, stride=stride) + information = tokenizer.encode_plus(seq_0, + max_length=total_length - 2, + add_special_tokens=True, + stride=stride, + return_overflowing_tokens=True) truncated_sequence = information["input_ids"] overflowing_tokens = information["overflowing_tokens"] @@ -250,10 +254,12 @@ class CommonTestCases: ) information = tokenizer.encode_plus(seq_0, seq_1, max_length=len(sequence) - 2, add_special_tokens=True, - stride=stride, truncation_strategy='only_second') + stride=stride, truncation_strategy='only_second', + return_overflowing_tokens=True) information_first_truncated = tokenizer.encode_plus(seq_0, seq_1, max_length=len(sequence) - 2, add_special_tokens=True, stride=stride, - truncation_strategy='only_first') + truncation_strategy='only_first', + return_overflowing_tokens=True) truncated_sequence = information["input_ids"] overflowing_tokens = information["overflowing_tokens"] @@ -285,7 +291,7 @@ class CommonTestCases: # Testing single inputs encoded_sequence = tokenizer.encode(sequence_0, add_special_tokens=False) - encoded_sequence_dict = tokenizer.encode_plus(sequence_0, add_special_tokens=True) + encoded_sequence_dict = tokenizer.encode_plus(sequence_0, add_special_tokens=True, return_special_tokens_mask=True) encoded_sequence_w_special = encoded_sequence_dict["input_ids"] special_tokens_mask = encoded_sequence_dict["special_tokens_mask"] self.assertEqual(len(special_tokens_mask), len(encoded_sequence_w_special)) @@ -297,7 +303,8 @@ class CommonTestCases: # Testing inputs pairs encoded_sequence = tokenizer.encode(sequence_0, add_special_tokens=False) + tokenizer.encode(sequence_1, add_special_tokens=False) - encoded_sequence_dict = tokenizer.encode_plus(sequence_0, sequence_1, add_special_tokens=True) + encoded_sequence_dict = tokenizer.encode_plus(sequence_0, sequence_1, add_special_tokens=True, + return_special_tokens_mask=True) encoded_sequence_w_special = encoded_sequence_dict["input_ids"] special_tokens_mask = encoded_sequence_dict["special_tokens_mask"] self.assertEqual(len(special_tokens_mask), len(encoded_sequence_w_special)) @@ -309,7 +316,9 @@ class CommonTestCases: # Testing with already existing special tokens if tokenizer.cls_token_id == tokenizer.unk_token_id and tokenizer.cls_token_id == tokenizer.unk_token_id: tokenizer.add_special_tokens({'cls_token': '', 'sep_token': ''}) - encoded_sequence_dict = tokenizer.encode_plus(sequence_0, add_special_tokens=True) + encoded_sequence_dict = tokenizer.encode_plus(sequence_0, + add_special_tokens=True, + return_special_tokens_mask=True) encoded_sequence_w_special = encoded_sequence_dict["input_ids"] special_tokens_mask_orig = encoded_sequence_dict["special_tokens_mask"] special_tokens_mask = tokenizer.get_special_tokens_mask(encoded_sequence_w_special, already_has_special_tokens=True) diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index ac765165e2..f860755775 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -744,6 +744,9 @@ class PreTrainedTokenizer(object): stride=0, truncation_strategy='longest_first', return_tensors=None, + return_token_type_ids=True, + return_overflowing_tokens=False, + return_special_tokens_mask=False, **kwargs): """ Returns a dictionary containing the encoded sequence or sequence pair and additional informations: @@ -770,7 +773,30 @@ class PreTrainedTokenizer(object): - 'do_not_truncate': Does not truncate (raise an error if the input sequence is longer than max_length) return_tensors: (optional) can be set to 'tf' or 'pt' to return respectively TensorFlow tf.constant or PyTorch torch.Tensor instead of a list of python integers. + return_token_type_ids: (optional) Set to False to avoid returning token_type_ids (default True). + return_overflowing_tokens: (optional) Set to True to return overflowing token information (default False). + return_special_tokens_mask: (optional) Set to True to return special tokens mask information (default False). **kwargs: passed to the `self.tokenize()` method + + Return: + A Dictionary of shape:: + + { + input_ids: list[int], + token_type_ids: list[int] if return_token_type_ids is True (default) + overflowing_tokens: list[int] if a ``max_length`` is specified and return_overflowing_tokens is True + num_truncated_tokens: int if a ``max_length`` is specified and return_overflowing_tokens is True + special_tokens_mask: list[int] if ``add_special_tokens`` if set to ``True`` and return_special_tokens_mask is True + } + + With the fields: + ``input_ids``: list of token ids to be fed to a model + ``token_type_ids``: list of token type ids to be fed to a model + + ``overflowing_tokens``: list of overflowing tokens if a max length is specified. + ``num_truncated_tokens``: number of overflowing tokens a ``max_length`` is specified + ``special_tokens_mask``: if adding special tokens, this is a list of [0, 1], with 0 specifying special added + tokens and 1 specifying sequence tokens. """ def get_input_ids(text): @@ -792,10 +818,17 @@ class PreTrainedTokenizer(object): add_special_tokens=add_special_tokens, stride=stride, truncation_strategy=truncation_strategy, - return_tensors=return_tensors) + return_tensors=return_tensors, + return_token_type_ids=return_token_type_ids, + return_overflowing_tokens=return_overflowing_tokens, + return_special_tokens_mask=return_special_tokens_mask) def prepare_for_model(self, ids, pair_ids=None, max_length=None, add_special_tokens=True, stride=0, - truncation_strategy='longest_first', return_tensors=None): + truncation_strategy='longest_first', + return_tensors=None, + return_token_type_ids=True, + return_overflowing_tokens=False, + return_special_tokens_mask=False): """ Prepares a sequence of input id, or a pair of sequences of inputs ids so that it can be used by the model. It adds special tokens, truncates @@ -820,21 +853,27 @@ class PreTrainedTokenizer(object): - 'do_not_truncate': Does not truncate (raise an error if the input sequence is longer than max_length) return_tensors: (optional) can be set to 'tf' or 'pt' to return respectively TensorFlow tf.constant or PyTorch torch.Tensor instead of a list of python integers. + return_token_type_ids: (optional) Set to False to avoid returning token_type_ids (default True). + return_overflowing_tokens: (optional) Set to True to return overflowing token information (default False). + return_special_tokens_mask: (optional) Set to True to return special tokens mask information (default False). Return: A Dictionary of shape:: { input_ids: list[int], - overflowing_tokens: list[int] if a ``max_length`` is specified, else None - special_tokens_mask: list[int] if ``add_special_tokens`` if set to ``True`` + token_type_ids: list[int] if return_token_type_ids is True (default) + overflowing_tokens: list[int] if a ``max_length`` is specified and return_overflowing_tokens is True + num_truncated_tokens: int if a ``max_length`` is specified and return_overflowing_tokens is True + special_tokens_mask: list[int] if ``add_special_tokens`` if set to ``True`` and return_special_tokens_mask is True } With the fields: - ``input_ids``: list of tokens to be fed to a model + ``input_ids``: list of token ids to be fed to a model + ``token_type_ids``: list of token type ids to be fed to a model ``overflowing_tokens``: list of overflowing tokens if a max length is specified. - + ``num_truncated_tokens``: number of overflowing tokens a ``max_length`` is specified ``special_tokens_mask``: if adding special tokens, this is a list of [0, 1], with 0 specifying special added tokens and 1 specifying sequence tokens. """ @@ -843,23 +882,31 @@ class PreTrainedTokenizer(object): len_pair_ids = len(pair_ids) if pair else 0 encoded_inputs = {} + + # Handle max sequence length total_len = len_ids + len_pair_ids + (self.num_added_tokens(pair=pair) if add_special_tokens else 0) if max_length and total_len > max_length: ids, pair_ids, overflowing_tokens = self.truncate_sequences(ids, pair_ids=pair_ids, num_tokens_to_remove=total_len-max_length, truncation_strategy=truncation_strategy, stride=stride) - encoded_inputs["overflowing_tokens"] = overflowing_tokens - encoded_inputs["num_truncated_tokens"] = total_len - max_length + if return_overflowing_tokens: + encoded_inputs["overflowing_tokens"] = overflowing_tokens + encoded_inputs["num_truncated_tokens"] = total_len - max_length + # Handle special_tokens if add_special_tokens: sequence = self.build_inputs_with_special_tokens(ids, pair_ids) token_type_ids = self.create_token_type_ids_from_sequences(ids, pair_ids) - encoded_inputs["special_tokens_mask"] = self.get_special_tokens_mask(ids, pair_ids) + special_tokens_mask = self.get_special_tokens_mask(ids, pair_ids) else: sequence = ids + pair_ids if pair else ids token_type_ids = [0] * len(ids) + ([1] * len(pair_ids) if pair else []) + special_tokens_mask = [0] * (len(ids) + (len(pair_ids) if pair else 0)) + if return_special_tokens_mask: + encoded_inputs["special_tokens_mask"] = self.get_special_tokens_mask(ids, pair_ids) + # Prepare inputs as tensors if asked if return_tensors == 'tf' and is_tf_available(): sequence = tf.constant([sequence]) token_type_ids = tf.constant([token_type_ids]) @@ -870,12 +917,15 @@ class PreTrainedTokenizer(object): logger.warning("Unable to convert output to tensors format {}, PyTorch or TensorFlow is not available.".format(return_tensors)) encoded_inputs["input_ids"] = sequence - encoded_inputs["token_type_ids"] = token_type_ids + if return_token_type_ids: + encoded_inputs["token_type_ids"] = token_type_ids if max_length and len(encoded_inputs["input_ids"]) > max_length: encoded_inputs["input_ids"] = encoded_inputs["input_ids"][:max_length] - encoded_inputs["token_type_ids"] = encoded_inputs["token_type_ids"][:max_length] - encoded_inputs["special_tokens_mask"] = encoded_inputs["special_tokens_mask"][:max_length] + if return_token_type_ids: + encoded_inputs["token_type_ids"] = encoded_inputs["token_type_ids"][:max_length] + if return_special_tokens_mask: + encoded_inputs["special_tokens_mask"] = encoded_inputs["special_tokens_mask"][:max_length] return encoded_inputs From 8df7dfd2a723465b0cca9f5e808a75e074482d02 Mon Sep 17 00:00:00 2001 From: Filip Povolny Date: Tue, 5 Nov 2019 11:09:16 +0100 Subject: [PATCH 007/505] Make dummy inputs a local variable in TFPreTrainedModel. --- transformers/modeling_tf_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/transformers/modeling_tf_utils.py b/transformers/modeling_tf_utils.py index a96e2765fd..110a590f55 100644 --- a/transformers/modeling_tf_utils.py +++ b/transformers/modeling_tf_utils.py @@ -51,7 +51,6 @@ class TFPreTrainedModel(tf.keras.Model): config_class = None pretrained_model_archive_map = {} base_model_prefix = "" - dummy_inputs = tf.constant(DUMMY_INPUTS) # dummy inputs to build the network def __init__(self, config, *inputs, **kwargs): super(TFPreTrainedModel, self).__init__(*inputs, **kwargs) @@ -266,14 +265,15 @@ class TFPreTrainedModel(tf.keras.Model): # Load from a PyTorch checkpoint return load_pytorch_checkpoint_in_tf2_model(model, resolved_archive_file) - ret = model(model.dummy_inputs, training=False) # build the network with dummy inputs + dummy_inputs = tf.constant(DUMMY_INPUTS) # dummy inputs to build the network + ret = model(dummy_inputs, training=False) # build the network with dummy inputs assert os.path.isfile(resolved_archive_file), "Error retrieving file {}".format(resolved_archive_file) # 'by_name' allow us to do transfer learning by skipping/adding layers # see https://github.com/tensorflow/tensorflow/blob/00fad90125b18b80fe054de1055770cfb8fe4ba3/tensorflow/python/keras/engine/network.py#L1339-L1357 model.load_weights(resolved_archive_file, by_name=True) - ret = model(model.dummy_inputs, training=False) # Make sure restore ops are run + ret = model(dummy_inputs, training=False) # Make sure restore ops are run return model From dfb61caf77a02a735af0bf430f6d6082b7d01cfd Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 5 Nov 2019 11:25:13 +0100 Subject: [PATCH 008/505] fix #1692 --- transformers/modeling_tf_xlnet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/modeling_tf_xlnet.py b/transformers/modeling_tf_xlnet.py index 8a25be78c1..e83674e214 100644 --- a/transformers/modeling_tf_xlnet.py +++ b/transformers/modeling_tf_xlnet.py @@ -539,7 +539,7 @@ class TFXLNetMainLayer(tf.keras.layers.Layer): assert input_mask is None or attention_mask is None, "You can only use one of input_mask (uses 1 for padding) " \ "or attention_mask (uses 0 for padding, added for compatbility with BERT). Please choose one." if input_mask is None and attention_mask is not None: - input_mask = 1.0 - attention_mask + input_mask = 1.0 - tf.cast(attention_mask, dtype=dtype_float) if input_mask is not None and perm_mask is not None: data_mask = input_mask[None] + perm_mask elif input_mask is not None and perm_mask is None: From 124409d0754f5b84ff47daea51dede1d3b37a8dd Mon Sep 17 00:00:00 2001 From: Filip Povolny Date: Tue, 5 Nov 2019 11:48:45 +0100 Subject: [PATCH 009/505] Make dummy inputs a property of TFPreTrainedModel. --- transformers/modeling_tf_utils.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/transformers/modeling_tf_utils.py b/transformers/modeling_tf_utils.py index 110a590f55..33cfdc503d 100644 --- a/transformers/modeling_tf_utils.py +++ b/transformers/modeling_tf_utils.py @@ -52,6 +52,15 @@ class TFPreTrainedModel(tf.keras.Model): pretrained_model_archive_map = {} base_model_prefix = "" + @property + def dummy_inputs(self): + """ Dummy inputs to build the network. + + Returns: + tf.Tensor with dummy inputs + """ + return tf.constant(DUMMY_INPUTS) + def __init__(self, config, *inputs, **kwargs): super(TFPreTrainedModel, self).__init__(*inputs, **kwargs) if not isinstance(config, PretrainedConfig): @@ -265,15 +274,14 @@ class TFPreTrainedModel(tf.keras.Model): # Load from a PyTorch checkpoint return load_pytorch_checkpoint_in_tf2_model(model, resolved_archive_file) - dummy_inputs = tf.constant(DUMMY_INPUTS) # dummy inputs to build the network - ret = model(dummy_inputs, training=False) # build the network with dummy inputs + ret = model(model.dummy_inputs, training=False) # build the network with dummy inputs assert os.path.isfile(resolved_archive_file), "Error retrieving file {}".format(resolved_archive_file) # 'by_name' allow us to do transfer learning by skipping/adding layers # see https://github.com/tensorflow/tensorflow/blob/00fad90125b18b80fe054de1055770cfb8fe4ba3/tensorflow/python/keras/engine/network.py#L1339-L1357 model.load_weights(resolved_archive_file, by_name=True) - ret = model(dummy_inputs, training=False) # Make sure restore ops are run + ret = model(model.dummy_inputs, training=False) # Make sure restore ops are run return model From 60a5babd57dd80f855df859abf006ee4488ff639 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 5 Nov 2019 12:01:23 +0100 Subject: [PATCH 010/505] adding files --- transformers/configuration_t5.py | 130 +++++ ...rt_t5_original_tf_checkpoint_to_pytorch.py | 65 +++ transformers/modeling_t5.py | 373 +++++++++++++ transformers/modeling_tf_t5.py | 496 ++++++++++++++++++ transformers/tokenization_t5.py | 214 ++++++++ 5 files changed, 1278 insertions(+) create mode 100644 transformers/configuration_t5.py create mode 100755 transformers/convert_t5_original_tf_checkpoint_to_pytorch.py create mode 100644 transformers/modeling_t5.py create mode 100644 transformers/modeling_tf_t5.py create mode 100644 transformers/tokenization_t5.py diff --git a/transformers/configuration_t5.py b/transformers/configuration_t5.py new file mode 100644 index 0000000000..a37a5b2157 --- /dev/null +++ b/transformers/configuration_t5.py @@ -0,0 +1,130 @@ +# coding=utf-8 +# Copyright 2010, The T5 Authors and HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" T5 model configuration """ + +from __future__ import absolute_import, division, print_function, unicode_literals + +import json +import logging +import sys +import six +from io import open + +from .configuration_utils import PretrainedConfig + +logger = logging.getLogger(__name__) + +T5_PRETRAINED_CONFIG_ARCHIVE_MAP = { + 't5-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-base-uncased-config.json", + 't5-large-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-large-uncased-config.json", +} + + +class T5Config(PretrainedConfig): + r""" + :class:`~transformers.T5Config` is the configuration class to store the configuration of a + `T5Model`. + + + Arguments: + vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `T5Model`. + hidden_size: Size of the encoder layers and the pooler layer. + num_hidden_layers: Number of hidden layers in the Transformer encoder. + num_attention_heads: Number of attention heads for each attention layer in + the Transformer encoder. + intermediate_size: The size of the "intermediate" (i.e., feed-forward) + layer in the Transformer encoder. + hidden_act: The non-linear activation function (function or string) in the + encoder and pooler. If string, "gelu", "relu", "swish" and "gelu_new" are supported. + hidden_dropout_prob: The dropout probabilitiy for all fully connected + layers in the embeddings, encoder, and pooler. + attention_probs_dropout_prob: The dropout ratio for the attention + probabilities. + max_position_embeddings: The maximum sequence length that this model might + ever be used with. Typically set this to something large just in case + (e.g., 512 or 1024 or 2048). + type_vocab_size: The vocabulary size of the `token_type_ids` passed into + `T5Model`. + initializer_range: The sttdev of the truncated_normal_initializer for + initializing all weight matrices. + layer_norm_eps: The epsilon used by LayerNorm. + """ + pretrained_config_archive_map = T5_PRETRAINED_CONFIG_ARCHIVE_MAP + + def __init__(self, + vocab_size_or_config_json_file=50257, + n_positions=1024, + n_ctx=1024, + n_embd=768, + n_layer=12, + n_head=12, + resid_pdrop=0.1, + embd_pdrop=0.1, + attn_pdrop=0.1, + layer_norm_epsilon=1e-5, + initializer_range=0.02, + + num_labels=1, + summary_type='cls_index', + summary_use_proj=True, + summary_activation=None, + summary_proj_to_labels=True, + summary_first_dropout=0.1, + **kwargs): + super(T5Config, self).__init__(**kwargs) + self.vocab_size = vocab_size_or_config_json_file if isinstance(vocab_size_or_config_json_file, six.string_types) else -1 + self.n_ctx = n_ctx + self.n_positions = n_positions + self.n_embd = n_embd + self.n_layer = n_layer + self.n_head = n_head + self.resid_pdrop = resid_pdrop + self.embd_pdrop = embd_pdrop + self.attn_pdrop = attn_pdrop + self.layer_norm_epsilon = layer_norm_epsilon + self.initializer_range = initializer_range + + self.num_labels = num_labels + self.summary_type = summary_type + self.summary_use_proj = summary_use_proj + self.summary_activation = summary_activation + self.summary_first_dropout = summary_first_dropout + self.summary_proj_to_labels = summary_proj_to_labels + if isinstance(vocab_size_or_config_json_file, six.string_types): + with open(vocab_size_or_config_json_file, "r", encoding="utf-8") as reader: + json_config = json.loads(reader.read()) + for key, value in json_config.items(): + self.__dict__[key] = value + elif not isinstance(vocab_size_or_config_json_file, int): + raise ValueError( + "First argument must be either a vocabulary size (int)" + "or the path to a pretrained model config file (str)" + ) + + @property + def max_position_embeddings(self): + return self.n_positions + + @property + def hidden_size(self): + return self.n_embd + + @property + def num_attention_heads(self): + return self.n_head + + @property + def num_hidden_layers(self): + return self.n_layer diff --git a/transformers/convert_t5_original_tf_checkpoint_to_pytorch.py b/transformers/convert_t5_original_tf_checkpoint_to_pytorch.py new file mode 100755 index 0000000000..608027ebac --- /dev/null +++ b/transformers/convert_t5_original_tf_checkpoint_to_pytorch.py @@ -0,0 +1,65 @@ +# coding=utf-8 +# Copyright 2018 The T5 authors and HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Convert T5 checkpoint.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import argparse +import torch + +from transformers import T5Config, T5ForPreTraining, load_tf_weights_in_t5 + +import logging +logging.basicConfig(level=logging.INFO) + +def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, t5_config_file, pytorch_dump_path): + # Initialise PyTorch model + config = T5Config.from_json_file(t5_config_file) + print("Building PyTorch model from configuration: {}".format(str(config))) + model = T5ForPreTraining(config) + + # Load weights from tf checkpoint + load_tf_weights_in_t5(model, config, tf_checkpoint_path) + + # Save pytorch-model + print("Save PyTorch model to {}".format(pytorch_dump_path)) + torch.save(model.state_dict(), pytorch_dump_path) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + ## Required parameters + parser.add_argument("--tf_checkpoint_path", + default = None, + type = str, + required = True, + help = "Path to the TensorFlow checkpoint path.") + parser.add_argument("--t5_config_file", + default = None, + type = str, + required = True, + help = "The config json file corresponding to the pre-trained T5 model. \n" + "This specifies the model architecture.") + parser.add_argument("--pytorch_dump_path", + default = None, + type = str, + required = True, + help = "Path to the output PyTorch model.") + args = parser.parse_args() + convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, + args.t5_config_file, + args.pytorch_dump_path) diff --git a/transformers/modeling_t5.py b/transformers/modeling_t5.py new file mode 100644 index 0000000000..fa3c22f24b --- /dev/null +++ b/transformers/modeling_t5.py @@ -0,0 +1,373 @@ +# coding=utf-8 +# Copyright 2018 T5 Authors and HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PyTorch T5 model. """ + +from __future__ import absolute_import, division, print_function, unicode_literals + +import json +import logging +import math +import os +import sys +from io import open + +import torch +from torch import nn +from torch.nn import CrossEntropyLoss, MSELoss + +from .modeling_utils import PreTrainedModel, prune_linear_layer +from .configuration_t5 import T5Config +from .file_utils import add_start_docstrings + +logger = logging.getLogger(__name__) + +#################################################### +# This dict contrains shortcut names and associated url +# for the pretrained weights provided with the models +#################################################### +T5_PRETRAINED_MODEL_ARCHIVE_MAP = { + 't5-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-base-uncased-pytorch_model.bin", + 't5-large-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-large-uncased-pytorch_model.bin", +} + +#################################################### +# This is a conversion method from TF 1.0 to PyTorch +# More details: https://medium.com/huggingface/from-tensorflow-to-pytorch-265f40ef2a28 +#################################################### +def load_tf_weights_in_t5(model, config, tf_checkpoint_path): + """ Load tf checkpoints in a pytorch model. + """ + try: + import re + import numpy as np + import tensorflow as tf + except ImportError: + logger.error("Loading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see " + "https://www.tensorflow.org/install/ for installation instructions.") + raise + tf_path = os.path.abspath(tf_checkpoint_path) + logger.info("Converting TensorFlow checkpoint from {}".format(tf_path)) + # Load weights from TF model + init_vars = tf.train.list_variables(tf_path) + names = [] + arrays = [] + for name, shape in init_vars: + logger.info("Loading TF weight {} with shape {}".format(name, shape)) + array = tf.train.load_variable(tf_path, name) + names.append(name) + arrays.append(array) + + for name, array in zip(names, arrays): + name = name.split('/') + # adam_v and adam_m are variables used in AdamWeightDecayOptimizer to calculated m and v + # which are not required for using pretrained model + if any(n in ["adam_v", "adam_m", "global_step"] for n in name): + logger.info("Skipping {}".format("/".join(name))) + continue + pointer = model + for m_name in name: + if re.fullmatch(r'[A-Za-z]+_\d+', m_name): + l = re.split(r'_(\d+)', m_name) + else: + l = [m_name] + if l[0] == 'kernel' or l[0] == 'gamma': + pointer = getattr(pointer, 'weight') + elif l[0] == 'output_bias' or l[0] == 'beta': + pointer = getattr(pointer, 'bias') + elif l[0] == 'output_weights': + pointer = getattr(pointer, 'weight') + elif l[0] == 'squad': + pointer = getattr(pointer, 'classifier') + else: + try: + pointer = getattr(pointer, l[0]) + except AttributeError: + logger.info("Skipping {}".format("/".join(name))) + continue + if len(l) >= 2: + num = int(l[1]) + pointer = pointer[num] + if m_name[-11:] == '_embeddings': + pointer = getattr(pointer, 'weight') + elif m_name == 'kernel': + array = np.transpose(array) + try: + assert pointer.shape == array.shape + except AssertionError as e: + e.args += (pointer.shape, array.shape) + raise + logger.info("Initialize PyTorch weight {}".format(name)) + pointer.data = torch.from_numpy(array) + return model + + +#################################################### +# PyTorch Models are constructed by sub-classing +# - torch.nn.Module for the layers and +# - PreTrainedModel for the models (it-self a sub-class of torch.nn.Module) +#################################################### + +class T5Layer(nn.Module): + def __init__(self, config): + super(T5Layer, self).__init__() + self.attention = T5Attention(config) + self.intermediate = T5Intermediate(config) + self.output = T5Output(config) + + def forward(self, hidden_states, attention_mask=None, head_mask=None): + attention_outputs = self.attention(hidden_states, attention_mask, head_mask) + attention_output = attention_outputs[0] + intermediate_output = self.intermediate(attention_output) + layer_output = self.output(intermediate_output, attention_output) + outputs = (layer_output,) + attention_outputs[1:] # add attentions if we output them + return outputs + + + +class T5PreTrainedModel(PreTrainedModel): + """ An abstract class to handle weights initialization and + a simple interface for dowloading and loading pretrained models. + """ + config_class = T5Config + pretrained_model_archive_map = T5_PRETRAINED_MODEL_ARCHIVE_MAP + load_tf_weights = load_tf_weights_in_t5 + base_model_prefix = "transformer" + + def _init_weights(self, module): + """ Initialize the weights """ + if isinstance(module, (nn.Linear, nn.Embedding)): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) + elif isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + if isinstance(module, nn.Linear) and module.bias is not None: + module.bias.data.zero_() + + +T5_START_DOCSTRING = r""" The T5 model was proposed in + `Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer`_ + by Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu. + It's an encoder decoder pre-trained transformer. + + This model is a PyTorch `torch.nn.Module`_ sub-class. Use it as a regular PyTorch Module and + refer to the PyTorch documentation for all matter related to general usage and behavior. + + .. _`Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer`: + https://arxiv.org/abs/1910.10683 + + .. _`torch.nn.Module`: + https://pytorch.org/docs/stable/nn.html#module + + Parameters: + config (:class:`~transformers.T5Config`): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the configuration. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. +""" + +T5_INPUTS_DOCSTRING = r""" + Inputs: + **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + To match pre-training, T5 input sequence should be formatted with [CLS] and [SEP] tokens as follows: + + (a) For sequence pairs: + + ``tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]`` + + (b) For single sequences: + + ``tokens: [CLS] the dog is hairy . [SEP]`` + + T5 is a model with relative position embeddings so you should be able to pad the inputs on + the right or the left. + + Indices can be obtained using :class:`transformers.T5Tokenizer`. + See :func:`transformers.PreTrainedTokenizer.encode` and + :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: + Mask to avoid performing attention on padding token indices. + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. + **position_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of positions of each input sequence tokens in the position embeddings. + Selected in the range ``[0, config.max_position_embeddings - 1]``. + **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. +""" + +@add_start_docstrings("The bare single stack (encoder or decoder) of a T5 Model transformer outputting raw hidden-states" + "without any specific head on top.", + T5_START_DOCSTRING, T5_INPUTS_DOCSTRING) +class T5Model(T5PreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + Sequence of hidden-states at the output of the last layer of the model. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = T5Tokenizer.from_pretrained('t5-base-uncased') + model = T5Model.from_pretrained('t5-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids) + last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple + + """ + def __init__(self, config): + super(T5Model, self).__init__(config) + + self.embeddings = T5Embeddings(config) + self.encoder = T5Encoder(config) + self.pooler = T5Pooler(config) + + self.init_weights() + + @property + def get_input_embeddings(self): + return self.embeddings.word_embeddings + + def set_input_embeddings(self, new_embeddings): + self.embeddings.word_embeddings = new_embeddings + + def _prune_heads(self, heads_to_prune): + """ Prunes heads of the model. + heads_to_prune: dict of {layer_num: list of heads to prune in this layer} + See base class PreTrainedModel + """ + for layer, heads in heads_to_prune.items(): + self.encoder.layer[layer].attention.prune_heads(heads) + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None): + if attention_mask is None: + attention_mask = torch.ones_like(input_ids) + if token_type_ids is None: + token_type_ids = torch.zeros_like(input_ids) + + # We create a 3D attention mask from a 2D tensor mask. + # Sizes are [batch_size, 1, 1, to_seq_length] + # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length] + # this attention mask is more simple than the triangular masking of causal attention + # used in OpenAI GPT, we just need to prepare the broadcast dimension here. + extended_attention_mask = attention_mask.unsqueeze(1).unsqueeze(2) + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and -10000.0 for masked positions. + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility + extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + if head_mask is not None: + if head_mask.dim() == 1: + head_mask = head_mask.unsqueeze(0).unsqueeze(0).unsqueeze(-1).unsqueeze(-1) + head_mask = head_mask.expand(self.config.num_hidden_layers, -1, -1, -1, -1) + elif head_mask.dim() == 2: + head_mask = head_mask.unsqueeze(1).unsqueeze(-1).unsqueeze(-1) # We can specify head_mask for each layer + head_mask = head_mask.to(dtype=next(self.parameters()).dtype) # switch to fload if need + fp16 compatibility + else: + head_mask = [None] * self.config.num_hidden_layers + + ################################## + # Replace this with your model code + embedding_output = self.embeddings(input_ids, position_ids=position_ids, token_type_ids=token_type_ids) + encoder_outputs = self.encoder(embedding_output, extended_attention_mask, head_mask=head_mask) + sequence_output = encoder_outputs[0] + outputs = (sequence_output,) + encoder_outputs[1:] # add hidden_states and attentions if they are here + + return outputs # sequence_output, (hidden_states), (attentions) + + +@add_start_docstrings("""T5 Model with a `language modeling` head on top. """, + T5_START_DOCSTRING, T5_INPUTS_DOCSTRING) +class T5WithLMHead(T5PreTrainedModel): + r""" + **lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the masked language modeling loss. + Indices should be in ``[-1, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) + Tokens with indices set to ``-1`` are ignored (masked), the loss is only computed for the tokens with labels + in ``[0, ..., config.vocab_size]`` + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Masked language modeling loss. + **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = T5Tokenizer.from_pretrained('t5-base-uncased') + model = T5ForMaskedLM.from_pretrained('t5-base-uncased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, lm_labels=input_ids) + loss, prediction_scores = outputs[:2] + + """ + def __init__(self, config): + super(T5ForMaskedLM, self).__init__(config) + + self.transformer = T5Model(config) + self.lm_head = nn.Linear(config.n_embd, config.vocab_size) + + self.init_weights() + + def get_output_embeddings(self): + return self.lm_head + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + lm_labels=None): + + outputs = self.transformer(input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask) + + sequence_output = outputs[0] + lm_logits = self.cls(sequence_output) + + outputs = (lm_logits,) + outputs[2:] # Add hidden states and attention if they are here + if lm_labels is not None: + shift_logits = lm_logits[..., :-1, :].contiguous() + shift_labels = lm_labels[..., 1:].contiguous() + loss_fct = CrossEntropyLoss(ignore_index=-1) + loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), + shift_labels.view(-1)) + outputs = (loss,) + outputs + + return outputs # (lm_loss), lm_logits, (hidden_states), (attentions) diff --git a/transformers/modeling_tf_t5.py b/transformers/modeling_tf_t5.py new file mode 100644 index 0000000000..deb453846c --- /dev/null +++ b/transformers/modeling_tf_t5.py @@ -0,0 +1,496 @@ +# coding=utf-8 +# Copyright 2018 T5 Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" TF 2.0 T5 model. """ + +from __future__ import absolute_import, division, print_function, unicode_literals + +import json +import logging +import math +import os +import sys +from io import open + +import numpy as np +import tensorflow as tf + +from .configuration_t5 import T5Config +from .modeling_tf_utils import TFPreTrainedModel, get_initializer +from .file_utils import add_start_docstrings + +logger = logging.getLogger(__name__) + +#################################################### +# This dict contrains shortcut names and associated url +# for the pretrained weights provided with the models +#################################################### +TF_T5_PRETRAINED_MODEL_ARCHIVE_MAP = { + 't5-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-base-uncased-tf_model.h5", + 't5-large-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-large-uncased-tf_model.h5", +} + +#################################################### +# TF 2.0 Models are constructed using Keras imperative API by sub-classing +# - tf.keras.layers.Layer for the layers and +# - TFPreTrainedModel for the models (it-self a sub-class of tf.keras.Model) +#################################################### + +#################################################### +# Here is an example of typical layer in a TF 2.0 model of the library +# The classes are usually identical to the PyTorch ones and prefixed with 'TF'. +# +# Note that class __init__ parameters includes **kwargs (send to 'super'). +# This let us have a control on class scope and variable names: +# More precisely, we set the names of the class attributes (lower level layers) to +# to the equivalent attributes names in the PyTorch model so we can have equivalent +# class and scope structure between PyTorch and TF 2.0 models and easily load one in the other. +# +# See the conversion methods in modeling_tf_pytorch_utils.py for more details +#################################################### +class TFT5Layer(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFT5Layer, self).__init__(**kwargs) + self.attention = TFT5Attention(config, name='attention') + self.intermediate = TFT5Intermediate(config, name='intermediate') + self.transformer_output = TFT5Output(config, name='output') + + def call(self, inputs, training=False): + hidden_states, attention_mask, head_mask = inputs + + attention_outputs = self.attention([hidden_states, attention_mask, head_mask], training=training) + attention_output = attention_outputs[0] + intermediate_output = self.intermediate(attention_output) + layer_output = self.transformer_output([intermediate_output, attention_output], training=training) + outputs = (layer_output,) + attention_outputs[1:] # add attentions if we output them + return outputs + + +#################################################### +# The full model without a specific pretrained or finetuning head is +# provided as a tf.keras.layers.Layer usually called "TFT5MainLayer" +#################################################### +class TFT5MainLayer(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFT5MainLayer, self).__init__(**kwargs) + + def _resize_token_embeddings(self, new_num_tokens): + raise NotImplementedError # Not implemented yet in the library fr TF 2.0 models + + def _prune_heads(self, heads_to_prune): + raise NotImplementedError # Not implemented yet in the library fr TF 2.0 models + + def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, training=False): + # We allow three types of multi-inputs: + # - traditional keyword arguments in the call method + # - all the arguments provided as a dict in the first positional argument of call + # - all the arguments provided as a list/tuple (ordered) in the first positional argument of call + # The last two options are useful to use the tf.keras fit() method. + + if isinstance(inputs, (tuple, list)): + input_ids = inputs[0] + attention_mask = inputs[1] if len(inputs) > 1 else attention_mask + token_type_ids = inputs[2] if len(inputs) > 2 else token_type_ids + position_ids = inputs[3] if len(inputs) > 3 else position_ids + head_mask = inputs[4] if len(inputs) > 4 else head_mask + assert len(inputs) <= 5, "Too many inputs." + elif isinstance(inputs, dict): + input_ids = inputs.get('input_ids') + attention_mask = inputs.get('attention_mask', attention_mask) + token_type_ids = inputs.get('token_type_ids', token_type_ids) + position_ids = inputs.get('position_ids', position_ids) + head_mask = inputs.get('head_mask', head_mask) + assert len(inputs) <= 5, "Too many inputs." + else: + input_ids = inputs + + if attention_mask is None: + attention_mask = tf.fill(tf.shape(input_ids), 1) + if token_type_ids is None: + token_type_ids = tf.fill(tf.shape(input_ids), 0) + + # We create a 3D attention mask from a 2D tensor mask. + # Sizes are [batch_size, 1, 1, to_seq_length] + # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length] + # this attention mask is more simple than the triangular masking of causal attention + # used in OpenAI GPT, we just need to prepare the broadcast dimension here. + extended_attention_mask = attention_mask[:, tf.newaxis, tf.newaxis, :] + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and -10000.0 for masked positions. + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + + extended_attention_mask = tf.cast(extended_attention_mask, tf.float32) + extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + if not head_mask is None: + raise NotImplementedError + else: + head_mask = [None] * self.num_hidden_layers + # head_mask = tf.constant([0] * self.num_hidden_layers) + + ################################## + # Replace this with your model code + embedding_output = self.embeddings(input_ids, position_ids=position_ids, token_type_ids=token_type_ids) + encoder_outputs = self.encoder([embedding_output, extended_attention_mask, head_mask], training=training) + sequence_output = encoder_outputs[0] + outputs = (sequence_output,) + encoder_outputs[1:] # add hidden_states and attentions if they are here + + return outputs # sequence_output, (hidden_states), (attentions) + + +#################################################### +# TFT5PreTrainedModel is a sub-class of tf.keras.Model +# which take care of loading and saving pretrained weights +# and various common utilities. +# Here you just need to specify a few (self-explanatory) +# pointers for your model. +#################################################### +class TFT5PreTrainedModel(TFPreTrainedModel): + """ An abstract class to handle weights initialization and + a simple interface for dowloading and loading pretrained models. + """ + config_class = T5Config + pretrained_model_archive_map = TF_T5_PRETRAINED_MODEL_ARCHIVE_MAP + base_model_prefix = "transformer" + + +T5_START_DOCSTRING = r""" The XXX model was proposed in + `XXX: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ + by Jacob Devlin, Ming-Wei Chang, Kenton Lee and Kristina Toutanova. It's a bidirectional transformer + pre-trained using a combination of masked language modeling objective and next sentence prediction + on a large corpus comprising the Toronto Book Corpus and Wikipedia. + + This model is a tf.keras.Model `tf.keras.Model`_ sub-class. Use it as a regular TF 2.0 Keras Model and + refer to the TF 2.0 documentation for all matter related to general usage and behavior. + + .. _`XXX: Pre-training of Deep Bidirectional Transformers for Language Understanding`: + https://arxiv.org/abs/1810.04805 + + .. _`tf.keras.Model`: + https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/Model + + Note on the model inputs: + TF 2.0 models accepts two formats as inputs: + + - having all inputs as keyword arguments (like PyTorch models), or + - having all inputs as a list, tuple or dict in the first positional arguments. + + This second option is usefull when using `tf.keras.Model.fit()` method which currently requires having all the tensors in the first argument of the model call function: `model(inputs)`. + + If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the first positional argument : + + - a single Tensor with input_ids only and nothing else: `model(inputs_ids) + - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring: + `model([input_ids, attention_mask])` or `model([input_ids, attention_mask, token_type_ids])` + - a dictionary with one or several input Tensors associaed to the input names given in the docstring: + `model({'input_ids': input_ids, 'token_type_ids': token_type_ids})` + + Parameters: + config (:class:`~transformers.XxxConfig`): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the configuration. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. +""" + +XXX_INPUTS_DOCSTRING = r""" + Inputs: + **input_ids**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + To match pre-training, XXX input sequence should be formatted with [CLS] and [SEP] tokens as follows: + + (a) For sequence pairs: + + ``tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1`` + + (b) For single sequences: + + ``tokens: [CLS] the dog is hairy . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0`` + + Xxx is a model with absolute position embeddings so it's usually advised to pad the inputs on + the right rather than the left. + + Indices can be obtained using :class:`transformers.XxxTokenizer`. + See :func:`transformers.PreTrainedTokenizer.encode` and + :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + **attention_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: + Mask to avoid performing attention on padding token indices. + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. + **token_type_ids**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: + Segment token indices to indicate first and second portions of the inputs. + Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` + corresponds to a `sentence B` token + (see `XXX: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ for more details). + **position_ids**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: + Indices of positions of each input sequence tokens in the position embeddings. + Selected in the range ``[0, config.max_position_embeddings - 1]``. + **head_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. +""" + +@add_start_docstrings("The bare Xxx Model transformer outputing raw hidden-states without any specific head on top.", + XXX_START_DOCSTRING, XXX_INPUTS_DOCSTRING) +class TFXxxModel(TFXxxPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **last_hidden_state**: ``tf.Tensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + Sequence of hidden-states at the output of the last layer of the model. + **pooler_output**: ``tf.Tensor`` of shape ``(batch_size, hidden_size)`` + Last layer hidden-state of the first token of the sequence (classification token) + further processed by a Linear layer and a Tanh activation function. The Linear + layer weights are trained from the next sentence prediction (classification) + objective during Xxx pretraining. This output is usually *not* a good summary + of the semantic content of the input, you're often better with averaging or pooling + the sequence of hidden-states for the whole input sequence. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + import tensorflow as tf + from transformers import XxxTokenizer, TFXxxModel + + tokenizer = XxxTokenizer.from_pretrained('xxx-base-uncased') + model = TFXxxModel.from_pretrained('xxx-base-uncased') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + outputs = model(input_ids) + last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple + + """ + def __init__(self, config, *inputs, **kwargs): + super(TFXxxModel, self).__init__(config, *inputs, **kwargs) + self.transformer = TFXxxMainLayer(config, name='transformer') + + def call(self, inputs, **kwargs): + outputs = self.transformer(inputs, **kwargs) + return outputs + + +@add_start_docstrings("""Xxx Model with a `language modeling` head on top. """, + XXX_START_DOCSTRING, XXX_INPUTS_DOCSTRING) +class TFXxxForMaskedLM(TFXxxPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **prediction_scores**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``Numpy array`` or ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``Numpy array`` or ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + import tensorflow as tf + from transformers import XxxTokenizer, TFXxxForMaskedLM + + tokenizer = XxxTokenizer.from_pretrained('xxx-base-uncased') + model = TFXxxForMaskedLM.from_pretrained('xxx-base-uncased') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + outputs = model(input_ids) + prediction_scores = outputs[0] + + """ + def __init__(self, config, *inputs, **kwargs): + super(TFXxxForMaskedLM, self).__init__(config, *inputs, **kwargs) + + self.transformer = TFXxxMainLayer(config, name='transformer') + self.mlm = TFXxxMLMHead(config, self.transformer.embeddings, name='mlm') + + def call(self, inputs, **kwargs): + outputs = self.transformer(inputs, **kwargs) + + sequence_output = outputs[0] + prediction_scores = self.mlm(sequence_output, training=kwargs.get('training', False)) + + outputs = (prediction_scores,) + outputs[2:] # Add hidden states and attention if they are here + + return outputs # prediction_scores, (hidden_states), (attentions) + + +@add_start_docstrings("""Xxx Model transformer with a sequence classification/regression head on top (a linear layer on top of + the pooled output) e.g. for GLUE tasks. """, + XXX_START_DOCSTRING, XXX_INPUTS_DOCSTRING) +class TFXxxForSequenceClassification(TFXxxPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **logits**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, config.num_labels)`` + Classification (or regression if config.num_labels==1) scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``Numpy array`` or ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``Numpy array`` or ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + import tensorflow as tf + from transformers import XxxTokenizer, TFXxxForSequenceClassification + + tokenizer = XxxTokenizer.from_pretrained('xxx-base-uncased') + model = TFXxxForSequenceClassification.from_pretrained('xxx-base-uncased') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + outputs = model(input_ids) + logits = outputs[0] + + """ + def __init__(self, config, *inputs, **kwargs): + super(TFXxxForSequenceClassification, self).__init__(config, *inputs, **kwargs) + self.num_labels = config.num_labels + + self.transformer = TFXxxMainLayer(config, name='transformer') + self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + self.classifier = tf.keras.layers.Dense(config.num_labels, + kernel_initializer=get_initializer(config.initializer_range), + name='classifier') + + def call(self, inputs, **kwargs): + outputs = self.transformer(inputs, **kwargs) + + pooled_output = outputs[1] + + pooled_output = self.dropout(pooled_output, training=kwargs.get('training', False)) + logits = self.classifier(pooled_output) + + outputs = (logits,) + outputs[2:] # add hidden states and attention if they are here + + return outputs # logits, (hidden_states), (attentions) + + +@add_start_docstrings("""Xxx Model with a token classification head on top (a linear layer on top of + the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. """, + XXX_START_DOCSTRING, XXX_INPUTS_DOCSTRING) +class TFXxxForTokenClassification(TFXxxPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **scores**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, config.num_labels)`` + Classification scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``Numpy array`` or ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``Numpy array`` or ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + import tensorflow as tf + from transformers import XxxTokenizer, TFXxxForTokenClassification + + tokenizer = XxxTokenizer.from_pretrained('xxx-base-uncased') + model = TFXxxForTokenClassification.from_pretrained('xxx-base-uncased') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + outputs = model(input_ids) + scores = outputs[0] + + """ + def __init__(self, config, *inputs, **kwargs): + super(TFXxxForTokenClassification, self).__init__(config, *inputs, **kwargs) + self.num_labels = config.num_labels + + self.transformer = TFXxxMainLayer(config, name='transformer') + self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + self.classifier = tf.keras.layers.Dense(config.num_labels, + kernel_initializer=get_initializer(config.initializer_range), + name='classifier') + + def call(self, inputs, **kwargs): + outputs = self.transformer(inputs, **kwargs) + + sequence_output = outputs[0] + + sequence_output = self.dropout(sequence_output, training=kwargs.get('training', False)) + logits = self.classifier(sequence_output) + + outputs = (logits,) + outputs[2:] # add hidden states and attention if they are here + + return outputs # scores, (hidden_states), (attentions) + + +@add_start_docstrings("""Xxx Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of + the hidden-states output to compute `span start logits` and `span end logits`). """, + XXX_START_DOCSTRING, XXX_INPUTS_DOCSTRING) +class TFXxxForQuestionAnswering(TFXxxPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **start_scores**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length,)`` + Span-start scores (before SoftMax). + **end_scores**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length,)`` + Span-end scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``Numpy array`` or ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``Numpy array`` or ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + import tensorflow as tf + from transformers import XxxTokenizer, TFXxxForQuestionAnswering + + tokenizer = XxxTokenizer.from_pretrained('xxx-base-uncased') + model = TFXxxForQuestionAnswering.from_pretrained('xxx-base-uncased') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + outputs = model(input_ids) + start_scores, end_scores = outputs[:2] + + """ + def __init__(self, config, *inputs, **kwargs): + super(TFXxxForQuestionAnswering, self).__init__(config, *inputs, **kwargs) + self.num_labels = config.num_labels + + self.transformer = TFXxxMainLayer(config, name='transformer') + self.qa_outputs = tf.keras.layers.Dense(config.num_labels, + kernel_initializer=get_initializer(config.initializer_range), + name='qa_outputs') + + def call(self, inputs, **kwargs): + outputs = self.transformer(inputs, **kwargs) + + sequence_output = outputs[0] + + logits = self.qa_outputs(sequence_output) + start_logits, end_logits = tf.split(logits, 2, axis=-1) + start_logits = tf.squeeze(start_logits, axis=-1) + end_logits = tf.squeeze(end_logits, axis=-1) + + outputs = (start_logits, end_logits,) + outputs[2:] + + return outputs # start_logits, end_logits, (hidden_states), (attentions) diff --git a/transformers/tokenization_t5.py b/transformers/tokenization_t5.py new file mode 100644 index 0000000000..3f8f4bf556 --- /dev/null +++ b/transformers/tokenization_t5.py @@ -0,0 +1,214 @@ +# coding=utf-8 +# Copyright 2018 T5 Authors and HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Tokenization class for model T5.""" + +from __future__ import absolute_import, division, print_function, unicode_literals + +import collections +import logging +import os +import unicodedata +from io import open + +from .tokenization_utils import PreTrainedTokenizer + +logger = logging.getLogger(__name__) + +#################################################### +# Mapping from the keyword arguments names of Tokenizer `__init__` +# to file names for serializing Tokenizer instances +#################################################### +VOCAB_FILES_NAMES = {'vocab_file': 'vocab.txt'} + +#################################################### +# Mapping from the keyword arguments names of Tokenizer `__init__` +# to pretrained vocabulary URL for all the model shortcut names. +#################################################### +PRETRAINED_VOCAB_FILES_MAP = { + 'vocab_file': + { + 't5-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-base-uncased-vocab.txt", + 't5-large-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-large-uncased-vocab.txt", + } +} + +#################################################### +# Mapping from model shortcut names to max length of inputs +#################################################### +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { + 't5-base-uncased': 512, + 't5-large-uncased': 512, +} + +#################################################### +# Mapping from model shortcut names to a dictionary of additional +# keyword arguments for Tokenizer `__init__`. +# To be used for checkpoint specific configurations. +#################################################### +PRETRAINED_INIT_CONFIGURATION = { + 't5-base-uncased': {'do_lower_case': True}, + 't5-large-uncased': {'do_lower_case': True}, +} + + +def load_vocab(vocab_file): + """Loads a vocabulary file into a dictionary.""" + vocab = collections.OrderedDict() + with open(vocab_file, "r", encoding="utf-8") as reader: + tokens = reader.readlines() + for index, token in enumerate(tokens): + token = token.rstrip('\n') + vocab[token] = index + return vocab + + +class T5Tokenizer(PreTrainedTokenizer): + r""" + Constructs a T5Tokenizer. + :class:`~transformers.T5Tokenizer` runs end-to-end tokenization: punctuation splitting + wordpiece + + Args: + vocab_file: Path to a one-wordpiece-per-line vocabulary file + do_lower_case: Whether to lower case the input. Only has an effect when do_wordpiece_only=False + """ + + vocab_files_names = VOCAB_FILES_NAMES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + + def __init__(self, vocab_file, do_lower_case=True, + unk_token="[UNK]", sep_token="[SEP]", pad_token="[PAD]", cls_token="[CLS]", + mask_token="[MASK]", **kwargs): + """Constructs a T5Tokenizer. + + Args: + **vocab_file**: Path to a one-wordpiece-per-line vocabulary file + **do_lower_case**: (`optional`) boolean (default True) + Whether to lower case the input + Only has an effect when do_basic_tokenize=True + """ + super(T5Tokenizer, self).__init__(unk_token=unk_token, sep_token=sep_token, + pad_token=pad_token, cls_token=cls_token, + mask_token=mask_token, **kwargs) + self.max_len_single_sentence = self.max_len - 2 # take into account special tokens + self.max_len_sentences_pair = self.max_len - 3 # take into account special tokens + + if not os.path.isfile(vocab_file): + raise ValueError( + "Can't find a vocabulary file at path '{}'. To load the vocabulary from a Google pretrained " + "model use `tokenizer = T5Tokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`".format(vocab_file)) + self.vocab = load_vocab(vocab_file) + + @property + def vocab_size(self): + return len(self.vocab) + + def _tokenize(self, text): + """ Take as input a string and return a list of strings (tokens) for words/sub-words + """ + split_tokens = [] + if self.do_basic_tokenize: + for token in self.basic_tokenizer.tokenize(text, never_split=self.all_special_tokens): + for sub_token in self.wordpiece_tokenizer.tokenize(token): + split_tokens.append(sub_token) + else: + split_tokens = self.wordpiece_tokenizer.tokenize(text) + return split_tokens + + def _convert_token_to_id(self, token): + """ Converts a token (str/unicode) in an id using the vocab. """ + return self.vocab.get(token, self.vocab.get(self.unk_token)) + + def _convert_id_to_token(self, index): + """Converts an index (integer) in a token (string/unicode) using the vocab.""" + return self.ids_to_tokens.get(index, self.unk_token) + + def convert_tokens_to_string(self, tokens): + """ Converts a sequence of tokens (string) in a single string. """ + out_string = ' '.join(tokens).replace(' ##', '').strip() + return out_string + + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): + """ + Build model inputs from a sequence or a pair of sequence for sequence classification tasks + by concatenating and adding special tokens. + A BERT sequence has the following format: + single sequence: [CLS] X [SEP] + pair of sequences: [CLS] A [SEP] B [SEP] + """ + if token_ids_1 is None: + return [self.cls_token_id] + token_ids_0 + [self.sep_token_id] + cls = [self.cls_token_id] + sep = [self.sep_token_id] + return cls + token_ids_0 + sep + token_ids_1 + sep + + def get_special_tokens_mask(self, token_ids_0, token_ids_1=None, already_has_special_tokens=False): + """ + Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding + special tokens using the tokenizer ``prepare_for_model`` or ``encode_plus`` methods. + + Args: + token_ids_0: list of ids (must not contain special tokens) + token_ids_1: Optional list of ids (must not contain special tokens), necessary when fetching sequence ids + for sequence pairs + already_has_special_tokens: (default False) Set to True if the token list is already formated with + special tokens for the model + + Returns: + A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. + """ + + if already_has_special_tokens: + if token_ids_1 is not None: + raise ValueError("You should not supply a second sequence if the provided sequence of " + "ids is already formated with special tokens for the model.") + return list(map(lambda x: 1 if x in [self.sep_token_id, self.cls_token_id] else 0, token_ids_0)) + + if token_ids_1 is not None: + return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1] + return [1] + ([0] * len(token_ids_0)) + [1] + + def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None): + """ + Creates a mask from the two sequences passed to be used in a sequence-pair classification task. + A BERT sequence pair mask has the following format: + 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 + | first sequence | second sequence + + if token_ids_1 is None, only returns the first portion of the mask (0's). + """ + sep = [self.sep_token_id] + cls = [self.cls_token_id] + if token_ids_1 is None: + return len(cls + token_ids_0 + sep) * [0] + return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1] + + def save_vocabulary(self, vocab_path): + """Save the tokenizer vocabulary to a directory or file.""" + index = 0 + if os.path.isdir(vocab_path): + vocab_file = os.path.join(vocab_path, VOCAB_FILES_NAMES['vocab_file']) + else: + vocab_file = vocab_path + with open(vocab_file, "w", encoding="utf-8") as writer: + for token, token_index in sorted(self.vocab.items(), key=lambda kv: kv[1]): + if index != token_index: + logger.warning("Saving vocabulary to {}: vocabulary indices are not consecutive." + " Please check that the vocabulary is not corrupted!".format(vocab_file)) + index = token_index + writer.write(token + u'\n') + index += 1 + return (vocab_file,) From 568c0ffb7ef73555567f8bd467cf80c2b1e6ac13 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 5 Nov 2019 16:40:29 +0100 Subject: [PATCH 011/505] adding T5 model --- transformers/modeling_encoder_decoder.py | 4 +- transformers/modeling_t5.py | 471 ++++++++++++++++++++--- 2 files changed, 412 insertions(+), 63 deletions(-) diff --git a/transformers/modeling_encoder_decoder.py b/transformers/modeling_encoder_decoder.py index a884abd0a2..713cf5252e 100644 --- a/transformers/modeling_encoder_decoder.py +++ b/transformers/modeling_encoder_decoder.py @@ -217,9 +217,7 @@ class PreTrainedEncoderDecoder(nn.Module): encoder_hidden_states = kwargs_encoder.pop("hidden_states", None) if encoder_hidden_states is None: encoder_outputs = self.encoder(encoder_input_ids, **kwargs_encoder) - encoder_hidden_states = encoder_outputs[ - 0 - ] # output the last layer hidden state + encoder_hidden_states = encoder_outputs[0] else: encoder_outputs = () diff --git a/transformers/modeling_t5.py b/transformers/modeling_t5.py index fa3c22f24b..d93e96211d 100644 --- a/transformers/modeling_t5.py +++ b/transformers/modeling_t5.py @@ -1,5 +1,5 @@ # coding=utf-8 -# Copyright 2018 T5 Authors and HuggingFace Inc. team. +# Copyright 2018 Mesh TensorFlow authors, T5 Authors and HuggingFace Inc. team. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,11 +20,14 @@ import json import logging import math import os +import math import sys +import itertools from io import open import torch from torch import nn +import torch.nn.functional as F from torch.nn import CrossEntropyLoss, MSELoss from .modeling_utils import PreTrainedModel, prune_linear_layer @@ -119,31 +122,389 @@ def load_tf_weights_in_t5(model, config, tf_checkpoint_path): # - PreTrainedModel for the models (it-self a sub-class of torch.nn.Module) #################################################### -class T5Layer(nn.Module): +class T5DenseReluDense(nn.Module): def __init__(self, config): - super(T5Layer, self).__init__() - self.attention = T5Attention(config) - self.intermediate = T5Intermediate(config) - self.output = T5Output(config) + super(T5DenseReluDense, self).__init__() + self.wi = nn.Linear(config.d_model, config.d_ff, bias=False) + self.wo = nn.Linear(config.d_ff, config.d_model, bias=False) + self.dropout = nn.Dropout(config.dropout) - def forward(self, hidden_states, attention_mask=None, head_mask=None): - attention_outputs = self.attention(hidden_states, attention_mask, head_mask) - attention_output = attention_outputs[0] - intermediate_output = self.intermediate(attention_output) - layer_output = self.output(intermediate_output, attention_output) - outputs = (layer_output,) + attention_outputs[1:] # add attentions if we output them + def forward(self, hidden_states): + h = self.wi(hidden_states) + h = F.relu(h) + h = self.dropout(h) + h = self.wo(h) + return h + + +class T5LayerFF(nn.Module): + def __init__(self, config): + super(T5LayerFF, self).__init__() + self.DenseReluDense = T5DenseReluDense(config) + self.layer_norm = nn.LayerNorm(config.layer_norm_epsilon) + self.dropout = nn.Dropout(config.dropout) + + def forward(self, hidden_states): + norm_x = self.layer_norm(hidden_states) + y = self.DenseReluDense(norm_x) + layer_output = hidden_states + self.dropout(y) + return layer_output + + +class T5Attention(nn.Module): + NEW_ID = itertools.count() + + def __init__(self, config): + super(T5Attention, self).__init__() + self.layer_id = next(T5Attention.NEW_ID) + + self.output_attentions = config.output_attentions + self.relative_attention_num_buckets = config.relative_attention_num_buckets + self.dim = config.d_model + self.n_heads = config.num_heads + self.dropout = config.dropout_rate + assert self.dim % self.n_heads == 0 + + self.q = nn.Linear(self.dim, self.dim, bias=False) + self.k = nn.Linear(self.dim, self.dim, bias=False) + self.v = nn.Linear(self.dim, self.dim, bias=False) + self.o = nn.Linear(self.dim, self.dim, bias=False) + + self.relative_attention_bias = nn.Embedding(self.relative_attention_num_buckets, self.n_heads) + self.pruned_heads = set() + + def prune_heads(self, heads): + attention_head_size = self.dim // self.n_heads + if len(heads) == 0: + return + mask = torch.ones(self.n_heads, attention_head_size) + heads = set(heads) - self.pruned_heads + for head in heads: + head -= sum(1 if h < head else 0 for h in self.pruned_heads) + mask[head] = 0 + mask = mask.view(-1).contiguous().eq(1) + index = torch.arange(len(mask))[mask].long() + # Prune linear layers + self.q = prune_linear_layer(self.q, index) + self.k = prune_linear_layer(self.k, index) + self.v = prune_linear_layer(self.v, index) + self.o = prune_linear_layer(self.o, index, dim=1) + # Update hyper params + self.n_heads = self.n_heads - len(heads) + self.dim = attention_head_size * self.n_heads + self.pruned_heads = self.pruned_heads.union(heads) + + @staticmethod + def _relative_position_bucket(relative_position, + bidirectional=True, + num_buckets=32, + max_distance=128): + """ + Adapted from Mesh Tensorflow: + https://github.com/tensorflow/mesh/blob/0cb87fe07da627bf0b7e60475d59f95ed6b5be3d/mesh_tensorflow/transformer/transformer_layers.py#L593 + + Translate relative position to a bucket number for relative attention. + The relative position is defined as memory_position - query_position, i.e. + the distance in tokens from the attending position to the attended-to + position. If bidirectional=False, then positive relative positions are + invalid. + We use smaller buckets for small absolute relative_position and larger buckets + for larger absolute relative_positions. All relative positions >=max_distance + map to the same bucket. All relative positions <=-max_distance map to the + same bucket. This should allow for more graceful generalization to longer + sequences than the model has been trained on. + Args: + relative_position: an int32 Tensor + bidirectional: a boolean - whether the attention is bidirectional + num_buckets: an integer + max_distance: an integer + Returns: + a Tensor with the same shape as relative_position, containing int32 + values in the range [0, num_buckets) + """ + ret = 0 + n = -relative_position + if bidirectional: + num_buckets //= 2 + ret += (n < 0).to(torch.long) * num_buckets # mtf.to_int32(mtf.less(n, 0)) * num_buckets + n = torch.abs(n) + else: + n = torch.max(n, 0) + # now n is in the range [0, inf) + + # half of the buckets are for exact increments in positions + max_exact = num_buckets // 2 + is_small = (n < max_exact) + + # The other half of the buckets are for logarithmically bigger bins in positions up to max_distance + val_if_large = max_exact + ( + torch.log(n.float() / max_exact) + / math.log(max_distance / max_exact) * (num_buckets - max_exact)).to(torch.long) + val_if_large = torch.min(val_if_large, num_buckets - 1) + + ret += torch.where(is_small, n, val_if_large) + return ret + + def compute_bias(self, qlen, klen): + """ Compute binned relative position bias """ + context_position = torch.arange(qlen, dtype=torch.long)[:, None] + memory_position = torch.arange(klen, dtype=torch.long)[None, :] + relative_position = memory_position - context_position # shape (qlen, klen) + rp_bucket = self._relative_position_bucket(relative_position, + bidirectional=not self.is_decoder, + num_buckets=self.relative_attention_num_buckets) + values = self.relative_attention_bias(rp_bucket) # shape (qlen, klen, num_heads) + values = values.permute([2, 0, 1]).unsqueeze(0) # shape (1, num_heads, qlen, klen) + return values + + def forward(self, input, mask, kv=None, position_bias=None, cache=None, head_mask=None): + """ + Self-attention (if kv is None) or attention over source sentence (provided by kv). + """ + # Input is (bs, qlen, dim) + # Mask is (bs, klen) (non-causal) or (bs, klen, klen) + bs, qlen, dim = input.size() + if kv is None: + klen = qlen if cache is None else cache['slen'] + qlen + else: + klen = kv.size(1) + # assert dim == self.dim, 'Dimensions do not match: %s input vs %s configured' % (dim, self.dim) + n_heads = self.n_heads + dim_per_head = self.dim // n_heads + mask_reshape = (bs, 1, qlen, klen) if mask.dim() == 3 else (bs, 1, 1, klen) + + def shape(x): + """ projection """ + return x.view(bs, -1, self.n_heads, dim_per_head).transpose(1, 2) + + def unshape(x): + """ compute context """ + return x.transpose(1, 2).contiguous().view(bs, -1, self.n_heads * dim_per_head) + + q = shape(self.q(input)) # (bs, n_heads, qlen, dim_per_head) + if kv is None: + k = shape(self.k(input)) # (bs, n_heads, qlen, dim_per_head) + v = shape(self.v(input)) # (bs, n_heads, qlen, dim_per_head) + elif cache is None or self.layer_id not in cache: + k = v = kv + k = shape(self.k(k)) # (bs, n_heads, qlen, dim_per_head) + v = shape(self.v(v)) # (bs, n_heads, qlen, dim_per_head) + + if cache is not None: + if self.layer_id in cache: + if kv is None: + k_, v_ = cache[self.layer_id] + k = torch.cat([k_, k], dim=2) # (bs, n_heads, klen, dim_per_head) + v = torch.cat([v_, v], dim=2) # (bs, n_heads, klen, dim_per_head) + else: + k, v = cache[self.layer_id] + cache[self.layer_id] = (k, v) + + # q = q / math.sqrt(dim_per_head) # No scaling in T5 + scores = torch.matmul(q, k.transpose(2, 3)) # (bs, n_heads, qlen, klen) + + if position_bias is None: + position_bias = self.compute_bias(qlen, klen) + scores += position_bias + + mask = (mask == 0).view(mask_reshape).expand_as(scores) # (bs, n_heads, qlen, klen) + scores.masked_fill_(mask, -float('inf')) # (bs, n_heads, qlen, klen) + + weights = F.softmax(scores.float(), dim=-1).type_as(scores) # (bs, n_heads, qlen, klen) + weights = F.dropout(weights, p=self.dropout, training=self.training) # (bs, n_heads, qlen, klen) + + # Mask heads if we want to + if head_mask is not None: + weights = weights * head_mask + + context = torch.matmul(weights, v) # (bs, n_heads, qlen, dim_per_head) + context = unshape(context) # (bs, qlen, dim) + + context = self.o(context) + + outputs = (context,) + if self.output_attentions: + outputs = outputs + (weights,) return outputs +class T5LayerSelfAttention(nn.Module): + def __init__(self, config): + super(T5LayerSelfAttention, self).__init__() + self.SelfAttention = T5Attention(config) + self.layer_norm = nn.LayerNorm(config.layer_norm_epsilon) + self.dropout = nn.Dropout(config.dropout) -class T5PreTrainedModel(PreTrainedModel): + def forward(self, hidden_states, attention_mask=None, head_mask=None): + norm_x = self.layer_norm(hidden_states) + attention_output = self.SelfAttention(norm_x, + attention_mask=attention_mask, + head_mask=head_mask) + y = attention_output[0] + layer_output = hidden_states + self.dropout(y) + outputs = (layer_output,) + attention_output[1:] # add attentions if we output them + return outputs + + +class T5LayerCrossAttention(nn.Module): + def __init__(self, config): + super(T5LayerCrossAttention, self).__init__() + self.EncDecAttention = T5Attention(config) + self.layer_norm = nn.LayerNorm(config.layer_norm_epsilon) + self.dropout = nn.Dropout(config.dropout) + + def forward(self, hidden_states, kv, attention_mask=None, head_mask=None): + norm_x = self.layer_norm(hidden_states) + attention_output = self.EncDecAttention(norm_x, + kv=kv, + attention_mask=attention_mask, + head_mask=head_mask) + y = attention_output[0] + layer_output = hidden_states + self.dropout(y) + outputs = (layer_output,) + attention_output[1:] # add attentions if we output them + return outputs + + +class T5Block(nn.Module): + def __init__(self, config): + super(T5Block, self).__init__() + self.is_decoder = config.is_decoder + self.layer_000 = T5LayerSelfAttention(config) + if self.is_decoder: + self.layer_001 = T5LayerCrossAttention(config) + self.layer_002 = T5LayerFF(config) + else: + self.layer_001 = T5LayerFF(config) + + def forward(self, hidden_states, attention_mask=None, + encoder_hidden_states=None, encoder_attention_mask=None, head_mask=None): + self_attention_outputs = self.layer_000(hidden_states, + attention_mask=attention_mask, + head_mask=head_mask) + hidden_states = self_attention_outputs[0] + outputs = self_attention_outputs[1:] + + if self.is_decoder: + cross_attention_outputs = self.layer_001(hidden_states, + kv=encoder_hidden_states, + attention_mask=encoder_attention_mask, + head_mask=head_mask) + hidden_states = cross_attention_outputs[0] + outputs = cross_attention_outputs[1:] + outputs + hidden_states = self.layer_002(hidden_states) + else: + hidden_states = self.layer_001(hidden_states) + + outputs = (hidden_states,) + outputs # add attentions if we output them + return outputs + + +class T5Stack(nn.Module): + def __init__(self, config): + super(T5Stack, self).__init__() + self.blocks = nn.ModuleList([T5Block(config) for _ in range(config.num_layers)]) + self.final_layer_norm = nn.LayerNorm(config.layer_norm_epsilon) + self.dropout = nn.Dropout(config.dropout) + + def forward(self, + hidden_states, + attention_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + head_mask=None): + + if attention_mask is None: + attention_mask = torch.ones_like(input_ids) + + # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] + # ourselves in which case we just need to make it broadcastable to all heads. + if attention_mask.dim() == 3: + extended_attention_mask = attention_mask[:, None, :, :] + + # Provided a padding mask of dimensions [batch_size, seq_length] + # - if the model is a decoder, apply a causal mask in addition to the padding mask + # - if the model is an encoder, make the mask broadcastable to [batch_size, num_heads, seq_length, seq_length] + if attention_mask.dim() == 2: + if self.config.is_decoder: + batch_size, seq_length = input_ids.size() + seq_ids = torch.arange(seq_length, device=input_ids.device) + causal_mask = seq_ids[None, None, :].repeat(batch_size, seq_length, 1) <= seq_ids[None, :, None] + extended_attention_mask = causal_mask[:, None, :, :] * attention_mask[:, None, None, :] + else: + extended_attention_mask = attention_mask[:, None, None, :] + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and -10000.0 for masked positions. + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility + extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + + # If a 2D ou 3D attention mask is provided for the cross-attention + # we need to make broadcastabe to [batch_size, num_heads, seq_length, seq_length] + if encoder_attention_mask.dim() == 3: + encoder_extended_attention_mask = encoder_attention_mask[:, None, :, :] + if encoder_attention_mask.dim() == 2: + encoder_extended_attention_mask = encoder_attention_mask[:, None, None, :] + + encoder_extended_attention_mask = encoder_extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility + encoder_extended_attention_mask = (1.0 - encoder_extended_attention_mask) * -10000.0 + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + if head_mask is not None: + if head_mask.dim() == 1: + head_mask = head_mask.unsqueeze(0).unsqueeze(0).unsqueeze(-1).unsqueeze(-1) + head_mask = head_mask.expand(self.config.num_hidden_layers, -1, -1, -1, -1) + elif head_mask.dim() == 2: + head_mask = head_mask.unsqueeze(1).unsqueeze(-1).unsqueeze(-1) # We can specify head_mask for each layer + head_mask = head_mask.to(dtype=next(self.parameters()).dtype) # switch to fload if need + fp16 compatibility + else: + head_mask = [None] * self.config.num_hidden_layers + + all_hidden_states = () + all_attentions = () + position_bias = None + for i, layer_module in enumerate(self.layer): + if self.output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + layer_outputs = layer_module(hidden_states, + attention_mask=extended_attention_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_extended_attention_mask, + head_mask=head_mask[i]) + hidden_states = layer_outputs[0] + + if self.output_attentions: + all_attentions = all_attentions + (layer_outputs[1],) + + hidden_states = self.final_layer_norm(hidden_states) + layer_output = self.dropout(hidden_states) + + # Add last layer + if self.output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + outputs = (hidden_states,) + if self.output_hidden_states: + outputs = outputs + (all_hidden_states,) + if self.output_attentions: + outputs = outputs + (all_attentions,) + return outputs # last-layer hidden state, (all hidden states), (all attentions) + + +class T5PreTrainedModel(PreTrainedEncoderDecoder): """ An abstract class to handle weights initialization and a simple interface for dowloading and loading pretrained models. """ config_class = T5Config pretrained_model_archive_map = T5_PRETRAINED_MODEL_ARCHIVE_MAP load_tf_weights = load_tf_weights_in_t5 - base_model_prefix = "transformer" def _init_weights(self, module): """ Initialize the weights """ @@ -238,19 +599,23 @@ class T5Model(T5PreTrainedModel): """ def __init__(self, config): super(T5Model, self).__init__(config) + self.shared = nn.Embeddings(config.vocab_size, config.d_model) - self.embeddings = T5Embeddings(config) - self.encoder = T5Encoder(config) - self.pooler = T5Pooler(config) + encoder_config = copy.deepcopy(config) + self.encoder = T5Stack(encoder_config) + + decoder_config = copy.deepcopy(config) + decoder_config.is_decoder = True + self.decoder = T5Stack(decoder_config) self.init_weights() @property def get_input_embeddings(self): - return self.embeddings.word_embeddings + return self.shared def set_input_embeddings(self, new_embeddings): - self.embeddings.word_embeddings = new_embeddings + self.shared = new_embeddings def _prune_heads(self, heads_to_prune): """ Prunes heads of the model. @@ -260,50 +625,36 @@ class T5Model(T5PreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None): - if attention_mask is None: - attention_mask = torch.ones_like(input_ids) - if token_type_ids is None: - token_type_ids = torch.zeros_like(input_ids) + def forward(self, encoder_input_ids, decoder_input_ids, **kwargs): + # keyword arguments come in 3 flavors: encoder-specific (prefixed by + # `encoder_`), decoder-specific (prefixed by `decoder_`) and those + # that apply to the model as whole. + # We let the specific kwargs override the common ones in case of conflict. + kwargs_common = dict((k, v) for k, v in kwargs.items() + if not k.startswith("encoder_") and not k.startswith("decoder_")) + kwargs_decoder = kwargs_common.copy() + kwargs_encoder = kwargs_common.copy() + kwargs_encoder.update(dict((k[len("encoder_") :], v) for k, v in kwargs.items() if k.startswith("encoder_"))) + kwargs_decoder.update(dict((k[len("decoder_") :], v) for k, v in kwargs.items() if k.startswith("decoder_"))) - # We create a 3D attention mask from a 2D tensor mask. - # Sizes are [batch_size, 1, 1, to_seq_length] - # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length] - # this attention mask is more simple than the triangular masking of causal attention - # used in OpenAI GPT, we just need to prepare the broadcast dimension here. - extended_attention_mask = attention_mask.unsqueeze(1).unsqueeze(2) - - # Since attention_mask is 1.0 for positions we want to attend and 0.0 for - # masked positions, this operation will create a tensor which is 0.0 for - # positions we want to attend and -10000.0 for masked positions. - # Since we are adding it to the raw scores before the softmax, this is - # effectively the same as removing these entirely. - extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility - extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 - - # Prepare head mask if needed - # 1.0 in head_mask indicate we keep the head - # attention_probs has shape bsz x n_heads x N x N - # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] - # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] - if head_mask is not None: - if head_mask.dim() == 1: - head_mask = head_mask.unsqueeze(0).unsqueeze(0).unsqueeze(-1).unsqueeze(-1) - head_mask = head_mask.expand(self.config.num_hidden_layers, -1, -1, -1, -1) - elif head_mask.dim() == 2: - head_mask = head_mask.unsqueeze(1).unsqueeze(-1).unsqueeze(-1) # We can specify head_mask for each layer - head_mask = head_mask.to(dtype=next(self.parameters()).dtype) # switch to fload if need + fp16 compatibility + # Encode if needed (training, first prediction pass) + encoder_hidden_states = kwargs_encoder.pop("hidden_states", None) + if encoder_hidden_states is None: + encoder_inputs_ids = kwargs_encoder.pop("input_ids") + hidden_states = self.shared(encoder_inputs_ids) # Convert inputs in embeddings + encoder_outputs = self.encoder(hidden_states, **kwargs_encoder) + encoder_hidden_states = encoder_outputs[0] else: - head_mask = [None] * self.config.num_hidden_layers + encoder_outputs = () - ################################## - # Replace this with your model code - embedding_output = self.embeddings(input_ids, position_ids=position_ids, token_type_ids=token_type_ids) - encoder_outputs = self.encoder(embedding_output, extended_attention_mask, head_mask=head_mask) - sequence_output = encoder_outputs[0] - outputs = (sequence_output,) + encoder_outputs[1:] # add hidden_states and attentions if they are here + # Decode + decoder_inputs_ids = kwargs_decoder.pop("input_ids") + hidden_states = self.shared(decoder_inputs_ids) # Convert inputs in embeddings + kwargs_decoder["encoder_hidden_states"] = encoder_hidden_states + kwargs_decoder["encoder_attention_mask"] = kwargs_encoder.get("attention_mask", None) + decoder_outputs = self.decoder(hidden_states, **kwargs_decoder) - return outputs # sequence_output, (hidden_states), (attentions) + return decoder_outputs + encoder_outputs @add_start_docstrings("""T5 Model with a `language modeling` head on top. """, @@ -342,7 +693,7 @@ class T5WithLMHead(T5PreTrainedModel): super(T5ForMaskedLM, self).__init__(config) self.transformer = T5Model(config) - self.lm_head = nn.Linear(config.n_embd, config.vocab_size) + self.lm_head = nn.Linear(config.d_model, config.vocab_size) self.init_weights() From 88e5bef58f34dca87f28ab489fdecbeaaef8b316 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 5 Nov 2019 17:02:52 +0100 Subject: [PATCH 012/505] share position biases --- transformers/modeling_t5.py | 65 +++++++++++++++++++++++-------------- 1 file changed, 40 insertions(+), 25 deletions(-) diff --git a/transformers/modeling_t5.py b/transformers/modeling_t5.py index d93e96211d..e1a1d019ff 100644 --- a/transformers/modeling_t5.py +++ b/transformers/modeling_t5.py @@ -154,9 +154,10 @@ class T5LayerFF(nn.Module): class T5Attention(nn.Module): NEW_ID = itertools.count() - def __init__(self, config): + def __init__(self, config, has_relative_attention_bias=False): super(T5Attention, self).__init__() self.layer_id = next(T5Attention.NEW_ID) + self.has_relative_attention_bias = has_relative_attention_bias self.output_attentions = config.output_attentions self.relative_attention_num_buckets = config.relative_attention_num_buckets @@ -170,7 +171,8 @@ class T5Attention(nn.Module): self.v = nn.Linear(self.dim, self.dim, bias=False) self.o = nn.Linear(self.dim, self.dim, bias=False) - self.relative_attention_bias = nn.Embedding(self.relative_attention_num_buckets, self.n_heads) + if self.has_relative_attention_bias: + self.relative_attention_bias = nn.Embedding(self.relative_attention_num_buckets, self.n_heads) self.pruned_heads = set() def prune_heads(self, heads): @@ -304,6 +306,8 @@ class T5Attention(nn.Module): scores = torch.matmul(q, k.transpose(2, 3)) # (bs, n_heads, qlen, klen) if position_bias is None: + if not self.has_relative_attention_bias: + raise ValueError("No position_bias provided and no weights to compute position_bias") position_bias = self.compute_bias(qlen, klen) scores += position_bias @@ -325,20 +329,23 @@ class T5Attention(nn.Module): outputs = (context,) if self.output_attentions: outputs = outputs + (weights,) + if self.has_relative_attention_bias: + outputs = outputs + (position_bias,) return outputs class T5LayerSelfAttention(nn.Module): - def __init__(self, config): + def __init__(self, config, has_relative_attention_bias=False): super(T5LayerSelfAttention, self).__init__() - self.SelfAttention = T5Attention(config) + self.SelfAttention = T5Attention(config, has_relative_attention_bias=has_relative_attention_bias) self.layer_norm = nn.LayerNorm(config.layer_norm_epsilon) self.dropout = nn.Dropout(config.dropout) - def forward(self, hidden_states, attention_mask=None, head_mask=None): + def forward(self, hidden_states, attention_mask=None, position_bias=None, head_mask=None): norm_x = self.layer_norm(hidden_states) attention_output = self.SelfAttention(norm_x, attention_mask=attention_mask, + position_bias=position_bias, head_mask=head_mask) y = attention_output[0] layer_output = hidden_states + self.dropout(y) @@ -347,17 +354,18 @@ class T5LayerSelfAttention(nn.Module): class T5LayerCrossAttention(nn.Module): - def __init__(self, config): + def __init__(self, config, has_relative_attention_bias=False): super(T5LayerCrossAttention, self).__init__() - self.EncDecAttention = T5Attention(config) + self.EncDecAttention = T5Attention(config, has_relative_attention_bias=has_relative_attention_bias) self.layer_norm = nn.LayerNorm(config.layer_norm_epsilon) self.dropout = nn.Dropout(config.dropout) - def forward(self, hidden_states, kv, attention_mask=None, head_mask=None): + def forward(self, hidden_states, kv, attention_mask=None, position_bias=None, head_mask=None): norm_x = self.layer_norm(hidden_states) attention_output = self.EncDecAttention(norm_x, kv=kv, attention_mask=attention_mask, + position_bias=position_bias, head_mask=head_mask) y = attention_output[0] layer_output = hidden_states + self.dropout(y) @@ -366,20 +374,22 @@ class T5LayerCrossAttention(nn.Module): class T5Block(nn.Module): - def __init__(self, config): + def __init__(self, config, has_relative_attention_bias=False): super(T5Block, self).__init__() self.is_decoder = config.is_decoder - self.layer_000 = T5LayerSelfAttention(config) + self.layer_000 = T5LayerSelfAttention(config, has_relative_attention_bias=has_relative_attention_bias) if self.is_decoder: - self.layer_001 = T5LayerCrossAttention(config) + self.layer_001 = T5LayerCrossAttention(config, has_relative_attention_bias=has_relative_attention_bias) self.layer_002 = T5LayerFF(config) else: self.layer_001 = T5LayerFF(config) - def forward(self, hidden_states, attention_mask=None, - encoder_hidden_states=None, encoder_attention_mask=None, head_mask=None): + def forward(self, hidden_states, attention_mask=None, position_bias=None, + encoder_hidden_states=None, encoder_attention_mask=None, encoder_decoder_position_bias=None, + head_mask=None): self_attention_outputs = self.layer_000(hidden_states, attention_mask=attention_mask, + position_bias=position_bias, head_mask=head_mask) hidden_states = self_attention_outputs[0] outputs = self_attention_outputs[1:] @@ -388,6 +398,7 @@ class T5Block(nn.Module): cross_attention_outputs = self.layer_001(hidden_states, kv=encoder_hidden_states, attention_mask=encoder_attention_mask, + position_bias=encoder_decoder_position_bias, head_mask=head_mask) hidden_states = cross_attention_outputs[0] outputs = cross_attention_outputs[1:] + outputs @@ -402,7 +413,8 @@ class T5Block(nn.Module): class T5Stack(nn.Module): def __init__(self, config): super(T5Stack, self).__init__() - self.blocks = nn.ModuleList([T5Block(config) for _ in range(config.num_layers)]) + self.blocks = nn.ModuleList([T5Block(config, has_relative_attention_bias=bool(i == 0)) + for i in range(config.num_layers)]) self.final_layer_norm = nn.LayerNorm(config.layer_norm_epsilon) self.dropout = nn.Dropout(config.dropout) @@ -413,8 +425,12 @@ class T5Stack(nn.Module): encoder_attention_mask=None, head_mask=None): + batch_size, seq_length = hidden_states.shape[0], hidden_states.shape[1] + encoder_seq_length = encoder_hidden_states.shape[1] if encoder_hidden_states is not None else 0 if attention_mask is None: - attention_mask = torch.ones_like(input_ids) + attention_mask = torch.ones(batch_size, seq_length).to(hidden_states.device) + if encoder_attention_mask is None: + encoder_attention_mask = torch.ones(batch_size, encoder_seq_length).to(hidden_states.device) # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] # ourselves in which case we just need to make it broadcastable to all heads. @@ -426,8 +442,7 @@ class T5Stack(nn.Module): # - if the model is an encoder, make the mask broadcastable to [batch_size, num_heads, seq_length, seq_length] if attention_mask.dim() == 2: if self.config.is_decoder: - batch_size, seq_length = input_ids.size() - seq_ids = torch.arange(seq_length, device=input_ids.device) + seq_ids = torch.arange(seq_length, device=hidden_states.device) causal_mask = seq_ids[None, None, :].repeat(batch_size, seq_length, 1) <= seq_ids[None, :, None] extended_attention_mask = causal_mask[:, None, :, :] * attention_mask[:, None, None, :] else: @@ -469,16 +484,22 @@ class T5Stack(nn.Module): all_hidden_states = () all_attentions = () position_bias = None + encoder_decoder_position_bias = None for i, layer_module in enumerate(self.layer): if self.output_hidden_states: all_hidden_states = all_hidden_states + (hidden_states,) layer_outputs = layer_module(hidden_states, attention_mask=extended_attention_mask, + position_bias=position_bias, encoder_hidden_states=encoder_hidden_states, encoder_attention_mask=encoder_extended_attention_mask, + encoder_decoder_position_bias=encoder_decoder_position_bias, head_mask=head_mask[i]) hidden_states = layer_outputs[0] + if i == 0: + position_bias = layer_outputs[2] if len(layer_outputs) > 3 else None + encoder_decoder_position_bias = layer_outputs[4] if len(layer_outputs) > 5 else None if self.output_attentions: all_attentions = all_attentions + (layer_outputs[1],) @@ -700,14 +721,8 @@ class T5WithLMHead(T5PreTrainedModel): def get_output_embeddings(self): return self.lm_head - def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, - lm_labels=None): - - outputs = self.transformer(input_ids, - attention_mask=attention_mask, - token_type_ids=token_type_ids, - position_ids=position_ids, - head_mask=head_mask) + def forward(self, encoder_input_ids, decoder_input_ids, **kwargs): + outputs = self.transformer(encoder_input_ids, decoder_input_ids, **kwargs) sequence_output = outputs[0] lm_logits = self.cls(sequence_output) From 151e4ab4e786b9b4b702205b5077ea2dfe67b4dd Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Tue, 5 Nov 2019 16:26:51 +0000 Subject: [PATCH 013/505] Fix CTRL past --- transformers/modeling_ctrl.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/transformers/modeling_ctrl.py b/transformers/modeling_ctrl.py index 1873040a8e..589a065a11 100644 --- a/transformers/modeling_ctrl.py +++ b/transformers/modeling_ctrl.py @@ -63,7 +63,8 @@ def scaled_dot_product_attention(q, k, v, mask, attention_mask=None, head_mask=N scaled_attention_logits = matmul_qk / np.sqrt(dk) if mask is not None: - scaled_attention_logits += (mask * -1e4) + nd, ns = scaled_attention_logits.size(-2), scaled_attention_logits.size(-1) + scaled_attention_logits += (mask[ns-nd:ns, :ns] * -1e4) if attention_mask is not None: # Apply the attention mask @@ -357,7 +358,7 @@ class CTRLModel(CTRLPreTrainedModel): inputs_embeds = self.w(input_ids) # inputs_embeds = embedded.unsqueeze(0) if len(input_ids.shape)<2 else embedded seq_len = input_ids.shape[-1] - mask = torch.triu(torch.ones(seq_len, seq_len), 1).to(inputs_embeds.device) + mask = torch.triu(torch.ones(seq_len + past_length, seq_len + past_length), 1).to(inputs_embeds.device) inputs_embeds *= np.sqrt(self.d_model_size) From 3835e1e651ebeeddaa8dd8cb5f4d30912ec5ec6d Mon Sep 17 00:00:00 2001 From: thomwolf Date: Wed, 6 Nov 2019 11:52:29 +0100 Subject: [PATCH 014/505] adding tokenizer --- transformers/tokenization_t5.py | 188 +++++++++----------------------- 1 file changed, 51 insertions(+), 137 deletions(-) diff --git a/transformers/tokenization_t5.py b/transformers/tokenization_t5.py index 3f8f4bf556..cff6a41baf 100644 --- a/transformers/tokenization_t5.py +++ b/transformers/tokenization_t5.py @@ -16,16 +16,15 @@ from __future__ import absolute_import, division, print_function, unicode_literals -import collections import logging import os -import unicodedata -from io import open from .tokenization_utils import PreTrainedTokenizer logger = logging.getLogger(__name__) +SPIECE_UNDERLINE = u'▁' + #################################################### # Mapping from the keyword arguments names of Tokenizer `__init__` # to file names for serializing Tokenizer instances @@ -39,8 +38,7 @@ VOCAB_FILES_NAMES = {'vocab_file': 'vocab.txt'} PRETRAINED_VOCAB_FILES_MAP = { 'vocab_file': { - 't5-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-base-uncased-vocab.txt", - 't5-large-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-large-uncased-vocab.txt", + 't5': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-spiece.model", } } @@ -48,167 +46,83 @@ PRETRAINED_VOCAB_FILES_MAP = { # Mapping from model shortcut names to max length of inputs #################################################### PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { - 't5-base-uncased': 512, - 't5-large-uncased': 512, + 't5': 512, } -#################################################### -# Mapping from model shortcut names to a dictionary of additional -# keyword arguments for Tokenizer `__init__`. -# To be used for checkpoint specific configurations. -#################################################### -PRETRAINED_INIT_CONFIGURATION = { - 't5-base-uncased': {'do_lower_case': True}, - 't5-large-uncased': {'do_lower_case': True}, -} - - -def load_vocab(vocab_file): - """Loads a vocabulary file into a dictionary.""" - vocab = collections.OrderedDict() - with open(vocab_file, "r", encoding="utf-8") as reader: - tokens = reader.readlines() - for index, token in enumerate(tokens): - token = token.rstrip('\n') - vocab[token] = index - return vocab - - class T5Tokenizer(PreTrainedTokenizer): - r""" - Constructs a T5Tokenizer. - :class:`~transformers.T5Tokenizer` runs end-to-end tokenization: punctuation splitting + wordpiece - - Args: - vocab_file: Path to a one-wordpiece-per-line vocabulary file - do_lower_case: Whether to lower case the input. Only has an effect when do_wordpiece_only=False """ + SentencePiece based tokenizer. Peculiarities: + - requires `SentencePiece `_ + """ vocab_files_names = VOCAB_FILES_NAMES pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP - pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES - def __init__(self, vocab_file, do_lower_case=True, - unk_token="[UNK]", sep_token="[SEP]", pad_token="[PAD]", cls_token="[CLS]", - mask_token="[MASK]", **kwargs): - """Constructs a T5Tokenizer. + def __init__(self, vocab_file, eos_token="", unk_token="", + pad_token="", **kwargs): + super(T5Tokenizer, self).__init__(eos_token=eos_token, unk_token=unk_token, + pad_token=pad_token, **kwargs) - Args: - **vocab_file**: Path to a one-wordpiece-per-line vocabulary file - **do_lower_case**: (`optional`) boolean (default True) - Whether to lower case the input - Only has an effect when do_basic_tokenize=True - """ - super(T5Tokenizer, self).__init__(unk_token=unk_token, sep_token=sep_token, - pad_token=pad_token, cls_token=cls_token, - mask_token=mask_token, **kwargs) - self.max_len_single_sentence = self.max_len - 2 # take into account special tokens - self.max_len_sentences_pair = self.max_len - 3 # take into account special tokens + try: + import sentencepiece as spm + except ImportError: + logger.warning("You need to install SentencePiece to use T5Tokenizer:" + "https://github.com/google/sentencepiece" + "pip install sentencepiece") - if not os.path.isfile(vocab_file): - raise ValueError( - "Can't find a vocabulary file at path '{}'. To load the vocabulary from a Google pretrained " - "model use `tokenizer = T5Tokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`".format(vocab_file)) - self.vocab = load_vocab(vocab_file) + self.vocab_file = vocab_file + + self.sp_model = spm.SentencePieceProcessor() + self.sp_model.Load(vocab_file) @property def vocab_size(self): - return len(self.vocab) + return self.sp_model.get_piece_size() + + def __getstate__(self): + state = self.__dict__.copy() + state["sp_model"] = None + return state + + def __setstate__(self, d): + self.__dict__ = d + try: + import sentencepiece as spm + except ImportError: + logger.warning("You need to install SentencePiece to use XLNetTokenizer: https://github.com/google/sentencepiece" + "pip install sentencepiece") + self.sp_model = spm.SentencePieceProcessor() + self.sp_model.Load(self.vocab_file) def _tokenize(self, text): """ Take as input a string and return a list of strings (tokens) for words/sub-words """ - split_tokens = [] - if self.do_basic_tokenize: - for token in self.basic_tokenizer.tokenize(text, never_split=self.all_special_tokens): - for sub_token in self.wordpiece_tokenizer.tokenize(token): - split_tokens.append(sub_token) - else: - split_tokens = self.wordpiece_tokenizer.tokenize(text) - return split_tokens + return self.sp_model.EncodeAsPieces(text) def _convert_token_to_id(self, token): """ Converts a token (str/unicode) in an id using the vocab. """ - return self.vocab.get(token, self.vocab.get(self.unk_token)) + return self.sp_model.piece_to_id(token) def _convert_id_to_token(self, index): """Converts an index (integer) in a token (string/unicode) using the vocab.""" - return self.ids_to_tokens.get(index, self.unk_token) + return self.sp_model.id_to_piece(index) def convert_tokens_to_string(self, tokens): """ Converts a sequence of tokens (string) in a single string. """ - out_string = ' '.join(tokens).replace(' ##', '').strip() + out_string = self.sp_model.decode_pieces(tokens) return out_string - def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): + def save_vocabulary(self, save_directory): + """ Save the sentencepiece vocabulary (copy original file) and special tokens file + to a directory. """ - Build model inputs from a sequence or a pair of sequence for sequence classification tasks - by concatenating and adding special tokens. - A BERT sequence has the following format: - single sequence: [CLS] X [SEP] - pair of sequences: [CLS] A [SEP] B [SEP] - """ - if token_ids_1 is None: - return [self.cls_token_id] + token_ids_0 + [self.sep_token_id] - cls = [self.cls_token_id] - sep = [self.sep_token_id] - return cls + token_ids_0 + sep + token_ids_1 + sep + if not os.path.isdir(save_directory): + logger.error("Vocabulary path ({}) should be a directory".format(save_directory)) + return + out_vocab_file = os.path.join(save_directory, VOCAB_FILES_NAMES['vocab_file']) - def get_special_tokens_mask(self, token_ids_0, token_ids_1=None, already_has_special_tokens=False): - """ - Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding - special tokens using the tokenizer ``prepare_for_model`` or ``encode_plus`` methods. + if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file): + copyfile(self.vocab_file, out_vocab_file) - Args: - token_ids_0: list of ids (must not contain special tokens) - token_ids_1: Optional list of ids (must not contain special tokens), necessary when fetching sequence ids - for sequence pairs - already_has_special_tokens: (default False) Set to True if the token list is already formated with - special tokens for the model - - Returns: - A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. - """ - - if already_has_special_tokens: - if token_ids_1 is not None: - raise ValueError("You should not supply a second sequence if the provided sequence of " - "ids is already formated with special tokens for the model.") - return list(map(lambda x: 1 if x in [self.sep_token_id, self.cls_token_id] else 0, token_ids_0)) - - if token_ids_1 is not None: - return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1] - return [1] + ([0] * len(token_ids_0)) + [1] - - def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None): - """ - Creates a mask from the two sequences passed to be used in a sequence-pair classification task. - A BERT sequence pair mask has the following format: - 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 - | first sequence | second sequence - - if token_ids_1 is None, only returns the first portion of the mask (0's). - """ - sep = [self.sep_token_id] - cls = [self.cls_token_id] - if token_ids_1 is None: - return len(cls + token_ids_0 + sep) * [0] - return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1] - - def save_vocabulary(self, vocab_path): - """Save the tokenizer vocabulary to a directory or file.""" - index = 0 - if os.path.isdir(vocab_path): - vocab_file = os.path.join(vocab_path, VOCAB_FILES_NAMES['vocab_file']) - else: - vocab_file = vocab_path - with open(vocab_file, "w", encoding="utf-8") as writer: - for token, token_index in sorted(self.vocab.items(), key=lambda kv: kv[1]): - if index != token_index: - logger.warning("Saving vocabulary to {}: vocabulary indices are not consecutive." - " Please check that the vocabulary is not corrupted!".format(vocab_file)) - index = token_index - writer.write(token + u'\n') - index += 1 - return (vocab_file,) + return (out_vocab_file,) From 73f2c342f53f2ff02124da23ba029d80c386e7ce Mon Sep 17 00:00:00 2001 From: thomwolf Date: Wed, 6 Nov 2019 11:52:39 +0100 Subject: [PATCH 015/505] fixing template --- templates/adding_a_new_model/configuration_xxx.py | 2 +- templates/adding_a_new_model/modeling_xxx.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/templates/adding_a_new_model/configuration_xxx.py b/templates/adding_a_new_model/configuration_xxx.py index b1614e71af..14c1c2c79e 100644 --- a/templates/adding_a_new_model/configuration_xxx.py +++ b/templates/adding_a_new_model/configuration_xxx.py @@ -84,7 +84,7 @@ class XxxConfig(PretrainedConfig): summary_first_dropout=0.1, **kwargs): super(XxxConfig, self).__init__(**kwargs) - self.vocab_size = vocab_size_or_config_json_file if isinstance(vocab_size_or_config_json_file, six.string_types) else -1 + self.vocab_size = vocab_size_or_config_json_file if isinstance(vocab_size_or_config_json_file, int) else -1 self.n_ctx = n_ctx self.n_positions = n_positions self.n_embd = n_embd diff --git a/templates/adding_a_new_model/modeling_xxx.py b/templates/adding_a_new_model/modeling_xxx.py index ff64f13f40..ee705e753c 100644 --- a/templates/adding_a_new_model/modeling_xxx.py +++ b/templates/adding_a_new_model/modeling_xxx.py @@ -280,7 +280,6 @@ class XxxModel(XxxPreTrainedModel): self.init_weights() - @property def get_input_embeddings(self): return self.embeddings.word_embeddings From 076a207935bfcc38416cd0baa887d3e025ebef28 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Wed, 6 Nov 2019 11:52:50 +0100 Subject: [PATCH 016/505] adding tests and updating model --- transformers/__init__.py | 11 +- transformers/configuration_t5.py | 53 +++--- transformers/modeling_t5.py | 151 ++++++++-------- transformers/tests/modeling_common_test.py | 32 ++-- transformers/tests/modeling_t5_test.py | 176 +++++++++++++++++++ transformers/tests/modeling_tf_t5_test.py | 190 +++++++++++++++++++++ transformers/tests/tokenization_t5_test.py | 77 +++++++++ 7 files changed, 571 insertions(+), 119 deletions(-) create mode 100644 transformers/tests/modeling_t5_test.py create mode 100644 transformers/tests/modeling_tf_t5_test.py create mode 100644 transformers/tests/tokenization_t5_test.py diff --git a/transformers/__init__.py b/transformers/__init__.py index 53f3c39dc7..bf896276d6 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -42,6 +42,7 @@ from .tokenization_xlnet import XLNetTokenizer, SPIECE_UNDERLINE from .tokenization_xlm import XLMTokenizer from .tokenization_roberta import RobertaTokenizer from .tokenization_distilbert import DistilBertTokenizer +from .tokenization_t5 import T5Tokenizer # Configurations from .configuration_utils import PretrainedConfig @@ -52,10 +53,10 @@ from .configuration_transfo_xl import TransfoXLConfig, TRANSFO_XL_PRETRAINED_CON from .configuration_gpt2 import GPT2Config, GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_ctrl import CTRLConfig, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_xlnet import XLNetConfig, XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP -from .configuration_ctrl import CTRLConfig, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_xlm import XLMConfig, XLM_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_roberta import RobertaConfig, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_distilbert import DistilBertConfig, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_t5 import T5Config, T5_PRETRAINED_CONFIG_ARCHIVE_MAP # Modeling if is_torch_available(): @@ -69,10 +70,10 @@ if is_torch_available(): BertForTokenClassification, BertForQuestionAnswering, load_tf_weights_in_bert, BERT_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_openai import (OpenAIGPTPreTrainedModel, OpenAIGPTModel, - OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel, - load_tf_weights_in_openai_gpt, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP) + OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel, + load_tf_weights_in_openai_gpt, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_transfo_xl import (TransfoXLPreTrainedModel, TransfoXLModel, TransfoXLLMHeadModel, - load_tf_weights_in_transfo_xl, TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP) + load_tf_weights_in_transfo_xl, TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_gpt2 import (GPT2PreTrainedModel, GPT2Model, GPT2LMHeadModel, GPT2DoubleHeadsModel, load_tf_weights_in_gpt2, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP) @@ -95,6 +96,8 @@ if is_torch_available(): DistilBertForSequenceClassification, DistilBertForQuestionAnswering, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_encoder_decoder import PreTrainedEncoderDecoder, Model2Model + from .modeling_t5 import (T5PreTrainedModel, T5Model, T5WithLMHeadModel, + T5_PRETRAINED_MODEL_ARCHIVE_MAP) # Optimization from .optimization import (AdamW, ConstantLRSchedule, WarmupConstantSchedule, WarmupCosineSchedule, diff --git a/transformers/configuration_t5.py b/transformers/configuration_t5.py index a37a5b2157..9db918e59f 100644 --- a/transformers/configuration_t5.py +++ b/transformers/configuration_t5.py @@ -64,44 +64,29 @@ class T5Config(PretrainedConfig): pretrained_config_archive_map = T5_PRETRAINED_CONFIG_ARCHIVE_MAP def __init__(self, - vocab_size_or_config_json_file=50257, - n_positions=1024, - n_ctx=1024, - n_embd=768, - n_layer=12, - n_head=12, - resid_pdrop=0.1, - embd_pdrop=0.1, - attn_pdrop=0.1, - layer_norm_epsilon=1e-5, + vocab_size_or_config_json_file=32128, + n_positions=512, + d_model=512, + d_ff=2048, + num_layers=12, + num_heads=12, + relative_attention_num_buckets=32, + dropout_rate=0.1, + layer_norm_epsilon=1e-6, initializer_range=0.02, - - num_labels=1, - summary_type='cls_index', - summary_use_proj=True, - summary_activation=None, - summary_proj_to_labels=True, - summary_first_dropout=0.1, **kwargs): super(T5Config, self).__init__(**kwargs) - self.vocab_size = vocab_size_or_config_json_file if isinstance(vocab_size_or_config_json_file, six.string_types) else -1 - self.n_ctx = n_ctx + self.vocab_size = vocab_size_or_config_json_file if isinstance(vocab_size_or_config_json_file, int) else -1 self.n_positions = n_positions - self.n_embd = n_embd - self.n_layer = n_layer - self.n_head = n_head - self.resid_pdrop = resid_pdrop - self.embd_pdrop = embd_pdrop - self.attn_pdrop = attn_pdrop + self.d_model = d_model + self.d_ff = d_ff + self.num_layers = num_layers + self.num_heads = num_heads + self.relative_attention_num_buckets = relative_attention_num_buckets + self.dropout_rate = dropout_rate self.layer_norm_epsilon = layer_norm_epsilon self.initializer_range = initializer_range - self.num_labels = num_labels - self.summary_type = summary_type - self.summary_use_proj = summary_use_proj - self.summary_activation = summary_activation - self.summary_first_dropout = summary_first_dropout - self.summary_proj_to_labels = summary_proj_to_labels if isinstance(vocab_size_or_config_json_file, six.string_types): with open(vocab_size_or_config_json_file, "r", encoding="utf-8") as reader: json_config = json.loads(reader.read()) @@ -119,12 +104,12 @@ class T5Config(PretrainedConfig): @property def hidden_size(self): - return self.n_embd + return self.d_model @property def num_attention_heads(self): - return self.n_head + return self.num_heads @property def num_hidden_layers(self): - return self.n_layer + return self.num_layers diff --git a/transformers/modeling_t5.py b/transformers/modeling_t5.py index e1a1d019ff..ce443cf882 100644 --- a/transformers/modeling_t5.py +++ b/transformers/modeling_t5.py @@ -20,8 +20,8 @@ import json import logging import math import os -import math import sys +import copy import itertools from io import open @@ -30,7 +30,7 @@ from torch import nn import torch.nn.functional as F from torch.nn import CrossEntropyLoss, MSELoss -from .modeling_utils import PreTrainedModel, prune_linear_layer +from .modeling_utils import PreTrainedModel from .configuration_t5 import T5Config from .file_utils import add_start_docstrings @@ -127,7 +127,7 @@ class T5DenseReluDense(nn.Module): super(T5DenseReluDense, self).__init__() self.wi = nn.Linear(config.d_model, config.d_ff, bias=False) self.wo = nn.Linear(config.d_ff, config.d_model, bias=False) - self.dropout = nn.Dropout(config.dropout) + self.dropout = nn.Dropout(config.dropout_rate) def forward(self, hidden_states): h = self.wi(hidden_states) @@ -141,8 +141,8 @@ class T5LayerFF(nn.Module): def __init__(self, config): super(T5LayerFF, self).__init__() self.DenseReluDense = T5DenseReluDense(config) - self.layer_norm = nn.LayerNorm(config.layer_norm_epsilon) - self.dropout = nn.Dropout(config.dropout) + self.layer_norm = nn.LayerNorm(config.d_model, eps=config.layer_norm_epsilon) + self.dropout = nn.Dropout(config.dropout_rate) def forward(self, hidden_states): norm_x = self.layer_norm(hidden_states) @@ -157,6 +157,7 @@ class T5Attention(nn.Module): def __init__(self, config, has_relative_attention_bias=False): super(T5Attention, self).__init__() self.layer_id = next(T5Attention.NEW_ID) + self.is_decoder = config.is_decoder self.has_relative_attention_bias = has_relative_attention_bias self.output_attentions = config.output_attentions @@ -231,7 +232,7 @@ class T5Attention(nn.Module): ret += (n < 0).to(torch.long) * num_buckets # mtf.to_int32(mtf.less(n, 0)) * num_buckets n = torch.abs(n) else: - n = torch.max(n, 0) + n = torch.max(n, torch.zeros_like(n)) # now n is in the range [0, inf) # half of the buckets are for exact increments in positions @@ -242,7 +243,7 @@ class T5Attention(nn.Module): val_if_large = max_exact + ( torch.log(n.float() / max_exact) / math.log(max_distance / max_exact) * (num_buckets - max_exact)).to(torch.long) - val_if_large = torch.min(val_if_large, num_buckets - 1) + val_if_large = torch.min(val_if_large, torch.full_like(val_if_large, num_buckets - 1)) ret += torch.where(is_small, n, val_if_large) return ret @@ -259,7 +260,7 @@ class T5Attention(nn.Module): values = values.permute([2, 0, 1]).unsqueeze(0) # shape (1, num_heads, qlen, klen) return values - def forward(self, input, mask, kv=None, position_bias=None, cache=None, head_mask=None): + def forward(self, input, mask=None, kv=None, position_bias=None, cache=None, head_mask=None): """ Self-attention (if kv is None) or attention over source sentence (provided by kv). """ @@ -273,7 +274,6 @@ class T5Attention(nn.Module): # assert dim == self.dim, 'Dimensions do not match: %s input vs %s configured' % (dim, self.dim) n_heads = self.n_heads dim_per_head = self.dim // n_heads - mask_reshape = (bs, 1, qlen, klen) if mask.dim() == 3 else (bs, 1, 1, klen) def shape(x): """ projection """ @@ -311,8 +311,9 @@ class T5Attention(nn.Module): position_bias = self.compute_bias(qlen, klen) scores += position_bias - mask = (mask == 0).view(mask_reshape).expand_as(scores) # (bs, n_heads, qlen, klen) - scores.masked_fill_(mask, -float('inf')) # (bs, n_heads, qlen, klen) + if mask is not None: + mask = (mask == 0).expand_as(scores) # (bs, n_heads, qlen, klen) + scores.masked_fill_(mask, -float('inf')) # (bs, n_heads, qlen, klen) weights = F.softmax(scores.float(), dim=-1).type_as(scores) # (bs, n_heads, qlen, klen) weights = F.dropout(weights, p=self.dropout, training=self.training) # (bs, n_heads, qlen, klen) @@ -338,13 +339,13 @@ class T5LayerSelfAttention(nn.Module): def __init__(self, config, has_relative_attention_bias=False): super(T5LayerSelfAttention, self).__init__() self.SelfAttention = T5Attention(config, has_relative_attention_bias=has_relative_attention_bias) - self.layer_norm = nn.LayerNorm(config.layer_norm_epsilon) - self.dropout = nn.Dropout(config.dropout) + self.layer_norm = nn.LayerNorm(config.d_model, eps=config.layer_norm_epsilon) + self.dropout = nn.Dropout(config.dropout_rate) def forward(self, hidden_states, attention_mask=None, position_bias=None, head_mask=None): norm_x = self.layer_norm(hidden_states) attention_output = self.SelfAttention(norm_x, - attention_mask=attention_mask, + mask=attention_mask, position_bias=position_bias, head_mask=head_mask) y = attention_output[0] @@ -357,14 +358,14 @@ class T5LayerCrossAttention(nn.Module): def __init__(self, config, has_relative_attention_bias=False): super(T5LayerCrossAttention, self).__init__() self.EncDecAttention = T5Attention(config, has_relative_attention_bias=has_relative_attention_bias) - self.layer_norm = nn.LayerNorm(config.layer_norm_epsilon) - self.dropout = nn.Dropout(config.dropout) + self.layer_norm = nn.LayerNorm(config.d_model, eps=config.layer_norm_epsilon) + self.dropout = nn.Dropout(config.dropout_rate) def forward(self, hidden_states, kv, attention_mask=None, position_bias=None, head_mask=None): norm_x = self.layer_norm(hidden_states) attention_output = self.EncDecAttention(norm_x, + mask=attention_mask, kv=kv, - attention_mask=attention_mask, position_bias=position_bias, head_mask=head_mask) y = attention_output[0] @@ -410,13 +411,41 @@ class T5Block(nn.Module): return outputs -class T5Stack(nn.Module): +class T5PreTrainedModel(PreTrainedModel): + """ An abstract class to handle weights initialization and + a simple interface for dowloading and loading pretrained models. + """ + config_class = T5Config + pretrained_model_archive_map = T5_PRETRAINED_MODEL_ARCHIVE_MAP + load_tf_weights = load_tf_weights_in_t5 + base_model_prefix = "transformer" + + def _init_weights(self, module): + """ Initialize the weights """ + if isinstance(module, (nn.Linear, nn.Embedding)): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) + elif isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + if isinstance(module, nn.Linear) and module.bias is not None: + module.bias.data.zero_() + + +class T5Stack(T5PreTrainedModel): def __init__(self, config): - super(T5Stack, self).__init__() + super(T5Stack, self).__init__(config) + self.output_attentions = config.output_attentions + self.output_hidden_states = config.output_hidden_states + self.is_decoder = config.is_decoder + self.blocks = nn.ModuleList([T5Block(config, has_relative_attention_bias=bool(i == 0)) for i in range(config.num_layers)]) - self.final_layer_norm = nn.LayerNorm(config.layer_norm_epsilon) - self.dropout = nn.Dropout(config.dropout) + self.final_layer_norm = nn.LayerNorm(config.d_model, eps=config.layer_norm_epsilon) + self.dropout = nn.Dropout(config.dropout_rate) + + self.init_weights() def forward(self, hidden_states, @@ -426,10 +455,10 @@ class T5Stack(nn.Module): head_mask=None): batch_size, seq_length = hidden_states.shape[0], hidden_states.shape[1] - encoder_seq_length = encoder_hidden_states.shape[1] if encoder_hidden_states is not None else 0 if attention_mask is None: attention_mask = torch.ones(batch_size, seq_length).to(hidden_states.device) - if encoder_attention_mask is None: + if self.is_decoder and encoder_attention_mask is None: + encoder_seq_length = encoder_hidden_states.shape[1] encoder_attention_mask = torch.ones(batch_size, encoder_seq_length).to(hidden_states.device) # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] @@ -444,6 +473,7 @@ class T5Stack(nn.Module): if self.config.is_decoder: seq_ids = torch.arange(seq_length, device=hidden_states.device) causal_mask = seq_ids[None, None, :].repeat(batch_size, seq_length, 1) <= seq_ids[None, :, None] + causal_mask = causal_mask.to(attention_mask) extended_attention_mask = causal_mask[:, None, :, :] * attention_mask[:, None, None, :] else: extended_attention_mask = attention_mask[:, None, None, :] @@ -456,15 +486,18 @@ class T5Stack(nn.Module): extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 - # If a 2D ou 3D attention mask is provided for the cross-attention - # we need to make broadcastabe to [batch_size, num_heads, seq_length, seq_length] - if encoder_attention_mask.dim() == 3: - encoder_extended_attention_mask = encoder_attention_mask[:, None, :, :] - if encoder_attention_mask.dim() == 2: - encoder_extended_attention_mask = encoder_attention_mask[:, None, None, :] + if self.is_decoder: + # If a 2D ou 3D attention mask is provided for the cross-attention + # we need to make broadcastabe to [batch_size, num_heads, seq_length, seq_length] + if encoder_attention_mask.dim() == 3: + encoder_extended_attention_mask = encoder_attention_mask[:, None, :, :] + if encoder_attention_mask.dim() == 2: + encoder_extended_attention_mask = encoder_attention_mask[:, None, None, :] - encoder_extended_attention_mask = encoder_extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility - encoder_extended_attention_mask = (1.0 - encoder_extended_attention_mask) * -10000.0 + encoder_extended_attention_mask = encoder_extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility + encoder_extended_attention_mask = (1.0 - encoder_extended_attention_mask) * -10000.0 + else: + encoder_extended_attention_mask = None # Prepare head mask if needed # 1.0 in head_mask indicate we keep the head @@ -474,18 +507,18 @@ class T5Stack(nn.Module): if head_mask is not None: if head_mask.dim() == 1: head_mask = head_mask.unsqueeze(0).unsqueeze(0).unsqueeze(-1).unsqueeze(-1) - head_mask = head_mask.expand(self.config.num_hidden_layers, -1, -1, -1, -1) + head_mask = head_mask.expand(self.config.num_layers, -1, -1, -1, -1) elif head_mask.dim() == 2: head_mask = head_mask.unsqueeze(1).unsqueeze(-1).unsqueeze(-1) # We can specify head_mask for each layer head_mask = head_mask.to(dtype=next(self.parameters()).dtype) # switch to fload if need + fp16 compatibility else: - head_mask = [None] * self.config.num_hidden_layers + head_mask = [None] * self.config.num_layers all_hidden_states = () all_attentions = () position_bias = None encoder_decoder_position_bias = None - for i, layer_module in enumerate(self.layer): + for i, layer_module in enumerate(self.blocks): if self.output_hidden_states: all_hidden_states = all_hidden_states + (hidden_states,) @@ -498,8 +531,9 @@ class T5Stack(nn.Module): head_mask=head_mask[i]) hidden_states = layer_outputs[0] if i == 0: - position_bias = layer_outputs[2] if len(layer_outputs) > 3 else None - encoder_decoder_position_bias = layer_outputs[4] if len(layer_outputs) > 5 else None + position_bias = layer_outputs[2 if self.output_attentions else 1] + if self.is_decoder: + encoder_decoder_position_bias = layer_outputs[4 if self.output_attentions else 2] if self.output_attentions: all_attentions = all_attentions + (layer_outputs[1],) @@ -519,27 +553,6 @@ class T5Stack(nn.Module): return outputs # last-layer hidden state, (all hidden states), (all attentions) -class T5PreTrainedModel(PreTrainedEncoderDecoder): - """ An abstract class to handle weights initialization and - a simple interface for dowloading and loading pretrained models. - """ - config_class = T5Config - pretrained_model_archive_map = T5_PRETRAINED_MODEL_ARCHIVE_MAP - load_tf_weights = load_tf_weights_in_t5 - - def _init_weights(self, module): - """ Initialize the weights """ - if isinstance(module, (nn.Linear, nn.Embedding)): - # Slightly different from the TF version which uses truncated_normal for initialization - # cf https://github.com/pytorch/pytorch/pull/5617 - module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) - elif isinstance(module, nn.LayerNorm): - module.bias.data.zero_() - module.weight.data.fill_(1.0) - if isinstance(module, nn.Linear) and module.bias is not None: - module.bias.data.zero_() - - T5_START_DOCSTRING = r""" The T5 model was proposed in `Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer`_ by Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu. @@ -620,7 +633,7 @@ class T5Model(T5PreTrainedModel): """ def __init__(self, config): super(T5Model, self).__init__(config) - self.shared = nn.Embeddings(config.vocab_size, config.d_model) + self.shared = nn.Embedding(config.vocab_size, config.d_model) encoder_config = copy.deepcopy(config) self.encoder = T5Stack(encoder_config) @@ -631,7 +644,6 @@ class T5Model(T5PreTrainedModel): self.init_weights() - @property def get_input_embeddings(self): return self.shared @@ -646,17 +658,17 @@ class T5Model(T5PreTrainedModel): for layer, heads in heads_to_prune.items(): self.encoder.layer[layer].attention.prune_heads(heads) - def forward(self, encoder_input_ids, decoder_input_ids, **kwargs): + def forward(self, **kwargs): # keyword arguments come in 3 flavors: encoder-specific (prefixed by # `encoder_`), decoder-specific (prefixed by `decoder_`) and those # that apply to the model as whole. # We let the specific kwargs override the common ones in case of conflict. kwargs_common = dict((k, v) for k, v in kwargs.items() if not k.startswith("encoder_") and not k.startswith("decoder_")) - kwargs_decoder = kwargs_common.copy() kwargs_encoder = kwargs_common.copy() - kwargs_encoder.update(dict((k[len("encoder_") :], v) for k, v in kwargs.items() if k.startswith("encoder_"))) - kwargs_decoder.update(dict((k[len("decoder_") :], v) for k, v in kwargs.items() if k.startswith("decoder_"))) + kwargs_decoder = kwargs_common.copy() + kwargs_encoder.update(dict((k[len("encoder_"):], v) for k, v in kwargs.items() if k.startswith("encoder_"))) + kwargs_decoder.update(dict((k[len("decoder_"):], v) for k, v in kwargs.items() if k.startswith("decoder_"))) # Encode if needed (training, first prediction pass) encoder_hidden_states = kwargs_encoder.pop("hidden_states", None) @@ -680,7 +692,7 @@ class T5Model(T5PreTrainedModel): @add_start_docstrings("""T5 Model with a `language modeling` head on top. """, T5_START_DOCSTRING, T5_INPUTS_DOCSTRING) -class T5WithLMHead(T5PreTrainedModel): +class T5WithLMHeadModel(T5PreTrainedModel): r""" **lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: Labels for computing the masked language modeling loss. @@ -704,14 +716,14 @@ class T5WithLMHead(T5PreTrainedModel): Examples:: tokenizer = T5Tokenizer.from_pretrained('t5-base-uncased') - model = T5ForMaskedLM.from_pretrained('t5-base-uncased') + model = T5WithLMHeadModel.from_pretrained('t5-base-uncased') input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 outputs = model(input_ids, lm_labels=input_ids) loss, prediction_scores = outputs[:2] """ def __init__(self, config): - super(T5ForMaskedLM, self).__init__(config) + super(T5WithLMHeadModel, self).__init__(config) self.transformer = T5Model(config) self.lm_head = nn.Linear(config.d_model, config.vocab_size) @@ -721,11 +733,12 @@ class T5WithLMHead(T5PreTrainedModel): def get_output_embeddings(self): return self.lm_head - def forward(self, encoder_input_ids, decoder_input_ids, **kwargs): - outputs = self.transformer(encoder_input_ids, decoder_input_ids, **kwargs) + def forward(self, **kwargs): + lm_labels = kwargs.pop('decoder_lm_labels', None) + outputs = self.transformer(**kwargs) sequence_output = outputs[0] - lm_logits = self.cls(sequence_output) + lm_logits = self.lm_head(sequence_output) outputs = (lm_logits,) + outputs[2:] # Add hidden states and attention if they are here if lm_labels is not None: diff --git a/transformers/tests/modeling_common_test.py b/transformers/tests/modeling_common_test.py index ddc0f9f3de..42bf9ac3f5 100644 --- a/transformers/tests/modeling_common_test.py +++ b/transformers/tests/modeling_common_test.py @@ -73,6 +73,7 @@ class CommonTestCases: test_pruning = True test_resize_embeddings = True test_head_masking = True + is_encoder_decoder = False def test_save_load(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -114,10 +115,9 @@ class CommonTestCases: for model_class in self.all_model_classes: model = model_class(config) model.eval() - first, second = model(inputs_dict["input_ids"])[0], model(inputs_dict["input_ids"])[0] + first, second = model(**inputs_dict)[0], model(**inputs_dict)[0] self.assertEqual(first.ne(second).sum().item(), 0) - def test_attention_outputs(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -127,31 +127,42 @@ class CommonTestCases: model = model_class(config) model.eval() outputs = model(**inputs_dict) - attentions = outputs[-1] + self_attentions = outputs[-1] self.assertEqual(model.config.output_attentions, True) self.assertEqual(model.config.output_hidden_states, False) - self.assertEqual(len(attentions), self.model_tester.num_hidden_layers) + self.assertEqual(len(self_attentions), self.model_tester.num_hidden_layers) self.assertListEqual( - list(attentions[0].shape[-3:]), + list(self_attentions[0].shape[-3:]), [self.model_tester.num_attention_heads, self.model_tester.seq_length, self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) out_len = len(outputs) + if self.is_encoder_decoder: + cross_attentions = outputs[-2] + self.assertEqual(model.config.output_attentions, True) + self.assertEqual(model.config.output_hidden_states, False) + self.assertEqual(len(cross_attentions), self.model_tester.num_hidden_layers) + self.assertListEqual( + list(cross_attentions[0].shape[-3:]), + [self.model_tester.num_attention_heads, + self.model_tester.seq_length, + self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) + # Check attention is always last and order is fine config.output_attentions = True config.output_hidden_states = True model = model_class(config) model.eval() outputs = model(**inputs_dict) - self.assertEqual(out_len+1, len(outputs)) + self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1), len(outputs)) self.assertEqual(model.config.output_attentions, True) self.assertEqual(model.config.output_hidden_states, True) - attentions = outputs[-1] - self.assertEqual(len(attentions), self.model_tester.num_hidden_layers) + self_attentions = outputs[-1] + self.assertEqual(len(self_attentions), self.model_tester.num_hidden_layers) self.assertListEqual( - list(attentions[0].shape[-3:]), + list(self_attentions[0].shape[-3:]), [self.model_tester.num_attention_heads, self.model_tester.seq_length, self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) @@ -214,7 +225,6 @@ class CommonTestCases: self.assertTrue(models_equal) - def test_headmasking(self): if not self.test_head_masking: return @@ -268,7 +278,6 @@ class CommonTestCases: self.assertNotEqual( attentions[-1][..., -1, :, :].flatten().sum().item(), 0.0) - def test_head_pruning(self): if not self.test_pruning: return @@ -411,7 +420,6 @@ class CommonTestCases: self.assertDictEqual(model.config.pruned_heads, {0: [0], 1: [1, 2], 2: [1, 2]}) - def test_hidden_states_output(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() diff --git a/transformers/tests/modeling_t5_test.py b/transformers/tests/modeling_t5_test.py new file mode 100644 index 0000000000..b8bb828ebd --- /dev/null +++ b/transformers/tests/modeling_t5_test.py @@ -0,0 +1,176 @@ +# coding=utf-8 +# Copyright 2018 Google T5 Authors and HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import unittest +import shutil +import pytest + +from transformers import is_torch_available + +from .modeling_common_test import (CommonTestCases, ids_tensor) +from .configuration_common_test import ConfigTester + +if is_torch_available(): + from transformers import (T5Config, T5Model, T5WithLMHeadModel) + from transformers.modeling_t5 import T5_PRETRAINED_MODEL_ARCHIVE_MAP +else: + pytestmark = pytest.mark.skip("Require Torch") + + +class T5ModelTest(CommonTestCases.CommonModelTester): + + all_model_classes = (T5Model, T5WithLMHeadModel) if is_torch_available() else () + test_pruning = False + test_torchscript = False + test_resize_embeddings = False + is_encoder_decoder = True + + class T5ModelTester(object): + + def __init__(self, + parent, + batch_size=13, + seq_length=7, + is_training=True, + use_input_mask=True, + use_labels=True, + vocab_size=99, + n_positions=14, + hidden_size=32, + num_hidden_layers=5, + num_attention_heads=4, + d_ff=37, + relative_attention_num_buckets=8, + dropout_rate=0.1, + initializer_range=0.02, + scope=None, + ): + self.parent = parent + self.batch_size = batch_size + self.seq_length = seq_length + self.is_training = is_training + self.use_input_mask = use_input_mask + self.use_labels = use_labels + self.vocab_size = vocab_size + self.n_positions = n_positions + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.d_ff = d_ff + self.relative_attention_num_buckets = relative_attention_num_buckets + self.dropout_rate = dropout_rate + self.initializer_range = initializer_range + self.scope = scope + + def prepare_config_and_inputs(self): + input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + + input_mask = None + if self.use_input_mask: + input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + + token_labels = None + if self.use_labels: + token_labels = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + + config = T5Config( + vocab_size_or_config_json_file=self.vocab_size, + n_positions=self.n_positions, + d_model=self.hidden_size, + d_ff=self.d_ff, + num_layers=self.num_hidden_layers, + num_heads=self.num_attention_heads, + relative_attention_num_buckets=self.relative_attention_num_buckets, + dropout_rate=self.dropout_rate, + initializer_range=self.initializer_range) + + return (config, input_ids, input_mask, token_labels) + + def check_loss_output(self, result): + self.parent.assertListEqual( + list(result["loss"].size()), + []) + + def create_and_check_t5_model(self, config, input_ids, input_mask, token_labels): + model = T5Model(config=config) + model.eval() + encoder_output, decoder_output = model(encoder_input_ids=input_ids, + decoder_input_ids=input_ids, + decoder_attention_mask=input_mask) + encoder_output, decoder_output = model(encoder_input_ids=input_ids, + decoder_input_ids=input_ids) + + result = { + "encoder_output": encoder_output, + "decoder_output": decoder_output, + } + self.parent.assertListEqual( + list(result["encoder_output"].size()), + [self.batch_size, self.seq_length, self.hidden_size]) + self.parent.assertListEqual( + list(result["decoder_output"].size()), + [self.batch_size, self.seq_length, self.hidden_size]) + + + def create_and_check_t5_with_lm_head(self, config, input_ids, input_mask, token_labels): + model = T5WithLMHeadModel(config=config) + model.eval() + loss, prediction_scores = model(encoder_input_ids=input_ids, decoder_input_ids=input_ids, + decoder_attention_mask=input_mask, decoder_lm_labels=token_labels) + result = { + "loss": loss, + "prediction_scores": prediction_scores, + } + self.parent.assertListEqual( + list(result["prediction_scores"].size()), + [self.batch_size, self.seq_length, self.vocab_size]) + self.check_loss_output(result) + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + (config, input_ids, input_mask, token_labels) = config_and_inputs + inputs_dict = {'encoder_input_ids': input_ids, + 'decoder_input_ids': input_ids, + 'decoder_attention_mask': input_mask} + return config, inputs_dict + + def setUp(self): + self.model_tester = T5ModelTest.T5ModelTester(self) + self.config_tester = ConfigTester(self, config_class=T5Config, d_model=37) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_t5_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_t5_model(*config_and_inputs) + + def test_with_lm_head(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_t5_with_lm_head(*config_and_inputs) + + @pytest.mark.slow + def test_model_from_pretrained(self): + cache_dir = "/tmp/transformers_test/" + for model_name in list(T5_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + model = T5Model.from_pretrained(model_name, cache_dir=cache_dir) + shutil.rmtree(cache_dir) + self.assertIsNotNone(model) + +if __name__ == "__main__": + unittest.main() diff --git a/transformers/tests/modeling_tf_t5_test.py b/transformers/tests/modeling_tf_t5_test.py new file mode 100644 index 0000000000..fac6763432 --- /dev/null +++ b/transformers/tests/modeling_tf_t5_test.py @@ -0,0 +1,190 @@ +# coding=utf-8 +# Copyright 2018 Google T5 Authors and HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import unittest +import shutil +import pytest +import sys + +from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) +from .configuration_common_test import ConfigTester + +from transformers import T5Config, is_tf_available + +if False: # is_tf_available(): + import tensorflow as tf + from transformers.modeling_tf_t5 import (TFT5Model, TFT5WithLMHeadModel,TF_T5_PRETRAINED_MODEL_ARCHIVE_MAP) +else: + pytestmark = pytest.mark.skip("Require TensorFlow") + + +class TFT5ModelTest(TFCommonTestCases.TFCommonModelTester): + + all_model_classes = (TFT5Model, TFT5WithLMHeadModel) if False else () # is_tf_available() else () + + class TFT5ModelTester(object): + + def __init__(self, + parent, + batch_size=13, + seq_length=7, + is_training=True, + use_input_mask=True, + use_token_type_ids=True, + use_labels=True, + vocab_size=99, + hidden_size=32, + num_hidden_layers=5, + num_attention_heads=4, + intermediate_size=37, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + type_sequence_label_size=2, + initializer_range=0.02, + num_labels=3, + num_choices=4, + scope=None, + ): + self.parent = parent + self.batch_size = batch_size + self.seq_length = seq_length + self.is_training = is_training + self.use_input_mask = use_input_mask + self.use_token_type_ids = use_token_type_ids + self.use_labels = use_labels + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.intermediate_size = intermediate_size + self.hidden_act = hidden_act + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.type_sequence_label_size = type_sequence_label_size + self.initializer_range = initializer_range + self.num_labels = num_labels + self.num_choices = num_choices + self.scope = scope + + def prepare_config_and_inputs(self): + input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + + input_mask = None + if self.use_input_mask: + input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + + token_type_ids = None + if self.use_token_type_ids: + token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size) + + sequence_labels = None + token_labels = None + choice_labels = None + if self.use_labels: + sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size) + token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels) + choice_labels = ids_tensor([self.batch_size], self.num_choices) + + config = T5Config( + vocab_size_or_config_json_file=self.vocab_size, + hidden_size=self.hidden_size, + num_hidden_layers=self.num_hidden_layers, + num_attention_heads=self.num_attention_heads, + intermediate_size=self.intermediate_size, + hidden_act=self.hidden_act, + hidden_dropout_prob=self.hidden_dropout_prob, + attention_probs_dropout_prob=self.attention_probs_dropout_prob, + max_position_embeddings=self.max_position_embeddings, + type_vocab_size=self.type_vocab_size, + initializer_range=self.initializer_range) + + return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels + + def create_and_check_t5_model(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + model = TFT5Model(config=config) + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask, + 'token_type_ids': token_type_ids} + sequence_output, pooled_output = model(inputs) + + inputs = [input_ids, input_mask] + sequence_output, pooled_output = model(inputs) + + sequence_output, pooled_output = model(input_ids) + + result = { + "sequence_output": sequence_output.numpy(), + "pooled_output": pooled_output.numpy(), + } + self.parent.assertListEqual( + list(result["sequence_output"].shape), + [self.batch_size, self.seq_length, self.hidden_size]) + self.parent.assertListEqual(list(result["pooled_output"].shape), [self.batch_size, self.hidden_size]) + + + def create_and_check_t5_with_lm_head(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + model = TFT5WithLMHeadModel(config=config) + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask, + 'token_type_ids': token_type_ids} + prediction_scores, = model(inputs) + result = { + "prediction_scores": prediction_scores.numpy(), + } + self.parent.assertListEqual( + list(result["prediction_scores"].shape), + [self.batch_size, self.seq_length, self.vocab_size]) + + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + (config, input_ids, token_type_ids, input_mask, + sequence_labels, token_labels, choice_labels) = config_and_inputs + inputs_dict = {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask} + return config, inputs_dict + + def setUp(self): + self.model_tester = TFT5ModelTest.TFT5ModelTester(self) + self.config_tester = ConfigTester(self, config_class=T5Config, hidden_size=37) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_t5_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_t5_model(*config_and_inputs) + + def test_with_lm_head(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_t5_with_lm_head(*config_and_inputs) + + @pytest.mark.slow + def test_model_from_pretrained(self): + cache_dir = "/tmp/transformers_test/" + for model_name in ['t5-base']: + model = TFT5Model.from_pretrained(model_name, cache_dir=cache_dir) + shutil.rmtree(cache_dir) + self.assertIsNotNone(model) + +if __name__ == "__main__": + unittest.main() diff --git a/transformers/tests/tokenization_t5_test.py b/transformers/tests/tokenization_t5_test.py new file mode 100644 index 0000000000..9362487d8d --- /dev/null +++ b/transformers/tests/tokenization_t5_test.py @@ -0,0 +1,77 @@ +# coding=utf-8 +# Copyright 2018 Google T5 Authors and HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import, division, print_function, unicode_literals + +import os +import unittest +import pytest + +from transformers.tokenization_t5 import (T5Tokenizer, SPIECE_UNDERLINE) + +from .tokenization_tests_commons import CommonTestCases + +SAMPLE_VOCAB = os.path.join(os.path.dirname(os.path.abspath(__file__)), + 'fixtures/test_sentencepiece.model') + +class T5TokenizationTest(CommonTestCases.CommonTokenizerTester): + + tokenizer_class = T5Tokenizer + + def setUp(self): + super(T5TokenizationTest, self).setUp() + + # We have a SentencePiece fixture for testing + tokenizer = T5Tokenizer(SAMPLE_VOCAB, keep_accents=True) + tokenizer.save_pretrained(self.tmpdirname) + + def get_tokenizer(self, **kwargs): + return T5Tokenizer.from_pretrained(self.tmpdirname, **kwargs) + + def get_input_output_texts(self): + input_text = u"This is a test" + output_text = u"This is a test" + return input_text, output_text + + def test_full_tokenizer(self): + tokenizer = T5Tokenizer(SAMPLE_VOCAB, keep_accents=True) + + tokens = tokenizer.tokenize(u'This is a test') + self.assertListEqual(tokens, [u'▁This', u'▁is', u'▁a', u'▁t', u'est']) + + self.assertListEqual( + tokenizer.convert_tokens_to_ids(tokens), [285, 46, 10, 170, 382]) + + tokens = tokenizer.tokenize(u"I was born in 92000, and this is falsé.") + self.assertListEqual(tokens, [SPIECE_UNDERLINE + u'I', SPIECE_UNDERLINE + u'was', SPIECE_UNDERLINE + u'b', + u'or', u'n', SPIECE_UNDERLINE + u'in', SPIECE_UNDERLINE + u'', + u'9', u'2', u'0', u'0', u'0', u',', SPIECE_UNDERLINE + u'and', SPIECE_UNDERLINE + u'this', + SPIECE_UNDERLINE + u'is', SPIECE_UNDERLINE + u'f', u'al', u's', u'é', u'.']) + ids = tokenizer.convert_tokens_to_ids(tokens) + self.assertListEqual( + ids, [8, 21, 84, 55, 24, 19, 7, 0, + 602, 347, 347, 347, 3, 12, 66, + 46, 72, 80, 6, 0, 4]) + + back_tokens = tokenizer.convert_ids_to_tokens(ids) + self.assertListEqual(back_tokens, [SPIECE_UNDERLINE + u'I', SPIECE_UNDERLINE + u'was', SPIECE_UNDERLINE + u'b', + u'or', u'n', SPIECE_UNDERLINE + u'in', + SPIECE_UNDERLINE + u'', u'', u'2', u'0', u'0', u'0', u',', + SPIECE_UNDERLINE + u'and', SPIECE_UNDERLINE + u'this', + SPIECE_UNDERLINE + u'is', SPIECE_UNDERLINE + u'f', u'al', u's', + u'', u'.']) + + +if __name__ == '__main__': + unittest.main() From 070dcf1c020e28e96e8e4b5acfb29fb818b0b4dd Mon Sep 17 00:00:00 2001 From: Diganta Misra Date: Thu, 7 Nov 2019 03:45:43 +0530 Subject: [PATCH 017/505] Added Mish Activation Function Mish is a new activation function proposed here - https://arxiv.org/abs/1908.08681 It has seen some recent success and has been adopted in SpaCy, Thic, TensorFlow Addons and FastAI-dev. All benchmarks recorded till now (including against ReLU, Swish and GELU) is present in the repository - https://github.com/digantamisra98/Mish Might be a good addition to experiment with especially in the Bert Model. --- transformers/modeling_bert.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py index 7c2c6f4602..2baee71f82 100644 --- a/transformers/modeling_bert.py +++ b/transformers/modeling_bert.py @@ -138,7 +138,11 @@ def swish(x): return x * torch.sigmoid(x) -ACT2FN = {"gelu": gelu, "relu": torch.nn.functional.relu, "swish": swish, "gelu_new": gelu_new} +def mish(x): + return x * torch.tanh(nn.functional.softplus(x)) + + +ACT2FN = {"gelu": gelu, "relu": torch.nn.functional.relu, "swish": swish, "gelu_new": gelu_new, "mish": mish} BertLayerNorm = torch.nn.LayerNorm From ba10065c4b44d733d135ad6dc1b8a77f88c6dbb9 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 7 Nov 2019 15:55:36 +0100 Subject: [PATCH 018/505] update model, conversion script, tests and template --- ...t_xxx_original_tf_checkpoint_to_pytorch.py | 10 +- transformers/__init__.py | 1 + transformers/configuration_t5.py | 13 +- ...rt_t5_original_tf_checkpoint_to_pytorch.py | 12 +- transformers/modeling_t5.py | 129 ++++++++++++------ transformers/tests/modeling_common_test.py | 41 +++--- transformers/tests/modeling_t5_test.py | 12 +- transformers/tokenization_t5.py | 1 + 8 files changed, 135 insertions(+), 84 deletions(-) diff --git a/templates/adding_a_new_model/convert_xxx_original_tf_checkpoint_to_pytorch.py b/templates/adding_a_new_model/convert_xxx_original_tf_checkpoint_to_pytorch.py index d50d129cba..9d389deaad 100755 --- a/templates/adding_a_new_model/convert_xxx_original_tf_checkpoint_to_pytorch.py +++ b/templates/adding_a_new_model/convert_xxx_original_tf_checkpoint_to_pytorch.py @@ -26,9 +26,9 @@ from transformers import XxxConfig, XxxForPreTraining, load_tf_weights_in_xxx import logging logging.basicConfig(level=logging.INFO) -def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, xxx_config_file, pytorch_dump_path): +def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, config_file, pytorch_dump_path): # Initialise PyTorch model - config = XxxConfig.from_json_file(xxx_config_file) + config = XxxConfig.from_json_file(config_file) print("Building PyTorch model from configuration: {}".format(str(config))) model = XxxForPreTraining(config) @@ -48,11 +48,11 @@ if __name__ == "__main__": type = str, required = True, help = "Path to the TensorFlow checkpoint path.") - parser.add_argument("--xxx_config_file", + parser.add_argument("--config_file", default = None, type = str, required = True, - help = "The config json file corresponding to the pre-trained XXX model. \n" + help = "The config json file corresponding to the pre-trained model. \n" "This specifies the model architecture.") parser.add_argument("--pytorch_dump_path", default = None, @@ -61,5 +61,5 @@ if __name__ == "__main__": help = "Path to the output PyTorch model.") args = parser.parse_args() convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, - args.xxx_config_file, + args.config_file, args.pytorch_dump_path) diff --git a/transformers/__init__.py b/transformers/__init__.py index bf896276d6..601a068592 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -97,6 +97,7 @@ if is_torch_available(): DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_encoder_decoder import PreTrainedEncoderDecoder, Model2Model from .modeling_t5 import (T5PreTrainedModel, T5Model, T5WithLMHeadModel, + load_tf_weights_in_t5, T5_PRETRAINED_MODEL_ARCHIVE_MAP) # Optimization diff --git a/transformers/configuration_t5.py b/transformers/configuration_t5.py index 9db918e59f..96e67758ac 100644 --- a/transformers/configuration_t5.py +++ b/transformers/configuration_t5.py @@ -57,8 +57,7 @@ class T5Config(PretrainedConfig): (e.g., 512 or 1024 or 2048). type_vocab_size: The vocabulary size of the `token_type_ids` passed into `T5Model`. - initializer_range: The sttdev of the truncated_normal_initializer for - initializing all weight matrices. + initializer_factor: A factor for initializing all weight matrices (should be kept to 1.0, used for initialization testing). layer_norm_eps: The epsilon used by LayerNorm. """ pretrained_config_archive_map = T5_PRETRAINED_CONFIG_ARCHIVE_MAP @@ -67,25 +66,27 @@ class T5Config(PretrainedConfig): vocab_size_or_config_json_file=32128, n_positions=512, d_model=512, + d_kv=64, d_ff=2048, - num_layers=12, - num_heads=12, + num_layers=6, + num_heads=8, relative_attention_num_buckets=32, dropout_rate=0.1, layer_norm_epsilon=1e-6, - initializer_range=0.02, + initializer_factor=1.0, **kwargs): super(T5Config, self).__init__(**kwargs) self.vocab_size = vocab_size_or_config_json_file if isinstance(vocab_size_or_config_json_file, int) else -1 self.n_positions = n_positions self.d_model = d_model + self.d_kv = d_kv self.d_ff = d_ff self.num_layers = num_layers self.num_heads = num_heads self.relative_attention_num_buckets = relative_attention_num_buckets self.dropout_rate = dropout_rate self.layer_norm_epsilon = layer_norm_epsilon - self.initializer_range = initializer_range + self.initializer_factor = initializer_factor if isinstance(vocab_size_or_config_json_file, six.string_types): with open(vocab_size_or_config_json_file, "r", encoding="utf-8") as reader: diff --git a/transformers/convert_t5_original_tf_checkpoint_to_pytorch.py b/transformers/convert_t5_original_tf_checkpoint_to_pytorch.py index 608027ebac..2b74d2dd93 100755 --- a/transformers/convert_t5_original_tf_checkpoint_to_pytorch.py +++ b/transformers/convert_t5_original_tf_checkpoint_to_pytorch.py @@ -21,16 +21,16 @@ from __future__ import print_function import argparse import torch -from transformers import T5Config, T5ForPreTraining, load_tf_weights_in_t5 +from transformers import T5Config, T5Model, load_tf_weights_in_t5 import logging logging.basicConfig(level=logging.INFO) -def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, t5_config_file, pytorch_dump_path): +def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, config_file, pytorch_dump_path): # Initialise PyTorch model - config = T5Config.from_json_file(t5_config_file) + config = T5Config.from_json_file(config_file) print("Building PyTorch model from configuration: {}".format(str(config))) - model = T5ForPreTraining(config) + model = T5Model(config) # Load weights from tf checkpoint load_tf_weights_in_t5(model, config, tf_checkpoint_path) @@ -48,7 +48,7 @@ if __name__ == "__main__": type = str, required = True, help = "Path to the TensorFlow checkpoint path.") - parser.add_argument("--t5_config_file", + parser.add_argument("--config_file", default = None, type = str, required = True, @@ -61,5 +61,5 @@ if __name__ == "__main__": help = "Path to the output PyTorch model.") args = parser.parse_args() convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, - args.t5_config_file, + args.config_file, args.pytorch_dump_path) diff --git a/transformers/modeling_t5.py b/transformers/modeling_t5.py index ce443cf882..6ed241761a 100644 --- a/transformers/modeling_t5.py +++ b/transformers/modeling_t5.py @@ -65,34 +65,40 @@ def load_tf_weights_in_t5(model, config, tf_checkpoint_path): # Load weights from TF model init_vars = tf.train.list_variables(tf_path) names = [] - arrays = [] + tf_weights = {} for name, shape in init_vars: logger.info("Loading TF weight {} with shape {}".format(name, shape)) array = tf.train.load_variable(tf_path, name) names.append(name) - arrays.append(array) + tf_weights[name] = array - for name, array in zip(names, arrays): - name = name.split('/') + for txt_name in names: + name = txt_name.split('/') # adam_v and adam_m are variables used in AdamWeightDecayOptimizer to calculated m and v # which are not required for using pretrained model if any(n in ["adam_v", "adam_m", "global_step"] for n in name): logger.info("Skipping {}".format("/".join(name))) + tf_weights.pop(txt_name, None) + continue + if '_slot_' in name[-1]: + logger.info("Skipping {}".format("/".join(name))) + tf_weights.pop(txt_name, None) continue pointer = model + array = tf_weights[txt_name] for m_name in name: if re.fullmatch(r'[A-Za-z]+_\d+', m_name): l = re.split(r'_(\d+)', m_name) else: l = [m_name] - if l[0] == 'kernel' or l[0] == 'gamma': + if l[0] in ['kernel', 'scale', 'embedding']: pointer = getattr(pointer, 'weight') - elif l[0] == 'output_bias' or l[0] == 'beta': - pointer = getattr(pointer, 'bias') - elif l[0] == 'output_weights': - pointer = getattr(pointer, 'weight') - elif l[0] == 'squad': - pointer = getattr(pointer, 'classifier') + # elif l[0] == 'scale': + # pointer = getattr(pointer, 'weight') + # elif l[0] == 'output_bias' or l[0] == 'beta': + # pointer = getattr(pointer, 'bias') + # elif l[0] == 'squad': + # pointer = getattr(pointer, 'classifier') else: try: pointer = getattr(pointer, l[0]) @@ -102,9 +108,10 @@ def load_tf_weights_in_t5(model, config, tf_checkpoint_path): if len(l) >= 2: num = int(l[1]) pointer = pointer[num] - if m_name[-11:] == '_embeddings': + if l[0] not in ['kernel', 'scale', 'embedding']: pointer = getattr(pointer, 'weight') - elif m_name == 'kernel': + if l[0] != 'embedding': + logger.info("Transposing numpy weight of shape {} for {}".format(array.shape, name)) array = np.transpose(array) try: assert pointer.shape == array.shape @@ -112,7 +119,11 @@ def load_tf_weights_in_t5(model, config, tf_checkpoint_path): e.args += (pointer.shape, array.shape) raise logger.info("Initialize PyTorch weight {}".format(name)) - pointer.data = torch.from_numpy(array) + pointer.data = torch.from_numpy(array.astype(np.float32)) + tf_weights.pop(txt_name, None) + + logger.info("Weights not copied to PyTorch model: {}".format(', '.join(tf_weights.keys()))) + # logger.info("Weights not copied to PyTorch model: {}".format(', '.join(tf_weights.keys()))) return model @@ -163,10 +174,13 @@ class T5Attention(nn.Module): self.output_attentions = config.output_attentions self.relative_attention_num_buckets = config.relative_attention_num_buckets self.dim = config.d_model + self.d_kv = config.d_kv self.n_heads = config.num_heads self.dropout = config.dropout_rate assert self.dim % self.n_heads == 0 + assert self.dim // self.n_heads == self.d_kv + # Mesh TensorFlow initialization to avoid scaling before softmax self.q = nn.Linear(self.dim, self.dim, bias=False) self.k = nn.Linear(self.dim, self.dim, bias=False) self.v = nn.Linear(self.dim, self.dim, bias=False) @@ -312,8 +326,9 @@ class T5Attention(nn.Module): scores += position_bias if mask is not None: - mask = (mask == 0).expand_as(scores) # (bs, n_heads, qlen, klen) - scores.masked_fill_(mask, -float('inf')) # (bs, n_heads, qlen, klen) + scores += mask + # mask = (mask == 0).expand_as(scores) # (bs, n_heads, qlen, klen) + # scores.masked_fill_(mask, -float('inf')) # (bs, n_heads, qlen, klen) weights = F.softmax(scores.float(), dim=-1).type_as(scores) # (bs, n_heads, qlen, klen) weights = F.dropout(weights, p=self.dropout, training=self.training) # (bs, n_heads, qlen, klen) @@ -378,34 +393,35 @@ class T5Block(nn.Module): def __init__(self, config, has_relative_attention_bias=False): super(T5Block, self).__init__() self.is_decoder = config.is_decoder - self.layer_000 = T5LayerSelfAttention(config, has_relative_attention_bias=has_relative_attention_bias) + self.layer = nn.ModuleList() + self.layer.append(T5LayerSelfAttention(config, has_relative_attention_bias=has_relative_attention_bias)) if self.is_decoder: - self.layer_001 = T5LayerCrossAttention(config, has_relative_attention_bias=has_relative_attention_bias) - self.layer_002 = T5LayerFF(config) + self.layer.append(T5LayerCrossAttention(config, has_relative_attention_bias=has_relative_attention_bias)) + self.layer.append(T5LayerFF(config)) else: - self.layer_001 = T5LayerFF(config) + self.layer.append(T5LayerFF(config)) def forward(self, hidden_states, attention_mask=None, position_bias=None, encoder_hidden_states=None, encoder_attention_mask=None, encoder_decoder_position_bias=None, head_mask=None): - self_attention_outputs = self.layer_000(hidden_states, + self_attention_outputs = self.layer[0](hidden_states, attention_mask=attention_mask, position_bias=position_bias, head_mask=head_mask) hidden_states = self_attention_outputs[0] outputs = self_attention_outputs[1:] - if self.is_decoder: - cross_attention_outputs = self.layer_001(hidden_states, - kv=encoder_hidden_states, - attention_mask=encoder_attention_mask, - position_bias=encoder_decoder_position_bias, - head_mask=head_mask) + if not self.is_decoder: + hidden_states = self.layer[1](hidden_states) + else: + cross_attention_outputs = self.layer[1](hidden_states, + kv=encoder_hidden_states, + attention_mask=encoder_attention_mask, + position_bias=encoder_decoder_position_bias, + head_mask=head_mask) hidden_states = cross_attention_outputs[0] outputs = cross_attention_outputs[1:] + outputs - hidden_states = self.layer_002(hidden_states) - else: - hidden_states = self.layer_001(hidden_states) + hidden_states = self.layer[2](hidden_states) outputs = (hidden_states,) + outputs # add attentions if we output them return outputs @@ -422,15 +438,36 @@ class T5PreTrainedModel(PreTrainedModel): def _init_weights(self, module): """ Initialize the weights """ - if isinstance(module, (nn.Linear, nn.Embedding)): - # Slightly different from the TF version which uses truncated_normal for initialization - # cf https://github.com/pytorch/pytorch/pull/5617 - module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) - elif isinstance(module, nn.LayerNorm): - module.bias.data.zero_() - module.weight.data.fill_(1.0) - if isinstance(module, nn.Linear) and module.bias is not None: + factor = self.config.initializer_factor # Used for testing weights initialization + if isinstance(module, nn.LayerNorm): module.bias.data.zero_() + module.weight.data.fill_(factor*1.0) + elif isinstance(module, T5Model): + # Mesh TensorFlow embeddings initialization + # See https://github.com/tensorflow/mesh/blob/fa19d69eafc9a482aff0b59ddd96b025c0cb207d/mesh_tensorflow/layers.py#L1624 + module.shared.weight.data.normal_(mean=0.0, std=factor*1.0) + elif isinstance(module, T5DenseReluDense): + # Mesh TensorFlow FF initialization + # See https://github.com/tensorflow/mesh/blob/master/mesh_tensorflow/transformer/transformer_layers.py#L56 + # and https://github.com/tensorflow/mesh/blob/fa19d69eafc9a482aff0b59ddd96b025c0cb207d/mesh_tensorflow/layers.py#L89 + module.wi.weight.data.normal_(mean=0.0, std=factor*((self.config.d_model) ** -0.5)) + if hasattr(module.wi, 'bias') and module.wi.bias is not None: + module.wi.bias.data.zero_() + module.wo.weight.data.normal_(mean=0.0, std=factor*((self.config.d_ff) ** -0.5)) + if hasattr(module.wo, 'bias') and module.wo.bias is not None: + module.wo.bias.data.zero_() + elif isinstance(module, T5Attention): + # Mesh TensorFlow attention initialization to avoid scaling before softmax + # See https://github.com/tensorflow/mesh/blob/fa19d69eafc9a482aff0b59ddd96b025c0cb207d/mesh_tensorflow/transformer/attention.py#L136 + d_model = self.config.d_model + d_kv = self.config.d_kv + n_heads = self.config.num_heads + module.q.weight.data.normal_(mean=0.0, std=factor*((d_model * d_kv) ** -0.5)) + module.k.weight.data.normal_(mean=0.0, std=factor*(d_model ** -0.5)) + module.v.weight.data.normal_(mean=0.0, std=factor*(d_model ** -0.5)) + module.o.weight.data.normal_(mean=0.0, std=factor*((n_heads * d_kv) ** -0.5)) + if module.has_relative_attention_bias: + module.relative_attention_bias.weight.data.normal_(mean=0.0, std=factor*((d_model) ** -0.5)) class T5Stack(T5PreTrainedModel): @@ -440,8 +477,8 @@ class T5Stack(T5PreTrainedModel): self.output_hidden_states = config.output_hidden_states self.is_decoder = config.is_decoder - self.blocks = nn.ModuleList([T5Block(config, has_relative_attention_bias=bool(i == 0)) - for i in range(config.num_layers)]) + self.block = nn.ModuleList([T5Block(config, has_relative_attention_bias=bool(i == 0)) + for i in range(config.num_layers)]) self.final_layer_norm = nn.LayerNorm(config.d_model, eps=config.layer_norm_epsilon) self.dropout = nn.Dropout(config.dropout_rate) @@ -518,7 +555,7 @@ class T5Stack(T5PreTrainedModel): all_attentions = () position_bias = None encoder_decoder_position_bias = None - for i, layer_module in enumerate(self.blocks): + for i, layer_module in enumerate(self.block): if self.output_hidden_states: all_hidden_states = all_hidden_states + (hidden_states,) @@ -724,9 +761,10 @@ class T5WithLMHeadModel(T5PreTrainedModel): """ def __init__(self, config): super(T5WithLMHeadModel, self).__init__(config) + self.model_dim = config.d_model self.transformer = T5Model(config) - self.lm_head = nn.Linear(config.d_model, config.vocab_size) + self.lm_head = nn.Linear(config.d_model, config.vocab_size, bias=False) self.init_weights() @@ -738,15 +776,18 @@ class T5WithLMHeadModel(T5PreTrainedModel): outputs = self.transformer(**kwargs) sequence_output = outputs[0] + # Rescale output before projecting on vocab + # See https://github.com/tensorflow/mesh/blob/fa19d69eafc9a482aff0b59ddd96b025c0cb207d/mesh_tensorflow/transformer/transformer.py#L586 + sequence_output = sequence_output * (self.model_dim ** -0.5) lm_logits = self.lm_head(sequence_output) - outputs = (lm_logits,) + outputs[2:] # Add hidden states and attention if they are here + outputs = (lm_logits,) + outputs[1:] # Add hidden states and attention if they are here if lm_labels is not None: shift_logits = lm_logits[..., :-1, :].contiguous() shift_labels = lm_labels[..., 1:].contiguous() loss_fct = CrossEntropyLoss(ignore_index=-1) loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1)) - outputs = (loss,) + outputs + outputs = (loss,) + outputs # TODO(thom): Add z_loss https://github.com/tensorflow/mesh/blob/fa19d69eafc9a482aff0b59ddd96b025c0cb207d/mesh_tensorflow/layers.py#L666 return outputs # (lm_loss), lm_logits, (hidden_states), (attentions) diff --git a/transformers/tests/modeling_common_test.py b/transformers/tests/modeling_common_test.py index 42bf9ac3f5..ee75da605c 100644 --- a/transformers/tests/modeling_common_test.py +++ b/transformers/tests/modeling_common_test.py @@ -59,7 +59,7 @@ else: def _config_zero_init(config): configs_no_init = copy.deepcopy(config) for key in configs_no_init.__dict__.keys(): - if '_range' in key or '_std' in key: + if '_range' in key or '_std' in key or 'initializer_factor' in key: setattr(configs_no_init, key, 0.0) return configs_no_init @@ -83,20 +83,24 @@ class CommonTestCases: model.eval() with torch.no_grad(): outputs = model(**inputs_dict) + out_2 = outputs[0].numpy() + out_2[np.isnan(out_2)] = 0 with TemporaryDirectory() as tmpdirname: model.save_pretrained(tmpdirname) model = model_class.from_pretrained(tmpdirname) - with torch.no_grad(): - after_outputs = model(**inputs_dict) - # Make sure we don't have nans - out_1 = after_outputs[0].numpy() - out_2 = outputs[0].numpy() - out_1 = out_1[~np.isnan(out_1)] - out_2 = out_2[~np.isnan(out_2)] - max_diff = np.amax(np.abs(out_1 - out_2)) - self.assertLessEqual(max_diff, 1e-5) + with torch.no_grad(): + after_outputs = model(**inputs_dict) + + # # Make sure we don't have nans + out_1 = after_outputs[0].numpy() + out_1[np.isnan(out_1)] = 0 + + out_1 = out_1 - out_2 + amax = np.amax(out_1) + amin = np.amin(out_1) + self.assertLessEqual(max(amax, -amin), 1e-5) def test_initialization(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -127,27 +131,28 @@ class CommonTestCases: model = model_class(config) model.eval() outputs = model(**inputs_dict) - self_attentions = outputs[-1] + attentions = outputs[-1] self.assertEqual(model.config.output_attentions, True) self.assertEqual(model.config.output_hidden_states, False) - self.assertEqual(len(self_attentions), self.model_tester.num_hidden_layers) + self.assertEqual(len(attentions), self.model_tester.num_hidden_layers) self.assertListEqual( - list(self_attentions[0].shape[-3:]), + list(attentions[0].shape[-3:]), [self.model_tester.num_attention_heads, self.model_tester.seq_length, self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) out_len = len(outputs) if self.is_encoder_decoder: - cross_attentions = outputs[-2] + self.assertEqual(out_len % 2, 0) + decoder_attentions = outputs[(out_len // 2)-1] self.assertEqual(model.config.output_attentions, True) self.assertEqual(model.config.output_hidden_states, False) - self.assertEqual(len(cross_attentions), self.model_tester.num_hidden_layers) + self.assertEqual(len(decoder_attentions), self.model_tester.num_hidden_layers) self.assertListEqual( - list(cross_attentions[0].shape[-3:]), + list(decoder_attentions[0].shape[-3:]), [self.model_tester.num_attention_heads, - self.model_tester.seq_length, - self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) + self.model_tester.seq_length, + self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) # Check attention is always last and order is fine config.output_attentions = True diff --git a/transformers/tests/modeling_t5_test.py b/transformers/tests/modeling_t5_test.py index b8bb828ebd..2c67b83c25 100644 --- a/transformers/tests/modeling_t5_test.py +++ b/transformers/tests/modeling_t5_test.py @@ -57,7 +57,7 @@ class T5ModelTest(CommonTestCases.CommonModelTester): d_ff=37, relative_attention_num_buckets=8, dropout_rate=0.1, - initializer_range=0.02, + initializer_factor=0.002, scope=None, ): self.parent = parent @@ -74,7 +74,7 @@ class T5ModelTest(CommonTestCases.CommonModelTester): self.d_ff = d_ff self.relative_attention_num_buckets = relative_attention_num_buckets self.dropout_rate = dropout_rate - self.initializer_range = initializer_range + self.initializer_factor = initializer_factor self.scope = scope def prepare_config_and_inputs(self): @@ -93,11 +93,12 @@ class T5ModelTest(CommonTestCases.CommonModelTester): n_positions=self.n_positions, d_model=self.hidden_size, d_ff=self.d_ff, + d_kv=self.hidden_size // self.num_attention_heads, num_layers=self.num_hidden_layers, num_heads=self.num_attention_heads, relative_attention_num_buckets=self.relative_attention_num_buckets, dropout_rate=self.dropout_rate, - initializer_range=self.initializer_range) + initializer_factor=self.initializer_factor) return (config, input_ids, input_mask, token_labels) @@ -130,8 +131,9 @@ class T5ModelTest(CommonTestCases.CommonModelTester): def create_and_check_t5_with_lm_head(self, config, input_ids, input_mask, token_labels): model = T5WithLMHeadModel(config=config) model.eval() - loss, prediction_scores = model(encoder_input_ids=input_ids, decoder_input_ids=input_ids, - decoder_attention_mask=input_mask, decoder_lm_labels=token_labels) + outputs = model(encoder_input_ids=input_ids, decoder_input_ids=input_ids, + decoder_attention_mask=input_mask, decoder_lm_labels=token_labels) + loss, prediction_scores = outputs[0], outputs[1] result = { "loss": loss, "prediction_scores": prediction_scores, diff --git a/transformers/tokenization_t5.py b/transformers/tokenization_t5.py index cff6a41baf..ae898ba0d3 100644 --- a/transformers/tokenization_t5.py +++ b/transformers/tokenization_t5.py @@ -18,6 +18,7 @@ from __future__ import absolute_import, division, print_function, unicode_litera import logging import os +from shutil import copyfile from .tokenization_utils import PreTrainedTokenizer From 8fda532c3cbab9e31fbbfa860f232b69e0f80633 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 7 Nov 2019 17:09:50 +0100 Subject: [PATCH 019/505] fix python 2 sentencepiece tokenization --- transformers/tests/tokenization_t5_test.py | 7 +++--- transformers/tokenization_t5.py | 26 ++++++++++++++++++---- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/transformers/tests/tokenization_t5_test.py b/transformers/tests/tokenization_t5_test.py index 9362487d8d..aabb21e443 100644 --- a/transformers/tests/tokenization_t5_test.py +++ b/transformers/tests/tokenization_t5_test.py @@ -18,7 +18,8 @@ import os import unittest import pytest -from transformers.tokenization_t5 import (T5Tokenizer, SPIECE_UNDERLINE) +from transformers.tokenization_t5 import (T5Tokenizer) +from transformers.tokenization_xlnet import SPIECE_UNDERLINE from .tokenization_tests_commons import CommonTestCases @@ -33,7 +34,7 @@ class T5TokenizationTest(CommonTestCases.CommonTokenizerTester): super(T5TokenizationTest, self).setUp() # We have a SentencePiece fixture for testing - tokenizer = T5Tokenizer(SAMPLE_VOCAB, keep_accents=True) + tokenizer = T5Tokenizer(SAMPLE_VOCAB) tokenizer.save_pretrained(self.tmpdirname) def get_tokenizer(self, **kwargs): @@ -45,7 +46,7 @@ class T5TokenizationTest(CommonTestCases.CommonTokenizerTester): return input_text, output_text def test_full_tokenizer(self): - tokenizer = T5Tokenizer(SAMPLE_VOCAB, keep_accents=True) + tokenizer = T5Tokenizer(SAMPLE_VOCAB) tokens = tokenizer.tokenize(u'This is a test') self.assertListEqual(tokens, [u'▁This', u'▁is', u'▁a', u'▁t', u'est']) diff --git a/transformers/tokenization_t5.py b/transformers/tokenization_t5.py index ae898ba0d3..93842d29f0 100644 --- a/transformers/tokenization_t5.py +++ b/transformers/tokenization_t5.py @@ -18,6 +18,7 @@ from __future__ import absolute_import, division, print_function, unicode_litera import logging import os +import six from shutil import copyfile from .tokenization_utils import PreTrainedTokenizer @@ -96,18 +97,35 @@ class T5Tokenizer(PreTrainedTokenizer): self.sp_model = spm.SentencePieceProcessor() self.sp_model.Load(self.vocab_file) - def _tokenize(self, text): + def _tokenize(self, text, return_unicode=True, sample=False): """ Take as input a string and return a list of strings (tokens) for words/sub-words """ - return self.sp_model.EncodeAsPieces(text) + if not sample: + pieces = self.sp_model.EncodeAsPieces(text) + else: + pieces = self.sp_model.SampleEncodeAsPieces(text, 64, 0.1) + + # convert back to unicode for py2 + if six.PY2 and return_unicode: + ret_pieces = [] + for piece in pieces: + if isinstance(piece, str): + piece = piece.decode('utf-8') + ret_pieces.append(piece) + pieces = ret_pieces + + return pieces def _convert_token_to_id(self, token): """ Converts a token (str/unicode) in an id using the vocab. """ return self.sp_model.piece_to_id(token) - def _convert_id_to_token(self, index): + def _convert_id_to_token(self, index, return_unicode=True): """Converts an index (integer) in a token (string/unicode) using the vocab.""" - return self.sp_model.id_to_piece(index) + token = self.sp_model.IdToPiece(index) + if six.PY2 and return_unicode and isinstance(token, str): + token = token.decode('utf-8') + return token def convert_tokens_to_string(self, tokens): """ Converts a sequence of tokens (string) in a single string. """ From 28d0ba35d73d5b8b31fdadd72686a3ac078a6143 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Louf?= Date: Fri, 8 Nov 2019 11:22:19 +0100 Subject: [PATCH 020/505] only init encoder_attention_mask if stack is decoder We currently initialize `encoder_attention_mask` when it is `None`, whether the stack is that of an encoder or a decoder. Since this may lead to bugs that are difficult to tracks down, I added a condition that assesses whether the current stack is a decoder. --- transformers/modeling_bert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py index 7c2c6f4602..6bd5ab6a2e 100644 --- a/transformers/modeling_bert.py +++ b/transformers/modeling_bert.py @@ -656,7 +656,7 @@ class BertModel(BertPreTrainedModel): if attention_mask is None: attention_mask = torch.ones(input_shape, device=device) - if encoder_attention_mask is None: + if self.config.is_decoder and encoder_attention_mask is None: encoder_attention_mask = torch.ones(input_shape, device=device) if token_type_ids is None: token_type_ids = torch.zeros(input_shape, dtype=torch.long, device=device) From cd286c2145221f3d1372aef103d0bc3ed03879da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Louf?= Date: Fri, 8 Nov 2019 11:31:16 +0100 Subject: [PATCH 021/505] add condition around mask transformation --- transformers/modeling_bert.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py index 6bd5ab6a2e..893ec51015 100644 --- a/transformers/modeling_bert.py +++ b/transformers/modeling_bert.py @@ -656,8 +656,6 @@ class BertModel(BertPreTrainedModel): if attention_mask is None: attention_mask = torch.ones(input_shape, device=device) - if self.config.is_decoder and encoder_attention_mask is None: - encoder_attention_mask = torch.ones(input_shape, device=device) if token_type_ids is None: token_type_ids = torch.zeros(input_shape, dtype=torch.long, device=device) @@ -688,13 +686,19 @@ class BertModel(BertPreTrainedModel): # If a 2D ou 3D attention mask is provided for the cross-attention # we need to make broadcastabe to [batch_size, num_heads, seq_length, seq_length] - if encoder_attention_mask.dim() == 3: - encoder_extended_attention_mask = encoder_attention_mask[:, None, :, :] - if encoder_attention_mask.dim() == 2: - encoder_extended_attention_mask = encoder_attention_mask[:, None, None, :] + if self.config.is_decoder: + if encoder_attention_mask is None: + encoder_attention_mask = torch.ones(input_shape, device=device) - encoder_extended_attention_mask = encoder_extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility - encoder_extended_attention_mask = (1.0 - encoder_extended_attention_mask) * -10000.0 + if encoder_attention_mask.dim() == 3: + encoder_extended_attention_mask = encoder_attention_mask[:, None, :, :] + if encoder_attention_mask.dim() == 2: + encoder_extended_attention_mask = encoder_attention_mask[:, None, None, :] + + encoder_extended_attention_mask = encoder_extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility + encoder_extended_attention_mask = (1.0 - encoder_extended_attention_mask) * -10000.0 + else: + encoder_extended_attention_mask = None # Prepare head mask if needed # 1.0 in head_mask indicate we keep the head From 727a79b305364522b6853679c5523efd9de7f772 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 8 Nov 2019 11:35:03 +0100 Subject: [PATCH 022/505] added TF2 model and tests - updated templates --- .../adding_a_new_model/modeling_tf_xxx.py | 2 + templates/adding_a_new_model/modeling_xxx.py | 2 + transformers/__init__.py | 3 + transformers/configuration_auto.py | 6 +- transformers/configuration_t5.py | 3 +- transformers/modeling_t5.py | 79 +- transformers/modeling_tf_pytorch_utils.py | 4 +- transformers/modeling_tf_t5.py | 783 +++++++++++------- transformers/modeling_utils.py | 6 +- transformers/tests/modeling_tf_common_test.py | 23 +- transformers/tests/modeling_tf_t5_test.py | 116 ++- 11 files changed, 646 insertions(+), 381 deletions(-) diff --git a/templates/adding_a_new_model/modeling_tf_xxx.py b/templates/adding_a_new_model/modeling_tf_xxx.py index c661975768..b58817e453 100644 --- a/templates/adding_a_new_model/modeling_tf_xxx.py +++ b/templates/adding_a_new_model/modeling_tf_xxx.py @@ -26,6 +26,8 @@ import logging import math import os import sys +import copy +import itertools from io import open import numpy as np diff --git a/templates/adding_a_new_model/modeling_xxx.py b/templates/adding_a_new_model/modeling_xxx.py index ee705e753c..9c3505f0cf 100644 --- a/templates/adding_a_new_model/modeling_xxx.py +++ b/templates/adding_a_new_model/modeling_xxx.py @@ -25,6 +25,8 @@ import logging import math import os import sys +import copy +import itertools from io import open import torch diff --git a/transformers/__init__.py b/transformers/__init__.py index 601a068592..b882f4d968 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -158,6 +158,9 @@ if is_tf_available(): TFCTRLLMHeadModel, TF_CTRL_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_tf_t5 import (TFT5PreTrainedModel, TFT5Model, TFT5WithLMHeadModel, + TF_T5_PRETRAINED_MODEL_ARCHIVE_MAP) + # TF 2.0 <=> PyTorch conversion utilities from .modeling_tf_pytorch_utils import (convert_tf_weight_name_to_pt_weight_name, load_pytorch_checkpoint_in_tf2_model, diff --git a/transformers/configuration_auto.py b/transformers/configuration_auto.py index edd21a670c..3bee5b84a1 100644 --- a/transformers/configuration_auto.py +++ b/transformers/configuration_auto.py @@ -27,6 +27,7 @@ from .configuration_xlm import XLMConfig from .configuration_roberta import RobertaConfig from .configuration_distilbert import DistilBertConfig from .configuration_ctrl import CTRLConfig +from .configuration_t5 import T5Config logger = logging.getLogger(__name__) @@ -64,6 +65,7 @@ class AutoConfig(object): The configuration class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): + - contains `t5`: T5Config (T5 model) - contains `distilbert`: DistilBertConfig (DistilBERT model) - contains `bert`: BertConfig (Bert model) - contains `openai-gpt`: OpenAIGPTConfig (OpenAI GPT model) @@ -114,7 +116,9 @@ class AutoConfig(object): assert unused_kwargs == {'foo': False} """ - if 'distilbert' in pretrained_model_name_or_path: + if 't5' in pretrained_model_name_or_path: + return T5Config.from_pretrained(pretrained_model_name_or_path, **kwargs) + elif 'distilbert' in pretrained_model_name_or_path: return DistilBertConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return RobertaConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) diff --git a/transformers/configuration_t5.py b/transformers/configuration_t5.py index 96e67758ac..83aab66fac 100644 --- a/transformers/configuration_t5.py +++ b/transformers/configuration_t5.py @@ -27,8 +27,7 @@ from .configuration_utils import PretrainedConfig logger = logging.getLogger(__name__) T5_PRETRAINED_CONFIG_ARCHIVE_MAP = { - 't5-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-base-uncased-config.json", - 't5-large-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-large-uncased-config.json", + 't5-small': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-small-config.json", } diff --git a/transformers/modeling_t5.py b/transformers/modeling_t5.py index 6ed241761a..6be0ae6863 100644 --- a/transformers/modeling_t5.py +++ b/transformers/modeling_t5.py @@ -41,8 +41,7 @@ logger = logging.getLogger(__name__) # for the pretrained weights provided with the models #################################################### T5_PRETRAINED_MODEL_ARCHIVE_MAP = { - 't5-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-base-uncased-pytorch_model.bin", - 't5-large-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-large-uncased-pytorch_model.bin", + 't5-small': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-small-pytorch_model.bin", } #################################################### @@ -442,7 +441,7 @@ class T5PreTrainedModel(PreTrainedModel): if isinstance(module, nn.LayerNorm): module.bias.data.zero_() module.weight.data.fill_(factor*1.0) - elif isinstance(module, T5Model): + elif isinstance(module, (T5Model, T5WithLMHeadModel)): # Mesh TensorFlow embeddings initialization # See https://github.com/tensorflow/mesh/blob/fa19d69eafc9a482aff0b59ddd96b025c0cb207d/mesh_tensorflow/layers.py#L1624 module.shared.weight.data.normal_(mean=0.0, std=factor*1.0) @@ -502,11 +501,10 @@ class T5Stack(T5PreTrainedModel): # ourselves in which case we just need to make it broadcastable to all heads. if attention_mask.dim() == 3: extended_attention_mask = attention_mask[:, None, :, :] - + elif attention_mask.dim() == 2: # Provided a padding mask of dimensions [batch_size, seq_length] # - if the model is a decoder, apply a causal mask in addition to the padding mask # - if the model is an encoder, make the mask broadcastable to [batch_size, num_heads, seq_length, seq_length] - if attention_mask.dim() == 2: if self.config.is_decoder: seq_ids = torch.arange(seq_length, device=hidden_states.device) causal_mask = seq_ids[None, None, :].repeat(batch_size, seq_length, 1) <= seq_ids[None, :, None] @@ -593,7 +591,7 @@ class T5Stack(T5PreTrainedModel): T5_START_DOCSTRING = r""" The T5 model was proposed in `Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer`_ by Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu. - It's an encoder decoder pre-trained transformer. + It's an encoder decoder transformer pre-trained in a text-to-text denoising generative setting. This model is a PyTorch `torch.nn.Module`_ sub-class. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. @@ -634,16 +632,13 @@ T5_INPUTS_DOCSTRING = r""" Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. - **position_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: - Indices of positions of each input sequence tokens in the position embeddings. - Selected in the range ``[0, config.max_position_embeddings - 1]``. **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. """ -@add_start_docstrings("The bare single stack (encoder or decoder) of a T5 Model transformer outputting raw hidden-states" +@add_start_docstrings("The bare T5 Model transformer outputting raw hidden-states" "without any specific head on top.", T5_START_DOCSTRING, T5_INPUTS_DOCSTRING) class T5Model(T5PreTrainedModel): @@ -661,8 +656,8 @@ class T5Model(T5PreTrainedModel): Examples:: - tokenizer = T5Tokenizer.from_pretrained('t5-base-uncased') - model = T5Model.from_pretrained('t5-base-uncased') + tokenizer = T5Tokenizer.from_pretrained('t5-small') + model = T5Model.from_pretrained('t5-small') input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -752,8 +747,8 @@ class T5WithLMHeadModel(T5PreTrainedModel): Examples:: - tokenizer = T5Tokenizer.from_pretrained('t5-base-uncased') - model = T5WithLMHeadModel.from_pretrained('t5-base-uncased') + tokenizer = T5Tokenizer.from_pretrained('t5-small') + model = T5WithLMHeadModel.from_pretrained('t5-small') input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 outputs = model(input_ids, lm_labels=input_ids) loss, prediction_scores = outputs[:2] @@ -763,31 +758,73 @@ class T5WithLMHeadModel(T5PreTrainedModel): super(T5WithLMHeadModel, self).__init__(config) self.model_dim = config.d_model - self.transformer = T5Model(config) + self.shared = nn.Embedding(config.vocab_size, config.d_model) + + encoder_config = copy.deepcopy(config) + self.encoder = T5Stack(encoder_config) + + decoder_config = copy.deepcopy(config) + decoder_config.is_decoder = True + self.decoder = T5Stack(decoder_config) + self.lm_head = nn.Linear(config.d_model, config.vocab_size, bias=False) self.init_weights() + def get_input_embeddings(self): + return self.shared + + def set_input_embeddings(self, new_embeddings): + self.shared = new_embeddings + def get_output_embeddings(self): return self.lm_head def forward(self, **kwargs): - lm_labels = kwargs.pop('decoder_lm_labels', None) - outputs = self.transformer(**kwargs) + # keyword arguments come in 3 flavors: encoder-specific (prefixed by + # `encoder_`), decoder-specific (prefixed by `decoder_`) and those + # that apply to the model as whole. + # We let the specific kwargs override the common ones in case of conflict. - sequence_output = outputs[0] + lm_labels = kwargs.pop('decoder_lm_labels', None) + + kwargs_common = dict((k, v) for k, v in kwargs.items() + if not k.startswith("encoder_") and not k.startswith("decoder_")) + kwargs_encoder = kwargs_common.copy() + kwargs_decoder = kwargs_common.copy() + kwargs_encoder.update(dict((k[len("encoder_"):], v) for k, v in kwargs.items() if k.startswith("encoder_"))) + kwargs_decoder.update(dict((k[len("decoder_"):], v) for k, v in kwargs.items() if k.startswith("decoder_"))) + + # Encode if needed (training, first prediction pass) + encoder_hidden_states = kwargs_encoder.pop("hidden_states", None) + if encoder_hidden_states is None: + encoder_inputs_ids = kwargs_encoder.pop("input_ids") + hidden_states = self.shared(encoder_inputs_ids) # Convert inputs in embeddings + encoder_outputs = self.encoder(hidden_states, **kwargs_encoder) + encoder_hidden_states = encoder_outputs[0] + else: + encoder_outputs = () + + # Decode + decoder_inputs_ids = kwargs_decoder.pop("input_ids") + hidden_states = self.shared(decoder_inputs_ids) # Convert inputs in embeddings + kwargs_decoder["encoder_hidden_states"] = encoder_hidden_states + kwargs_decoder["encoder_attention_mask"] = kwargs_encoder.get("attention_mask", None) + decoder_outputs = self.decoder(hidden_states, **kwargs_decoder) + + sequence_output = decoder_outputs[0] # Rescale output before projecting on vocab # See https://github.com/tensorflow/mesh/blob/fa19d69eafc9a482aff0b59ddd96b025c0cb207d/mesh_tensorflow/transformer/transformer.py#L586 sequence_output = sequence_output * (self.model_dim ** -0.5) lm_logits = self.lm_head(sequence_output) - outputs = (lm_logits,) + outputs[1:] # Add hidden states and attention if they are here + decoder_outputs = (lm_logits,) + decoder_outputs[1:] # Add hidden states and attention if they are here if lm_labels is not None: shift_logits = lm_logits[..., :-1, :].contiguous() shift_labels = lm_labels[..., 1:].contiguous() loss_fct = CrossEntropyLoss(ignore_index=-1) loss = loss_fct(shift_logits.view(-1, shift_logits.size(-1)), shift_labels.view(-1)) - outputs = (loss,) + outputs # TODO(thom): Add z_loss https://github.com/tensorflow/mesh/blob/fa19d69eafc9a482aff0b59ddd96b025c0cb207d/mesh_tensorflow/layers.py#L666 + decoder_outputs = (loss,) + decoder_outputs # TODO(thom): Add z_loss https://github.com/tensorflow/mesh/blob/fa19d69eafc9a482aff0b59ddd96b025c0cb207d/mesh_tensorflow/layers.py#L666 - return outputs # (lm_loss), lm_logits, (hidden_states), (attentions) + return decoder_outputs + encoder_outputs diff --git a/transformers/modeling_tf_pytorch_utils.py b/transformers/modeling_tf_pytorch_utils.py index 88ce4d4610..6330c2748c 100644 --- a/transformers/modeling_tf_pytorch_utils.py +++ b/transformers/modeling_tf_pytorch_utils.py @@ -156,7 +156,7 @@ def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None, a e.args += (symbolic_weight.shape, array.shape) raise e - logger.info("Initialize TF weight {}".format(symbolic_weight.name)) + # logger.warning("Initialize TF weight {}".format(symbolic_weight.name)) weight_value_tuples.append((symbolic_weight, array)) all_pytorch_weights.discard(name) @@ -269,7 +269,7 @@ def load_tf2_weights_in_pytorch_model(pt_model, tf_weights, allow_missing_keys=F e.args += (pt_weight.shape, array.shape) raise e - logger.info("Initialize PyTorch weight {}".format(pt_weight_name)) + # logger.warning("Initialize PyTorch weight {}".format(pt_weight_name)) new_pt_params_dict[pt_weight_name] = torch.from_numpy(array) loaded_pt_weights_data_ptr[pt_weight.data_ptr()] = torch.from_numpy(array) diff --git a/transformers/modeling_tf_t5.py b/transformers/modeling_tf_t5.py index deb453846c..c1de4745c2 100644 --- a/transformers/modeling_tf_t5.py +++ b/transformers/modeling_tf_t5.py @@ -22,24 +22,21 @@ import logging import math import os import sys +import copy +import itertools from io import open import numpy as np import tensorflow as tf from .configuration_t5 import T5Config -from .modeling_tf_utils import TFPreTrainedModel, get_initializer +from .modeling_tf_utils import TFPreTrainedModel, TFSharedEmbeddings, shape_list, get_initializer, DUMMY_INPUTS from .file_utils import add_start_docstrings logger = logging.getLogger(__name__) -#################################################### -# This dict contrains shortcut names and associated url -# for the pretrained weights provided with the models -#################################################### TF_T5_PRETRAINED_MODEL_ARCHIVE_MAP = { - 't5-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-base-uncased-tf_model.h5", - 't5-large-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-large-uncased-tf_model.h5", + 't5-small': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-small-tf_model.h5", } #################################################### @@ -48,33 +45,294 @@ TF_T5_PRETRAINED_MODEL_ARCHIVE_MAP = { # - TFPreTrainedModel for the models (it-self a sub-class of tf.keras.Model) #################################################### -#################################################### -# Here is an example of typical layer in a TF 2.0 model of the library -# The classes are usually identical to the PyTorch ones and prefixed with 'TF'. -# -# Note that class __init__ parameters includes **kwargs (send to 'super'). -# This let us have a control on class scope and variable names: -# More precisely, we set the names of the class attributes (lower level layers) to -# to the equivalent attributes names in the PyTorch model so we can have equivalent -# class and scope structure between PyTorch and TF 2.0 models and easily load one in the other. -# -# See the conversion methods in modeling_tf_pytorch_utils.py for more details -#################################################### -class TFT5Layer(tf.keras.layers.Layer): +class TFT5DenseReluDense(tf.keras.layers.Layer): def __init__(self, config, **kwargs): - super(TFT5Layer, self).__init__(**kwargs) - self.attention = TFT5Attention(config, name='attention') - self.intermediate = TFT5Intermediate(config, name='intermediate') - self.transformer_output = TFT5Output(config, name='output') + super(TFT5DenseReluDense, self).__init__(**kwargs) + self.wi = tf.keras.layers.Dense(config.d_ff, use_bias=False, name='wi') + self.wo = tf.keras.layers.Dense(config.d_model, use_bias=False, name='wo') + self.dropout = tf.keras.layers.Dropout(config.dropout_rate) + self.act = tf.keras.activations.relu - def call(self, inputs, training=False): - hidden_states, attention_mask, head_mask = inputs + def call(self, hidden_states, training=False): + h = self.wi(hidden_states) + h = self.act(h) + h = self.dropout(h, training=training) + h = self.wo(h) + return h - attention_outputs = self.attention([hidden_states, attention_mask, head_mask], training=training) - attention_output = attention_outputs[0] - intermediate_output = self.intermediate(attention_output) - layer_output = self.transformer_output([intermediate_output, attention_output], training=training) - outputs = (layer_output,) + attention_outputs[1:] # add attentions if we output them + +class TFT5LayerFF(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFT5LayerFF, self).__init__(**kwargs) + self.DenseReluDense = TFT5DenseReluDense(config, name='DenseReluDense') + self.layer_norm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_epsilon, + name='layer_norm') + self.dropout = tf.keras.layers.Dropout(config.dropout_rate) + + def call(self, hidden_states, training=False): + norm_x = self.layer_norm(hidden_states) + y = self.DenseReluDense(norm_x, training=training) + layer_output = hidden_states + self.dropout(y, training=training) + return layer_output + + +class TFT5Attention(tf.keras.layers.Layer): + NEW_ID = itertools.count() + + def __init__(self, config, has_relative_attention_bias=False, **kwargs): + super(TFT5Attention, self).__init__(**kwargs) + self.layer_id = next(TFT5Attention.NEW_ID) + self.is_decoder = config.is_decoder + self.has_relative_attention_bias = has_relative_attention_bias + + self.output_attentions = config.output_attentions + self.relative_attention_num_buckets = config.relative_attention_num_buckets + self.dim = config.d_model + self.d_kv = config.d_kv + self.n_heads = config.num_heads + assert self.dim % self.n_heads == 0 + assert self.dim // self.n_heads == self.d_kv + + # Mesh TensorFlow initialization to avoid scaling before softmax + self.q = tf.keras.layers.Dense(self.dim, use_bias=False, name='q') + self.k = tf.keras.layers.Dense(self.dim, use_bias=False, name='k') + self.v = tf.keras.layers.Dense(self.dim, use_bias=False, name='v') + self.o = tf.keras.layers.Dense(self.dim, use_bias=False, name='o') + self.dropout = tf.keras.layers.Dropout(config.dropout_rate) + + if self.has_relative_attention_bias: + self.relative_attention_bias = tf.keras.layers.Embedding(self.relative_attention_num_buckets, + self.n_heads, + name='relative_attention_bias') + self.pruned_heads = set() + + def prune_heads(self, heads): + raise NotImplementedError + + @staticmethod + def _relative_position_bucket(relative_position, + bidirectional=True, + num_buckets=32, + max_distance=128): + """ + Adapted from Mesh Tensorflow: + https://github.com/tensorflow/mesh/blob/0cb87fe07da627bf0b7e60475d59f95ed6b5be3d/mesh_tensorflow/transformer/transformer_layers.py#L593 + + Translate relative position to a bucket number for relative attention. + The relative position is defined as memory_position - query_position, i.e. + the distance in tokens from the attending position to the attended-to + position. If bidirectional=False, then positive relative positions are + invalid. + We use smaller buckets for small absolute relative_position and larger buckets + for larger absolute relative_positions. All relative positions >=max_distance + map to the same bucket. All relative positions <=-max_distance map to the + same bucket. This should allow for more graceful generalization to longer + sequences than the model has been trained on. + Args: + relative_position: an int32 Tensor + bidirectional: a boolean - whether the attention is bidirectional + num_buckets: an integer + max_distance: an integer + Returns: + a Tensor with the same shape as relative_position, containing int32 + values in the range [0, num_buckets) + """ + ret = 0 + n = -relative_position + if bidirectional: + num_buckets //= 2 + ret += tf.dtypes.cast(tf.math.less(n, 0), tf.int32) * num_buckets + n = tf.math.abs(n) + else: + n = tf.math.maximum(n, 0) + # now n is in the range [0, inf) + max_exact = num_buckets // 2 + is_small = tf.math.less(n, max_exact) + val_if_large = max_exact + tf.dtypes.cast( + tf.math.log(tf.dtypes.cast(n, tf.float32) / max_exact) + / math.log(max_distance / max_exact) * (num_buckets - max_exact), tf.int32) + val_if_large = tf.math.minimum(val_if_large, num_buckets - 1) + ret += tf.where(is_small, n, val_if_large) + return ret + + def compute_bias(self, qlen, klen): + """ Compute binned relative position bias """ + context_position = tf.range(qlen)[:, None] + memory_position = tf.range(klen)[None, :] + relative_position = memory_position - context_position # shape (qlen, klen) + rp_bucket = self._relative_position_bucket(relative_position, + bidirectional=not self.is_decoder, + num_buckets=self.relative_attention_num_buckets) + values = self.relative_attention_bias(rp_bucket) # shape (qlen, klen, num_heads) + values = tf.expand_dims(tf.transpose(values, [2, 0, 1]), axis=0) # shape (1, num_heads, qlen, klen) + return values + + def call(self, input, mask=None, kv=None, position_bias=None, cache=None, head_mask=None, training=False): + """ + Self-attention (if kv is None) or attention over source sentence (provided by kv). + """ + # Input is (bs, qlen, dim) + # Mask is (bs, klen) (non-causal) or (bs, klen, klen) + bs, qlen, dim = shape_list(input) + if kv is None: + klen = qlen if cache is None else cache['slen'] + qlen + else: + klen = shape_list(kv)[1] + # assert dim == self.dim, 'Dimensions do not match: %s input vs %s configured' % (dim, self.dim) + n_heads = self.n_heads + dim_per_head = self.dim // n_heads + + def shape(x): + """ projection """ + return tf.transpose(tf.reshape(x, (bs, -1, self.n_heads, dim_per_head)), perm=(0, 2, 1, 3)) + + def unshape(x): + """ compute context """ + return tf.reshape(tf.transpose(x, perm=(0, 2, 1, 3)), (bs, -1, self.n_heads * dim_per_head)) + + q = shape(self.q(input)) # (bs, n_heads, qlen, dim_per_head) + if kv is None: + k = shape(self.k(input)) # (bs, n_heads, qlen, dim_per_head) + v = shape(self.v(input)) # (bs, n_heads, qlen, dim_per_head) + elif cache is None or self.layer_id not in cache: + k = v = kv + k = shape(self.k(k)) # (bs, n_heads, qlen, dim_per_head) + v = shape(self.v(v)) # (bs, n_heads, qlen, dim_per_head) + + if cache is not None: + if self.layer_id in cache: + if kv is None: + k_, v_ = cache[self.layer_id] + k = tf.concat([k_, k], axis=2) # (bs, n_heads, klen, dim_per_head) + v = tf.concat([v_, v], axis=2) # (bs, n_heads, klen, dim_per_head) + else: + k, v = cache[self.layer_id] + cache[self.layer_id] = (k, v) + + # q = q / math.sqrt(dim_per_head) # No scaling in T5 + scores = tf.matmul(q, k, transpose_b=True) # (bs, n_heads, qlen, klen) + + if position_bias is None: + if not self.has_relative_attention_bias: + raise ValueError("No position_bias provided and no weights to compute position_bias") + position_bias = self.compute_bias(qlen, klen) + scores += position_bias + + if mask is not None: + scores += mask + # mask = (mask == 0).expand_as(scores) # (bs, n_heads, qlen, klen) + # scores.masked_fill_(mask, -float('inf')) # (bs, n_heads, qlen, klen) + + weights = tf.nn.softmax(scores, axis=-1) # (bs, n_heads, qlen, klen) + weights = self.dropout(weights, training=training) # (bs, n_heads, qlen, klen) + + # Mask heads if we want to + if head_mask is not None: + weights = weights * head_mask + + context = tf.matmul(weights, v) # (bs, n_heads, qlen, dim_per_head) + context = unshape(context) # (bs, qlen, dim) + + context = self.o(context) + + outputs = (context,) + if self.output_attentions: + outputs = outputs + (weights,) + if self.has_relative_attention_bias: + outputs = outputs + (position_bias,) + return outputs + + +class TFT5LayerSelfAttention(tf.keras.layers.Layer): + def __init__(self, config, has_relative_attention_bias=False, **kwargs): + super(TFT5LayerSelfAttention, self).__init__(**kwargs) + self.SelfAttention = TFT5Attention(config, + has_relative_attention_bias=has_relative_attention_bias, + name='SelfAttention') + self.layer_norm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_epsilon, + name='layer_norm') + self.dropout = tf.keras.layers.Dropout(config.dropout_rate) + + def call(self, hidden_states, attention_mask=None, position_bias=None, + head_mask=None, training=False): + norm_x = self.layer_norm(hidden_states) + attention_output = self.SelfAttention(norm_x, + mask=attention_mask, + position_bias=position_bias, + head_mask=head_mask, + training=training) + y = attention_output[0] + layer_output = hidden_states + self.dropout(y, training=training) + outputs = (layer_output,) + attention_output[1:] # add attentions if we output them + return outputs + + +class TFT5LayerCrossAttention(tf.keras.layers.Layer): + def __init__(self, config, has_relative_attention_bias=False, **kwargs): + super(TFT5LayerCrossAttention, self).__init__(**kwargs) + self.EncDecAttention = TFT5Attention(config, + has_relative_attention_bias=has_relative_attention_bias, + name='EncDecAttention') + self.layer_norm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_epsilon, + name='layer_norm') + self.dropout = tf.keras.layers.Dropout(config.dropout_rate) + + def call(self, hidden_states, kv, attention_mask=None, position_bias=None, + head_mask=None, training=False): + norm_x = self.layer_norm(hidden_states) + attention_output = self.EncDecAttention(norm_x, + mask=attention_mask, + kv=kv, + position_bias=position_bias, + head_mask=head_mask, + training=training) + y = attention_output[0] + layer_output = hidden_states + self.dropout(y, training=training) + outputs = (layer_output,) + attention_output[1:] # add attentions if we output them + return outputs + + +class TFT5Block(tf.keras.layers.Layer): + def __init__(self, config, has_relative_attention_bias=False, **kwargs): + super(TFT5Block, self).__init__(**kwargs) + self.is_decoder = config.is_decoder + self.layer = [] + self.layer.append(TFT5LayerSelfAttention(config, + has_relative_attention_bias=has_relative_attention_bias, + name='layer_._0')) + if self.is_decoder: + self.layer.append(TFT5LayerCrossAttention(config, + has_relative_attention_bias=has_relative_attention_bias, + name='layer_._1')) + self.layer.append(TFT5LayerFF(config, name='layer_._2')) + else: + self.layer.append(TFT5LayerFF(config, name='layer_._1')) + + def call(self, hidden_states, attention_mask=None, position_bias=None, + encoder_hidden_states=None, encoder_attention_mask=None, encoder_decoder_position_bias=None, + head_mask=None, training=False): + self_attention_outputs = self.layer[0](hidden_states, + attention_mask=attention_mask, + position_bias=position_bias, + head_mask=head_mask, + training=training) + hidden_states = self_attention_outputs[0] + outputs = self_attention_outputs[1:] + + if not self.is_decoder: + hidden_states = self.layer[1](hidden_states, training=training) + else: + cross_attention_outputs = self.layer[1](hidden_states, + kv=encoder_hidden_states, + attention_mask=encoder_attention_mask, + position_bias=encoder_decoder_position_bias, + head_mask=head_mask, + training=training) + hidden_states = cross_attention_outputs[0] + outputs = cross_attention_outputs[1:] + outputs + hidden_states = self.layer[2](hidden_states, training=training) + + outputs = (hidden_states,) + outputs # add attentions if we output them return outputs @@ -85,6 +343,19 @@ class TFT5Layer(tf.keras.layers.Layer): class TFT5MainLayer(tf.keras.layers.Layer): def __init__(self, config, **kwargs): super(TFT5MainLayer, self).__init__(**kwargs) + self.output_attentions = config.output_attentions + self.output_hidden_states = config.output_hidden_states + self.is_decoder = config.is_decoder + self.config = config + self.num_hidden_layers = config.num_layers + + self.block = [TFT5Block(config, + has_relative_attention_bias=bool(i == 0), + name='block_._{}'.format(i)) + for i in range(config.num_layers)] + self.final_layer_norm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_epsilon, + name='final_layer_norm') + self.dropout = tf.keras.layers.Dropout(config.dropout_rate) def _resize_token_embeddings(self, new_num_tokens): raise NotImplementedError # Not implemented yet in the library fr TF 2.0 models @@ -92,51 +363,56 @@ class TFT5MainLayer(tf.keras.layers.Layer): def _prune_heads(self, heads_to_prune): raise NotImplementedError # Not implemented yet in the library fr TF 2.0 models - def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, training=False): - # We allow three types of multi-inputs: - # - traditional keyword arguments in the call method - # - all the arguments provided as a dict in the first positional argument of call - # - all the arguments provided as a list/tuple (ordered) in the first positional argument of call - # The last two options are useful to use the tf.keras fit() method. - - if isinstance(inputs, (tuple, list)): - input_ids = inputs[0] - attention_mask = inputs[1] if len(inputs) > 1 else attention_mask - token_type_ids = inputs[2] if len(inputs) > 2 else token_type_ids - position_ids = inputs[3] if len(inputs) > 3 else position_ids - head_mask = inputs[4] if len(inputs) > 4 else head_mask - assert len(inputs) <= 5, "Too many inputs." - elif isinstance(inputs, dict): - input_ids = inputs.get('input_ids') - attention_mask = inputs.get('attention_mask', attention_mask) - token_type_ids = inputs.get('token_type_ids', token_type_ids) - position_ids = inputs.get('position_ids', position_ids) - head_mask = inputs.get('head_mask', head_mask) - assert len(inputs) <= 5, "Too many inputs." - else: - input_ids = inputs + def call(self, hidden_states, attention_mask=None, encoder_hidden_states=None, + encoder_attention_mask=None, head_mask=None, training=False): + batch_size, seq_length = shape_list(hidden_states)[:2] if attention_mask is None: - attention_mask = tf.fill(tf.shape(input_ids), 1) - if token_type_ids is None: - token_type_ids = tf.fill(tf.shape(input_ids), 0) + attention_mask = tf.fill((batch_size, seq_length), 1) + if self.is_decoder and encoder_attention_mask is None: + encoder_seq_length = encoder_hidden_states.shape[1] + encoder_attention_mask = tf.fill((batch_size, encoder_seq_length), 1) - # We create a 3D attention mask from a 2D tensor mask. - # Sizes are [batch_size, 1, 1, to_seq_length] - # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length] - # this attention mask is more simple than the triangular masking of causal attention - # used in OpenAI GPT, we just need to prepare the broadcast dimension here. - extended_attention_mask = attention_mask[:, tf.newaxis, tf.newaxis, :] + # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] + # ourselves in which case we just need to make it broadcastable to all heads. + attention_mask = tf.cast(attention_mask, dtype=tf.float32) + num_dims_attention_mask = len(shape_list(attention_mask)) + if num_dims_attention_mask == 3: + extended_attention_mask = attention_mask[:, None, :, :] + elif num_dims_attention_mask == 2: + # Provided a padding mask of dimensions [batch_size, seq_length] + # - if the model is a decoder, apply a causal mask in addition to the padding mask + # - if the model is an encoder, make the mask broadcastable to [batch_size, num_heads, seq_length, seq_length] + if self.config.is_decoder: + seq_ids = tf.range(seq_length) + causal_mask = tf.less_equal(tf.tile(seq_ids[None, None, :], (batch_size, seq_length, 1)), + seq_ids[None, :, None]) + causal_mask = tf.cast(causal_mask, dtype=tf.float32) + extended_attention_mask = causal_mask[:, None, :, :] * attention_mask[:, None, None, :] + else: + extended_attention_mask = attention_mask[:, None, None, :] # Since attention_mask is 1.0 for positions we want to attend and 0.0 for # masked positions, this operation will create a tensor which is 0.0 for # positions we want to attend and -10000.0 for masked positions. # Since we are adding it to the raw scores before the softmax, this is # effectively the same as removing these entirely. - - extended_attention_mask = tf.cast(extended_attention_mask, tf.float32) extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + if self.is_decoder: + # If a 2D ou 3D attention mask is provided for the cross-attention + # we need to make broadcastabe to [batch_size, num_heads, seq_length, seq_length] + encoder_attention_mask = tf.cast(encoder_attention_mask, dtype=tf.float32) + num_dims_encoder_attention_mask = len(shape_list(encoder_attention_mask)) + if num_dims_encoder_attention_mask == 3: + encoder_extended_attention_mask = encoder_attention_mask[:, None, :, :] + if num_dims_encoder_attention_mask == 2: + encoder_extended_attention_mask = encoder_attention_mask[:, None, None, :] + + encoder_extended_attention_mask = (1.0 - encoder_extended_attention_mask) * -10000.0 + else: + encoder_extended_attention_mask = None + # Prepare head mask if needed # 1.0 in head_mask indicate we keep the head # attention_probs has shape bsz x n_heads x N x N @@ -148,14 +424,44 @@ class TFT5MainLayer(tf.keras.layers.Layer): head_mask = [None] * self.num_hidden_layers # head_mask = tf.constant([0] * self.num_hidden_layers) - ################################## - # Replace this with your model code - embedding_output = self.embeddings(input_ids, position_ids=position_ids, token_type_ids=token_type_ids) - encoder_outputs = self.encoder([embedding_output, extended_attention_mask, head_mask], training=training) - sequence_output = encoder_outputs[0] - outputs = (sequence_output,) + encoder_outputs[1:] # add hidden_states and attentions if they are here + all_hidden_states = () + all_attentions = () + position_bias = None + encoder_decoder_position_bias = None + for i, layer_module in enumerate(self.block): + if self.output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) - return outputs # sequence_output, (hidden_states), (attentions) + layer_outputs = layer_module(hidden_states, + attention_mask=extended_attention_mask, + position_bias=position_bias, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_extended_attention_mask, + encoder_decoder_position_bias=encoder_decoder_position_bias, + head_mask=head_mask[i], + training=training) + hidden_states = layer_outputs[0] + if i == 0: + position_bias = layer_outputs[2 if self.output_attentions else 1] + if self.is_decoder: + encoder_decoder_position_bias = layer_outputs[4 if self.output_attentions else 2] + + if self.output_attentions: + all_attentions = all_attentions + (layer_outputs[1],) + + hidden_states = self.final_layer_norm(hidden_states) + layer_output = self.dropout(hidden_states, training=training) + + # Add last layer + if self.output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + outputs = (hidden_states,) + if self.output_hidden_states: + outputs = outputs + (all_hidden_states,) + if self.output_attentions: + outputs = outputs + (all_attentions,) + return outputs # last-layer hidden state, (all hidden states), (all attentions) #################################################### @@ -173,18 +479,26 @@ class TFT5PreTrainedModel(TFPreTrainedModel): pretrained_model_archive_map = TF_T5_PRETRAINED_MODEL_ARCHIVE_MAP base_model_prefix = "transformer" + @property + def dummy_inputs(self): + input_ids = tf.constant([[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]]) + input_mask = tf.constant([[1, 1, 0, 0, 1], [1, 1, 1, 0, 0], [1, 0, 0, 1, 1]]) + dummy_inputs = {'decoder_input_ids': input_ids, + 'encoder_input_ids': input_ids, + 'decoder_attention_mask': input_mask} + return dummy_inputs -T5_START_DOCSTRING = r""" The XXX model was proposed in - `XXX: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ - by Jacob Devlin, Ming-Wei Chang, Kenton Lee and Kristina Toutanova. It's a bidirectional transformer - pre-trained using a combination of masked language modeling objective and next sentence prediction - on a large corpus comprising the Toronto Book Corpus and Wikipedia. + +T5_START_DOCSTRING = r""" The T5 model was proposed in + `Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer`_ + by Colin Raffel, Noam Shazeer, Adam Roberts, Katherine Lee, Sharan Narang, Michael Matena, Yanqi Zhou, Wei Li, Peter J. Liu. + It's an encoder decoder transformer pre-trained in a text-to-text denoising generative setting. This model is a tf.keras.Model `tf.keras.Model`_ sub-class. Use it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and behavior. - .. _`XXX: Pre-training of Deep Bidirectional Transformers for Language Understanding`: - https://arxiv.org/abs/1810.04805 + .. _`Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer`: + https://arxiv.org/abs/1910.10683 .. _`tf.keras.Model`: https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/Model @@ -206,67 +520,50 @@ T5_START_DOCSTRING = r""" The XXX model was proposed in `model({'input_ids': input_ids, 'token_type_ids': token_type_ids})` Parameters: - config (:class:`~transformers.XxxConfig`): Model configuration class with all the parameters of the model. + config (:class:`~transformers.T5Config`): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. """ -XXX_INPUTS_DOCSTRING = r""" +T5_INPUTS_DOCSTRING = r""" Inputs: **input_ids**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: Indices of input sequence tokens in the vocabulary. - To match pre-training, XXX input sequence should be formatted with [CLS] and [SEP] tokens as follows: + To match pre-training, T5 input sequence should be formatted with [CLS] and [SEP] tokens as follows: (a) For sequence pairs: ``tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]`` - - ``token_type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1`` (b) For single sequences: ``tokens: [CLS] the dog is hairy . [SEP]`` - - ``token_type_ids: 0 0 0 0 0 0 0`` - Xxx is a model with absolute position embeddings so it's usually advised to pad the inputs on - the right rather than the left. - Indices can be obtained using :class:`transformers.XxxTokenizer`. + T5 is a model with relative position embeddings so you should be able to pad the inputs on + the right or the left. + + Indices can be obtained using :class:`transformers.T5Tokenizer`. See :func:`transformers.PreTrainedTokenizer.encode` and :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. **attention_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: Mask to avoid performing attention on padding token indices. Mask values selected in ``[0, 1]``: ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. - **token_type_ids**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: - Segment token indices to indicate first and second portions of the inputs. - Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` - corresponds to a `sentence B` token - (see `XXX: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ for more details). - **position_ids**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: - Indices of positions of each input sequence tokens in the position embeddings. - Selected in the range ``[0, config.max_position_embeddings - 1]``. **head_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. """ -@add_start_docstrings("The bare Xxx Model transformer outputing raw hidden-states without any specific head on top.", - XXX_START_DOCSTRING, XXX_INPUTS_DOCSTRING) -class TFXxxModel(TFXxxPreTrainedModel): +@add_start_docstrings("The bare T5 Model transformer outputting raw hidden-states" + "without any specific head on top.", + T5_START_DOCSTRING, T5_INPUTS_DOCSTRING) +class TFT5Model(TFT5PreTrainedModel): r""" Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: **last_hidden_state**: ``tf.Tensor`` of shape ``(batch_size, sequence_length, hidden_size)`` Sequence of hidden-states at the output of the last layer of the model. - **pooler_output**: ``tf.Tensor`` of shape ``(batch_size, hidden_size)`` - Last layer hidden-state of the first token of the sequence (classification token) - further processed by a Linear layer and a Tanh activation function. The Linear - layer weights are trained from the next sentence prediction (classification) - objective during Xxx pretraining. This output is usually *not* a good summary - of the semantic content of the input, you're often better with averaging or pooling - the sequence of hidden-states for the whole input sequence. **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) list of ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) of shape ``(batch_size, sequence_length, hidden_size)``: @@ -278,27 +575,68 @@ class TFXxxModel(TFXxxPreTrainedModel): Examples:: import tensorflow as tf - from transformers import XxxTokenizer, TFXxxModel + from transformers import T5Tokenizer, TFT5Model - tokenizer = XxxTokenizer.from_pretrained('xxx-base-uncased') - model = TFXxxModel.from_pretrained('xxx-base-uncased') + tokenizer = T5Tokenizer.from_pretrained('t5-small') + model = TFT5Model.from_pretrained('t5-small') input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple """ def __init__(self, config, *inputs, **kwargs): - super(TFXxxModel, self).__init__(config, *inputs, **kwargs) - self.transformer = TFXxxMainLayer(config, name='transformer') + super(TFT5Model, self).__init__(config, *inputs, **kwargs) + self.shared = TFSharedEmbeddings(config.vocab_size, config.d_model, + name='shared') - def call(self, inputs, **kwargs): - outputs = self.transformer(inputs, **kwargs) - return outputs + encoder_config = copy.deepcopy(config) + self.encoder = TFT5MainLayer(encoder_config, name='encoder') + + decoder_config = copy.deepcopy(config) + decoder_config.is_decoder = True + self.decoder = TFT5MainLayer(decoder_config, name='decoder') + + def call(self, decoder_input_ids, **kwargs): + # We allow two types of multi-inputs: + # - traditional keyword arguments in the call method + # - all the arguments provided as a dict in the first positional argument of call + # The last option is useful to use the tf.keras fit() method. + + if isinstance(decoder_input_ids, dict): + kwargs.update(decoder_input_ids) + else: + kwargs['decoder_input_ids'] = decoder_input_ids + + kwargs_common = dict((k, v) for k, v in kwargs.items() + if not k.startswith("encoder_") and not k.startswith("decoder_")) + kwargs_encoder = kwargs_common.copy() + kwargs_decoder = kwargs_common.copy() + kwargs_encoder.update(dict((k[len("encoder_"):], v) for k, v in kwargs.items() if k.startswith("encoder_"))) + kwargs_decoder.update(dict((k[len("decoder_"):], v) for k, v in kwargs.items() if k.startswith("decoder_"))) + + # Encode if needed (training, first prediction pass) + encoder_hidden_states = kwargs_encoder.pop("hidden_states", None) + if encoder_hidden_states is None: + encoder_inputs_ids = kwargs_encoder.pop("input_ids") + hidden_states = self.shared(encoder_inputs_ids) # Convert inputs in embeddings + encoder_outputs = self.encoder(hidden_states, **kwargs_encoder) + encoder_hidden_states = encoder_outputs[0] + else: + encoder_outputs = () + + # Decode + decoder_inputs_ids = kwargs_decoder.pop("input_ids") + hidden_states = self.shared(decoder_inputs_ids) # Convert inputs in embeddings + kwargs_decoder["encoder_hidden_states"] = encoder_hidden_states + kwargs_decoder["encoder_attention_mask"] = kwargs_encoder.get("attention_mask", None) + decoder_outputs = self.decoder(hidden_states, **kwargs_decoder) + + return decoder_outputs + encoder_outputs -@add_start_docstrings("""Xxx Model with a `language modeling` head on top. """, - XXX_START_DOCSTRING, XXX_INPUTS_DOCSTRING) -class TFXxxForMaskedLM(TFXxxPreTrainedModel): +@add_start_docstrings("""T5 Model with a `language modeling` head on top. """, + T5_START_DOCSTRING, T5_INPUTS_DOCSTRING) +class TFT5WithLMHeadModel(TFT5PreTrainedModel): r""" Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: **prediction_scores**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` @@ -314,183 +652,66 @@ class TFXxxForMaskedLM(TFXxxPreTrainedModel): Examples:: import tensorflow as tf - from transformers import XxxTokenizer, TFXxxForMaskedLM + from transformers import T5Tokenizer, TFT5WithLMHeadModel - tokenizer = XxxTokenizer.from_pretrained('xxx-base-uncased') - model = TFXxxForMaskedLM.from_pretrained('xxx-base-uncased') + tokenizer = T5Tokenizer.from_pretrained('t5-small') + model = TFT5WithLMHeadModel.from_pretrained('t5-small') input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 outputs = model(input_ids) prediction_scores = outputs[0] """ def __init__(self, config, *inputs, **kwargs): - super(TFXxxForMaskedLM, self).__init__(config, *inputs, **kwargs) + super(TFT5WithLMHeadModel, self).__init__(config, *inputs, **kwargs) + self.model_dim = config.d_model - self.transformer = TFXxxMainLayer(config, name='transformer') - self.mlm = TFXxxMLMHead(config, self.transformer.embeddings, name='mlm') + self.shared = TFSharedEmbeddings(config.vocab_size, config.d_model, + name='shared') - def call(self, inputs, **kwargs): - outputs = self.transformer(inputs, **kwargs) + encoder_config = copy.deepcopy(config) + self.encoder = TFT5MainLayer(encoder_config, name='encoder') - sequence_output = outputs[0] - prediction_scores = self.mlm(sequence_output, training=kwargs.get('training', False)) + decoder_config = copy.deepcopy(config) + decoder_config.is_decoder = True + self.decoder = TFT5MainLayer(decoder_config, name='decoder') - outputs = (prediction_scores,) + outputs[2:] # Add hidden states and attention if they are here + def call(self, decoder_input_ids, **kwargs): + # We allow two types of multi-inputs: + # - traditional keyword arguments in the call method + # - all the arguments provided as a dict in the first positional argument of call + # The last option is useful to use the tf.keras fit() method. - return outputs # prediction_scores, (hidden_states), (attentions) + if isinstance(decoder_input_ids, dict): + kwargs.update(decoder_input_ids) + else: + kwargs['decoder_input_ids'] = decoder_input_ids + kwargs_common = dict((k, v) for k, v in kwargs.items() + if not k.startswith("encoder_") and not k.startswith("decoder_")) + kwargs_encoder = kwargs_common.copy() + kwargs_decoder = kwargs_common.copy() + kwargs_encoder.update(dict((k[len("encoder_"):], v) for k, v in kwargs.items() if k.startswith("encoder_"))) + kwargs_decoder.update(dict((k[len("decoder_"):], v) for k, v in kwargs.items() if k.startswith("decoder_"))) -@add_start_docstrings("""Xxx Model transformer with a sequence classification/regression head on top (a linear layer on top of - the pooled output) e.g. for GLUE tasks. """, - XXX_START_DOCSTRING, XXX_INPUTS_DOCSTRING) -class TFXxxForSequenceClassification(TFXxxPreTrainedModel): - r""" - Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **logits**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, config.num_labels)`` - Classification (or regression if config.num_labels==1) scores (before SoftMax). - **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) - list of ``Numpy array`` or ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) - of shape ``(batch_size, sequence_length, hidden_size)``: - Hidden-states of the model at the output of each layer plus the initial embedding outputs. - **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``Numpy array`` or ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: - Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + # Encode if needed (training, first prediction pass) + encoder_hidden_states = kwargs_encoder.pop("hidden_states", None) + if encoder_hidden_states is None: + encoder_inputs_ids = kwargs_encoder.pop("input_ids") + hidden_states = self.shared(encoder_inputs_ids) # Convert inputs in embeddings + encoder_outputs = self.encoder(hidden_states, **kwargs_encoder) + encoder_hidden_states = encoder_outputs[0] + else: + encoder_outputs = () - Examples:: + # Decode + decoder_inputs_ids = kwargs_decoder.pop("input_ids") + hidden_states = self.shared(decoder_inputs_ids) # Convert inputs in embeddings + kwargs_decoder["encoder_hidden_states"] = encoder_hidden_states + kwargs_decoder["encoder_attention_mask"] = kwargs_encoder.get("attention_mask", None) + decoder_outputs = self.decoder(hidden_states, **kwargs_decoder) - import tensorflow as tf - from transformers import XxxTokenizer, TFXxxForSequenceClassification + sequence_output = decoder_outputs[0] * (self.model_dim ** -0.5) + lm_logits = self.shared(sequence_output, mode="linear") + decoder_outputs = (lm_logits,) + decoder_outputs[1:] - tokenizer = XxxTokenizer.from_pretrained('xxx-base-uncased') - model = TFXxxForSequenceClassification.from_pretrained('xxx-base-uncased') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 - outputs = model(input_ids) - logits = outputs[0] - - """ - def __init__(self, config, *inputs, **kwargs): - super(TFXxxForSequenceClassification, self).__init__(config, *inputs, **kwargs) - self.num_labels = config.num_labels - - self.transformer = TFXxxMainLayer(config, name='transformer') - self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) - self.classifier = tf.keras.layers.Dense(config.num_labels, - kernel_initializer=get_initializer(config.initializer_range), - name='classifier') - - def call(self, inputs, **kwargs): - outputs = self.transformer(inputs, **kwargs) - - pooled_output = outputs[1] - - pooled_output = self.dropout(pooled_output, training=kwargs.get('training', False)) - logits = self.classifier(pooled_output) - - outputs = (logits,) + outputs[2:] # add hidden states and attention if they are here - - return outputs # logits, (hidden_states), (attentions) - - -@add_start_docstrings("""Xxx Model with a token classification head on top (a linear layer on top of - the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. """, - XXX_START_DOCSTRING, XXX_INPUTS_DOCSTRING) -class TFXxxForTokenClassification(TFXxxPreTrainedModel): - r""" - Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **scores**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, config.num_labels)`` - Classification scores (before SoftMax). - **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) - list of ``Numpy array`` or ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) - of shape ``(batch_size, sequence_length, hidden_size)``: - Hidden-states of the model at the output of each layer plus the initial embedding outputs. - **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``Numpy array`` or ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: - Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. - - Examples:: - - import tensorflow as tf - from transformers import XxxTokenizer, TFXxxForTokenClassification - - tokenizer = XxxTokenizer.from_pretrained('xxx-base-uncased') - model = TFXxxForTokenClassification.from_pretrained('xxx-base-uncased') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 - outputs = model(input_ids) - scores = outputs[0] - - """ - def __init__(self, config, *inputs, **kwargs): - super(TFXxxForTokenClassification, self).__init__(config, *inputs, **kwargs) - self.num_labels = config.num_labels - - self.transformer = TFXxxMainLayer(config, name='transformer') - self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) - self.classifier = tf.keras.layers.Dense(config.num_labels, - kernel_initializer=get_initializer(config.initializer_range), - name='classifier') - - def call(self, inputs, **kwargs): - outputs = self.transformer(inputs, **kwargs) - - sequence_output = outputs[0] - - sequence_output = self.dropout(sequence_output, training=kwargs.get('training', False)) - logits = self.classifier(sequence_output) - - outputs = (logits,) + outputs[2:] # add hidden states and attention if they are here - - return outputs # scores, (hidden_states), (attentions) - - -@add_start_docstrings("""Xxx Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of - the hidden-states output to compute `span start logits` and `span end logits`). """, - XXX_START_DOCSTRING, XXX_INPUTS_DOCSTRING) -class TFXxxForQuestionAnswering(TFXxxPreTrainedModel): - r""" - Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: - **start_scores**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length,)`` - Span-start scores (before SoftMax). - **end_scores**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length,)`` - Span-end scores (before SoftMax). - **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) - list of ``Numpy array`` or ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) - of shape ``(batch_size, sequence_length, hidden_size)``: - Hidden-states of the model at the output of each layer plus the initial embedding outputs. - **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``Numpy array`` or ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: - Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. - - Examples:: - - import tensorflow as tf - from transformers import XxxTokenizer, TFXxxForQuestionAnswering - - tokenizer = XxxTokenizer.from_pretrained('xxx-base-uncased') - model = TFXxxForQuestionAnswering.from_pretrained('xxx-base-uncased') - input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 - outputs = model(input_ids) - start_scores, end_scores = outputs[:2] - - """ - def __init__(self, config, *inputs, **kwargs): - super(TFXxxForQuestionAnswering, self).__init__(config, *inputs, **kwargs) - self.num_labels = config.num_labels - - self.transformer = TFXxxMainLayer(config, name='transformer') - self.qa_outputs = tf.keras.layers.Dense(config.num_labels, - kernel_initializer=get_initializer(config.initializer_range), - name='qa_outputs') - - def call(self, inputs, **kwargs): - outputs = self.transformer(inputs, **kwargs) - - sequence_output = outputs[0] - - logits = self.qa_outputs(sequence_output) - start_logits, end_logits = tf.split(logits, 2, axis=-1) - start_logits = tf.squeeze(start_logits, axis=-1) - end_logits = tf.squeeze(end_logits, axis=-1) - - outputs = (start_logits, end_logits,) + outputs[2:] - - return outputs # start_logits, end_logits, (hidden_states), (attentions) + return decoder_outputs + encoder_outputs diff --git a/transformers/modeling_utils.py b/transformers/modeling_utils.py index 063f52365d..5b1d3bb458 100644 --- a/transformers/modeling_utils.py +++ b/transformers/modeling_utils.py @@ -160,8 +160,7 @@ class PreTrainedModel(nn.Module): base_model.vocab_size = new_num_tokens # Tie weights again if needed - if hasattr(self, 'tie_weights'): - self.tie_weights() + self.tie_weights() return model_embeds @@ -458,8 +457,7 @@ class PreTrainedModel(nn.Module): raise RuntimeError('Error(s) in loading state_dict for {}:\n\t{}'.format( model.__class__.__name__, "\n\t".join(error_msgs))) - if hasattr(model, 'tie_weights'): - model.tie_weights() # make sure word embedding weights are still tied + model.tie_weights() # make sure word embedding weights are still tied if needed # Set model in evaluation mode to desactivate DropOut modules by default model.eval() diff --git a/transformers/tests/modeling_tf_common_test.py b/transformers/tests/modeling_tf_common_test.py index f636c42889..6c3954a088 100644 --- a/transformers/tests/modeling_tf_common_test.py +++ b/transformers/tests/modeling_tf_common_test.py @@ -69,6 +69,7 @@ class TFCommonTestCases: test_torchscript = True test_pruning = True test_resize_embeddings = True + is_encoder_decoder = False def test_initialization(self): pass @@ -156,7 +157,11 @@ class TFCommonTestCases: def test_compile_tf_model(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - input_ids = tf.keras.Input(batch_shape=(2, 2000), name='input_ids', dtype='int32') + if self.is_encoder_decoder: + input_ids = {'decoder_input_ids': tf.keras.Input(batch_shape=(2, 2000), name='decoder_input_ids', dtype='int32'), + 'encoder_input_ids': tf.keras.Input(batch_shape=(2, 2000), name='encoder_input_ids', dtype='int32')} + else: + input_ids = tf.keras.Input(batch_shape=(2, 2000), name='input_ids', dtype='int32') optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5, epsilon=1e-08, clipnorm=1.0) loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) metric = tf.keras.metrics.SparseCategoricalAccuracy('accuracy') @@ -189,7 +194,7 @@ class TFCommonTestCases: outputs_dict = model(inputs_dict) inputs_keywords = copy.deepcopy(inputs_dict) - input_ids = inputs_keywords.pop('input_ids') + input_ids = inputs_keywords.pop('input_ids', inputs_keywords.pop('decoder_input_ids')) outputs_keywords = model(input_ids, **inputs_keywords) output_dict = outputs_dict[0].numpy() @@ -216,12 +221,24 @@ class TFCommonTestCases: self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) out_len = len(outputs) + if self.is_encoder_decoder: + self.assertEqual(out_len % 2, 0) + decoder_attentions = outputs[(out_len // 2)-1] + self.assertEqual(model.config.output_attentions, True) + self.assertEqual(model.config.output_hidden_states, False) + self.assertEqual(len(decoder_attentions), self.model_tester.num_hidden_layers) + self.assertListEqual( + list(decoder_attentions[0].shape[-3:]), + [self.model_tester.num_attention_heads, + self.model_tester.seq_length, + self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) + # Check attention is always last and order is fine config.output_attentions = True config.output_hidden_states = True model = model_class(config) outputs = model(inputs_dict) - self.assertEqual(out_len+1, len(outputs)) + self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1), len(outputs)) self.assertEqual(model.config.output_attentions, True) self.assertEqual(model.config.output_hidden_states, True) diff --git a/transformers/tests/modeling_tf_t5_test.py b/transformers/tests/modeling_tf_t5_test.py index fac6763432..33f6f895f0 100644 --- a/transformers/tests/modeling_tf_t5_test.py +++ b/transformers/tests/modeling_tf_t5_test.py @@ -26,7 +26,7 @@ from .configuration_common_test import ConfigTester from transformers import T5Config, is_tf_available -if False: # is_tf_available(): +if is_tf_available(): import tensorflow as tf from transformers.modeling_tf_t5 import (TFT5Model, TFT5WithLMHeadModel,TF_T5_PRETRAINED_MODEL_ARCHIVE_MAP) else: @@ -35,7 +35,8 @@ else: class TFT5ModelTest(TFCommonTestCases.TFCommonModelTester): - all_model_classes = (TFT5Model, TFT5WithLMHeadModel) if False else () # is_tf_available() else () + is_encoder_decoder = True + all_model_classes = (TFT5Model, TFT5WithLMHeadModel) if is_tf_available() else () class TFT5ModelTester(object): @@ -45,22 +46,16 @@ class TFT5ModelTest(TFCommonTestCases.TFCommonModelTester): seq_length=7, is_training=True, use_input_mask=True, - use_token_type_ids=True, use_labels=True, vocab_size=99, + n_positions=14, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, - intermediate_size=37, - hidden_act="gelu", - hidden_dropout_prob=0.1, - attention_probs_dropout_prob=0.1, - max_position_embeddings=512, - type_vocab_size=16, - type_sequence_label_size=2, - initializer_range=0.02, - num_labels=3, - num_choices=4, + d_ff=37, + relative_attention_num_buckets=8, + dropout_rate=0.1, + initializer_factor=0.002, scope=None, ): self.parent = parent @@ -68,22 +63,16 @@ class TFT5ModelTest(TFCommonTestCases.TFCommonModelTester): self.seq_length = seq_length self.is_training = is_training self.use_input_mask = use_input_mask - self.use_token_type_ids = use_token_type_ids self.use_labels = use_labels self.vocab_size = vocab_size + self.n_positions = n_positions self.hidden_size = hidden_size self.num_hidden_layers = num_hidden_layers self.num_attention_heads = num_attention_heads - self.intermediate_size = intermediate_size - self.hidden_act = hidden_act - self.hidden_dropout_prob = hidden_dropout_prob - self.attention_probs_dropout_prob = attention_probs_dropout_prob - self.max_position_embeddings = max_position_embeddings - self.type_vocab_size = type_vocab_size - self.type_sequence_label_size = type_sequence_label_size - self.initializer_range = initializer_range - self.num_labels = num_labels - self.num_choices = num_choices + self.d_ff = d_ff + self.relative_attention_num_buckets = relative_attention_num_buckets + self.dropout_rate = dropout_rate + self.initializer_factor = initializer_factor self.scope = scope def prepare_config_and_inputs(self): @@ -93,61 +82,53 @@ class TFT5ModelTest(TFCommonTestCases.TFCommonModelTester): if self.use_input_mask: input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) - token_type_ids = None - if self.use_token_type_ids: - token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size) - - sequence_labels = None token_labels = None - choice_labels = None if self.use_labels: - sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size) - token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels) - choice_labels = ids_tensor([self.batch_size], self.num_choices) + token_labels = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) config = T5Config( vocab_size_or_config_json_file=self.vocab_size, - hidden_size=self.hidden_size, - num_hidden_layers=self.num_hidden_layers, - num_attention_heads=self.num_attention_heads, - intermediate_size=self.intermediate_size, - hidden_act=self.hidden_act, - hidden_dropout_prob=self.hidden_dropout_prob, - attention_probs_dropout_prob=self.attention_probs_dropout_prob, - max_position_embeddings=self.max_position_embeddings, - type_vocab_size=self.type_vocab_size, - initializer_range=self.initializer_range) + n_positions=self.n_positions, + d_model=self.hidden_size, + d_ff=self.d_ff, + d_kv=self.hidden_size // self.num_attention_heads, + num_layers=self.num_hidden_layers, + num_heads=self.num_attention_heads, + relative_attention_num_buckets=self.relative_attention_num_buckets, + dropout_rate=self.dropout_rate, + initializer_factor=self.initializer_factor) - return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels + return (config, input_ids, input_mask, token_labels) - def create_and_check_t5_model(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + def create_and_check_t5_model(self, config, input_ids, input_mask, token_labels): model = TFT5Model(config=config) - inputs = {'input_ids': input_ids, - 'attention_mask': input_mask, - 'token_type_ids': token_type_ids} - sequence_output, pooled_output = model(inputs) + inputs = {'encoder_input_ids': input_ids, + 'decoder_input_ids': input_ids, + 'decoder_attention_mask': input_mask} + encoder_output, decoder_output = model(inputs) - inputs = [input_ids, input_mask] - sequence_output, pooled_output = model(inputs) - - sequence_output, pooled_output = model(input_ids) + encoder_output, decoder_output = model(input_ids, + decoder_attention_mask=input_mask, + encoder_input_ids=input_ids) result = { - "sequence_output": sequence_output.numpy(), - "pooled_output": pooled_output.numpy(), + "encoder_output": encoder_output.numpy(), + "decoder_output": decoder_output.numpy(), } self.parent.assertListEqual( - list(result["sequence_output"].shape), + list(result["encoder_output"].shape), + [self.batch_size, self.seq_length, self.hidden_size]) + self.parent.assertListEqual( + list(result["decoder_output"].shape), [self.batch_size, self.seq_length, self.hidden_size]) - self.parent.assertListEqual(list(result["pooled_output"].shape), [self.batch_size, self.hidden_size]) - def create_and_check_t5_with_lm_head(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + def create_and_check_t5_with_lm_head(self, config, input_ids, input_mask, token_labels): model = TFT5WithLMHeadModel(config=config) - inputs = {'input_ids': input_ids, - 'attention_mask': input_mask, - 'token_type_ids': token_type_ids} - prediction_scores, = model(inputs) + inputs = {'encoder_input_ids': input_ids, + 'decoder_input_ids': input_ids, + 'decoder_attention_mask': input_mask} + prediction_scores, decoder_output = model(inputs) result = { "prediction_scores": prediction_scores.numpy(), } @@ -158,14 +139,15 @@ class TFT5ModelTest(TFCommonTestCases.TFCommonModelTester): def prepare_config_and_inputs_for_common(self): config_and_inputs = self.prepare_config_and_inputs() - (config, input_ids, token_type_ids, input_mask, - sequence_labels, token_labels, choice_labels) = config_and_inputs - inputs_dict = {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask} + (config, input_ids, input_mask, token_labels) = config_and_inputs + inputs_dict = {'encoder_input_ids': input_ids, + 'decoder_input_ids': input_ids, + 'decoder_attention_mask': input_mask} return config, inputs_dict def setUp(self): self.model_tester = TFT5ModelTest.TFT5ModelTester(self) - self.config_tester = ConfigTester(self, config_class=T5Config, hidden_size=37) + self.config_tester = ConfigTester(self, config_class=T5Config, d_model=37) def test_config(self): self.config_tester.run_common_tests() @@ -181,7 +163,7 @@ class TFT5ModelTest(TFCommonTestCases.TFCommonModelTester): @pytest.mark.slow def test_model_from_pretrained(self): cache_dir = "/tmp/transformers_test/" - for model_name in ['t5-base']: + for model_name in ['t5-small']: model = TFT5Model.from_pretrained(model_name, cache_dir=cache_dir) shutil.rmtree(cache_dir) self.assertIsNotNone(model) From 4321c541254bdabbda631520cff0a5a376ad9f48 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 8 Nov 2019 11:49:32 +0100 Subject: [PATCH 023/505] fix tests --- transformers/tests/modeling_tf_common_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/tests/modeling_tf_common_test.py b/transformers/tests/modeling_tf_common_test.py index 6c3954a088..83a15c137a 100644 --- a/transformers/tests/modeling_tf_common_test.py +++ b/transformers/tests/modeling_tf_common_test.py @@ -194,7 +194,7 @@ class TFCommonTestCases: outputs_dict = model(inputs_dict) inputs_keywords = copy.deepcopy(inputs_dict) - input_ids = inputs_keywords.pop('input_ids', inputs_keywords.pop('decoder_input_ids')) + input_ids = inputs_keywords.pop('input_ids' if not self.is_encoder_decoder else 'decoder_input_ids', None) outputs_keywords = model(input_ids, **inputs_keywords) output_dict = outputs_dict[0].numpy() From f03c0c1423d4635f3e71a6c24053f01f6f02063c Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 8 Nov 2019 11:49:46 +0100 Subject: [PATCH 024/505] adding models in readme and auto classes --- README.md | 3 ++- docs/source/pretrained_models.rst | 20 +++++++++++++++++++ transformers/__main__.py | 18 +++++++++++++++++ .../convert_pytorch_checkpoint_to_tf2.py | 13 ++++++++---- transformers/modeling_auto.py | 13 ++++++++++-- transformers/modeling_tf_auto.py | 13 ++++++++++-- transformers/tokenization_auto.py | 7 ++++++- 7 files changed, 77 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 40b08583b1..d6f6e426d8 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,8 @@ At some point in the future, you'll be able to seamlessly move from pre-training 7. **[RoBERTa](https://github.com/pytorch/fairseq/tree/master/examples/roberta)** (from Facebook), released together with the paper a [Robustly Optimized BERT Pretraining Approach](https://arxiv.org/abs/1907.11692) by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov. 8. **[DistilBERT](https://github.com/huggingface/transformers/tree/master/examples/distillation)** (from HuggingFace), released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter](https://arxiv.org/abs/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same method has been applied to compress GPT2 into [DistilGPT2](https://github.com/huggingface/transformers/tree/master/examples/distillation). 9. **[CTRL](https://github.com/salesforce/ctrl/)** (from Salesforce) released with the paper [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/abs/1909.05858) by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher. -10. Want to contribute a new model? We have added a **detailed guide and templates** to guide you in the process of adding a new model. You can find them in the [`templates`](./templates) folder of the repository. Be sure to check the [contributing guidelines](./CONTRIBUTING.md) and contact the maintainers or open an issue to collect feedbacks before starting your PR. +10. **[T5](https://github.com/google-research/text-to-text-transfer-transformer)** (from Google AI) released with the paper [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/abs/1910.10683) by Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu. +11. Want to contribute a new model? We have added a **detailed guide and templates** to guide you in the process of adding a new model. You can find them in the [`templates`](./templates) folder of the repository. Be sure to check the [contributing guidelines](./CONTRIBUTING.md) and contact the maintainers or open an issue to collect feedbacks before starting your PR. These implementations have been tested on several datasets (see the example scripts) and should match the performances of the original implementations (e.g. ~93 F1 on SQuAD for BERT Whole-Word-Masking, ~88 F1 on RocStories for OpenAI GPT, ~18.3 perplexity on WikiText 103 for Transformer-XL, ~0.916 Peason R coefficient on STS-B for XLNet). You can find more details on the performances in the Examples section of the [documentation](https://huggingface.co/transformers/examples.html). diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst index 43c08228bd..c6240dc850 100644 --- a/docs/source/pretrained_models.rst +++ b/docs/source/pretrained_models.rst @@ -144,5 +144,25 @@ Here is the full list of the currently provided pretrained models together with | CTRL | ``ctrl`` | | 48-layer, 1280-hidden, 16-heads, 1.6B parameters | | | | | Salesforce's Large-sized CTRL English model | +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| T5 | ``t5-small`` | | 6-layer, 768-hidden, 12-heads, 66M parameters | +| | | | The DistilBERT model distilled from the BERT model `bert-base-uncased` checkpoint | +| | | (see `details `__) | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``t5-base`` | | 6-layer, 768-hidden, 12-heads, 66M parameters | +| | | | The DistilBERT model distilled from the BERT model `bert-base-uncased` checkpoint, with an additional linear layer. | +| | | (see `details `__) | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``t5-large`` | | 6-layer, 768-hidden, 12-heads, 82M parameters | +| | | | The DistilGPT2 model distilled from the GPT2 model `gpt2` checkpoint. | +| | | (see `details `__) | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``t5-3b`` | | 6-layer, 768-hidden, 12-heads, 82M parameters | +| | | | The DistilRoBERTa model distilled from the RoBERTa model `roberta-base` checkpoint. | +| | | (see `details `__) | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``t5-11b`` | | 6-layer, 768-hidden, 12-heads, 82M parameters | +| | | | The DistilRoBERTa model distilled from the RoBERTa model `roberta-base` checkpoint. | +| | | (see `details `__) | ++-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ .. `__ \ No newline at end of file diff --git a/transformers/__main__.py b/transformers/__main__.py index 31dbd24908..6136d768f6 100644 --- a/transformers/__main__.py +++ b/transformers/__main__.py @@ -6,6 +6,7 @@ def main(): "This command line utility let you convert original (author released) model checkpoint to pytorch.\n" "It should be used as one of: \n" ">> transformers bert TF_CHECKPOINT TF_CONFIG PYTORCH_DUMP_OUTPUT, \n" + ">> transformers t5 TF_CHECKPOINT TF_CONFIG PYTORCH_DUMP_OUTPUT, \n" ">> transformers gpt OPENAI_GPT_CHECKPOINT_FOLDER_PATH PYTORCH_DUMP_OUTPUT [OPENAI_GPT_CONFIG], \n" ">> transformers transfo_xl TF_CHECKPOINT_OR_DATASET PYTORCH_DUMP_OUTPUT [TF_CONFIG] or \n" ">> transformers gpt2 TF_CHECKPOINT PYTORCH_DUMP_OUTPUT [GPT2_CONFIG] or \n" @@ -21,6 +22,23 @@ def main(): "https://www.tensorflow.org/install/ for installation instructions.") raise + if len(sys.argv) != 5: + # pylint: disable=line-too-long + print("Should be used as `transformers bert TF_CHECKPOINT TF_CONFIG PYTORCH_DUMP_OUTPUT`") + else: + PYTORCH_DUMP_OUTPUT = sys.argv.pop() + TF_CONFIG = sys.argv.pop() + TF_CHECKPOINT = sys.argv.pop() + convert_tf_checkpoint_to_pytorch(TF_CHECKPOINT, TF_CONFIG, PYTORCH_DUMP_OUTPUT) + elif sys.argv[1] == "t5": + try: + from .convert_t5_original_tf_checkpoint_to_pytorch import convert_tf_checkpoint_to_pytorch + except ImportError: + print("transformers can only be used from the commandline to convert TensorFlow models in PyTorch, " + "In that case, it requires TensorFlow to be installed. Please see " + "https://www.tensorflow.org/install/ for installation instructions.") + raise + if len(sys.argv) != 5: # pylint: disable=line-too-long print("Should be used as `transformers bert TF_CHECKPOINT TF_CONFIG PYTORCH_DUMP_OUTPUT`") diff --git a/transformers/convert_pytorch_checkpoint_to_tf2.py b/transformers/convert_pytorch_checkpoint_to_tf2.py index e673b77dcc..19629172ff 100644 --- a/transformers/convert_pytorch_checkpoint_to_tf2.py +++ b/transformers/convert_pytorch_checkpoint_to_tf2.py @@ -33,7 +33,8 @@ from transformers import (load_pytorch_checkpoint_in_tf2_model, OpenAIGPTConfig, TFOpenAIGPTLMHeadModel, OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, RobertaConfig, TFRobertaForMaskedLM, TFRobertaForSequenceClassification, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, DistilBertConfig, TFDistilBertForMaskedLM, TFDistilBertForQuestionAnswering, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, - CTRLConfig, TFCTRLLMHeadModel, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP) + CTRLConfig, TFCTRLLMHeadModel, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP, + T5Config, TFT5WithLMHeadModel, T5_PRETRAINED_CONFIG_ARCHIVE_MAP) if is_torch_available(): import torch @@ -46,7 +47,8 @@ if is_torch_available(): OpenAIGPTLMHeadModel, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP, RobertaForMaskedLM, RobertaForSequenceClassification, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, DistilBertForMaskedLM, DistilBertForQuestionAnswering, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, - CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP) + CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP, + T5WithLMHeadModel, T5_PRETRAINED_MODEL_ARCHIVE_MAP) else: (BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, GPT2LMHeadModel, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP, @@ -56,7 +58,8 @@ else: OpenAIGPTLMHeadModel, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP, RobertaForMaskedLM, RobertaForSequenceClassification, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, DistilBertForMaskedLM, DistilBertForQuestionAnswering, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, - CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP) = ( + CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP, + T5WithLMHeadModel, T5_PRETRAINED_MODEL_ARCHIVE_MAP) = ( None, None, None, None, None, None, None, None, @@ -65,6 +68,7 @@ else: None, None, None, None, None, None, None, None, + None, None, None, None) @@ -85,7 +89,8 @@ MODEL_CLASSES = { 'roberta-large-mnli': (RobertaConfig, TFRobertaForSequenceClassification, RobertaForSequenceClassification, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP), 'distilbert': (DistilBertConfig, TFDistilBertForMaskedLM, DistilBertForMaskedLM, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP), 'distilbert-base-uncased-distilled-squad': (DistilBertConfig, TFDistilBertForQuestionAnswering, DistilBertForQuestionAnswering, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP), - 'ctrl': (CTRLConfig, TFCTRLLMHeadModel, CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP) + 'ctrl': (CTRLConfig, TFCTRLLMHeadModel, CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP), + 't5': (T5Config, TFT5WithLMHeadModel, T5WithLMHeadModel, T5_PRETRAINED_MODEL_ARCHIVE_MAP, T5_PRETRAINED_CONFIG_ARCHIVE_MAP), } def convert_pt_checkpoint_to_tf(model_type, pytorch_checkpoint_path, config_file, tf_dump_path, compare_with_pt_model=False, use_cached_models=True): diff --git a/transformers/modeling_auto.py b/transformers/modeling_auto.py index d98110d4bd..a2129176d3 100644 --- a/transformers/modeling_auto.py +++ b/transformers/modeling_auto.py @@ -27,6 +27,7 @@ from .modeling_xlnet import XLNetModel, XLNetLMHeadModel, XLNetForSequenceClassi from .modeling_xlm import XLMModel, XLMWithLMHeadModel, XLMForSequenceClassification, XLMForQuestionAnswering from .modeling_roberta import RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification from .modeling_distilbert import DistilBertModel, DistilBertForQuestionAnswering, DistilBertForMaskedLM, DistilBertForSequenceClassification +from .modeling_t5 import T5Model, T5WithLMHeadModel from .modeling_utils import PreTrainedModel, SequenceSummary @@ -47,6 +48,7 @@ class AutoModel(object): The base model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): + - contains `t5`: T5Model (T5 model) - contains `distilbert`: DistilBertModel (DistilBERT model) - contains `roberta`: RobertaModel (RoBERTa model) - contains `bert`: BertModel (Bert model) @@ -70,6 +72,7 @@ class AutoModel(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): + - contains `t5`: T5Model (T5 model) - contains `distilbert`: DistilBertModel (DistilBERT model) - contains `roberta`: RobertaModel (RoBERTa model) - contains `bert`: BertModel (Bert model) @@ -136,7 +139,9 @@ class AutoModel(object): model = AutoModel.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config) """ - if 'distilbert' in pretrained_model_name_or_path: + if 't5' in pretrained_model_name_or_path: + return T5Model.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'distilbert' in pretrained_model_name_or_path: return DistilBertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return RobertaModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) @@ -171,6 +176,7 @@ class AutoModelWithLMHead(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): + - contains `t5`: T5ModelWithLMHead (T5 model) - contains `distilbert`: DistilBertForMaskedLM (DistilBERT model) - contains `roberta`: RobertaForMaskedLM (RoBERTa model) - contains `bert`: BertForMaskedLM (Bert model) @@ -197,6 +203,7 @@ class AutoModelWithLMHead(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): + - contains `t5`: T5ModelWithLMHead (T5 model) - contains `distilbert`: DistilBertForMaskedLM (DistilBERT model) - contains `roberta`: RobertaForMaskedLM (RoBERTa model) - contains `bert`: BertForMaskedLM (Bert model) @@ -262,7 +269,9 @@ class AutoModelWithLMHead(object): model = AutoModelWithLMHead.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config) """ - if 'distilbert' in pretrained_model_name_or_path: + if 't5' in pretrained_model_name_or_path: + return T5WithLMHeadModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'distilbert' in pretrained_model_name_or_path: return DistilBertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return RobertaForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) diff --git a/transformers/modeling_tf_auto.py b/transformers/modeling_tf_auto.py index df0ad6e401..b24623dcdc 100644 --- a/transformers/modeling_tf_auto.py +++ b/transformers/modeling_tf_auto.py @@ -27,6 +27,7 @@ from .modeling_tf_xlm import TFXLMModel, TFXLMWithLMHeadModel, TFXLMForSequenceC from .modeling_tf_roberta import TFRobertaModel, TFRobertaForMaskedLM, TFRobertaForSequenceClassification from .modeling_tf_distilbert import TFDistilBertModel, TFDistilBertForQuestionAnswering, TFDistilBertForMaskedLM, TFDistilBertForSequenceClassification from .modeling_tf_ctrl import TFCTRLModel, TFCTRLLMHeadModel +from .modeling_tf_t5 import TFT5Model, TFT5WithLMHeadModel from .file_utils import add_start_docstrings @@ -45,6 +46,7 @@ class TFAutoModel(object): The base model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): + - contains `t5`: TFT5Model (T5 model) - contains `distilbert`: TFDistilBertModel (DistilBERT model) - contains `roberta`: TFRobertaModel (RoBERTa model) - contains `bert`: TFBertModel (Bert model) @@ -68,6 +70,7 @@ class TFAutoModel(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): + - contains `t5`: TFT5Model (T5 model) - contains `distilbert`: TFDistilBertModel (DistilBERT model) - contains `roberta`: TFRobertaModel (RoBERTa model) - contains `bert`: TFTFBertModel (Bert model) @@ -133,7 +136,9 @@ class TFAutoModel(object): model = TFAutoModel.from_pretrained('./pt_model/bert_pytorch_model.bin', from_pt=True, config=config) """ - if 'distilbert' in pretrained_model_name_or_path: + if 't5' in pretrained_model_name_or_path: + return TFT5Model.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'distilbert' in pretrained_model_name_or_path: return TFDistilBertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return TFRobertaModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) @@ -169,6 +174,7 @@ class TFAutoModelWithLMHead(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): + - contains `t5`: TFT5WithLMHeadModel (T5 model) - contains `distilbert`: TFDistilBertForMaskedLM (DistilBERT model) - contains `roberta`: TFRobertaForMaskedLM (RoBERTa model) - contains `bert`: TFBertForMaskedLM (Bert model) @@ -195,6 +201,7 @@ class TFAutoModelWithLMHead(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): + - contains `t5`: TFT5WithLMHeadModel (T5 model) - contains `distilbert`: TFDistilBertForMaskedLM (DistilBERT model) - contains `roberta`: TFRobertaForMaskedLM (RoBERTa model) - contains `bert`: TFBertForMaskedLM (Bert model) @@ -261,7 +268,9 @@ class TFAutoModelWithLMHead(object): model = TFAutoModelWithLMHead.from_pretrained('./pt_model/bert_pytorch_model.bin', from_pt=True, config=config) """ - if 'distilbert' in pretrained_model_name_or_path: + if 't5' in pretrained_model_name_or_path: + return TFT5WithLMHeadModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'distilbert' in pretrained_model_name_or_path: return TFDistilBertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return TFRobertaForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) diff --git a/transformers/tokenization_auto.py b/transformers/tokenization_auto.py index ec056de17f..5be2562448 100644 --- a/transformers/tokenization_auto.py +++ b/transformers/tokenization_auto.py @@ -27,6 +27,7 @@ from .tokenization_xlnet import XLNetTokenizer from .tokenization_xlm import XLMTokenizer from .tokenization_roberta import RobertaTokenizer from .tokenization_distilbert import DistilBertTokenizer +from .tokenization_t5 import T5Tokenizer logger = logging.getLogger(__name__) @@ -41,6 +42,7 @@ class AutoTokenizer(object): The tokenizer class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): + - contains `t5`: T5Tokenizer (T5 model) - contains `distilbert`: DistilBertTokenizer (DistilBert model) - contains `roberta`: RobertaTokenizer (RoBERTa model) - contains `bert`: BertTokenizer (Bert model) @@ -64,6 +66,7 @@ class AutoTokenizer(object): The tokenizer class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): + - contains `t5`: T5Tokenizer (T5 model) - contains `distilbert`: DistilBertTokenizer (DistilBert model) - contains `roberta`: RobertaTokenizer (XLM model) - contains `bert`: BertTokenizer (Bert model) @@ -101,7 +104,9 @@ class AutoTokenizer(object): tokenizer = AutoTokenizer.from_pretrained('./test/bert_saved_model/') # E.g. tokenizer was saved using `save_pretrained('./test/saved_model/')` """ - if 'distilbert' in pretrained_model_name_or_path: + if 't5' in pretrained_model_name_or_path: + return T5Tokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) + elif 'distilbert' in pretrained_model_name_or_path: return DistilBertTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return RobertaTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) From 15e53c4e8712260b016225310c397e19a5f7b21c Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 8 Nov 2019 12:43:21 +0100 Subject: [PATCH 025/505] maybe fix tests --- transformers/tests/modeling_tf_common_test.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/transformers/tests/modeling_tf_common_test.py b/transformers/tests/modeling_tf_common_test.py index 83a15c137a..20ccfd8ce0 100644 --- a/transformers/tests/modeling_tf_common_test.py +++ b/transformers/tests/modeling_tf_common_test.py @@ -131,7 +131,11 @@ class TFCommonTestCases: with torch.no_grad(): pto = pt_model(**pt_inputs_dict) tfo = tf_model(inputs_dict) - max_diff = np.amax(np.abs(tfo[0].numpy() - pto[0].numpy())) + tfo = tfo[0].numpy() + pto = pto[0].numpy() + tfo[np.isnan(tfo)] = 0 + pto[np.isnan(pto)] = 0 + max_diff = np.amax(np.abs(tfo - pto)) self.assertLessEqual(max_diff, 2e-2) # Check we can load pt model in tf and vice-versa with checkpoint => model functions @@ -151,7 +155,11 @@ class TFCommonTestCases: with torch.no_grad(): pto = pt_model(**pt_inputs_dict) tfo = tf_model(inputs_dict) - max_diff = np.amax(np.abs(tfo[0].numpy() - pto[0].numpy())) + tfo = tfo[0].numpy() + pto = pto[0].numpy() + tfo[np.isnan(tfo)] = 0 + pto[np.isnan(pto)] = 0 + max_diff = np.amax(np.abs(tfo - pto)) self.assertLessEqual(max_diff, 2e-2) def test_compile_tf_model(self): From b4fcd59a5ae8d12102db106d3b03849ef86109bd Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 8 Nov 2019 14:38:53 +0100 Subject: [PATCH 026/505] add sentinels in tokenizer --- transformers/tokenization_t5.py | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/transformers/tokenization_t5.py b/transformers/tokenization_t5.py index 93842d29f0..3847aeefbf 100644 --- a/transformers/tokenization_t5.py +++ b/transformers/tokenization_t5.py @@ -18,6 +18,7 @@ from __future__ import absolute_import, division, print_function, unicode_litera import logging import os +import re import six from shutil import copyfile @@ -31,7 +32,7 @@ SPIECE_UNDERLINE = u'▁' # Mapping from the keyword arguments names of Tokenizer `__init__` # to file names for serializing Tokenizer instances #################################################### -VOCAB_FILES_NAMES = {'vocab_file': 'vocab.txt'} +VOCAB_FILES_NAMES = {'vocab_file': 'spiece.model'} #################################################### # Mapping from the keyword arguments names of Tokenizer `__init__` @@ -56,15 +57,27 @@ class T5Tokenizer(PreTrainedTokenizer): SentencePiece based tokenizer. Peculiarities: - requires `SentencePiece `_ + - `extra_ids` add a number of extra ids added to the end of the vocabulary for use as sentinels. + These tokens are accessible as `` where `{%d}` is a number between 0 and extra_ids-1. + Extra tokens are indexed from the end of the vocabulary up to beginnning ( is the last token in the vocabulary) + (like in T5 preprocessing + see: https://github.com/google-research/text-to-text-transfer-transformer/blob/9fd7b14a769417be33bc6c850f9598764913c833/t5/data/preprocessors.py#L2117) """ vocab_files_names = VOCAB_FILES_NAMES pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES def __init__(self, vocab_file, eos_token="", unk_token="", - pad_token="", **kwargs): + pad_token="", extra_ids=100, additional_special_tokens=None, **kwargs): + # Add extra_ids to the special token list + if extra_ids > 0: + if additional_special_tokens is None: + additional_special_tokens = [] + additional_special_tokens.extend([u"".format(i) for i in range(extra_ids)]) + super(T5Tokenizer, self).__init__(eos_token=eos_token, unk_token=unk_token, - pad_token=pad_token, **kwargs) + pad_token=pad_token, additional_special_tokens=additional_special_tokens, + **kwargs) try: import sentencepiece as spm @@ -74,13 +87,14 @@ class T5Tokenizer(PreTrainedTokenizer): "pip install sentencepiece") self.vocab_file = vocab_file + self._extra_ids = extra_ids self.sp_model = spm.SentencePieceProcessor() self.sp_model.Load(vocab_file) @property def vocab_size(self): - return self.sp_model.get_piece_size() + return self.sp_model.get_piece_size() + self._extra_ids def __getstate__(self): state = self.__dict__.copy() @@ -118,11 +132,18 @@ class T5Tokenizer(PreTrainedTokenizer): def _convert_token_to_id(self, token): """ Converts a token (str/unicode) in an id using the vocab. """ + if token.startswith(u"', token) + num = int(l[1]) + return self.vocab_size - num - 1 return self.sp_model.piece_to_id(token) def _convert_id_to_token(self, index, return_unicode=True): """Converts an index (integer) in a token (string/unicode) using the vocab.""" - token = self.sp_model.IdToPiece(index) + if index < self.sp_model.get_piece_size(): + token = self.sp_model.IdToPiece(index) + else: + token = u"".format(self.vocab_size - 1 - index) if six.PY2 and return_unicode and isinstance(token, str): token = token.decode('utf-8') return token From 268d4f2099f90bb62949988c3b78596242e1d753 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 8 Nov 2019 16:41:55 +0100 Subject: [PATCH 027/505] fix position biases + better tests --- transformers/modeling_t5.py | 11 +++-- transformers/tests/modeling_t5_test.py | 62 +++++++++++++++----------- 2 files changed, 42 insertions(+), 31 deletions(-) diff --git a/transformers/modeling_t5.py b/transformers/modeling_t5.py index 6be0ae6863..2a74333d31 100644 --- a/transformers/modeling_t5.py +++ b/transformers/modeling_t5.py @@ -408,7 +408,7 @@ class T5Block(nn.Module): position_bias=position_bias, head_mask=head_mask) hidden_states = self_attention_outputs[0] - outputs = self_attention_outputs[1:] + outputs = self_attention_outputs[1:] # Keep self-attention outputs and relative position weights if not self.is_decoder: hidden_states = self.layer[1](hidden_states) @@ -419,11 +419,11 @@ class T5Block(nn.Module): position_bias=encoder_decoder_position_bias, head_mask=head_mask) hidden_states = cross_attention_outputs[0] - outputs = cross_attention_outputs[1:] + outputs + outputs = outputs + cross_attention_outputs[1:] # Keep cross-attention outputs and relative position weights hidden_states = self.layer[2](hidden_states) outputs = (hidden_states,) + outputs # add attentions if we output them - return outputs + return outputs # hidden-states, (self-attention weights), (self-attention position bias), (cross-attention weights), (cross-attention position bias) class T5PreTrainedModel(PreTrainedModel): @@ -564,14 +564,17 @@ class T5Stack(T5PreTrainedModel): encoder_attention_mask=encoder_extended_attention_mask, encoder_decoder_position_bias=encoder_decoder_position_bias, head_mask=head_mask[i]) + # layer_outputs is a tuple with: + # hidden-states, (self-attention weights), (self-attention position bias), (cross-attention weights), (cross-attention position bias) hidden_states = layer_outputs[0] if i == 0: + # We share the position biases between the layers - the first layer store them position_bias = layer_outputs[2 if self.output_attentions else 1] if self.is_decoder: encoder_decoder_position_bias = layer_outputs[4 if self.output_attentions else 2] if self.output_attentions: - all_attentions = all_attentions + (layer_outputs[1],) + all_attentions = all_attentions + (layer_outputs[1],) # We keep only self-attention weights for now hidden_states = self.final_layer_norm(hidden_states) layer_output = self.dropout(hidden_states) diff --git a/transformers/tests/modeling_t5_test.py b/transformers/tests/modeling_t5_test.py index 2c67b83c25..091bd742b5 100644 --- a/transformers/tests/modeling_t5_test.py +++ b/transformers/tests/modeling_t5_test.py @@ -45,9 +45,10 @@ class T5ModelTest(CommonTestCases.CommonModelTester): def __init__(self, parent, batch_size=13, - seq_length=7, + encoder_seq_length=7, + decoder_seq_length=9, is_training=True, - use_input_mask=True, + use_attention_mask=True, use_labels=True, vocab_size=99, n_positions=14, @@ -62,9 +63,10 @@ class T5ModelTest(CommonTestCases.CommonModelTester): ): self.parent = parent self.batch_size = batch_size - self.seq_length = seq_length + self.encoder_seq_length = encoder_seq_length + self.decoder_seq_length = decoder_seq_length self.is_training = is_training - self.use_input_mask = use_input_mask + self.use_attention_mask = use_attention_mask self.use_labels = use_labels self.vocab_size = vocab_size self.n_positions = n_positions @@ -78,15 +80,18 @@ class T5ModelTest(CommonTestCases.CommonModelTester): self.scope = scope def prepare_config_and_inputs(self): - input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + encoder_input_ids = ids_tensor([self.batch_size, self.encoder_seq_length], self.vocab_size) + decoder_input_ids = ids_tensor([self.batch_size, self.decoder_seq_length], self.vocab_size) - input_mask = None - if self.use_input_mask: - input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + encoder_attention_mask = None + decoder_attention_mask = None + if self.use_attention_mask: + encoder_attention_mask = ids_tensor([self.batch_size, self.encoder_seq_length], vocab_size=2) + decoder_attention_mask = ids_tensor([self.batch_size, self.decoder_seq_length], vocab_size=2) - token_labels = None + decoder_lm_labels = None if self.use_labels: - token_labels = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + decoder_lm_labels = ids_tensor([self.batch_size, self.decoder_seq_length], self.vocab_size) config = T5Config( vocab_size_or_config_json_file=self.vocab_size, @@ -100,21 +105,22 @@ class T5ModelTest(CommonTestCases.CommonModelTester): dropout_rate=self.dropout_rate, initializer_factor=self.initializer_factor) - return (config, input_ids, input_mask, token_labels) + return (config, encoder_input_ids, decoder_input_ids, encoder_attention_mask, decoder_attention_mask, decoder_lm_labels) def check_loss_output(self, result): self.parent.assertListEqual( list(result["loss"].size()), []) - def create_and_check_t5_model(self, config, input_ids, input_mask, token_labels): + def create_and_check_t5_model(self, config, encoder_input_ids, decoder_input_ids, encoder_attention_mask, decoder_attention_mask, decoder_lm_labels): model = T5Model(config=config) model.eval() - encoder_output, decoder_output = model(encoder_input_ids=input_ids, - decoder_input_ids=input_ids, - decoder_attention_mask=input_mask) - encoder_output, decoder_output = model(encoder_input_ids=input_ids, - decoder_input_ids=input_ids) + decoder_output, encoder_output = model(encoder_input_ids=encoder_input_ids, + decoder_input_ids=decoder_input_ids, + encoder_attention_mask=encoder_attention_mask, + decoder_attention_mask=decoder_attention_mask) + decoder_output, encoder_output = model(encoder_input_ids=encoder_input_ids, + decoder_input_ids=decoder_input_ids) result = { "encoder_output": encoder_output, @@ -122,17 +128,17 @@ class T5ModelTest(CommonTestCases.CommonModelTester): } self.parent.assertListEqual( list(result["encoder_output"].size()), - [self.batch_size, self.seq_length, self.hidden_size]) + [self.batch_size, self.encoder_seq_length, self.hidden_size]) self.parent.assertListEqual( list(result["decoder_output"].size()), - [self.batch_size, self.seq_length, self.hidden_size]) + [self.batch_size, self.decoder_seq_length, self.hidden_size]) - def create_and_check_t5_with_lm_head(self, config, input_ids, input_mask, token_labels): + def create_and_check_t5_with_lm_head(self, config, encoder_input_ids, decoder_input_ids, encoder_attention_mask, decoder_attention_mask, decoder_lm_labels): model = T5WithLMHeadModel(config=config) model.eval() - outputs = model(encoder_input_ids=input_ids, decoder_input_ids=input_ids, - decoder_attention_mask=input_mask, decoder_lm_labels=token_labels) + outputs = model(encoder_input_ids=encoder_input_ids, decoder_input_ids=decoder_input_ids, + decoder_attention_mask=decoder_attention_mask, decoder_lm_labels=decoder_lm_labels) loss, prediction_scores = outputs[0], outputs[1] result = { "loss": loss, @@ -140,15 +146,17 @@ class T5ModelTest(CommonTestCases.CommonModelTester): } self.parent.assertListEqual( list(result["prediction_scores"].size()), - [self.batch_size, self.seq_length, self.vocab_size]) + [self.batch_size, self.decoder_seq_length, self.vocab_size]) self.check_loss_output(result) def prepare_config_and_inputs_for_common(self): config_and_inputs = self.prepare_config_and_inputs() - (config, input_ids, input_mask, token_labels) = config_and_inputs - inputs_dict = {'encoder_input_ids': input_ids, - 'decoder_input_ids': input_ids, - 'decoder_attention_mask': input_mask} + (config, encoder_input_ids, decoder_input_ids, encoder_attention_mask, + decoder_attention_mask, decoder_lm_labels) = config_and_inputs + inputs_dict = {'encoder_input_ids': encoder_input_ids, + 'decoder_input_ids': decoder_input_ids, + 'decoder_attention_mask': decoder_attention_mask, + 'encoder_attention_mask': encoder_attention_mask} return config, inputs_dict def setUp(self): From ef99852961b9f3bb87a18a58093c9f513c86b683 Mon Sep 17 00:00:00 2001 From: eukaryote Date: Sat, 9 Nov 2019 16:32:40 +0000 Subject: [PATCH 028/505] from_pretrained: convert DialoGPT format DialoGPT checkpoints have "lm_head.decoder.weight" instead of "lm_head.weight". (see: https://www.reddit.com/r/MachineLearning/comments/dt5woy/p_dialogpt_state_of_the_art_conversational_model/f6vmwuy?utm_source=share&utm_medium=web2x) --- transformers/modeling_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/transformers/modeling_utils.py b/transformers/modeling_utils.py index d51eefab58..61dd2546c6 100644 --- a/transformers/modeling_utils.py +++ b/transformers/modeling_utils.py @@ -417,6 +417,8 @@ class PreTrainedModel(nn.Module): new_key = key.replace('gamma', 'weight') if 'beta' in key: new_key = key.replace('beta', 'bias') + if key == 'lm_head.decoder.weight': + new_key = 'lm_head.weight' if new_key: old_keys.append(key) new_keys.append(new_key) From 90f6e73a35ee85e94b898a6867f19707b264d387 Mon Sep 17 00:00:00 2001 From: eukaryote Date: Sat, 9 Nov 2019 16:46:19 +0000 Subject: [PATCH 029/505] Add DialoGPT support for Pytorch->TF --- transformers/modeling_tf_pytorch_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/transformers/modeling_tf_pytorch_utils.py b/transformers/modeling_tf_pytorch_utils.py index 88ce4d4610..aa74fcc10e 100644 --- a/transformers/modeling_tf_pytorch_utils.py +++ b/transformers/modeling_tf_pytorch_utils.py @@ -118,6 +118,9 @@ def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None, a new_key = key.replace('gamma', 'weight') if 'beta' in key: new_key = key.replace('beta', 'bias') + # DialoGPT format + if key == 'lm_head.decoder.weight': + new_key = 'lm_head.weight' if new_key: old_keys.append(key) new_keys.append(new_key) From 7246d3c2f93c4461f3ec8ada7a26a002d8f196ea Mon Sep 17 00:00:00 2001 From: Michael Watkins Date: Wed, 6 Nov 2019 13:18:16 +0200 Subject: [PATCH 030/505] Consider do_lower_case in PreTrainedTokenizer As pointed out in #1545, when using an uncased model, and adding a new uncased token, the tokenizer does not correctly identify this in the case that the input text contains the token in a cased format. For instance, if we load bert-base-uncased into BertTokenizer, and then use .add_tokens() to add "cool-token", we get the expected result for .tokenize('this is a cool-token'). However, we get a possibly unexpected result for .tokenize('this is a cOOl-Token'), which in fact mirrors the result for the former from before the new token was added. This commit adds - functionality to PreTrainedTokenizer to handle this situation in case a tokenizer (currently Bert, DistilBert, and XLNet) has the do_lower_case=True kwarg by: 1) lowercasing tokens added with .add_tokens() 2) lowercasing text at the beginning of .tokenize() - new common test case for tokenizers https://github.com/huggingface/transformers/issues/1545 --- .../tests/tokenization_tests_commons.py | 31 ++++++++++++++++++- transformers/tokenization_utils.py | 5 +++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/transformers/tests/tokenization_tests_commons.py b/transformers/tests/tokenization_tests_commons.py index a921696b77..287e6fc7b3 100644 --- a/transformers/tests/tokenization_tests_commons.py +++ b/transformers/tests/tokenization_tests_commons.py @@ -110,6 +110,36 @@ class CommonTestCases: self.assertListEqual(subwords, subwords_loaded) + def test_added_tokens_do_lower_case(self): + tokenizer = self.get_tokenizer(do_lower_case=True) + + text = "aaaaa bbbbbb low cccccccccdddddddd l" + text2 = "AAAAA BBBBBB low CCCCCCCCCDDDDDDDD l" + + toks0 = tokenizer.tokenize(text) # toks before adding new_toks + + new_toks = ["aaaaa bbbbbb", "cccccccccdddddddd", 'AAAAA BBBBBB', 'CCCCCCCCCDDDDDDDD'] + added = tokenizer.add_tokens(new_toks) + self.assertEqual(added, 2) + + toks = tokenizer.tokenize(text) + toks2 = tokenizer.tokenize(text2) + + self.assertEqual(len(toks), len(toks2)) + self.assertNotEqual(len(toks), len(toks0)) # toks0 should be longer + self.assertListEqual(toks, toks2) + + tokenizer = self.get_tokenizer(do_lower_case=False) + + added = tokenizer.add_tokens(new_toks) + self.assertEqual(added, 4) + + toks = tokenizer.tokenize(text) + toks2 = tokenizer.tokenize(text2) + + self.assertEqual(len(toks), len(toks2)) # Length should still be the same + self.assertNotEqual(len(toks), len(toks0)) + self.assertNotEqual(toks[0], toks2[0]) # But at least the first tokens should differ def test_add_tokens_tokenizer(self): tokenizer = self.get_tokenizer() @@ -160,7 +190,6 @@ class CommonTestCases: self.assertEqual(tokens[0], tokenizer.eos_token_id) self.assertEqual(tokens[-2], tokenizer.pad_token_id) - def test_required_methods_tokenizer(self): tokenizer = self.get_tokenizer() input_text, output_text = self.get_input_output_texts() diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index cd14cc4582..fc31c10d25 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -512,6 +512,8 @@ class PreTrainedTokenizer(object): to_add_tokens = [] for token in new_tokens: assert isinstance(token, str) or (six.PY2 and isinstance(token, unicode)) + if self.init_kwargs.get('do_lower_case', False): + token = token.lower() if token != self.unk_token and \ self.convert_tokens_to_ids(token) == self.convert_tokens_to_ids(self.unk_token) and \ token not in to_add_tokens: @@ -605,6 +607,9 @@ class PreTrainedTokenizer(object): Take care of added tokens. """ + if self.init_kwargs.get('do_lower_case', False): + text = text.lower() + def split_on_token(tok, text): result = [] split_text = text.split(tok) From 14b3aa3b3c300eb1fcc4f3a0c046c87bdabe0afd Mon Sep 17 00:00:00 2001 From: Louis MARTIN Date: Fri, 8 Nov 2019 16:47:21 -0800 Subject: [PATCH 031/505] Add tokenization_camembert.py --- transformers/tokenization_camembert.py | 120 +++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 transformers/tokenization_camembert.py diff --git a/transformers/tokenization_camembert.py b/transformers/tokenization_camembert.py new file mode 100644 index 0000000000..9facf7d911 --- /dev/null +++ b/transformers/tokenization_camembert.py @@ -0,0 +1,120 @@ +# coding=utf-8 +# Copyright 2018 Google AI, Google Brain and Carnegie Mellon University Authors and the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Tokenization classes for Camembert model.""" +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import sentencepiece as spm +from transformers.tokenization_utils import PreTrainedTokenizer + + +class CamembertTokenizer(PreTrainedTokenizer): + """ + Adapted from RobertaTokenizer and XLNetTokenizer + SentencePiece based tokenizer. Peculiarities: + + - requires `SentencePiece `_ + """ + vocab_files_names = {'vocab_file': None} + + def __init__(self, vocab_file, bos_token="", eos_token="", sep_token="", + cls_token="", unk_token="", pad_token='', mask_token='', **kwargs): + super(CamembertTokenizer, self).__init__(max_len=512, bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, + sep_token=sep_token, cls_token=cls_token, pad_token=pad_token, + mask_token=mask_token, **kwargs) + self.max_len_single_sentence = self.max_len - 2 # take into account special tokens + self.max_len_sentences_pair = self.max_len - 4 # take into account special tokens + self.sp_model = spm.SentencePieceProcessor() + self.sp_model.Load(str(vocab_file)) + # HACK: These tokens were added by fairseq but don't seem to be actually used when duplicated in the actual + # sentencepiece vocabulary (this is the case for and + self.fairseq_tokens_to_ids = {'NOTUSED': 0, '': 1, 'NOTUSED': 2, '': 3} + self.fairseq_offset = len(self.fairseq_tokens_to_ids) + self.fairseq_tokens_to_ids[''] = len(self.sp_model) + len(self.fairseq_tokens_to_ids) + self.fairseq_ids_to_tokens = {v: k for k, v in self.fairseq_tokens_to_ids.items()} + + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): + """ + Build model inputs from a sequence or a pair of sequence for sequence classification tasks + by concatenating and adding special tokens. + A RoBERTa sequence has the following format: + single sequence: X + pair of sequences: A B + """ + if token_ids_1 is None: + return [self.cls_token_id] + token_ids_0 + [self.sep_token_id] + cls = [self.cls_token_id] + sep = [self.sep_token_id] + return cls + token_ids_0 + sep + sep + token_ids_1 + sep + + def get_special_tokens_mask(self, token_ids_0, token_ids_1=None, already_has_special_tokens=False): + """ + Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding + special tokens using the tokenizer ``prepare_for_model`` or ``encode_plus`` methods. + + Args: + token_ids_0: list of ids (must not contain special tokens) + token_ids_1: Optional list of ids (must not contain special tokens), necessary when fetching sequence ids + for sequence pairs + already_has_special_tokens: (default False) Set to True if the token list is already formated with + special tokens for the model + + Returns: + A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. + """ + if already_has_special_tokens: + if token_ids_1 is not None: + raise ValueError("You should not supply a second sequence if the provided sequence of " + "ids is already formated with special tokens for the model.") + return list(map(lambda x: 1 if x in [self.sep_token_id, self.cls_token_id] else 0, token_ids_0)) + + if token_ids_1 is None: + return [1] + ([0] * len(token_ids_0)) + [1] + return [1] + ([0] * len(token_ids_0)) + [1, 1] + ([0] * len(token_ids_1)) + [1] + + def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None): + """ + Creates a mask from the two sequences passed to be used in a sequence-pair classification task. + A RoBERTa sequence pair mask has the following format: + 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 + | first sequence | second sequence + + if token_ids_1 is None, only returns the first portion of the mask (0's). + """ + sep = [self.sep_token_id] + cls = [self.cls_token_id] + + if token_ids_1 is None: + return len(cls + token_ids_0 + sep) * [0] + return len(cls + token_ids_0 + sep + sep) * [0] + len(token_ids_1 + sep) * [1] + + @property + def vocab_size(self): + return self.fairseq_offset + len(self.sp_model) + + def _tokenize(self, text): + return self.sp_model.EncodeAsPieces(text) + + def _convert_token_to_id(self, token): + """ Converts a token (str/unicode) in an id using the vocab. """ + if token in self.fairseq_tokens_to_ids: + return self.fairseq_tokens_to_ids[token] + return self.fairseq_offset + self.sp_model.PieceToId(token) + + def _convert_id_to_token(self, index): + """Converts an index (integer) in a token (string/unicode) using the vocab.""" + if index in self.fairseq_ids_to_tokens: + return self.fairseq_ids_to_tokens[index] + return self.sp_model.IdToPiece(index - self.fairseq_offset) From 6e72fd094c98901cc90d146d3fe3cd5a0e879911 Mon Sep 17 00:00:00 2001 From: Louis MARTIN Date: Fri, 8 Nov 2019 17:09:48 -0800 Subject: [PATCH 032/505] Add demo_camembert.py --- examples/demo_camembert.py | 59 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 examples/demo_camembert.py diff --git a/examples/demo_camembert.py b/examples/demo_camembert.py new file mode 100644 index 0000000000..df28f4f267 --- /dev/null +++ b/examples/demo_camembert.py @@ -0,0 +1,59 @@ +from pathlib import Path +import tarfile +import urllib.request + +import torch + +from transformers.tokenization_camembert import CamembertTokenizer +from transformers.modeling_roberta import RobertaForMaskedLM + + +def fill_mask(masked_input, model, tokenizer, topk=5): + # Adapted from https://github.com/pytorch/fairseq/blob/master/fairseq/models/roberta/hub_interface.py + assert masked_input.count('') == 1 + input_ids = torch.tensor(tokenizer.encode(masked_input, add_special_tokens=True)).unsqueeze(0) # Batch size 1 + logits = model(input_ids)[0] # The last hidden-state is the first element of the output tuple + masked_index = (input_ids.squeeze() == tokenizer.mask_token_id).nonzero().item() + logits = logits[0, masked_index, :] + prob = logits.softmax(dim=0) + values, indices = prob.topk(k=topk, dim=0) + topk_predicted_token_bpe = ' '.join([tokenizer.convert_ids_to_tokens(indices[i].item()) + for i in range(len(indices))]) + masked_token = tokenizer.mask_token + topk_filled_outputs = [] + for index, predicted_token_bpe in enumerate(topk_predicted_token_bpe.split(' ')): + predicted_token = predicted_token_bpe.replace('\u2581', ' ') + if " {0}".format(masked_token) in masked_input: + topk_filled_outputs.append(( + masked_input.replace( + ' {0}'.format(masked_token), predicted_token + ), + values[index].item(), + predicted_token, + )) + else: + topk_filled_outputs.append(( + masked_input.replace(masked_token, predicted_token), + values[index].item(), + predicted_token, + )) + return topk_filled_outputs + + +model_path = Path('camembert.v0.pytorch') +if not model_path.exists(): + compressed_path = model_path.with_suffix('.tar.gz') + url = 'http://dl.fbaipublicfiles.com/camembert/camembert.v0.pytorch.tar.gz' + print('Downloading model...') + urllib.request.urlretrieve(url, compressed_path) + print('Extracting model...') + with tarfile.open(compressed_path) as f: + f.extractall(model_path.parent) + assert model_path.exists() +tokenizer_path = model_path / 'sentencepiece.bpe.model' +tokenizer = CamembertTokenizer.from_pretrained(tokenizer_path) +model = RobertaForMaskedLM.from_pretrained(model_path) +model.eval() + +masked_input = "Le camembert est :)" +print(fill_mask(masked_input, model, tokenizer, topk=3)) From e44b939e7198fac5ce4085a1bda8e17b3934e67f Mon Sep 17 00:00:00 2001 From: Louis MARTIN Date: Tue, 12 Nov 2019 17:11:00 -0800 Subject: [PATCH 033/505] Add configuration_camembert.py and modeling_camembert.py --- transformers/configuration_camembert.py | 33 +++ transformers/modeling_camembert.py | 257 ++++++++++++++++++++++++ 2 files changed, 290 insertions(+) create mode 100644 transformers/configuration_camembert.py create mode 100644 transformers/modeling_camembert.py diff --git a/transformers/configuration_camembert.py b/transformers/configuration_camembert.py new file mode 100644 index 0000000000..07ebd6e82e --- /dev/null +++ b/transformers/configuration_camembert.py @@ -0,0 +1,33 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" CamemBERT configuration """ + +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import logging + +from .configuration_roberta import RobertaConfig + +logger = logging.getLogger(__name__) + +CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = { + 'camembert-base': "https://dl.fbaipublicfiles.com/camembert/camembert-base-v0-config.json", +} + + +class CamembertConfig(RobertaConfig): + pretrained_config_archive_map = CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP diff --git a/transformers/modeling_camembert.py b/transformers/modeling_camembert.py new file mode 100644 index 0000000000..982c349531 --- /dev/null +++ b/transformers/modeling_camembert.py @@ -0,0 +1,257 @@ +# coding=utf-8 +# Copyright 2019 Inria, Facebook AI Research and the HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch CamemBERT model. """ + +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import logging + +from .modeling_roberta import RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification, RobertaForMultipleChoice +from .configuration_camembert import CamembertConfig +from .file_utils import add_start_docstrings + +logger = logging.getLogger(__name__) + +CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP = { + 'camembert-base': "https://dl.fbaipublicfiles.com/camembert/camembert-base-v0-pytorch_model.bin", +} + + +CAMEMBERT_START_DOCSTRING = r""" The CamemBERT model was proposed in + `CamemBERT: a Tasty French Language Model`_ + by Louis Martin, Benjamin Muller, Pedro Javier Ortiz Suárez, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah, and Benoît Sagot. It is based on Facebook's RoBERTa model released in 2019. + + It is a model trained on 138GB of French text. + + This implementation is the same RoBERTa. + + This model is a PyTorch `torch.nn.Module`_ sub-class. Use it as a regular PyTorch Module and + refer to the PyTorch documentation for all matter related to general usage and behavior. + + .. _`CamemBERT: a Tasty French Language Model`: + https://arxiv.org/abs/1911.03894 + + .. _`torch.nn.Module`: + https://pytorch.org/docs/stable/nn.html#module + + Parameters: + config (:class:`~transformers.CamembertConfig`): Model configuration class with all the parameters of the + model. Initializing with a config file does not load the weights associated with the model, only the configuration. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. +""" + +CAMEMBERT_INPUTS_DOCSTRING = r""" + Inputs: + **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + To match pre-training, CamemBERT input sequence should be formatted with and tokens as follows: + + (a) For sequence pairs: + + ``tokens: Is this Jacksonville ? No it is not . `` + + (b) For single sequences: + + ``tokens: the dog is hairy . `` + + Fully encoded sequences or sequence pairs can be obtained using the CamembertTokenizer.encode function with + the ``add_special_tokens`` parameter set to ``True``. + + CamemBERT is a model with absolute position embeddings so it's usually advised to pad the inputs on + the right rather than the left. + + See :func:`transformers.PreTrainedTokenizer.encode` and + :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: + Mask to avoid performing attention on padding token indices. + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. + **token_type_ids**: (`optional` need to be trained) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Optional segment token indices to indicate first and second portions of the inputs. + This embedding matrice is not trained (not pretrained during CamemBERT pretraining), you will have to train it + during finetuning. + Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` + corresponds to a `sentence B` token + (see `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ for more details). + **position_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of positions of each input sequence tokens in the position embeddings. + Selected in the range ``[0, config.max_position_embeddings - 1[``. + **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. +""" + +@add_start_docstrings("The bare CamemBERT Model transformer outputting raw hidden-states without any specific head on top.", + CAMEMBERT_START_DOCSTRING, CAMEMBERT_INPUTS_DOCSTRING) +class CamembertModel(RobertaModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + Sequence of hidden-states at the output of the last layer of the model. + **pooler_output**: ``torch.FloatTensor`` of shape ``(batch_size, hidden_size)`` + Last layer hidden-state of the first token of the sequence (classification token) + further processed by a Linear layer and a Tanh activation function. The Linear + layer weights are trained from the next sentence prediction (classification) + eo match pre-training, CamemBERT input sequence should be formatted with [CLS] and [SEP] tokens as follows: + + (a) For sequence pairs: + + ``tokens: [CLS] is this jack ##son ##ville ? [SEP] [SEP] no it is not . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1`` + + (b) For single sequences: + + ``tokens: [CLS] the dog is hairy . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0`` + + objective during Bert pretraining. This output is usually *not* a good summary + of the semantic content of the input, you're often better with averaging or pooling + the sequence of hidden-states for the whole input sequence. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = CamembertTokenizer.from_pretrained('camembert-base') + model = CamembertModel.from_pretrained('camembert-base') + input_ids = torch.tensor(tokenizer.encode("J'aime le camembert !")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids) + last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple + + """ + config_class = CamembertConfig + pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP + base_model_prefix = "camembert" + + +@add_start_docstrings("""CamemBERT Model with a `language modeling` head on top. """, + CAMEMBERT_START_DOCSTRING, CAMEMBERT_INPUTS_DOCSTRING) +class CamembertForMaskedLM(RobertaForMaskedLM): + r""" + **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the masked language modeling loss. + Indices should be in ``[-1, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) + Tokens with indices set to ``-1`` are ignored (masked), the loss is only computed for the tokens with labels + in ``[0, ..., config.vocab_size]`` + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``masked_lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Masked language modeling loss. + **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = CamembertTokenizer.from_pretrained('camembert-base') + model = CamembertForMaskedLM.from_pretrained('camembert-base') + input_ids = torch.tensor(tokenizer.encode("J'aime le camembert !")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, masked_lm_labels=input_ids) + loss, prediction_scores = outputs[:2] + + """ + config_class = CamembertConfig + pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP + base_model_prefix = "camembert" + + +@add_start_docstrings("""CamemBERT Model transformer with a sequence classification/regression head on top (a linear layer + on top of the pooled output) e.g. for GLUE tasks. """, + CAMEMBERT_START_DOCSTRING, CAMEMBERT_INPUTS_DOCSTRING) +class CamembertForSequenceClassification(RobertaForSequenceClassification): + r""" + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the sequence classification/regression loss. + Indices should be in ``[0, ..., config.num_labels]``. + If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss), + If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy). + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification (or regression if config.num_labels==1) loss. + **logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)`` + Classification (or regression if config.num_labels==1) scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = CamembertTokenizer.from_pretrained('camembert-base') + model = CamembertForSequenceClassification.from_pretrained('camembert-base') + input_ids = torch.tensor(tokenizer.encode("J'aime le camembert !")).unsqueeze(0) # Batch size 1 + labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, logits = outputs[:2] + + """ + config_class = CamembertConfig + pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP + base_model_prefix = "camembert" + + +@add_start_docstrings("""CamemBERT Model with a multiple choice classification head on top (a linear layer on top of + the pooled output and a softmax) e.g. for RocStories/SWAG tasks. """, + CAMEMBERT_START_DOCSTRING, CAMEMBERT_INPUTS_DOCSTRING) +class CamembertForMultipleChoice(RobertaForMultipleChoice): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification loss. + **classification_scores**: ``torch.FloatTensor`` of shape ``(batch_size, num_choices)`` where `num_choices` is the size of the second dimension + of the input tensors. (see `input_ids` above). + Classification scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = CamembertTokenizer.from_pretrained('camembert-base') + model = CamembertForMultipleChoice.from_pretrained('camembert-base') + choices = ["J'aime le camembert !", "Je deteste le camembert !"] + input_ids = torch.tensor([tokenizer.encode(s, add_special_tokens=True) for s in choices]).unsqueeze(0) # Batch size 1, 2 choices + labels = torch.tensor(1).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, classification_scores = outputs[:2] + + """ + config_class = CamembertConfig + pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP + base_model_prefix = "camembert" From fb6c70a91d3183742ce0a6d97add68103253ca3a Mon Sep 17 00:00:00 2001 From: Louis MARTIN Date: Tue, 12 Nov 2019 17:11:49 -0800 Subject: [PATCH 034/505] Update tokenization_camembert.py with urls --- transformers/tokenization_camembert.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/transformers/tokenization_camembert.py b/transformers/tokenization_camembert.py index 9facf7d911..0a6e751351 100644 --- a/transformers/tokenization_camembert.py +++ b/transformers/tokenization_camembert.py @@ -11,7 +11,7 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. +# limitations under the License """ Tokenization classes for Camembert model.""" from __future__ import (absolute_import, division, print_function, unicode_literals) @@ -20,6 +20,19 @@ import sentencepiece as spm from transformers.tokenization_utils import PreTrainedTokenizer +VOCAB_FILES_NAMES = {'vocab_file': 'sentencepiece.bpe.model'} + +PRETRAINED_VOCAB_FILES_MAP = { + 'vocab_file': + { + 'camembert-base': "https://dl.fbaipublicfiles.com/camembert/camembert-base-v0-sentencepiece.bpe.model", + } +} + +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { + 'camembert-base': None, +} + class CamembertTokenizer(PreTrainedTokenizer): """ Adapted from RobertaTokenizer and XLNetTokenizer @@ -27,7 +40,9 @@ class CamembertTokenizer(PreTrainedTokenizer): - requires `SentencePiece `_ """ - vocab_files_names = {'vocab_file': None} + vocab_files_names = VOCAB_FILES_NAMES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES def __init__(self, vocab_file, bos_token="", eos_token="", sep_token="", cls_token="", unk_token="", pad_token='', mask_token='', **kwargs): From f12e4d8da783a535cd8978656f0a3ca108a2e2ed Mon Sep 17 00:00:00 2001 From: Louis MARTIN Date: Tue, 12 Nov 2019 17:14:50 -0800 Subject: [PATCH 035/505] Move demo_camembert.py to examples/contrib --- examples/{ => contrib}/demo_camembert.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename examples/{ => contrib}/demo_camembert.py (100%) diff --git a/examples/demo_camembert.py b/examples/contrib/demo_camembert.py similarity index 100% rename from examples/demo_camembert.py rename to examples/contrib/demo_camembert.py From 3e20c2e871db82f81c3b2b814265a481be15273c Mon Sep 17 00:00:00 2001 From: Louis MARTIN Date: Tue, 12 Nov 2019 17:16:24 -0800 Subject: [PATCH 036/505] Update demo_camembert.py with new classes --- examples/contrib/demo_camembert.py | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/examples/contrib/demo_camembert.py b/examples/contrib/demo_camembert.py index df28f4f267..28144d5167 100644 --- a/examples/contrib/demo_camembert.py +++ b/examples/contrib/demo_camembert.py @@ -5,7 +5,7 @@ import urllib.request import torch from transformers.tokenization_camembert import CamembertTokenizer -from transformers.modeling_roberta import RobertaForMaskedLM +from transformers.modeling_camembert import CamembertForMaskedLM def fill_mask(masked_input, model, tokenizer, topk=5): @@ -40,19 +40,8 @@ def fill_mask(masked_input, model, tokenizer, topk=5): return topk_filled_outputs -model_path = Path('camembert.v0.pytorch') -if not model_path.exists(): - compressed_path = model_path.with_suffix('.tar.gz') - url = 'http://dl.fbaipublicfiles.com/camembert/camembert.v0.pytorch.tar.gz' - print('Downloading model...') - urllib.request.urlretrieve(url, compressed_path) - print('Extracting model...') - with tarfile.open(compressed_path) as f: - f.extractall(model_path.parent) - assert model_path.exists() -tokenizer_path = model_path / 'sentencepiece.bpe.model' -tokenizer = CamembertTokenizer.from_pretrained(tokenizer_path) -model = RobertaForMaskedLM.from_pretrained(model_path) +tokenizer = CamembertTokenizer.from_pretrained('camembert-base') +model = CamembertForMaskedLM.from_pretrained('camembert-base') model.eval() masked_input = "Le camembert est :)" From 694d4fcbb61f15b66781219954112791248d832e Mon Sep 17 00:00:00 2001 From: Louis MARTIN Date: Tue, 12 Nov 2019 17:28:37 -0800 Subject: [PATCH 037/505] Add CamemBERT classes to __init__.py --- transformers/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/transformers/__init__.py b/transformers/__init__.py index dd9e18a050..cdf0669b39 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -42,6 +42,7 @@ from .tokenization_xlnet import XLNetTokenizer, SPIECE_UNDERLINE from .tokenization_xlm import XLMTokenizer from .tokenization_roberta import RobertaTokenizer from .tokenization_distilbert import DistilBertTokenizer +from .tokenization_camembert import CamembertTokenizer # Configurations from .configuration_utils import PretrainedConfig @@ -56,6 +57,7 @@ from .configuration_ctrl import CTRLConfig, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_xlm import XLMConfig, XLM_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_roberta import RobertaConfig, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_distilbert import DistilBertConfig, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_camembert import CamembertConfig, CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP # Modeling if is_torch_available(): @@ -96,6 +98,9 @@ if is_torch_available(): DistilBertForSequenceClassification, DistilBertForQuestionAnswering, DistilBertForTokenClassification, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_camembert import (CamembertForMaskedLM, CamembertModel, + CamembertForSequenceClassification, CamembertForMultipleChoice, + CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_encoder_decoder import PreTrainedEncoderDecoder, Model2Model # Optimization From 035fea53157b43d311b1e1164395398c4ec0dda5 Mon Sep 17 00:00:00 2001 From: Louis MARTIN Date: Tue, 12 Nov 2019 17:41:41 -0800 Subject: [PATCH 038/505] Add CamemBERT to auto files and docs --- docs/source/pretrained_models.rst | 6 +++++- transformers/configuration_auto.py | 7 ++++++- transformers/tokenization_auto.py | 9 +++++++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst index edb47e7f1c..b0a578fd80 100644 --- a/docs/source/pretrained_models.rst +++ b/docs/source/pretrained_models.rst @@ -155,5 +155,9 @@ Here is the full list of the currently provided pretrained models together with | CTRL | ``ctrl`` | | 48-layer, 1280-hidden, 16-heads, 1.6B parameters | | | | | Salesforce's Large-sized CTRL English model | +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| CamemBERT | ``camembert-base`` | | 12-layer, 768-hidden, 12-heads, 110M parameters | +| | | | CamemBERT using the BERT-base architecture | +| | | (see `details `__) | ++-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ -.. `__ \ No newline at end of file +.. `__ diff --git a/transformers/configuration_auto.py b/transformers/configuration_auto.py index edd21a670c..2906136139 100644 --- a/transformers/configuration_auto.py +++ b/transformers/configuration_auto.py @@ -27,6 +27,7 @@ from .configuration_xlm import XLMConfig from .configuration_roberta import RobertaConfig from .configuration_distilbert import DistilBertConfig from .configuration_ctrl import CTRLConfig +from .configuration_camembert import CamembertConfig logger = logging.getLogger(__name__) @@ -50,6 +51,7 @@ class AutoConfig(object): - contains `xlnet`: XLNetConfig (XLNet model) - contains `xlm`: XLMConfig (XLM model) - contains `roberta`: RobertaConfig (RoBERTa model) + - contains `camembert`: CamembertConfig (CamemBERT model) - contains `ctrl` : CTRLConfig (CTRL model) This class cannot be instantiated using `__init__()` (throw an error). """ @@ -72,6 +74,7 @@ class AutoConfig(object): - contains `xlnet`: XLNetConfig (XLNet model) - contains `xlm`: XLMConfig (XLM model) - contains `roberta`: RobertaConfig (RoBERTa model) + - contains `camembert`: CamembertConfig (CamemBERT model) - contains `ctrl` : CTRLConfig (CTRL model) Params: pretrained_model_name_or_path: either: @@ -116,6 +119,8 @@ class AutoConfig(object): """ if 'distilbert' in pretrained_model_name_or_path: return DistilBertConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) + elif 'camembert' in pretrained_model_name_or_path: + return CamembertConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return RobertaConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) elif 'bert' in pretrained_model_name_or_path: @@ -134,4 +139,4 @@ class AutoConfig(object): return CTRLConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', " - "'xlm', 'roberta', 'ctrl'".format(pretrained_model_name_or_path)) + "'xlm', 'roberta', 'camembert', 'ctrl'".format(pretrained_model_name_or_path)) diff --git a/transformers/tokenization_auto.py b/transformers/tokenization_auto.py index ec056de17f..4510159905 100644 --- a/transformers/tokenization_auto.py +++ b/transformers/tokenization_auto.py @@ -27,6 +27,7 @@ from .tokenization_xlnet import XLNetTokenizer from .tokenization_xlm import XLMTokenizer from .tokenization_roberta import RobertaTokenizer from .tokenization_distilbert import DistilBertTokenizer +from .tokenization_camembert import CamembertTokenizer logger = logging.getLogger(__name__) @@ -41,6 +42,7 @@ class AutoTokenizer(object): The tokenizer class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): + - contains `camembert`: CamembertTokenizer (CamemBERT model) - contains `distilbert`: DistilBertTokenizer (DistilBert model) - contains `roberta`: RobertaTokenizer (RoBERTa model) - contains `bert`: BertTokenizer (Bert model) @@ -64,8 +66,9 @@ class AutoTokenizer(object): The tokenizer class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): + - contains `camembert`: CamembertTokenizer (CamemBERT model) - contains `distilbert`: DistilBertTokenizer (DistilBert model) - - contains `roberta`: RobertaTokenizer (XLM model) + - contains `roberta`: RobertaTokenizer (RoBERTa model) - contains `bert`: BertTokenizer (Bert model) - contains `openai-gpt`: OpenAIGPTTokenizer (OpenAI GPT model) - contains `gpt2`: GPT2Tokenizer (OpenAI GPT-2 model) @@ -103,6 +106,8 @@ class AutoTokenizer(object): """ if 'distilbert' in pretrained_model_name_or_path: return DistilBertTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) + elif 'camembert' in pretrained_model_name_or_path: + return CamembertTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return RobertaTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) elif 'bert' in pretrained_model_name_or_path: @@ -121,4 +126,4 @@ class AutoTokenizer(object): return CTRLTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', " - "'xlm', 'roberta', 'ctrl'".format(pretrained_model_name_or_path)) + "'xlm', 'roberta', 'camembert', 'ctrl'".format(pretrained_model_name_or_path)) From 26858f27cb352b6bb1cda2b090413d1d2206a9ee Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Fri, 15 Nov 2019 23:23:31 -0500 Subject: [PATCH 039/505] [camembert] Upload to s3 + rename script --- examples/contrib/{demo_camembert.py => run_camembert.py} | 0 transformers/configuration_camembert.py | 2 +- transformers/modeling_camembert.py | 2 +- transformers/tokenization_camembert.py | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename examples/contrib/{demo_camembert.py => run_camembert.py} (100%) diff --git a/examples/contrib/demo_camembert.py b/examples/contrib/run_camembert.py similarity index 100% rename from examples/contrib/demo_camembert.py rename to examples/contrib/run_camembert.py diff --git a/transformers/configuration_camembert.py b/transformers/configuration_camembert.py index 07ebd6e82e..3ff64454e5 100644 --- a/transformers/configuration_camembert.py +++ b/transformers/configuration_camembert.py @@ -25,7 +25,7 @@ from .configuration_roberta import RobertaConfig logger = logging.getLogger(__name__) CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = { - 'camembert-base': "https://dl.fbaipublicfiles.com/camembert/camembert-base-v0-config.json", + 'camembert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/camembert-base-config.json", } diff --git a/transformers/modeling_camembert.py b/transformers/modeling_camembert.py index 982c349531..a0f5933bc2 100644 --- a/transformers/modeling_camembert.py +++ b/transformers/modeling_camembert.py @@ -27,7 +27,7 @@ from .file_utils import add_start_docstrings logger = logging.getLogger(__name__) CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP = { - 'camembert-base': "https://dl.fbaipublicfiles.com/camembert/camembert-base-v0-pytorch_model.bin", + 'camembert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/camembert-base-pytorch_model.bin", } diff --git a/transformers/tokenization_camembert.py b/transformers/tokenization_camembert.py index 0a6e751351..de587ac863 100644 --- a/transformers/tokenization_camembert.py +++ b/transformers/tokenization_camembert.py @@ -25,7 +25,7 @@ VOCAB_FILES_NAMES = {'vocab_file': 'sentencepiece.bpe.model'} PRETRAINED_VOCAB_FILES_MAP = { 'vocab_file': { - 'camembert-base': "https://dl.fbaipublicfiles.com/camembert/camembert-base-v0-sentencepiece.bpe.model", + 'camembert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/camembert-base-sentencepiece.bpe.model", } } From f9abf73e319d99ed74f46671e98b00b9328cd245 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Fri, 15 Nov 2019 23:40:20 -0500 Subject: [PATCH 040/505] [camembert] realign w/ recent changes --- transformers/modeling_camembert.py | 10 +++++----- transformers/tokenization_camembert.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/transformers/modeling_camembert.py b/transformers/modeling_camembert.py index a0f5933bc2..05538926e2 100644 --- a/transformers/modeling_camembert.py +++ b/transformers/modeling_camembert.py @@ -37,7 +37,7 @@ CAMEMBERT_START_DOCSTRING = r""" The CamemBERT model was proposed in It is a model trained on 138GB of French text. - This implementation is the same RoBERTa. + This implementation is the same as RoBERTa. This model is a PyTorch `torch.nn.Module`_ sub-class. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. @@ -94,6 +94,10 @@ CAMEMBERT_INPUTS_DOCSTRING = r""" Mask to nullify selected heads of the self-attention modules. Mask values selected in ``[0, 1]``: ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. + **inputs_embeds**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, embedding_dim)``: + Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. """ @add_start_docstrings("The bare CamemBERT Model transformer outputting raw hidden-states without any specific head on top.", @@ -143,7 +147,6 @@ class CamembertModel(RobertaModel): """ config_class = CamembertConfig pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP - base_model_prefix = "camembert" @add_start_docstrings("""CamemBERT Model with a `language modeling` head on top. """, @@ -180,7 +183,6 @@ class CamembertForMaskedLM(RobertaForMaskedLM): """ config_class = CamembertConfig pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP - base_model_prefix = "camembert" @add_start_docstrings("""CamemBERT Model transformer with a sequence classification/regression head on top (a linear layer @@ -219,7 +221,6 @@ class CamembertForSequenceClassification(RobertaForSequenceClassification): """ config_class = CamembertConfig pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP - base_model_prefix = "camembert" @add_start_docstrings("""CamemBERT Model with a multiple choice classification head on top (a linear layer on top of @@ -254,4 +255,3 @@ class CamembertForMultipleChoice(RobertaForMultipleChoice): """ config_class = CamembertConfig pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP - base_model_prefix = "camembert" diff --git a/transformers/tokenization_camembert.py b/transformers/tokenization_camembert.py index de587ac863..ae1b322941 100644 --- a/transformers/tokenization_camembert.py +++ b/transformers/tokenization_camembert.py @@ -87,7 +87,7 @@ class CamembertTokenizer(PreTrainedTokenizer): special tokens for the model Returns: - A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. + A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. """ if already_has_special_tokens: if token_ids_1 is not None: From 0477b307c7501ea76e01b03cb387a2312db752b3 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Fri, 15 Nov 2019 23:54:11 -0500 Subject: [PATCH 041/505] [camembert] tokenizer: use additional_special_tokens --- transformers/tokenization_camembert.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/transformers/tokenization_camembert.py b/transformers/tokenization_camembert.py index ae1b322941..41d3d74cff 100644 --- a/transformers/tokenization_camembert.py +++ b/transformers/tokenization_camembert.py @@ -45,10 +45,12 @@ class CamembertTokenizer(PreTrainedTokenizer): max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES def __init__(self, vocab_file, bos_token="", eos_token="", sep_token="", - cls_token="", unk_token="", pad_token='', mask_token='', **kwargs): + cls_token="", unk_token="", pad_token='', mask_token='', + additional_special_tokens=['NOTUSED', 'NOTUSED'], **kwargs): super(CamembertTokenizer, self).__init__(max_len=512, bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, sep_token=sep_token, cls_token=cls_token, pad_token=pad_token, - mask_token=mask_token, **kwargs) + mask_token=mask_token, additional_special_tokens=additional_special_tokens, + **kwargs) self.max_len_single_sentence = self.max_len - 2 # take into account special tokens self.max_len_sentences_pair = self.max_len - 4 # take into account special tokens self.sp_model = spm.SentencePieceProcessor() From d08a338c3bbe9964a4d44e5bdb15a45e985256c0 Mon Sep 17 00:00:00 2001 From: Yohei Tamura Date: Sat, 16 Nov 2019 18:47:37 +0900 Subject: [PATCH 042/505] modified: transformers/modeling_utils.py --- transformers/modeling_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/modeling_utils.py b/transformers/modeling_utils.py index d51eefab58..e7fd593bce 100644 --- a/transformers/modeling_utils.py +++ b/transformers/modeling_utils.py @@ -728,7 +728,7 @@ class SequenceSummary(nn.Module): def __init__(self, config): super(SequenceSummary, self).__init__() - self.summary_type = config.summary_type if hasattr(config, 'summary_use_proj') else 'last' + self.summary_type = config.summary_type if hasattr(config, 'summary_type') else 'last' if self.summary_type == 'attn': # We should use a standard multi-head attention module with absolute positional embedding for that. # Cf. https://github.com/zihangdai/xlnet/blob/master/modeling.py#L253-L276 From d32ce2c8df7053c19061b709465cdcc765e45a15 Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Mon, 18 Nov 2019 14:14:19 +0100 Subject: [PATCH 043/505] camembert: add wrapper for CamembertForTokenClassification --- transformers/modeling_camembert.py | 38 +++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/transformers/modeling_camembert.py b/transformers/modeling_camembert.py index 05538926e2..f302346f2d 100644 --- a/transformers/modeling_camembert.py +++ b/transformers/modeling_camembert.py @@ -20,7 +20,7 @@ from __future__ import (absolute_import, division, print_function, import logging -from .modeling_roberta import RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification, RobertaForMultipleChoice +from .modeling_roberta import RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification, RobertaForMultipleChoice, RobertaForTokenClassification from .configuration_camembert import CamembertConfig from .file_utils import add_start_docstrings @@ -255,3 +255,39 @@ class CamembertForMultipleChoice(RobertaForMultipleChoice): """ config_class = CamembertConfig pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP + + +@add_start_docstrings("""CamemBERT Model with a token classification head on top (a linear layer on top of + the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. """, + CAMEMBERT_START_DOCSTRING, CAMEMBERT_INPUTS_DOCSTRING) +class CamembertForTokenClassification(RobertaForTokenClassification): + r""" + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the token classification loss. + Indices should be in ``[0, ..., config.num_labels - 1]``. + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification loss. + **scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.num_labels)`` + Classification scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = CamembertTokenizer.from_pretrained('camembert-base') + model = CamembertForTokenClassification.from_pretrained('camembert-base') + input_ids = torch.tensor(tokenizer.encode("J'aime le camembert !", add_special_tokens=True)).unsqueeze(0) # Batch size 1 + labels = torch.tensor([1] * input_ids.size(1)).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, scores = outputs[:2] + + """ + config_class = CamembertConfig + pretrained_model_archive_map = CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP From 33753d9139307d9635db0309b6ddb9c53192c60a Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Mon, 18 Nov 2019 14:14:54 +0100 Subject: [PATCH 044/505] module: import CamembertForTokenClassification --- transformers/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/transformers/__init__.py b/transformers/__init__.py index cdf0669b39..5c7b0a6197 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -100,6 +100,7 @@ if is_torch_available(): DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_camembert import (CamembertForMaskedLM, CamembertModel, CamembertForSequenceClassification, CamembertForMultipleChoice, + CamembertForTokenClassification, CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_encoder_decoder import PreTrainedEncoderDecoder, Model2Model From 44455eb5b61b2fc4a0bdbea6775d328c45a597e5 Mon Sep 17 00:00:00 2001 From: Sebastian Stabinger Date: Mon, 18 Nov 2019 10:07:05 +0100 Subject: [PATCH 045/505] Adds CamemBERT to Model architectures list --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 40d6acb76e..d1b37dbd48 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,8 @@ At some point in the future, you'll be able to seamlessly move from pre-training 7. **[RoBERTa](https://github.com/pytorch/fairseq/tree/master/examples/roberta)** (from Facebook), released together with the paper a [Robustly Optimized BERT Pretraining Approach](https://arxiv.org/abs/1907.11692) by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov. 8. **[DistilBERT](https://github.com/huggingface/transformers/tree/master/examples/distillation)** (from HuggingFace), released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter](https://arxiv.org/abs/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same method has been applied to compress GPT2 into [DistilGPT2](https://github.com/huggingface/transformers/tree/master/examples/distillation). 9. **[CTRL](https://github.com/salesforce/ctrl/)** (from Salesforce) released with the paper [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/abs/1909.05858) by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher. -10. Want to contribute a new model? We have added a **detailed guide and templates** to guide you in the process of adding a new model. You can find them in the [`templates`](./templates) folder of the repository. Be sure to check the [contributing guidelines](./CONTRIBUTING.md) and contact the maintainers or open an issue to collect feedbacks before starting your PR. +10. **[CamemBERT](https://camembert-model.fr)** (from Facebook, Inria, Sorbonne) French language model based on RoBERTa released together with the paper [CamemBERT: a Tasty French Language Model](https://arxiv.org/abs/1911.03894) by Martin, Muller, and Suarez et al. +11. Want to contribute a new model? We have added a **detailed guide and templates** to guide you in the process of adding a new model. You can find them in the [`templates`](./templates) folder of the repository. Be sure to check the [contributing guidelines](./CONTRIBUTING.md) and contact the maintainers or open an issue to collect feedbacks before starting your PR. These implementations have been tested on several datasets (see the example scripts) and should match the performances of the original implementations (e.g. ~93 F1 on SQuAD for BERT Whole-Word-Masking, ~88 F1 on RocStories for OpenAI GPT, ~18.3 perplexity on WikiText 103 for Transformer-XL, ~0.916 Peason R coefficient on STS-B for XLNet). You can find more details on the performances in the Examples section of the [documentation](https://huggingface.co/transformers/examples.html). From 3916b334a86484af8442d1cfdb2f15695feae581 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Mon, 18 Nov 2019 09:29:11 -0500 Subject: [PATCH 046/505] [camembert] Acknowledge the full author list --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d1b37dbd48..a49e8086d1 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ At some point in the future, you'll be able to seamlessly move from pre-training 7. **[RoBERTa](https://github.com/pytorch/fairseq/tree/master/examples/roberta)** (from Facebook), released together with the paper a [Robustly Optimized BERT Pretraining Approach](https://arxiv.org/abs/1907.11692) by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov. 8. **[DistilBERT](https://github.com/huggingface/transformers/tree/master/examples/distillation)** (from HuggingFace), released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter](https://arxiv.org/abs/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same method has been applied to compress GPT2 into [DistilGPT2](https://github.com/huggingface/transformers/tree/master/examples/distillation). 9. **[CTRL](https://github.com/salesforce/ctrl/)** (from Salesforce) released with the paper [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/abs/1909.05858) by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher. -10. **[CamemBERT](https://camembert-model.fr)** (from Facebook, Inria, Sorbonne) French language model based on RoBERTa released together with the paper [CamemBERT: a Tasty French Language Model](https://arxiv.org/abs/1911.03894) by Martin, Muller, and Suarez et al. +10. **[CamemBERT](https://camembert-model.fr)** (from Inria/Facebook/Sorbonne) released with the paper [CamemBERT: a Tasty French Language Model](https://arxiv.org/abs/1911.03894) by Louis Martin*, Benjamin Muller*, Pedro Javier Ortiz Suárez*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah and Benoît Sagot. 11. Want to contribute a new model? We have added a **detailed guide and templates** to guide you in the process of adding a new model. You can find them in the [`templates`](./templates) folder of the repository. Be sure to check the [contributing guidelines](./CONTRIBUTING.md) and contact the maintainers or open an issue to collect feedbacks before starting your PR. These implementations have been tested on several datasets (see the example scripts) and should match the performances of the original implementations (e.g. ~93 F1 on SQuAD for BERT Whole-Word-Masking, ~88 F1 on RocStories for OpenAI GPT, ~18.3 perplexity on WikiText 103 for Transformer-XL, ~0.916 Peason R coefficient on STS-B for XLNet). You can find more details on the performances in the Examples section of the [documentation](https://huggingface.co/transformers/examples.html). From 0b3d45eb64607158977f546d57f90eae268c7836 Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Mon, 18 Nov 2019 15:49:44 +0100 Subject: [PATCH 047/505] camembert: add implementation for save_vocabulary method --- transformers/tokenization_camembert.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/transformers/tokenization_camembert.py b/transformers/tokenization_camembert.py index 41d3d74cff..bf2a6fe993 100644 --- a/transformers/tokenization_camembert.py +++ b/transformers/tokenization_camembert.py @@ -16,9 +16,14 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) +import logging +import os +from shutil import copyfile + import sentencepiece as spm from transformers.tokenization_utils import PreTrainedTokenizer +logger = logging.getLogger(__name__) VOCAB_FILES_NAMES = {'vocab_file': 'sentencepiece.bpe.model'} @@ -55,6 +60,7 @@ class CamembertTokenizer(PreTrainedTokenizer): self.max_len_sentences_pair = self.max_len - 4 # take into account special tokens self.sp_model = spm.SentencePieceProcessor() self.sp_model.Load(str(vocab_file)) + self.vocab_file = vocab_file # HACK: These tokens were added by fairseq but don't seem to be actually used when duplicated in the actual # sentencepiece vocabulary (this is the case for and self.fairseq_tokens_to_ids = {'NOTUSED': 0, '': 1, 'NOTUSED': 2, '': 3} @@ -135,3 +141,17 @@ class CamembertTokenizer(PreTrainedTokenizer): if index in self.fairseq_ids_to_tokens: return self.fairseq_ids_to_tokens[index] return self.sp_model.IdToPiece(index - self.fairseq_offset) + + def save_vocabulary(self, save_directory): + """ Save the sentencepiece vocabulary (copy original file) and special tokens file + to a directory. + """ + if not os.path.isdir(save_directory): + logger.error("Vocabulary path ({}) should be a directory".format(save_directory)) + return + out_vocab_file = os.path.join(save_directory, VOCAB_FILES_NAMES['vocab_file']) + + if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file): + copyfile(self.vocab_file, out_vocab_file) + + return (out_vocab_file,) From 56c84863a1a20dfb82b928c5c9f77c21d9def8c7 Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Mon, 18 Nov 2019 15:50:16 +0100 Subject: [PATCH 048/505] camembert: add support for CamemBERT in run_ner example --- examples/run_ner.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/run_ner.py b/examples/run_ner.py index 4359e587ae..127d63a6cd 100644 --- a/examples/run_ner.py +++ b/examples/run_ner.py @@ -37,6 +37,7 @@ from transformers import AdamW, get_linear_schedule_with_warmup from transformers import WEIGHTS_NAME, BertConfig, BertForTokenClassification, BertTokenizer from transformers import RobertaConfig, RobertaForTokenClassification, RobertaTokenizer from transformers import DistilBertConfig, DistilBertForTokenClassification, DistilBertTokenizer +from transformers import CamembertConfig, CamembertForTokenClassification, CamembertTokenizer logger = logging.getLogger(__name__) @@ -47,7 +48,8 @@ ALL_MODELS = sum( MODEL_CLASSES = { "bert": (BertConfig, BertForTokenClassification, BertTokenizer), "roberta": (RobertaConfig, RobertaForTokenClassification, RobertaTokenizer), - "distilbert": (DistilBertConfig, DistilBertForTokenClassification, DistilBertTokenizer) + "distilbert": (DistilBertConfig, DistilBertForTokenClassification, DistilBertTokenizer), + "camembert": (CamembertConfig, CamembertForTokenClassification, CamembertTokenizer), } From f3386d938348628c91457fc7d8650c223317a053 Mon Sep 17 00:00:00 2001 From: Kazutoshi Shinoda Date: Sun, 17 Nov 2019 18:08:51 +0900 Subject: [PATCH 049/505] typo "deay" -> "decay" --- examples/run_squad.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/run_squad.py b/examples/run_squad.py index d7fdc32ae7..69088d73c3 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -402,7 +402,7 @@ def main(): parser.add_argument('--gradient_accumulation_steps', type=int, default=1, help="Number of updates steps to accumulate before performing a backward/update pass.") parser.add_argument("--weight_decay", default=0.0, type=float, - help="Weight deay if we apply some.") + help="Weight decay if we apply some.") parser.add_argument("--adam_epsilon", default=1e-8, type=float, help="Epsilon for Adam optimizer.") parser.add_argument("--max_grad_norm", default=1.0, type=float, From 4193aa9f813c7868117214c1261b01c79fb9420f Mon Sep 17 00:00:00 2001 From: alexzubiaga Date: Tue, 19 Nov 2019 10:32:08 +0100 Subject: [PATCH 050/505] add TFXLNetForTokenClassification implementation and unit test add XLNetForTokenClassification implementation and unit tests --- transformers/__init__.py | 8 +- transformers/modeling_tf_xlnet.py | 53 ++++++++++ transformers/modeling_xlnet.py | 100 +++++++++++++++++++ transformers/tests/modeling_tf_xlnet_test.py | 26 +++++ transformers/tests/modeling_xlnet_test.py | 56 +++++++++-- 5 files changed, 232 insertions(+), 11 deletions(-) diff --git a/transformers/__init__.py b/transformers/__init__.py index cdf0669b39..ecfb8aeff2 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -83,9 +83,10 @@ if is_torch_available(): CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_xlnet import (XLNetPreTrainedModel, XLNetModel, XLNetLMHeadModel, - XLNetForSequenceClassification, XLNetForMultipleChoice, - XLNetForQuestionAnsweringSimple, XLNetForQuestionAnswering, - load_tf_weights_in_xlnet, XLNET_PRETRAINED_MODEL_ARCHIVE_MAP) + XLNetForSequenceClassification, XLNetForTokenClassification, + XLNetForMultipleChoice, XLNetForQuestionAnsweringSimple, + XLNetForQuestionAnswering, load_tf_weights_in_xlnet, + XLNET_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_xlm import (XLMPreTrainedModel , XLMModel, XLMWithLMHeadModel, XLMForSequenceClassification, XLMForQuestionAnswering, XLMForQuestionAnsweringSimple, @@ -136,6 +137,7 @@ if is_tf_available(): from .modeling_tf_xlnet import (TFXLNetPreTrainedModel, TFXLNetMainLayer, TFXLNetModel, TFXLNetLMHeadModel, TFXLNetForSequenceClassification, + TFXLNetForTokenClassification, TFXLNetForQuestionAnsweringSimple, TF_XLNET_PRETRAINED_MODEL_ARCHIVE_MAP) diff --git a/transformers/modeling_tf_xlnet.py b/transformers/modeling_tf_xlnet.py index 4733ea8589..b5e824e924 100644 --- a/transformers/modeling_tf_xlnet.py +++ b/transformers/modeling_tf_xlnet.py @@ -939,6 +939,59 @@ class TFXLNetForSequenceClassification(TFXLNetPreTrainedModel): return outputs # return logits, (mems), (hidden states), (attentions) +@add_start_docstrings("""XLNet Model with a token classification head on top (a linear layer on top of + the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. """, + XLNET_START_DOCSTRING, XLNET_INPUTS_DOCSTRING) +class TFXLNetForTokenClassification(TFXLNetPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **scores**: ``tf.Tensor`` of shape ``(batch_size, sequence_length, config.num_labels)`` + Classification scores (before SoftMax). + **mems**: (`optional`, returned when ``config.mem_len > 0``) + list of ``tf.Tensor`` (one for each layer): + that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model + if config.mem_len > 0 else tuple of None. Can be used to speed up sequential decoding and attend to longer context. + See details in the docstring of the `mems` input above. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + import tensorflow as tf + from transformers import XLNetTokenizer, TFXLNetForTokenClassification + + tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased') + model = TFXLNetForSequenceClassification.from_pretrained('xlnet-large-cased') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + outputs = model(input_ids) + scores = outputs[0] + + """ + def __init__(self, config, *inputs, **kwargs): + super(TFXLNetForTokenClassification, self).__init__(config, *inputs, **kwargs) + self.num_labels = config.num_labels + + self.transformer = TFXLNetMainLayer(config, name='transformer') + self.classifier = tf.keras.layers.Dense(config.num_labels, + kernel_initializer=get_initializer(config.initializer_range), + name='classifier') + + def call(self, inputs, **kwargs): + transformer_outputs = self.transformer(inputs, **kwargs) + output = transformer_outputs[0] + + logits = self.classifier(output) + + outputs = (logits,) + transformer_outputs[1:] # Keep mems, hidden states, attentions if there are in it + + return outputs # return logits, (mems), (hidden states), (attentions) + + # @add_start_docstrings("""XLNet Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of # the hidden-states output to compute `span start logits` and `span end logits`). """, # XLNET_START_DOCSTRING, XLNET_INPUTS_DOCSTRING) diff --git a/transformers/modeling_xlnet.py b/transformers/modeling_xlnet.py index 658048a660..2f4f883905 100644 --- a/transformers/modeling_xlnet.py +++ b/transformers/modeling_xlnet.py @@ -1046,6 +1046,106 @@ class XLNetForSequenceClassification(XLNetPreTrainedModel): return outputs # return (loss), logits, (mems), (hidden states), (attentions) +@add_start_docstrings("""XLNet Model with a token classification head on top (a linear layer on top of + the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. """, + XLNET_START_DOCSTRING, + XLNET_INPUTS_DOCSTRING) +class XLNetForTokenClassification(XLNetPreTrainedModel): + r""" + Inputs: + **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, num_choices, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + The second dimension of the input (`num_choices`) indicates the number of choices to scores. + **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Segment token indices to indicate first and second portions of the inputs. + Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` + **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: + Mask to avoid performing attention on padding token indices. + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. + **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. + **inputs_embeds**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, embedding_dim)``: + Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the multiple choice classification loss. + Indices should be in ``[0, ..., num_choices]`` where `num_choices` is the size of the second dimension + of the input tensors. (see `input_ids` above) + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification loss. + **scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.num_labels)`` + Classification scores (before SoftMax). + **mems**: (`optional`, returned when ``config.mem_len > 0``) + list of ``torch.FloatTensor`` (one for each layer): + that contains pre-computed hidden-states (key and values in the attention blocks) as computed by the model + if config.mem_len > 0 else tuple of None. Can be used to speed up sequential decoding and attend to longer context. + See details in the docstring of the `mems` input above. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = XLNetTokenizer.from_pretrained('xlnet-large-cased') + model = XLNetForSequenceClassification.from_pretrained('xlnet-large-cased') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + labels = torch.tensor([1] * input_ids.size(1)).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + scores = outputs[0] + + """ + def __init__(self, config): + super(XLNetForTokenClassification, self).__init__(config) + self.num_labels = config.num_labels + + self.transformer = XLNetModel(config) + self.classifier = nn.Linear(config.hidden_size, config.num_labels) + + self.init_weights() + + def forward(self, input_ids=None, attention_mask=None, mems=None, perm_mask=None, target_mapping=None, + token_type_ids=None, input_mask=None, head_mask=None, inputs_embeds=None, labels=None): + + outputs = self.transformer(input_ids, + attention_mask=attention_mask, + mems=mems, + perm_mask=perm_mask, + target_mapping=target_mapping, + token_type_ids=token_type_ids, + input_mask=input_mask, + head_mask=head_mask, + inputs_embeds=inputs_embeds) + + sequence_output = outputs[0] + + logits = self.classifier(sequence_output) + + outputs = (logits,) + outputs[1:] # Keep mems, hidden states, attentions if there are in it + if labels is not None: + loss_fct = CrossEntropyLoss() + # Only keep active parts of the loss + if attention_mask is not None: + active_loss = attention_mask.view(-1) == 1 + active_logits = logits.view(-1, self.num_labels)[active_loss] + active_labels = labels.view(-1)[active_loss] + loss = loss_fct(active_logits, active_labels) + else: + loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) + outputs = (loss,) + outputs + + return outputs # return (loss), logits, (mems), (hidden states), (attentions) + + @add_start_docstrings("""XLNet Model with a multiple choice classification head on top (a linear layer on top of the pooled output and a softmax) e.g. for RACE/SWAG tasks. """, XLNET_START_DOCSTRING, XLNET_INPUTS_DOCSTRING) diff --git a/transformers/tests/modeling_tf_xlnet_test.py b/transformers/tests/modeling_tf_xlnet_test.py index 12a8fbe36f..a00a965570 100644 --- a/transformers/tests/modeling_tf_xlnet_test.py +++ b/transformers/tests/modeling_tf_xlnet_test.py @@ -30,6 +30,7 @@ if is_tf_available(): from transformers.modeling_tf_xlnet import (TFXLNetModel, TFXLNetLMHeadModel, TFXLNetForSequenceClassification, + TFXLNetForTokenClassification, TFXLNetForQuestionAnsweringSimple, TF_XLNET_PRETRAINED_MODEL_ARCHIVE_MAP) else: @@ -42,6 +43,7 @@ class TFXLNetModelTest(TFCommonTestCases.TFCommonModelTester): all_model_classes=(TFXLNetModel, TFXLNetLMHeadModel, TFXLNetForSequenceClassification, + TFXLNetForTokenClassification, TFXLNetForQuestionAnsweringSimple) if is_tf_available() else () test_pruning = False @@ -258,6 +260,26 @@ class TFXLNetModelTest(TFCommonTestCases.TFCommonModelTester): list(list(mem.shape) for mem in result["mems_1"]), [[self.seq_length, self.batch_size, self.hidden_size]] * self.num_hidden_layers) + def create_and_check_xlnet_for_token_classification(self, config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, + target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels): + config.num_labels = input_ids_1.shape[1] + model = TFXLNetForTokenClassification(config) + inputs = {'input_ids': input_ids_1, + 'attention_mask': input_mask, + # 'token_type_ids': token_type_ids + } + logits, mems_1 = model(inputs) + result = { + "mems_1": [mem.numpy() for mem in mems_1], + "logits": logits.numpy(), + } + self.parent.assertListEqual( + list(result["logits"].shape), + [self.batch_size, self.seq_length, config.num_labels]) + self.parent.assertListEqual( + list(list(mem.shape) for mem in result["mems_1"]), + [[self.seq_length, self.batch_size, self.hidden_size]] * self.num_hidden_layers) + def prepare_config_and_inputs_for_common(self): config_and_inputs = self.prepare_config_and_inputs() (config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, @@ -289,6 +311,10 @@ class TFXLNetModelTest(TFCommonTestCases.TFCommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_xlnet_sequence_classif(*config_and_inputs) + def test_xlnet_token_classification(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_xlnet_for_token_classification(*config_and_inputs) + def test_xlnet_qa(self): self.model_tester.set_seed() config_and_inputs = self.model_tester.prepare_config_and_inputs() diff --git a/transformers/tests/modeling_xlnet_test.py b/transformers/tests/modeling_xlnet_test.py index d97ea6a425..8f35d34e14 100644 --- a/transformers/tests/modeling_xlnet_test.py +++ b/transformers/tests/modeling_xlnet_test.py @@ -28,7 +28,8 @@ from transformers import is_torch_available if is_torch_available(): import torch - from transformers import (XLNetConfig, XLNetModel, XLNetLMHeadModel, XLNetForSequenceClassification, XLNetForQuestionAnswering) + from transformers import (XLNetConfig, XLNetModel, XLNetLMHeadModel, XLNetForSequenceClassification, + XLNetForTokenClassification, XLNetForQuestionAnswering) from transformers.modeling_xlnet import XLNET_PRETRAINED_MODEL_ARCHIVE_MAP else: pytestmark = pytest.mark.skip("Require Torch") @@ -38,7 +39,7 @@ from .configuration_common_test import ConfigTester class XLNetModelTest(CommonTestCases.CommonModelTester): - all_model_classes=(XLNetModel, XLNetLMHeadModel, + all_model_classes=(XLNetModel, XLNetLMHeadModel, XLNetForTokenClassification, XLNetForSequenceClassification, XLNetForQuestionAnswering) if is_torch_available() else () test_pruning = False @@ -107,10 +108,12 @@ class XLNetModelTest(CommonTestCases.CommonModelTester): sequence_labels = None lm_labels = None is_impossible_labels = None + token_labels = None if self.use_labels: lm_labels = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size) is_impossible_labels = ids_tensor([self.batch_size], 2).float() + token_labels = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size) config = XLNetConfig( vocab_size_or_config_json_file=self.vocab_size, @@ -129,14 +132,14 @@ class XLNetModelTest(CommonTestCases.CommonModelTester): num_labels=self.type_sequence_label_size) return (config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, - target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels) + target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels, token_labels) def set_seed(self): random.seed(self.seed) torch.manual_seed(self.seed) def create_and_check_xlnet_base_model(self, config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, - target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels): + target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels, token_labels): model = XLNetModel(config) model.eval() @@ -164,7 +167,7 @@ class XLNetModelTest(CommonTestCases.CommonModelTester): [[self.seq_length, self.batch_size, self.hidden_size]] * self.num_hidden_layers) def create_and_check_xlnet_lm_head(self, config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, - target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels): + target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels, token_labels): model = XLNetLMHeadModel(config) model.eval() @@ -204,7 +207,7 @@ class XLNetModelTest(CommonTestCases.CommonModelTester): [[self.mem_len, self.batch_size, self.hidden_size]] * self.num_hidden_layers) def create_and_check_xlnet_qa(self, config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, - target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels): + target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels, token_labels): model = XLNetForQuestionAnswering(config) model.eval() @@ -261,8 +264,40 @@ class XLNetModelTest(CommonTestCases.CommonModelTester): list(list(mem.size()) for mem in result["mems"]), [[self.seq_length, self.batch_size, self.hidden_size]] * self.num_hidden_layers) + def create_and_check_xlnet_token_classif(self, config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, + target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels, token_labels): + model = XLNetForTokenClassification(config) + model.eval() + + logits, mems_1 = model(input_ids_1) + loss, logits, mems_1 = model(input_ids_1, labels=token_labels) + + result = { + "loss": loss, + "mems_1": mems_1, + "logits": logits, + } + + self.parent.assertListEqual( + list(result["loss"].size()), + []) + self.parent.assertListEqual( + list(result["logits"].size()), + [self.batch_size, self.seq_length, self.type_sequence_label_size]) + self.parent.assertListEqual( + list(list(mem.size()) for mem in result["mems_1"]), + [[self.seq_length, self.batch_size, self.hidden_size]] * self.num_hidden_layers) + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + (config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, + target_mapping, segment_ids, lm_labels, + sequence_labels, is_impossible_labels) = config_and_inputs + inputs_dict = {'input_ids': input_ids_1} + return config, inputs_dict + def create_and_check_xlnet_sequence_classif(self, config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, - target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels): + target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels, token_labels): model = XLNetForSequenceClassification(config) model.eval() @@ -289,7 +324,7 @@ class XLNetModelTest(CommonTestCases.CommonModelTester): config_and_inputs = self.prepare_config_and_inputs() (config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, target_mapping, segment_ids, lm_labels, - sequence_labels, is_impossible_labels) = config_and_inputs + sequence_labels, is_impossible_labels, token_labels) = config_and_inputs inputs_dict = {'input_ids': input_ids_1} return config, inputs_dict @@ -316,6 +351,11 @@ class XLNetModelTest(CommonTestCases.CommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_xlnet_sequence_classif(*config_and_inputs) + def test_xlnet_token_classif(self): + self.model_tester.set_seed() + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_xlnet_token_classif(*config_and_inputs) + def test_xlnet_qa(self): self.model_tester.set_seed() config_and_inputs = self.model_tester.prepare_config_and_inputs() From 337802783f0ba87a58e2e9adddb47ac5cb00646d Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Tue, 19 Nov 2019 19:50:32 +0100 Subject: [PATCH 051/505] distilbert: add configuration for new German distilbert model --- transformers/configuration_distilbert.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/transformers/configuration_distilbert.py b/transformers/configuration_distilbert.py index 2a8a149acf..57b9e57fe0 100644 --- a/transformers/configuration_distilbert.py +++ b/transformers/configuration_distilbert.py @@ -27,7 +27,8 @@ logger = logging.getLogger(__name__) DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = { 'distilbert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-config.json", - 'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-config.json" + 'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-config.json", + 'distilbert-base-german-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-german-cased-config.json" } From 22333945fbd1f5eff8cfb664ac6f63363e6b72d4 Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Tue, 19 Nov 2019 19:51:01 +0100 Subject: [PATCH 052/505] distilbert: add pytorch model for new German distilbert model --- transformers/modeling_distilbert.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/transformers/modeling_distilbert.py b/transformers/modeling_distilbert.py index d30f493c69..cb2fa1915e 100644 --- a/transformers/modeling_distilbert.py +++ b/transformers/modeling_distilbert.py @@ -42,7 +42,8 @@ logger = logging.getLogger(__name__) DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP = { 'distilbert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-pytorch_model.bin", - 'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-pytorch_model.bin" + 'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-pytorch_model.bin", + 'distilbert-base-german-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-german-cased-pytorch_model.bin" } From f21dfe36baaf316675a4d2f8c918e9e8afc11db2 Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Tue, 19 Nov 2019 19:51:31 +0100 Subject: [PATCH 053/505] distilbert: add vocab for new German distilbert model --- transformers/tokenization_distilbert.py | 1 + 1 file changed, 1 insertion(+) diff --git a/transformers/tokenization_distilbert.py b/transformers/tokenization_distilbert.py index dfa02926d8..61f967bf0b 100644 --- a/transformers/tokenization_distilbert.py +++ b/transformers/tokenization_distilbert.py @@ -33,6 +33,7 @@ PRETRAINED_VOCAB_FILES_MAP = { { 'distilbert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt", 'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt", + 'distilbert-base-german-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-german-cased-vocab.txt" } } From e631383d4fb55ed2dff53f2b80a29e51cd2f563d Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Tue, 19 Nov 2019 19:52:40 +0100 Subject: [PATCH 054/505] docs: add new German distilbert model to pretrained models --- docs/source/pretrained_models.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst index b0a578fd80..3b25a79802 100644 --- a/docs/source/pretrained_models.rst +++ b/docs/source/pretrained_models.rst @@ -151,6 +151,10 @@ Here is the full list of the currently provided pretrained models together with | | ``distilroberta-base`` | | 6-layer, 768-hidden, 12-heads, 82M parameters | | | | | The DistilRoBERTa model distilled from the RoBERTa model `roberta-base` checkpoint. | | | | (see `details `__) | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``distilbert-base-german-cased`` | | 6-layer, 768-hidden, 12-heads, 66M parameters | +| | | | The German DistilBERT model distilled from the German DBMDZ BERT model `bert-base-german-dbmdz-cased` checkpoint. | +| | | (see `details `__) | +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | CTRL | ``ctrl`` | | 48-layer, 1280-hidden, 16-heads, 1.6B parameters | | | | | Salesforce's Large-sized CTRL English model | From e7cf2ccd1567615513013d5fc9f9002733f70e13 Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Tue, 19 Nov 2019 19:55:19 +0100 Subject: [PATCH 055/505] distillation: add German distilbert model --- examples/distillation/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/distillation/README.md b/examples/distillation/README.md index 8efd1ea6f4..c2765c28ff 100644 --- a/examples/distillation/README.md +++ b/examples/distillation/README.md @@ -2,6 +2,8 @@ This folder contains the original code used to train Distil* as well as examples showcasing how to use DistilBERT, DistilRoBERTa and DistilGPT2. +**November 19th, 2019 - Update** We release German **DistilBERT**: 98.8% of `bert-base-german-dbmdz-cased` on NER tasks. + **October 23rd, 2019 - Update** We release **DistilRoBERTa**: 95% of `RoBERTa-base`'s performance on GLUE, twice as fast as RoBERTa while being 35% smaller. **October 3rd, 2019 - Update** We release our [NeurIPS workshop paper](https://arxiv.org/abs/1910.01108) explaining our approach on **DistilBERT**. It includes updated results and further experiments. We applied the same method to GPT2 and release the weights of **DistilGPT2**. DistilGPT2 is two times faster and 33% smaller than GPT2. **The paper superseeds our [previous blogpost](https://medium.com/huggingface/distilbert-8cf3380435b5) with a different distillation loss and better performances. Please use the paper as a reference when comparing/reporting results on DistilBERT.** @@ -45,10 +47,11 @@ This part of the library has only be tested with Python3.6+. There are few speci ## How to use DistilBERT -Transformers includes two pre-trained Distil* models, currently only provided for English (we are investigating the possibility to train and release a multilingual version of DistilBERT): +Transformers includes five pre-trained Distil* models, currently only provided for English and German (we are investigating the possibility to train and release a multilingual version of DistilBERT): - `distilbert-base-uncased`: DistilBERT English language model pretrained on the same data used to pretrain Bert (concatenation of the Toronto Book Corpus and full English Wikipedia) using distillation with the supervision of the `bert-base-uncased` version of Bert. The model has 6 layers, 768 dimension and 12 heads, totalizing 66M parameters. - `distilbert-base-uncased-distilled-squad`: A finetuned version of `distilbert-base-uncased` finetuned using (a second step of) knwoledge distillation on SQuAD 1.0. This model reaches a F1 score of 86.9 on the dev set (for comparison, Bert `bert-base-uncased` version reaches a 88.5 F1 score). +- `distilbert-base-german-cased`: DistilBERT German language model pretrained on 1/2 of the data used to pretrain Bert using distillation with the supervision of the `bert-base-german-dbmdz-cased` version of German DBMDZ Bert. For NER tasks the model reaches a F1 score of 83.49 on the CoNLL-2003 test set (for comparison, `bert-base-german-dbmdz-cased` reaches a 84.52 F1 score), and a F1 score of 85.23 on the GermEval 2014 test set (`bert-base-german-dbmdz-cased` reaches a 86.89 F1 score). - `distilgpt2`: DistilGPT2 English language model pretrained with the supervision of `gpt2` (the smallest version of GPT2) on [OpenWebTextCorpus](https://skylion007.github.io/OpenWebTextCorpus/), a reproduction of OpenAI's WebText dataset. The model has 6 layers, 768 dimension and 12 heads, totalizing 82M parameters (compared to 124M parameters for GPT2). On average, DistilGPT2 is two times faster than GPT2. - `distilroberta-base`: DistilRoBERTa English language model pretrained with the supervision of `roberta-base` solely on [OpenWebTextCorpus](https://skylion007.github.io/OpenWebTextCorpus/), a reproduction of OpenAI's WebText dataset (it is ~4 times less training data than the teacher RoBERTa). The model has 6 layers, 768 dimension and 12 heads, totalizing 82M parameters (compared to 125M parameters for RoBERTa-base). On average DistilRoBERTa is twice as fast as Roberta-base. - and more to come! 🤗🤗🤗 From 2e2c0375c3e725483d2ec297632dc52ed6a9f5fb Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Tue, 19 Nov 2019 20:41:18 +0100 Subject: [PATCH 056/505] distilbert: add German distilbert model to positional embedding sizes map --- transformers/tokenization_distilbert.py | 1 + 1 file changed, 1 insertion(+) diff --git a/transformers/tokenization_distilbert.py b/transformers/tokenization_distilbert.py index 61f967bf0b..c82ac09727 100644 --- a/transformers/tokenization_distilbert.py +++ b/transformers/tokenization_distilbert.py @@ -40,6 +40,7 @@ PRETRAINED_VOCAB_FILES_MAP = { PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { 'distilbert-base-uncased': 512, 'distilbert-base-uncased-distilled-squad': 512, + 'distilbert-base-german-cased': 512, } From da06afafc87b03f7588b6bd319e1b7592b091339 Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Tue, 19 Nov 2019 21:57:00 +0100 Subject: [PATCH 057/505] tree-wide: add trailing comma in configuration maps --- transformers/configuration_distilbert.py | 2 +- transformers/modeling_distilbert.py | 2 +- transformers/tokenization_distilbert.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/transformers/configuration_distilbert.py b/transformers/configuration_distilbert.py index 57b9e57fe0..1c35f5c0dd 100644 --- a/transformers/configuration_distilbert.py +++ b/transformers/configuration_distilbert.py @@ -28,7 +28,7 @@ logger = logging.getLogger(__name__) DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = { 'distilbert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-config.json", 'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-config.json", - 'distilbert-base-german-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-german-cased-config.json" + 'distilbert-base-german-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-german-cased-config.json", } diff --git a/transformers/modeling_distilbert.py b/transformers/modeling_distilbert.py index cb2fa1915e..4ba75248a1 100644 --- a/transformers/modeling_distilbert.py +++ b/transformers/modeling_distilbert.py @@ -43,7 +43,7 @@ logger = logging.getLogger(__name__) DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP = { 'distilbert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-pytorch_model.bin", 'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-pytorch_model.bin", - 'distilbert-base-german-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-german-cased-pytorch_model.bin" + 'distilbert-base-german-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-german-cased-pytorch_model.bin", } diff --git a/transformers/tokenization_distilbert.py b/transformers/tokenization_distilbert.py index c82ac09727..6574e0bc4d 100644 --- a/transformers/tokenization_distilbert.py +++ b/transformers/tokenization_distilbert.py @@ -33,7 +33,7 @@ PRETRAINED_VOCAB_FILES_MAP = { { 'distilbert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt", 'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt", - 'distilbert-base-german-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-german-cased-vocab.txt" + 'distilbert-base-german-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-german-cased-vocab.txt", } } From 3de31f8d287da44a40566fb1d5c44107708b87ea Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 19 Nov 2019 18:14:14 -0500 Subject: [PATCH 058/505] mean does not exist in TF2 --- transformers/modeling_tf_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/modeling_tf_utils.py b/transformers/modeling_tf_utils.py index e08605d154..8be7eaaf67 100644 --- a/transformers/modeling_tf_utils.py +++ b/transformers/modeling_tf_utils.py @@ -454,7 +454,7 @@ class TFSequenceSummary(tf.keras.layers.Layer): elif self.summary_type == 'first': output = hidden_states[:, 0] elif self.summary_type == 'mean': - output = tf.mean(hidden_states, axis=1) + output = tf.reduce_mean(hidden_states, axis=1) elif self.summary_type == 'cls_index': hidden_shape = shape_list(hidden_states) # e.g. [batch, num choices, seq length, hidden dims] if cls_index is None: From 454455c695ff38df1ed3670a43677fdd1abcedf3 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 20 Nov 2019 09:42:48 -0500 Subject: [PATCH 059/505] fix #1879 --- examples/utils_squad.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/utils_squad.py b/examples/utils_squad.py index c04dacf6d3..4f1c581588 100644 --- a/examples/utils_squad.py +++ b/examples/utils_squad.py @@ -240,6 +240,7 @@ def convert_examples_to_features(examples, tokenizer, max_seq_length, # The -3 accounts for [CLS], [SEP] and [SEP] max_tokens_for_doc = max_seq_length - len(query_tokens) - 3 + assert max_tokens_for_doc > 0 # We can have documents that are longer than the maximum sequence length. # To deal with this we do a sliding window approach, where we take chunks From e70cdf083ddb8bfe298d43e6d70d698a3a2f56d3 Mon Sep 17 00:00:00 2001 From: Jin Young Sohn Date: Wed, 20 Nov 2019 22:33:26 +0000 Subject: [PATCH 060/505] Cleanup TPU bits from run_glue.py TPU runner is currently implemented in: https://github.com/pytorch-tpu/transformers/blob/tpu/examples/run_glue_tpu.py. We plan to upstream this directly into `huggingface/transformers` (either `master` or `tpu`) branch once it's been more thoroughly tested. --- examples/run_glue.py | 35 ++--------------------------------- 1 file changed, 2 insertions(+), 33 deletions(-) diff --git a/examples/run_glue.py b/examples/run_glue.py index 19316cb0ec..527e440075 100644 --- a/examples/run_glue.py +++ b/examples/run_glue.py @@ -158,7 +158,7 @@ def train(args, train_dataset, model, tokenizer): loss.backward() tr_loss += loss.item() - if (step + 1) % args.gradient_accumulation_steps == 0 and not args.tpu: + if (step + 1) % args.gradient_accumulation_steps == 0: if args.fp16: torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm) else: @@ -189,11 +189,6 @@ def train(args, train_dataset, model, tokenizer): torch.save(args, os.path.join(output_dir, 'training_args.bin')) logger.info("Saving model checkpoint to %s", output_dir) - if args.tpu: - args.xla_model.optimizer_step(optimizer, barrier=True) - model.zero_grad() - global_step += 1 - if args.max_steps > 0 and global_step > args.max_steps: epoch_iterator.close() break @@ -397,15 +392,6 @@ def main(): parser.add_argument('--seed', type=int, default=42, help="random seed for initialization") - parser.add_argument('--tpu', action='store_true', - help="Whether to run on the TPU defined in the environment variables") - parser.add_argument('--tpu_ip_address', type=str, default='', - help="TPU IP address if none are set in the environment variables") - parser.add_argument('--tpu_name', type=str, default='', - help="TPU name if none are set in the environment variables") - parser.add_argument('--xrt_tpu_config', type=str, default='', - help="XRT TPU config if none are set in the environment variables") - parser.add_argument('--fp16', action='store_true', help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit") parser.add_argument('--fp16_opt_level', type=str, default='O1', @@ -439,23 +425,6 @@ def main(): args.n_gpu = 1 args.device = device - if args.tpu: - if args.tpu_ip_address: - os.environ["TPU_IP_ADDRESS"] = args.tpu_ip_address - if args.tpu_name: - os.environ["TPU_NAME"] = args.tpu_name - if args.xrt_tpu_config: - os.environ["XRT_TPU_CONFIG"] = args.xrt_tpu_config - - assert "TPU_IP_ADDRESS" in os.environ - assert "TPU_NAME" in os.environ - assert "XRT_TPU_CONFIG" in os.environ - - import torch_xla - import torch_xla.core.xla_model as xm - args.device = xm.xla_device() - args.xla_model = xm - # Setup logging logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt = '%m/%d/%Y %H:%M:%S', @@ -509,7 +478,7 @@ def main(): # Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained() - if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0) and not args.tpu: + if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0): # Create output directory if needed if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]: os.makedirs(args.output_dir) From 2cf3447e0a3e3fd04b715f1e6f4ee43575e1e7c9 Mon Sep 17 00:00:00 2001 From: Juha Kiili Date: Thu, 21 Nov 2019 12:35:25 +0200 Subject: [PATCH 061/505] Glue: log in Valohai-compatible JSON format too --- examples/run_glue.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/examples/run_glue.py b/examples/run_glue.py index 527e440075..ea5ac5bbb7 100644 --- a/examples/run_glue.py +++ b/examples/run_glue.py @@ -22,6 +22,7 @@ import glob import logging import os import random +import json import numpy as np import torch @@ -171,13 +172,21 @@ def train(args, train_dataset, model, tokenizer): if args.local_rank in [-1, 0] and args.logging_steps > 0 and global_step % args.logging_steps == 0: # Log metrics + logs = {'step': global_step} if args.local_rank == -1 and args.evaluate_during_training: # Only evaluate when single GPU otherwise metrics may not average well results = evaluate(args, model, tokenizer) for key, value in results.items(): - tb_writer.add_scalar('eval_{}'.format(key), value, global_step) - tb_writer.add_scalar('lr', scheduler.get_lr()[0], global_step) - tb_writer.add_scalar('loss', (tr_loss - logging_loss)/args.logging_steps, global_step) + eval_key = 'eval_{}'.format(key) + tb_writer.add_scalar(eval_key, value, global_step) + logs[eval_key] = str(value) logging_loss = tr_loss + loss_scalar = (tr_loss - logging_loss) / args.logging_steps + learning_rate_scalar = scheduler.get_lr()[0] + tb_writer.add_scalar('lr', learning_rate_scalar, global_step) + tb_writer.add_scalar('loss', loss_scalar, global_step) + logs['learning_rate'] = learning_rate_scalar + logs['loss'] = loss_scalar + print(json.dumps(logs)) if args.local_rank in [-1, 0] and args.save_steps > 0 and global_step % args.save_steps == 0: # Save model checkpoint From aac35514075290a46419b9bd969e6f94fef9d43b Mon Sep 17 00:00:00 2001 From: Aarni Koskela Date: Thu, 21 Nov 2019 12:37:39 +0200 Subject: [PATCH 062/505] Add download_glue_data.py from kamalkraj/ALBERT-TF2.0 Original source: https://github.com/kamalkraj/ALBERT-TF2.0/blob/fa90194e5fe729dbb19f32ac29c8d6d6372c0f93/download_glue_data.py Original license: https://github.com/kamalkraj/ALBERT-TF2.0/blob/fa90194e5fe729dbb19f32ac29c8d6d6372c0f93/LICENSE (Apache-2.0) --- utils/download_glue_data.py | 141 ++++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 utils/download_glue_data.py diff --git a/utils/download_glue_data.py b/utils/download_glue_data.py new file mode 100644 index 0000000000..86a4e8951f --- /dev/null +++ b/utils/download_glue_data.py @@ -0,0 +1,141 @@ +''' Script for downloading all GLUE data. + +Note: for legal reasons, we are unable to host MRPC. +You can either use the version hosted by the SentEval team, which is already tokenized, +or you can download the original data from (https://download.microsoft.com/download/D/4/6/D46FF87A-F6B9-4252-AA8B-3604ED519838/MSRParaphraseCorpus.msi) and extract the data from it manually. +For Windows users, you can run the .msi file. For Mac and Linux users, consider an external library such as 'cabextract' (see below for an example). +You should then rename and place specific files in a folder (see below for an example). + +mkdir MRPC +cabextract MSRParaphraseCorpus.msi -d MRPC +cat MRPC/_2DEC3DBE877E4DB192D17C0256E90F1D | tr -d $'\r' > MRPC/msr_paraphrase_train.txt +cat MRPC/_D7B391F9EAFF4B1B8BCE8F21B20B1B61 | tr -d $'\r' > MRPC/msr_paraphrase_test.txt +rm MRPC/_* +rm MSRParaphraseCorpus.msi + +1/30/19: It looks like SentEval is no longer hosting their extracted and tokenized MRPC data, so you'll need to download the data from the original source for now. +2/11/19: It looks like SentEval actually *is* hosting the extracted data. Hooray! +''' + +import os +import sys +import shutil +import argparse +import tempfile +import urllib.request +import zipfile + +TASKS = ["CoLA", "SST", "MRPC", "QQP", "STS", "MNLI", "SNLI", "QNLI", "RTE", "WNLI", "diagnostic"] +TASK2PATH = {"CoLA":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FCoLA.zip?alt=media&token=46d5e637-3411-4188-bc44-5809b5bfb5f4', + "SST":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSST-2.zip?alt=media&token=aabc5f6b-e466-44a2-b9b4-cf6337f84ac8', + "MRPC":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2Fmrpc_dev_ids.tsv?alt=media&token=ec5c0836-31d5-48f4-b431-7480817f1adc', + "QQP":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FQQP.zip?alt=media&token=700c6acf-160d-4d89-81d1-de4191d02cb5', + "STS":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSTS-B.zip?alt=media&token=bddb94a7-8706-4e0d-a694-1109e12273b5', + "MNLI":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FMNLI.zip?alt=media&token=50329ea1-e339-40e2-809c-10c40afff3ce', + "SNLI":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FSNLI.zip?alt=media&token=4afcfbb2-ff0c-4b2d-a09a-dbf07926f4df', + "QNLI": 'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FQNLIv2.zip?alt=media&token=6fdcf570-0fc5-4631-8456-9505272d1601', + "RTE":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FRTE.zip?alt=media&token=5efa7e85-a0bb-4f19-8ea2-9e1840f077fb', + "WNLI":'https://firebasestorage.googleapis.com/v0/b/mtl-sentence-representations.appspot.com/o/data%2FWNLI.zip?alt=media&token=068ad0a0-ded7-4bd7-99a5-5e00222e0faf', + "diagnostic":'https://storage.googleapis.com/mtl-sentence-representations.appspot.com/tsvsWithoutLabels%2FAX.tsv?GoogleAccessId=firebase-adminsdk-0khhl@mtl-sentence-representations.iam.gserviceaccount.com&Expires=2498860800&Signature=DuQ2CSPt2Yfre0C%2BiISrVYrIFaZH1Lc7hBVZDD4ZyR7fZYOMNOUGpi8QxBmTNOrNPjR3z1cggo7WXFfrgECP6FBJSsURv8Ybrue8Ypt%2FTPxbuJ0Xc2FhDi%2BarnecCBFO77RSbfuz%2Bs95hRrYhTnByqu3U%2FYZPaj3tZt5QdfpH2IUROY8LiBXoXS46LE%2FgOQc%2FKN%2BA9SoscRDYsnxHfG0IjXGwHN%2Bf88q6hOmAxeNPx6moDulUF6XMUAaXCSFU%2BnRO2RDL9CapWxj%2BDl7syNyHhB7987hZ80B%2FwFkQ3MEs8auvt5XW1%2Bd4aCU7ytgM69r8JDCwibfhZxpaa4gd50QXQ%3D%3D'} + +MRPC_TRAIN = 'https://dl.fbaipublicfiles.com/senteval/senteval_data/msr_paraphrase_train.txt' +MRPC_TEST = 'https://dl.fbaipublicfiles.com/senteval/senteval_data/msr_paraphrase_test.txt' + +def download_and_extract(task, data_dir): + print("Downloading and extracting %s..." % task) + data_file = "%s.zip" % task + urllib.request.urlretrieve(TASK2PATH[task], data_file) + with zipfile.ZipFile(data_file) as zip_ref: + zip_ref.extractall(data_dir) + os.remove(data_file) + print("\tCompleted!") + +def format_mrpc(data_dir, path_to_data): + print("Processing MRPC...") + mrpc_dir = os.path.join(data_dir, "MRPC") + if not os.path.isdir(mrpc_dir): + os.mkdir(mrpc_dir) + if path_to_data: + mrpc_train_file = os.path.join(path_to_data, "msr_paraphrase_train.txt") + mrpc_test_file = os.path.join(path_to_data, "msr_paraphrase_test.txt") + else: + print("Local MRPC data not specified, downloading data from %s" % MRPC_TRAIN) + mrpc_train_file = os.path.join(mrpc_dir, "msr_paraphrase_train.txt") + mrpc_test_file = os.path.join(mrpc_dir, "msr_paraphrase_test.txt") + urllib.request.urlretrieve(MRPC_TRAIN, mrpc_train_file) + urllib.request.urlretrieve(MRPC_TEST, mrpc_test_file) + assert os.path.isfile(mrpc_train_file), "Train data not found at %s" % mrpc_train_file + assert os.path.isfile(mrpc_test_file), "Test data not found at %s" % mrpc_test_file + urllib.request.urlretrieve(TASK2PATH["MRPC"], os.path.join(mrpc_dir, "dev_ids.tsv")) + + dev_ids = [] + with open(os.path.join(mrpc_dir, "dev_ids.tsv"), encoding="utf8") as ids_fh: + for row in ids_fh: + dev_ids.append(row.strip().split('\t')) + + with open(mrpc_train_file, encoding="utf8") as data_fh, \ + open(os.path.join(mrpc_dir, "train.tsv"), 'w', encoding="utf8") as train_fh, \ + open(os.path.join(mrpc_dir, "dev.tsv"), 'w', encoding="utf8") as dev_fh: + header = data_fh.readline() + train_fh.write(header) + dev_fh.write(header) + for row in data_fh: + label, id1, id2, s1, s2 = row.strip().split('\t') + if [id1, id2] in dev_ids: + dev_fh.write("%s\t%s\t%s\t%s\t%s\n" % (label, id1, id2, s1, s2)) + else: + train_fh.write("%s\t%s\t%s\t%s\t%s\n" % (label, id1, id2, s1, s2)) + + with open(mrpc_test_file, encoding="utf8") as data_fh, \ + open(os.path.join(mrpc_dir, "test.tsv"), 'w', encoding="utf8") as test_fh: + header = data_fh.readline() + test_fh.write("index\t#1 ID\t#2 ID\t#1 String\t#2 String\n") + for idx, row in enumerate(data_fh): + label, id1, id2, s1, s2 = row.strip().split('\t') + test_fh.write("%d\t%s\t%s\t%s\t%s\n" % (idx, id1, id2, s1, s2)) + print("\tCompleted!") + +def download_diagnostic(data_dir): + print("Downloading and extracting diagnostic...") + if not os.path.isdir(os.path.join(data_dir, "diagnostic")): + os.mkdir(os.path.join(data_dir, "diagnostic")) + data_file = os.path.join(data_dir, "diagnostic", "diagnostic.tsv") + urllib.request.urlretrieve(TASK2PATH["diagnostic"], data_file) + print("\tCompleted!") + return + +def get_tasks(task_names): + task_names = task_names.split(',') + if "all" in task_names: + tasks = TASKS + else: + tasks = [] + for task_name in task_names: + assert task_name in TASKS, "Task %s not found!" % task_name + tasks.append(task_name) + return tasks + +def main(arguments): + parser = argparse.ArgumentParser() + parser.add_argument('--data_dir', help='directory to save data to', type=str, default='glue_data') + parser.add_argument('--tasks', help='tasks to download data for as a comma separated string', + type=str, default='all') + parser.add_argument('--path_to_mrpc', help='path to directory containing extracted MRPC data, msr_paraphrase_train.txt and msr_paraphrase_text.txt', + type=str, default='') + args = parser.parse_args(arguments) + + if not os.path.isdir(args.data_dir): + os.mkdir(args.data_dir) + tasks = get_tasks(args.tasks) + + for task in tasks: + if task == 'MRPC': + format_mrpc(args.data_dir, args.path_to_mrpc) + elif task == 'diagnostic': + download_diagnostic(args.data_dir) + else: + download_and_extract(task, args.data_dir) + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) From 05d4232f63f121baefec9a87704ea7a15933f6e9 Mon Sep 17 00:00:00 2001 From: Juha Kiili Date: Thu, 21 Nov 2019 12:38:17 +0200 Subject: [PATCH 063/505] Add valohai.yaml --- valohai.yaml | 94 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 valohai.yaml diff --git a/valohai.yaml b/valohai.yaml new file mode 100644 index 0000000000..2573551b4e --- /dev/null +++ b/valohai.yaml @@ -0,0 +1,94 @@ +--- + +- step: + name: Execute python examples/run_glue.py + image: pytorch/pytorch:nightly-devel-cuda10.0-cudnn7 + command: + - python /valohai/repository/utils/download_glue_data.py --data_dir=/glue_data + - pip install -e . + - pip install -r examples/requirements.txt + - python examples/run_glue.py --do_train --data_dir=/glue_data/{parameter-value:task_name} {parameters} + parameters: + - name: model_type + pass-as: --model_type={v} + type: string + default: bert + - name: model_name_or_path + pass-as: --model_name_or_path={v} + type: string + default: bert-base-uncased + - name: task_name + pass-as: --task_name={v} + type: string + default: MRPC + - name: max_seq_length + pass-as: --max_seq_length={v} + description: The maximum total input sequence length after tokenization. Sequences longer than this will be truncated, sequences shorter will be padded. + type: integer + default: 128 + - name: per_gpu_train_batch_size + pass-as: --per_gpu_train_batch_size={v} + description: Batch size per GPU/CPU for training. + type: integer + default: 8 + - name: per_gpu_eval_batch_size + pass-as: --per_gpu_eval_batch_size={v} + description: Batch size per GPU/CPU for evaluation. + type: integer + default: 8 + - name: gradient_accumulation_steps + pass-as: --gradient_accumulation_steps={v} + description: Number of updates steps to accumulate before performing a backward/update pass. + type: integer + default: 1 + - name: learning_rate + pass-as: --learning_rate={v} + description: The initial learning rate for Adam. + type: float + default: 0.00005 + - name: adam_epsilon + pass-as: --adam_epsilon={v} + description: Epsilon for Adam optimizer. + type: float + default: 0.00000001 + - name: max_grad_norm + pass-as: --max_grad_norm={v} + description: Max gradient norm. + type: float + default: 1.0 + - name: num_train_epochs + pass-as: --num_train_epochs={v} + description: Total number of training epochs to perform. + type: integer + default: 3 + - name: max_steps + pass-as: --max_steps={v} + description: If > 0, set total number of training steps to perform. Override num_train_epochs. + type: integer + default: -1 + - name: warmup_steps + pass-as: --warmup_steps={v} + description: Linear warmup over warmup_steps. + type: integer + default: -1 + - name: logging_steps + pass-as: --logging_steps={v} + description: Log every X updates steps. + type: integer + default: 25 + - name: save_steps + pass-as: --save_steps={v} + description: Save checkpoint every X updates steps. + type: integer + default: -1 + - name: output_dir + pass-as: --output_dir={v} + type: string + default: /valohai/outputs + - name: evaluate_during_training + description: Run evaluation during training at each logging step. + type: flag + default: true + - name: do_lower_case + description: Set this flag if you are using an uncased model. + type: flag From 6f70bb8c69cdbe1207e4909ccface32c8f51297b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Louf?= Date: Wed, 20 Nov 2019 18:01:03 +0100 Subject: [PATCH 064/505] add instructions to run the examples --- README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/README.md b/README.md index a49e8086d1..9cc783fd71 100644 --- a/README.md +++ b/README.md @@ -86,6 +86,18 @@ When TensorFlow 2.0 and/or PyTorch has been installed, you can install from sour pip install [--editable] . ``` +### Run the examples + +Examples are included in the repository but are not shipped with the library. +Therefore, in order to run the examples you will first need to clone the +repository and install the bleeding edge version of the library. To do so, create a new virtual environment and follow these steps: + +```bash +git clone git@github.com:huggingface/transformers +cd transformers +pip install . +``` + ### Tests A series of tests are included for the library and the example scripts. Library tests can be found in the [tests folder](https://github.com/huggingface/transformers/tree/master/transformers/tests) and examples tests in the [examples folder](https://github.com/huggingface/transformers/tree/master/examples). @@ -253,6 +265,11 @@ print("sentence_2 is", "a paraphrase" if pred_2 else "not a paraphrase", "of sen ## Quick tour of the fine-tuning/usage scripts +**Important** +Before running the fine-tuning scripts, please read the +[instructions](#run-the-examples) on how to +setup your environment to run the examples. + The library comprises several example scripts with SOTA performances for NLU and NLG tasks: - `run_glue.py`: an example fine-tuning Bert, XLNet and XLM on nine different GLUE tasks (*sequence-level classification*) From 26db31e0c09a8b5e1ca7a61c454b159eab9d86be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Louf?= Date: Wed, 20 Nov 2019 18:13:38 +0100 Subject: [PATCH 065/505] update the documentation --- examples/README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/examples/README.md b/examples/README.md index abb4cb6e5a..622fa07f8f 100644 --- a/examples/README.md +++ b/examples/README.md @@ -3,6 +3,15 @@ In this section a few examples are put together. All of these examples work for several models, making use of the very similar API between the different models. +**Important** +To use the examples, execute the following steps in a new virtual environment: + +```bash +git clone git@github.com:huggingface/transformers +cd transformers +pip install . +``` + | Section | Description | |----------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------| | [TensorFlow 2.0 models on GLUE](#TensorFlow-2.0-Bert-models-on-GLUE) | Examples running BERT TensorFlow 2.0 model on the GLUE tasks. From ea52f82455a7ca0f979768204dfeb38b5fff13ad Mon Sep 17 00:00:00 2001 From: Lysandre Date: Mon, 18 Nov 2019 14:42:59 -0500 Subject: [PATCH 066/505] Moved some SQuAD logic to /data --- transformers/__init__.py | 3 +- transformers/data/__init__.py | 3 +- transformers/data/processors/__init__.py | 1 + transformers/data/processors/squad.py | 318 +++++++++++++++++++++++ 4 files changed, 323 insertions(+), 2 deletions(-) create mode 100644 transformers/data/processors/squad.py diff --git a/transformers/__init__.py b/transformers/__init__.py index 5c7b0a6197..b859e18c53 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -25,7 +25,8 @@ from .file_utils import (TRANSFORMERS_CACHE, PYTORCH_TRANSFORMERS_CACHE, PYTORCH from .data import (is_sklearn_available, InputExample, InputFeatures, DataProcessor, glue_output_modes, glue_convert_examples_to_features, - glue_processors, glue_tasks_num_labels) + glue_processors, glue_tasks_num_labels, + squad_convert_examples_to_features, SquadFeatures) if is_sklearn_available(): from .data import glue_compute_metrics diff --git a/transformers/data/__init__.py b/transformers/data/__init__.py index e910d6da2e..827d96ed29 100644 --- a/transformers/data/__init__.py +++ b/transformers/data/__init__.py @@ -1,5 +1,6 @@ -from .processors import InputExample, InputFeatures, DataProcessor +from .processors import InputExample, InputFeatures, DataProcessor, SquadFeatures from .processors import glue_output_modes, glue_processors, glue_tasks_num_labels, glue_convert_examples_to_features +from .processors import squad_convert_examples_to_features from .metrics import is_sklearn_available if is_sklearn_available(): diff --git a/transformers/data/processors/__init__.py b/transformers/data/processors/__init__.py index af38c54beb..4e322a2ca8 100644 --- a/transformers/data/processors/__init__.py +++ b/transformers/data/processors/__init__.py @@ -1,3 +1,4 @@ from .utils import InputExample, InputFeatures, DataProcessor from .glue import glue_output_modes, glue_processors, glue_tasks_num_labels, glue_convert_examples_to_features +from .squad import squad_convert_examples_to_features, SquadFeatures diff --git a/transformers/data/processors/squad.py b/transformers/data/processors/squad.py new file mode 100644 index 0000000000..c1a1034f17 --- /dev/null +++ b/transformers/data/processors/squad.py @@ -0,0 +1,318 @@ +from tqdm import tqdm +import collections +import logging +import os + +from .utils import DataProcessor, InputExample, InputFeatures +from ...file_utils import is_tf_available + +if is_tf_available(): + import tensorflow as tf + +logger = logging.getLogger(__name__) + +def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, + doc_stride, max_query_length, is_training, + cls_token_at_end=False, + cls_token='[CLS]', sep_token='[SEP]', pad_token=0, + sequence_a_segment_id=0, sequence_b_segment_id=1, + cls_token_segment_id=0, pad_token_segment_id=0, + mask_padding_with_zero=True, + sequence_a_is_doc=False): + """Loads a data file into a list of `InputBatch`s.""" + + # Defining helper methods + def _improve_answer_span(doc_tokens, input_start, input_end, tokenizer, + orig_answer_text): + """Returns tokenized answer spans that better match the annotated answer.""" + tok_answer_text = " ".join(tokenizer.tokenize(orig_answer_text)) + + for new_start in range(input_start, input_end + 1): + for new_end in range(input_end, new_start - 1, -1): + text_span = " ".join(doc_tokens[new_start:(new_end + 1)]) + if text_span == tok_answer_text: + return (new_start, new_end) + + return (input_start, input_end) + def _check_is_max_context(doc_spans, cur_span_index, position): + """Check if this is the 'max context' doc span for the token.""" + best_score = None + best_span_index = None + for (span_index, doc_span) in enumerate(doc_spans): + end = doc_span.start + doc_span.length - 1 + if position < doc_span.start: + continue + if position > end: + continue + num_left_context = position - doc_span.start + num_right_context = end - position + score = min(num_left_context, num_right_context) + 0.01 * doc_span.length + if best_score is None or score > best_score: + best_score = score + best_span_index = span_index + + return cur_span_index == best_span_index + + unique_id = 1000000000 + + features = [] + for (example_index, example) in enumerate(tqdm(examples)): + query_tokens = tokenizer.tokenize(example.question_text) + + if len(query_tokens) > max_query_length: + query_tokens = query_tokens[0:max_query_length] + + tok_to_orig_index = [] + orig_to_tok_index = [] + all_doc_tokens = [] + for (i, token) in enumerate(example.doc_tokens): + orig_to_tok_index.append(len(all_doc_tokens)) + sub_tokens = tokenizer.tokenize(token) + for sub_token in sub_tokens: + tok_to_orig_index.append(i) + all_doc_tokens.append(sub_token) + + tok_start_position = None + tok_end_position = None + if is_training and example.is_impossible: + tok_start_position = -1 + tok_end_position = -1 + if is_training and not example.is_impossible: + tok_start_position = orig_to_tok_index[example.start_position] + if example.end_position < len(example.doc_tokens) - 1: + tok_end_position = orig_to_tok_index[example.end_position + 1] - 1 + else: + tok_end_position = len(all_doc_tokens) - 1 + (tok_start_position, tok_end_position) = _improve_answer_span( + all_doc_tokens, tok_start_position, tok_end_position, tokenizer, + example.orig_answer_text) + + # The -3 accounts for [CLS], [SEP] and [SEP] + max_tokens_for_doc = max_seq_length - len(query_tokens) - 3 + + # We can have documents that are longer than the maximum sequence length. + # To deal with this we do a sliding window approach, where we take chunks + # of the up to our max length with a stride of `doc_stride`. + _DocSpan = collections.namedtuple( # pylint: disable=invalid-name + "DocSpan", ["start", "length"]) + doc_spans = [] + start_offset = 0 + while start_offset < len(all_doc_tokens): + length = len(all_doc_tokens) - start_offset + if length > max_tokens_for_doc: + length = max_tokens_for_doc + doc_spans.append(_DocSpan(start=start_offset, length=length)) + if start_offset + length == len(all_doc_tokens): + break + start_offset += min(length, doc_stride) + + for (doc_span_index, doc_span) in enumerate(doc_spans): + tokens = [] + token_to_orig_map = {} + token_is_max_context = {} + segment_ids = [] + + # p_mask: mask with 1 for token than cannot be in the answer (0 for token which can be in an answer) + # Original TF implem also keep the classification token (set to 0) (not sure why...) + p_mask = [] + + # CLS token at the beginning + if not cls_token_at_end: + tokens.append(cls_token) + segment_ids.append(cls_token_segment_id) + p_mask.append(0) + cls_index = 0 + + # XLNet: P SEP Q SEP CLS + # Others: CLS Q SEP P SEP + if not sequence_a_is_doc: + # Query + tokens += query_tokens + segment_ids += [sequence_a_segment_id] * len(query_tokens) + p_mask += [1] * len(query_tokens) + + # SEP token + tokens.append(sep_token) + segment_ids.append(sequence_a_segment_id) + p_mask.append(1) + + # Paragraph + for i in range(doc_span.length): + split_token_index = doc_span.start + i + token_to_orig_map[len(tokens)] = tok_to_orig_index[split_token_index] + + is_max_context = _check_is_max_context(doc_spans, doc_span_index, + split_token_index) + token_is_max_context[len(tokens)] = is_max_context + tokens.append(all_doc_tokens[split_token_index]) + if not sequence_a_is_doc: + segment_ids.append(sequence_b_segment_id) + else: + segment_ids.append(sequence_a_segment_id) + p_mask.append(0) + paragraph_len = doc_span.length + + if sequence_a_is_doc: + # SEP token + tokens.append(sep_token) + segment_ids.append(sequence_a_segment_id) + p_mask.append(1) + + tokens += query_tokens + segment_ids += [sequence_b_segment_id] * len(query_tokens) + p_mask += [1] * len(query_tokens) + + # SEP token + tokens.append(sep_token) + segment_ids.append(sequence_b_segment_id) + p_mask.append(1) + + # CLS token at the end + if cls_token_at_end: + tokens.append(cls_token) + segment_ids.append(cls_token_segment_id) + p_mask.append(0) + cls_index = len(tokens) - 1 # Index of classification token + + input_ids = tokenizer.convert_tokens_to_ids(tokens) + + # The mask has 1 for real tokens and 0 for padding tokens. Only real + # tokens are attended to. + input_mask = [1 if mask_padding_with_zero else 0] * len(input_ids) + + # Zero-pad up to the sequence length. + while len(input_ids) < max_seq_length: + input_ids.append(pad_token) + input_mask.append(0 if mask_padding_with_zero else 1) + segment_ids.append(pad_token_segment_id) + p_mask.append(1) + + assert len(input_ids) == max_seq_length + assert len(input_mask) == max_seq_length + assert len(segment_ids) == max_seq_length + + span_is_impossible = example.is_impossible + start_position = None + end_position = None + if is_training and not span_is_impossible: + # For training, if our document chunk does not contain an annotation + # we throw it out, since there is nothing to predict. + doc_start = doc_span.start + doc_end = doc_span.start + doc_span.length - 1 + out_of_span = False + if not (tok_start_position >= doc_start and + tok_end_position <= doc_end): + out_of_span = True + if out_of_span: + start_position = 0 + end_position = 0 + span_is_impossible = True + else: + if sequence_a_is_doc: + doc_offset = 0 + else: + doc_offset = len(query_tokens) + 2 + start_position = tok_start_position - doc_start + doc_offset + end_position = tok_end_position - doc_start + doc_offset + + if is_training and span_is_impossible: + start_position = cls_index + end_position = cls_index + + if example_index < 20: + logger.info("*** Example ***") + logger.info("unique_id: %s" % (unique_id)) + logger.info("example_index: %s" % (example_index)) + logger.info("doc_span_index: %s" % (doc_span_index)) + logger.info("tokens: %s" % " ".join(tokens)) + logger.info("token_to_orig_map: %s" % " ".join([ + "%d:%d" % (x, y) for (x, y) in token_to_orig_map.items()])) + logger.info("token_is_max_context: %s" % " ".join([ + "%d:%s" % (x, y) for (x, y) in token_is_max_context.items() + ])) + logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids])) + logger.info( + "input_mask: %s" % " ".join([str(x) for x in input_mask])) + logger.info( + "segment_ids: %s" % " ".join([str(x) for x in segment_ids])) + if is_training and span_is_impossible: + logger.info("impossible example") + if is_training and not span_is_impossible: + answer_text = " ".join(tokens[start_position:(end_position + 1)]) + logger.info("start_position: %d" % (start_position)) + logger.info("end_position: %d" % (end_position)) + logger.info( + "answer: %s" % (answer_text)) + + features.append( + SquadFeatures( + unique_id=unique_id, + example_index=example_index, + doc_span_index=doc_span_index, + tokens=tokens, + token_to_orig_map=token_to_orig_map, + token_is_max_context=token_is_max_context, + input_ids=input_ids, + input_mask=input_mask, + segment_ids=segment_ids, + cls_index=cls_index, + p_mask=p_mask, + paragraph_len=paragraph_len, + start_position=start_position, + end_position=end_position, + is_impossible=span_is_impossible)) + unique_id += 1 + + return features + +class SquadFeatures(object): + """A single set of features of data.""" + + def __init__(self, + unique_id, + example_index, + doc_span_index, + tokens, + token_to_orig_map, + token_is_max_context, + input_ids, + input_mask, + segment_ids, + cls_index, + p_mask, + paragraph_len, + start_position=None, + end_position=None, + is_impossible=None): + self.unique_id = unique_id + self.example_index = example_index + self.doc_span_index = doc_span_index + self.tokens = tokens + self.token_to_orig_map = token_to_orig_map + self.token_is_max_context = token_is_max_context + self.input_ids = input_ids + self.input_mask = input_mask + self.segment_ids = segment_ids + self.cls_index = cls_index + self.p_mask = p_mask + self.paragraph_len = paragraph_len + self.start_position = start_position + self.end_position = end_position + self.is_impossible = is_impossible + + def __eq__(self, other): + return self.cls_index == other.cls_index and \ + self.doc_span_index == other.doc_span_index and \ + self.end_position == other.end_position and \ + self.example_index == other.example_index and \ + self.input_ids == other.input_ids and \ + self.input_mask == other.input_mask and \ + self.is_impossible == other.is_impossible and \ + self.p_mask == other.p_mask and \ + self.paragraph_len == other.paragraph_len and \ + self.segment_ids == other.segment_ids and \ + self.start_position == other.start_position and \ + self.token_is_max_context == other.token_is_max_context and \ + self.token_to_orig_map == other.token_to_orig_map and \ + self.tokens == other.tokens and \ + self.unique_id == other.unique_id \ No newline at end of file From 72e506b22e90feab6c410136bacc27f3d65284b9 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 19 Nov 2019 09:49:55 -0500 Subject: [PATCH 067/505] wip --- examples/run_squad.py | 29 +++++- transformers/__init__.py | 3 +- transformers/data/__init__.py | 2 +- transformers/data/processors/__init__.py | 2 +- transformers/data/processors/squad.py | 122 +++++++++++++++++++++++ transformers/tokenization_utils.py | 4 + 6 files changed, 157 insertions(+), 5 deletions(-) diff --git a/examples/run_squad.py b/examples/run_squad.py index 69088d73c3..d4219c3096 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -23,7 +23,6 @@ import os import random import glob import timeit - import numpy as np import torch from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler, @@ -45,7 +44,7 @@ from transformers import (WEIGHTS_NAME, BertConfig, XLNetTokenizer, DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer) -from transformers import AdamW, get_linear_schedule_with_warmup +from transformers import AdamW, get_linear_schedule_with_warmup, squad_convert_examples_to_features, read_squad_examples as sread_squad_examples from utils_squad import (read_squad_examples, convert_examples_to_features, RawResult, write_predictions, @@ -309,6 +308,8 @@ def load_and_cache_examples(args, tokenizer, evaluate=False, output_examples=Fal examples = read_squad_examples(input_file=input_file, is_training=not evaluate, version_2_with_negative=args.version_2_with_negative) + + examples = examples[:10] features = convert_examples_to_features(examples=examples, tokenizer=tokenizer, max_seq_length=args.max_seq_length, @@ -319,6 +320,30 @@ def load_and_cache_examples(args, tokenizer, evaluate=False, output_examples=Fal pad_token_segment_id=3 if args.model_type in ['xlnet'] else 0, cls_token_at_end=True if args.model_type in ['xlnet'] else False, sequence_a_is_doc=True if args.model_type in ['xlnet'] else False) + + exampless = sread_squad_examples(input_file=input_file, + is_training=not evaluate, + version_2_with_negative=args.version_2_with_negative) + exampless = exampless[:10] + features2 = squad_convert_examples_to_features(examples=exampless, + tokenizer=tokenizer, + max_seq_length=args.max_seq_length, + doc_stride=args.doc_stride, + max_query_length=args.max_query_length, + is_training=not evaluate, + cls_token_segment_id=2 if args.model_type in ['xlnet'] else 0, + pad_token_segment_id=3 if args.model_type in ['xlnet'] else 0, + cls_token_at_end=True if args.model_type in ['xlnet'] else False, + sequence_a_is_doc=True if args.model_type in ['xlnet'] else False) + + print(features2) + + for i in range(len(features)): + assert features[i] == features2[i] + print("Equal") + + print("DONE") + if args.local_rank in [-1, 0]: logger.info("Saving features into cached file %s", cached_features_file) torch.save(features, cached_features_file) diff --git a/transformers/__init__.py b/transformers/__init__.py index b859e18c53..9a767913b3 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -26,7 +26,8 @@ from .data import (is_sklearn_available, InputExample, InputFeatures, DataProcessor, glue_output_modes, glue_convert_examples_to_features, glue_processors, glue_tasks_num_labels, - squad_convert_examples_to_features, SquadFeatures) + squad_convert_examples_to_features, SquadFeatures, + SquadExample, read_squad_examples) if is_sklearn_available(): from .data import glue_compute_metrics diff --git a/transformers/data/__init__.py b/transformers/data/__init__.py index 827d96ed29..50f2e768f4 100644 --- a/transformers/data/__init__.py +++ b/transformers/data/__init__.py @@ -1,6 +1,6 @@ from .processors import InputExample, InputFeatures, DataProcessor, SquadFeatures from .processors import glue_output_modes, glue_processors, glue_tasks_num_labels, glue_convert_examples_to_features -from .processors import squad_convert_examples_to_features +from .processors import squad_convert_examples_to_features, SquadExample, read_squad_examples from .metrics import is_sklearn_available if is_sklearn_available(): diff --git a/transformers/data/processors/__init__.py b/transformers/data/processors/__init__.py index 4e322a2ca8..924b4a1245 100644 --- a/transformers/data/processors/__init__.py +++ b/transformers/data/processors/__init__.py @@ -1,4 +1,4 @@ from .utils import InputExample, InputFeatures, DataProcessor from .glue import glue_output_modes, glue_processors, glue_tasks_num_labels, glue_convert_examples_to_features -from .squad import squad_convert_examples_to_features, SquadFeatures +from .squad import squad_convert_examples_to_features, SquadFeatures, SquadExample, read_squad_examples diff --git a/transformers/data/processors/squad.py b/transformers/data/processors/squad.py index c1a1034f17..1900e9f0ce 100644 --- a/transformers/data/processors/squad.py +++ b/transformers/data/processors/squad.py @@ -2,7 +2,9 @@ from tqdm import tqdm import collections import logging import os +import json +from ...tokenization_bert import BasicTokenizer, whitespace_tokenize from .utils import DataProcessor, InputExample, InputFeatures from ...file_utils import is_tf_available @@ -11,6 +13,7 @@ if is_tf_available(): logger = logging.getLogger(__name__) + def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, doc_stride, max_query_length, is_training, cls_token_at_end=False, @@ -265,6 +268,125 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, return features + +def read_squad_examples(input_file, is_training, version_2_with_negative): + """Read a SQuAD json file into a list of SquadExample.""" + with open(input_file, "r", encoding='utf-8') as reader: + input_data = json.load(reader)["data"] + + def is_whitespace(c): + if c == " " or c == "\t" or c == "\r" or c == "\n" or ord(c) == 0x202F: + return True + return False + + examples = [] + for entry in input_data: + for paragraph in entry["paragraphs"]: + paragraph_text = paragraph["context"] + doc_tokens = [] + char_to_word_offset = [] + prev_is_whitespace = True + for c in paragraph_text: + if is_whitespace(c): + prev_is_whitespace = True + else: + if prev_is_whitespace: + doc_tokens.append(c) + else: + doc_tokens[-1] += c + prev_is_whitespace = False + char_to_word_offset.append(len(doc_tokens) - 1) + + for qa in paragraph["qas"]: + qas_id = qa["id"] + question_text = qa["question"] + start_position = None + end_position = None + orig_answer_text = None + is_impossible = False + if is_training: + if version_2_with_negative: + is_impossible = qa["is_impossible"] + if (len(qa["answers"]) != 1) and (not is_impossible): + raise ValueError( + "For training, each question should have exactly 1 answer.") + if not is_impossible: + answer = qa["answers"][0] + orig_answer_text = answer["text"] + answer_offset = answer["answer_start"] + answer_length = len(orig_answer_text) + start_position = char_to_word_offset[answer_offset] + end_position = char_to_word_offset[answer_offset + answer_length - 1] + # Only add answers where the text can be exactly recovered from the + # document. If this CAN'T happen it's likely due to weird Unicode + # stuff so we will just skip the example. + # + # Note that this means for training mode, every example is NOT + # guaranteed to be preserved. + actual_text = " ".join(doc_tokens[start_position:(end_position + 1)]) + cleaned_answer_text = " ".join( + whitespace_tokenize(orig_answer_text)) + if actual_text.find(cleaned_answer_text) == -1: + logger.warning("Could not find answer: '%s' vs. '%s'", + actual_text, cleaned_answer_text) + continue + else: + start_position = -1 + end_position = -1 + orig_answer_text = "" + + example = SquadExample( + qas_id=qas_id, + question_text=question_text, + doc_tokens=doc_tokens, + orig_answer_text=orig_answer_text, + start_position=start_position, + end_position=end_position, + is_impossible=is_impossible) + examples.append(example) + return examples + + +class SquadExample(object): + """ + A single training/test example for the Squad dataset. + For examples without an answer, the start and end position are -1. + """ + + def __init__(self, + qas_id, + question_text, + doc_tokens, + orig_answer_text=None, + start_position=None, + end_position=None, + is_impossible=None): + self.qas_id = qas_id + self.question_text = question_text + self.doc_tokens = doc_tokens + self.orig_answer_text = orig_answer_text + self.start_position = start_position + self.end_position = end_position + self.is_impossible = is_impossible + + def __str__(self): + return self.__repr__() + + def __repr__(self): + s = "" + s += "qas_id: %s" % (self.qas_id) + s += ", question_text: %s" % ( + self.question_text) + s += ", doc_tokens: [%s]" % (" ".join(self.doc_tokens)) + if self.start_position: + s += ", start_position: %d" % (self.start_position) + if self.end_position: + s += ", end_position: %d" % (self.end_position) + if self.is_impossible: + s += ", is_impossible: %r" % (self.is_impossible) + return s + + class SquadFeatures(object): """A single set of features of data.""" diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index 4fa26a26f8..ba10e6b311 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -605,6 +605,10 @@ class PreTrainedTokenizer(object): vocabularies (BPE/SentencePieces/WordPieces). Take care of added tokens. + + text: The sequence to be encoded. + return_tokens_mapped_to_origin: (optional) Set to True to return the index of each token in the initial whitespace tokenization. (default False). + **kwargs: passed to the child `self.tokenize()` method """ def split_on_token(tok, text): result = [] From 9f374c8252330bffd669c43749b5e937ed31d90a Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Fri, 22 Nov 2019 16:27:15 -0500 Subject: [PATCH 068/505] `encode` and `encode_plus` handle attention masks and padding --- .../tests/tokenization_tests_commons.py | 51 ++++++++++++ transformers/tokenization_utils.py | 77 ++++++++++++++++++- transformers/tokenization_xlnet.py | 1 + 3 files changed, 127 insertions(+), 2 deletions(-) diff --git a/transformers/tests/tokenization_tests_commons.py b/transformers/tests/tokenization_tests_commons.py index fdaf8cc137..d5b70d5266 100644 --- a/transformers/tests/tokenization_tests_commons.py +++ b/transformers/tests/tokenization_tests_commons.py @@ -335,3 +335,54 @@ class CommonTestCases: special_tokens_mask = tokenizer.get_special_tokens_mask(encoded_sequence_w_special, already_has_special_tokens=True) self.assertEqual(len(special_tokens_mask), len(encoded_sequence_w_special)) self.assertEqual(special_tokens_mask_orig, special_tokens_mask) + + def test_padding_to_max_length(self): + tokenizer = self.get_tokenizer() + + sequence = "Sequence" + padding_size = 10 + padding_idx = tokenizer.pad_token_id + + # Check that it correctly pads when a maximum length is specified along with the padding flag set to True + encoded_sequence = tokenizer.encode(sequence) + sequence_length = len(encoded_sequence) + padded_sequence = tokenizer.encode(sequence, max_length=sequence_length + padding_size, pad_to_max_length=True) + padded_sequence_length = len(padded_sequence) + assert sequence_length + padding_size == padded_sequence_length + assert encoded_sequence + [padding_idx] * padding_size == padded_sequence + + # Check that nothing is done when a maximum length is not specified + encoded_sequence = tokenizer.encode(sequence) + sequence_length = len(encoded_sequence) + padded_sequence = tokenizer.encode(sequence, pad_to_max_length=True) + padded_sequence_length = len(padded_sequence) + assert sequence_length == padded_sequence_length + assert encoded_sequence == padded_sequence + + def test_encode_plus_with_padding(self): + tokenizer = self.get_tokenizer() + + sequence = "Sequence" + padding_size = 10 + padding_idx = tokenizer.pad_token_id + token_type_padding_idx = tokenizer.pad_token_type_id + + encoded_sequence = tokenizer.encode_plus(sequence, return_special_tokens_mask=True) + input_ids = encoded_sequence['input_ids'] + token_type_ids = encoded_sequence['token_type_ids'] + attention_mask = encoded_sequence['attention_mask'] + special_tokens_mask = encoded_sequence['special_tokens_mask'] + sequence_length = len(input_ids) + + padded_sequence = tokenizer.encode_plus(sequence, max_length=sequence_length + padding_size, pad_to_max_length=True, return_special_tokens_mask=True) + padded_input_ids = padded_sequence['input_ids'] + padded_token_type_ids = padded_sequence['token_type_ids'] + padded_attention_mask = padded_sequence['attention_mask'] + padded_special_tokens_mask = padded_sequence['special_tokens_mask'] + padded_sequence_length = len(padded_input_ids) + + assert sequence_length + padding_size == padded_sequence_length + assert input_ids + [padding_idx] * padding_size == padded_input_ids + assert token_type_ids + [token_type_padding_idx] * padding_size == padded_token_type_ids + assert attention_mask + [0] * padding_size == padded_attention_mask + assert special_tokens_mask + [1] * padding_size == padded_special_tokens_mask \ No newline at end of file diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index ba10e6b311..3214699e12 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -190,6 +190,11 @@ class PreTrainedTokenizer(object): """ Id of the padding token in the vocabulary. Log an error if used while not having been set. """ return self.convert_tokens_to_ids(self.pad_token) + @property + def pad_token_type_id(self): + """ Id of the padding token in the vocabulary. Log an error if used while not having been set. """ + return self._pad_token_type_id + @property def cls_token_id(self): """ Id of the classification token in the vocabulary. E.g. to extract a summary of an input sequence leveraging self-attention along the full depth of the model. Log an error if used while not having been set. """ @@ -213,6 +218,7 @@ class PreTrainedTokenizer(object): self._pad_token = None self._cls_token = None self._mask_token = None + self._pad_token_type_id = 0 self._additional_special_tokens = [] self.max_len = max_len if max_len is not None else int(1e12) @@ -696,6 +702,7 @@ class PreTrainedTokenizer(object): max_length=None, stride=0, truncation_strategy='longest_first', + pad_to_max_length=False, return_tensors=None, **kwargs): """ @@ -722,6 +729,8 @@ class PreTrainedTokenizer(object): - 'only_first': Only truncate the first sequence - 'only_second': Only truncate the second sequence - 'do_not_truncate': Does not truncate (raise an error if the input sequence is longer than max_length) + pad_to_max_length: if set to `True`, the returned sequences will be padded according to the model's + padding index, up to their max length. If no max length is specified, no padding is done. return_tensors: (optional) can be set to 'tf' or 'pt' to return respectively TensorFlow tf.constant or PyTorch torch.Tensor instead of a list of python integers. **kwargs: passed to the `self.tokenize()` method @@ -732,6 +741,7 @@ class PreTrainedTokenizer(object): add_special_tokens=add_special_tokens, stride=stride, truncation_strategy=truncation_strategy, + pad_to_max_length=pad_to_max_length, return_tensors=return_tensors, **kwargs) @@ -744,7 +754,12 @@ class PreTrainedTokenizer(object): max_length=None, stride=0, truncation_strategy='longest_first', + pad_to_max_length=False, return_tensors=None, + return_token_type_ids=True, + return_attention_mask=True, + return_overflowing_tokens=False, + return_special_tokens_mask=False, **kwargs): """ Returns a dictionary containing the encoded sequence or sequence pair and additional informations: @@ -769,9 +784,37 @@ class PreTrainedTokenizer(object): - 'only_first': Only truncate the first sequence - 'only_second': Only truncate the second sequence - 'do_not_truncate': Does not truncate (raise an error if the input sequence is longer than max_length) + pad_to_max_length: if set to `True`, the returned sequences will be padded according to the model's + padding index, up to their max length. If no max length is specified, no padding is done. return_tensors: (optional) can be set to 'tf' or 'pt' to return respectively TensorFlow tf.constant or PyTorch torch.Tensor instead of a list of python integers. + return_token_type_ids: (optional) Set to False to avoid returning token_type_ids (default True). + return_attention_mask: (optional) Set to False to avoir returning attention mask (default True) + return_overflowing_tokens: (optional) Set to True to return overflowing token information (default False). + return_special_tokens_mask: (optional) Set to True to return special tokens mask information (default False). **kwargs: passed to the `self.tokenize()` method + + Return: + A Dictionary of shape:: + + { + input_ids: list[int], + token_type_ids: list[int] if return_token_type_ids is True (default) + attention_mask: list[int] if return_attention_mask is True (default) + overflowing_tokens: list[int] if a ``max_length`` is specified and return_overflowing_tokens is True + num_truncated_tokens: int if a ``max_length`` is specified and return_overflowing_tokens is True + special_tokens_mask: list[int] if ``add_special_tokens`` if set to ``True`` and return_special_tokens_mask is True + } + + With the fields: + ``input_ids``: list of token ids to be fed to a model + ``token_type_ids``: list of token type ids to be fed to a model + ``attention_mask``: list of indices specifying which tokens should be attended to by the model + + ``overflowing_tokens``: list of overflowing tokens if a max length is specified. + ``num_truncated_tokens``: number of overflowing tokens a ``max_length`` is specified + ``special_tokens_mask``: if adding special tokens, this is a list of [0, 1], with 0 specifying special added + tokens and 1 specifying sequence tokens. """ def get_input_ids(text): @@ -790,13 +833,24 @@ class PreTrainedTokenizer(object): return self.prepare_for_model(first_ids, pair_ids=second_ids, max_length=max_length, + pad_to_max_length=pad_to_max_length, add_special_tokens=add_special_tokens, stride=stride, truncation_strategy=truncation_strategy, - return_tensors=return_tensors) + return_tensors=return_tensors, + return_attention_mask=return_attention_mask, + return_token_type_ids=return_token_type_ids, + return_overflowing_tokens=return_overflowing_tokens, + return_special_tokens_mask=return_special_tokens_mask) def prepare_for_model(self, ids, pair_ids=None, max_length=None, add_special_tokens=True, stride=0, - truncation_strategy='longest_first', return_tensors=None): + truncation_strategy='longest_first', + pad_to_max_length=False, + return_tensors=None, + return_token_type_ids=True, + return_attention_mask=True, + return_overflowing_tokens=False, + return_special_tokens_mask=False): """ Prepares a sequence of input id, or a pair of sequences of inputs ids so that it can be used by the model. It adds special tokens, truncates @@ -819,8 +873,14 @@ class PreTrainedTokenizer(object): - 'only_first': Only truncate the first sequence - 'only_second': Only truncate the second sequence - 'do_not_truncate': Does not truncate (raise an error if the input sequence is longer than max_length) + pad_to_max_length: if set to `True`, the returned sequences will be padded according to the model's + padding index, up to their max length. If no max length is specified, no padding is done. return_tensors: (optional) can be set to 'tf' or 'pt' to return respectively TensorFlow tf.constant or PyTorch torch.Tensor instead of a list of python integers. + return_token_type_ids: (optional) Set to False to avoid returning token_type_ids (default True). + return_attention_mask: (optional) Set to False to avoir returning attention mask (default True) + return_overflowing_tokens: (optional) Set to True to return overflowing token information (default False). + return_special_tokens_mask: (optional) Set to True to return special tokens mask information (default False). Return: A Dictionary of shape:: @@ -883,6 +943,19 @@ class PreTrainedTokenizer(object): "for this model ({} > {}). Running this sequence through the model will result in " "indexing errors".format(len(ids), self.max_len)) + if pad_to_max_length and max_length and len(encoded_inputs["input_ids"]) < max_length: + difference = max_length - len(encoded_inputs["input_ids"]) + if return_attention_mask: + encoded_inputs["attention_mask"] = [1] * len(encoded_inputs["input_ids"]) + [0] * difference + if return_token_type_ids: + encoded_inputs["token_type_ids"] += [self.pad_token_type_id] * difference + if return_special_tokens_mask: + encoded_inputs["special_tokens_mask"] += [1] * difference + + encoded_inputs["input_ids"] += [self.pad_token_id] * difference + elif return_attention_mask: + encoded_inputs["attention_mask"] = [1] * len(encoded_inputs["input_ids"]) + return encoded_inputs def truncate_sequences(self, ids, pair_ids=None, num_tokens_to_remove=0, truncation_strategy='longest_first', stride=0): diff --git a/transformers/tokenization_xlnet.py b/transformers/tokenization_xlnet.py index a4f1a6e3ba..3ea71f4438 100644 --- a/transformers/tokenization_xlnet.py +++ b/transformers/tokenization_xlnet.py @@ -74,6 +74,7 @@ class XLNetTokenizer(PreTrainedTokenizer): self.max_len_single_sentence = self.max_len - 2 # take into account special tokens self.max_len_sentences_pair = self.max_len - 3 # take into account special tokens + self._pad_token_type_id = 3 try: import sentencepiece as spm From a7dafe2f41222469797f1a67232961d67bd2e519 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Thu, 21 Nov 2019 11:30:40 -0500 Subject: [PATCH 069/505] Padding strategy (left and right) rather than boolean flag --- .../tests/tokenization_tests_commons.py | 43 +++++++++++--- transformers/tokenization_utils.py | 58 ++++++++++++++----- 2 files changed, 77 insertions(+), 24 deletions(-) diff --git a/transformers/tests/tokenization_tests_commons.py b/transformers/tests/tokenization_tests_commons.py index d5b70d5266..40d68d0ab2 100644 --- a/transformers/tests/tokenization_tests_commons.py +++ b/transformers/tests/tokenization_tests_commons.py @@ -343,21 +343,33 @@ class CommonTestCases: padding_size = 10 padding_idx = tokenizer.pad_token_id - # Check that it correctly pads when a maximum length is specified along with the padding flag set to True + # RIGHT PADDING - Check that it correctly pads when a maximum length is specified along with the padding flag set to True encoded_sequence = tokenizer.encode(sequence) sequence_length = len(encoded_sequence) - padded_sequence = tokenizer.encode(sequence, max_length=sequence_length + padding_size, pad_to_max_length=True) + padded_sequence = tokenizer.encode(sequence, max_length=sequence_length + padding_size, padding_strategy='right') padded_sequence_length = len(padded_sequence) assert sequence_length + padding_size == padded_sequence_length assert encoded_sequence + [padding_idx] * padding_size == padded_sequence - # Check that nothing is done when a maximum length is not specified + # LEFT PADDING - Check that it correctly pads when a maximum length is specified along with the padding flag set to True encoded_sequence = tokenizer.encode(sequence) sequence_length = len(encoded_sequence) - padded_sequence = tokenizer.encode(sequence, pad_to_max_length=True) + padded_sequence = tokenizer.encode(sequence, max_length=sequence_length + padding_size, padding_strategy='left') padded_sequence_length = len(padded_sequence) - assert sequence_length == padded_sequence_length - assert encoded_sequence == padded_sequence + assert sequence_length + padding_size == padded_sequence_length + assert [padding_idx] * padding_size + encoded_sequence == padded_sequence + + # RIGHT & LEFT PADDING - Check that nothing is done when a maximum length is not specified + encoded_sequence = tokenizer.encode(sequence) + sequence_length = len(encoded_sequence) + padded_sequence_right = tokenizer.encode(sequence, padding_strategy='right') + padded_sequence_right_length = len(padded_sequence_right) + padded_sequence_left = tokenizer.encode(sequence, padding_strategy='left') + padded_sequence_left_length = len(padded_sequence_left) + assert sequence_length == padded_sequence_right_length + assert encoded_sequence == padded_sequence_right + assert sequence_length == padded_sequence_left_length + assert encoded_sequence == padded_sequence_left def test_encode_plus_with_padding(self): tokenizer = self.get_tokenizer() @@ -374,7 +386,8 @@ class CommonTestCases: special_tokens_mask = encoded_sequence['special_tokens_mask'] sequence_length = len(input_ids) - padded_sequence = tokenizer.encode_plus(sequence, max_length=sequence_length + padding_size, pad_to_max_length=True, return_special_tokens_mask=True) + # Test right padding + padded_sequence = tokenizer.encode_plus(sequence, max_length=sequence_length + padding_size, padding_strategy='right', return_special_tokens_mask=True) padded_input_ids = padded_sequence['input_ids'] padded_token_type_ids = padded_sequence['token_type_ids'] padded_attention_mask = padded_sequence['attention_mask'] @@ -385,4 +398,18 @@ class CommonTestCases: assert input_ids + [padding_idx] * padding_size == padded_input_ids assert token_type_ids + [token_type_padding_idx] * padding_size == padded_token_type_ids assert attention_mask + [0] * padding_size == padded_attention_mask - assert special_tokens_mask + [1] * padding_size == padded_special_tokens_mask \ No newline at end of file + assert special_tokens_mask + [1] * padding_size == padded_special_tokens_mask + + # Test left padding + padded_sequence = tokenizer.encode_plus(sequence, max_length=sequence_length + padding_size, padding_strategy='left', return_special_tokens_mask=True) + padded_input_ids = padded_sequence['input_ids'] + padded_token_type_ids = padded_sequence['token_type_ids'] + padded_attention_mask = padded_sequence['attention_mask'] + padded_special_tokens_mask = padded_sequence['special_tokens_mask'] + padded_sequence_length = len(padded_input_ids) + + assert sequence_length + padding_size == padded_sequence_length + assert [padding_idx] * padding_size + input_ids == padded_input_ids + assert [token_type_padding_idx] * padding_size + token_type_ids == padded_token_type_ids + assert [0] * padding_size + attention_mask == padded_attention_mask + assert [1] * padding_size + special_tokens_mask == padded_special_tokens_mask \ No newline at end of file diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index 3214699e12..dbbabd0e1a 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -702,7 +702,7 @@ class PreTrainedTokenizer(object): max_length=None, stride=0, truncation_strategy='longest_first', - pad_to_max_length=False, + padding_strategy=None, return_tensors=None, **kwargs): """ @@ -729,8 +729,12 @@ class PreTrainedTokenizer(object): - 'only_first': Only truncate the first sequence - 'only_second': Only truncate the second sequence - 'do_not_truncate': Does not truncate (raise an error if the input sequence is longer than max_length) - pad_to_max_length: if set to `True`, the returned sequences will be padded according to the model's + padding_strategy: if set to a strategy, the returned sequences will be padded according to the model's padding index, up to their max length. If no max length is specified, no padding is done. + The strategies are handled by the following strings: + - 'left': pads on the left of the sequences + - 'right': pads on the right of the sequences + Defaults to None: no padding. return_tensors: (optional) can be set to 'tf' or 'pt' to return respectively TensorFlow tf.constant or PyTorch torch.Tensor instead of a list of python integers. **kwargs: passed to the `self.tokenize()` method @@ -741,7 +745,7 @@ class PreTrainedTokenizer(object): add_special_tokens=add_special_tokens, stride=stride, truncation_strategy=truncation_strategy, - pad_to_max_length=pad_to_max_length, + padding_strategy=padding_strategy, return_tensors=return_tensors, **kwargs) @@ -754,7 +758,7 @@ class PreTrainedTokenizer(object): max_length=None, stride=0, truncation_strategy='longest_first', - pad_to_max_length=False, + padding_strategy=None, return_tensors=None, return_token_type_ids=True, return_attention_mask=True, @@ -784,8 +788,12 @@ class PreTrainedTokenizer(object): - 'only_first': Only truncate the first sequence - 'only_second': Only truncate the second sequence - 'do_not_truncate': Does not truncate (raise an error if the input sequence is longer than max_length) - pad_to_max_length: if set to `True`, the returned sequences will be padded according to the model's + padding_strategy: if set to a strategy, the returned sequences will be padded according to the model's padding index, up to their max length. If no max length is specified, no padding is done. + The strategies are handled by the following strings: + - 'left': pads on the left of the sequences + - 'right': pads on the right of the sequences + Defaults to None: no padding. return_tensors: (optional) can be set to 'tf' or 'pt' to return respectively TensorFlow tf.constant or PyTorch torch.Tensor instead of a list of python integers. return_token_type_ids: (optional) Set to False to avoid returning token_type_ids (default True). @@ -833,7 +841,7 @@ class PreTrainedTokenizer(object): return self.prepare_for_model(first_ids, pair_ids=second_ids, max_length=max_length, - pad_to_max_length=pad_to_max_length, + padding_strategy=padding_strategy, add_special_tokens=add_special_tokens, stride=stride, truncation_strategy=truncation_strategy, @@ -845,7 +853,7 @@ class PreTrainedTokenizer(object): def prepare_for_model(self, ids, pair_ids=None, max_length=None, add_special_tokens=True, stride=0, truncation_strategy='longest_first', - pad_to_max_length=False, + padding_strategy=None, return_tensors=None, return_token_type_ids=True, return_attention_mask=True, @@ -873,8 +881,12 @@ class PreTrainedTokenizer(object): - 'only_first': Only truncate the first sequence - 'only_second': Only truncate the second sequence - 'do_not_truncate': Does not truncate (raise an error if the input sequence is longer than max_length) - pad_to_max_length: if set to `True`, the returned sequences will be padded according to the model's + padding_strategy: if set to a strategy, the returned sequences will be padded according to the model's padding index, up to their max length. If no max length is specified, no padding is done. + The strategies are handled by the following strings: + - 'left': pads on the left of the sequences + - 'right': pads on the right of the sequences + Defaults to None: no padding. return_tensors: (optional) can be set to 'tf' or 'pt' to return respectively TensorFlow tf.constant or PyTorch torch.Tensor instead of a list of python integers. return_token_type_ids: (optional) Set to False to avoid returning token_type_ids (default True). @@ -943,16 +955,30 @@ class PreTrainedTokenizer(object): "for this model ({} > {}). Running this sequence through the model will result in " "indexing errors".format(len(ids), self.max_len)) - if pad_to_max_length and max_length and len(encoded_inputs["input_ids"]) < max_length: + if padding_strategy is not None and max_length and len(encoded_inputs["input_ids"]) < max_length: difference = max_length - len(encoded_inputs["input_ids"]) - if return_attention_mask: - encoded_inputs["attention_mask"] = [1] * len(encoded_inputs["input_ids"]) + [0] * difference - if return_token_type_ids: - encoded_inputs["token_type_ids"] += [self.pad_token_type_id] * difference - if return_special_tokens_mask: - encoded_inputs["special_tokens_mask"] += [1] * difference - encoded_inputs["input_ids"] += [self.pad_token_id] * difference + if padding_strategy == 'right': + if return_attention_mask: + encoded_inputs["attention_mask"] = [1] * len(encoded_inputs["input_ids"]) + [0] * difference + if return_token_type_ids: + encoded_inputs["token_type_ids"] = encoded_inputs["token_type_ids"] + [self.pad_token_type_id] * difference + if return_special_tokens_mask: + encoded_inputs["special_tokens_mask"] = encoded_inputs["special_tokens_mask"] + [1] * difference + encoded_inputs["input_ids"] = encoded_inputs["input_ids"] + [self.pad_token_id] * difference + + elif padding_strategy == 'left': + if return_attention_mask: + encoded_inputs["attention_mask"] = [0] * difference + [1] * len(encoded_inputs["input_ids"]) + if return_token_type_ids: + encoded_inputs["token_type_ids"] = [self.pad_token_type_id] * difference + encoded_inputs["token_type_ids"] + if return_special_tokens_mask: + encoded_inputs["special_tokens_mask"] = [1] * difference + encoded_inputs["special_tokens_mask"] + encoded_inputs["input_ids"] = [self.pad_token_id] * difference + encoded_inputs["input_ids"] + + else: + raise ValueError("Invalid padding strategy:" + str(padding_strategy)) + elif return_attention_mask: encoded_inputs["attention_mask"] = [1] * len(encoded_inputs["input_ids"]) From a5a8a6175fb5cc1e993366add026ba06386bde10 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Thu, 21 Nov 2019 19:18:20 -0500 Subject: [PATCH 070/505] Works for BERT --- transformers/data/processors/squad.py | 507 ++++++++++++++++++++++---- 1 file changed, 432 insertions(+), 75 deletions(-) diff --git a/transformers/data/processors/squad.py b/transformers/data/processors/squad.py index 1900e9f0ce..a0f2408a16 100644 --- a/transformers/data/processors/squad.py +++ b/transformers/data/processors/squad.py @@ -3,6 +3,7 @@ import collections import logging import os import json +import numpy as np from ...tokenization_bert import BasicTokenizer, whitespace_tokenize from .utils import DataProcessor, InputExample, InputFeatures @@ -13,10 +14,68 @@ if is_tf_available(): logger = logging.getLogger(__name__) +def _improve_answer_span(doc_tokens, input_start, input_end, tokenizer, + orig_answer_text): + """Returns tokenized answer spans that better match the annotated answer.""" + tok_answer_text = " ".join(tokenizer.tokenize(orig_answer_text)) + + for new_start in range(input_start, input_end + 1): + for new_end in range(input_end, new_start - 1, -1): + text_span = " ".join(doc_tokens[new_start:(new_end + 1)]) + if text_span == tok_answer_text: + return (new_start, new_end) + + return (input_start, input_end) + +def _check_is_max_context(doc_spans, cur_span_index, position): + """Check if this is the 'max context' doc span for the token.""" + best_score = None + best_span_index = None + for (span_index, doc_span) in enumerate(doc_spans): + end = doc_span.start + doc_span.length - 1 + if position < doc_span.start: + continue + if position > end: + continue + num_left_context = position - doc_span.start + num_right_context = end - position + score = min(num_left_context, num_right_context) + 0.01 * doc_span.length + if best_score is None or score > best_score: + best_score = score + best_span_index = span_index + + return cur_span_index == best_span_index + + +def _new_check_is_max_context(doc_spans, cur_span_index, position): + """Check if this is the 'max context' doc span for the token.""" + # if len(doc_spans) == 1: + # return True + best_score = None + best_span_index = None + for (span_index, doc_span) in enumerate(doc_spans): + end = doc_span["start"] + doc_span["length"] - 1 + if position < doc_span["start"]: + continue + if position > end: + continue + num_left_context = position - doc_span["start"] + num_right_context = end - position + score = min(num_left_context, num_right_context) + 0.01 * doc_span["length"] + if best_score is None or score > best_score: + best_score = score + best_span_index = span_index + + return cur_span_index == best_span_index + +def _is_whitespace(c): + if c == " " or c == "\t" or c == "\r" or c == "\n" or ord(c) == 0x202F: + return True + return False def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, doc_stride, max_query_length, is_training, - cls_token_at_end=False, + cls_token_at_end=True, cls_token='[CLS]', sep_token='[SEP]', pad_token=0, sequence_a_segment_id=0, sequence_b_segment_id=1, cls_token_segment_id=0, pad_token_segment_id=0, @@ -24,57 +83,184 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, sequence_a_is_doc=False): """Loads a data file into a list of `InputBatch`s.""" - # Defining helper methods - def _improve_answer_span(doc_tokens, input_start, input_end, tokenizer, - orig_answer_text): - """Returns tokenized answer spans that better match the annotated answer.""" - tok_answer_text = " ".join(tokenizer.tokenize(orig_answer_text)) - - for new_start in range(input_start, input_end + 1): - for new_end in range(input_end, new_start - 1, -1): - text_span = " ".join(doc_tokens[new_start:(new_end + 1)]) - if text_span == tok_answer_text: - return (new_start, new_end) - - return (input_start, input_end) - def _check_is_max_context(doc_spans, cur_span_index, position): - """Check if this is the 'max context' doc span for the token.""" - best_score = None - best_span_index = None - for (span_index, doc_span) in enumerate(doc_spans): - end = doc_span.start + doc_span.length - 1 - if position < doc_span.start: - continue - if position > end: - continue - num_left_context = position - doc_span.start - num_right_context = end - position - score = min(num_left_context, num_right_context) + 0.01 * doc_span.length - if best_score is None or score > best_score: - best_score = score - best_span_index = span_index - - return cur_span_index == best_span_index - + # Defining helper methods unique_id = 1000000000 features = [] + new_features = [] for (example_index, example) in enumerate(tqdm(examples)): - query_tokens = tokenizer.tokenize(example.question_text) - if len(query_tokens) > max_query_length: - query_tokens = query_tokens[0:max_query_length] + doc_tokens = [] + char_to_word_offset = [] + prev_is_whitespace = True + + # Split on whitespace so that different tokens may be attributed to their original position. + for c in example.context_text: + if _is_whitespace(c): + prev_is_whitespace = True + else: + if prev_is_whitespace: + doc_tokens.append(c) + else: + doc_tokens[-1] += c + prev_is_whitespace = False + char_to_word_offset.append(len(doc_tokens) - 1) + + if is_training: + # Get start and end position + answer_length = len(example.answer_text) + start_position = char_to_word_offset[example.start_position] + end_position = char_to_word_offset[example.start_position + answer_length - 1] + + # If the answer cannot be found in the text, then skip this example. + actual_text = " ".join(doc_tokens[start_position:(end_position + 1)]) + cleaned_answer_text = " ".join(whitespace_tokenize(example.answer_text)) + if actual_text.find(cleaned_answer_text) == -1: + logger.warning("Could not find answer: '%s' vs. '%s'", actual_text, cleaned_answer_text) + continue tok_to_orig_index = [] orig_to_tok_index = [] all_doc_tokens = [] - for (i, token) in enumerate(example.doc_tokens): + for (i, token) in enumerate(doc_tokens): orig_to_tok_index.append(len(all_doc_tokens)) sub_tokens = tokenizer.tokenize(token) for sub_token in sub_tokens: tok_to_orig_index.append(i) all_doc_tokens.append(sub_token) + spans = [] + + truncated_query = tokenizer.encode(example.question_text, add_special_tokens=False, max_length=max_query_length) + sequence_added_tokens = tokenizer.max_len - tokenizer.max_len_single_sentence + sequence_pair_added_tokens = tokenizer.max_len - tokenizer.max_len_sentences_pair + + encoded_dict = tokenizer.encode_plus( + truncated_query, + all_doc_tokens, + max_length=max_seq_length, + padding_strategy='right', + stride=max_seq_length - doc_stride - len(truncated_query) - sequence_pair_added_tokens, + return_overflowing_tokens=True, + truncation_strategy='only_second' + ) + + ids = encoded_dict['input_ids'] + print("Ids computes; position of the first padding", ids.index(tokenizer.pad_token_id) if tokenizer.pad_token_id in ids else None) + non_padded_ids = ids[:ids.index(tokenizer.pad_token_id)] if tokenizer.pad_token_id in ids else ids + paragraph_len = min(len(all_doc_tokens) - len(spans) * doc_stride, max_seq_length - len(truncated_query) - sequence_pair_added_tokens) + tokens = tokenizer.convert_ids_to_tokens(non_padded_ids) + + token_to_orig_map = {} + for i in range(paragraph_len): + token_to_orig_map[len(truncated_query) + sequence_added_tokens + i] = tok_to_orig_index[0 + i] + + encoded_dict["paragraph_len"] = paragraph_len + encoded_dict["tokens"] = tokens + encoded_dict["token_to_orig_map"] = token_to_orig_map + encoded_dict["truncated_query_with_special_tokens_length"] = len(truncated_query) + sequence_added_tokens + encoded_dict["token_is_max_context"] = {} + encoded_dict["start"] = 0 + encoded_dict["length"] = paragraph_len + + spans.append(encoded_dict) + print("YESSIR", len(spans) * doc_stride < len(all_doc_tokens), "overflowing_tokens" in encoded_dict) + while len(spans) * doc_stride < len(all_doc_tokens) and "overflowing_tokens" in encoded_dict: + + overflowing_tokens = encoded_dict['overflowing_tokens'] + + print("OVERFLOW", len(overflowing_tokens)) + + encoded_dict = tokenizer.encode_plus( + truncated_query, + overflowing_tokens, + max_length=max_seq_length, + return_overflowing_tokens=True, + padding_strategy='right', + stride=max_seq_length - doc_stride - len(truncated_query) - sequence_pair_added_tokens, + truncation_strategy='only_second' + ) + + ids = encoded_dict['input_ids'] + print("Ids computes; position of the first padding", ids.index(tokenizer.pad_token_id) if tokenizer.pad_token_id in ids else None) + + # Length of the document without the query + paragraph_len = min(len(all_doc_tokens) - len(spans) * doc_stride, max_seq_length - len(truncated_query) - sequence_pair_added_tokens) + + non_padded_ids = encoded_dict['input_ids'][:encoded_dict['input_ids'].index(tokenizer.pad_token_id)] + tokens = tokenizer.convert_ids_to_tokens(non_padded_ids) + + token_to_orig_map = {} + for i in range(paragraph_len): + token_to_orig_map[len(truncated_query) + sequence_added_tokens + i] = tok_to_orig_index[len(spans) * doc_stride + i] + + encoded_dict["paragraph_len"] = paragraph_len + encoded_dict["tokens"] = tokens + encoded_dict["token_to_orig_map"] = token_to_orig_map + encoded_dict["truncated_query_with_special_tokens_length"] = len(truncated_query) + sequence_added_tokens + encoded_dict["token_is_max_context"] = {} + encoded_dict["start"] = len(spans) * doc_stride + encoded_dict["length"] = paragraph_len + + # split_token_index = doc_span.start + i + # token_to_orig_map[len(tokens)] = tok_to_orig_index[split_token_index] + + # is_max_context = _check_is_max_context(doc_spans, doc_span_index, + # split_token_index) + # token_is_max_context[len(tokens)] = is_max_context + # tokens.append(all_doc_tokens[split_token_index]) + + spans.append(encoded_dict) + + for doc_span_index in range(len(spans)): + for j in range(spans[doc_span_index]["paragraph_len"]): + is_max_context = _new_check_is_max_context(spans, doc_span_index, doc_span_index * doc_stride + j) + index = spans[doc_span_index]["truncated_query_with_special_tokens_length"] + j + spans[doc_span_index]["token_is_max_context"][index] = is_max_context + + print("new span", len(spans)) + for span in spans: + # Identify the position of the CLS token + cls_index = span['input_ids'].index(tokenizer.cls_token_id) + + # p_mask: mask with 1 for token than cannot be in the answer (0 for token which can be in an answer) + # Original TF implem also keep the classification token (set to 0) (not sure why...) + p_mask = np.array(span['token_type_ids']) + + # Convert all SEP indices to '0' before inversion + p_mask[np.where(np.array(span["input_ids"]) == tokenizer.sep_token_id)[0]] = 0 + + # Limit positive values to one + p_mask = 1 - np.minimum(p_mask, 1) + + # Set the CLS index to '0' + p_mask[cls_index] = 0 + + print("new features length", len(new_features)) + + new_features.append(NewSquadFeatures( + span['input_ids'], + span['attention_mask'], + span['token_type_ids'], + cls_index, + p_mask.tolist(), + + example_index=example_index, + unique_id=unique_id, + paragraph_len=span['paragraph_len'], + token_is_max_context=span["token_is_max_context"], + tokens=span["tokens"], + token_to_orig_map=span["token_to_orig_map"] + )) + + unique_id += 1 + + # tokenize ... + query_tokens = tokenizer.tokenize(example.question_text) + + if len(query_tokens) > max_query_length: + query_tokens = query_tokens[0:max_query_length] + tok_start_position = None tok_end_position = None if is_training and example.is_impossible: @@ -82,7 +268,7 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, tok_end_position = -1 if is_training and not example.is_impossible: tok_start_position = orig_to_tok_index[example.start_position] - if example.end_position < len(example.doc_tokens) - 1: + if example.end_position < len(doc_tokens) - 1: tok_end_position = orig_to_tok_index[example.end_position + 1] - 1 else: tok_end_position = len(all_doc_tokens) - 1 @@ -101,14 +287,19 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, doc_spans = [] start_offset = 0 while start_offset < len(all_doc_tokens): + print("OLD DOC CREATION BEGIN", start_offset, len(all_doc_tokens)) length = len(all_doc_tokens) - start_offset if length > max_tokens_for_doc: length = max_tokens_for_doc doc_spans.append(_DocSpan(start=start_offset, length=length)) if start_offset + length == len(all_doc_tokens): + print("Done with this doc span, breaking out.", start_offset, length) break + print("CHOOSING OFFSET", length, doc_stride) start_offset += min(length, doc_stride) + print("OLD DOC CREATION END", start_offset) + print("old span", len(doc_spans)) for (doc_span_index, doc_span) in enumerate(doc_spans): tokens = [] token_to_orig_map = {} @@ -183,18 +374,20 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, # tokens are attended to. input_mask = [1 if mask_padding_with_zero else 0] * len(input_ids) + + # Zero-pad up to the sequence length. while len(input_ids) < max_seq_length: input_ids.append(pad_token) input_mask.append(0 if mask_padding_with_zero else 1) segment_ids.append(pad_token_segment_id) p_mask.append(1) - + print("[OLD] Ids computed; position of the first padding", input_ids.index(tokenizer.pad_token_id) if tokenizer.pad_token_id in input_ids else None) assert len(input_ids) == max_seq_length assert len(input_mask) == max_seq_length assert len(segment_ids) == max_seq_length - span_is_impossible = example.is_impossible + span_is_impossible = example.is_impossible if hasattr(example, "is_impossible") else False start_position = None end_position = None if is_training and not span_is_impossible: @@ -222,31 +415,32 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, start_position = cls_index end_position = cls_index - if example_index < 20: - logger.info("*** Example ***") - logger.info("unique_id: %s" % (unique_id)) - logger.info("example_index: %s" % (example_index)) - logger.info("doc_span_index: %s" % (doc_span_index)) - logger.info("tokens: %s" % " ".join(tokens)) - logger.info("token_to_orig_map: %s" % " ".join([ - "%d:%d" % (x, y) for (x, y) in token_to_orig_map.items()])) - logger.info("token_is_max_context: %s" % " ".join([ - "%d:%s" % (x, y) for (x, y) in token_is_max_context.items() - ])) - logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids])) - logger.info( - "input_mask: %s" % " ".join([str(x) for x in input_mask])) - logger.info( - "segment_ids: %s" % " ".join([str(x) for x in segment_ids])) - if is_training and span_is_impossible: - logger.info("impossible example") - if is_training and not span_is_impossible: - answer_text = " ".join(tokens[start_position:(end_position + 1)]) - logger.info("start_position: %d" % (start_position)) - logger.info("end_position: %d" % (end_position)) - logger.info( - "answer: %s" % (answer_text)) + # if example_index < 20: + # logger.info("*** Example ***") + # logger.info("unique_id: %s" % (unique_id)) + # logger.info("example_index: %s" % (example_index)) + # logger.info("doc_span_index: %s" % (doc_span_index)) + # logger.info("tokens: %s" % str(tokens)) + # logger.info("token_to_orig_map: %s" % " ".join([ + # "%d:%d" % (x, y) for (x, y) in token_to_orig_map.items()])) + # logger.info("token_is_max_context: %s" % " ".join([ + # "%d:%s" % (x, y) for (x, y) in token_is_max_context.items() + # ])) + # logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids])) + # logger.info( + # "input_mask: %s" % " ".join([str(x) for x in input_mask])) + # logger.info( + # "segment_ids: %s" % " ".join([str(x) for x in segment_ids])) + # if is_training and span_is_impossible: + # logger.info("impossible example") + # if is_training and not span_is_impossible: + # answer_text = " ".join(tokens[start_position:(end_position + 1)]) + # logger.info("start_position: %d" % (start_position)) + # logger.info("end_position: %d" % (end_position)) + # logger.info( + # "answer: %s" % (answer_text)) + print("features length", len(features)) features.append( SquadFeatures( unique_id=unique_id, @@ -266,7 +460,48 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, is_impossible=span_is_impossible)) unique_id += 1 - return features + assert len(features) == len(new_features) + + assert len(features) == len(new_features) + for i in range(len(features)): + print(i) + feature, new_feature = features[i], new_features[i] + + input_ids = feature.input_ids + input_mask = feature.input_mask + segment_ids = feature.segment_ids + cls_index = feature.cls_index + p_mask = feature.p_mask + example_index = feature.example_index + paragraph_len = feature.paragraph_len + token_is_max_context = feature.token_is_max_context + tokens = feature.tokens + token_to_orig_map = feature.token_to_orig_map + + new_input_ids = new_feature.input_ids + new_input_mask = new_feature.attention_mask + new_segment_ids = new_feature.token_type_ids + new_cls_index = new_feature.cls_index + new_p_mask = new_feature.p_mask + new_example_index = new_feature.example_index + new_paragraph_len = new_feature.paragraph_len + new_token_is_max_context = new_feature.token_is_max_context + new_tokens = new_feature.tokens + new_token_to_orig_map = new_feature.token_to_orig_map + + assert input_ids == new_input_ids + assert input_mask == new_input_mask + assert segment_ids == new_segment_ids + assert cls_index == new_cls_index + assert p_mask == new_p_mask + assert example_index == new_example_index + assert paragraph_len == new_paragraph_len + assert token_is_max_context == new_token_is_max_context + assert tokens == new_tokens + assert token_to_orig_map == new_token_to_orig_map + + + return new_features def read_squad_examples(input_file, is_training, version_2_with_negative): @@ -347,6 +582,124 @@ def read_squad_examples(input_file, is_training, version_2_with_negative): return examples +class SquadV1Processor(DataProcessor): + """Processor for the SQuAD data set.""" + + def get_example_from_tensor_dict(self, tensor_dict): + """See base class.""" + return NewSquadExample( + tensor_dict['id'].numpy(), + tensor_dict['question'].numpy().decode('utf-8'), + tensor_dict['context'].numpy().decode('utf-8'), + tensor_dict['answers']['text'].numpy().decode('utf-8'), + tensor_dict['answers']['answers_start'].numpy().decode('utf-8'), + tensor_dict['title'].numpy().decode('utf-8') + ) + + def get_train_examples(self, data_dir): + """See base class.""" + with open(os.path.join(data_dir, "train-v1.1.json"), "r", encoding='utf-8') as reader: + input_data = json.load(reader)["data"] + return self._create_examples(input_data, "train") + + def get_dev_examples(self, data_dir): + """See base class.""" + with open(os.path.join(data_dir, "dev-v1.1.json"), "r", encoding='utf-8') as reader: + input_data = json.load(reader)["data"] + return self._create_examples(input_data, "dev") + + def get_labels(self): + """See base class.""" + return ["0", "1"] + + def _create_examples(self, input_data, set_type): + """Creates examples for the training and dev sets.""" + + is_training = set_type == "train" + examples = [] + for entry in input_data: + title = entry['title'] + for paragraph in entry["paragraphs"]: + context_text = paragraph["context"] + for qa in paragraph["qas"]: + qas_id = qa["id"] + question_text = qa["question"] + start_position = None + answer_text = None + if is_training: + if (len(qa["answers"]) != 1): + raise ValueError( + "For training, each question should have exactly 1 answer.") + answer = qa["answers"][0] + answer_text = answer['text'] + start_position = answer['answer_start'] + + example = NewSquadExample( + qas_id=qas_id, + question_text=question_text, + context_text=context_text, + answer_text=answer_text, + start_position=start_position, + title=title + ) + examples.append(example) + return examples + + + +class NewSquadExample(object): + """ + A single training/test example for the Squad dataset, as loaded from disk. + """ + + def __init__(self, + qas_id, + question_text, + context_text, + answer_text, + start_position, + title): + self.qas_id = qas_id + self.question_text = question_text + self.context_text = context_text + self.answer_text = answer_text + self.start_position = start_position + self.title = title + + +class NewSquadFeatures(object): + """ + Single squad example features to be fed to a model. + Those features are model-specific. + """ + + def __init__(self, + input_ids, + attention_mask, + token_type_ids, + cls_index, + p_mask, + + example_index, + unique_id, + paragraph_len, + token_is_max_context, + tokens, + token_to_orig_map + ): + self.input_ids = input_ids + self.attention_mask = attention_mask + self.token_type_ids = token_type_ids + self.cls_index = cls_index + self.p_mask = p_mask + + self.example_index = example_index + self.unique_id = unique_id + self.paragraph_len = paragraph_len + self.token_is_max_context = token_is_max_context + self.tokens = tokens + self.token_to_orig_map = token_to_orig_map + class SquadExample(object): """ A single training/test example for the Squad dataset. @@ -423,18 +776,22 @@ class SquadFeatures(object): self.is_impossible = is_impossible def __eq__(self, other): - return self.cls_index == other.cls_index and \ - self.doc_span_index == other.doc_span_index and \ - self.end_position == other.end_position and \ - self.example_index == other.example_index and \ + print(self.example_index == other.example_index) + print(self.input_ids == other.input_ids) + print(self.input_mask == other.attention_mask) + print(self.p_mask == other.p_mask) + print(self.paragraph_len == other.paragraph_len) + print(self.segment_ids == other.token_type_ids) + print(self.token_is_max_context == other.token_is_max_context) + print(self.token_to_orig_map == other.token_to_orig_map) + print(self.tokens == other.tokens) + + return self.example_index == other.example_index and \ self.input_ids == other.input_ids and \ - self.input_mask == other.input_mask and \ - self.is_impossible == other.is_impossible and \ + self.input_mask == other.attention_mask and \ self.p_mask == other.p_mask and \ self.paragraph_len == other.paragraph_len and \ - self.segment_ids == other.segment_ids and \ - self.start_position == other.start_position and \ + self.segment_ids == other.token_type_ids and \ self.token_is_max_context == other.token_is_max_context and \ self.token_to_orig_map == other.token_to_orig_map and \ - self.tokens == other.tokens and \ - self.unique_id == other.unique_id \ No newline at end of file + self.tokens == other.tokens \ No newline at end of file From c3ba6452377f085d0f59e15b97ac247bca24367e Mon Sep 17 00:00:00 2001 From: Lysandre Date: Fri, 22 Nov 2019 14:36:49 -0500 Subject: [PATCH 071/505] Works for XLNet --- examples/run_squad.py | 38 ++++-------- transformers/data/processors/squad.py | 84 +++++++++++++-------------- 2 files changed, 50 insertions(+), 72 deletions(-) diff --git a/examples/run_squad.py b/examples/run_squad.py index d4219c3096..634b566a46 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -16,6 +16,7 @@ """ Finetuning the library models for question-answering on SQuAD (DistilBERT, Bert, XLM, XLNet).""" from __future__ import absolute_import, division, print_function +from transformers.data.processors.squad import SquadV1Processor import argparse import logging @@ -46,8 +47,7 @@ from transformers import (WEIGHTS_NAME, BertConfig, from transformers import AdamW, get_linear_schedule_with_warmup, squad_convert_examples_to_features, read_squad_examples as sread_squad_examples -from utils_squad import (read_squad_examples, convert_examples_to_features, - RawResult, write_predictions, +from utils_squad import (RawResult, write_predictions, RawResultExtended, write_predictions_extended) # The follwing import is the official SQuAD evaluation script (2.0). @@ -289,7 +289,6 @@ def evaluate(args, model, tokenizer, prefix=""): results = evaluate_on_squad(evaluate_options) return results - def load_and_cache_examples(args, tokenizer, evaluate=False, output_examples=False): if args.local_rank not in [-1, 0] and not evaluate: torch.distributed.barrier() # Make sure only the first process in distributed training process the dataset, and the others will use the cache @@ -308,9 +307,11 @@ def load_and_cache_examples(args, tokenizer, evaluate=False, output_examples=Fal examples = read_squad_examples(input_file=input_file, is_training=not evaluate, version_2_with_negative=args.version_2_with_negative) - - examples = examples[:10] - features = convert_examples_to_features(examples=examples, + keep_n_examples = 1000 + processor = SquadV1Processor() + values = processor.get_dev_examples("examples/squad") + examples = values[:keep_n_examples] + features = squad_convert_examples_to_features(examples=exampless, tokenizer=tokenizer, max_seq_length=args.max_seq_length, doc_stride=args.doc_stride, @@ -320,29 +321,10 @@ def load_and_cache_examples(args, tokenizer, evaluate=False, output_examples=Fal pad_token_segment_id=3 if args.model_type in ['xlnet'] else 0, cls_token_at_end=True if args.model_type in ['xlnet'] else False, sequence_a_is_doc=True if args.model_type in ['xlnet'] else False) - - exampless = sread_squad_examples(input_file=input_file, - is_training=not evaluate, - version_2_with_negative=args.version_2_with_negative) - exampless = exampless[:10] - features2 = squad_convert_examples_to_features(examples=exampless, - tokenizer=tokenizer, - max_seq_length=args.max_seq_length, - doc_stride=args.doc_stride, - max_query_length=args.max_query_length, - is_training=not evaluate, - cls_token_segment_id=2 if args.model_type in ['xlnet'] else 0, - pad_token_segment_id=3 if args.model_type in ['xlnet'] else 0, - cls_token_at_end=True if args.model_type in ['xlnet'] else False, - sequence_a_is_doc=True if args.model_type in ['xlnet'] else False) - - print(features2) - - for i in range(len(features)): - assert features[i] == features2[i] - print("Equal") - print("DONE") + + import sys + sys.exit() if args.local_rank in [-1, 0]: logger.info("Saving features into cached file %s", cached_features_file) diff --git a/transformers/data/processors/squad.py b/transformers/data/processors/squad.py index a0f2408a16..fb3d2ae4d4 100644 --- a/transformers/data/processors/squad.py +++ b/transformers/data/processors/squad.py @@ -83,6 +83,9 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, sequence_a_is_doc=False): """Loads a data file into a list of `InputBatch`s.""" + cls_token = tokenizer.cls_token + sep_token = tokenizer.sep_token + # Defining helper methods unique_id = 1000000000 @@ -136,24 +139,24 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, sequence_pair_added_tokens = tokenizer.max_len - tokenizer.max_len_sentences_pair encoded_dict = tokenizer.encode_plus( - truncated_query, - all_doc_tokens, + truncated_query if not sequence_a_is_doc else all_doc_tokens, + all_doc_tokens if not sequence_a_is_doc else truncated_query, max_length=max_seq_length, padding_strategy='right', stride=max_seq_length - doc_stride - len(truncated_query) - sequence_pair_added_tokens, return_overflowing_tokens=True, - truncation_strategy='only_second' + truncation_strategy='only_second' if not sequence_a_is_doc else 'only_first' ) ids = encoded_dict['input_ids'] - print("Ids computes; position of the first padding", ids.index(tokenizer.pad_token_id) if tokenizer.pad_token_id in ids else None) non_padded_ids = ids[:ids.index(tokenizer.pad_token_id)] if tokenizer.pad_token_id in ids else ids paragraph_len = min(len(all_doc_tokens) - len(spans) * doc_stride, max_seq_length - len(truncated_query) - sequence_pair_added_tokens) tokens = tokenizer.convert_ids_to_tokens(non_padded_ids) token_to_orig_map = {} for i in range(paragraph_len): - token_to_orig_map[len(truncated_query) + sequence_added_tokens + i] = tok_to_orig_index[0 + i] + index = len(truncated_query) + sequence_added_tokens + i if not sequence_a_is_doc else i + token_to_orig_map[index] = tok_to_orig_index[0 + i] encoded_dict["paragraph_len"] = paragraph_len encoded_dict["tokens"] = tokens @@ -164,35 +167,40 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, encoded_dict["length"] = paragraph_len spans.append(encoded_dict) - print("YESSIR", len(spans) * doc_stride < len(all_doc_tokens), "overflowing_tokens" in encoded_dict) + # print("YESSIR", len(spans) * doc_stride < len(all_doc_tokens), "overflowing_tokens" in encoded_dict) + while len(spans) * doc_stride < len(all_doc_tokens) and "overflowing_tokens" in encoded_dict: - - overflowing_tokens = encoded_dict['overflowing_tokens'] - - print("OVERFLOW", len(overflowing_tokens)) - + overflowing_tokens = encoded_dict["overflowing_tokens"] encoded_dict = tokenizer.encode_plus( - truncated_query, - overflowing_tokens, + truncated_query if not sequence_a_is_doc else overflowing_tokens, + overflowing_tokens if not sequence_a_is_doc else truncated_query, max_length=max_seq_length, return_overflowing_tokens=True, padding_strategy='right', stride=max_seq_length - doc_stride - len(truncated_query) - sequence_pair_added_tokens, - truncation_strategy='only_second' + truncation_strategy='only_second' if not sequence_a_is_doc else 'only_first' ) - ids = encoded_dict['input_ids'] - print("Ids computes; position of the first padding", ids.index(tokenizer.pad_token_id) if tokenizer.pad_token_id in ids else None) + # print("Ids computes; position of the first padding", ids.index(tokenizer.pad_token_id) if tokenizer.pad_token_id in ids else None) + + # print(encoded_dict["input_ids"].index(tokenizer.pad_token_id) if tokenizer.pad_token_id in encoded_dict["input_ids"] else None) + # print(len(spans) * doc_stride, len(all_doc_tokens)) + # Length of the document without the query paragraph_len = min(len(all_doc_tokens) - len(spans) * doc_stride, max_seq_length - len(truncated_query) - sequence_pair_added_tokens) - non_padded_ids = encoded_dict['input_ids'][:encoded_dict['input_ids'].index(tokenizer.pad_token_id)] + if tokenizer.pad_token_id in encoded_dict['input_ids']: + non_padded_ids = encoded_dict['input_ids'][:encoded_dict['input_ids'].index(tokenizer.pad_token_id)] + else: + non_padded_ids = encoded_dict['input_ids'] + tokens = tokenizer.convert_ids_to_tokens(non_padded_ids) token_to_orig_map = {} for i in range(paragraph_len): - token_to_orig_map[len(truncated_query) + sequence_added_tokens + i] = tok_to_orig_index[len(spans) * doc_stride + i] + index = len(truncated_query) + sequence_added_tokens + i if not sequence_a_is_doc else i + token_to_orig_map[index] = tok_to_orig_index[len(spans) * doc_stride + i] encoded_dict["paragraph_len"] = paragraph_len encoded_dict["tokens"] = tokens @@ -202,23 +210,14 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, encoded_dict["start"] = len(spans) * doc_stride encoded_dict["length"] = paragraph_len - # split_token_index = doc_span.start + i - # token_to_orig_map[len(tokens)] = tok_to_orig_index[split_token_index] - - # is_max_context = _check_is_max_context(doc_spans, doc_span_index, - # split_token_index) - # token_is_max_context[len(tokens)] = is_max_context - # tokens.append(all_doc_tokens[split_token_index]) - spans.append(encoded_dict) for doc_span_index in range(len(spans)): for j in range(spans[doc_span_index]["paragraph_len"]): is_max_context = _new_check_is_max_context(spans, doc_span_index, doc_span_index * doc_stride + j) - index = spans[doc_span_index]["truncated_query_with_special_tokens_length"] + j + index = j if sequence_a_is_doc else spans[doc_span_index]["truncated_query_with_special_tokens_length"] + j spans[doc_span_index]["token_is_max_context"][index] = is_max_context - print("new span", len(spans)) for span in spans: # Identify the position of the CLS token cls_index = span['input_ids'].index(tokenizer.cls_token_id) @@ -227,17 +226,17 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, # Original TF implem also keep the classification token (set to 0) (not sure why...) p_mask = np.array(span['token_type_ids']) - # Convert all SEP indices to '0' before inversion - p_mask[np.where(np.array(span["input_ids"]) == tokenizer.sep_token_id)[0]] = 0 + p_mask = np.minimum(p_mask, 1) - # Limit positive values to one - p_mask = 1 - np.minimum(p_mask, 1) + if not sequence_a_is_doc: + # Limit positive values to one + p_mask = 1 - p_mask + + p_mask[np.where(np.array(span["input_ids"]) == tokenizer.sep_token_id)[0]] = 1 # Set the CLS index to '0' p_mask[cls_index] = 0 - print("new features length", len(new_features)) - new_features.append(NewSquadFeatures( span['input_ids'], span['attention_mask'], @@ -287,19 +286,15 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, doc_spans = [] start_offset = 0 while start_offset < len(all_doc_tokens): - print("OLD DOC CREATION BEGIN", start_offset, len(all_doc_tokens)) length = len(all_doc_tokens) - start_offset if length > max_tokens_for_doc: length = max_tokens_for_doc + # print("Start offset is", start_offset, len(all_doc_tokens), "length is", length) doc_spans.append(_DocSpan(start=start_offset, length=length)) if start_offset + length == len(all_doc_tokens): - print("Done with this doc span, breaking out.", start_offset, length) break - print("CHOOSING OFFSET", length, doc_stride) start_offset += min(length, doc_stride) - print("OLD DOC CREATION END", start_offset) - print("old span", len(doc_spans)) for (doc_span_index, doc_span) in enumerate(doc_spans): tokens = [] token_to_orig_map = {} @@ -382,7 +377,7 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, input_mask.append(0 if mask_padding_with_zero else 1) segment_ids.append(pad_token_segment_id) p_mask.append(1) - print("[OLD] Ids computed; position of the first padding", input_ids.index(tokenizer.pad_token_id) if tokenizer.pad_token_id in input_ids else None) + assert len(input_ids) == max_seq_length assert len(input_mask) == max_seq_length assert len(segment_ids) == max_seq_length @@ -440,7 +435,6 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, # logger.info( # "answer: %s" % (answer_text)) - print("features length", len(features)) features.append( SquadFeatures( unique_id=unique_id, @@ -464,10 +458,9 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, assert len(features) == len(new_features) for i in range(len(features)): - print(i) feature, new_feature = features[i], new_features[i] - input_ids = feature.input_ids + input_ids = [f if f not in [3,4,5] else 0 for f in feature.input_ids ] input_mask = feature.input_mask segment_ids = feature.segment_ids cls_index = feature.cls_index @@ -478,7 +471,7 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, tokens = feature.tokens token_to_orig_map = feature.token_to_orig_map - new_input_ids = new_feature.input_ids + new_input_ids = [f if f not in [3,4,5] else 0 for f in new_feature.input_ids] new_input_mask = new_feature.attention_mask new_segment_ids = new_feature.token_type_ids new_cls_index = new_feature.cls_index @@ -497,6 +490,9 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, assert example_index == new_example_index assert paragraph_len == new_paragraph_len assert token_is_max_context == new_token_is_max_context + + tokens = [t if tokenizer.convert_tokens_to_ids(t) is not tokenizer.unk_token_id else tokenizer.unk_token for t in tokens] + assert tokens == new_tokens assert token_to_orig_map == new_token_to_orig_map From e0e55bc550a16289763b4f656790e30ed86e428f Mon Sep 17 00:00:00 2001 From: Lysandre Date: Fri, 22 Nov 2019 16:18:18 -0500 Subject: [PATCH 072/505] Manage training example & refactor the refactor --- transformers/data/processors/squad.py | 368 ++++---------------------- 1 file changed, 51 insertions(+), 317 deletions(-) diff --git a/transformers/data/processors/squad.py b/transformers/data/processors/squad.py index fb3d2ae4d4..3d8f48c1bb 100644 --- a/transformers/data/processors/squad.py +++ b/transformers/data/processors/squad.py @@ -92,31 +92,14 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, features = [] new_features = [] for (example_index, example) in enumerate(tqdm(examples)): - - doc_tokens = [] - char_to_word_offset = [] - prev_is_whitespace = True - - # Split on whitespace so that different tokens may be attributed to their original position. - for c in example.context_text: - if _is_whitespace(c): - prev_is_whitespace = True - else: - if prev_is_whitespace: - doc_tokens.append(c) - else: - doc_tokens[-1] += c - prev_is_whitespace = False - char_to_word_offset.append(len(doc_tokens) - 1) - if is_training: # Get start and end position answer_length = len(example.answer_text) - start_position = char_to_word_offset[example.start_position] - end_position = char_to_word_offset[example.start_position + answer_length - 1] + start_position = example.start_position + end_position = example.end_position # If the answer cannot be found in the text, then skip this example. - actual_text = " ".join(doc_tokens[start_position:(end_position + 1)]) + actual_text = " ".join(example.doc_tokens[start_position:(end_position + 1)]) cleaned_answer_text = " ".join(whitespace_tokenize(example.answer_text)) if actual_text.find(cleaned_answer_text) == -1: logger.warning("Could not find answer: '%s' vs. '%s'", actual_text, cleaned_answer_text) @@ -125,7 +108,7 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, tok_to_orig_index = [] orig_to_tok_index = [] all_doc_tokens = [] - for (i, token) in enumerate(doc_tokens): + for (i, token) in enumerate(example.doc_tokens): orig_to_tok_index.append(len(all_doc_tokens)) sub_tokens = tokenizer.tokenize(token) for sub_token in sub_tokens: @@ -138,56 +121,19 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, sequence_added_tokens = tokenizer.max_len - tokenizer.max_len_single_sentence sequence_pair_added_tokens = tokenizer.max_len - tokenizer.max_len_sentences_pair - encoded_dict = tokenizer.encode_plus( - truncated_query if not sequence_a_is_doc else all_doc_tokens, - all_doc_tokens if not sequence_a_is_doc else truncated_query, - max_length=max_seq_length, - padding_strategy='right', - stride=max_seq_length - doc_stride - len(truncated_query) - sequence_pair_added_tokens, - return_overflowing_tokens=True, - truncation_strategy='only_second' if not sequence_a_is_doc else 'only_first' - ) - - ids = encoded_dict['input_ids'] - non_padded_ids = ids[:ids.index(tokenizer.pad_token_id)] if tokenizer.pad_token_id in ids else ids - paragraph_len = min(len(all_doc_tokens) - len(spans) * doc_stride, max_seq_length - len(truncated_query) - sequence_pair_added_tokens) - tokens = tokenizer.convert_ids_to_tokens(non_padded_ids) - - token_to_orig_map = {} - for i in range(paragraph_len): - index = len(truncated_query) + sequence_added_tokens + i if not sequence_a_is_doc else i - token_to_orig_map[index] = tok_to_orig_index[0 + i] - - encoded_dict["paragraph_len"] = paragraph_len - encoded_dict["tokens"] = tokens - encoded_dict["token_to_orig_map"] = token_to_orig_map - encoded_dict["truncated_query_with_special_tokens_length"] = len(truncated_query) + sequence_added_tokens - encoded_dict["token_is_max_context"] = {} - encoded_dict["start"] = 0 - encoded_dict["length"] = paragraph_len - - spans.append(encoded_dict) - # print("YESSIR", len(spans) * doc_stride < len(all_doc_tokens), "overflowing_tokens" in encoded_dict) - - while len(spans) * doc_stride < len(all_doc_tokens) and "overflowing_tokens" in encoded_dict: - overflowing_tokens = encoded_dict["overflowing_tokens"] + span_doc_tokens = all_doc_tokens + while len(spans) * doc_stride < len(all_doc_tokens): + encoded_dict = tokenizer.encode_plus( - truncated_query if not sequence_a_is_doc else overflowing_tokens, - overflowing_tokens if not sequence_a_is_doc else truncated_query, + truncated_query if not sequence_a_is_doc else span_doc_tokens, + span_doc_tokens if not sequence_a_is_doc else truncated_query, max_length=max_seq_length, return_overflowing_tokens=True, padding_strategy='right', stride=max_seq_length - doc_stride - len(truncated_query) - sequence_pair_added_tokens, truncation_strategy='only_second' if not sequence_a_is_doc else 'only_first' ) - ids = encoded_dict['input_ids'] - # print("Ids computes; position of the first padding", ids.index(tokenizer.pad_token_id) if tokenizer.pad_token_id in ids else None) - # print(encoded_dict["input_ids"].index(tokenizer.pad_token_id) if tokenizer.pad_token_id in encoded_dict["input_ids"] else None) - # print(len(spans) * doc_stride, len(all_doc_tokens)) - - - # Length of the document without the query paragraph_len = min(len(all_doc_tokens) - len(spans) * doc_stride, max_seq_length - len(truncated_query) - sequence_pair_added_tokens) if tokenizer.pad_token_id in encoded_dict['input_ids']: @@ -212,6 +158,10 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, spans.append(encoded_dict) + if "overflowing_tokens" not in encoded_dict: + break + span_doc_tokens = encoded_dict["overflowing_tokens"] + for doc_span_index in range(len(spans)): for j in range(spans[doc_span_index]["paragraph_len"]): is_max_context = _new_check_is_max_context(spans, doc_span_index, doc_span_index * doc_stride + j) @@ -254,249 +204,6 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, unique_id += 1 - # tokenize ... - query_tokens = tokenizer.tokenize(example.question_text) - - if len(query_tokens) > max_query_length: - query_tokens = query_tokens[0:max_query_length] - - tok_start_position = None - tok_end_position = None - if is_training and example.is_impossible: - tok_start_position = -1 - tok_end_position = -1 - if is_training and not example.is_impossible: - tok_start_position = orig_to_tok_index[example.start_position] - if example.end_position < len(doc_tokens) - 1: - tok_end_position = orig_to_tok_index[example.end_position + 1] - 1 - else: - tok_end_position = len(all_doc_tokens) - 1 - (tok_start_position, tok_end_position) = _improve_answer_span( - all_doc_tokens, tok_start_position, tok_end_position, tokenizer, - example.orig_answer_text) - - # The -3 accounts for [CLS], [SEP] and [SEP] - max_tokens_for_doc = max_seq_length - len(query_tokens) - 3 - - # We can have documents that are longer than the maximum sequence length. - # To deal with this we do a sliding window approach, where we take chunks - # of the up to our max length with a stride of `doc_stride`. - _DocSpan = collections.namedtuple( # pylint: disable=invalid-name - "DocSpan", ["start", "length"]) - doc_spans = [] - start_offset = 0 - while start_offset < len(all_doc_tokens): - length = len(all_doc_tokens) - start_offset - if length > max_tokens_for_doc: - length = max_tokens_for_doc - # print("Start offset is", start_offset, len(all_doc_tokens), "length is", length) - doc_spans.append(_DocSpan(start=start_offset, length=length)) - if start_offset + length == len(all_doc_tokens): - break - start_offset += min(length, doc_stride) - - for (doc_span_index, doc_span) in enumerate(doc_spans): - tokens = [] - token_to_orig_map = {} - token_is_max_context = {} - segment_ids = [] - - # p_mask: mask with 1 for token than cannot be in the answer (0 for token which can be in an answer) - # Original TF implem also keep the classification token (set to 0) (not sure why...) - p_mask = [] - - # CLS token at the beginning - if not cls_token_at_end: - tokens.append(cls_token) - segment_ids.append(cls_token_segment_id) - p_mask.append(0) - cls_index = 0 - - # XLNet: P SEP Q SEP CLS - # Others: CLS Q SEP P SEP - if not sequence_a_is_doc: - # Query - tokens += query_tokens - segment_ids += [sequence_a_segment_id] * len(query_tokens) - p_mask += [1] * len(query_tokens) - - # SEP token - tokens.append(sep_token) - segment_ids.append(sequence_a_segment_id) - p_mask.append(1) - - # Paragraph - for i in range(doc_span.length): - split_token_index = doc_span.start + i - token_to_orig_map[len(tokens)] = tok_to_orig_index[split_token_index] - - is_max_context = _check_is_max_context(doc_spans, doc_span_index, - split_token_index) - token_is_max_context[len(tokens)] = is_max_context - tokens.append(all_doc_tokens[split_token_index]) - if not sequence_a_is_doc: - segment_ids.append(sequence_b_segment_id) - else: - segment_ids.append(sequence_a_segment_id) - p_mask.append(0) - paragraph_len = doc_span.length - - if sequence_a_is_doc: - # SEP token - tokens.append(sep_token) - segment_ids.append(sequence_a_segment_id) - p_mask.append(1) - - tokens += query_tokens - segment_ids += [sequence_b_segment_id] * len(query_tokens) - p_mask += [1] * len(query_tokens) - - # SEP token - tokens.append(sep_token) - segment_ids.append(sequence_b_segment_id) - p_mask.append(1) - - # CLS token at the end - if cls_token_at_end: - tokens.append(cls_token) - segment_ids.append(cls_token_segment_id) - p_mask.append(0) - cls_index = len(tokens) - 1 # Index of classification token - - input_ids = tokenizer.convert_tokens_to_ids(tokens) - - # The mask has 1 for real tokens and 0 for padding tokens. Only real - # tokens are attended to. - input_mask = [1 if mask_padding_with_zero else 0] * len(input_ids) - - - - # Zero-pad up to the sequence length. - while len(input_ids) < max_seq_length: - input_ids.append(pad_token) - input_mask.append(0 if mask_padding_with_zero else 1) - segment_ids.append(pad_token_segment_id) - p_mask.append(1) - - assert len(input_ids) == max_seq_length - assert len(input_mask) == max_seq_length - assert len(segment_ids) == max_seq_length - - span_is_impossible = example.is_impossible if hasattr(example, "is_impossible") else False - start_position = None - end_position = None - if is_training and not span_is_impossible: - # For training, if our document chunk does not contain an annotation - # we throw it out, since there is nothing to predict. - doc_start = doc_span.start - doc_end = doc_span.start + doc_span.length - 1 - out_of_span = False - if not (tok_start_position >= doc_start and - tok_end_position <= doc_end): - out_of_span = True - if out_of_span: - start_position = 0 - end_position = 0 - span_is_impossible = True - else: - if sequence_a_is_doc: - doc_offset = 0 - else: - doc_offset = len(query_tokens) + 2 - start_position = tok_start_position - doc_start + doc_offset - end_position = tok_end_position - doc_start + doc_offset - - if is_training and span_is_impossible: - start_position = cls_index - end_position = cls_index - - # if example_index < 20: - # logger.info("*** Example ***") - # logger.info("unique_id: %s" % (unique_id)) - # logger.info("example_index: %s" % (example_index)) - # logger.info("doc_span_index: %s" % (doc_span_index)) - # logger.info("tokens: %s" % str(tokens)) - # logger.info("token_to_orig_map: %s" % " ".join([ - # "%d:%d" % (x, y) for (x, y) in token_to_orig_map.items()])) - # logger.info("token_is_max_context: %s" % " ".join([ - # "%d:%s" % (x, y) for (x, y) in token_is_max_context.items() - # ])) - # logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids])) - # logger.info( - # "input_mask: %s" % " ".join([str(x) for x in input_mask])) - # logger.info( - # "segment_ids: %s" % " ".join([str(x) for x in segment_ids])) - # if is_training and span_is_impossible: - # logger.info("impossible example") - # if is_training and not span_is_impossible: - # answer_text = " ".join(tokens[start_position:(end_position + 1)]) - # logger.info("start_position: %d" % (start_position)) - # logger.info("end_position: %d" % (end_position)) - # logger.info( - # "answer: %s" % (answer_text)) - - features.append( - SquadFeatures( - unique_id=unique_id, - example_index=example_index, - doc_span_index=doc_span_index, - tokens=tokens, - token_to_orig_map=token_to_orig_map, - token_is_max_context=token_is_max_context, - input_ids=input_ids, - input_mask=input_mask, - segment_ids=segment_ids, - cls_index=cls_index, - p_mask=p_mask, - paragraph_len=paragraph_len, - start_position=start_position, - end_position=end_position, - is_impossible=span_is_impossible)) - unique_id += 1 - - assert len(features) == len(new_features) - - assert len(features) == len(new_features) - for i in range(len(features)): - feature, new_feature = features[i], new_features[i] - - input_ids = [f if f not in [3,4,5] else 0 for f in feature.input_ids ] - input_mask = feature.input_mask - segment_ids = feature.segment_ids - cls_index = feature.cls_index - p_mask = feature.p_mask - example_index = feature.example_index - paragraph_len = feature.paragraph_len - token_is_max_context = feature.token_is_max_context - tokens = feature.tokens - token_to_orig_map = feature.token_to_orig_map - - new_input_ids = [f if f not in [3,4,5] else 0 for f in new_feature.input_ids] - new_input_mask = new_feature.attention_mask - new_segment_ids = new_feature.token_type_ids - new_cls_index = new_feature.cls_index - new_p_mask = new_feature.p_mask - new_example_index = new_feature.example_index - new_paragraph_len = new_feature.paragraph_len - new_token_is_max_context = new_feature.token_is_max_context - new_tokens = new_feature.tokens - new_token_to_orig_map = new_feature.token_to_orig_map - - assert input_ids == new_input_ids - assert input_mask == new_input_mask - assert segment_ids == new_segment_ids - assert cls_index == new_cls_index - assert p_mask == new_p_mask - assert example_index == new_example_index - assert paragraph_len == new_paragraph_len - assert token_is_max_context == new_token_is_max_context - - tokens = [t if tokenizer.convert_tokens_to_ids(t) is not tokenizer.unk_token_id else tokenizer.unk_token for t in tokens] - - assert tokens == new_tokens - assert token_to_orig_map == new_token_to_orig_map - - return new_features @@ -592,35 +299,35 @@ class SquadV1Processor(DataProcessor): tensor_dict['title'].numpy().decode('utf-8') ) - def get_train_examples(self, data_dir): + def get_train_examples(self, data_dir, only_first=None): """See base class.""" with open(os.path.join(data_dir, "train-v1.1.json"), "r", encoding='utf-8') as reader: input_data = json.load(reader)["data"] - return self._create_examples(input_data, "train") + return self._create_examples(input_data, "train", only_first) - def get_dev_examples(self, data_dir): + def get_dev_examples(self, data_dir, only_first=None): """See base class.""" with open(os.path.join(data_dir, "dev-v1.1.json"), "r", encoding='utf-8') as reader: input_data = json.load(reader)["data"] - return self._create_examples(input_data, "dev") + return self._create_examples(input_data, "dev", only_first) def get_labels(self): """See base class.""" return ["0", "1"] - def _create_examples(self, input_data, set_type): + def _create_examples(self, input_data, set_type, only_first=None): """Creates examples for the training and dev sets.""" is_training = set_type == "train" examples = [] - for entry in input_data: + for entry in tqdm(input_data): title = entry['title'] for paragraph in entry["paragraphs"]: context_text = paragraph["context"] for qa in paragraph["qas"]: qas_id = qa["id"] question_text = qa["question"] - start_position = None + start_position_character = None answer_text = None if is_training: if (len(qa["answers"]) != 1): @@ -628,17 +335,20 @@ class SquadV1Processor(DataProcessor): "For training, each question should have exactly 1 answer.") answer = qa["answers"][0] answer_text = answer['text'] - start_position = answer['answer_start'] + start_position_character = answer['answer_start'] example = NewSquadExample( qas_id=qas_id, question_text=question_text, context_text=context_text, answer_text=answer_text, - start_position=start_position, + start_position_character=start_position_character, title=title ) examples.append(example) + + if only_first is not None and len(examples) > only_first: + return examples return examples @@ -653,14 +363,38 @@ class NewSquadExample(object): question_text, context_text, answer_text, - start_position, + start_position_character, title): self.qas_id = qas_id self.question_text = question_text self.context_text = context_text self.answer_text = answer_text - self.start_position = start_position self.title = title + self.is_impossible = False + + doc_tokens = [] + char_to_word_offset = [] + prev_is_whitespace = True + + # Split on whitespace so that different tokens may be attributed to their original position. + for c in self.context_text: + if _is_whitespace(c): + prev_is_whitespace = True + else: + if prev_is_whitespace: + doc_tokens.append(c) + else: + doc_tokens[-1] += c + prev_is_whitespace = False + char_to_word_offset.append(len(doc_tokens) - 1) + + self.doc_tokens = doc_tokens + self.char_to_word_offset = char_to_word_offset + + # Start end end positions only has a value during evaluation. + if start_position_character is not None: + self.start_position = char_to_word_offset[start_position_character] + self.end_position = char_to_word_offset[start_position_character + len(answer_text) - 1] class NewSquadFeatures(object): From 041a901f324eea7e7ee04b0f7a563c7ed5c8a03a Mon Sep 17 00:00:00 2001 From: Nikolay Korolev Date: Fri, 22 Nov 2019 22:56:43 +0300 Subject: [PATCH 073/505] Fix typo in documentation. toto -> to --- transformers/tokenization_gpt2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/tokenization_gpt2.py b/transformers/tokenization_gpt2.py index 4bec515903..a4798b0129 100644 --- a/transformers/tokenization_gpt2.py +++ b/transformers/tokenization_gpt2.py @@ -184,7 +184,7 @@ class GPT2Tokenizer(PreTrainedTokenizer): """ Tokenize a string. Args: - add_prefix_space (boolean, default False): - Begin the sentence with at least one space toto get invariance to word order in GPT-2 (and RoBERTa) tokenizers. + Begin the sentence with at least one space to get invariance to word order in GPT-2 (and RoBERTa) tokenizers. """ if add_prefix_space: text = ' ' + text From 176cd1ce1b337134425b426207fbe155099c18b4 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Sat, 23 Nov 2019 11:18:54 -0500 Subject: [PATCH 074/505] [doc] homogenize instructions slightly --- README.md | 5 ++--- examples/README.md | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9cc783fd71..52e74a6f80 100644 --- a/README.md +++ b/README.md @@ -89,13 +89,12 @@ pip install [--editable] . ### Run the examples Examples are included in the repository but are not shipped with the library. -Therefore, in order to run the examples you will first need to clone the -repository and install the bleeding edge version of the library. To do so, create a new virtual environment and follow these steps: +Therefore, in order to run the latest versions of the examples you also need to install from source. To do so, create a new virtual environment and follow these steps: ```bash git clone git@github.com:huggingface/transformers cd transformers -pip install . +pip install [--editable] . ``` ### Tests diff --git a/examples/README.md b/examples/README.md index 622fa07f8f..0c57990ea7 100644 --- a/examples/README.md +++ b/examples/README.md @@ -4,12 +4,12 @@ In this section a few examples are put together. All of these examples work for similar API between the different models. **Important** -To use the examples, execute the following steps in a new virtual environment: +To run the latest versions of the examples, you have to install from source. Execute the following steps in a new virtual environment: ```bash git clone git@github.com:huggingface/transformers cd transformers -pip install . +pip install [--editable] . ``` | Section | Description | From afaa33585109550f9ecaaee4e47f187aaaefedd0 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Sat, 23 Nov 2019 11:34:45 -0500 Subject: [PATCH 075/505] [doc] Fix assets urls --- docs/source/_static/js/custom.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/_static/js/custom.js b/docs/source/_static/js/custom.js index 2c7836fd20..ec804b3704 100644 --- a/docs/source/_static/js/custom.js +++ b/docs/source/_static/js/custom.js @@ -1,5 +1,5 @@ function addIcon() { - const huggingFaceLogo = "https://huggingface.co/assets/transformers-docs/huggingface_logo.svg"; + const huggingFaceLogo = "https://huggingface.co/landing/assets/transformers-docs/huggingface_logo.svg"; const image = document.createElement("img"); image.setAttribute("src", huggingFaceLogo); @@ -24,10 +24,10 @@ function addCustomFooter() { social.classList.add("footer__Social"); const imageDetails = [ - { link: "https://huggingface.co", imageLink: "https://huggingface.co/assets/transformers-docs/website.svg" }, - { link: "https://twitter.com/huggingface", imageLink: "https://huggingface.co/assets/transformers-docs/twitter.svg" }, - { link: "https://github.com/huggingface", imageLink: "https://huggingface.co/assets/transformers-docs/github.svg" }, - { link: "https://www.linkedin.com/company/huggingface/", imageLink: "https://huggingface.co/assets/transformers-docs/linkedin.svg" } + { link: "https://huggingface.co", imageLink: "https://huggingface.co/landing/assets/transformers-docs/website.svg" }, + { link: "https://twitter.com/huggingface", imageLink: "https://huggingface.co/landing/assets/transformers-docs/twitter.svg" }, + { link: "https://github.com/huggingface", imageLink: "https://huggingface.co/landing/assets/transformers-docs/github.svg" }, + { link: "https://www.linkedin.com/company/huggingface/", imageLink: "https://huggingface.co/landing/assets/transformers-docs/linkedin.svg" } ]; imageDetails.forEach(imageLinks => { From 7485caefb09cb7f4c4b720b40ec69fed345a6b1c Mon Sep 17 00:00:00 2001 From: Lysandre Date: Mon, 25 Nov 2019 09:33:39 -0500 Subject: [PATCH 076/505] fix #1894 --- examples/run_lm_finetuning.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py index 52a1b75a65..aded521c1d 100644 --- a/examples/run_lm_finetuning.py +++ b/examples/run_lm_finetuning.py @@ -68,7 +68,7 @@ class TextDataset(Dataset): directory, filename = os.path.split(file_path) cached_features_file = os.path.join(directory, args.model_name_or_path + '_cached_lm_' + str(block_size) + '_' + filename) - if os.path.exists(cached_features_file): + if os.path.exists(cached_features_file) and not args.overwrite_cache: logger.info("Loading features from cached file %s", cached_features_file) with open(cached_features_file, 'rb') as handle: self.examples = pickle.load(handle) From 99f750d64e78d20fa5213ea11235b6a1b084481e Mon Sep 17 00:00:00 2001 From: Evpok Padding Date: Thu, 21 Nov 2019 10:35:07 +0100 Subject: [PATCH 077/505] add Camembert models to modeling_auto --- transformers/modeling_auto.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/transformers/modeling_auto.py b/transformers/modeling_auto.py index d98110d4bd..a5828148ad 100644 --- a/transformers/modeling_auto.py +++ b/transformers/modeling_auto.py @@ -27,6 +27,7 @@ from .modeling_xlnet import XLNetModel, XLNetLMHeadModel, XLNetForSequenceClassi from .modeling_xlm import XLMModel, XLMWithLMHeadModel, XLMForSequenceClassification, XLMForQuestionAnswering from .modeling_roberta import RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification from .modeling_distilbert import DistilBertModel, DistilBertForQuestionAnswering, DistilBertForMaskedLM, DistilBertForSequenceClassification +from .modeling_camembert import CamembertModel, CamembertForMaskedLM, CamembertForSequenceClassification, CamembertForMultipleChoice from .modeling_utils import PreTrainedModel, SequenceSummary @@ -48,6 +49,7 @@ class AutoModel(object): The base model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertModel (DistilBERT model) + - contains `camembert`: CamemBERTModel (CamemBERT model) - contains `roberta`: RobertaModel (RoBERTa model) - contains `bert`: BertModel (Bert model) - contains `openai-gpt`: OpenAIGPTModel (OpenAI GPT model) @@ -71,6 +73,7 @@ class AutoModel(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertModel (DistilBERT model) + - contains `camembert`: CamemBERTModel (CamemBERT model) - contains `roberta`: RobertaModel (RoBERTa model) - contains `bert`: BertModel (Bert model) - contains `openai-gpt`: OpenAIGPTModel (OpenAI GPT model) @@ -138,6 +141,8 @@ class AutoModel(object): """ if 'distilbert' in pretrained_model_name_or_path: return DistilBertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'camembert' in pretrained_model_name_or_path: + return CamembertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return RobertaModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'bert' in pretrained_model_name_or_path: @@ -172,6 +177,7 @@ class AutoModelWithLMHead(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertForMaskedLM (DistilBERT model) + - contains `camembert`: CamemBERTModel (CamemBERT model) - contains `roberta`: RobertaForMaskedLM (RoBERTa model) - contains `bert`: BertForMaskedLM (Bert model) - contains `openai-gpt`: OpenAIGPTLMHeadModel (OpenAI GPT model) @@ -198,6 +204,7 @@ class AutoModelWithLMHead(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertForMaskedLM (DistilBERT model) + - contains `camembert`: CamemBERTModel (CamemBERT model) - contains `roberta`: RobertaForMaskedLM (RoBERTa model) - contains `bert`: BertForMaskedLM (Bert model) - contains `openai-gpt`: OpenAIGPTLMHeadModel (OpenAI GPT model) @@ -264,6 +271,8 @@ class AutoModelWithLMHead(object): """ if 'distilbert' in pretrained_model_name_or_path: return DistilBertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + if 'camembert' in pretrained_model_name_or_path: + return CamembertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return RobertaForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'bert' in pretrained_model_name_or_path: @@ -298,6 +307,7 @@ class AutoModelForSequenceClassification(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertForSequenceClassification (DistilBERT model) + - contains `camembert`: CamemBERTModel (CamemBERT model) - contains `roberta`: RobertaForSequenceClassification (RoBERTa model) - contains `bert`: BertForSequenceClassification (Bert model) - contains `xlnet`: XLNetForSequenceClassification (XLNet model) @@ -320,6 +330,7 @@ class AutoModelForSequenceClassification(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertForSequenceClassification (DistilBERT model) + - contains `camembert`: CamemBERTModel (CamemBERT model) - contains `roberta`: RobertaForSequenceClassification (RoBERTa model) - contains `bert`: BertForSequenceClassification (Bert model) - contains `xlnet`: XLNetForSequenceClassification (XLNet model) @@ -383,6 +394,8 @@ class AutoModelForSequenceClassification(object): """ if 'distilbert' in pretrained_model_name_or_path: return DistilBertForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + if 'camembert' in pretrained_model_name_or_path: + return CamembertForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return RobertaForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'bert' in pretrained_model_name_or_path: From c8eb8157b86291413fe8096217f4defc168aa73f Mon Sep 17 00:00:00 2001 From: Evpok Padding Date: Thu, 21 Nov 2019 11:01:20 +0100 Subject: [PATCH 078/505] fix docstrings --- transformers/modeling_auto.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/transformers/modeling_auto.py b/transformers/modeling_auto.py index a5828148ad..ce36f6dc4a 100644 --- a/transformers/modeling_auto.py +++ b/transformers/modeling_auto.py @@ -49,7 +49,7 @@ class AutoModel(object): The base model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertModel (DistilBERT model) - - contains `camembert`: CamemBERTModel (CamemBERT model) + - contains `camembert`: CamembertModel (CamemBERT model) - contains `roberta`: RobertaModel (RoBERTa model) - contains `bert`: BertModel (Bert model) - contains `openai-gpt`: OpenAIGPTModel (OpenAI GPT model) @@ -73,7 +73,7 @@ class AutoModel(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertModel (DistilBERT model) - - contains `camembert`: CamemBERTModel (CamemBERT model) + - contains `camembert`: CamembertModel (CamemBERT model) - contains `roberta`: RobertaModel (RoBERTa model) - contains `bert`: BertModel (Bert model) - contains `openai-gpt`: OpenAIGPTModel (OpenAI GPT model) @@ -177,7 +177,7 @@ class AutoModelWithLMHead(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertForMaskedLM (DistilBERT model) - - contains `camembert`: CamemBERTModel (CamemBERT model) + - contains `camembert`: CamembertForMaskedLM (CamemBERT model) - contains `roberta`: RobertaForMaskedLM (RoBERTa model) - contains `bert`: BertForMaskedLM (Bert model) - contains `openai-gpt`: OpenAIGPTLMHeadModel (OpenAI GPT model) @@ -204,7 +204,7 @@ class AutoModelWithLMHead(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertForMaskedLM (DistilBERT model) - - contains `camembert`: CamemBERTModel (CamemBERT model) + - contains `camembert`: CamembertForMaskedLM (CamemBERT model) - contains `roberta`: RobertaForMaskedLM (RoBERTa model) - contains `bert`: BertForMaskedLM (Bert model) - contains `openai-gpt`: OpenAIGPTLMHeadModel (OpenAI GPT model) @@ -307,7 +307,7 @@ class AutoModelForSequenceClassification(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertForSequenceClassification (DistilBERT model) - - contains `camembert`: CamemBERTModel (CamemBERT model) + - contains `camembert`: CamembertForSequenceClassification (CamemBERT model) - contains `roberta`: RobertaForSequenceClassification (RoBERTa model) - contains `bert`: BertForSequenceClassification (Bert model) - contains `xlnet`: XLNetForSequenceClassification (XLNet model) @@ -330,7 +330,7 @@ class AutoModelForSequenceClassification(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertForSequenceClassification (DistilBERT model) - - contains `camembert`: CamemBERTModel (CamemBERT model) + - contains `camembert`: CamembertForSequenceClassification (CamemBERT model) - contains `roberta`: RobertaForSequenceClassification (RoBERTa model) - contains `bert`: BertForSequenceClassification (Bert model) - contains `xlnet`: XLNetForSequenceClassification (XLNet model) From fa963ecc59a1dea59c2d0e952b2c4483e1828176 Mon Sep 17 00:00:00 2001 From: Evpok Padding Date: Thu, 21 Nov 2019 11:03:12 +0100 Subject: [PATCH 079/505] =?UTF-8?q?if=E2=86=92elif?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- transformers/modeling_auto.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transformers/modeling_auto.py b/transformers/modeling_auto.py index ce36f6dc4a..5866420001 100644 --- a/transformers/modeling_auto.py +++ b/transformers/modeling_auto.py @@ -271,7 +271,7 @@ class AutoModelWithLMHead(object): """ if 'distilbert' in pretrained_model_name_or_path: return DistilBertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) - if 'camembert' in pretrained_model_name_or_path: + elif 'camembert' in pretrained_model_name_or_path: return CamembertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return RobertaForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) @@ -394,7 +394,7 @@ class AutoModelForSequenceClassification(object): """ if 'distilbert' in pretrained_model_name_or_path: return DistilBertForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) - if 'camembert' in pretrained_model_name_or_path: + elif 'camembert' in pretrained_model_name_or_path: return CamembertForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return RobertaForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) From 07bf43074f39379d8c7f6afcca105e69685f7531 Mon Sep 17 00:00:00 2001 From: Bilal Khan Date: Thu, 21 Nov 2019 20:52:06 -0600 Subject: [PATCH 080/505] Fix GPT2 docstring --- transformers/tokenization_gpt2.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/transformers/tokenization_gpt2.py b/transformers/tokenization_gpt2.py index a4798b0129..5fda709448 100644 --- a/transformers/tokenization_gpt2.py +++ b/transformers/tokenization_gpt2.py @@ -107,10 +107,10 @@ class GPT2Tokenizer(PreTrainedTokenizer): """ GPT-2 BPE tokenizer. Peculiarities: - Byte-level Byte-Pair-Encoding - - Requires a space to start the input string => the encoding methods should be called with the + - Requires a space to start the input string => the encoding and tokenize methods should be called with the ``add_prefix_space`` flag set to ``True``. - Otherwise, this tokenizer ``encode`` and ``decode`` method will not conserve - the absence of a space at the beginning of a string: `tokenizer.decode(tokenizer.encode("Hello")) = " Hello"` + Otherwise, this tokenizer's ``encode``, ``decode``, and ``tokenize`` methods will not conserve + the spaces at the beginning of a string: `tokenizer.decode(tokenizer.encode(" Hello")) = "Hello"` """ vocab_files_names = VOCAB_FILES_NAMES pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP From aa92a184d2b92faadec975139ad55e2ae749362c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=B0brahim=20Ethem=20Demirci?= Date: Sat, 16 Nov 2019 16:49:37 +0300 Subject: [PATCH 081/505] resize model when special tokenizer present --- examples/run_lm_finetuning.py | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py index aded521c1d..c33aa94a32 100644 --- a/examples/run_lm_finetuning.py +++ b/examples/run_lm_finetuning.py @@ -215,6 +215,7 @@ def train(args, train_dataset, model, tokenizer): global_step = 0 tr_loss, logging_loss = 0.0, 0.0 + model.resize_token_embeddings(len(tokenizer)) model.zero_grad() train_iterator = trange(int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0]) set_seed(args) # Added here for reproducibility (even between python 2 and 3) From 5d3b8daad2cc6287d30f03f8a96d0a1f7bc8d0dc Mon Sep 17 00:00:00 2001 From: manansanghi <52307004+manansanghi@users.noreply.github.com> Date: Fri, 22 Nov 2019 10:31:54 -0800 Subject: [PATCH 082/505] Minor bug fixes on run_ner.py --- examples/run_ner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/run_ner.py b/examples/run_ner.py index 127d63a6cd..1ab1236d94 100644 --- a/examples/run_ner.py +++ b/examples/run_ner.py @@ -127,7 +127,7 @@ def train(args, train_dataset, model, tokenizer, labels, pad_token_label_id): "attention_mask": batch[1], "labels": batch[3]} if args.model_type != "distilbert": - inputs["token_type_ids"]: batch[2] if args.model_type in ["bert", "xlnet"] else None # XLM and RoBERTa don"t use segment_ids + inputs["token_type_ids"] = batch[2] if args.model_type in ["bert", "xlnet"] else None # XLM and RoBERTa don"t use segment_ids outputs = model(**inputs) loss = outputs[0] # model outputs are always tuple in pytorch-transformers (see doc) @@ -217,7 +217,7 @@ def evaluate(args, model, tokenizer, labels, pad_token_label_id, mode, prefix="" "attention_mask": batch[1], "labels": batch[3]} if args.model_type != "distilbert": - inputs["token_type_ids"]: batch[2] if args.model_type in ["bert", "xlnet"] else None # XLM and RoBERTa don"t use segment_ids + inputs["token_type_ids"] = batch[2] if args.model_type in ["bert", "xlnet"] else None # XLM and RoBERTa don"t use segment_ids outputs = model(**inputs) tmp_eval_loss, logits = outputs[:2] From 0669c1fcd15051ec6fe2d950079886faccf2fb33 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Mon, 25 Nov 2019 19:22:21 -0500 Subject: [PATCH 083/505] SQuAD v2 BERT + XLNet --- transformers/__init__.py | 2 +- transformers/data/__init__.py | 2 +- transformers/data/processors/__init__.py | 2 +- transformers/data/processors/squad.py | 180 +++++++++++------------ 4 files changed, 92 insertions(+), 94 deletions(-) diff --git a/transformers/__init__.py b/transformers/__init__.py index 9a767913b3..f3f81f1dbe 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -27,7 +27,7 @@ from .data import (is_sklearn_available, glue_output_modes, glue_convert_examples_to_features, glue_processors, glue_tasks_num_labels, squad_convert_examples_to_features, SquadFeatures, - SquadExample, read_squad_examples) + SquadExample) if is_sklearn_available(): from .data import glue_compute_metrics diff --git a/transformers/data/__init__.py b/transformers/data/__init__.py index 50f2e768f4..b351bf625e 100644 --- a/transformers/data/__init__.py +++ b/transformers/data/__init__.py @@ -1,6 +1,6 @@ from .processors import InputExample, InputFeatures, DataProcessor, SquadFeatures from .processors import glue_output_modes, glue_processors, glue_tasks_num_labels, glue_convert_examples_to_features -from .processors import squad_convert_examples_to_features, SquadExample, read_squad_examples +from .processors import squad_convert_examples_to_features, SquadExample from .metrics import is_sklearn_available if is_sklearn_available(): diff --git a/transformers/data/processors/__init__.py b/transformers/data/processors/__init__.py index 924b4a1245..1e52776629 100644 --- a/transformers/data/processors/__init__.py +++ b/transformers/data/processors/__init__.py @@ -1,4 +1,4 @@ from .utils import InputExample, InputFeatures, DataProcessor from .glue import glue_output_modes, glue_processors, glue_tasks_num_labels, glue_convert_examples_to_features -from .squad import squad_convert_examples_to_features, SquadFeatures, SquadExample, read_squad_examples +from .squad import squad_convert_examples_to_features, SquadFeatures, SquadExample diff --git a/transformers/data/processors/squad.py b/transformers/data/processors/squad.py index 3d8f48c1bb..39ee00ae56 100644 --- a/transformers/data/processors/squad.py +++ b/transformers/data/processors/squad.py @@ -46,7 +46,6 @@ def _check_is_max_context(doc_spans, cur_span_index, position): return cur_span_index == best_span_index - def _new_check_is_max_context(doc_spans, cur_span_index, position): """Check if this is the 'max context' doc span for the token.""" # if len(doc_spans) == 1: @@ -92,7 +91,7 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, features = [] new_features = [] for (example_index, example) in enumerate(tqdm(examples)): - if is_training: + if is_training and not example.is_impossible: # Get start and end position answer_length = len(example.answer_text) start_position = example.start_position @@ -105,6 +104,7 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, logger.warning("Could not find answer: '%s' vs. '%s'", actual_text, cleaned_answer_text) continue + tok_to_orig_index = [] orig_to_tok_index = [] all_doc_tokens = [] @@ -115,6 +115,18 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, tok_to_orig_index.append(i) all_doc_tokens.append(sub_token) + + if is_training and not example.is_impossible: + tok_start_position = orig_to_tok_index[example.start_position] + if example.end_position < len(example.doc_tokens) - 1: + tok_end_position = orig_to_tok_index[example.end_position + 1] - 1 + else: + tok_end_position = len(all_doc_tokens) - 1 + + (tok_start_position, tok_end_position) = _improve_answer_span( + all_doc_tokens, tok_start_position, tok_end_position, tokenizer, example.answer_text + ) + spans = [] truncated_query = tokenizer.encode(example.question_text, add_special_tokens=False, max_length=max_query_length) @@ -187,6 +199,34 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, # Set the CLS index to '0' p_mask[cls_index] = 0 + + span_is_impossible = example.is_impossible + start_position = 0 + end_position = 0 + if is_training and not span_is_impossible: + # For training, if our document chunk does not contain an annotation + # we throw it out, since there is nothing to predict. + doc_start = span["start"] + doc_end = span["start"] + span["length"] - 1 + out_of_span = False + + if not (tok_start_position >= doc_start and tok_end_position <= doc_end): + out_of_span = True + + if out_of_span: + start_position = cls_index + end_position = cls_index + span_is_impossible = True + else: + if sequence_a_is_doc: + doc_offset = 0 + else: + doc_offset = len(truncated_query) + sequence_added_tokens + + start_position = tok_start_position - doc_start + doc_offset + end_position = tok_end_position - doc_start + doc_offset + + new_features.append(NewSquadFeatures( span['input_ids'], span['attention_mask'], @@ -199,7 +239,10 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, paragraph_len=span['paragraph_len'], token_is_max_context=span["token_is_max_context"], tokens=span["tokens"], - token_to_orig_map=span["token_to_orig_map"] + token_to_orig_map=span["token_to_orig_map"], + + start_position=start_position, + end_position=end_position )) unique_id += 1 @@ -207,86 +250,10 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, return new_features -def read_squad_examples(input_file, is_training, version_2_with_negative): - """Read a SQuAD json file into a list of SquadExample.""" - with open(input_file, "r", encoding='utf-8') as reader: - input_data = json.load(reader)["data"] - - def is_whitespace(c): - if c == " " or c == "\t" or c == "\r" or c == "\n" or ord(c) == 0x202F: - return True - return False - - examples = [] - for entry in input_data: - for paragraph in entry["paragraphs"]: - paragraph_text = paragraph["context"] - doc_tokens = [] - char_to_word_offset = [] - prev_is_whitespace = True - for c in paragraph_text: - if is_whitespace(c): - prev_is_whitespace = True - else: - if prev_is_whitespace: - doc_tokens.append(c) - else: - doc_tokens[-1] += c - prev_is_whitespace = False - char_to_word_offset.append(len(doc_tokens) - 1) - - for qa in paragraph["qas"]: - qas_id = qa["id"] - question_text = qa["question"] - start_position = None - end_position = None - orig_answer_text = None - is_impossible = False - if is_training: - if version_2_with_negative: - is_impossible = qa["is_impossible"] - if (len(qa["answers"]) != 1) and (not is_impossible): - raise ValueError( - "For training, each question should have exactly 1 answer.") - if not is_impossible: - answer = qa["answers"][0] - orig_answer_text = answer["text"] - answer_offset = answer["answer_start"] - answer_length = len(orig_answer_text) - start_position = char_to_word_offset[answer_offset] - end_position = char_to_word_offset[answer_offset + answer_length - 1] - # Only add answers where the text can be exactly recovered from the - # document. If this CAN'T happen it's likely due to weird Unicode - # stuff so we will just skip the example. - # - # Note that this means for training mode, every example is NOT - # guaranteed to be preserved. - actual_text = " ".join(doc_tokens[start_position:(end_position + 1)]) - cleaned_answer_text = " ".join( - whitespace_tokenize(orig_answer_text)) - if actual_text.find(cleaned_answer_text) == -1: - logger.warning("Could not find answer: '%s' vs. '%s'", - actual_text, cleaned_answer_text) - continue - else: - start_position = -1 - end_position = -1 - orig_answer_text = "" - - example = SquadExample( - qas_id=qas_id, - question_text=question_text, - doc_tokens=doc_tokens, - orig_answer_text=orig_answer_text, - start_position=start_position, - end_position=end_position, - is_impossible=is_impossible) - examples.append(example) - return examples - - -class SquadV1Processor(DataProcessor): +class SquadProcessor(DataProcessor): """Processor for the SQuAD data set.""" + train_file = None + dev_file = None def get_example_from_tensor_dict(self, tensor_dict): """See base class.""" @@ -301,13 +268,19 @@ class SquadV1Processor(DataProcessor): def get_train_examples(self, data_dir, only_first=None): """See base class.""" - with open(os.path.join(data_dir, "train-v1.1.json"), "r", encoding='utf-8') as reader: + if self.train_file is None: + raise ValueError("SquadProcessor should be instantiated via SquadV1Processor or SquadV2Processor") + + with open(os.path.join(data_dir, self.train_file), "r", encoding='utf-8') as reader: input_data = json.load(reader)["data"] return self._create_examples(input_data, "train", only_first) def get_dev_examples(self, data_dir, only_first=None): """See base class.""" - with open(os.path.join(data_dir, "dev-v1.1.json"), "r", encoding='utf-8') as reader: + if self.dev_file is None: + raise ValueError("SquadProcessor should be instantiated via SquadV1Processor or SquadV2Processor") + + with open(os.path.join(data_dir, self.dev_file), "r", encoding='utf-8') as reader: input_data = json.load(reader)["data"] return self._create_examples(input_data, "dev", only_first) @@ -329,7 +302,13 @@ class SquadV1Processor(DataProcessor): question_text = qa["question"] start_position_character = None answer_text = None - if is_training: + + if "is_impossible" in qa: + is_impossible = qa["is_impossible"] + else: + is_impossible = False + + if not is_impossible and is_training: if (len(qa["answers"]) != 1): raise ValueError( "For training, each question should have exactly 1 answer.") @@ -343,15 +322,25 @@ class SquadV1Processor(DataProcessor): context_text=context_text, answer_text=answer_text, start_position_character=start_position_character, - title=title + title=title, + is_impossible=is_impossible ) + examples.append(example) if only_first is not None and len(examples) > only_first: return examples return examples - +class SquadV1Processor(SquadProcessor): + train_file = "train-v1.1.json" + dev_file = "dev-v1.1.json" + + +class SquadV2Processor(SquadProcessor): + train_file = "train-v2.0.json" + dev_file = "dev-v2.0.json" + class NewSquadExample(object): """ @@ -364,13 +353,16 @@ class NewSquadExample(object): context_text, answer_text, start_position_character, - title): + title, + is_impossible=False): self.qas_id = qas_id self.question_text = question_text self.context_text = context_text self.answer_text = answer_text self.title = title - self.is_impossible = False + self.is_impossible = is_impossible + + self.start_position, self.end_position = 0, 0 doc_tokens = [] char_to_word_offset = [] @@ -392,7 +384,7 @@ class NewSquadExample(object): self.char_to_word_offset = char_to_word_offset # Start end end positions only has a value during evaluation. - if start_position_character is not None: + if start_position_character is not None and not is_impossible: self.start_position = char_to_word_offset[start_position_character] self.end_position = char_to_word_offset[start_position_character + len(answer_text) - 1] @@ -415,7 +407,10 @@ class NewSquadFeatures(object): paragraph_len, token_is_max_context, tokens, - token_to_orig_map + token_to_orig_map, + + start_position, + end_position ): self.input_ids = input_ids self.attention_mask = attention_mask @@ -430,6 +425,9 @@ class NewSquadFeatures(object): self.tokens = tokens self.token_to_orig_map = token_to_orig_map + self.start_position = start_position + self.end_position = end_position + class SquadExample(object): """ A single training/test example for the Squad dataset. From 8e5d84fcc1a645d3c13b8a2f64fa995637440dad Mon Sep 17 00:00:00 2001 From: v_sboliu Date: Tue, 26 Nov 2019 12:04:31 +0800 Subject: [PATCH 084/505] Fixed typo --- transformers/modeling_bert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py index 7c2c6f4602..81d92d8f1b 100644 --- a/transformers/modeling_bert.py +++ b/transformers/modeling_bert.py @@ -278,7 +278,7 @@ class BertAttention(nn.Module): if len(heads) == 0: return mask = torch.ones(self.self.num_attention_heads, self.self.attention_head_size) - heads = set(heads) - self.pruned_heads # Convert to set and emove already pruned heads + heads = set(heads) - self.pruned_heads # Convert to set and remove already pruned heads for head in heads: # Compute how many pruned heads are before the head and move the index accordingly head = head - sum(1 if h < head else 0 for h in self.pruned_heads) From c0c2088333e2e8ce2b24d0c7f4bf071dcccbd7ea Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 29 Oct 2019 19:57:38 +0000 Subject: [PATCH 085/505] ALBERT model --- transformers/configuration_albert.py | 72 ++++ ...lbert_original_tf_checkpoint_to_pytorch.py | 44 +++ transformers/modeling_albert.py | 331 ++++++++++++++++++ 3 files changed, 447 insertions(+) create mode 100644 transformers/configuration_albert.py create mode 100644 transformers/convert_albert_original_tf_checkpoint_to_pytorch.py create mode 100644 transformers/modeling_albert.py diff --git a/transformers/configuration_albert.py b/transformers/configuration_albert.py new file mode 100644 index 0000000000..c86c9565cb --- /dev/null +++ b/transformers/configuration_albert.py @@ -0,0 +1,72 @@ +from .configuration_utils import PretrainedConfig + +class AlbertConfig(PretrainedConfig): + """Configuration for `AlbertModel`. + + The default settings match the configuration of model `albert_xxlarge`. + """ + + def __init__(self, + vocab_size_or_config_json_file, + embedding_size=128, + hidden_size=4096, + num_hidden_layers=12, + num_hidden_groups=1, + num_attention_heads=64, + intermediate_size=16384, + inner_group_num=1, + down_scale_factor=1, + hidden_act="gelu", + hidden_dropout_prob=0, + attention_probs_dropout_prob=0, + max_position_embeddings=512, + type_vocab_size=2, + initializer_range=0.02, + layer_norm_eps=1e-12, **kwargs): + """Constructs AlbertConfig. + + Args: + vocab_size: Vocabulary size of `inputs_ids` in `AlbertModel`. + embedding_size: size of voc embeddings. + hidden_size: Size of the encoder layers and the pooler layer. + num_hidden_layers: Number of hidden layers in the Transformer encoder. + num_hidden_groups: Number of group for the hidden layers, parameters in + the same group are shared. + num_attention_heads: Number of attention heads for each attention layer in + the Transformer encoder. + intermediate_size: The size of the "intermediate" (i.e., feed-forward) + layer in the Transformer encoder. + inner_group_num: int, number of inner repetition of attention and ffn. + down_scale_factor: float, the scale to apply + hidden_act: The non-linear activation function (function or string) in the + encoder and pooler. + hidden_dropout_prob: The dropout probability for all fully connected + layers in the embeddings, encoder, and pooler. + attention_probs_dropout_prob: The dropout ratio for the attention + probabilities. + max_position_embeddings: The maximum sequence length that this model might + ever be used with. Typically set this to something large just in case + (e.g., 512 or 1024 or 2048). + type_vocab_size: The vocabulary size of the `token_type_ids` passed into + `AlbertModel`. + initializer_range: The stdev of the truncated_normal_initializer for + initializing all weight matrices. + """ + super(AlbertConfig, self).__init__(**kwargs) + + self.vocab_size = vocab_size_or_config_json_file + self.embedding_size = embedding_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_hidden_groups = num_hidden_groups + self.num_attention_heads = num_attention_heads + self.inner_group_num = inner_group_num + self.down_scale_factor = down_scale_factor + self.hidden_act = hidden_act + self.intermediate_size = intermediate_size + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.initializer_range = initializer_range + self.layer_norm_eps = layer_norm_eps \ No newline at end of file diff --git a/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py b/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py new file mode 100644 index 0000000000..04877d41b9 --- /dev/null +++ b/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py @@ -0,0 +1,44 @@ + + + +from transformers import AlbertConfig, BertForPreTraining, load_tf_weights_in_bert + + + +def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, bert_config_file, pytorch_dump_path): + # Initialise PyTorch model + config = BertConfig.from_json_file(bert_config_file) + print("Building PyTorch model from configuration: {}".format(str(config))) + model = BertForPreTraining(config) + + # Load weights from tf checkpoint + load_tf_weights_in_bert(model, config, tf_checkpoint_path) + + # Save pytorch-model + print("Save PyTorch model to {}".format(pytorch_dump_path)) + torch.save(model.state_dict(), pytorch_dump_path) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + ## Required parameters + parser.add_argument("--tf_checkpoint_path", + default = None, + type = str, + required = True, + help = "Path to the TensorFlow checkpoint path.") + parser.add_argument("--albert_config_file", + default = None, + type = str, + required = True, + help = "The config json file corresponding to the pre-trained BERT model. \n" + "This specifies the model architecture.") + parser.add_argument("--pytorch_dump_path", + default = None, + type = str, + required = True, + help = "Path to the output PyTorch model.") + args = parser.parse_args() + convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, + args.bert_config_file, + args.pytorch_dump_path) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py new file mode 100644 index 0000000000..b006cbe8fd --- /dev/null +++ b/transformers/modeling_albert.py @@ -0,0 +1,331 @@ + +import os +import math +import logging +import torch +import torch.nn as nn +from transformers.configuration_albert import AlbertConfig +logger = logging.getLogger(__name__) + +def load_tf_weights_in_albert(model, config, tf_checkpoint_path): + """ Load tf checkpoints in a pytorch model.""" + try: + import re + import numpy as np + import tensorflow as tf + except ImportError: + logger.error("Loading a TensorFlow model in PyTorch, requires TensorFlow to be installed. Please see " + "https://www.tensorflow.org/install/ for installation instructions.") + raise + tf_path = os.path.abspath(tf_checkpoint_path) + logger.info("Converting TensorFlow checkpoint from {}".format(tf_path)) + # Load weights from TF model + init_vars = tf.train.list_variables(tf_path) + names = [] + arrays = [] + for name, shape in init_vars: + logger.info("Loading TF weight {} with shape {}".format(name, shape)) + array = tf.train.load_variable(tf_path, name) + names.append(name) + arrays.append(array) + + print(model) + + for name, array in zip(names, arrays): + og = name + name = name.replace("transformer/group_0/inner_group_0", "transformer") + name = name.replace("LayerNorm", "layer_norm") + name = name.replace("ffn_1", "ffn") + name = name.replace("ffn/intermediate/output", "ffn_output") + name = name.replace("attention_1", "attention") + name = name.replace("cls/predictions/transform", "predictions") + name = name.replace("transformer/layer_norm_1", "transformer/attention/output/LayerNorm") + name = name.split('/') + + print(name) + pointer = model + for m_name in name: + if re.fullmatch(r'[A-Za-z]+_\d+', m_name): + l = re.split(r'_(\d+)', m_name) + else: + l = [m_name] + + if l[0] == 'kernel' or l[0] == 'gamma': + pointer = getattr(pointer, 'weight') + elif l[0] == 'output_bias' or l[0] == 'beta': + pointer = getattr(pointer, 'bias') + elif l[0] == 'output_weights': + pointer = getattr(pointer, 'weight') + elif l[0] == 'squad': + pointer = getattr(pointer, 'classifier') + else: + try: + pointer = getattr(pointer, l[0]) + except AttributeError: + logger.info("Skipping {}".format("/".join(name))) + continue + if len(l) >= 2: + num = int(l[1]) + pointer = pointer[num] + + if m_name[-11:] == '_embeddings': + pointer = getattr(pointer, 'weight') + elif m_name == 'kernel': + array = np.transpose(array) + print("transposed") + try: + assert pointer.shape == array.shape + except AssertionError as e: + e.args += (pointer.shape, array.shape) + raise + print("Initialize PyTorch weight {} from {}".format(name, og)) + pointer.data = torch.from_numpy(array) + + return model + + +class AlbertEmbeddings(nn.Module): + """ + Construct the embeddings from word, position and token_type embeddings. + """ + def __init__(self, config): + super(AlbertEmbeddings, self).__init__() + + self.word_embeddings = nn.Embedding(config.vocab_size, config.embedding_size, padding_idx=0) + self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.embedding_size) + self.token_type_embeddings = nn.Embedding(config.type_vocab_size, config.embedding_size) + self.layer_norm = torch.nn.LayerNorm(config.embedding_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + def forward(self, input_ids, token_type_ids=None, position_ids=None): + seq_length = input_ids.size(1) + if position_ids is None: + position_ids = torch.arange(seq_length, dtype=torch.long, device=input_ids.device) + position_ids = position_ids.unsqueeze(0).expand_as(input_ids) + if token_type_ids is None: + token_type_ids = torch.zeros_like(input_ids) + + word_embeddings = self.word_embeddings(input_ids) + position_embeddings = self.position_embeddings(position_ids) + token_type_embeddings = self.token_type_embeddings(token_type_ids) + + embeddings = word_embeddings + position_embeddings + token_type_embeddings + embeddings = self.layer_norm(embeddings) + embeddings = self.dropout(embeddings) + return embeddings + + + def get_word_embeddings_table(self): + return self.word_embeddings + + +class AlbertModel(nn.Module): + def __init__(self, config): + super(AlbertModel, self).__init__() + + self.config = config + self.embeddings = AlbertEmbeddings(config) + self.encoder = AlbertEncoder(config) + self.pooler = nn.Linear(config.hidden_size, config.hidden_size) + self.pooler_activation = nn.Tanh() + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None): + if attention_mask is None: + attention_mask = torch.ones_like(input_ids) + if token_type_ids is None: + token_type_ids = torch.zeros_like(input_ids) + + extended_attention_mask = attention_mask.unsqueeze(1).unsqueeze(2) + extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility + extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + if head_mask is not None: + if head_mask.dim() == 1: + head_mask = head_mask.unsqueeze(0).unsqueeze(0).unsqueeze(-1).unsqueeze(-1) + head_mask = head_mask.expand(self.config.num_hidden_layers, -1, -1, -1, -1) + elif head_mask.dim() == 2: + head_mask = head_mask.unsqueeze(1).unsqueeze(-1).unsqueeze(-1) # We can specify head_mask for each layer + head_mask = head_mask.to(dtype=next(self.parameters()).dtype) # switch to fload if need + fp16 compatibility + else: + head_mask = [None] * self.config.num_hidden_layers + + embedding_output = self.embeddings(input_ids, position_ids=position_ids, token_type_ids=token_type_ids) + encoder_outputs = self.encoder(embedding_output, + extended_attention_mask, + head_mask=head_mask) + sequence_output = encoder_outputs[0] + print(sequence_output.shape, sequence_output[:, 0].shape, self.pooler(sequence_output[:, 0]).shape) + pooled_output = self.pooler_activation(self.pooler(sequence_output[:, 0])) + + outputs = (sequence_output, pooled_output,) + encoder_outputs[1:] # add hidden_states and attentions if they are here + return outputs + + +class AlbertForMaskedLM(nn.Module): + def __init__(self, config): + super(AlbertForMaskedLM, self).__init__() + + self.config = config + self.bert = AlbertModel(config) + self.layer_norm = nn.LayerNorm(config.embedding_size) + self.bias = nn.Parameter(torch.zeros(config.vocab_size)) + self.dense = nn.Linear(config.hidden_size, config.embedding_size) + self.word_embeddings = nn.Linear(config.embedding_size, config.vocab_size) + + def tie_weights(self): + """ Make sure we are sharing the input and output embeddings. + Export to TorchScript can't handle parameter sharing so we are cloning them instead. + """ + self._tie_or_clone_weights(self.classifier.word_embeddings, + self.transformer.embeddings.word_embeddings) + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None): + hidden_states = self.bert(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None)[0] + hidden_states = self.dense(hidden_states) + hidden_states = gelu_new(hidden_states) + hidden_states = self.layer_norm(hidden_states) + + logits = self.word_embeddings(hidden_states) + + return logits + + +class AlbertAttention(nn.Module): + def __init__(self, config): + super(AlbertAttention, self).__init__() + + if config.hidden_size % config.num_attention_heads != 0: + raise ValueError( + "The hidden size (%d) is not a multiple of the number of attention " + "heads (%d)" % (config.hidden_size, config.num_attention_heads)) + self.output_attentions = config.output_attentions + + self.num_attention_heads = config.num_attention_heads + self.attention_head_size = int(config.hidden_size / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + self.query = nn.Linear(config.hidden_size, self.all_head_size) + self.key = nn.Linear(config.hidden_size, self.all_head_size) + self.value = nn.Linear(config.hidden_size, self.all_head_size) + + self.dropout = nn.Dropout(config.attention_probs_dropout_prob) + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + + def transpose_for_scores(self, x): + new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attention_head_size) + x = x.view(*new_x_shape) + return x.permute(0, 2, 1, 3) + + def forward(self, input_ids, attention_mask=None, head_mask=None): + mixed_query_layer = self.query(input_ids) + mixed_key_layer = self.key(input_ids) + mixed_value_layer = self.value(input_ids) + + query_layer = self.transpose_for_scores(mixed_query_layer) + key_layer = self.transpose_for_scores(mixed_key_layer) + value_layer = self.transpose_for_scores(mixed_value_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) + attention_scores = attention_scores / math.sqrt(self.attention_head_size) + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in BertModel forward() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = nn.Softmax(dim=-1)(attention_scores) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = torch.matmul(attention_probs, value_layer) + + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,) + reshaped_context_layer = context_layer.view(*new_context_layer_shape) + w = self.dense.weight.T.view(16, 64, 1024) + b = self.dense.bias + + projected_context_layer = torch.einsum("bfnd,ndh->bfh", context_layer, w) + b + projected_context_layer = self.dropout(projected_context_layer) + layernormed_context_layer = self.LayerNorm(input_ids + projected_context_layer) + return layernormed_context_layer, projected_context_layer, reshaped_context_layer, context_layer, attention_scores, attention_probs, attention_mask + + +class AlbertTransformer(nn.Module): + def __init__(self, config): + super(AlbertTransformer, self).__init__() + + self.config =config + self.layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.attention = AlbertAttention(config) + self.ffn = nn.Linear(config.hidden_size, config.intermediate_size) + self.ffn_output = nn.Linear(config.intermediate_size, config.hidden_size) + + def forward(self, hidden_states, attention_mask=None, head_mask=None): + for i in range(self.config.num_hidden_layers): + attention_output = self.attention(hidden_states, attention_mask)[0] + ffn_output = self.ffn(attention_output) + ffn_output = gelu_new(ffn_output) + ffn_output = self.ffn_output(ffn_output) + hidden_states = self.layer_norm(ffn_output + attention_output) + + return hidden_states + + +def gelu_new(x): + """ Implementation of the gelu activation function currently in Google Bert repo (identical to OpenAI GPT). + Also see https://arxiv.org/abs/1606.08415 + """ + return 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) + + +class AlbertEncoder(nn.Module): + def __init__(self, config): + super(AlbertEncoder, self).__init__() + + self.output_attentions = config.output_attentions + self.output_hidden_states = config.output_hidden_states + self.embedding_hidden_mapping_in = nn.Linear(config.embedding_size, config.hidden_size) + self.transformer = AlbertTransformer(config) + + def forward(self, hidden_states, attention_mask=None, head_mask=None): + hidden_states = self.embedding_hidden_mapping_in(hidden_states) + hidden_states = self.transformer(hidden_states, attention_mask, head_mask) + + outputs = (hidden_states,) + if self.output_hidden_states: + outputs = outputs + (all_hidden_states,) + if self.output_attentions: + outputs = outputs + (all_attentions,) + return outputs # last-layer hidden state, (all hidden states), (all attentions) + +# config = AlbertConfig.from_json_file("config.json") +# # model = AlbertForMaskedLM(config) +# model = AlbertModel(config) + +# model = load_tf_weights_in_albert(model, config, "albert/albert") + +# print(model) + +# input_ids = torch.tensor([[31, 51, 99], [15, 5, 0]]) +# input_mask = torch.tensor([[1, 1, 1], [1, 1, 0]]) +# segment_ids = torch.tensor([[0, 0, 1], [0, 0, 0]]) + +# # sequence_output, pooled_outputs = model() + +# logits = model(input_ids, attention_mask=input_mask, token_type_ids=segment_ids)[1] + + +# embeddings_output = +# print("pooled output", logits) +# # print("Pooled output", pooled_outputs) + +config = AlbertConfig.from_json_file("/home/hf/google-research/albert/config.json") +model = AlbertModel(config) +model = load_tf_weights_in_albert(model, config, "/home/hf/transformers/albert/albert") \ No newline at end of file From 91ccbae788de11f0f5ffb862421e345b27a20a76 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 29 Oct 2019 21:21:57 +0000 Subject: [PATCH 086/505] Accepts multiple sizes --- transformers/modeling_albert.py | 143 ++++++++++++++------------------ 1 file changed, 60 insertions(+), 83 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index b006cbe8fd..f3cebdc3d9 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -5,6 +5,7 @@ import logging import torch import torch.nn as nn from transformers.configuration_albert import AlbertConfig +from transformers.modeling_bert import BertEmbeddings, BertModel, BertSelfAttention, prune_linear_layer, gelu_new logger = logging.getLogger(__name__) def load_tf_weights_in_albert(model, config, tf_checkpoint_path): @@ -32,14 +33,14 @@ def load_tf_weights_in_albert(model, config, tf_checkpoint_path): print(model) for name, array in zip(names, arrays): + print(name) og = name name = name.replace("transformer/group_0/inner_group_0", "transformer") - name = name.replace("LayerNorm", "layer_norm") name = name.replace("ffn_1", "ffn") name = name.replace("ffn/intermediate/output", "ffn_output") name = name.replace("attention_1", "attention") name = name.replace("cls/predictions/transform", "predictions") - name = name.replace("transformer/layer_norm_1", "transformer/attention/output/LayerNorm") + name = name.replace("transformer/LayerNorm_1", "transformer/attention/LayerNorm") name = name.split('/') print(name) @@ -84,44 +85,22 @@ def load_tf_weights_in_albert(model, config, tf_checkpoint_path): return model -class AlbertEmbeddings(nn.Module): +class AlbertEmbeddings(BertEmbeddings): """ Construct the embeddings from word, position and token_type embeddings. """ def __init__(self, config): - super(AlbertEmbeddings, self).__init__() + super(AlbertEmbeddings, self).__init__(config) self.word_embeddings = nn.Embedding(config.vocab_size, config.embedding_size, padding_idx=0) self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.embedding_size) self.token_type_embeddings = nn.Embedding(config.type_vocab_size, config.embedding_size) - self.layer_norm = torch.nn.LayerNorm(config.embedding_size, eps=config.layer_norm_eps) - self.dropout = nn.Dropout(config.hidden_dropout_prob) - - def forward(self, input_ids, token_type_ids=None, position_ids=None): - seq_length = input_ids.size(1) - if position_ids is None: - position_ids = torch.arange(seq_length, dtype=torch.long, device=input_ids.device) - position_ids = position_ids.unsqueeze(0).expand_as(input_ids) - if token_type_ids is None: - token_type_ids = torch.zeros_like(input_ids) - - word_embeddings = self.word_embeddings(input_ids) - position_embeddings = self.position_embeddings(position_ids) - token_type_embeddings = self.token_type_embeddings(token_type_ids) - - embeddings = word_embeddings + position_embeddings + token_type_embeddings - embeddings = self.layer_norm(embeddings) - embeddings = self.dropout(embeddings) - return embeddings + self.LayerNorm = torch.nn.LayerNorm(config.embedding_size, eps=config.layer_norm_eps) - def get_word_embeddings_table(self): - return self.word_embeddings - - -class AlbertModel(nn.Module): +class AlbertModel(BertModel): def __init__(self, config): - super(AlbertModel, self).__init__() + super(AlbertModel, self).__init__(config) self.config = config self.embeddings = AlbertEmbeddings(config) @@ -129,6 +108,7 @@ class AlbertModel(nn.Module): self.pooler = nn.Linear(config.hidden_size, config.hidden_size) self.pooler_activation = nn.Tanh() + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None): if attention_mask is None: attention_mask = torch.ones_like(input_ids) @@ -166,7 +146,7 @@ class AlbertForMaskedLM(nn.Module): self.config = config self.bert = AlbertModel(config) - self.layer_norm = nn.LayerNorm(config.embedding_size) + self.LayerNorm = nn.LayerNorm(config.embedding_size) self.bias = nn.Parameter(torch.zeros(config.vocab_size)) self.dense = nn.Linear(config.hidden_size, config.embedding_size) self.word_embeddings = nn.Linear(config.embedding_size, config.vocab_size) @@ -182,39 +162,47 @@ class AlbertForMaskedLM(nn.Module): hidden_states = self.bert(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None)[0] hidden_states = self.dense(hidden_states) hidden_states = gelu_new(hidden_states) - hidden_states = self.layer_norm(hidden_states) + hidden_states = self.LayerNorm(hidden_states) logits = self.word_embeddings(hidden_states) return logits -class AlbertAttention(nn.Module): +class AlbertAttention(BertSelfAttention): def __init__(self, config): - super(AlbertAttention, self).__init__() - - if config.hidden_size % config.num_attention_heads != 0: - raise ValueError( - "The hidden size (%d) is not a multiple of the number of attention " - "heads (%d)" % (config.hidden_size, config.num_attention_heads)) - self.output_attentions = config.output_attentions + super(AlbertAttention, self).__init__(config) self.num_attention_heads = config.num_attention_heads - self.attention_head_size = int(config.hidden_size / config.num_attention_heads) - self.all_head_size = self.num_attention_heads * self.attention_head_size - - self.query = nn.Linear(config.hidden_size, self.all_head_size) - self.key = nn.Linear(config.hidden_size, self.all_head_size) - self.value = nn.Linear(config.hidden_size, self.all_head_size) - + self.hidden_size = config.hidden_size + self.attention_head_size = config.hidden_size // config.num_attention_heads self.dropout = nn.Dropout(config.attention_probs_dropout_prob) self.dense = nn.Linear(config.hidden_size, config.hidden_size) self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.pruned_heads = set() - def transpose_for_scores(self, x): - new_x_shape = x.size()[:-1] + (self.num_attention_heads, self.attention_head_size) - x = x.view(*new_x_shape) - return x.permute(0, 2, 1, 3) + def prune_heads(self, heads): + if len(heads) == 0: + return + mask = torch.ones(self.num_attention_heads, self.attention_head_size) + heads = set(heads) - self.pruned_heads # Convert to set and emove already pruned heads + for head in heads: + # Compute how many pruned heads are before the head and move the index accordingly + head = head - sum(1 if h < head else 0 for h in self.pruned_heads) + mask[head] = 0 + mask = mask.view(-1).contiguous().eq(1) + index = torch.arange(len(mask))[mask].long() + + # Prune linear layers + self.query = prune_linear_layer(self.query, index) + self.key = prune_linear_layer(self.key, index) + self.value = prune_linear_layer(self.value, index) + self.output.dense = prune_linear_layer(self.output.dense, index, dim=1) + + # Update hyper params and store pruned heads + self.num_attention_heads = self.num_attention_heads - len(heads) + self.all_head_size = self.attention_head_size * self.num_attention_heads + self.pruned_heads = self.pruned_heads.union(heads) def forward(self, input_ids, attention_mask=None, head_mask=None): mixed_query_layer = self.query(input_ids) @@ -248,7 +236,8 @@ class AlbertAttention(nn.Module): context_layer = context_layer.permute(0, 2, 1, 3).contiguous() new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,) reshaped_context_layer = context_layer.view(*new_context_layer_shape) - w = self.dense.weight.T.view(16, 64, 1024) + print(self.dense.weight.T.shape) + w = self.dense.weight.T.view(self.num_attention_heads, self.attention_head_size, self.hidden_size) b = self.dense.bias projected_context_layer = torch.einsum("bfnd,ndh->bfh", context_layer, w) + b @@ -262,7 +251,7 @@ class AlbertTransformer(nn.Module): super(AlbertTransformer, self).__init__() self.config =config - self.layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) self.attention = AlbertAttention(config) self.ffn = nn.Linear(config.hidden_size, config.intermediate_size) self.ffn_output = nn.Linear(config.intermediate_size, config.hidden_size) @@ -273,18 +262,11 @@ class AlbertTransformer(nn.Module): ffn_output = self.ffn(attention_output) ffn_output = gelu_new(ffn_output) ffn_output = self.ffn_output(ffn_output) - hidden_states = self.layer_norm(ffn_output + attention_output) + hidden_states = self.LayerNorm(ffn_output + attention_output) return hidden_states -def gelu_new(x): - """ Implementation of the gelu activation function currently in Google Bert repo (identical to OpenAI GPT). - Also see https://arxiv.org/abs/1606.08415 - """ - return 0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) - - class AlbertEncoder(nn.Module): def __init__(self, config): super(AlbertEncoder, self).__init__() @@ -305,27 +287,22 @@ class AlbertEncoder(nn.Module): outputs = outputs + (all_attentions,) return outputs # last-layer hidden state, (all hidden states), (all attentions) -# config = AlbertConfig.from_json_file("config.json") -# # model = AlbertForMaskedLM(config) -# model = AlbertModel(config) - -# model = load_tf_weights_in_albert(model, config, "albert/albert") - -# print(model) - -# input_ids = torch.tensor([[31, 51, 99], [15, 5, 0]]) -# input_mask = torch.tensor([[1, 1, 1], [1, 1, 0]]) -# segment_ids = torch.tensor([[0, 0, 1], [0, 0, 0]]) - -# # sequence_output, pooled_outputs = model() - -# logits = model(input_ids, attention_mask=input_mask, token_type_ids=segment_ids)[1] - - -# embeddings_output = -# print("pooled output", logits) -# # print("Pooled output", pooled_outputs) - -config = AlbertConfig.from_json_file("/home/hf/google-research/albert/config.json") +model_size = "base" +config = AlbertConfig.from_json_file("/home/hf/google-research/albert/config_{}.json".format(model_size)) model = AlbertModel(config) -model = load_tf_weights_in_albert(model, config, "/home/hf/transformers/albert/albert") \ No newline at end of file +model = load_tf_weights_in_albert(model, config, "/home/hf/transformers/albert-{}/albert-{}".format(model_size, model_size)) +model.eval() +print(sum(p.numel() for p in model.parameters() if p.requires_grad)) + + +input_ids = [[31, 51, 99, 88, 54, 34, 23, 23, 12], [15, 5, 0, 88, 54, 34, 23, 23, 12]] +input_mask = [[1, 1, 1, 1, 1, 1, 1, 1, 0], [1, 1, 1, 1, 1, 1, 0, 0, 0]] +segment_ids = [[0, 0, 1, 0, 0, 1, 0, 0, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0]] + +pt_input_ids = torch.tensor(input_ids) +pt_input_mask = torch.tensor(input_mask) +pt_segment_ids = torch.tensor(segment_ids) + +pt_dict = {"input_ids": pt_input_ids, "attention_mask": pt_input_mask, "token_type_ids": pt_segment_ids} +pt_output = model(**pt_dict) +print(pt_output) \ No newline at end of file From 139affaa8de89c6ef16a4712d24a22a8fea91eda Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 29 Oct 2019 22:45:24 +0000 Subject: [PATCH 087/505] Albert layer/layer groups --- transformers/modeling_albert.py | 80 ++++++++++++++++++++------------- 1 file changed, 50 insertions(+), 30 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index f3cebdc3d9..440ccf2bce 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -30,17 +30,19 @@ def load_tf_weights_in_albert(model, config, tf_checkpoint_path): names.append(name) arrays.append(array) - print(model) + for name, array in zip(names, arrays): + print(name) for name, array in zip(names, arrays): print(name) og = name - name = name.replace("transformer/group_0/inner_group_0", "transformer") name = name.replace("ffn_1", "ffn") name = name.replace("ffn/intermediate/output", "ffn_output") name = name.replace("attention_1", "attention") name = name.replace("cls/predictions/transform", "predictions") - name = name.replace("transformer/LayerNorm_1", "transformer/attention/LayerNorm") + name = name.replace("LayerNorm_1", "attention/LayerNorm") + name = name.replace("inner_group_", "albert_layers/") + name = name.replace("group_", "albert_layer_groups/") name = name.split('/') print(name) @@ -104,7 +106,7 @@ class AlbertModel(BertModel): self.config = config self.embeddings = AlbertEmbeddings(config) - self.encoder = AlbertEncoder(config) + self.encoder = AlbertTransformer(config) self.pooler = nn.Linear(config.hidden_size, config.hidden_size) self.pooler_activation = nn.Tanh() @@ -133,6 +135,7 @@ class AlbertModel(BertModel): extended_attention_mask, head_mask=head_mask) sequence_output = encoder_outputs[0] + print(sequence_output.shape, sequence_output[:, 0].shape, self.pooler(sequence_output[:, 0]).shape) pooled_output = self.pooler_activation(self.pooler(sequence_output[:, 0])) @@ -246,18 +249,18 @@ class AlbertAttention(BertSelfAttention): return layernormed_context_layer, projected_context_layer, reshaped_context_layer, context_layer, attention_scores, attention_probs, attention_mask -class AlbertTransformer(nn.Module): +class AlbertLayer(nn.Module): def __init__(self, config): - super(AlbertTransformer, self).__init__() + super(AlbertLayer, self).__init__() - self.config =config + self.config = config self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) self.attention = AlbertAttention(config) self.ffn = nn.Linear(config.hidden_size, config.intermediate_size) self.ffn_output = nn.Linear(config.intermediate_size, config.hidden_size) def forward(self, hidden_states, attention_mask=None, head_mask=None): - for i in range(self.config.num_hidden_layers): + for _ in range(self.config.inner_group_num): attention_output = self.attention(hidden_states, attention_mask)[0] ffn_output = self.ffn(attention_output) ffn_output = gelu_new(ffn_output) @@ -267,42 +270,59 @@ class AlbertTransformer(nn.Module): return hidden_states -class AlbertEncoder(nn.Module): +class AlbertLayerGroup(nn.Module): def __init__(self, config): - super(AlbertEncoder, self).__init__() + super(AlbertLayerGroup, self).__init__() + + self.albert_layers = nn.ModuleList([AlbertLayer(config) for _ in range(config.inner_group_num)]) + def forward(self, hidden_states, attention_mask=None, head_mask=None): + for albert_layer in self.albert_layers: + hidden_states = albert_layer(hidden_states, attention_mask, head_mask) + + return hidden_states + + +class AlbertTransformer(nn.Module): + def __init__(self, config): + super(AlbertTransformer, self).__init__() + + self.config = config self.output_attentions = config.output_attentions self.output_hidden_states = config.output_hidden_states self.embedding_hidden_mapping_in = nn.Linear(config.embedding_size, config.hidden_size) - self.transformer = AlbertTransformer(config) + self.albert_layer_groups = nn.ModuleList([AlbertLayerGroup(config) for _ in range(config.num_hidden_groups)]) def forward(self, hidden_states, attention_mask=None, head_mask=None): hidden_states = self.embedding_hidden_mapping_in(hidden_states) - hidden_states = self.transformer(hidden_states, attention_mask, head_mask) - outputs = (hidden_states,) - if self.output_hidden_states: - outputs = outputs + (all_hidden_states,) - if self.output_attentions: - outputs = outputs + (all_attentions,) - return outputs # last-layer hidden state, (all hidden states), (all attentions) + for layer_idx in range(self.config.num_hidden_layers): + group_idx = int(layer_idx / self.config.num_hidden_layers * self.config.num_hidden_groups) + hidden_states = self.albert_layer_groups[group_idx](hidden_states, attention_mask, head_mask) + + return (hidden_states,) -model_size = "base" -config = AlbertConfig.from_json_file("/home/hf/google-research/albert/config_{}.json".format(model_size)) + +model_size = 'base' +hidden_groups = 1 +inner_groups = 1 +config = AlbertConfig.from_json_file("/home/hf/google-research/albert/config_{}-{}-hg-{}-ig.json".format(model_size, hidden_groups, inner_groups)) model = AlbertModel(config) -model = load_tf_weights_in_albert(model, config, "/home/hf/transformers/albert-{}/albert-{}".format(model_size, model_size)) + +print(model) +model = load_tf_weights_in_albert(model, config, "/home/hf/transformers/albert-{}-{}-hg-{}-ig/albert-{}-{}-hg-{}-ig".format(model_size, hidden_groups, inner_groups, model_size, hidden_groups, inner_groups)) model.eval() print(sum(p.numel() for p in model.parameters() if p.requires_grad)) -input_ids = [[31, 51, 99, 88, 54, 34, 23, 23, 12], [15, 5, 0, 88, 54, 34, 23, 23, 12]] -input_mask = [[1, 1, 1, 1, 1, 1, 1, 1, 0], [1, 1, 1, 1, 1, 1, 0, 0, 0]] -segment_ids = [[0, 0, 1, 0, 0, 1, 0, 0, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0]] +# input_ids = [[31, 51, 99, 88, 54, 34, 23, 23, 12], [15, 5, 0, 88, 54, 34, 23, 23, 12]] +# input_mask = [[1, 1, 1, 1, 1, 1, 1, 1, 0], [1, 1, 1, 1, 1, 1, 0, 0, 0]] +# segment_ids = [[0, 0, 1, 0, 0, 1, 0, 0, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0]] -pt_input_ids = torch.tensor(input_ids) -pt_input_mask = torch.tensor(input_mask) -pt_segment_ids = torch.tensor(segment_ids) +# pt_input_ids = torch.tensor(input_ids) +# pt_input_mask = torch.tensor(input_mask) +# pt_segment_ids = torch.tensor(segment_ids) -pt_dict = {"input_ids": pt_input_ids, "attention_mask": pt_input_mask, "token_type_ids": pt_segment_ids} -pt_output = model(**pt_dict) -print(pt_output) \ No newline at end of file +# pt_dict = {"input_ids": pt_input_ids, "attention_mask": pt_input_mask, "token_type_ids": pt_segment_ids} +# pt_output = model(**pt_dict) +# print(pt_output) \ No newline at end of file From 12290c0d5ce8475e884190b6ba480a8b3e671b3e Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 29 Oct 2019 23:19:02 +0000 Subject: [PATCH 088/505] Handles multi layer and multi groups --- transformers/modeling_albert.py | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 440ccf2bce..90e9b162e6 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -136,7 +136,6 @@ class AlbertModel(BertModel): head_mask=head_mask) sequence_output = encoder_outputs[0] - print(sequence_output.shape, sequence_output[:, 0].shape, self.pooler(sequence_output[:, 0]).shape) pooled_output = self.pooler_activation(self.pooler(sequence_output[:, 0])) outputs = (sequence_output, pooled_output,) + encoder_outputs[1:] # add hidden_states and attentions if they are here @@ -260,12 +259,11 @@ class AlbertLayer(nn.Module): self.ffn_output = nn.Linear(config.intermediate_size, config.hidden_size) def forward(self, hidden_states, attention_mask=None, head_mask=None): - for _ in range(self.config.inner_group_num): - attention_output = self.attention(hidden_states, attention_mask)[0] - ffn_output = self.ffn(attention_output) - ffn_output = gelu_new(ffn_output) - ffn_output = self.ffn_output(ffn_output) - hidden_states = self.LayerNorm(ffn_output + attention_output) + attention_output = self.attention(hidden_states, attention_mask)[0] + ffn_output = self.ffn(attention_output) + ffn_output = gelu_new(ffn_output) + ffn_output = self.ffn_output(ffn_output) + hidden_states = self.LayerNorm(ffn_output + attention_output) return hidden_states @@ -303,16 +301,16 @@ class AlbertTransformer(nn.Module): return (hidden_states,) -model_size = 'base' -hidden_groups = 1 -inner_groups = 1 -config = AlbertConfig.from_json_file("/home/hf/google-research/albert/config_{}-{}-hg-{}-ig.json".format(model_size, hidden_groups, inner_groups)) -model = AlbertModel(config) +# model_size = 'base' +# hidden_groups = 1 +# inner_groups = 2 +# config = AlbertConfig.from_json_file("/home/hf/google-research/albert/config_{}-{}-hg-{}-ig.json".format(model_size, hidden_groups, inner_groups)) +# model = AlbertModel(config) -print(model) -model = load_tf_weights_in_albert(model, config, "/home/hf/transformers/albert-{}-{}-hg-{}-ig/albert-{}-{}-hg-{}-ig".format(model_size, hidden_groups, inner_groups, model_size, hidden_groups, inner_groups)) -model.eval() -print(sum(p.numel() for p in model.parameters() if p.requires_grad)) +# # print(model) +# model = load_tf_weights_in_albert(model, config, "/home/hf/transformers/albert-{}-{}-hg-{}-ig/albert-{}-{}-hg-{}-ig".format(model_size, hidden_groups, inner_groups, model_size, hidden_groups, inner_groups)) +# # model.eval() +# # print(sum(p.numel() for p in model.parameters() if p.requires_grad)) # input_ids = [[31, 51, 99, 88, 54, 34, 23, 23, 12], [15, 5, 0, 88, 54, 34, 23, 23, 12]] From 1b92564330aa2f40b065a0b9a2a94a28a595bbc6 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 29 Oct 2019 23:23:18 +0000 Subject: [PATCH 089/505] Reorganize and cleanup --- transformers/modeling_albert.py | 287 +++++++++++++++----------------- 1 file changed, 132 insertions(+), 155 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 90e9b162e6..c6662cb6d3 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -100,6 +100,138 @@ class AlbertEmbeddings(BertEmbeddings): self.LayerNorm = torch.nn.LayerNorm(config.embedding_size, eps=config.layer_norm_eps) +class AlbertAttention(BertSelfAttention): + def __init__(self, config): + super(AlbertAttention, self).__init__(config) + + self.num_attention_heads = config.num_attention_heads + self.hidden_size = config.hidden_size + self.attention_head_size = config.hidden_size // config.num_attention_heads + self.dropout = nn.Dropout(config.attention_probs_dropout_prob) + self.dense = nn.Linear(config.hidden_size, config.hidden_size) + self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.pruned_heads = set() + + def prune_heads(self, heads): + if len(heads) == 0: + return + mask = torch.ones(self.num_attention_heads, self.attention_head_size) + heads = set(heads) - self.pruned_heads # Convert to set and emove already pruned heads + for head in heads: + # Compute how many pruned heads are before the head and move the index accordingly + head = head - sum(1 if h < head else 0 for h in self.pruned_heads) + mask[head] = 0 + mask = mask.view(-1).contiguous().eq(1) + index = torch.arange(len(mask))[mask].long() + + # Prune linear layers + self.query = prune_linear_layer(self.query, index) + self.key = prune_linear_layer(self.key, index) + self.value = prune_linear_layer(self.value, index) + self.output.dense = prune_linear_layer(self.output.dense, index, dim=1) + + # Update hyper params and store pruned heads + self.num_attention_heads = self.num_attention_heads - len(heads) + self.all_head_size = self.attention_head_size * self.num_attention_heads + self.pruned_heads = self.pruned_heads.union(heads) + + def forward(self, input_ids, attention_mask=None, head_mask=None): + mixed_query_layer = self.query(input_ids) + mixed_key_layer = self.key(input_ids) + mixed_value_layer = self.value(input_ids) + + query_layer = self.transpose_for_scores(mixed_query_layer) + key_layer = self.transpose_for_scores(mixed_key_layer) + value_layer = self.transpose_for_scores(mixed_value_layer) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) + attention_scores = attention_scores / math.sqrt(self.attention_head_size) + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in BertModel forward() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = nn.Softmax(dim=-1)(attention_scores) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = torch.matmul(attention_probs, value_layer) + + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,) + reshaped_context_layer = context_layer.view(*new_context_layer_shape) + + + # Should find a better way to do this + w = self.dense.weight.T.view(self.num_attention_heads, self.attention_head_size, self.hidden_size) + b = self.dense.bias + + projected_context_layer = torch.einsum("bfnd,ndh->bfh", context_layer, w) + b + projected_context_layer = self.dropout(projected_context_layer) + layernormed_context_layer = self.LayerNorm(input_ids + projected_context_layer) + return layernormed_context_layer, projected_context_layer, reshaped_context_layer, context_layer, attention_scores, attention_probs, attention_mask + + +class AlbertLayer(nn.Module): + def __init__(self, config): + super(AlbertLayer, self).__init__() + + self.config = config + self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.attention = AlbertAttention(config) + self.ffn = nn.Linear(config.hidden_size, config.intermediate_size) + self.ffn_output = nn.Linear(config.intermediate_size, config.hidden_size) + + def forward(self, hidden_states, attention_mask=None, head_mask=None): + attention_output = self.attention(hidden_states, attention_mask)[0] + ffn_output = self.ffn(attention_output) + ffn_output = gelu_new(ffn_output) + ffn_output = self.ffn_output(ffn_output) + hidden_states = self.LayerNorm(ffn_output + attention_output) + + return hidden_states + + +class AlbertLayerGroup(nn.Module): + def __init__(self, config): + super(AlbertLayerGroup, self).__init__() + + self.albert_layers = nn.ModuleList([AlbertLayer(config) for _ in range(config.inner_group_num)]) + + def forward(self, hidden_states, attention_mask=None, head_mask=None): + for albert_layer in self.albert_layers: + hidden_states = albert_layer(hidden_states, attention_mask, head_mask) + + return hidden_states + + +class AlbertTransformer(nn.Module): + def __init__(self, config): + super(AlbertTransformer, self).__init__() + + self.config = config + self.output_attentions = config.output_attentions + self.output_hidden_states = config.output_hidden_states + self.embedding_hidden_mapping_in = nn.Linear(config.embedding_size, config.hidden_size) + self.albert_layer_groups = nn.ModuleList([AlbertLayerGroup(config) for _ in range(config.num_hidden_groups)]) + + def forward(self, hidden_states, attention_mask=None, head_mask=None): + hidden_states = self.embedding_hidden_mapping_in(hidden_states) + + for layer_idx in range(self.config.num_hidden_layers): + group_idx = int(layer_idx / self.config.num_hidden_layers * self.config.num_hidden_groups) + hidden_states = self.albert_layer_groups[group_idx](hidden_states, attention_mask, head_mask) + + return (hidden_states,) + + class AlbertModel(BertModel): def __init__(self, config): super(AlbertModel, self).__init__(config) @@ -169,158 +301,3 @@ class AlbertForMaskedLM(nn.Module): logits = self.word_embeddings(hidden_states) return logits - - -class AlbertAttention(BertSelfAttention): - def __init__(self, config): - super(AlbertAttention, self).__init__(config) - - self.num_attention_heads = config.num_attention_heads - self.hidden_size = config.hidden_size - self.attention_head_size = config.hidden_size // config.num_attention_heads - self.dropout = nn.Dropout(config.attention_probs_dropout_prob) - self.dense = nn.Linear(config.hidden_size, config.hidden_size) - self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) - self.pruned_heads = set() - - def prune_heads(self, heads): - if len(heads) == 0: - return - mask = torch.ones(self.num_attention_heads, self.attention_head_size) - heads = set(heads) - self.pruned_heads # Convert to set and emove already pruned heads - for head in heads: - # Compute how many pruned heads are before the head and move the index accordingly - head = head - sum(1 if h < head else 0 for h in self.pruned_heads) - mask[head] = 0 - mask = mask.view(-1).contiguous().eq(1) - index = torch.arange(len(mask))[mask].long() - - # Prune linear layers - self.query = prune_linear_layer(self.query, index) - self.key = prune_linear_layer(self.key, index) - self.value = prune_linear_layer(self.value, index) - self.output.dense = prune_linear_layer(self.output.dense, index, dim=1) - - # Update hyper params and store pruned heads - self.num_attention_heads = self.num_attention_heads - len(heads) - self.all_head_size = self.attention_head_size * self.num_attention_heads - self.pruned_heads = self.pruned_heads.union(heads) - - def forward(self, input_ids, attention_mask=None, head_mask=None): - mixed_query_layer = self.query(input_ids) - mixed_key_layer = self.key(input_ids) - mixed_value_layer = self.value(input_ids) - - query_layer = self.transpose_for_scores(mixed_query_layer) - key_layer = self.transpose_for_scores(mixed_key_layer) - value_layer = self.transpose_for_scores(mixed_value_layer) - - # Take the dot product between "query" and "key" to get the raw attention scores. - attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) - attention_scores = attention_scores / math.sqrt(self.attention_head_size) - if attention_mask is not None: - # Apply the attention mask is (precomputed for all layers in BertModel forward() function) - attention_scores = attention_scores + attention_mask - - # Normalize the attention scores to probabilities. - attention_probs = nn.Softmax(dim=-1)(attention_scores) - - # This is actually dropping out entire tokens to attend to, which might - # seem a bit unusual, but is taken from the original Transformer paper. - attention_probs = self.dropout(attention_probs) - - # Mask heads if we want to - if head_mask is not None: - attention_probs = attention_probs * head_mask - - context_layer = torch.matmul(attention_probs, value_layer) - - context_layer = context_layer.permute(0, 2, 1, 3).contiguous() - new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,) - reshaped_context_layer = context_layer.view(*new_context_layer_shape) - print(self.dense.weight.T.shape) - w = self.dense.weight.T.view(self.num_attention_heads, self.attention_head_size, self.hidden_size) - b = self.dense.bias - - projected_context_layer = torch.einsum("bfnd,ndh->bfh", context_layer, w) + b - projected_context_layer = self.dropout(projected_context_layer) - layernormed_context_layer = self.LayerNorm(input_ids + projected_context_layer) - return layernormed_context_layer, projected_context_layer, reshaped_context_layer, context_layer, attention_scores, attention_probs, attention_mask - - -class AlbertLayer(nn.Module): - def __init__(self, config): - super(AlbertLayer, self).__init__() - - self.config = config - self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) - self.attention = AlbertAttention(config) - self.ffn = nn.Linear(config.hidden_size, config.intermediate_size) - self.ffn_output = nn.Linear(config.intermediate_size, config.hidden_size) - - def forward(self, hidden_states, attention_mask=None, head_mask=None): - attention_output = self.attention(hidden_states, attention_mask)[0] - ffn_output = self.ffn(attention_output) - ffn_output = gelu_new(ffn_output) - ffn_output = self.ffn_output(ffn_output) - hidden_states = self.LayerNorm(ffn_output + attention_output) - - return hidden_states - - -class AlbertLayerGroup(nn.Module): - def __init__(self, config): - super(AlbertLayerGroup, self).__init__() - - self.albert_layers = nn.ModuleList([AlbertLayer(config) for _ in range(config.inner_group_num)]) - - def forward(self, hidden_states, attention_mask=None, head_mask=None): - for albert_layer in self.albert_layers: - hidden_states = albert_layer(hidden_states, attention_mask, head_mask) - - return hidden_states - - -class AlbertTransformer(nn.Module): - def __init__(self, config): - super(AlbertTransformer, self).__init__() - - self.config = config - self.output_attentions = config.output_attentions - self.output_hidden_states = config.output_hidden_states - self.embedding_hidden_mapping_in = nn.Linear(config.embedding_size, config.hidden_size) - self.albert_layer_groups = nn.ModuleList([AlbertLayerGroup(config) for _ in range(config.num_hidden_groups)]) - - def forward(self, hidden_states, attention_mask=None, head_mask=None): - hidden_states = self.embedding_hidden_mapping_in(hidden_states) - - for layer_idx in range(self.config.num_hidden_layers): - group_idx = int(layer_idx / self.config.num_hidden_layers * self.config.num_hidden_groups) - hidden_states = self.albert_layer_groups[group_idx](hidden_states, attention_mask, head_mask) - - return (hidden_states,) - - -# model_size = 'base' -# hidden_groups = 1 -# inner_groups = 2 -# config = AlbertConfig.from_json_file("/home/hf/google-research/albert/config_{}-{}-hg-{}-ig.json".format(model_size, hidden_groups, inner_groups)) -# model = AlbertModel(config) - -# # print(model) -# model = load_tf_weights_in_albert(model, config, "/home/hf/transformers/albert-{}-{}-hg-{}-ig/albert-{}-{}-hg-{}-ig".format(model_size, hidden_groups, inner_groups, model_size, hidden_groups, inner_groups)) -# # model.eval() -# # print(sum(p.numel() for p in model.parameters() if p.requires_grad)) - - -# input_ids = [[31, 51, 99, 88, 54, 34, 23, 23, 12], [15, 5, 0, 88, 54, 34, 23, 23, 12]] -# input_mask = [[1, 1, 1, 1, 1, 1, 1, 1, 0], [1, 1, 1, 1, 1, 1, 0, 0, 0]] -# segment_ids = [[0, 0, 1, 0, 0, 1, 0, 0, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0]] - -# pt_input_ids = torch.tensor(input_ids) -# pt_input_mask = torch.tensor(input_mask) -# pt_segment_ids = torch.tensor(segment_ids) - -# pt_dict = {"input_ids": pt_input_ids, "attention_mask": pt_input_mask, "token_type_ids": pt_segment_ids} -# pt_output = model(**pt_dict) -# print(pt_output) \ No newline at end of file From 67b422662c7002319e54679a73d654ba313702ef Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 29 Oct 2019 23:33:53 +0000 Subject: [PATCH 090/505] Documentation + improved AlbertForMaskedLM --- transformers/modeling_albert.py | 85 +++++++++++++++++++++++++++++++-- 1 file changed, 80 insertions(+), 5 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index c6662cb6d3..9b3c51fe25 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -4,8 +4,11 @@ import math import logging import torch import torch.nn as nn +from torch.nn import CrossEntropyLoss from transformers.configuration_albert import AlbertConfig from transformers.modeling_bert import BertEmbeddings, BertModel, BertSelfAttention, prune_linear_layer, gelu_new +from .file_utils import add_start_docstrings + logger = logging.getLogger(__name__) def load_tf_weights_in_albert(model, config, tf_checkpoint_path): @@ -232,6 +235,70 @@ class AlbertTransformer(nn.Module): return (hidden_states,) +ALBERT_START_DOCSTRING = r""" The ALBERT model was proposed in + `ALBERT: A Lite BERT for Self-supervised Learning of Language Representations`_ + by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. It presents + two parameter-reduction techniques to lower memory consumption and increase the trainig speed of BERT. + + This model is a PyTorch `torch.nn.Module`_ sub-class. Use it as a regular PyTorch Module and + refer to the PyTorch documentation for all matter related to general usage and behavior. + + .. _`ALBERT: A Lite BERT for Self-supervised Learning of Language Representations`: + https://arxiv.org/abs/1909.11942 + + .. _`torch.nn.Module`: + https://pytorch.org/docs/stable/nn.html#module + + Parameters: + config (:class:`~transformers.AlbertConfig`): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the configuration. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. +""" + +ALBERT_INPUTS_DOCSTRING = r""" + Inputs: + **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + To match pre-training, BERT input sequence should be formatted with [CLS] and [SEP] tokens as follows: + + (a) For sequence pairs: + + ``tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1`` + + (b) For single sequences: + + ``tokens: [CLS] the dog is hairy . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0`` + + Albert is a model with absolute position embeddings so it's usually advised to pad the inputs on + the right rather than the left. + + Indices can be obtained using :class:`transformers.AlbertTokenizer`. + See :func:`transformers.PreTrainedTokenizer.encode` and + :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: + Mask to avoid performing attention on padding token indices. + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. + **token_type_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Segment token indices to indicate first and second portions of the inputs. + Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` + corresponds to a `sentence B` token + (see `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ for more details). + **position_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of positions of each input sequence tokens in the position embeddings. + Selected in the range ``[0, config.max_position_embeddings - 1]``. + **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. +""" + +@add_start_docstrings("The bare ALBERT Model transformer outputting raw hidden-states without any specific head on top.", + BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) class AlbertModel(BertModel): def __init__(self, config): super(AlbertModel, self).__init__(config) @@ -274,6 +341,7 @@ class AlbertModel(BertModel): return outputs +@add_start_docstrings("Bert Model with a `language modeling` head on top.", ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) class AlbertForMaskedLM(nn.Module): def __init__(self, config): super(AlbertForMaskedLM, self).__init__() @@ -292,12 +360,19 @@ class AlbertForMaskedLM(nn.Module): self._tie_or_clone_weights(self.classifier.word_embeddings, self.transformer.embeddings.word_embeddings) - def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None): - hidden_states = self.bert(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None)[0] - hidden_states = self.dense(hidden_states) + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + masked_lm_labels=None): + outputs = self.bert(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None) + sequence_outputs = outputs[0] + hidden_states = self.dense(sequence_outputs) hidden_states = gelu_new(hidden_states) hidden_states = self.LayerNorm(hidden_states) + prediction_scores = self.word_embeddings(hidden_states) - logits = self.word_embeddings(hidden_states) + outputs = (prediction_scores,) + outputs[2:] # Add hidden states and attention if they are here + if masked_lm_labels is not None: + loss_fct = CrossEntropyLoss(ignore_index=-1) + masked_lm_loss = loss_fct(prediction_scores.view(-1, self.config.vocab_size), masked_lm_labels.view(-1)) + outputs = (masked_lm_loss,) + outputs - return logits + return outputs From fedac786d401a9da31847b45df842bbee185afe6 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 30 Oct 2019 14:39:16 +0000 Subject: [PATCH 091/505] Tokenization + small fixes --- transformers/modeling_albert.py | 2 +- transformers/tokenization_albert.py | 210 ++++++++++++++++++++++++++++ 2 files changed, 211 insertions(+), 1 deletion(-) create mode 100644 transformers/tokenization_albert.py diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 9b3c51fe25..a1c0d5f610 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -298,7 +298,7 @@ ALBERT_INPUTS_DOCSTRING = r""" """ @add_start_docstrings("The bare ALBERT Model transformer outputting raw hidden-states without any specific head on top.", - BERT_START_DOCSTRING, BERT_INPUTS_DOCSTRING) + ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) class AlbertModel(BertModel): def __init__(self, config): super(AlbertModel, self).__init__(config) diff --git a/transformers/tokenization_albert.py b/transformers/tokenization_albert.py new file mode 100644 index 0000000000..f2e37222f6 --- /dev/null +++ b/transformers/tokenization_albert.py @@ -0,0 +1,210 @@ + +from .tokenization_utils import PreTrainedTokenizer +import logging +import unicodedata +import six +import os +from shutil import copyfile + +logger = logging.getLogger(__name__) + +SPIECE_UNDERLINE = u'▁' + +class AlbertTokenizer(PreTrainedTokenizer): + """ + SentencePiece based tokenizer. Peculiarities: + + - requires `SentencePiece `_ + """ + # vocab_files_names = VOCAB_FILES_NAMES + # pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + # max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + + def __init__(self, vocab_file, + do_lower_case=False, remove_space=True, keep_accents=False, + bos_token="[CLS]", eos_token="[SEP]", unk_token="", sep_token="[SEP]", + pad_token="", cls_token="[CLS]", mask_token="[MASK]>", **kwargs): + super(AlbertTokenizer, self).__init__(bos_token=bos_token, eos_token=eos_token, + unk_token=unk_token, sep_token=sep_token, + pad_token=pad_token, cls_token=cls_token, + mask_token=mask_token, **kwargs) + + self.max_len_single_sentence = self.max_len - 2 # take into account special tokens + self.max_len_sentences_pair = self.max_len - 3 # take into account special tokens + + try: + import sentencepiece as spm + except ImportError: + logger.warning("You need to install SentencePiece to use AlbertTokenizer: https://github.com/google/sentencepiece" + "pip install sentencepiece") + + self.do_lower_case = do_lower_case + self.remove_space = remove_space + self.keep_accents = keep_accents + self.vocab_file = vocab_file + + self.sp_model = spm.SentencePieceProcessor() + self.sp_model.Load(vocab_file) + + @property + def vocab_size(self): + return len(self.sp_model) + + def __getstate__(self): + state = self.__dict__.copy() + state["sp_model"] = None + return state + + def __setstate__(self, d): + self.__dict__ = d + try: + import sentencepiece as spm + except ImportError: + logger.warning("You need to install SentencePiece to use AlbertTokenizer: https://github.com/google/sentencepiece" + "pip install sentencepiece") + self.sp_model = spm.SentencePieceProcessor() + self.sp_model.Load(self.vocab_file) + + def preprocess_text(self, inputs): + if self.remove_space: + outputs = ' '.join(inputs.strip().split()) + else: + outputs = inputs + outputs = outputs.replace("``", '"').replace("''", '"') + + if six.PY2 and isinstance(outputs, str): + outputs = outputs.decode('utf-8') + + if not self.keep_accents: + outputs = unicodedata.normalize('NFKD', outputs) + outputs = ''.join([c for c in outputs if not unicodedata.combining(c)]) + if self.do_lower_case: + outputs = outputs.lower() + + return outputs + + def _tokenize(self, text, return_unicode=True, sample=False): + """ Tokenize a string. + return_unicode is used only for py2 + """ + text = self.preprocess_text(text) + # note(zhiliny): in some systems, sentencepiece only accepts str for py2 + if six.PY2 and isinstance(text, unicode): + text = text.encode('utf-8') + + if not sample: + pieces = self.sp_model.EncodeAsPieces(text) + else: + pieces = self.sp_model.SampleEncodeAsPieces(text, 64, 0.1) + new_pieces = [] + for piece in pieces: + if len(piece) > 1 and piece[-1] == ',' and piece[-2].isdigit(): + cur_pieces = self.sp_model.EncodeAsPieces( + piece[:-1].replace(SPIECE_UNDERLINE, '')) + if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE: + if len(cur_pieces[0]) == 1: + cur_pieces = cur_pieces[1:] + else: + cur_pieces[0] = cur_pieces[0][1:] + cur_pieces.append(piece[-1]) + new_pieces.extend(cur_pieces) + else: + new_pieces.append(piece) + + # note(zhiliny): convert back to unicode for py2 + if six.PY2 and return_unicode: + ret_pieces = [] + for piece in new_pieces: + if isinstance(piece, str): + piece = piece.decode('utf-8') + ret_pieces.append(piece) + new_pieces = ret_pieces + + return new_pieces + + def _convert_token_to_id(self, token): + """ Converts a token (str/unicode) in an id using the vocab. """ + return self.sp_model.PieceToId(token) + + def _convert_id_to_token(self, index, return_unicode=True): + """Converts an index (integer) in a token (string/unicode) using the vocab.""" + token = self.sp_model.IdToPiece(index) + if six.PY2 and return_unicode and isinstance(token, str): + token = token.decode('utf-8') + return token + + def convert_tokens_to_string(self, tokens): + """Converts a sequence of tokens (strings for sub-words) in a single string.""" + out_string = ''.join(tokens).replace(SPIECE_UNDERLINE, ' ').strip() + return out_string + + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): + """ + Build model inputs from a sequence or a pair of sequence for sequence classification tasks + by concatenating and adding special tokens. + A RoBERTa sequence has the following format: + single sequence: X + pair of sequences: A B + """ + sep = [self.sep_token_id] + cls = [self.cls_token_id] + if token_ids_1 is None: + return token_ids_0 + sep + cls + return token_ids_0 + sep + token_ids_1 + sep + cls + + def get_special_tokens_mask(self, token_ids_0, token_ids_1=None, already_has_special_tokens=False): + """ + Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding + special tokens using the tokenizer ``prepare_for_model`` or ``encode_plus`` methods. + + Args: + token_ids_0: list of ids (must not contain special tokens) + token_ids_1: Optional list of ids (must not contain special tokens), necessary when fetching sequence ids + for sequence pairs + already_has_special_tokens: (default False) Set to True if the token list is already formated with + special tokens for the model + + Returns: + A list of integers in the range [0, 1]: 0 for a special token, 1 for a sequence token. + """ + + if already_has_special_tokens: + if token_ids_1 is not None: + raise ValueError("You should not supply a second sequence if the provided sequence of " + "ids is already formated with special tokens for the model.") + return list(map(lambda x: 1 if x in [self.sep_token_id, self.cls_token_id] else 0, token_ids_0)) + + if token_ids_1 is not None: + return ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1, 1] + return ([0] * len(token_ids_0)) + [1, 1] + + def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None): + """ + Creates a mask from the two sequences passed to be used in a sequence-pair classification task. + A BERT sequence pair mask has the following format: + 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 2 + | first sequence | second sequence | CLS segment ID + + if token_ids_1 is None, only returns the first portion of the mask (0's). + """ + sep = [self.sep_token_id] + cls = [self.cls_token_id] + cls_segment_id = [2] + + if token_ids_1 is None: + return len(token_ids_0 + sep + cls) * [0] + return len(token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1] + cls_segment_id + + def save_vocabulary(self, save_directory): + """ Save the sentencepiece vocabulary (copy original file) and special tokens file + to a directory. + """ + if not os.path.isdir(save_directory): + logger.error("Vocabulary path ({}) should be a directory".format(save_directory)) + return + out_vocab_file = os.path.join(save_directory, VOCAB_FILES_NAMES['vocab_file']) + + if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file): + copyfile(self.vocab_file, out_vocab_file) + + return (out_vocab_file,) From e3ea5d1d8db36091ab840328cf6691c4f46f2e89 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 30 Oct 2019 15:03:30 +0000 Subject: [PATCH 092/505] Docstrings --- transformers/modeling_albert.py | 40 +++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index a1c0d5f610..ad8b979cef 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -300,6 +300,25 @@ ALBERT_INPUTS_DOCSTRING = r""" @add_start_docstrings("The bare ALBERT Model transformer outputting raw hidden-states without any specific head on top.", ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) class AlbertModel(BertModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + Sequence of hidden-states at the output of the last layer of the model. + **pooler_output**: ``torch.FloatTensor`` of shape ``(batch_size, hidden_size)`` + Last layer hidden-state of the first token of the sequence (classification token) + further processed by a Linear layer and a Tanh activation function. The Linear + layer weights are trained from the next sentence prediction (classification) + objective during Bert pretraining. This output is usually *not* a good summary + of the semantic content of the input, you're often better with averaging or pooling + the sequence of hidden-states for the whole input sequence. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + """ def __init__(self, config): super(AlbertModel, self).__init__(config) @@ -343,6 +362,27 @@ class AlbertModel(BertModel): @add_start_docstrings("Bert Model with a `language modeling` head on top.", ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) class AlbertForMaskedLM(nn.Module): + r""" + **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the masked language modeling loss. + Indices should be in ``[-1, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) + Tokens with indices set to ``-1`` are ignored (masked), the loss is only computed for the tokens with labels + in ``[0, ..., config.vocab_size]`` + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``masked_lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Masked language modeling loss. + **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + """ + def __init__(self, config): super(AlbertForMaskedLM, self).__init__() From ee20201d339b62948acd05783efc48c4e2b466bb Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 30 Oct 2019 16:19:49 +0000 Subject: [PATCH 093/505] Tokenization tests + fixes + init --- transformers/__init__.py | 5 ++ transformers/tests/fixtures/30k-clean.model | Bin 0 -> 760289 bytes .../tests/tokenization_albert_test.py | 78 ++++++++++++++++++ transformers/tokenization_albert.py | 30 +++---- transformers/tokenization_xlnet.py | 8 +- 5 files changed, 102 insertions(+), 19 deletions(-) create mode 100644 transformers/tests/fixtures/30k-clean.model create mode 100644 transformers/tests/tokenization_albert_test.py diff --git a/transformers/__init__.py b/transformers/__init__.py index 5c7b0a6197..152d520e7b 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -42,6 +42,7 @@ from .tokenization_xlnet import XLNetTokenizer, SPIECE_UNDERLINE from .tokenization_xlm import XLMTokenizer from .tokenization_roberta import RobertaTokenizer from .tokenization_distilbert import DistilBertTokenizer +from .tokenization_albert import AlbertTokenizer from .tokenization_camembert import CamembertTokenizer # Configurations @@ -57,6 +58,8 @@ from .configuration_ctrl import CTRLConfig, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_xlm import XLMConfig, XLM_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_roberta import RobertaConfig, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_distilbert import DistilBertConfig, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_albert import AlbertConfig, ALBERT +from .configuration_albert import AlbertConfig from .configuration_camembert import CamembertConfig, CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP # Modeling @@ -104,6 +107,8 @@ if is_torch_available(): CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_encoder_decoder import PreTrainedEncoderDecoder, Model2Model + from .modeling_albert import (AlbertModel, AlbertForMaskedLM) + # Optimization from .optimization import (AdamW, get_constant_schedule, get_constant_schedule_with_warmup, get_cosine_schedule_with_warmup, get_cosine_with_hard_restarts_schedule_with_warmup, get_linear_schedule_with_warmup) diff --git a/transformers/tests/fixtures/30k-clean.model b/transformers/tests/fixtures/30k-clean.model new file mode 100644 index 0000000000000000000000000000000000000000..c91b8acfa56ccfc80e1cdd854ddcaf9b6c44ab2a GIT binary patch literal 760289 zcmZUcd7RhN_s5k&L$Y)#Q6X7UnrY9TB`SONgwHlJpPA3r=l%K2G+9#FLzbju2}KB5 z$`(rYuO&r<5?PW$2?_Z<&pq$Y9goNF^VdDE&t1+v_uTE=bML!s=d$L9=OXbVWgC9& zwk=b-n5hlvrb-V#_M~AW{`3FvVaK25=~iXi4L^C{u%R9(D{FF~MQ&C`_@MuU_x(?} z`+vfRl)^ieZTs~Vx0R?J%bLFWK#9=5#?3BKWl)pHO8a&@EhuYQwjJR@bwc9fj$2T+ zLs^qmS-d* zyS00z^k;P>E-fc`vLRcP^H)HUNnC#ZY(XhiG$B_vUFh&b<-!54mWr)U4bbd2dY)s4l4WVWUN|(=K-uSsv~s?xpJ@f90w6;dKn@62YqXhE3UJD zj*@F85VA^CC)3hx;rlL0LcZK)iT{I!KBOa1Ko z12jZ^8Ut8O2uXp^&)>m z()ek)qwS}qoTvw_6`ctXcGdcm_#^qH)5&aNz_PiJ9v zY5A8W?qX>CuahCER)h5VnUGh2jJ$lws!}%f33=!iDB6P#{P?zCl_EwXv0B-2zNhQ8 zLq8Y!+C`Qh~xv=F2AnS zg6RmN8wW|RK{~R>7lbYIc(y(x`~3}F(!D&ow_W@7r3mRPbxEBJZKX#Og?jBM*@Zqz zeUffF<5OtU?o75`X4U>w%CeY=qi$R8^#Bz}FdWYH0E12{A$?Z>7+3WPa%{cR&rTRU zrB#}c{S!+T8l?Sazm(>J3fcSMUolBTiN_q$8k)~JeH6ZpN?c2zU-O6^P*)x$sGMkB{WZ0!( zmSvELWvSJOk9;a7puBY3ZPS9X)@3b8Rm>zAJOY(kurH|P9c2iRp;ex)PB4viUJh-7 zL`ecvYPa7?2~?$I+X2uz*alKHjG&379_8w_5Ykk!Dl3<4_j@U$Vy2Kx$xm$nEOu3b zsr(Q~x>bcd+>4O0l&C|g9vSXas`N85c)CxSt*n$??)N0(k^e3B$?LQETG@Mvrz?}G zv@C1&M@dh5%58^3SXqTcBrSWyp{=?BOr~U?w|#_E7FD?POXyM^^YUen&8`D!!8>0_ zxYV~}5yp-)MuUX8B{Q*vG#%FnF#cA{QCImiG)e*8sF(<1QY!}X#-C8eUv-l1z2a|Z zE4~^QEU;CmBPJ_{t}%&T0Vzn{ROqj?#@9LeR3z!(Bn{y9gj`x&}0&_b7|YYGKF?1dXqN zj{f7jknmM2_Z{$8smLG^39=bE;s}tiQL@=u**qB9ipga2NNv&OK0+#4nUIN#8ne(2 z7s$PDfY`#Ymz=%HN6%=pyx8jRQvQL`nSP#5MCiX4PJ*`1h>+32ylkEdWZcJ5?XCAh z8b`TQgogd&X#jcwA{_mi1yP<&iwRmq(mF8A*y8DGomAPcxcjWNRBwSL?pu;QpRYB`OPY%aPFGSjC!BmkrI&12VOVv0OuqcTa0*$KNv|J#4(SL>fa6>ITPieD zOh;j3kbszA-0a)B%G0R`L+Y*LppElbHl5S{G2}`Q1=`YR^@y8%s+w$dMjrkE8bS7w zCzca3o)ZXWq{Ci|+U%h*e$(;e(a5I@Cup1zETS z+9VK7RM8KHZQG=jYd(>pXOl;#GB7H}5_umX2%}YUY6lDTk$hJ|R%j|&C@?RM831C1 z7P48{coZb*85z$yfuQy9-qQU(ke(0(9QuX*Y2kxqZ`8aS=8<2k#AOQ+Jgv`!neT67PF3sBs;nb(2&z$OGv} zp7G|MD1fn9&^BvU1#LVO<-WlG$w(8GifPc+IS>MC$%2e|3<#aoQK0f&!n9XW=By$V z){WtrWbPta}4c5%ZPt1rJwxvc57rHC~}BGxzebiG#Ti5;MghpI%z z^~Mj&e3U?^ym#9{TZae|xkAEq{KxkJgt^|bv3uj+46)i8Rnnq2pv|eQxX(Eh%H$St zbNU`f16boCSQ#=1(rVM73y6_}p~*Zar=3p7G;;}Dijy<1`iIRMKhU_tX)BU#BF za7mtjt!mj=A*;hx?}CSPoybB?4jK<+W!DtZyPaL;uNF=nHxBKG#dh}0UlDY_7? zRwMf#0A^)Ulh?XInW!qMs*JDjDdLHkRpgxBK1D?zIqe9aBASgi$dsd?tx$G+Y?jJT z{7(c{e6ESpU%Cj|W#cSP075OH4h#sNddV! zO;)*lLYE>;a+BZpk4lYRi!+zk&v}WB+$8c z`)#%_DL|o^&wz~B76kqnF}jx#wAeXzGI_aO0V_)_9WB~GYrI~rXQG9Jf5tq#1uxwA221uRWv0+m^t@M&TcTVxJc)2_eHc0{K2k3) zJ>tn6gV#lKAx%^^KL^t76(BY5!R-9S7RK%)&3+^t%DsTutsuYr>=SdiP{<;)Q~w3B z(yJ4xoa`VwloF+F*Y{&PkWd%vBedtw9RNa2hQSNUeF_c?w7L%|Vx6#`yj0sx8Z5p=r9uRAYc(N`T=sfr2 z3@~Gr4H%r!$nIYP zguPrBT$~MI$N2%!n!~p830bxt$ZFqT+HQa_1`4{4t5w=^O}RBeQ32}6>B#C9ElU6C zjy#a(S^|UvX@42o8p?{oM4c(ncmvx2S?(!anr*wMr|C3tIiowYm6qj1BO}M01Yi|r zO&mvL+0#KxA~8%8(;7+L-+z8K|5}+rE)mlWd;5q_P+#33HF=+7CEcIZ;FvleAf(UyncG2yG%`%sBLB|8GIf$`2`R@<4=)jMb9>tW6`a7}J-k?`|`EtVmi{g1bKIv*}sR z%JUwjQIdK|KW3p3Qu!<)Q^zb;GB;#p7J*rDY-%WO?%M#yePt1T4td{4sA53P$Xg#a zMqrgH)ytqQ*HFDsA;rQGfLTrTzV$agS`;HnCV?h@{e#a6TUH?JmA0m55t3rpandG_ zBr@(;VdZZis*HJ=+oG%_$co-_{_apFC)U1cHDWf%+_r$B_UWqd^Sz)O6*R_rf7*UP zR;aTt)rA<^cVRh5m?52Btv1K>29mO!*>lF}(Wz|C zZIb^!1~}9bj1pHI2Wf(@&#KY(`V#=CSf0b|GYA^j9ARhVw_(sGV&-mUBUE^=v%wfq zb@;D6*T+xkdB|fIKwI8v*MrAj1VG+wLPkd{m`F*7%LrP{vDxZ=q3UuFQ_XZzy4(mA zszQ?OYX3==v=EWWVUQE6in~FrUKx0aJ?68h=q2+PSdw%^9$rk)a>MMJk!wHkG#XnW z-zeYE>ey@=g3= zQIip=+vxv{q%Z+8Vl>;a#2iBeHsa2nuGabFg#)0Cmt^3^u{KBiCkFlGos*%%VW^nP zX;)f)IuOiZURZgyPajFSGpno41F`C|*<(oQ838Cw7A#+w`ec`ck7Nf#6Ker%SQ;Qd z6(B;17HOZcS9ltU_LSyRp^a&0eF&sR5$hv~c#&PF_ROcJ0a~7Qa>i_ku;)hdm_OsP z^Bf@KR6A~!9Q2Hj!}(4j&q?9JML@<%&+_vA((4UqE1Ie>XvuAj2!~2)3Tw8MI9B1H zP}Bu!AWJ{>={U*LIdseSAfcR+38tqzegr_O%miIF_}F^D0+4iUmbO2b#G`Y&D!I#WBgx; zy07#--u{Y5WcpzLS6b)RcZNY*p2?)lI@2OzhUqfgN2K*QaUB62j!kODzYwxbSt~}c zSal?wLra3{Bvz*=xV1$gTVSYrQ~^sD!BqP|0@CVbgBig?SDL> z)&n;>KBcJXou1bwIRQIxG>{iOd*bV$ZCs56K{l2&^^M*DOzXN+`#Y5|4sGq&=SH7N zjFq$O{UEfl$e~h_?QerDn(Y(ivkg?M=Ntel37aTtV+*OmGr-1UB*g@U&7kQ^9>PxH z7I*5e1grKyu()Mj-5ZS*L{c8@547jN{ub1qsI$UEaXn zn&m(!1FV0til8s7x19FXf9Udlvhq8hI+xE@grM#!6PRCnHJ`ah zaV0B7-AbN59mvG*c1D5By5C2N=54n$X&$q^p{8JAUwteUJ| zUebuzOt;fA5Mzg>9F?#5#7A+RKPAgP2eF*diyEd(Nw4;iYS_t9){q}SLQy;A0c-gq znAHqDiOW)@5eUnwzc%;`botGQQMM7prj%Tk5rfm|lw|(^v=JhiQDd!*MSZsdhayw2 zwx@U5xkS^>1!>+I+BlA>nGIf6w*@lgs+D3#2rDa@)-{BlXT8-KFdS8LIp!o%{nZ7` z3euIbr1yt5X~o&##-(ddAEB5-dNU985u#aL(cEwtfaMz6Ap>cC9GJCBT8)(j=|0$J zgPJB}-btQjiP2ZSy5PTbANlJ-pF=d_&dr{Vf*5ByM+dBFe3S&%PL%e>N}owpGM|!E z$|s3pXpKv|kv>UKOsiqM%Q%l{qp*CMbt8z0R*kbd9L&4fL+Zz(Ej{3NAnQR@WISv# zBqJAw23hkEVUv9%!(Ib}WL(xh0caA5V>PSF$h0RxO!Cnb*1SnCL8CkL>f>Mbe_=I1 zoQt5XFpf*5&)bkDaXUaI|GPf`+o7yUoou(5kd@@TyD~CQ5u3exl}qk(DASX2S--+Z zuFOZ&bh_^<5HDsn=94x6s7`04&F=(_4K3+PRuM*TVxrP7f_V z>jx0_M?H0ZxIZ*fsgda6giI=&8>-&+KguVj)6rwkRRC6<4^W>IAi_<+0C{96lu0~M z%(G*o_V1qzRvI+eOrhjG&H%DPf{|D`6Efgz5L*B-vQ)BHWdx#|T{YY%R+l{O;*Ujv zLcue>;=@tn0oGvbHd?2AB36Uy-69`kfWm6o)@D@RN3sPop3C(>#=Ubk(HWukN-(Qm zJTE&>fHIXuJ7f7|6mgGH&=jRwj=vScB+GgzC%aFAHdTnSgYI|-WXMOpLHo@6_ke`G z0OJMwS!(ddOu(>R0T}Nxpz9w6wVLZnIWE6E?sH?=m|<8S{FINvYKx^ppQi!na11W8 z;Ta1jgR~UpHYR2B&VY5y^B~4~E>cWM@0Xxa!fYTbUh#ji9WKby1<+RGynL>NwQs-- zuYF^;_koO2hQ>rzcKe_)1J%Eul~x5$OS8rPuPQZ$y|conWvXB$-fbm-Nw5KP<%*wt z{CZsuRR7}t<_t-?Y=X4*>e)vIZ-Fv#r<3tG3-8E(kSyILBe|`OaX8?oS3S~Xmy#X? z+J7|GVcU6(HI*%hy6ymG5_3D9^ILeDMJ(>^EupC;UgxWK@qg)Qirlfgr*&g3|0->j z}r;t@|r71@MgdJTCKrN1f zHi6_5x-pVDM}t^VY+)FeDe9ELU@(q>;@Oi4hP{_l0M*!I&Id9^P*;|x_f=YS`q}XG zfBw_G>@^vHus>+mAD@Lb&2|TkwjLgr2enqiRs6{!lnH}ld>LH_Y2~R{@jk5wF!eh^ zKE4#f__uycp>K|}2zVq^$%yNG@=7MEyqr7{fG(ky&LwvfHujt}mfd6jB^kJWyw4|u zRd^pg3&8lQC@+`Uk3pJfSm-&giR|M(iMkZg<(>ebEOq10&mnAOv5)K{-JgZF?3le| z%e(*in^VT}4=t%S$aS9(wo0;`tt#rNRri(lKl^wYuHjJ6F%VP-a@$uPaTfMKZu%O; zWSq{+h;N~2V*IP8tnpFv>YPxJSJ#4A%@U*7j?!=X`~>8gFP9HCc$%r7iG8C_l8+?S z#v^}h0y3szYG2}f@-Lr=gVDHbY_V%82IESR1Dua{1+kJt&mnoLjgQ2sQyQE2L+yQ} zG`ABXtbuRe6Uf+O(G!=)IzUrBjKGho-K7QvVLd8C%@G?hL6NS_5Ww{2)2RLlx?4hM00=s1wD zf4Q^yyc{qH$n-t#{O7hC0$|J))N614P-s(}fF;7{Qz4DlNIY(Po3hiu&^w(l=MN)n z`MJYZIrBmQ<06wt#NjoW1qnyZ3i&`O`=5D+vp)GW0HMb@KkU=8ZGVK6*z3W$k_`5>>l z|M^rPnu1$2>FGXx-n>;_QN+ZP!dqnRgFbnzn0Kd<`#%a)8c}q;dxZj41ilMQS#i1P z>Hk1MPS@fao&};*_9&k`Pta!VFjE$0kG%k>fvA8KD>sZ@d z{|1bmz(7Yezxf0p)apb6*Rs!{{ou*@rZd_1Tn*}#D#oNgYMn>0GOO<1 zwB92)fT((Q{RzbKroIt5=nv?S;Vg^HKmFfnUBylL3%a!OaW;1HSg4@YfDsTcv*cC) z^rWB{I?mf+w~|oZlu3e(3L(KvQV!aQpmB|LTKyS@w*=9|dYfghwh&g;@O}ddB(ErH z1;!IPrnYYfV6|XO#T&3awAF%jq})1HeN8Omh%4Iv&V($N4z)w?w7-6kUHzeO^H#v3GorZ-!$ld3 zjzN5R-bc5i)CF}Oz&1vAKj~77T$`1+1}!TF^}McLs}w>d2nW*h3ZIrG6Z1gxu>fHw zBmv_#Q|0vWfF{OxQm(!l%ETC8XPa;hq$)sChFt3tM>sv;5E`MLub`kO>tRd|SO|!ypQ4xn?0KDUa}Q2J3C{M?n?e`1|0IFn>B`0Rv3urZ6M49n94 zo>_A|jT2cQdp-wkT;ocgw0`#SB0%Gs2?dARKVJ8#@F6J3ON%|N-bymA&p}MaYPZ)NZvG1(xUAv0R)hAhnrn3dv3fO-3C8SlNHh73k7ft?fy`S2 zWc6dG##QAF(AIFY{-J+ESw*-y%W2Br3RpopOsFf#7h8eCf;aIsStW<>3S=ef-rSjCr7f8C zmpTT++mu8$47mI6LD0%Y)vz+IZx3L?tt@J9mGe7*7$=b!*EpFs&*=mdj_W;T!@f`^ zrCJ=>W#E3$VO@%7tDJw3k3hGH%Yq)zp~{)<;kF8&q$a|37Wu0efN73e%FO4ibzeZn zF8t{wuHmpln2C+OaG;>gUpPMK*_1krZ5jYtl5GPG_w}IwR(@b6f`<!uPSZYMMFJZ zF`$3BbUoG6%w2e_D{btlS3i{u$2YN5lt+}++l#-@Nws(Se# zft|z0iA8KI99i$7N_J-I-Lhm9P$+vnp$O#Au^<$ii^%a;5wtwh%<3umaH7wEJPYb% z`sY+2?&D?(6~=L0%8Kfh%DnChcrE5%46@ZJ@9}zJ{49s=X^RB#3Yl_ z9e4e6&{kn{+2X3aLe}b)46byIxcqOvPZrgM!oUTd4w9oeym|}TDx}T;)w1oo9;kHF z+i4#HSV7MFn%QvJCqQ;Mq%5QS?tpLPx4>b=*eiUeq}7~_rTW+0yB>t@p)We@vXQXW zr&8^TZI!lSumDDMBieP7kHRrgQqKL|)2U=Gkad4U(=P0TcWox6ex+M>b=XG1&T0;7 zWfZEJQ?K3uTJ@*DyuBkqD~`R8JMucNrH{=dT_Jz&0&V<9^(?DecG)9L-&a<*_mS~g zWQTl!r%BvXCLHK#Tsl+IsT|tc#*H6(zxRRuU{-*;)5)2}VIVd<;Ag>>)Kz2Z;h@GV z#&zee^Z1cIVj&T$&SX>Bss_1ppwE!gU}pgu>Jd%?l8iv#pXM{uGo(1HtPjrsGTuT5 zw?M|94Q6`5T7xzpIvfDmvAFqFgES*6J!yLZ;ZQp;AM5x&QlYSqu$5&FxDbF2QR&=U zURBB(g%h^^GCb;|?a_3L9Re^Dbiz4Rx}ANGI#6THttu|6hqeZC(`8oPyu|0l_Q%%I zZN~UmdXb8DxJHpsx0SzkW1&ljsrZCochQPII1U)Os9$_Sve!eobuZ67OduR;MkJ{X zdG}2osnhEU_%~1T2u3>H%#5B4!Xn5i&8t%&Ol0b%diZomYm{8I4u)qaU_4d~=vOYs z-3N^fv(@s}41!j=ZdEV|-#6R-tcpm(;{)A`X&4IFPF`;ln{O%_K!m6v@w|^Si z^3ofybfQi3L40%A1a|rYFk?2#p~9#|kh%?3;eGWEgtaKnG{`NGpB1u9dAdVA)sydo zgo9{xo+(oe%Kbh7w32Z8h~;@uMD|+jlVXQZbv^wf5ah094gJKY;BsItqUWJaz67%x zVU|cp#VVfzRwZ=_8Tvg4q8lBNBi0ZMyK0dgBwVyu2V`7u?VHsd*a7Q(9=hP8?G|kS zfeQ_vw#lc!=uzi9Y6kxXW*Ji`Yw5V``@7GCYrc7oM5q7Z5snU3CI0vmBy4tkb66lM zVBF{y+#N4%_qUJ4+a)OG@+NIc))<@@BAMi9voBt-EqH0FQtxM7K(19VQ~~zG)l$Aa zfVECMp&lZ#Z<$A!V{|`t@{S&H*RccHucgn1mw=GbTMAe<(sfQSTOQ(2rVsNRI?_#JaVehn%Q@Htq#v&8h}usow|Msxh*tUM>Ch^$}3*26lOYZ_xDi9kklXGv(X#gTtHL6%E9c1yKy&YtCl`9K($`!4x&v@Ax|Gd*b33T zn(E3!y@0I4p}%GQpe(z9@mn4{9NMy@uTp<$IS|C8RE?#@Iq~*V%u;c;F)rT^f<|Mb zS&_jO<<^A&RyP&_m_agf z%SS+D6GUYBGJ>TsCaz|>23hFyM1=oVT%P&~%*s&bc=_>XpAVKob`xL!;Q`#V7=|AC3m_b0^fimP zJfn!^!P8=l0)In?TH!9o2GX`k+Y-b<6T6>4_HPPQ5-C1{+d)zp9V4F6u>DobT#?%e z((=cmg$GAVXv;s9jOKNBRoBWxmH6XoZ3naivf8@0LUi1HdQUKXNJdKiUW6zvg{Tr8 zd}3C-?ym6t9ev_FHV)_K@lY2qV?!k>OZWH5Ftxf(Ui=^*g^4gDYm~NDQ75KAUhd`r z)|Lzf($8nYDwPZ4Ol1Qa<7hKUkHZ1N)<2DyFXlGtJ@fr6Aub zZS0|4MLq7l{Td*OR8Qb#w-Po1Gg7Fr;-}j{;4Ph%4!2uGJs&&g4#JkNt3qUoPl!gU zP4(C5AXXA%o%6hU?s1^7A^ORKbD*?b)}g(gB5VTZp(vFamwn{qxu8~GCV5tg44BIm zvw|6v;}Uz;a?u09zn=HGIF}Dw20izJ&xMJtpa&sx>MLN>ihCZj->_h$T5eiG&;+hy zHg!q@j5+<~t5wj*NnLDr{D!azHN|PY4uPE;AvNq}nCxCyODtm&ewetJ zo7O1iTDc(2))Nky;4q^g{eSk^vb%P2?)nQ565{^msT#CqOUGo}4N%4&9_$4ft~9f< z+mfBR5yFN~%$4%mCLgzAK;Lqi`vnU7MwdhPcRCK6y^6`1g#+YJs11;ILzmuu^Afr9SdS|=CZTIj`t~W=;k2k_Y(n(lakW| zIxyuFP%BqoLgVZ^CMORAGkG&7T&q-xGgW3(ey5AUoDc(<9U*MxCK?hlvI^3kGhywT z&yHd)!l}MNwy*Z_QtT-1P}11Z(^$OT*o(`O6sWZboBvn@&ynU?AlrJ=QB4(z`EU+6 zlt>Pr3|Xc?m{gZIKu?bHar67XTMGV)PzXBP>?pZxPRwJTXcZyaQ#N0%f&~yE7M^hw7IC8u#iG?gNEm~4BO%MVTWC}qzCT5mL|DtgP@ZTYJtF|~NpSnoIU7{dowX=shFntP1irkms{ zY25Ma=&(UY|m{;a1WV*}g z^FJ5(2x{w5QJoqGGQHygIJr?t6Ej{OfefpJwnEgLg(h5^Y($vq7y>t>d=@&x?UwTq zIIz(kd0!sHGS$N)c{_kMIpH)0<3ASwObygRge#A>&!bm>ny|tb!Dxx{u|6##MGUEN z0AZ!w5d}AV8X=zIbvIs~zLHqzFN59h8Z=dK??CBT*Wp@^={ZGOU8jyw%&MMZsA9Nz zRS~O)9%snW*ZCN{Ql-nWt&Io^Rb2wCn?M|^tFv#)0XG6zXVfe6v1AUj8jFywH-TGi z$s3!b&S(QC0$PcTWj%3S_PKGyOZIo(^4|!3WWnvw#0`L_+d!VV7bF}Eih1|U-SzhYSv~YA)j;l4z{IFaqP%)aG`-(PitF=Na_>w4 zV<(=)fR~V?AM{9|PB^>-qlh(rViaSIT_N~*mXGHiH;v0?Mf^C^# zA!Weh046dXA}dJE6Q0&Biu8UG+UjOo+EFq=A(Vq&UH1$@V_5eZXyKo8L2Tus*O8LE zMxiZp+_S)8zoB>TKOfTAVAsPrX4jWJz~yxM&6ouMxG%e8s7676)&HZn09K=U27|GX85YROhPY{uUr6 z1AU4|UMTa4>{ud@UONNnyxU8TXho0~*Ow4S?dD^$UUN5GH@60|TJg*cqgAw>2h>s? zNL>d2s}+x?Ch*7(GQWNU85f-5t-{_xCxe#VU{`3Rs`8MI5~r5nX>U zu*pU5s%6|{e)b_gKX<-B(>|_H*pg@{!rQe6h%v9%>uRN-fN`OZr|Z&qK)Hw1w@lqb z->mQ%v53>zX{92Tksj6sQta!aV5!t9PdEm|8VU!B>yGvRGlZ*Cz&|H?B$IWXxfh=X z!eWcXNFc4wfS?)Fd^29dRvL+D^SZ*m26VPieQ|QYMG(ua%A@ah-IlZE(Qd) zwTf9L0lS_~Gkl^13sa0>O=g0ap!D>L>rhVulxmA*zvolXVSAyjddBk6LLl0g4ZFl& zwXoV)hb#)iC{^b-eKNP8rkM|X3n*N|gl{E)PEpjjR_9>nzcT9`FsqwgNtQYP^RZG9 zuEFY!?Pc$S8E5P|-PZBK4}jX0?TSxfByM-L%oHaJ^q+77=qA&b`s{)t3ap<)nDB+i z^f_4fxWsFUSt`!mZdeJ$NK=nf(JF$zHF`-zX%nOFR|4txl?S+s#13ST0+v;(CXLc3eVoj_@C_J(*6q%<-xH?rnxq`LmY@~Jtq8`kL5+a3F3FGU ziDB&FXb{K^KSNvL>9pMSr%z9BL33Je0kFXe3mgZ`YOIaR9)ANHllX+FFX&$X_>`)0 zfsEVgfv8(+22J-Yl^o47alE9oRf~MN^{wZR&B2U+#)O{Tq{&VI)RV=jEZK#y70M&H zY8}0ESCCRa<@GOjxc;xU@AF{Wi0g=Zo9~PPptc6r_fTvpNiR9( z2=G$dl@H(+MkC<79AvQqei))Q=-;zPf|(R~giu{V4>=0NG?F`!6$AbMTtLfF&R55R z&`R8OEI*N8IEK}7GCJ7Db@YC zguK-=`L{|7s}JVFBw?!>&(6wGHISB*yVqY(+Hz6@ryjIUX@u;GL?E|hiDlKdj~q13 z`D8WDvo0ljDP#?xwj7F{SOlQ~V3{ttR;f^BSO9cufHX!JKY3gC68|?gd)e+%NXsp% z8_{aHqY;Vg1I~FFIfgi;j$g$n!^CmI6=23GgENtS8S68uw0HgPALpY4xYhGA>6HM+ zSryj#TdsjLk?8KD3Uwbe0SMu$W5(qgE~#fl?`|^uMgXfxPH(m*Zt^sJuU_`M724Qz zFQ)1~v*m4IRz0(5O2zFUmaRHyG{`gsLJ1@|F66b^$sVcX0uz1u`Kch5wV4c2>ocZ# z48yazZeOUFv6!vJ-mq>aG^)>hy<(OHE0gljV+4(bRJO<}>vIK&hPVG+kNf|ax)8(B zPXbt$Y-rf6D(wbXzQXg2|1YiI3i@)c{ikEoKN_}l6$9`)_-P)rRUtI$1ai%@VAcZe zu>+FLeGbTY(-DSE?Q1?C7DRlA=?!QTmwulJLG1NrV-l9@897CPQbVvX)U&*Ki+vmCu981~gRqe-nv@g&fU;_^`!-j|BmV@W zrP%ZSqCpd$8mrSx{FiU>5Mw220=Z5hYX<1-)DQp!3ZR~zIb zC9M{9xTxuqMVGhsIdIUX`;6Nau}W~A2`1b6L|iiG8d5~gYUiUQIS!EX_VhGP>;vR4 zrNgFSDb8MRF955kdhoCq|Fkm*vaL(XYy0`sjB!uJ z)*m|D`w=+G0-l#X!&Ti-D}snW(mpL0Q3-*%-6JU2%ZY=+M8v zJXiy5b>Rj_7H>_~335U$sEJ)~#o-!;J8=puRC{N=WEfE>EGsE9^H5d=*3ShBTcwC) zVW zJpfksnruQQC~0lQ$ft`kHR^t6>xf7Rd?*|GOfr(5eXh?C>hkeNj zGr;Ksd=_x&Lj+AAyphK%tB*okv3liZ;cQ52MZ4#tt@5O$=JAm?G-zBivi9vJmpu(& zOz457cH_Cv`ZzWCy2*oO z#4tHjau6rG?FelmRo^MMIoWP!u+olD?+L4(T;9q@4BT3hb-~`df`ubgk|r`Qfm0Q= zEZuz>O?`7)FqqUAh)!!y7(L)L>h8S=8aGw?tYA$CXsaA9`Bd(dPCf$eBYYHQA0Hu? zWD1aOoqdFW$E0P9(%Oz}CEIl&94Z1gpv_fjxq_h%X+PEDp}GUWLX}Tc$+-tYTHE26 zmSyB#)QIrq0o@u->rNcXVW5iB=!pCuwhRmq+KuuAP&bwE6 zK%Zm_WO+3Jv=KSB$vc1C_~2mM6Iz0Zh$U)%BHICqf%r9Aj|dLLxVVSds2# zj}FE^C>D+btli|Q$7|Q6G5Z0S{f01|i^lWU| zgMh|1t9?!s@;vGXD$X5A!vN?DNi-CJ5?eQ{zwFg@#bp{CJ z^jCZ&PL5z}_yPdyusD$4w+PBts<$+G4ceGyp{^&`i{J7HcD}nZt3?~210&%-y!`eVJlnD>*TgZI$mUE zOE34SosA`s!e=1XDlxtBB<(&2uo~4Saadp&obm;b@quC3SvcgU6+Rt`Nf*0$C4eb$ zJX@5PGpaxf-3Z3-+E9#M50!{f*kk*pfxNEUGpQ9 z?sfG7@^9;Wd~B}lj@4W;`6ocD7QGigllOk{0KR}IbKi{smLqr7vPvO!-d(<0gP|h3 z1{?G{K&Z%i_Hsuf&2dm56`T1#>pVATE2w3wPwBAisA<-*)HLc{q^FagHTMvvH`WroIkN+hrNb`9T}e5j z%tzv|-FatE+8L-MA6@ed6c4cKWzDlo8vv6Jmfi_%{olnr*8$|*y#RdQ>LcwtK~qI8 z>`vL2pj9V)2v+YO%~Z6cM7RvAOWeZF9-~j|gT;JT{17l>zlt*oT6;}z5YrNUY{6XP zcI^XbG;2TnX*5y`{r3J|Zq&7Bjte5P0oKNel-AvE-o#@lDL#vl|LwpqXushq@4-{fJzzij;hY~hf>LDtt z^ixjuNUrF$iCO$?(%5O&Dx0N0~`~je^E9M*|JAKoOcOTPMekCuoAgf{nlC)jl%bdTIe2sffj3w5^q?*LoUrV1q=jgND`c z1yn8v-T*-Eyj*zKZ3IhWN0v|e(1lh`viO`4>FHn~LvIJLme2=*be3D8kd>L^2^i@- z)zj`lo%g0en-Hm{K6vDh9Iq0vFZ>VM%2pe7T7q{0 z!j@pw!*byJMx-jm>tHEq_8xJl38SN&rQt9GcS`-J0^ul&yCc>nQSql$o@+` zS>R;wuaA5d>SM|`vOWf|s&O4ECHF4(5nvMa*`k0ksdw&>#_`{Sn1Go6+&gJwe+CL0 z6cg5a8z60v>)S^j`4x&P=z^xxCc@SgxuiClf6c1{)Da9vXW!q!~3ZqLk9RP_|~&NAG7RGkLUo3Aln=c zVj74cpk9_A0Uh=L_hoC&1&#zVC5RW*^YsScOKdoOh*yT9rQXg>NHQM`l5X()M@~Qw=C}h=9=OMfv^*&PHL7>J5#wr^3uAx39 z>r7`B_~9fVlaU&b#-0jkYGRH_?7u!%%s7Z*R%GNDa+=S_o}>vR(+Dxps(#6JrxVMH z4|2H7%F_y%9;4NXGkmr+`j{K`u`@x87hP!Xr=&@OKHF1fo$YBxUL3p5g*IMwhILom zdY%VnW$I@%xSu&3fHL`BDGf1ygiqpb2dO{uDHnq=&DIp<1`S$G)x5%@Xigl&^5Z=_ znChSKIfP$f3FL<=Fk=)e4QC|PK91U<+8?BEP$1dLzK=N zyX8Qvt`F$uu?eD9BS#Spf#s5N%OwQ0OEG(VH`ZW|9fNTMt$6m& zjDl^(1DJlYSJn&NTK{_$waVe>LqEhIa;1kNQT_T1pU0a3WL%k#ft;y`=@HLG?R_)U z|M#+DayV80DJcjKf2d3N;kSU9j@97JvFELjmbtCc<#|P@9p`)B-bT=>5pf5TH%@WN)9zNVRG@Am1L^vJkB7ws%dCDF{)idAfNExRI^7eB zP+wk}+V3HPw5C37dqx=6dnMm%xYQB*4UoCBpiTZ5a5gJtMPsUPO}6}D59!xj^wRF# zvwa@)5gzb;%;$k$#NChkf1TQYtu&l)rYu*@A#C}xRn!HRjDN~QJUof|;^y)UP^j_A zQ54v0o(J^9&vCg@0n3L^eshyDE|)I=3OlZT$#36Rp-r^NFk;_bIz&NBr~QpqkQWuQ zw!}prvm|*neI3k%!m{eHXTT*2lp2-e<0Z>{ zF77KefgJV;h}BYW=W!iBE<3LBse)>D{O2fb9ow0e^d2iuPbz90@zDh?v`O%d&x^SM zyCL24JH@Or?1r^Ej0y2?0j+=Pxk2rENaHfl+k0y_`o9Y-_s`!1X%%&!3MIvy-#x^b zrdA|5`wyV7?z*CpwwnP;b?;vxPj2-QI5lA;Jmy~y(8XB!RyN(cR3kMW&{p%Bfs|Gd z*s$DlX>$1Xp!BAyqzv1^M<_jpf;O*i0cvW(Yd>%71ZAA@Z3=uJo3#Wm&e))(B&W1R z&?!8S$9M4&Fj1xChFyJxN<1m}R7Yz7IH)elLJgMGv`X(Su#HyCnkHRzM>C&yY((^) zXGS{i3u02``T>tZ@F0~!mW;y!eX??4XAoQBHC_S2?6E&^NwocX$}**`J$VOGIv)UO zS+jw58<#l;`V@S3hmQqxg|?!iygq``eR;4?L=V(QUnJQbDAdh3$G|$oOgaRNK|?=i z+PVkfP`BK36l!#wSpjIgsaKAibd;xg!K%O9FwoQ5X=T=N(8dXFfZFk6$Ag5k2ZAa; z(I?=9BqddYp;^f3*Qw;_Ax7xwr7Tv$+EEX;3i4+ofK3wM-l4>Rd$!kd@yQnE7(xC$ zjj$;U#=2k6fHL{jC!%yDuHX&>GDUQs^T7EOoj>wS(6ArTeuK|}wDR=@d0Yq12QXPM z(&gp0;hxqjbaM9yXcHh_XIKJsiLw4-KodyN5Z7rUAMw$c>G8J>sH9W3r zM7TVnv)I5Waf~7K8d;_lutaJ*#7H!)5(J5GnM>wsIBbd_uOARtq=;3Jxl^u7LZbFP zdrGmEplK$3ITM7Ze$೟ijeo>0}xRnJNf|R4JFfvQn>!Mj$Guwf$l$b7R)0Zj0$ zSutVjrmpQ6K;wbol$8PNqW2ZE+mLW>FCjT=R@W=|!|I^#bgB&`KMsg0>TYg=hK*qa zmy_48gbwS+kuc^74g;?OGZ7{-IDls}F6SMu0k*ou@e;w7euW~X+V|@%S6%C4(A~MT zB5fytSbf|k(=F47-vDT}#xokz%M%J%Yeu6nxo9G!@x+|3cUt%W>CIqPl)gLRk<;SnEId+e~sAT2Yu4?uedKH;IjJr9M8fI?PV_e`Z4 zDP}+EF=r^${`lcj9xL4tl6fzHg(HhwF*zk@_adkXlr=2F>iGEp#t55)sD3@cK}e;d zi`?-F|5`KYj*1bT(ms0?&;-M}rd9^N1x>YiDo*~@u!*slaTcX(-}8u$m9YjXTMR<} z(V|RSLeTOry~u)x=_eixidDGf>e^+Ig2uRes#)Io!UHTL*-=hk31BsGPePFBnJ{^{WDiX!5U6a-)JyYJ*MVrdHNdwYYd&d^~gc?{tnFe zN7dO#=`b`-L5rxXemv=Uk$4T5aZ<2%5S~!Pl5(b!#E7p9H-a44*(GUtPeJ1(U#!AW zal#KiGkbzcdj8^(m>zZTG2=#rw`=s_g;O>Vhh|D*`MYc*VQa#AwZzr@3T-Uo+Kwdz z%R#5#eH=b~oRD=&n|u+aI|18qGZ0y(IMaAe!Q)e%f*UTA|0QUm;yONkS*Nq5+w4=?Nmt{t!S02pJyCcfNDpRQ^RWGWcrMveFgI@hXcdyzunpmJ?4RWCGDu?CMh>ja$rvw3@D>7M})awZb~XxmR4yIvvPLR5yk|ZdSnJu+4Bb zqSqSZa3zDc+Bcj<9OEh*b6@v~%UkDwS^1PEr=1Jw1=>r_9S&{fnA7?b3K)Z>H(D6u zF7P1F@-x8dba%oCKpo)gWa5PcE!UdJ=+Sa_1R7B=$H`I+TZVcdYzFX7Q4hLTmo-b7 z83VLNi`qDQQ5A?)+I{_%F4?vg$hsug`_(e04hR#|pcKdsE{fRQ!VT37ySxa%>U%tB(k}7ctRrb1!u$4Gk_3@_5 zp^a1hb}GguGHqn^hQ=k5kH6 z#m}6NIn06%B@lkD=YdCn!fuE8G$Ma0V0k(R)j;-s%;!?8A6JPhZ87++rq*_?A|^)N z&y1M^88$Z0%Hii6$keBOCRn5zWYqJ}#tKifxGD3C1wIO!{|Z^5G)uBzX?W%Jdm!zioT#6>hHyA| z*h|l}P|F_xt+se->wBD$wIE0rHR-XQpcSc;^AaVkNPL!9It6mh1|aKhdK;aOw%XA0 zqT-?Cxpc~rD-W4%1hb;_dk-LwD`FYy;!LH#wh@ZNv(Aut$L~H}6}=y(x7iG0qQO5S zA=m%y^Wl;=4=u;#?WX&dqOdTmlD&6;wyGp~E@?uU|C_f32gu|dJ&k)d-y>8y6iYVE z2*yVQd<^FoGqELf*uKT^t)?$pfms7`OZ2N(kyDm6+ov<;Up4j=@7+zgez<%B7&R~NaeH^Hz! z(loewOzI0_MHMsJBQx@?Le{)RoJx1<4{7Y{2PasY915W9vsF2aU}?a#w`6gQJ=*^7 z*-y4R-luKAYlQofg8<-*!wYG3g8x$&{5+E}7@(xoDK+5q9s*+AagyU6=DB$&kS-{R zAb%)ftnwilNuT7S>-tic3MZTlWO+uj?!!GZ6fvQpuKi@wX+Du22kN4_^%+2>dtB0+ ztCTUpOskIK^4Bn+P^I0o`O@S}AIS`dIHsKqWP-yfo3XFcIRG}z>g{oT;Z9k6`+Q(y zC`tL!YlKh7YL;{hF9a|tGx@QR)9K*mivW!)_5ah^uG7Up#+5te4x~-gXT{bKQ_g`3 zSSoH`QI|lbDHNts=XGwuRDmF3+=WKUze>>B+>9Pl?PJ>7P99JsT%K{gQ168xo#r*5 zR-yVV9~rnv>9F%UkE3q20IHFF{b6BcTi`Kw5Qp^Flvs*tQPD##MDBz<6vY%Ci*@E6zrbz4RJI!k)@}iJwos z2iPg_-pk-o05(!#8@Jb_ij7$K(Yiq1YapJ}^Q?5ajIfo1Qu5f7RF1WO)wSza4V$pg zc)TdaLt6iO6((I<2JBDJL5dovhf9?(S&yoI1wAr^i5 zgwFK}hQqFX5j&6<{)6Gr;HHGfZnJ#qveJGMLF1OCT1u8R(zQwYcgHFGPrY>}Nom-) z#(2x=#hR%A)>0{2>a{!l{|pB-`SN=j0ll&(zbRm??cO(n=N9(@8PB1wZ6GtIgN1Yb zp)zm=lqzgmsx)X~!d9wJ8vdh5CN*s}BIfYR|%pT06Cb&o?C4{CM9qI>lm5UWDCr)iK!6bt35{z&CdL0hRg zJfsWqj{;Vbe;ZLwe+JOx6IyYk$6O$706txM_B_JoBH&*XVd2I|F-!kJM*X_T%IAC@ zjBffii0u8m{hKXH@+~!6gQkP`~{PIi0uqyNEJeG8_RgqB1 zb<@^!k&oeQl7XE28i-Y;G>^iy!5Wa_1+j8i*y^Q(Dk=LA%qoe~4MwxROMEhXWiwn- z9=g;=;ge)oFHiXBKT%vaJ^B+6Yc#BPT;b5HM}O+0MN(0N$fZthAEUG4ZEfDmj$f)}}eQ-YlID|1(`C5-m)#h$-3XvuH^;b5*odj1Gx zO`eOZBXkc4?wT;iQK-6L43AC)&^u^4mV$~_ZbDqMb7*gI_&0sVZ~R;@4E@e#KSoTGsEt`djR_rBUH!D0LFVP%eig+U(kp^jgnIF zkB^;GOO9+-pmY@FkBiv7^lRGL@u8=!FKVA0*{?XYw1?m*!XNe{nKNo!M1m-KDwmaRd;&Z=+AsJuTC zFk^!^%~+r8;p4D~;$8+zt;HR|{*S7&4zRPh+BlFxaqU493M5zvkd238En3_up547S zyPH+^?k3sbS{#B?tdy4E?!lb^!GpWIgy8c1o_XFom+!AT=ehHaoH=7>&YW?=<(|fP zMkK}J>mODk=wh)YW89p)GJt!NJZ9^J6+2sjg%mf&cY~m4FVq2}SpD?G z>1zNXAQhze>mh`*GypueHUN2Z(6YojwNK^78-|S%l>2g_X6!!{(CeR)-o@29Mkx*w znoZT%74Z7Gw#%mrcY#=DVV+Woesw@*WHOIJ={^iZ4Kp^j9G?H=N|>xLCkvtcj3#l` z2*TzZXX5e6s7#J$#64&Xr1Ro*ii1+Nr(=O!1C*M{K{yJ-^#IKU22;2&VSNBM8h6^e zW7!RJgt82sV#Z%HkZzB}Mwq+8iXcO5M=-UPK`}Jk!o(bmJD) zzlV;Av$rw9+HSL!}0LWU-`Qpyq zAlzETI$EVW{~w?(P#oM0#2>)MAvG+a@&cjwdJpx>)R^6ulUUVV(>6uknz0Q6a zGaB(VmpnJeXK1FBJNR1azL#4JSQ%|{DTvv|JtCa4iFReIm}GZ$IS>VFr3Bk4 zOlryf+zMAfxB}eFjo~6)y2dt~B*iXQIx_1Ho_-VIe~~t(*Y>LbBpgY(bYVd;2DbOc zMOX7*AsOY#8iB7DVtLduFFoYPxBb4I(T9ip?hcx&`qPTpFT9xP)?S_y;YYods+zjOUj*Zq^A)F;U{@CX> zNJ#=OIp0n&PsPl7JaMnl0rr;rQ4FP zfw-ovY_ipH`2O3%ZVgK zL$cS|QlN7h-;3eg)n7rogsr&V5=#ev4J0k#gcn~DeQ&fMq%zV$vz$fMY_j<^9-uy#QlYGtk@6Q<5Oh8urCIf}I~Dh{MxM=t?Dp`^%}e<{MQSur81PsP%pRT;2F{BeSoTX0DR zqW`afz>k&wsrVa`FKyFcMVWa`^HV&MUq#^9nHo3E5$ z)XLCqC!QwLiQEB$atzeWt+Csx&@Lzn7;f9+!9x(4bA3E!Dr`QcC$M}W(Hg3>wFS656;?3YiY%lBEmZ8d6*NLA zKmFa?5-y~=4Fy3@EVCVmQ{(D5vhw3D0PdB#E{Gw7+W6_+ax`6vY>oT&04SutMeknV z&|z;NCr(nj)^X&1AVsCM+L|renN?5{lwzrIIYNauM50S{p~EViQfQ@fNIh`Tq8!n} zPt2(sGS*d(mn@k)eAft8tsd$FE>n~<($X=Zm+uBQWrVpxj$2 z_1mNzL*A2E2wXTBq$3a4fK@qJ)s3{F`w4OZE(J~;r`rR4&5OFQ?CZa*+X8t65o zZgmh46-+lxPd^yKZIDg|;&K6Q@bL^%#TIM$A)xdIG$8vb=u8TUh_?>|a9Vr3v-%;eVluE+m^T#t6y4 zwwL73xcICQX_09l9$a`}f-RK>yR`&5L)PNyiOC%WQ3)9nj=YR8lEmGTrr1D)xzFW= zI7ei`S@aK=LzzV=U53SDqDh6NdAy@AvW4HtH?J@(H<6>SBJ7&!`VK|B;c5`G#6GU* z9fPj}%LxyUmv4l2DTXnMuXhuqtP;8Rok=q@{y=crgov_(OFJL=^kj3Hxo%u)%&^M})4ZVHw>K**+w7LF??s?ri zYXC;>nC0IE@VmxjL31yNi|9A~@SlAjP%$3ZGaGh0_h+nL_u=Fs^F?9iOu9C<>jOC{ zRzR?zFMfPDLmj?~-Pv$H|&a*)mUgt<&i&3O3~jR0vG9<47zm_Nx#B(8c1K)h#`eOzI)2K`uxCEA%U zgE)PLZRw^;G3He;w@UTyK_!k7=ybbyPO3Ae)c|%SH1cAv5knF{DoWMa-0r z`V5N7Vq*7F5p3#>{l5Tp*4$n1X-?O!&JbA58vB^Vzk+tlOVyl)5(;pPU`K0Cic4dWfcYUC9`WOy6@iM?qMod0KLYxd zP!|^8Vx957m2xZ}J10HbS}TKA%c*0G?FE=4N$=hfE3E=TEt|VzO9hJ-q6u}mu@d_W z^cQV?EHrqRdmxC*nBGldx>;;cj?~qOlQ6tptO`;cNtg@Osa9Pr$6|igOK-cbnGsH# zRj_A+M=Crs@yEqq)- zew=> z+X0xue8Jt!dTid^{%9U$TE57SHck`%)_-WjAgoFxQkty3R!w z-WkL-9Xm9x+9gvRz448xqQ}uEQK;kbdbf;e{hF9@r!X_aZwP2)I$-x4k=iD|5wY%G zU@juAbep1Ev@4R%2HdHC6pC=s4`Z&q37bFid4^5BfU3yQPswTieH|Zxk8j2imdMG` z)XEJIh1AMmX#>$nDiwWGLE^Cl?28K;b36uqv@GMH%@fWPuATrzad=xXerhFbdBmqj z{H?wCtCM$S6gF!ydXuTze>!Ujml7C2pA_N(;S~lSbzPaBz)~#k*&n(P5{*N}s(054 za4|eF^LJJ{@#P%n_JF!9vg=%GBBUFOONBbrY>c-jffZvDBW80PI0b-eCzap32jqX# z(mBQ+1mJpet8NcbRne45@lS{NV@_>2$2=SYUbak$-xMsm2R=MDT zYH{D0&@P6Z|5W0LvwEXl#8kl$$*{TW^mmrh&j?t@mQTOY`5!Ni3S4%RpRgy6N!P=yGVv zAm`-mSLE0g-Rr^*`l^h``#)uVNJ!BE&_JmJta>$&<%9dR)m`&Qh$tARQFRm1u7cFv+z0;q^*IL5PqxLF8=+~A5o2Prn+TdsJfX^cjvWNJJUr@+ zlvQH=n}N(~E~&5=r_ui|+X1LAYN$A5dXA6KB)%Pe2auCwDC~>ZL_10L>1iCB_f8;}o1Ry0 zl0(wR?*=1PDIoW_m$3P)(UX%&HOOc8ffiGN&bQMsfBpS{MJsV4sy@*t$dwk4r#=Ab z@?cff7Slx+OyHP7wU2lX z)E$s9beCCBh2ar}Pph|d=yBLmJLVN@Ws(7~_PzCdQs zyS)bBv}r_aS5VKTn9ID*AFc*^fx37|wDazM%-0qu4I- zES@mO*JG9|z6I&@xjfh!i@uXTLCvfkL@C~`VT}K+JVd}dQ}5<@seyDn(k;v~P#kc? zY3~EUV^*JW=Ldw%&91H(_8}BK9k192e@w7i#}T7q=%>&wBDV~p>odn3IwI~;u&696 z_tq~UNnLhgzkW$LCxS`KSJ1AzY=IcPD$(@~nA$)m825Y&K>@h&xc+wp-TGSY*Q38c zdpU{vFaVmxJ4Ko zWxaL4`~-`#ac^J4!BjyWy%;gtYy|PZLV(5Cs>?fU2LYV7yo$!*HJTcXijPFQsn8H5 zeG9`%+ftx3O`4>7m(EF`^+1ch3^Y>0dp&XUviUz=;+Yg@F9&Ub;#eWR5a}}3%hO9T zmbjdA%vg60P<3%GFJ9^VOa z))TmoK5KPIbzEh!{~8cQSLux>)`D`Yp%h1XPnOFLb#xR%!();N@^4^NTda;S`A_MK zUq%otn9G55U&KiN#|7${3YyRGbxbTi2HG{J1E5;bp!=;rm%6(w1q_?qO@Lf^U6w+K zcit4ls~4W&Nq)4`^Flugrw%PsRNmp6J&q9OR+9sjV?=r#Ep?Pm1EiY2QLuAkHcA8eTeT16UiI3Y zbK3a3@*MZ?-N1?hsW&h2kDQ8xl~FA26Xd$VMph_tn7{cRfMyXVY7AiU=3XE!R!c`S z&h3~^m}@WD8`w3(j-DFsun&NHQ}J9L+nLyMUr$`!$vcTda|&6lHwMngA2 zXM)XMa<~153Q#dr>Jdnt^{v+fmPV4cn5wWdYwD%_V%v!TZUv4fBr9z~d=udR4evCB z{xMS~X~szdw~1Yjg2wo0D>;A#vkQ%o*<`)S1}odqbB&_ zkQ~p(Y~}dhp+K4Q>=ds!0-)G@P!vz*2BC*+iy(99n% z5^mrjaKOm~i)NRf!I)>N|EqMxISM+R_7*H~@I7@Z2yMvw8*#P5F0I~YZaB?aMqUV1@Iv<#Dsib0 z>M(pPXHq$SBi&VpFWX)QWY)HIvOQh*@=Rj~Scw+V&LSPm(wzrf0pvn!UKkR$2yh1& zv0jY1GJnC|8G9%!%&r1*ChQUG;tkOhi3QrLR}*x*B!zcca}2u%&{Dw@wcZpid2LQZ z%bfT^v}}*FIp!jzY{1=?d_OaCBQz!~h*o_J%*nHpVSDyCw7UqNS)%I+NOO&)D%X_d ztKma|P7Ojq1U^zXu}Ba>c@3qc&|AXESgpOw7lg^U zp&@Qouo&g^4l3Mzt@xKq!tW{Q7LvNhR^A)U&R{yf!p~AEzMTVT_UKfsK0f#!fc!dX zmZN{jU(lv)C0*{1w*{L2*k_mHkH12@B+2}aMECg(sAx`Dvf$jSS4)%a?xItmImd2A zkM=$+#N%Tzut=`@%>(M>QFoPEQ1oshsOW6Ixy<71+kSxNX*vqjw@>{6%tb|&-xdqa zpJ`SWt#Qu+nbvDQvDbpo7RuI1vCKkHMJqLPpI9jlUKkAS=vAwxMF`Ux!|USo#R+<_ zDO`W29oKDw)pqoU#}pQSPU##qrirGs zTywc<06}L`nple2BB_&>F`KMJ*vYfxW`)szWdIjEc_`JIf1y=!q#iV~tucEbK<l;U8jWFHQ=BQ+U zIj&@DzKtc<1!90o)mnEbgfl`}8>iPnx;?NYX=>(TJ_Rnt^}?MreH)usr1#3<`BRZ! zi8Kd%(=HYq1p>=3eWyd5Evn#9588H|{EP)r)3J(LY(2sf-qG2rG-zJ2*wP1WSbekNTIj}WG zYzOVcW`2)| zo^d%glU%yawOT!p8In#P+Tx5x5Z6LtD1|X?Ij9Wk`l1`-7BdUx+iMu(uBwX@%keo2 zW?($1Vu&orSy+7#Khz?@hGZ2utBFW(Ni)gm%??kYHe*@|n?pQ%#85lA&3|(K9$T~% zcKvkx5ly0t)|KHc?>csZ!iae3{`oUp-F3wM-Ov&N)-hizT+ABse#=^F@g5*a z*YFsrVCs>KS))W0xp4rDfb1-!D*qkPERuBF*%>|9>n~wSN7PM(D9Wv;(0&kI3|S*$ zU~?cScp z?2f4KnA+-G+e zyNE2hB@^tk=~x97k8ic9Pir*h*L83Hh#S?ifmr)Ke>7@r zyr!UeElUBN(4TxiklU!Etb>0T*70DDhJ|R-?A}p>CUwgfeJS>Qh*+)!izX6Nzq?qV zMAVLBbZ#mE3Z*!!7SFFwX$-sRcVMAoXNYY>wj1oK>{8Vdwn@=t5OJXi|mpTx(QQvy}-=2m9yD4NPh> z03I|CVG2pttXHrY?Jy;X2SgW5H<=A>GKX074}j*ypt|^EUMNyhb>}GTewlh|{3w92 zk;Nb0YUYD*V%bwg93#*TkRBQ3IADRC2fpSwHPJ0g$KuIvg84(LEk^l8o*yVo*aCC*wU;)k(U0`e=5*D>gWFJq^ zwcuh2hRwP~5laEO785WyVsW?EpFmt9ox=@@dzJ<;*GQ#dStz)stM22L%TalEjG^LY z0ZvWt%poSHEe~Qb(IXa;)A?7(kz|O%HRnf#c<@gSdwqOfiG^1LcTsc_!`=EC(12C} zH@*VQQl1~>fCCBX`7;>hW!cT*=KGbPT!3U7&3N(5%0LApbPS(tz|USK$Ezf#oy~E} zKp=R>q0{z*9OUTwc?B(L6Lk9jrfAohi+H{8v~4vISj56LUQ^h4p?;r)Y8Mu7ygHyu zjeSU3Va0F4%w-NdHe3UWMrT@UjRQou3OHirc5x-{734y20@55Wi>9c(_0cgT$CkqZ zO@KEFA~ktvjyWQ#a!a@1^7K0Xc+7~Hrl9}CnhDeHctS`apggnM5nHbd;Ks`8F*WPW z!d*&@)av=42(m0PAH>+9kWO5@V7z}oh!bbvMEa)<18}`Ktm=uIMVnJxz?B6EJG)vO z{0z0m=n=&6@S)>vjZQSelB04;aD3tGzoNx z)5#l70mt}@Qg=*JP;!>$y1`?iimG($Tzaek^CS5RrZbe+19I)SmDLod`~@08;a!Z_ zW&QjgT8g@OPz{#w;pK>9VVQSYaQCrn_`$9!U*e+M0ig z0gqcrZ6e_iO~PpSswfw=t&?7`#kSC9ZiPog=*{m5p*eUIWSQ;qpR5PvYH@iXPP?0T z@7JkCz@J2gwa^YbEP{jJ#s0&hqg-Ilh~<6i1B%GE>tog8f;X*8m2;aJ?3vDr^R$vSD)irV+pfLu@pq zsqN(o0rI#=i9hJ5x~>zyl%0gM89O&QjGb;gFoCeMOP6}p`JNSK-s;fo6EID;T62B2_cIiPjp+{WxwW{B_>WAo^rU%$7@jGK|VSUA(r{ao>5(8*~3B1G2X|n#2KQg{g|2%)Zh~cI?1}RC`B7i%-;^nanvhb=do~C~!F2CF z1s&hUB%JFCq8-|q5Pv%-e~ax4*6wk~xgd0u^uS)+d>)i*g8nGBKOYj7u{InT%Uu9L zI(h~9fqxS=Gf;c9#?=>Rx}nU-_~U<}U0dI3962pV!0e?GGcPx-rL%0vUbzCAoYGu( z+*J^+gzs?iR^c^3mNCZOG*iop`}J#qs}h3IeI-u34#YXr1q9p(K+5!o=D6x+2-i~_^$2}o2XM(98Rc=# zHg3*I%HOyX)YU8Xb+E(ii_PxIkS_Ua$ZZ$osxdXlCT0D5b0j^1qDaf$mm}dVLweo$ z9|W;jRi6;RqV%EsC2Okm5Xwc5<}aI>s7Y(=3=r3JLNAW4_Ymy@a0;zm-vvU4B+_{WL1UbtF>_6j+C-dhx_hztP z`lyqkOrAq82z}1z5 zusB_`Mo1R&OTR$a0+bXFcJ&pbyt(YmJ{Z-?`N5nY_g15;KYIZ>|V3gycH~7a@(jb25n~<)<#7>>FUGp{oOkma2 z#`!aHv(7spWY^UkYrdCb;{<}D9Pt4_ZJy%ZRcGw@p;N%`4DX{Ug_ni64wONU6USGA zNOV$5jQlvK;oGt_(e|H!xmXq4FZRTMPXWx@x zX}_Gz1Vr!(k;OWL8`muRsqZR(05k`1*e2oLPe|2jofCS72ZbiuL~Eg5dw?VR8F{`Cl2R=yJuNMS)x(ahD=C2=NHO zu3QIuNqh$`R{OR%KD=0v8I=|^tVKD59=#MO@}M^;7#SX2I)CTa>DVDG=NR&+gdNRC z%R{=sxwDGp%L@7LIxJ=z20&7NX2a1d=6{B9zjJSq{4;WRoT8waB|poZ@y1F3Rlm`^ z6U1S0gCC`SZDsy(cAV=r$2+3^DYux=Vb!eyP)u*d92Hv&BP(1ZEjNf@A+j8D>SST? zYWa&wSr@;NhP8wtkaY;;)HU+|v|L}3r{xBNx!cH;{xFd)Rg?V9KPlSn!T>Kr?PJ!; zF&gDf;kO~sMU`>2fyKZ&If5?c$B=cQ3xnC@09IxjX@B^!{&C`Z7M^dpSgOdSDYK4{+40CmEs3?ZPr`%@oZjs~(cY0N5|7R2piYss-Q zVvX^$5IBt4#wC9t=$^yYP#0wIg168H`3ss;KJQi*;%FF@^G1>;(u)MTRGO}({%GG2 z$gJh1z1G<3Z<%h8)yf|>`Q0b7IWNchg8WH3I{H?$877CNH2I;M<&WACbeXKYEr=H9 zOf@#yf-toiJ1Tk=EV7q}`01jZ7GsM(T<-5V29NiZ;gqGKlk-xL=q*U~<< zxP`tQ7HqmNkSoKrQf79}J>mv|{*c#DQ8w&74!}AuyD}`C5)ZG>Uo+}4i%1y93#2(< zakMB*jWIjfrGcO;Ak_;Oa_5vlJc>h2Ekn6*tS`C7(h-Bl1GzmalUjPPOh`TW5Wy}p zx_IAUi&4#hE^|ZT=l2r;%*6Ecl8W|^Rxm^Y+vN1<`sJei5t7Ub=>2v8H`wr@L*j%E zNSC>Z{)+7H-UZ^k8Mc#rRN`%c)Dm}YPxcUWigLn+{qH0IxWRqp*tiI5RC!!sSDuHj zm!AUVd|8Lb8wWya`J-X+ClQ4-a3vAL5Il$+3??2k#a^qh|EK-1RHh#-b|{#c$ucrM z0rl4-fLs?wN}fcBTaR=iGVnO{pM(n;M^(S%asG=fMr^FG|KemQP88{+Thg<_xFbFu z$d%?kC)%<}Cjz)gJST2l^_xH;HsoGTbV~4+5xZBQK|a zb_!uv0dZqn9+#d9BF=Zlk*7JL?n4bei?F*sh6r6J zxwLa1?iE%H2e>A=6^at^ zJVNYyTmHPUk1H(kvH;gXTQ&M2jkd;gK$i;3k`~khlZCic+%<$5OWy@Rze2?_Caw|T zHlxvb-lIm>jjiMVyZK*zj1}Jt_Yij3FgsZHUMOd;9>+d|?5`A9496NqlU_5v5o*q% z7l?K4&*`w^eDnb*mmnPk&VCfSnl47g>0;x9GXOpI7DoV9xNr0rIK^wh(dRn~BW}D% zwA$kYUB%HcQ-l*_R_bKvU~KqBpxam0cxYbZgqc8v)Nq>xm#k0b4~tCMT750hvBuZS zvDK8P0L+!1mN@tsC|9)${r}&eg`|A-lyAdXgqn!&MA2qlS4;fxAyjp-#uk~uj(tlU{1LF5ucAxLNz8v1XgSltLoa(o zHnJ;zlB2SDYwMFrHU@kK=z3Y_S&4yy%u`-@X8n7U02tGS4a%1UDHDzH`+GUOWK6of7OAa_Sls-RzaKd|W+-S*GJ?}u7ySYN zE3~ql@oP>DSLQ4?)tnpu21p4hU`zaIp21aSReF=P%nTte0P9L!-lijN-4Du3@sFROudA9LzlD+#r4y z>Bbx{BUxNtE&*a`((813C}LMZmST?H`!5Ama5Oz3v6t7UwUvvFNU*EJch}o+>X}Zz8$|_(qlQvgzmx(us^#Wlf-dzj8RUXE&^{{mzsq&QGXdOz>S*ImeTih(fA2TvC_&;0+;K7H3F3i`F z2V80`KLXfk_MnJ7Z)E8p3ZQUB3(Zjb)X=8s}_Np z#AWLffnv2b$Cev8$a~AgaaH_ZYt= zVM*1P7`hcf_gi+P9g{e`EXOo~#6^!BqqcUqwR2(&-JY;>sbeoWYX?Z`j<4wEn7AWE zktGTa^|?Vi0Xc1%w#tkmt)xpg-Xmwg!ep>y_tbZYpu(iL@f(83(08CA(a zdqKKTEMKcxKD&30BoD7E?~^0Rr59aFy8Jo&f|?l>+5RC2kL?F!iA#NRk#W$p0n&MT zeU6P13d_uLd??7}$BQVh97cB|h?&8DVZ{3f4`Fpiq99N@J`Zve!6KenCXFGFUCQcVpL}F8cmL=|- z1Oy+)4vU#ZuoLfnlR-T!uq{G`#05{<<%lW36p}4m8n<2-T};LZK}DB}OEKes{IL$G z<@-iQz;_=4T=X%HNVhl)k_yU0-hm3cT#RXQa}@&)2XdWZs2-K+65>cr_1GXYQ5+)3 zOk?KfG_e*19RvHDN}O;cQPfw5#ViGtBlpr?RM_%c-9*IC!dyT!!m;AfkWL(@BHX(= zHh)qzA&bev9H*p*f_@e4GO~0|OT*?9fZWp3Voi>Jo|pliu)|N$uEludR87gDi#~Ou;d&m6GaoKqtqVFl_>sJR^UG4O2WX5@wg1V}-K_S~Tq0 zGo5H%cs3}#h&|ZQ_(4QbA5M34zvUM}6i)`%)6Q|2#b?~Cu!Y=J;#ezi*~)amplFH{BNa`MeNttK*QV~S>`Qs9Yof+aN5k9&=hsZ z^}y~{Qm28OTtnz*=+xptA(A-kMp5%h+jBE2$oD#fceg1F4s zOZCO-H$xYzEG=tg&QveVanSql%86(vEc<)C2Xx3SK(qkHHnI7wge^-r43X@WI82@@ z)CucIOouUXzA(PRTs%fhCv0Z7qlP&74oIwm2gO=<7Lm{rd{Tt#@t3geUEr=u(o1y2 zmqJ`F3=h#<_QgN$0kXJMy)af{dJXJsz`qcNIxNNo`r~rPgyekUg?oW47hN@%g_T(0 zK5*yEk#IfL|KULpR}NM)fWo!49|BY5m|S0fgfMNS-35LWW|)$us4qSeMT=pQa{prl zU0T_S@Z=tyX_LnRi`-ZTkn(&_l|7%0lv zv!C%4R8{Qt-Z!b4JMdlH6n}mL(iP>_h@U!|_Ga<<@EG+D z!KyeRO^o%O@BR+bQl%2lz6a#`vPbWWYd?f`eJlIN>!Oknw8y<45q8~WoVwn}ko5nw zk@@0N2p5GN1!i<~?7O~l1hhdNJX{0e!qD8ESRutfW&@Ek`e7Xp#ZjUiWfJBklAVjb z0kS}8{J;|d*GOXypjnFN4~DYqzXfpS4LtJLtV{Dx2y&5H)TQK!~t01*aPg)%F69hc6$J!r6S-^Q0j*Aa5>t~Rn(a}Zp z#oINYaf;q(xae166kNxeD|Vb~txQYF*(%z_$GkLc&wB+q8$Dh?q|byn8!pyR{%>nQ zQpLwoa}$FKjAdgu4?!~tgBAKFEzr{sOhY)&y~nYC$Pw_-i**3P`1pKaE@rtMo6cFH zb5+x|(Ld&|FgW3YD5{uVVTFn*JzmTa3^VNE^8;J5+8Ux~0VpS(cI3~CM$+VKV$6d1 zKO8gQi(WMUv8s$`79w0spd3Fie4|jfcriei3p-_L>sMSnCxHf`Csq|rRapv*jL{-I z8c6NTE7KCTmL))|^35Z)OF}v$od7EuPpt_w7pt#i#P&;NOnwOZl-9<70&{kw;x`d) z%r3?NvFNh^a?@dCNzWX#G=OV>F^djAeil+lFl$oX>YryBAexhT@{ncozg^m@A!NHP z59G4R%Pl+oO8l|Ezvyj<%@uSGr7{`_Znq5paUqi9fOuYjv#ctBeq1p_n1iOpUj89?vQm%}<{FReFyC7>N0NgVuF1!Q)pBad$*lJj9YW0m zty;=)VGU?+;lsE_fb%OnxW+fD13ABRD9G978X#0&zIkH#H6dIpEVtqzk)sp z84Tct(&<52mW;dn4rA?=-tX8~7>VjpRU8fERAjX|goj=MT!{3(PhWg2#QZ{G)GO^b zBk!F1_Ab8&^n40~(vA1D5g0lQ+zc3x24CX+>u?}fiLEw0 zbb}E9nzcs9Ulnw841TeEV#PWNsNk1u<9Q5Qh*>1pD7%k_ba~RzfegmuLV+$ghSsc9 z&_3-n1`L)oqkoGbf5{Q5PTynE^}#%>Fl^$+dV@@J(lRzattdP^b5Xl5$@GA~1CgtiVC!v7xL}yChazbE3Up4G0gu7z zKc}IV*Hk%EkfU%6#G8xTLDO!${MfmDj*UiGSEl0Goq;@J<6d~#U7%bwR8$&Ne-`4p zbBf3lPLM7XD|_s5wBUbEpug&F?(1o8k5hLCsS1c*59^6D1W`5ZPd`)GAC$Ce?uUUFO$ZGHHWkSDJfIpUUjHHho2U65w1osE>2N+Yqfa~h>2kmTR zdb?sogFi^-PVt2(GQ{3bwioj?Lb{6SkyiE#yjn0G&=u0zpE}6ff{@cTY3q+`f~fX) zZi;CQd~pkyi#5JI)}N45V9AP3sxLMdWScRj)n1vjVuMT4v-Jnu7&{`yw&u_Lzz+7$ zZ5b-sAm$h3>^VJ<_eTuU+riAoMs~PUJ0S}Nj1GxuO-&F#2__boo22xaxPSf#v5j{{ zR>h46Bo^w1c4IVRkJ%j8R{$t!OGDK45X`b86SOHp9Gi1jGW%{KfJd+7UQGToD)H!~ zjFveFW2lti{bzDc4u|Tx|JSb%#FZohs~0%6j<8(7(Q)IHOk>~C84rs#6WGtifP*rr zXH#z%=^|@RVLF`d$(}FRO*yfGi3C)^?OyIyC%dOZ!HNNdf-!O(4xj;q z)0syUb}2b=D#t-b_&+h}vLgvQG3}K(nCF`IQDEj1^AV1hlQR1lp)T!shB8*boBb2S zrKLq>c9~WLj|nDg42EOqF%HW``|}Dr+2lHe8JCgn!DB&PINb@2=R{j7(W$fV6$i9! z*z-94a37Ed0kMCP@-MVMt814{9u z0QWrE*!Y;M-$^+M)R-ew%-5O0*h2DTe%9N;6*(h9BF!85Ur+}@WHDc zX1I=(t|9E4@pxE{o34j;9d!3fIzb_P$xbm^Zy@Z1>roFhQId}YxtVY;g-)8x7QG3~ ziE{O-GaeP~&$#d@Rc3rF$Q3}#%38a8O9n8A#ZmPLLaTrITrcrbs;Xd*MyJs!%Ravp(s2*NjICYXy*-`yB< zKM7eBK`xeQsj?t=AZIJ{JeB`uNh}W!u}KZW{S%XKC5{(Vj8S?=i>;5W^}wu;nwV~+j>h`>nEVX1D?$gt!cI%w)t&`(jd3CGwG<@MHU-^{TJ+s{f=-T`x-a$RicIpy8_>BN?9Jydwodmt_<+uqK2?0smu zeP3@heL&EK;aCaQpC!NbAYj1}yiGEy*#N7+|t@-}<2sR(hQTd3^G*qSS zum~DLJHR(SA?!kzuyWJIm`beqIbcn7R=5=Gj_tnihn(od%&!Pj4c4gIzQxO;J^0`N zezUI~vyGAQN`+ksooj0{sUhR_mU?&05mFsViknVYTYZzioY1R>U8Apt=rjVk8$E|O zW+&$6yf!M@a>(v4juz=QuU=+}Pldq@8GyyeZ}a!;0EWhPh@Ba?U`8A>T{-s8iSgo$`y;Ijnq4MwZ1GJ0N+1gf*-i}wYe*)8(#2Vu7KNGgZ zCr^%i)rD2gNxuRkBV&fgPYSxen1+mx<$r^A1M*aY%r!7hXcg#eD!u7#FK$rJJ*0|h zZQ81(^C`kCP3(&&%CGl`d2%$rltJs9_XjZNq0J_)ieUT-^X7flXCnC7b8TRy$@=gKbc3{xy zD6#aCj%u>tqPJjk;cF2`3UsAeWen#@XlSxymB1{$3 z2~1)&Nnc<1&mb<8o?C2)vH(ilK>M9m`;#*XP6Bpcg+I++Y-Oy$Uv^*(!|?+{a$9UT z2+XzdZ9Cp26I93~!$P`9caSjGkdtHP?AU%)5SZ85DlP455QsfX;=0xI*K!`gP=!W9 zn2TOwq{eV!4FHQ9CS6QRRIwE22z4~tl9^`FH33{WJQ*cnK2VTHydrrY0X7fLQ5jv* zJ&8kp2lbTMhTt8(2I=HSnI5nfA6A88*qB&)2z0Kg4tc2 z;)xN^5?P&3o;ec2MV0Xel_`$`anmIok^0O`Va^060HoP78o*pl>Sj`!EvTrp9{Q=o z(PKbdEAFd}8e5|}a_fx)wt^}OSZ23Oz0VR^5&X^m|~Q_fL!(O&xt!Qt7TZTYL{PWDn)i50iYpO^G(+QXs? zMr+LyF9>jJ59dv(8RH+q%83?kBI9F0MZd2OnLq9hX7=Jy zsByLJx&1Px7Y}tJ7Pks>dX)~D$d%%)`W&sqsE~}*{?q_wj!a5s z^y>+5IU6Wwtk4KeXJ)+tyF~a;870OYC1^P&ciyp&`^E6XVKM2f8qHc@-9-5=baYDBq5PkZH zjNvz@y-Qk)`;W|@F(}7%M?ty*+5>uy`$V97FrGALi^}7fj+G8P8pN%x^Z1x9n)rV2 z?PpQuymYlZFnTP2`H>!YMvON;}R_s0_mn-91}il0w}EHa~PdNiy5NkC2rKiT`ofK#AdK$aq!IR%*ONR{3rO?tWk zr{<)1O&WK+eX-hU8N%v3Z8yfA4&=5$XOPaeVq0P6YN=kn<26}y3v@2kX9Q%Lccx&M zMpr+EMBf?tJNggg4| zB>t=OoeUe-So1={mMD&$j}S#m)=!C36r}Z#a$RkBOQh>0ab$V$pA0l4oxuzNZf^Ny z-0XjlE`3)or<$Fy@+BZHF^ju~I9)VZV*o~%xKfleKtYa_EjA56-A|k14@6XJJ8VSE zb(R0bysRhU8VJ}TN7koa>py6*5pjhGFETNx$MhRxKi&hq&e2(K#j-az+{CBfDQwx` zecH4`Yl;PL1f&$n0DF!ISCLEqSR+BYimdZGI8<&FVqw6rg*`4WpWg~Z;X29fV}+fA z9VuxD`O|d3B2m_w!$dk!y&ggjyzUOC(2YxyrxmW&D0%3UwxoP#M%%GTz~)j;4Vr~I zB~~z+7C#W8valE1^De>`3@-nnc!%Ye+zsaPRd{vMy7!arseuSQ<{U`=}Rp=0xh{YR(#D{Uo$ zB$z+qkVgo+c&v6i5MO4=+XND!2g|MAM+v*eXjf!-+J6R!bMrOwIm zjxhHDF6H-s9Et>yQ5<~`u8JLZY)_a8?c{jep5^*00-T&*RwU0~g%zc+l4kBFYe*^i z@Lf%a6Jqbjz1#-{(DN9|o>$O9)Z5CrE5k$or-3XZn5D#YkG<5po zr@Rg3GH`#8Lx5>DKucyj;BZjM#J=?G z{1GdM&!uQzX7;$>}ocx4z8bT?dn%0#^k@w4VeoxGQJk=Z14)MYk$Yj146p# z5KiOibODrVB4f%u3cCzaeScH@HHqPLpML!1>^av;n=T4<$h@Ggu)NAi@-LkasLJhj zF6qAd#|%``-L;_$0Jumv}oI83Bt}57c*1X-;?V^f!5k^Pan;TOvMnOQ)-b}f=(>UI4}k+ zMP%oOp`f&t#|Uv5F`{UWpKCM*5~Jej;D5IgiO0mHA?ZQM_~x@^@_$^%i^Z4CNhSAz zYl=2Ma0J6?-Jg~N`2Wj63B|t4=XjGiXQT`@07v#*51)GlVwiC}^o0L)nxsKk(S13j zQC(&Uf>$_ikEQ$n{%?HD^u_*yTvbkInq#RIp$o-_UTS{$@yv?~=~f8Cs}?u9hQdjg6vi6?W0Lb;;yPMA)9liR)l zb^PI)G4QYkY2w?>1QspM;M63tdJe*oGWI&jqV$Mi;7CI1Wd{z2aEDFqO*&A^R^p|R zpe}l;6V>D?qo5g{rc8;AMiW#gWIctU{d!OqOIE;oNNsZ==2Szn${F<+0Q0+^M`rax z*m@i0IFn!mM%SCyTY{M{ z%+rX3EEMkC3J~eidDV{F5Uw`L(6MoZXiK)vf;t&>wGDf0TYsYS!v5P6c8!v!wb)L8 zD~)|UMzc{C;#4Pe(hm+6U2qj`RmHZn7wrh@yozJPG&n!B6R?|Y0&BfpcZQ^SSV!Ps zeOCxqn{)SM`cStUkgJ$RXHM$cV!_=3T}Cb)Gq{WrP>fETnz5z3bFcgzL$mf=@N43} zfV3v2o4TpBACya?rQYaqkWN&$jJaYgz{1Ij3><|;DJcc9E_k;92M z>w#Sw6iw;%mB#!rDtBg1+WbEyFr=&(6Voz5n9!uxo7NZ);d0bt;9ic;1<;^c()DX1 zY!0!rOpcqI1vxia7r~y#gz$kB0-r1Fwqk6@b9FPcE5*g8w?q|uqf?RlYAqnnU?StR zjxF{R#3y)K#HeykEkX~MWaNV3z)m8#jief3%44*=QlOKjDQT1VM^{cdZ9|!Pnrjed z;=DY-gBklfH@r=8h2}F+FbUVHvlSE@&@5cvO}Hx8!$!rcqMex@SYm$Iu>#`ek`WY} zueNxv2h5zMmoYZ?_GW-r&oEwmUVv-Qs|oU58%s?Ds?PD8O|$-E70@m;w|p9HG>#6K z1Y9lG&=Ju(8QLPoIz3(%2`5Jmk>9g7MY+%np)#tutS=*oY&{nvj-3MH0?DO_M)WrY zxfX1**kBGh0Dx4JsZ|^+%70_dPoj4n2;efYAsrvP9Rv*@wdv@phH=Djioz}qD_?E- zcR3uyveMNU4~z2ucqQuQ$Y=K>K%5m*k~VQi0;q|o>zPO8KXF{cGVH$s%nLeqd@Iu9 zwSA>(bfA&mKH6zu`4s2mzw3x$WAcS%D z$zVmlgcCaQSm=}t>C$;Sda`2$nPGa|9W)K1 zNS;LlSCKCTa9WMJy3!GUy&S|6v_C^<%lKsS-|q@g%O=I>;3lzm#=XL6uKJ0w$(00M zQXFe>K3|R#1UWlqGp=bLcU1;3TxZlc{2G8_4yTKy6NjPK=7d-Za0_sJv|X3K<6MFo z?|D4{wPJb9yJR=yM06d7ajQGpZv>-Y*qFA>-IU{3nq?#dBfb>oHmv9H z;GJ9ZH+WDXj}@i^m@Qg@Hn2*;-SJSNW*^-}@xHtxBT0!@iH+|DDdvK9nZ!#)+9J>& zcAz_CZhi0`5Vsp!|8jgHIyXGax1QMM0U#>PC4|xO!h;YVitxxV?ja~=B0m|(J7e!z z4?Dgb7{wPN%xU{crqahflE3024r>{Xq~cD2mdwJJEkk^pVdsG5yXJM~((@iCZXt>- zl@6Bb{}F;Ha_utaDC{iLjeDIjE&T+TtEHWliusZts?}VE@w?1~a69Ngea9yuNe*ui zar;va>jYtqrwO}FwW#Fkser23!{8auL6fLf$8pm05SCuNa<^2lV@osg9P6Uv(Y zT3-M_YB&`?Q9%;W4l(+_m?KIL-5URU$^UZYk2kYLST-72lCi^6*Lqf{Ymc2;9x_$^m#u4aW?EPSdPhT{-{p@%^G=TL-5Mc?=vuG$1qfh3q%(m zqDSD0AoaNSbHIYrzQHhsS_hO+BbwPQO&9Sn&$wxklH7vb=_&g_@I%4?1jE1>;k6MTolEiTi%d5X-vc zvAp58oC5wx>m%k~r*9(%cohigG#Sb1rSa z_7!G^VHC|?f2sf%yo0L8qoSP+yNME$2?O^5^8&gB*@d*Q8Ial5O7rEkc!iJtHBNvl z%A<7TSBhH%!N&&F({C#5^2nzGIwh|8*3!U96K*E{{6``o7WkOrU1kwZkP(1n~Yp2(=Lku8bt3ed(RImPI^-MEc*T=a?mSip=*nPb zp}Vu&49j1YdOPC;kySa<=?HVydBT(lSDSi=6L#G(tQO(oGdB0cog@4w3%I)YRD|=# z8C64Py2WzHNI-LdshKvsvo*i;=qiWshi&fH!UNq%~}GnO6=;ryh$gRQVZPN}_<17OXZ(>DYwItJ74 z2O^z0LjsO5MLTor^WoI;8|6=!!(*aI7^kJuYYMwuXgSiFt7l^%bI9JJHEo?DkP6q0 z4L-JW6@>>Zg^Qupe$|P6U;+*^-ha)38xBn z;u?_{w{O@7#3iN=;I(syeF0n~+Y?t}h9G~14wMcayN?5Lmc{tm!*#s;At!k0cx{#- z8ccV1XEx-&txLryq6Fl0Tcr-_ju}FVs_MY~fHJg8$G!y#KTLp2SK@d5%*j-_#F6U%%zi2leKI7wYk**X6?a4qaeiY{Xv_I7;`~O;i zT&86B9=o*WIB2oyIi+~5EywAQ`;%yIuK^rX@)o!N3m%=E4QCH+|Aasn5Sjzr&P=oQ z!{NVZmynzBI=sLmoG_;Fa#MVzpyjrL(Hwd?$=Hrvpma~^oML|wPN8~HIM&`D%(BAR z$->ju948Bgi%qD#uj?jUZK>pjet`-IsiIo#i)kXv5(W|WNjvxC_z011R{UWih%+ws z&!yN^Sk(!9ak0REcQIG zfVm0&aS?7}X*>ELaEd-&-b0r?7{WDUbtw_sQHU$Ui9$n5oO%d=woQ%koq`qzPJlRZ zXp73BIr^kBZxhDA!!m#cGesIv1K5J{$zewlqiAMMyW$nmPPLhHdMs6AS|6~^gABF%~?)fm6>1_>sDcINL>o(L^E;*2<_I^ z6(=a@{L)DiJA%G5fr|XFAB#gp`+Mv&G%v4qHV7r)?XNgiVPuB$>{2Xo4g`aGSDLs6 zpKA&W1jdvh=b1vC-N>UKqWDbf$JkwjbHtYb3XB?!WnxEscs~ETR=irt@eWh=xC@;? zNqeV$|Auf8aUG;PBIAV=ZNba;-Eq-HISM18+Wo5kfWU?Qd*dzz-4wnAtDVu#7Xw;c z(2>hOUpltgTQDV0hbw1^aLF;w!foz7@ycEQZlx{oF7q1cIj94Zh2~#cua7nlkG#p3PITibZih3cm z8q;9>bOmAO+tkZyPK$(bSAtpYi!(HFVb!ZZivn|wR=OIp=tKEouS!h32H3^X-QCvM z{8|8Wlup{_SCdCw59pf7>n=m^0ylwZfa1>ZAcb8^?Qj`IVhuq>7ip0Zzcd892y{)+ zEa8&(W@x7>FZY}ipCqKZ!e+im{ea=+JmE!aN$E1HMpNp}SmYM|FItcFI*yJ6R6BiF zj1%cK3x|C@qMTZz_O9pN3T^(?T!_Suw}HA=Ec(#4^~EDK$kugu8NZ!a$YSsK_^*QM zQ0n`0Pls?GEWD|^X!ylqBmQ&;VdvM-k`A=xVdcQPK%E~al-xlm#lP;(P<>ltr#3>% z+>;@O%e0u@{$3!91dc&DRY2W6Lzr2Mk5V0qJ|@HkYSxQR+uoNGLg$p0Kr;nZqhOYH z^8GoH&IXy+#jipvL`WK4o)s5L;h_%!R~g9Bw)JUmKMd+}X%9oM>WSqZ0fR>rydzG2 z6vC}m>Xf_J*2e%~66PZD$Hxg5?BIHa)FO`y^2Q9)=fbI`7MOp0f zv^|~zvalon#d`@WKb_+x4GCEddJc$UYGD}fh^Tfvo#lQffO1QV+Wox491X|66}Bjg z*H^y)DJheU$nh^h6mrkB(HTp=0)U`%X`Zu)heQ{x(?`n?~-z8{1N=JzO1XF=9m5Vi6{8A818H3`8_xz9CXvNy^ zLzvwxCEDY8(JFA4c(CpVQ1AdQ#s&&k$FCOdrsfY3rLNfbLui+<9WS*q9UA!&kh#Hl z!Vq}c#{gouyqqro352^u@=VQBP(qx9oV>{dWV#?HLGhRu=K2(%7&Z(420Ea-{xe{6 zKz6Bkr>ewqpMz0&tm)$a$U5sN%Zuaf<1PtF;ch_|SQmo(;;spBXXcLVD063acXqMG z3GM-c1OfziC%8lK0Kq-M{fCG5^YqiVFXz1H+&{XW+uxRIudb@TKw+mdDXGv-`m_L; zFEaUk^mzf8L+SkMd;tJ|Y4U@P3x;`5i8*Gjoag8ibxPBqyqtKDax)&G+~G|f)fg++s6AkGm@3A&BUni0FIq4=*LwEzl5Uo2Suu2tL8Cg{Ci0WlZHwP{ zJ^we3jnI;rM}*rd`FFvNPe|rH7j^Ki7FYkaU?_)Sb&TcDhCCdheDtvMpu#S)UMB8A z)O)HRug+6D;_FPwB{J6$iMbvaMaQuuJ1rQuJd(G*T0NY>6sRb zE=$m*K(b6nxI=(bkP;0%+`De3086``FjcBZUj zcM5O>)>>ZXUJ05;)te--#>x;>Yp^5MQ7~64SrqGzcg|J(|N6g+eHF~3k1K!pcH^oA zVNR{!`DesxAm$`%N(-;O$K=(4a`u|jcN+#x^J*`(>{D z$H$a($eecoH0ahJ3uvAqfs~5kH~<&XHiAAhd?46`C|izLbUdK@f$k=7{D*PX?+b>T zY|b2XxrKz88J;?m_Di2oXWiA+JJb?S3xI#gh-MwaW$w{|iOrM>ywtk<;XyKbTr9mF zluN)>j&Evi7UpyjrBI~@)(3FYaWCJcQvP*=5{In{!+zrp0bD(uUY^Nop7&sxg1Xu@n#AUR^w2zybGE6aH)9UV|l zVy--Svl6$pqmAP=f+9h#9a24BYPxD7fLVawD=TBJy%iXo#4$k}yKVWv)6A<4pwiWc zqvxjWiQ(?28J=sc({=!<_tP{FBvq-icPdG-J0srB0@UManDurh?5d(^nw+d1FUTFR zzcMKn-UZS;o+3A7$*yqft_953^Njo%yOp$1SCHU5P7>s$xkt(qI0i_sI3BVmcpker ztALy5?p2T)H&M(sd;Fmwm70v;W0!qEm>iJf|E!=B(1H)w$2Wo0YoCq;bi7*3+YD^Z z`;@j<3xI2kh)bRU;@fJ8LpNhD_V?BjhgmzRq+BP+873DkcnEC+p)`G6aY+Y3v#f@h z?>{F&!r(z%6TjO@IQQ#{94x%m1>l5`Q0tMI#kw86iHoMa6|UDJUAZPD+I4z?-KQt% z)ssp4`4jAhVv~NtPEeMYjNGq<CuB$Q#DRpJ?KIv%i~A3PcKI<|=SdH?)g3bm#{EgPTlWZu915n*x%$|> zhe0?8B;fR)TMh^Cm_d-Dm(*zZS&snBJqQ^zTZf$*2FQ?G;Z1q>EPv;|c|dJDDYPhm#1Ct}G)4;tNqO4nA>oqDs^7;->)T-oo8VCC(J! z=ITV0`BCG4WU#pT9ypc%U4C{?tf+CI5VB{|-u^TP(Sy8R!MsT8@P^hR`=K3A2cMzLE{sD}u~awBV)3(GriJ+rZfFBpdg=&jZUP)}RyVlw^mF z4M-dzh+La>tXll9B?)%I$rD1XazVkG5hW!{jOGi$Tv05mbdKy5lGiar)Lh##B38Hv zF!!onIn?~*;_{vL;56C-mz3}7w+Fm;Au#tLDpHMMe+PgYY|dkeOUs{Zi`jk6a~VK= z>gtUrE{8O0@@qlNPgnZ;X~_EbRoLSt$wN1~s>IjSnN;x46XrNg+VFiP+66);OS>L= zb%}s$wehjxwIxEeqFejv0$hrur+`{u+#}4Lc?w?5L}~Jpl;eTxAY6wY#`NqrK)Rif z=ChaWkG&d@b%$`*aAWzX9hbhli6p^x=?OkB!Y#x&K^rKRy1As8u1D!H#|m;kqB%p zo)QV*{f?}wqZbs;Qw5jDh)kGe`yK$Si?{5bK>I~@B@gn43nSZZGG6r|r&Z%PH0hC& z7BV@#g~<4vCeWq9?GUoF%A)`@4*EmYI9h}gOlF0(Sl}@bYN9K<`Tt4S1;=Y|d$fpl zY#LIg-~SVm2OEBGn6Vj2Yd={)O=1|oV2HZvQ-I7roa-hjNEV&!yf%5-VWb=Jy236h zXE`Krb3X&%{P7Vs5Wf-a{1HOp4KX^N1)(xYCOIG?&rY09Bz$dqu8~5uX&{al?T$}h ze@avy8a?cSbwf7S3nh^(&xz$<1ah{#xWY|Q{7DEkpjXDV|0Z1DWzc>y!=Vi~eYwDu zX6^;h0#$o`AlPM;;fDItkFNl^+Q}z$eEKSYnJWG<%~GyYUIQe{W@IZ1zE0TXmaZP6 z9lV6U>Hl@MoawESK-$VDHRkQ!2CPeV@{U#u?qA*k_qb?6jLB9-7s1a6HS2qLgera% zZ5~K3iizj^cR}jhw!@=^|5HAK1-zXgx?U04gzD&g#RnxN7|cCmto|WLorC508#fE~ zOoZ)G+#}NY;i89>7!Ds7h&v>$NAa~Fmy^e5#h61sDM&I6?~1XXf|%jDrD%`41mwz9 zv9fE8H9iB$vmo8b8GlTyD$2j>i}GCy7MyHvw)?Um$$xq+?)X2a#E8^$ajSm| z=~gJRAIXmn5KhO`O)(lI-<4#fCr-Uq_yNT2fn%l4m@b;`(8Mj|n+m%=D7BN=w*u;X z&mLCe%^v~X^~a2k??pLEkxaz}GYzk2!jZo}?iXz?=yvJVSn(rhvNUIC#H~F)I*u zlG6WY+1gf!xy@k3@I3)JSK_eTovJ7lD@$S?Sh0M;{(%PjKLM@@remo|H&_Ws^ETb% zxRo82q0?f&Bka;L=G(B9ABgo<0n0s+=u5=doXE?iw4+iSu8wzx)h_dN_lr1&{%gtG-b;eo^x~YRL;QFE|Dnk4C zPGQ%hvr4gI$&CSMdzlNxG9vu17IZXc1@NhqKt5MEH@G!)V*gG2aq843i7=OYFrgmW z6w;+aV$KU$qDiU^A#m5N>=#x zj<{n>Cou+_!$)(tf(CofR^T2FYS?}jHD;Dmg}Sv+%*x6KF9~#-%p36KsjUHA6lUjI z8=7@%Zd2mac)kf0Z~?AjtJLeh7VZ34Ls+wuSZKTL%4ZyOcocp9?E&0|d51C-O9-PG zH0{Kw9U#ofDfIZIcZ7s&`iN9S9uI)KWye+^*?CjuW zj>UI|&Uy2K8yeZwxKtQ@w-5Hpkn#Q@({ZUA$u8awJGe+y(eQ~K1`Ui=+b(1 zex}ivMe@}i|Js%R%@u4=d21o{tlM@kDIu}Mn+_?RvDF;(zkSPBEpj6y4U?OMk&kZKr&kEO7z}EohJzVy4Xk6kU*D4 z%9lx|^N=vtkX?S<+zhGe;9KfFg2wPJEOZ5nhK`t zwm40c8WVS{BOgW19%0Z(`V1y zwHVg}TB~bbLda%-o}RpP3pw zD#TscxBNV!Ajr(AFn4{`h{5(luYbhQ{v{dtum(A=fwXc%339U#Hw&WCj(A11YcV8^ zq~rmn{~$+VT&BI-G|_I~v{9Fbqi+u`ALb1`6?t)nKVMQNVPue}9_uhRY;nf%gk3c@OPpKb(Y_}XEInQz)ujn1`WsEtLpdx_ zZ{;L#7ZwXd+MaEP;6tqu1UU$DIM7fzinYp~7Zc7kj}o&T1r} zN8pnc^gQPQx`*&s8Gg-0J3)zghT{42O9b>Fc>MdX(Ah9h*I8O?EvG+M|2U~MWM;jf zB*h_(^Wm!k+|99jO80GBURWUM)aU`j{sx5V7nh&?@mK?D!X>9PL-+d|AA0c|8@G#a zMX`m8O)i3T|6oGU&5<_r6NQ@B+*!-%;p;-+m5zXSTte7oqx&XV&>DXSa?_$D)glwi z_`5KD!eDB@jIeWU#+mH=mzNZ}xVRJV#t3qjlGk#m8+%1T@=KKKTnR!&(v{|&BJxC_ z4K;&m!K;AWAUKtx=KTW9X5DDTtD;GOL(M?!dUc7d_dK{9xwZf`**?Uy>kEL;8V#~f z1US7B@*XkEjnJeg%kp?O6C);|WVhQ*{9}foe9QF$J$SpD!Q3lREk@Z!f}ewKbrS3| z@jiYV1fzj%#l+hQ!accl-s&HOT`wLI$6vtd0xW3dcKsKR4ENsIh{<~tJi>EV0XTM` z68Y5K0Iq^wl!~84J8e`CX!{@U18_q$GeQ|Suig)0PIF9Pzk#Isfda8BUC!pBg}MS_D9N}$eKqDi_rqS#~Cv={{ASG%h0OFqMPGWA-My= zwSPSZNm^358+hDdU31KMf^cr|-WEJoiH4!A{jube1oH@IZ0`FMq#H;#i%IM<+tXlP z#94Z=g!EutNw8~ze+#C$zGndPps5YX`$R7Xj0OeUNUuZWrRd#pac3 zOS#CO`BgyIv3pQPuk3Il#8*>bc2R|$kxCPYi;d3Fmn~jS3 zK7uMOHZDF9?SDIaV)>67U-zf`|BZyWMY$f7I#2xblLATiL}m%IdGsQ*lsjciO7mg}CJ8Aa!pAaIx~ z>{#imk_d|jZqPUa3GxSUTdULk4Mayl_7}5#O&n)b??f^F|G;v4%K-7QZ=fma;I#Nk zK_|&Rk2h=L-fv3+*=iOal;455&m%9wKa9uP8-gh=Zc*YJh0P(}eBnGd%l80ug!B~I zDnFEenJY#Pi>C!RUldPdg!qgQw|ghsnA(q!?t#AB)t={f|7(D_{qU0BDKO6hI84AF z9&^*rV3Z80d#s+rhzH^kh0TfP3hP^SrV;gGU?ixc-Aprs@Fm(TvDPevU50d8=}{Su zo3(s`rk9NNIMUW7D=gqrTI3dkq41+{lH%?wVh+M^BN--NHYbGBWkmORTe{ZVfM#GT zuUgx`+6`^IW#%if<)E_`R|qis`i}854dD;r3=U3?v6~EJTM1cE4sqr>0%8-$MS3<)I}W6yYi`p&`WY z{|!LdxoW>C4)z788;m)%n(j_!Sp>|LWm(k|q00(!d6|FWbCE7D$NfsQFA5EFb&XCnX!^JcRVSI zHm>x>+A9*wLpk}SWZpSdn6pdW_0^T2-BMD*Y~l7n7X*v10_-f9DPjeYWTCfm+TsOK znyF~MRSCjHseVsbov@RZB3n!q?W8NMZOk^bIPihPfa|S?`vIjmX$>$s52BqhF@8;m z%xJD#cpzAS3nN+4Eo(V4Fv{khqUUvkD)7QaLSCG*wjAB-XDsl8k!tR7g zb(g;T{%HR#KX@M~?7t;Z)ta-@SRhK5cDajhI$hs@Y8`mor<2UIVTqL`8!?|C&t_Kg&>im58v)YCnvo@M zLXbKlu5P1Fi)`wD@W-+5W(3{Tj6i-AW1h_mrt_B`B8=Y(Gncpni|ra@H<|&p#urGF zq&mmLLR;`f?#rmb#tx$4M6&UZJw=%lJWedD;Z`9Ocd$9O*^(fA4L|pBzrt=&Eo0nE zZB+p7JvfbQC%{RyV{+Be5o>M@l6%o0-Z8EaU2j9)an!rtJdV6g`L4CX8&teACd6#i zJFz@{M7c(6TR_||^LZ=?6gF!*xyiUoq>Gb0X<*p5T_Xa0{27tCbTW@P zX?tkbP*PK#Hy1#k(4I8D7vT;xV(s{B2PpEbRj0<(9SOS+rg^J>Fg+b{yI^-BN%onG zV!oY#NDzgYj<{lHh_WEin+J8xy4fzEE(DK&aFx&^K;LL%^(PUzW47xg+Y=Y>T9V2B zVYR~DA(TeMa``0%{f)LYQmwvu51@Siq1|;)KE`jkXGu#emMj^@UoTX3MUmr#~3Uib;Tn^vc4oO|K=cc5wVh1$M zq!P_g*q`eeV>CRuMx6xaY;`4&o|cZ;I|1DZaZezn2klBO>;`p>xh_HY_3Iu0($>2Z zH}6N-JVeqyDqa%pBy<4Fw!L2ptq;bYII|xzk7##7N_2xjrOG|je-w6!Ie&8iMVB;c zV9p7JitZQ^%@Aa{VWm4xlp7t54$X9r32`ReW#NNLYxI8x0m+2ZTwFMrunUC|Q9Lfv zZIib5oI`e>0^}?w_44)jsnF)cD80V%Q=oz{n#u|G&=9;&(R{G;3{Z-#%dO`QA?!M}54FIDqYedd4n4@bH$MW>O_4-b+SSZ`B$!#n zE|NDdhGL{33duIBH|`hV#$bynHya-aAxGUZuX04r=_b?6;==2#3IE@!gwfb?OvD+zx^W?__DM8Ftrvl|s=rtDiMW=%~T~>9j zza{+I@(e&1hF-{G`stqwz}}l_;*&E0oO4dm20`17CyP4xG_lxQ@MWT&y^ zIVCB?lr-!W4FLWl9r{^79=JTIz>AHs*ttNosa|Qv1N3<%2{~`5#zyA@xRLt?)dABz z@kF5-zzA)kMFgzD`DI+Guv;9Ty?XHV%LbCxqFNmFSK^oj^fi2nT`M5>KPi2%m|p;5 z)=8|0(1xBeLtwo(CEwj22q`^~Q)W7vtZ@-2g=4r?;tCOY)N@%FKV1xMR&tNY7-v8p za!J7ue`pZa;#FZ(3Q<7(?(YsynKC6VQaHDLQg0uKn*_N&lG*pg)0YBJfIi6rJ{D2$ zUGl1|33u9MWr@HE5T)F?uPOj1SCqFW3&_(Z2B5274M{V~ z$f6d*Mdi^UHPeZrT|_B1@}{W(hRa}kOum+&*~)#F>={110fh2v`HRJGgmC#~@W#ae z<)13h>EpY1LYyd?FPJ4;V)U&L7VB|A!}d#_E3{91Pk%TmoB~Lwigf>dfi9Z-UJS(w zw*zDjw^!57dhUM|NOMtzL0%P9F9F`ld+sdBu))Q7*f#>qpC$~E7r(3gkMIdnjyn^`uv|M?DPS zax-1VJ0fX`bj>mOQ4>iwYM#d+NCnO6cw1qQg7lUvHHy9d3FeHk*h+lg?uimjo)s#* zKNs^p3Ff+S0>pbwI=c&WUDzL`8mnzrdm5N5i4`NBC0tkE?U8;A8zvUje+YMa`L!f8 zqFxi|KH0`;htDg|f#e=7d*P~Rmlk_r7VDunPY`{Y(Q>1Lo)~Z>B=62T+&&}JtZGA? z_Q}5>orxUnauf8!^Cd!0dsiIxLWv+feKo4ik1zsauW~)~1WB6t*p=J1qQ?I=a;>OA9)7U}_bM^bAs5kiy=v@7dt~C3< zCa9h~QgFHxO>p&Fp!5eFY2((n%U6;!spXb`r$E>Xv`fdJ_FjP)wk(MQ@xXt9oOZge z;E73+`&R=c-az|)gF1H|^geOnQ4J-_miR`Lo3|#_4ETtriVq5iZwndr13oR#jG#d! zbiMjRkUT_Xoxk))(B}Twk+JZ{P-c^qr4W(#$GXBaQYhUf3cIzr_>H|9(X&-7&o+KS z6n9(M(^grI#HK%d>QqK?vsY0V{z<_mZWQ6lqyuZ(zkVXnP2uawq3HO`AEAWY9p`-x zk^3?iWE_Lu5K?bo{9DNB#KvEM)mu*wMx}N3v#&v&c2l(j!}+#Y`~QGkts3&Z6Gb}p z=6+L{{gw~#Ysdh z&KBy{p4=M`it=ZzQn#YRtT)rh6ahz8Fz_}}&J5SP-PJU6-Yw82Vx1Zv^UMsL`m?gICgxcD7c`G2o+{ek%TokO0zI+z68^rw zqciqY*fl|B*3})={HH7v&uje26vv}d151|JP044#*#h#w7^Hs47=$qKX{C;*MdaMI zuF!xn(^BOV`6_@-vkK8%DaFu*mxf?9u1$*#mm%n~S+j3P0q{xAcvf4Mu#-kHLDOto z+#oD>P_oxL*%DtY2k53r(lNQRMQdq!P_ubjZ*RQ00whJ?5@B@Qyds1vB~@$E`1MK! zsZPb3uNwQWToA^>V1H9wCWOMVafk(1A?(!Azm45jgLDzl3!(IMcwSQ%>THmAHZh@{ zw>pTYt`$eero+nr+{eqXdteO^mvU6>y%vNSVBcYsbB^I)FmIqaey?C&qd39K!@DF+ zD~|xK&q3;_;@fTlofJCNL-wyj z*g4}LD{<_Ib%ES9r6=7VUkNa~SO;a;{=Jag_jxW7nfg%tdcBfTR_%*ng8U&ev_Z1F ztbvRep3GVP+xQF>jHI^um-YF~U4;%d--b}soR*6vHzHhSgb}go#?USYz8Z120TVtF z<{BjfQ$G8q5sgJxMwHl`Zc4OL%f5lA2p}KyBI6o`UGP>ZH({c%Ifxrn{sLnQNM~Yg zM~Ji}s*E`oO!Y(#%+5d%)77x{}&Y zh%3}7PhyOm(*-$o+;sRM;t4wxOj-?T$Me#TU@)$aeM0L_B^hd=j)6A~AXiM=rj8`Q zM3KEjWLv8%jc95>wgMT53A;FH1pD#G?u2uj$Pq_NydgwA1K*Sv?LpXmhMSyo_AGJr z^a(ZIV=oZ1MLWKj_YWloJ8MnOCkP=0bLc@)dHq!Bm@4tozBgd*Gkr46*hGM1C0SlN zKC}yTQShervM3iDy>HDvZwql(LKYX}_kr}phil{tMCLiua}w46wF%6*_Uf_0cu@fL z=I$`&Y%YJIl*OK9r#1jmPtzD4mf9hmJ&e$3ZfzjaG+;iNxsy1~G}-OzrKihPo`MX9PKBt!}kW$DU;hFtw2T%bgT<&KPCH;Ub*~r#)(bG<2$Cck*RV z4oe#OiNY>qa-=zIsL(76oiS0gE6OY)%>Wp<%QQe|hquSraC(UYOU>n&C{SG zsh}e=zxTyjCquhbylEG^i*(m!*EC^RoGrlddCZ~`N1OufHs{68Cn_g$A&M@AncM;5F2Ow?$2Q4p*t1!b1gALfb)>aZ=508A5GS#KpV^tgt?rE^y0QN z3)zBD_RnV(5|f;GS)@7GH`v!fISx7-#GRWPIxAh1^J0G~5t~r3W^6AbBoAflDM=S| zTq9l|r9u3#ju7{!Npjw-CmUn?Mzki}MRB5hUI@HsqFM`|N7&s3F$xx)2+%dX+#}po zYr;_Ti}NAPVeb%m$lwB}B2~?Q|BZ0HLGZpj*G15-E0c)$zjFhTJ`2TvN=Qz!4|_Rk ze&8iQEj%l&a-yyXIGqMzPzd)$06Se@lUmC43&!td|!nBKd_i_m8 z+{Jd`_$vszvU~Bo~B5TJ9Av&A0~8J)n;%IPK1UYG9}{c#nJ5 zwZwD9*y2gt9iIux{HaW;#1hv*Q?#Tcajpnwk62UpV-GZt-l;=Ovhn2gB?0F4DZJ|C z1UP47$#nyS`7nta9Gu?X2w;|R?HrJpJ1d<@WS zA1bH>o@l7a#3iBTt>u;sy_p{-CS_pP7_%zu0;7+bT)SNUNP$ozPsjFa1myztvJk!~ z+I@rD4Q;^RX~aO6r7!k*lo+{9*{>$)$$P@cmo|;XA9EP-YAp6RVMZ|C+BQ+pJ(u+m z8Ouuoa#wBV(b{H75xy7b;!t@W^m(Fug~W)7cl475;PEOhcoumIKv`5`DFtoWQtLp@ z!2zJk>5IMjnfm7!E2XZ2)CDdQPPt_)cjME9T^=c#v?Q75R(q%Zac0Z#~}9 ztT1I5>WEGSbIfY1B=vEOAh)pYRM~*n;;j62b8{u`QrLZDNJiT&vEXYUZrr|#BpC|} z@O)5}Hhr_*KCUG=b4cFec6%L~`e1-IZfv|M%0)!y%%VAU6I7ub{@;5;# zJ_?2}E9i{)fLApBm@Bk}$@4LoXlVaod z2)mwm<})Q;QYilJ}i}v>_f;*lc0;rbi@z7DFznbJSuV z@ITdOIi^VKeB>{u(Lg;4`%7si^v4MT+#L}9U>d3!Z_Q6iECeSQvz;%%v9g*ov2Onq z(ACiF$24)!XCPD_Ie5INFua`HAIE%7(Cwc5wCP3azYEJPHh{m24@DEbw-Rf8Ns!7Q zw$bH$RitC$?qDDe5KSh?dE$J9-LlPklZ=EP`U=QH5;-XYh|6fnlAiw>9A@G)q849> zAPq^vJO7Wc>n3ll=&#ZfR{RFgd@^v#w*bzmg)@#`RQdEfAd1}2Zu9#_be1FDnEwI5 zLkJb&_~Va|o#n>2+V z^D~gkDP6IlIBKR*_1aK<78wM^vC}LClpcA~#6x}$2t#BY9Dkh+!kM9YsR!Lg&0Y|> zt|m^C5I*RvAkJTZP6#;I#GY}-xd^-W@(dIL0j9{I_@B^R+T^lmD1J4!Q^06!LQEA= zMi>i{+v(4O%{lCk;vtc)Alh@ohsB)p0Aw>m$>I^)W&OmR^-pG4rCYnyDGg*my#uA$ zd5Pmf`8?JfRYBCGjec>j!cLq;p{J&Oj*Io^d^r|psMF>ziRdC&>&x|mNMwM?d!q%) z-*R}YX6hDX4)x)Kg!d^mq(t=rv3`p9i9akGcVDkjc4~bL*d*Fr2#;d zHu*JBy{iqyWg^WmM2YB9eUSg}#*ZTd{`H0U--D6sZgrzdRDV-2nUD6iw$@@$m~-OR zfRTk8p8%o*Op;OfX(Dn-dUY95i+?Q)y^vHRjcuC8=>UT=`nM*UKx2Qv&sqAv0IsY?QC zpmxTi3OYfZAvBEpmnxqigU1)J=FF!B=5?kn*I;xlyL5?*3_K6v*96i&Si;90%Q)P` z9I(`~g!8mF*wP}4m-t>-9%q^e+GDTfK-^wz-PyVgTONSYws*&<6-sQo&W;C#Ftei} zh9cRx6`?Z!q=kH{Xs2Zd3m0a96GYpf6m>2W;EpKGy#D4PIb}b1Rj0~AGpQr0t3lDs z9F#}I#UgTNPg3NXr7~v>${JSFy z2FWH2;lXFZ+*R=Aq74fDCq@;pw`o#VFK83!Qcvz35*z!3cz-cw?bvHHR2JW}yY8=Y z_mu=;3ydirqKP2Q5h;8x@5lvrhaAyL4I zHAFg54%HnD93Gik8%zzyGCWG*)MGG(t%-1b*y6D(cufGz!i0yJa<1P)xskccpB8hA zb~9uhV-g=Oki>f_>z)cT9$K4Y<8=tSBAj7*SY-aRZuy$=KF~YBTU~Lsz#L8PDQfYz z^*|_~?rmf5^~*PvZtajz0FT=?03=6=Sq|8cu#1Sqn?5Sq>9O`}=H6jrklaLh`xqw( zbJH?YGhVClfuQ_hrBjvpy73{Ll$JTqqisekvgUp}j!@W%=rJ(0-YxlW*5z1cbN)7O z`vt$qh2_Z`)6>+V?&eCEPq)HQkoN6~m ztfxf!d!!Bc(2FIv1aUc#cq5I!u>sVk$c|*jRwYJ1Lh_;br~&BFx@w%V4KbXrd>v_6 zUAb+^h?B64^=8`+go5H!A$k?gOy?>#ZG0~i=H6pNGjVLb9RTZtrDqTY7D@Zj`xIX9 znS%%kPZZ!`jNE>a9ieiqIYX>2vc5mTgJTvwJS*5aqSnEtvll(QcLh?+L7MpBod`P@ zH2M*TpC`cGslUpBuy1E*=ZcQsN4r4f{@;W9n$#uM+ZE7nOeDcV{kXvr^!wfT%axOQ zA$kbb*Vsh3o1VG0Zsh%i&^#Sbm=5IpvD)rnE@0wvtg}Y}xEyAUeRfX(H@L4@E|8!=ZFmlB@1l8<9Au48*d{B@(tUHE##6Z2@Fbn!=l)Rf4%&qI{C3>oZ%y{=XjB z67LCjpZ9f6OZ-QWdmWbzv0NKuZX8~X8Hfi3z*nhJ|4(6Om2Rih*-vjT7;inJ!p=5! z?@3@zb27)plSI11F}Hsx3Z^7Yd1i;zJE7g!cobBUj|(BbJdw1=kD_RzF5N1w*5$uY z5MeIAK$Q8*YCIwayP-?hXK(e0fYP}K*?_VhT-?)$g%FfG(Ov+UIXE@u=_6R5z}Ow^!;CNCyPfi(MJyMFWP-rUK1;`Cvf=SB))%nRi%~qp}_gO$DK-x ztJRj6dm5N?YfpC`wK!god$=Ck?&dMw)bXbS*IAJq^y&a}$Nr#hT3JiqaDbCTp$*j! zj;}4T?14be6&Y(64t8sCi7MW_7`NT-Y5rxcKnXdr%>G?qTJeA0%|&0Y->F--hXBwbIB^9;n2 zhe4TH+%BfttsX0`dtk>yJqLMJ7fa|D^qy*xhNkBc+O1x6hO z>^{U9n<9@n8lZIQv2mVg*AYi4(SKY?ramicV}GPzVrdmZ{CtW?r`j^a?mJqK2XLwb zGTPqt1Zet+9=2#xxIV1ekVx!9@~h-5`hbk{-~V6 zcP@ve8EG)i|L2lC9%tCme9{2;66gB=IFlGI5>icx)t?dK5^!4~%bR##kQ0|qpIk-K zHnW{oKIPmYor{%)z!*fsamLw%J)4dhA3yvB%2UbsSp8f_)6QkY`Tk={6;-r1{tDp+ zk)N?70zTmaFw((1tWu53MY(W#i$=%R^DYE(J4>Pxm;MbpGr7_%KPW3+4B};G5Q~{+ z)L?4z-}=KP;O??aCUM+lkZudzIo9Hy%K@AjO9n2%I-~CjAQ~1&k*6t`MmYU;^Q$1t zq2%*YYHCc7SA)V2wncGh7<0GUPmRpJ^^E;lbZv1f@ph4_;+5LpxsxDLog z?qm`E$Muk|J2#G<@t$ZiP|~TYF4KM#=4wgy#VeLK061B_d#jm#H9<~?DVR6mMLQjy z^W=EKmESTqmUOrhXE%3<0H;$)YUh>sNKh^h+l{#W=8}%zl%+hszXi-w6K4^5H>LnX z0-YAyvh?og2@MFH!em;o{jJ0@i+CPnpuHA92!X?TIO2rc9ByOG?0S1ikBinGnPhk0 z0hH&OBz(M30N--Li1q$Kn8r-~WxhMh$JA7M>d`ES|I>2Tl?mi?W8zQ|q*5E|j%O8i z`E>n954hlNkle+Z@p5wyr0cF_h%0cI{j|XPxfoqqS!*XR2z%XIV#`yoyk=D60D-yn zvSaGS?kn+<+N3siaoq!eZVH^oHb?Ix&~!E(ZMx%3Q63VAm6IK7()4*)xEsE?H+hn5 z;|8i=^3s7KIohoa49y&>!Lj7vM~Ua!qb@hZqH&@SYVVnmhcHV7rtemU&5#fm5=P_J zRDuW9c=jznH;_s)bBR5kC623u zm*|1GQFNZq^rVljAfFXRw$jmyUq1&y9?9U22}h(`CB1%3gQwf}4~4tiFgmac(WTFb ze*xx$7t>X;8R8fC7TedD9CQk@stV31WwPZagu1uwUIg^O6)!Y*ar{{pxOoDD|DrJ}tij2#zOzYpaO(Su== zt|8$n`}GgNseC&}p*s|Ip&MGUdT24``jCHI=QqG4`Q~D=&wyQH zwy%A0+vm`vi3<~63w5}Yu5*#XE~4J3lQ3GUP2n%$-iQ{8PVy)a)w*z<5Ixg_tJgtZxasnzD(K75vQK0l8ds!uHs^ zL8GM0n{%SghPo+UOI#w546zND)$FkEAzgCC!B07-mmk2~oXo!%r(P$3a>$rdWlyXiLg((uv4Mj14#;etHcvax z4d{ANK2EnS(Jm_ntb zH}AaV3r;-pMkxShXg}UGA7K}c1vm5WEP*bGTu#OfqMbzt4_SO5vc7Lkol}~ro6Qdj z!{FaQ^oVc@lco6bq-Yq(Eg96C3qZLJ2+io6b1&$>xg;4oHhvIA`FO%19{3euXRYhd z%0iHNdf?JAy%xmL>3hNCoxG;Z{Tm2#K(a7e|2rZ4t*va}maJYh`bKBmEV{mv(xosjVXiE~R$iW71lol}OD^qJbuB+#u-R7k_yOCl z6lz|uCt}xl@1g)=m9{0Z)nZWa0z0?3RAJ2rGK7l-mw?I*%}c=axt1kCT;OhWIHx5k z?&$)}K-q>axfEnQJA5+4ga#l*%JvwsborngDadxqK{^|zb9L=GmM=+g*knV}DZqW% z>V&m;Nsw8>5Ua#GE0k}Tuu$>8Q2_PQ#%u2t37au3JP=%0yI%>E+E%!lY>Pug5x=9e zH=0)_>^9&qK)0m_yd%tIU>~8^ed7J!0nzGo`QG^Tst`I{_r)g)lCT%)%U6SPNjc9C z#oVhyyYXzxZxQfjg1unKssovjFO(M=2Chshh^G%&(*!~ZHcTSSxOS{=P|00%EfC7k z+!q@uNOkZm6;l*;(I?9wh%N59f^rwu?h4aLA?m`s`W!Dc{+5*@e?K$4q`{M9yu5=3 z;Rql%{3NVsZWQT;AE->FJ3cAEc{FL-?2p%kxE5i&EJ!uA5LI;LwwuU2H6*-n|$llSJ2hAc`zh=2$_lFHbbHh_APHiPMKkp{%+7Eyy`D z6y#*)-E}~m@Q~E@w_3M+ge-(DKBiUc15tC<@k;b&0-;`4TrR>{avw)APuc*$43+0D zcDSryFaY^7M`qI;XKc)$W{G_Y#pOcWqPkYY;qXA5x=D$J zG=P;mUK8Tp*3~f>%WMh>lPcY)$Q>lYX(op(+{Mi zx2-=wAuXQRzWj|JYEF&a@tYlhG$GLghhjBRF5zUhXS8;^klYF7R-5w75ahL`*b^mL z-!VJ#hx$LP>)45~tAY@N=haw2u`>`#41^f5&d!8$ch`BgDjH_zjs1qZK<0%?Ze7#G zbvneUp(XmfNVvytuo|=O=I^EThD1|T?gS}Uy8CUy z+>bgrE%wA7y8}2ARKv1^*7)xpB^vK9ae&%)PXKd{;jL5kg+l71j>&%D574fV9z)in zRkb)>pxN8XETHS7=LEqcY4^|aN5b{?;1YqyqlCE3cv+Y8q-tEhHyATjWomq+AhnfT zGS=A#!a2%$RWqg``0EqsVzF`Ri{nJ|MU4iES4GrY+fDZ1zQ2eMKQ3x2G>@2}6(QQ6 zAg1n+&fwC`KrS`Xb-HbQA%qgLaXYGopc$+~T6c`E0=P!H%SdcFP#ArILdnI!tgR+# z(jIiY|0Fhq7DJB9P<}iAiR~^31t2u3T=tgx}e=ZlKvbs2`l$Z-6bVt4@h383&_#f z>0mNQkBg&)=JFyYNe)%jNC}5u7~Ye7EGNW+-X>1)K27oaOy&5mt9P>A%YStY zOpA{db`7|ZMic0(2A~T~-aMOH62{PdC^|)(3kWAVu;f^4T1g(?gfUU1rU^jH3nD?FW^w(*pr+i8T%`U*k?8T^StRfLPfj!j0Kx z5Gzk0>FowO#5Ka4FBh)3?&d^x2oPl-pG<4*;1>KrNu;u6u0!)di6J)F1v z8DQ?@_!C+)Q#cuC-FZ=zGuDL(3;kC@a>i0)GLpYKaCHWQ416 z==A_z*6Ooc64pL1+&t>(^&46j-T>s`jAzy4?29tKBQ(z}pzU0_0f;4dpkMhW!me=( zmtFgc%r#{5K-0f=Gl;X1zLN&b#9RD*4~7O`DeMo>*3+6g&#nGf2HhJg?5ta*@Ef1y zKe?ZarSEV=rid7m!;*scD$IIaZHi|Vtk3LZ=ec>we}KA3T<2i4Rf|Uixfty{JwiJ? zB?J!Z^bw2S3E_t1e1dt5XgJW1e%-8hJ0jd487qq551GpyuCUWgx(Vot2+1>5I{D6g z4?v#EeF3c7i5Ws2t+z?7z3jaJ&K|WBUReMRevN z3U33^)rf#>e^eYTn$&dA=g=-nBa7yXH6MU5r_hG$kap6wf}9+ySaWae_F##FH4OWU zTKrd#%ZtawuJ~LuC2YqikX2-rhar^(va?uA1i2$rV5a^|6#Q%Fy{|FKhtjp5vlgGM(`DJ#ng`NQOyxXRE zsK?vvT?AA0DLkxp*^?y-Q}cix71sgfdBNo80&VgXRL*Y*pXt4bWVR4SjJHmo(T0<2 zQ7}4bkfB5SGi%OVpF; z)jUNdFgGr0p7H~`-m4`Gj-DuzU`)lC?-If8K3F)(QRN*%{4Hw@3nmW{yaq%wRJe2M zif2V-#w3UHqhAMbk8Ed14Sxd?&dSH*8*e&{KZ0m_i?FleajY6f7&i+cW(BLH+Z4{j zzP%;(cpK8yvZr-)tKt}exn~ct?TAYSI7#e6cqS+Q`!0yPg|)QOUUl?)B_3?!%n(!m z1CmFUZtk@m{F^W*kEa6iyH9{wftd+A7dGR9V1=1_?EB>lBvmZ0*EWC=6Go327Poys z4CkZgG*fA|`w+}^Q;%d}Vig(j5n$p)U(EM0VRwflK_q2Oq)P~P`M5tu)xE0$;NXF! z>=^-RgdoECUPNQ1)U{ViX6a8Hi7A_VHW6wp*7t`Mc3Baja^U9w?}eG4NLn~1B(K}c zed@2I^02kSZUYQ$qyw_aX9a;Pl7{Rd#Er|86eo!^UsRU)+y4Di6o z|3QlxtOJ{VOV~wjVqsOs2yJMsy!|~9T<4Y{ zHa&3e9YIbLaW{;6_ZM+`2f0W^AO8p9(4s?3z_GIkrj`M86BYE#$P=P1qMSI~ONtjC z^1tk-q|kMfAZO0TYV5FhtN}FZd7F6RCt|ot90F?i=UC={AY_V)Gs=fY z{tN}TdSTCJ3cKsHN^ey?deTf|>V?Aoirq?A>^n0MwL#3yoAV-^ev9TJe0RS7Oy??Tj2J&|eDux@P3h$}28+Rxhz}cL>TLpDB^Ftkp-_NXh^|H) z{u>ZYF=&Ry_}@CIZmuNPQP^F8nL-B}loO(0I6`E%Oeeo%;etuVpJwCni-9>Cyv|7U z^{fy#subPnFkJn8u{fantQ2CZF=B}ltJ1Zk%P@c*cALt(%2qLd#MYMyCneqSUr-?a8vdrg9xg!D3MY$3}1M203_;cgOGZ(Mms+Gi~Q%E|V0 zD4IpUzZUHT^S<#=VQxxk>LIdar-RqI5x_3R*wMq{VUZ*wCAnxH4dG1Wjd3VW72x_) zcLpsYctI|m7IO7to|X|<7_FIb^D!W9(R6CT^fKNSrnb@b(XL}5T#hMKj-?{q7(9Qf z4TLhAF4U%fB;fJ@U;oub!9xs=?!0VZ)( z>S6Z^G!q#zgR;lK!sWh3R8Cx20Z6F*+bv3_BXA8nUZg96NFb?4&oi+k#@2~z|1{Qi z*%H+3#{Hp8nx>i-W zQ-HaZrp=|cbp%~JvfXTnkpdinNx9v$Q$QC1hCZkWZcVh&V!7=g%dLUm(9!^$a|dur zA;H|^3OfzNz9?qRyAuGTa|(9G3X)t_Lz^g)&&DChctK%*hR+`cOzaFmH8H^(iUUNL zsWKCgeb2(X0GWNYYBIFCcvq0jzU-MQOy>#+hZ>g{$FoRNGtZ12E;j=oA>_Majz1q3CW$N(k4N!*c4e!RzRzYXLGVxOI)0tB|fN zErv~XtSrPummx-594sJjJpH5#*Mle$3wA;@T>LE3EmZ4bfw-g%+6nXEdna7nVp56G zJjnKFl}>0E4*T_VCv%!0SUA9*b;qvqH|ew2t;K5H{+neoR_Y<_>?d)bcwH}~`9i6E zdC`KmLr`~^yzjS z;lyAK!WCuul0VWB1LYI6G6vHVeh)V=&Go6Hy)4X}*7kB*JTVAB^;@t+dRgH-K{Hg^ za6iGMG1KINqO-&)E@`~nB-BNa8Y)ZXzl6BdoDdP*)6YAmly9>ODa&0vGPmseJq}5Vwf-E7cfwc=^gAzgk=^D6g^dU7B*7^@#Fe z4dE7UrUg(}7T1orzws}pP2?hbA6cTK+*u}!9pCn-)3UQVU1^YsY zw~z8CgS=Kb&(Va<73oK=AkuvpMQo0aqA3iT#F$ky;&xKdtNz4)t}>S09PQ#5fZQPl zd$3uIsX`nDZ;_Pt5&`O`%%!7_Bb-Mu3W3ttI7bi*VlUpe8voLS@D$rGLAh}lhG{oa zJs!*zZ0hJ`Z=jb=zc`_!+*O&9jKsD+5r~Y3I7m!Vn6b@t%XDynD3`js17Ag5@v#th z4V(a|18P;Bd=es=IVH!JF4wlFL8L6Dc(A)D z3e(3VFzIx{RJpS=9#$|jR96MD;GY5VK}nC{u*uXJeWYOLhSUPJJxaHWFf&uW|LD$f z;h6V}uu2pth?XkPa^6l6-Rdnt{Qmh|{M}@eB zD=o=)G}S%f+!9S^273A;=YizTAxX40U#ALl;`l&geiH4ZywTI#a+N?grDUtOh;$>g zrfZ6!^8w7+3fJE{Wc^8y^U@SmlY^h9gt_3{)kx*$_zOx(xJ&08ccuWEy@HMO;Dv-; z6f_-pHc%(r&jjX%<<2JQzwGT_YHqSlSQxtTwFOi1RT)}dC(K_L>%(fzW)-JeB&I!g(Ozzq<$BYsT`_>p{)MYF85%&{y6FU{0grEW?wS_oiP!c-??g zw+OP#fr8@Io24Pu{C79=hr1?Q@Yqfy9B0RlzQgyTTtF^0q=$UTEsl;tH!`kR=2l1s zb_=JO+X$K!x@%{X)Dy1@%RP{JMKTpM!EOh17Mz!l6QvHzboRc&v;wN7)wt^ph+KVN zkZV{>`Uj{>gyO5D497GOE!|}BKm%#j+oS#Qr4Ta>FQ6z#O}MipK6w!L&%9K?S@h3B z>GpC{B^OwCiKaX##&pC?cbU+t7f;^};mW2-N-FUm3!^5?3-Q0k|4HB`vzZO=;s0E8 z?va?(_7vh_#__!~*1WgGK&mQ^ye=fiKqoHVZO~I#hvL7DFPITfRErJn;|o`{saLAe z+X?U(N&A%kd^O!GoXR108Hkfakn#P+C`v0lP4CUwFx0F+Ecfr2a2R*O-k}I5xEtc5v(y-;|^i&o!u?*u_#X$ zRBrMR5*}UHJ@+uM%ff1bbL~e8J!1Um;cYYA?iS!uJ%yaIWQ%+#Yv(mH+E+o@qXh^oPc)5evkW438Uj25u}2eRdoH6aGo3J zP})_TCn)!ier_(}`X~G`vm>u}h~Q&A#<|XugmbeX-b|~<8^UrwVFw^N@()h|F$MJW z#V$`1bd@A3huv#D1LA7nJgG0X67Ad(?Xnzo3UF7zm^BU)nG1}ESos_~S&*9`yMRv; zwN0J{rlj(_w7tUqN}l%BwD125$Qi*hGWu8m3~bjzdinDZd2X4;D=m=@=?0WQpVGT(Q4gY@$ef2b5~>k9>={|=t+`hB-)(7n^-?j4=nIjiGl7>E%p*^ z&h$>>5j7|{&1%>mtB7#>HgRt3ZIB#lV#9a&*VREiCfzi}mcraNMIeX(RVaz_AUIx> z8lPeGpKg_6{`ZJLg+{Cut0)KuQ9Ot-3O5!}tUzTxaYh5}=;1zD7eBRF>3!n5qVg5V z`t`C9x0$rAtu6fE2PIlEKJ1SjJ_Iobaq>Mr+C;nAIPP#66#ChtvM^#p;{a*mPRgx<<3C36a63kii6cBx24FQhCF54ccpA_aCxbc@i zjB#IqP<*vyd@I6fp!mRCwEWiqv{`RQY_Fg>#I_lm9?psX2jr$>%c99fvV%>8=7MlP z)fqp213-=2InYh}mN3~>d$6o}Uqoq@aq+QedTFvGjOpJ&<*iAT;|-#-ftdMwKsc5r z?OH4?Qpss9-Bd)q4K0g1L=ah{uMubd;P5!!U0C=>!e%^n?Xi+bXT@-=PUHMniG2mS z9-O+l!02y$#O5sa{K@I{p|~{n{|Gy4IWgB;H@_3)&w2fWb?eTV#?~{z9e=tb8;Db8 z2FycS4x4A21)2hDqp*X*b@q%K86VAB=t&iv+x;looujXp{dw9hqVYPLli>Nict>Hk zcY4YSpNg}WLrsa3t`9R#G z*|S;2eivctGpHW5#Ggf}Kh&BBW7GK|)c6!@dxbNfyD*NMz5uj&*P|T+Nk1)ZTw28wzb5RKN3)ta0h6tT{5J|6v6aGRFKP>MtjIiw zXm06RO#Us9v#btbxhD^3%Lf?M7&&~z_*i>k2y&Bx++P%?wz|J3IAm>T>7-~N@q9i+pxMd}xP{eMm zf1Xs>`5+q7y+ORRa!Hb7^FqMOj{qB6#eCW_l8?H1C` zaZ`aVIdafS>?GQ~i1Td|8>cA^q>n)Z%P$n7DNU!8RmVV>|5|6%En zohZ@`?CD~aaUiZX!p_)zJR~V%pfC`Nt_@MIA$kYMA~c6B^m|~Jl&67OqFJ<)Kn@u5 zuLJ4Yab#u9)Hf%r`-^YbJe($)rqiJ(Yv4R5$jNAGeLcrOmVupQ654+v$Tg(&T%U1O z)V)3!>}Ee5`zcJ}(fw?Xjt%^WB0ZGDQgfcZ5qRF1)UD0X;ucCWx^&%jW5RjXoytb8 zgA0V+Hz^ou0ad2hTI?&V-YwY{vSm&?(b{Iwq7LOOdALLn-3 znhk_HOM9nBLuQ&V_wJ-#%W3U8L8MI6F_XQQDLAy^nAmK~l5h{os@Wg@eL~&kxyoW* ze^h`AEn%s}IyBJCTZ6l;>!zO?jaLgMNAf^V>swK7TA7`*lrXuiyDea@Aj{A{L~6th z;GtvI?I7G$q!1QMigp5scG&4i4*sz~X8Pu7vGn!?-9OQ^kh7m%b^xJano$edM`5Ri z-D5wFU7^i2bf5K5L`&R~idqnAn!v z9WYN|*b?Z>eR2arIXd;Ty9BwsdM0QnuGymiST*y}K6?Vd^T}$mw?#N7+)JU&E1Abl zdjaMSW)Vhz-0%l5=ibVSpl`SQV}ZDHPZxk!3v%vBX(JZc8z9diavX(@)IK1&9fo@F zPffRP5|&$@3)py1bgn9w!4g~jdtV?Jh|`kTx!>k$MoxwH z+=MW78&UqGySyN^`4C|4yliuM20(zPgmT?U557aNKTc1_(@9$5t;WZlI5kHlZW^(u zPn#~c7oHB`(%@FU8e5BYR!kzy-_sirrHS7Ind^Xp>AaoxO{0Z5`F=fnjJe8zAn>P4hnVOR1rg?zGYIEpvr^;wCf%zH9^x3fFF!(I7l3(P?vZ9X6vQk- zMWi#95ba^bJyY!8Amvl$)!$2sHydD3faOM>97MTmJdw78m zJ*Hu>$`L@hSJ!F`uj2*yGaewW=uzC$j|9qnjgH3U4in*+qsxb_Rcxu_@IL|MWsTcelj}k^mtDcMHC_`|p10seYU5`~K;x z&eQdlobprWoC2h25%ts2z80mS*~XfA$f*!6G#8ldNpbl5(~2~j*zZQhRGu&mrvEA@%aRz_IgrfVEM=HcYRCd{@Uxjk2`ju)6X2#G~HnntY}U zAk0J#xY)SdRtH$3SrpzBK)yV$7^_`K*zMDfLh+>{&5ObQmTt_D#|~b8A}MLbR*bjCPvF($Rv<6Zs!yuAkw05a);G4&HrOn>G|i zf%t^C9Yi=44xQ|bs&VWMKsh6hm$AW(^)%RUBtLeS3zJB4DDF})Hw1SGIM#SXh$|$& zvFKmaQ=sp4B~67zZz7U2mU>KudQZ3+h=S20%5ks=Hw4~t;+#6!+}IXR-dcRE-mK<- zQIM-DjWAAS2V}RzoG#Iq4*x{c*7C8M*D8P`M?_d1kqDhWL)Fp+$0_1G%@mCRMU>CB$ zQWfQ_3x(NOQmG>=(ivTXx{LFtvapP`pCFoB2X}tem{ON^7K}H*cLh>tr~VV$!&%C^MNeVA`E+T8A3ZB2a%?<2hnc zkL=~@X_5XY&w0`CJW%f0df|sV5GFW*<{d97x5tH|&8s1afn(_x04S`47O%cYnCi)C zb4#r85>)0~Z+~ngvev%I!0M>qAr*Y3Rbtmyia1GSj+^q=KwK!yDG^6WOKzmVTw^3P zIt;)0I#6wG&R%IMvD+Jw*DPk`hqE9-;oYi|deAKY+}gS|+Q@e+W&(V7_k&^0oq8CO_twkO!;QFJP-Z}*>tV+E5B_LA@Qx>;GnP`;V`ix)k59#uPQ4IT=eQAAT2sr9zVrN1hG8Ap&*QA(=VoFCxv_CY}I^PSHgtX=-QZ+8L7t z=J~@OYLEQ3h=yQR215+Pvjv(nT3Xrj; z^0nXiony$JJ=uXDEdU-2G0}bceet=}hIugH2LKlV4??Jx^~S|N0=Wni$3^@MdSv&>?XR_Kodrfh*kkkP?9(tKN+HV(zU_7zZU6DX={t|zZKuhzf(sXBfze}$~DR(gU&I;hcqRo6KQ*rXK7q`yFA5I?y1g1>dZpAskTy?fY z$TR!m9YM9-2ao!t%_APu=K^-M&@996ZX7f>h;v3{2Gh2m2Y`ObD+DJg2>*~wc0}{M z5N;3Y%AEU$;yc;yj)*Dq0n~EjJ|gc#mz&>daJcD>;UbDS7_o4VLKz+xoVz1?T0M)h z;sOOkLO>m{x1ikN)4NPET-|v=Kv*kT`Ed&u-{^fK*#_Pq$c;K-Z2UQ()F0TwM*kuZ zx#9y-$eC#|04kUjo5qQ(F-CH%;{>>rHN(OM7Y8ys<>Z4JUa$m6(foQh6yuI1!OS`7 zGj%TwNhx@*u_Iy`h}`ff50vBHVE|M}FS7h?+2T)5hopW@9pG(Ww&9OwK&(u{%NLg8 zlRQh{buE4tZH`NHrS}sxdgfgoSS^e4d@Liv-{9V@678bR=+uRofU0p$9pvsRO9Y+} z=+aBWt~Z)jr~@UeO5;iaPK)l^$YWoePNuF5Xig%#lz~xvCMeI5%5rRogKBAQ(nuTf-$sLApz8x}gK1|311%jAC&uh0V{H0!l{Z6!wj= z1(d9pf$kd*ln3O5Sbahfp;0$z6mt_n=3&yKta)%hwH}qXR+G)gOM;vhu1+qW2<7%d z@o)lfDnhH7*(`p!mcw$Gd+OSRb9LxtY3n%CI$&;-^zKbHwh=;;v0iq^KI=l5bv*ru z$!crd^5^1}^GO1WL@<71IRNX81N_81b?Yr_~|b`|0%< z$Q{%H58R9xq^{%Os+$+FTL*i(qf4|iXvP+krSK9g3tXc$XmOhCVlL+qN%gG_uOEv1p1WClj&d z4iK&-FZyup0n;|w3CwBswQ8S_ZQ#y8xiOe(c+p7!r9wS1?pD~R%Osvimex9E;-8g& z$O^3k%n35V7QY%>?#e&zA}wrDJ`!15K4rp#y%-HT?k~VDiM8c;-$sxNjBW+e=$7s{ zZ8tEph3i7OTo}D4i027}!kTm8-g^Qa4by%(Bj~=zR#f`06>|iuAW!`?SaG`}%Jg1~C_JC6a7-+ajha>{$RqIgXEM*UULe zc;+xlepBOh0q_o4TsihCLF8J?VGRz$<9T6p6LgljC7YoUD)%XN2pXi93nJS_Ch7+j zcJda1oYMqA@@dOiEjy;J_OH)jJZr}374W>q($YNTB^UCsOEa*mE6?3=Xq|4DFnVPC zvxWb2<4U+9($NtGw8pBf#owrZ#Lgn!ESl5lTwe*H;L^-Eu#K>nPhB|6tb$ekb)jxw zydmg*nGW`mK-Y0fTh|~@kxwdO;RQA`x}_b^*~v*V%VVdIJll?r;T=UPB+N!;O93u^ zr@W{AAlfC$RzzC)e-R2l<#8e=cS2Bs4z#L{Qkc{mwG71R`5(3`aZHy#;n3S17j`=u zhdrK=Jfp{dA-NqBb5DkF)8c+xD{Jg3$Q86#!= z5wUr-h~LESPEQrSEy%S*65kW+_d~nKktu^225%(H1?j*JNqy`9K~x(_I`eV;e@BIyv&SZ_NhS5NZ!ln&E;6Pj>#sfQKyt;2c1*$lssHSZhs4E zDBfoR-Li-xbh9|e5Qqzmv4_T6TqTGC_v`9wt^JB*=z2O!kj#q$-RVgqc_q;7@w#A_ zv#W%txB@n`KbX@;-5b}xeQ}qd+?mG2e?*a6i*C*4IRL^ep41Jyb`)Kkq_mfjc}k~1 zw-w7TbKuDWiv6P>G3QcMsOw6Lck;UW%m)^TgtAoRZXv~&-Yiw4=imZGeGLCT14846XcB#K9u0lYO2dS?m&rft zHX_|Dh~CPw&1yTUfFv81lqJ1gI+d`ZO8_r&Grn{eY zViAp3`y^~V^<)sUj>Gq0+*YTX&~A?(L}zBBXHN%M_ml$R$pu~dd@96gB!5ujPla}0 zFFeA~O1}warQ<1sM@}Q?{#@;f6;6jTXAwiN+wP2ZK~A#7d(g4-83oOx&>S0{4NY1d zQ=(l#GZ@t~pVwwQ7tE#BbJ{4$93{k!*@+yhZiI_ z2sa;luvm~eC+o)r#phUn$WC>!3qdH*6x4eabam1@aKwK~kOvLc0L+YO=KE174ag;W zEPoMUmw3QF16IEngieA}`BH2ng5t9TtFXt5bQZj7gep^4oGl2JA|~gS=TcF%4M96f z_MGlz>07x3JWm8{!rG%a#;b%2cNtOPP-h)Wn-U*)<#?b0DlHqy)RT4ZqUH3 zG44tTv$Kk8X$mE#vHcqz#IVWUutZ5B*0OSRxA&$PY_(Eq>=}5mO$Y1Dcsv6%-1IS%~ z#v64vB!!#Y-yH)A*X9&zPGM@VM;(*RVammBJP{#zt~FAW-gx&W6$#%EeT ze-q>(>4)z~V~b~sXj*wCs60^+d?`1^rS*T=4fF3{yi$i)KGDv_)=-eELqj4RjwhZ4 zagB9hRF2i2E3%OimEHh2UyxhYUT{^%yX)~7-Moj+%UXi+ge0k%Z)Di9{%{SrB1{uW@Z#jUi&RD&kcHOD4r4Rn$Z5z=w9aS;;UAs zaXMo)7Jdhe65$su{-toP0*BP3Z8PqFV88R*Tla*!+PnNi8?;Ft{MYxK49dT;)BA*- zG4{T)V3_>_5Z9p(V~Q4OdB=hR-9?$Clj8ePf+&pk5hp8b-Z6>O^;Z25fc)hBIL3*{ zm8D0;ZlZJ1*&*W%xi_Z%E{?S3M}0&b4|8tbV;_<5pQa+gdqt5tzMVUxcRGa2>nTi~ z^OjElDO?F@_spOA+ky1{83b3X~JV+Cw$VeOAQdVhR!0pa3$ZA*$F7 zLbwV#SRrfR+0@U$Tx8AzG5m`nO0|+6qOQhYg;A__YWxp{UGAa_MZ=fHN0h{i_nyLB z;>P|m+t>XCxchNvFQbCN%T1CPKcYWiG`6_OEanN(6ghd~8unF@3^vbfCeIMSznI1L z#@ixX8XkaQQkUiM{NMPKNhMs_>lws3z6EeMun8_cTm_jE1Kr)_q`0=tcVJYr6+Njg zh2aV;Qo~%G|HAAvu2I;Z_V?uf&-)&%HVWmK`U?Rbz_?q{iAwq~OZ-rerhBz032^!K zQY8xw^VWHFu+-6l30YrjXKejb@o77E*G@aE zNxu{jbBgYn;y6L>;@ly#EQxm2usGq6-Q6Bjeg$*YIxv{yrLNxqoN*Zq9uByx2w}40 z9qubSH2P1DUieRvCfJ0ughbVH+;t4e$z!X?x~kH4Bpvw>(}v5>u`Fl8!DjvZ$&BI;#w zR0Hmu1B7;G>m{dI&x_2%ZcyTZHwB2tl_|0GoP;So+tX3eBf<=)Jjv7MG(ov>xkQ%# zyh@bk0we>BHKf|uEYcZBznm9x1h`@_aAaIukHFOPprb+AVYj!31;@qxhHA^L}gqG!J1%Vbl|hdk^j z(Cn2_D(n7qAuf|vXZAsOuAU!^=1iv7an1roj0&=&)cL;^<}t#A+h-3;vG9VRp4ez< zj(3RnNN(yF?kUPR}Iy3y#}<$Vg(`z5|y3nk05zyS5V;gzHgLJmrovqc@(1#B)^Zb z#p1*uqei{lu&W3OEu$)Sx zZ6hIGEZBC(XChrr1{&`6en*S9yD!K86jJ^UW6>2LoC6XxWPPl_e-h?$X-|m|gKV1x z(slb$()&SSXV}RzJ!`KBnG&Qh4~xjfYimRkB^Fx=#N}y`oObq=ix@q0XU5w!A*79O zzgT=#!t_w~3W#BM5ara^W)H&Et%f^pBNwEvE9`$chUqeXjS(Q`F{g8sJPr`xy7UdO zvBH5+Ydj&)ou#E4Z^c#&TxGQ)s?H5|>wc=hq6Z<8U3PT<*QST1EdC*gwdNKHRTq|uU&n#D98L7hm~(tVOQI78i1r+!o2N9rF=ZSl*riCK z=IEUO;PP|k;eCg`=o94Zbl7H3-4%}r6F-~UV!eq)8d!iciiQMG6WOA~q_rX3O_+$= z=w!zVF+VsR=#eXptOwQs&5YBVd8G4}pggT0CPCX*d&U{oEl`9U>=Dvk$7(`dzkJt` z{1_c9n4(B0H;(=@gp0z~uQbT%ObFG$%X4(CN7&WuY~qZrz57uc_;WpX5jTl2i~CzB z-X>T(nCj z7nn(Syqhp**;4TuL=H!74qEFO7$l>j_N1Wvao42ogg7H~Mz~9FjXAdj@^Gl49WiyJeY$ecRq^~z7 zi*m=tXoLH-7q$X$xsV~SPl(oS`~lKY9)3Jnlyl`mBiZ>~ylwFbZ;|NUVUg`YoU!hE zq@7ufLv{dk;W(VfIU=1i{hzTi-;Mz066X)JU`S}n&R~{j>K;N~b2N3=ljIA+T1GkDJJa&soJ&o$4v@nURBc5sLSpRLh|V4^~ISYh>F{nxK&}N(u?P@ z&vHCAxuavTefZB+L*b}BR^Asn_mJe_S99BQjeu@#y^Yt=He?l_9h$)1Rm!{!Fc^mj za34m>b(tvV%MLc46X~RpMq!HDj~3!_6~N4FwBo4qLd_tw@8~t+0tH>Tc3o_o*aq!h zlT09x(y|7eE;M&oxW+zJ0E}So(G$ZuAk5WCTPOvj| zISs`7JaZz#g_Q3A3VBd>f#esrKNsT4q8Ak>_CUH&_^#xORVM>D)20&g+HFPCKs+24 zKPc=nCV^#hHOBT9(b0+IrcAVx$0R526$z6@tr5>F=)(2cx?+44#EfpP^rGse&Cr^C zfO!-Tq6)#h9_vm4%Ok`-hha}gR0YF7ROJU^g#i;BpE_fd2p1H6y>zpCsUT<1>o}7* zs7@2&>^b^KJdXZ7+#Z!)P@h!cq6&p zEG%E_QVj-bc2)mX{c~ghEv?n49#JHyS96%V&JmJ(Sz5QsG53)`6qqYN&X4S(MrA-d z%5*pLg8+ZT_~I&{BUU)7NI9v`U>j9O%Cd_^A@S(qv+5ugpu8dU%rQWDmXW(3y*Q8Z z(y{f3Y1!aK-Q$7GMfRGgcl`YX0Jk4I!=_i%eRH2EdfYLtN zPsXhx%u+_M#&35la%zDgsjW!b7fT3p36PMo#H=iUSp^NW*z7dIq==xo6ly|@*_rOht$OwiIs$EK4V9HjKWTa zCygjqN8BpNy>_6j53z{MX}&xg(8cGHgK@OzIRGAabh);e@!TRIrjkCr7B)f<`7lw( z2KB%DFg}$s!a0BEZ`X-yTU|bUc0LH)&szO6UH}PK*n5{^B@rZsGECflp~LI};=zk* zk+`Zp=wgUm%GM5ilU*!;$rq{qB?^*SW2v<(=D!5O?c6YKWE>*WeB$Aj^aL3l?oOdD zpezZHNImyEq=|QPO7WTy^AGd-F5NJEAt2qH} zBsi*fJ@-`DWy0({-p_w^AiuitO8z1@HhH~q)zuL0?Z_OYhhB}{t|`9dMTah`%lOXQQ$et^;!g>3Lcrkp`3a#(?Q5P&OQcLMeQ@ocSyhT<*5Tfj1N>Nb!Y@^3aWrt4qANScDmZ=i#_tWG+}+ zRqiROvHVS7W-@Mjx?+rI$H6LDGTw6Z3v=Olp^lNPW92|SA{Uh~k@5ASF!v*g`6PsW zQ;_SZ#e-UpycvWR>+PNzyD043Boj$3dW^8lPN<++**)wInXWAJ-Rd$AJ?gg<739#{Ga69P4T!Wmq1rT49SMuL0nEYd=ehw zTJMfJhH`0|RJX5Vy!(`$D;c$4lIp z)Ajx>%q8M(Q7_+}csB@rf(KdKVvBnr$U?pvZ@kZcFtD5AIT4;p%H7f-d|rT?lEVmd z9P*xcU!YU3GLh4etK458jW`xUPIIylH*Gfyv$kz-34)<=4c8SHJOJga*yxXEKlC7g zv$C$AmeFqn=EkMN(;cpPsEAhLWQPESa^E1({HABggIG(<`42Ev6943@C`{I@NnFm1 z7L_N=ejTq8P+J$dB+(#b96k4N5f7mNqP5QjxWjOL9VV;=*|=Nh(tC|@=TND|KlGAav@vV8Z|UO5~3c*;cK2J2(xyyHpS)& zx}!-D6ubP6mOA!{qWMgUo1f<}j*#N8e}kL7+-phEK!EFJ=IT^`+EYd95`@LCqUoTD z5$pU1A~(CN$2yE;M#>Z#=YkWSCX)ISPdjqs@LwqB%DWcG3I6m9fH^`hVMpR#xVm5` z)ZfDfo6)+HAom_tye2fzdET^+NhOjFg&Trri%&HcWdgVCb44cXsPyCreWw!V33f-( za~cOfUyu!GCZG2LBt0Oz*5PohdG0x(xfB@N<65vBUkS@I6td{`UW9}#^)5On#I<35 z#=xvU?tcl0W|3#}bzUV5Q`y|MMxO{bgRDT?xVXd+PN3N~AcgnX`85Ezs%x8a{8N*T2Df$ zrQZT^MX(i3r;VKiDM8*2?o&7~J5sE9Tr{O*d)?9(U)Mj@%CFJ?ogf!n%DkD?u=KFE z`NI`1m6@7i@Errq*r}(>;I9R^a9T9kobLM`h?|ButT8m#Jnw_Kz`W?51SBH_d1NDI zLGGQVy%|0Lr495FZ0u7IV<*IwAM%eYg^QUM)6SSzYju16m}0;J?9XD%M}%GFacjor zqFfDBN$H6@3Gl~KuAyna7vd5kC+a{l5NCc2?**}AF9ax?QO8wk{#Lu4>U=wGN zM(%?U$9)0le9`-q#Z5UL7UnUBBj)(#OGuhvhzsy7zak7D^h#5_B7#5I631p=6LyZp zdxEjQK=UQ39U{8wRDh#dzu z=Wo{mB*d)W9|~}3yLc1qfZrf<9rK$ehYE5-ryA@qJ91()OcIOq=$POFDUhe`b;A^xIa zW2hA(vH}|u#7o_y9LFe3`ZU4BI6*{hF0ImbbN1oGlD zaa63ZI25hb(-MseI&KxSLEVyEBFL<$aHdICAg?X~M&Y24YAk7@UwwJRQV>?flOBC< zGHg$Wqa&Agw2JQiZxLaR%hN90m?nhURB^yQ+c3fu&jus&Ws0i+=J*XnR-^u-vDjSmubC<$echrLo7O~0MNYRfOlmb;AWkf zgM6+QZ$SOq0L&c@`bgcoO=hgd~SRZUFPiU#}=Ic zLgGAn7<(w}#3lQWPejvlTv@cnyb~cvms0@kahWJr1I+`3K2HlE9jV!GvL<0?T4gJ? zxkxuJ+H|_v&}n2ys2h+T*{UAJHUC<6}_iq%G3tj4xSH?Gl*bqw(0DqhtfoTDd zx0u&6Aq||^Y-9d$Z>!AgS*XUv zc_N%OrcjJ02AcgE}g_Q z&y2|Q`=o5a(dWc{L0v%P5K?5jRtT9U^T#_xIF;%EdUP?aRHQ;Lk`27(vA2X$wpO%) zk7_JFkwVpA{8xZ`4W@i?RuiNvJYiJ4AOVb;gJZ5TAVo-5(6@`o{av@`7=H+H zAL*05FGJvoN)d_GL}&lonn9dvJKAb@iF8He|Bw3@0Zs%RWM=V7R9lKfI5mqGrwM_d z%&@=I|3ti~;WMGNND>a?2XuTJ09@dzDMqvtrk2U&V_YT5jgG{#D?S%Z%t?eF>>%v= zC8L6Bd@979P2SP8fsBPa!Calr?$+2zq!a8xe<^P7g3e5x7|V7SksDcv$B4|GP%npe z#55r=m}}okY}5l$>nY>MkBD<81DG{>bz2M4AA9}5U_VRuY9jo9b+8kQe(njYKvYQ2 z8g9}@*x4cZm{O`lmyle2mA0ebu}9%M`dWqEEjfE*E}4AE%r)@4G_WDTkVA;m;NGSy zJ`qh7QuFUMwTPbf#??4d2n>U3}iGcgw1AKH_w-E~ybM7)Mz8ZO-ea<6{?Bqiym(>{8T z!)+|Zk13p&Vh)>r7IKSPNA>|x**AIXdWBe;C&f3g0^O^Rz=Juv+bLGBjJ47jJ7=Lmq@ExAQ<4 zY8`YO>iCNv3y}8+6(%7`oZ-lJg54tWNg96_O&^d#P;WFH2LYK7-`HKW z^OW^79sVF8xhHUi$MhiQ({~6ZjRrdIJN0j*(@sTx6yib1v^UT3P%cd?9*1M@6QJ|3 zz$k(84hbO!k837o&UO;C8$nYPPcFs6g7UZ?6YZkR%qn+*w~BO%+&W0?apB1zxy?FC zXp4R%z)`q!)Z04?ol+n;@zi$bOd-ye!ILg+_B##8wNF=aez0cI(~F4gmy-jG(wRW+ z6Ygj*dVzDmT=BH|zFM^7AQzx}vMV_LTre2fgsoon?}VK{0=}erceo&r21z{nc;%Vh z-FblU!s;bWqH4L|TlLlRq3MYn1$j>R0*K6O)ZQr6dl!P3Gsp~C@;AN+z`3G+lk6Km z5M=h!tKyc6i%(E$;k32KCH`MZnW!y1Dv}&g1Zs~@MC68Qi0?&FIF#LzmD4Yxof%Gw zD^2X{+Aalhfp9~`6gqgBzhO&Skt2y)M0>0jX9GMulw+35`6GADnq-4DQ>e4(sdOh> zn6v5-9&k7~m1D6hidfPKZD*mpMi7%s6Wv?E+`llCdQ_x~NN+^0G3iQxJV`Y(rjkC` z@mB#lWBE|h;JHnZTb$|FZ&l27HJH3P8#m72*)pMWe_erbj;wIt?Tt@`(et{v44C73!tN&Y30($V zUq`s>kudxoA;hO&#m5S}&vgtk9UOE6G(DhU;^>KSjtKv$7rW!!{AUYq`@}OM+-4YJ zVKSPOeV-Sqw2`d;^+v+3BeM-VaRl~)oMH=4cf|TP<>=f}E`GDaQ_(E>Rbg7NZF0KhFCsqMR5f0{YVjcNIXXT;ft|zi9w2qhA53#^r)spjL!7&xmyEC$CoA z7uCN(KaH-7Pv^UfNV$M3-vdGc8+9YFwFr-htSHYn+X*z|5vok#l8pnTFwYO;Cq~1) zP+q`_8*u7=w_q1gHkI#-%mw78AHB*|?*nmmop`0Ou%P??0+j7~J07F@qDNq!0sG4X z`26V=;>7Xp6MYXsnm=sQ(drqDeg6UEa%V3j)i_m{8C@Znu2|qv0IJ`^SwB7#k*klG z!8@Ms1Zr+aM$z;bVHcx{tp}a8SBMjwf}{*l{XYRnkcpD##6&njHd);FYvrE#Ux4l? zL_%8thaKG9Jzk*9vNb_oLY+W`Ud2da^L+L1qm~eur_p!%6C~g%D7gT`zl+cE9dg8pNpy3Hd4e_?ZR9LP- zV$uIVm>ozb*)}u?aMxhRfjJX(V>W3M>;#eYHN-=G+a$(HbUT@&uEztEx z52+Gsy#Y;mkvYaT3K#7?A=*Tf4TdIdal)GrZdu9BxiN@O1esSd-9-qu>02OL6(+?c z3cB(wsDD2o($z?xBh!vAg^@SLUa{uegq=5c)QyF)Xs0}YL`_TmP9Ffd-0au7@M^-Fioz%=kHW`e3j32%8T%G`@ogUh ziOo7CZ~PI2ha#H+q#ayH6K7Mw9mmYfV@8gModm!z zG+eu4wr?PErKBV>HTs08H`oK?mfu2>T=JYiMPB*NG`%Cu>DBLx1eht+wH_5hd(k!G zONGs$o)T{`ql?!Ttv>*&0LU`p2NA^9ZheWL9Ok4N!+s&`dLpClkB>#WMR+b9;M zJ$@}P?EksX=R9@uZw1CptS)BIftq2>TCYGMP{+k|{yuglV6(fUdk`tXV7w@Zq9HUM z74L}1MW8dqtTRJqABpCy7v& zwDV&*+{m0h^IYK0jfuD|8A{wMEVH^akc{E~HaC!SZ0_b}3>C;~#5{mx!2Sl2sR%M? zrA~J#tZijS{HS2A9#_4+an!s3wGqdT6agm{{F_i`shhY)=K8k;`Ah8#tMOGm4(|ac zo0`4nBTi-nvQn&N*zoKp&|5(~eZ4q8l=}rL{^+7C0PQh^`HuXZZn>c2r}-yZMAgd2 z2B{S-rmh&fFlc5;@@Y|x$$}hF+Z$exsHUdps=KtZAbXja)OK zY$uQs^|YdBz2ssLW-zBdT-Yu1M*zw$#p#a}CMGQ6YJ71Br%fNld7S7x522e!^!6n{ zJeAkZ8sf$=g4J)6wrD&rk_>PY5tEiI(qO{oqu6;VM`p6Z15}hmI#ZsL>W)(~ZF5%S zb4%BwqdvX6NJr-_=!x;pGSIo5F(0FhXAdh7};QV`F!bZYw=#jdiR5mn}Izz1nI)z zY6(^0j|4bH1`ls5#DOapU$kQgahFI^OSjhdW`b$FA^s=AS+f&M65ekFxhHdN(}L)M zB6h4&B#4!MT(T-8V;(aW#`wI$p-7&gG8m7GaOoKEoB$3R0Z^-5^7$IC2yz@0w@}Uh zMu7P=AR&{E2sf_==32L)3i0mh#W&=`8RU>Pif`)bpBi!0C{U`0sGuu06+uT*3653RU6I3LbNpR&t~z%5 z9KZ#@rYbq`yvk@OmjsUux``f)_3Bs~Z%C)-(YwYF&kc~I|LLv6{|O{tj3i>%Si;Qr zJyYV3;|NkVx?3q$6XD{shfEGq*f{Mo-e30Ne)r&sgv}pqFtqVJUJz`ylr7E^=|XZ@ zuGNr*XH77t%Qb@38?l83&3f`0R z(vo2F64}mRtgdJ-2NrsLX}5N}Fr7)UgYSx8*M;&I%(iS~{tWF=P?bVlH4a`61fF#w zE;wd=!Y*w)JH;h8@L!n9Mu);A+m7e|rZ`fRS;TRk^|Bf-YzX8AVK%8&c@(rWHroi$ z1*F|rCt%58!d#?*q=&vSw0Xk86A}KY0&;{(V`Ds0ryJpR?6V2~yAG1aFt!?nxH#QN z4^^!pK~$@qdzK3oCI|eYQB>BwOX?|b<0G$TJ8Vi6=gCofT>K{5t*I?3+S{6yF5Jus zrPV+xKgVni;Ebync*!gLc3S|sbZiK;!0jT0?#F#&W1LX`yNcgLW{qh=Tz{?oNCUP6 zfYp*}SE3@yCBt*^U^Hz7?O};UrF%vjE6FKagS+jzFn&*@ONP%B)?@j_FJfRk8Q*OK zS<4qWK&-JXfIs2Q=Sna8z83|-#WwCT7Tk_-W;->Z-hHXLow% zdCHzZxq8TGI&hcH(dY`H9%Z`6;l&g-UI}#es1lZX;#I|>UacH5cHfJzhai$2yLEY1 zn2X=bf$gk)AoC<79fYjj@`_*=3h`o_)`^ApEg&9jqny<^vs6H(uBj*)pdZu(l#@sF zuxp)c$FJN0BApwD;w;0%+gTadJmPH79ZS?{M&zitU9?M+jQAMW4-3lEUu$-X{#gag zY-1BRl>sI9N5h(n#I>u933Ymsq%a4F&V#CUGa=Xgw+PSEC3i~nx+PmcT(?PB{TwPX zXD8iCJF{544U~?{Do}}&L^us;7SY-ZP6BaN*jqKX^>)Tz1UXrjlNfA=bj^CCRIZ+V zmoOb*s_}t>Zg7mVIo$jvz}IgQ9tGgdcHWcg1a+y(-FjRgrU;=3 zW35Zv*HuKoFii?!@79q4d`)mWcXT&#$X51%?eU^0Cz*_UDzRn{2z+SK2ECLClo%^< z<7ECZ3pz_&ozQFM?FGuMiCg@%2YF4HlSI>qYs#2b1<6(K=8`s+?gMbYVTmcl`l20y z^@f@b>j!ZC@!KTDam^l$Lg8j_Yb-W}Afsg>Uf>SMVLcza^+1u*APR~MB>^s>+L`se z8jDRWB4N#f9@hi`E;3s%$y3f0;u4~5S&2(TdknKvXu*ygr#E*9q!2R6yN5nA!RVw(e!VEyLJmi1>)UbylTc&7N8vo-aH~>^X9c)9)fMy%$m07KDKriCAd8Nr zgt%I=yJqZ{V?|*qIH|6wFddL(Pq()>))SE5MXV+^Jb(!9<<-I1S(IymV>P5pI!jCw z=#r&3k>i{L0mN5E$yWyvb}4%+T!k^~uY7O;=^|6jvyC7+tt6E(`Va_bxx%>E`%oxn zsmu4aK3ToqBhXp0igMfjpa2r!NUaC9O|XWIiDgC93Q%p5cKZr~^4F|arMOXm^YDH} ztv|!zfNt-`4y*x|5bfsa>rQv0Hwy6tQblF!rTPbDM1wKuh$4X&yJhKe~v7U9XNxl{iya9u7#3D9TL7 z6rW+_jtu1>0p>AcxmjZ)4nK~6 z&1b!>t*hee{su&a@hKfE98Xx&k{%Y=Rs^}TePg4)y(lvpAwpZ+EZV$~+&8VvKMHdR z#Rp_VlyHd?09`_lpw$F8mR?4;&GaOJ<_7038M{6sBxiuVCQ_M7ymum)=B6nz_elg@ z9tIwF3a1naAXLV{A!ZRmeFlway0gq^1P(4rQ`X*P)3c-+3b4 z?8t5P2tllM4v-7hg4I$CiuPpxXlsH|o##Th2#EAhdY7NK8wI)s zllpPu8KcfC0G+i<(I>$9w<3CzK>jvC=A10uIaT2<`+P8J&gLzKUr?lB8O#&`PJ`!W zl84rb7Xs13ZT&GtK`{+`+HVv#kGXx9j^q{>fp}+??s!;TwB6WSu#=RLjs(E>3CiDM z%RqC?b1?`h5AibD#tNIQrKyO8L^`rMJ0o`YOF&#Ji7)7fWn5wSqcPq-{2~&5w&;ps zhf5*cdS%u~)%AC}JTH<&dD=`PTv+i^D8e>{nk2;POpDlA3$hf>#&ONQ=bA3os*a zd#1DNF+!*U4uRrXh0W%Yo|a~HS?C)7tsT#Z9G2I(uN8IyP>^HVneo~JkY^iiRs}dQ zW-zWQcs@dq1{~J&agm7JuVvEuK>icmbmk8cZXT{dyO5%^MD@A?(`g9{mvX#R$8=?m zeiwSE!pv6gn52!($BSGKSTqD)B_|5VbxpTuy|F<(4rV6Yc0Ey#!~$lEMxw3=UJ>D(QCCN0X78H-TyY++ug15c-B&T2jL|nkdVEV} zS$}N%ulmFBu;-3P>m(O(@!Bo?o4X&K&|1Fm6|Yt_7#Mw5rlW z%0kVx9ttw+G)UJSAL4A;Ce{HgZ&;tU6TpDrX{*>@VTz5zUj)h*iE`bvuFFA@?!&Lo zA7CaBH!JMQurKC7AD;>#LuL#gf?DndaP1I`(QDN`@3{xirNq5ujJ($$u^DJYP=Tmh zfZ04K&ChM`DJWBEa zM6DArym+99PH>=5^Q(&PS$jMvfNyxM2D{x%Xwi@FGl5+-E%adV9dbT)z7+v{$IT17 z8xbxod1GW9FV+!eXnB_(dWbl*Ln{THrm#za4>s;^|C9gZedL(%52xHL=Mfu;@WjEc zlB-Xcd#W(kOjk;HyU{-TJfZI1+?poSOlH!KcqAu>ZDwyQ_b8EQ7w$T>*MCT4(RNcY zd1Pbxvp{Ob{&;lk@EC+k3fJl0o&RzQH09Wle~Q+}i~mv7B8PqfHMY}h?JTtI2~aZ# zch_=TaEuV~U2F0qBIuX4vHCFo#ja;$9QP!Dx!JR7Ykd0@kUNiDG{nfKp>vz!&a4tY z3dj*;;6Cerq0L0ipXs%(#f0Ua&88D(EF;8UA>J4r>poM&V1pxPQ5!r9ViQq&1!ZFL znRBZCsSPDsL<-$>p>v)C_AqGDeYZ%uKxwEgR(#%JULlV03cHTIcw18&Z}9>U>}f_@ zIPQKC!e!+l6e-@_C&Xn%H>w;@h&BrljoG;o@Bn#4I^QD5A!HTiezs;kRTK7rx9Nt{@VNcv__U6I=Hr*}_2R6~H`& z`nl*Nhu>MS<7%(K5Mi=ls3_GW>g0g7Q(>?*{`^>k}XzncKMBYnK3Cw>-1;+<`b+y&o) zaDkZZcw&g5SB{m0(y(~ijopiIQ+%T^pYqgCU(EClgn6aMO8y|Sus6;IxT%dc(?9O6 zI2hF1oILa?%=KWeEd!rpg}9lLyBAvHEI}^!#4)k>yQcJnKzl4FiY@EdHDWIX{bf~4 zINfc3LAfz{xwfKTRO8s3KqCUvmKggUQF32q@7*RccO;Zju*He}1(}bX7{vF*bpmRm z0=GfTI_kV@y74*A$grg`pt5oUM?txPBTvW^U~ z?z0t|Z+ekvD>h)LX$tU1lv7GcPu8PIJ$XcYD}Zt%)$NVhK82u?4ZLeyQP@3efM;>> zW~j|y^JkzmX|k-2k3>0rDI~H1?)@Bu0izoqNl(reP4YOk?v5`-xY9bxlwzqb9h;6l zrLCA= z5mCo-UR>iSlfYakH01D~!UYhif_PHGc&2T!SdRFU}Y3DHBJW3=9@DG&p0zPvEdr zlX`qD!Wk!P>X`Xw|I36}iN!>?6ll9}i0g}K`G0opJh&jjS@NO_F7oT~H5X~1KbPYt zt*W;~WOiUIolbRE{aWO~H&mn=(*#k|QLLBO{YE&m7gd*z7(T;VwSm9`ezD|?kp6!X zCjVUN2=M-#Lmsm``<2v{{x~y#xU==(_Mc^-NeG;vCy$H|M0m29FfOLd0_Bo2@=INM zE#{P2!7_u{_eRrf&`waoC^oA%2%+GO+)J%KM-dyjel_+I?OulaJ$k3wZ1y<;&EP&x z0NN(cDJZu{xx_Wah60>zwUtfX-lCn5{MMor#B6roTtz}HSm@|*al<@CJTBLhO&01B z0zHlBG3r(3g(Ph@1j#~Y9no-vD)+_SBB)I(b+0JwQlVN-FF)W904}+l%{%H)Xx_}8DZ|sZXL@?iXw8V1? z=7UdmI+z6Ik1Yu9#F(3V(!GaUyb0L1&7!Xv05(cqvJC z8F}Bew_vA77l~s7qn}~Wu2?(W=~t1pxv8(Ms-vBz`^}aGryP3r`0N~>gx%}O%YnPyu%U%DUHI3J zS(gWO$}5C@mkp)MAb>%~~TQY0OqwUq};2-ao?HtB;gaa9m!Iawt=RoM@S1^k`(6dMz=d|5^UVk`ke48e*+ZL+>Q5@Ng zaFRPDlTrs^jQ=&0+KqA|yhx-+g3i)3S=uy?TLaj|kqQk`r@=^=v*K!6rySlyAsWN1 z)CK>35#c%|-&xT)vH+&E$?xw&0+{3x5z_155JfrlEENxmFVFz#PR5xkIen!|E-I8R z*~xf2Phn?}+^@x)qYFS9fsL`KfLfa&9ZJ2rQ6Pmx9lte}9s}WuHOc{^&b0dpbIx3? zu(L^Gsu&CE+;nst5sl-VPGgDAKTQPX;_1CuWP<k!|gJYk`pg9@(S3Heu4N4kF6wj82hGQx2r#cG0j^PH8zk ztOKQ~Py2;+MYuGbO*l}^|52fd@sSA60u%+)&o&u-m1FjGom`U1#BHKTA!|hcvK}PG z(lh2G)+g)^k+qDYOOTnSMFce!0p=~m)7u?V?0Z9~D=bwi+PfU*ZUE*H%i^VN+0Gk+ zP?K^u#C{^g4^FsmC=3IV0xZki!W%(56ZYQidP{&C(T#+1qY+?pk!#Jkq5c_%A*|0=*o6PZaxGAsig4>p()HnGqWy8h z__6VZNSBJ^if#)wb$mULqYiY7Xex@k&scaf!g;A6G78E=bp*u*y@C7W=EU*NsdFHn z5=BvvoVCUuwt&bpG&UA`kQl3{Egd7hCHt)i7mnBaVwtTVonLw{wkzHf;+8^!)YmE} zqHAwc#F923b=X~qWAOqHcTr3MnJ^Iw&6n z*uc>6*9wFmm~6!o+ZSJP_aP5)RA>`{W_Y?f+^bGEw{ds=>F++0Nlu#JPT8UO2rKP8xQDa^3!xF!~Rd<7_PF+6{_Iv-gcP6sFQry^1|W^#6TvCd{s0FU|m4F#iM&dk~77Knf4-F zYYqG?bE~}X?;-4k;QQFUiB%imCDdkS+Q&&$@Mp1%!g*LAoED|3$V&WD5yc#gheh3; zj@j2SkeODaEW#~XdnZTxqiKR&M|nMrjT#GD?2Z|l3L429-U38ZC9IHRw8E+_M>%$6 z>xeXabbRE#>l-1iGx{ssZsJ6t4CMSeWze>}Xws7k*vN{*dZpxOg|$1!jbt-Hr;=QW zlgc%Ms5`rX!5Gm3K{2!?r^|G=nq)VCJS92eGSsj@}&Jg6zhr$Ktb5x#qtVgIz zCfQ`n-d4~pQj%G05;S=-9k;~iBFt|zyV~WX2KTD%fVuIw3F?k_1ehZgbPL&VJlFw3 zS+k+u%Oa^g-lAi4m%|-&obwbW8g&>bO^t6v)>=2&dN70z>n=dHMfo+>-vs7y#Ziu4 zti^ba9#E>LLzo(UFVT4(OKR>13h*G}bZg7?V}-f5F}XEPb`s($%^k0Zb_r#Sc~}+F zCCuLH=yEIf6`&6D^a-6}1-c*BPt%ncBRI1PRXVsiK?wO{XosEf38LUW2hUQRmjA-t z1FP(>qF^E}`D2BC!Y&rBp5nwQkWLk+x{{U5Hvr=7>9*SVqgUDzy9qV}x^TyqmdJ#e zm#myUSQzT9?}U<@yo7Y?D+#9C~y$VjJzR+(lF z2{WhrTjM=Z?yy$$WfB-FK0=JeO&{yicEU2plY2sK+m5YAWHe|iHWXI~BXuufZ`VIl zrMfZv=Bd8@aug1^mDp{6C>H>shHi!9F+r*{{wUv2*!4umA9q5Z34rHq&FRGdQ~h&o zuq5SQ;lLurq!vY;=ROF?4A)jEZJ}-vmivHj_)3Qqv2Z@&>$FpZxp?VTm^;Yx1?B$C z>c!FH&?1hkv2^lyUXU}SfBBy1iNnAc7d7LIiw-a1Fr<8dd{r3iWUIB~5&YAx$a%?4AC zgLZyQ2D;*m2Lw53nE|IsW7zTjj2QqQtvIGS0b1h|e_MDbIuVM>C4=SzM5Nj@#_#-2i8 zr>4{Rwf;9=hNNQp&-3^@4@jqi%b4?vL>js0V~?@F5TEF1N6tKIT~^4OFDSm2@lVof zSouOQPXmlr%Cw3A^RrtL(|tu#4Z85u{&-VV9;My=O|41ICqBOjlu{$hrUfsC$bGPv z`JgE-76Kh4AVJFXqdu$wfiT|5K`5sr;|DisQlEI|~&3wnZK%(3UZOq zI%Ye0RT0Gw8msYVLC#7dtZPJ?p~;vkZm)lm9vR^N=FO{%Pf!kxm9K%U^$m9H3~`h} z1X45lW1M|$5ua0Sd@Pc`P|l9w*AaFtTHF&$Lim>{CazJ~#b@R!#p|MLV^UKwi{hKY zXgnU<;&nn1t_()r@wrGhx8@zh_3G>MUk~ak%J(|AF9OK28)wBgDC`1p4bsk2GTtq|nf z$c@v{3TrA)7DIRCf6A2NGlk8=Dgv!}Zi6&Kq=8ex!bXT&r3>4j_><_oQc0JI9?E5u zK=LZFC;lXd)k1A|fV<=zQ1SGS@wXt#IG`ttuFQlU3yOzDIH#U|WU&2rI_98s=I#^W zl*(-_X-oIEAeV$in7yX%lnvOAR8#&_h4=$0!Mrchtteq-SG;mRfSQ|Z=X;?3Cx$+KafkpHnGJJ( zdH0M50cjkZYjHq)QIvaI(wEGp=3fbBO<18JW_yUB%h;jkAy@oILF1I4dtK2kVZ+FA z@v=ylkV%L}d8G~@GlsYC2%yNEC1Rb237gM4C`f~I@kiuVh|A*FLZ;Z?TVaFG9Kf!;L4AYQOw1`kkl%~YwTA1}Po&?WLhEp8|o`5{Q z&=k?7^S=bSRwb#5Rb$3~J2j4uv4g^7gZKPu?41d2(7IztgufeQozq+ENYd`F#_d9= z{vzCDN`41$O zZ@^0JJca#zm6vijsQ1PF!kjHznQAmV4V}~D@r~BlUw|XvWN>6WE7~bH1mB|+#?#^&JeBAgsnfgBc5MVKtiEJR`|LEW|WSbF1v6H#w$`8Kig zsFX8l65HqMYFR|?@c1e@_Q2kMQV zg;GCVj-!fWf-A0;I7Ebo5gvkH&0%&5amWYY&Rk-_n?9_QSX4hDvNo8|D2@X@0&u&l z)%6;9Uz}YBd%AnFg!C4H{UM+-%OAh1MFD$6HRolNOwPO!dPR!hIXF#;pbBH6#@4Euv7{gAY|NA1T+Lnw{Ph26`y-<(XrYrV0 zh2@zp+k=*3rXK*^uE};!=4GuHrm}&E;MW#QXgUnR*eoIQNV2e>}z%kL3T-6!@eF^8x`89=$&n zK>J9V*&5&1|HdOOGK)QCSi4p}-sphIoZawbBQ)1T3rabj6+*h6cbIDzpAn$e)w`SY zHui>s{FNazZKmV6MBAmgf!!LVuEFG${Z2uS$?V>* zH)DE8;6H-RNohenEz-5g4rCcn!{z~WF<7S<*>Q&uf2Ha1ZIS*82l33{S`?b+1*H2* zjjmf^7rWHPi2PUnuQH`OrYr2m=t|Z>@BRTG7v3+u#N_#$7IOQzMq$Sm=X9>|BVr{1{v0;~99=2XslbBB4sApQbz}UptrgBwbE^RuB zSzug^&6WXoXBlFPh4Q`tCs--7C6o;9WnnH11`#afE%Aq81=P-+lAx6YIVre7ukR7y znwL5-FR!5B1VT6H=6q99*#9Cg)Q<9$^3&$`(x$-D$$i zK{iq)G=%6RMAXp)a!%6&WuOZ}oe$`DHzhJWeEaQbK|#-de2%Kw|fPLLyO74O(xvBdgd=CY2l2w_KW00Ng$6s*Kp z5&oohF&rNXBSX&KB>GjRH~|p9x*`ANMGhwZvrfwLa@~#iH}e#oL6oB62tlrOQ-347 z!g~d{0^GWG#Zemq0x8Vcl*euts z2xFGp);X{=@=US_=PAvJqL9{y48hfGlMUiPk z!?;*&C&F$Xj@a4YcsGI0nO!;~=dU{hxa~F0IP9G+#NXLaLsffGAlZybPE5+XKv7Yx zyzw^?PLll_%iK!>Jb_4?jG0|$$ftG%H4oB5q@UL5PM%YUd;Y@zUVwEMidd1ucyFA! z8*uKf60^`cay&gb!$iYGmp~Yb=r5-08<&tV$99kgpg2DWcE#jVCWh?|4Xes&r@4tJm!D%q z61MIj$c?S_KV8I37e+peff&0FVHX5b5hiV>*eDBhJ0WP%-f^XUL6lFLGM8#BJ}T)o z3D(Khct)Uel8c07Q2SdwB2M_Ehs$iZY!eallENwSRgo?Cy-tJ#(m92lxB&?e-iCl*50?lS8fZSyAI`HpS_$xR{0JuNKpwAMp{|BtD&4wI|M+Bhyjg5&_f7S}*B@gTd{q6;kU(mmZH znVIg%bWbJ|ba8iCZ1Lb5U~!kl-CY(4`yq?VV$1h?>%D!O=jlJ{yj{2I)G0r8>J-^B zZ$(We<3q`+UONREDb#lphk0-ASc2wi6-Hx=2sj{tKI%0q%k;R zP^5$A9V>ZCw45|E6$CN1wlkS4NR5nc85K7+WAX~G)&=pTAPX(|WI;UY+OG(92{p(d zvkj+!m^q!@jfme1u-G{A$1PhQG$q!ej)adE2@93B_(DN{%E}l`_plm(%g#;G)8gh3>sVF6-*&SHTo?(F4I; zIT$7BY=5*6GR1NVlLrx5%hC)?&WAI>(+)COWnuTO9&qzI9`S3pe#x@lY^w;4!#3 zM`35rd_c?oM1UJ2^Ka^_YKZIp1vqPE7=g!gheKx#6O(Ir={N$!nalANmelc?AdeX7 zz#K&uqvhAlh)l2;a`1?#Ag9I|sHX9WM}m-_{84FRe`ZEAmG)|Vv;9&0-(XX>0KHrL zS#Tx+ZBWnyEpRlDWmIBCzEG1a_P54gj^STw48>9{4i}LL1Jm%qSngN=w-X$Gvj&&h z$wmVCgeU&GqD_Q3FQa*+mJSf&M0j-#1|G}ev-v2*xa0WO^-)A^p5F6*UYMC?_O0mo ztrrBTAtA>&`gjPJfHi--C6fQ7_K7u5fN0!-L2<~mXA8}f^@lTJW5ihwLOoTOGs1d+ zfvgl43Ua-+mAU0U;iR0VQ&eJ8(X^EohOG-uCTwXit}_A11bOdM04)G+;Z->wxKxO9 zK|78^61D(>T!Ib^6!$+ZkC7fyq8!%=%6h02UN4AF`A&>mPbZvBJPqfcgK_H_p!5w| z#3~cw>x{A;?TcsfU&bk`Uz>w1dKO@#3s-pJg*j(sVdfs4ic~GfG-Gkzj~#%{Ergl5 zsq*gobCc#4V>uQ(hyTsW%41^BbD>~eJ}+k|?1F3e{)|YoKz*pCmtu$Wz}#MHNpo=7 z`8iNePnq8*fC{fliTp2|W5xo%160Wi5(OtB1%q@_T_YGF!uL@_Y zfXt!QbOzazE&^8L#l|8o7GXB%Tqu-8=PL}3ebbV1tBXOi4u#*xdh8`YontCJ<(7<~ zU>y95$onf3rdsF+0$u#l}M&{t?3?D|e}VP7!F!}Hn3Hv*BE4o)st*ojpr;8!C3Jv%c^Mw|QtB>#iP#k&d!`WC+@PqDkePNq8^65@Q>+F<;J_=n#MM(tv*TZs!qQ1E(ZSKN|? z%iu@fmnVgfElpip-49}^X^7H1w7no#TGFSbBl}$hl43h1f`>G}85q>#bnUU^19>vN z%=og%*+39qpl*w6vvAA!QSs4(dGfTJPQ9Ws7<2s-(5(vtli0aQ)_5H84w2?49RVNs2Qjg+G2f?{w}FU z7{R&(c@e>5M%7|x<7J1p^#?5r&sfaSOFasebqv|ta;Kw-*oHR#DFo59jGhorDcopN z%;_JAcHN?h)m0IWR|;eJ;^Au=~HplbRjwJnPIZhYg3T>^*TdJm@Gn=twM~NCn0Ak|yP%Qi`VKwjM_*_I*2Dt`; zkt3f2au%{uCeP;q%)?~aknAYhg+zmb4ggar5Kq3m`JipNbO~{y;H*MWw8qsh<{wI# zL04PsB@pL@RU1I3fGq6}J)|AqWl^MTN(!gvU(Qoaqt)Om`6tW`eZ1~_gb)k32mh10 z(uw1M8-=<#Os;nLI{PZGf?0a>1lgXT0C`KGdB8|VBg}}`@&ri^Gt9IB-$-a0C)j*m zD#9fhK+#o;F9oP&P8c6Ay^(*9DTUO0QlsDYX8wr`AG)IaRt})DKzls&Z2&ble67U4 zGJ?sg94EZPKZq9XRJFgoBmO4Z1!;Iys>Fi=U62yKMyROe{srQpyR{fKQT|Ae7Q4mW zAkZJtTBys|?Y$h4_iZ>|?tKs!c3i5Hl?7PhT%>Pn!%UU>KVPT>gndNJ_5p;4g~`mQ zONrz^=I_qfLIkqn95|j(IBStSda9>c-W6m%d4&f3rL43LnFZ*^Rnon`k3~EEcC??f ze+bza*lDBE{4XcWMU!V?M!rXdIAt9jqLko_=Q)8+xx2kskLR18u^^Iv@?rl+44Btx zY0MO1Vb~k*>rH?=bo8i0KgwgsYN0QF_c4H(L(##)lVm0eYp_BAQ`>%8pbJ%KYmF;E zh0I1Mu9L{pY|4M~q*xrfiTs(Ovv~C0wNpjP$D2hapjrZ3kIM6P{>EA#&UB3b9yNI_4b_r3>X^A%&L%s#GT)54gPQEu0 z)My5M+=W^5I{=qKF3C0$Y4(c!B@DOYZ6R=?J5#ab_xa!LdKnWl<^O`19qB)joXQy! z1)H68+fLQAi!hXKBU@ti9|+QwSd_%t3Y$yLI8ztcMUamxSntS-9N#nMCEVQU)RI%B ziefq+7PY(qLbYu5V~(+A?oaELpPN{EY^s3ZNB@+^ODj;uytr7{Z)QCjFEJSY@`JxZyA(D20+%(K zuQJ<)jr2SE{i5zVf-LJcEfteaqZZo<&gv9{gL3TN1mtRkqg(-5Z{W`KVzWb15%sn> zV2=D-+%DoKY?c6O2G`y(-<h3jZ7zUp%#jbvPSKVY8ewLz>8j^Lgt|0Lyj;Db znanj07@}#FEo)tbs}c9p%zSa05OUSye2dOU*n(wlr6-nRmmxsTCp{>K*&40ky9N9a z57x&b3lb)My`&yHECgXTIgYExaZQ?yMIkN{O>X#?i*FS+FBLfeWzoMdh#Q}5H99N( zT>2)vr-fV0NCR84r56EkIi(?BTxOvo4AB-R;|Pf$EHNvV<8l%16wVW8o}Qy$&JHk=2*nDE7sHi-S4kbZ;}+mfbFpytxaG&Ezl$ zGtL=Htv?PHKqj0p#J)k;nb1{`c2tG%FUrzbUpBDl5dk*k?%)#LT>f zD-wpOjAosx?Ni(@+WF%8j(RwKDFC=Cm2pddfe5$BG3*l7S~@3rxHqPYbW>;Ksvz!i ziAnyHCY*WnkVQ)@zf2w*qdn&FLb|GQreI3K8EO2eutmpY=0ovsmIZ4FMt3geTMob# zqN{dl%ohk^Q0JVCwlU}Oc`C_#mo;(h^1J+_qy<6`e3T$3j7=kEE_Vxn6P&xn?^Yme zLCWbp*^X}_JdCZA@q>go0WAS|ZDmD(Y?7#OH8j<7MW9pCZ3(951Dl9uDA|&&L>zKv z2NQo)*fQV%3(Fds)W^c86I`2fr)3ogvMu&wGNICMw<nOZd$0@4DB0d(mbVuF(HME9Aa$f6YeTy#@qn6kDTfF$myC5B5^-UQXQ%xTVa*NgbMGLM2*%*|HsX0(l9YW3HKx-_zE)>#5IX@WxYY?&? z9y*eLoIaPE%l$0+C0(9wH(nFb-VRIUF1@A5VlH9KzI5F zqqKgWPCA<#x&gHF!uZR}r+b7^pS6Be`iE}_;f%7;SPk+2gl3%*LnC_CT)zjQe2njs z?4=^rBUx^Z{zIMw*9p6OF(#x$PYZRs>|peZg*Jj#g>oi`UR{*`O;3ogE7}#)%1S}z zPTg^rK(|#HSwFEcBz$wZ607n}^8b1nh(BwR18mjT*_3}RC|36@e)biRHM?#`xFS1} zH8ull%#qTMY9~b#zfU;%V3<*gwf~rZ$XLWgOncp4Sc6&Cb&Rp)=oiQ=!)EVX18I^|lVG&JjP0b)Mob-P`pzmaoG@iJ48zhPGG;!tzl zEkVq+cGtOwlySn%HAnnxORn0Id>>vA=?3z{SV7R$(|} zV_J`4e}Zr!bqfId>I;OxE5}~R2Kw)!E#d;E1@6*|ZVQ-oM%gaMu>xE|u6(z|8%-K# zS9n_4j{oNyGW)90su0<;tZs=5L}WdXv9D%d97}Ew>JnzpI{Fy&7~!s@LU(c!5qq18~6Vz})W|0MFA+{BbOv?4?v5UfP4MpaWy+x+VLyUbQT)U`2 z=$p(W(}g)Lc0Bm8Y{t>ZpDy>ly<;AShunxGnnc-K)?&S#AS^&R$LAjW&H%0m%vb6i zHGE2em?z!Q{ACwNvr)ysIaT?pyXH}lT%0b_T(N4Y#3iEL`82yVZW=AP8?bAXduNRJ z@`<#UKr@6H61(f3-Sb%3Ot*3GoIbJV9)OgArw96Ch6uBqUUjdnE1<{pN0w2tEY`^Xu|RWdhmc4Q zW8M9JgQk~!bcy+4p$uCr;+UHj-ye!H$obCs3L``g>bv455te&V=KgU@7L#Lvcu`^V zi_QlnORQ1=$;K(vIa-?TE64>+&ZNX>MVLRVaNx!SGPDR{))_UV!jXaH?5#N}4|v>^ z(p0;kP^YLWNjK*g6Xv$XO0zGfigpg%lGCB^uRNl{lmg#EpZeMB`5)NsWpt#T6nfBo=;LLb^wQYe)KO?W1ZjrXA2MA!cpH zdWAS=94C}@<2Gu-;0z^PjOifkJW)r)S0Wuq1~gN51QJ=l{~QyJ*5k3CZ~Bs178* zd1niQ*wNlB?g3$ikfts()=5XDP6#OyNMLb zsb%iimnWgu?qTF$Az5kB3&?)tM?qPy>_?*&zY55z0;71YuHn+44o2m1p*VW`2{)=1 zwHl+A5H}TRlj9Z9u939OPlx!YP6ecKup7EbLC3FYn!axU+AI{9n0Zcp8i3iPbtbLL zN`m~2mZq_aXmiPBA3O3GEzIfQ0)^q{X8~GIF;LXv`RPz*oHut^aK+jOfMly29Ltm| z12i5Q@@~Q%lLHWS^1lipCEb~;#Q7Q3TN}g#W$6R?&#Xwz+sjB9txb$~w3(6+(3_Tb52vaR>p>8g+oBx(pVBU4EtAZqP2e9KiM#14WSlFI^TOvwA;9BgS*Q% z+DitV%i zdN9?yQeku3GHOD6EYfNBRp~?VZ5Fr6lfR1`#h=ax#c%<)0z9aDG#L4`Od1vYE9?UG zXl}&6gAlqF%Jb1N=~xJ}k@g_FH))osBJ|U06lvIeTL^~C>6w}dM7Gkp#!CZVDbzp{l zQV!J0I4_7TPX@5O(7{Cgl>C48ZN(}cw*bsB8^UB5!eB7(X`to?oeV7{mJ#B@7SLA+ z8X?G;<8GGBs{)W7k1G}83K3?PzRPuN)>6Bj4uC16AzpPAZWE~n!)3$w6eiD>bTl{Wd?+`Tu~E1H!tBvFCPkNM z=ggZ-uu>7=@)qbRyxWIS$^F?^y_^_Gr_uShJQdVW%ypt0t$kt)zYx;w4$vdhC5nTE zxdI!i1D1WK359Fq%)L|N(&ZO4X8GXxG2JAK6GT&exGv@$mzyDAlpb2?h)+fNMn}3QVji!gXCO>hM=1^q&Q06=^f*O8h9yRiXvX9JfI_YerKU{H!YkfwVJv z?V>PQpsMSSyYGN-7Rg(DlEs{N0uCRt7zJSf)KO%9K(_f^{FSv3B%nj~L+%DLL-cgc z7wPB2?g1kYEoEbC5!qNSlO@DWZ@oZhCJo-CVJFSIlJ|nS9JOjO{wNZGB3*X>cZqVT zSe45e&)Y)GJkF>v{hp4}ZH4>tXnKT?ZP_1$xJ9yx;@VRJoXvDh9Jk2+m#qU!1Ut1w9))y4ndYhX76BGe zQiibM>x&ZvnyoJGCd6%zL7N3#Z`JEgTRaZrbak+Oph#z&lzr?NguoZqEC%C85zdxF z2>Q}(PXJJY>C>xm?o)&I+WYz}Rlxjp{3;DFuz*kzVuW(lL z+}S`Udzg@{Q>%*qDLU)ajm;Msy`LpgL5uRmGkGF6Vv%170Gwu?dbtUfqvi>N=EMd-? ztLEKpvB4_{(rZ8&vy2z9iD)y+W~7_N6SjBFIL!CBxPARMaeCx)4oj6L zh7&$CmlW~=VcBpth|-D0*kH{02B7O)cis4OT_GwF7A3nYY`L(9<1Xo6o6#|$tuY>c zCd8>?=fRAs9$0)c2l3F^C6`6DnAHTi2#p3q+Q^>;G6o}1wmIdup)3(;vos$)BZvmZ znNn|D_YQ>l!Q_seE$Nl1a+}e#uzb7SeA{nJrrZ*-$WzZ zbUk#uFz1F28;gr4-Uo2)bZG*e^+BGFN+g~AkXb;MXQe!#`Ri&S83XJHlCEjgzroy6 z{QO7SS^OXr$>WkIX8*rD9VrVLB)%7tRUcEe9E0Hd;-fqo_9QZ{pDBbj*P|l#Ra$$+*z%jPIKV7*exTN^95K|_&~?g8c(o%+5C=;yvB&wQJBBu?RYxN2LfQAm&YThtItF_nIgv$ zy2motS3qW&+}YTQraWy7E8bnbT`?~p##Sw^UfAR`D36k7>$PJS*GEPe)aPPzflwU{-9 zz#My59#9nFdPFs!Y@KWIy}(S6D0pJTFVIdm9df8G?N|jkvQw*fns#b4&}sghs)aTXaN?dtk2a1PZwks$@KR5<;uO zZkR1XtTnI6nk+}k(Js;_I_MMO&h`U1&D9^7V7TX}P2~ae0l6?bO4UNV9Ipw4M?6?5FoemUMk@E1t6U$?MD+nTDv!!AR~&5fN59%U67b= z2U1^a`%Hs;ej!j=7j7Z?V%~*Klyru(kqEk$Tu^VWu=(O>tvwDC?PX==51pZeRyF5P zaOXjD$VRi+Ola03nBUVDZ#!XbNt|0ykcR|VU|8ic)DFf!7sg*#D^D6~uu(6Na!CsSBp_rz4dx5!cGk~6lfd=-# zM~)_jn-&*0(hVB8?i!n;^gBlGv|UeVR@;27dA>&xDLLK)@vf8+ESL-NxQMJ7XXikt zjstR0yQQ74OL70OK*y6dQ1(u9j|Xxy>_c_VN@uDNjYsIw_W=Gz;a}UIdkBAEx_k0A;qj{SsxoA&64->#2ba2|HI7db*l(rx5<- zo@mVTdxwkFt{ASc`IT4nnEemXWMfaF!$p}__>!}LQXC}ABH>Om%c;|v0G7LSk8_#; z2RZEe|1N;Rsn{0xD`?L2`XCa0O$ZXT^WS(wWFyO}H$NWUSp&ih1p8^T#NXXveXS4~6N0$cJA*$uFaSyy3K2O7??BjWD@oPQk?z&2J7$881V z%-AQlCl7`^g$Rh;nNnqKbJNz4ma&>x+%Gz7rab&1dzaYvPk9_>ol3kdI_n*Hc<76f z+X6Tv1_>$6Nckv%Zroi37AGA{DV2CcC}qX-04aaJ9h7t9qzbQ%G^1JDgJnYR!55f=#7YH(A=!=6D_Lr#3SiD#8(gkEm zREz26s7@7M+7;ANBnP&np9vt95)XlWt1z|1^$M7Z8M{G}n+EN;NrcCu7M518iFApu z6V-lm@!dgOr&^H4A4RLt>%8@Rd9&|o5{ZbshCzyFg#gZ1ar#2xr{Atm@>RwKU zy*4j3itr~j%m<}KJwN{+mG41&=h+n4Or?X0tA)V_T4}c9B3%4Zzn%e`vJZgE!7URy zP@KIlNVX%vZKSQ?wZ!5EURJ=^cuM%^nxx7>JK=1)%y-825I>C!(Bfnr`~yCm~r!Dsx+TlM*zA zMXkmuM;XGYO~t81IbLl8aHC^o!R}WY#jV=`&1Jn4V`I9@?t+{TuYq@>fZM1e|1$Hk zO8eMSpfhIzF2^OJGpVrfMYZ!>CkX5$E#+pD^S?Q!la6~AA(W2YJr~n{5@klv&Gt&| zHlhp2ZJ5;~W+_Q|^!sj5%R1>(rTksK2Mj6JQ9!PwuoJ>LS?_cPt{VjT6BY!O&JykK zCSkCTrDamXty_V13uo6Idx>;u2Ff)Zx_l$V;$zLgnuRr0%vW_P45KJ=$HZEqNd+yA z-uT;IBob(qS@;Pe%$SV)xp)>U_X0Uh**&t{pfGy~OntH*{Zj~g`IkJaCo^WWpMAiY ztmQ$Z7B32+c}^S|v)2idX&V=ZVsR0!CC)~BV>Qtvt_#DPD-5&kR5A;^<3%=_eljmp z4`MlcP4*2=&0}akk({%xJ29rLD1>&Vr8@fx?xzT_wB-hC5Y2IXCoEgWatnoSqrc*? zsd-d(Pnbnf>njg{Y0SgdWLK;;4GJFRcAMMz>xp!uuBAoLT>@M_7It{|6pfIR1||)x zF*pchmTA4xLVPa7b(*Nw z`yL97{4qLY`+LY=pj>fGvy|Jz>P3Rdwu{r5M-?`o((uFQ+d{GyfE)a@kXZR}e^F#r zI8I?o-=jrOIVOp80*pV>r2IjM<$;fd_DVY8*|!<3q8oa(xL63~z|R<0Z!$uTsN;kq z_{Y_k^b6d-C)qiUbo>%V5ivxB%9-31%n;>FG8(&F0hWSR zWK7Cb_-e-jIwx7u$bnZ}Doi6OtA=`9B?_q^%dYt3IEbv{VJY5N9^7(#o{C&Jv5l_9 zQ^L$IeOV)ayxWA@s!|hnMBfR-q6kv-|D>?LV9c(tYKgf{1ajBHGCh_&2@=j2AKQDP zcuM|f1&z;Yr$Sl+?7XfLoF2!0X-how zY@kNZWXU(=uh6dgw2ssP#RI}fuY&`pw-k1KPTdPJa1ONV-kw^0dHo^5PR6h5(bQ(0 z3&!*@eq;RW^=hoWA`FB#RY82z(O`zoGM1*UAN*peTBxIPYAz+4RdIU}iIBVSKGXKl>3`#w5v}nty z*w?Q!!|R2($Vn$mb1cQ&mp4D_XC1M@3~2M!Tj=fQE~5Z5qP3Qm%44qpa&bw*4rWdk z1~WYP3TvBP3FWMKSwIKDj3!lqS!=;MGF^x{S{U_KZ7;-e3cGzz;t8!MMbeP=SwH@* zptanrG-(9ws+dyjcon!))N7ULF`So#x!gJ#pd0=w#I>4fLN%w(b2TVA7yG;79fcbM zsC-%%t^sguVR6U9cGp5@em6gP?OUlQi%13KuPcmuSVL6eV2H_HI8nrWW53ZO!2oCLh9uts8)ea9P{N$EPzfuf06 z=<1Jk|KW)A)crJpOnEe0F6KC*teiO35VO>u#Hh~vC@-jbmWQJgQ#BGe@gSsq+0 zNL7Lg;d+IUh;-O-s|e?zQ$xne?}a!;rq8m>=Hl>s!7QVeQIq0!ky(M+Z>BeXUdf{1 za!%Fp^L<2d^Foi-N;>RGA+=n17xW zCfbD_^G`6!B-6|H6m~J>SDEp9#fL!L-FY-DoviB=X=TCY3-90P@E(Sym$XieBNepR z(w2f5{WL-5V=~(P?zrL+0JSFhn7&1XrGjoLUK8o0b*(SPJqq2Z8r>#DEEfyo6PBGg zw2Ci8@=4O4@%MI*`ETY33^w9Gk#IzV#;z?LUA!a6iL$ZeD3RfQmN54t<_#7ta*T2I z6l!Z6YCdIfXAj~}Hah3R^=uDv8jb+I*LkO%F%W4+! zsz{4Q*Vs`3JPBa&vFBxN&vhL^X3Q6nG&wEzR1UJOWc*N5*hCS zh^$gr7HF-!@H0SJ&br+diwQ8VZJp^vc4;Bb*t3zuwAizGG_6x`ka~y^3zIRrj^n6u zto|GroQ%V)epiKE!KjR4^z)D~CNtsa6G6k7G%^->fuQq8?E6 zPms%y+GgA@z*3OEJx*xkF zNCK-yLY!M)Xq5ny< zi-B?YEW(XOpBD7(@ue_xfuCrp$@l*-N0PM-?wp1AyV@W}_Bhvw;r|9B-Ih@;algWu z7?=Y3STU^m5fD6-3LV`sUPRU>z0qeb`jbF6kuHo(?-1!m)yZ`RY#ju+a(TlG8j``p5a-I#G@@MK^emm&$saBgjSL5*OoO9Qp}JCIGpG)O7XTr+F;KeqFa);XgT2 zkRy>&Ocdf8WF0o0COwtOE{+i_*|H_RUt!uF9yidqKau^z;Yj@W8UMJTcxH{RFCbme z$!I*7OsKz$}$QPTjnCEN43`RI)U0FeY+gHxc zj879rRGw0d3%@075tYhq1yrK(h#)h=<5A2Ko!`Ciljz*;jg90NHqZ-2BHz&Ex?jxch4{05MrTp;UX-q{S_L4aep{IRoGdy zO2GpwyUq58)aba`h4ruq;5|hSxnb=e2mhiIE2DWfDDWcD~I9 zD7q}xX$a>ofzCWFr0Ck?<^VFUxFjsbKBAo`w$$jK7z@9eGskKL&IwRz#X@s|S@3cn zI3iAJ0(#D?9<$F)47aajSXD`WCgz?ePm2Oc`8yc%G_l$sI&W!lwiagoQE=9H%}fnI zpflw{K2yUJ0y1f`CDb|GO7rF!aJqtxJl{Pl%u{j0o*5ajRuoQm@9RX@^^J&364DGT zJs)&q>B<~4-OBCD1!r?q?z}HX&JXIATE{YUdXrS=dsn0Ip z&V-YESu0~sFH8w(U-m*Xf{uUE9r$4j5h1HZSWSvkJO&CCZ#1e?6n0Jn8XsG-oV65Nu=jgeh0Ex#zZt9URS_#7#~$@g>drJvo_<$|)w zq@a_Xcu|POpH^bcZ!#mp*myBwz%g&g#DNOCrdfIP#PgyVrN)koRfiFDZl$aq{v1H& zNam6Qv8Mpbo4ab9t?G1qsU-m2T2du-GJtT-u#;fdJkLaYF3PE60Gw1nt1p>D=`I0D z{UFH2Lm!nKsw@TK9FtzX4LD8}m<^7cj@;KIyRrTFT%@^X>&754*Khzhu?xzfqRg)C zh3OBdVRjMh+Lnzg7T7w+oO@|dvyH7lk8HqK66EBuOc*^PUJ_t#`g(h!V;M*vZOQD> zHLP1`8!gKpxtx6HMPt83u(QyhAZPLiF9+l-P>f-WeqIyMMc!ImC%|c8szt}aoS!-R zy5)i0&N+KxvqSn5eg|guYW-8=6OpurI{W|gRv_FMX|#yOTUp#9&~=~--K4tliXi3? zB~f}(@-ab7*BJtWZl-?jIVfo|V=DrdVD@^Xz4w9I6-0?FQpcK?szir+8IiTAJ0FZ>gp)@(OWlnUKmwyjjAOgs-u#>Sn;TJTN9xOzZEajtcXt&NjjLbvqK^~ysR}i=AvXh{vevstec086?C4oPTt%(dJKT` z?5&oy(K}-tNY+|764Em3YC(?FS{*1>2cu&=2rTHmKL)5*MOsGM{KPxIrTcp04-@#` z0$^>W`|IkKTMLDmjqvO#!sTJq88aeo z7XT;hmg<@(?-T8u(TK_XRkn7!Oajfu2{*x1^aMM99i{21xQ`XnCBxNyTZ~-~+Wbt{ zHV<1nA&rkJIDak09qR)j9n?j-F#C>3H+YUxX|B{DEi_o0p^3*yF2FF5ZbvW2O?D5u7aFvOUvkZK_u-|u2<&%Jz-?l-^W9hS7ihm zkM?*{gn7dNh&>k;6$|_UtkEXY^u6syAkK|xDk(dj-57}EwdL7<6T;?}^=tpMI7Wb3 z>8wwU+eD`RuhsYPO`+Ufc|RG|iB_kd3v~HfCbENEVl!y-kZc^dBPqn3`k6jzcX?qj z(ThK&y28$LvaZ866TnAC6;0-IS!B!{V)j208G-2BEY=Z`4M@o)ELAvsb4S3&u@WOi zWZj&N38%W8BUQC$eL?@YEOrPim-gRo2?$pc7$JU8I4dkqarnC6@T~yJor9#lcvOUA za7387!(L@;KueURGFBHC2%vPyGPo2oMY>t%Mkz>k8&J}fVN(=DIEgy9q8Ky_w{=VFvu!d$F z0pJkg&roobOpa%YP?OU@QmQ>*7 zxK)5tf|o-4D4JY2IgQzN$|K{;hV$DH06%Ec%CW2nSe>5EAZ`@Z;0U9PU3Z4Ii1AQ3 zol`elt;gO%oxNINEd~TwgdK(HIw5^ch~>=5CpJdt!UU0Ro0KzOW`tf^izj#CpR83U zLpF+d!>(X(($6MGR}XF!Z8`LCbKBPxZrTmdjBpE&J<}Sy=PAi&G{a+GyeG_MO7FmZ zA=<2YAm`9~4R!NJtg|4L-h+99fhOT* zmCFoBOe1yzz4YVUx8_R@nz+*+W`|ael(}`Jge55*Z~Q32|1(J9O%cniy>kc!Xljgm z39=-xJK@$%rvMj;gBK}RF5f3dv|^#|pV>D@Si)n19bX8cHE_B`y==H2R5n8-r;&08 z*caCcrg6258y`<746BS`^;mR&2s6P%gz?Ht0{FL3#0=nd5w2y<$Jqujg8$gWbVCKr zK`AyX5YL^o_jCzXJ*X_v>wSw84w z^+ur-lzkFw=%+>bGxSc$ir^bT=C+Od!#Zz`p=B`l_;yV99u;Xu(T(zQK|CYGAC)nh z<}kPv)7x?kT{ePcusytqaXr&D_!~@aJ#c;vmx>3MT%u)?jSeWJ$>Oy=R>}z74v$G9 zoG&g-b>>)$orGl#Ku!;7HbN{3T3X{$d5=(+9rv9K34JlG`Kf)m$EiY8<>SZ43(bE^ zg|t$fuhaj^?)+AToe8WoTx)(L)ROD&FUDq*A>Bi0iv8{F>E?w%_fs7272~jGBw6D7 z+2y&rh~!dXZ-{T0*iJ~23N>_n^KUtYX9jpih;!x+SZvx2>73I&qGU%fTMuBP)o_f% z{5+>1OEh_Bq$$M~mHbBHT=|n{Jci$@r-GEi3cRc(yl>!@{DH28YF`fSD!C z>2>u#)^0*w+(hc(aQW>fgqd@Cu<1QPmUdS^woCW`?#q)$jVRmmj#yWi+1HD4(IeU- z(Vnmpr#2(-3|mLMCfdoS1p}^4Yk9J0(mIoi<5lZmw7GT^s{8j7b{4W7jG3aH0zE|o z_M<{98<~Kycj%5~raFZRdaQ8@TL#@#c31SSKM86~ZgRm;m+@D-P$$H3xt>yJkM8Ej z3{%W1T%Ho-hLnyllgZm619^)2u3xT>TjNln&RS=;s_`cUnSZnZa(p&y8VEvT$smu4 z+>aHIb(u2i6_Ng;(1t^WO57yG$rieGoKTHrrUS8fF80Lf3O3plo@9RzjWmmck-6V*DvXj$^)x9Fk3YUVr<5AIMjdmw~RoHcteA9L^ z8$I)vJU;VdUwk9l6~j!;PDAU!B@P3$Fb8;TceqHi)rL7QehLLxBIt_MIEEb#l4XS_ zHSGc8Az^MXs&g8uH1yjZ0cwVOLE})-4BBXyi~Vu$kx**I9p%{eD8g{T#d%G1HIa>F zg>2DL_W_?SxKT*Xs&wKtQy8gKnAvYu*t{|V$O>o0qd`1gpgr$HJz}Noc;U($tHVnb z_E)Hqr4nB67!V6euF^EsPLF#9@<}V^bt@iAIIBGN+~qh*fJ=&9#X!t)Tu#$y*_Dbm z=W;KWv}B7M4`#`?F#2sNG82}jv5T62e#~U;kqh?XJ?n zG>S7doS6bEySG!PkmT4l;Rsf7A?AZyoI4f{XrNFO`FF;YG+qx| z<6+Urn~PnQ`13gs$hRBq!A%M~b-b{#|F}nhQ{oo!*tq)KobE#>ADf?-f1rgAt^a%g zl4jxG7sD>dW0$387%AG#M6(elgxIgej?Gvs$)`tAbXLT)wjJ^}0E>Z|O-koQ1d$D% zrwcJ%gg@Zufj!*`0$fcT70Q9)MHhm&96fAs8tSb)SZX;R5#i>G_ zozBF{@#o6{D6mw+@nS}>=$R0Y&fp(2La*0q?+GEqC6CUCDziH#KULIc_t3H8u~?{^xx zTt=O&s3g}ia&*i&ldvVh^W!l>q?%^|73l9pxGkgFESWZ|J-S3so9j>%B{i`n4#>j1 zoE@hs?0m7&<4&vGf?VDNH4C?PuqTOsHbIOmQGX(%cv+w;g!Pv+t?vslFD&*@a9y9% zs0;AYCmP0kJ7{KSi^!Ue_Cj_TyUh)t@QPhbT&=J(KzYJr@EC}KxSp_@EOe@URd38= zp)YLJ4FTkWPs7+0@QMJ)8QHsq}_ok z|FC-;#%v~@SD4|@)86QNq0AEc2?pma1i0*6Oe)2mqOC&b+F z2vmnAr4zDm810yTqVOm&+0Tz76WRq zmJxB4fNUv-PNc*;w8>=X6XC97&Lr_}t}81lNcw|Gs>}{8P83abPRUmM?00i4xlh!I{r$oup~>2@JeyHm>fp#=(!cnRn$V^>Hp>Xj*0GNW z4NK!E#<>c+u;`!IW5;4C==e^gdk}rT8W+xjwrpiQZ~viB3v?nZD^NtdCcs}| zE{crONh+MnyQ}n_ogw5KZE=#>H+A=Bk^>a_A6zhEo3t>;LLb` z7QaYov095ggr|yRMYi7;Ce%=)PXyz^imS)>3R-OaJUqO^mylW6Scu1eL}y~tVL2!0 z@yJ(T&X4=AsJQXdjF{pR;pRNOWEWoxAZ=OTBa~gfhHSJ)&QJzpKOz2tYp)pYPZf}L zS}C!S7i45uss(>R#gq!kGv%E5Z0unEBL;JzRo_%8x)!k-^`KvX z%p&`9ZDFz15=N_Hg@j;V{2x@t9nyl|p}+p-8+0#Patv=`-iR?ZtR>7}4kV}8Q5NEE zBMq#Y8}V*wew*3fVHG>uMyWS+p7FHB=N5T66WxwW$* zpr)Ri(;qV7a+78BTu^Q?U3l%S$IAjpY0Q{Waq8R-kDfRx9#EM4x;ThK8!yT-w!$V{D-LI05vLdZ0F9Q6TCqU0;tqM45T~P+(9RZQ1`QJ}{J?CCc4ust$+KzULQ#u-M%> zE#_T}aF&B?q+n?cK`y?|)O1jaE}d2FCi=s@h{>{tJ$7HLzBriqXQf4_K1BfhL`}0NQc@;P)juvRx_8d` z;|w9rpl2%jQjz8cGn46s_)37q&EQyxt(I_n_6r#K#dacLmX+`5xK>0qj=@;cC*Pk% zpdE4|w`87}wij{9QqXRgGUP3kZECXga3@OX3bBm{H3t+jlNGkCnPJllE6XgMLmZ@Y zldcx03o^?%$-st2bXI7zRk-P2V;K;avWI*=*_7UggO^_J1WG70?Y_?ud|nXmj@vay&oL2MELJ9 zzS6hU3LxZymWGjQTTw1OI*U?`wz8KXv!%zdV&CiowIr^cteAa{uJp*kdDXI z2GL0+-X;PO_x0n>aFr~~8Ps1C&Q>ohiBN=VP5fV>E|#1Y4aQ=t`V&^MF-&1Igok=` zs;fh1HC*qJmP3Zqv4>!%f!#e9;@=ejb9(tH=34_I8-h?0?j#cSFn5cS6}Ajy^Po2Y z?h)j=lQN4@sTK#U36@QUwn1bdzf+(y#qlIcvj+wEOZH>kj3%X6X)Q1q!7j2AYp)G~ z{LvyZWmiS{d(Nu4jF$l7a+9Dlb{fIIS>^KnCrKUFM7VFH&a?hH#K{E8we6aAW4Nyn zn(>96XnGtcz}$Cnb(3CziO|980-9Oro6y;o<3wQ|4${8bN<@5>30GBfG8>1EgrE?3 zx5pdQnW9`F$tEZ5hU)Qw;EV@OyK1pOOAavVp_Usa!1-4=c9BcASh^Xhos(gFsS9xG z4E$1^FcX&JW5I3}Xw|CGIT|_}Guff&p287Lh$g~%sS>{*ljq2(C6Zx>xM>q(b&FtF zIe#vU_Q4gZD2>g(VdLYAA1??)0PI9*`LBplW5>MwpmBsPXiTrwX;8s^F3_UT;}|1i z)$#fFjL3(HLbiHGCqA73;R>aX>foV+MdwdKoossUCp}y;|3pB`k9k%XsY}r>%xwl& zgL*{jcp;WlT1VH{gU&`gI-gF-&tD%XYebx1(krwM_)4fFavqYdlHa~T9+7PmR|H!% z>pw5lovQ=a6R{zrGi;fN_TpF36pz8Q9-I8Z1X|K+>?OhOJC#8fTpAP!^U8Zf>fqr>MH)u6H0BR=c(dtQCY8IcEx_=Fj*VjzIaJUUS51z)pZA9 z_N_pjHxnkx8~pMJa!zfG-1G-b-h^eXGTDER*cyOBq8q`hw+R2+E^CRnTmTipmEc%@ z8^SIM3yw0r6OI+)=#+#r)GGy8DmX~#^mWotnz5wj>WdZr_i#aG~E3n zq)|LQ%LfisydmCRhN$w-ZkH3je3nJZx|D4C4j0x&4 zBH^fo_gIcp7@-eHfw0XE{)4<@ABD*S|Bh}wc}cXCY4AhE>G zj@g9>E>_r*qrFLqsom`}Ld{ZvD-5_ktjDv$+&T+5w3F%nds#wN>`*S)g}==)X2BBn zn@xn#N_GqH3xUPaJQe<_!cGn|)8xYCv|WKH4y#MvRue%HTJdY0I@s9V02&iYW-39a zo+X%%(j~0%yF<8sJE)2+MY`2sO+v>MO>1MRR_l%_dqBC6J*_B!b&+GnZ!qLWI~>G#R0g8C`hw z4lq}0eUSNXwCKjf!Ojbj)#6HlPLJ&nHeE6+8;oZJyMUb)JH7l&TDaPvGKS-|OPHBbhh-UiNH+)^^m6a)EQS3AHY*UH~(Z+gm7*55X-Bx@NH9!-qChfxr*cTxFVdOf4!j>d(jc$&8~})T#^XWduUQxy z8w?=xw5AoS)9_~r?b_Cbw+ipWopCUbYa3M#1{N{u5GOX7{;(-yli>wOD zW-593WHp;s^%n_u^Wzw#v6g&AsCks_6-jP>Y#uN7Qdx=VLMdt$we4{VH|lgC=^kzr zL~RtSRgU)mPgGVLa>|sh`L`bjNZHYcaF^!zJO)0@87Oq{xQ9U3G{;3+Hl|I$;wJ!` zMV%tCh%O4riil0AtgB?}6h{h&$?j@7UdqBP<0eM&MB^-5kVa+V1un5nUl2y@u!>G4bI|+F3!m+e z(Nq><7ZDa-y-MD3xd8LV&Yat9e}#7Hyb{Be{&-S|Kk3!^UmST30Ia2H^eGYk7h_vY zdp`Uvpym2cO`yaG^(I5`IQ|kaS18J>a`avb z?M!&&QBC|zA^xh)D=TSS|F(&-LPApPp0~X$kB1vPY353ChA__?%$=9cfO1|M>|&!U zppgtNOJl0S@I6_c=57<=h*R)@63>Y?3+drzc!)2Wm{eBje81~T;<` zq(foLQT_#%ydKhQ75f={7PB;Kp3j2R{2!3+s5n&$v+nCMpBCS4i z+)bda@_LmwT1n__K``2Z_CoSm`DSPj*gn=b4=uL1nU3 z<~H;;{xDa%bEBz+qaJ~XrdIBXi?VRb*zxhY!sZMe?)VWg`>8Cxsf3Cb8btr{H)x@b&vJbgMCbwTjQyIFCw z!Y%+Fe)S+4ONoW<0?e8QY@nwZEyUT%&r_Tt+MH`Yl1^Ov?gld}B!%L=E(5;@g}Tu4 zir*ctH39U$w24#hAx73FTW~Td_KygsU4IA7?3^R!| zr$#|E<-}cW9H1~oVr=T?z9%E=Uxb*k9`t&Zc&iDdWm(LBzrSxqf77n8)24xU$6)p! zZ!6>Zlm9dVB~|`?4?tOX104OZbRGI2h)csoYdv0sTLNL0JEKjusT%la{wc<#y4rfK z5X*+kmUwdGakPhl%(oOx7=#OOQe35COQEYFQw72k25eAwipr9~)rlOfz9lHDQ9bp! z&?C?uW3We#KZ+z5ZUbt|-yQ7&oJ3L90Hk_c6Vg$LTJMX3TzKh!sJ0=Gf=~%9VdAPtYh-{DW>Ag4Xp+pC3En_!WpqV(paI7J}& zB&nvmIdpBF)d@`Xd#lt_(^f)Q-m!~Ic`zET<>WhcOdCWd%~0gNg*zu)5pg3gc0zYq_KCT-nbjU`@$ppMXOaszky zm!K$gE0g0j3R@)II3i*vyT{9UTuzF+MVYIXG2>$CS0POn zbWT#s=g^Jx2g&JT4ztCRB4LtdQjfV`hj58;bxREPc00TQ<~pfyNxs_MWyrbFzSJ15 zRU{X^9dm=YuK7_i##eWHlPDBf9v#+ti?F3AGmLutLx5AnR78)lyqF{8$w z3B|9B7Zd1Guq@TBlfn2QOS%j5(i7izHht`dE>+lZd$fB89}NqEN2#-Mc8GL^+%~wcihXdjbK}^ z$UIT%YN|y`fHU=vnbd9Z0X^5g>W5%vsiH?g{w`Xzq8wwce?vJ7&U&&rn`PYpIZ{sn z)7RKk6s4G|?)ATqAS_9CiDa#gu+GPhAqSSxCW0K;6q3V%qFhF9Wy%9=9F;|2Y^69i zi89kD_#CsTy01ysbTQ_$EHdi2 zO3eNrB3u4kt&;qQ2;p-Mn0O7bjz`fv2J=eyAkRI24r1<;Apmv!wjlG%G|?5_>LJg-vF)#v-B86$0mR}tXKC6{#mhbA+IeG8oRdK6mh{?=|H=&q)- zZah>-<5PJEp@G5l_kvwcy#@kskQN2w4g-Nq zB4ZVymYE)!uD0_ks33A{c;ebav^m7$LHdT-e*w|pLbm%UY*{iTQ7`ffTZ<$9=P0cC z;$nr($aKiBMP?#E6~|q?Ujdu}WzkNrI@`wXs4PWqSC}@f+m$SqL=l%qePk<4GhA&B zBgqOA4Gg!Xguq@I(n!7+Mx|>BA7d7QFeig@je>mk5M*J{2l132$2E~6vlIPtRuf^VJFaDX zY_e#cljcjk;1PQZBPTB9wc}P4!d;<_<&4G@&aOcXjF)r`7q6Ju=gQtWeK*@s=S z#G0{0{$Vo9(VgWhmdqpJH-MGP`vRQ3?p(xwL{r;ER02IqIb6oo+A_ll(`bt6J=MoV zHCnCpC}_Z!26W*DGB1J$3L>>uEW=+_*ww?FBsY2=3CTFd)}RmzE|W(o6tVCr#y3p_ z?*pov1(zj`vu5CA$)l!G5@;D=!;HBV3#w_q!Pqy}tIR-03o{StV7@2*C!|p$ZB!&? zdCN5;$x3Q4O3UZT_vnEhb*oE-`9s~k!Ov36_B$Z6rVDN~TUI9tbRl64PbYILR(34H z-7g+^@z!^2^J_bTQfv4@AT5!VQz4dK0m2;o`G{7y}%N5sGx8Ftdu~4Gyh_tOU@QXE}3|2Z&M>ggP1v2adRxSsB25 zQR?_vq&tl6kRvZ{W3|TGs{mU%_I;xD;xz*Osa)ROCz4hsm!5_AUX(NH9@M&E?NxJt zXD+DE0|mGgG;y}44EQIk2IeF&dy0ESrcncJ&>vTaaDnuY9|t0u_-JM|-g9jIu$7|)uaUfWYLq$6|*~u1o1gaEQ3UrlN zAxJ`kEH_Od`0Yu{g6{;Y32^2WE3FM-R?(=GV=d86oBbEI)f6DE6zGQ0Q$pHh@ zhw4;aU0p3g%SF;ruKrUfM)bIeQ=+D@YmNewC3^lf3IHW1{Qj+g%t<6w*_~mmB9OwQ zSBNiP6T)ScdngiFer%D*ROE1MBuaoyA1B{`5Xj6k@ud zD6%+5;qtg37oiL0q?Ka(wZZb_z&1>0y9f_&uT#WCoUcvSBsTAQV6G-dLyVB^1+XGk zJL4P$Dfo0e6JT6hL_}LpPmc8goNovB@#L;F_7g@A?8O)C7v*;}=BT{ruh(u+BqHZ5 zqp{Y8MI!3MD6qIzE{tqDSwG@!5qbQ#ca6jcqG?#2)U!ibdn3r)6ZBYb+$6vaUqkcq z7m?z9g}HF{jUilOZN6bYmsN#10an8e!jbz|RPq|Gbi!+;80^ z$R(^`l@e{66bW?J2i>P^Bct^hi_tyBIk%0RHuK-?!}{WgOt6AXh~qNRkg4tuHYb{E&Fw37 zjNODdCk*=JFFWoLL`itrtR9b+e=xgsp!Aq^3qI!#pW&}lYcZ#aCb3SG>09kU*j1Mw zU_Ugttw6UnkG#c6BHgt(tYG&w>y7}F4L@u&>vU0$J%k5=w?(=D^6Z9^KmMKLr)%2r z!<~rWYV#tRr2qAWn7drYW!NxX{i?)dGLcRwe=RK64Zr*nS&k^7OC$L%hBVL>~SqY?|?o0LMp!l?msIdnXRd@Y2A?m+1@=9h#u<#xxm3i{6xR<_YtY7fVgCxQQbO4lu5veLR_gTLy0%gN`R?| zd;KTuOANO!1H*Q?6DnZjgp?XvDol}(j(I4dt%%%-*QA>K;XT<7=87O`N}@n4Odu)f zRoJH$!6RXSdHkxXNxiz-&O^8&UFIPy2&l!oH4y&oK~cS62Vs}Lljh^Rv;;;*CG{LC zpb*A&+9|R0JfUW|&Ud6&x?PZq!G(xet`m}6q$a#2hsRACAJ645uR+ISUBsd8>AAXf z>c#&Z%$6I7G~eXeKDnA)wA=B~#tlYAL~e7;vUIflmLP9LcFo1sT}KOcQfZK%P}1lcxvo~yXshre(N|=} zgu=4DivTw#`i0!Ft`_J<8j-GXD8@HHnC$Qy#ay~$i;`pBI7?KkYje_{UQ^j%05JDN zBr0w~3UHbT$vStxT8L}O`$@^ib%MyI*`uA$@`FS$&(n?{$!6@%exA6{gNp>ie4ePQzG>DV3_oRAU9WMKN84XQ=!vnqnrzhaOt_~ zEn^F9>bDb0f%HTY{onwRR2Q@LcqsoZb$L`q91lSoWc7xj{^z2SZpSB%VxR|7fbI|vQ3L81nF-C@OVxH6=nOJRBK!R8roU+ zRpjNjR{+VQe2bTVL%5g@+Up3&^2YL`gMiJ!+BD8BQbOYe=AtA|>#>s%7e(po!R*6Ps|fdWG?$f_d^B{ojrijK8K$~~Q*@E0jIZ24$iFEZCs)#Rl3vgqj_G&4}n(gGGKN9cuW9)aWkh&+P-;@ho-$a&52HhU7AJ( zJIa_*0=$38y#KlYGXhmKF6K}0-{U4th+}_C7>=?3Lq{*d>_|o~dXn$H5|c$hdb#Em zVQx+)6$vaICxYZ2*3Y^?nYTR&$hp!-Fkli*PRM3l{}Wg}8D%y>%Z3rEMm1uK;nW=O|D$*^)X|wm!p|1`kPY~+T;zCEleq8_z)TNA=^(=p7ZlFIhe19l_%T`lBrq$i)-m&mv}e=!CV9b6H(DS@~?fxHvjn;_@eV%7iI*UAXE0 z9fa%0n?R`Eur(57*63ibk9#CS@(^TfH{x6YIa$`mqad)iFP^qP8|G33Y`U^&Z{3xx9?0XWFC^!j}E;3y9~Q%GpL=JR=$wqN4AN zw?$BtG-t=67eY~0j)LR23g@26)ZE^c6sBhhbz*%ya2cQHKR6wavo7LWOYxmwb7v)J z#(kIY!OURVU;=ue1mq{P60ZnwQ=_?Mmrc=rd?^^^O;)SaqT8tW;7VnGZBU-(B%zib5J53U*(=NOj z4Z)cUFoC|*jK_tUcQ{dH;C?57gwp+k*{^}fC6nt|*!EjN=9HXQppvGqoGg$!_YU{O zSqkS|;BDGeob!hQ!+npnWp@k91E$iHHN_Iwf;j0mU1v{609#!L=mIwNh*6v*pw(Mp zU^)nm^Mz7^5$@W|bvPhn{(#tC-H>E@8lHr5qt zPB$fYY$@9HvqVOM`{y`mI-vP(6-}Y#mW?H-J(jxxz(pEvbVweaD8$K5??PSGCOVCj zVa{4RG9_7SB#sv4g6W11x)M}_vDF{_2XR)I5<7|VWTm#%Rjy=iv5Ro$tgDEWWnUq* zFj|ggoFpQ5G&JgIih8%i(l$|+X>B(W&&6fJVSW|u|L}QT(?za0yTmZtlWx8mj|UD9=iq5 z1(cCz{O`}uq?w)!Nu6-ZTaCakuHdGYA0nK_xJgN@S^KU6Nf&}f<3J%)WjgcDL*>`$SWRGM z)c8vgzcwx1AAd@S>!1@G#0p)8*zIn|E3A*=PeRmn5*y29It}HuG?Va#=Cd}1I%gbnt3Q=u&uQv`7Vdm2%(|U1e2{A>> zr(zr}h&*+2$z*VbNT;dc%3k_wA-nU*-{pQVcPYd<-p^ql8`lb?xXdx}^9KmG z+Ci@u|3d)%4aI2EcXvGq?Nl{E(=(cJt}yq>e$Kh#bJ3YgS%)Y|YKOl9J9EYi(}gq~ zlO6(eh69yRR-^p|kQnyrX(H}?7}^zUSn|!ON&HJ;PP{Fd z;;i-v0DQ&?c{6r=)L)BBP%jAJD;80l0zOv!%eIVl`BMRI-F~<`iBw4E3R4UW?W(0g+9S4Ki!qexf4W{gSZc!7?D5MGb} ziFU!e`qeAec&3QJk#RLX`8zbcv3*wj(&JvQi*}!3 zoU~j4#211+7UYz#*@6GA=Yi<1IK_(JDC}fWDlzmf7vL(h*6QXyPMBT*b7|nRw2Gey z$*I;j2SdF3Mi4dNCarqTUN4sEFf$}qEf)&&Up^60A3xMesnR#gePDT$i_h1RMvLLUb%qN;6wcHNHAIpDPELb=9e2s{ici0Qi z)MpD}4>?ke84Bic-tw~w@0NTW*d@Xah-b?iB|xKLyLjUp#PCpB0sFLl-h`yPaV;a6 zsNNvj=~8F7&k+H8@H++PR^x85n7hqe1td=zvSd5Bgz$-+Yy%$=fs z%bOxxSe^Xw%Jx41Tw}xw-D2HZNY-I!Gtjz-H)AFoqp(80TKMfEg(@d*v|5i43ee$q z1+N#GmmX=@>sZCtV)z}<+)PQkG!)YWx!|~ai-X>UbXjpuiuURS0WhwWM9+KBx$GR9 z(tZi?FX)>X@AoT8P9xn8-|wGJ1973fAN!8mM7tQsZ|U%U&i4z5i-;&Su_+PcX#=*S zW0xJw5B>$1I}7g>!pymZz-qJ)6>P;JTpPVoGC`y>vPJ`U(H|5EjH0$gi}#`svr12Q z;F6ESn%L*VBBi!Ic`+zv72pZocBrb8;7;}yvnBqQe}ofAlg!hOUnit zYt7G~nGZN=K|{emboDcY5l2ckjmw)vn^y=NsFk=KDaf^uDVDr3RAYgE10r3^pKgEL zC`t=EI;7VWZjFHnvHs_fP7q-}sn)83=ozF&e?rgyuQE=ulc$CME68&fx;#G_{;U6h zr96=`;to;dInv)B%YH%FnP4EAyeo|eGGDn5uJLw&kesB%a2-8-F3b~vN`sp;eTnaX zS->cZ(6(A{GP~fM4R3VFJ9w`k_wuO~u9!q~rZ{Lt*c``kf+*=!E&;CozanJ`bZnI5 z%sX}y?55&ELtFewbgM#k3v-7c7r&2_y>yW9xG-0ejk-K1%>6ZpxsL&2>?hL2V zP7#oMusqpaDB3A#g=fZI;2RK4S`ECMz9|BhOPm}u{}$J@h8 zeh=Cc15c_^n6zwPFVw7%)v{ib{iC2(k3+@Rj7NpIo)UTJ?W3{o_X`XyvP>eH@pEBj z8CD_e>9!L<={n?N;x!R@$(GeBM|_ld^6bSw*r1TeHYL#H5D7;jKOlz7!wY^gLYpqc zDd`MWyC&kDQpS^l3t{ITAuhKKR%x?M<^W3!!|-Y=5l*N%+}@57K%&e+Lj4;Zp*LBG z^v~(vaCSH;hD6YtIDBA1d`(p54jm(j1Ww zL=df(t~U|oh#waJjnvcY<4lpa2+XN-zksI}<^iBrVT&I#6sB;Tgd@1Um#NWy_A~Ke zrdYZ<+oC=%F<}LDnj9yIaChN=Z7|LjZLTxlNpt$S5SKPx8=@e6KLW~fhYUfO3N-m1 zBA6o7>(YuIQAX0{O|qfp`SS!j1CD968eX4&BPHgpRp;Y7rAz0ATZmvfpe>|y_C+OB?6i&&>MHB)dYm{>u4D6msjVMvB_dm#OM~c?2(2G7 z&0sty+J)BJzcN2~cM%{Ot*tKaT8l3RMRCSWSUaXEOmTSLwGpRgLWWrJx`@mwuA4Pt zfyDt_EG))&8cltAIf3d9I;4!JM7i?$$qkm~CBV!u96E| zZeT#i!qFLr33Rd8<>3@%D2@~4v^vn5JS@_k8l!TIaAVzNiht-W+}az6i9acz?%_5j zJ$!AOyH4>h0& zr^q>fe_SY9DXu;--pJn&3b`QG8|$uA#KnozKx`zMxcE_uLlriQ=`A?Ks>YuLX+mS$ zJ3Vd}We&3}>$vzOA$efyQ5R`Ez7(cCS7#hNh9Ifq?JaIqIQQ4&&-l?2;3^BJ59hB; z4CjxHU7E}95#*lN$!08H=$N<)DCMwYY4wgRg_s50+QR-#G#O808h%P)8UxL^USIi7 z85P5cwPNd4{h4{W%6qWqi*zxO3z*BU6hP8!&!nA?r$obg&gm=ho(OYO^wwX}{Knvr{Pb@r9(= z^rw(&?THiOTZNs^0OrmZVl2HTNN!W+DyD&$B#0lhKJ1~e`G!`W2V+W_2tq5~(XJjRQn z>PXp9SGYB-5M)%hsrh@e(jK2F?8LO$(LVC@N&a02iI3%QKNl*0zE+VG>%Kjn?I6s* z+fp3wT7n8aM%o0WPX37S++}Loo$4$#f#wGn^TxLqo6}O^T zW4PNUNb`eM+V!VEsxZ(nc_D$FHhMP#1ml{rC_kZVx&ctC_J)S%JEtm|(G zM-yTj5iTr_b2aIlC&cxUW4dNsFMt9do}goTB~uLO$?HVVdd2S&C|Q5s6GCj}6Evqc ztPe%j$X!~*hBkmSBjh}YOEXEi^)JD#iJ2pr^u${NU8|}Tjj&?IhQw2?^uFOoB3gZ% zqQ`GH0&rD&?FVY{jX`KttOH|nh0O%~Z^xM;9l1w-4x4eA5a)*^Q_%)?#?OH0CCDMk z+HI~){1aY7x{w%X*{g))aU^TPct?Q75teOO^A}&a7%(3F?WP5QqaKbPZxY~QV7;~$ z((3m&1EI{kg&8j@>>{<}*afv;Y`l2^F^jipX4_m49H&K^IENEuULs&%Xru|evIU?A zH2Pp{G-UA^Ckllnv<tE}^G1L{$Mr$43+=GlXVxl#-h zO#(zrVE(-y8I4m%Mqfgm51QHGUe3I3(ciFiYDaJUHHYz|7ISV3o+km>qbw%cb>PJx zj4W0Z;G$3t?ID{nW;-w!h4m`GWxdY!1*naHl_(ntG?UP#skdw`B&RprY){*-gn39< zW6t3%8o)k9ZB;`O>1f;|NIip<@p*+^3LGYN4Plqn7EA8{==z{<9*8@3gl=_C`VSWS z0JJduxf-AT0>U}y=9UhLSKk@P<&yzg^00ZJFh|2+k#W$7tIKGdU2uB}+4sg0V~gT* z*uMxPyEJc{w~GmOB3X53g}sCO*(eJPCOWX>E%>;_@x@}_4E zZ#K6IapAR0;zVgEKHMG5g@+^9%vWN$J%G$P4j0(dG5BMQKywbMWlHQSnvp?MsHfc` z(mib|D^k{4d@R(7!znsQXZ&JMAUCG;9+)=tYJYj4UB%{8Uyco z+%4KwLve^xop^PxB3a6u-qp~C4 z7o*b4^Q#Azky!BCtiJkh8HvYur^L^HRb+`_mmaa;&1_*gOH3WoRz6-4NMo_cjEh6| zfhZbFew=<+BFuZ(?Zylt<|ZwbHrwY3$_eXnDYnu)i&XyEUKwUFjXQ+6b@Z&KlsL=- z2kr|>0T?aSxLQOmKxIU}YwpRP@nXQ`-U^?+Ht03_!$tBdp2-JeY#W3JkuFSNaxK7- z^=j?!%kKym+JFi$Uokwa#Gk9s&YLsezRI*%yjG?_cFUGzFSe;Gh%_T`B+Pq|JW3^q z0wI&KsB}P>D_FM4DHQVE>Yae@c-gyOa`9XhY}POVO+|Q-ZQP?m^IQ#EaVOCQK>xH7 zcl|n)n{KV>7U6^ul68JZ&%-U2V0S$(QS&@99*U0X26JXK0(qsi{dl1+Zl#?IjcJZQ zXC_b*=YXz}{6U}@J8s;>n7;>-jZS-KbSP++Ab6*d{hA;OiB!Z>8odx^Ii5G$Vjj`V zl)YGJj8)hsRJx^GXAyP5w&>%76UAQvd&gSrEy(F`dt)>`gh+{f?Z zuQAddW}k4azB^wgmGT4SwsiH_ zLx4NC=3ka+=C&?@B%T}<$A~CruG@-CASVfNm6HbWEYU7L4;auDk?CgNzrY5t2IxWz zGe$A-%ikZXC_)}q)wC!{ge^hLa4=()B(drN#N;>K2aIb)P$hc?VCX-8paF!(-hMq( z^p*gR7sghjT5Wl4h40&>?upy&Z7jnwK3&_H8TW=-%{1zQT`$u0jKkxK=MLDVYXQ5 zFen#O>SAdGw=N+@JDxc3!6is7;7jH_doIA!2F{1$5|L&P?;%Ucu;LK~f)ZC7>1~9# z{KJjmyyu_oNI*9Rd2u@~(~LM>d5dL_D!!v-!@;R&=SvgEdLpT9j}#TdBHSfW3~~HV z$9S~F`q=5{0kC*pn0bay1^vu=M>`d^0E6*CCi?W=)WOG;QE7_)_~EeyAm?QD*jRvT z&Y3W)ee5NKBI24z4Kn$-A_7u|c3am7$@%xzdFtya0l5vZudP>cDRcr*9$tuF>>b+U zY+;UuKa}LZV};)WnWr|zkji7sNku%Vt8@~&lc3CIESKUN(eRBIni_G)$;HQJjXMy> ziDnjJXBRgr>o6bV0y@r1&zOdU_3axpyL*+A|!Jrxb`I7f&Jh{*+|Npi?TWxTG+ zuuhg|JBN5VGx}Rn7Of%71(G3P)Xs&5J4}x~ZE=z)%8p~d-grucGnR^x&Ns&l5I4n? zu`9*`B3)Se28yRj?0udSkn_AaS_ILVJtS9NBvSRn$H4US9oE&?+Z3hgk3RJG*w4QRf$hTm~+WvGYeGM`amnOEn4BR7(gP)-x6eO5f2&CRM z`UVb8E{AflFwsiZ!W)-RbAT~GNsqsxh=rj>x_q&~l|Zl(NiNn^7+xUlS7M+1p;76M zHicb){P-`j-RZ4Axi$9MRYY(P=|%GY{?$&Dw=Cj9h25Chr>sWL3qms+c}QN+9=`{2 z%M7s^#*w0(X7XTh4WznDx^4eIQEq2BG{FMwTOm%7xe!O~opH$@3P|?nXnf*BK~%R3 z_2KWYB}{cWU8}~wM0oz;Y>hSkqU#*92QBJL3KO#jktIGC;jFPgl6lACHx&OOKH=JW zj}l;+(WPDi>SY`U#3drkO)38^KOgW%K(i@%-scV_x{OPNng_aQ@rFqEPpM;Y`_qU; zZv^vfmdq${h=5sgOcw0=G#eN+#q$DkvHMwMSU~;+;&Nakz`Hedy2yaQ%tmsM4e5v= z|Jgmv1u5Q0EfG`_Tmg((ZX!-=Sh419B-&{Yqf4sGA5|P8FgH3)j3@RuQxFx@9L&hS zxlBgJtnr>`_{Iv=6OCIST%|Vp40rYZ3}ANf7=Z20-Vl}xiyv&Jqg8GNF@usRlsrIF zhaSQ7gkfGFJWOGie%$1>C&clh&31(QxF|WpG7**+;0^hc#CXeJJ(6SzyunZESp-SgDs- zawb7mcM4O#Mlg)KGr_&{Mtmf~9PDDz?TiU`7Qocx53eGi;KbPY-Awcp-V^>xv^mqm zQ`Fns1({2aB3o7+NSp@?WvH^wq1ZS@l#31P2h-z7SLPT@s+?t>em60kDKqX+JTDqv zwK0`^t*|*EqXw^ybKC=Hmb1L~HQ1z27Q{a&DRD1OnEcaLF|I5lx7@6}DhLj7vOYcj zEy69zS)?Y@!F&BbE}rOA|Y*K7U|7WrqA%6j>r>#s3t}e48?E?U;N&v^lMH zXq~u8fEhQ6{lhIHDFJGl{&+})tAon1jzi}7$pb(x6h>JxvY9M|KM>k`V)sn&z;Gq@ z%S4hscYUx(tF;L~N01u{A9|dvqz(HeLS5qN>Rp$WKUUg!kpn(FTK>oxW)kcflk@(S zAE^mu)UwQ)Br>-JV_Mz+4}#p{$+@n)UGDOb|Kb9qgn}1Ex_-!vb@WZMJq$wqa6B{^ z^NYx1iux*n(*&8LddQ$R?i0{rk!09d<&h!^X5}19tRcWrm|!F#jtFTLExWt8SD2dy z54`b3`4esd^ziOMj}k!*joM|i$BHCz;HyKhy@j}7a^OE2_Y0u=VW`GXzVGqk2P_ce zwtB-S3WPyfr6(OwA}BnyC%#088Gt!40%0}2|F;6d2Mvd9JW?;nC7;M^Ty>FdXl@>G z_>o=;I9aG0y3WJCaYO#8y^9?XZ^%AbBr9Ex^y_B|aMZ8ZQggfI|`S zfx^xL`!DoMPeFUoO7h3V3fXV2rvYhTj+h4HYY}9TCT;FWJyRsWG1)*2izaS*f9CE? z@ZNPLz7gRDd!N zhh*_fAdDuYagan_5k+Ybn7DxZLJ?h;cUd{BvEYkf?gHGS(EKfc|CwV+?Fdo)ujiLy z_Lm^cO1A!N{(d?F|TbyFs>#bBpv4 z;O$ft57k&kP@Xkpi$2YWw`aJzmBuwYyIxaKT|UkQY4$S&kQD+V{RMcS*4 z1iAXGxYJ_G{}%<5TkISKnXl}?v07ng5)TNZ-aXvxKKm_)Yu)`=;1hQ78~6;VA^&_L zhzg>E6;Hk_V=z828mY#2C6v|yCgJ8koMt!IbIw)Rm10&vgncc4tYXyut-@3gP2o`N z_;wMqo1=Q3BYp=!Jz7t##08>Ull1sf>LK?Dbf@T{$URzh{w0h=la2Y3??SjrQX9(y z^ooKq*SKvQx0O-)aTcoaw&;6AfkC4P(wpb7u3a3Wuv25)CAn#Ze-@CR?Q2G#APRsF zww}0NgxkL=uNqh{zYpU4xr^5s3;e5~QM;}cJC!s!)Z$ptPC*xNXq2age3=2)X?lipCB_4vp*C$@`KVZupC+Nuo+`N1al#~hOw2qMRe}Q zS`)ajAjE&A?Tord!kj!7OWDwM*N=*%*j%>96G(Urp zI$t45Oyzhv#(YvF(d-&#ldL`M8UkHJOuCR#VnZPwopNY*oNu6?9WJ-6i;!z}#9quc?!rk4vbEn+BaJuKgcknd?;A>XiG1x%DL|F;Zn<`hZZU zhg8+4_e$FS3j_m(IQTkJVRK70fqyrD0ip$9P@2wLMfrD`yQy{W5|r~ss>M97H|`ad zdmg39PGTP|f5ufOqQGE0DaiRHHD5DU`4WT{WxDT(HNJ)*^Yq4493{$`bL^7bU%gR6 zotzMHz8BvK%EJJY12%%+0=R%=iOQX$kc2rY-Fed;`iB+GkZppSvFt3HB+hj8^+mVB znm_8uxnGKKv9(8~TOK;AqvAC?{-AJ9iA7I)+m}j6!mOmNR|Gi?1W+Nb2_Xx;l@wEE zGrF%$*do8QnhsGV?g4#17zg0qPt&Why^%c#oRrp8}i!dtN4f9`qGNNwC3ZAv|#o zC~73dX`Ca%D<=v`8DUV`X51y%b(4dk)U1={1oIF@eHD9(bg6XnvOkV50Ty*G5nWaS zbV@qb*tz({++||Y^DMUsX)WBzwlqzLPYO0?5Gpb8)+F;*85aw(>{P88U(8KhDmT;&kae7p-b-=5tpbe&t>z$jQ+Gxcqp$vGL!gJv{){?^05m*xZ}u8 zI8z~ljW9EvyZ${fU9@Y$DkJgXN+D*k&nhKS?z>=-l+J-U^&O2qVJ;^p3@;4jOsh5BG}Z81F96{dr>L>eB95*$=4C)p zd>QT;jwKX!F|;k#JuYf-qEOn9`y9P2LdgD$0=R*6(;J7ev6P_P@^S}Mjg5pbtfuy3 zqE(HD%YXEsS)CKIMHefQ!E&vuie<)8<(x@n`58Swl8$_5dwO($i^A^=IAWlZ!`1TcPmb1lY22ZD= zb)l}6jwj-W%R;MHqnxQkMU)d|<1~o3j8lZTg81L6sm16236RUr{Kw{aaRIqsF!5=r ztVO#(mu7%F7{`cIHzChs6?QK2DMNoeS%`DNmI>zxqcMK@0zv@dN?uo-D#-cNxN1cI zW?nf%APJ9PSF!F2gkg{#o{MutxTffdQGDl*D7`S%Tak}uj?5#Ffiy6?S1M3MC6?Kw zl253MD_?gM_k6`8ALO&4SZ@r32e5A5q2lDFY+)`beMc;zqaG!Yzp#UVA0k|1^pV)7 z;*dv>3(Pwf?J<63X!BXJWi7Uv3E*k7uILuvlA&H0;av1yA+8rznQXw)yRYx80!q$! z>SkVDwfKomi*C}-vl@tN!2FF41Z-Jh{8DGD6k|oWuvLUSsV)yF|3RGN?Qo83t`J6( zv)F61e^VJ{c!09V*Ts`%N-SP_uOprkL{9jR=FyAQA!yACHxAcRnEGQk&cw+)y7W(- zzTNuA#v+L;Rm%dj8MKI24FGoOcOulO;pW#l}KAZxk1DQbK3AOJIw8z6iGPI3O2cxRDRe zt1&DzHz&86I+}5F8P5(T($IQWXg1u@fpzOinR&(+|5xit7rulL=QB|6?Ueg47RD8Y z!g0NU6|0GG6_85WV!iy4Ii@Q%$pl*Sdh9WQn63wlgUs@xSCD%SJN;U6eXznrFf$UZ z-{jR}iU8Va7oBz_-~-X#y!pz;K>r-5b?A~ z`)kwd8gdkq*63!utuHPQa{`wLW7D-koMp13VWt}rGs#s z?@{_sfvyOhQWE~yjX+!xiqWnu;O>Il_?&F9&@HtwK$41i2{?8ZMMq4#fqg~zcl1oM zSdQa!Y$Tm(Uu?7q5pu6)wu-Gvx>GaR<)U3o-7WVH?@57V!Et9j-V@OpG!w?I9^VR} zo3*1?q zcor2-iP(=LD~oW6WI4M|%)2=NCF(=Xx1z$i8G46f?@ZdGVeBW8bhtT%x@ijtPeIx( zaTY8=@4FJ^&RPRU8k%2YOXB91Lxjh4z9Ga#^#y=Bul^x--7K8K;1NGw%3+39oUv7r z+tfNYO*P!u;R$uRRjf!^-ZtGD#I@rkJ=U|X3`jvF8;UqofHj#&2+ZZ&I!S}~4Sg_w zIhc|E%&wSqn<4||B`ui?3ULOS%&PHh83l&p#3()%qM3v<=yBT?3CQ&uBW*~C6X0e7 zv(pCxB&m(k(c_lwpwvOJ+>J*>@QL*t-mJcT@h2}9Qk97UTw)EPB(_(C@jp9jJWuQZ zVg4kSHqBU3h`T(yHJpD|VlP3AN*vU_R(`ccyb}BG$XDmVhQxWCEUdNA;#n5{As8;Q z(@aTo(TP4%7v#DQrAw)&{{n#0>FPrKSA;wKAX9M6w{sDj69g;(Y62*}9fe3hnOX<2;3(1~MW$gLegZ zn~1lx$WMZ5VC zj?v4;J3{h=$yP^doo)62a@qR&(V9h<0P|I5h+0}=KVhEeFr~m0L1y633Z$s`OsL0m zBB+sv@vVEB;#>(m%RGCT>c<`(mANRVPo=Eg>JaAilLz1AmF#ALG=wheFiu_&N$oH( z!J|SgcGw%*wZrw8u0tLpq*Y2?>yvbTia<))Sz+=!Qv{9Dfo}cvl8LB7=?%gsP zk3VSB#9?3;(OfAWOw)~az!e0#22-Wkyg{_nqR>qg!0|#ID0K)#s`a`d%uduj$pZta z>>)IFEoz3Q=4v5Mx7mxe>pnfuPM3!<(5Z9bUXTYR=|~aEf;MW{X<@^x5qDT`krors zxOhdh8H>YIEcO=b1E7O(*MUiVb5Uf?twTJ6U`Z^xm0%jedo@7lLZkSqJzZgclYO2XyEu|!Kt8> z4|PX->>$GIZL6TPJ6tpkkezkI$=igubkcRmGUkq)Lf24VZ~S?L@AR-XvcFwnHy`tY z2Hp3jfw%~8lR=y=CT<~gra2v5Aoj2VT@-xU48^^ob4xH}G(7Gb1@buJNDQ9};&GLG z5CuHY9Hq&b=P1qog7d7DOv*1V0qn->J@Nhhi!AhZrLLN6ASgE&*K&|X*ntT1>@S^0 zqm#v4J?X6ScmC}|qaF1Fi{B;o!{VVhO_(c^-aBN1y+DxKfL&>PBqCS5A2rxlWg6M_ zc{Aq!HF0QImIAgaD~ocEtW+w|SCRw$wK!EIiPMvKn(8-2;v8SIj)*29x)7u5KcZYT zY;!c^;s*x-xeXcVHB@Bt3ZaE%WVorqc~Tsu3vy!-M*ODy2U%067&C;JjnWWcZ^A2( z!pICS*=ZZek536=8X3E0{OMqVd02V zFBU!wB6laQitBi;W$jYUXo2b3*KRI5hl8?p|Z| zBcaG zRjnQI{gc3)f3pi2g@e`-VE}PTA1j^ggrwbg#l<_#39OhK-O(C9V>TH(|I}=i4gM%78RfuvTQ@Op8 zukx&PR`JUKGRJ-*&F2o5YutvOozq|j?v3q4QRXfVFRoOWsL8CFTT(IRcK~jIbl|Ej z^^?NP9j@RrJv8HMLDW^xuf#Fu6#oo2BuVnmKTFIz2(UNLEz-eQpH{&E_>2N#-cCn7 z&k7=G)~$N1ejbFYg4&89w1EH$&&zBvsxUE;#p`j22(zsR;r8nM4aZlNxUT%hL!j}3 zNUI#Mbi}FEfb^NH5ViC9kSi~RuMAsm7iNy@QLrJd?ot02h35Vx6P={WIP*e47nTDj zwnR4waIBc3(Y4gPqe;GNBmO-x_2@t z+TxlrQuZH*^zyPmc-5%k5aklWZbS7xrl0sgkkdfR+Yw10o1%^ zLeyOX>XkxH}x8&z;gm4vk%qE_^ z36f!@>m#*T>So83uhUpdgxRfUTWw@8B+zW_t8}oZ8WCVFGC6UBn8zl|A7w*}luGA6 zQJ`yv;)Ko>X9{u6IO-hiXX=kjgt>IcJ@I8p%HH?JTlh+05#(rw*>8n%QJOU@NMlX` z{uw`gafwLhs3D0A@PrW8hd1T=?Wx z9+5S^yA6!|HG?6oo^l7I^XKSR9p^zI?oWDd2T?uU+L>}EFxBGPHsZp$qWl~3fSmR$ zcNd83HNbt#jYZP7XwN#MQ-sUWkJqIEos+cv#SyqC-V^(aaGiBN;>P%mz}y(gT~wSY z#Oa_=^Va%BVQyOa|CTO)vAcm<0%bcF4y@x|MX=e0*VJ0pddjotp%_$}G@j;h3I+z9azFOEHEW-UCoA(NUIF_QSR1gTOg|PE56kRRuYJ zgsn&;IB@F-f1Cm&}GZaGtT@ciM+EH^C0*GPPkRk2Jvk)#h$2+LT z9~O|ShZz_%Y&GV3&VO(`R*79in70kRZ`Gz}lj1O;E^pUxJ#H51Zi972CGN^8px>>< z^CB{H*!k%sY~|;HoGaVEv=@)9g=u<_3FjV};07f2qas{rylrD@wZ#hsz_G`K)niuy zt+Q)&0pu7C;mB#;7x_X#En~-Ykz~SwStBOD1mRNf6c^1+-*~pfSk!z%__;6|svVP} z#a|}u!AbYk0=JY9O6S*jndgoa;GTrmXh`N5UAo+K-YbsC=#wA2W;`gsX>r;)84pB@ zz6z3upQg9{h+G_uTPem#NqlZ!0WM6tcZZ z&)aS%P+h2w>C?_4oHA+_8vi5#&YD$kDCU3LvC)U3hZEu0xVNL&Zwa8-EE^R2r|&@J zfijGuljPPEln0hxwBAItIn8$|&@L-u;1F&g{vetplV|gIx%`U54K}~;@|6;#ZK#xO z8wkifjN3Hyp;~M&%xxp3gYKP@(XK*Wl>Q1AZ;uk~(&`4ArkWE3xqb+0h}zc+aAzDq zt2X^TNCpypJ(m7w@k#e6Hx&t2`*5QZ11&<2k?t=dH>Qq2cYYt*nY5+Jw<>^Cr*n(+ zG=*Iq^=|Bo^@Iahs}TM1evYoE8K(S;PfnYgBIDvD(G~=xrDAKC)|$rzo9mc*YMS^) zh3>{UV-tg*Wp zia!gYVtPeGO1{5|hA$k(YNQQ+4#^(w(Lqoqc>Qn#7pw(Wi8NQG7+<>yL9plemm4&s7)(sBgs|GokCHanP51am^8& zSS%L!3cy{i4o{gwR~16G7~-m2v88A=0Rm9B!fpbDc={g?yT@Sufns+twBu0GxfkR5 zfIY%vLMV^o$EN=)K652X?+INh#D&r&$!1LX8UQYGrl3LaOOgKG#l;G4ObT#cVQT82 zlrlWKzC_z_k+IjTbBGLf+0oh^QDkuy0Y&NB_Q>o_p4mkDT@$oZ-$vGwQ7ZI)%YZYq2t!DXo zD`O7Jumh(DRT@Sqi19G$w*MpkN9hK^_#av}PO2~w=K8SBhbMCIwlHZ}{}2yycLI6h5^sojLZ6b^G3 zO(NX=u#!nH9-l49jfO=p@5qX#OHIVyck6i_9%RwIT4BfIqG?}zEZURw`0-<7;*TKB zTExKd>%?>c{*JGWSYSR#*Ne6j&#s#vgp80<;sJ%7I49rOU@TD3J-TXqh3GsrNb7)$ zkB+!_LBQO2T-@a*ZfCqIEO#-kG$CBBwGfC4Hh|fFC4MWwQP@xMT2L&%a1o`epJd{9 z0&-nzG|^k4%{FFud0`KnUbmnQgl7an!Md8P;_ zi2o;Yct?PnV+6Ci_eE;$!1x`D#WHr=C3Y*;BH$VFG%vs5_)D~hggk0%K+wIf6KZ~7$fw8Ye!istlW3vEe>c&bWw!Oi2@1Lt zeZ$u0V%Q}RW*}F`7Ye&wFv`&hX?$D$!}#szIs>cD8%q(-m2York3E-$bVKR%KAkG| zl$ajXO-sYEKPFyU6M5())?NlmO@gIRJR>5{HYVlfNCtaJcK#M>?j#_h7gLyWgA{ee?%0K z+34|D?FtZ1xef334~V2qP+TW}`dh6C?VNPOmdms=0qMTO*#caz^tv?6+jk|W$>fOQ zAod+oq`}QXn)E>d{-qn2%qZUDD?whZco|k+{dDIf=3Ke_m4Vk2V+7Dgk|6JMK25Je zsSzH#D)EO*pp55;UxYWMx*pFgQs-huXZ%OFi-1;?WxpCHtl~tb>x^yoRUtf{u?LzG z3yUTPSWu0L{o&j$D!mLOwK~rD^tiGxHa21XNrjuLdvvwNSMcg%!`;mnd|3Y9Z>T$@0V|` za2}b%gB+cY`6)nV2(NyK`R599BKRCaW3P?VVru~AiH>8w7$@2ZVL*lze*(-LOi^T) zGf7ceZOinNPwx+yDJ8ajaf)b{Af3lhf;-E^dCL^-(lavwh!|MEuUQm_wXQ#&7oCfP zLZuO#k2Szhw_W_);{fRJ01ckAK0%}_DI;d;O$R++V9}iT1YK=>0q{gH)7?V?;AM5X z?rIPw#X2tvvvYq)Bwd9Qm3llT!j+OGryNJGGZBcv$bBKY?kl3qE}l@(sp;~QK-}_N z>+MYjG{*>YG8h6eF(QizazoTHs)!dvyUfhc)O>--0Ir$rVYD$Fjm=9CJ8;><(Lh@Y zbdJa->=j5pVQw@OFiB~yW1IxLxU%-)=QH&)wU>bohoZ|&fu#0&|2Q@g(X#9A8^|*< zwVtLAu1_-4U__oS2#)p-48($KJCPxEpG8Gz;i;>it|ThgiWZe)WVR6kogCdXox`3c zC>PX{)O%$lb{0+9Vl2N7kzDakc8STfx# zh+=c5SVxGbRUT@IIoF4DskKw2>#Csz%9Ib!Itsc1Sh}$XL>nx~yeqBoC3E-cLzW!t zJn*jjQtdG!6!z)<-?$A3=PKacy&3xopiy+CE|%O70)DbTVsCc4D0O7~`+cpjYwBJW zV>SYDxoz-*$QcI+BsEURx?;k{#Xq=d#vUQ&`I$eXPHM)gnZSIpKh_iB^as)|CC)4n zG|JiwZ?*|>a_`rqqtow`g}D@{lr^8C`9D=4)$l_&XOvNpx9d#|l1JlQfo35VsGzr|zxKALtq?-n9JDK3pHTHcXTxIrX-AKU;|H7Y{PzU0Wogthn z$4+=V;yS({FX!@%u8l|Ptqbl7?&*|XhCJw(a1-o8((!W7PGFvnTe|o$`oA^kIkA-J zT8(Xmny>io8z0Y$cK6{f6Y8@=b_1Yy`R2@}qTnDJG08x4?hfhV*Ssx=g#_hn{F0d> zj`?K~F3p+<3BcYeWu=^Eudmyyc1`ydE7X&BM$!5G)Mv&+$Rlscf62Z)s zlUIiJCPJJFW~l5uxqwwEAY5)}$%-yP{tqcY4~YJ!jMRa{P}$fFwGqiX$K>*aU0U8b zp06*?guSC)!i=i~(S`_kwfHI%bvZt&wEK%r>*5j-&X`7%W&N_%A}!|oq^JC04G6Au z;eT}ng>zQ*X=&%Ro*)g_E*7oNGQonv%Gb#k_4lN@-$)ekD(ufLG}AmS0%7RTzcppoib7Fe$T3>WS;>CV1`>SK!?cZu-JR zFrE zboV31HUVIIFIy=ET}CY9vHe4NFDNrZ%WN|?8Y&R2JxoQj4HpPIIIO^8DIu*)@V68j z37{T09*%7lrjz39knP(6nZhW5hB!@Bu}+{7SCX5hirdORxH z&ClvUGtdMZO#>q%wA3gdwiDHw2=PT42MX{n8j8u&=F2697>el$)5kue#4}gg`bXR4 z5bKE&q6a1a(N79;fpif#)|?JajioKxS>fD7)6x>6-hPfySBlF*O{A(N_Xlx_Mb~1U-NEq2D`R zWbVS+;4>rAwVKiVHMp}z`OA6x#R5oT7(X+=A?PY)`wdlTrGr3Qqd%QjurvF23E~&7 zHmDP(IO$+wnyKtNlC$2^OK7SdXkeO75KKg`6Cdld>%%PA5nqXRB?iztFLFrnSw10R zo09IUjP}Q|qCJ_XEr{MJu{F~>`r>LL_cf-)!G{(p<4ig(7n!>n7fa)t!~8iJh~mQM za7cPtC%aqrf1)Vyxbc%>_Z%KV1pMt0;EciaD&|cha%YvR&KjEeCyoR&Lu4SMo$_f% z0eLXs2UfR{Il5}@BIDKV~f8%F`>U=!oNeX1_j=d`eWDQ zpz=zMkja~AaJ?#!vT~+`1Iy!!7z0ub)?`HYjX+q^%ZV`8=wqQ1pmSH|wmnMm#f6ZK z%d)d5SC||6hna7!K#AoO@e?0$g(|#yh zFf9J*>83h+M+ds%GEq5m?zpPuJ{H7al;ty582>BMS#u*)``XXW0C8p0^Ca?PHttL? z`k7w3k4a~l&@oqBE+Y4*;pxLY^|<|PkYZ-hlb0m=```H=REg8$I}v$4sCC9>=a}q5 zp~4z9Akr!Fz^F9#8=qUmAH=oJwj#}9e3fw!vA+OHM{gdC7e%-^(evOaIDbr{_>wb< zU)$=;zp;q`7dA<*$z1+;fiA&xj6vD$E_+__AC^KmUAkL<89M}{qr>(7 zRG^tDU2&hRVHdan%++u3gvVr&&JF2FGj6vKCr(RqIfXY-F9f2IQN=MJe|Zs>6?S#l;`Ije|m|a!CR3P8jw+*9kD+no<>fDVpCg=&r_Y zmzuyFczWz3!UdL(HN=*yg?Kg?ZgBR2H7Oc_wJs~-NvSe08b1@_ipdYGZoV}xFCyt& z71N(Hg-}%=GF>CeRZaaf{wu&$)iGmjly&R)E5MvC(+8J+c%|XWBATBhqZVrmQ`grf zyB3>?^az$IG1Evo8QEK~6XOCw>R>ktvU(@`k&NUR>o2 z`AcgWN-km8V{+p~KeXhS#AzL=A9qcG%GzW|0x<4`gt`)3F4YixMu-clr$rO}YmZHc#}v zGuu*m_j*>Sr&VNiHdVacAzmfi#YPFz$t4X2UIHoh2QYn{zm*Vp*6bfO>V+I8hjhJ48AK zPM&nt@?#-nhn@k81QC+J)YJduFvdV}>P>`Q%;AA5t6($E66Bn!H65-$CcqqQvJakc zvq?M&sp$_E>3)nY?ofQ5KeJv>V8B>K) z8>xcgz)WC{+Z$u=;7jhaIF{j##nwWohjg56cltm4DOO@Ygp(#qs(ab;29@_k*<^q2KbV;mqiKT=%wa$JXT;nm*?FE?~ z3^_Xq)R?%k{8`56bd8x11S9Su#*2#MTqV8tVkyAZ=x#uFI2IRxsRA5}8MTQ*p3}+` z1iB=g(dZgT%(w^0P25#s?Yu!WESBTxSmfR!5lr)SD}X=h0$p};W=7Al_^@y%E=Qu8 zX>S(fUf;o`=*P<#5-G*@uSoRe{%Hj`VOiHn7ryQNK#Y8OOT?p( zs5}hdA{vGj9sqG0YtCWH)l9UZP`4pYx<`@|h2u+%O@Oh>g=x@8nYG%3#ZUNO$F>`z z|#eb7Mds^)Mit&^0H-I|{m0^l2E^LFq8)bK$vTaAB_&-wJ3g z%IuDF>&^BEFh#(8Ywh@n2-?j`l-O9L(~v-o;hg}+7T#y#r;ieL!FYSB$IW_t3DRWf zFJ;<1^qBwZ=-{H1!f+5TD|{P`D<6j@d%AO5ypsvN9~d8rFmD;FY?y{(g(rYq9(KbR z$nGe>%pPtajc^)8@2d%Qx6o}_M6jbu1o=SorK$}gtXE| z+uVsY80xtvLEV*5;|-*9(!+%LU-~f@ui{%Fq|Vb7>9FJBr-~RDk}93mo(6I4da#{j zG{po#t{tyrA@84A#=%1@?ue@~_8H>9t`VKP?Ipr#;2o7V#!>g*fyfvSa}=)@&xt0! ztXg8mvk)%3TZ}<{)^lLF-t1$@;sYW4(xE}ncpk!?M*3(pT`Y6VCxUaoXNgpW?h#~m zHSJ|Z=;!7C$uTFRLDSvWg3bLdxgK2lMQDaKWkB*)?4#A`dDwyG-EBfGfWY=o;uiF>eA_ z5z2p5vL4%sV0`2YG7sc$JYL(3C0^m1d6LfBffjoe$SsPYFVes+0!WN=6@0CU$h^fc zvomQ;aa<(Wb!D1qpeNvvT}<2kH9oYM#(~!-qL~DGkd0q2J`HMAT`1ZaORL(9Zv>Dr z%8N>jdBd?W%H~3Rd-RDmzbkltWh%N=h*`nls7=Q^I~tGO-vo7mrNWj)itI1;7wq3h zv_Rs;@huRhiSd)xihXlfK6LvOcCt8p)HJLsMYsI})P=w0p_G+a<+xB;M+iM zFq~IQ*+?z#5J;tDnG(;4aOX}-oI27f?-UqPxgW5u3d@70GJrQJ&Nu|+nGEqpXG_(% zRG_)dOw}9nz667nfuq5A zOktM-p?#|6QJgY-0B8mz6&}nTBgl&uq6}WYn384P`I2z)fk$Oywhtl5m>!Ii%b^$% z?cSrq5ScRb%M~AiI!g%&DA$=!migGfQpPr7#do49BUkEp$bqnBlg|E1TrZMOpri48qi&U4C4?*y z`4A)GYLV_CNblTC`8PC0=;ZKYhQh6HteqQt4sGUf^wu2Klr6}aCWDOhM&)?|Nuy_U zIG+6vVK-`5a#{140BXUpRd0;@0)n}g$()g16ID#`!#3KiMqM!3a(Ab~$&@srpMD9H zy9!-{%a~sQkfm(*UQ*Z}HHY`aTLR1kM!syCzAO=0e=$;qzb20B)`LnEcQUcvH^pxq zTz{ndT~H!y(Xsed+{Wz*Im(mPX3 zf4p8fYk?tR>dx9|d{$yfQKnYiYc}Gwx~#6p3?e5mrQaspxx*Ez@TdUS9QA-?^i{tH zl6p0by^e_7JCsZ{ZW7}9BtwSVMZ0}{Rv=#8C(zmSrHg06!8eAcZ@|^& z1#~Vn2NNLJX2M(+c^5^9I8jJz)#YkjazDn*^dn%mhK{?G`>lc;Go2=6J6z zmtk?E2&d1UUXO{8{%i{Y=8if%eY)i4$$~uT^QvL&DAI#j#bTq((K>}+eqnG*hDBX$ zps-nj6uQH%DL66~KX(#I zZc}g?bcMn$GZ(ker&Qu*K^~rM+&=5_^8JcX*BFIU9s|E$43LD=MO=>K7KfxD5_)0- z5vmg#sIL5h<(6j`ML2z&Z_}E|+B61BX8EHmVRfl$yE5{CRF_BP$XHo%^*$3MT#2_uz%1knZM!yE2GR`0 zCkYq5Wm&YJK;rnGZ%h;El3-*tF}}BK@edbsb^d^H_x%Fh;tX$mq9C&*%pJ|{H;|%a zu>IFE0gMu}#o>zrsUJ26z439G8b^1`4s@6w{e%d4w9$l|T^=jMEamAb_O&wsJW_#? zzzO9)(y$|q{8@;&&N98pa)e!ZHX2e#(@pmjsD?%K$FHR*btZOd@v4YC1)vQ>EQ+~T z0D?CI?DpYZEFju!z`2JGhnlgVFc%!(IdTN_i~JY;s-Y_{b^Baj(0`4lOW_{M)0|SJl6Vgsm%yk|EEP_^zbQhzUVwEh7onrS>uB_ z5;7r9yN2Q^(YdK5!E5RKOqkPVI>r!(ZSWdJH15CZl1nX?6Gma>>||77YQ?L}U2%^H z_{si(J?{TRd3j*%veJkh>vw`_16|8y9$9xyNax8;f>j7XN05uAck3CVow37MFxM$5 z7~C`^9Z_A+|c7wPRu6lo%6r zatjm>3M17nrpzZ5cDs-&z>M(#?pk=sjekuj(&R!eeUy&9<3uobnJVi(0_Cnka$#^z zE!%(x%SaV8Jsh8`Gl@vBxM!F@W{7Yan8nd>alQ~r)yAQ5buGf~Ryd3qh`(leBhD)!4z$?|55;{Yq{n4-fIW6fkso)H24Z*7WR#pN3hIHlu%xBLj1|`*x+~KLL$GL=D8$W-(PeKxhncl_`v0gp>o{Ah>i2;%1D z0S6R@Se`xsNAPI*p%Xf(juHCH!70h+?%9Kv6L`R6FvvC ztL1Sap=M`0o=~(yUa<_A7g-5cHxx=K24$|RZNq zu^5&++$ZwC1GTaE?pFLu70~dl#9<;F6g79rSiTnGYizE~v@5iT5{s{``6riywgaU^TpUb}M?{!0SqhU( z>?Usyoa-ccnXC>#%;Vt-VzlVr0l?*B`!y8*E9tgr)27CWJI*A5|j7h6Mk^%q*bx8HzWZ9Mv>?x+ofyl`66xYm9c)=HIRGGc3%#!MuSwiijsrmz)1=_ml6d!Vrf z1*o$i-s_1CszsVo;$b5OM5X-i`Xhs6YqwTi`@qaXOd_jh76mozEx{ zj-hJ9-$FsweBO8=x_w-KiaQh8PY+-1h>h<8PJe01X;eV!&HsSaU^!!a3N z^I(!Yk@lr65iS$X-MG7IVTa%tIvG-HdgAhZAl{V7*;`Mm3-Lc}JXttBn2f3oSMGnRTq8*#@U^lvt?VJXz?RqA{zB?h!}88nPY}MVQ?Txg^#7 zg&=3yZJX4sXE+9L=nlrtBHWj(?;%y6-Z&zM;fU84JB$*BCfDPX6nTG91pNaGhwF%+_Yz7&rvN8T&mO z*2r1H9D#wt!x*B+cfM%h;MzkAVk;&dS_GV~dj46otIqWSVz(JF|6xGmb36lz$Q1If z5|uYY>CQ1H&3iausfaz3o+OH5R%9k{nF!a4DNSx;7CfRr@aCgC(nW;0*l3FI#JXtD zV)PiEKb5F-zh3~lJTixt0YAIFgqjPSTxd0xgL4(q4a-(vn*sq&zg=#Z);zMHIX%|m zfRY}qU=z|I+T7qBs-)$4rXbQxud1l+&J&<2C+AkX99_gf)|qVTjun)f6P<5Xw&R4j z&!QE=t^wcYbh{e`yL7DX)wo|YDa)xvBaS`>$`$l;^RVKUnP9FU4%C^X7d#dKj`KEO zGZqu!-bUfk4#`@d>j^hknHN}|b;|wLaeywRcA5;6-AkldOFOSMj(6$|l1dydBJ-xZ z#uH5gvG556l-x6s?3f@5G*U-jWcw2#@}fiwStI8Ffz%rJfLcW2IMFT$hr2P-IJu0L z9>;`Ncw!Z3cF1)J@Bg6AT1Lf}m6rXdgpfEdTS`wLKFKsQGHX#^TzV4!Q@ms$|NU7I zu2HA-QJC-E6J!R%KBkgYPA&j$Vps&vD*?8Un9={>mxOR7Q7DwMDV?MxkkYaP1LKE@ zZ>v+l$QwVhOgWk0xKxR6{)&HcLFn`HSh0d2`hui67$}wjIO*d`uNij;A~n{y(fCkA zG1KCEkX|<$^PCFml5uIp<0V?46X+VEx{a)AS%q@ux)@dIH!qLY#%|j^x4L6=-Hi^N!u?Iyu<`P zF!=$IMRuG`w!Wx{!K;@N^xrLn)w!*0TAcn{$7Vl(!bMyl(mhFcYs{8|al0@#1)Yt4 z-xm*-5W&#atj04Xq}#kY&m_%#aS@Sjsi)E7dxBg#nwhzLdjaO2yl=!$N*ar_O57#d z1yVPr(FbGn5-=JQA3^A8i!e7k8NB%#a_P%J%_eq$J=E-HLa21DITlao8* zkuFpR>;FKka(NM3qHQ|EjzU}vPBT)c<6tJt(NGdbuD{k5MIc?sq-Vo>g^?8#t31#T zWs-9ocZG3g8Ka$v8JScwZYpC8VnV&#l|=%)Vi#|gBo5N!yI1iq?30SN^k)tf?Q-&@ zWmh`HwuE`yNP3Q14O@UAfv}@KHWugV8)j){@2InZiyMWwRTv~#4VFnTne*a?>fd>F z8Ay}GyP`?59Z#9P*YNE;S(K@oCaxz1(r(FN>(e5<*5j{)jSt5i*&Ya%44@frmuckt z0~+Ezfw`@c2fc~E10YwqCjVI9WH5&NsMz;f2zLRQO|b6&Nr>aJIif)SEdjI_=n3)r zA_NN#hCR;+$lZ_myQfAUp5r>O+~YWKBCZlQ))MNF@K|NvQz5x=}pt4=zkp&YxdfXce{{e!u>%801t8Wi==`3lDTk!uJmQ7w?EGfbq9AWlt z#Vi4iAg>5=d+@X%vq#cywnW;~_Wcpm6&~ZR;2M#Rmv409=05>t26B+^ljZf|Hx{v` zb0fa1$kc&w&~|$M4`pp#9A1#gYE4{;4@Ks|IU;8^^ZmtfF($-8vj}J3ne7*F8#B&U zT8dq}b_Uy*;SgDJMjDNu3i8n4y1yB>mO=0kgTCcxeE(*Gz)^OG@w~p>Fgj$>$M(7f zgi5e`9gHP!g>a?wyXCcbP?)!;uvku&?UVEw;ZBPC5)Mgm&21oLglI1gxSelTiHo$l zoE8oXa!b@uK*vl~TelVN0L^1tE9e`dJrEI4w&Ujs8AdA>xD(hVYoRrOF3o5xD=fEm zUt?I^V?#lyt~_e0Z}S{E67C&ZWq2;jyL8s=zsupd_ges_7g$E5<@c2pCFQi$Mq>=;zI^l zXNGzAW+2)H&@f|ci~p=|YR=2+#Jy9L`xhO!&Mw%8^0tS;DT$0G;twLmk79(9Iv73R zA;GQ?vMC-enf8byG3dHt2NC4SYl1Q7qkNm0NrbL;TuK-@bVx+Ig9tZ3y~$pOg}qCV zQ^zktuZ#=;+#LpL%xXF}Rh^vKAo*o2$jmQ0sSrirm zd{Yh?IL4e$K)9eB>$$NbsS#nGtvdRL;&hRD5bK?V(YRfRY9d9o4@JP6jF4&pOD&fb*!d#lGLRsh*C%djNnof z4OYWY`xbpu0_{qqse{x*i1{Ut{4H&=;%6@ak|yVMJW8Z&t`WHNgulOU&zK5RTq06l3Rmogi$h5PhDaPiN7KiPwg2K)R4^)7D)t-V^O| zNneNiNa=TfA=CxovX5g8d!vN8AXxMJ|jM&8cnYX3i z0aNnYwf9DkNQy(HXpf(ZGT+c)W`+3bUH^@@IrPovd(Uyv|B}A&X(Dqk8fs5oeJJq# zLUW(Azh*kz>jHB<8N&>y_*9Tt123e0-+Uh=v$Q)ef6ohZMCJp8H;q{R10XjSN3@tG z(q+bxJ6rQz1UO;buILTRgN4A8ejW?BvJ8$tF1p6i|KLwg{tPS}^-?~WLp&+mwUb1V z`BNk3ZK0lN*`sU3rs;9ghecr4Jz4l^P5P5iO3Jvb)7bBb9Pe1`O^L<+X*%Z_TZpEZ zC%hkyn?WG5uo`M&!6$_k07$VZrqkhrMuu`aOZ&UhgLi#nn<{+AB;IZfiNqP zveP^9VkF2MC*9VdtVGu-F(&Htj$m9T%;|CqGZv4OVWe5d=BgEc6GmP&Y1SIJdqA%l^EdL%T=u5D1zI+CFb3&_&M| zPrb2|Fei?(M>pos#|d##9AiV&fLlA)?itYq{wcq8LNnJH)4cwwLYl5~0jg9SB zf^x~^`9x>2Z~Yfcb!OtI#6F@NDc!xnu<0elom8^sJwi|(5;S`%XvU@E3G?WV!Tz-9Buky?r-wH zYE37I14Wqm-5lt(eL75#ONmL>K-w)lB+NCJx_m9E-mE!q5m!2j+W2fBh?bS*OINgu za;$W@m^9y7WvoFxS?GPuq}xicc8m}giMLU7 zsW=*E3v&~c?~o&nPPlo9(k!l`DzW$i1f?^zP`gT(TWbm+dwJZ7r$sm+u8B}OWtXRo z={nyAr>``)yS-lDt}>UV=&&W-gk=_Th*55X*V1?98u$Kbid#n**)X7|#JM6|F5Ism z5~{{;O9<^_EHy@B!orR+UA9`AWr9LgVn-3qK*F8$l;rLu#@AY<8upwO&K#Yb{Pc7phG#NI)%R)F8 z&XU|M$6-Ru?deF5yW?a5ZuPEef4nY|9LRv(!E#0PiBporzD)9`!LyIA|Q~mwQHM(fz4`= z?uSfujksa;A`VJmy07G9aHl{AYV^_T_WL1p?wL5GN*Y%e3UinqdSTom+MKeX-YBmj z0BH=aHzDR+lW+3`6=P04Drr&C; zR3ehyqA#_(5L!=*Ok62~X-%*Ca2qAc>>XEFrPDrBCV^`QrkHp^h&4M9lH_}Sb@pY~ z;g5W#;R$BWoh+8Try|%j!H!~Td@34_BYV(cf6WPyZk1lt)8VHfYeGb5Z`Ze zEWDK}ujwLTM?YiwMSW8zt;I1k5yHvo1tFdvgbzmxBm~mR{^*?q;bP)4gZ9L3niSWX z3``54xMqJ_hYKK!3QNn0`p$(!h=V>la*pQ(k}9XP{@8sAglmblPAw)*g?67{t1L5R zEJ20253q$#8Wv6)Kz*j%)`hXlG<>j-c>n5c6YZBdKk1&`-| zmIbH!Qv|_oPW?Q`9={byp*ghn#C0N^EJs|r#}@+3ch*{|lwZ28!=ozP8^h~CkQV!p zUc}6gi_D?%-A(>q3o)P2cIZm#MzO{EfG(8gJ$bh{xrDS6<59afTae3whj9G$Us;Ah zM9GCL4i7gVOdjj_qexmRs|#}>m}NQH$Av=N6k|1p(nT9Wn^!HiR%|>pDa6Zy$9W|c z-fFxl$T2y#Ap-ajw0apfw8!b2!NLZpif-aYP%f<`P&mmIkQbIN2AhxhhY2-5kcCTU z3F*k;0y95Y{k287NDvwE{8HSYZXvUPwy7v@T$;we+Z?`;g?rl2FmWRWfp;`Q!U z{3`#S6$9TY|Ho$+<7wqhiWGH$LFIoeL}eI5rpA;_`A>r!EkQ&U{}AaW(A3L?NitOaw_tNZTX&88xim=>#r5e* zceCEpmn%Y?V^gy94$id+i_nha$~ccK19=A$okm>tLy6LmsUX)LX>%X?KJ`H|@9t zTpOq(1-n`;wkBy7E+I4$BaSV~TS~H=JS7knI z#ZC(5mZ<-zk#8svf};raKpD3`-vql>5eyg}Sv+nv%%s^D(>E|E)Lyexq1`qLw#*@pW-xLh!gyF=?8Y%bWzvxfG?_4`7*Mu@l(LyyKj zKQ_ep(M+J@^PnKj4eYgPox??XYC+~MW!k}5?k8XrMQ8VBtRc$P@2m~U>S!||Ztfm> z>pyaEtxLl(aXv*94_3r4icSJ3iKvcX)pAK{xN5{ zC6dxNz2{Sb&IC!7#0Od}7e4^h{6(#fRBtH(ULZl)?Rf#z42b}n@>fMssOW1{P{1117alO8Y#nOSMPIufd+Wg_|5H9oLzMq27ct}+HV!jH58LVfM zIF<5RoG>?NjRk{dNg|3P1e56seRgbmWA*JlV(EEU2>Ixw!l9~6%8q+kk=N=XgvXx@o@kKRPLOL2FcQy-hSPFT zA8(7u4M-PGiUMnNgOLlvhADqglta+ln*DY5o`(o?$uJohCf_S^9C=tBjQd5o$yntX zE6ddZ=v9*^PKa6h_UVCQ=R<-9)cyLyu^41Xt{)cQk|WF+igTw!Q!plL)|X2}QIDi_ zyG)P&s-@RrH)#|RxopJKh-f#o#;NB2u2`%W&=te;Es_$dv8*u4$pfabeEzO7Ra~U+ zJOQvvL5YRH4+!)WRE4jbH6dNvbT#^{XfvsXsT*t=>H~3(x*%c~R*fTsxxwH!qM5On zIa@GvC9t*SSyW-U04N35lcmauv&H2)LyDYN3eqq_$tE5YVTN)w&rsQ+9{>gp;%K-d z{ws>GoG0jm0}#$Yx4kuOUt+lypobPWQMy+~4HP-%3BsLl4-QWH;z9u=)gc}5?+iki zFE#X?*AdBo%@($VTZnK>8ip?YW&RiW-^93LsQ9<5-i`~F7%l)_{ABVxT!4$oNkXD_ zczA|DTCEq~95$7=xKV&PrCY}a?iTR>{4+r3 zt0!EgKHjbla>>sNx($agvD(4VPQR<( zm0o^YM^LTbk#CRXwJOSnYMzspAwj70*=zDMVlqo*DgsjNsi7ISv#b3 z0r#vx`VEhd48;2)T-gpx3%FuF0>BJGWlb6qF;qfaH_1s-943fv*xep~(kIQqyoCfi z_WebXU}Fe#v$*%j0_aSV%b}wH+%x5Ejn!z2qn!l%njS{Pp&}_D(r7o6xjA)PHe&)7)MAFx=wqnsZK$O=j{GVWB%1VMn`Kg%>G*Fq&y784L zbDFy=32@=yqsIc89aJ#QG4Ba7^AR8F{VwdYj|0;f#ffAbC&F9}h;;^li2xXRX9$qA5~Uw<;2uzm@}}OLOQAW)Z;L zNUDY^(u=~1K9mI7yPph1hUp#AqeReq)|nE={*q7USDC@Ib(3hyiBTwWhIm#a(Rc&6 zJAQBqgt^f|RgKKQ62jkHeX;rc6@jg_FSCci!Ecf{%}INw4v>PCPY({ z){8h>RAz$S;9K%EXs0Yc9M#xHK<>hRG-tmuNPm3>=j_V`P&C>R#dQ&8NH?b|9!QES zP6u*>tTbst+$z#pB5_F~&DdKs@l#^cMR=-dO5Vs-D?RcAp*fX-O1gi%QIIRz+rg7*jhOo^5DF$uGwr}* z*$gmv(N*Rk`@R5IWu-Qz*z11{?H)3I42ypYgWr0~u@)Pg1LfhV-Hvu-QmZ&fFhy=L z@5Pu1`b5fab3efDg9&6AA#Q0NSU?oaY>UjM8fWPbk7?B3s~W%fW}Pj*90~Q# zjv#>@n!#OG83y61&Pn$OaVOwWiOK87O9CifUD`qmo?HCOxtQ(ATIT^UsD^l&c4K`L zwKd9aTN)e}3m_`5KGfo&{I3|uW5Sh~c0T{+*&WGUy6)IX7!1>uRB!xQl*`hdOk)2b zz${@gm*vZb7Zi!0tiuKp+3BvrC=+s)Xy{urcvp;-LC4?PjvItILCyhHo}5|vLJ&G^ z)~nlJqysQ9%ILl+#Jon($+=il!C;QX4U{fN@5yAUYb7@REm5cyjm38Q7*Vb_x)D7C zcqGFlrO1X?L?l&zt_;PMxaMiR&s^P>YDk!I{FfSUGU0NkT*Co6Fb{Heyl!rVtY z^iJ*ZqFtLo6o`=vTv#G{Lz8{SMM9hjwWHVVc?kdJxF@FSH|2(53R*Bf<2FP zc67utR}{J6K}9NH)mTj!m6c7D2E`_27`O zv7OkwUZ8I?je#X)+*ivea60?y-R4Sy(b+Lk>55ZCI8k&{Sx``bjlT)>bVTOK^$9^g z7Mi;tk9m;%r$Wraq$La!V)LuYP+0NOEZYciJFvr>vVI&=0tjVsB>o!#Zv93DUnJkV zx=1%&lc=B}%(=^vIbwpvgt+K>no0|QH7+iLNduJzxmw5)^M8*nu4ZPqs;!a~X|8Md z%lYDvk_R9}!%S&<{ZZd;#}N$2ZWoz*Gx|w`l)C@Gw>uU7DYzx=ik}I}Y^CopQGHn=yhx&K7WpG# zoUi0Oow02R@H_&&wJN}=P))^oW_Bb=#@{R>(GlU`5`Q8R!x7)5Ou-XGIg=jtkGwX2 zBLD>_Y`m`TeB#IN7)oXNUTBw_90)ywI9}gQf&C(){ssQxKPOF_636S?RnK>6aoEj3 zZZavn;{K4m*MkDhQaKQzzl_EU!Wb;Lb7 ziLZ1U6wQwqgq@rS^Q_g)y+u!KDa3h`0)Dv=Je(=aLlMsbXmPQ7<9V^$b3%Ijc?}V6 zacmm7;SlYF8x?uoh+ZMCfK=l2#=@*J63?nN;w#Z`TOJw)?wBosWMMZ`2w5MNj#`q@!`FWC?)4O>plplt&8DWAJ1Ra;x3_P2seNz z2h`$KK_2g2DERa};6$o2P+#;xh`a=JBdtIli~%T1uyAvF1RuHDeU}Jsxn$5WQwY^a zeR}SPAaYOY)$<&?tIt%GkrS^#YGn4~@x;6kEDkW{%gc&rFh zYYwB*vttQpN<|?|02!iLI}ocr4&ef3;y!kkNgHDG=Lqcrwve}SCAjHd3j{wR z^6FX<;%08qe_HXm0JERX5#sE0;(p|LP@0WBDyG&iI1XB8%+U1aI|a}dO*C%WUgSGx z%38y^{-}^VtJF9iGB4fy5|9~DVaJ6;pUGmwmjTJ6V~G31_eA88R~gky9LEikYLydPz=F-Wc&HG^_E{%9V zfcS`PcAhu*&J18K$2KE26Xf{qNIs%f2B9f9z2bONkQs|ziDpU4w%VHoNKHBX>C*HN zfp8nGy{goE~UtzE+@_%hnEQB{ST;!rTGrP<>p@rJie9p=GlfwvG(!jD`7!`km#CX^H}f}PPDV0jM*bHvv9E|6RP*aV z+>rm~u5~nC6hV=b%dUZVIseyzvT!W*4*!)hjHJ?>_L9r$k35eLVf4yeD$S+U1iLRF zszV^XPMHAL&fbhBt%V(C3pvK2Cw3R$QtDkEZfu_~gUGicj)Th}vJ_*TcvA>vmoFAB z2Y>voNi{Ni;Zc!htegQS@$Cn~$|+PjCcFWaLG5Jf(zMmc+W0*}x{8JW7ZM#2?0zAR z>4BTm1-aJKlg8iK0-OoH8JJttfBzwnCS@|h&yol=79Mood7p2F7uSrfp;+SsC@02^ z6RGbgz+I}v3o|E(bd#_ap}vDWJ3U6V`#<=DBq!4EUeUJ;KoKQoV+Q!ahk%|yq_b%w z={QO#(Z&#p-=c3yqP3zDYy1<60<+f`jBP%G$fH)y5?Z3o9dS*k05#FDV7DU9x}>dk z;m07Z4-O1if&VE$<($wK`+veWS@g=Y&_^QjNachp4T_lO(;{|vWrSO%?+I`qTo%gZ z{b+d(8u{qbi}PPW#Z z-p8|*p9wJ&kZSbB(Vs)RFkZ1p<5Xd0eOiNYg{>-9$THZ?+6u;NL(_8}S`Y+?fG&8(d5c#IXV#360h? zdw(LxMdrpwr}Yj03qtMsaC5zzzVnuo<0RSy+G#N2g`2@-gn*e=TljxqZ~~3u?pR#} zgTvm>V!m%|J)VnwuEe53Xwr5bpy-LIqMROEiE6S#*jN}z;Lby3zf6@UZR2CFy4+S>ch!pUzpt<#ar~FCG&BH!36b zxNa`KT?AD*Sw-`>o=`Uz{Usk8ZV+g;PRD^RJD*$2U|LQQp3yz_nWsoG?E~#kswUK( zM&735LeVaA*AVY(2)8R_6nS<6IJ>&2o&W zJQLN?^VFOf@0K7MTpVR|n0Qa1JB?nRV9ORC3(B2FN}Pl7A0aMsXCJbgwdRL*nbZaP z;`aim15<;#=6(x8=1mhTQmd7Anx6=zh+VuMfyc~+OyPfo>2@Je&NDf-uEsBfVCDI(xyYo_-8jSx5bUBl{ zJX0!zaq4$L-Cen4#?1jW`ca9o%ppMLOuDTw7eeldC73%`kgJgdgne;I8L=jhW*UQ2 zzDFpk(vB`*thz{%o1_D2(K<#L0m!EjSBWsQc^7~a{wMigy$T*%Eb5?`TJcPSh+HDP zCkXmLP#(+pE=*(Z+X3ckrzJ^hUaikw`gN3D*pfF#T z0i-9AE*sbSJ^|bmFa{T1q8VlVxG&m&F}}z9n?_1kd@jftbAZ(}3HZas0iCfn?A*3C z&qb3c{;jwj{mxPjsbzYqzCEOSF~HbQWUdP)xUhl! z_|jl*ReAZW#jgcWDDJ@VRk92O+4i!-++W|W4fk2n2s%=T3&LG3t_ejutt4KJ`IjxC zbg~m-d@U%%wd9P;%fF+sPzm8^tW&$>50)d83!2QE)exTvb7ovpVa~Wbbl!qU-8pN| zKPfnOlyOz{l~!<6dQS2qb(LtZH>@I9BxFOUKMMEeicM&m`+ahJSbw9$N}t)mG)%XY+s20dCs*$KQqjY0zK3?bn_fG;wUU^RxW~~@Sw9i z%>T(MfG#ZOD0J7@B5${9foZ?Tq+N^ch0!-ql#G4ycN=!5UHYbg-F0rCju&Chb;zuf z>n<4lQVe2z81oDV|hVI?t5 zkZZ(|Uk$fW83^H8QfWO=5b0u?7C&2)Zxswjd~ux!$3{XWfmPpHAnpfrea75XC7|R6 zW5czJ5QsnsF-w3MM5FNg&}eKY&;t+VquMT7?T*lD`UJjdOk@lkHQyHHs_0sv5(~E#QRN#u zHW8f*KFp0yGWcJ7BB1%sdCYcTs|X`q&hUuX4wwW*j`9cjq`uuEoB(iM){1up5jeGT z%$y9B$5jKp>6m{?0dTlx4NZVck?xk5Yq{c8s9dxetVT(NNeN<{Nt=TM*CwCk?x00`WRb= z4TP9)t#rEHN&sBfJ0jPWfA+97tHdV)NRAV)T%>$a{*Oiq-D>U)iuCpL71P2oLd+B? z!_&8@(pLg=6|p;r$2Wx5Sm{F0x95)_T;E|1Ns^6oFdr65YcoO9BKK|N2<+YKan{BV zt^&&jE&Uq-uBFBrJ#fM%AWoA*UU$8{F7wqjHU)Lru;N7Gq`hkbDO);vF!A;&(0Bvl z*GF5B6NJGA)*sm>+_4#?^JALqWpSIdIS3WvspeXYh;aD^&+K-cezGT33)0pnvuxd%JkG42M1)nqMtL<}(X_dwi}SF6eN5 zKw`T&wgh28>l}$m`gGvB97FwBv{zcLY`bwJo8*E^Yz0i+QK-{Ne{YerOJBz*^ZL76 zLpg)w1gkwh6_UrL#CNgVHUO|jn$+<-5oRtEHQkmJ-xoG6Ku))rj3h=)YzywRaSx)o zZ*3vs5|hxM^zG8?;k~$DG!+{`_OP9UO>A2?D%G_G-*tC?gs5Tp!E)? z-Vxw_OOYB{Ll)c}hz9FpJG+Fw^SIJtC7I4`!ra6|%umQEAKn85wy`flHEk~lijGYY z>F>5Tl$(d+t&I6$(McuPo}}A4QC=$0Mbp!c`0NxwLP(3*I=?5%9X&1WwPtdvvFtv; zxuu8NyI^R4{6!eW;QEd@qV}?c4jXj%L)pCdl2SZ5m2S zpZplwwPjDhRwX9?1jJRU%1TeXjwu3N4ix*i#X!Y%YKh6`zmDoLO_+N!CIuXPG@8^^ zV3q!I&Rq8O#Lq>$a9Qmz4lh$o8dGBCRl>|SW>Kc*dG;&f;ieXCXY?ZlIcbJRZ>)Df z5d?8LC&*@eR6_C>!84w*!GVMtZ+w-L32^35!N?7Pp#%-BI8gw-Wqq6;oThJ=7)Bvd zZpFER%s%WEbO-#X5E=xX77fcqDvsaJ14sYY&UfyeFq_B91voZ#EnTQ+eImrnr+FKr zBl5Msc?Td3&Z!%-gsKU~a9dmL5 zA!uRqd%+;^Wu#WV^9w2X*(j{SO!?eEsp^Ea96F(P0@(6~R z=xN~v8UQXXUfXqx&9bzVKo>%H4b9k70Nob1tZ|9H%_X#1V~*Y;EM6iwN^dS87Z``8 zN%U}~Fc+B3Ij5y+yeEiGgI7kz(;Cep06S-1+G#{pka^ZUqFSEFM7xQT=1)2c9pV(^A+-@p917*0k+$;53EV0I-3n4Ray2TM|zZUGwRQSdadyeZ51EaX?5V>@laRijp)j~VKzP<7bAXi_uHF2|O z+6S{Vshd72(rI=en&vdqz}5iFMMi+v>eeYPZaoT+jPSXS6lU(Dq1*wHx3Hwo5a3qf zs4p|M<1&Iq4Qsp@bFmOMupJ}uh(76aY>682L?*gq`?Ah4{G$p<;VaG%<)SbSS$gFR z_;G<$K^=uH*sCH*i^X~%Ce4H}2RO>7z3e7};6R=~<5bbEae(dIoCj9lRfGyR&P z%rm5F^;r5iXqO`CtFVvZx?7-G(4QRa{X&3Q!1$9=-b^8+iq;VdQzwXWzSIq=4EgpR z59WEB{h<5?Sxt#Iu}x^Hh+Z)_%VyhGuZMq8Bs2y;eBt-Kmnom52X>BEVrNC#k3pnmy$iO8~r zjoaHoT#&AGmoV?F0vO|-w<{JF;H;%{lH9aU6*j(gDAgGb6Fh!-&tX;aOpJg=7loQ@9^%qWG?ZK!rnEPOHqqfbg|zty*k#;(&(fL|kw zzzv;#pAg~NHL#iE^^;!$xPRD)j$LYOEYQ3jM1yO4k!}jy$PdI%%Q)0X8w`BX+)@T& zbsmZjOPa~772iIEM9lkPHjRv(g?18BP;szKkTL;?YP@ zZUuxe2m<=zbOm$mS~|$bon<_^Jd_{LmjqE6JtL@Y{F76m&45;q-agh@_E>?=PnUH% z4Xkn+5Cej<5u1;LPKR>t)V3}&>8PFoW;U@as>GmZCxR{+YZy0@We|LIU>YUbOc{_4 zM>ReZLTgS$E%!TT@~xRhR=Asr;Ezi0P#l;4!0Qx>a3Uxe1*QAmB+9!paajvPx;b3+ ztQ;!IJes0hRKy6a_`PVCgL8s>q95>U5Opz>9C_!%1eXU~5{htGyvvFi)pJ0c8riXQ zi*_^gB)>ol{|3N?;!uwl7e=e@WqMVGbzw1oFF%FDZe}XUYUSvwKVsot{99ym-2d#!ABS zEm7I00^#*^-s5e?M&}iY$VZEgP4CSHu_=}tuC>o6l)0mq2V$ydm$B8?>2wb-F$R_#nF7c5s|9)M!7_w(ae7Df<}zUSP&?CO zHEtC|6SuT*u62tnwq1iWCjLv(SND9u91$hN(> zh&$9oB^4!kA!IdzpQ%{wlHzZ<5Mb^&K?r~IPBFI;B0Ohy<95AOo57x1@`yDqoh=@R zr)FFw#Pig&DJ#dbB6I5_AwwSw%l6B_%%Hv&Cw5l9Z(af9?kP8It=LF_;}MBTmlFS2 znCs8(kSl9m=P07FJ)^JQSORoEqzreLfS$eNp$q}k3)xOzOuiDr?ZWvOiEV0^p9!UR z=+QM2iMK?%yy&x3YrV~w=PDrAvW|hrS|Z_eXJtnGsQjy!xowfF`HTE`47w82MYx*m zs+rVI5+ng@u8LuipY*o;u1_5*+?b{>x#dFHg86BH0X^@ zg_whA$28KrGP_>uSS;8{KDe7`hv5#8AqJNZ5~fL&tKNY)T$HmSu z5j8&W7@!ojJ7$z1?%vqeXzlo`K=|FoO=~Q9J%qDQ7Z4YSrr9JmXG3*^NVq=<$7xsJ zz_*)*-4>GiJ^lcYd84aA_6c#1Fc(8RH*1mX`^O??+Vw|8fQrs(6|Hqq6sTAIbEQ8) zC`a1+R(+Gf2+x1;UN!n~U2(>ZV9o%q2E2ZyLr((TwzU};AGP9Q1#_z;Q30D-iP^6f zOda&dq~0@)KMQdExf^ba1^*09_HAvGVnpAraA)5ru}>0E^uKfw!Fuq`n?TKm5gdZE zqvcvan6pnCSSei7*n0_firnwA$-k)t@Z-s-{_bB0k*mk#EaQe|oFx!;;kghFy(h{= z<`6O#ABlG5F;|hY%bYiZ@bxBuCaDd}+rovS##@bN zL(*9PKY`9fvUCJqua~J{_Q42xuLLS1mKW=a^1>+gbc zh2+Ujin;%o4Pvh?q1AWpCX_oX3bp8Lq?DEr>UwgA#fcjR&k&Z=t9KHQEeCmHH#I4=2s0oq^8#V_cC4xaq*zbHcYMW~QG z#HUNcbW=^2hqnkb+q%%!;R645A#@i%uNMnF3`yQ7TJ*$$B0Ly4PV+jsXvgM8SbNil zg}CY1Kp>Ml`w;*oN0T50S^QB*YA>T5F5@)oJt)Y{so_kApocH?n8Q&iuK9G*jay1c znX4bYFNnhNKvgA{db~&nBNy(;by_-ApiA5?8R;@lKvQDw<1wrazD;?)Dort zRps1*vd-U}2Ii_f)ECuPix6G7R@KeQs30ek zwt9VWq7d?B&mh&>dqh%KmJDtpMVL{T7YySmJGBrxkl*to07x{uh&Q9ho*0;u^`23jY;l5M-dbetz}cB75{glHx8Kh!fuf z^@zbIpA?B0-4_XTYSSw{^!>{Om%BfX)-#KxHJ&ohu2`{MQu? zeZn@aw6Xe!MOZzrBiqQR3o~mNqZs+8Jy1e3|NGiWG@kuu5qE$|lXmKecZB7R(TC>= zdeWqifHbX7m=s&#CDMI2LJ6=+5b>tWla{|n7Gf(4CEii5la6qdVRgY6c<(PXe2?g)EaCO#U5~{U71n68NXV z#~ypHrZD)?i}3I!ebXV+i+S<5DD$$H-nh>Q%65e&ew>f4jqj?mWj_qQM+y$Z_&B=9xv~}AiEHz_4t)gmlvsWrkw*0RMTv_ z`8-9C_CDJ4e$h;2xcmAr|0S7%KbPA8wyg(EhVWiwN zo-C3O<1Yy^LOv0cCyi{sz$UX5Z+{EieRfc5D;D_kIWj#5Z4fEB#d3nEb_b0*S>NMb zCOzn>&FU#RM77=%r_Nu5z{eRDKy)_Dw1kc~oSaN(d|fWkRY6Ho@07-qg4~wUJx(S@ ze-)OQDns6n%e0ch0h2eT0}B*^xR&Q-Ytf9fE(yOs7U3?9jsW6T3H+-O-v;(lgcO3a z#L5DQgqO3vm?DB|53}M9>KhKDealTi{H~1HsC03~_HYU4J-^Xdb-^NO^cv9h*j|7+ zU2CS3%!5K)Ei6EKVzGswU3E@2eQ3fp+4Pmm*)8SzDf3Tb9g~(=o%nMD5<8tCV zk->hn5^+Tsl=x98b4q6)GQx+pK=)D^Q_zt$nOrV3*STN2@IMNm#FM5@iTm|!?n@CR zIlNk65is}9Dw{0Lfgd_Zb~?Y_@HxlnXRo@WH8%qMX1nLKv z{=|WToGhn0iW;+oxKH*VCp=FiDY9BsRvE}%W{xcaTVJqv~5%u~z4-p7(p_4ma9+#teC+8?*ECVO#+YFOLiOzez5Upl? zQjL2~6itGtxi7xGJOp{QwM~ij^-a^|7oUz1$pY0g5=XDVrxK9-DCS-f%0=K6pE@hA zCe?X`x_6-Jj72?<*9&q<+HsM^ytastaYI@^ZDJL<<@E={Tl&*5J!WK@<%o_o5Lbz& zoDKX~t@8uEDUFUG)i_2JrRG{oB8xRwf^MBk+SYtlRLDH-$@!tlk6hU+mNV9w_#Hr|^6;!cTErM{bC85%aC6eTt zyjmc{jMRYgRaY#!S`m|zCGX8epAeV7%2VTUqG*q?%1~!qCX(vtKoLudTcMk zeUWENeP7xiN|OVHt=w0I2qFgt*!$oZ}(w z&jMUj?%x}6muT7wEm$b&Rvqb$iYVv6SyDPj@vSw1oE{eqtl$Zt-I*<7G5yK+ zM9vA@>Dx84GQ9Qz=LmEqBJt@S!K@LH1HpCmYYk`^TI+r78P#;OCb7qswvb^@% zMO1E%(duHlT1S}o6U@ytncPIT;%9sX z;9RC}GmZ(GM{e_f99m+R2}NSaF?86O&<5frd$RTRrPz}{a1dOf8Q+fF8g>zTT$T;EXMMWz7L(R7m@2Z(T`dAg=0-)2_` za-$64+DtCR@VZ=v!%AAFCWc&qu2CNSifa`n=U1&s(LG(t-%>_pV=UF8xV40ETY)FD zTHG!OE_S1}_2g8(%|#rxjl|zYtIM>t#a`3+CRgj%atyjrfF?p%8c&Gu%!C>ZyAcfo zDYHymmp@=$XN^S!HQ!9qghH5?`aJ==C0RMn3 zji3RIuH7)ls(vAksJS|MZjOTlI`O{hSez!3ZpCG~*W!!I|1s{2w?%puOc#qXJX&Nc zV=_WdTF-Fq1Yxuh?i|MA+O3O7ohW}M=LmNR%T2{bT6>Fk1-ah59nup=Yy+(U!GQ_U z*tSsSYu!G{wHr#Pli^kbL%Di%DWGQnRN-b|A(S)#`UIOp>;mdc0J9-Hq@`=f%M^xI zLdh_uZpSw@Kmt#*i{Ot*D6lWReUT8R4d}$hib9;DE-7k*a^#Qa55ca_NO!#}-WE-Y zy}WxJ?}{+rI~WZw?co17q159e5v~-*;Oty^A8AJ*^BvtQ+6K|s(!!jr{J?XxNtZ^e z3C`K-renJ@YF6Q9-z#jK`*b2k=s_ja&RkyMdhu34l(Ef9E{p60=@v-4c;(iLlZ56L z=pL$Ssk})L73I1WYw%qlJW(YFU2%K~Fex&FEWT@*uBB~i*}aA49?i8P$o@h!15cb7 z6@9xe^ZW^BIXW%P6qp!HPyM?Hb4>2L25QKD-pc`)=x~C#o0H>J@wivtc{b(xq%Xd? zI{>-Ml>x^#`o`u$ayEK*WC3-;p5 z8GT4r&B^fh*TUT=nhcxUM7jp8p?2O9m^LeHzP7iz3w4_{3k-4Fl!SYt*h{B<9!QoB)Kop)+m84R;2^c zhzw#jRY2E>odLEXdUAwImB)qCA`(u<%OW(oDkE6X_JUgQ99crXPoKUICZ0%`xJUVoY}58g37CUU@m&^dMKp&-#=%$+F|wql5a*y^Vc z6cL9iTF5#@@*heGmH3~CoJBWc5eXQh;s8ioCd8H^oc=J+!sWSgJE7(Sesi>6+ewHS z+mFHxdfPdbVRpxHLMFQOM%9%f-09QKw$6*bbg&l%kF$Hzh=_J0rbtQju~rS#3_(H6 zkGHQa%!#qY!NSW+KodE776RN^=t9hmYWNL00iC4|$GDuntgA>!jn3P@bpZDQ%r3Dz z>*_`!408hE)DXEf{=!8#9ds0VsVgoJ;!c3vfVD!jv-jiM;?jxJ0U6ZngS&g9s|U*5 zr7zJ9Tk(H_97_*eVCO>Bo)Jj@>qdrjWP@+_ZHxfo6p?v3;h~Zad#H@t1-k)cQ!fKQ zT64Z$(5%4C@r7G+23E4YSVjLhyo@kAbd^<&RfW2e-F>)gsKr`B%=JNT6)RB_;3}l! zHnFyE0=Y=(BA0{c7@6-Z*jY%f<(KDU524Nil}m{ycI*T3;Kx**=BD%vGW3c2`2*gi z{d=q-%6#ZW>wSHZPQB7E{VUEDwOHvOP&Z|J-8=eput>Ob?H{VOGtDK0#+6&T=vOqg zG`EIvnh1A3mL%IDPNzHgt&h&5KsW1+r%LPT^wvoSas7mcbO4TdQIDU6XPs> z=a~RGLoJ>Y;NFUNgTa_<1~lxzIXLatLSE_ z7E?x{Twv}+FpQ4Q65;VTIU{*qh#4x8pF3b@e@D{|`+I`jfl&ELy7pc@RyvqJ@`R9Ncxs#k=0t|t-SZiT>y%+rzf2R@ zT8EH;i^Rg9mHqc6!ebv?{>39g%sF&=xM7_CPylt?v^dXo7!-WQag^@d#zex_Vd)9a zeYgphuz^@XM4lUl&{$#4Sw)bW8Q};6BR%GoP_wgD(ZjT~?&=EZ!NFx>Y*VHsgPG>( z0zE^B4XN!91(|mf4-M#=w#xIiOZGYp_@vC%O=x!|n!(!%6b?Gj`Uko8xd6G)ez zIAOiGVkX}%Ae!wOajNfH$AY>e$$1jqUXm*91%e%zX)kRx2z<9tmj%THge%q9=ePpG zI0!c(197Gx7mw{7My?2sVx!{$H4<^{8Rwi(MC#{~Smn4v5dZUX4lc$;Wd0Iep08q; z6TvikTZ~S9n~{2cq+-AY!+;NYHm&*gF3pX()*-eD2&McA+s5qMu=wy-ENGaogLKs|j zCi~r!p%xx`;Fb@>!wqsLBZ>YqDf?xo9mS(&zf&ya% z*TJ=}?W;dIV1H|m?1$(r@%{6`zteNW~;ctl||hR!6EYt$nN z@hs7)J*1U$iYOOd_u5I4wdh#|mZYze$Zjocys>%uJ+0B(lmU?rbD*bf_7-NA>xD^> zkrI*Te{5(D72-~SN*7zCEq@JQwlH_Gp+ZnB$i+tMWLoTSc0nUl7>dV4=k7_tQxE<~ znA1hOokvTLJO{w(O1}jgj+peDA|3R}hT>PEVIo>{eQ~LX+>i(}al*wLJ?DaXY_Vra z{slF*7CsLcHcE%J65EM#X)!EqkCy0^Dw>Bs7vW+t72@jm4FS%E(VkvbiEo_`M#FL* zjOFy51)VtX#q>3;j;**&f4Eceq$`*AM8^%u1qI5A!w^aP#o5qtYc~2%LfwwcP8vJ# z7i;K4Ax9e_>`FCmyD)O59pPND*c86AuV-YwLJ`lg1q{1aW222t^8URSC!l zxE0%9Lce>RAn{ zQ-640nRa@~PF%;h5@$n?u8>wM%pVi(tXj=cNn55Xq!F|!q?B`5nVFvAODssDp(Dzm zQU$>l`A8vjk6zUEk)Mk4V&BfdeO9Ea$-2~xqb`A_w^U{z5<5qP|HK3s|AeC5ICyo( zl{Q0a>ZM?$hmNrPf^BwL5rmzf?r70c6y}B=-@?FuZRPifUiRG^ra^y_vzbdvIUc)Ws37=4xTPrV%qO=N{~(yIGg1CM)+B}6+!Z`#qtR3Yx-GzRy093iT4zff0WpizV7+pza_NwlzB&@hg3+4LUmTGAF`pL+PE# z`egy;OHZY{8SC8#?UJL+E1~|@C8FIhOaFZL6UHTIH8{4VsdL>D#NI4jdgCQQpc%?5 zOA<^~2l6V+%g@uJU5dNmx zC1PGdkopWa?OW;F1&}M>Budyb|KHX&IXcSdgXlABKvc^pvVLO$Z^ei(vrUhiq6JXK z8$iKY@t!FmSc36%E2chJ6i#OZ^@BAY0;2CA{i5DmipqmejEtKFxH_`$*E3`Y6xum* zE07!zJp33CWvo^7w)EqoNS!Ob-nis(hn5o$`lsAmZuErz$Buq5Iz%}2Ug?z`E;^4& z#1$~$;}StxS`TwB`Xphf%A^UC;s^S6Wi^T=&!o$=3C-;1a6B3l1vpvsoV0g1NQkS! z18;2ndG}5b1xKZ_8ZU{UP7D}sSl$sumOQu`-+G#FGfDnLxK+Wwj4-0I6Im}-dIrK} z;>v>enV7a)v4Kzz0?topm+bBn$NL`%axo*NM1Zk4w4)wo^=S&YFd<#ZBB&gU_k}pF=5{2;sn0L_4zO#0)&ok9 z2u_w4=BE^@=18um$94!|HCd%yR!MLSd)PDu25GXRUV~^px9u9|4(_q%<0v zd<^aKr_+qef4wj_8yW~K5reP^X8uRGlbcR+vLeP@p8&ZWoVFKtM zMK}XIPx~7255nA8@mlbF);rDi9s4Xz`7|2JwWW1H9i#HEM3K z^~2Dq5?d>vGe~<>b%hziDA#~(aYQ*g^-;RqLqc3q66gIV$@{?$Goeq-v1GX>=al-Do{){1`#%-ty&WPM!%%vk8sFEhusvvW@Row^|FRkfQLjmVQww8FpS2L!kjQR zf@xyBRgl>|9d>U!Hzc)S1WCD~Bc{w#BqP7AtQ({8nlSpd{G7%j--Ix0laIA#EGvX^ zNUDv)f_b6L7f$S4c=XH$aKOPJL4euG!f4CYxJ;nir-uGq7t&_B)t`jA&!DL}*iC;| z&EFTyWZE@^EcIJ_k2ke+5M$2yLGpx`rZGebnhTB)PBjou)Z-`-Br&ONYTT&r+9V`%{H7+qxgb_lOS`qmnAug!k-uBXoKVs?caum52z1vxp}@QmkY74&cqj$2p4)< zIsQ*bZg{q1I^RE-6OmxOKk5q-z|7G3z@M$NceSRn(KvJxwp+T_zR zF#3*2#pr(tbEE276eos7$UMuiu-S>hJl`Q*vw{?p1Zk()Au^(K{Y}t#GvK~NP4kv8 z*e*FiA20hr&;an%ifp}7gHu%c{BXZBA_m2Z3cn?R0wOa z)S`gqWjE{8_@OSrE|*@O(7G5aD5&$n_@arUlg)%UA7p*1<;k7*z3k~BAgK`5pE^pw(o;@1Qgm^5;i<9Ja?x)J;?U50P&8E zc{3he42o2-CByqhCUoZ=>n&a+NU5_gP*$TYgqug|3cBOWpqWAc98FJrq=4hyrkBS) zlL{Z3`}BtcBVwx$j?ps{Hr{RIKPGjsKPoRq>?msX-AnN8D&rzh`pZ2+T(u4w3VBQk z^dq|OiGK;u4h%8)BL9!8^MI19sQy2Kh>}&VNS3gYac6dC6G#>j5CK7gfb>lFUt7E|gP>#pOB5UfhvB!VuX1uT6`wla_8 z-&xxX$MYimx`KEjC$9)G6Ie2`c3W{>0Qcx#`RA>_p0i}K<#0RQZ?U}~=fV-C%pMy; z+*Y;udB2qw`YU{*_`&lMq3zRB@;my$hGYmf(*R7xXM+L!GX0Xv*v5RW9=})uA<8<3 zEbFXaq+$D{x4ybTf#?f#rf&#=!+jNEl$)>DAJXzY4eDaBSW& zFIL>lZ)q-$%|&?4z-?`@r^q~tq?zP=0mUrc$1y_MFJ3Cxg9Gt(xZz1#<9h_U78(O| z{IuET1=Kx|qSC|}g4{j~A)a7rTNKH(?qkayhYNE3vFFtEIVl7t&zdtmp4PLACC6nn z94WfndRqeL>4$!f%88EhErG5R{fvxW72tAOD4Vh4RzUt5PV)GB$Dj~+OV|&N3qA#b zXeh`aR@b6Pj2Re_-_<$q6EZdWGfy-2(}@>OcRJcV-E@x;w-6VbQ^*EJ*uqD$-w17e3(+B_#6|lU5eRT>!2feHTxG zo^gr)(SqIJcH?DKeNdR+W%!XD_v`}Dg1Q;iIRLcwjE))cA;j6!Em6sj?T#_2rpXsA;@o9a#vvZpH^+fHgQ}wA(nfqol-W{6M;I50maxy55!-%-XLZ66 zmWw-OLZ`+ke1B4b`$0bP7)N5!3a~Z1ZGYsmRpsr+QCX_K1v6o^?vKK zKzVA%fJUPzoRERa75Zh0ZM~c;X)*q+Kv$KJTndVO$gV}6yozljL3#z5LBnlLjjxw! zdeu4DX0G%((wMpI2a^9zah{gqQ^>9$r#8{Z}Au!yblP zf_I0QDj>Q|IC{GvejjdQjq|98+@o-{%~RcbLUX=~?ZaWfa2&Ms3mU zB|-71v`u0l;R`?2FIf@>nzA4xXiWmm&_OJ3{-ZS1nk%;!#AaWBaFJx>tLWATmXHR> z7PtF6EXcg6Yo}C8s-9;`G}&4;Sz+U(u~(6!I-+EwV{d8>R)1aCl3NQyB(_uYZ2wS?sL3+b&_6PH*Fg8wrna_x( z3aAIQ&SWavSBOi+(gs#4M9F~yb1#^zAp!}YatzO{`7xezfg*kr!%)8N8)UuZU&uAv)Yb#1Ub9z@e~nc+k-(|TREcFVow3; zlSDF!?;HX_!8%zrT%qSIy0+b>4ek->*O}j#Umh30>*H83#Zret@Hqre#=lIIANdLh zoWLoXn#Zt6Dx2&UqbbV0vVz$nhVDsgagksOF-*_CQqL9{Mb4CE8cv3Exu59=*R0mV zj*&Bf65;EL+tR3z#x+30l851+2{e~DQqgT0@vb0BWz(as9wIm)li_*G9(MHltwDKlZFtq}9i?Nvq^M%-~ zEmq>mL}-p+^Io^GV4NnvJV5i-kKy)M4MhB9E*Xzqd!bx^9kjt*hUzE#09_^6P1u7< ze5@ZReH`gtON2SD^R|XhpP4?l)(?EXp3GpGJ3lKx!qPb_mLA|)QaqmlB=!>Jo`_a} z7-1#Nbrg+wR5Tw<3fzwmI-7A8T6^f(RbLB_(Lpxne(@*15lZnctPH)XE%`6^}X__>1-4) z;9k9;65|4#NEZdC4Zx+sTs}sMM(jQcZC?6dO?q^#FwfQxep%5D@zZN9iKUB<6_Yll zbHSh}pub2fyvrn$C3Q?CJ3fJ)7IfoiW|(a($}+HQBOkMHlmoKnjVid?i*L zcNU$~XUEBUcK(=X67y^238%zVWOnSFiH6*G-x6V7c8{@W-g*++UzeYRbgQ+vT_ANP zs5O39Je#MZY_Z0et7DBeSZ;U)Ng0bHgitgLAX{4;RYUBeeI*bTm{rJS~vk7 zEYJz1+?R_-9Dg(zg+yUVG&>P|rE~hsxaU}&J+Tb+FNpg^k`AZ*N_^`$2~Xu?Nr#PVemA+A+(ymugW679lr zFe=yR{&?oAU>*>M=!9}oV;7iEOCWkq%)y)Syb_w7gsH6|T`yz=PUx0kM%R~@NKE-FJyrcvW!p{|U=xbwj+zFuUh zbFXyG#;7nCf3X`7q|=|XGRO~>X;z`n!~w;hvFkJ7H#-_WW^>XkH)X{PDYb&itk1bpuXJ6nm~RR|4@@ zR#wNJ36UiVy)!d;+$P8wFu!tadzS#WtM*E$bKVrpB8Hy>FQ)txOml-W24| zVsc2k{O8=_t$JmY(K1#!uRw+ds(`V!5KEHI%%x*y5c!8tMroGp8_tS4q<6;5uf#HgU{trdDUK7wMHv4nJ$PbG=;*ftA!j;67Q@ms! zc$z?e1RX*vz9pKENN3$*0}+=AppJ53z?gwLE`Je-^%JsWW^S?G=sUulzkH(^H^~2H zfi6|@V6QQdyeTa8Hg4i8f5*~^P{_1z<9qyGIOi9BS>*PFelbT`U*PW7AMXfq1*9H` zp6^3jWVDLTCz>1l0F*HSI|`OHv8hN(Q0LO~pXk|*m_m6Iy(1?^zAC-lvBJe9FyFbW zx`h#_0DpqRUMV0nsQyE!Ys#(>v15BQbV-pnH+)HNGQUK+8{v0NAug$C z+KhWlxY7R7A_E3Zo%Nh^8Hkw2)co@w@@$SLqm^_3^Cy9>4@yIYjaltSATBTCxfVJ) zq@63&=~$Lr-bG4?h`iF5-n#AptPg(bw~fs0@@PR}pFz!7vF zZ;3=si8V=;j=3C)`f+FVQ2amyKcT5$)3+#73OroF$BExuY&H@-98E|#re6W-Iyc+g z;G2}hy9o4*r7bkU9h3HNM6gSX!*8-r`ja3UGhHC6$f0jv36Pn?+2pt^H>)x4suGnE z9ZGnWBniV z3@`feritT3@B^V1W9Mslwp6z&oS%PP^b{{@n6jrSb@5L?oj{jeOu* zlDJLjkPPNjW1g_0f9b*^88XnlW7|di;tC=uZ0Y0cLbC8D>*!Yn0vptc2V;>aHOfRhmVY2t3L!3+zRAfj zl;`xdAvgunFMc@Zn#OVv6V34X~U1 zB?u*wr$l^01T5ct^Eh9RW(p&Kj8&4NWV1Vf-Lkk(pd1rThUpYLsa(5j5&;?1}a=gkZ|b%Ao6T*5WNMq`JHiy4;9cahq{qopl+b#@!O3Uoz?8=*1y zuVpg!BdkCbwe;A#i)1>vpfs-uLB>YWV7B6_UuQ~Y|8ch{XDU}dI_6>bfVdM7Xe=5c z=>)yC@i^>PE?k#~{9LAdQKnduU z%7IwvH{{`(SE|W21NnPYpqba?ZiDOqZP;aRjrV95yI2Wx+1VjHBa(^_v&-XJby1dB zsVil~_t)P7xx}2Rcg~Bg?*nj_3W5kbtG@$r>(>W{8d3Xw@e125R9Q%~MB~Zig6%WA1IIJXpN5n)Es@c6OEi_EY> zV+p*S0ChOdG}-ExKhr8GOy3d!BXvp<)1H9vhewFejk^#2o;wNjH`t14i+z9)vr3oy zrg`;HVdT?5M<3%N_-2ZO7FUTfLkI}Z&)1iLJa`7;s3-k0W=I3^?o%c(53R&ie>TB= zJ@{Y#rN~HDm9K8s!IS$=TYsbN!hVZC1vMR2gMm?hFxnKE2#$83RSnI~jURhx05P>+Y=xy+iR z&b?SLCD9RZEEb_Y$Y!}6n>^>A*!B*`4kBC@dK3oQm~1~^K#Uz|J@BLvL+@J$^7PUlT&bR_0V~=2AHiyR_S-sy zBLJ*`+-C9)M=Q=CakkJxc#|>UuLQXc3QCj~ahLrA%yp2JnUB677f5-u!WoLqUxR{2 za>8h!$J$Y}1W>E<`!m7zu`gZ`K`{qdk8b);=YcM)zZR3C_+=c^l~eW1`|PEOhB-yZ zxqGpQTo~euIJ(fZe*wCB^ex>?t7ET(nhWWOFYZz@a_yC=$yTpJNR}u+7YM%LpD^Sd zh@XhC95wrxO8#w9UhxcsCyS(<9n+`9srh*(mt|h8XZJJq6KS%uO~%P@@`H=4DaJaz zb4ySb@mjAF152PA1dX2dU2&@*H%R|DdPt1`bN>TIJr>Bq<5CfRN0#@Bd%yaBf&9Tf zbh{icGDtot5N=qbtZ_%4APmDPHy+b-F-%}{LxEWkzarR`=+Exb-~V3$u}otet;Hfi z=0zQA#-yKmU6{YvBs}GCio<@yJAfW-F-hkHC+;bc)Ca5a{&$PaxcQK^HABuG!eFet zI^u2-2ne@i5QTf4_n=%;R)vk&MYPL;Cz8y_`(kflye@aRXo_&ba21lWgW+)B|0_VO zWXHL_V+u4+bT&M7S4T=~?&I!+E!YCRJT-@;W0p@XwOitBjV?1&&wLzf?!MSfg!#!e z%k)VW4(|i9crqR0qQG$IbAf`Q%pPdXav06rHSA)FVp+`NOJ5Up?AFTsB$z z4#%;}0KlGpRL)wFEEMfmk2RujBMhb$YZV+Fhu*(ATIiKV2>WNw0A=+R`85*>%DHxz6pBtX-EW zL3V-zb%rV3n&h_>PNGB4R2IJ*-FNr6NE%f({_&yZAuB(?jl1VQ#hKGR~iM`Y&ZIEzkhx)qW~6O{DYAg#1=0VzD3I^s-0QzfbB z0@uD66h_@UXLNF>Q1QEZ4~ynz{6m<(HOO?$u5bBOKys&77+)4eLnR~=d*yqgAip<> za*MY7#Ht`}3_PA?L8D8G4iE~flB(q;5ze|=(P3F7KK)@Jw+FGy#&BKZrH+pPI&*GS zlk(%}jIfqt0Mo*L*^iROJ-^EJAhC;R=iQxxQQ|frNFARLW3l3D5Ue8@i?dgNJn`Gn zB|IgfEiV;e4kE)uoJX9l5Jq9xjHg*_I9?axPvLr&G*CN)6VYW@3e-87{;m*ka!n>26p_|G2A0Pp?D8ld!_Ojt<^k5-TA1&)I*30@s0N}ZVn{5A!H{TC z^xLW^+Ij{~4h!^5wRD$Nd{uiVK&%d`R5378J$h_$S`d zvlDA%q=jSMwf*yKtfxMsXaCH?y{!%WnS#g*F-CE*NR)*SUTA8Nv)gZkdW;&PsoyRW z6&sLdEL?{~Fh^0`v0&s%`E>!z97dG3LJ;@GP6Ay5H1JjJuy-z@bo&Km9u{Oq_qRvT zKaLBajxEmnzoBP0y}a1O^f*UYmSU!uxV229n7e5(`L!^(I;e6yE`n-vm;n)zAXSH$V98T@#wP&$367jb;uO&oX>Ql_m|Wk9X%J4M`rC1tfGlpf7dHg<=d}!qwrID>1e()jH}cOMus8XS z4^4xnvfREm9G@3qcJghkYxdf>0EhuLHf$g_iz23BYx`BDF`E$s6lfIUVPO;kH-%(6 z{9LA~H%onV=AvC~He4L~#@9C~5ZesLAB0#ew2)Oy2=0X2)ESczJ#r+r*bExx&c^)Y zGCk+Em1g$Al4iTth-Eh~GL#p9baE)H1-kS}@YLq}2+KpsD4|DnuEJpTZNXkTB)4Pi z%F!VY_YvK#^pFVuPLHWJQJPIm2XgySY!+{)37{0+ED(^@_jly?B^XzU z@cWE`Xg~U6#ZDkNm|PQ4imxr2*AvY~MWkDV{*0PXA!fG~C;?KtS46l{Bg9QlUK)=J zbD`j)w6rq{08vKDrq2YR5j#6)#9Tc)HAb5(V)_NZb+lRBuDC?t>0|VRI|{aRj7mwT zeY{YLgJA<#M2n#O_~N(P!|{nOlh&!n{vt^_z>X$v&%Ys9lasx%-YoxyMIVD*Cb(L; z5({VZi`j-%M@1g_7YH(6WLbh56juB}AeHK1d%Wcw=hi!p2i6`UC=_2B!z1X+A{k&b zY1ZOQQI-?ghY*-$afz^kWXy~=N`$#j*46!Uo&PAe2gSuA%)y}ww=l-z0w_rzD!av7 zJ0&}vO1ys?2)7oSFE%oq<&S*2z%UY8fW1PWF^VWXN&pjpK-dl=)4*l=4QH`*k|DCts#X3P-ko{Na|#KEG?1&r(ErEs#4+{M%(b!2nW zc3{+tePeuA&+f7e%2QGjF9`PMuxTg!e+nb?q3uDP{h*xx#$rVw)JYa7c-1hhG1hD@ z(8br@R4cwMN=Va}ri6XV%toyF+F}Q1Mgy}FA(fUFj|+4KSPLpT&Tu>{Y-*?{siQG1 zfi^M9oNUqt?RZsbt(KY6j-(i{tL_Nyy5M52DJ6vXOT>(7V#mgPX=N%|g3ztDERj(h zEp)msgzg}FnD}z}Gb=)rsqvVQ+?eC_L9}P_lbs8Qos@1cjx#<3WZ}T53%f`SD=!JO z#4G6IKQrkSUo5{1zq;`;scOZlqWvLa{iPVyhm{b4gBW0|@v@*?OSXEc8yxvruq-xo z9a4Qqi1XH=TihVpVv@X3*?Y)F`Q%-J5pXL6za`2A;EDvTtYh})3W(hx=K$3>QxI9_ zgYNi?Xbn95oW6d3Hwdj76cZ+fMA0LekJ%v8@Z5*q63NWBfxygXMsjKu zNACfYnTk1la!`0!n3FLDBMAh95=8AfwwT4?#vY%C@^~|vnEq`6&IP9z^e9ZJj|p>` zaxlKT_XHu|nnpD;Mz_5E7l2$$bW_;B4HGM(BG9=dpCyQs!E-0>n9N#=$0_dL6%r4xU6#kaXbyF0qZTBfK=1{B**bDAh zVj3V>HgeC_7fI(1nwBdJHOvk*BKICxuvY zIlF4c#-b^_Vuhx_lKW(UQHGAkwPjZeKmeNyxCQgcOki+{&7_E&7yH7Py&rUL3}zID zx~sqB6xv&9P>cPY7qZfh7e!d`P_|>+gi#3V%mYAOi*B7Jv5#H?>T6W725K8}lQO!Y z5R&KRe&wr}(`dMjg9J?tg*3(xlR0;+2z9}G=t;3iv>O&r6=r>TMYZEm!PLB0^X?i4 z7VqGBQ;jc|bhFLxBciiBX&)_x*TrQTt`+W$H$@{%v$>?^Ej^oARk@i|W9NfFvcSXn zYFsFQ?+?`Qg4z7bCb$`>5|4;5ldyAZ)Zykcg2>PIBRclo>R=G($2eAP#Et^UPe+My zYbFRyQ;&N^I6wAxbZ+EMkU1>Z=85DHoj|hemGo-~^LY8Noc(ViCM&teKinw<_Q zGG|Z78nqc`3!)@g9oOSY5e(x}HBjsa4uyn~nzquhIX(D~!WbFYG}0y3{|c0wkCUxR zoLth}crqNf9menEj<&fH8&x38=MnC0=I7oLA$~Fj{&Bew7a1uf$Njtgg=$+G>z;Tb z)4jSbf~H!ohuW417Np=1+hm&F@`;sDYMT@v;ari0l8@^K8&(fb$(^! z5MZ8g{GjlKYxftKaA67h$n!FQT_`F;W|+qWctUS796nc)M0Shg2l$l|v8o{GiikY+ zDE8DR2BBR827dGo94rhu3CC#3hJG#4E)j#UqB348gdEd>*!?1$14>O)ADFic7jH12 zXhHXY5PEQ95^diI&t?j-j&bY%Rvo~zGeUKxD7Pg|l+FFF7nnr>%lLd+@vKl+rmw=Q zA8bHpc}z-@I|VpFOa2$@jzYUC1MIdDxXpz4YseVhWO$jyF#_QyU0R0YYaI?r;XCHc zj%j+%eMn>Z^f72yL65D(=S5gVdWjd5E%P;&Vukc-K!yZ}ECU>3^J zU7G4yEg)Ao*~$^Gk^aBMIH=3dNsiWCcTRx#liDvQ1HPU~Fb~T~HAS0gGx(Zde?`k` z-C(oId>~4%>o8(SL}nIch&S5MQ&}?ws7tK6z%MZCU^*+prD0{vX=t3Z1cEw-Uh_*K zuB%qh)%d3Xc&mdNl=v$g;S8~3s>H@3TtX^6u22Def{+c{>t8L(MH-WJhZiXdT0mly90N+;qq4``PlVs2FU2a_##xTfHqhw+j1`TCDk1XE0go zAB(j!!F4zTv9X9!=qBR=0l8SLmKgFIadCcw)lDPjokRkE9~DFDMNL62Hsk#`X8vil z@h8D~UCD4cM(Y!^vlUMWFFFG*I`NVaH%_h2=1NB=)mZstP*)2p)K+XF8a7~e#e^t= zCQA3%$BczVQjGl;#4km=kb}DJoM`3ioC4&Ejn^l!ed7!33Uloljal_k zIkarrL?|tX8<})jr;Bv?b(W79ZTAfjL~&l%-1xqp-A`m9&Ver?bfrLlJh@eCZ2Pe= zvxiVc=moUfCi_=~Q!_<%YR7WlgmO(R$msF4`=qayo%EUHJ;VnUneWqY&Bwzt%>~KEnj6 zqgE^yk-Hs(7A4vI%mR^e6k)zl2)ydUu;6w*XSO6iTx9irVX8h$)`9rNSx!AZI1t-^ zi)V9|4WmpT7-shtD521Z`ArcnJUU;1a|O6S9Li`|{+^J`PgW7c!-^|%7X7W(Xf)2| zcd@R&%|h-PQO;0XNba8bHh@d4$%X(EBXPnxU@jL14j6jL7~(#G{!X)j2_`j5kPDNJ zu+fJpk&$kzbM879#2>`2fq7@8^8hT7avjo*rqp6(p)Lll{6o4EIF2ttHO6PGLv++1Wq>DpPWCDA9U4U6YbLgD;Jt3CJ9*zg&;0vMs z^%TlY{pp*+s01R5ZtF%-&VyC1Opg-cJPwxve3kDOc{mh5>fgD-{1&!uM3Iljg}887 zqvE*pv;aQCp*d~;;zdxdvKI?dH@zaz^&3RpQu!XFn~M{a#&|C}7lI!Ponjz?v+?FJ z=9C1ZN_)vEG>M|ig+KQl$mi7#Eu&lEo0KZ+(8c7!{cSzF3RR3?lG6PNVHA%A z$ozO+1jR$Ei%C&ljP7wMAm7hUHyGc1O^9n@^Ac^VPcA{cY*`G+=Rbk40%r*{)vqWC zWz$_Jl7Q7OgQl$<)0l&qE&&t@@<3oBjGkS63i_XiTE!zwwI(LZk1o256J;r0P2(y|~c9~Z$cLj)Bm{m_b zHSt)iBa~_({)~oiiL%IH`$s(wyAr@hHOxpX9_6Z}P3z-V@xxRDZ~?f!u+G(BW+rn@ za-!*r?jHkMBDt+To)pRVbR}O;yele8A_}_{5qphmz&w2yM%b*-HV|wE%-1G*P_#r- zo(FO2PoP|IR#M37de;`YFwr}{Cl&mXV1E!5wPujvXbUqdaS}{}*X4r5m1IzNn<#&p zz2iW9aS^mR$su}+tDLmj`uufeHsdYYuNhw!l)t8L$>He`VP+C5KPjs@cbG5GIZ9L+ zQzcZF3gx@(P~{&Nn_ds?N>V2V&yxkXV4R{3#ZPX4c1?&+pUmtx{wa{<0F9*vqvwVA zCAOfw(RX9<5(WSm*fH`5$_+_$B+5uvd0wC^BWn>hox|~ViB(#xSu~}sbW@QZQzl^+ z$D+H0I6qDXiMI&yCs0Y~E?8P(zncN6p!9voCvMlD0Z`U_Q%5WiojS{mSos#7T{8t3 zVHbIz5LcfL!CkA(7%UTE!buGAIN??jxf*l>8je&6b74>|$U6@3^pK98&EEMK; z>Q+pgCRQ)~af47wsUYQ=#r`N7={dYoV-dH`ZBQWKPd!`X4^ub`N>yC>rAJ9%_v z*ukV-SFCpzn7>PwsA1zQAzp1tn3`?e=;hRThj4!jl__Jl93MOH2BhL$Gp5HvJ?Fm1 z`hC8wAHOYBf|Iu?ajgJG9o>@-NB)?9r?iTGQj1-G&F`=!UBY?FJrLA|@fbGWE^2DX z(>}q$xVS3%UVfk!D4%F1xvMM?l1AJfoa97k%otNI+vqo@SmjWy{Y04oX!6;A#o~y$zPpHezvY3NMVyoX@KvFnJ zG}jPBzMKq=ARTeEXmgbs6ILK@6heaxXdU(+5tMM8a4j4CfoFd}_7C!~J4+CS#wkhL zz<-If+%`FhMaM^Uya&L{V@1EJ#Q}dTfM$w%u-t4Ic9(0FLiMni2H+&c(i!*NN zi2HUz+~nFa$5%_5ecMpnk!hCD>1b`QM@a8UzCyI!?#ci+>hK z5~bO$#zz+yFG+pQOI<>;^l{`9?Z$#6g}>#v zYr`i&;UGJ~YFr~CXNBG;=>j%<3QYZY=B(IF&t^oM7>Lh_r1Z%mOEb`+e|FkF?42Ic zvs)0pGVppNzWlGid~b4`rN#bFLzr(w5kebZiIyN20j7;4gVpPVxs`B#W=9ln{LQI2 z{-{@bbqx$>%g=yOILxod>57+$G;5RUHLexloVmJ?cuS}Qo)+e@T)nMQ+-q^uv;2@{ zB}Im+#@_^)@2HQvqwBdMKXl0~6A<&+!pvhrx1jlxy`E8L#{4^hzIaI}?URwkt@W;J z*-ZdMq`}raX0nL*mG2vZa0CY*a<+{Dpi_kRTAS2*? zdd_*$n{hLx*l*?pyv3qWwa0o=q`*|6GC`nJY3D!kj?2cydTk<=KiY z0drwwnk{9(kl;MftCcnJH{!3#*o{QM(jlUNRO4x3)Hq!y6)%YLe5X?$2k9o&w3mxF zbt0qYjO&G21}bBNxa@u4?**V(Db4tjfGiZqTUwRB?-elEHNrKDbM=h44iTm_wiB_$ zqC`$Y68q2O7dmE4i`Vq*YV{IIHah^X_$nx}i3Uc-IV=1F(hWq+z@}-xl;X9p1V)#3 zic(o1SoF&>Vcpk2C|M6m$o2K?c9~!e!S0%oYCnONG?<@mYW&qdf#4p(fmh{fuS2;9 zV-p&~B3=Ckx2{waV&>n%oUu+L`xX$_CFZ0mQVr*Yw<{hGscb7Bgk>h- zWZsU;mB=E-N&HyLhh*=TuM%j=s)`T1M@kE4$Ly{-aiM6nl&mer;$#0;{ESrM&MMli zjbk0V<2XYIjWtFL{NL)C3Mjf?CH^HUiyNz|WEXn$QoB#R&ncrWxL;;z5Em8JumRE) z3bDOVvo;$UGq&tdCQa*3d#fKMGjhiXAunNQk+FROuv0M^pb& zI*W)6kkr`-gIVUT^9tE=B*-O7k^VK1t*{(#=Tg$klY8l>h50)u*a?x>ka5R;Wl99c z=PJX(Tm_;jas<+j6+Q^$V)yBW&9PYJLm=?4M)cFMctjLctatasoy+qKJ2-_SB$^1n zNDZ`8|DO=VGF6LF=6x%eX6f9aTU9?QI`drCg_YPqh--slmJ1ljWlLf17ucgA8NC8% zWX&bW05dGLo4F#OI6t#1_SbXnlN_|SqF;bNG>MLwTGUHK*FK@xIZ}w3#`Z;C#A5F) zWxDEEavZx$h*_3YdVG8Nm5NtcEF|}T2IWr+r9^`$@bA^LON1H&MevIm;L0CGJ{pem zS0;tKFyn12{wmrSCC{3kICEq+;3}Ecn1m+%-%r=q7{|HluWieQdFRY%aNyW$=_&8MlxGWZX;?#`b zC?U-T=l1^-;ym$#vyavEHNdF7mJ(>PW4kp?)0Y((EIuPzQ;zN#q>V=fQCGrUAucrE;93TFrLLohXbM1D<`HDypZqf_Gk8y)JBu#mG}kTMxhuK`B@ZsJ)I62s=|a*|<Ro8n&sLR06P8ef?T9%Mz?>1vh?W^y`706%y4>)3z(#!%*=RC#HOe1tHwjMLs6dhj0tU2HW8TEBj7 zo!Vp*P^RAbJ@L98U3Z0h;Y+J*3gWz_Fs0uT#Y7kdB|KH!t7p|w*JCcVS@AlGV(w0h z4+_aeQ(QoeM5}KB=Gw^CxD^KpV6p>9MW4%v-z$liB&A3v8Sun&EyUlo4fD6>D z%x`E-Dfu1jD6+vSzmFN}NFljBu**(ckzWXNZd%^9($>j-plx*KScHxTX25m@?mbO|vpbD+n00PZscN!Au!$FDC6_2kQe(Inrb zDXsAK<{c=C(*vT)USz@n)!U_GNAjE`|UTj^S6GgGKT!Q7@ z_KT?R3~ILba@|#Gx=~^Nz7(YLa6VFy#iDPdyFY$d(j#*J82=2vo4Z_Z&xG`Q0d9W9 z$|`hnAKRtKTcLK;sV@2~P!=&Ph-FS7B!6vucr?DaD>S*Yvi64JasjjjbGokdS(JZg z(xor61^S%;4R|x=#AAB45OH)zcohxMZwmA`y{CxnK35`Wf@tBd3!!}mSk}ciMDRro zJgfnKB+@M6OfoGWci*jmIEqVAguW%nT;ZGo9pJSBs2GjJwu>&fdyxR6bLNK=1-P&Z z`RX&hlLXR^#8c+D=(0@N8K!Fwexs&}y0nv?EqLj3O$tkdC!x9it$wt+lAeDE)Wph) zmX*SS&qJCWj0{o_yeq^NGaWY05aa?-!70CkMwt*l zA0p6UJ|T!w&tuQO^cQ$GlL+v|DsDo6CQR*j_S?%TJ7;#pH}y;@YbdB%I-8DH^RM$d zJ7a^r`PGsvZ!re`c5E%o1=1bO&@iC8)r#U%`h^w6=wKYEN0+-P z|1&ku>B4e%(ySKemjJmij4r(|Df}_+UsAY)6@<8;SVysMuf*DdGHbanp%N>9sd$Uc z8A_)JA^tj{*-;oF4c!M6uPIJ_J022X5yI0;+0Pa!4aHG!@o7JzJhs7B8_iV-?yfAdg!dpT*p-K*JVfuM&rdaN)b*phmv)1v!21KreSb9|nz7 zd}>atp+|nIFt)c*7M>xR3Uy7N7VB1cc21n+5fZ2#8bHdC z@j)9i4(sScBi*131)S$-uX>h9aSqqwOZ5y_FlojKg&J>O1)x;2zfthRtwd)5t8g_} z>@6S*p^k=0Gb)G-$E)Kpp5JE(K}Ft(--xD+jN`=np?Ac6LM)Fc_PKSCebVVQFn=XQ zVvKVHI1v}B%b2YAza%GeqSUZwz<0Z zg*dO=(pTij29(|5So}f|4a)AYBl`P`-?gpP^7w}XK;lnx3|E4I7$1a&}@L~50D%b_HdmDr-#=FrJ5CXPjOfI5;HbihWObq zKf_n-Br#@+4~{_cg;7SFb@eIywr-tQa^>V(N|`qjHeCiZex1Ar2L! zJ^72BtKOyv=o_@`X_e?+`BaOb{@pwrD zA~3IOcD#Q+&n{GjC}y(#Vif3TgSt?%+0$lmNsQJ*7fZSsN@r$Bomk_$-AlYGrM>3Q z9}45^1g>qwZ6c@~F#_o`{qaKnk)22_R$5SG$W|y>C2uGQ{&Q}pA?G-euDjwnQTGdl zP@Rrh)8ZpXI59S>qpSlzn@Pr$CNt8FBGpsmM1PM6uX$DJu_=3n2ZdXnF#lv>zVVR& zW(I;{NjXlK6UlHB^H~AvR(O|QbQI6Ih{Nn@N8$rVgHQ*RW4tHU5a}F;6o5!VG$M=) zy0{o1jwzD&va!8DWX_MVmrdlALb8Bhy&a!9)-R8Yj6{Ea?&PAqwx0dn9#%}}iF7|; zZB*-(V`BVOU~X%=DQ5+EY#tBhb|ug(8#kOC1X)n%&#X(Cx!X&ktOJN6pIpCET3(lL zEmNlsC7vvEB!(KXuv+n~AXkg)MuF(pLR-<1 zlZy01DC?pwnvT^<>-*Eb28BrV%d+%B5oViiv#d1w<2pgP0@NEne?n!Q3})d;YM3}; zAplvj8bZR)6vfBqP4A3r^*kkK6Sx?^B!JRmZJD-4?~3O8tZTV@=@bZ87K6+N5lTKO z#FfS9U?9ds=Y}QbhU$96l1bRSE-I7MdRQ-`az6D`-l7updo;C3Aid%=Va}O-;ZR&6 z+9jn!V13ec8i0kZk~BNgGn^yPC6ql7?EJY9RYN_l&?5b8FED^EUp zW|1v13nZ)6=o02JibY9_?g`WAO>=}JFl?MqS|5-pT{YZ~Uy6TI7C0T81x3kO8w^BE zM8OhWSHVRN!pu_kJQMiENGLWv+pl4rU5{-G+Zems1hwG(H_tLbB}A+_v61q_Uu*LSs+4r2v05T^mxwAHQ5b&^kjD_v!pRZ^rD{ z_*`&HHpbFpcv@m1RToI&4o)IY$j=oHFIPAp+;vyb>|~DpNnw*z9J(IuUr}uSS)kuh zKoVy2-+vdxT&N7OQjFOb6-g>1%rZl<%=bXdPOd^7(Am}Ug7^{^29>y01O=bqnD7}r zJ3AbhFymY2`%Z~{_DFnE1ivD~tQ+Wu8w;SG+5-E)-Dht|PI}yJ1+>&2sDrv{_LR>~p0>&g_G4oO|ReDZW z?4{?-AhtD{0Q&^NAnXD#U>8A+*<8{tr{td|weP-Igxi!}jHI4*836hA(8XWYbN&(Q z!@gMhhXCZNa6a*45w2>hHouixm77igT@)5>$vqua5+R?#SNn)8OYXzt6fp#tjn|+T(-Uv#LeI6Zq(xUqO&lOfl!)p4w1U^z2=uPTgw650;j) zCD_?=2+4`a*95@-WYUYv3lwFfBd0RoDmv$aLVwDF`whWf)@?*cyVb6Noa#S>9ZXA@ zwM(!vIiz6+>HV!h^MHY&5)YR&&hoW*RkSN9haGnFYhDXt=}s4dvO=M;HxTNkWb7ay zd@BwSB%xz~JVMW|B_^0?$gW=GALn&-#va!d?-D5^d0)>LWFh8MmzV`uaW53+Jerjj z3xg*FxNKYtz<3l-l?Y3Pev|+&3ZW1ar2p9Uex8SU>ad>SazCn|R=n>9C z&5Bu`Fv@_<2SeE|qFg;@NX%P*D8QL_<0e3osl z;zjgx@;cu3=ODEC42*n+^z6@%>Lw-@fR%VppueR%P8m4=@(U0OHLvIpB!AeDzRb7Grc74Hos^#ke^<{+Ws&oEQ+oV!dk zbZ@~qnZP^|`z=#ruVuhte|soFH^ zCPap+1M#Agxd9k&P|Xj;o%a-wO8uTM791-d#qmX{Us5)iR@Qi!_!mCAxWq~qjGeg~Z;pH=yAY$1TE z=(wA^Vnmt$=#J!h^Qqs1P&wjn4aIRHNQ~JXs<{7>*s0kLF9trsn;iQxd{nCpqSw z@CUz`B8$a0MZs2r&BuB97nVy%#2S`DAv@VF1Mfh8s zoAmVeph=3=1m=#$P_0>VEkWuX>{+(dbMES>DB}Rp60mWm;y6PTLo0SEI@G>Sr1_zX zKj}3$3vmf`uGbeg{1HHTcg&7E^qj?4tCL3jpAaXn;`>pJr5^;~18fr&c5yF}REKVj z8d?NJlhRW|#_T^qPf2XL)G2ll#Me6J&W)2E^7DA#KrGaAt~5bTn2BTchk=TN3wX!+ zjqSxJ1oOoq8I0~E!UY~=eq!TveTkr>CQxJCBg9!VVPFM?(b!ZTidMDPBhG`BGCF&) z4SU z!~xOt^OBG}-Jq^a8?LkgdHJ#8Z6Y2KY!l0HLGEZ6pQKAb<2_-PXqFPJ3Yo)Narxur z%j^>*c=rf$1*ESfe#;X8F0M)!A=<^on}QLzC)O84)6K(dchx70pEa~j?2T382)9Ipr&Cmk{QI%2zPug zg58?zI^};ubWdTfESoIN3@me&dm7MuWh=_5+dcy5|MX^t|8qsTxLB4a!#=+0zZ0ZbLH4)Ax z#Vq7TtGHhnLNk*Y`KIT1cF_qF2QQu$;1{$_C15?CBhQ1m?=_n!f<^irosNveR|UX) zMq&<(Qoto4lr#k*V1%3bBD5EXBtj;1yvnlrm%=k2h#JNag&5pjf?Aqbx%E&9X6dG2 zYre$m=B$i?BpW?~U?Tb`ELqZcuH1kqLvHg^+o(%~_~-~h*5X|e&UuvC z0HtwU_7;%)u{>_lgjkKW-ky4kqCWLM2o3a{W;~;3S1qlgQ#9dr?D`IH9s-HuM_+zY zh(E>JBP~T=5>yU#s2b8;4ePuMO!l2~I$}c4a0Ocs%^^pMr0@i|8;`Gvph0x)IyY8G z15V_4f&Pq6Z`DRWEg?n`LWyB?{vN65XJ|7>aG5B-gVKpbA?F~1%mtZDF+XYEU-$n& z6YtpiZITJ(Wp?bD2|4M-kO)^EHJPT3lbr&xU>alQwBfU38QoNd09Lq1`9N2K`>cH4Xw8aIgds zDtE@ZKGxp*|10y;*%YfXaP?0o=tO3)0WI%vh zbu=D7i)@Vtq5fXwU{Wb7n$1@%^GtjEp&@O-X3nuk6)At?2)6% z=75z*;6{`6e+3Jh5kVB6UY`X1f=sheA$-z!Tq`*+m&bQ+S4+b7vdivm}315G?4I9puuhK$v6LUSm8Es|%s(Ow`EnQKHOYxs%2{ zq9yclO?abxnV3s*E&TMV{0yfQkQjM*Qe@6uQyxodA+$4^TTMRISoOmI@U4SsV4G9&->FrJPg{-Gg_sS!x?qE$ z>jNJJ%FIQ}oVvxJFlx)UQ}B$tL{lAPQei}wSieFE++DgzUHbj2^$dhVsWn!H!l8OQ?m1jQ1z6V z(_-Isd8Xo=&T2p2*dB3+h99Ig_q)dw5xG!kDS5bkO9%Wb=&%nYI_upnic{mOb^ zE-yZlGFZG^h`5i7`0w;=zA$C6#bJ)oSpEs%%z5lDS$qocY}7X{2Ntas&$<>^#L;*@tfuovxMZv&6DJ31vztkF$p@z z`lcC|2vz17#Kj&PI-hiYnzr)a6+-no2_d%cMm!^lDYkhtVj86R!%-W z+okxo2y@}+``9J?o2_ns=`BI;UbFq>X!6)Mg+AR8($D>Z|Y>Sh8>HuMfpp; zI%TTGa+`rzj1%?8cTppAjq~dU4^8y;HF_^JXUao|6 zjfD5c`QXpBQ!(VrG#La~4*&~z^rfEIersSuEp^p{B2fN7{`eAw{ebRe(epq&SSQCz zA0H$9|I-iDyf#7k=WY#=hkg##5W}<_-DwFJOO=Yk@Y7Om zMZ!9JX2t00Gw6gI_ee>HON0`-0^~NpbxPX{aVxVmS6r3&T$x*=JNd~|(d|A%I;w?X zV1ImU7YHgqOe_v84-}PqhR!X1BHABj-BiVkZzMMVESNu|I3+UUyHrrFoh_#TpDWWb ziYMpHopvQ1jft&JoRy!)7&dOwbMd~8C@JPGIROFc;)9S&&^L4C#?Fx5n)_MGABlj+An$ zFjBxGs^7C0lpC6!iM5PgJ5eBf?cw>X{5%hf))(|_jJ>EQTk0dS;S zTH^)#K&UimH7jwkD3>?w+u&eaB5bPe1MEi9dj9iY1a+zEnB?kku^9<8d+|JDe;+3) z8J#03WLfgOC_1+|Mk;Kt;uS$IUz@5m<6i%UlWDc010Fb&PNe4Z-fE zltkK%<}NBj1{s3WMN?vPy}~Jey*(1g>V2RjH$?4VS9X%1sd1LfE*I^F(PT6hD;)~J zOhT(Fdgl%z{X54;aNs`zU@j*km3ZG*AY8z7Cz7Rl4WVWMdM%vMV_1lD>FFP9H{)3W zE-XtGro5qO99E=Z35Zen>fJ>)EHpHR9VLXf`Z(Ib+N&QaIbV>Ofei{Fnxz__G@LsY zyEPPij8eBNt=)w<*MOhWXeAfx-{p)Mau~{4LSjuR$W$r_|9ff_5FGfI?p5lkvd z)yCUtlU_)=*o=;jSgdE44iAkcM|hTZG=!JB&qd33RK#JpAudNaYPjKoJmK3KAnCTyZw0^W)U~Be4-G@lN^UAhT9{ zpiW|QcY)pMP7>k~2ktt+-7UhXdagU(f6^BU_ zdmQf9J7;pNp=Xzl4Mro*6K%oN@_@NDO;nS9Mj&@5_ooekCmxRNXjWPP8mP(4Yovrld>G_MdOpf;>k;>v9( z*CquZ;~1nKn+c??IV9?gt3(t_~J1z=OG(_N9(xIhruVi-xGM~;A^HD__@ zT3yfnL|Ppsd(`;?ox^1IRr{%+sohtvwg>it+~!Dd7of?iC=GQ9%LQ=2+SpymT+{AG zbpSy=aYf9dWjwevg%R11+%dV*R8T^XObY((kt&M(s?#|II|$4i4a(W0WD6* zBHRe<(6FH%6F{}+A;iy=?`m@3*L{xWSFxx)iiJ~Ml=B|q5_p#0>m37<8P!i`O*^Cq z1ZI(8XQ(pocq|ZK>73OOV|sQjIt#!U?>HeIDaLynags=j54Y|zpyD)vyNFP_$kg%H z@5@vuyi)(5mvtZKR7B7omnB=j=vl+H|Un}jtGBx0lUc6j)!!8#t3{^ zPkXTqPbff!Km}Fc1Y}&Gdn}!&5+{i^W3}HLj$aALLRcfxgVw$KpIBf%31V*jt}s|U zf-z++t`=qHNR_|>?bY%fRFkGJCB&z$QA|r)g;GC*Y zh}lX2DKbx2ubf<>U@S$_bequ3Hr?rhxmSYVRPQ+EGh=Z|nSW&*w+8}q3gPQ)%Hljd z7k!=h0*k%|KrKgE<27Qbg~iW|kld^|ARtNMtawJxv=5en6S3SW5NxQ&4v&xN(M{D| z8I0>hsu>laa{aGExNazCZ4FS3YlY@+h*v&#c}-((w{E?;NNI9O(ZUVI%K}_=jGzgP zQ;AmuxlwR4N&%El1+kp)J*G!|c;edvoeZ6XG)OC)R^){OFS)mVM3BGF_N*NXMNgHH zs5%-cuMtMo*!0bf+ePH5pfQTM&sf^5-I4Q2vD}zV9uQ5j^Q9)T6Wi?@KzSHuQDLQf zY<)T?Eus6K$Kn=I{`?@3tdNNwoW}}`5t##JmIX^-+7e8tmIUzTGB-NUE|L*xB83NmF>}8Ss`7Qr zjCp#7W%kK^;2bD_37eq0?gMyRki~(WM?X@77tHPFI)~1gbEEe>o?Rxb z;Q@kMoGKy@A8ZQ;Vz~`VTqR1q^M(ixeS#btUxPb98 zD;6U<>C2kU}?l-+7Zqc)g%A%fCGCI7Umzd5z*tS>WDPdO75H8!KzSy}lO5Es=))BDp(D5Y55JfqHVM z??O@b>C>mh{ELbV6+BDV*#Ak8g@9cY8<@0^s(ugDm87u-I4EsL!pvVS+X!jE_I^+a z-3&6_V#XLQ5nk-4ub^VYeW+O3y zwDBZ-&agnL+u79-l^^l!+V~8n8H-AYF^+fey%l6W5NgR$PS(2|jM6!{)cT^FkL>1T z>~p3dOA{05D5JinNB)#Dvuxd8^oW$*Fs z5-+mbmzJ^fC(d5wA^eZ({A)qZOOX)M=DF)yFwdC7-Mw+CNVBCE<$%WTdVGHoVD4B9 zFkEq&6zRVd3?opAvht>hm$?qW`D64zgiprggk_oP!Ad$-zrOf_cYbQxrvGGXvH$i1-VmM^$9S8gj$TJU9NxI{n zDTZD!s2|Kaf@Ibe_n(NhhFvqXFuKgqC$K4bZXvXq=-tL*NLg4_rDo*T7>1W5{3q%c zIyr|qJ@K#zN~OCfP)KTCcq0Ekh~GhU-OBIg5Th2ubiZ_)dk7^9BIk9*10u|rzOe?{ zsy%K4po?~Obwz)Et`B0Rt7j)fA3V?(j|wnLuyJK^UyZNa?sw_-QPlaj7 zn=v=u*0Xwb|2U@4^UR|hX(;v@d}lZ|7ix(?4Udfy>)f;?IYmF@PJ{tY zU9EJEFy~5^EkTO~IDGQq1@<^lb&g)qON>)7VU^j`xwS4MaA z#G}zE$dyVLn_$kyvhjC9VIc9&8JxGc2TDTGF)b$bY*w`2g2%Zq4(-xQbzAVtuLxx(^jjlu^F550S#K#GO0DF(5B73JDU zv5t7h5x)hRYB?+q6pj(^*RusXGaXM8E|ae=66lg(hlO&TLFL2u6$xc~Nej`VrwR0T zIJQqmPKW;v$ebk!PM&jrU!=iaGM0!6`GQR z1iDygwdf~wiNl1^0P{L$#^?V~B|Q(2 zBscuH4{7;&pU^zJqI8U(il(9j?_pAb8Qp&bP*G;ii47j~kFsBZgSdEQ0IdNUV)%KX z0HvNWE1uUg{aOQPno!~+e*%F+(h$bxBKWdm0qO+kP|;IuHi8hTAukYU=HSJ^^)k%7 z*9mi5$lIzNca(|Pi`3Hr=&cVqcj@@$Lvhx_&=jnSS=3W{Mx;?XXbSo8BajFg>w4zd zK2cNSJbF|GO-W2-4KHi3{cXd?5L-l5sg5%#J&&zy+0RFWrV)Z;J&^f*O&9xiCWrjeZEX5+0ojj zuY~d)0`dQ-Iu9sIi{kA|kf0#3Kr+HENRr)|*+dW}=LIAR0{70`$uoDDJ7Mnv5=1}| zR0Jdk0VN2EC@MKi6p##mat;zCDe?O~{q(%=e&@`Y^VV}aR;TLf>gt;X`&&KHY{K7pVZrKnj;GC0Mi>gT_5| zP_UC6(GwHWARZ#j?2@RCp~SI5oC(u#y2Jg9Ae3gk{jt(Zd{P}t_VG=T?r>FICYvI< zC=SOa+Na+u(43YeND-wedP1mk=4HhO8|EiVge^33MNEE~FfK7Z`+O_x6k#qOHpR*A z_W4)mNoq_7jO)Ax;x=Ip$2NI00cMGvdNb4aH{vqEUIMWniJW*N^W>gpo@uk#CDzO?NwnG64_JgsEM5&%y>_@t6i-$F=&nT-T+Y# zK~2Lx^21CvYU#$yM{gG4@h8gb$??HkAkIzB!_r3JBViMK$IV@mw7>Q?Ahp2;vl6pJ zkV`M3A?wwvBArO3Q!i(L?fi~|Fs3JAE4BH%1=&}x#uXx6a84!kaMpb96=~#qV7SE; z=7MNn)UNZ3_;wjkM_s&SH4=LXb2D~ul&PbqPC?EAPtP5kN{+q{;%3x#xg*y90Kmn@ z=et|~G-344Nv$#ZVG$m8nRK$-1u&f3M=P=6|MT{4k<(&`p(S?FtLOdzZ$tcAhpMbfpxUfuy z*#m;8Ytp!&C?APmWkfToF0Au%O&QJ2jP|Jv)IQ3sis4x5Qxc;1EXC{-wh=`ub6pBQ z88I%>oiI<`h#&|I{}Qfs1XloG{fuwdgVm@<8es`ETM4G~ATw8Ddl9gBtSe5^r<(*P z1IQ;Y3drLT_a*6$L>UH64ztW+pBG^mG2J~F1?>0*kQPtJNnxuEEV;xcC z!%sY{*)bf`1yG>mKxUQ*b6NG~Y7>_Ygt_ZsAm8CzzUR#c=xi`6;Cwe;6hhsZa$>#t zi(e7VwPWuV0y|mZ+4$cg3U;cd97{eXlBm6v@%TjF6bE(<#-a;AmK1iE2GhxMXvg zO!hZWWQWtsNV>|ijUcK*_o{L=@oJG2Uzg0{ZV_e@uOzY0iHFMs29l8zFN~DXTzZjI z28(^U$R>qCO58_~M@eIF6zfLOPP&(BAy|$Hfy0$Ka#w!cLQooSQ)WiX!v1dMd&Nch zcJoDF?1UXNeUut^z@>j|{DTb~pE-%P+?9&>&nP~Nyw3X`@vaPAN(}VEuC1gU^;>~uF zP1j!>Tw?}v)ObP!#YVDBx9d0invTFt^eW=a)RZF$ix6a7>*wpgn5FnI+a3tX0xs-oimRvO~I@8Y_w*6;6l| zkQslQ3vg+xwn^HmL|CP&tSc7@QGcxu!iDf?VC2;;(-l)ZJFo8QsM0 z{|;RN+^p!5Z%56MKNRLx;>0Umc{@Rni=fFqz7XvqbE>JMjr&(DA~m}4G={7RVS59PVOEPJTcqKghE2Xy8`pz#G+XYS$H)d zsy2#vT+z3ggZpflT@&DP$$A{ZB6An&j>hUxnVZ^J(tWQNWENp< zoVKXQ(*F?Z;xX&UvPAXFA!1^u;igyX4Wpqf3)nYs%npFF=i{4@~MCKq@x2#~{cw=pc8sf~m_D#NBdQ7glE(j0TUB@x!%5Pbl2r~%NNi66SiM1!tNZi5E+V=P26cNa`IyRH6&bbbOqNezK0G*5#KxIJxtf zp0QFNSXy}Q7O-aYP9aYfx;avd)mt1LrTxsfS%eFLVc48lrLCaZwbf%Y(QX8thz<8tb>pQg zb`)+66NeIG>id@L(+?gM9G_uy<)g<0yC&9q4R&BaolsYKkS5n*V1jZ#)`dG##gJXO zIUKWNl0@4}XbAT%tXT0wL2e}}mn5FX(Zb9UroiseI7fifXRDC)Z~s>&R_T|e7%vo; z5FQhxVCQ)#qPinhYxp=Bj-980&=T!3-Tz)Dc!|3wP7*=0w9T3v@8=(|XKTh1)A_*+ zAF7P?$9bZiDEkuWtge^=;>x1pk@7A+669J?FZMiF%q+qnr=ig9h))EO<1mIP^;yNw zjoL^u-n~SSnL|S+!>hQw42CfeFQX5~A4&*6Hq0`$xJ8gT#w5zDI>LtRE@7UA&h69W}ro~wK zyO3_S-sFO4Z2>NojN!TwYNnKs+>vk%Ikpf)O|e43QP)6xCYn?`Fb}5Ut85OL`KS3= zv;PLdC{j({aN+<_L>TVo@OU8ZDJk+zIiVJ6Qd6$4zK+R(bB{dA(G>2HDsQ&&3(>g%NCE6mZYV?aV(b_*w<$uj%a`a+oww)(BTh)4OxLpxUOdn2;YMw4ZIrGARj}6{S0w>0c zjsoKiAzH!o_*?vMp5Hl6p=hgaU&N@eahI~DMA#&1|F`H4gmELFaO0NSe+8)PC)2}I zcPxHRi$EN_6M(tfjgD11wFPztqLJo~;-X`=2zL!ij#h<(#cgMUxnrrr=Tp?}gyxi( z--hV`^FX+o8`PA$r$Xh9id(m2nR}=(ml<^?$7LUs0OJyE;EKBt!n|TaQ?GbJh?_&= zsM@J7o)+lotbtr{@UA8+df}-%ev?U@=ILV9BH!cJ%qh-^=EhS3TmfGM8IE_#FgT$@ z!dYi`!oVB0RrNSSgzJJ%PCl}maiuUP!chaW!<=~`{4OTL#J?+)=DlgtW0yVnb~*Go zPd#=OKpt}X9mk7MNG|iQv!_GmTSDxPGr%N(9*$F_Hv|y6bxK<-x0gfnBXbm^<1Yf- zY;ut?XzzcY2qib9+QC?AZ-|Md4gu#|qWQ7fL<%`rgbPZ4^X8Um&fMdv> zYP?ti)P?d5?Ng+W>#|xrD%w*wFQMY^R$g^0vD){6-Q38B@&vPi5DFr#Lmad(gxlU8 zj5WuO37lvQT{>lUI!PEJPdm0XvKYi{H3>a&vVLGtp!3AM`K&Qnu=If)E!<2c~t>N{}0{+QU^Rc3sR07b~Dq9pzAbDYUN73DWvJ zi6Xh`jWR(FRM^-K#|;M(m0C)w!Y&ch{EC6}$g&bYBm>0}%$ij~h;!Ox$4#Q#Us$I` z;ttW~NPj&&+xV^!HC#LIpDxx8k&CLsFkG*a=}tmXQ}vHWL!X+gFwAMj?Hy3&IcMEG z_R7xZO=0DMEb=B!AH=UO^n>|@g$*YGxF@ZZ5FBT`0ad7Nr3xkV<6NepTJ#G+@ntVp|_44iFvO!L! zZEUQ_?H!tuam6Q-HcU=W&^Ni)^uqlY zBAkX!=hQ{WZppqPJ6mc=6j)6l@#e}G(~cq>4=;K+_DUjKzfky98N`gB8C&%`1`g#1 zVh<6H!M3u!4?7^X<9$M1F-ft=TkMVR6-c|1D3)sH4??+@P(C>aj$;ZPxf^1TS$?Lqg|CS!&7xw_p zpKCgliB*NstNVCWwWob<98iYg*%$5<)3?qLHZk^je-+PGCDM`H(Z%8i5r#JELr3#Z zeS2x*$TYScb{6z2rirn*py8iq`FW#%iS)Q?uT@9m!Vzd%iu*mh!Y~Svi(K#P!``|c z`wBBB@G8K0N4ku$pHOoGC#GD!D%csYS?VO*#vNJX$;^)FKIj;oC}x-uf#G& zd%_^M-A10=&Bq^61>HzO~tV4rh1vE$|d>?*{Obj?-T;P1~9hy$#lM*KjCE8Wh) z_)wfQPY|7>4aV9>7AZ?Z$u{jDLUNO!w{OOa0?Y=aA|@u9eW4!}fzs{hSXhADZElaQ zEy~zs$r42J&QF5wE|BIQW&N(}8_sbW-i%F;f`BWXcz@(Brzq!IS956R^@ksWxza3^ z_^ja7aY4>EnQJo*^clpCinaCauo&Iw!q=b>GSLM{olj#lu+ zeo43qX$Ug`Sv+weq%T&`4^)<;BP@YM+W)55f5j1 z_(c%y>frzvHzycBjR_+o)Qt6bSA_eG^r<5;`&a(8Pp+g+I0+&*Uw68~cC`>1cO#5; zTTgaSTtsk5Q3NH{{H&`i*NM&v)AvW>DFH4@r=FkI^Dq0J0_OA)5e5+;J4%G1S3~qT zu|$xtBvAfJh?5>1mHzZP0dxY5C=_#_{2G!@pqDXtH~ds6vx+@Xy2z9C)B6ZFi-*w5 zMP0O0ttX4UHBJX`fi>YLkF0l;SQ3>m_K(%iAe<*1lqgMk*ScGfy9n3Ty5;_W9(+%U zA|WNgAF(i@6rO8?oY}^M3S^dyCHEHbj1XAfisSaz^zEsOb7(4KS#iNLfnD|yZW(vP zx&mA#?5&a4hU0EQj#|}y15Tf1$?~aS*Aiz2Y|2p&FLM@{S%Z^~{#Z`5Q%jwd+eYUK za*jOUG1QDp1;F%lbvpiB{#+#+eBR}sr>@_|%D2YZ&HxDqok*>qkx8-D=b?K2PU`8t zsm>lYI0wL`M4QQILY&yP#4v~BXo!Vkm-C8n(lFD@`eKg~OK*)xVREuCxICc@X^;1X zXjI5^$;rQg@L+BPahw8}ljuuTDb_m#emdc)`pNugj$+s_7?%s7fwB74!SS1!#M!O3P#a%Xghv=< zd5P_Xdkt&>we- zbXV-J^mWHeqMZ{Hgs(V1r)I;jS$UQq;T;0`1<674E;uU1!j4;SJV zuBIo~&lBLBdMhwq4z=GXqv7FHcPUo;Q<>0cKbwu%RtOVDFYVeM$BJ@Ubo+!Yb1ZeG z18~%Xfb)VVm!xJ7iL?F;0^?dR;`^??`4uL!Xh_bvPYC?!!B;JbFBPV0CyJNc-Nb9jQHyXI!`<&v)POrg^;S=d1BZ+C6XeH^LXdi|Hij#;9Hpp zwt}2!-(Y=?wRgJ+bfO$1ps$X-g_wimn9la8?;KG^VGeC zK%B`3ZL3^efXU?_Z)Nw#K?2=DkXY(*a~Vjc@641;K%bW&f!e!M51f4sF*P&ln(*Ty zU_cG)0|xOX*Fu_)tVEct?k9j&<@yTK!|S5V0D5$PGfCtNT?c4JbvL^(5voOk9FET@ zXT7T{ekj^i(W3s)vM3!f5u-Ui_;kzb@7IYS6ZwajRZSM}{? zN~#AY9k{u`?Vx5Cw{UfxcO*6}G4@U98VR?VF!Q7jBegicq{q4T9nXq(BQhs3>c8?2 z04g?I(?f9Sux&d*ba0&$v##`uEZB^uP_&EUn_^U8-8;a{GYZW?g_hm*1UfJ5beOjLuNV^Z#taZX$>(A`2-`r!7snsDcbt3!^qn8j~USYs7LXWe&-n}p@A zQ0cC4S{hFaq8iNfc!ZCZe?r1B3TQSU1y(JQM>0T^OxjTEPBVLJirkv-JE|5M!}|PN@>h+z;gj z#rZu?zLhjym^$O&2l(A$GFAaey3kUv$AiEW809Dm`xq7JoOu~ej)CG!4;N5&IvN`Y za+7mK3TY#@7eX5&^h}Et9)WOC2RlE&{e1v3An6NzU^dTJ}XjJyA@NJQHtmX8CU2AP;%sJf16@KGbw z4AjXHp8Gjf9W?AR$c}*N@XUcK5AEiVFD{lH;8x2q;DD(|5m9G1LMZ*t%v$kj#H!tkOQzIYjI zVhTmU8k1fDU|=A&%kjdHNdJzylK01t%)hg!OMtpqBvsm|b!yx*@r^696JG_V;+zO@ z=Kp{w=PZeiX^KmNuYtMJ_?kcykS3CZy6u=PhOnGWy*$C@1-a^;^!fV;(O(E!rk6BfG8@v#DO?20|pAZr+V!;NK<4jM70Z??-GM)KKet1QwE7?IzkjWo{ zxSMjEfr9)A0TX;cpWhMZe^dmK;=Mm^7GO5WG9LelB((W|i%_zsL}w|0gfJ33Oc8py85fUcqN$_C7eu5{ zUY8~3H$QPeO!J#@oCv!AIDLZyK2fe2GrtbJW5rJ$nXWQ8jF0@!9Jy^H=IEBsA>0tW z_*|*R;Q~mF&WX~n@kQ}7Dh6hT(*@8`W?)*%oL_D zlmJLZyL@Tq38v`knyd^MPYHE$bW)yV`5DZeaXjc{lah9CN=uM_r z|I0=4sz}qO(zQj@~E2J2&*c=n!iT z=|XZWz&6u}dCG}Dw&cq#L@u;za`io1L|#17qowIS%PvBRGr;}WGOHJs>amv)l209+j&Pu8 zbA`2h90yDS+#m>b%;pk#o)%~Zb0U%s)ITpHO4r^W8+^qnr-_##|DI@fF(xPc%#zzT zz6$2@v`m>IA_Vys4$NmCfIvW!lz8#Z)mN9gX6b*kDplPQ*)EU2Dyhsqw)0N~(@)2RK ziChZM<^LzdB~Ozv3vqu;`x>CD$ILSpeWG2CdKWwE__+Ww!k}{??q34JtH-fmyY0Ojap?rz2x8DNU_ zHl%BrB8f(24(f8z&Xt~qIuS|wxMjgSM)2S@;dbUxp^`-qkWLSO_cDpTr*Fz=_x$m> zNVv&HVrrbSoWsq;3F&A0&fTg$$Z9i8ffMNLb$FW8&tH@Qb;2o8xE6;k?_}9n^u#$L z+=Lz2Yc}H&0l5}vY0|dm*sp_8-WIgD(^lZyT@2kvZ3g%wKG_0QDt1Zt6*XT(Pg?%B)5!;*Ta3Z!^1<1K`q{*f57+VcQ z9GTV<_vzbhj3al{0QesmiNV#&IFjGF1pEyUcUJiaOUel8y4L`9elpHd*?R@KK9rKX zX<&1NxeT~tl*Z{KA@0KRm=q6|389EfuM<8Y$VJOeo|M$vWkC7~Be_2oT9be-8&}*L zynmT2z5glLWkcHNV*ZSG1i8b|49q=klK@-{{7Uu3Hzz~8Y0G0vOYG~c#ZRt-OdW9r zdA5+;*z%c{jHs?!8_-$spd~xSe+bCy6kTL^NW#=`Ec?v@W)q)!o`li_=R%{Wkz~EV zI)uy_qg7#HT0@8f4IwXxExQYHF4zcT_~ka&a(D73FrSRAoiL zH1R)SPIv&X71EG?XFVWKgyWda{841C9t#Q95bFbwDB3EHtq$4%D&;qQM%*pJ4Jcm# zIP{YglTdT5zn6P{GIKQO!mtU%1dg3r3g{|fMS>Gx+Lkijr&JAVy6Z>u5CfL}7O=Ub zXCW~JJwk{pLyeP}{VxT%^i>{t^u|1B}UMb(+n1=&1V66HsYCfP1m|>IS}K*-CKq@#W+T!<8peO zMX#Rx-frvM^cNZd1W?K?inKO^(jW95yCpCzd>aZD3Je@jObahE7fq_d{QEBba_ zBp|XS6+mHZRvDj(ba#}q(!8{uzA>oVOYU7z-Ng4dDH5t;bIjAr0^B>%R3V2kDqr6e zjGU*nOpU9*UHr~PQCo(6Tw-%;s3xS%O;~PxX|*U=Ef(C2$Od~^#DLR{6hQ1(yT)Q= z{q8PU>tLw-Jk#tlHR;5kMZ4NqKyZ2Q9RVI7n02Myum;F7-vM^@Sr`)gK&J?G>P>vF z##GU+s*b05W)QXCcfnve4ry`l_-9dhSaRsto_B03Zvjj(^tg`H=Fu&{;h10OZ`R|k zz9n$xCG$(V_nENV25d3th_OQ%jEeT4cIc143(M@Ip?nYK8iDRWJPX6R6nBK#4o zYlOIupdB2ImqoiL(XnKxq$%O`GSploub8qup(gBz3O83|Fjqgg8Iz3GBOiJR^n_Bp z$E@`*;m(VF8pjgp+TT1$pbDXL$z)<45CfmVF6>;o;$US)*)^@|%x>8oip=SeH4bZ# z0e>&hvsQbpJ#NV)+az5mUVcY@)%4Ac+1dJbXV7C~>ijVx%elRpi!&3%-l5W7C^FK zFs9R^O_hSnD1#aqRmGMi#x@DtCbo0i33F^+h~+^=nZoWa)GT5Hg9;6$bP2KACGGxU zf~W^h&MNWK@^|GXne@*kB>Qd5y5j~xx%2iUhxS9e7Fpq4P?kn`(!EJNe6}`P1zqq+4$w+p$??Icv!|7pqvEn*`6rB^rsj=Jc5X_%G zhPhP~b)UF<&jp|P_H+Op&T()nDr>;gO|%Bb#!{H3A120~-U~?k#jhy1M~+^`!nq5> z5f#{)!d*)?)~e-tLh|H-G6(M~0^AH}aXFzJj&=7c!k|@QB7R1IGu6#VoT+O+`GdVd zQ~hSlh;|XV2V>Mq%RIUd2sQ5In1X)#eFsJw9*fB$T!=CDU_CKK0EIw-8aiAa_^P{!GO)Dv^lc#%?SVokZY*QTyP&KfN7@PFLh&?-A0EE529~=$<3iS} z-1X@!l82T2#dShR9<8tk_4^q>HKEC|9{Y3^iC5{l9Ef7rCCo#B=Wv(R_a*%#- zuGsppnU*7*c|h*VF^F)IX3)cQ)`s}DnWH2p_Rq-%$97q#2{%VNx$28R_LdNOUq4Q2 zKhd|j!>*Md&+9INGZ(T0iuHSpVC%-zh;}i~>~#VyjE#%*?F7I0Cv9~ zuE`5al7td!4sy*Jt(@kH^%OJ@oe9q{ZNl8^^fV|=eq!S?MMkSU!)eQ~qfiFBERE4U z)}b_ZdAaWzeN)%|9yIhf4?#F%NrSfddQhO(!b!L~{LVp;ZVV1Obq=vzi44i-Mn`;K z$V6w9-j@Y@Oqh8jbFO+EDj;Xp#9kR&sDZdlpoX+=R9#y}>10pSANLINtJ{L}jz)YS z+8suwIJoz=RoBPDoiv*0I(I`TR07Q`HqNY9{jqF=aIht`FXO#e z-)^K%Zu_y~YJDCSYCh<$Jwvb|#JRF?Nbo1u1UgqKbtSB9J>qkH+gFa(-zmhT`jB zAeu2XYrGXfjH ze`zYcShx#k0}&JzWl%YsFOJ?nDwE}a4;dAacBR9J?#_j;bu>eh1UNlT!scRV9yLKZ zYi^`6DN!Rt%1edAqcM5g)Hj`sRW-eQz1`tOGOXb!%sQf1ko)0)c4Uhl0qyyYIaq>F zQVlO9m|!^SOTJ+*$h402D;R*kA=*8MiIx0f*B^nn4qYfd2jfWrn!ILBjjtWWw_6vM z@rsh>)R7XjK0=HBKLzGw@YKLfydML&z_Lq-ABc9N_oD7OMx;Az=?)++Jlc_1XV~yI z;tA0%uHRdXE01xAv=3*7={OeJCG<{R8tB&qI+~_g8fL?vfVg-p1NcQ=^ruCT>Tr6E zWV(>tQIj$fx1%M*_~FiDJ$4r4a-$od+Z_9$#X$|bSS=eTmdvk8>5|3F9Eh1 z>@?>886jYEFUP4nT51$`OMWfXjtwY__$=)<5Ul%wy{0=u(F|YyF}CWy8Y43Be@QG?OpZt5 zIMH;U!~~2aL^20W)A95zC;B(!BkCE?iF6s#9L!p>%Sk{k0}o-T{ZA3%K)M>CLM(W) zlR+x$8|{moM7mA!`b&?VEr23*k2Z7^^jKjkzC4+*&x=8mpppqGVe8sVL*SMzu4)LHSqzRgz7a;xci(Aj}M~$zzS4vFlkyR%tEP!?uf@ z4d(I;SL)oOxk>}F91>Zk{V$+E)i)ipwDAne?kB)%=^Ce zcuj<(V+$8c*L2aNL&*g)mrIykmd%K!Q;BNf<3N4*97+* zv?TG)7v_==(G-$aZW82zjbR?$A3I)D#KRSQ9~bC4Vq6%x%XS2R8W%&kv<-}~(2CuW z5jhG-?~cW@LR=&{SYXe?1hDWWfSCzc)nm>mz~$+a&l$QozJE&$)Pa2Z^WU$cN7Gxfw)72IXrbb2Uj0vn#M^- zj4NHn?=CFXaSV=cWPtR$>m68_%oIQ?FrSTfcSgTR4^xcD*+iwq{8*u`B1}NQuf$IT zW!|tPSb6+AfmEkUs(^b$n8S=i`f@x{CPPOa6F1)%3M&JC_5v-=eU z(RMmB-~l92PI;)&WKoLEg)si`9};`&+jUNFipJ3bsA89Pk~n-837eA>dHlCy`r}JM zm*@xQtMe$PnfS9H_)BrqqbXmAcCFc5utaF9-S~q8vG>-q74fiW^AI&>dax16bl*P~ zQMF6bwVlDJ3xr{8t8g_T!Uad2LGQRyfD;+ zfBU2`wS{aozx5{wSb$nctM|V$iJ7C`1A0j0#6-${1szc_`q#M<*p<}*3}zZA^y>mS zZ&|M;PSvLi!i=uHVf?F%!fT_L@~d>umEl-+SlowWn?DoP1(woK&ydCUgt_{703zOx z1W;%xk9i8=((-pk6Ye?h65#yxz_=E#@B9Ubeqy`CQBkf}x@^GSR#Ngqf}K%KRvxkL zRgNW3r(>~+sN8S8*6k&PjJi0mzzGCK7jcRpctgcid_Ny@xj-k3I5#{L2_T|8w##ap z%xmn#g-&w-dzSsKgK%O=CrmFtSrEU=Z+YA-0&Zac#vPtX*F(81gNS^U zI9PytAlnxiT`-GoaYF%u$m9*SCeTeifbfI)9H&<(+RLzD>46cayTuxrJ}nM{k&7@!Szt3Uj~K)Q&{B(5)cmw?wjG9!~;rTlS2$)&36M+e>?ZwjEeOkF$-9sd<+e#n`Qrn>uM zP3b-{lXa|qH<%|~{EXvBf%OG*4WYC=!UJ;8H|{BNlM6gL`jP(x$$deGi44kf1(|u= z45P75xevf)?qlN>qxVBQcaA7g1c5F6K#>kRoOEAi)e>SFW^DJz3_(u1(qBcfd5!>g ze;J@n+ywYn8Loen#|h)%66nB0w5)CWJr*D2 z9|w8Lf%z=ndeA>s^|0gt4?(y-a_5I*-exTFFqrv*8?E#}_3?s8kduH~eEAUw4Tjdq zF`{o*B5gi+s7rJD5|0*1pk82!?TB3k<&H~EmFO1WtmXJG*?b%-%vIweh+aKCqC_g( z(?V|lX_*kly-aQw2yhS8B?PqTbn&Gg19T;N>$nJ`|E>FYkzs>}JF%qdh`WVpFk;77 zdxCHJOGlmSCh>(R*NiDr+OKpr^pz(ap>@ivSXtjhK-bRxdp}VV<4m(gHTp}4$wgf( zZp`spFd^Fc|D4t|p8Gsi-=sCH7kA>Hq8NjnY?_$r-WTZ_R{H14evFhp73cyMiWZLB z66|7dnKL~LnjpGIIwpoUpO?w$_<>~(@C$*Chm8l@;^A2EsWMx=Z(Eg#?!p2cuDdqG zj6E1jln~N>8UQy6%JTu$W@vpR#LbP1B*b|}*CS7Zd313@4u=%a6lv3I=cf2+54|7h&c#BNki2C0{R6V=Q6Y$<0nd6uvT-edKi_==|8h^3(U<@K5M0`r^kT za(BzF2A6yjh>CGqr`r>;n`pA*Pze&o{xAn%f7D&+#`ak>{R6s-_sjkx(iLH)8gAkO zoI?NmEl{^7F8b%uyrv&qo8J1|T4%hakREvGX?PqD*D?&Cw*{LM1C?Ga1nu!3VJ`gu z*QiDt?eX6-m@Y%5r$iQiyNHTWla`Q53<=94i1AJdf0jda;Iv&AnWd^aTbMH#uZ$!m z{BO&M@`lYB!j10~Nug@c;{2EpxZkCF1pj&01oJ>WrvC?m39%h_pkw+b6dv(wvBG;f ziZ&ygh%gHqxIkn6(c#BuLd~f@67Gwo-v^+j^u`_&VEkCL18`)+)rflqI2}3ZNirKg z!i2g%ewR7fh{y33tdAa?7-9Im5aEl0`>|OzsdivBE@qxZ` zTQmo`jGmT^$^Qd&CJk9s$8sM7xDwNN}D9+A$zQ31H03 zA7K)}Ii#Vyd{4Z+N5~IzF{tifrN&IW*nH%|+{om0Po@Aw=&v{B zt$WdhiEJjK$K()J`RmY;9TSh|6IcBe9UN_$oK&eMen-vR-nBX?cIFH!J8NzZkaehz6 zHwvMfXjvT{?vF`J79p63wN1WFi1SIdeH7--rGQ*Qj_I|*C)~nI1Clux9qTbgggcOo zBeR7EySlF|!w()I+FLLkF}`>0Aqx@D@I6hJV&WqNCzrEgb>;n1!NEGVc1I!m4(r#|b;tDO9l96Grt|gf*5LRcX{utquY);HY=*EHRzokp zLJ^R8Mg|;IM3~Et)f6cqzBDyEv5>|>=@xjD^LCaxX@g}PAklN zE6)S9r+w-7R{^4xxn)l3BXN{yxj~*2E3C?=Q>007>bsTzidB<4zaNO89{HXIv+P={ zfjGlntkBfL>j-k$*`FX-=n!phf#!hr7VI_l6=E3yeGF0zBAn#efik)zEEY*Pf1yBU zgY6c_aD#EPAUaYT4>&CS4ZfWvvviXCy5h7F#Hg3O0;jTZuRyBKz9jhpS!fMts)xOH zPmGFiA=$u-<-c9CNVvU!v});OgGqoAvXgPk2cq1`&~)PCh*f>&WU$N?-Gk*cKS7x* zT=-$*z;H{T<6_du#*p1X!aP_xzd{T{3X$2Zl5|kGFSn4DI7XPui<4D3?7N}Nn0K+@ zYe(EB$Qfhrh(1j9>Rua=YIh9t>fKMjX@boS8cGpr;t8Xh_>B;%+m@WYZoUqbtJaz3 zQH;+85nyP9*7Zgfz7Df(k4L9TVz=v+2nv8xd0f%3~Y%Xbq-_vpq0^7-ZOHt?qVT_VgKofU&E z^W`Pbm!1N6rUbai#kTf;0u=C#Fvt{a(>7Jq@O1B^nz5 zQTpWPT)L__>RUx1Zr-IW)OR-of}^NoF}2)5l#9ws&an6%0px>(qZijd4O42aSV$Eq+lij+U{K*6a@8IXH7%qOfS2Mcqh5CQ3( z!*Qq}7YNtOmEL;XB?L~$91=M-UJy-(V@1RZXbXfZgn28&dtU)rgp=VweK4K8{YkKM zl9w2*4quc}c>}YPIlh%JwDYu<$?>7S;eyp7aY`E`d2+jhCm5zdxM{f4H5hM-mT*&J z*%^%arkj*$G)jiuMY=-F15#)nk`XDZQCEWeFh^ROiP$sUwPNw@~d_v-qYeC|dQ{ z!hc1YnTS@@Y{>cty7Dw4>gscrF+NQdz@i>Wl8*aVQQX3m%r zb2jChmaFvhetDBKuahz`h&^ah53TWShagM$5Zmo}|PID@*P3`dW^ zxK_1FpqIH(uDQ0eO-*KK3HM}TCkkm^zOqb}8$xMH|GOY}T&^z+#?_)7v0C$i9OL2| zq2^2#UsP=7uPc+{=vUn!t{0S>m#rq(R*+3@6bMJCZIyQhUKh!rfr%7;%OceVF4gJ3 zD{T#8p6baj&cqsVvoPn4bc3-#th5b?%Z&255}S(7A#j^TdABP=3{<$zx4&qo+eHXm zvG>PW!f0Due#QRV@@+O^_9;E#;DF@OA>p}4jOYSgTqVSL(+Yhtc{^y=ZwSv*dy8~N z43{RY&O?bM=JOM7G95#$o$5Qg``${=!YkGL0$&L)U+;($p~!(&2nzraIB4e_@l-V^A4*4JcY z%thZYXw-#5Uh9@9yTC3*dOXg6BSh{3ftbgfHQ99RRAEeC+VjkfS424(?i|R7JMP{U z$cq9u6&SuB3xHLlb4TOK-S~!8oSPx@e-8pKaD*f0mD6`GB6i}UzZTmFFb}vYfF(gm zeq@~71<{qW1&dhCVejmuZU=Jtq{H+icXtc?0BVBCXm)jgaQGDD5um0-HXh%E@tvIV+a4SWfua_!s>mPd^=qsw#@ zG*{TC#5FbO%o4doOeQDGcLFj-Td4gC`ZlX|`I|PrUWoaDI}@g~*UKQ1IvHRe+8`{O^dd~=!7aj3SVJ;r8*3j^b*B9;V=%+!2y%+JmFwmyUvDxrS#0?cKoiPDWFbB3}*^=EqT&`*U#hH5|Phd zmfi(=in1`5vsc&CQA-OnGrAD3`(t$hW;oZkDzU$43d`xd_Czwi|LSZP*%d(4*Uq9dWQ=Cxn|Bgu%E#h}LC(U=!D`Qh^Segen3G_#kv>go4rZK2+$P*v zle6xX&kI#+&2+!;PKC;YjcXP0Kp6}3u3oYA89{EMs;};?Q3o+ohI-gY$I?Ru!0`=N zZ+c2#pw`pd6Hf|YRzrD-BEIz?hsR}-U%TB>fa@!x*F&ASeE6A-gjy!_cpY%3I31;?~f%n7n^zGgFr)^frpnyQf6 zV28%UHUgXr`%R`ShTt!9AO#IV%L@<^cI(8xyy^DV6AP_j2> zid|k=+*r8lfe6YRMfC3qa|0tkup`k0>zxI<@Y*JDsq7XZ6caZfsq?IO2sD*Jmo;4H z_BwUJPP9!Zp*R9UQC>6cbR>Q<+is%}8PRdva6BNy9fu+@d2D$ofJ>S5hw7`p5av>H z+KGHCaK#@2lAEmKw65$b+9{!gM~pj1fHOn%lMwcBjzX*9o_{Fb6m8~l$e7mIBt&XSZbR%JIWvqF0qg6YPJ&!0*WCpy8 zBpaB7k`<>>EitYj6YW@)<~fclV=+&$dBujKBhD*Py;{qOrA}kkIsM9nmV%T()_FCgY6-rb6=Y$G8afPTu)FP2XmB z6;Cw7ah3qH8xdIQl52%HOU#Viszj>EbUBv$F=5;oGCE2-=5>X6aB2G#w~40s?d|RHiN2kt{GZbDUp?BPCrs#f z6Ak~c@{oq;L6NR6j?i(|`I-P%1)DKW#l)`Tj{$W1%VSt&DA`0L)QsSIPI@2s1O+ri zYwIPixK)sgLT>JMCpiSMK}ZwSn9)7jf}koPGpDLfl<{Z zC|4J6Y^W2CEMsEc#;GX^l(@Bw$&nI^Vtd?^W9GX!G}l5uaRG3k*dL3FATMNIYLS#k z9X|!ioWhc&Cx%Oag8~c!-YNlj$)@fT0Td525^ZCY$#Eb|^l0P8aTO}ct3V$+^Sgcq z2@CquG5?ttSXH&I$3KLcArdGwjP4TTr6)bh3Ey-n{oXQuYCko7 ze=K|gfn6gcaDdGPOvsnaW0?1*jP@5!50kgOrGCkGDkSd(|5Zfh36te;+$6vS&09CQ zT)0)J1{mf8|EF*FUWCx0IOxP837jTVtA_-T0YkYfe)%g1rzgiKGI^l`&3{sX%D7N^ zc{XPY2zA|c?IuY-B=Kdz=2Anw4XAG(DDO1>Dk%3jnvPYHx9T1c=;~GaRJM0d2B2*C zsThuDe+?lyrVAsZSpHNfmz2|c^vj8J-9iZ?FPl8mBnm#X%xaC*PUD-T&=_&WO9UN- zTMaCLa%Z-ikcm~T!PB6O!tb75VA{az%xPXs{#Vl_p)D0I*SH7kb0ok7xq8e+TxFzy zv27U+AvgKi_(_=s$|G^B62B2fsi#kw8t?1d9HGacd06m_A}`i8ytfU;qJkWz%CR*^ zy(NH=7i|2hfV>#u7A{$kE-B0<)UkwR!Cwe;XPGNIy8o1Phlc5@XYxBmlgIbJ=^Iuf z9*xY6+eA4g4?E(}LNtZafM#{UDj`R0;nnlc^zHKG(?Q(up5^3NRM(sp7mD&kqsKmJ z{YB3PF_YPyr)upgh~Yd?o1=U0Tb=`HJ`5lg@IF8ZaLI$=v&^}UKpxfZq1Z_z5p?e$ z+36iAfIh;-@Ui&Hc@Pu<-v#Irq{f{g#2KJu;<%{)0stzVPUn~nR=&_tNS~!p^TQ@W zGMCwT`Q5>51(Hple2rcw!u=R033Q&c>AUT>psp?Vu<89c!xLntq2wReEE$i?gW;D) z$IsFH9pPvro?3}Z^zFpCF)d|}d`Vp=)QRib2*PN*SO(N)DW6GfaS;LC3XFO#)x`IO z(C7>XP8n9YxcHsLQrESAF2osD)8-TlGESz1sz=Y59JgHJ6r?92Ur{a!jhKYOl`eJQ zHau}G{d>OQRI+>AUj*D9$K*+~%Hg8T5abSSpm1`z@?{0aMBHDO1$MVEnl0^r5O?EB z0Tic=UVMYTT>uu)EE%GV)=1&Vt?feGxtqL685fJ5n73(XPB4-f{uklyAaiT@ zs*Gm@z%HrdhvP?ofFj?%N^d;;N4|3d>w-CQ?k}$ZQf1K+#iJt3NCY+v%#wcQdsh~* zYWPypgfI`pp#s~ViXm4(7Ngv8+e7hO0q&>Ijl)D!F8UDWSS$S*igBbH*JvHr6zN)U zI#6ev2*^E;>7!G6woYM`Uzc`ST(!hE1UcJ243wBaA1;w@`6}mn$X^KKg76F{9^yos z4RXVyW?knhAh^`pIx|kzw}aE?2V%p&7Qtl)ErIGMg5VlkNY2MZc(~CdO=;+wG5R+^ zuTz{F6o*tV2sVdNL!lH;=8wnyy+GLs(j$?wF%A~$B=C!X?=w57`+snr}qG_dMB+!g= zu7#W^emW`F?&YTIK(XRDs573@CvBV*vhji_Qe!?&^}OkNXx9j<4qg>NRYv?bZYW@w zlDrqGO%j|ZY;GwcUt!#+pcbYE2YTX1H$u8bgEedn;xYlQQKQ0q*%M3M1ma<%>7uyZ z`6uCSWqD(Y({6!Q517>!m+CvWC=vli$|wwPxfRe!OIJ-V_~vZ|(!kd=UEz-5HZy@|?*70mqa-v&LXd-yb)Y5Er(RhJLv_iQ&d( zG14xWp%UK`N;=c0w!{f{@ogSXosR3|3q?~Be1m9Xt46#pL#$46rl*x);kyY%AL&RF z><*$_dYnC^1BG1$IawTq*5e_1~rahh$?P82_JJ2fduRj22R^!%J!jfd&4TVzgPF$7Tpl`-lZ&y67Pxy!O zhEvCx|AO+EQUf7_rtA363wAcRiICv{-QvX(%y%YW!Ak;N-HysAra*)7h9GzK_PXAS ziVuY_?sOPSs~++oq+1BvTFz8T8XJe<`07La?%dfb=YzOu0?8=PEfP*I6XbMIkb`FjEJX5 zW3)$)5KkF;LAMe|32+%OdB@mBwEK?rPv~$z73TI4lat?vM$CE~)U{-^%aH)nL;@Y1 z`KH3-MstnjJh@EL9UCd81JO4JlF|Pr!eH?@)^C_yb&7PeG!V};yIdp4r9coH!XFds zQLOv~s4I;@U!C>s8X=CFRzT#lTgpI>Yn1lyb>}Y=q!lg@RfHqk>Tc}!8-i)smp1(JInuF&D^Oo83>Fz#OeU zh!1gGj|+0657p46#Q6f;P-xy|y>!HLAaJ;kn-J}BwW!1@jvc=9JcKKUf{t@0?uO9V zHNj3)cJQ?r6_C3La;I8$(HEUgJ9h=+ED`SAIF#;=`Cfun6R?rvL4fT=QYI$&POh3E zKD{j@vrqH2=CF5iBqs5}&h}XK!%cQ@0ujz;WL2J1q{ z&sun(5Necn^TTn!05>+Gl%5ArrT!;$q8_}?_s%QOE+#dx5LbT{j4spKIwOvr_-?_t zNZ*djZn_>nd<`15S7@ewt~wRJ4x!$Rk3?q{qk`w|&FcVe3i+)=sglkc?iQT;fu1)? z=5Y6wpk7fONjs5s-Y8NWulHjsaj5`Wy1R$dw?Acq1I2pWl8IR}Sf-yBk#nHjx*E#6 z;co(FMx)Eai&;V(46ltkzoQ@TtYGFMSM`{PlJ;cp5@njkpK=m@zxHp!<{1#H$ksafKQ&THh*yrPAO7MTnaUQMijUP2IEtqY1dd zijB>3Z=2!-lyic~qMWSW8NtahJC4l+I(HfEvLab{7}w!F(BlgEV9G`v1ki+dEvgG=^U5?=l(NYm1+miW9(1C!-J zJWDVwQ;^1cML~F02`A+gC7J?VTy7L{v+523j6|LM(Crp^ACipGsJF#oBFr)bDH+sr zY*S)Q86q(_V(tfo%Wc9A4)#AQgtla5(^*?A@FBFTz#5o@!_@@2BvPzl*D8Rlxw}>8 zY7>MTpZ1fTwvrhU>WpwQq>1H6LNqI~-JoL zuWum49A%@z-hKa10bDWmXz7w8FI|0B#6uR5PT(yeo>_FDfIHPR9-ChPn^m0uyI4mG z%bgq0ijSWN%&o#v3Kd-HOS>eN>rKV@g9!TS4E!%&sqYDHRk~$u_=O!7Z}dFJM|PwoGr|`Ve5c08~slSC69QBqjw8( z0fwYQ?1&W?DC5z8ISG$TONiruWVE~Yf`powpJbAbexM^pz6_c>7}AMOZsQVROfwv^ zvjzI6D92+C(HXmpl%Fr-F|5*Uig-m>9yPKxhg+|f!FZZ?6p;=02MZO!*y6HG55)!x z1Gz~jZhlDCXcMd@hr=BnXB1_&Vn5UwM`apq076|Y9v5xKj5L$qui-^N=-Jr6;0P$@ zUldZa6fVb1xLWBr0DiM8x+4eS7@-pHh%j%kDw3$m(ehWo+@ri%(K!EsK+0JkZ*c4M zZc*+J9I-IxwtN+U9#uz9p0XI<&X$F#o?p+{NvLzuEtmA>bHcox)n!ns(D1{^M<}r8 zyw+HLaR@U7!?M0OOtdp+a3C(;EWk;RaB!PEHNGIs4TZDo^<^&fm9GI6dNO9$WFk?E z>B1eI>k_&P!d7&NBCLFtaueMbfnOJ%c{A*1@P^~kB>|mRlKYqhZxS>yK493BqKT*d zGTRL=x}Ri_S!tA852NyFAJdj1B%Ew(X^C_74bz!TP;sojG}J^}4KmqbI&iJ9iMGN{ zBOOg7(7mLARt~HGgt&xs*$(;Dct|mGPaf@0CwluYQ)EC#&|!La^a@iE>O3nF?}_p} z+FPr}0?Ycxsp%fZCZe2e^4G^`7vkX6@y@t7Ae|-4YTDbpxm*!v>eSRq?+M9Wiz%`% zzO}p);o2>Vk~mVNYs{+=G6$hgufIY8rkhboy4p~nM;v`iI!iioBfc$oqL`@jar7mG zI&y@9%yC8jUG6NkIqrx*2{Ge)B*JSv+$YS5vwYT(Pk0ieBc`qhoM#fYr_8*vZQ4et zD=yc3GRiqwkkgbblYBFtEsS2TTP$&783!xvbbR=%Aa^RRl2_ySm7vWkR4Dk`zd?X` z!;X*VSTxnmS{abqG5X==?V{*jv$$RSy1sKiv0u?0Rsl(~2@~U394#{UIGyKe8*rj9 z_0OrTvG%Hbdql}VHC?usNOOzuG<>Jcj6GI^aH%W0oOg_9v#3WZCONAdiF1WIE5s#) zD$07Eu-t3uf6}jBA;>u^gBQHvcngrHS)BB-Y}ju!3y%B@`TH|CrwvT9cOWgWvlD7lC zM}+4doR69pZz~|_0o8OH333_Pav;;QKS?dOm41L5J$)=8JBrB5UUQhUy4?i026!&< zv~-g&&$yFVK>kt2VTMPi^?(2m#;G$<%)Tkw1!V7*o{jlXkhv^pBt4z+<7q`89Cjr; zrPl;G9r%triJ8zY97hry($AU&KtV8Blt{U=$lP=o$S`Vl5#m9h-KM%L;?OZdJ+1ce z#2y2=gk4qkB67Xg;h=69+i7qG%Aiwafp0X>KjFyiSf%RHaRC;$7{4qt$7z-h8+3!| zDxqeE-0SetP~1}nOnMgVCs>1?5bE0C%4m$8-E(DBt{`xPHXLsYa{2VGFcT`KE&Fd& z#Dvx{?!iJ_Vx&yj!pbCW;f+CEV)=!t#xw!BY&yOp%nO1v5_!mJ!A<#giL`ZNR3#+M z!H7Ov>t}ZhMpm36lHYiyyC*jOHiT1_t2O*p#iE-5F=%actTo^6h+A=*(*@kb|5GO-udw#Gj0FBy> z@;WXM;gWMnzB^tP?G|ETkpR2yzSb)y8AP?ID$k zY}C)p1Uot2O%_28T0fiU!JF~l9YDllbj|hHbVn$cSK&M2-vV3!j(WT5@<_YnPGD{a ztSq$L$&b8Y_aNQ_%WQ>E+x>lMyA4jijh1&u(i!@371u5 z*ISoGxtJHL3(BQq8BfKSD~ueG8<^6L6lKm#LYUfqXGl0B|4H%X*+pzA%F-5ghOpcO zddaIjb}U25hy=@r!Prfh*@TB(c4!xu(RgNFM^jG-AtyvYY+^)E3ZzwCFSu~mB9nd$ zAQ5NR-OT}55V^t{+lX}f7z9`GxKl<{z9wzqY?*b48hgkAaiK^MFv*rhSz~uddVEXU|J+yzFBG7RT)c z0dF`GVl*&GEWEb?cvt0djL|q3`HZr0w22ZT5q46vE_8HrB| zqnmak()7f;WgLdR*5dws`JFG2i}< z!#I|p5~ql!xN;rK{Ku-zn#E|oOSmHFO>6b0n_J2+h{dgB;BMe=8AQk{iG|;=@AB zcof;(Zc1AE11nC8n6#N@sf&cTkTUb(CRj5b5~fZ$b#m-b<=X{jHLpZXwCAC8NQhih zBVzZi0?`d! zim{InmsLK<2Bf*_6$VdqJn?gVn z3+kbpl*m96b!kV`gg74Bdo;>~Y6x>%=uIjOou8Eu`)i=V*t||CR|9n*ClblrtuB=I zo{q1}mV=74k;>R+P^Q_!VAqID5nd88(Z^=Smxdj;tz~k2Pv886@=^W)cWXd;0g*tJ zJUJ(W%|GdfyuQ1cOJi_95MLLOJ4^Bgjp#ERs|t1J!?~HfUw6eUVNMA>$(W$M1)0x?ygCX> zBIw70-5>~xdj4jgLqN2SuoJ|klPIU2?B=ACxk^}0ol{TkvR@M9nFNuXzV?m)v!Y+G zhhxCkh}xk=C{{6+mzM=lBW54@)6UZQ+J}KUeNa~D!Pr3%Y|9=l;*X*!Bq!mRUL1co zlyhg(&!Rbe1b|D;ks|3bW1KC_sY)LsC%)&F!4M;CclmmWr5C_t>N@#IqI!tQc}TlR zr!J#f?v&!2r^M1j(_;BrKPqz6qa<<$IsGUgH$P1}5~qtc`(dQCWvd=tL`wHUNoPQi zXEk>GGC#31gx7@AcV@Lti9L@g5|WQGTIpgT?kx=e9*p)ifHyuC)a@dtVYn~ilGP>x z9aMa&4Rfr_wf-_FuEXgL;%oxVMXp}bZfrgj&ONK}+loMzV2709g{J*S6w-X`|3u#tlCu;N)``^OH2vVBPnh1S;hqv)Fo|~|q)T{4V6Kn$qSWVw65>83 z4!GLmRY5MtU=J_e#>zhj$Spi7*L`tziJ(~0t;P?8IBA^?D7U?T0p!Zd%~t$IbZ#OH zmGUi$M}Ya&(}Dj42o#T z8k96%33Z`)PDOn+hjiU^RZ7>oPF7?O1SuPF@@DMdQ+KisHt%KV`)bA~tR8A0p*+6f~D_V$cMkI39_`zz!0 zio=Asx3O{K{36|W;fHf_Oaw}8>ibTeCuKGRaioBWGRkHmZW2P5ml_S3%H<9o65%Rs0Y^^Vm@Y$}lKMsT87 ziLFF2usABMM^l9J)_rw+-Nef!gn$pDd*Yr890lv_@%TuDhkjRmY$Vpc5SpHx+>eYz zRitany#rQ_`2BA|^3vYKr!G5exeR?raOR~vOh@PM09+Q`qUdKmdiElq|HsvNfLT@) zZy!N23IYWZCF~-ZjkCK%m7E1!B;%c#J2Q9E9p+BhJ0uAr8AL=71VIszAW8;NlpvWu zIjZCwM9D$E-`j7`IpfE(56|>_yZdxkS9e!eS5@aZs^DIjp7irvPyx!uL#DR`-4 zL)uH(Bs70>m2@}R36p}H_TTFI%jfBy@tQtq&|$J(BQ>WB}B`)%^Qqm zuO*iWG{7AILX8}6HWBJIPeHHHwzMk1BaeORhs>K*uG!e!B<0*H*bRXlco+WJ>j0QS zy)~JD{Z}auT8-Da;&Y`a46)!1AZBJi8A3Z^xYXZf7cSwJ>)F4nc;AMg9b0B5)=fiS9p>HqD6ehP~6J z0$ejvK2hWFNV&Nv!!-$SyDIHg!c%f(<@4tWq74ih5}_v0LT5n8eKqzp#;Nb;33@EzH}N6{XOpY2RrR=oYYBCF8Rn38F{! z5v*G-xs8O0tT+WU@V`P_0{vhzoA~DK1tb-#7KuIo2IN+AY{kk$QdQhlRK{dc@SX1q zp)!O+-atn8#MASFAON=YLkXDgZh*Xw_G8n;W6f+>3H1k10_lk*mYIc=723`qCihbM z+wH_NW<&v`Ly*hp@>J@T?L4a2YPoqw$I9ZpT zzLy`wb~SWj8{JnFYbWe+h4*Fv^7pV8Rmax5A1F_L&WFg*o0kZY^>s)1st|uZy;c^x zJx~=p;>oo*<_Gx$r70d@IWsjUQpWd|U z93lwnReHD`IjyWoiV_U><3d4RNc1{Z+^BS3ll2Y*kELjHcto&i$^{_<7OOr~Agp|H z8oyEqpPD)w$Bz$`Fo7LD_fr94*eTOvgNKVBvFIe+(_8_xLff>Izr#6=94)yc9#)#M z;C_J(bC0lQ-og&EJ?W^kpV_ree7M{kEM|K}B%PF>l`mo58MeVFC2=;gm z2{I>fpHYvSl+GN$YR-c@0-$Z8R3+AViX>yIplxzhgL!%(nj|>9+9n6!I9e{5#Kp(WMT=Wg^BXGSZ_e$N)Q$=Mc#SRC@4%l8xA88yJMO8R?D+^XUx zJ|xHl?86^2J}a{*PKnGB8_$zPn>0r5ztrEfp9#Ww>j|Y?xV6&7dkad_TqfmkZ(j_& zNRkDcm5(=mYv~A-7VH!B1GV()(`YJgfC}usiSS%;n zhY>|zR3fi$(%ZMigRgj|dtbUoMC`mV6FVUN;Ta!+taFQ|ZjSM5xE2GoBaZ zu^DCiZ+VksUZ!w0<%@?1VVI}QnH5LtZwQr6vi+>?7YlF`p}jT}%vjzkF`|j48;Q4t zK?HVo*<8djZ#xY~^lNoYX|3_-)mAEh>*Wo@Sm9qq-nQfgue6(pdxBn#IZX)7@@*#O zF4qb$l@t|&DbLbTji3Gp*i_- z65uK^V@b-b8w8CrI8xe>;@8ah0MHW;qgS5p`LIZ1jM*9QEA5IqP>$52>mz{NRur+i z+uPw|AdAiIIyNzNSfs^!`|6j>J#Ac#LHVj6w?YpB4yA)g0?ibdQede}*$!nimC<|l za@vyPNYn(6XC+jOczHya>8#k6ImZ6a%AB-(BrQ_Qeo}lL^)yGoc35(nK+~Y7A6fi^ zPm4@CswJDuIN-m4?uy&~tYFf0RN{RBv<;VX#$O)(ow4eCyNy@QPQw+W z(u^V}C7sIF`U1(^bT(po2tlGns5?0#e~*5i*_ePe(A6LkS7R&Xg9oN>jOzW$vQUF- zGLE>talf$KYK+>n(Ofw{(D;01Lv6(4LbNQ;m=UKg;2PO_2jem&JXTto{5nn77XeKz z4EKiPG^L$s{57p=d}%>IddcG%l5fY9z&v?Fd@h-`GVK||tRCEN$OnIW1zoe?SVIN$Tg&EpG`$6ePcQe*L%AQL;C!EGhB z_adO~5CQjl;s~Wpd|7a$_g~%@<_}^C-p4$MMM1nXh~cVz3OYqYD#F>3lBI2&t&|(V zomMM*R-7xyGm4!Lf56MiJiY8w;)=z+wI)0b&N+?brU&_}?w@sQxwu&^K~~ z!=(bm0|y(;lU<5THJ%Y{e(0-77?g2Q0{vxP5Fz+_hMbQ%&%UM;c*^JIw074qY=1+w zxga;=aAjL8yJP`L6)HQojZ287H^n5|OOR_K{s=|}R|gq3nN9Q2uhYA?fmQA$e4i26xVKMHWQ+G2&Qv31NC- zDN76*CE%esX!CYlp1=J87SeRc>@6V_naRWHbwQMvHFH)R{#E`q=W3Udt}B*Wp{NCe zP^=HQ&{`3QiH4Lyn3mgCBIW5geuKu*hWB4r1~-#-5N=^y9hhK$gUiR{G!oaBSz+4- z@zNA^BpwuOVlu#4c^a__s3+toHd*mzna0|uCl*|l-{GD5yrj#Stz`Zl?c>0Q+Dw?~ zthadBj2Hb{Q7O8&j@V%}(xx3M_M|qc3UX`YFjQ+K``EKq2Nl7mOpYbi;BQyU(Lm9? zHWcFFgIKk`m|Ftv_^@UDQN)It> zeH|87tceXYZ4!kEha=p?f*psWSynbwna9F)mjHLPufqI=3pbur)ZfP$L=P0kkA%5v zIDklFdSQuRyeG5moz@}`gs<@A6^G76)+U)*7rzg#kA;ww{K6_3$3 zVg)5kA-4TilpNLAti)Kn4jPulT*5qtsBkgzlWH@8UI|e3Gjo#jOth;!ADh`SIePWC zX~DGWiFx_AX+#AY)8B+(8R?EMP2*1!ggcXDy0eKOccO`BDVl*D%S`OhDF{zpkekmM zZ^Oo4mDn(W@Cn0&`;suI+u)KS`JpX6y(lO{pmx~}P7>xmaxE>N6=;2gK#zK7|8RV) zq+7*eh&C@K%>Z%J(`!R8z?w6G+$t<2y$$au%-Q5Tr-Q(yf*4b7t?RLD3kg?@YLp1G zv6c{X1d5NgS`3tc0;SjEQUR`$MKm1`mYL-`LNnCHa42Pw$4ogi^e3Od0JHvp7eI(WMSq7oOswLY^0cASBcub z4b$Q2%IH>1n}Q3*+_Eywy`*UA5N4te^6FD_4oMG815J{n>ps7+^Kh-hop-#bq&R%W zoM>!PbO!%O^FAayFZ1wA`mN&1i%t+caUAdET&2rGElru^-O^X~f8DbwOFo4Ewb z8ah8VJ}A8E=3ul;hKdU7kmS`}1o9=l#fqN!5G6B@vi)%{GB1|e0xXlIQbi_?WrcX5 z*HJ^%lLkFjFLNeE;Al({B5_!Uy4XdjalRTBF`}!4bkQh5<;w9gdL23*P)dtQ7gs^m zZ;){FvLg*I_X*1zG{-Ww=n9>t1Pfammk=33M=8iCDw6^W@Es)Grs6m_2+|pU( zc}fLe}$TFFjSXf0zxOyEk-)4)348!%@n8(;IE<>V9Rd zrfeik@<&RH$%kHv-TE0}uDf57>R|j=h#ST#kI{(cP%OROgnXRX2x=+BePl}-QJBun zw+Et+=$+H?ZM7YWEIet;$#EdA6+|nMqS&C`SBe4+If+A*b#^4}DXG5Wj*;s5n_zQV zS93t#94roS+S@yUn_7%x@-)ZTPoUc+m1Co|8T0Q91oyPpurdfG@?iB#M>$IX%s<3s z2ZMjHlBO+(p}wSBSZxIWJpO5aJOB=Ul3xdaj3=|SbC)dx-C4C%6W~eu7uIp9wuZg@EuB-1O$;y zmJRC)$`vK=C#`BT1?FotRD0}7c?x|)Ke+KcFv}UZUe+krG_#Xn?GSog)L|pp4fEWgLI~I@al=AxBr_dDFDNVn4_k%~U_bEzu zVfNFD8uuwd9$MVJ)7}oVCDar`x=L@G#Y<&jW*H`K?XkqZ6gE+~3dRUlCIm|gbd8)e zw3=(~tygA+{S`|X5*#4V^aH^q1ib<(o8CA?KZtek9EqEhfGcM3SnbRD+YM8Q%rsp$ z*$>PgMARFwH*lOJH1m&wy2U91%p6>kqMwh`%Pa^2OwpSr0&Li^!2U%gh;~F0Us!;9 z%#o>Ip8D1JxXh&|zU%R;1Bzl8MR9byQ-FsJd9g=*Klwl)dNpm%?6^gL`)3(^5*Rsl zr~oml@E==xMJcZp)p=4Y;Q^`AGg(OSVzYC`%P{h?54pk5KAvpp>k} zmP)Zi;Cv!&>b^?454>K3qBzbJ;?}X&C%N^CGE-YU88f_4B6t+HGo-!k6f5XQBon@T zoG@3;ny zFb(SjFpl}UNzn2hR_dWRA*a~Puoyl`DQ{Y+iF7lQB)hn;7o63|R+DY9VBKl7&gmWM zgOv7k^1CRa*?9uZ>{2x9PE^mIJtWwRMcOUwM8K6*`eA%&=F?~vniD|@b#_CXkh8ss z2#`WYKB9U}LB51`r&5o{g~2o_7D`t0~4z&9VbPZY1twX#*#cN4zfBG=%w? zkFnT6ARdf%9w@5E?gHFg!bi)G60OY&gGF^*(<^2{l2vaL?CLnY4H3+3@u8wL(GWQ4 zN!WR81kmln(g~|do|+Tn_9Y_*E-5tco)GMhGvQ^?16REw&=f!bM5v?uj-z01HnRvS zCjiErLv*F%^N*uQ0!hw1Q-B+Q_8!(D|2!brWe9E2Q2Z;Wc>b43@atw#d4o&ufp|)Q zRv9kDmpYigVY;caXHJh!CHP^+?3UQz5dKc{T6@`UN_p<{7$WM_hGdSWPzcsqZ~Do` zSnf~|%|K2XO`a@{djv4HDRwokM@pI^a9i7Os~Vrc6&71shl%8oRR+>cKhX+Zdp6=7 zmG%Jfl4~8U>&HS&if&G`G6Pxaup$#{66->IO^CY1vpKW$w+mnmf(uQ1eBp2)nwTED zug7*uI){S65hgs^N=Q!qiX64__sQj*e~@K1&R5!{q=b~k#Os0}En#^&ZLM0-s_^Az(t zT|6wIoW{6}Ma+fi-Y!dXSu#`|K_02cia#j6oz{cFm@dR!?HNWXHoMG$p`Xkl4iGZl zIR+lVjL-sQwxK4w_fX6@hHMa;ah^1FPV_37g(r@9ai7xewPKVbwD0yq5Kjwi*I5Vl zEdd1~YmL7Ma2bRFj6ULT$AZ8y#1X~u>`A5kdxN(~v|I}j^W%y1mE-u^R1r-vW>Cns>lVkIf_}dl8nYt3Y z3-H%rW~q}9hRzj8i7rfVetv3Ef+>tZ@reM@hZiP(^dtUu7l~}jnnt9(5|Wp`RFzMi zMlSOv3I;CI($>1bk3qeGh>UV}Kwv=H{9^e7tFgjZWka1M^aEoB+jVo&WP4s(h+9uU zLez>n9xp2}ub*6HB$gi###F<2gjdH-C*f9iqKuR~`uaa9Al7=BSjSXBZnc!D_@8U^ z_ZKSe!91fYPCSE@Kh<6*c8%g*taK)rIf{E@&GQq_axMy`o|m za)-z{T43I4rFmgIbr?#p+pAMWFGge&a$;1WCw`Mo=BK$ym-zAb!W1xZzGmD2h0SvJ+t&9lLcn_!Nc-sTkz*V;;|_+V~+mzz{*oI zS)8TT{Y*db0W>EK`j(6Wo)P9L&wI`BoRV%C=d!fx!Vx3qfkJd5Y{5<+|H9SKNzA$k z`~f4Ze?HhaiQ1BH{)U3w^rYlZcXv|-X5PTU3KtIl&8CU(5zrAQ9 z6Zga;N;`9d5M$N&)g=IQT(2&1Ak}vEn@bJRQzi(j#!&2W8IZfdvc?H59YkLc>dLW+ z!vUd8^O{;SCSA_&X-6eg!s5Rxs%S;arN=_y@;0HFEtqueUAzNnaD3vSo?^WwcQlnZ zbIWk2ue#1JYwbgc$*Lgv>@Ot>Q*{#FMWLQ%T|s$sWzu@_`P&4!N)+ZO#}Gf}{Jy}X zb7J@Cj5~#8vT3D`rv-Rc@fZU~9yCTtKlp!t;0IHRdp(++w0YfEf|^o1^V=B>rKwM@ zAqw#w^IrwPm`!h)8VBod+R4HymuG$dRzaSZqiv%+VnGtZ67OR$u2I5$K>dcT4U)!@ z*MPYdXjHJqO-uO%w1MFs+$HHVq2@cjXmbe>@if(8X71+hie>I%A^wt1aBwL>4TPC8 zU7Q9R@nTs_rvdKJJ{DrSH;CV>{;Yc~kcVB5^YW4};0%EfiBq`D1Y*~mh7aYC7pExg z&kt5x5iuFSk?YF3YxB~PWryp5Amyy7ljCRl+eMmOw{v^Z6MNnOmbq4Uv&Sn<{+ZLK z#~u3H7h~gcTMke8Z~a0)u>)%&-1fzpHv*VE?KqMmFP-qm0)fA|GKxiRDv+dz=IT(0 zS(_DtNZ@c22hIxxn?%G^L9}TYGKu6Ctana1;+Dme*Nsj>C>s;(w%{EsH^RRuqdSS1 z#mVSTq_rks)U%WRc5bc*xFCqn^RLs1-M8AGi>6@Vr}&96D9BB*x3$1W1)3$%jmEBj zB`F4;Ju9xz-+6Q^1Z~5bqY|$R%(G7c#?k*Db&IP&@EXFx{um{-nIjpt#idGR#^MgG zJ)X+|$_&B|!XNwI>Kw>|ob}-Ps{~~Jl*e!lGTtpLFTgU4^fgC9ol~K<5zGj1s+^hN z7_Z<(j6gO4Oi_JU{n`z>G@7|ul6_uKX?IUCQrK2v?rlYd?F49Y?CP@xn)s4OVBtRr zaYqq&*=ZKNy#RE@I~1!4$aQPha-Vp~-+*!lxU=Wh5kF>Ot{RQG^vY-4St6WIM_%Cq>p>=^iq=amY-1)ZjQF&V*hw&F1q2xmVKpC#T87?gfJ@su<4?#uE3DqD8Yf z4pj6vgp=nUHu3~xz8|0%YdfCuSf)@aX!OZR?X(9-xgFf6j#27M4;7_Qoe#5#u!zQL zLfsUcWO^|2s!YVNzsFXBvsB9GUT~&B@7^A(t0=28A-aZR-LiHz&LN3Ch)~}aYF^VD z8O&n%e=mWqO0gjrR70L*{X9WE<6&~R@i=MBW8mfqaWj!D`t`s|9A4(a{#YmW<4Qzk z^pv}yMB0b-Xj?oWMARfSSc>}jv;dl;!NP1$`nfG00fPEnweHwae`kK+D~S3ELf$LX z{DrZToucUPpkVVj7f=lThL0Bc6tYU2uLRe71m}FH7JA~=#{e>u@K~i5o;gndnF#6H zpE&G7Oihe>hKTE+*&xu{8i5rCiF)CX9p8Qu+!TXvhy||U5C)4(Tc1b9V=txrfkt{y zGJYb&t-#$iMND4mDWGCHU}u`F-PRZCmf&>C{-_53P@p?Cb;`zk;35H;&AF>dBFo(q zFdm&vw_PheU3_EQgMgvitVGj;m%c9Q6v~3HfX?EupXW67C(EvnmG(EWsZNqf%=o*@ z$dz7)#d%6=su7B`8HYYg$~{CZL|#a)c*hI%0JAm`IY0&vjW|iLX;zn|4N)wFn01Gk zPk&X?l##`xEfgQld0I{8{huR`$;{PL3YK+>AZp<;Bi8ts`#i}!G@~`e;6J)V*dz$w z5w{3&EwZQ968oBc>z;m#E4^F0Wg)(HXx9p;;+Q@ zLR}Ikm1IqvAQPT#xx!mWxDH~*r<}QKIvg~((YRTdv!dRWPAP??dQ`B#jrBP<`LC8= zX=O{6vb6DP@nO~_f>T%G2PMS4ZDOv2Un@Q;3lt2k+T&zlta@lBr^V??m~u=35t!BE zJb_Sd#`LLix&DS!SdntSRn6YN)|;+i zl$|zn*YSc(niT$4j+HkG)9AGjljyDDclcyTYfvm%Vttt3q}!%--gedLJO^X6Z(*C5*!Mf?V_TNPr zJ^_7?{||tt?Ese}FDWT{O`8)7zsuj5y=8r|xY9JwPdL)g+3x|E;M_C9jC8ZD#BfsQ zdLO}Uz4u*Ef<0DSgE2*b3DgPIkv%v%KQ7Q*%sSWB(+U?c$u{}`I4`^$L#2@7u#!-B ztphtgtqch=rwsEx9DG8wQTS>CZWS^T4?=xxqKe%J6fU~j=Pdj~fvz{5X7Das>BF)C zOgFt8jtIW0P!A1Z>SQCig%H>0N0M~&a=k#)kkj^9eDo1%$X#b5Fk;JpLedn}Z7mk{ z1fLKFnOSZ*!f=lLPylsENYGvRQlF95yqYpQuF~Ibude(=ngjn^K&^Ex5pB^W$TTFx zA^w|5EnF9Bg6km2y1~q9#vx_-UInpaP>&VlUMNak{6uMYeyD;yM=gF|A{AWD(7>G&<(neY|2BAVsgzo>J-EpdmeKiIOfJ(hn}pj>pMcC)tiK0J`%- z^>)^f!8l%!_;=RqSZRL$J%=F3=jd-!Kv%;Ox9HcT1&Rt}GKRDkI|=f#))IT?1P7LS z?D0i@F}V<;`G+=@;TjNZ+Vo4MeX7#q$7^IznapT4{KJCa?g5%enM`pay0QdeE0hW- zJ`B2BY*bBM8egq^uGGP&kryPjeq|wW^A8gZ#}KqK1H#zVd*tRHH!Msl@1S-kd{qcN z93uimEU_pF=rwcJAJA~vO4Py_d3)lFt z1QA)=txow!AS}&cn!^k_;AOvLKuc_j%(9UH*NzH`H^pKzA>$*43}y-9G@-e}(g=vV zFBC>QrJZeyyObKA5h&H5$aY@_qLZ_DGVU$?O;Z{eyC-v=U6urJFXU&RUeSnq1(J*Q zA$Mcfbzhka?`dKHXr?|Z(7ZA=-YbjfJxweWm-4Tbc5D`QR3i5emqYH}*ZK;`IP<2E z4$&ou#Q_rOWtnG`q+cALSg225nw04QKgnrS-1iHiG!(1fT6(yxdl`UoQtL@bEehXS zwkXYMPLiBl94-{f7qoTN;(4VqtMEJ{*ZQ9eF>Q3aX+e9T0Q3;x7Y`S*Y<{=I(iNeU z6e}%9zB~|3MOEvK)xHX(*@ph(0 zpke~ubzN_zHRNs;7Zc1v)pCy#nfhFAvVoys<5^n4?jO<06(?ZY${?O=c!i>HRT|37 zKqK>W{q2@Z>!mPb7p&s!=`jqpfIlkj3cIV=wPRDYDu~<5C39Bf+#t}|rp`zK%1-|p z2rM?nd(OB@wyZ|d9pW}^v|ZsN#)Q$776jg(=f7vSOo;{705`YKio=xf=aShl7M7iH zra(6wT?c2;dORY?AK>a^4AoF0o+-;}AxB?;jPRB~Hxx<)=8~JwcxIupn*PXfl91 zi8m6)5IPgj5Xi_i>X`Sk&-Jb4G)qOJg4*g7rQHz;&&XVl3Gt_pU%4i$#4cY4GAGJ+ zko`fks3|n_9P?A!ZP;F*<_wWE@6g|TTOx?m1<$QbnqErqoA<0)(Kn$rQGcU3<2n9$B3>E2;K&E2%pn#4RLefUiN_E=!$n_T~17vv8}WQ2y> zwSZ)zR@zzBP8UFDQW(mbeC?4Tx1I&aS33!-%_91gk|qPjV7+?KlGeXC8`NWlb&GxS z6K1J_>m_2>5!~$m%iG6&H`o8zq^rC&;hdmp9ihhwBEh0Jp=LhUgU@=-?KZH(YnfBM zFR6|^rv?UaHmbxcf;>7}rX0F0$BBUM!OR8e}e=AJcszeU)SI6EDyjkhnulu3CXHg9v26fP>mVZtzVIcUT$1k$-#EtBIO{mpb~)Q1Vo`ERA%;^eWf zUGu-+Dypauw1UNvL-{#dx{B$urpJq0@wb^znqnSm@eSTTw=OEn7sJiCa9c3CyK*TB!DrxCvsXfj}z$5vta=8uBSApL{5MBG}IZ4-+nWye1!fW`j3-gcDC z6IEWFtot_#a&x4|(EI23+IfDcGHqy=1{ZE4Y6`N70W=ktcH zjVFMLQGi)F^LcUQ^he;WdM(cSHUPBacARamFYZ@bTS3e8XxWXwX)*7S^u>@8<~0S8 zVk&Uo!?@roX#LzN{O9y|)S!C(Q${uehZCbPHAQ~mL%rbL9 z%{&Cp*8PE3uE9OQ-Ai=M$&#f#5@`0stE;znfWX?r%9yLwV<~7YIoh5jG`EL4Jr?o! ziy%{@6P58$+$F%Dmi|KXCB-rP>UWE>G82+Z?qorx#|RI?ur6LH#7%*1;3f9@7lpY( z0z{|XCRvevYcFtrnZ1x(l$wA{Ue-#gyM1pEMxuqi{}cVqNbsi5%(#Ca63|_j^$h7k z`xYhJDoyUJVgn&$fTQ)i)x!Ic%uJ3EsUkR?DacKQyl7fjI<6Du$(5op)wEu)#_V5| zM#4&x{Feloz6vy)W#PY1K(S`fRS>l%)E{rf=u^RfmN<~&bY^G_mF8MX_;=o!kkx5C zCdk#xZZ16&)D??-ugHjhR? zm{l~!D6lZ<;oC;3b@Hb#s&t`T6W(1PTiGt54+Y*&$w z$VAMh6$L;h2X;?;RMOPEESw?MBETt|kHq8}5N%0sy9{vw8T$z_A2iU-RtO6Ba5s>p z5-Fassq58U0@PfjhhI!5X=K^piE`Hq=2prYn6oK&Aq~SGqps>$t2Ck5F_d|ksD-eT z;Jm=ps+}^MZI4@&cYLbw*fS=YF{cmAq-@C2u0OsZ#GJ*ZJ{(&s%?ROmjtgIzozmQY z6Rp3fl(Eo0h6?a;!P%O1oO|lr<1HVbBG_zxj67pBini<66!iwAvsUUWg~%}5i$}$ z|`JzO^Zae8N3Ac>dzU-Bu;5-thKd*JM#~@iXfjnN_mEr!lPHC;+2vx0P{(JhyQ)A78 z`P(d&EYed%0Aa39FPPLhQCu!)oE^~^%9qw4!Wqs~+-qH-EN(bUXBtR4c(qXXTldz3 ztZm}tR|Jo<2Mvyu4gqjGTJ;#7EN6!f1@lL7@6+l*$^OHOk_4qf&vvH(m!zqFl6uM! zU{Gg(_hDk%ktE!mR%WRDcy<)zw!(KDpE-_HW4)t5Ez_j9n~rzO@-j<5xFN%MGI_;K zpufvq7t@?zFIy_3yVX@`w)e-j0-OnBcbjS3V{8JEB?0xxG#FlLu=oP#{eT!h;};8aF-4V=Y^_g~qPCfpS6Ym+Oc9-m5JZVXSfb;MOF?QRVBSG8goazWj*=s_~1 z)$+^CDvr1f4uEA@>*;Xt7sTY&nM`8-2_#(=3slEYLrUybP6RZWnDPS(0}N+Q6Y7RE zFvypM`Rjt*TjJ&tSde9O{*%EVRx5HzC6-Z&&oPpSIWg-LlKxz-1MYrXU}9V}hsT^s z0@|lL0>1h?rKtdWZmFBU_#=|0WkrU=$l*yOnl9WeX;siy@!XWt3QT8eS=ktK*U~vV zy@FxX&BA$=lZ~YPRNP<^eQJ^i~D9xpB1kw0dZ{V5+5k(&-7HA zc&An43#S8#brtO{HdV@u-BTN_#c!3SGQGnRS1XZQ)P>=N-E{sb)SYW*4x!Hb8P~TnPC13cH{)$O$I>Mo&{=JEA*|; z8xz4MHs~x4jYy!^E2vBS{{ejKB+{fG#^5*Qi46)2_g8%_w~K`h_o&g> z@MmD2cQ~kEbUIf+9#y?9(;gQIfm*$?AG+*p=VyQQW%d&SEG?z4>*V5nz5N`1VU!3- zpvPdkb8;X(02Dv@IluZlQ|HWSA#UH4b6qv(I+zywvqJ(Vn)bygqptSzfgP(b`V7@I+Qv0o$pdD1Z`Uyf6tdcjKcU z33rvu3C>(>=^vL^AJ*;=95=q}it;Q8Y@ChiJV_FZ3U{NY#STii(VDzmo^jN>M4&s$ zGafqS%y&b9bdwKF$(MwX3&09fDF^<&)(8abd?hxotOr4%+_J)wmF+6HR z$W2ThZYzprQwHYwl)n`qx1|`9X|+k9OS68UhT|K1DkH5UB(Su+9c4N7I@$Th(E?3* zu_QAf+U@`{t$J$BdR(it$^621!ur(R)f)r)+Z}I*-DJcP zpOzTPM{H#1{iF9470cR36@Pv|5cFvbX=RAn|L0`Vq4&qE2b^S2T65}@*g{Dyac%4= z@ja#TvH^pk3yMnwnM^FCcoe=Mz+IDTLp<~#X+EcE!_<0LN#`Fz^EwojhX7nBl0cfu zH9>hO>fJnr`*2Z)V0_4D-xL6eM=Dsg9ifB?g&nGVJS&fYjIXkK9x9p5b_;brofJ5= ztp6y8Kb^EbtTs~xnQK{Jb#upIcc=uBsBz;YNLvgG^hfm64}nGHotRL%mz=zGg*sAx zBk9}t#t#M2dPI!jI7x}j!6{fc8Gch@dizjDIuo%5f#y-sRpZGrA1YRzRIBltFf$y6 zC2*)V4j$8f3^;Rhy%(>;0ReeILy_v>tuH-MGzO;$DMqVt%#&b@A2h=%KK}9)Nf@F{ zy@`jEG6j(fG$CTn--|j>{zy~O9xn^?*X6I?6K@Id*KP17#$5RyfF7VeY>GK|uT>(f z3({C`BE+mw)wSZ>GRLTcM;s&s>LM=;#}AayT7XWPHRSWMbdStpSNJDCd(VM*rJ8Cx z;cm$Q+9=k_n@fb-CaI8?e}+6BI3f#}#ddtIz}SpAUXZUMEFb5336RZO1?f1~Ej(?= z)7g-lO66I=h;?#T`x8NKgyN-eEMN0^5U(sPv8xg;p;s7nFnLyx4-US|9T(3((+_-0 z_jUC+<^@s^g(==cSo7syB$=0Q?hl}gM5H@}<`F_KR9~XG>LBiZ-#WiAak301pvj%Ex){Dbm0&ur6mtu{HH-+RZ+`ywx5#XwI z;2~lnj(;x~nWTDd#%U!$m<9xlpOyfj3At`s;B|^*nqwfP2Y!w#p%hDoihnG~ALHM3HJFB`_Wi5CYy`~-;6h>JlK`2pHSQ5YSV!&S`2#(;X!{K;b2~V)jN_V`Xq(8z=ftj1c)wSOT%>2-k{eN49xn3l0xy8nA z9{~B&Qbot5N>gtuYi~6S3WHYwQ(!P>qgHAF)a~2h;NZt<5{6@y)>fC?|4p-JK37# z3r7Cc2^gwPV$XNQJHoWE%_0cmil32i1K77&wdV>j{UpK3;I$Eb0_iz-uS8rt_`gM# z6eV0%UvZs4Xl1p0{8>rY+mNqE+&uptsR}}IA=~^zDcXR}6w~(wNSN*LzLN#X)k65* z2tkyparzfY<=LhKBqylEWoHY|T!vdx-LHwBBh-{pR8sQCxq?iY6v0P(^!dWPx~7X< z$STggu!K9S*zhuPx&-{4;5@BhXAJ&f-QNrJ_s6Z{5duG75EQ|22KK4l3-LEZfzvhT z9#GP?B%g+gR>9td$MesKm-M%LDyI)rY`(63UAQU4vN(&lT5l`@Ld~d_WMTQXlI|h5 zT*yWoQTi7JgT`3<6Zc@Tq6Xd@hDl>LAuhrLOcr`?7sdo&l~8Q7=v>IS!N7lGtbm=q3qyf-eu&BbS+);}-y72=QK(agmk4LC_yW?_ZqLH2vIEXbjQ z*Z<-(A>L(KQ5AkEdpBDJtThig?Ga)byFE>+guO&(Oi(K)^O1>E@dB?hSk_p zs6WKUi0Lb;kmJhY@-T!`I8>b?(2YdR!zsEqmYNI%rDhX(YgPU2pRsvPO1-uc5_fU0 z9zPNUsavZkUE1RbCH?Vk-i1g3c{g7RjCD_OCZk7*Ohw&vvZ@m9yTDLlXfh7QaqEJ) zZ&)36N@)vSS6L6#qXh-(DBG(-a&LK*oaL?=^MtwO$;b#%Cqd4J8XF}I<_`%o8)0w6 zN&CDFC}*Hw#u~BDh5#_8A{I&>IYVh?C{j1u;6wRWy)mZ7J*u>e*Rg3Ij18v%xE&aq zCR(gJ6-ax13(u#$tAq=*bxG-X*0cg3>{P1YI?e`qxiRnnj4@=Zp+6`Cl1JIo$Q;nk@z^LlcyBKTX-w-XR5Q@VU;3) z>eBfY3G1mO;wgaOTEJ4lmV#oFZOvV%s#Pi@Wp{^StNGg|SOJqo52q8PRCy&U% zF`MV|b|5vr^|K%mCeISgL;9guF)%FExh8>bTD4!NAqez{GP)(5b+oz63cm7aSzMlZ z$rU%A5NP6X{!Vf0*WIxw&3q;tOHHvHTZCqIV9?kgB_74$g6SRsabWW+zLVyVzDzCp9wPE(uFe5RL>B# z3NuMK@vwI2g83|gE{eKMK4SmM2-K0wD=c%tw~NY{ja692l$(!&`TK3VfSAdo>KSsbl|8;!1cfb%@JJ9EK2T0E1^IzALv3i5|wBQCCD z>YgC}5c>u)>U04v#e&{cl%!?%D)Mxun8m*qKs9pcMHS3J>G_PHGU8H8b_p-n+~1?5v0noZ$Wh8RCGly2&)q67oyVpS8}d_df^SR zud62)*Kr&i5s!8d=0RqiXCg?hk)y#s^@FD#`gI)@H03wiuRys~PbSq2)(?bwq3Wtv z=0)>B(rz=7rHqZ^3*RdsnfzB#M14(=nL$yh(8Kb9H7bCnHg0WPVg8?h%rZkwoc8qw z-S=AoJ%=F*CaV_P6YLF-!+yHh?vV4>5z6jC9n<@tZE=(kbHltEk`^LfoG#39Pt$=U zxx>JsaQtR%{Nz><%ZkS8fGm zqB&q8;5>ouGwLNmLvlazVTo}D9!&?LxjjW;MqU?}nq2~EOA0QMPU0W+f-plxX_~fp zKS_E!gLC7>`rBXPgvuoNS>c+40Ngvgtq8gm`wxP+)huP)&&gaMfq75Y z&Wt*nm9e6%?sJ7_m-QT9yP)yzDIzOfKDo@+&(o8N&G!dEc}uD4eN>sj#p^=N!Ca(Z zan~s7MelA8jmXfxk!4A{Nqq=FUi|3F>!_D&uBQ!e*V z+rco(zyC!!sBjER8|?0?(Rt+%iV?6*;d3W^XAr+phWE%4V*avU_r^8{7p-B9VQcP> zy9JpHY)^^-_J>12@~B{y!kX3=R}1qvBb#&3kgzcIP(WHTaq2bW)WblbqGBN+i^Tmo z%@Wq5P!mfX&hI9aw5-x2%j&XU@Hj8jdy-4wCBi&@C^{H{+rAIL_COSN?pco@;mv-m z*%`+u>17ccH0?q8G_d`V{NN8VC{=CV5IGX;k?CSFS16JZ<#ZclO2FMue;g^y*_c*I zU_POQxWh?yNT-w#(}xADH!c)p!Zx^9gSGD!V*cu?jNxH?R8ciW`lSolLJ)nbV13O2 zY&RuM_Q84+L@V1q0>^a=@LHR8d8$7{ILtkJ>g0%{`P(0_H+YJUaTemfb)=&bB~JSx z7+>Rw7+w0Fr=-cPAkFQu!Lb048>RB>Sm-zsCWO{QXc0|8>G$a~;t?fWdQ3Ek_XRK@ zl5x;yNsWTg!kpOZ_@btPD%*a2D7F@4rq)_JIHWx-wh`(o`*?L38#e)TB6UDg5F6=N zPA~?4rHg7x#<9Z2d!l>llLeR_xQ|yD*B_q<;_f8N5k7Z%33XsCq)W^51kuB`v1U)~ zdQwqqD|4Dt7wIaTA`oo%4kTWH-(Q3Re1U^7mN!@(<)FKC4F|kr( z&S@YfM~#gj7 zXKrx2;v+$DUoS@|y0Yu(MJ0r>RaoGko>3s^R+t~L*iS(c!%dkHt0-Y2sriH4>7mcM z<;()L`yI>kKwNiLfif;oJ?Z+l{~4%ztJ@pvxf5t&Rp%krjVJ{CP$ZZ^L-b?GLhw^r zXTyC0`QJ1>K(I;CO2kQAKx#s~F3J9zt!p%e>5pOk;Ne6lMF45Um>@TZ_gs1NS)0RE zxk?!WvaWHd|GE(Od<-kp{I=7gXBUOJi0GHWVtWh;G&`_urSL~_Uy12rCq-s><6QE& z-AHH?6K(d&L;Stk;HBvF_H~a-ZFm9tLfNArQ=a(h91z@sWA%ww49bDll>uM}_ z0f4)uU}&b)h63F(-7%(TD|Qp+6>@a6R*ONU%|AFfwH{mim=N$1Y1e{;Lr5o2+d@v4yQM?<+VvDc{y2Oh)6Fx=ObWD(A~LQL1mEEK zuH(hy6Uy-J3YwX?%pXz=J-JD@Tu$2k?_@GmbOPI7Snej3qLyCm_aNpG4##MuzVQcu zye^<%=3O%hQ+-0m3CH&lrnMdQ`iD$-!ig)EPs-~L27h@Q3%LPk!@N@iMM z{3sb^hwFj7YVwe`4LcZvc{c!?@36%qmaI1RBLq4xW~&Y_g82MDu+&hn20L|vT?L1R z^vq<=>lUC3mFWkug{jqK>%d4zG9uqPscZlWndBAFh_lPAjUF|sO;(wYmRXsTxW9=l zSGlq105KdI1WbY#P7ugg@cJR4N#TSOZvyZ*>q-sEUr-_)+#q6Ov*4eKJlqP;oYNA= z3o(Ne?F$C~_MZXVRZRJ0BgOi)r$Bd=NI_hu6Ndg~Ack>(S7T%7FU9Y~H-V}9H2!4$Rb4c&je+D$80Gn6wfZRDLfc8$FEC>!IZ~STr9{f zW<|~qo2+pQp!!x*XiQux&Y8fvG@t38QtH;8(6@>;W^F*6Nh45t}XxHP%xBF*=1P%a_OwScU4bqFi zKWsXGbCz}EeDNUN)18t0Gfzod_4Xn;-n~N`s499%+chgn4dN+Pf8EQl{X4x(8HyDTnTd^*1a> zxcRm*OmIp0bFGL*uDJRrK zb~~)hahL`;o>%(E#E3Jte2P< zgn;^3n8{RSVS(?z{1Av)hpmVx`ML;h73eQe8^q$SQIJbwDvXvRt`cId?BrgvD^`CP z0Q#U)>5VOwaEnG2fNHKy5^mzo5R6JK7q^lGNEnmkfhNieU(mI7TX_dP?|;(BYL+O|;| z!vs&iKvI(gp^z)UxcEiVCL=LWhmcTW@=HZMbA3i>w|6`mp9}6Dq{oq!N1w$sItp(-VSd4|aRKK%tWySL8`E0cDx4 zp@Wc!`#&f_Oj#b`=!A~Xok1)cdH6?_x;L`ELLIV~5cfv*PTZGNV_#t=Gg49AdW!=y zD02>!U>>_G%KB1x4<6}$gR)+!M(f?NypsMP-c%fv;+sO;D&2re`q-+32Dp`wLBYyz z7P$!ANYmS5?-C;FE30Q`{Prz!u?mmyrH}M?<`8PHRIb#d0Rp2i&`6T zDBX@8`JbXb83Ckw5H&yHCIyeoHWd2s7-cmhbr9DwF#`hB34-&wfKDU1Z^y|3={;79 zaj*U!=ONDR*weC^{-02Hmo<&+UaYU)73Pm}^5M>*KjwQENZUyZ5Bb!UfE+0Gb^pg$ z94f>maBxd+4lMLu39*eaD0IcV<;Uch)tf4f7Bx44ex zF8VQ`+A)h9mngxE)l-LEu`(K*rJ|#AK~5+ zJeC&3@1wQup4eUqx}%k~+Fgl#l!h1s0|RlP{>~1xHX9wpQC_zMi+si}=1p!A_1f;o zLQG~%XYdpm_`-L_JCKYukR4tsA(3eimxlWmAeZ?=@)uG5bi4VEP@eG&QAyN!s9rzF1(oU0~DejIRsxr}2lxP9(+6-%;o|MbT8t zqb2@Yf^uHq6vPqxiiKQ0DZuHEWxu|KL20Em(|t zQI~v;{R&8jXM+8b(o*d~-;<1>b zY`GmgNN7*w1Ues1Z;8;GDxXM6AwU_R9T1yMVXkKg!4(dj2ua>WIE_Y`%f4$}@hd@QIb19oNRzsU|4gX6!!lg2 z4v#Ry-UvVs0zqp>V%6nHGih79Q9P`*d{GY~&A5&8ZbE%Q%aIM+3I!at*8BXg@{?yL zfm#x6>`P`W>4w9XRbLd-0! z4;Kru#8Abo?VPWXgj0H|IDo`^N|~S|sHYln+iC!=vadoYDQJAf>OhQZI`C-a)E01q zKo>*@j*G<20&)KX z`s7>$p)Ot@)C1M2Ye#Hw(*4RM!p%bUF^N-m2ypT8_#V#-bwhf(nkXmY+Z%wm?Rr~# zNO81x6Xvpg$cBi4a>Lp~Xy#@-b7iHxpCEr#@fx90nTK-)lOL0g3@PX+i?Z0_PC;&k zR8cq_KUkJ)t#M4oV&|PQ6UPQ!bj;q6Vs5IQM&%q%&~ag|3Q11k+8-*B6- z!;|I>&1kP6CN76C6hSAJ8758hJ>s!bif=)Cy=>GLhff7#%n1C=!qtekm4+q8^eWc6 z)BGbF3wq{xF>N~OJo`|>rKiSY#TkI^0hu5O+Od^{nTD9^L{80%i>4o)&g_&%+fnxD`RtU`)K;e;= z-c?%wEa>STPsehd)mRiZnZ++|40rEXfTb$87YcRQt`LzcaqMi;d1Md?Fc8^i1IHtosN^0WWSU^FGWmI4n*v`@M?nn861{7I(TAn^P<9Z zc?j!nD99Be5pr3m;rm7jrVEnfO?O*aShj8QvHXW1(+?R@RsyKWgwa{7t@YeK~RO2aJ};$kcD4H9`wuu0`po$M_rHyrIVeLqsvJ`ykd4X2jhZm ziws07l9m4a+Z70g!GtD=C4^|jojrX{RF&{7K@i6D_-O&mU{pp37;h+L!eB*?C1qmN zIXi&5W!>_2$ux?M^Tzc{uyzMH_z z8!U5RfC8ed*ooBc6Y0E9zL#bsEJQmzO$M3il*aWN{` z{o-m|uJ!deQ<$5<70aj=jcA7l48Fa9YUOGpaL^+COSQ5g0GR_Ww!jL5HrIh z9?{usch{mfeRYfao|rSp;?Ef|`&W~}mEK(kSLTuW~WVErtVx@YAWDVr#r34xg` zn*Z2G5be`ripUvFC0#jtR}GhW7P@Q8VjSdJWtwtpE~XbW$?>d`rh2+?lN#}ay}%eQ z>56OdoKo&4?uCSv*t;l!;h)yTfw)ANJ3-`XRNrYXY_Jcg`GD{o%;lDS0T?@U#VL+H zF92jjRF#bx4^);2Rdf7_rT23wsWfdv*pd-S$-#(q{stDv1VY{`B^X^6vi|O$f8e8%q@^|(!a#{o5BkCUED~~t zNY{%AHf0sGQbEPzL}j%orI$!$yMW5H9_I>oUUdDa(+ER*k+96xiao)Qzga@OY^BX2 zJ}FBof=ue<*S<$duVS;~7fN^#*!ytjS*i_y(M9xvxEr)PFTr@G#NJ9|F4G+bLjSoP z&Q0Kb8gOe)wc}dWjD0%!%@pP`Nq6qts{Tiy>%j6)i(RalAGx$jwG> zKwG0r*+&GL%iAk>&_9;bNY;r?|54fvL$!c&>BT(-(9IUzjQRTjXlZGn5pN5E-?Ivx zCE=ca5ZcbAN;6(o!rV{@CtA&`1K@5TD@0E>ZRjy*LbTx}sSUTI%%_tg7O@P<1P~d3 zg~yfCq_OU9xDhYNMvuz_Knf$8G+ zWmfJSdfQvmkl%khBE!>3DYiN3XG(l!QmQI!#IYMOEIf+21 z=)3WE6?RTN)|ag4VDeYXIy-CjI$F9(PM;GitsIfmn`FAVx_%g62T)#Ug!dL^K^I%U zsysflxpUy?)GzR%m{pNmRv05~dN+m%yvOa;VEj#>hm%EQgeMLTE&yDEdOPB~0#Ki> zzWVC%>qGe4tj+Wrju(_R$ysydoO0Kp&OtB@Ss_K+VWhPIu;kX__3x9)WI;D6tI=;B z0Yn2DO#%x@vG!d7Ccegp(e<$_;r@2B4|=x#Z@x5+`r={2O(a=1;HWbkX9#mU8%=J4 zaC6KEM`lh{x~e|qNb;CGJnz&?XiQpnrodboO-T~c@kbR{a*)AxFjhGlj9ze4M#Nf{ z=Pfhv96r7nu`@#jjFW0a4Sh|BJA}RWKy3YkB2BEYTHL2Jq~jV5L2u5nq|CO+K5h6( zN|N72s2k1o+YlBi$m)+CS7b$X0Q05DcN0)fDcUhOM>wx5NG3qZnEDA-;9l!yg4l9x z@nng$VZWV(>}P~|I63K|;cUj+C4>=8IxjDCe346r7jo6SQji9nz?KN2T`_h7Kpr5h zxrirurVzGQH02#gzSR?50(uT$^E(uW3vdZ^lX9=9o>XM%#@e|n?mW53(owBSyT08i zMHWo2Ik_KmY60+?8kgmd{-`L^nv7&)$$AIL_>&AKZIbMHUcIFCmVc zm>l+C@9JEb^XU+(yZo6y2J)P19qvI{bgcjng6{kXC(m|uhd>X4?B6ha>4`VXj5;fk z5k5NhbTUE-g%Os`d}rJt#P#VeL)JrteHEBb#JG$kxr+4=ChGhNzgRpS8*Yu`lytuk ziFkyGhsp$*`BGSKA5a!O3)G*Xb-E*lD$oC{C=K(q*RUy0kXJsr0XG!T2|a9rU7UD4 zRH-A$3o7njbN`|1?5Z(qfr&SZlTeo=qhv~uvNK0G3W_}Cx2s%T$%twKG+0AF!S&X^ zl`GFgtio}1S@LWU&&1BkXiwa)v|HAP*;hI>$MXU`Y8Y~HW53}!0H%{{=yl7pry#c; zyL825ti+n<7L`#IGeR*D*ZW10vB5ftYN-+RUltf9Rhnc3;uB`VQy4*0R)pP%8s%_LAE8^H{uJI z7C5gu6E;3wBUX)@Z_qxAm`NI*U)c?Fe3pZBT5%KXHo&zg&!qiWPV8@7%JO{mGA|8CXUAmOx~Fh<%GS}5*h_!=6X}`8 zj!JZu5KaX6iZv$P2l#uj{dcbI(%P7M1MQM*s zU0kB1sjPr9LtQAiv?yFF+$~~^kocQ%T?ujL!@UB29+wxnn7*wlxY6%`+(?YC(PqWl zLg?sB;@n?!Me*xE+S238zXvhR$t>NA2)VmZ6LQkbd=&k;FfTll2q^N7l9}~%bRCMg zvZ#uKJPRYfDD_xdsHbap9pSg*D$?#*b&RVY)Tn0)avOL$Q2I_CC=%*6pn#W&Q^S}G zb8$j^foA-ytgi!KEX}xc1-WTFFGq~_IA6#(yFtC=HF#lJl7n@MAN6ZN=2qVqh`XC{ zS(%ejVspUuuB?&A0eAzILw0;x=0ow0y)=e@*A#8(uGHj{-Xp|I6LyRYY*T z2>aq5L0Q_UR9a*0Ye};$B;h}J8*t`9OfdaK*FysH^2mCH z(*{;)yssGdRrkJZp_n%Sd#1f7w0QM>-0%U|ELx22hq@RL}Y9*J9Q4WJ&bA*i{pV3L3WXY z)_Wz=ULzn!HP(25Qh7c!kj@dE+hVf7+?Xz|ve-jsmynE75C>wl2gxN~LEx;8#yd*7 z{~TM9@wr)nTwi+(czhaiOl5`Pm@*NSF3^)Rp&kPK-*MY_K9j1O=m~h7G$&$xZRz5% ze4|Aq=<}Q{sI=Wx~cv+5! z%L?LfmmqhGhvJ8-DQ;rScmgy}T12i^?Gi1*GGFl^EBu}ia}dH6Lur2?U$PSTQD9PiWv>}eS%yyF7h-9=DlB-TZ*ckb11l+q8};qiWwyd#q+{o2hRAi1l{Nvl5WRn zKaXW>qcp8V6s|Ype5L#`PFcJJK&2}_3uZ!bH&2YWc=S0Cw-Unu4jqJ?ejbRgRQRFNuBfNhj($N$`tQF1NXti)VOkvf7D+df z#eywo`+pUcXupj^mCn3G_>?3_w7gSbc$-R0hQEaQV|p9Kp|EF_pxg<#d0{jYmw9@TUR>pM&*Xi+~5LU2* z^rqB(|0V$gP%YG$?>$mb!>W=v@O_eQ2<9-%^H}Z!5NL||H!7ZHtfMqJ}rO^drD@W$d_ENkU) zA_nO%LLf}Lj6lD;kP}l|=1h$%l`sv{t#N!l5l|$K&P9qG$RylKKPN!TWFOHW;&Y{# zC8%;m;e8h+nWs9k1ZQxnn74Qd@_4haG!y8?x8om>Oi0G_df2K1+~*H)nKGgLP^H>4 ziXlfxD8!Sj&*JxA0&{IS9_{H% z7R7q%?YO$0E5OB&+A7Ffl{jvxqL^YnFfYfJaWMjUYK^SL^2-8vMANAh1HvSje@pl{ zrD{YB=3gYxT`2UEXkjMg<(N%xxeGU#8CF+0+1TVe$L z??=ev3{bX+aK3c0iB!6mz)WUj79O)Zutapt)Y})Amz5z=V57*AwQ3aQcpkbPd1PlH z9*5+i62BIZR|x`zNZ+#Paz!bQh&sTXAjGBcxo5tfEx;r~1}2cGBq@A!mIr2iX{~hi zckyo5NlLrZysRoimPWkx)uJee8VpRCnhlPEkOsf+gB_KbFSxV3MwwRa#@($ zYq{r04fT~(fZd5I_msM{N{~Ozs} zOf6y^Lh1fkV0AENwj9ejGak7{QAdT4zVXPKqR63-o}i?M3Svn+b*2#4&|MuQ z_Q$(93-*07&tXmaFxN4_g*7vD+Se(O*^ZlncCm%|4X1hN=Q_6`&VyJ*fP)Z4NF?Z{uTMOWVr7gIpkFVM(84JCWn)5zmB5|wWT#7 zNb}nmMmQ};Z=ppE|GPL!PAlf*_|q>9j51aH(?F&fMs`-*}{$afmRWO11x2t&3 zK*v77e4lkSZ^qVPZHpaN;gH)eg&RvUkZ1{Ktq`L>t1oO?Vxs=n7?Zx1{<+*=+b>BkcIi}3AhrELDin*%>bGBv5QJ7l8gcw%*Wibo7ir?+ zm+H;vRjM<&zlpwTAR6n72BZ?;v=S}qBLUC=Yx$*q*g#&abV%PiI<-)Um5vFQuA0g7 zzqk&}Xv4c*SeVUFQ6t=V8`%m`-G;vDf-MUAZXzL*Glf~vWg>?Onx7YY<0>)M>X4}a zB+U9zZ2{1Q4vdfgjRSKbEqah_|AWD)G-$VIyiszFxE{^3;?hdOWUVk6bv<(FqE>8 z;a`ozvYAquMuH#t6FjwaTT>Ac+(-Z`uS`p5FU$FYZ2Suoz@QCR#rD<*%MJQl1P}F% zBL$Kyxj-(=^?OFyyEG9h!Y}UN-AXUp+|=_yAr=jTD93&k&3^={wpk~XTKLL4Q^Dw! zHEZ2G8U%KxR=7gp{JVK0-(g1(96}?w@~1#np_&_TrAF3h51ajHBgtpD5^ zgPJkrvIDrn=L%Hur&gC@ip}r8Clyk$A<{88|4SeZB&>av`HV5VTE6H7#!9&)td!O4kelSc5MU`VaG`4OY55>g z8R&i4Mp5Yp2#cyvT6kuPSf-9W!okRl=$)P$i^r%t(P>FL$vL1Pl?bB&joJ?rViD>d zpQ5mMpH~RT$iz-0w%qB(VnP{%C={kEe-cXmTP%zEM~VO}|FZLmz}2$^#7bR+ail8N zar+9gB5^(s(S@oCnB^d}o=MrXx2s~0O0={L_U#{IjK)EG?5`1KnM2fE4r*n}W$O(wpHY!Q=)g6p z=_16^LKs7SSzHNV%3w*z_vF*0*s^P}Lco5C|X4Kefai zA~G3S?T5?1(NCR_Rz3WP7(?>}XS2y^q%g}kLKW&-KS~IVmCj1)syp>+xn?sz#C;|P z*w&1`;Wq-T_AvZesq2QGnhX>%pH8mol%n4vkcbiNDzm5hkyBEBka5Z?tLd8kHv;h^ zgNiWr&DFd{td4lhF5DhJ1EE|g0|xNLML!3#EEa3Ug`Hjia3zzEQlB}^UO2>E=s(ko zaRJL-OwmIhe$4}*My!%tl2Nk@Sk!8n3S(t6j0yoJE$z$-TI~S}Yo#Y3J02_iWIm7$a!<@EBh5ERQNX^=FMV56yWfH z|42y0sT^v|_R}8$fK$gpec$O(ULxky7E=USx{W*Lf?6HfO0BwbcJByosUvM+fo_p! zBRMw#a*Z==7JM34;fD?22s4`f->DdkBy8;smVh3LA+|7-GTiuQ1X-4`!3KB#N`RHB z3QJtj`=!_m8%iP44gpccGlvW##hw`B=}oX zK(IgyO?Orwho%9BlL?x7J}B6-R#1jwpu*1UPKDB$E{ zk$+5xHGsm2EZ+HF3NU`4g`skgDId80L>zP~({?hL@jO1Q9PD5XL(Tny;$~Z38uVdn zgj$xdLZc%iRZCMyF2~GJDs5g4V)-jcs)b%i%DhL>I%XnpB%Ku8`~C)yl#iXL_t4_yjcK zSTqjY>)bj{_LQ*20p(@zX4J<_%Otk(uu$77tQNd5;Da@~^~xI9LFQQGLBF%%XyVXlk&z*a&1bfyHU81o$E$ zWB|Q^nXS5bO)8Xn*iw$1Kf@zO3k-r54#*w8B*RQIdZ z0XoQ)AI30iKFI&3pgkg(4nW(COZ?|?bX;~4ltZgGh`wQ&4mBgIG&}ibg_eS`6uCvp;=c>FZV>j)fjy>LHyA#ce|8G#csY8; zO7kCe-#TL@vt<8N9G1~auAM6KX}bW7BK@?bsGZ)8B0228VP`^5Ay%o>3L{V+Q+HVP zhp|u0eQ68xhzLnpk??DU7^wgQsV2MvWVvEK9*(H{3A@3_RblB^NB}**`)Vqj7QwYz z*h^5vhs;&z(sks#f9#QE!yv1@Cx{-sq6mfjy?ZQXDtuhN8qX^NEcv1`bfcR6f9v&B z)KxiH1VV0v+yOP8RoMDQmyy>RW8sy~1MEVDCO!5SbA&)I#@AT1Vg^X<`N47gNVIff zp~o2#dqpG9T_@UWbCW;`YQ;qi?0fZQ46!F7*?w99od#PSSR9hdpX1Q}Y_B244iPMjIx- zy0$^9A_28p;6}l0?o=sk?NdwhVP6wQN|jVg8`0{OM*XJ+T0t@YPfzHV2(bvmL`6Es zWx^~=Y%Pk-JiZ{@axc52itsNnREeQX9Z~wPAdPownGmAQpHrc-brO{E=71o)Ol7@D z*V3H-KJS*-aHlUlMvs$7nWc*=)hMl)SulRw2jEX3$g z!mYk+=k`klS=x;AEz&^IVNfd<(a|%AYa0YcJivI#X&is~M?l6rC@WF=-w*?nP^NRc z*2nm;OvnT_i%*MUgiAZHZQ$P*LM&k?za;4Ie3D9u#;FY>>`?;>l@kPfKlq5flMzN6 zmRF*8Sdng?J*c1ZodI+L7fa00@0N_CDyJhI|5qR*E(^Hoe)pZ@U>Zwm=>S=gB=w;TJy)KLT=^Ow!c3~WK&uiuS9T6jC1YE~hOvK{4c`(&IuTuo zK6}RJmIsL;RV+3C_Y2-F!C(oi+TU?3g%n8Ssqps;u>wtE=P)%Ejs9i&0C8#kTJORT zUjz`l?tQBAo4+(48eGHfj;sC-KtHI7sF}yonynx64`3sOMn;M6WrCuD!-l44sTOD~ zrF*o)dPs#|5qm{JN@u~8(Z3?l8Z5JCyK7Iq#U9}%c3ds7Bm1AcThEM+lW{@Tx$^u2 zUjrLqFmk~WR(~=^&~9nke*6S}j74<}u*^4plM19`o%H5!35gmMd&`g>O8o=>0;7qd z3Xs*+ReH6uvt5XNB@_HDCxI+a#bs4_j5k5~y86GsMv{tr?%B{53?D!X&Y4Rj5sn|o zpHYeaD_F9oL~Dea$D;D;3Cc-;tN^eKD>gf0KUy?b>-bF!Fgpgvi z((jM`p?0g(D)O)09WC@>4H=E*zajwJF&L01Pgf-H0Vz1oRa(Z5I=xzc(Egy3cqk2k zzO8TAJpN!hZ$?apDD=NHqSG_LQk=sMOsq)?iiUa}58?PlST;(uOL;|oSSZUL0kV>? zek~yminaoo{jx=jZE^;}5A11T8=;szP0>aDqWVf1O-PTEkcdW{{+!rbxPeSPLBJ?B z?WB~h?lyU88SXg!iOH_8Q{uP6jR(ak6;u5iQT!D_SfKhV{a!&<*^KUGL05(O&{IH- zEF5g0Bg^$8W5}9gzW1l%i+rGp!SbNLO|KeBw4{d(Jo~7V+i>#84!l}AG+t!JaOK*= zok5Y*`droTO8>=aDK9x?*!4$WPIdhBbkK-frXYecIQ)Mgxkg`%5hC*$b!)-o0ZP;m z+6q!Y|EkX7{y4(GH)z5!|02+ubv9?de3{-YE_SIR#v{S57G`nj=#Tb3{W%IIAy=&J z&ah~h@MB1%(zn-plzdPdU_2-C!qIQKg+jkc?=Y>M31q-L=S%=AuvAHC=Hc|gX8}TL zNWx^&->+ArCL)0}LK2VLpKVd8cV-Qk!%yd=yfNM*2?o{Rw(~(-Mi`_Amgd6!8~$Z| zh-#e@Pt70ix5tnUf@m2qKoM!!po!!bVU}llUq(~9690?d5n3k=NN1km{DZ}SGprGcZLZGR!J)~T|aLLy~2v(mjM zuoaSxLMSoWW#0-5=bY8f0v7cu6hR8LAToVB4xhtTzHsL9dohNw4K3BcpslcKO~VlX z=L)kD(( zz7R63HXuD)Sf4p55Mr}8CrlgYuKQdGMA_NC!8HOtTd$Vjg?+-!PW63Lh)lmB1YUd< z2%N}O=A3&jxtcdP9b1-kfO&qo-i?=}2G1^uZvFX2xHW9uw?>X6F>qgI_+Y$?U|-2) zHri)zun|}5dvwdituc6vbEp*Hq~ayV=+>A3>Ks#gKai+jH?z-eEAN0J_2Vf*+B#WQLb0KU0(zp0R}+e>p}KuxMD) zfvdoQKJg=bT5y)}B5PXb2((OcuUC<_wswml-WA;6D##+F_d;Sr^PlIR78*5KwxhDF zKIE2EY|IM7WgX86vOcMZ**{wA+W;VT{uEh0%lDPHn+J|cgniKdR{~m!Sz0zF7~v}# z2}oL6=`~v!#X|6=6@Bg-vtA^CMh(sRb?2uK7#us1X-}p9nSxoPWhaZyrdY@)q69Q( zmxN8SPYW^9a62Rlj9h=pC?JcT^AMq7`X0i}+t}>tpys(ukP)U7Ll_J@bA%WHQC72y zma!f!<(;5Ln{gQVsrLN@SU^?*SuFA!g;+q{mxw%}u5o7+Q2V`@DwX?x+=Vye7|I?gl2mR2maZc9>9W+sA-G{|eeO1~L4Xdbgkq>fr>DnofWt#$Y*B>q!B!ib0Dn z4Jw0dE09i6*2ziZE5&Jnv0lXO&>W`})It*k1uPuKNZEvJS#3M>Ui=t&nO8D%A*!Fo zSUA`qecN02rQ*T_3@y~3dOrwFPithXFQ;XvJTREBBJ5wwPa$-HFtzVE7R2%xtS+Nn z{KYsRD`2TCL4{TqK+CvnADtxP-s{-r(}x%wqEg|!yQ4K z^p!#^=HlvH77>;RFy_yf9V5oL3f=d%Eea~;w|fct72?yVquolfO?15=OK$39tU!(D z)riHttGWaZgKM290$SG5aj*j>n_Y^AnpfBm8B~+D;5DLa^*CnLt-WDJw4AkJ+DNMy zbw&mOUtMo2lyL}yK8TeS{xqd-nWozySNZ+|K~^(nMu@@Vq1aoFELtIvJ|QToQY6wU zmK9@9=ubGR$T{&iE(!1&Vz@6i6+cW>BBLJul5#L3g4ReY%*J1?P;#QpqG2uz4PK2f z=&r-LC{2oe_X_+`X;zK!$07k%Dwvu1=_LPH6%bWXiGgug<9m~MrH3oWVqcEmHkmhM zt-?IYzrLE6mhz)9m(;mpY6{CM(^m1a9K$Tn?9yS`JnXD*N!^K%f&pNIAht^c&tcW!r;hUe1p&-4yOIiZW-d!R0r7CA@OE%Jv1 zSz3shl$^;$hnXpFGFiub!k5efB3_LwGzGt^-rm*Gun*T?(ZGAu_vzLIzj_XU5dewO zZf5YCVh}+SaYn-85rLKi77%m;OQm5Zc{AiiH#AFC6Ff&8FoC@z^x3VTaj5*YBP&2r z;<6H2sC_4`(y5|4wJX6e<6K?ZEJ=WN|2O5xQr93Hl$Aj*ddkCmh=y1en6zn0Gp+B& zu(T)AT59Lh6@o3VVK_$4e_D{0wW>6_dhdcqVs8l08eud4E~?3I1&3G+lKO+B=uc%C4m0Ed)yIg<=n z1l005sgP~9SY`50$fsen^1?hBFj^?g#vv9SGy#{9dB&&MFEb&@I(h!|CsT=6aK=f5 z9?}YuNlm1fje;x{1SeSl9p`t&-k8M(+NLc`MTYrAscZbRF~m8PFtOV$$oPP+pSCLA z`bwaYB`n->H}X>;MuRYgvE!uYJ`GCW%&}ya9{jn_@M=uK08!4>9@_CqHUCK;@E2CA zu*v-^UabHKC~!GLLH)B}#tGTl*^O@X#Rw|NMu?G#2v&y|p%N3>)O8xcnW*EbOZI8KMm_=4D@vYcTj3c8{ z;p(DN|MM8)6bPqeCi`iEj7!mrRB;Kg06Lo@cx$YGMsE?F^D*y|>(|C{$O|eYze$jl zPU62QScX3b2t{=>488t(y;{*Y(X7k4{7ymf#H!`U<}*uxEk~rsHsZIQ2Ve+g<+{?} zxs(^n5QjvN?fiBj=D!FDr^uhS41f|$Wppm^mn=_x9*@>F-*1ZmA~&8ZUzv&$73E}1 zPt*vp7C{lQI6zMWyZGa)z^xEeE*dZFtHux}@wL=okgXq&&-{%Oiyi`mUi{|+%K~@h`LUe}zF@Z*& z3UrX;{fN~77QMC~ha_)b17^`{3nX-Yd}GQN`UrXj`l5Bh@I`~+bcjZ(9YQQ}b{*rz zZvwEgvuy#HvzW=bzaU0Y9Ld_57Fn}5rvh`Rj4}T)0hSAH4MuI?-;0rOS|{88XFGl@ z_hX?et*gS9?@VD@n$UH1zB0z>w^6-v7G;L8mYzhKFfBsY3A98RIPpCy6-N6O46zd8(RI_dN*q1R7tJ#Cjl0>cCymuuY3i>vagG~;m|xkURcBi zk@7`a;j?}XY)lw7;nQJ3_Huvq>)=)jtc@~GNq-+eqkGuqY&)p>$tyg9SY6GGK*L;r zL1Z8--ZH|7H3MeZ%;9ht-C;mlqSKLvz$F6}*2*`Y@>TmmQDh87>?UKc8mGQ*jIsP& zgfVG0$HxJ=@P<}XEx=M_8Cw@!d=-0{fEbMdu{SL5wj(S^bDiHbh~#whRwd`T8=F%Q zJ3I0?kKjkWm5Mv~l0702BYJiY#^t6m(6spjn3a%42CZ?ix@f3X1$tek->jQ6+62#3 z^mwcjYiQ3<;Hd5Cgf>!dfUHDZT`RRfJ9)CWf>Jm-D{*NI(>rjs5-x@b$^{#3%bCE@ ziwLj?nPkYO##MhzA?jn}}`O8fEFeR33&>c_c~QA1*zg-zOmQ zq~i{fsq(l&=zQv0~LsJAaD(E&m5b z&)+XBZGNRUO9SDReJO>0?x7SUO*EEg{m8>Wk(JNXjY%QpwfbUYz>Ec3z0_6$XcB}h zaUg0$y>|Fu31KZY@R~C`#2Sd#+C^kLyAon?=cl$i$kL`RD z#^?Kog&2uV=0z~_j|jBfu?sMqfL++yA1cqVNb#aT;|v0`O-Y>%J^EY1jRwqG!ge8_ z{Y5Gvbcc~ig%<=yO-%D*RMbL@2N7JeTkiK8^Chqq8fR~>Dw}jDap`fC(W)usskz$;> z1v<)#*e4sIf|6F-xB}e5e()IjTWThtMqjF4Yf5M#suY8Hovih~JSO|&`~yNGuVK?M zarixPVoZK9*Fo=%klX(o{2JldeatxOe=Ed7kJA)95a@RTE%#bxW3QXir-hHvb!6)@!l73OdxC1c21s`J+3AD*nr>rC(A-=TYgGxI0^>%(=3d@`2v* z!J%bFP2H$9Az=uVFJVg@3dZ+^kiBrqB2<$9M39k2Q%R1gXiLYKbWmd`Z4wA}{#s!< zvWvwHnLa!Amm{+k>eusy84VE2*-cjN%LEy%H141?WwJI=U<*l^AF$^?jt%v*36YzG z8JnPz)wSs;sh5NPe488mW5Fr=7oxK)o7n)LL! z{>O0yvIQ-b0A~*c8uiA7ZCIuYygl|?iG>pxzRB~W1lrJ=Q#^^yrh2zTSQbYngr)Lg zOqOxz^Nan@VvNazW?iNJzfoNHVKXDU?m7~e5oJ7^|LAL95)yG3b;qHneE*%(EeD}B z(U#pAywkswbIFYFsTYev^Ivoa?+7uvhPjD0ru%Eo0<=n@Wy#{U^8Kbz#xVBB!5I=R zHR~tyH=dn}LzA0Q&R$fs28olaX=UIl__XvnlYspQf2+b;5$LkjBDi!wnAKRc8eDUMODE`X!eCsyjCD=ke45{jsY|RA!Q>$)0#D-X%(rAjLA6bP^Dv$(zwRS zPG!EoQJ|HbD|E{cEpCakqn$LYvbKh_4F#*G{#N<2bW_Xp>V<9>ZdBp|W)?4h6#*6b z=oF-nTJ`r-Fe5i;CrKFY3WVHP1wOTQ8JN)HoiMA77yJ#tjs%>pR|M?eshA>EH+Upjbb zgVQYntsoPMtF@ZpM+!1(qF2;Z@a{OIv^mwpm+S9~L(*4p?@6AYBFK6R6t)xm8+vEF z8qb(Vjy~3F)HibFlBQD>GxK6l%K>w{a9-s)LCU%jgPUG20f-ngzqC9k*o!W+aL_?E z!F6EqhRdUHMf4mcC^@nPP!w#jF!fbxIah#?B~4b@FH(D@c^C&rU=9i`^N7A6mPw8z zqwZi`1eLW=%Rp`w8%UVa8`dHNOY{mt4rBI z7|i2rU4I}boX4)2u#B)H0;Hb~=My_$YY`)3ilGWhe72CN>9GpH5*J&aas(P-5N=p- zM++V>%cj!Pq%A)VtCM21aKZ-A{7tTAc*CGif&5WDoTik@|H(LZgej>iSp*XnHGwMG zb`0zO>*543a+6QbMJsJ{9bSz%=q{N(`rbn9h%__WaB^K2`D)g5l|EUc#~3yO;)cPg z(6EoTE0j{q?;Y7whKzV-{Fcv45O29xJ6 zzui3Lmj-D(^sM`YMS>yz7}mqqgatoI!8CLUvkW`cz7hT92gA^|kw%R(NTYPrZcPM`d+ zKOSq#F%&j@BE2@=o605K1INWm{I>g2r6F8&;TsW;yyABd{eLMEme{IAAZAoZSXhe0>_!8i~r4%f*saKjJ*vvSKq z3Y(-TPymfY>`!EOSD3rD@@l5}MUIaYr!j!0NitR5r2r_B;Ht{iKxwl~(|xc4){w-sN(P-C@?`|k$;=>CT^eH?;-_ze+IPmWA@ef7FYzx4vNp)lEHwM3 zRXE0f-N)xouSn%JoLQt_FB23sIKpX5QicDyKP1J*nXr4qkvqq%C4m1)|fVv&`0L^TjejRY*}oR`8hvYpv8x> zn#}`wzCsZ7-{-PES6%8W^=4VX?2@)+O^Xq9|H{+@LM#VNFwv4ij1|HxHH<=OT!aJv z8H3sjZ*2XqI5gWeY(wgIrV-jm%^Dn1?-v51`p|&`?eY9dSVRjg-y*~PLl8Y@tDVu( z9aQ?tWXdlpTjZ)-Kc^-I$%HIxXTJAzAnUQ%GJqDuOrFh}32Zb#fnAJ(Iv`fMl3W&i zuu)WLpyg3zWAV7mKdhjJq1{DukNow7Fv~aNFelh@{X#)-5_!{k;f;E=_OGKp43Wqs z{-s&wNfQtT<~8-a(%+bstZF~|LEfYK#fK>b*9f%qkqn>&iR}_bVkl*>tL^{x5U*A` z`gjc0`5(?n1=2>)Ad~*?Q83Fi3|n4Cs|=9kQunAASB33x0ksq{^~{N`oP6^c1*XJ0 z=-j0E%$o`aAvog_W8v-|HJA6IbeSU2&xU^R{n%RtAZsNYs}yF1=fD~Ls4PBZO-05jDP87dkmVN5gf6(jAdnWhY1)GD-zs=G#Kcjd zR#0W(c&w~PiL^8s4%#x7+P2NE{5GEEEW`>@Wa~(*_E^EXKmY zJ-R~wLTe1cNZrD3g;_1-6tXg-4hh=-S#ZQUFcke>+Okw;U=}!Amy>@k(7b8a4gzh- zau6c}Cknz;N(UtUrd8ndqrqTaNSgXOsM(t-Px(vyYixc$Z$S23dRL3N@8Vg2*;-qc{tD=#v;qOAR&O+D3%h zom44aE8p53hhKI00X3?YvvsIKH>tG3(Ans2@-oAWD>=xovzwt2xAH1sk{w!D5>U2m ziPC1iG^b*$lveml^u-cGI$`rHJ=tY(A;!w=x+Z=V{(7OtBsrRLXzQ3C8iOiqb-6_M z`vn>?8G1@8eA^uWRs~YMF?Q4mF)wHYiYxt|T>w_2x#64<@?}sL&UlRvmTUoM45U*o z_5rk9m*g@N5c{<9NNLa+J6yxlQ)`D{fpU%?#e@V&Odr;YGni~8!kSK$+IBtic zMyACfC$qf(MXqlaWTYoW&Xvd(&;)-%uw|OHYi-Lc_E*0L2ES=Kn?Jt9H|X86#;S+R zR{69)0~zg@?qKxtQUUl8Ltrsao#WGjEU#3bt`ivee;`C0beD|%#nHR$aO>d4&Vzin z%uFhaw$PM+02Y;xqg?FcgADHrjJhBly5TreB*DhMbvLPwCl#4*%RhIFaPBX=bp$NI ztFtMu3M)4uW~DH4F}(_?WejUsg>1yi_Xm`eVeH!HU~ztq|GzkSZ5)tJY@9zB zd(6=#&Qj}rAk1hpwz`ZNoBuFII5Jz#g!;MQe7@}{HC49~vT5sO(f2Zm>A=ZBwySib01!@Z{v zXw2W@pwNdKtPHY|5r5j|O)Fy@#rlOj!;RQT1%UPLjYFd?WH<0+gCdHT73m~PD^DCT z=6YTZH!7TYD#_O^lYVv_Qw7s))#uU};@F5Lz5WA1Rxw<*P=IO+6X&0U!R!b!sN72Y zYQ0i^pM8INJ~S%HD}% zDnhlt@;L9-%ta#d%g|iyKl~clh|HVoipw^6Zxf3^Xpd2oQpy7itl{RUx9fck= zfY#?f)*n(eT9edM%gFrk?<|Oww^hjV(i<6*TZiesvC0stbi$tf<2zIdMQjm}QC~ zC!B5iP>@v->K17p{v*W7JdU;90^j$WR4ArOTE+F(3bK6Y_}OH}pc0>#epgFAQ1^vP zD=1xsP$M|H2BvK-yJy0Os1axz6FWW0Vk;CaA{z}kGRw|iMPh}4O;|8uNcn{@%Y)|p zToa65>H&d9x3DFLnUpLw-`57%3XbJl858j11X=Pj(H->-g+i^4V%;@$m&gD&GD(Tc z$)tV}0+UiGb^cFe;>R*1?PnN-{E&8lE&V?O&52Warz90{1yhnRk7(P`FbzHk4xE$2 z(r1iSB8Y^A*?>j0Pk$zLW3_p-qDd#Hg@Sf**akREy2_pPmk4BQtGsod4!*-lG~SM`+fTcCOjD+rCgblgOB@&Rn&Bo3Ow z@H2bjBPu1LC?evX?-)W~!a%G(G&zot#ZBX(HL*7gcE97(8Ot8a zDqNZG6UPR3Ndr>qhYPdV^4O%Jj%>6b8!{%9jl&8<9882Rl;W}#sO>%Y z#ZpE9t!B7w8a_qM7_g|JqXyF+UrO8xtzydp#|71owF=MBRYm<5`;U~mQH%5R?6UG7 z$NAAHFfPn*|0>Xu)>#7VUu65wVhjmfc3UuKGCCC|l@zgkOo?iMJ6LX zm;^5OZSPJ6ga+tb{2(C~P?KC$!eK$sGZH@hE%(`b;aKzz%sXe@&pY&`Mwlhoj&}cX z1R|E9V$#Msvali7d;&7-Vf^Q`jo&EHXesHCH6*cNKrOfQ7@T`&&~Jq?^ONp2!Ynkh zEd#lBk(p1ca-<4o8Vq%QlM=F`m6fv%ni_jaP{g%WS~xSrJh6QM2`=C7jy*AB)&=r! z3bC0bi6TY86k#gt&&lJ9WgDy06IjXTq8nkx_~X^u{V)NRDyRCTrN3K9lq&OM?P|{z zWCeq;lDgGRY!qtole=KW$oI+zW2!x&YHC5YpQbnSp{=CM?oi% z3N!qo-azwEfVC@S!(eWSo`O_^#D`%l!NwC~HDa*crr6IB7}XbgkDv(+n0d7I8_Bqj z%418Nm2)`AuMlF@%0^1;YyUcq>%zD7YB=rFbb{9A&oL^LM5qLR#?KM|OUpUgg&JpV$uyofj)lkM$_i?87A zylUQ%IgpK5YvxQ~r2ZNwoZ_d<8#46|m=zy$iWLm7uxyJ*@)!($s8Rw30kW$Q=dq0I z3!>Q6yo-yp2PWSa#uz5jDyukF=BEj>6p;2<*;n8Dpg=2jh0K~`W#pyUpY+`lCn|mB z)Re!n91><^$r1z|=?<$8pJPD#|ZKL7ZKD zDfj8IN9IzDf8<{Z!_z#GQc5o!g0fTRJu5Dq%;gy059WhV7a?+32EE7|y$HWOZ{9XQRo>#j{EI@c%C9!(s);K@Yd z*+l0RiNv}@VoM_NO2U1a$oT0ExBO&`oKtZ`>u{ zB+~zqocB#4?bRjvIem3~TH?_(_exsg^|b%C{3YIQ^?P5M`%7BJ1%IiTb-|};83UNR z4Ei!HeL%&aZ_?TfxU^!>^z@7Y$P9zd`{%BXr?f3Csl2arY*Fs%AMEPrHl{gO>-3jQ z;F|uO=C))wH|tyc{*d1To%@-7e`feY`aQt8=Izcs-OeR{iCc`D(UrjGI#+`m*wrQb z;)dgf;*x#9anIwgf#<-x(%pcY(%p5Zxa6RMbeE_j3yFkFpW|FwgL4@RoNF`JxvWLb zWiE8C-Ad=$E_UwJM(0jh>RgAd&b8m+Tqj&dexJ6-xz4+tJEPgT)8BB;?{)6Xcbq%> zfOBWMU2qvjq3Rw1k`4-X#Yo z)7<}_>XJ8{-A%;__oIGU?&dGD-0%rm?v^20 z?#ExWaknINZOLMbI(%sBa>8_?f^^%|F&L)o!Cfc}$ zz3Hz0Xu5l(opUP)+Y5X)zc1z4&?nu^<#*FQ=SC6dG~AB!T=EuZlguZs4#YXoxfxy4 zT~p^Y*G##l_riZ=x?56{>Gt3^xt=`Wn)zMBx24qSUVb-ylJ1)CPjj~u*9z+GE}r+| z@^A;_w~%<=Z{wz)mT-r89}j+%=Sf_~`4pGu258juY#Y~%zoxbcx0&~)lyyv}71N69ns zHGSR2b>+P${${DG~txrE!wvnTij{Pg8n$M@N)>l2yoRG!JbnXYDcW}ruNn+!L7 z1ZAwsf>_{NGu_O-S#DOZELS_&xrd2kIqhyB@zfv5at+N{Za&x}yR+PE(t3*DpTbMa z$B;fe2@5Be;g$@}bVCU<3O9{3hw~hRYn+(r@_Ftek4<007Z<0ySNXk!Hok}F?BUM6 zNciRO$rhgL2WGli#5;2VJlmB#H8{5ueELf2X0LONi7fXf?{neJ=?fF?b>e7lp8=0# zxEioobqQC$BjKt@a}RV_LtbaY8-vNy4{^<4$>Z<`{-)O?+>AvD(nz@1h;!!3gqt$Hh;!5U_62b-=>^~PB)(AzR|mGdFZ}X6{6e_@ z!T%gE@#6dZUehk&KH=H)3EwYDa|`&kgmTwrIJb!3f8kpr&lQ_!OA1H2n|L<;8~+SS zJNUgvcwM?Xjk44nPIrGJ%&a5n?suZ)Qs~R~Z@^aZzE^%1(v}aTyV*PVb|HDY0H^kR z8Eu!obQZjj90hGhJNFggW>CJ_@WEli9>BdtJT=qsGm-Xm0dcIO{ZC7G?|?5kE!`c3 zo;AG_Zv9E;meZ#+4o-K6!Isb`o#c5Oca(UR^PYqbHC>%s&+q!)&OJ#PmM;J!?j-5Y z;JM^D@8qem#<}B!Ti-6-H9&{jfuv76tfjs*W+mIHuXFAsewyCSa!vHJ8C~Fu2K?QU z>6+WX?-O`d^1Lp~J>9#LYg&=znpbDJnbhU1QJvuTPOcqwI6cwc&EP#r-PO={JWswJ zp^j%Cb?z~KAD|2mk+1raPOgElod~m0;d(kZT=f9Ge!|mtqo2X`#C65>#%17=y(kmz z--PKv9&4)^^LW+_hj$hy+&SP&PLjWV)C=QmSAByX%|B0rmpZr$`97NXSG>{5&Fb&m zpP+ri;!Z9deC}z^{gQ9*fi2|yK7J>+cA{N(a(l`5j5(d$Kfrq`UHGgw&u+Z;<@r6H zHH(O|2lUt^`jGx7q`9#@{cd}=teHC9LmBb=NKGdl1PiFXw5 zLvgdHr!hPyl8>wSz9fUZoS;rtq`Q3Zo4{rdqn?k^zIdOEzw5!o2h(`3!Zodi&t7-# z0A-q0=-fe`H9Kg7(Cq_ZTNCb9!pvQ2{IrNLBYCeUA4~cD^jyYG-ao>x+WsH-{TtjX zxF+0;p~OWTHMr!`gd0d5&7A1m{d}t>Zx4Cr9;6%%(7GPKkC6YdeA|YbO&ZJTvzHA^ zxP0=yjyx?H&DgLKp2XG2QI;gGabm(1;paKUI|n+F&QRPE=skw#C|o|SrUd`E^%?LU zY4!!Hqa4MATS0mY_&o>L*bX|bBj2>;ByL7C?G1brZU?SuJN*RXbOZfCQy1|v^1&YJ zly=iJ09U~8A!%;LaQa^S&)QGj^(Jk~SOXtTC#`bQ+{?FS+yTW2Pu4?^`a_mS$rw$w z#B*E1Eg72bYWp*W+@t2Kjpo~oyC0sSXC*lr~e}(@u z&*T{V=EFA$@{rAMcrQ83xjy(i4_du|AIT;c^ZRn#XmI+?4I{}A8O{495f za7|m%-HK6J?g+n2_;!qEg1)*P?u34WEoo%lz;kxbOqZqSaOYN%ujwVw1UX=O0{H|w zEe4y#bE}@GAs;j{-tfEWHu8gvS3|ue-+-1p)&D0?Qg?e3ZXteVGhQ_Gb*_T=1CK%b znZ2PGSnV|CBIIX!9bx&M)Yw9QaV}Ws{|`A=O}Gber{L;4Ku_vqy>QE^n`8&lg9m2yfu6&tvvX~1?$7s* zgqw^1#u3o{UHJIxgj+G#xxsvEY0ouB)7=yJS%jNS-j2dIOZZ*0hxSifU&-(3{2oS_ z+BebzkIW>k(Y%YsNzxd>?^)T*sdyIRJo#2%-Mrf51C7%ih^I5|4BTw!wu$G-?Wu_#x1X7zKH+n z`Ml##Y5ZMreb&V_|C{k2Y)zwc`w2Uj-%YPDuD^}^xdVDVn&tivyoUO3I;X2^z97wM z-tLUeK7|D{)JX;{Q7QBMWWieK)SA9sJKT;BTk774!jTk}sv9 z{Ggk5)zq8-z<}TuW)~4=^O?+R7gHAJc z@LtF7hVJed@k!2lnld-={RHnCQ!}XxvXV@8pngs=zvuUo{oUP?>TYfcb=RKnjeWbj z<=Nfba?)E&_*(of=eY#@`=r}Q{IwIodUSW|8F1H=<^sMSqCBS){w3A-M(TTA7k3-@ zddhPL&u75TrTnMidf|HDl3OW1Zh8;sh|E?CUHX8}Y@jW{6ZN#=h9e2LpST}73Ed9U z&y$~a_<7_2VW9b3`j8s@Cgm3M8+U@V{)-b0PNR$s3li?>8oo6X5GP$r^B1&>!{{jZ zUUQUoyHx!m?eiYy;a$;L5Pro8=X&t{yZpX~GUVWT@hmEn;v}YuQ)~!{wu=2 zop8OuBtMM+zk=U`d0s1waHIJ>3^(f-^89|B%RV6Rs%_ zJpt{0Hhv2g_8oKqbNPNC)6G4Of7iw}e@4HMPkEr#j|ejne^ZHXsKTEEE&EgFVAZ&$ zdx)#2bC*z`8h<78FX^A|`tW`wZZUqAk0xGp1ogC~B_q<^O0Y4+HKU}1TfWG-WsTrG z*LT1lmX5^s}#M?j{sL9B5!}tyRCP(86 zaWklgWDmYk{-kKY@BZKoq67J=VSFtk&6<8^^fvzS}&2Y(0G75X>ViMYUuV46F`_j+hjGn~BOMiBN0 z*quD*_Rn%xj7EMa$#N%%yD3ROb77iWhrBb8IBN*g zgZx+XtR`smxX)sV)VAu$1>(;&9mYC6`8KC z5Zx_U9~?iEO<$zDES`OF?~vB|1;|4M;`;KtAMQHbHMmP~Rrp;-etYtKlKgk= zfzD`Dy1RjI=knX*vM+!Z#m>#o@8BMDY3>>7dC}5@d#Ztc8u#Qv+9A)T3mA9Xkhkl2 z-oq23HKU7d+Mea=lu&{>X$SPg#J7zwbs5f0*y2fJYxc7?2l{uL&9^IE7f##GwV9K1+(FahK}xuA&e=|VdWM_<&{f0ND7Ddzx{?k&lFfTFV{akPiMVQ9Ju?1M;t2Dz?a05(3w}&| zGm!%p@jXc!t~m*v6)*X$8Ae`d^Ud!=1D;FqvlxfymTYGHoln0qhxnSwXH$D zF@yeN-co)y631TRSj2NBV+4M0Cq30w@(tW3@ET|WKBKS=dN}HNBY9Aq1JIq|cV40m z{Rg_Huc$xDs(j67Onrwu?8PBoGG0TcE~F2?Nr$$Q@S=4R`A+k+<^p(VK5=wzhd%Wz z*K}7$=K1i&nH}ioPk}$sV=*ymI(RmY_kO3k=C9fY{u)Idk|P*XafdD-Uys5+)YTDw zH|-;y&ydN5J8;r$W?iWnokH_S;x8cWV&qG{XVcc^FQjjI3dh_hJh{B zZ|FNh-$(Ji#L}NPlzPXn+E#zQ&Bb46f5f{m3O^KlaTI>I!XJTt$KYM!TPa-m$Bp9K zKz%RaJ2Vxq%vhRm9h85{tMpIfcLuHp{+eEg9`C2Q)sz|WJ{jb^{;Xl~n<-ZE9o&B5 zltJY=#P6Ppe-H0_q5mt4^~ga<3c~y zgS7Tu>lRS9#3B__c1vN$QQY)0!h` zZUk|U#w9Zby1Hr13&?M8=%+QmUJPM1jOj_n2+?nPGf!wVU1JP^tPk~193lVE zZ#w0YPB47$#qWvGm$t0_K;h@$SADv8tAXEhaT7^5tkHZpkaE!9d_tJ^#LpNd{l;{7 zWyaxaT|4}qhC3J6hcMqJk5xPe>YaQubxvLkmZY3DgRgbdcrWC84bOa@$)4c7u66U~ zLWgP2eFwkuY5z_8s7v%dCHR>)ioA(W$=@dGbQ+laG!jSX_ZM5<#mBYO-zIR?;|`wn zlzsX;gx!yyW*mH20}a$(_wao^ZYyp#Zr*zMpE%y&cW7rm^4}Z(1JS=Jzv2_()A%=| zhTm@wbbT+N{i5s97*>Z|KNo+OQjUc@8_EWR|8p z^3s&%hD7l-wI@GqX{)CZ*E4)yN4SmjKdj1fz4{H@ z6kHq1HQG6L(vTnM_FBT-%`9!c3mahh(@Mo(~4Ll#JCjDuwtyi{jGvT$J&okes0_*DBdxY6`3-!)i zVavbakrl)R9k+vPZEBWez6SF;LHrT@9#cF~emCg1aN^s{?^(y$m+&rnEyh!(A<6xe`43DN_u~GH zo8K3D_CZ%&kmlHql-y6;^C|cL@|*VT=F=W->x|CsEavfNyW}5uDvqo9y=h;%+sNEm zaqPb<&D{;Au=__62kH0W{a__}jn^r=hc-M9Ceb^B*E~yIf0pK+e;vK`J$%RiLiw*v zlm5dk;`j4kr@4b)p|@#DcMINT4H9g_!F0EbFw4oqWBZ)T1KYXVxgwrV@GR$9jZ5Ox zewwyl>vkfrKg9d?Dec`3esBA#y?c3Nd$)z>W8j-{FKub>w(;D=MwAzyMMv>$d$*YP z_1C4lHSqmf+W3|X)-P{jod|p_Zp9Gj3Ju=l_i~=kL&K%@@Jiow_tL<0_Za!y#B<>o z=K8~#^V7zjDntjs^BLT)>Jsjk%{a=aRZT8i!X-hxGw|?$d z3)oKtUoFD#Gsn@ntWUT#Q_!=K?oQmof%M-zpN4myYDai}{}FfarZmU?Ik)>NJG@aNBUZaj)Z+;*uMU zZw|GEzU1o|7<;zd9P0WwzYjTh9Uk~#erKoo#}f_65NFblOrei;ZQPANVEqbrEADpO z9k@l%>hswJ?wQ32cQ^0%;+|VbIe3o6jmMSZmN(F5cuvBt9F}lXd9GgR+%I@e$IZlT z=t&#q`7rJ=+)K>&Hupz=)FO}p)ixUr>^NYlT`_4$rIk+@2`-{61kNo?uiHGlcB(ZMq*=~KJ zz`Z!92pw#Z+c>PqZCX&~IuiH62?=*P&$Dpn;kx0Hv?J-0Li^gaya~5?OqP3T6i?{6VK;kSiR(MK&0RCy z%{*_#Z35p`nCV_7U$^sq2X5y|;>B&p?Z7P|?z_S7#pU3};+Bnu2H@lQ{X&AW_0Dvq z{4U3>J(}g#;2LqOaVtn;W$%PrMLT+7ApHQ(M&8%(TuZy2M3||#U*M+WX6m_vI>04y zTNa^DZ-l>=Cfv*Pt=p^MuW7Uup1XKoM?bOtBy;86j19Prn`oOmXrnw|BHV+-_b~1; z+y?rC$9Znzxf%Bqzh}Z*vydydk++xoqR*!d@4)SxmT?9xIfeVf#?6=-ot%9 zGSmGp&%fYyZamL@#Pd_!-*7Vvy0}^N>0k2uAGmLD|G~BBLBy|V186h2j=0ltXW>?K zWt<+I?pEPmz^%qL4x_zKq`l*6FX-yd!(TVtw{YLVeGhjD?sD9gBlK;@o!fTOxtEVY z|GCf~x0Ake7jE4l^t8AaaT{BD^x0sdH0_{};IFxS6;Iacl9nhBnklxYf89YG@A&;rY3lZUxUw{7k|O3Dp55FAem{l3U*ev_EyHa_IkgG5 zir+8q@8-6l{942B7je4^ySts)-Q5nJ+jnb}3Y3Kc z6*C3}s-XfEsrU1U;C65K)cmo3eCzi+=XvgV&Uv2m_B(I4YTGN}s&KC125usjc_mci zHofksx>Hz{-_xf5g*rjHlM}=-dKbGG(F@fDC|@am9W4%b{3f+myq69M_vjDs2%XcE z-$}l?a-1Qv$c;9iw^;hcEiR6AWIcD{xH76A(XdNCIMHq#4d8uQ= z(>&ItPUz;&a7TG8Cx16qtA7&8KiQH||IkyyPal@|R;xcO>>MMUaVW+_#K!CM`2LgW zb#LW`_#1g)Dt$T{R_ZJ8OQ3#JUYN;mHs+!f^RNJmumqnh|I4^nVGZ`^yH|cAKUDlh zept)B4i)(Yq4zUG!g_j7jWR>_{dip1Om4+?l%X7z*oT&rL&E{`5RTv&(mV7&aDv{o zQvYM=&~S<#Lry)BMKmYct*)tAYHgD=#ib`oYkf-EGSWykB-r9L&xN?(CDDv%?Rc8p zNk;P(?aFfpI*0qd__%(i=g@e?9PjnK&`h?x<^8LVQ%KX>-mw3m9lw4h^nMX9{F!=Q zUdI`n$3u4u z5tGqZF+NNsd-MyZlf8e;H`wUCjepvInaQu_zPbqULUmJtItq#5>MZqPgL-hbd*))W z{C#R@D5cNC0xUxNNOj>@aa4D9q5GkJfia9Mnfp-xW3jq)xjJ>Vx|M9@PR&r~q77Bn zGAt3!GOWNV#P(YMu>ZNRhF)j=Vw`&|eH|Kvo8(UH(tjY=^K0qRzaTdweQQ|QN^VE$ zzIbKC%eyS&F2`WIt!#E5_72tmHRGj!($hXl9a%5k3F%0RZ-cm2`prHZz#$w#d51oE zlW}*l{jU7CTmIWG|B>Ctm47nJog*8C*HkF|-fPP(`OkMxk!iGDQU0q7!ZBf!%6(p!OdPxac>RaTV84um667 zyouXrKRQ(ZOBt{BoDIU=B3xxZOXj#6?+*=4kF9MuZQKB@P3ne>aex&2vvz2>BaC}^ zfJb3?#+o33IrEf>W?xK)9TI3rQg?e(QK5jHG6|EyKbG`4=!$Ub) ziGApOwJ;nYd%kWwC-OH|w5Qv61pOFlgi|Y=_o&0N7D}QguqxSmM$*1)j#-QU7MN{l8bXgwy*{QP@sC zvn$K!<@)u}7}B1P?6c#p#6BEA-#3fGA@T^0VfQz7gcIZ`^nS~_2(qV@?ML?gRDLF7 z>a~hLsI%#5bfQI@vyRN-49+3VFQx50PwzUZ4b#3}q+dpk&5%uKyU?Aou3rA@v%c+F z{>y%2Xt?UW%0I|&-!cA5zk!>$jXUW5a(=i+_I#iHUOg;4pg+PB3?Rm@TG)B7`)=C8 z$X*#nAAtsONlw#7qFx+}__Y{k7)6djdf{_n99fK%^%$M#!bI-L$j;I)m}C54zJ5Xp z`(GHf+Q>Lrr$1AFT-f_Pqe=UVseUsZGcg;3^?#H44s*HNy`K*6Vz8YT=WhDP^{e`} z@sjoz%~!O)uC=-+)uI1UtN$ULRheO-R5eJjBcTpR9#=*Vvtj*aBQ|3zwxbN?s6@MaI^5fNQaMHUCgmSl2hx?*yNTT9P zZ(FdUJ^EAsvYB7@qqjpRS^1S6#^l6{eg@}o9yQ0MU%cX{g^zlA;)3w+3m+HVa~W50 z4etg05Ar5%qy4nAT`iWJglL(dn7g*(FOwa(xk z`2dgb1QkEt5wi7fhXH!tJasU#?eByUWZ$QTgd%bj#$X(Z(euM0VItZ4W#jQ=GUZyWnX0OVQu$_My%2A2f;vJzH z`{;GM%R(Fn=!eiiPa<)+EYu^ae_CdgeO&*f@4pi&%u5_`Z)$>PUw9{UU3%wfJ^Jft zT<#t9@;mwo@8~DI6KZa~6KWs66XN~vggUbR?mHnd%QH&KLIZhBI45ulF~pHXGty{( zzbth0cn-SIeM{Otl*XfFA&17rW%gy4g=Vy%^KUx|JJscO@X!8wSi{g3U=3nPSI zgzAIJft$t4x7)z!{W1^kLJde>&AgA5R zUNZh6%x?a}_-(~0#Bg#*e)#D~?_z8Sr_Sbw8tqlBHY)z1_OV}^b64AQMH_To`>6eF z)Q&bSf6n-tvCs|1_qQ0|A8lNJyzzaqeVX}!S=!+l+TlX&?pWIgkmYW;&;CBh{yxh7=70G5mhj`#e;NMv!gs@wY5#XP zxcb}Shs(bme)RrdhC_?r3de8#Rd{#&{}cYBxSi-SBrO-wwwfza7fo z_)hq1i_Xso`y47qZV7u(VN!GV{%_m6`93H+c9NSb7{w`AxuDX5=H_&4}#Jk3TdY#L0 za<}rzugYS{Q`Uc14|(3cUHe;Vhlf~Zc&M!&9cs+g)FIwKIwU$qhWe(Fp~1XP^5myN zlZ~y7x5kH-566b)A)gMZA%7lPcZ>}cFZs?G+&^(x{!vff@tn%9+N1rg5bn_*p!16F za@}`2?K>emBR}NO?LBAEobX*!zBAcW?YkcLT_5_+^weSB=B?+9o9RcP7D+Uq9?|}a zN1pKn1IYWbHVh+Bgi)v&SrBT+8)KML5b97rtso@4zvRM#(C~Ue7~`ICD8@ug##Btl zOti07|GcICVcU11o2{S0pq-G?{}^tMDVlrKKWN>e{@J1aA=@_aYswE5K|N$s!ffHq zMfy^HNLA;DQhL{gd~Ha6m`7iLT!*lmJY$w;VG+NohxuU%xeT$x`C%2g2Gu?J>aTqD zm%6K_FjW5Pg^+w`T%+GShrUq)k^SHEh37)=m&`?c{WPiTp+Vl-M?2GKxS8}|L+!t z+Wx{2e^?mmHn0n~unTvv7sv+g6@GgJ$8Z9t5JMbEv_EA3_OnsRuGiV$$Sh=kZ(@I= zaWwmPJo}eyxxoIt!nVE6{w3SEt9BHIX5pmKi7aBmtHv36-TQ3VOU40e^1?aphOy$t z?oE*Or`f%Wy|3lo7h0CS7|y#V<+*8Gq*s2RACu2MU2S~8xeeX3#BYxH(R21?)GTNJ zul9_??Bs+nQ^L-82HC)USr}Jw4L5KTw{Zve@bUPs^4x{)x4dIy^#yZR^#86%f3@@r ztL3Eh(^K^Haq0K|tHu|F2f}%T+fi8|VW*;{$f}`uEH9u#X7707ZK(HMLey5;|j{Ga_$``a@# zl(|-pO6&xVmBP=rxP_iO(#jvm=0ov8RAKNNH4PWtW{c@*8K zDbT);e`;gaO?zrnCrY&Mv$XG1hJ~2tl&kM|-BLGUj4&pm@@3_!(fggm{ynR$zF?f2 z?A9;HkXh~=*|?JZi)LZ9pmh%WcM|&-ZJt|od{~$)ys4OunTTby|Cmj$b3Kl^^inkR zh+Drt2DpUs%Dt3fV3C9Xs?xc4^)Byp zAODv7)*5~L>G(i;wf-Q#Lr5+6?H8(-aD@99vL))J3BLUtd0>{c1N=^*LLHgi`ErQS zdtUZTvNuS>^V36;-i)ZdNt2z(;tbB=JgT&*7s=}PtzjguB6cY+TqAFw?yNG5M2E7G zQbzQrqcO(X4F#cQrMAD&b=M->KC(d?3S8%x{r329)9-KN4hGkCN8`;$&98EI^vHu# z*dAo(hsrh?^#P)~C8}eZ7PCRn!ZvB1rtP1k4=_jDKT;onUZcF$if0^kw>-OF97Zea zU?fTgr!@XzsclllS4DS_#_Wwimzc%4x``@=|ov&;z zX8)6IzD4Cv$6McF4pvxw|JfOJWHBa2?q3WG72kO|^!$S|!rj|wJc^9$gXxj`%ktD% zb_#tq=3@6ppVPK_hWmL&4(H9{1p17SBN3z>}RW`OilD0*zEx|IxMi+-_ zhb^t3*A`gb$pm+jY#9AYIN+W`ID%t1fm4Vfj`l+H|6|SnPcZ+FZgx#% z+hh&Y#unY-cx`O2Dyh6~`&zp-{-2|Uqj`ml=HtF5^m6MqJ2j+{=~*Z7rK#= z_UwJ>N8@q%Z@%#-d9HBFCM5-xz`N)=f_4 zHx<)S^`X2>RzGHc(`VD?q7?Bf%BT9Ee)Q8iVsQMAd*>V8JEHtI&wYzf5%TqI^vUU! zKi8l6Dtny10;{kFYq1VB3FSYd{HLPuz5mhP|9J18jLLsB|FPbE8?hN%u^nY7N4tAF zrYU2Sl>ZA)%T(sN@1}g^l(WVi-v4s%f3^2dwl4PmOT7PC-oM}Ve7ZQAo7vg>#p1A! zJb*(uf@3&=Q^-sX1)UBk6`6SQCb1>qj|13bbL z#8zv+FhH+csx8@OEPtUqgTeM|q4w)_?H5_&dhJU02)_>XNFa#@jPRQxjKUZ^9e=lf zsY(BayL~acb-8wawRWED=FVv2v)cCDCU)ytcIyRpEZK6M-Abmo(`4H=ejh#;iiI-~ z6+idg{;~xH#`3S>ShzumsDn0;{kF?FZ#QZDFT= zT^G7HsoRl74vqKaKQup-|K!Wov+`e)G@=bHx9lkq&N{5eMx=WRq`x3+rgtgFo!Cm> zj@%aIcZl*kT=|`l7s~imSuapdRw5?f>?04LI>D~rq+H<;clNXT?en$mGnC^umE*yw?R*!e=plIaW^ zV3&2t^e%QmC(hB&BS+67gKpHkuKbHfT)d*SGW83^YlpsvbT)KE{g1+M(eEzfDk{G7 zLbyiWz)iF(&mGEjC!)1}-O77rwD&*K`$yw@#y`=lu4_RnJ%zOU+O{bFf5rCy2|KLa zIKVfQFJa!nJv_i8Ji!3+7KNQ3Nq^rr@H=ILJ8FBQI}UTT{FD7#%cf+XCfKJ*vVrYAOgLjO4#k*=$(V}iXy0P~ zeTVsXvdeh_-Dla?7ueTiuAluc%qF&UGg@}Df7!h$c6u6ZXkq`(6wYkSMJdwL+=F@a zu1&@kHWY;g^hL-W7WPW>-%HJZvxArLtKwcpu0V`^yoy|dYI=MMyMC6leYw{m>3I#8 zUJUh4v|B<9O;FI?AdcWU@&B*?xGC_X4{$`i&kM7;R|9;I<6y-t~r;{^Q_8s3n$ZPK_&TFDr{zTdnU z;$#xdNTU;3oWVJq$3%^SqGvKWiH>0!0{wF&Kxa{VgUZVlsL^vKEo-`Jz5B*|L0Sm`TpYT$Ca` zUirs7de?B}f6mabfW8PHum5|juQXEm-=+L3Gx6iff0MFtMH#uSY>*8X)K3@e4;e0f z#>1ER{W7dV_PYgP4cYG7cA)bu-v-^@X$Dz(PJPg*Z_%`0`QNSlZ}9zH`*{7Idn!NU z{XDM@7S=kf$3|>MRL-OH_t?z0mAlvb-cFXG9F?eAZVr_^fI~Qf!S$Qt&7acihD+NV zdyVL)@bUV;$G-na<^L`1527}@4)yc|lE<}Ehlhm~QFw?WiDsnHi7eXLu^o@~1@^Ol zcWV>6*HY;u6K`sN zXG#AY_wAB?)FXi;8iae(Z*Job?%@F*;Ry!NF1>^E|DN?PzQ6DL@%(?3Rt(PnXQW@) zZQ|N8T;J#~t4&ex?AyV%Il9-oTu*)z*#u55fY?D?K{?`uQDT=$n^9u{B` zmS7oHU=@0QZ;XWO`8-=Syc*Wh*I_+2Vl#Gsp(Jc2x1+Dom^E3BO6gGZxv2 zF5`pU{G+k^!F7mJ>`U^z6F7x9dcJAj-9HQsyA98u|VE(DO(2Hg_koID^V> zy%zR-+g|@KOtAi2KZD=1^@Zo@7jYS#`Wap5Mh02raMiVIxPhCvjXSu9zCW-(zWr+G zdDR*j?r8ntBk~Cbkhj>kA3Zb-BS)YJqY#_p43`<^Kk0SqusE{cDh>`$oi7fkRX%X4P?x7${*U}@2r5G7$b~{7+jb1uJ`M{jv1w)6J6-O zRLZt14O!$al!nIDrJ-p;X=t8PYOg?PXq{3TQb><24Q-?ORhNdz!kdcen29vMRH`)0 zrgz;E_R-QXmtKn8U11}GZp`CX)l(W4kc$wbFCmwq`o45smo8^Vtl(aSWUX|e9tr%Q zU{Yw2&NZ&BMS4nUSVyi$6wl5b(lx&{Y~8x<{{^!F_3TK4BeRD=Qa{BY(@XBf7$BUo7F?ZcK1~2DD!`qNFX zhM%3aR`>P%5HoN1pL&Le*}m7whx*PR8r!*UKFyx5#MzM{`Mx>BIiCtOZ$2Mt%`?Uw zhE_McFzlN#J^ande;3MxU5*1C|1SJsWFXVhsu-x zL)bHVdZ-vXHS9h(H5_}RH2j_KQ29&k*l&i0eZuJZMrn9=uQC)$!=blkhaYYG_u&Bl zzE78iBa8n-_}h-x%-Q~jaCr6HP#u4XUkzvI=g`pM{-#%rW4@{`dNrKqcM&Z+ zhK9>zwD0jMc@5D%$Q$HMbe&ZPA=9A_N*SN@O{00l+pgWgJ@kD;T^zNA#*(7_mVMtZ zecJy2u{1nzz4ztP@Pv%U{{|xWXY~6Y4h?xr!p=S3Q{{K^!!UYmrSiXYXc$4SW8=h8 zL?4BQNo+9nLqh#fU!uMj&LfO*Z5)a*5tA_$HQJims2$m_9oenj(B37;XZr&i`?c?{ zv#-&DR-_iPui4&hXrI77N9QE=`5gB76!!UO_W5}BdB5LG(l&X{Ow7hyJX?QQO7CJ% zN9zyg(H9`crq1%qq=bb<{Hort4vt)g7<~n~3f1BsUoGA}*79?&MUvjYwy(c$ei}7R z(wA}H1@~Q-J~EnHNIum5v*UK^C+*S)EI+GH_bv9+5A`#GJu1T5i1thRfA8!6_2~bh z`=I`xb}M^W|8KAMU!SjOi~j!({eQBRJEd=)MjKkR*Xx9{8C$U(=^fgC{$=#8CSlil zKD`pr+Py4y=A>tl`}kE285RzZhY-`3IYJ&oHUGhJ#fSR;+@~-&uDD6x@9xua#-7iz z0r>ZRRXY(~M>EpsL>6ao4(AcAeUJ8__PxvA+!n$`ewR`4OZ{8<`x-q(rt!4@&;I|Ob2vAl3ep138i7~|qL0@mOWQPQM7?pC!S(NK{WzPyj;v=7 zC&(ms1G)2-a|Z9e7|O{7!dr$FScNrczry}!_ji(AAF}@+v;UE$=g@ea{clY`^L)1c z>K8*RnOe;LC)2-sKr&_rVuK8NVvy z0Oe#QV)T9F0aP11h_9Be(=UcY+((c+`y#uJ{V%==vWDGX`@Z|;7=K6o6y*X*WjNaZ zd(3Z6;1pumS@_geDbE;(;%;|OhkO5N{LT2gGLl6OjmlZm!#|C`sY6k+t%FbZQZ_`bAh(fGhP?qc-)?8PvVoQj&Uf7<^$ z-~7J5dA+tOp)E~r(Z4_L{G*?Z4%xT0Q_t&Pylfx3-^|2p%tiad=!_%tA7s~lHvKN| zA6ewkc+39ZyY~N*Eo}Z)dg_urd{^xMz2KY)Va&q8C$U( zW$62@cSTlWA9_Dk6b_I*U$@?$jMnlFj<IM`=rW-YI(5 zpWaQ39!Kt^-^sg~>)sV=v^BLU^=?MJOV%gU!`kPh_PK#P;x|b&BaKdEaR%qmu1)Vi z=X>g1bic12M)rbw`Kgn^sxrocSir8WM@6^}V z=ye%m=&9$z4f;(qXup&C4TVM@d`d_AIw4n8&^53tFlWl(2vf3PiaHeA>W+NJ#Pi?X0fgbHC=|m}g9&&T!y&0ad zTRnj4Y36VEFG5uAmypX)RiGb4Cd98EgX4bcfmQq_dmnWdxTMXcrNVYK7d0wf@3&=Q-~pso}cSmk-eXjC&)B9 zk;NID!+Bi9Wz-nQuSFbns7C@xG@wzv(WK63CR@}Wt@LN>kJ{8N?QHyxS^C~3(O8oH z#}NGwbw#uXIHxXX(LP@FoNKs&n@CSk5A(11>iE#5d`Ej5?(n~doU)zemqB-vbHoGm;XMA|z{zrI%0pu-{esgof$X@Sq1X+Yp7=v*r#zag;pMHMD zN9O;(E01vZe$|*KITN!n7tx$)DLD@dun5uKiE92!=+E|VFQaE`UJn1?{o6G&>_0~w zb*Nu$UV6Sc@x|t;$=l*xe!-dk(cyLd&e0dnDh#gw`Gfa+wm5W{&+a6vK+Bp=2OXis6HrN8R^2?95(+xMfwq6 z?!FEDxA5N~{bYmr`y+mH3@303F~pHXGul6t{)gh&Fa2cqU2*f=EOKZZFa5WqpMMM4 z>b?}2=58ZfuG59ni7d|G+4%_-UstD05Vmw?KWpyRwH$YLj%Up9Eb>|Yt2%5gg8WyV zDh}t|cM&oAR^RV3y-r%<;*;1WjhmjPElCcJ;~1ZB*`*w~FGXG>Zy-(op82Mm^e%a- z^R9e?CO$oK)>zU9aIP-`+a=?ZQ(uc-e3*EA72R%=sm9%hrZufL-5U4!Xxe{ z7(m|t>Rh7&eNg47j*KHws9#Dp%+gOq#diwAF!zl>5k_GQ#-SJ;AIgJ0%JySroXl`{ zlR0%kmTa10{m&$Q?-G6R3CgwpZ>zp<+xu*kk@g)5V=|_qYPbFcSxv?;lRg)vhzqyw zminh({Uh8Y8m_R{7iw#kYqL?MPRRZw-*}9E0p|H#IW2PUkcU9 ziTCWK|=3apCU-}8S2>dBsulo9UUmy5$%GP3{IMefh5zoc#VH*bga+#9hO zY4v&vTj^b?r+w4y^fE*?X?B|OfK1{3YDc93v z+csyUEMy0wbEWZlbkj36QAw$bQcJ+t>8tF#<&xg>fjxV1IL#{9WQaIPS_HvLkI! z@h$#q>Fcl_8?hN%@yY(ncJ8vs4;9+n%D)%?&l{)YK7g!o{zK#u97B)#?gZIeGc24U zD?akyb!6Y)izk^xGt%fpG^dv(&)^)+Bdv@_{qKwPt`g;Mmj8{TUq!Cp{fO2GbmJPo zolKGp}$`7h;86#4rqH~{$TpNYh zQMO~v3!%~&LY;abo>BK4R|lbCi#iB3Z0lOY+1$}QdA)h}#8UP@8c^w(k1$4f<4}x= zn2f1tzia*f!#}P6-)~O-u(|o;)-97c?#3DH|2gLWhnW9I>tnV((#8eau9$z{=J|;4 ze^EQlE7FqYvoRN?n1=;egmz;99Y>83oHYJ`ZfgoM#u>5;jX$7~ zjo-x9Z#G8JLbh_Jnv6f74blGlCBj*T6)p2zo3Rz! zQHC+zc{y3*8MTOy6%Oj@i8p*7MB@`5k1xnmEf3ijzJJPheYU>jUmc&iuiPVtO3&Db z12}{ui01c?k&!(go#_{?4>-Yn3NgfyL^E3M7lt(1i7d__y{j;!dJ4lide>6@hJ|_I zJpCdD`wiUD99TD5<2(M73-;<~(C?r@o1NrN=$k}) z0H2-1Q@+@qCf9quH7fM}g}%n~|6Pdo-#y^h_o}|gr^kdx^vbXL&$q7(4NvF;7@YqX z);}Bn<4*3@2N~}_RaPi(7=a>;!WfK0MSDSL_pUp3DC1<;aAjIuK6pMGJvYSnQ4ch! zBbvz;_qD#|`>ghTmb3pB4++J>nTW}lis_h%zAvyTeqI!!x&NNOc+vTi)(E{@6lS|t z9<`f0lzYtOE=Bei+TZ#1>(WzXda;*b{_fz2~C7%j^du3!e(lk08E_f;Y_}1v~qtw`NXvU|* z!8xA}KP-GX{NRn3!&UFGh5fuFdbYN40l5rm?eYq86}k>i~u=gvBtHv_!sjF zyPnR!SJe2wOxM?=YEnViNLG`vY1XIFw_-cWP>xFM!vP$^5gfw_oI+J?K{$0OKb(9& zKm7Df>((ZW4}U$&d1tQ0QF-{&VbArK%x{eiyRoluboiU5mqOCDW*jh=_5=Tc^8E{+ z^54uO!=X0{!;iKUI)_pl?D~~k+?z@Kvc!VbyK;HkR{@qq!|3`uS9|fU{>^A3^ zA+y}MyUr#*njf0bywF$`TCMd@Ax&>X)%g67{na1CFky{A5n?OzL-nluFp6HcTHG*( zJ`N4X?Xf`u^(f}oVmy5!ITdO3&2(}mQXNn0p04H5_;-KU^YcH3ESn<77U)(+GGxtl z>0hXAH(yq_L7&68W8$zrh(1O`hyMM3^FPWrm zGWs}V&T|{}VVbTOpFoT2t^A*@4`}0FB#b4fx}={&RzDmXR*ny6sZLi2?ax1o@@9WOzAS`_?%_aaJlL;)-XEILg4X2&^5}s2Z$SMwU{82|I3m2tdhH_{@)$k6sXwH) z^@kJmuI2rq6Q}4g&(ivM!1@&F7l*Vsw7IWkHfdxhlWwyV=xZIsQ7d9|9_qzCer)9G7u({QxTorGo8$SSvmev zL6}KTD~~DTWV7jA+UU-VIVyT7ax3KXn-Y?N?G+ z&VI|>uiq`eA}ql&tiURCJeCLVD|?HTK{E5YaD^S6J(8t24Oa%0^A_c>d5CzuuMA$Y zHi$c#pO41l)(EHf_x7)l>+nhYVLkUoY{qt!p&Wgm(|01PK4eRf&-VWvpby$7N1ZWF zkM=Ca)dxrD$1phlCS8N+Jm`!3hXGG)Q z?CUAo-$~B0^PBSb?EgFcoIjy3-p+jogX4dH^zQGnuODjLwfkLUH+N>~)3!aDmu+0l zKA*tGp2I#DZtGaKHPYAF=U40>IO^OJ;XS}3JVC6<{(%nr2k6n5e02U#-b(#2H27U~ z_CSJvbQZw~el6lvM25rg{zxVv|lr`Tftf132q@{AJc z7r()E`SU#k(f+^5elr!*F%z>f7p0ho_7DG+^dB@vaM=3C{eRm3C$7!n+CsMWOFx`qx0ou_YCgup|7P^ zeo4LhEq#+I`hSyr^Ag{Ff;@oe{LyvpS&zZ}f8mADtNrNvz`1mv(vI-&`=NPh{pZc} zo?jbNBDbRq<*39yRGrQX2gqvj5P1YK`Z4kZ>P{-(3HiQS`PMEsDErYF8vo7lKlbaX zC@jR0L^IOp#Q)#%zrp=KSHuk;@Bh(vADsW0<~@1F8Jx#OT*g&g!ws~1j~)0I{cHX0 z+%5fo{rslY`u}8$@LFdnn{)L4>21EpPTyl!wegjN{=RhpH-&i{chIM8&wkGNgZBU3 z>T|Kuj4@xMor`vrMQ{rQBx=gZC)_zP~l1}&%VCPzJAP>e#pK) z%)Vy3r_q*rF|@PmJKm80(1mVfkVOtvs|){W|Lb^r)IBq{*Zy~72-E3xN7=py?Gd2Q zMngZ__dXl=F9wE^seGc9wC0NyZ5l z8Yh_lBKwp5&wm+KU=?ChJbwcFpI#>}@eKQ)z77pNe|o-W*Bjqh&u=3(V=K0!4CScA zJ{-Uy9Kqmx_!Zy#y8SNPHLn~0TWV|ub#EG**X|y5QlhQ1l?02=7+VC_zJyIX_ zp>~_zRia&;O^l)xk6AUCMd1c3?JrE^_0Q zKRnydQOd99tMnKQWhh4__C>N?dHSL6Q)v8utnvR1#{adO z_1et@nM4Eoy>XiH|Fg#b^$-5V{yNWScdbMJq4Ry~1MuwJ{_JsM0Ehj?^#h)F2uE-X z(cZk&73cNPyCytcTX2dVLvD&Xkza=Fo+V!VsRD|Gq!y z>yT41J#y;<%p{|>W;WSt{mdR>G}T~g|!)5(W1WEPNuFKKhqy5qnD!+ zozfERW$xB~XXbcs-fIrA{r2CuegM_G^?k@_PW}ja49QLM5$ZR{S8qh+Q$GVITswsr zdMfns$lgEKw~O4~d2{4uV`P6fKBVcL=re{~@keV5e*bdFM((dX7tWC9a2^-&EFWE_ zM|+|xzhphaFJ2DUxNqPlZsQK_p~f6>ZGrvw_R-go^+W8xA8!Bs9Q*Ha)jbdJ2v0D8 zyw&PVj6nN)_TOKz|NfHw_s-t$e(e1HyUyP~>im5)PO|@gn*H~}YO&wGm7emfv}d(h zPtdaal~5#{Q5b`9NbhwIis@09ojtFFiS)_Hy(esb8GhZKGnHS}=~u#ZawcLIUJ0|w zxu~YctEDUPN+{)?ha|mWzjW40r)xFl?`si%Z2lkh^aPS6zMF6T$@RY#-^>dOgs}*F zjIUI_>-@V*`hV{$V?FxU_!sMc7c2kEeb3d(KRLMn`@a4k2G{?ZgI^-NWmth#NWW!` zz#Ha#>0Jkfy;onJz7B)yf4MWeJd0c(`THhS)&gv#S5L84g4~LjwF2A8GSrD<{Iopy zp1epl>`=#%<*rp?9}eIUj^G$h;1ptrBZ+2oA{(L4(DPe!)aLwqfA>m=_T5Kgo4XLr zi=1)Kv;BYP=;v_}m(lkbd5gS;8@L(y3FkI>2lwy*(fa>K$$v|2WAx!}N*B>9b_~it@Q0^ z7|rfNVgfsCim?~AZ!Oz5zMTF0Ci`~>+mamIC-;Un!Eee?j!F#nU-cWK^KbWY58A(M z;LgW?vj5q=nd6_Ze|y;fXhusv`fx|(g5EVr z*r?PG+hdF{XIvsX!86EiaX;mrXbd4n#t}Oy?qoBnjUzleS0K&ZiR7E?`VG>#Njk|v z`@h6}d!--s!W#VlBYFd}esc!ra2^+N8CP))?fcpP(j3igbXAK}M%*^}{utbU@4Yl# zkp3vGXnjxGkak~Nir)}(zrwkR+qi@DSoe%%^U(*-9}vF>^hd}QiW4&E#uI*3^Mx0c z4|;U=Lf-!&ZyN8dUM^kRq-(L?sIP|cPcD=$L}xD~u+u#IuJ^26yKb)ig0nZ>6P@7` zqZiRDeegX$);^z&>|^b-vXdCEeI^^Yd*1!Gq4(3q2wtLp!Pvp4oyqBUV=xZAfA8!M zvd8{{iR5JTea?NqofM|hqqDdZOP`MQHM}W54pAP5%d;yzZ@E6jM}=X!`(|P`vVSNH zbIDT7!{Gl2=r3g7Fy@E$yT;?$70=Eai~b8Wb%k9heQjhTdwFm?p1XxRI^S!7aH9X= zSVUHQ&6)b-GOWPK5A(uLM;oWH-r$t<_8CK6#eWUfVjb3F_ZKFI=q!Sb^wd$$qi?2H z{@i(sKlZ{OzX?BnYm_r8M};F7{!RGn<-ZN(um3h|71lBA**`v19{hCJZGNxf z@YwJ-BVYC(eJ_U-uARaUc8?ARj+*U%N1OXHpO`2NN!OZ@Mklg(_rCve+)xkGzr?Ww!;*>Fyn z=W!92aTV9FGwN$c{mJ)@iE`JBG`?Y6BW}$=-E!j_#tIUfjBji)zOmuOaML|^@bv#j zoH<2*fJbKcaQ5u^sWM97{*ve(Z?WX>?J$I*u`jL7#PQ|>fno^n4E}M>cudb zoQmpQ(sf+AUbl{ddnS_erAr#?mr5sUb~*nManzw62_&2RSH($ZV*1TolwuwhpuEF7 z)VJP=_F2x~EHPG~zUVqEfA5#SQ}Va=E=M+QP~PP8=K0z`?NF?5XQ^+*>!1FA#C2_SG)B2q7~4^X za{P)NUPNt~NcOFvqhq~G(?-c4IAdqEhz z6-D6)+4B$9U63bm3NciC#@aVBiDsnH_mTWYX3_G{8H42D{KNP2*#-F_ngdNq?|J@R zJH%~^v)SmEk(=Y&NN;BO)4Hf?RzX<7{~WI38e#|SDV7qkE8v)Na(-Ct+JBZnuh4d8ZE z2HsExFakvwg~9%bHZuN}b`15}v%&sJ)Q%fNXuS2O^@n8iKLD*C8aqIGrS*r#C)(Nh z9q2?CqO}JZafsF)MDr6>r;UMl&NvifB4WlLsxLYJhh8__vps(*eL5Pnxk)5YuZ{lX z`DZg-&;EqXj@k6NsI(`3&o8Xsr_aLzEW#2jLyh*jc7ta~&%ZYQu);m7um)?f4(qWI zo6){p|8uqeC)xEL`}ckJFS7L9Y4&du`=f*Xv6uaU)=kC+klv#Ixq+WK`mMs*jxv-Z zt^G}*lKw3J?4ut*RHn0t+OTe9*+kKribJj)K}=acMxH=5Jw8LZlIKowSA3w~pgct9 z>er7|4w3zydfl~W|K}O)NloalC#UIOkb~`AzxrBQs`OQoQMhCb&GgD|dcR*&SI|3= zMX&L?Gi1-tjNg!b-%u9Fi@1!dxP}|JiQey<*CKmf%@23Tdw76HsQ8TWb#efCYvoV$ zedoC_f-FMIG<}pWmxNLDaVSQbUkVfHUGFJ>%Kc>eROGm`+!?a_l5xf8{GXp&+hhM< zbpFrB>zOV&tKa=IF&o*Ry&C3{rRaOvT!DF+dGx{b=a+;9^p;KfFl1`1?>pRH5c)E# zK<5zo1Kr3Vi)ejP&F<1rn<`btOVy91>c`TMAd{y{LqmUQSmn2CuommE9viV4ThYGs zHSOPP+P~Mdf3KcZYEIN`Hsv;D|7e;RH@0hB%UFM*ADe zA3D*6Ze)-}4viP2|BCb@I_seIr1aNHzjx7wD(^cjoK9qM2C?C#&R;AI=je4K#f|$s z{URDRh?{sOHi_$y(r}sIRb0aj+{A6%!99FDUR@Bv1MaMTZUvsu2k_74zm&Q9S<0U> zm?Rsx^ZrkH9wShMQ5b`9C`S8-`rGK-tAC4becueS^c)(O8-H7^tdcDsntOk&KaMoL zt%u)ghm#{hyT4PL~iRiz>vdEx&mvjHaOQCA4wFa)u zMNHVG(Te+*Pfm~xIzBgVQel~=ke>3s~ zncz;64Lh74P@g||hVUWJI)Y<3fm4W~Mtfe1_!R9AqB-U0Y}vv2f9`eei6e<-q|u2i z&S22~J;?sW$M&yw{#pC}@%g`Z*}DDgUov%F|KQ4>>|gf!IpLhgMO;R@!#z#b%+tH{ zB|34H{z!h!jb#&w*P#6`o>954x~1NHU7r*&&$!{z@chBIKGDC}CjHzExGkJJxQ7RLgeMq4-nXRxP33=s@=tc{Q2xmbca|KS ze;UjFNAoV}M=MfDqYcr22M!a?2ozxy(vRJ9-O1A4<{Mi8{>Yo{QuV=|a?ZNnr5*Z!T%4Q8(>hmj6<}nTP1?gGJ;L zEW-+{!apnjyOqh~%H}R*6uH!YyZrCmVEmz4`AMk1m)k!p%v6*50c+_M<{{RRox<%p zEd2e#pCNqY5S9J)uJ8WtTVW%)8L@uz5ZFqudtV)J$y`CS|K0p>!wz*DqCLj-r|oBS zPZ`QliM~c-6Th&ghkgLPZyVcSi}bYHhvPi9L;Q~57*606VyLi>Ap7;gFgQLoOL|Jo z(eh8C8H4}-_YLC!^eoQc9M0n+F5@b$;RbG^kG)?}Uugf2e8}Cae{hGqhX;6sXwCN% zasYX6s;d#5EgYROH-bJm|2V;(0QxB89_kw+Gf7^Zrv0C0efcbN+$En_|G(S3A{siZ z(--a-zZ-{Q>>R6{A(~s5$lbou{5}Tf*|)Jt-(s60JJviq8mq1UueJVPSS?M~|8HX3 zB08VH4b|q>v(|o36;_PBJei!1s)RXzGWsuq=syV>5^QoT5TDun>-fbZ`?>MwFO9`0ZaU58CGBw z)?h8xVLjTtr;h#XF|zA8`=6eP(v)KVZ;R|d=|?kK&PxAjdkc_88>$``g^j}5jIG#? z*huRS#ukM#dbB4aj&gb>8VbY>3HL;27!KyQ(Z0uc~Ph@`=X%?BDs?|HayRvURogpGk!D^^Aj0`x-T4LX>0z%`{KTBbubpemo&6|3j3WD<4`IyUb>l8%@3-vh(B|*? zwspwdEg5|oGL>N8SL^fBr(!xfU)MgMTiMJk_HD@AW8?p>&Bk1mVjdP?5&Aw5kMHD% zo?pv{+`TV5=aO83Rak?n8OA`!ia)mwm|Tz86#IWC{v-O2*E-bG6NuKBMQ8O6^PBPuPuB`WYX-J*m!b00PuB;uUo!svzBZ~y`;YE} z+NR^$sKeI3?X~`Gq4p2|?EiP~YyZ%O#vb-NnuoLB(K?p>jx@augXaqrvfuf2p_^X@ z&(8NP7ndWxZ6&!62XH9z!!gv{w|;>=A7|&+T{rLl-Y51ip7hKq<{8{~3NgfyUakF| zpbtRrnlEi=j{K3!NEJE$AtoI>u)NIv#nQI7;_ z{I<5q+}uLz59EpZ)xHgxlt&uKtA6vz^#RwoZ{Q}fUmFtIcbWg*Z~l9?`R}9b|Gnz= zj5z9;(6oI9R% z4;8LIAk(*$Pj$m1`rx^luiH;UA3&1cuu%D0s(c~u|MGs6g<<3f#7>%5AV(ouA5>fG z42F~TK8kN#yc5^$fkeZ7eYJk+_}t53jC;qS7!xrWQ!yPKJB&dhI(xel(b?PG$kC(! zI%_)Y9e5`#i@gJSYPENOHg4yAg_**bjkzerJS@N>EWs!3r)Au$um*en;GMlaUcVqe zWPkax|L++V*79G6_1K8b*oy5aLpdt3579Y@2gpM>f@6`Nxrh_wDa0bTFymyjKQKu~ z=l?ZFZuNRPa(`=B*g5h~?eP?8?Dr1%KU-gtrJq5?cgD*f>Yt+KgmH{&RHJ|j3MfVe1yoSX z4cx#D+`u(sm=5Eb8WM;hffyPX&+g2g*?WHOJ-_z^6won>(E$}yP(c}{puh=CK>;05 znB33W80w_w-v05`^So=n@7nKr-*^3Zp7o2~G1$0(VK0Q6j&CFHxJP)`D1G?C8?G52 zaLf1rve|fm7BYLo_<%9S2cYdm;|q)%=$IoOvTKCv#u)=3jvjH{73Y0i_w79-2QcVc z+CdnKYU2`WP#6c${RjIC9DX5GE+`Jeoi`GrF&0T-RVb#{N_*<%;xK_e3H8D=j?=>H zJfn}-zmxqw71L4P;u)joSBWs&afPuOZ<*6o-eoOt$8(Wyk^graf3#ozM=Y!B)JJJ# z==CSf4|m>tEW~0g#WJkGDiqeAKQ8~H6J7Yd_5Z!^Ml`+1*N2ul+J+N-KT{r$KkZz{ zKZP~oT8pRa^IztlLYz0hUie0AM!9b>w?+Qnpzq3YN?jAzm@1{0p|Ef0PVI<;+7YM_ zR_Xj*$n^{f>48^401t$9bGUeE2%YU^4%0 zHZ6G-*U|qe{=cH)aFc!;cX1yNF@Qm@`~EQ$+vg}BhG-vtV2r%u$_M%}wabo9`=jvF z59PmCH-yH-8=+~)8=-mpw$PIJUdWF6UYIGKBu2U>|9S14XW7s1*k94{SQKLdCgG^^ z)ZEZexh;hE;~sXS%$dITZ1{)7Cqwmk>rI~gBp>>BL+z07@>zX1q{nRvnJwSphxtyZ zzNYn_qqQxulbI}aA|Z4`wS^i}8? zrR_0G8)TZc$d010Mp)&XqOg`+kK}`*u#w!1DthXCk#*y^|xnQg{RF47J;X`bCC z;}2hdyl-A=i}uMy?SgyS1=3VlfAE^}{R8h0(cUU^eZ_b5Co4yG(f6QjoAeKs{$bKj zc8!yMGL8}GAsY^Be{E84IIo#(S)^^VT>6(t{{mrmimcHoKjILMAh*pNJshKVHk#Y_ zn)#OW6nYNwZS6P5@2I(cHQtl3%4|_+B6CR2az2?y6+Jc1d%ZU}oN#;!nX}&OW$*cx z_Z;7SyFNJWfZ7k_Q8KMesKb->KTf;m49?*KF5wEU;yMcB|JnG__gGl}*MqJ^CEt+`fvxQ;HmlpR)fOUtyoV%sxd6J-dK?x`BO)wjJzSbf6Pm z=tdqrxbGRw75_ZUM}^~sX)#8cd~7#cm|_FVf^bd z*Cd5i?Kl3OUfaw5H3okbeGTewxK>#Dp1819So2Qf|G%Ytdfq#wuNSrv*}ZK4-NyeL z>%ZCYR^+GgOW?`)|7v!0Vf}+w-QO$XTrTdH{RiqdvB6zah6?P$9_+&b975Ys&j%gD zJm0~d?>Nsl;rXI4|9hC{`+?_s(epj;`8ImK$klkhuB$vgJRA|vF(i?SQdpI9GxS>5 zr{eYW9O6C2y&2O9_v*I}VR@XuDHPfxGkj0V{nL)m;2g?*GkH{eQ9I;+hHy#P6;vPB z20ttBp;o)Tu>aKzJ~H{Dp1kUu>$r*AxQqLEhyk?uw%g^unD;u-wfC`YUf6$=-oVap zT*CfeF0YY=^||gm&U>w7XQOLtqjp}#&P{3_6M21W^A*tN^OowPsM}qiTeH1F~%VN8g`su)jzf z?OkJ?g_of3pX@6{#_|92$%R;qrC5fFZ)&$#zkdZit_8A+%x;taHkn&RUyJqV(AMrm z*8|_jS?>Zp*yy)P`EWD270GVETvz(++v#*>E5p|K-qdh&+P4GNk{D!@@Co66IfVTxWdBEcF8l{^=U^gSzwS{`5z! zpL0X~f{qW=FDKP6m(?%F)i3CoW8Xb-ecV1xInJO77knSp>WdnrUgd{I-D~PMJQ;^p zomT&4+2`HrKeEno`m*u-+AH~JNOsSq@HbF&A*`j&8`$L;`o5(8EkX$oWliNLc#u@sLe%xV*g*IUq@_H ze*mO<|QAjCCOIBHI7=$%m*~`EU0Bfa5{m_G})p|9d~O|LU0uRzZ%?{J3vy+r;-yg~ce?QTjLwc<=rm}qb z^sT6WplrG)Z||4Kk^h-JM1_~30v|8`b~)aIeet*dCOyIYa{3V*LlP-u(1aX1kjDwc zeE`aT{YHrP_9@4GU;LwRnmmJZxPa!@&9f!1;3}>ow~~z|{3g9H|6;sxWAwXtI{(Le zkN)@j!X9D(u^bq*&UZs5)5?gizY!|7yKnc9y5;-0=KJXNecWIlDm&^i)cL*agz};{ zL*M7LAO7)~Fg%8Ru_%lr``g|^+Lp?!a;Hcn~itSaTx zDpfv}DxXS~Po<$@b!li^R~nkOl!oRFrJ?1uQoi-lkaJ#Zwlq{PDGfC%OO4?u4YjC4 z8X44Mg8w-nZIdGOXD}I4F&)+7tU(I3sN4Id@BdBy_c!_9-wZRIGaDtCi+Pxjg;c`QP8(MjVvq86Wx+dC>eH6RL|HVC+;xkTq9vhvP zH%7f``M;%Wv#_nmyzIH4@SL1mO|MzxJbLXm;huNe^Ugf*e(xyv<*B#6q)u>c87i;~ zd$13`xBd(};opt_+0Xtz%>E~v7qI`A_%@fTpUAlWO!MV!;edD!;Ruc)cWYb7B1!Lj zjelT~_5nSE!upqcwSN!t54@~REv)k3wvZz`kgVAj^5h9r(Nm+eE5_OX*70d%&THFW z*3KBLtxuj2b`BSC2~Wpo&I;j*5fsPgS|kZudo|9ml!Jv7U|l?@n^wWK6|$ z%*1Szplz7`-x2(7csjrOyz4Hj-^m8$KqH#Ac-9*{|JOW!?G8Tz|?zx3WuDHr|`|5VDJcr&bxVP7i^ z>&d>)m4@q{;Zk}nTQP;gxHht0y%_h3sZeIV^?Bv=PV4PBuZrAEZbgzF_wOmCm!Sf? z@MQm3K=vZ zhqfE)e{?)h|D%iEjr?-;zxw@Y`zFEG9jARGZCSQ)-0QIQqVa!?%Eyn{A06UM))@bH zR2=l$N7^PhK`(#bH}QeC$#HEIvhKWde&HQT$0@&`My^JCf;@+~4q;(li1NR%{@1g< z3woS47U$k|<7xU|lYX{&oi;-pmypTI|26u5^Z{KE*CkxRRb0nS+{S-6{^+9b@2Kw& z&1gYkjq$y{KUA{E?~3O>9%2AVwt1C4ph5qq^skmT~TA$r~!Du1yR z(52n-w0}dpXR+V=zMBY3$>={@My|jrtif8W$3|?%R>ZZ;OUW`+FOdIZ`G(pWY2uHF zdlYBr^(eRge}(gQVGs7<01hGl-C?0^o%}yo9#sxJwf`LF9rut87xjOl$$8DIJ^yW< zKbb@866^mPi`Dy=Z2p$$1N_9}@qa_uZ_2+T{^J-@$e;>bO|-+iUeouEC+qVy4;db= z`=6V*jl0OrU?UIaTcCGtaxEUx2hg*G?*ZMbACI*k^d0rj0{aw_Ly_DtJPapCqUvS$ zJ$vd#o!2~Nc$grbNtleO$h|E6 z(m0*osSN0FJd-{fJ&yD8a5vf2Xk4+d%Gb>wC+8tKczBpkE<}}drX1Hw!(zuvk-6sG zJFdH@9ZCM%{CjpZe^%XLZEG@v`mA=lYgS+t)?h8xV@-gw>4 z1pDH3>+1+Rgq|VZv#_WAm#eL@Bm5YWNFgUIyGa`9P4O5VsC--AA*&9W|3p7QFV`=6 zicIZPKHO41JcxNn`9QC~W?Zq~&LIEigG2e(28VO>3%GoK8HM$k|9Ws3N*|7q7>)inhKK%Y-_IuBqT^yrz$E0R`Nl>i z!en~qINuL_Dt$V7dfA4Ll%eRteg8F6_-vG*!tq>k9_FKZxB6NAmO89{x~6`5q<%W9 zenLGO8u<^}`43)YOQYpA_Vp|5YsCF^pYH!NOZ|+({y#6OH^-=-5!e4;=zkVtDV8C* z!TL8#>?=pF)eeYr6IapmKlM$jmowdu`|#_??TfU#U-SL=4x82YJ2^nI+x-6~;~`vA z5uR5*dInSc|3v>kNBe-xBl_~I_evj9s70N$q>)jN*JG_X)?*_!fzy?_KjgTy=>Oj@HaT%_d}@IusOVSaC0ahvNh~nw>cc*fBA`e<%r*oA&Fk|RNs3| z|JovG^B+0e|faIVb6pg;oWUR!~PlnDb(y99jbSZ4z-C-DXT|^^oEh4 zE;};Rw@(e3#23QRZStCHD~}e3V>S9^8^?zC@4XQI;o6^uB)$9>Z1k@^ADWz#LkIFW zfm3KSFRbC>)X=PtvFXO-ki9o4v?M+k%D*!%oc4QRelHs?`mkd9&j>q*x7zeweq~&^ zM8AToxQ?5+jk~yyhZw-1jq)q{wwsSk_WsnEJF?%{!I9)>j72dfU=k){D&qcr)5$*m z|C#Zak9KxEuG9ac9T0uv@&5Xs*LN&@F7jDq|lZ&J|OFV^zC<&U5>k#7$306|18kIDUL=op&2bz z#uwBWU!d>5b-(`q7p0c@Ro1`B( z=QJNI4jp73Ipx3!@)WX-!e3P`;I!j2$j{L(P=<8P@(z?E)vpwXnmEP+@xSV}iTj}Y zc-w#N(EhIr;hgI(;1aIjDz4)u+S1w|+7un@{0~{!f11oY?jajK(EdkLjc=t&`?Fi! za8&yPt^181di{C*7te=|c(F-o<@9!~h2UU-AluVmL-3+W3`=#)Z-Jsx9L}a{r(( zmR^hrcsf7-j{5hJGU1;3*S9~}Z@s@!zma`!Bno?R?ARo~Y%{h;SYiEv_lv?z`fQY7 zF6N>C9et0lYX2j(T>Y*btJ@-eWV*$l)%N)5;4t603$YmauA;D%T!s~Bo8tLDUPFU~bs zi}l!u&De@k+=%V|-`G##Yrb2@&HI&8hDcC&O{v7tB>o*H2Z)1+r|c>?4bHz{n_N4 zC;bOGdI$1RE>Qn!6a3-)zv`FOzY8AM@&9i9fz{816Rtgl(>Q~3xPVKzg0|P#|LAyK z{OH=Een)=wE-v@nHoAJk>neZxtNFf$j$QoOtX#;z4I0I2Nu(pqUW$SGP==)oo|@Kq^?=! zw-rckoG8YXASQ^8W_;AFb&7dF-##*Z)1f`!4%U zi?jSQd=72w`meN$tUBFhto%5G~zq# zT<)Eb-P!|rGLApq?E0#0=1XrH7OvCBI=2`TP`S>0@crh4A2uJHtbft?eBqOka$LLH zn1gH9>2o|CGcg-EVcC1bLkYd}DEs{)`;HZ5tq#rH4>VM?Ws(z|W z8=E1X44RNb^1Su;*%2M|+DCrB=R2gIK>b8{1L@1Y4QyX7|0)B^R6qx`Q_&CoFNPQZ}I7M3GYV!qP~7q@3j8cetmqX9v=DK)p z;x@|lf8HhU<00~27#s%3L7U|<^nF$Pp6va+xdCLQJT;OWjjg8I%cB;bJ1^ahI!=tsGENf#~duA z7v}#zJ2)(+=Y&Nc(^7ipEq%jSMqhy*&n2(C?dAvQA`A5o`?9d_YPWiwom$vmhpZp- zi7?YOtFQ)Zu^t<-8C%iD&Td!Vb_`Sh4p#q;Q~#2A^vvLYSI0J@iA~<@+!pp?*7&y^ zT2VPg|ABbQP=Q@Y3XA*N?V-m#hf>%_KY;rE?Ef0}KR-Yn`SJ4Mkl&Bs7|Ory8~ZE$ zar6`_ex)A%n(vCQ~3xPVKzf~&ZWe*H{0$=hhYWo;hv zJ|1EKao>Y%uXT9-Z_j_XJ^(yJABvur_3Vz}!^qWlZuFa~fez3q|wFV3g)OX!95r_?Ql^@oP=KcGooZ$7F1*DlSHagV50 zvf`&-331)^VsTX((>H;vN*nKIjNT;rWaK|@jy@klE#fzu9_Je->xZdZUp79#Inxo_ zoHNPhqsGsWCCI-b?=FxpH_7*82j5%gI{tpY&BJ^w#3^}fF}W1WumY=4t)8k`^0D^0 zx+^VDXBw3c{D=+8ipIy71n8NrZ=XNSz3i}U}dBpvPBQF0o-{rfB zaKLe&@%@L$BdGX>=l+*-!ZCUhDU_Gn^PFr#4*mKD^56bS=vB{mIPUvv$K(l|!f7<0 zHD;JRhYPrb++Ock-F}7M+3nlFRr+=GY!U~a>>pX(Jtx$dPn;T465?F*x@jdLy{IH) zP`|*ud)It?e9ukCw{aKu(PmzI`{k0*L3Y0Sl5hGY{(bZ7={+yMr2ozQWi*Mq87+6r zUq;US_SRe4j(hECE}jbI>03WD=8~Q~TVnn1k}&9dvHo#yc!oX{^)uWX;u-~Y7%r^& z@JnGNIU2dDm%>=G7}@koq2u+JLZ@{GCODpiynE}h7C_fJ_qqC|Fj?4COvg<0ebYJ% zWN(-gO341-&IxnLd6vi(iYWZuMd`{*a_mB-)_OCdbcCdfZvO)et4y}mu|Bi^~ z7?Ozl0OgjjKV}%;=UeF1Chyqb{`{6foKqb4)_AghO_Q)V|1U>&ASrL=$rGrG*NLZg z4j;bb)5zRWZlKP2X;jxpKT_?^LmfSh%q;H;+m(|~%HK1tKZm@1e=d<%&~`)liw?XOqaUrX2`%ay<6AFlt{|7m-?iKj6C;hzVG>-6$(s9(tbpO=K& zWG{J_?E9iM3X~!DW0-csLo&B*P#7QwZSlQt(oah^uVx33!%^@1Rm%s6BERMcnHP3n`lS9={fUD!;_MhQON zzcJTw{@eO=$wL3P{+#*rg;2by=<<3K!H zQHnC;q%qFXub_8cQ}!O@f2Z$3&n@E@(0#-Bg?q*|2s?m7ID%tHB8C1R6om}g`_JlM zvhUaYB`>NkqCKO(E6-mQ_v!8MTcP}8f8^=O8QMRd?+JSJiKO%qoT8sbT)Qx?VNme@ zZ!*?EShMo|9C-n8UCB%26=dm!b%A_)R~=tR-m~dJVIC@3Esxa1e1clkWqtqhYhit? zH2>GHpA9!%cN=$c9}h8rL0dh4eQ)jP(C^lXr|o-s?2BtSFaKYb|Hq{X#I^j_im&{~&xZBnMr_7bl%fn3h`!D3=>M!a z%j7O$d$14rpFJCPw%9N1Q}#R4_U#L!LoYwnA;0x~!~ACZ3m%~#Lq+&xc#Dsr`S7qX z$#D`XWRPS3WwXOV6TR~RJMfM@-sm0Z8N;^LuiWjNF64z(UbKcEc?!wB!@_Cu460n0 z5>IWnJ>DE&K!#0Q&$g{|UK-W(nw9Kd)S?bg_qU^8a?KT7#dX}oZQR9uv@Njyz#`-P z(TT1N#up%u9yA>PgZ1|uXOTlInr}Q89*QTwJrM@T1IYC{NBE%sNBRe|;W3o{40`B! zbn{zup|Jn%%ihPg>|>xWvf^Kjum4AVKh7PA(J23pK3cLEy}#7oPxk$V=T26>{#=+$ zPQ`T0L~%k8TtNgYGasK>Tay{bs_`>{o^~pxZaSVAJ`xD0i)hR2| z`1_;Jh9g^shQqH94ey=(WcW$r%y4Ma%y97Rm%@*Gr-vV{n;s6Vp02O&rSR^`lJLU~ zC1Kx;{~Gpgm>qtwWp>!UUwL>~d3aR+zyIj}bM58tJ{L;qeb3wX&a*6|_x{rN`&Y_K z`RSPFef0A1@Y8m_oEkm?znA~4DCCaP|J8W>e<@O~Tf;+m_XcH>vZjZflBe$xmQ!{` zUqOemsO+-&hl&4cZUme4XUhB$FNL?pyk!32e+@gA&knn^oBnR%e-8UxcL00ZXN2#M z`Y+*~8UH!#-XZ+xe+fUlI3(nMJ|-NB=QIwpw&ru;-7Wtq>|gnx!lA*>g`a4TAMx8U zB*8rq-qTJ>(R=@bKZ%TU`N`mp{aG}ru(L{s8diEQH6iIx%nBhmlP z@NmlSH7E6{QAhy>sK1<=h`zkhYKj!+l}5AE;(*{dvIt+ z$6@=w=_lxFH#ZD2x6HpIn;+T#jh)PU8T7GFJE~5822VDjk;-r!)@m@uVjOh_mLZKO@FR57^X|0=Pwu;e-ibD}OICLe4K?YZVW|Hcj*%FRu_(p_OhRFQ1?`^H6z!i^wSN|9|9C%{M)p4@ zJ7+4UVGbD9qu|47d|FQ@I$ zsvS{TRrGt~ALolRd95f^oh%9q>9wy))9d_7^kt~eO0%#u>hv2`eAD=fPui!>Z_PXT z{>iw9c-%XD4Sg-vh~I9h4FZcv_I;sqwD(3*owIReJNRn z3UtZGF^|W*9ryhDLw{h)UhNG$8E@1;Z$uNCu}d6#unz}t2yxxOBV?aCA^)xC!!ddi z{r{vbMrP2294g;dhD>?fri*<89m0~k&0j#CUOQM?aDsjs_4Ev$`2Q+C>-qhBTsY&m zbGU#@xPpH3ey@_f)*HM|_I>vg;U;+-cX1!h%9e-Z00wRIjU(5njnS^pi5|zK#6IZZ z^pSYF{?9Aw08}orU%YEa`)w?Wk=&sFTR;BJ3Mf&p5 zjFvn4|B*v0s`cmB?DqfCQXAJaAk&UB?fySo9M<}e_1K8b*osn=p#p8LX;16tuPWBR z>6vQVb(4klU9<@s&ia2ep&2dW%yt_$K(@NBd04SJ#y5z4IDnk6EDq5-M~NFp=*Q6G zz2r9(^RX6(E|huxm9KgaOS}g@_#{35p8iDVR?$<7qziAop>6Ut^NMCjmuFiS*Dye} zyjP<>Pp#Iz-lkrEJ+`lXqo_wYUuQ;KO~|1Gd7Qv0{9%6UIxj!+$K}bM54Hc#YX71M z&GBLfy;KiRz2eKi=~~E$Wa)2K5+l&7e~G3bFk&lpKzc7>l;u z+W+!k2Vy&`tK0k6F87Q=57}^y{eO%7|A75}hy8z1-b7CRY|Zlhf5ti!;_3VR3toY?xdv;o9cP>;H{Fbjt0md{;y9m<4Zh1Q%74crk)NhrFvoYg$aji=*Z-|7Nbi;Z zQIFBC8H-{}z$8q@R7}V3uYawry-D8GwvBs)_3#1yN9$kD6vu3opz@KqFl5yY&tROr zR_ODw5UE+pC*{JEb%Qfo+@HL@*l$a*4CTfS6#P#^e1Eg}Vuh{3TC7L@m(PZco_lW&}d|0?i@6>19kr*Cs)9<3kao%w^*~JfVUsxZX#Y3|9x7M;J2W?kg zVkm}VBt~N_iqZegXTt%}6$y|IfDFO0N}13Z?Wi)Q@*>v)tnx>E{Qi5Ej>c z*hTI^?twKb$pgq9_g&V=|H_9$j*p=gQ+zxrO6J6(xFS*PIPxi=H4KJ$y{oW+bX0pZk*-`9+ zan`>lJH%1`p7Y7d9oorc)!XuaBcDCJ;C^s^D*uoA{nkM9+d~Xs&<@`{`t)NBC41|XyJWxmZzMSy1^fA${C7+LZ9M0j zd>mgj4(QALcHbKwiiJ%;{#|uG`DFb6RQhzxME`G%{Wt!9HoaLHUP9hi{?8@nAva7J zGsJoY^v+&3$r$Yc`eO8)l?QI{kt6!bbMAYR@TFLW6jj!;pvs;_hwJl_-^FyP`~%1{{I@bvVMb)_y6zm`yT8=+&?0Z z1N1{E^gp~~P8Izal1QQY@JH#Zl0Ipvlh(rghV#<*p>#T@2|08ij}thB(`Zwsx5s>j zPQ*P`yXD8H_IG?fyFaqOS1bRwc{gMnqtQw>PvOHC&pBMcCFB-qBTTgZ0KL<5=uppJ zrC&$SQE|8XkH%PsYdZ+5)F!x1-bHeh^#{m@D6Bs)N4ka?C+2uinLMn{slTS3>%8Bb;o-`Q`&pY8plC4R(Y?7z1Mr*Gd;WCyLp}80#DX4%Fydk zhcvdgTdROAT*e<4`*JG&^0$fTFUWsM9&ta_tJS}o)W6&0FS2f({GH|B@77O84tLH- zjK)|LV*(~>Ih>)c>e2p%Yyr4N=CCJ>AMIa=zadqWtHN68WF~HnEt#6wA=B9$G=h z{{K~EpY0-CQ4BtGu{xFjqmEPHPtafRNw@^^`jivx+P^zwi4PG0|A zA}_3a)9(^pTYi_Qel(D%nK+P0O&Lhkl67+i5@|ByxSq^^fltu2h4J@%0XONlaToW| zw&Qn+c61#4U7{0RZ;LzYKN^3R=t=)B(Xf0V(Rl55iKbh>OEmZXF402I(sLJomuN+0 z)$bAy#WR3Gf1~_HQdpHZhSF=tyEpL;r;kMai|$QWdWL(QHINuBtog=3Vk}vV-2Q>Y z1acCx!aEuV5}oY>iOG(qqOdQFGN5bkK%yJfx5N#KgKj^rn{OJ`hx{*f@8m39V`)o=KB%0A8&u5hZIkNRe9RK$yu~FDjjj{zH?ELO$!+PhG+f%gs zAB)09dc{}O-(NSDhQ1Z0D2w69^VL_2s|In5T%mox-}t}7@;_Ox+}P#3J=li>C^uI2 z5P1a0(8mAYjyMOf6J31z-N@5>&>&woDg&BM^7p0nKeqD+B1dnnvX6*3QplhQIdq`^ zSK=Uh^`D&}`@XGhL2lP>-k}dSj)^(rxcPzoj>(E|>G%07ed+Yva%~)3qIbTk++eF* zp5p2<-Sed=riO{3`hUJ=*K5V zqf(hRmaIB&zcq3KlJrUBWYo&EgqW7QWYw`E}Wl`w=+amj|7|$Q= zCw9VA*Tr!w)5)2bjS}>J&AI<#&C~A|sq^haBCIfv`B+bsV~zf)w`&_XWx6*EW-+{!Wyi_dTd17CieF>|3P+s zsC{u>y`?_OlRb_b)V+=3ZaU2VCJXD|sW)RC+ltC@pU|K4>%?YpZAB@PuNeP7!<+_s z?ILyDtDguJ^j#>7Kiscwds!VP{KzX05`SOyAo1SbM~TA=9wZJed6f9c@<)ju55AK) zs4O^8bvN;&m3I>FuDh4mKjU6vU+;~?4|m>A{9wti6ML86NHjK%3k_A{Li5{WLetCU z)+9a`TJDVrd;Hfv96(u(Jf?3ejx{@`-=T8wpb+<@JKAjy;z@0L=Tx%ej;YtHthxC9 zed76$*)+m9g#N_eyx5;OB93FI z=)ISCYsCFT`K+6Xox5))c5S(z_`BXaiMak;%I|v~JW70Dnext@yNTT^?3LeCuiG#m8O zpb5=rS)?BaIkYYi_DWH>DxT}OiQC9+D+*cRcj=wkBEFrXaG(AVJ^PFJAB)&d+DI4> z7Wb|jv{T+hauy#UIUH5OQ+J9&Ek-(ydl_cV7KM7$^%jLRdd>MOe^&qh=Y6yM7h|2* z&z>oa|Np|UQ0%x*`(Xmv`&(mr<<&{?od2e8h@6V)hmr6d@MwAn>9RzFQ(VN?Y-}KHY}wtL%rjS<1|^fm%nAy zb79B0=R(94X?7l(exVo8!hxKa?WeL z_gtu4=ibE=*T&mL?m<%DdzJ8g^xEBg11PjVzOC&{&mfID9B@vv`#(e;K^*^oj7%ci z%P)|X?(W!!uZ}@p-5=-WJstl);-mQw!wKm66s$q{d@-H)z+B z`2VV5;R?Mr`UZ>-xJti{`tj-wb=TAJ|HIUWe7&(gyy^GbxQn=Ve8n5`%CC#UL&pOc zR3VRGD28Js3iC4DV__Z2iQbF!w;r}nBAIpE(e1r@&TUoRGd(}WdmZIHd&k`qp9`bK zF&4#`fJvB)shEybmFMamRG;*mFMF56-|#W9qX%FFxS%^K%#MJbZ6>3>%~meFgKkttNr zccFfVviYDgdatsY+#@XZA?za$AV)t$9zpCw=uoG}b$O3DP9pEP$8lFy8-T37#{Ry= zzd+VLQvZ`r{R_*5yC#DsZd@~&jd>0dqzjJ&Yd3w(j-^n!J2^!M;!zcNNkMjkfg`WLTJa^2HLg|qsF&+Gp_tDpF!erj=C!WCS_b=<^l+(p|; z{qN{NC%Vv$JbKX3t^eD9H0{>cO|~4?w@v08x020cio<>JJj4J7{jK-D#5r?{!!z{G zm-T;dDGo#F!_nh7|C;{qMf$(Vk-{pM7l+a0SR_a2PbDWHj$ufNr*^P$1n+C_@IPh7 zNtfd~GCfNFw=~yCa|*Q|2)`&C8G8Nk;@?~UXOcK8tZnhucde~LpNju*{hwF+aUiS46o+Nx z3dCnI&of;`uRX4h`vdQtjC%~kxy5mg@fyFa#d>VSW|V)%+WurI%20t_*n@pIgd@m* z=dn#vnE$fHT$fN}{GEE{yVjWZd;ha9ge2Kp=lPO-|LVDtO~|1Gd7Qv0oJRlG%@-uk z;R5>JXGfE9{hur3Rb0nS+(z6}=q`C54>5q8u16}jep$k&Hq%p+;Pu~z5`+1 zGqi(e>HnK0|0A_q{@*G8t3T6Z2GK??tp9Y+7(@(reT6aIZxsy=Bk7~j=KZ!S<2%UC z<=)AwAJzZzcn=Em|EtvhY5D)C`k!8ypX+~Gw;2~Ij$%x}BuvIsOh^CUn*Xi+)cf_~ zFw=40*X<=imS8UCq4E{wC%F)du@p&RRm!zx^jdj5^{Rc<=&MlwqO=K1&yZH>F6^(X zZb-3(YuBkC_?ip*>(c92o44kgHCT)F_;~-%M#pW}*xxtQ^AFVX=%UB5@o|hm4|}^| zoch-}O}Es)WXmJx$X*Di7o;@E+Gg*0&pkG2<(?5@H=X6cfPx7*U0^wJYeD#?^TYS5`SlR3O zjk2GPu$b1k=V1orUosa>9*OTdh5Rq9fA8A3j>K&;d(r$?@;-9vzl!a~(|?(7?WDFO zo~*0YGfFhYji5Oe3QXuJ_GW zkCPt<`({xqti1Z!FvfyLYsrdV z#d!zzAEze|8vnP~xJ7y`dpw2B^sR_}sxhr`%wru&g_WTKyRZlQZ~%qt3*9Mn)IV?iMU_!ABb!1 z4|UCOjKpY+MKLB|5(??R_Xp|U%fEphG)QOTHR<6f;t z^jdy_xc>iidY*mpwEsWe>nwf1!sD2qtommrJbeHyOY=l|9@!Vw!NTI$gB%Luf0T2nd(t;7 z_KRvG4VI@6=l>n?AIFeF22IGJ19=qI|C!^N-tug(dB%@CYclV+hisUl{WpVs=bUD; z#qr12|Jg1)j`@rGpZ42pprXP2g@5Ar6n8)0KwUQSlen5ms&%(bIVw-xSc+wcK8(2k-wJx?LG|x`>krb`phvx$f1v)oqy8n= z3ai{?3_ zGQNv?a=S4aJ3jP%oYcPlrtj)|{FLH(tIoHgZ>r6Bj`oYvZ=6RQ>(RB>*beE=@0b3! zrGJ6%Z;9`3x$lpBJpYuR=#KUO(Y$F;*dxAuIDkXQW#yCA=H=2m_dfO^9iu1FGef>o zPIT|!Ln2ec;vSS4vfQ|r#M3G(HTg$7jf)$O+40IKA^l`ER0rI{IkzjB|hJ9`FAA*7v2E{C;fz2Ib;c?DOoLVobm! zlz+io`k$JoPM?bYH~veQPR@+_XV?Q|2`au-B7eOU=F;b3KKkD^cAkvo#bPp!4O~i= z-FV5GOD~1(cU}sW=8Atm@wsqcnwI%(1q%Dm3tvUAHGe#{x+JWjuSNax6783gkT#dA z&TmInemVU8rZ0#0j(;f}&dv&ls%C_rY%=$2%NN7J4b#Gb@l(Q&c1{cLuKPmRzxoU2 z%YHumuxd*9!J^NFz4XRmpAHR!KOLIQOKMv3X}*!qgqG}Q!g~L;5u33Ur6|L(vtKbz zD-nLWX_o%4uY{u?$Opn>AIvd%zG_^N^&yMGKRozkNa_o#Njw{>*XjTIP(KFJXFv70 z|LgpSkh%CN>o!abe=}ub*d>lVs5t(G@YaYap?utnVdtn9!>$*ngujdT?Y9H*oN-?a z-BdiZFtR!sLR^5TJ!PXh{e|dfNy$y51 zC3@dity}PAeb@A>=>N@2;W~K}6+dPNo6p@WZRMXV3b%#b#eL+4%rX8As*cYI!yS)Al0KRoi`qGJ_;Tii^prWF4)vp~ zm4V#r`n&x;0a^MaviVi(0+Le^_Y&$s*KutFbUe^jz;wUOMD-Nw4;gPzSbr$3T*?}M z(9KUo)?eh`P!>Gt-%sFG1>nHzbCm2D-ib|Uq$x5As>?czbXl9$@SQX&8TGS zY$dBU*$;&*Ly}%W?n3P~Wiis1wXg7`|G%1@S);CtW9)0^sFx|Zp9vvC;vcQ9M`7W?;}y0!Ns`~R}A8g}ZG z$Ip2CIOV-KbHd{o`3$}D197Woo9N}n7ZmJ&b$0hG^{=`$?j2q!-V5=&b58Q0^T|A_ zZb(w9>)dl;1zjSU>|GMuQzo%SR82?X?^>ShUU7Bq!o>MrDGdPC} zxP&WcQb-o%dNSp==k)hXPtZRxf0h2Km(~)n}-%PgMVsG!1uXe~+h22 z-W3{N(*{SA^P08CTOP5;$sAhG%U|l{9fD$PRJ^4 z>ZJT!LEnWvNXe_U_`~%P`hISW@=sa6&ssfkp5FEi(t}u@?sHwm&-ewtJ}exfA3?u+ z?hoeT(vwIbgC^wAfjmy2|NXfB<1lNVTmQjv->0l2OP;|wTtM?Z{Y~T*T*YEh?ZKCQtHPh!zjHpae|NBdce8(YvVXI~!vWXa$3qO@c zav1w}$nY?dUOR&QORtFYUyQ-3-{RgjxW@$hSNLe>j72e;v(JSIWY#_8+`}aIJIV26 zbQtU3i7tKnPw&q?R=>^$MM|8th;s_k$epo<$eE zTl(_S_1OO>{dc4v%_rIa$E6=Rw4(BXK40-H#!@Upa)k7cF_wZ}JHx$U6@3lr2fH_< zol`e)`0v$EYyG|+8&R&!mq*1{jZ3p8`c`43C_@EyVGpWz%m4f3f7GhK>XZkuj)-GK z>mS*x*Et7p2uIKpJ{^vcNus|0g4!Cgji&<#Kf-^7L4ybf`B^&`+VTujfJecd!2&Upy;7wtDzhGMx>5J$8+x`K2{Ly27fBsYEm-{{c`J!-_ zypM;7Yj6&bgZ6rsztj%0XTdY{q3AOQW;ogV%jd#Kvj5Y@D3D`Oj0u>8$(V}in2Fga z!CcJ4d_@1>&Z3dV==nw*$29^9>&K`kWBIaJSo1V~ezN=*=Jb)PkbBYdV|%Tk$2Cdf z8W3yg>rq($OB%b6%l~Bc4dbtndSLt&>gZ`?)D88xE}ctVrw=H^{nIwmH)AVG(WVY= zzo?GtRsW-Fue#~5I!gT-_xfpgRsDyiEiv`MU83 zD+h;t^jLSNaDaXYg?)*{o8F-gBaa9>hGu>8Nis{O$P9AVJ`py%|0a58w>bz$jSVn% zuZw@7hs>ioq5hxfKgO#kQHS&t^}n#V&T-DQANOx`IL_lllwV|jdq(Xj?2q_?^m`WF z$kTh!Ag;z~(mzA`he-bz=|_&HUA;{a^I{ z`FHAk18HP@L$R;uzH1(00E700?e9c8mmG@WXuGBTE01?5M>= z{j5#Ga_B&uI~mKqJiT^^=Ql_BNk4^p-$L9wFzzuQ*FdY@J1*25w8sC@aiJDx4m5dKiT?facKL{_<{4r4_r2Wfb6g)l%4 z`l0qSh9Y;%e_%MhbB45yH_m`Q8a*4NO}~1#|LpP~Wv}aNJNiP{{+jP^k@@#EFNCqq zD@GMPwc35TrZA7|*Pgv}$8`d?=KC3N_IM?gx`v2Z{Gg_4SS>(nTe}ZQ3c&>QnVLlcjCoGG_ z^v+?vdn~0dLk~TVZggRVuuAWD6}bjU`dV^5s_3!rxfUB8Z$^e*k06hvt>$c#(&21jmrmu8(UBCh48p z8?i4iMbDt8*Rzq|y6-&p6<5mlP4OCZAdh6b{L?7U(4Uk)r|72<*93@b1E#0RKjax< z=g|MxFN6!^C0xN(T*uS&jp9>u-22Pt!)>zfZ(j&^$- z8vnm34!r??X@7t(*e9T&I1GB%cZ#7Hj*%FR^75h(_ZJ*X&+;$kMkPWqeF7%o>G-jG z(*EfGrR?rwtgi0G->(4%g~L$c2o>s}7%}3naD)_Z@j57`7%^gs5hJD;F=E7@c*PV` zjDrkk7=$BKs6!npR2XN@JncP?d(Y$ETTC%xiYca;Vv6ZF#fTB7aEn*G!nv=t5w5m9 zx4-+xJD<;5`}VY?DY(wp}hW9;1Nn28&3 zBj(^%+=dJL?-m(CwpvyDwMt+_>*?0I{-72eo zmE)UT*dJ*l`;U%=@*mRZMC^a4l}-baXv6vbhcvynlnq}6dKS4#b_W^HsSj_6i~SF8 zl5ZpSKfFV}i+uls^y9k1R9apwU z8^C3)ecJ@5bj`CQvV&qitU2V%WH?*|A+pZ%<3hg8ZJ7>PJ{w+5r> zjjpYMUow_H9?il@;RM;_dI$c%#y0WaUtwc0(J|%L2AD)%g==sfreh`sFW1jX-WYZE zIgYhBzn)`Ue_)RI0ev&Kl5zcwJK}LqYNLKjJHTs2;V$93aWC#ikGfYcx@rnTx492l zb*~)y=oxjhHu3EV^BBg+M@Z8YJ1 z2p+=|cnaB;QQ>K_ZZiKD4QMPm=Tj!8@iF<9ajyMsj(G-aunra2ge};HF8%!7eC-}S zckc@R^(y|gzV$4b<7>C@Z(Dcsf6 zb=A@vkXymG7MEGY*GA0u)yMfovHnM|J?7dS*M=H;gZ6^hhn^PhL6RQ#F*rZZziz4f zmrg^e~lQw`G2$fuXO)A z-K+ckyZv|l$9vLwA0OaDq*uBBqsBbYdl$O@`J=-p^v{s1lJ-{rG0**rKPj%d)jAC1 zm#CdFI-DhkYI*Jk;jFW!R->uaxKUrN~*vgMfkk2ZO|{e=8~SRO~ZTmDbU z`w8zY#r|jXMeeqixHMB_T+44JJ?u%V-+n-ubidypz(Xh>HY{ZE2tDrk^%(gCp2C3o%F|^3J?c2*8mvPF zzS+O9Nq7sk;pbbw7Y1K3|Km@}rr(=iB5vR>qr)z84=OIxuJhusP<8aX;br{ln6(CH z|4Z0;#yX5A{w4fce%vKrZk>8Xc=5pZ!Y`&=5w@voS6UnJrK$fKemUX0_G0_j@IwD3 zVe^zr!j@^5grA=_)=Jvjv1#_D;Xf)b4dur!3+oFn3l&E%3mZ;c8fyL4fORu25C1;y z^6*Oiu#gmPLmEBEV#mQ@;aBESy<~ll%A*&D?f7Nt;_!yw-o&=Wqrxw+dus@93s)CB z8}?M`zw3TByteh(@Sjyb3$^{Fp}uT&sN4Os(75rZp`qZXzS$>3)8e0o=Jij8hmVG~;>Y<0kLqK1G`u6tckv$H#|QWjAK?>xhLfl(G5$b1Km!`J3p5>3 zuh?h&0h)`oyF2Ea{{NT4XEEfL?o)q!7aQ7*o~hbDv>EhG=Kmv$99opYt!P6#^6ek9 zl;b7*e{BQp#TSK<(#o&@Yp%~|dfYoPrEWBq-kXv(#?vPvcS73g5&89(YKDhN;;I*4 z6s{t#L9II1b>wu^jB#I%Z^TUD8<0G}K1eUWjzd4YQP2NpOByo#e|}=q6zzWtwaK9w zH#%nyZpCdVzsVZTLU-E6L78i$Zyo!t}G1=;s^Cz4PM5+)7y|n589V_ z7Pmhevh+9bCgQlJxEA=^^j`NL*JgQ#{x0I46v2xKhFQ3Qg=A>To~(r z#$zHTp?2~b>-Vn-SJ4}5{5P(lUx(&0_rKNsr`G5TS`(&=n~586Bj(^%+=e@lUvEjf zRII1mC44u^^%Z4X%;lulC5->^J{oXg{MiKc^bzbd^6du0jIDF*19%9J;4wUbE&&If`p9@2(#3?U^lOr)2V^RKiVHi(N#3WpW!ONZt*O1qt z{lt(kot%jqF$d{mo?pMUL+HIL`G(8YA?+hr@!MhN#$zs)|C9r9^6R_oRKHXI zjO)8RBCdLoHJ!;PP`kkUCZ9&lJadQK{|)qKumZC8GOp9R*HS!H})skV2rcfR%kwkm@xJ_U2hG_Z%BU;kzws zUraZ~)04_>B9jcN}V+(}OJDz?*m*@8Df@&FB9kj;Za% zh4~-q*s-i_@r*7!pUs|+XPH8p-nrg0^ZR?!c^@C(L&Um4YPNQKdhc@fe}Z*>=$|3C zg#AZmvHM?XjDWam>3vRqiQ3i11du~^xc^f3<=Dn)=5Yv*L{eJKXqx3di{1aBulxUf zuZ?J$C0~}PBa_Y3)Xxv_(Z4RgMmv8j;{2fOPmK+rUl?z{QT=?qGGXUAA3a;9ey)Ds zvPk`ZiTXd;&JXA~t^SWRy|Z7v{7QcDCB}tH>ndD>>o6UI=JU-Y`{jol$N_u#Wxp^+ zfIbJeB97(h;a6l;q;C_>`L8T8=#zf?2xDx---WwzFVaU`1Ma8SdB^q0uW!}uxzO|N zKjy_YkJG2-n1}EP9>WuO3Qyx1bWPy5PqY5OlyiQ3-{Et9d-ec7ejmSmMPX=N#g9k( zJbwEUetU`a2WIi(rxu2Ob0YiiVecO*_zaZSU9gT^z^8_-;;K7<6C_G<&JTTCI`80JyoWe9Acgnoz52m>#*7Xh(8nXUnE%fw&inuT z*AK;2^MgMkKS3@1Gx8*A=nae9*AjlzYeU25;*#UsSF!u#KgV_c>RR1D8V)#apLo}r zAd`sxe}4Yg3H`zN()n@B?S>BPZPJJQ%Ka;6^Xos7z4CmY@;GybeXo-LE9L*q?td=( zi*|G%g*Z>R6YZ{JxO7HhG{zz=F6Fw%)8n2p(f^-FpM;!n78z;x;VN;}p2Icdb*QCJ zCugE&%-8<^UTs#wHzG-obN`wSvJc3Qe=Ff5%kPcya8s51k0iZW9-iZzTX7riz+Jc- z_u_tZCHVi#)IV0Je~^8p>K|lQI7hbZm;cdLFE7@}{~392_c{N*vr_qzQEq=-zC0kk zhmidP|CxLOab8|0gK8}Hy<#JxDm+;iNEqx`Z$ zdu(Vk6`#G-y7uJ2?}mmC$$st7ACVQmQ%||i+V=F%a1x*6OPs}!SJZzo5`#^~|9?8n zejcO4XyG{jZ!9?;6EO*K&D5*NYj7Q=qjs_Uzfjppk82t<;0F4QXr3bf&zArB1WhIK zf7RFaJ;6?7lz;4af}SK}`}cdUDfZ38wzai;FA8&{aVz57&)di@cDx(;b#~ePJ~ltI zl8t8XbK3k{l&7s|Q?|CVtrzUyYVH4sebjeI=Ptx?Zxz?G^FQ&elnf7di_4F5Tg8{2 z$L`?~*K@DacnFUmwt?5+F?zIp4S0h76rxZ6&Gm!p7P_6h1$nDbEfnC^xbUcP{UQ6$F?LBCqCy|>VZQ%_1ux?!Je^mcj>K!4D9cXi0 z+<&;H#(B%#7s|C^_QW*kaW3Hb`^@{ln|&jd?0>g!LVX}V{)cQnpx(b$Kl8)#Oic5^ zq2UekO}veF(6vhbFO|p0UirN5u)KvVJ-3(r&+z}H-?pCrFCTU+k~f#g|753d9RKsK zbjmMQcPHP+`SCv=(9f@*^&$Nuq@joZY&ygSh;~Q@* z8WO$~K8xfb<=PSD+_-q3<}2=$-!Kw`O~b=zaxBJUBC;P}9ww1j;W|u5|JsYfOmg5s z&t07}KKFHKFICQpPpweCFEciuUU6ly{BG|4Y4(8)$@dQ(R))8-4;XwkggK786}RCI z+=aUl_d6+n{PNKMTm4_casJ=H?e_Ad_g}BhPgYCwA#%NOS&xv9q4t<}-*0RJz0ox_ ztSSgk(Vs?h3EvWlMQi}-j_d!|U*0fV|1+AV>X$}xr~c=y*8O+RGgyOlsEG2kd)9v% z_bTnG*Z+^6`TFnCw^;xEJpK0z^xu;$C(QppZT|mWbN$iLt^XhCg#P~;;{(R%-$xtT zwNZ8;rOh(EMF0N6D?-;w;{#S3A23(?=&NL3cCs&Ij*q*^^`yPmE#x+A#|~6{q21wk z`Zu+C)*DyQaM1qr8T->G>|G$uB%0Cgo_9HZ4{FhXwEIrYi_gIKmvV1ti}A>9*8g9o z|KBlvuD?fIb*VV=4b&d8p91+dY6?78_uq(jgx^JSpXa*YbME(?9b4xe)R!6ej7Bse zvC%l?R(qY`J?FfS5AY#A!YB9)C(*^mcC)oTWSk4rM`nbxWX|{9vfJF3gt-PazRzy{ zzq)10^V1)ap9}N3G}aY+pNQkHzZ5=;A-jD0m)b{;EU(j-MvldJOvEJoT;6M+KQvrL z&+k9WR$oI;i%a1;dhb+a1DiXYJ`=eK^6xbH7k%0gZV*?!*?1jt4&vS!x01IZ?xE43 z4&QiOo1^euNFMt-|DIA$p=Zm7hP%bzi~I2a9>ODd4Bu=&ctZFoJcIHd7$;A*PgA!e zHyC$NaVH<0o)(wFCVFr5?N%3rE%a^3W#nCCy5;4C{P&+-5w?pT_}vv@2e}LV#$E1- zagQ2rh&AYBX>OM-fpOc@9FTZqHsHMlTi(d*imMI66H;I+L7wpeTAM$HuCPt!*t;)~;-~2cBPdK-p{gdXm#{aW{ zDWs3He@Fd(!2G;J=I0&M7I5770KR$ZjP`?L>Idl39^XAh{bHK-12VsEKbaNIk)vJ1 z*mymzYdkp-lW-NTLESj_$7VK2yHUDLWI~!r|JmH_KMRdM-P8i2nE8WH}!pKmLB2=Q_(d`VsFJpL9L34w4^d zNM2Zf!7*{1e$y(?zs9>sct_;gWAY}}osplf)*g8MckFLvtb?>3Lf1a||A72Y_8yV{ z=^4l7$9L>h|30GqZwmjz_uM|2|A7?J=tO>8oBH4*(s~TJeSBJ7-Rqp?+<5?9S9-b8Le?ObDu$?d4AieoJw9wW_FemLjSuUq#)nP2$A3LMi*onpizkH_s{S=>9y2LyTRAcO z;^@TC;J3k#uMXQ!3=hBb?p_jC>AmlmIwbtc{UrU~hBW%G{a$$Gh_OXiekc5TvNasl zYj@5PH|=}jSN6r)ar%l-HT#P2a_bf0mjz?ZVf)umsn6`Ch2z67^qXy)@!hbs^}FH4 zssA(leBu8L1I7=O4>L~xa^q2oysuf_SBZDxzp{8E$}8=QN4|r>JB@p}PrJv1_Qex^ z5AWjxl>g*A;X`szd)Hw3*W>?M*!@2l|M##qA-{cs&u|i-<4a@%U*Q&gzw{xm$`2Tc z(HM*In22iipFP;E?0r=py4Ue@_|fecgQboqVcFIb1^#r5uER=C?S_oAcVTJ1Db!~OIJ@DLus zV|W5j;c4ud7uV3TPXvAU0`sZ0`}}9Y55sHbBi5AaS1Y(FG|nptb@OM3`lGXzxidm@ z>5R~{#9X$5>(#%sKe*;N$Dt8T6I>UPMXqz2dGgX-hYD=M7Hq?I>_Ats_7~5&$FaR+ zUx{ZU&&NKV+yU(mi@cvD?vu>FpIM%*h5p_kr6 zuXvu{+NVz6FW!29nbY=vbnd$OY|p+T>&9v)q}N(|v@NEAnnS)DG>YpH&LWvAQmz$+ zCg&tF_J36Wub;>NSfZZ3NV&0Ey&U=WhmHIV=e&ux@ebnNHSdz|;eB*DrhAz>#tQxi z`FHyd^FdnChIVuyg)}##&bc2C;2}JM$M6K6Le~QJc@g`(kbPdxKJ)1_ z%h=}?>@(T2pZ!PM5x(vr{_g?ye--;)%D$5^?rG^fgEd%(^l8T^$1CW)uCE9A_W$q8 z|FQjl1p8aS|0Uy|o7G$SVRMyPsLdF2?Yu3>um3ZjUBx!x?MNvs_m8*f?;y8AT|&Jhv&#LiHZMS2 zZ2y0cd>^&qKOjFuzWslp|HVhbpCCEe|4wxsC9cE&{9XJ1LiK7S(TvZWa}wE4t@T5m z#gIL|e~iRGZU3Jk|DkP`{HMPDx9$IH<=Zm8NnHOTjsYAk&D0_JlkYN?9>@QTC(qB@ z?c)o?zQ-(b*xrjDB*n+_y`EJSn-)kcx2S!=@oZN*y zs70Lr*FYxGhIC|}I^I#=EIq&8%T(Vh{S91Le_(=dc}i^mHNSk;9M?8SJ#|iK5>Bk2 z6Ow4|Hs9DeZ{lsdgKy6Ndsn#2G2Q5ye>4C8X8!-pA%pCKn?r8y&7mb>etOFM^j7oJ zH=CbcWqvx+8_iEgwcq0Wm-nRgK0ZKg;T&yObHa!8#+9yZ`J8k4pFeeB{ei`X9u5YL7t9O0=vtzgG zLmHjvI_mz8yFaq`jQR8Q%#k_ltNHVu;gD*10V6RQWAV8>GM=1>x+R|Pai?+qUoZ|;++DaE_u_s$fWMpnr_P^1ew_|>;vvU8g2(U#V*URq@@YJS zu449=9sO_T|83O%hqkHmKRVc^6w>rg?Ee4^HIiHW)#AUx27AmyWZ@AN%qoeo% zzg1U{=bHbQCEviCcpLAa{QK%0A@asJ;2 zp@?G#rVG!+`T2i0(9h5RyOBNz|9|KIrCswi&db-ojpezOejDzlwxfl220n{!(H~;S;`uX{PkI)~(-_HMgLR`DHgQv)+kv?%zc!peq6u+*=Z*hN{ zb;1?M^7(V<+ozsFb}iR_fu4=pFVM%9W)R2N<>xPsQ9meFKVXB~9oNC;rjVw0E@eOG zv+s+(QGbf{3N|>olAXnE)xPa-*pE%FVGFj!Yk89IP{~K7=g0ruZoMgbZA$;edgCAH zjjpA^F}3stG@p?+5@;%;i;MIB+Q@XI#J!MNd~^Qa8^UkK_)_(jWy*dsKmRZD|2_Zj zZRfm$ckv!#{(qnR03YJQ{69Ybx90z~8lQ*%mHB@kN#_%MhLfmWeQy5W=k)XQZ@#3T z#ox~VD_bFdV@QqXT&UgSC!@kh`e+Qswe{5x`Y$)fPJQ)vx{k9~mF;q&9$;&5iNf-MAO`BaYvCfP4s#;4z#Z|Mvv_{P@48=+EHq z$Nv?D;o_gh8mvQp{day^1^xW`@A>vW{Xfm)_@+qk>zc6E8vA9|*;}{U`u{bf!zSmX zcJnXIwckSDhVAH?XMBM1$bDqSIssWSXIyn1zqFoz+OU-WTgv}kChgRBLNfE6&`j=d z?k?;>EgFzS8`9`9&bfP)@dw5o^pbtX9c0ExA33xf_>T5PeY!_nAK4+Ca$I_)_5V?Q z%(>FZ;tjlsT0U~kaciy7&#yo54*gv;Ynw>&)6cIz@SeDD=I{4~Kfs5G^Y7!ln~&%l z{-o`Wo%w|R8BSum@40S;``@SzS7jX6R&BGsnS^gAd0P7)%3lrPbH{#(vlvp#USK3f zQb#}b6mS}_rm^f z^v*^44<0m^!1@evZhx$Qq@@|h{gkP*#J-Cj;n;?sCEtj;y~1MsWAJyz%?Rh$A7KCA zp|>k*-X&9|#!s(Or>4J;574tv`HVjPLVkP-nLEzM@!OkdXf^&D`EhZ@>|KP1` zYzF9#<_MK?khk4-o14e#QFE?2I^B~DRnE%lxFR2>;H?tf!?S-5!e4Of7Muc z;pW(PpiYsHev|%*IpX5JLizRoFZ2FV_CpeP8&VnfK3}`RBK2?KyO5pjxub8M_dwRU zu6lKb{5m}g+4PlcJCfbu^i-MuuWzPL-+l(!QvLs{^#AMcZatv?oNPa$ z|DQ|=r^!yowV#?49+S=!cnVJ=UF;Y3SZT;wL=RZn^{rHp zbt%thy>vE9hivj}5}r%)jC5Rki}=<`_lNcx_m^?)-*~_I?_;6w192^;zb${;r?{Vs z^+EZsJ@J2$o-B=7{*&Ij()};@fAluQF@+cQKUnHHimN7j$Si7?O$u+2Z=%Mz`E?Pb z^|tUk$nSq}P(8!{#{Cca@0l3h6+iHkk>Ne^eSClq@ew{j)*d*Yktgx<8P|m`$-&?J z-{CCzi>1@Uw&H&aTMu6sUYvhj_~l{!Wvj-9$|C(`d#8tA73)u%J3ds+A0J*W`fm8O zYuYvDdts-3yH{4LJM6YMM1z&pBUxwC#FN_`eJT&)z9+Cxr4T&iyL(}n-p|RjEp+TLl{*bYqM@H+5vA@;+ zk?M0J^~qcus{8p>C+wYe)YzYT-&Dc>2>*`Z^pU8T@o!2M-s{e7x_^_=+|Kr^6h4QxwLwUF;Y+tC)5HBtKUZ|WsG3+Rv zWFDM$O=*usnX+Q-qJr?sV(p5)hwNK?67dr;30L77T!+EmPYTn?{;TZ~K@L1>d=Ysg z=HOQJE>}NWEPvDQK(1Jsh0FxDex)+!xHX-{-;H~5KhmxzCH?_=b&>slyvK*=kKqy2 zvhOwGpP)Bx)Q8n=Ttw{ue^&kfj5!*JYs55dEeva0?H9<8UZ?Fnj_-TQIjP0k_h(pp zhyD!Kpob0ZMISQABFC22v6uDT?B6Q(e?6PJlWk?kliKK;CrH=171)F=*oN)cfnE64 z{DMO17fauNXOTmTK8aSet=89o4*ECeCsg~dJ<_Q~1CmiM<$sS;XP`GGTpQB#9>hHX zlEQI}WD~OD-oTr98yD;Y+xL#}yLcb@{fG4{eL(*ZAK??!Z8rX=O8x(^`hU!qC)EED z*O-XwPsjTIXO26G&+#SBVn_pjz^s^f&`hz516 z#zo%aeDf%gJmS6Wi~TF!;~{e!UE^qs#du7_BwU4S(0-@cHf7$8yPB;-#6L2j{eQ{NB-gd!>#K7 zXhvLr?=ETGjeBuF9>7C*1YI-OfAk=>wg27xzk}-kXhXc$jxnA;(&$9LzWv9fGZ5P< zZuw4lf*$)HpNes>>l-7V!5R$e2l!O`%etsPYzzUp30trY?cL^Hk*Rw1e|dZdeHZp1 zmJz+^bN?CS$ER1R|E)4_!tb?cKr-sBMWJSyeq4HE<+;B9>}%?3!g24kBogcS|M=GY zzh!)B$E3(KKRip%&;Pqr9gIKS%ir%)HfQ+ZS^jvAtXs_g)ppQ;Ml>OTB%1MtG~UG9 zcn9y|J-m+(&{e?yM-O_7rC%idS<)|w>HGd&ODliBjj!Lnfd9XU|G&ib(L3kz|EHQm zAf1o!2|h#ZWd6VSlk`UZeuMUm&*@(xjva_=118A0hum3lLmK5pjKtv8MPW2K7UMC% zPn$^gUuJK2a_uxw=T zH{eG6!})(@@;`EDSuFpf?YQz;`S5r1|JdI-(zzA4;SQvIN2ykJgWkK7eZk%Id-1pP z|JHwT%Gfh&(E05F#JMLAk&mF}l>B#?Uya9vpTOVF|LZ!Y{6f!isRqDN{tgFdf(tEYD_N=n58~q*RV%q>R>HvM{fATxwUGW25=BT`G{=i$td0+%$fuC~b3u5TtoW00N&Nn2=xeYp#$gi%TZV-#WbD7)Mh^UBSlCYP zz%J}TUDen2zsUP2l>dvpFEYRX#4Pszd1Hzk*MKD2kVX%(=-Mm)pa;F^Lk3yokYE3Q ziu^zM>+yf|IJWh?{dq$gZ=!uOKbws6e%~SAMOxf@D<|@BkgY;J8 zlF8{vIi`*+tVhFQ*M|JMWUE~BO7?$+dwTRt$iDVpVW#76z>S!LTX7q@PFw!~`F5W9 z^7ImUd#*f=Tq*y59Q$9){*&$d+5f}JR;1~j7`T%EuMQCRHygNQ__xLn-XZ)9TWFj zN}{gxY^X0g>$kI^5l#H81d`ony}PsFUg&+rlcvc-%y5g!ANssJ7?E=4(Z=hzL^X9uRlwUhEyeTd@+kK(Qaf!KS&2Lryv%%}o z?z-M~Ov-hp51kF~&?_EfchMt%_R5!i@@Iz33dj8&-*wD;cpo3&L-fCXCVWH=yzo`{ zg!~LAG1&1{_?-L_XE7ux|4;S&(NN+#XSpu-l348dyS`@EKin}RF&bkr9uqMMSK(Xp zkJ+6I?SB(o-#FjjKK9=|wvp{e)c?u+{-R{(>Vj~MG_J#RRCk{VGs&8RXF~0vGok!< zL&IF*tT8yT4Y0A$x8qw*kV)ZYvTVM5K35ye2>$F6DP@L{4?8J z#6}}gz(yDIBgv$5TKbI-LL1^-z7BqQ${3&g`rsw@_9-;Jzg97ULsj#u9QVmZ9#1`oU@cUGEyu)a@FOq&L_2&$&awa>uQ}O02^9@dvBv=g04r z(#w#WuS{@GtV8$7KkLP9#Aa;8;0r@TC0T`?7+hj9e`Nc%?P8h}UX zy(PX~9HSpcuAgo2U+339%#Z(H$j3$F0Y2>^zU>h{F4?@F?wk`ih0{2Lg66Qc5F_xd z_5aif^8R-L8;$(>e|-B^{CDgBRUaN6ilj3J<1hhn4Zs?dzv7$Yi#H&)9~b+rnaxgi ztAnUJG?8n4fAJe!w^;p9XX`@r_^o%U>zl3afGj;XSskfr zc$n(AX_$dosMR-7qb*}Lz4563cT5R=E}D;fHsTVeJS+77urSOMKXCQ1FrQq2g;<2e zSc0WkhUHj+l^Fc%u&|1(JFWd=g#Q@h`A?JPY-vyNAM|E&wPQ+AhV?k#PZs@$jlx|= z`JczVW3u;*{7+_tvt(|u{O>uo>PKlqJ30{eWsZ9ccB1Ye`^O%|yxAyE#k`pylW3kL zpFKPzZ1q2tsKQR{Mh)uGHAViM=J}Gn@@pTNU!O~!%b{g~{5@Iz7T@mo4l?EVG}-CD z+SeO{^OiD2T5%3wE14FTssug1uh&N9$i@2WUy2Uq`^f!hOeh2Py0_iu%7A7vmH`L+b`Xbf7=v#O4M)g+9K}G#&~Pk<)xVFE zajaF@D&Jda2q%P3;WT33UUnGUP8PJtABb}RD{d(Y8>}bTd&)cVelj)QQMY#^-)89} z99M)fDA)cmj+}sE4F0wtOeXukKRiq!W8X;h|EJQYVFqTQT3voNS%NtCZ!S3xHS~s+ z>XoYt!hGQc$orM-LX)~>VygZpW5Mf<9d982ZvDT?(cz4HTj;z+Sd1kw4=|KrExPpM zck`w5{rijg-}Cw3$nUeq=WaP+{P}6)&-u#j-R1|>pBvks-f8}SDgR#^K>ZQ^{|Np+ znkMl73;F;2_-1mc|67LT$Qqluf?SCWKl4rK2kjc?|BLy|42_9@$aGbu8CK>N9K^5g$5_3iEFAE58B=YuRg=eYBB zxqgJYaJTe}-@?SRQLB#oean5gC z`#-J$8q0#t+48>l)LdnO@EPgUZZ%%LYG^2E^(~WyVsZ&KsQXk5 zv!+uV!#8Gtx&vw^}5#iVF_mv`RKOP?uc9@&_m8J-zj}^|EIpX|113B#HsM&nbTqG@zY^l@qg+6`ZARGueqog_j%Yb`}0sf z;>)o9#9zavy?+V+p>J)T-{xaW|ETcuaihZvvrdN11=3554!FhRPEo!;XW)!>^Pzi~P12WyM2f)}i{pv=f|?p9?OLpT~##&ErEInk%R4&-=bU zps}H;YHX;knH2V5x3-8Q%8LJ-FwxkPiJ``Raj)&XA}o>SQY^!AtiVdF!fLc1y)Lw& zHSuqu<>>TKDy|Hvf~&Q~Ul;0>U-h$;DJ8CjY?`L5NhxzO>ObUq$8N-CY{iB0qdSC3 z;jRVz|0VqY724NF@ChcI>wC-6a|Omf9#HljQU)GT7Lpypac#cXKHo`JNu&I&;;@^H z>-g7@^%%U}_jaGPKjZiSL$TJYJ|gCVQ2;J>)*W?Z-6i!~q<{AsogL^rJ3eeSnm8+|f8i|96T1E!H2c@3?uD z{_i6F-_!Je`>kDHct@qa>z(?g$xh*}l<@%>{nKQx_`db}?iX1f!2JFk**)=yt+7?Sa}tN$-vP(%n-q$iHwO9f&yN>` zar6l&#$<7yHmD+%U~^omCvBNNXyR z!p)VJ`ws2HFr`@gjy@WDG)(i`4AhP(4ztMFs4*#}ebJ;)LZ6Fyn2+=X&kqaey=2eq zNns&<5pw;mjUCFkw!VT%VX?R+Sc+=457h;s4K4 zSD40sU3Tud$1#xQfluSu)!6W)`q@{*LK%HMHexfjq7qfui8ywBH(7&vB+!Z!2LEX7 z2QrS~%a8+?7lgg!KJ3Q<^se$fFIVS~Uk(cAlr33g&?jHSJaZo(p9zNGk@Ei5-W5OqJ3SIT`KQ6TYEq1MJ zYZm$TzXRXs|8rc2I#vp4dgpF?4qa=k!w-ym`l6&G4|y zam!I7zF`@^dWCY6Z?)kT`CqtMI6u#xT;aIEXA8qhaursi6lGYCjo6H>s6-V89xV(z z$$srP?FWq|q1T`u38dX;3a#|sde3IJFufZ&_C6~vgFervuKVlp4-;G`^6T=Ic>c3o zr+dse=bPh;_X_XBejG%Xx@0%<>&wX37wVVFrQ9Or&lJ}&&GRSYno1q?RI%qj&hz(8 zSFH71{yZccl2)~GABV{r<3MY1gx-&%XgF?s=uu-n;@A%|iRMGru~UaX=J)d7$W!x% zhU4_umvDkSg*etOhd9}MyZdCs-Q}3r&`&LJgMHqwT&E6RjXhPnP zPYeyEzRPibpMYYl? za%Yqcr<4(e-W4{B+loq*-!dvxkvp*)gYtR}*?+Zfgd8xJtKvoP>#I?rRXByUe7`vE zZ5{uw`iO5qT#fW={ZEFz7yA&`{%AzgM)iLrU0<{7Jm4La_3_n=7r{4T@^kw+I zHGE+8#vEB!#{WkH8qtKrX8!;BbL0PyxQ2cl#W5Vm37o=dbnWE-C;0z0{C~dvxAq^@ z4$*Ru|Bp7bqa(%t&+z}rPUp4rug^%QAgzqW2;|ov6kkN|RoCso82UKmR=f6yEk)dUCG!hNgwy zBT8K3T+G9KEWko6!eX>9EDB4=rC5gLNUya1!0Mu~g5Jwk_UJQQNneGW`gL~9xjJ?q z`?*?N^|+!?N|vE^R#8|_ZbVIqu`BFj;~C?4J{}sjii>?-&Fo_n`R6LUQ^Kcl2Kn)CS8C(w zl+Q2%MW|zg>r>h%GGC7`7@=){2LB(;(~P^X8ydzqZX70{7?UvtQ}Iv7ziAt2ox=Z@ zR(lEmpG=M8{}=QBwG~d2M!#{_GsuCz7KT~mY?NRw=3zb-U?CP^F_vKPW6zwgyKb7i zubvq1eX01kc0g*cdI-Ij?e9@XSWeIKAL5+I3-+JwuTBjQEB#h+t$coGL0CnvouU7s zK-)jPajEA&W_T#2m!Wx~=fB+ZM_j96y}0&_b|i8$(#rL%WF?{v>XFZTmlT95;hl(m zKe?mIiFv+*Gse%X^!+10|AqY9{cX!Xy(#Q=UJdGzKr2${M%NPlrh4{&yZ*)@_1`1< zzsdF${NL5;364+GJIP|-Mn*b&5$m%1$eJ0(vtU2{01l#|SpFR6`ikVwlsZ8x|6DyF z?|+P7|0aZR$Z?0!kL)Ekg`;G}>oCGx_qw~fg+5-I7~n> zzFq#xUxz){eexHgt^d3Ee}7Z{_P?fWNIC;Q_Fb&KDU@G+6M1u(B5o?CVFqSlHcBuT zaqRy*vaHm1w=#tJ^w=k~kX#ghzcYlzWNfQkLdN+|OUY$ej*9zkR*$$TwC^klM?8lG z;#OcK(ueHdo+=5e=)I@y-+rPbtfrSDH(`$QcTUJSrVnM}st=ch_2fp>ZY>F$$*rhy zUW0TRGbN!?xC+TdbCkbxLX+bXs1vR?m%m}@oY06SB#=ZiV*cOhyxlQ9)FXjbq|i0* zX6-NL`J2n%yTZJF^ZhgC`)A3VaLeACLo3?Qjt-=dMkm&GE8h{HUAOf5*Ukxh$$@K2 z!alM>8}Ei;#-LwZ67~xpz(E|sVYJu#Kk^9raTMv@{zLpRdhfhBp$Et5Cy=9Ok#XOB zs9t_^I3@lx&Y+;%GhBLes1aXCZ(Q%$USxVc?|#lwDma- z=-c1NU)Bd3=i|+g&W0xS=e}W~i#_h1#zs%k{&h_I*J(BgacocyEy|SE)$;#b`5ztf zb*hAYn#KQ}%1>QU5@tznHfl>tLJ7IT_gHPsq`CCOW_cdX{cQ4ay86&O@%eo`68z^c z^lb?b{FRM=eOOpPUx-Cmj3ro#Wmt|CScy2VXcbwfOsRJd4X*#f{wLG?mv9^-v)VDG zD8qVe#AcMY3=3Pyu7mFXu={s?z3AKO{&%|nO839f{Y$IWy|$sf(ETgNQYYL$I??~L zo5R2l;+%&$_7At7gR(5H|6M8VxQ192xf?aeUO6n(lY@Vl69#|l8=$wM-E+>*e{}!( zy$8f~BPG7aGwMybCgHuvy0+X(*I6UZHBJ+^6Z^0q2XGLFa2WaZ^VqNkAr!%=#Tc?PvOMn8@dIE89)?dl1R;TqpU`n%}mW39!Ai6XZvsVH2ZJ} zMZ)>@bJ^3(1pfDA_IIN>2FmO*tkrfm#xeQ%f9o#_hjj$d9s&GU7g zjjl7+sS**V@8Cy}O@2|c}`}=;ry)q`hFZW*d z9nIRbD;-mXo!E^U)FXjboZEl=>+z4q`}V2dWRR^=&q2#(?eFu>^+Eo_{m19ZkJ9Nz zZWy13OvkuI>d*AJra{kq^%DC2c#LqiO5WKj50MAN9mF9V#^7tl50L#hih)biJ!1Gi z?W0!~`7k^m#5SE1F`nJ;f7Sf{SGDo~_@c1>rBUIO_`%!17f#3DuO1T4kONOn3yAcCA;f2RW+9cRVB%iLa5r$B^StD{cZ=j5r6P!L=tAdv6PsZNhPnr@CeQ zwpIL3{``gg?^66YetKO0|G592?EER1ifQ;EZo)0-(y!jVg8#jYAIUH58|N7o^CQWe z{`?m00Ii3#4;;}xKz0bHR`Mg!>Adz+qr;D+b35+DkCC3`82;ov^xiT2Z@&0_^q(Nd zH_ZxX^qcpQdH=ih8~!)n`9a6!{qNoGix1woXmof~T)FWRN#W)b{8KV9Z*=(eyo~t@XIMP!gf@am4ugyO2RKT-W;}#`(fBxGCRD8 zpD&vd2Aiyv5Qc>fuNH*I{m+y5DF#0-2tOnHjd6XJjOF~=D2-()kMS25h26Ug!t?ZM zW#1lk;+g})!fRu!Ay_;i)S`ZOu{MGUZ1d&DuxWeWIwmyfGixrm)cYF~T2GJD#yUT9Q}VYLzCZHu%-0};pa1M2rrb(44bFT4BMvM5Pos+;!yr0We5AS zecZosK4ScjtDY53l2uvJid~ z__JfG5045Tlb@ni|L9-HzoN!@4bo}Mj0#@}e}&{C9MK8)yBKT1B1=x6tgVE6H)@K2FtPjl!)2EX@AekA^9cou6>j_2_L>h#^` z+XqtG2adD3&PgCiZ$7Gjzn*`Ow)y;fbS&oIBTer_*J|?*=DCi!u49S+rDqlxA3)}2 z8~;~4JiO?CUc$?G1+|O$-|Xoh|yGO@41d`OEAt zzpH~@5%kyo#y)}cKD>_K;4So3yzE3(%5 z|AI^}^8L&j627AMu9i1f4w0zu6gl5k7U$QmyjWaZe}5Et32Mg-373+Wp~iU)(uwuz z%Z0B*@-*9nCdVakt+?+aJ4~N;m=vz3{{TP4O<2p8m$|of?x&m|U(w=uzBDx4;S2Ynjd}FI$5{-&|aOOSL0x_(O51uvY)eem}qd)+_Yh1D+pVrN4$8 zJ&O$b(B}QOFH?_>^|VFm&86ziJM9xh#xW#4n~kwj56OjMqJ-)n7F#h`seX?`x6xCpGON?(S~+(6zcmPVf=x<{mxm& z9~2pXFwS)pyAHBXICIqagX6{@kkwnS2p9XGifh&HZ!ZX==(QV-!zjBVTtaVjOam^Z zUxsGqCHw6maLQf+xLjQOv`OJg@@m9!57(04M;r&y;~sl)z3>l^U8P=Ls-C{vb)fE` z`#bFZ`i(z8(?a79kVG?n=$xBy3x0&#aVLI^d(d@6{%7CvN#`ec5D%mFp!a{cI6O*kJmvl4ar%?U@2BaTPaO6Bkzap) zvGPX!XPy3!pE>4PtVMa`)1l#ca>MiT?x(}T3-lN95?)5#diDRxb7T4%YuLq%zP>~1 z{|DJgyyBQw@mf3E5dI_ngpYB4 z{jE>w=hxpVzgph?6Z?Be{>SOTKgFN% z6+%uKkBc!1m!NJX`-BEGqDdJQ*UinZqf6%3_dm-2Kdu}jJI*M#$n)#_cOBvXuVjB$ z{bT>X{qXQ`ssFhQm*YyLoge$pucr4FC<7gHE&cn*31=tp-^cLZ$?L^ctBd@A{2^jH z!A;~Xs5znTlG2WlDGWanz8%Sh`mGVyE=VBmof~a;et)C>f^er}YM1GEExjoGnBG{% z|94DW|0mnTCJV=ev{ruk>H?bNQzQ;Zf&3jwkU`{0z@xExOq7ZuE@sKZVjQ zmj4Unf8=KI|K;-w{eO=8mi=EUeQ7+87f`J}_aa$yOnVF4{1W|Tyn=6yk7?yQBF>M0 z)o-sMtL!c}#-fhigcfulKVDwnNc4BRg!}M1Ht-i>|9e~$_$}ez;&=Ex(k1dJ{y^_l z-|iVRH2jhNC*->2?N)i5>?;@={wywz5BiuK&_44i`4{{ZU*Ic**L~Nx7^83rF2&%T z{D(!_|K#BYb>v3;ZU0BsI$@VN_HtZ_tFcagig{`+x;(>feSjdHw(G z|M2{M4=LY6zW(LB{zuh6{=)x{>;L@KcP{qfu6^|MhU>(7G=(VNVRdF9Z&S`8|I3)ap{vhI-a>*0wm}Jvj_rIS1znPy*=J)@j zC;0x!6hE2I{sZScj7RY}p2Sb_Gdzo~g#LZU_Bf}P?9*qQDdnS+IsSc1J^x>Po8Q~X z4#&r}{qy7h$+-XDTIrPIdAxvhiDL>!hZpI+{P~_0{7L%D$Su_WzMTKR)PIpN@5cTA z^8G;c+SQ}OD~@{=HFMq9RQEMY|GTy8UlW&{=Dx=1e+CVBZ041{)rhPo3L@7IdHseNnFV{?T(l{@dsMAMyUleE*NU*CM^v&F+7_d!6t7 z|39SNdvKQ7o!I%KTN^x5H;lmskLZR8OkhT2U;?wEBX!disG_&%TBtx3WMLO-p#r{0?h#5#A3khsc1X-9s1PLUNf%bmR zi(z`~BvY0B<6FPqInVQ+=Q+=NuD^3z>iNmqRem8Inw^7Z%YV(`nE}B+Hh{3;1-c_IZUo{ZK#C zeh-sfE5bC)Kru=%8*?xZ^RWPnumsDn0;{kF>ri8EP%Y}%2@2V*!F{jS4?a+4=o4k^gZH!#u)#AnVGCMI*=fl3sq72v|7G;F zasEAIcNhBvQVr_vo$7)Y?C4M2^P%`J7Jrln>-T+w^cde@llcM9bHiXsPN?wvxc+ZH zc@T$j6vuHAHAtWtZAc=8p)1n5JU*~V8sU!X|IU)<(T_{0e3Tuokk@bnw@`gg{vXH= zx9Rnp<$v6z-^c6r8~a~Ec~>(<{e!yFU)Z17)8p8F<5v9yl>1GzvAwbnJmfBy9v+iV zk-VUON35fJkfQe@tr!U%qqkF+7kF{s}6Mkpl5BkrMFw^zI{k$W;4=}kzW0U7)I z%3svK+xf5Nzh3;wrekk}BEOl287M{xW@8TKA^Al7wN<*wU)eu+$A4x1qk6y7`_C7~ ztNDTp=y8tZB610qp>LM>P8VM?j#s=I&lu&Kt#G}v!Tf`*FVkRIOoPkJ{o)trKCW>+ zp8Xf+cJ7+@N!Wec`T_1Zw=*+;VB;HMoqL9Eu+Na2um#&thH~tQ^0+ii_D$Rx_LGBa zz7h_S1DpJRm$SoRdZzv_+a8Y6JM{147}s%n9DC|kN1deCpif?XZ9g{#RhuUb%2#o0 zKaTOoxy5l`>?U@aOte0bO?U$7iW7!VLj_bxXr>H*;#We}Op>$r|t34vzepIz9cOEErgnfzo8X9jZ zcf4z2;AKB$*NW|7_m=I(HnpX~Cn28gdc%FU5X<=6Lz%S0d=$oQEpQXLx z+R%+R!b9>gp5i&eKjc46{ww-83>fRr<&Jx!4U2M92>IlI_GaaUtT2K;3OhG!4P(ga zGv=Q0FQnI#b?36ec=|-dJwO^;l>hEYsIzKrZucL29O|+@4)sW&0gY(F74I_H?}{)D zGw_$z|4sckbPaqQy3vCadXYvSnm_nBw0!h&IJNZS(7O2J&^Ghq&_3y7m%P4W!&XxJmb4w^qqa1 zhO54hHe?Ii)+uxKTG2*tzi9p5Bm3JRsZZU7Zv4vme$wh5VOC&2R{5_El812=$5ErL zQHwftU;Q06$`|I4k&S4&8p28Ue0hFx4R->~NH1bHE>oA?&kjlM6i(wT&LeFN=HRA} zLqB~WxjkGWuizSP;1+Jwo7BX`+ncenMHgWEn1Gq?xJvO+Ogg4r0l{k1TMY~7d@=8^NU0E-a!R%u_U zA4uZZDiOr!6S z`fsNCzf}ECwic-WN2}Y%ssG8&;nu#LSQlFLr8D+7`wMoQu$P?YB+%Z%Zg7X)fUUoo zyA5f2+yf}}gdLy$lV@APkMC@;pXAnXbi%gqqet7q4_9tCez!3kzP=&+U|>V|{?<+5 z(E3f`AU&&<{;lS24VgI=QJ$yQ0efZX$KL|--r?BgzHDULm&7u6r+OQMfuG$>F z)3rIAcI_;_yKzJKY{#}xF>7PkhwpjT=e66zdDr^!{k2=e4}71416#tO?OW}AnP>f* z@g>);;2Lh=Xl)3$$Vz8u{nY<)0=Kz;)-pU)+o!V1xoann7}Gj1THkDRs5RbKvvGK6 z8aE;|-W#cp_HrDs)jGISj|xM}8MY$!qBeQzu5j81#<0bV4fp8}@fc6>dVc1Moba5x za&%q@=fr!D{-d_)FnaYQ<3BU=LO#8IEgQ-$@0LccYn#x9BvLqyvpA2M z>H5d|k9Ar4$9O$IUH|g+{QpV%xB9WIWZMLNcXXf=$#d)j`q6O>e2+eLitOc1+bf_C zt+j84e$TmtE4YS^%fh(+X1GC*X9jk^cr)Chr;GI2{5~DSa8JrTvENzw;LVWfhc489 zJM?C_?cTenI_AB*y!TH1c>Aiw{qq`kcyAWlUVz!-Y{dSk#Sq>o2asXZvsusEMB zmA!1Z?~A%|-gC6~n&7?Ah$c+*o5?7`G|WITN-!HqZMd$h>KOg`9x}zF~Tx_K7*dn1}hORNgEgtEvk0A)NDf!<;(y`z8G9?kS6qFlV5_ z8Gwywx@kVkT=mgM>LWIQO6hG`{QU>cy~PT8X8)g{9-gWmo+Le%%3ow7nnv4mV5vU9 zPS#zZvum#&thU9Af`}NugWY2bO06f43hYNmfjo+X^vwRc`pJjsN73hf)ADmK;{LkF`JF@!5{ToD z&15|Lw~frq3)ioRc7fQBPV!6PG~)b&xG&_uLG%BMVg7&Id*Hll{kViHxP}|Jh1HjX) zM?~D~HqoyB?^XYkP26Yw_Bp~olKwFa`51vwNUk-$j#uOSsj=#1qzly3#vGfks{geK zPLZwdYujpk9&vx3&XvaBT^l2uLX5{mRF~?%Vlq9hiHq~!i|EtP)XV;k1|+iHFu_y$ zTQB{s)_z{-x!3%^8-<=(YyJ3p=GcO@M&ih>#|3mB_$2^1l&&9u3 z7$umEIhcp}Sb#-HZj}GfjUK$-|MjYGEH7r_-y;5K-KYKIIqeJC|Impg!l)F-W#kG} zYfCNiy;sqz=yjvL*G7GL?sdqt%MNJ6#XkJZ zZFI*z`W^Z6h5Y$I{?s<_!(G?z;~^g7DV`&om)=mLOsK`{@`2ufMl_-M1Nr+S`FpOs zyjY&w>iev=$ZYAT#IvgGUGlYm0c5F?ps(z4)ff6R8PqcBgiqR zKfyl1_SaCW@888fafp2a<;D=w^TvhUtBn8Ow~m($tI++i92idy9nA?7$;l|fG|WIT zN-!I9Fc0&w0E@5$Lv6ABJ}wMwlRmiP89*z@Rak>{Xw@FyKyJboY(t#e+5RNUA0*$O zkT$QY4{jHRJ=}c_^eW{*i?pggP&3;2;aKB`Q;Z)Xv%Y_l@x!Uc56Mw}^Xap9jLE+v zQtyOv?h5R3KGAM#DU)Z6ucI41NTC;L^r3mV@k`;H5?OV5dG)H_jm;_a|s zcn5J9M^Rn%wsnthhvW1(CSG^s?QoJ_gQf@G?T#_dN8a@r__ZE!&H&klj^1}dl1!n! z-5Kky_24x3S){ib_eTo7sPRs<3q518_^%b-R$-%&-h}gh(~nEIf@`>eTeywnb@BI3 z-QJ}Kse9s&^fB>2B>rQ?AE)@WqRl<+=%9C^^5)Au3|_4tx+~1Mhe4J2-KW>PUWb$N z#6#}5cR?fK-Uac$ihE~8+fzC}D?D|*gZ`Wh7sOxr5^YXBldL10=l?Q)EKTp5>c5{Z z4ItNj!w~0Qr1kal=|eT%_Bw8nlZOXp*1e89TMwpf%m02OjvDV+d)c#5U!Yta zt&g0iZ=9u%j3u783@fk(@$A5LyBtSQ`mJ}+z$)=(;hZDbkBqr1ZYTVxNAhi`{H+=GSAJtqhB3HC>~F~C;e z>LC3vj-s3mpyQlp+{_BcxqB{qw~PAk^cwWFNCOxA_h-B-S$RD#tm2IDo#K9WM6@Bej7`|m^Z z0~ebgxX@fJb^q)Ai6?1)Xa}^aV@C*Q6vm(saSng`fj7c()mdv&gI!hH>0>e_qkbEtb}{vQ(b2E5w; zzf2o##|!P7Ga<|oM%?pk_a-(c`eXV0zqtPKfW8v`s{Q|mj5*I2)&eX-To<#1T!s}G zy!Dl^ij4NJHPLN6YF%_ovm3}b|7{bw1>ZLhZX5Y({#zOS)%>?|`X2o9{I?2z`*9GJ zFSN7Cst4Bp>8l>4AIC}5DGOp5kw7dXV%gAyFV8QiabE(>NH59{Npja|{p*R!1o~;5 z#d-AO65`lHt$bdGdL-oi2IW9z|K(Y+{w)Ys+;wPeBeVbVLiu0bZ=E9l3$uNa{EyDj`u64=J`~QNxrC3&r-*AXo|EAp%g^iH3b|x@ z#am$*nU4_|(iRv+mW`G^#!4Skjs5iNL#)uA<{#T1h4DUP1>?!g`3L3>Or{rM8fKtU z-?PoLCen*hg6aa}$CyozebRU~{T%u{G_BM(-L8*{MAn;OKEK$9UqCKG2Ym^-4DIx8 zYw&xpf_oLx^gg`WC$MI@{_lFvSgrpX!z3HH8_A}wx565~S%(eSge};HGL$1}jd_DYuAn1NJ*jTIM(Jd0o}FRsf07-*JpX0FT!B?sgLT+|E!c)K z4BfW&-`s$5dgU?w7Gpy@*ummnkmX0s-P&mUzdbwbaXqvDkGSoc=X>_&hS&3d#5;jZ z|F=p#rLSL4#yRZ`C)DxurkihsO~RR#N6WX3r>r>0i_A9dcZzJ>oq z@t-dLp$TOdv^DW5Gx;sQPjAsC#clds{FU=>7s`J)g}7JDFVDY?XA<5QPGg# zS6=Uf64T@m8`#b{u8+{BaR1=7-wC@Gl!o2L*p_h*owRNw%Gudr6}blMumPK}1?43r zVH;T%b?m{Q@yH4?j(@i<%nAGH2XPoj(Xln!fO5iddhGLePiGUS*Wi`itC#ID7446u z^8XI`|A_p5K%USZ$ejNv&o-gF##(YD{I(fwNFs&PQD3P1MK_|2Ew#~qgEac^E9?LH zwd2+Ke`W3RZm;^AJTJUm&DQ1LDhd7cp+#(TMb-r`lBT%__k?hTyoMXNh02A}BH4HC zx58cWKB~9M{~MLV^!i!iHa$B$rawjI{6A@=VYIkn|I~@$K;Go=y@JW%^Xrqt!JSjX zp_xVD`{#%@BL8w;(cwb_k9EDY~1_$KhCZ% zcwhhL{V>OG=3zb-U=fyJ8CD?azOK>sKS0kU`yZ6rkKl!G{^Y&T_u#$Iyxsl>AK3o@ zt!P90NA^E(U+2vCL*mESL+>H`KUh;)JI?+C z6YM`wYX5;r_8*vP{{gaTsc;MA4{hAoHaP6}M{yiAF|4G%zx>*zJ<>ha_#0B%D!oX1 zMtn}Quv%u8hEr(GD-CU))qX)I(tH?E|^4mIXkGQUX9rp$_j??~lqMb28J7cu? zcj^D*_4!|=%H~P_+o|ela+BYD*}k%cdmGA--e6tf4)cTdv48Aj|2V?_v6uZreUet6 z^eJbWmur70H&3AzZD_w|e$i#~i;%G&R%M6Get45#=8HYTu0T8kES~+UT>qGj}$MmoF+E-2=zH6uc`M%gcmp*;BUZm-L*l|pKh!vg{+t{nfHCTrY z*cHd7v==wgw_qE}P#%fj{)j$2y#o7j5FN|C0}j)B7JByfx582Sar8~sm;X?o-X3+) z22r_G-<*F95@<&CG4>C%(d!TBoA;W>Ku@9Rk^cK${r4UE>f~vDXK^0=`11JLCGK)| zm@DM&yKyhS+;EM41GjJ+HPT3}G*d^`Px9ZC{^k6C*FE>~5RdT`&tcJNSuWUULl?TQ z%LDTBuk3#z%ob(IDY8}AZS;6HOo!)nK2s+MBOfEsdcfLU^40#dW9Wq#k8U=(IEN(l zLcK+%5!bR+KgtdhU7w6N=e&rlypSEHku%WPtFGFoty!h+qSv)+XOqRQmEiSyPyhXF z?zHwq`IwwAk3Js@un0@A3@fk-yW$*sV{2>Z>#zZvum#&N)XX+WMq7C~8P^5xAuF&S z2XPp2-NRAxI8LGl0}1~F8SU$3)1}eUm+g%HY=CVdyXH@#;}BYo*Z0YE)01IS8xqCa0|C_7x(cH zk1=%ntx&z1ZH4|ERqL}u_$R)9gLysVFfxg@PG@$MS-yiv|jX=#2^)Kpb)l*1x zy|ha=(wi{KJzuVW#&8#6Jkp!AzZ%s4?dnK&gB~))-OJXVMjx6_=>OvsTG58MF0P|W z{qMPRymzz##QkI2=pChO?ZTdnXak6S{~qib#_p~Elcx6}wb64B`~F4lorW1GM)g+X z7p|4i>xCU{0JG_H5YNvgEm5y_uQ`k_d@*7iT_3MC);j{KiR?ENwz*PwlAD@*nmyw$aBxg zH^LTrX8q$V>(c3E_~rGF<@_rBPkYD;RDWPU1F~GYuUaf#+r?|H_fZZV=HIwnytor& zgX=X%^ncGt7pO-94f1cJeBFc{#_G#()Nen1W{$vq>+k*V$GJ};dCvIyMSX0t=caZ6 zz1J9j`iXYJv0seO8>2skRec$Pp`vxzC zaGSh~`xq*=em2U->P+%!)SsGvqAq*YKPeC6!gK!NpUN8}#)Vgy( z1j~@DHU5q6<;K|;8du+FTpj7n#@W{zmw#dW`=ashbH=|LjDL3-|3(MB(>Q-CeT8sV zVGR!YejVf7gLT|J!-YNT?XZC!_wVo9ZfqTy{r^WAJJ*J&eDU_nd1wpG|6A(1dSR3M ztLSy^t^erlm*d~c^G0DcAz{2fv;Y5cw+Lq&%Hs2}2Nl?l zgLu9EcB^(CQrd#O$h7mW%KtmXf1mi1t;fWlZ0GJEJGI4HAG{q73+E_~<0LxxwWEgK z^P%{E;JxV0=;Kav_mZiF!sAz2Yo7x$g=+d~@+_+8b$Q}dU>*Z^KN`84xD%trldPGb z4URh0BY_4qq6wG$<_fMMy}~#Ec?)-OAIZJ)Kf2}ho|q@QRN+q&eyQ-uH2OyC zf7~;V+4Ws()W6z_;filCU>_y^x%B$A;0+ljYch3PhXoe)1p=<0vxg z57a%!>2XY=8z<>C=zF0KMI0lKXT@jw-|Clm#$EjZWk$QQ*pAjRB;|zH#dRy~-|}(l`H=`G4-IRL`Y^a~g3T+F7!C z;mi4d=jpHZ-^tYf+oZ|fFU|j>_q*p3;@&@3$hh~x*Z{d7RrI=Z%C(C* z;UM>6G;%j_CzL-8WM+TwXkV0n%Y!)w?in~K?~sGWUTeq%nlUtu?SM=oh11w|%lv}J z+VS-B=*J}t^}i9WkOR%qBN@;3y+Pi>ZQR9uJj7!>#dCyzreB9)i2KLw9AkfQVdZlV z7HV%yQPzy}zWPfe_+|PZ{q~<*V;?}{`vaH$NlwQjeY|JJc3khdF8^Pa25y^!&K=vN zY4jqVRa5iG{%NQ~JrdLPZPAD(ob*nk{BArZVls*_4Kt9uXMR7r^@V$OnisF%+^cV# zMqj(W(qi+^mz#f1wz{wFnfd4Fpmz?KKfkpg6bq*WvoQzNE&AVGZ-#mF`Xl<^2h1~| zFF@QozYz^cc*Y`rt!xfU$YtobXRZOc3hnf6Hj5s%iW%auhF_X(qL0lW#lQE2ed{L} ze@7jD<@`hMyv}bnU=y|=mhn4=zpSseaVKYqKe{i7KjNB%Uc@v0`p`U8{AY^4_syJ- zFk1Y_ivMu&M`i7s4iJ`4%hV5SJUiJ4svE=~d-&Hs@^1I6UzlusVGO(2P4T}h{&&Qm ze}#KmAE*P!gNSoK50gg`_xa3>ORtxnxlbZZ?_2J_k9|;jjXC_a=JnT2kpGcD1L7W^ zO{npk1e(!?pN?cR$8qm{Lccb-m;D_XJK-UHT7CQ8&GP7W`JZgQWBmJp@o%#Ag}Hb1 z_M7Zve%tBZxPQfG=e`+|!cO5ds!y;V@;^(@>_6%~&eQwR#0K6t#XBO-A^7nmdpPFCHuZQnle%m;gIj8O4 z3cus${ej)td^8Yz$0J-#Gc%A>XP2*hka`!(*zH1{e3S&@+@u<UTkn>wx?aSeaxnK*|~{fO?Z+6Tz&zc)$UJ5n87 zp#J|bFHH2D$tc1!RDWds8D`MycdGwTOfNx`y08&(@3I7D^P7Wtn2!Zmge6#p_^oyz zmX|BI~UMg5O-e@^HZ&Lv#K4cx+Q4Ba*sMn)UneR8nK{0Qv>Mqq@U+hU;lZ| z&$6E8jNo@-v^`SC>06X4V@CeRoQAB2IgL~QeNI!>zt5=|`R6&cNB%sgF6(hl{kZ>_ z)0TBNr+xLm$w{Amo>PwItou1Fi-&SfeKeHQy7AxUj1t}$6kLTu4;<$U;i<~9&YGrJkL%EDz zzutd~74%ie%&T6i{9Gt*E0yp2zR0OP^hHkHu`hD!$wbQ+ISsvEvDkH>!V6weW^OT!q3e2hTyL-S9B)xF$v$doX9 zH@|G-MO$;TvcKh`@`r4_sr(_^x#Jx1&h6%Zx4+04C7dycHrPV4`ofExDvYPst6$>W z+sX8_@$n|^IQO=JjC**NeJFh{_J1I*|0{A|6(JrZcBRX!b1KB0Ms@@1d$Wv9HZylUGkZ}%!+lrf## zm0#YYOZm`E_AFN>kiFb#vhNvlfik1DKpyg}C5UzBGI9mt8o+LOq(}Z)#k~e`?O)$= z?VQcpIrMe>HeeIBU>jopp^S`cfy&7}sKAhVCo^C77wz}`{4(u6bE$WaVH+sQwPszK z$MJZ7jea{hu<47O8glTV^g%YG4N0VM8fVdZAS;|FU#&0er(ePqbo;M+q=(dc>Eng+ z`H8%6Pyc3$`fss%0vY@FA@$!ebu1b82|ehy*Kh;3a2t1V9}kh4|Cgt4tuF5&Q~Lb9 zWM=%mRR8~p_JKO}6xq7k`1^L_?`#Dfi1YsjN92Xa!Wnp&8=jJ{+t=#s=iK3+`$oT* ze~?Qbvc_>3xtl$9m-cX_cg^P>fl(NPYISNA3hDJ*#ciX0GJPWAp23aJ%rSUmt%3GR z%>nU89Wv|xy2QUq{2Rodoa{G6n1&fBMhRwP4w9?oKi@Rd&i0Kn`}%Qb_W4&AH{TS0 z;hYj)E14PpsMS6?@^btm?uj%{IP1)QoU#LL^dy?-(FPFr7>zc76u;9ri;2?Pkny|o^nP5zAiK#Ga-bqBTqAGb z7VhFcGW$PoR{m{gKbUQv?L+o}Ed}ABYmf01&k_EGvKzyYj}aJ!F(|}%OvGdqVH$?| zwa@SC)7uAV2KV3?dytSNn2kATy<;s1Szi8Dm`^UitMz|N=yC7K%=*9M`C%D%-vnuM zmNHnlDe;HYhJ*dE-c@+D}0eenV-my0z{Va*5t^RoJ*fzlOe+!7ZB7SiXylG@H;@P()9@*l6)|J;!{zi>`w#D-SD^2@cf0HzNBe$=XTHwhzaIy27}e(LRhj>Hl%CoD|508z zPCtnzby1u{-{77E$`)$>Emi)lWQ)*dsBzD$_4f&SX8qqH<@W>a19fi-ao?WwW&Qt) z+6V4wLlP;R##s!_&JE|ufoW{~FaEZKvYq)`1h<)GN>y10q zX=^;CKSz^sfX3zOe|`zqYP>_Ocd0{MC-9ff-#unt{Y~i|!;p_J=l>Dh$-V3w$c+DM zL!`RY%}6iQ|3~v3_5TC){|oiMHc-Ysa!&n!M*Z);8vjMD|DDiZBiF+};`FtMfIA=_Qzr%=sM3gVZs3Z>K!y-aay}@tx!PJXFco^U2Dq z_M{~jp^@Hn^JSZ&fvkHVKVD|9e{5e7*K4L5e?%SXkvL!uAR5tR-rh3ztiUR)!8&Zf zCTu~n*Zil|=0B}B|A|bghkMC1cOTiTt()0j?V|KqtAE&K4q%lzIw#BlL|L!(9mA#H zvDPjMuMFuW=0lFMp8u%x*Vl~;XO(?>_*cYhslu?IJcz?+eWsm49>+=4AnxrD_jpa; zFedKXcW?E-@^40_{wEJI$9pSV|_m! zazDmXJV(v&ckEAN5BzcOg!<9%ghYY8?~U&_q6y6--!Z@Jop6f%wDrO};_^;tKl6_C z{SNz|wfN_p;m*JNs&7$ien6M`0Z7yPPWX*A`r-eP$1n`}=(sG5Dr?Z`J@L0s?7vSR zgFeqm&k{zdu-sS3Z#*VqGKw&;;+-&!9BeBLGst3;U?_|Wv&lJ_hxw?S`F2=9F2WKl zL$z`Bs!8T0(Cdw}*KKAeuF(hNZmKdjpv%~KuQ7D@m3{OV{omFL*2uc2-FNMvucPle zuAP)C4X6`(8vN(&{(Ij(O~$<#YL3YNs4I~F^ZX~{{5ND~{pn2Qyf8Ll3$~#Q<=BG? zBqz!LQ-wcE_()9=KGHX(|EtpfNAkaWPPwO*Y%7&E$&Lx~zh~_i#z7oLJVWXzS#?Cd zrXQ!DL=EcX^?G?X@kG7(;$`2UDY{*2MjK-LH%X>&8fTHQ7fN5Bj+2hL`*8`o+v5J; zxf%Q4HSQa@g`vCV!IO7!AA`@$fhS}7e@s5bbA*2>&7gIqv3)WhBQOfD*8hy5$3A~I z3hCp~H=t}#CiEhXe>ZojpK)rq?}N6nzE7U-gHGQnNoLMhTkrd>_I+3SzIe61a_6$I zgo(nPj3SJZXQq)eP>h=C>=(0?dl!5^B(D2@XrwnG?%6X?_I4<7?_kPV1ol{%9d+p@ zJt8;EqtC}szH~}1!V)Ztew*Gl|IPZpSOyN1a)MasB+{Rx0#1`}Ogs}$eumPK}1=~=Dq&l*zLH$={{Cl|i zeXRN&nf-tA)bE$oe>jC!{POvLaZkW<;p{;L_M>CC{%;oB1HEUouyL4v6n$IOtsB*` zdFomm=QnUXH=HB~muZg__!cvzw^FtZehDs+GbkMJm*U(OX zW&gXueG9*0|I;6@MP{91sqa6@_n+$ff9U&vpl#zfcX1yN@fc6>9O0Jl|Dp7WxVKYg z{n=LO6KUxu+JY^MOI@J{j&*wmNSN|E0BKkq~CGUFWJc5 zME<~-+V@u*TbnyF96a{*@cG8Chwm*OZGXjole2&6pXK~y_vOU-+lYj2wsAruOKaQ=H?JXzV29e#S%eu(W~3CH(JJ6Xk{dH8$UmhXmB z``$Hn@LQoRYofJ3cF@~BkaET zjZl8PAQbuiwr4G-(qIYy*_eYN zV@~tPxc+ZG8TbEQKrX@(EJHl|a|O8yYp@O-$Mm0$WQPs(p1t}{z4{3BEy(O&won^x zskYn!?f)qSVH^K4lw%L7ms>}$v>;T_>pwI$yj_2rei%(h?EQAY9&lu0VZmQ+|4%iS z+CA-L#|7)(>6!NbFl8`z&pGyiGwcIMqYo3^a}Xy{g9MtAFwsFqHV3Y|L7QJ{lRGK58T%^lYIg`d7jU|_mQ#{eW+ZiAMH8i zXS2d-@+_)%N+3Zxxju*~EOhyqp)b;I$^r`7_&RyL5eg?f5 zne*?au)!g&0ZRAhhDv#?#I@O|KBGTC&O_B@b&1Vw_-_|~vX8r2 zd|RG~KU&X;zqq%HcSpPU>kkZU(FWJAj_vHh68&u9uS0rNZrDJUC$hs9a`zVY`TV@F zjb4T!d-V*>)Bk>?4(Hy33hYPb4Bi3jqUdo?xbC&)WYEix=Z3yn^7&F_v_3*?OB{{< z6}jPfl%Y7BBx{gBGwP(jm)Ol&+YRqRUaU7g%* zWcy~eqvg>qRS=S7*WzfGDq#07(Do?^y-x~4JpXl*=d>Y-6spZdj&;CkdR)6!H^n-8 z`gz1N0UF1uqeiN$)LZ@hF5wEU;mh{>8{D^W8=3t#*$eN|@8cmJqo(Q?=lk{Qe_v3i zURSrGk=}GhJ3>3?sr#NI{3~rf3`0Ifz*_9kb=Cim9%($b>3R?O|Bw0q$!2wV%WCWY z^!Ho0>ub^5*BgHzJD*4sGtILT&KMM8JUW(Ye=Ib2n%;9l9sZGV0D2MnYPCQ3^^&P} z`~L8ofuVn^hM)7K!{?xPLAfsD3_b^Ka;^%KZV=uoF`A-AEO-o046V0l(3<1WW5 zyFlE7zL$NVW~25g>QIkgIsYb66889A1@_}04&x|}Be~Z28oGzG&m%QX*^sAPC}961 z|I+z4?t8sIAH7q#a8ejGNT5>RqM59kuK$5HdJ-wbvjO560SWDnxNrBX{dtGRX#bA( zjpg-di$Cq&^Vs#YM7e2h0sRuL;2H+keI?{dO9N}Hz2*L^?0>i1a~pSY9}n>uPw^bd z&DvkveP^=gqCBk}@4YFHU(o(K!~WC$U$FoAUphZh_E_5&2EEr~^FYb+F{NP`nU4_| zg)u0^cud4(3|XsCMDExw-Wx-hMvrq`ipi3AUA>WBWbcLjSz$K!pgjcVkn=DfyRP{@ zkFA+MRvONV#|(Z8un5&xOWFTQ!xDP^Gi$RSm4;>X6=)i(T{ZH((BPf~R`F{+@m^R% zu0uz|dtn2)3GFS``hRF$I<|0cLz>=)6nas!_`OiO`aN?L-V60)V!N={dPbM^?bmvsXrs5I!#$lF-_N9- z$^p-5QKz8#!TaI3aMSvmRgdDjeD8(w+s0z~H`1GsKm%HDmV_GqnfV3AYZCMh>*Cv= zl!Rt_kG1pNE!L9LljvLO-4=RB_oPs>Q2ddJ|4QMa0k6;hIxbE5-D#Y~dGzBFt{|x{ z>_V(#dXSRmGwpwRADUl$sr|3MZ0pj$MF+i8+q3mfX}Bhw8@Pqr=oqj@-rD`U^mrz3 z_pI!2pZ*Yi+u8omyH)#fqO_w8mg-c^#TUS{*;9(Zc)D>)t$F&Raeh8ZYE-2bP9 ze6|12Z2BC`!+-1kKXL!T`F_6ui?9UC5ZC^#Apd{w|1$~aC5$;Ia7>0a|z~H9bFp7+O;*B8-F&+~!8AX_e87Rh3 zTW%;JXQM`+|MmQfTIGkl*?>lR6XN{+IqsW>`B;EOSb}9pu9yGy6T0;wdyt~{7AX6t z%l~M;F8||{`&w7a|LUrCvICv!pdGzAY~aS;jpalec-)(Qg>cjIUxzw)4SgMY)Vb1t(*`dnyUn}=%Px9OKlszS`*LcKMa8F*>9!`+4EQou8ZE?>w zl%X7ZP=P_;dOsQ0;T-}7yr`9?_7`_Sxq3r->KmDo1&<=Ez{{r|5C?*?w+Htym+9%AT< zJ)p?g|9?sjp0qbY{^;AL*A|)i1fJpF(zC+t$~1$qM-5!X5xk9wW{67FwCd5QdOU&kapInMX@eYz+4{`chn zXR%%HI=v6gANu|aeSfr~ZKk*hw}b5TyUM-#g2LHlY&@R-ahhK3Op+@8XX*8a*njrf zBaq&Yrf2Lw)`>LiWdFe>eyzvk{dw}$?A*|?I44};cMX~SLpJIcm=wCdOhc61}oZsE_J2K7PN8T3ZUEIe*?A$gs zJSLxFm9$$kLH?HS;vQJ9?U#@AAJBy7?hF5~yo6yGT428^as)M4S?YCdjJ6hI-wnpV$;t!T7!}4RCdutt$blJo;WjzwOvk(AeLTctRE~50-qe?Eh>U%YU-eb* zug&qCUjNAZ-_yst>wlQ6PCw(j-t_(h-k<+}cKw0>qEWa_WcohapWkNIKaQ6+=_4=- zW02f#{KbFRUEn_;Q@(NUaKD>n{{b{>54CLe?cLW(wk`D^%=PX4znx^w0r}&IGJu`o~)SO=W3N$H8V%~2ctzqY`_-mUSobh;kb~Lr@HoOe~~@%R*L?5e0MAR(?abpyx#xu zk@god=da%}hyR+kg?evCmHH3cgk6Sm44t*F13ADpP(cnBTO(k8!2akbEgd8eBQxL6 zw<(`psEua~m)~)mMBi$0K<`!WhF9x>YFta8(i!5L$Y!)5iE3lwRaMR+r`L}*{*BZ0 zvuJ96J2avp&-gda^XtbYT)~&e->z}rz%8^ZFE@VEd$;+<*R+ASH*?n*|F0eWrTGQZ z%`Y&v-)K%j(}Z`}NB-jet-tj9?cDZQ30>FO|Jb;DkUGHrhcq(#Ukjr}n5WQ+w&CU$ z;MMthned%+Gqh-2@F~KM_>7ciUh-8{3mXi6X#oXS8^B)bPELzZ(v$pAz=t2hT=^?>`zD4(%+mc5SM;0Bqm~+0Y*SR`}Mw z-wx%6emm?OC)!|o&pqTlzPasOt>&j&k4FuZHjBhwx3!v%iBb_xFK!*@4qp;UCkJ>%SWQguH@( zj+?jz%MQyX3+G?cyXXTa-wwaSJ-Eoe3={q^^xK1N$Td@9%l;jnKv$})zsz`o^#gxS z|K2F~tgwbLWQQ-fzv}nNM_&yE*BtJ>`Tg-5q3mDc-=UEEKauZ{U;Robn~E{`5BOcw@_(0XBL91`T{-vn$j*(w zo&ER87Pbgmrj@l8Ov&Cy_TCHOIQgH@fUdE>ll>3Lo~5I*-x(Il{;qc@A%EAm{2$3# ziozY_M)Dt#zeA39Pbs;K{M+RJM9y{Xf5nH~jy)**JLFdWOUd8#ZvQjcQ}CPFm+1YY zen0yi?!ML1ihS0~R(!`hl&#gj+id)GV^;Xq%?a7%Yrhh9?rpR;!ybkCuMZ^&a=;RpKNhc9P^BkJ8B zs(*i^{5`t%4g05#%Kou?euCqhM`fSrdL#VwLu+w%PR%|!f!%Q`J0$;~wX2^`jLfcG zI4V1yPt-C&JW?~WFP6;A_WEUYy6y=y#V)*SXYnuauW-Tjw~#!cZQ#G`gsP+LPh8V9 zExVcLKjqh*r;NHPJ>0Vg%?o?aTu`^(`9^lCKwX=q-X5^W6`J+yTg(;hu>P~ty=_nI zLG!@6&{A#3GtSW?YiHUw@}fB=t|iEZb9v#X{44YF!g0?yG0VJyae3iq+$YJZ(e}IX z4nOky_sGTg@y)#Olj*s}$+N?;t9jw?^V^G)-g(+@#;M@yHu`7&yMK#+hyDKZZ2$io z_`dXikh@7<`~%m%gCF3hsEfGv=hE4)za@R6Klg8DUu>S1-9MhZMUKE4ESZ*lu_Efz zvoDUCp51>X%HPlK-!MM=;-$Zy-9L>!k$-+k_Qf^tXZMeJKl|d0C`+?1p8mb;{y8P| zS=s%?zn6WnhIc0oaANkw z5fiif%m0?&|1H<~r}!t_^VIWi6=z@ERGj_S#o<5rJ}ag1J-9gjxBd2avg`O2`Ck1e zz3*}N7ynlFALnuxzbpL+Z^W$Zi!0vC?)Nd%rN#mu6lMQ2*L#$yxxP_X>2GHLsW1%1 zl~wUK4WjH6`3KUC=7{s2L)m6}Q?K*vj{J@6hN}GV54h{k{bqKK>t|j6Yvd)8S*&ad zeK{(##$+Go_f6rB!%vpyhaWG@H$Nml{B`ahEuE0vM7E#}N&Ghc1efsn)Zfbfr{p^S z^&jA~y!`Opjrn2U>in>k`#W>TXRq-7duRS`_WwfuJ#6E@f&4=-b(IC*WrydMtwfOj ztt0w0-`hRE$h?8*HrCf_en*@69e>K*G2HwQ z#Ph?G3d`hN-=F=J{_PT!G^h@M8nEN^8U5eE@38;D?=~&+N914q=1JB8{PG;(j^Co*_nWE6kAHBB`p9o44IKM; z7rz^M>z}h9Ne*Q!7b-){=lJJ#M)!z>Z{nMXKsP3aIk-j2H+nk06y|y2SF;uzDsFJW zC}@1EB%8i~Hatjjj;*5q6)d_L^rEI#q5p*aA-;e57KNO*T@<=fio;flf%CdchPJ5E z`BhSuwvxW(O8%+4QYzD_uh_p(9l26!bX<%1IAdpH;!3Hfuh+o!ja+Zy`r~)ww-z1K zeXGPmU#SH-8oWZReL=i0>%Yj+Q1tV&vGd(@7&#+6mHr~n;dI~^jebDHUjQ^y)zM2O@WL--GHh!o6*h(0=6DeA1N;3i0dhu~*W=3FJ^yModG3YN;f!BPd) zm}~Y0OKn)N)W!SXKUnIK4WaC*2uBx=wyz1<974P32ocLN?c^fuWB~0VZKI8Fc4QmZ zwIdzaJCL0bw5$3M-rs*dGyei!4w2q~5b4X}eB}(WwF*vA?oIopE8@+|*R?mHyU@jh0}w!7)$VE!8v9So8g+T3*kv@drD z$<@dWuo)PFYHkF&mzsWIE$1u#xrmuzm}dB0)_2aXkoPcu1n##<|3>Cr3Yd3k=f6(4 z4IDWnt;d685%ZUjBSG>{q>l9z+Xm`m;wrc=NJ5Zn;Bx4R<=I|DM`R;&J~1nVX+RBI zM`;Hb!|q!l?;s;-XZ{NwgWo_2yhgZ>A(*h&09_1?mv__7+@NyiZ%~?L%E8hN>O#Jg z_qnsH7zgp)lio#80D36Vg~)fgws@9uU4*A1#NX^Sc|-k{2lUMw}7`%#;*MO7UQ zmFmgOsvdj8^!2I^Y0O+9k8tf%(71fDWFgDob1?I6R%nO0pD@-#_5vzm=o(QD!e}=z zd*OL_5nh7hZ~{)kDfkP#4e!DSK$l;07CwV7;45I*pwTuSk|3mc@gZfIen_<}JfvC| zGsJp2gR^fBh`suNwD}I8AACR@FhV0<4{3co4O*B>Di+)zTLyxN8$59q%$OY!_+F`PauCCJj5!%}8D zEalF_Qh}`W9hNF&HTIhHLsA=iNb155iIMlLUpOQU(71GnvH2ls!rVN6NX(Ik#1ef- zTA(%YkXVs6uCpWCQV&V{zC+?5t`1};>Dq;KV(&)w;NKg4i2WC&KXQQU2a!Xh^Ki;< zWMnEsMwc_iwU8n1r3~>*XNniI5BIUj3>lx!pdcKO?8F12-E}~6b{>%2qywT$J|KBZ z&q#jO0meVfYhW)-IlzC=pOa$38BEVfNgwC-L_Q~F$#3N#W)0*$^IOUH{#Lp$kHYit z5}bnf;S5~0m9YhEfr?)qlpBz@K>2+K+9njx(PzLTi$XYJ* zDeyA90)xLeC__O9DG)0p+q6Qo$Q%c{@GknlkVnTq9|{&$h#m@;82>^sW&@P4hF%I~ zbkxhCg7Ir5RHX#)-wXbOfw>myG8z9uJ>~{z#Lomxn47`uLf0EwjxqiP>-5)qm+Y9^ zpnZuo3h2Px3I07xPUyzm1HCThSD}9w>)SAx!u%=>WA^W_{LlVO%pUOK=YuiKhS%XucnjWv&*92jX?q|F z?t(lhgbr}S3-AtH6wka0JP1F9U%_+088DhWD1;WU!wc{-yb78t*it#f*2@P*4>u+TY zsLJ_YO9ktUl~9$khWuYc{?j*xy394?{~Gdt4f)SGUHF-x33D@;U7RBZEyq}k1nV@q z*I>uo2JK6%k3t9LPUynV3Eh}`px4FvCiL%OeG>*#Sl@(U%p)+G$@(a`F?+y^pAW_` zkNfSJLE?ikeUOZs=vQV0q0bk@dPb0VU~qqs48bsrKri$`KMX(@VVuy7xd+;lg2VwG z&hR;Yw)i{ zmSHasroS0Ve-l|8M|p%2D1`#hLm?DFF6bZ+@<9XHpoJW`_M3dua2xyz4!|MMLLT(P zFpR;g@H%`7U&48a+Af;!z=QBGJOPDJ0!{D&yaI2*=WrfW0^c~Sg3I7ah=Om!ci=8~ z0-l7Y;1Fmb4@#f{YM>sPpc{S$ZN=C1jQg00ZhhT2{8P36D+&9O(u=W@1tLHi+{iw0;;!a;m`*ZHaxZ8vK z8OG>-ooa0^I%JF?=jb5L(K#H)Z;pK$zw=AR$$#)m*5Ur>zlJ&dKlvs0Gk36N^GY#UOOdHty zb$BE8v0U67%=H#4d6FH#AP9AhmFt&q`z3Qe zR|iRx4l`@k{lq&=|7@&_@n1}cTsWVRHC}BoW55rBM9W;zueitPS*~F{*qF{9t-Z)h z){TkVZ}#7B>`v~9e-jgT7rs~9zx;jJuf~5p{;aEv-ACL|KSNk(K6;VNt;WrZ{}TQ` zVZGur{LkUf`b;x4O(UlW7ett7=EpY@=d+j-zjo7pm+-jz-|b8z&lF7f@QcL1KeRrB zXT`4NTEG7sG~w19O1uGIKjYl3#D(2|X0^XeoH>ghZ2KnxrxL+=5&~S7G>55L_ zLEMA&_Q*G^z2jQ@VdnH9Vz*9iCiIoCOf!^L%&{bBs??$3Lp zgKHk;`WFdfB)mT^z6Wg;b_05xm^~t-J35rCCpGOyZvwmaA?DtLD8IxvKp8XkG57yV z+GyT)WIN$3tZChfKhM(19G2gH=5hc1&>2`uIG)|Pp6hfxR~l)+Gx+n=|Bn8;z;5Ol z%LtF(9N)A54E#9;oO28$=NL3zs80->Nx6;ZJ^uAIM$URX!*ic`i1PnS*bLVZmN=Vy zgyo$t(EJAtK@pz3+~{Wjl%aO z@*MUv*U|UkK4)$utqZt+2KNsasi0o0K0&(2VxJT#A?(;o=r8`9vp%WkwD>nA5l;+y z0r+Wizs`rTmE0HmMc47%yj#<0(%=N~?*BSoe>wC2m)XcOoFOmQ6RwAER2zhD;eA(2 zuUySJsO)jV+>}cEd%52Gb@(N|r~TB0{=CaT`tG<|v?!{KANhu4`s>KS$glH@J@#{F zBQLZQS-4-wSJLl7{_);=zneb~`5ydvwBWB#|HAY0JuO{9JaHh}O|3$X|T8^O0GQ)lt`qVc1)^=pu zGWx0XxgF3k$vii7#i6?b-OvNQyV0M6e(?Vbb8tWUH82b#FuEK49B_jN{{7#sf9v7- zNP{DA6pq0JOu`gQ!wk&A*$qnOkqdAd&cPxyF5V`l<=bQlb9tGPWuyak!Y;ogkKJjf zWEV0SM*D6P7wpEo2T~vvb`y3lav$u$oC2w^57HnV_Cp3_LKb9$4)kDva;OG>oohs< z5!YGF=|~fH3)mn4doYwgUp6d4BL5CM^3V-9H#^8*g#5pY5cW4SXg z?NCFQXRH3EX6s&6b5(y=b8COEW~+wPY?(>T<+iA~mH8@y`$fW;(E)X4dyz76&D+Fx zW<#wy^Y+_H`>v8S>?@gzb03|SHAsrP=1L&RHP^z8a5HR&Z^L)t9(Vv|HP5QE^d0Bc zKc%#ql`?C-M9r>T$Nkr+xht+w4-@9rK<;zSD1CeW|(&8O^iB09`cA zgAj{34ro?1Prw1tLIIRO71RUOndT=DPuK)V$NV_=O2nK5RF4{}1`XAa<~gW?C!qqK z2C4zgNzkwco}Eg&yodfT?NM$fIvJ3MIUfpAE)u?{$qF> zDniyvCHuCk&awUuHBbw6leB?QpRisUkd3VCo1iI+wmxz_bv1Jo(2&R+1(+~5!baY~ z-=8a|KT|e&kyJ)g#-SR&DyWTQjsx`Pc*cc`qzGBKbdebP(AC0T>bOWsT)Z3Zn$x@+ zQj5QCl6S;EADK7pKRdyHh8ktlM;QelW-p9W7mPt4=3W@UuOEgm55kCvG7kQK2e$Vy zH8^ zBENtr%&VXYvj&ENCQWl5@_?pL(}AShp(#dQhP)a=_@+N%F6?FIGjdnZwgpqJU^CnT z-+_DK?2Y_G7MTh^g$kIv`{}?2vrZo?qjUChjHuAxo$sY z@;Szr#f(9ld9HSn+|9jSWSwS$`@hb7*I)UrJ?O1;^DH*eEKv^i%q3co^NWD{kfK89ZkY{z^fl$$S- zPq;S1ABH$H@S7lgdU==GlRE-uAH6bg?k`&d`Ttko?D|gwXYYJ1aPH0z;dg-pgc*i2 z&wdnm=J8JhU6|+n_V2zGc&08oaCWdLaQ2Scz`27hfpc}3wgC(Lo;ThxzBOp#8f< z^rx?s!uhMDXk(-lCq#+?b4l7-DV@4f%5+yrc{1mn@4Q4RS+lJ|R%5S0)@Jb!KBN(Q zJ+dL~D*k(WiI|`Xn!$YJ60txFw1U-jvDm=wxLDdIIG=x!{|_#+{~y_jy=$8N|ImGm z{r}MGV*fw%r?LMZ2KTc6ABGcH|Ao;o_Wy&Me&GL2KmIe);_vV!EQ3&7H^6KV-#hl# zU=8+L;a>PT9EKX8cxgU|O9*=@-+0nW>Sx%2Ibk#NDbNZpf$LhO|N=29rbuN*4qFIC3U|B0vn zLtS1&onDKqOJ}T>@h|-+WFvM{JmWORdCf@k1bTW?=s!`r8Yr>Wzp zH~o9iJE1E9-D$Y+ACE-p#+}rW$zPu>I*>};i2XnR7s;Y--9`TgF6{ryLO*&R`hq-z z{|r?P@9zlb8w=pu;v4M%C*Nro^jO&EfTEiv<6<%Iy;xdLUo0(rleYPb#R_eFTlSQd z;v(EAgA@k5@P;vU1^hrM??l)Vk1(vN)zIf(xVav1w|!rcu&gnsZsIauHX zWSr4AZc|@GbZO{EoWdy&&3#VAg|R-9^*`@Br}1X>y?f<_O17Qxi_u>)=~(KkS8P zARp>Mh~pBt24dk3`0+~0EgT^pmfJOvA^!Seo!o-F9`{&?^M}FjFSI=`Z=)YiTjN3Q zq+ig;IHuY&6d69R7&?qm2fZm3~s!U>m$+=Kil z;qA!s&Eyqf-obqfG{U10&h?++K96}lG{XmQIUIu)_!zE&<6wus!!n>MpeaPAAOiul z1x*s{g4^H<_=31UBTOvj)$lI%SFx8sD%Who?GpF^_bJ@U;YYYTkv~Sx-F+#2XZT{B z%#Fgwn9cADc=l2`^L!|C2Jj5#2cRAP0;Ktw4eNxZFb%)w(X1gpR9iHsFr%iW*@$@s zem_I5Mjk-kiu@*YU>*ikSr~V$<$nL-nK0%|Ft3Dn;j^`J#tU!5ao7%P;g|p7KEmYk z%pbru%=pdy2=kp_g+ZS2CGg8SI0%o!eXt%rSWDe%tQyFEUZ` zkp=ueT_2yw_%~6Ckj2;yxR>BxiY!Y{l=6&3smMx{N@NxGYTRq^uSM1o-WZ=q-#Sqm zkd4?)xHsY7j5ME46w6|wv@9n|E7FSHhPxgAHe@^D9r5fV=lV`$7j`G^-T3z)d(-*9 zdj|h^&*J~?$U*EwxDSUV$_R|cC5j8&-~n$K|DA_17zYjichBbCwY+;y=nlyZ-yyol z9g>I4$6gS>L-dI|q!3wzy%=``{w2s#>}44{q@1$^Dv*`ft8lN5*da9`JERus5_X6Y z>M=J!BYq}m!rTnzr5$2{7R;?+#m@$I%x%z~utOZsfw>d9@N+^p<{s$H-6@7o? z2=>to##mYCKqEcay}0}EA485~?;`J7+N(N?zM`k{`+B2hxn) z5=?%P*R4peiS)FQo?JhU9L4U^k=_Q<8#y>be&9Zg{|K@dd*4y=Yl8ej7Vo=V4C%K^ z39>Znb}86-JNg#4OChoddoJ!e{PU3c*fqho3;LR(Mdn~{Ox-S~dA_I9d{4+a?8cqj zSwGw^4ah3jcGeHKvwpap^~3E_#&zXfUxBO)NRa2LgHFL)a25_wKSfbrG9|5Hn5=mS zvkCb}@WG4JZGWL{`YmnID-eQxE8GP?hUcIh-k^Ou3n7G!f_S(aeh5!PHdMivfEt|U z3fKa-!A`gz9)TZ08dO6Qr1Fdpuc1wV+ewbDRkSe|a~Yw&vk_bS@r=dkBtzmj+^=2^yw zdr)(w4CO}1aKU>rQvB|Px!k@g{ue4&$Ux&&=<9wgKI<9Qbv}`C>uDL4x5dR_B5n@G z@o-p^^?4DStQPxAte3M1;vM2&!8-(VeY z{~Px{!hFL0in!)d;_)kdW`j0itK=**{+Va|bDHrFGC%7kDM-3W^c$5FX0rb&ntuzj zm(YML!L2lsHH>}eKQnJ!fvn6zzZ+SNy#`r}KkEF9y_uv8U5!B_9tec|H>RrC;Xr1L;UA_2gbJ-{x6joCI8<4J8~NR zmnrrq&agib`JeM=nG+f#u5qxCJ_hpPJZbe)()8P;qnq@3557SfU4z>WWH4q%+*k{#2g{iVKzcNehtuw*#u24^q;`Ii#(`i zZf+-YbUcFG&LJeLM%smwhA_nZ#UPBpFu1}=L-2qb{Qoxeg8$!!V_;!iZ7$~= zJx3_}L^(IX6^agfsI+0XBOUm+<9^{k#7Ji*b@G1JZqk^aMV3b}J`ZDj9?SR~S)IuE z9BQ%GfH9KsJv5{=lHvcuZQ@0dA7a`^dGaR|B-{(hj1Upe*`&7cvn32Ki7MZ-szjf z$NP+h4a%}!3-9#L)#MV9MT5$?1gUNo^?p0?iy**Un9-X zh~0#}BAETdi|nHcp{y7Zq^9p0#+8gSSR=0vxLOK*%u6OQPpM;`(!{(Z=F(lvQyxnY z1M8jIaP}M1&T4{{WFzw?XxIB_=dtUKQtp?~mtjJ@57|4*o(UAnhr_4`BB%$jd*Uhg z;EJXm0RKP7$H4y&axb(pY1{(-Jsf6eGqHCAY~a`Z>tw>dqmMlt(Obm-FU=k!dpF91 z#fj{)pu-n{E+6+U&q$E6MRfeI*F>Q67l!U1vVnaX^$EdZj6*jN*&K>%j@9@coOW;0gCeYCR{`LybW?FTV)?o&bih_Tj8UOX8Fr0?1OUHNLfvUFv{mf zNCYFi3uj?9W%g>gn7G!#f5DHT5;~y)df*IvkMjEi_#xy#2xU1Eo}m0a40&)F@Ai&g zQqG=)L(l@hhnL|)&=@HH)s%k|eQ=Bh<%UEe*uin0U3A9F%|JYXSIe z|M$`UXVLy6ZP@MSBBhObrrpB)H~jzpe@8ssPTCL~^WS?Tq>nlB{%OiT{no)nzFqpU z!^jcrqem(KlYGN_BiZN6H%mBQ;^i`Sly7-5K)xgm){suC{NDi#!+UTI`FKCv%s22D zyaJbz?@{m_xCS-@@FbK#D9l_&e!v2pfjRgX-iCFTlg_Y+`BPvBsL`O8mJR>FoWr_$ z?hxmr5+<(?ePJjlN1qr9p$Ll4vHlJvPzq(`=>J!v|IZ{rCGk}0Sbt~YpayC&*ST1K zXX2oqcpLgyf9D=1^y2*baLrs}-n&99`&Mu^JO2VsWB*?U`_J~T5IeMCZr{cEz{#A! zzmxNUli2f>%HFRO&IiW52lw7M&i9SyeBVU&pCxeiPc-NIZsdGH7{NRmfo@$SXA_09 z|16CCXQAvr3qcp}96Cg_E1D4UKa}zxN&ZKW|I;Em^o#PQIFAS&t^&g830Jr*?0?{V z#RbmspzSG{XHV-a?UIT8zeaSwoap~L(ElBxUFs9sDUn*tb$WEr4Cry`L>jU=|AO!) z!Z+dGjJr8Q#Im2gPFd`4%@nbwJ)msq4=DRy_PL_>)Skkg&s4_1dqg^SbIwRI|Hs+I z|8Wv2{|S`;m;sD9KQt@gJoy-kTm=y@{rH3G8^~ygg6rT0xD{@KRtxvG@eJiW zOEvf8ne9Bg&BwF2xF^rp!LvGe_IBQ(2f8skaqGfwU>UuwCC=DBhfdofdRs>HwoK@8 z!60Uj6TK}5dR*LN6fT@cL%rtz_qaEWGd+2RN8nDNTGHGN_rvWledBI*FY+PCT37UYev4r)_Kd(x^JS(Qqf?MF+-5N$NhR|=o=&lN&imt{ZrWwfE>a;jQj}yb_;F$=_KiJ($?E((|xq< z$ZqUCNV9{!L3t8uhj)tq{|Z+4_x}~zF#nS{uHd;Jg;Zc!U1Qo4EQgWJDYVtdN@()S z;D7B0K-PrPS3r*9{zo_i8^fqW;U-9g`#`gwz5!?<2Xc4QH-Nmm|AwEyFW_0I zfLdq$GV24EEa2q)lUxG0>pO1Kfe3%lT9_zj?csTqXl;V4YNC$N(5=5n|d5+Dig z0)7v#c>**}%0KO?)<^kA=3>`PQUA|S|04@d)BZ0~|1Z=2Ba5*ca4*5X6j?_2@*(Pf zuCGK^VXxkU{$DEkf5^IY^#5ox>a)=QLpEYJ;ogLQGt%s23^|1UpO3yI(u&2+5ZyE{udYRKa3`@ z|Ha_%!?OP+jQuXzlsy>Br2h}Oq(K=pLMOZg6YvT7|4%rGnc}V~0;&g1Gf>#!T=`5w7{15%R~ zB-xc!W%?j=41(zY21zAU z^##%X1u_4{`j0b6>Z*gp$YOPU8tXsMn92GNG@<|2%pPcSDC<9w=vG9q{<9O^ieT1% zlF|Q(XZ>f0Jq}^4|3C+H#4Hcvk{U6*bP5!bgJG-9tuHsIfgG+}Q-HtT3}vuJy>`QD%v ztY8B>v_U)l7Ke`S&A|7D>@x8^TKFEJ2YNH;|3E+er2!rL|0jZFXp#N@%Zz1@QT~_s z-oQ;;iu&(7_1}5wzw^|8=f#+IUg|T?GnPFsjhIc)lyY90!E8G(7Uy~T zAm?cx&(l7h7aL|fZf(eR<9X)L&P&I_dFecTUb+^~i}T!h&hR_W`JLyb*Kl6?s?SS5 zVFqAOe_r^%JoVps8S$N$QQdj!zw?~WcV0ZE^WsJNvd_yHjDvlee*p#4?>w>+-AKmr zoF~x=X2#Q&p_S4dKtB_@=w~{iBawb+H2qSfBc6U{7X3`Fzwlpce?A=J9EN@LOUaKu zxbPoruJ>}S58U)iJuu4kF6?>qS@R(`mVJ6iZ7AO%@oK{OE{V5_pnD87Yk@W3G4Kz?5V-++hV`;ZF9;2V4g`N-db6*?giw+G-^I1G8v2446b zprWAp9L~Uf)XQ0r3l~$zN5G}Pk#d@7WE>oX9H@jJLK-xK9iD`HX%8NRzhV9orr~Ya zgxl573s3W2eb51K!5gracME}1cmg!Eh1sBm9LP1%{_AM}^|b#`K%1_oZ7)O?8EF5j zY5xgRf-J>eKpfxX{VSGf{}*ZhPt*S6S3{UuWSxWYAJn^;|AR)%Ci)0XHpc&iG4Esk zKmGqQ{>Qy-hCcT^eQz7(-%0sLcKIm($p4K0XJ~KcX>X{{2atorKeWexzU05g|5KEI z!nlzh;_y!VYyO{j#)kss1kYXq-z8uEN&0Um4Xa5zJ&b@0dlBvdm|rH$iS6QQV$Hew%pT$88Bip6MGN1He!c4ipp<8_~sx9_cn+Jru8+Euo0pm24W!&{>(Lh zg-PPnB$NNU$$!YfoSV%0-$v^Ho#cO3fV@GNmO$PQX1L}x>{H<5Su&9;crU6yO+J*v zYU~ZjE08wiW@Ha?8`6c0NB)Vv&K}aC4jSMF-uYoL!*TckK7zkNC~2`4eho)pn?Jpv z7Mh`#@2HROsUPY856mEP2>UQ{B$fH!ea!!)GtZ0k{|nrU{4>|x&U2)JCfFs}v*@Nn z4&*{R?=}GW3+T2_qkq5DCxwfB=(zXM|LqgQqQCt|$9);yc>K$u0{=$htI}7f>T~FS z5T+LD@H0;JOZ`m0G|+EobkY7p6Xxb@^gk?&|Dk1LzqHz@hiMaSj4SO)12RQ??X;DS zYU*7R^)B*$zMYrhHq!0ed^f$z)U!J3UBdkw`=F6}6FE%S5!^Q4z-;0jP>a|7D8z=e=uZcz@E<2=%Aw|B#-Iq=|`iYa)%BF`Gl^|3C}@B?{|DSWt0#&6Pb&Q%%wxFu^40gXa3$1!TL9{GmLX_LfPAx$zH$=_Qr;v zmEN?o(uaFL@eB}dka>OT1sRTIE+U%oeFSq4vwZ)j`Tpnm{+9ye6yMbhj4$S?HrB!3 z!Tdh_iSjuO1C;eyK8-Bulz&7~CQs_r@wavApO~{_D8rj+TfhKb(#8j4yz{uxi!N}m zxWKKazXgM&!O&En4A1n*2w{5r`lXMw>px0Afplvoju+u~*HI_IIViYRG&=5+1~K@> z!t_tfivJL3rXS!NMRN3zrV#SsW9q>(Jlki;x_zW~SigLQd3u{!RV9?G>SXd`C;5>| zex;FLP?17@fo9n&*;9>5yU!~*N4=6;?iHQMD|t4rF=tA$> z3Eh}`pf?5mf77J&S5MOaeO(6oCS?eQr%%cVj4qty{F{^FUOFir@M8ACnCqmBgJ##1 zWP>(kN^&3VvAOiN$*G~fTU3^?Cp{6Ee6zsdUl zUu0xvMn>mn#I-mh?&TTU{~7V}^FN<$27R>|ImA;s>^mj3>8GSB;}m)@r=&jh6#u_mmKE%0t54ffLRFE`$_Gc--0==7C|173wf@Ca` zdA}dg7iRx^F&oAVT|u(Ub-u+y)xL3D9I@ll0iB8C)UV^>+%qoSyT_Sd9v5@OO0h5* z+CsS2xB;;e#uhy;_Rs-oLyj&Ch->1HoEI`69#@EX>1+EqRCg?CHS4IH&z3Mw`{YAE zbDZ^=Kcb&G&iu@{44)og+%V4g(#QDHCxs{^7e)J|7=>$tkMs4S$D~xhTGRq{PH2c| zZT|TH&LzuUDLTkAaz3SVwbbqPi4p44Ls(xnsYVtCOrc)RDqw%j{tzh(ACvO1W>pd2 ztSUpBRaMBCR71^vpVWeOG)QV`cU>=%hL57lbRW7*Z1Bn@jd~L~>yrDg0IDhpL!QOo z9mbe%GwJv)b|K25yEOp!_zQ>+~eYi(~|DvjvCHPfkTLhs#pHbUw?yGk0qxLoRGJ>w(#;h^F9x`=;b6@7}; zj15AWYl5fX0H8LZIS7YA3*}G?^({P$M z=%1`*j`>aGsN5*7;;rH?xJlf!J9&+l%kgKahwEZR_xR;9g4@ebAm}07xL!^Np({hR z{a3h>=KD@~1fGE+*v_lm0e8Z8;cmDOtn~dJK(=JC&PX57O#j6K-u-Uzl@H4pa@;m7 zqvuA&mEjh*@fGnP2lu*VDBUf?nQrEjRO9ZuGHU5ySkjltk34Qj1#( zK%X=$g~+0eVaW{`WjyDWytG%C&vMh>aZ5JOp>?<=$A&$d{%@#9wBaL?6W}4dM|2xU zBoCRN=#hdDkLWQMMtGQS^GNZ2j~KE%Qj+G8(tRE&v$&)@+9MUm{v?&is*N7%{}HJ{ z)-H}n-NK*5xO0T@UxR8Oo<^4k{SOcQKaVsY@rXIhL;tTzwe0dptD#m|xu0#QR@oCg z?9m)y{ii`Wc)pIK#BmG!T7`61t}&S#Z86YXDk zfhsWRl-^=x{o5)<)mACqSfC7X1*!yDdfH0+XP5G1t5oc1m&zooR6+ILcBw(uW?7|f zmsO0I>r<@Ku*WKmaaJ)UTBRw*D$UVWG4HjDW!@?+At$62X`Qu-EudZO$hP=)X^%O< z{8u~rZ@H@Th*i2Wt^A+CD&54>6JTZkyH)yj6{-FZs|-vOs6p;Gv{;~q2{&@Ao&I00 za`Aj_+H%i)f$}EnlyAS4{;yTWcUmc+dX*iZS6XDwjE(+Jo9H5K!Z8<`{4g8!vrY72 z4(9)C@+fYPg8}!F5Sx?++n8^)p-*j-iU^xjhTE9`Y?B`mre=B){U4ju5!QIlA@#_H zqYi0Ynv|d5-{jD#X57rXZ0M8Qq=k4|<7{F@+9C>-J;jDDiC(oQ*m!@v>fnBzE{AmO z(kUm;*PS>iJwtlco0Y5jl5EnSX+!_VhW?*k4NdCRFmiM&v(t{QWC!z#tWm`B4e`CyI}4T3SE%Zc`B`?! z<9@nb+%woNIX=5+gY#7OQU`sh*v9y;O=>3Fq;{6L&b6`rU!dw8ZPJilq#766SpO+fP19}Y@O6qg zGhbO^+t^R=nzZVQlr=U_*}~ey&U3Xzx3RaoNI5J;ssq`X)y6!eO`N_q>9%!BPhT75 zzm2umPU%MuoNklBr8XHl*CxZGjb+jQUo2J{pGIY8TSV)! zu>NP^yEQ0Xm_g+s^Cv8{C#|B7wMb#Sll{*Y>OYGZ0-REUERD1Nm<)TKG9d2fp}IZKquSE3q`IceR@U9?DcYB!zK7RiUaV6D(w zO2J<8KfM+G|5o}Ftx`Nm{_kl;|N9LoMV1k^e6olBNUK!N_i+B{8&aLoBQ^RQRf}7l zgZ!UwmHOFMX_#zf{MRa`rDE0OYGq!&RGA}M$^TN-!u?u}J?MYrC|hDH``_M>w(3&V z9;sE1jjhr_xX$oabOuY6v#(TjBYSwR-Xsh239ZsU)r09w9x^|}ECp$&M1Pd}YZ?8WxZg8>iT(}~za>oKmL3!8e~V-% zuNUn`=6`2LRc=Cu=pZl5r}B5NXMUgg|EW<`I6?W3^{C>#>%}l_P$lX7|7PEMDa&O2 z-!-Z#cKKB0PLHYzT`$#4^43H!|L+)8b*UX<4B#I<#NQCSp8l>+nRKJ7Dbu5x^`pw1 z?NgS_QPsjBSgqCTDgWJK)2|o1k@_GnycA^UizWB?< zn>4OGF_*Jfa8$VhM%5_uaU-XlGK?IGyqvj(%Vhxip$~eQqw7f-Ro%(V$)#P6ZW42J z$(J)%>l8=Q<cid?V_dh??JH5WLoHGh+>K79R#g$MvQMijGP)T9WUI2+ZuZP|OG$D!dosGEm~chD zZYc~dQTipy`<^!?e;4(Eshc^S7Rg2C7^yEZ-jr;lCbCP8x96&f`|{NB4KAch9lx_g z-NpBIyiUh|GIBZ3K1WSFkfV;9k;TYu8vfUz;oly}T#c%~g`p$w7st1%ryr0EkHG&x zC1ijJ_QGK}ewO>>a-RoA)kK#^X(l-Tk#A8uBa(y6#jaaK|BUZ7A6bC?5x(^!++zxz z$qASN7h~0pl$El5jK$MgCn~sJ;;|>fPS^#>&?4J7WBnE}f51N5kM9us+1sS;v+Kkv zw~Fm63K~j4+4N7esbP|{Crol}UO-@MXJ3Rr!#Q?|)2Hx_qi?_o%9N_*BiV)l$oKb&gIkW`?o;&%8`>7~g-8n9|2o z6ZU45kIarRu|SI}Oj^OJ4-#8Sr`TOV(xw|z?Y=SPFoj9Syi0YKhe_ADFxG#=q`RE{ z|6-W*E(A$mj8FAb=MBK%^r#v#hSC2Cl95=S8Vzno-(pO;iQ5xE|KAqI{7e}8UxH*j zz^60;F8+TpuCy%3msjOQcT0ZeTIvtxzmqwCbE8WY&3ILD+_*9v@v4#p zmnub;p`%fb;#Ng=hg9xfD^*-yjWSFPvUcaVs@uh(hX!Z_6Ey8*?zgW)%*+p4 zOyjDhdR(;*trhFcsIocMihX)5`af%>T~Gi2^jhi2M!zh{tGf7ZozUIqQa$I^O7A)J z)8f6VKi0wcXIu?tua%($`u~QtGNM~6qtnbsM|hQc!KFO&)4~VPC9iC+IiL{cJYj?JM^GT(9QE&lGd@t$=b$#uWCWs_N)^tvJGhmV+LuY^Qs19 zy}_+a`gQ0?FgL1O#~xval!fO#wUu4ts%FTo^u~4QsB$)H z{J1JyR)@4^(*I8` zQnrjD^glbK4ccQI^#4fP5C{9e8UO9<5NA@6>fTqRdQyv2FKtyH^e2)3Fu30#LogiM zAtNzG)Zsg?&sPE7h32Y2eo6eXox zQ%1P*g(#^QijvC7D5+YCV*D2+H8W9CI}s&y*p1lh@oO+fiGzFh!7TUoA*sIEI}*kC zKT297qU0!Uwv=eGCqzkGSd_GfbItx}nZUg>E?T+*qS1efmhQ+X=0Bq7|3}exkCOiQ zC>hutC4-sl|J)NL!>Lg+k{l(Y*j?D&_<16tWYK?bI1T4OW7tanX{%_R?0@xbW&Lj} z>wjBW|Jf?}Gh3x#eyiwDZ{_@_t+GV;0NS}=2!(JcE9c*WPyv-t)wflup=M|+_1`U= zO>>JF_uRty54T7|#4XZjyoK-o7RvTEY2LMs@!vMF9NWhDk9LX;W7dgnV&fot`}{WM zf3`{cP@Fhianf-*PCEPQrE9WAoPD=T_t5PU!To!as_6gL(EqKc|64=w58v4IAa@3#xSsSD!dxOLfpVhfVVvz~ZR=z<} zk&bg)`2WHd?!QI4vbTs6x=kCThiiIW8>DYxgY*w=kiCQ*T-v~T-v-wEHps}t2HA<5 z3wtusW8A=617!^|AQLnRTO@mVgJ=UbNKV8CNykl>vPJR|Hn0w}K?=e*py#uN^xs1N zbqn(-TNr9}q5qmv;4G|lt zTW=Cm@az14;U;MYjdO=&5A9$KwS)05=U-sgxn4s5o%1gyIR9di^DmY;|HAkZb0?fv z5b!PT|1Bvoyd zjY!k#>=v$Fc_M4G0z=jKMhge{Pr^ zkKR?{*LqhM^spT0Ww9obzi*23Ptbp0{k;&1SdTAWIwpop^dC~tf1tf9OFKqil=c5a z+9>3O_5Y)+|EK;x^nW>bq$ve`OfWmpH-#1#`hQ@>YyAfW9eoETR9ijQ$Tg zK2CIfx{*EUzmr~c0Q=xS|IfxeH1VDcqw6zbL;vsCd#p#J|C5IP4|p@t|A8^gMZ(>Iyc6z&FR?#_ zd=$P(U)Dj`Utng*Lz4?ZA5xDXLy(QgIxs^o41$yJFCve_2{;LTxUC`l5c2QX{{jDm zDcYYaaDOwN@gnzf;r}h{p}6nB{yp&F_Cw_VK*z?<`0p?0e%LtoW8!`3-gy?iO3qo? zb(T4b&!lDFJJL$oFuOSS1MJnD17hM_5W?G#?vOj=e~D)l>4M*2&WEGi;|TQ6aLz|C z=hP&hkv_`QaMoGoSl*=%#W^);cSt19R!aC*XoC^>Bm55j0NSB{$e)m!Wy&dKaSrqQ z@K0F5IJn-&xiZTWQinA19!)8n0|Jd;f{JX;rAb?0ocgYm_i-)__S)>vSf_Yjs*OA3 za^koFZsa+(Ag_X3;9Ky0h{OF6R_agrbj`3dqjrFZ>uLaLY&DM)(2bB22<4vAqr)-WY+s7+4`RJ%=>1fz^p5G+F`D z2(Y#GAPF(j%;*AfgyrbE@B6;*>Z7{4yY9PcnW|EE&q9rc4y2hLBhO6NnThORW7!Dp z25Wzww_2<>?5Ovb`5gIV<~N_`ec$KF{N-C`*2l%yX`6n}_W%1#Ux)ZR*1sZN{UQD+ zTKBxZL7SGhX!{BNc)I!L`|hv*jK7q>o)+u)>uK4(6}o(8eEtr9{;U2!#g+ZIw$Gpc z=l(y1_{R5)?;G$N`$hlXLOe0IACrIcqcKI(bcuSO@%xC>|HpnGkp?Xf(ePin2bNFk z_foWd+3zcU-R~>@f!|l;pXjo`oA05AY8_KY^*{7u(Ljwq@ng|M%|G&I(efid=1bDf0pm2=Ih3X z`E07h{l?%2{@m|ev>E5Bx);ONN30M2C7+MD{YU&R2JcR*xXm%$;-0_tI~n3G>sR$H z%)Fn^M||ZmzMfC%Q?PzbJo^d0pig}xCVqfV$i6YhOk2LR|4K~q{7ljM6MjRTR%wm4 zUv=K7G|y2yFW&{_+!W8rWBDiaY0%<7)TeRwLL}G0zoa+m-_eh{&-(}TQ}omHhpv$y z7XK2xhra8Y`gQUDYWw5juTYugYWkYxdMc;Ck*--hO#L)XyRN-uaf7rkh`Yq!68~%J zxBSEU5`L6MEKkzUS$-G&t-O9o{5|ng;@_sa}eHHS%ON zQL{E|`TC2|`s8=?e>>0A@!da-&aXSKpLL#p@}Eb~Gp>u@;=_3Oh3Nl;>*VKM7tXx4#Mc3x?mCTQ|+oL8ErOLX~V*9Bdr8M^kc>w>P+4Z8Vj&MVE+0xf>od8K7q zq19h=UC{dX{`c5Whs`I=YgdPDb=WE5?|Rnt+_)*;@|?MThfj`H_y0w#iI;6(v42s# zWdD28w>}{r{T|=n z&|k%{c!UONi27-Oda3x|38i@y;?6JsRdk8l9or$VR&krWn#Cw^yBoO{+xSzdd>Eq zw*C{AU$=ZjiMnn3hqjH2r|JK)?O)Nh<)5_ON7lu^u>39hN7nzH^?zykXD$Dr&-{O5 z`G>{-iSOi}692sIZxcUGAE1Y=|G0hsB|T>OzZU;9+kaC0dGXiir!D_uYPbA$aliPV zB=_yo&s$!mcUxW;|L5)t9+T!>^nPhROy9BnU+ZJO_~-n;f64#*SNea&mDa2NmjCHm31|Ib7Ee;(5R^N{`@|D&Bop8h{#bnBrQ+keRY??e0$59$ATD5k{I)-Q=K|J_4z zMSRuzjQE=Uv*PR4Z-{R?e(wGLzx|UB`JD#;-~R9J#L{m)p6`BK{152AqMxE))@DB~ei!``{T#iAo}$O; z!}L-5b$Xw)AEgf!()M4m?Q8VE(eKz+F8&SillJ`@{ifwl(I+gA+jo_&(>g8F3$#t& zpcm;c=(nW3C;kq7%JTmy{+vAio%jcx&!=tsQE`{g#b<5*GJV#zXXstFf6aQkxSOui z#U(_UEukA+rr@j{*^68x7 z|DWam7x#Du=>4quR?nGlB_8;?`G4Xe>%-!aCwBG!x&NnenxILVqG`HBm)~&zPk%rE z&+;r?ryKPDd;dRXJ{qmi>eGB#v`!neNn5l%wHrHQyRlxhfj0SzGwp2o1v;Q_TfZl+ zvagSv=Pq@#w{&maq;tSSS#2*%y+4kGw-=*j2YgA?1rv1~F=c&$m6D?c*Q`_4te?$B) z=$q7K{a@SnZ|Dd8&cF}TJ=Ro>{=I)2y{?Hq`}(CR z{r~nm|2B%>Z+gG{T70)&bdOO+IdqivcG}um z+S*y#+F9Cq$=1uX5N|2g|6D1r%H!3>La(+7n+lbdnk_Y#mRc;el$Kg8wU(BOv~926 z6t>4(?w9u!L4W*Y{G`_}$4|w76aQ`eqxk9g?-=HO=HZ{Ew--Ta=|ZveND(|*1dkQL z<3;d}B6w#JJRvBOT_}z!k`^h8ghjd{S&^zp^lo|&y_ft>;Lp6DK0v=hAEXb_hv`@8 zBlIMFl>RAwjDC%No&0d{&wQLdK~K>q={M=O=(p)<`W^Z-eTF_uze}H^-=k;f^YjIJ zmY$>M>5KFw`Z9fmzDmDOe?VWOKcqjRKc+vSH!3fd!tzpBQ3|U{VRb32DTQ^Vu%Q$- zmcr&z*is5xOJQ3nY!@D`7I_}77P%d+7RMg0>FoWP!}a3W!}ZQ0+?XiB&9YKh9D7tX zQwnE`uzs-!n~J?hO-1^n)}|tC+bqI?fg&9A4|RU#Xm(n7+%r{#gSDlwDDimkauJSR zEy9~cZYO0${wH;7ML1BDelodTgi}S`PNs@-PiuMHw&&@ z-dltXlSMdG)F)pm%FQ>6dj56!wIb}QEW+N&zaD+C@ZEvZ!f?@6e?3tg_18Cx8vgZK zQMd0_6ghvlv1p&~cFF%|+?~_LK6p zcsL$@Ydrk6c=+vJMWiK)aIjWbN+3nC|KZY(hf6yiF73Dw4_~n3LTSf^(vAzI9goDr zkJ#}@X~!d_O^?RIkJ|KTY15;nO^?OHkJs-bdq+$KsL4-weP=xSL_Au`phWa&acGI^v3TsQ z;k@dCf6V^JOW`}C1o~K!s&u9wkH_B{PIU?U@nUo7Y`-Jk@z!v<-%+YmsVUx3>@1!4 zcg8#48cuwvVcuD6E}i=);)%D0lV574CyLFb^Z%}R*Smf({`>eJ;uphDK>s5BQv7oK z@=wMu|8o43cz3+}6Y=ghYrCF{_r!a?9Pf?yzAxVU#ka+KZTrIK;(hVH56Am{JKpyv z@xJ{n+K%_f2jT}K6_?7p>uYB-N;#WTQbo|O6#|PtsABzt@6(9Uu*!ZFNQ1PM< z$A>-ar79aayeC%US(o^xVUyqM{{QV*Iuf?y& zuYWauBYxw3@f%OZZ&>(Rd^|q>{*T1RW%%(Y;}h{zJoO>3k_092PsS(T8=w5`_~h@z zZ^mza$m>J#Tk+fRKgRzQpNfAL|8x8=@pL@>w&&g!PyfQF>8tVdi`(%#@jFj{B|aUW z{)I2nAH=7hel|XB^)vD5=e#b^Bk`H|%zNT9PseATjn6#ib%AXCllY95{A_&osrc+y zJ{P|mzgxVXp=YCrgwMt2;&Zb6z4$$BzxQ-J6VDVc`g}a|d_42Dc;=7d^YMlF!n@-O z_`V?77oLhQd@{aJRK>~rg77)73-m~Q;rHVUTJqU=_Ve-VbMfr+@$45r7tj7tJgf1a z3#lY}?%na+Q}Ns<e#J`-P+i#i{1LzZHV2**lmj4=GbkC z-L}|mkKK;g?Tp>7*zJzpzS!-L-GSI0jNPHw9gW?w*d34EiP)Wt-Kp5U6uXyW_e$(u zjoq2py%xK(v3ossZ^Z7+*qw{r`Pf~E-No2lirwYdU5nlI*xiWT&Dh|KeytFbo|d)H!b zHukQ^-i_G18GCcFw-|d%v9}z1E3vm0d+V{c5qq1lw-tNav0oAUm9bwH`_-{u6Z^HX z-w^wavELN?&9UDS`>nCx7W*Bs-x>Q|v2TE|C-!?|zc2RtV}Bs_2V;LI_J?DCB=$#R ze=PRLV}By{Cu4sq_Akf&mDs-;`!lhBE%s+)|9b4-i2a+fKNtJ+vA+=ei?P2H`^&+h zzrPy$Yq7r`ocQ~j;SKo1Rjn??aXnbQVzA+QuxDnsD8Mm6^R&(5HiCe94t1WJ| z$E}XI)fuWN#uajP$G^~bG&xHTBJhU3;q+!~ErV{vOdZcW6k>9}<%Ze5OB zSK`*yxHS{EuEnj{xOF{l-H2O?Sd3dsacebht;MbNxU~_tHsjV-+}e&?J8}DB+%AjT z<#D?rZdbSpw`<~dZQQPl+x2m~A#OLu?WVZh9JgEIc5B>ji`(sSyCZIQ#_g`S z-5s}k;&yM`?u*;~aeE+c56A71xIG@Xr{ngexP3WpUy0jSS2#O;;1(;9c$;!b5MyFai=ft48)zmxHA%W#^TOI+?k3y)4}|I=StkU z8h2*m&TQPd5qECJow>L(A9ohw&SKnIiaV=uXDv*Hc(E*AERPo};>Egnu|8gGh!-2< z#nyPSEnaMo7rWxcp?Gm5UR;Y8H=hrWH=b=g8VNe$pgIm}f@j>4w~bjB@SBS zpe+vC<3NJWIOvLl?l|a)gWfpki-Z0+7>I+xI2ek9;W!wHgV8t`i-UUc&R2{s*RWG z;-&g{sUco!jF+0?rRI2PFkTvpZ(fXVmc=(K;+r+`&BpjosGNKU+!lPh)$3a;MX*`~n}Tl-#<$zN zij}r+?Zmg+y^56%uOjaBD&nqhtrdZ#?jqc)T(Z zuS~`(Q}N2Q*DIHb;93#P7Qyu*m@9&L!K+n8P+bHyMNnG=bwyAwc&*s+TCwA`V#jN( zMbH&RmX}_wc;#}D&!tx@{Yi7lBfVfTUb*a5lz6!)QBdS{xhPn0zU;WmMfsPD@-G+V zUoNt^Toiq|NOC1!x$0HqA}CT7k-bG^|J5RE!Fht&^8`w`s)SPF>*t9}rPZQDv_-B(RHQgx zwuIt(QS|lVs8TbO8llt#H{z9>UV2W7+RweR@M`U=l@>}HUTZ9>6a~_PSCNOH$VF6S zzi_@>s>wo8j^KQIDf5LQ&xN9Dg3{6FP}a^jf?X)3FDmQ;Su7F>N~=ZQiv`VjRHP`1 zSS&JKENwkMLzd6ilnpaVrptO2kS?lL(rOH_@jw>QZuAMJiTK6iB zT`$rKz8^_jtQ9GWwCk2$z39CwXf6e>wiQuJDL7wlEtZQ8I8iBRk5|jQWG|R4g6pL~ z8LyT-2#S2mlpwHF?p35Lf3;HUilh{rZz}3mtd$ZLhm;pf6<)7gDgwn+6e%l;Vk%s! z0!IkG9~H@q11pLH?XUDXD3~aMsUnb1Ws#5IK~x-St#m+LRV6C{|P)QbbY|kt3_m_ms-Be7>i|SzVM?{mNVs z%nM4#%D6c8LH<7HEaR;{-zq;VHAR^A$r^7n5rpFE3EWKmb_ zkf6ReOmM!e?)B$wUtiRzzNk}uQK$MMTfz6E;>e;--n~U2!}?MVMeYvNl8#)9SJ%U& z*H^dW)t$I^G455wy~?;(9rtSDUTxfKjC)OSuQ~3u#l7CRHx~CM2#>+!X^c&#B`YYg9#uJG47;`Pa{ z*C#hc-RDuyc{F$)jh#p1=g}3(X3iIHoJVuEEXM28W3NxkV|x5Nx>1TQHN@+e8sqiL z-LGG^^K#F5Go=^=*|%_! zYN%MYZ^6DL@lp{>v(it4G(^Mk`dZWLtMl>t`poO=edp0wyuQ)&`nrvqj@i2S`u38j z{Ppb>QR8{kb{=(~M+4{4-0NEo+7f{mZL7+5Yw7Uq_EM7V!SiVBJeod_X3nGezdvUF z{LqEJ->^_@c;n)9yiqY2`n#!&%Bg}XsfwzphH9yf>ZySmsfn7Yg<7eN+NqPesGo|w zhiI5aX^h5cf-cb&nxShnOV{ZJ&C_zcQKep0%CAyhRS)%2AIYmqURCm{l2_FvO%a|d z`Bq&fc~@N}`B%xmO8!;yuabY&O`0R+R4vdVEyWwvxT|qj!Q3CF)vBGtj?>_u1dQq-KzAfSXPVe z%2-vWRpqQYhpWn3bq-gRx9S|ODs$C2Ty+jtF|R6rRh?IHuR4#b%3M|Es7Yxvjj zui;8v@UP)t!@q`q4gVVcHT-M%*YL05U&FtKe+~Z{{x$q-_}B2S z;a|hQhJOwJ8vZr>YxqqLe`5{*8vZr>Yxvjjui;8v@UP)t!@q`q z4gVVcHT-M%*YNuhp*Pm>ui;aUdZsPQ{^2R34O`MxJH*s#_ z+{C$wa}(z#&P|+~_%`uv;@iZxiEk6%Caz6fo47V{ZQ|O*wTWvJ*Cwt_T${Kyac$z- z#I=cQ6W1oLO~J@NVJV!n=ic z3-1=*Eu338w{ULZ+`_qqa|_=VzAb!P__pwE;WMG(jVMVr7QQWfTllu{ZQAkwsCE{CbzL|W822Ijcps-HnweS+t{|TZDZTUwvBBY+cvgsY}?qj zv2A19cJ8+EZ98|{7`HLHL3v{v=eF~3n7B&5KL+7Ex)b{7Sr8?Qmc7Z~B^lvsyOQ{WL(= zWt}YR zWLY;#<1|ZB*4?0+G)If1@^!1U7H`&HB*)h~zP^fTNgnmr$no`#uXlWdJR8PHz6}#} zmDI68x(4YwVRcK>(-3b?%V)Ye-kcfwGo>tz#r=zAR8F;2NA=V|jnqWV)IzP)M(xyD z=r`{Bd2}(G!}pig8y_qkH4J@!>H7Eg9@lj|*mm62^Zlj4YY&zTSATzL;KAM^_ipNm z`(Qk&ptL@9xw0yeI{qid|T2k-w%Z2W1lkyv-!kiiFiY-^3n!2v< zmtQSZ$`VSkq(lu+UX(ghDx*l@02x=DSJ=wz{fZ)~J=Y4AHWbBLQl$cQw$S~GslQKX z<-xXD)i32&lv8ZJUoloHxAq+Ne3LpqDDS#t*UwA3e!t)^%C)2(7tdF2*dV?7g3~NT zLDpr2rpZdtT>rkz2TN$q8_S6;pDWU`JKQgyr-DT*pGT#$1WS4KrR_zoLzWh4sZdd9 zsRE^y(pf1LT>!dL%3UetE2WlraOzfVD9Uo_ly4L&wOes2&zEW~6_mxbv`ssKwb1u# zS)BHYa;l(8s-kMDDO4)4sP+TG3a7rJuGG2(LPh=kie^y*HIf!8tvBC)(Bu^@wzQO* zvdFu&PzlKSow4FQe0~A{`|3Wppo`?*TNNFJ&J%Z9?7Z(XcO@3}DKfBLoUijGtz6-K zT6zn7|2iuw>@6`-DzyOVD(xwfb{D#@rkb?;(s?dDws{*@^cMM+@05-za__bKynVb4 zEBZ>USG1)!Wbr1`yrSZgD(Y;>>2X?$B^QH}R54Jfbd?m5^Iqh7zEsp@uu!RtqK-p_ zO0DKl2_D&Cl-=W!pBR+1WgtyffcnpUn(;4iBjPcrNSqQ z&!?gam*_HGp`vKzs`i8PX{Fd+9I3rKWq7`%!JMjMb5WgZ=M@wc?c<$xo^#P01+@;TB%jODT~WCiE#dk4{uWaX3&DhtrO^G#!UmuG4ZHUY(D_YYTCBqd5*23eD1L z94;x}v;CoGd+tH1qk2-7=l4U;G2JOU=dxfKmGjZgb^T_w(k&FMxTk^;?c2s$lq|u*qR8>Vy)Jc;xMbhZA zInrmt@kq0@5=Yh2SIf6rdN*lD)m_w0J*13k$5%_QPv%J9%TcvsYU-$w25Fc^$-Y{7 z)mBgqIlgv?lvk_1wHt9%S53;Pne3+Z2IV)%yJ3yw z(}=B6K8@R7uh`$)~A}v_q40P3qgEK20}Cc}?EbA^n7NHL3hbWlw6CNp+rdyw8H8Np+fZZB1S}o`o@~d8eREL7!61DNR2m$&~Cb z$^Md}F01Zkhg`A!%5)rEbsBtx99>hDYuj-&t9fQ&%-VNdRj)gh*R9`hYHrBmhGTE2 z-VJ%)(A+nizMHCcQz*5_P8b57r!W9Br$yr$6OeB>X79WBXoNqI|< zm+W8KjH6}9%jbB##xxuc*(8<5!%<74`ISaOACjq|50@m(tPN!f~yv^eP?c zK|0cVbfnklNUzP29+@LOF-Lk}j`W-y={Y&lJ94CVRirMhm|Dlf5O<-gU=a-E@ssEyGL2xamPIqG!@6a&Bybq z(W|<>cJyI>#L_E+cQ~jALK=$8(qBcpm8dP8=_2pap3cG~a@*TI%W~% zqV$XEzohO-KBrhCQX}qx%^Ms=HjGXecSD8?~RiVj2-rMOvg#5V>)f?9HB{4?@sKU z^6Xq7ZP2OSo!Y5OeY=#`Wq-G_yW8W$+u@`aOYacLqpzHl*@w4J+x2Uoe#`^P^rd9F z16?B9y%A0Z%ShdQL7fcY9g@zM)XA_kBhrt^-vxg%rhUh><#;2t#mR(p6P72`ZSpE< z8zY^?^?%}<=41-%lzRDv71Jt68B^*$r7qLToNgj{PPdS{Os|q-FUi-p(#a)dUDifd z)aiqJ?FL8 zylo5O1@&6c77IAN4^G^4o-8_cQJohtFFOBA+Hy%7EJ?R4pJm6cIIk<}VZiCcP3eh2 zniDsfCk9&`sK=oj5J8qXwF#`8YLTd0J^-O)pK*Wx7iC*T|=4Gfr#euLti` z&)sRQblw-IwIgw=tL;=*+o_wrQ{8K)uKm+`dDpkm5Y3T1>bK&wVTz{Xw9)oP~HId(+*{K*w!Kc4t4D8j?=Em zIPGa7ZQ^F?v{(MU^6kalXM0~O$+J(MuI~e%fz&U_4F-n+gAr?L%0GJ= zUQ)M9j=wCwtMa<4&9ADbYw>i(z8Ps|)O|)?GmgC`pKIzgtNqhwOlI6>IdD_7B)UXnhdZP%WvKFZVNF?q@^N4q+Zq zhR?jSkx^2<&%Lveoj7w1osCwJ{iEtVD(&b-oVi&)^WAgKTFT zj(1yoW?=2iz}cDa=QFpqXPfG3KDo%9+4e9Yj>@QxARxO`4;5TA)Q*qGiJ92`1fK zqjlP#P1>Su+KJ@+q&fU^_~-D?;h)3reV*pDl`+=nOk;Yp`gTEM@6e*wRHljO5LE#P0k?>;3h;9tPM zfPVr10{#X33-}lCFW_Imzkq)Mzx$fBfZu0DTEOofC%MN-3-}lCFW_Imud^*J;9tPM zfPVr10{#X33-}lCFW_Im@BSz);9tPMfPVqM&!Mz{e*wRHr?iNlVJ9u(U&OzNe-Zy8 ze&1bb5&t6oMf{8S7x6FRU&QaeD=p$*#J`At5&t57&nJobCoSUFJCGLf`wmRZK#3VB zd2UII_!sdn;&<B^rA7RU_!sf(Bu$=$ z(jxvv{EPS(@h{?E#J`At5&t57&r4|${}O)BP02lD(u0wf@Gs$C!oP%n3I7s)9=gOs zmw4#X68P(zJx%zgSO8_?Pf6;WvPp zmhih@P3{rW682j_?Pi7<6p+VjDH#b zGX7=!%lMb^FXLavzl>jJby~*1jDH#bGX7=!%lQ49`Lv9G8Nd7Uw2Xfl|1$n%{LA>4 z@h{_F#=nez8UHf=W&F$dm+>#-U&glsUW#?lJ@75sY0k{+_Of`0}73VzS- zX$AiZ{uTTy_}w=q_l-$UT3W%sg5NVvTEV}9e+B;ve$V{L{YF~Bzk*+{Thi;6R`Bbr zPde+9=c1&;J~84YM!dv`ml*L9BVO{nlyu@Jo%o3nFX_lnjChF=FEQdJM!dv`ml*NV zDt<=1#E6#|@e(6m(&?Y{IV48Bw2Ge*FS++gtN0o55+hz>#7m5Li4iX`;w6Iwi4ia9 z8BC0Li4iX`;w47BWZ)n%;w60;i4iX`;w1wKi4iX`;w47BWKbb_#!Chkl7WT9h?f}g z5+h#H)0i0Xl7WWAh?hJECPuu(h?f}gl4ruih?f}g5+hz>#7m5Li4iX`;-xkGjChF= zFEQdJM!dv`ml*L9BVJ;}ON@Al5ic>~B?B3W5ic>~B}TktP$MzoB}Tl&h?f}g5+hz> z#7m5Li4iX`;w47B#E6&H@H65iM!dv`ml*MqzMI5|ml*L9BVN*Lni%ntF4N?mGBM&M zM!dv`ml*MqZq&qxmvp5jM!dv`m)v(I{XdBjFBxP>jChF=FBxn}23ry%URuY`h?f}g z5+hzR0FxN;5+hz>#7m5Li4iX`;w47BWN;=i;w47B!vK`$}rCHLn^7j)7EojkiG2ED|fml*UCgI>}t zof!0zKDET4ml*UCgI;3LOALC6K`(9KXV6Oquo8n_V$e$rdT9ecgI;3LOALBR_jY2? zOALC6K`$}rB?i63pqCi*lCJN>pqCi*5`$i1&`X{j(gyww{2TZ^M5Gy0`1{Q9kuVT{D^m$vXT{iQAZjDJagR??r9 zw(v6mrY-zi__y#IAWh7Gi5W04114s`q~9wUG)>Hai5W04119}o$q-30M3OvLCeM|L z889&eCT76I44Ag@GXp05Wr-Ou8A?gafQcC}=|4;QFOp}`#0;49VQ!o)zBcJS}u-@(sJm<$;uX2Qfwn3xIE4*nhdJNS3-@8D-J zObmvJ!7%OMXEIDohKb2AF&QQ%!^C8mm<*GC_rz$JcJS}u-@(6we+NJFVcNmZe3%SV zB?iRAfS4E%lOe05-!kpsXBtR5`1RkXot?-RFH#wmQw3F06;)FW)lwbRQv)?p6E#x{ zwNe|kQwMcY7j;t)^->@8(*O<95Dn7^jnWv6(*#Y@6iw45x=dH-D$UR}nx*S>gKp9s z&C>!c(h@Dx3a!!_t#Zv-IUM{A_)>3_ojMF2m2>m&@?8_~kPEY<{^6zdqw!hM(Op zm*FqN&-Rzg@U#BqGW`1Wav6T(E4d7R8U8Z-`kJ%8=FAqD_4{Skz|0z$4NGSBz-$~Q zm*Hm-%#04X48Oj?Y-}d$pU(QHb2)xPn7JH(IsS6|<@n3-m*X$TuMayj6K4I`xg38v z{&M{K7;`y({fxOBe>r~rjkz2@BT6pEUyk26P-Z~P<@n3-m*Zzf%;osY@t5N-$6t=0 z$1pP!=5qYy_{;Hg31%+A%n_K&@t5N-$6t=0uP&G4*SDUT`P?#3TV}e-JZ+h$Emz>@WXl!!EAVr&WlpwSf!~mBHg=VH*D}*u=2puU_$%=1 zQ_U6lIny#TT4vwMEN8g_KPy@05z7_$4GCvsUzxcivvOsAt<0sB^~YwOs?0By4H;*4 zsLZ&N*`YG)Q)Ye2%qW@RDKj=@hNsNMl)0EP_g1dJ&$5&mm9l=_%&3$Zm2w6C3j7uL z8J04`Qf64n3`?0|DKji(eZZM%DOcb(nJSx1l})C~#wN1KRJjtr$yC{7s%$7bn@p7} z@taJQO{U5wQ)QE>vdL80WU8z`Ih#zCEAg99l})J1mG~?1SK_b4Z-Q0U=bTNl$|hN5 zldN(jeiN;-alu@P-$biiiN6wmC4Q5xawYys{7gC7w)vdLk&3O`d)uEJl1pY0%5;jhA9g}(}a6@I3rT#cV`DOck+sVtjRmaFks z<2Si1SL3h7UyZ*SKZ8?dVaV0^*%&fcdS-aaTh{Hzn%#I|f= zTV|oiCbnhc7@4^$SL0`}%GLO*@mJ%o#$S!U8hWN!Ox+&UX4$;Pg;v65Va-{iSmgTDrU4So~pvT?p_GF`60UxS~~ zE1OuC**&t!b(!HS*Wj#>_038P2kC$6SlQ7C-Y@ zW<$wl5#(C@wfK!m=34x<_-pYSx6fu5WU~u$E&f{kwfJlCGp=Rhm$?>yEq>#exfVY& zTdu`#Of%Qwuf<=BzZSn43b_`)SqhnXG4lgt#<$E9ka+@fE&f{kEHSwje=Yu6{I&RN z@f)+sd;^(pAlKqIewAzSGtOl`f?SKg7Jn^%o`PJ5-}HuDhrbTLX%4v#KZ9N7H^_DP zO?k+5_*rjq9e&2UT!+67KR-gQ!(WG=EhpFEuft!5zYc#Letv~qho5I5*WqW~$#wYa z@Ymt5!_Ui*>+sj%uft!5zYc#L{yO}1`0Mc3;WzCf8w<|Hf^!{y(=M_x;arE`^owlz zMK)%a83VI%;%uBa*Ws_jZyYbz;jhDAho5&Mo1T&D@z>)wO(UD8k?Zl-<2PL+GZ*H1 z{KlGdJ^p(9_4w=Y*W)+-oa^z|+#p)ugA~hkxd)Prj2A?k6e%6xOc9{Z|plWKIVG-_4w=Y z*W<6pUyr{Ye?5NV9Og0@RH{fr;-+;dXe*^vo{0;b-JaYs72K){989p<^XKujH z7n2+Cvxnsd{EhgTgL5PPM*PNhGh=9G49)CgxedZ^Yk-zY%{U{zm+b_?et@BYu8|+=!o{H8E~&e>47O{JdPb z8GkeWX8a7uxfy>m{$~8m_?z)Jpv&`l;{B8K# z@VDV_!{3I#4SyT{HvGJWxeb3C{xXB;KcjAL$KQ^>9e+FicKq%5+wr&KZ^z$`za2l{WNydLLY&+2 zx8rZe&&Z$K@wek|$IsNC+wr&KZ^z$`za4)&{&xKB_}lTf<8Q~`j=vp$JN|b3?fBdA zx8rZe-;Tc>e>;8?b8F8p2iyYP46@50}OzYBjC{x1Ao_`C3T;qSuVg})1b7yd5%UHDBz$*l3Y z3x5}W{@>h%zYBjCev_?o7yd5%UHDDB%3b)o@ORF8p2iyYTbveVa{N4Dw@pt3z#@~&f$1r!}@5XPMLGH%ijo&1n z+>O5*zsWwC4>5P+Hw__oF!QX?w2R}b} zHa#Nu;P1iTgTDuV5B?tfJ@|X@_u%ir--Evge-Hj1{5|-4@b}>F!QX?w2Y(O#9{fG{ zd+_(*@4?@LzXyLW{$BjO_l@$<0fUi`ej z*?f-Ni@z6tFaBQqz4&|a_u}ux&lj3`-g7U0zV~eYNAAVni@z6tFaBQqz4&|a_u}ux z-;2K&e=q)C{Jf>P7k@AQUi`iId-3<;H*qod;_t=Zi@z6tFaBQqz4&|a_u}VM&VBg% z@blm2KKy<7`|$VS@5A4R-^7&Mho6@<_u=Pf&3*X$@bfrllgM%({yzMD`1|np;qSxW zhrbVhAO1f4{Lr}%e;@un{C)WQ@b}^G!{3L$4}TwizUgdoTIQY3efasOb07Xb{C)WQ z@b}^G!{3L$55EaCxevd21i24C&u#9*--o{sKmTnuzaaPF@5A4ZzaM`;{(k)Z`1|qm z=;nU>{Mxx6e?R_y{Qda*@%Q8J$KQ|NWS!iPzaM`;{(k)Z`1|qq=#hJG__v7!!-;ci^zlp8c{E9q)e*pgg{sH_0_y_Qtbd(4158xlbKY)J# z{{a30`~&z0@DJc0z(0V00RI5~0sI5_2k;NzAHY9=e*pgg{sH_0_y_P0;2*%x_ni5j z^8o$<`~&#S_sD$Ec>wC8#od@unFqH@J z58xlbKY)J#{{a30{3cUnb4l_b{z3eM_y_S1;vd9sqE$B0Di7ix#6O6C5I^sJ9>hP0 ze-Qs5{z3eM_y_TuK$%U>%7gd^@ekrRQ7aGPAH;9&N;Y>T58^j@D-Yu5^Uj0#2k{T$ z=T*+;2joHggZKyW58@xhKZt)2zsZ?-5I=u-9>hP0e-Qs5{z3eM_y_S1;vd96h<_0O zApSx8L->dA58)reKZJh>{}BEm{5%fNA^b!5hwz(dmWS{U;pa)uL-FMA%sKZJh>{}BEm{6qM8-188A^Ca>R z{vrHB`1!c=5dIdA58)reKZJh>{}6t2EAkM2li~ck6T&})e+d6DeiPL5F#ciu z!}v{@%jRO_Vf@4Rhw%^NH@_$k;~&O9jDHyaF#ciu!}y2s591%kKaAg`yF84482>Q- zVf@4Rhw%^NAI3k7e;EHT{$c#X_=oWi<2RQn591%kKa77E|1kbx{KNQ%@eku4#y^aI z82>Q-Vf@4Rhw%^NAI5K9Ngl>OjDHyaF#ciuBlt(~kKiA{KZ1V*{|NpO{3G~B@Q>g( z&mfQBAHi=ja2~-wf`0`62>ucLBlt(~kKiA{KZ1V*{|J6_6!HlE5&R?gNAR1okj;0= z=4<5<{3G~B@Q>gh!9Rk31pf&B5&R?gNAQo}AHhF@e+2&se)Ar(3Cei{{|NpO{3G~B z@Q>gh!9Rk31pf&B5&Wb0&6CKZ_($=N;x}g^kK!N2KZ<`8|0sU*DDo(N6QJ`b{!#p+ z_($=N;vdC7ihmUUD1H+yvw0VJ6#ppxQT(I$NAZv1H@_#3;vdC7ihmUUDE?9WqxeVh zkK#9PBah-A#XpLF6#poGKdO*N@tcgDO~%fn_($=N;vdC7ihmUUDE?9W=6>W+{G<3s z@sHv+zbKF4AHzR}e+>T^{xSSx_{Z>%;UB|4hTmMIJci%Ax;%z|4F4E@b4T+S{xSSx z_{Z>%;UB|4hTkOdJcfS^{}_IAQ1Tdl^P2J){xSUKH{~(>WBAALkKrG~KZbt{{}}!; z{A2jX@Q>ji!#{?94F4GZG5llr$MBEgAHzR}e+>T^{xSSx_{Z>%;UC9u&TAgWKaPJK zzxl6u9RE1}as1=>$MKKjH;*un<2OGxkK-T5KaPJK|2Y0}{Nwn?@sHyl$3KpL9RE1} zas1=>$MKKjAICqAe;off{&D={_{Z^&<2O$ykK-T5KaPJK|2Y0}{Nwn?@sHyl$8X|& z9>+h9e;off{&D={_{Z^&;~&RAj(;5gIQ|Lz6Zp+5%MN|@K4~Mz;C`$p1?nWe*(X`M|lFjIc(V+ zq&$It0{;a53H%fIC-6_;pTIwXe**sm{t5i%EM-3|kSFl_ae+L6e**sm{t5hkY#^H( zmnZO>AD7Mf%;q@d3H%fIC-6_=pTzG+3GyWVN&J)e{XjvU#6O9D68|LrN&J)eC-G0> zpTuv@U7o~0iGLEm`O?{s8stg*llUj`PvZ9@2YC{|A3DgB_$TpC;-ADniQkVOQK2|nYHBaK7#6O9D68|LrN&J)eC-G0>pTs|je-i&B{z?3k_|0?8 zllaYb&6D^i@lWEP!f(!CHpe?p;h(}kg?|eF6#gmvQ~0OwPvM`!KZSn^{}g_6WwT#r z$W!>I@cWg9JcZx<**t}R3jY-TDg0CTr|_Fso2T$k;h(}kh2Jkb^A!Fm z{8RX+@K52N!as$73jY-TDg0CTr|?hV_sbA@3jY-TDg0CTr|?hVpTa+de+vH;e)A)< zIlXxrzj?iR8viu@Y5aa(Et@-;r}0nYpT<9p-(1Q(jei=yIl$Q*;5?0g8viu@Y5ddp zr}3LVm#6Vh|36jTOKxS2x@Kjv7Zhm&P0SQo@TC9@@$=ifVgw;?Hk5DnrRujD51 z?1Y4*;2B;aYel@F2zz ze(l};vHy?#f9(HbzZUQQ*sndlKlcBz|BwBD?Eho`AN&8<|HuA6_W!Z}kNtn_|6@Oq z`Tp3iS-wB^zu5m`zrOEY?0>QU#r_xjU+jOe|HXcN^S#*rV*iW%FZRFK|6>1({rb{- zvH!*X7yDoAf3g3?{uldS?0>QU#r_xjU+jOe|Hb|n`(NzWr{0VGFZRFK|6>1({V(>v z*#Bbxi~TS5zu5m`|BL;)#Cx&-#r_xjU+jOe|Hb|n`(NyTvH!*X7yDoA*D>CU{d(AY zwV!KnulB#%|7!oM{jc`F+W%_*tNpL`zuNz5|EvA4_P^TyYX7VKT!eeI|JD9i`(N#U zwg1)rSNmV>*M{G#{jc`F+W%_*tNpL`zuNz5|EvA4_LEKS)&5ueU+sUj|JD9i`=Lqq zYX7VKulB#%|7!oM{jc`F+W%_*tNpL`zuNz5KXm(E?SHlZ)qb7&z1ja}|C{}9_P^Qx zX8)W0Z}z|0|7QQ2{crZa+5cw$oBeP0zuEt0|C{}9_P^QxX8)W0Z}z|0|7QQ2{crZa z+5cw$oBeP0zuEt0|C{}9_P^OrbGbMB-|T<0UvqkI_P^QxX8)W0Z}z|0|7QQ2{crYb z?eES0H~Zi0f3yG1e*OKu+5cw$oBeP0zuEt8|GWL~_P^WzZvVUe@Akjj|8D=g{qOd_ z+y8F=yZ!I>zuW(A|GWL~_P^WzZvVUe@Akjj|8D=g{qOd_+y8F=yZ!I>zuW(A|GWL~ z_P^WzZvVUe@Akjj|8D=g{qOd_+y8F=yZ!I>zuW(A|GWL~_P^WzZvVUe@Akjj|8D=g z{qOd_+y8F=yZ!I>zuW&||A+k__J7#_VgHByANGIP|6%`!{U7#!*#BYwhy5S+f7t(F z|A+k__J7#_VgHByANGIP|6%`!{U7#!*#BYwhy5S+f7t(F|A+k__J7#_VgHByANGIP z|6%`!{U7#!*#BYwhy5S+f7t(F|A+k__J7#_VgHByANGIPuV+8b!hP8PVgHByANFh9 z@5BC2`?c=(Y5%AFER*}R|I_|Y`)NY=X+QM)KJ5p8->3cX@B6g>)BaEUHPH8I|EK+* z_J7*{Y5%AFK(qU_|I_|Y`#&x%U z{xAE#?EkX=%lzwBqv+?W0QnftQ;%lzwH0A|I2>1!F}8RZU49Z-}Y0Y?%V!v z`@ik~wx54+-}Zmo|84)b{oIcGw*TAyZ~MRP_kR-HxBcJtlcw(5{%`xg?fZANzmo|FK{HfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~Cz zPyc`V|I`1U{@;J0>&<@s|LOlv|9|@b)Bm6T|MdT-|3Cfz>HkmvfBOH^|DXQ<^#7;- zKmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V z|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd z|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U z{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va z`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{Qs< zr~g0w|LOlv|9|@b)Bm6T|MdT-|3Cfz>HkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24a zpZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmv zfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>! z|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^ z|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y z{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ< z^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~Cz zPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;- zKmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V z|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd z|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U z{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va z`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{Qs< zr~g0w|LOlv|9|@b)Bm6T|MdT-|3Cfz>HkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24a zpZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmv zfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>! z|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^ z|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y z{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ< z^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~Cz zPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;- zKmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V z|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd z|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va`v24apZ@>!|EK>y{r~CzPyc`V|I`1U z{{QsHkmvfBOH^|DXQ<^#7;-KmGsd|4;va z`v24apZ@>!|EK>y{r~CzPyc`V|I`1U{{QsHpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq z|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq z)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ z|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJ zr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c z|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUc zPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnx*Y@lG z)BmUcPye6(zkhAN{y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ z^#AGq)BmUcPye6(zkg%D{y+VH`v3I*>HqsT_Ur%C|EK>?|DXOp{eSxZ^#AGq)BmUc zPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>? z|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm? zpZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v) z{y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6( zKmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp z{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7n zfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH z`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D z|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ z^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ z|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I* z>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq z|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq z)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ z|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJ zr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c z|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUc zPye6(KmC9D|MdUq|9i0i!TtyPAMAgy|H1wT`ycFou>ZmS2m2rFf3W|-{s;RX?0>NT z!TtyPAMAgy|H1wT`ycFou>ZmS2m2rFf3W|-{s;RX?AQON|4;v){y+VH`v3I*>HmAM z|H1wT`ycGr|EK@&!TtyPAMAgy|H1wT`ycFou>ZmS2m2rFf3W|-{s;RX?0>NT!TtyP z_5bPr)BmUcPye6(KmC9D{~qmswExlmNBi~v>HmAQU;p2u{g3uP+W%<(qy3NeKidCj z|D*kn_CMPHX#b=AkM=*>|7icC{g3uP+W%<(qy3NeKidCj|D*kn_CMPHX#b=AkM=*> z|7icC{g3uP+W%<(qy3NeKidCj|D*kn_CMPHX#b=AkM=*>|7ib{{ZIBk+5cq!ll@Qj zKiU6e|C9at|MdUq|9i6k$^Iw%pX}HFr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ z^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ z|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I* z>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGqd$#}C{%8B2?SHoa+5Tty zpY4CP|Ji>1fBOHP?SHoa+5Tty_5bPr)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH z`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D z|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ z^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ z|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I* z>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq z|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq z)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ z|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJ zr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c z|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUc zPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>? z|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm? zpZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v) z{y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6( zKmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp z{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7n zfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH z`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D z|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ z^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ z|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I* z>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq z|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq z)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ z|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJ zr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c z|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUc zPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>? z|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm? zpZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v) z{y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6( zKmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp z{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7n zfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH z`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D z|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ z^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ z|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I* z>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq z|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq z)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ z|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJ zr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c z|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUc zPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>? z|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm? zpZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v) z{y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6( zKmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp z{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7n zfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH z`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D z|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|DXOp{eSxZ z^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIa z#r}V>U;m%}KmC9D|MdUq|I`1c|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ|LOnJ z|EK>?|DXOp{eSxZ^#AGq`^)~n?AQON|4;v){y+VH`v3I*>HpLJr~gm?pZ-7nfBOIQ z|LOnJ|EK>?|DXOp{eSxZ^#AGq)BmUcPye6(KmC9D|MdUq|I`1c|4;v){y+VH`v3I* z>HpLJr~gm?pZ-7nfBOIQ|LOnJ|EK>?|L^VJe_6r+gaHTx5C$L&Kp2290AT>a0E7Vu z0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx z5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S z1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rX zAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv z3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L& zKp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST z7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhl zfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuw zFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp229 z0AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPU zVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I z0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy z!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a z0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1Da zgaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!- z0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu1JJwu@Akjj|8749APhk7 z_P^WzZvVUe@Akjj|8D=g{qOd_+y8F=yZ!I>zuW(A|GWL~_P^VY0SE&S1|SST7=SPU zVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I z0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy z!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1Da^kM&p{U7#!*pC4S0}uuv z3_u_Df7t(F|A+k__J7#_VgHByANGIP|6%`!{U7#!*#BYwhy9=Sf7<_P|EK+*_J7*{ zY5%AFpZ0&+|7riH{h#)K+W%?)r~RMyf7<_P|EK+*_J7*{Y5%AFpZ0&+|7riH{h#)K z+W%?)r~RMyf7<_P|EK+*_J7*{Y5%AFpZ0&+j{yh+(5L;M_J7*{X+H)a3_uuwFaTiy z!T^K;2m=rXAPhhlfG_}I0Kx!-0SE)om;GP%f7$a0Q6=5m;GP%f7$a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhl zfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuw zFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp229 z0AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPU zVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy!T^K;2m=rXAPhhlfG_}I z0Kx!-0SE&S1|SST7=SPUVF1DagaHTx5C$L&Kp2290AT>a0E7Vu0}uuv3_uuwFaTiy z!T^K;2m=rXAPhhlfG_}I0Kx!-0SE&S1|SST7=SPUVF1DagaPpOf2ysc0Yn3c1`rJ( z8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2 zKs1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4Immo zG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaM~?`)L5t0HOgz1BeC? z4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1 zAR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ( z8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2 zKs1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0jb`1?r%hz1Z1AR0h4fM@{G z0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLaw zq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V z0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?W zL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz z1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$ zhz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c z1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh z5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC? z4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1 zAR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ( z8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2 zKs1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4Immo zG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4 zfM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCF zXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks118 z0MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT z(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G z0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLaw zq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V z0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?W zL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz z1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$ zhz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c z1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh z5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC? z4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1 zAR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ( z8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2 zKs1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4Immo zG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4 zfM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCF zXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks118 z0MP)V0Yn3c1`rJ(8bCCFXaLawMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR*uS=)1~3}HXaJ)Dj0P|o zz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQt zFdD#U0Q)!g(*Q;T7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IX|*iQo(4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IX|+D`)*4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e7Y zll@QjKiU6eKMi0sfYAU(1K5-OPxe3A|78D@{ZIBk+5cq!ll@QjKiU6e|C9Ys_CMMG zWdD=>Pxe3A|78D@{ZIBk+5cq!ll@QjKiU6e|C9Ys_CMMGWdD=>Pxe3A|78D@{ZIBk z+5cq!ll@QjKiU6e|C9Ys_CMMGWdF1M&-Opt|7`!W{m=G4+y89;v;EKZKimIo|Fiwi z_CMSIZ2ze+fM@+4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y-~ZBuLWlqUABPVI*LfU%9Grb|cyjPM-r;}$$Kn5Q@TJ|s&;1Yo#eeJV@Bet` zzx)5as9@IU>>5B}%>_`$#P;NSkQKJ*_?`G3FT!9Skq|9$X}XZpAQ z+kg0wfB(*Z_>lkbz=!;kt6RVS94i64`o!@^E2>kx#(*5sW&V&B`<%HAkU(V7v_`Uq)e22q--R6=(C=UW&Y$vM?$!PM%S}$df4OVr_b<18{Qk|k>fgUP3HkdsXJvl>=4Ha~ z-~7A#{TmM8_it{(|NWa=!hiqf-VO)9_}^Vw|NXnmln(!kgNuR=|9H>o_20icC-D1s zU-WT-*6$zROZ@)v;_>$nQ^LV-?a!y@`}2?A+8=jl|Ne11?e8D=6aD^i z1Ih1;E4aTePV@Y}_`3W1A~*ki5##>8xCQ<9#l2m>FK#&cef3Ju!5`SG&q2Sh`ikFI zciR5Gxor6R=9', '.']) + + def test_sequence_builders(self): + tokenizer = AlbertTokenizer(SAMPLE_VOCAB) + + text = tokenizer.encode("sequence builders") + text_2 = tokenizer.encode("multi-sequence build") + + encoded_sentence = tokenizer.build_inputs_with_special_tokens(text) + encoded_pair = tokenizer.build_inputs_with_special_tokens(text, text_2) + + assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_2 + [tokenizer.sep_token_id] + + +if __name__ == '__main__': + unittest.main() diff --git a/transformers/tokenization_albert.py b/transformers/tokenization_albert.py index f2e37222f6..0785e55ad2 100644 --- a/transformers/tokenization_albert.py +++ b/transformers/tokenization_albert.py @@ -8,6 +8,7 @@ from shutil import copyfile logger = logging.getLogger(__name__) +VOCAB_FILES_NAMES = {'vocab_file': '30k-clean.model'} SPIECE_UNDERLINE = u'▁' class AlbertTokenizer(PreTrainedTokenizer): @@ -16,12 +17,12 @@ class AlbertTokenizer(PreTrainedTokenizer): - requires `SentencePiece `_ """ - # vocab_files_names = VOCAB_FILES_NAMES + vocab_files_names = VOCAB_FILES_NAMES # pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP # max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES def __init__(self, vocab_file, - do_lower_case=False, remove_space=True, keep_accents=False, + do_lower_case=True, remove_space=True, keep_accents=False, bos_token="[CLS]", eos_token="[SEP]", unk_token="", sep_token="[SEP]", pad_token="", cls_token="[CLS]", mask_token="[MASK]>", **kwargs): super(AlbertTokenizer, self).__init__(bos_token=bos_token, eos_token=eos_token, @@ -142,15 +143,15 @@ class AlbertTokenizer(PreTrainedTokenizer): """ Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens. - A RoBERTa sequence has the following format: - single sequence: X - pair of sequences: A B + An ALBERT sequence has the following format: + single sequence: [CLS] X [SEP] + pair of sequences: [CLS] A [SEP] B [SEP] """ sep = [self.sep_token_id] cls = [self.cls_token_id] if token_ids_1 is None: - return token_ids_0 + sep + cls - return token_ids_0 + sep + token_ids_1 + sep + cls + return cls + token_ids_0 + sep + return cls + token_ids_0 + sep + token_ids_1 + sep def get_special_tokens_mask(self, token_ids_0, token_ids_1=None, already_has_special_tokens=False): """ @@ -175,25 +176,24 @@ class AlbertTokenizer(PreTrainedTokenizer): return list(map(lambda x: 1 if x in [self.sep_token_id, self.cls_token_id] else 0, token_ids_0)) if token_ids_1 is not None: - return ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1, 1] - return ([0] * len(token_ids_0)) + [1, 1] + return [1] + ([0] * len(token_ids_0)) + [1] + ([0] * len(token_ids_1)) + [1] + return [1] + ([0] * len(token_ids_0)) + [1] def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None): """ Creates a mask from the two sequences passed to be used in a sequence-pair classification task. - A BERT sequence pair mask has the following format: - 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 2 - | first sequence | second sequence | CLS segment ID + An ALBERT sequence pair mask has the following format: + 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 + | first sequence | second sequence if token_ids_1 is None, only returns the first portion of the mask (0's). """ sep = [self.sep_token_id] cls = [self.cls_token_id] - cls_segment_id = [2] if token_ids_1 is None: - return len(token_ids_0 + sep + cls) * [0] - return len(token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1] + cls_segment_id + return len(cls + token_ids_0 + sep) * [0] + return len(cls + token_ids_0 + sep) * [0] + len(token_ids_1 + sep) * [1] def save_vocabulary(self, save_directory): """ Save the sentencepiece vocabulary (copy original file) and special tokens file diff --git a/transformers/tokenization_xlnet.py b/transformers/tokenization_xlnet.py index a4f1a6e3ba..c01fbbbeeb 100644 --- a/transformers/tokenization_xlnet.py +++ b/transformers/tokenization_xlnet.py @@ -185,9 +185,9 @@ class XLNetTokenizer(PreTrainedTokenizer): """ Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens. - A RoBERTa sequence has the following format: - single sequence: X - pair of sequences: A B + An XLNet sequence has the following format: + single sequence: X + pair of sequences: A B """ sep = [self.sep_token_id] cls = [self.cls_token_id] @@ -224,7 +224,7 @@ class XLNetTokenizer(PreTrainedTokenizer): def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None): """ Creates a mask from the two sequences passed to be used in a sequence-pair classification task. - A BERT sequence pair mask has the following format: + An XLNet sequence pair mask has the following format: 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 2 | first sequence | second sequence | CLS segment ID From 1e5b31c3881cb1313216ce0f3cffae89b0845d4f Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 30 Oct 2019 20:25:32 +0000 Subject: [PATCH 094/505] Several fixes and improvements --- transformers/modeling_albert.py | 36 +++++++++--------- .../{30k-clean.model => spiece.model} | Bin transformers/tokenization_albert.py | 2 +- 3 files changed, 19 insertions(+), 19 deletions(-) rename transformers/tests/fixtures/{30k-clean.model => spiece.model} (100%) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index ad8b979cef..371a2e535c 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -7,6 +7,7 @@ import torch.nn as nn from torch.nn import CrossEntropyLoss from transformers.configuration_albert import AlbertConfig from transformers.modeling_bert import BertEmbeddings, BertModel, BertSelfAttention, prune_linear_layer, gelu_new +from transformers.modeling_utils import PreTrainedModel from .file_utils import add_start_docstrings logger = logging.getLogger(__name__) @@ -37,18 +38,17 @@ def load_tf_weights_in_albert(model, config, tf_checkpoint_path): print(name) for name, array in zip(names, arrays): - print(name) - og = name + original_name = name name = name.replace("ffn_1", "ffn") name = name.replace("ffn/intermediate/output", "ffn_output") name = name.replace("attention_1", "attention") - name = name.replace("cls/predictions/transform", "predictions") - name = name.replace("LayerNorm_1", "attention/LayerNorm") + name = name.replace("cls/predictions", "predictions") + name = name.replace("transform/", "") + name = name.replace("LayerNorm_1", "full_layer_layer_norm") + name = name.replace("LayerNorm", "attention/LayerNorm") name = name.replace("inner_group_", "albert_layers/") name = name.replace("group_", "albert_layer_groups/") name = name.split('/') - - print(name) pointer = model for m_name in name: if re.fullmatch(r'[A-Za-z]+_\d+', m_name): @@ -78,13 +78,12 @@ def load_tf_weights_in_albert(model, config, tf_checkpoint_path): pointer = getattr(pointer, 'weight') elif m_name == 'kernel': array = np.transpose(array) - print("transposed") try: assert pointer.shape == array.shape except AssertionError as e: e.args += (pointer.shape, array.shape) raise - print("Initialize PyTorch weight {} from {}".format(name, og)) + print("Initialize PyTorch weight {} from {}".format(name, original_name)) pointer.data = torch.from_numpy(array) return model @@ -177,9 +176,9 @@ class AlbertAttention(BertSelfAttention): b = self.dense.bias projected_context_layer = torch.einsum("bfnd,ndh->bfh", context_layer, w) + b - projected_context_layer = self.dropout(projected_context_layer) - layernormed_context_layer = self.LayerNorm(input_ids + projected_context_layer) - return layernormed_context_layer, projected_context_layer, reshaped_context_layer, context_layer, attention_scores, attention_probs, attention_mask + projected_context_layer_dropout = self.dropout(projected_context_layer) + layernormed_context_layer = self.LayerNorm(input_ids + projected_context_layer_dropout) + return layernormed_context_layer class AlbertLayer(nn.Module): @@ -187,17 +186,17 @@ class AlbertLayer(nn.Module): super(AlbertLayer, self).__init__() self.config = config - self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.full_layer_layer_norm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) self.attention = AlbertAttention(config) self.ffn = nn.Linear(config.hidden_size, config.intermediate_size) self.ffn_output = nn.Linear(config.intermediate_size, config.hidden_size) def forward(self, hidden_states, attention_mask=None, head_mask=None): - attention_output = self.attention(hidden_states, attention_mask)[0] + attention_output = self.attention(hidden_states, attention_mask) ffn_output = self.ffn(attention_output) ffn_output = gelu_new(ffn_output) ffn_output = self.ffn_output(ffn_output) - hidden_states = self.LayerNorm(ffn_output + attention_output) + hidden_states = self.full_layer_layer_norm(ffn_output + attention_output) return hidden_states @@ -352,16 +351,17 @@ class AlbertModel(BertModel): encoder_outputs = self.encoder(embedding_output, extended_attention_mask, head_mask=head_mask) + sequence_output = encoder_outputs[0] - + pooled_output = self.pooler_activation(self.pooler(sequence_output[:, 0])) - outputs = (sequence_output, pooled_output,) + encoder_outputs[1:] # add hidden_states and attentions if they are here + outputs = (sequence_output, pooled_output) + encoder_outputs[1:] # add hidden_states and attentions if they are here return outputs @add_start_docstrings("Bert Model with a `language modeling` head on top.", ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) -class AlbertForMaskedLM(nn.Module): +class AlbertForMaskedLM(PreTrainedModel): r""" **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: Labels for computing the masked language modeling loss. @@ -384,7 +384,7 @@ class AlbertForMaskedLM(nn.Module): """ def __init__(self, config): - super(AlbertForMaskedLM, self).__init__() + super(AlbertForMaskedLM, self).__init__(config) self.config = config self.bert = AlbertModel(config) diff --git a/transformers/tests/fixtures/30k-clean.model b/transformers/tests/fixtures/spiece.model similarity index 100% rename from transformers/tests/fixtures/30k-clean.model rename to transformers/tests/fixtures/spiece.model diff --git a/transformers/tokenization_albert.py b/transformers/tokenization_albert.py index 0785e55ad2..7b16bb573f 100644 --- a/transformers/tokenization_albert.py +++ b/transformers/tokenization_albert.py @@ -8,7 +8,7 @@ from shutil import copyfile logger = logging.getLogger(__name__) -VOCAB_FILES_NAMES = {'vocab_file': '30k-clean.model'} +VOCAB_FILES_NAMES = {'vocab_file': 'spiece.model'} SPIECE_UNDERLINE = u'▁' class AlbertTokenizer(PreTrainedTokenizer): From 5680a1106302b1ebeb960de0700d6379c0aeef5c Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 30 Oct 2019 20:42:49 +0000 Subject: [PATCH 095/505] Activation function managed from the config file --- transformers/configuration_albert.py | 2 +- transformers/modeling_albert.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/transformers/configuration_albert.py b/transformers/configuration_albert.py index c86c9565cb..15437dbbea 100644 --- a/transformers/configuration_albert.py +++ b/transformers/configuration_albert.py @@ -16,7 +16,7 @@ class AlbertConfig(PretrainedConfig): intermediate_size=16384, inner_group_num=1, down_scale_factor=1, - hidden_act="gelu", + hidden_act="gelu_new", hidden_dropout_prob=0, attention_probs_dropout_prob=0, max_position_embeddings=512, diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 371a2e535c..7e9f7f1c46 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -6,7 +6,7 @@ import torch import torch.nn as nn from torch.nn import CrossEntropyLoss from transformers.configuration_albert import AlbertConfig -from transformers.modeling_bert import BertEmbeddings, BertModel, BertSelfAttention, prune_linear_layer, gelu_new +from transformers.modeling_bert import BertEmbeddings, BertModel, BertSelfAttention, prune_linear_layer, ACT2FN from transformers.modeling_utils import PreTrainedModel from .file_utils import add_start_docstrings @@ -190,11 +190,12 @@ class AlbertLayer(nn.Module): self.attention = AlbertAttention(config) self.ffn = nn.Linear(config.hidden_size, config.intermediate_size) self.ffn_output = nn.Linear(config.intermediate_size, config.hidden_size) + self.activation = ACT2FN[config.hidden_act] def forward(self, hidden_states, attention_mask=None, head_mask=None): attention_output = self.attention(hidden_states, attention_mask) ffn_output = self.ffn(attention_output) - ffn_output = gelu_new(ffn_output) + ffn_output = self.activation(ffn_output) ffn_output = self.ffn_output(ffn_output) hidden_states = self.full_layer_layer_norm(ffn_output + attention_output) @@ -392,6 +393,7 @@ class AlbertForMaskedLM(PreTrainedModel): self.bias = nn.Parameter(torch.zeros(config.vocab_size)) self.dense = nn.Linear(config.hidden_size, config.embedding_size) self.word_embeddings = nn.Linear(config.embedding_size, config.vocab_size) + self.activation = ACT2FN[config.hidden_act] def tie_weights(self): """ Make sure we are sharing the input and output embeddings. @@ -405,7 +407,7 @@ class AlbertForMaskedLM(PreTrainedModel): outputs = self.bert(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None) sequence_outputs = outputs[0] hidden_states = self.dense(sequence_outputs) - hidden_states = gelu_new(hidden_states) + hidden_states = self.activation(hidden_states) hidden_states = self.LayerNorm(hidden_states) prediction_scores = self.word_embeddings(hidden_states) From ce9eade29c75fa676ac528d1fe21d9f4ac3c5622 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 30 Oct 2019 20:50:44 +0000 Subject: [PATCH 096/505] Initializer range using BertPreTrainedModel --- transformers/modeling_albert.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 7e9f7f1c46..b45208b696 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -6,8 +6,7 @@ import torch import torch.nn as nn from torch.nn import CrossEntropyLoss from transformers.configuration_albert import AlbertConfig -from transformers.modeling_bert import BertEmbeddings, BertModel, BertSelfAttention, prune_linear_layer, ACT2FN -from transformers.modeling_utils import PreTrainedModel +from transformers.modeling_bert import BertEmbeddings, BertPreTrainedModel, BertModel, BertSelfAttention, prune_linear_layer, ACT2FN from .file_utils import add_start_docstrings logger = logging.getLogger(__name__) @@ -362,7 +361,7 @@ class AlbertModel(BertModel): @add_start_docstrings("Bert Model with a `language modeling` head on top.", ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) -class AlbertForMaskedLM(PreTrainedModel): +class AlbertForMaskedLM(BertPreTrainedModel): r""" **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: Labels for computing the masked language modeling loss. From 25a31953e8820eb5c88d8ad35ee547efccfe577c Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 30 Oct 2019 21:18:06 +0000 Subject: [PATCH 097/505] Output Attentions + output hidden states --- transformers/modeling_albert.py | 58 ++++++++++++++++++++++++++++----- 1 file changed, 49 insertions(+), 9 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index b45208b696..52cab2ea69 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -105,6 +105,7 @@ class AlbertAttention(BertSelfAttention): def __init__(self, config): super(AlbertAttention, self).__init__(config) + self.output_attentions = config.output_attentions self.num_attention_heads = config.num_attention_heads self.hidden_size = config.hidden_size self.attention_head_size = config.hidden_size // config.num_attention_heads @@ -177,7 +178,7 @@ class AlbertAttention(BertSelfAttention): projected_context_layer = torch.einsum("bfnd,ndh->bfh", context_layer, w) + b projected_context_layer_dropout = self.dropout(projected_context_layer) layernormed_context_layer = self.LayerNorm(input_ids + projected_context_layer_dropout) - return layernormed_context_layer + return (layernormed_context_layer, attention_probs) if self.output_attentions else (layernormed_context_layer,) class AlbertLayer(nn.Module): @@ -193,25 +194,45 @@ class AlbertLayer(nn.Module): def forward(self, hidden_states, attention_mask=None, head_mask=None): attention_output = self.attention(hidden_states, attention_mask) - ffn_output = self.ffn(attention_output) + ffn_output = self.ffn(attention_output[0]) ffn_output = self.activation(ffn_output) ffn_output = self.ffn_output(ffn_output) - hidden_states = self.full_layer_layer_norm(ffn_output + attention_output) + hidden_states = self.full_layer_layer_norm(ffn_output + attention_output[0]) - return hidden_states + return (hidden_states,) + attention_output[1:] # add attentions if we output them class AlbertLayerGroup(nn.Module): def __init__(self, config): super(AlbertLayerGroup, self).__init__() + self.output_attentions = config.output_attentions + self.output_hidden_states = config.output_hidden_states self.albert_layers = nn.ModuleList([AlbertLayer(config) for _ in range(config.inner_group_num)]) def forward(self, hidden_states, attention_mask=None, head_mask=None): - for albert_layer in self.albert_layers: - hidden_states = albert_layer(hidden_states, attention_mask, head_mask) + layer_hidden_states = () + layer_attentions = () - return hidden_states + for albert_layer in self.albert_layers: + if self.output_hidden_states: + layer_hidden_states = layer_hidden_states + (hidden_states,) + + layer_output = albert_layer(hidden_states, attention_mask, head_mask) + hidden_states = layer_output[0] + + if self.output_attentions: + layer_attentions = layer_attentions + (layer_output[1],) + + if self.output_hidden_states: + layer_hidden_states = layer_hidden_states + (hidden_states,) + + outputs = (hidden_states,) + if self.output_hidden_states: + outputs = outputs + (layer_hidden_states,) + if self.output_attentions: + outputs = outputs + (layer_attentions,) + return outputs # last-layer hidden state, (layer hidden states), (layer attentions) class AlbertTransformer(nn.Module): @@ -227,11 +248,30 @@ class AlbertTransformer(nn.Module): def forward(self, hidden_states, attention_mask=None, head_mask=None): hidden_states = self.embedding_hidden_mapping_in(hidden_states) + all_attentions = () + + if self.output_hidden_states: + all_hidden_states = (hidden_states,) + for layer_idx in range(self.config.num_hidden_layers): group_idx = int(layer_idx / self.config.num_hidden_layers * self.config.num_hidden_groups) - hidden_states = self.albert_layer_groups[group_idx](hidden_states, attention_mask, head_mask) + layer_group_output = self.albert_layer_groups[group_idx](hidden_states, attention_mask, head_mask) - return (hidden_states,) + hidden_states = layer_group_output[0] + + if self.output_attentions: + all_attentions = all_attentions + layer_group_output[1] + + if self.output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + + outputs = (hidden_states,) + if self.output_hidden_states: + outputs = outputs + (all_hidden_states,) + if self.output_attentions: + outputs = outputs + (all_attentions,) + return outputs # last-layer hidden state, (all hidden states), (all attentions) ALBERT_START_DOCSTRING = r""" The ALBERT model was proposed in From 870320a24e28f187d0dfd10b82c4d60d5269374d Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 30 Oct 2019 22:30:21 +0000 Subject: [PATCH 098/505] Early tests --- transformers/modeling_albert.py | 59 +++---- transformers/tests/modeling_albert_test.py | 191 +++++++++++++++++++++ 2 files changed, 219 insertions(+), 31 deletions(-) create mode 100644 transformers/tests/modeling_albert_test.py diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 52cab2ea69..f906352311 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -11,6 +11,15 @@ from .file_utils import add_start_docstrings logger = logging.getLogger(__name__) + +ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP = { + 'albert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-pytorch_model.bin", + 'albert-large': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-pytorch_model.bin", + 'albert-xlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-pytorch_model.bin", + 'albert-xxlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-pytorch_model.bin", +} + + def load_tf_weights_in_albert(model, config, tf_checkpoint_path): """ Load tf checkpoints in a pytorch model.""" try: @@ -39,6 +48,7 @@ def load_tf_weights_in_albert(model, config, tf_checkpoint_path): for name, array in zip(names, arrays): original_name = name name = name.replace("ffn_1", "ffn") + name = name.replace("/bert/", "/albert/") name = name.replace("ffn/intermediate/output", "ffn_output") name = name.replace("attention_1", "attention") name = name.replace("cls/predictions", "predictions") @@ -114,29 +124,6 @@ class AlbertAttention(BertSelfAttention): self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) self.pruned_heads = set() - def prune_heads(self, heads): - if len(heads) == 0: - return - mask = torch.ones(self.num_attention_heads, self.attention_head_size) - heads = set(heads) - self.pruned_heads # Convert to set and emove already pruned heads - for head in heads: - # Compute how many pruned heads are before the head and move the index accordingly - head = head - sum(1 if h < head else 0 for h in self.pruned_heads) - mask[head] = 0 - mask = mask.view(-1).contiguous().eq(1) - index = torch.arange(len(mask))[mask].long() - - # Prune linear layers - self.query = prune_linear_layer(self.query, index) - self.key = prune_linear_layer(self.key, index) - self.value = prune_linear_layer(self.value, index) - self.output.dense = prune_linear_layer(self.output.dense, index, dim=1) - - # Update hyper params and store pruned heads - self.num_attention_heads = self.num_attention_heads - len(heads) - self.all_head_size = self.attention_head_size * self.num_attention_heads - self.pruned_heads = self.pruned_heads.union(heads) - def forward(self, input_ids, attention_mask=None, head_mask=None): mixed_query_layer = self.query(input_ids) mixed_key_layer = self.key(input_ids) @@ -225,7 +212,7 @@ class AlbertLayerGroup(nn.Module): layer_attentions = layer_attentions + (layer_output[1],) if self.output_hidden_states: - layer_hidden_states = layer_hidden_states + (hidden_states,) + layer_hidden_states = layer_hidden_states + (hidden_states,) outputs = (hidden_states,) if self.output_hidden_states: @@ -367,6 +354,8 @@ class AlbertModel(BertModel): self.pooler = nn.Linear(config.hidden_size, config.hidden_size) self.pooler_activation = nn.Tanh() + self.init_weights() + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None): if attention_mask is None: @@ -422,33 +411,41 @@ class AlbertForMaskedLM(BertPreTrainedModel): list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. """ - + + config_class = AlbertConfig + pretrained_model_archive_map = ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP + load_tf_weights = load_tf_weights_in_albert + base_model_prefix = "albert" + def __init__(self, config): super(AlbertForMaskedLM, self).__init__(config) self.config = config - self.bert = AlbertModel(config) + self.albert = AlbertModel(config) self.LayerNorm = nn.LayerNorm(config.embedding_size) self.bias = nn.Parameter(torch.zeros(config.vocab_size)) self.dense = nn.Linear(config.hidden_size, config.embedding_size) - self.word_embeddings = nn.Linear(config.embedding_size, config.vocab_size) + self.decoder = nn.Linear(config.embedding_size, config.vocab_size) self.activation = ACT2FN[config.hidden_act] + self.init_weights() + self.tie_weights() + def tie_weights(self): """ Make sure we are sharing the input and output embeddings. Export to TorchScript can't handle parameter sharing so we are cloning them instead. """ - self._tie_or_clone_weights(self.classifier.word_embeddings, - self.transformer.embeddings.word_embeddings) + self._tie_or_clone_weights(self.decoder, + self.albert.embeddings.word_embeddings) def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, masked_lm_labels=None): - outputs = self.bert(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None) + outputs = self.albert(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None) sequence_outputs = outputs[0] hidden_states = self.dense(sequence_outputs) hidden_states = self.activation(hidden_states) hidden_states = self.LayerNorm(hidden_states) - prediction_scores = self.word_embeddings(hidden_states) + prediction_scores = self.decoder(hidden_states) outputs = (prediction_scores,) + outputs[2:] # Add hidden states and attention if they are here if masked_lm_labels is not None: diff --git a/transformers/tests/modeling_albert_test.py b/transformers/tests/modeling_albert_test.py new file mode 100644 index 0000000000..46a2eeb729 --- /dev/null +++ b/transformers/tests/modeling_albert_test.py @@ -0,0 +1,191 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import unittest +import shutil +import pytest + +from transformers import is_torch_available + +from .modeling_common_test import (CommonTestCases, ids_tensor) +from .configuration_common_test import ConfigTester + +if is_torch_available(): + from transformers import (AlbertConfig, AlbertModel, AlbertForMaskedLM) + from transformers.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP +else: + pytestmark = pytest.mark.skip("Require Torch") + + +class AlbertModelTest(CommonTestCases.CommonModelTester): + + all_model_classes = (AlbertModel, AlbertForMaskedLM) if is_torch_available() else () + test_pruning = False + test_head_masking = False + + class AlbertModelTester(object): + + def __init__(self, + parent, + batch_size=13, + seq_length=7, + is_training=True, + use_input_mask=True, + use_token_type_ids=True, + use_labels=True, + vocab_size=99, + hidden_size=32, + num_hidden_layers=5, + num_attention_heads=4, + intermediate_size=37, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + type_sequence_label_size=2, + initializer_range=0.02, + num_labels=3, + num_choices=4, + scope=None, + ): + self.parent = parent + self.batch_size = batch_size + self.seq_length = seq_length + self.is_training = is_training + self.use_input_mask = use_input_mask + self.use_token_type_ids = use_token_type_ids + self.use_labels = use_labels + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.intermediate_size = intermediate_size + self.hidden_act = hidden_act + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.type_sequence_label_size = type_sequence_label_size + self.initializer_range = initializer_range + self.num_labels = num_labels + self.num_choices = num_choices + self.scope = scope + + def prepare_config_and_inputs(self): + input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) + + input_mask = None + if self.use_input_mask: + input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) + + token_type_ids = None + if self.use_token_type_ids: + token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size) + + sequence_labels = None + token_labels = None + choice_labels = None + if self.use_labels: + sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size) + token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels) + choice_labels = ids_tensor([self.batch_size], self.num_choices) + + config = AlbertConfig( + vocab_size_or_config_json_file=self.vocab_size, + hidden_size=self.hidden_size, + num_hidden_layers=self.num_hidden_layers, + num_attention_heads=self.num_attention_heads, + intermediate_size=self.intermediate_size, + hidden_act=self.hidden_act, + hidden_dropout_prob=self.hidden_dropout_prob, + attention_probs_dropout_prob=self.attention_probs_dropout_prob, + max_position_embeddings=self.max_position_embeddings, + type_vocab_size=self.type_vocab_size, + initializer_range=self.initializer_range) + + return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels + + def check_loss_output(self, result): + self.parent.assertListEqual( + list(result["loss"].size()), + []) + + def create_and_check_albert_model(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + model = AlbertModel(config=config) + model.eval() + sequence_output, pooled_output = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids) + sequence_output, pooled_output = model(input_ids, token_type_ids=token_type_ids) + sequence_output, pooled_output = model(input_ids) + + result = { + "sequence_output": sequence_output, + "pooled_output": pooled_output, + } + self.parent.assertListEqual( + list(result["sequence_output"].size()), + [self.batch_size, self.seq_length, self.hidden_size]) + self.parent.assertListEqual(list(result["pooled_output"].size()), [self.batch_size, self.hidden_size]) + + + def create_and_check_albert_for_masked_lm(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + model = AlbertForMaskedLM(config=config) + model.eval() + loss, prediction_scores = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, masked_lm_labels=token_labels) + result = { + "loss": loss, + "prediction_scores": prediction_scores, + } + self.parent.assertListEqual( + list(result["prediction_scores"].size()), + [self.batch_size, self.seq_length, self.vocab_size]) + self.check_loss_output(result) + + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + (config, input_ids, token_type_ids, input_mask, + sequence_labels, token_labels, choice_labels) = config_and_inputs + inputs_dict = {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask} + return config, inputs_dict + + def setUp(self): + self.model_tester = AlbertModelTest.AlbertModelTester(self) + self.config_tester = ConfigTester(self, config_class=AlbertConfig, hidden_size=37) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_albert_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_albert_model(*config_and_inputs) + + def test_for_masked_lm(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_albert_for_masked_lm(*config_and_inputs) + + @pytest.mark.slow + def test_model_from_pretrained(self): + cache_dir = "/tmp/transformers_test/" + for model_name in list(ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + model = AlbertModel.from_pretrained(model_name, cache_dir=cache_dir) + shutil.rmtree(cache_dir) + self.assertIsNotNone(model) + +if __name__ == "__main__": + unittest.main() From c14a22272f3fc17bb2eaeca62986c31a7d26bc85 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 31 Oct 2019 14:04:10 +0000 Subject: [PATCH 099/505] ALBERT passes all tests --- transformers/configuration_albert.py | 4 +--- transformers/modeling_albert.py | 9 +++------ transformers/tests/tokenization_albert_test.py | 2 +- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/transformers/configuration_albert.py b/transformers/configuration_albert.py index 15437dbbea..04f9fa8d60 100644 --- a/transformers/configuration_albert.py +++ b/transformers/configuration_albert.py @@ -7,7 +7,7 @@ class AlbertConfig(PretrainedConfig): """ def __init__(self, - vocab_size_or_config_json_file, + vocab_size_or_config_json_file=30000, embedding_size=128, hidden_size=4096, num_hidden_layers=12, @@ -15,7 +15,6 @@ class AlbertConfig(PretrainedConfig): num_attention_heads=64, intermediate_size=16384, inner_group_num=1, - down_scale_factor=1, hidden_act="gelu_new", hidden_dropout_prob=0, attention_probs_dropout_prob=0, @@ -61,7 +60,6 @@ class AlbertConfig(PretrainedConfig): self.num_hidden_groups = num_hidden_groups self.num_attention_heads = num_attention_heads self.inner_group_num = inner_group_num - self.down_scale_factor = down_scale_factor self.hidden_act = hidden_act self.intermediate_size = intermediate_size self.hidden_dropout_prob = hidden_dropout_prob diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index f906352311..9bb38dead9 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -202,17 +202,14 @@ class AlbertLayerGroup(nn.Module): layer_attentions = () for albert_layer in self.albert_layers: - if self.output_hidden_states: - layer_hidden_states = layer_hidden_states + (hidden_states,) - layer_output = albert_layer(hidden_states, attention_mask, head_mask) hidden_states = layer_output[0] if self.output_attentions: layer_attentions = layer_attentions + (layer_output[1],) - if self.output_hidden_states: - layer_hidden_states = layer_hidden_states + (hidden_states,) + if self.output_hidden_states: + layer_hidden_states = layer_hidden_states + (hidden_states,) outputs = (hidden_states,) if self.output_hidden_states: @@ -247,7 +244,7 @@ class AlbertTransformer(nn.Module): hidden_states = layer_group_output[0] if self.output_attentions: - all_attentions = all_attentions + layer_group_output[1] + all_attentions = all_attentions + layer_group_output[-1] if self.output_hidden_states: all_hidden_states = all_hidden_states + (hidden_states,) diff --git a/transformers/tests/tokenization_albert_test.py b/transformers/tests/tokenization_albert_test.py index dd63f6756b..59eb3bceb0 100644 --- a/transformers/tests/tokenization_albert_test.py +++ b/transformers/tests/tokenization_albert_test.py @@ -22,7 +22,7 @@ from transformers.tokenization_albert import (AlbertTokenizer, SPIECE_UNDERLINE) from .tokenization_tests_commons import CommonTestCases SAMPLE_VOCAB = os.path.join(os.path.dirname(os.path.abspath(__file__)), - 'fixtures/30k-clean.model') + 'fixtures/spiece.model') class AlbertTokenizationTest(CommonTestCases.CommonTokenizerTester): From b21402fc86257feca05ab050d061e15441a49929 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 31 Oct 2019 14:19:31 +0000 Subject: [PATCH 100/505] Python 2 tests + licence --- transformers/modeling_albert.py | 16 ++++++++++++++++ transformers/tokenization_albert.py | 19 ++++++++++++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 9bb38dead9..b6d1291725 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -1,4 +1,20 @@ +# coding=utf-8 +# Copyright 2018 Google AI, Google Brain and the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch ALBERT model. """ + import os import math import logging diff --git a/transformers/tokenization_albert.py b/transformers/tokenization_albert.py index 7b16bb573f..7cba99b9e4 100644 --- a/transformers/tokenization_albert.py +++ b/transformers/tokenization_albert.py @@ -1,4 +1,21 @@ - +# coding=utf-8 +# Copyright 2018 Google AI, Google Brain and the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Tokenization classes for ALBERT model.""" +from __future__ import (absolute_import, division, print_function, + unicode_literals) + from .tokenization_utils import PreTrainedTokenizer import logging import unicodedata From c4403006b8301a24bfeec99c39ce8d5d47df570f Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 31 Oct 2019 15:30:11 +0000 Subject: [PATCH 101/505] External MLM head --- transformers/configuration_albert.py | 17 ++++++++++++++ transformers/modeling_albert.py | 35 +++++++++++++++++++--------- 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/transformers/configuration_albert.py b/transformers/configuration_albert.py index 04f9fa8d60..b72bbb971e 100644 --- a/transformers/configuration_albert.py +++ b/transformers/configuration_albert.py @@ -1,3 +1,20 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" ALBERT model configuration """ + from .configuration_utils import PretrainedConfig class AlbertConfig(PretrainedConfig): diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index b6d1291725..487455e561 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -401,6 +401,26 @@ class AlbertModel(BertModel): outputs = (sequence_output, pooled_output) + encoder_outputs[1:] # add hidden_states and attentions if they are here return outputs +class AlbertMLMHead(nn.Module): + def __init__(self, config): + super(AlbertMLMHead, self).__init__() + + self.LayerNorm = nn.LayerNorm(config.embedding_size) + self.bias = nn.Parameter(torch.zeros(config.vocab_size)) + self.dense = nn.Linear(config.hidden_size, config.embedding_size) + self.decoder = nn.Linear(config.embedding_size, config.vocab_size) + self.activation = ACT2FN[config.hidden_act] + + def forward(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.activation(hidden_states) + hidden_states = self.LayerNorm(hidden_states) + hidden_states = self.decoder(hidden_states) + + prediction_scores = hidden_states + self.bias + + return prediction_scores + @add_start_docstrings("Bert Model with a `language modeling` head on top.", ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) class AlbertForMaskedLM(BertPreTrainedModel): @@ -433,13 +453,8 @@ class AlbertForMaskedLM(BertPreTrainedModel): def __init__(self, config): super(AlbertForMaskedLM, self).__init__(config) - self.config = config self.albert = AlbertModel(config) - self.LayerNorm = nn.LayerNorm(config.embedding_size) - self.bias = nn.Parameter(torch.zeros(config.vocab_size)) - self.dense = nn.Linear(config.hidden_size, config.embedding_size) - self.decoder = nn.Linear(config.embedding_size, config.vocab_size) - self.activation = ACT2FN[config.hidden_act] + self.predictions = AlbertMLMHead(config) self.init_weights() self.tie_weights() @@ -448,17 +463,15 @@ class AlbertForMaskedLM(BertPreTrainedModel): """ Make sure we are sharing the input and output embeddings. Export to TorchScript can't handle parameter sharing so we are cloning them instead. """ - self._tie_or_clone_weights(self.decoder, + self._tie_or_clone_weights(self.predictions.decoder, self.albert.embeddings.word_embeddings) def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, masked_lm_labels=None): outputs = self.albert(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None) sequence_outputs = outputs[0] - hidden_states = self.dense(sequence_outputs) - hidden_states = self.activation(hidden_states) - hidden_states = self.LayerNorm(hidden_states) - prediction_scores = self.decoder(hidden_states) + + prediction_scores = self.predictions(sequence_outputs) outputs = (prediction_scores,) + outputs[2:] # Add hidden states and attention if they are here if masked_lm_labels is not None: From 4f3a54bfc8fa8749f6d5b29f110148738a646fcd Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 31 Oct 2019 16:37:34 +0000 Subject: [PATCH 102/505] ALBERT can load pre-trained models. Doesn't inherit from BERT anymore. --- transformers/__init__.py | 2 +- transformers/configuration_albert.py | 9 ++++++ transformers/modeling_albert.py | 44 +++++++++++++++++++++++----- transformers/tokenization_albert.py | 25 +++++++++++++--- 4 files changed, 68 insertions(+), 12 deletions(-) diff --git a/transformers/__init__.py b/transformers/__init__.py index 152d520e7b..bdfb1a0922 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -107,7 +107,7 @@ if is_torch_available(): CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_encoder_decoder import PreTrainedEncoderDecoder, Model2Model - from .modeling_albert import (AlbertModel, AlbertForMaskedLM) + from .modeling_albert import (AlbertModel, AlbertForMaskedLM, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) # Optimization from .optimization import (AdamW, get_constant_schedule, get_constant_schedule_with_warmup, get_cosine_schedule_with_warmup, diff --git a/transformers/configuration_albert.py b/transformers/configuration_albert.py index b72bbb971e..c35426768f 100644 --- a/transformers/configuration_albert.py +++ b/transformers/configuration_albert.py @@ -17,12 +17,21 @@ from .configuration_utils import PretrainedConfig +ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = { + 'albert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-config.json", + 'albert-large': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-config.json", + 'albert-xlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-config.json", + 'albert-xxlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-config.json", +} + class AlbertConfig(PretrainedConfig): """Configuration for `AlbertModel`. The default settings match the configuration of model `albert_xxlarge`. """ + pretrained_config_archive_map = ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP + def __init__(self, vocab_size_or_config_json_file=30000, embedding_size=128, diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 487455e561..4da10ed1cb 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -21,6 +21,7 @@ import logging import torch import torch.nn as nn from torch.nn import CrossEntropyLoss +from transformers.modeling_utils import PreTrainedModel from transformers.configuration_albert import AlbertConfig from transformers.modeling_bert import BertEmbeddings, BertPreTrainedModel, BertModel, BertSelfAttention, prune_linear_layer, ACT2FN from .file_utils import add_start_docstrings @@ -274,6 +275,29 @@ class AlbertTransformer(nn.Module): return outputs # last-layer hidden state, (all hidden states), (all attentions) + +class AlbertPreTrainedModel(PreTrainedModel): + """ An abstract class to handle weights initialization and + a simple interface for dowloading and loading pretrained models. + """ + config_class = AlbertConfig + pretrained_model_archive_map = ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP + base_model_prefix = "albert" + + def _init_weights(self, module): + """ Initialize the weights. + """ + if isinstance(module, (nn.Linear, nn.Embedding)): + # Slightly different from the TF version which uses truncated_normal for initialization + # cf https://github.com/pytorch/pytorch/pull/5617 + module.weight.data.normal_(mean=0.0, std=self.config.initializer_range) + if isinstance(module, (nn.Linear)) and module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + + ALBERT_START_DOCSTRING = r""" The ALBERT model was proposed in `ALBERT: A Lite BERT for Self-supervised Learning of Language Representations`_ by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. It presents @@ -338,7 +362,7 @@ ALBERT_INPUTS_DOCSTRING = r""" @add_start_docstrings("The bare ALBERT Model transformer outputting raw hidden-states without any specific head on top.", ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) -class AlbertModel(BertModel): +class AlbertModel(AlbertPreTrainedModel): r""" Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` @@ -358,6 +382,12 @@ class AlbertModel(BertModel): list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. """ + + config_class = AlbertConfig + pretrained_model_archive_map = ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP + load_tf_weights = load_tf_weights_in_albert + base_model_prefix = "albert" + def __init__(self, config): super(AlbertModel, self).__init__(config) @@ -369,6 +399,11 @@ class AlbertModel(BertModel): self.init_weights() + def _resize_token_embeddings(self, new_num_tokens): + old_embeddings = self.embeddings.word_embeddings + new_embeddings = self._get_resized_embeddings(old_embeddings, new_num_tokens) + self.embeddings.word_embeddings = new_embeddings + return self.embeddings.word_embeddings def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None): if attention_mask is None: @@ -423,7 +458,7 @@ class AlbertMLMHead(nn.Module): @add_start_docstrings("Bert Model with a `language modeling` head on top.", ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) -class AlbertForMaskedLM(BertPreTrainedModel): +class AlbertForMaskedLM(AlbertPreTrainedModel): r""" **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: Labels for computing the masked language modeling loss. @@ -445,11 +480,6 @@ class AlbertForMaskedLM(BertPreTrainedModel): Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. """ - config_class = AlbertConfig - pretrained_model_archive_map = ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP - load_tf_weights = load_tf_weights_in_albert - base_model_prefix = "albert" - def __init__(self, config): super(AlbertForMaskedLM, self).__init__(config) diff --git a/transformers/tokenization_albert.py b/transformers/tokenization_albert.py index 7cba99b9e4..acf67c1154 100644 --- a/transformers/tokenization_albert.py +++ b/transformers/tokenization_albert.py @@ -15,7 +15,7 @@ """ Tokenization classes for ALBERT model.""" from __future__ import (absolute_import, division, print_function, unicode_literals) - + from .tokenization_utils import PreTrainedTokenizer import logging import unicodedata @@ -24,8 +24,25 @@ import os from shutil import copyfile logger = logging.getLogger(__name__) - VOCAB_FILES_NAMES = {'vocab_file': 'spiece.model'} + +PRETRAINED_VOCAB_FILES_MAP = { + 'vocab_file': + { + 'albert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-spiece.model", + 'albert-large': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-spiece.model", + 'albert-xlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-spiece.model", + 'albert-xxlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-spiece.model", + } +} + +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { + 'albert-base': 512, + 'albert-large': 512, + 'albert-xlarge': 512, + 'albert-xxlarge': 512, +} + SPIECE_UNDERLINE = u'▁' class AlbertTokenizer(PreTrainedTokenizer): @@ -35,8 +52,8 @@ class AlbertTokenizer(PreTrainedTokenizer): - requires `SentencePiece `_ """ vocab_files_names = VOCAB_FILES_NAMES - # pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP - # max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES def __init__(self, vocab_file, do_lower_case=True, remove_space=True, keep_accents=False, From c9875455929a63f12b81e2dcc8fe30f72137c06b Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 31 Oct 2019 18:48:02 +0000 Subject: [PATCH 103/505] Converting script --- transformers/__init__.py | 2 +- ...lbert_original_tf_checkpoint_to_pytorch.py | 36 +++++++++++++++---- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/transformers/__init__.py b/transformers/__init__.py index bdfb1a0922..db98d5fd44 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -107,7 +107,7 @@ if is_torch_available(): CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_encoder_decoder import PreTrainedEncoderDecoder, Model2Model - from .modeling_albert import (AlbertModel, AlbertForMaskedLM, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_albert import (AlbertModel, AlbertForMaskedLM, load_tf_weights_in_albert, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) # Optimization from .optimization import (AdamW, get_constant_schedule, get_constant_schedule_with_warmup, get_cosine_schedule_with_warmup, diff --git a/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py b/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py index 04877d41b9..5bbaab8c21 100644 --- a/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py +++ b/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py @@ -1,18 +1,39 @@ +# coding=utf-8 +# Copyright 2018 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Convert ALBERT checkpoint.""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +import argparse +import torch -from transformers import AlbertConfig, BertForPreTraining, load_tf_weights_in_bert - +from transformers import AlbertConfig, AlbertForMaskedLM, load_tf_weights_in_albert +import logging +logging.basicConfig(level=logging.INFO) def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, bert_config_file, pytorch_dump_path): # Initialise PyTorch model - config = BertConfig.from_json_file(bert_config_file) + config = AlbertConfig.from_json_file(bert_config_file) print("Building PyTorch model from configuration: {}".format(str(config))) - model = BertForPreTraining(config) + model = AlbertForMaskedLM(config) # Load weights from tf checkpoint - load_tf_weights_in_bert(model, config, tf_checkpoint_path) + load_tf_weights_in_albert(model, config, tf_checkpoint_path) # Save pytorch-model print("Save PyTorch model to {}".format(pytorch_dump_path)) @@ -31,7 +52,7 @@ if __name__ == "__main__": default = None, type = str, required = True, - help = "The config json file corresponding to the pre-trained BERT model. \n" + help = "The config json file corresponding to the pre-trained ALBERT model. \n" "This specifies the model architecture.") parser.add_argument("--pytorch_dump_path", default = None, @@ -40,5 +61,6 @@ if __name__ == "__main__": help = "Path to the output PyTorch model.") args = parser.parse_args() convert_tf_checkpoint_to_pytorch(args.tf_checkpoint_path, - args.bert_config_file, + args.albert_config_file, args.pytorch_dump_path) + \ No newline at end of file From 0d07a23c04c9837234cda402b32246d7581e5bc4 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Fri, 1 Nov 2019 15:07:01 +0000 Subject: [PATCH 104/505] LAMB implementation --- transformers/optimization.py | 93 ++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/transformers/optimization.py b/transformers/optimization.py index 99e6cc75e4..90f3dbca9b 100644 --- a/transformers/optimization.py +++ b/transformers/optimization.py @@ -167,3 +167,96 @@ class AdamW(Optimizer): p.data.add_(-group['lr'] * group['weight_decay'], p.data) return loss + + + +class Lamb(Optimizer): + """ Implements the LAMB algorithm (Layer-wise Adaptive Moments optimizer for Batch training). + + Adapted from the huggingface/transformers ADAM optimizer + Inspired from the Google Research implementation available in ALBERT: https://github.com/google-research/google-research/blob/master/albert/lamb_optimizer.py + Inspired from cybertronai's PyTorch LAMB implementation: https://github.com/cybertronai/pytorch-lamb/blob/master/pytorch_lamb/lamb.py + + + Parameters: + lr (float): learning rate. Default 1e-3. + betas (tuple of 2 floats): Adams beta parameters (b1, b2). Default: (0.9, 0.999) + eps (float): Adams epsilon. Default: 1e-6 + weight_decay (float): Weight decay. Default: 0.0 + """ + + def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-6, weight_decay=0.0, correct_bias=True): + if lr < 0.0: + raise ValueError("Invalid learning rate: {} - should be >= 0.0".format(lr)) + if not 0.0 <= betas[0] < 1.0: + raise ValueError("Invalid beta parameter: {} - should be in [0.0, 1.0[".format(betas[0])) + if not 0.0 <= betas[1] < 1.0: + raise ValueError("Invalid beta parameter: {} - should be in [0.0, 1.0[".format(betas[1])) + if not 0.0 <= eps: + raise ValueError("Invalid epsilon value: {} - should be >= 0.0".format(eps)) + defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay, + correct_bias=correct_bias) + super(Lamb, self).__init__(params, defaults) + + def step(self, closure=None): + """Performs a single optimization step. + + Arguments: + closure (callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = None + if closure is not None: + loss = closure() + + for group in self.param_groups: + for p in group['params']: + if p.grad is None: + continue + grad = p.grad.data + if grad.is_sparse: + raise RuntimeError('LAMB does not support sparse gradients.') + + state = self.state[p] + + # State initialization + if len(state) == 0: + state['step'] = 0 + # Exponential moving average of gradient values + state['exp_avg'] = torch.zeros_like(p.data) + # Exponential moving average of squared gradient values + state['exp_avg_sq'] = torch.zeros_like(p.data) + + exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] + beta1, beta2 = group['betas'] + + state['step'] += 1 + + # Decay the first and second moment running average coefficient + # In-place operations to update the averages at the same time + exp_avg.mul_(beta1).add_(1.0 - beta1, grad) + exp_avg_sq.mul_(beta2).addcmul_(1.0 - beta2, grad, grad) + denom = exp_avg_sq.sqrt().add_(group['eps']) + + + # Inspired from cybertronai's PyTorch LAMB implementation: https://github.com/cybertronai/pytorch-lamb/blob/master/pytorch_lamb/lamb.py + step_size = group['lr'] + weight_norm = p.data.pow(2).sum().sqrt().clamp(0, 10) + + adam_step = exp_avg / exp_avg_sq.sqrt().add(group['eps']) + if group['weight_decay'] != 0: + adam_step.add_(group['weight_decay'], p.data) + + adam_norm = adam_step.pow(2).sum().sqrt() + if weight_norm == 0 or adam_norm == 0: + trust_ratio = 1 + else: + trust_ratio = weight_norm / adam_norm + + + state['weight_norm'] = weight_norm + state['adam_norm'] = adam_norm + state['trust_ratio'] = trust_ratio + + p.data.add_(-step_size * trust_ratio, adam_step) + return loss From 6637a77f807615ef2427c0390a015f4eb4814fb4 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Fri, 1 Nov 2019 15:17:31 +0000 Subject: [PATCH 105/505] AlbertForSequenceClassification --- transformers/__init__.py | 3 +- transformers/modeling_albert.py | 77 ++++++++++++++++++++++++++++++++- 2 files changed, 77 insertions(+), 3 deletions(-) diff --git a/transformers/__init__.py b/transformers/__init__.py index db98d5fd44..51995942ce 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -107,7 +107,8 @@ if is_torch_available(): CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_encoder_decoder import PreTrainedEncoderDecoder, Model2Model - from .modeling_albert import (AlbertModel, AlbertForMaskedLM, load_tf_weights_in_albert, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_albert import (AlbertModel, AlbertForMaskedLM, AlbertForSequenceClassification, + load_tf_weights_in_albert, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) # Optimization from .optimization import (AdamW, get_constant_schedule, get_constant_schedule_with_warmup, get_cosine_schedule_with_warmup, diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 4da10ed1cb..bba6767079 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -20,10 +20,10 @@ import math import logging import torch import torch.nn as nn -from torch.nn import CrossEntropyLoss +from torch.nn import CrossEntropyLoss, MSELoss from transformers.modeling_utils import PreTrainedModel from transformers.configuration_albert import AlbertConfig -from transformers.modeling_bert import BertEmbeddings, BertPreTrainedModel, BertModel, BertSelfAttention, prune_linear_layer, ACT2FN +from transformers.modeling_bert import BertEmbeddings, BertSelfAttention, prune_linear_layer, ACT2FN from .file_utils import add_start_docstrings logger = logging.getLogger(__name__) @@ -510,3 +510,76 @@ class AlbertForMaskedLM(AlbertPreTrainedModel): outputs = (masked_lm_loss,) + outputs return outputs + + +@add_start_docstrings("""Albert Model transformer with a sequence classification/regression head on top (a linear layer on top of + the pooled output) e.g. for GLUE tasks. """, + ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) +class AlbertForSequenceClassification(AlbertPreTrainedModel): + r""" + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the sequence classification/regression loss. + Indices should be in ``[0, ..., config.num_labels - 1]``. + If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss), + If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy). + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification (or regression if config.num_labels==1) loss. + **logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)`` + Classification (or regression if config.num_labels==1) scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = AlbertTokenizer.from_pretrained('albert-base') + model = AlbertForSequenceClassification.from_pretrained('albert-base') + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, logits = outputs[:2] + + """ + def __init__(self, config): + super(AlbertForSequenceClassification, self).__init__(config) + self.num_labels = config.num_labels + + self.albert = AlbertModel(config) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + self.classifier = nn.Linear(config.hidden_size, self.config.num_labels) + + self.init_weights() + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, + position_ids=None, head_mask=None, labels=None): + + outputs = self.albert(input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask) + + pooled_output = outputs[1] + + pooled_output = self.dropout(pooled_output) + logits = self.classifier(pooled_output) + + outputs = (logits,) + outputs[2:] # add hidden states and attention if they are here + + if labels is not None: + if self.num_labels == 1: + # We are doing regression + loss_fct = MSELoss() + loss = loss_fct(logits.view(-1), labels.view(-1)) + else: + loss_fct = CrossEntropyLoss() + loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) + outputs = (loss,) + outputs + + return outputs # (loss), logits, (hidden_states), (attentions) \ No newline at end of file From c110c41fdb3b363528336ceda3b9fb46f026ad9c Mon Sep 17 00:00:00 2001 From: Lysandre Date: Fri, 1 Nov 2019 21:59:40 +0000 Subject: [PATCH 106/505] Run GLUE and remove LAMB --- examples/run_glue.py | 14 ++++-- transformers/optimization.py | 93 ------------------------------------ 2 files changed, 10 insertions(+), 97 deletions(-) diff --git a/examples/run_glue.py b/examples/run_glue.py index 527e440075..550a0b8175 100644 --- a/examples/run_glue.py +++ b/examples/run_glue.py @@ -47,7 +47,11 @@ from transformers import (WEIGHTS_NAME, BertConfig, XLNetTokenizer, DistilBertConfig, DistilBertForSequenceClassification, - DistilBertTokenizer) + DistilBertTokenizer, + AlbertConfig, + AlbertForSequenceClassification, + AlbertTokenizer, + ) from transformers import AdamW, get_linear_schedule_with_warmup @@ -66,7 +70,8 @@ MODEL_CLASSES = { 'xlnet': (XLNetConfig, XLNetForSequenceClassification, XLNetTokenizer), 'xlm': (XLMConfig, XLMForSequenceClassification, XLMTokenizer), 'roberta': (RobertaConfig, RobertaForSequenceClassification, RobertaTokenizer), - 'distilbert': (DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer) + 'distilbert': (DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer), + 'albert': (AlbertConfig, AlbertForSequenceClassification, AlbertTokenizer) } @@ -99,6 +104,7 @@ def train(args, train_dataset, model, tokenizer): {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': args.weight_decay}, {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} ] + optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total) if args.fp16: @@ -317,7 +323,7 @@ def load_and_cache_examples(args, task, tokenizer, evaluate=False): all_labels = torch.tensor([f.label for f in features], dtype=torch.long) elif output_mode == "regression": all_labels = torch.tensor([f.label for f in features], dtype=torch.float) - + dataset = TensorDataset(all_input_ids, all_attention_mask, all_token_type_ids, all_labels) return dataset @@ -361,7 +367,7 @@ def main(): parser.add_argument("--per_gpu_eval_batch_size", default=8, type=int, help="Batch size per GPU/CPU for evaluation.") parser.add_argument('--gradient_accumulation_steps', type=int, default=1, - help="Number of updates steps to accumulate before performing a backward/update pass.") + help="Number of updates steps to accumulate before performing a backward/update pass.") parser.add_argument("--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.") parser.add_argument("--weight_decay", default=0.0, type=float, diff --git a/transformers/optimization.py b/transformers/optimization.py index 90f3dbca9b..99e6cc75e4 100644 --- a/transformers/optimization.py +++ b/transformers/optimization.py @@ -167,96 +167,3 @@ class AdamW(Optimizer): p.data.add_(-group['lr'] * group['weight_decay'], p.data) return loss - - - -class Lamb(Optimizer): - """ Implements the LAMB algorithm (Layer-wise Adaptive Moments optimizer for Batch training). - - Adapted from the huggingface/transformers ADAM optimizer - Inspired from the Google Research implementation available in ALBERT: https://github.com/google-research/google-research/blob/master/albert/lamb_optimizer.py - Inspired from cybertronai's PyTorch LAMB implementation: https://github.com/cybertronai/pytorch-lamb/blob/master/pytorch_lamb/lamb.py - - - Parameters: - lr (float): learning rate. Default 1e-3. - betas (tuple of 2 floats): Adams beta parameters (b1, b2). Default: (0.9, 0.999) - eps (float): Adams epsilon. Default: 1e-6 - weight_decay (float): Weight decay. Default: 0.0 - """ - - def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-6, weight_decay=0.0, correct_bias=True): - if lr < 0.0: - raise ValueError("Invalid learning rate: {} - should be >= 0.0".format(lr)) - if not 0.0 <= betas[0] < 1.0: - raise ValueError("Invalid beta parameter: {} - should be in [0.0, 1.0[".format(betas[0])) - if not 0.0 <= betas[1] < 1.0: - raise ValueError("Invalid beta parameter: {} - should be in [0.0, 1.0[".format(betas[1])) - if not 0.0 <= eps: - raise ValueError("Invalid epsilon value: {} - should be >= 0.0".format(eps)) - defaults = dict(lr=lr, betas=betas, eps=eps, weight_decay=weight_decay, - correct_bias=correct_bias) - super(Lamb, self).__init__(params, defaults) - - def step(self, closure=None): - """Performs a single optimization step. - - Arguments: - closure (callable, optional): A closure that reevaluates the model - and returns the loss. - """ - loss = None - if closure is not None: - loss = closure() - - for group in self.param_groups: - for p in group['params']: - if p.grad is None: - continue - grad = p.grad.data - if grad.is_sparse: - raise RuntimeError('LAMB does not support sparse gradients.') - - state = self.state[p] - - # State initialization - if len(state) == 0: - state['step'] = 0 - # Exponential moving average of gradient values - state['exp_avg'] = torch.zeros_like(p.data) - # Exponential moving average of squared gradient values - state['exp_avg_sq'] = torch.zeros_like(p.data) - - exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq'] - beta1, beta2 = group['betas'] - - state['step'] += 1 - - # Decay the first and second moment running average coefficient - # In-place operations to update the averages at the same time - exp_avg.mul_(beta1).add_(1.0 - beta1, grad) - exp_avg_sq.mul_(beta2).addcmul_(1.0 - beta2, grad, grad) - denom = exp_avg_sq.sqrt().add_(group['eps']) - - - # Inspired from cybertronai's PyTorch LAMB implementation: https://github.com/cybertronai/pytorch-lamb/blob/master/pytorch_lamb/lamb.py - step_size = group['lr'] - weight_norm = p.data.pow(2).sum().sqrt().clamp(0, 10) - - adam_step = exp_avg / exp_avg_sq.sqrt().add(group['eps']) - if group['weight_decay'] != 0: - adam_step.add_(group['weight_decay'], p.data) - - adam_norm = adam_step.pow(2).sum().sqrt() - if weight_norm == 0 or adam_norm == 0: - trust_ratio = 1 - else: - trust_ratio = weight_norm / adam_norm - - - state['weight_norm'] = weight_norm - state['adam_norm'] = adam_norm - state['trust_ratio'] = trust_ratio - - p.data.add_(-step_size * trust_ratio, adam_step) - return loss From 70d99980ded1565c9e8efa2cd04c21572b664f2f Mon Sep 17 00:00:00 2001 From: Lysandre Date: Mon, 4 Nov 2019 11:34:30 -0500 Subject: [PATCH 107/505] ALBERT-V2 --- transformers/configuration_albert.py | 12 ++++++---- ...lbert_original_tf_checkpoint_to_pytorch.py | 5 ++-- transformers/modeling_albert.py | 16 ++++++++----- transformers/tokenization_albert.py | 24 ++++++++++++------- 4 files changed, 37 insertions(+), 20 deletions(-) diff --git a/transformers/configuration_albert.py b/transformers/configuration_albert.py index c35426768f..de665c9b1c 100644 --- a/transformers/configuration_albert.py +++ b/transformers/configuration_albert.py @@ -18,10 +18,14 @@ from .configuration_utils import PretrainedConfig ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = { - 'albert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-config.json", - 'albert-large': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-config.json", - 'albert-xlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-config.json", - 'albert-xxlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-config.json", + 'albert-base-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-config.json", + 'albert-large-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-config.json", + 'albert-xlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-config.json", + 'albert-xxlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-config.json", + 'albert-base-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v2-config.json", + 'albert-large-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-v2-config.json", + 'albert-xlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-v2-config.json", + 'albert-xxlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-v2-config.json", } class AlbertConfig(PretrainedConfig): diff --git a/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py b/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py index 5bbaab8c21..b6476b4fb6 100644 --- a/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py +++ b/transformers/convert_albert_original_tf_checkpoint_to_pytorch.py @@ -26,9 +26,10 @@ from transformers import AlbertConfig, AlbertForMaskedLM, load_tf_weights_in_alb import logging logging.basicConfig(level=logging.INFO) -def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, bert_config_file, pytorch_dump_path): + +def convert_tf_checkpoint_to_pytorch(tf_checkpoint_path, albert_config_file, pytorch_dump_path): # Initialise PyTorch model - config = AlbertConfig.from_json_file(bert_config_file) + config = AlbertConfig.from_json_file(albert_config_file) print("Building PyTorch model from configuration: {}".format(str(config))) model = AlbertForMaskedLM(config) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index bba6767079..51cb0a6d23 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -30,10 +30,14 @@ logger = logging.getLogger(__name__) ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP = { - 'albert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-pytorch_model.bin", - 'albert-large': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-pytorch_model.bin", - 'albert-xlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-pytorch_model.bin", - 'albert-xxlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-pytorch_model.bin", + 'albert-base-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-pytorch_model.bin", + 'albert-large-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-pytorch_model.bin", + 'albert-xlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-pytorch_model.bin", + 'albert-xxlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-pytorch_model.bin", + 'albert-base-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v2-pytorch_model.bin", + 'albert-large-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-v2-pytorch_model.bin", + 'albert-xlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-v2-pytorch_model.bin", + 'albert-xxlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-v2-pytorch_model.bin", } @@ -538,8 +542,8 @@ class AlbertForSequenceClassification(AlbertPreTrainedModel): Examples:: - tokenizer = AlbertTokenizer.from_pretrained('albert-base') - model = AlbertForSequenceClassification.from_pretrained('albert-base') + tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2') + model = AlbertForSequenceClassification.from_pretrained('albert-base-v2') input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 outputs = model(input_ids, labels=labels) diff --git a/transformers/tokenization_albert.py b/transformers/tokenization_albert.py index acf67c1154..2f9af0b0bc 100644 --- a/transformers/tokenization_albert.py +++ b/transformers/tokenization_albert.py @@ -29,18 +29,26 @@ VOCAB_FILES_NAMES = {'vocab_file': 'spiece.model'} PRETRAINED_VOCAB_FILES_MAP = { 'vocab_file': { - 'albert-base': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-spiece.model", - 'albert-large': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-spiece.model", - 'albert-xlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-spiece.model", - 'albert-xxlarge': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-spiece.model", + 'albert-base-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-spiece.model", + 'albert-large-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-spiece.model", + 'albert-xlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-spiece.model", + 'albert-xxlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-spiece.model", + 'albert-base-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v2-spiece.model", + 'albert-large-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-v2-spiece.model", + 'albert-xlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-v2-spiece.model", + 'albert-xxlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-v2-spiece.model", } } PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { - 'albert-base': 512, - 'albert-large': 512, - 'albert-xlarge': 512, - 'albert-xxlarge': 512, + 'albert-base-v1': 512, + 'albert-large-v1': 512, + 'albert-xlarge-v1': 512, + 'albert-xxlarge-v1': 512, + 'albert-base-v2': 512, + 'albert-large-v2': 512, + 'albert-xlarge-v2': 512, + 'albert-xxlarge-v2': 512, } SPIECE_UNDERLINE = u'▁' From 4374eaea786399a948b4b34d39fc614ded5e1de6 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 6 Nov 2019 20:47:42 +0000 Subject: [PATCH 108/505] ALBERT for SQuAD --- examples/run_squad.py | 12 +++-- transformers/__init__.py | 1 + transformers/modeling_albert.py | 93 ++++++++++++++++++++++++++++++++- 3 files changed, 100 insertions(+), 6 deletions(-) diff --git a/examples/run_squad.py b/examples/run_squad.py index 69088d73c3..59683c0668 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -43,7 +43,8 @@ from transformers import (WEIGHTS_NAME, BertConfig, XLMTokenizer, XLNetConfig, XLNetForQuestionAnswering, XLNetTokenizer, - DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer) + DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer, + AlbertConfig, AlbertForQuestionAnswering, AlbertTokenizer) from transformers import AdamW, get_linear_schedule_with_warmup @@ -65,7 +66,8 @@ MODEL_CLASSES = { 'bert': (BertConfig, BertForQuestionAnswering, BertTokenizer), 'xlnet': (XLNetConfig, XLNetForQuestionAnswering, XLNetTokenizer), 'xlm': (XLMConfig, XLMForQuestionAnswering, XLMTokenizer), - 'distilbert': (DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer) + 'distilbert': (DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer), + 'albert': (AlbertConfig, AlbertForQuestionAnswering, AlbertTokenizer) } def set_seed(args): @@ -128,7 +130,7 @@ def train(args, train_dataset, model, tokenizer): logger.info(" Gradient Accumulation steps = %d", args.gradient_accumulation_steps) logger.info(" Total optimization steps = %d", t_total) - global_step = 0 + global_step = 1 tr_loss, logging_loss = 0.0, 0.0 model.zero_grad() train_iterator = trange(int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0]) @@ -537,7 +539,7 @@ def main(): torch.save(args, os.path.join(args.output_dir, 'training_args.bin')) # Load a trained model and vocabulary that you have fine-tuned - model = model_class.from_pretrained(args.output_dir) + model = model_class.from_pretrained(args.output_dir, force_download=True) tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case) model.to(args.device) @@ -555,7 +557,7 @@ def main(): for checkpoint in checkpoints: # Reload the model global_step = checkpoint.split('-')[-1] if len(checkpoints) > 1 else "" - model = model_class.from_pretrained(checkpoint) + model = model_class.from_pretrained(checkpoint, force_download=True) model.to(args.device) # Evaluate diff --git a/transformers/__init__.py b/transformers/__init__.py index 51995942ce..81e659329d 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -108,6 +108,7 @@ if is_torch_available(): from .modeling_encoder_decoder import PreTrainedEncoderDecoder, Model2Model from .modeling_albert import (AlbertModel, AlbertForMaskedLM, AlbertForSequenceClassification, + AlbertForQuestionAnswering, load_tf_weights_in_albert, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) # Optimization diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 51cb0a6d23..2540218e69 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -586,4 +586,95 @@ class AlbertForSequenceClassification(AlbertPreTrainedModel): loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1)) outputs = (loss,) + outputs - return outputs # (loss), logits, (hidden_states), (attentions) \ No newline at end of file + return outputs # (loss), logits, (hidden_states), (attentions) + + + +@add_start_docstrings("""Albert Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of + the hidden-states output to compute `span start logits` and `span end logits`). """, + ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) +class AlbertForQuestionAnswering(AlbertPreTrainedModel): + r""" + **start_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for position (index) of the start of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (`sequence_length`). + Position outside of the sequence are not taken into account for computing the loss. + **end_positions**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for position (index) of the end of the labelled span for computing the token classification loss. + Positions are clamped to the length of the sequence (`sequence_length`). + Position outside of the sequence are not taken into account for computing the loss. + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Total span extraction loss is the sum of a Cross-Entropy for the start and end positions. + **start_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` + Span-start scores (before SoftMax). + **end_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length,)`` + Span-end scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2') + model = AlbertForQuestionAnswering.from_pretrained('albert-base-v2') + question, text = "Who was Jim Henson?", "Jim Henson was a nice puppet" + input_text = "[CLS] " + question + " [SEP] " + text + " [SEP]" + input_ids = tokenizer.encode(input_text) + token_type_ids = [0 if i <= input_ids.index(102) else 1 for i in range(len(input_ids))] + start_scores, end_scores = model(torch.tensor([input_ids]), token_type_ids=torch.tensor([token_type_ids])) + all_tokens = tokenizer.convert_ids_to_tokens(input_ids) + print(' '.join(all_tokens[torch.argmax(start_scores) : torch.argmax(end_scores)+1])) + # a nice puppet + + + """ + def __init__(self, config): + super(AlbertForQuestionAnswering, self).__init__(config) + self.num_labels = config.num_labels + + self.albert = AlbertModel(config) + self.qa_outputs = nn.Linear(config.hidden_size, config.num_labels) + + self.init_weights() + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + start_positions=None, end_positions=None): + + outputs = self.albert(input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask) + + sequence_output = outputs[0] + + logits = self.qa_outputs(sequence_output) + start_logits, end_logits = logits.split(1, dim=-1) + start_logits = start_logits.squeeze(-1) + end_logits = end_logits.squeeze(-1) + + outputs = (start_logits, end_logits,) + outputs[2:] + if start_positions is not None and end_positions is not None: + # If we are on multi-GPU, split add a dimension + if len(start_positions.size()) > 1: + start_positions = start_positions.squeeze(-1) + if len(end_positions.size()) > 1: + end_positions = end_positions.squeeze(-1) + # sometimes the start/end positions are outside our model inputs, we ignore these terms + ignored_index = start_logits.size(1) + start_positions.clamp_(0, ignored_index) + end_positions.clamp_(0, ignored_index) + + loss_fct = CrossEntropyLoss(ignore_index=ignored_index) + start_loss = loss_fct(start_logits, start_positions) + end_loss = loss_fct(end_logits, end_positions) + total_loss = (start_loss + end_loss) / 2 + outputs = (total_loss,) + outputs + + return outputs # (loss), start_logits, end_logits, (hidden_states), (attentions) From abb23a78bab17ec09dde4635c32f4aa21c15fa83 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 7 Nov 2019 17:09:16 +0000 Subject: [PATCH 109/505] Head pruning for ALBERT --- transformers/modeling_albert.py | 42 ++++++++++++++++++++++ transformers/tests/modeling_albert_test.py | 12 ++++--- 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 2540218e69..89ece4b61e 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -145,6 +145,29 @@ class AlbertAttention(BertSelfAttention): self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) self.pruned_heads = set() + def prune_heads(self, heads): + if len(heads) == 0: + return + mask = torch.ones(self.num_attention_heads, self.attention_head_size) + heads = set(heads) - self.pruned_heads # Convert to set and emove already pruned heads + for head in heads: + # Compute how many pruned heads are before the head and move the index accordingly + head = head - sum(1 if h < head else 0 for h in self.pruned_heads) + mask[head] = 0 + mask = mask.view(-1).contiguous().eq(1) + index = torch.arange(len(mask))[mask].long() + + # Prune linear layers + self.query = prune_linear_layer(self.query, index) + self.key = prune_linear_layer(self.key, index) + self.value = prune_linear_layer(self.value, index) + self.dense = prune_linear_layer(self.dense, index, dim=1) + + # Update hyper params and store pruned heads + self.num_attention_heads = self.num_attention_heads - len(heads) + self.all_head_size = self.attention_head_size * self.num_attention_heads + self.pruned_heads = self.pruned_heads.union(heads) + def forward(self, input_ids, attention_mask=None, head_mask=None): mixed_query_layer = self.query(input_ids) mixed_key_layer = self.key(input_ids) @@ -409,6 +432,25 @@ class AlbertModel(AlbertPreTrainedModel): self.embeddings.word_embeddings = new_embeddings return self.embeddings.word_embeddings + def _prune_heads(self, heads_to_prune): + """ Prunes heads of the model. + heads_to_prune: dict of {layer_num: list of heads to prune in this layer} + ALBERT has a different architecture in that its layers are shared across groups, which then has inner groups. + If an ALBERT model has 12 hidden layers and 2 hidden groups, with two inner groups, there + is a total of 4 different layers. + + These layers are flattened: the indices [0,1] correspond to the two inner groups of the first hidden layer, + while [2,3] correspond to the two inner groups of the second hidden layer. + + Any layer with in index other than [0,1,2,3] will result in an error. + See base class PreTrainedModel for more information about head pruning + """ + for layer, heads in heads_to_prune.items(): + group_idx = int(layer / self.config.inner_group_num) + inner_group_idx = int(layer - group_idx * self.config.inner_group_num) + self.encoder.albert_layer_groups[group_idx].albert_layers[inner_group_idx].attention.prune_heads(heads) + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None): if attention_mask is None: attention_mask = torch.ones_like(input_ids) diff --git a/transformers/tests/modeling_albert_test.py b/transformers/tests/modeling_albert_test.py index 46a2eeb729..979e0488eb 100644 --- a/transformers/tests/modeling_albert_test.py +++ b/transformers/tests/modeling_albert_test.py @@ -35,7 +35,6 @@ else: class AlbertModelTest(CommonTestCases.CommonModelTester): all_model_classes = (AlbertModel, AlbertForMaskedLM) if is_torch_available() else () - test_pruning = False test_head_masking = False class AlbertModelTester(object): @@ -49,9 +48,10 @@ class AlbertModelTest(CommonTestCases.CommonModelTester): use_token_type_ids=True, use_labels=True, vocab_size=99, - hidden_size=32, - num_hidden_layers=5, - num_attention_heads=4, + hidden_size=36, + num_hidden_layers=6, + num_hidden_groups=6, + num_attention_heads=6, intermediate_size=37, hidden_act="gelu", hidden_dropout_prob=0.1, @@ -86,6 +86,7 @@ class AlbertModelTest(CommonTestCases.CommonModelTester): self.num_labels = num_labels self.num_choices = num_choices self.scope = scope + self.num_hidden_groups = num_hidden_groups def prepare_config_and_inputs(self): input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) @@ -117,7 +118,8 @@ class AlbertModelTest(CommonTestCases.CommonModelTester): attention_probs_dropout_prob=self.attention_probs_dropout_prob, max_position_embeddings=self.max_position_embeddings, type_vocab_size=self.type_vocab_size, - initializer_range=self.initializer_range) + initializer_range=self.initializer_range, + num_hidden_groups=self.num_hidden_groups) return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels From 16263f9685eaf459408f33c9790a967012b93fa5 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 7 Nov 2019 17:29:29 +0000 Subject: [PATCH 110/505] Headmasking --- transformers/modeling_albert.py | 11 ++++++----- transformers/tests/modeling_albert_test.py | 1 - 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 89ece4b61e..6682930d89 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -224,7 +224,7 @@ class AlbertLayer(nn.Module): self.activation = ACT2FN[config.hidden_act] def forward(self, hidden_states, attention_mask=None, head_mask=None): - attention_output = self.attention(hidden_states, attention_mask) + attention_output = self.attention(hidden_states, attention_mask, head_mask) ffn_output = self.ffn(attention_output[0]) ffn_output = self.activation(ffn_output) ffn_output = self.ffn_output(ffn_output) @@ -245,8 +245,8 @@ class AlbertLayerGroup(nn.Module): layer_hidden_states = () layer_attentions = () - for albert_layer in self.albert_layers: - layer_output = albert_layer(hidden_states, attention_mask, head_mask) + for layer_index, albert_layer in enumerate(self.albert_layers): + layer_output = albert_layer(hidden_states, attention_mask, head_mask[layer_index]) hidden_states = layer_output[0] if self.output_attentions: @@ -283,7 +283,8 @@ class AlbertTransformer(nn.Module): for layer_idx in range(self.config.num_hidden_layers): group_idx = int(layer_idx / self.config.num_hidden_layers * self.config.num_hidden_groups) - layer_group_output = self.albert_layer_groups[group_idx](hidden_states, attention_mask, head_mask) + layers_per_group = int(self.config.num_hidden_layers / self.config.num_hidden_groups) + layer_group_output = self.albert_layer_groups[group_idx](hidden_states, attention_mask, head_mask[group_idx*layers_per_group:(group_idx+1)*layers_per_group]) hidden_states = layer_group_output[0] @@ -544,7 +545,7 @@ class AlbertForMaskedLM(AlbertPreTrainedModel): def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, masked_lm_labels=None): - outputs = self.albert(input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None) + outputs = self.albert(input_ids, attention_mask, token_type_ids, position_ids, head_mask) sequence_outputs = outputs[0] prediction_scores = self.predictions(sequence_outputs) diff --git a/transformers/tests/modeling_albert_test.py b/transformers/tests/modeling_albert_test.py index 979e0488eb..466f473332 100644 --- a/transformers/tests/modeling_albert_test.py +++ b/transformers/tests/modeling_albert_test.py @@ -35,7 +35,6 @@ else: class AlbertModelTest(CommonTestCases.CommonModelTester): all_model_classes = (AlbertModel, AlbertForMaskedLM) if is_torch_available() else () - test_head_masking = False class AlbertModelTester(object): From 9d5c49546fedb3d52f76f97f4043a07e08ded918 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 7 Nov 2019 17:32:52 +0000 Subject: [PATCH 111/505] Tests for AlbertForQuestionAnswering AlbertForSequenceClassification --- transformers/tests/modeling_albert_test.py | 45 +++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/transformers/tests/modeling_albert_test.py b/transformers/tests/modeling_albert_test.py index 466f473332..da87709df1 100644 --- a/transformers/tests/modeling_albert_test.py +++ b/transformers/tests/modeling_albert_test.py @@ -26,7 +26,9 @@ from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester if is_torch_available(): - from transformers import (AlbertConfig, AlbertModel, AlbertForMaskedLM) + from transformers import (AlbertConfig, AlbertModel, AlbertForMaskedLM, + AlbertForSequenceClassification, AlbertForQuestionAnswering, + ) from transformers.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP else: pytestmark = pytest.mark.skip("Require Torch") @@ -157,6 +159,39 @@ class AlbertModelTest(CommonTestCases.CommonModelTester): [self.batch_size, self.seq_length, self.vocab_size]) self.check_loss_output(result) + def create_and_check_albert_for_question_answering(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + model = AlbertForQuestionAnswering(config=config) + model.eval() + loss, start_logits, end_logits = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, + start_positions=sequence_labels, end_positions=sequence_labels) + result = { + "loss": loss, + "start_logits": start_logits, + "end_logits": end_logits, + } + self.parent.assertListEqual( + list(result["start_logits"].size()), + [self.batch_size, self.seq_length]) + self.parent.assertListEqual( + list(result["end_logits"].size()), + [self.batch_size, self.seq_length]) + self.check_loss_output(result) + + + def create_and_check_albert_for_sequence_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + config.num_labels = self.num_labels + model = AlbertForSequenceClassification(config) + model.eval() + loss, logits = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=sequence_labels) + result = { + "loss": loss, + "logits": logits, + } + self.parent.assertListEqual( + list(result["logits"].size()), + [self.batch_size, self.num_labels]) + self.check_loss_output(result) + def prepare_config_and_inputs_for_common(self): config_and_inputs = self.prepare_config_and_inputs() @@ -180,6 +215,14 @@ class AlbertModelTest(CommonTestCases.CommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_albert_for_masked_lm(*config_and_inputs) + def test_for_question_answering(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_albert_for_question_answering(*config_and_inputs) + + def test_for_sequence_classification(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_albert_for_sequence_classification(*config_and_inputs) + @pytest.mark.slow def test_model_from_pretrained(self): cache_dir = "/tmp/transformers_test/" From d9daad98c744e115bfb425f316a5e7d4f405a9a5 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 7 Nov 2019 19:55:43 +0000 Subject: [PATCH 112/505] Re-ordering of group_idx/layer_idx + Python 2 tests --- transformers/modeling_albert.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 6682930d89..640af537d0 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -281,11 +281,17 @@ class AlbertTransformer(nn.Module): if self.output_hidden_states: all_hidden_states = (hidden_states,) - for layer_idx in range(self.config.num_hidden_layers): - group_idx = int(layer_idx / self.config.num_hidden_layers * self.config.num_hidden_groups) + for i in range(self.config.num_hidden_layers): + # Number of layers in a hidden group layers_per_group = int(self.config.num_hidden_layers / self.config.num_hidden_groups) + + # Index of the hidden group + group_idx = int(i / (self.config.num_hidden_layers / self.config.num_hidden_groups)) + + # Index of the layer inside the group + layer_idx = int(i - group_idx * layers_per_group) + layer_group_output = self.albert_layer_groups[group_idx](hidden_states, attention_mask, head_mask[group_idx*layers_per_group:(group_idx+1)*layers_per_group]) - hidden_states = layer_group_output[0] if self.output_attentions: From f6f382532bf40a5869dc14e3eefa451646c19ded Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 7 Nov 2019 23:40:45 +0000 Subject: [PATCH 113/505] ALBERT in TF2 --- transformers/__init__.py | 6 +- .../convert_pytorch_checkpoint_to_tf2.py | 13 +- transformers/modeling_tf_albert.py | 723 ++++++++++++++++++ 3 files changed, 736 insertions(+), 6 deletions(-) create mode 100644 transformers/modeling_tf_albert.py diff --git a/transformers/__init__.py b/transformers/__init__.py index 81e659329d..a409ef772e 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -58,8 +58,7 @@ from .configuration_ctrl import CTRLConfig, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_xlm import XLMConfig, XLM_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_roberta import RobertaConfig, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_distilbert import DistilBertConfig, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP -from .configuration_albert import AlbertConfig, ALBERT -from .configuration_albert import AlbertConfig +from .configuration_albert import AlbertConfig, ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_camembert import CamembertConfig, CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP # Modeling @@ -169,6 +168,9 @@ if is_tf_available(): TFCTRLLMHeadModel, TF_CTRL_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_tf_albert import (TFAlbertPreTrainedModel, TFAlbertModel, TFAlbertForMaskedLM, + TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) + # TF 2.0 <=> PyTorch conversion utilities from .modeling_tf_pytorch_utils import (convert_tf_weight_name_to_pt_weight_name, load_pytorch_checkpoint_in_tf2_model, diff --git a/transformers/convert_pytorch_checkpoint_to_tf2.py b/transformers/convert_pytorch_checkpoint_to_tf2.py index e673b77dcc..d1776e9c14 100644 --- a/transformers/convert_pytorch_checkpoint_to_tf2.py +++ b/transformers/convert_pytorch_checkpoint_to_tf2.py @@ -33,7 +33,8 @@ from transformers import (load_pytorch_checkpoint_in_tf2_model, OpenAIGPTConfig, TFOpenAIGPTLMHeadModel, OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, RobertaConfig, TFRobertaForMaskedLM, TFRobertaForSequenceClassification, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, DistilBertConfig, TFDistilBertForMaskedLM, TFDistilBertForQuestionAnswering, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, - CTRLConfig, TFCTRLLMHeadModel, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP) + CTRLConfig, TFCTRLLMHeadModel, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP, + AlbertConfig, TFAlbertForMaskedLM, ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP) if is_torch_available(): import torch @@ -46,7 +47,8 @@ if is_torch_available(): OpenAIGPTLMHeadModel, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP, RobertaForMaskedLM, RobertaForSequenceClassification, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, DistilBertForMaskedLM, DistilBertForQuestionAnswering, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, - CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP) + CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP, + AlbertForMaskedLM, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) else: (BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, GPT2LMHeadModel, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP, @@ -56,7 +58,8 @@ else: OpenAIGPTLMHeadModel, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP, RobertaForMaskedLM, RobertaForSequenceClassification, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, DistilBertForMaskedLM, DistilBertForQuestionAnswering, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, - CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP) = ( + CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP, + AlbertForMaskedLM, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) = ( None, None, None, None, None, None, None, None, @@ -65,6 +68,7 @@ else: None, None, None, None, None, None, None, None, + None, None, None, None) @@ -85,7 +89,8 @@ MODEL_CLASSES = { 'roberta-large-mnli': (RobertaConfig, TFRobertaForSequenceClassification, RobertaForSequenceClassification, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP), 'distilbert': (DistilBertConfig, TFDistilBertForMaskedLM, DistilBertForMaskedLM, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP), 'distilbert-base-uncased-distilled-squad': (DistilBertConfig, TFDistilBertForQuestionAnswering, DistilBertForQuestionAnswering, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP), - 'ctrl': (CTRLConfig, TFCTRLLMHeadModel, CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP) + 'ctrl': (CTRLConfig, TFCTRLLMHeadModel, CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP), + 'albert': (AlbertConfig, TFAlbertForMaskedLM, AlbertForMaskedLM, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP, ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP) } def convert_pt_checkpoint_to_tf(model_type, pytorch_checkpoint_path, config_file, tf_dump_path, compare_with_pt_model=False, use_cached_models=True): diff --git a/transformers/modeling_tf_albert.py b/transformers/modeling_tf_albert.py new file mode 100644 index 0000000000..8861b7add8 --- /dev/null +++ b/transformers/modeling_tf_albert.py @@ -0,0 +1,723 @@ +# coding=utf-8 +# Copyright 2018 The OpenAI Team Authors and HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" TF 2.0 ALBERT model. """ +from __future__ import absolute_import, division, print_function, unicode_literals + +import json +import logging +import math +import os +import sys +from io import open + +import numpy as np +import tensorflow as tf + +from .configuration_albert import AlbertConfig +from .modeling_tf_utils import TFPreTrainedModel, get_initializer +from .modeling_tf_bert import ACT2FN, TFBertSelfAttention +from .file_utils import add_start_docstrings + +import logging + +logger = logging.getLogger(__name__) + +TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP = { + # TODO FILL THAT UP +} + + +class TFAlbertEmbeddings(tf.keras.layers.Layer): + """Construct the embeddings from word, position and token_type embeddings. + """ + + def __init__(self, config, **kwargs): + super(TFAlbertEmbeddings, self).__init__(**kwargs) + + self.config = config + self.position_embeddings = tf.keras.layers.Embedding(config.max_position_embeddings, + config.embedding_size, + embeddings_initializer=get_initializer( + self.config.initializer_range), + name='position_embeddings') + self.token_type_embeddings = tf.keras.layers.Embedding(config.type_vocab_size, + config.embedding_size, + embeddings_initializer=get_initializer( + self.config.initializer_range), + name='token_type_embeddings') + + # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load + # any TensorFlow checkpoint file + self.LayerNorm = tf.keras.layers.LayerNormalization( + epsilon=config.layer_norm_eps, name='LayerNorm') + self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + + def build(self, input_shape): + """Build shared word embedding layer """ + with tf.name_scope("word_embeddings"): + # Create and initialize weights. The random normal initializer was chosen + # arbitrarily, and works well. + self.word_embeddings = self.add_weight( + "weight", + shape=[self.config.vocab_size, self.config.embedding_size], + initializer=get_initializer(self.config.initializer_range)) + super(TFAlbertEmbeddings, self).build(input_shape) + + def call(self, inputs, mode="embedding", training=False): + """Get token embeddings of inputs. + Args: + inputs: list of three int64 tensors with shape [batch_size, length]: (input_ids, position_ids, token_type_ids) + mode: string, a valid value is one of "embedding" and "linear". + Returns: + outputs: (1) If mode == "embedding", output embedding tensor, float32 with + shape [batch_size, length, embedding_size]; (2) mode == "linear", output + linear tensor, float32 with shape [batch_size, length, vocab_size]. + Raises: + ValueError: if mode is not valid. + + Shared weights logic adapted from + https://github.com/tensorflow/models/blob/a009f4fb9d2fc4949e32192a944688925ef78659/official/transformer/v2/embedding_layer.py#L24 + """ + if mode == "embedding": + return self._embedding(inputs, training=training) + elif mode == "linear": + return self._linear(inputs) + else: + raise ValueError("mode {} is not valid.".format(mode)) + + def _embedding(self, inputs, training=False): + """Applies embedding based on inputs tensor.""" + input_ids, position_ids, token_type_ids = inputs + + seq_length = tf.shape(input_ids)[1] + if position_ids is None: + position_ids = tf.range(seq_length, dtype=tf.int32)[tf.newaxis, :] + if token_type_ids is None: + token_type_ids = tf.fill(tf.shape(input_ids), 0) + + words_embeddings = tf.gather(self.word_embeddings, input_ids) + position_embeddings = self.position_embeddings(position_ids) + token_type_embeddings = self.token_type_embeddings(token_type_ids) + + embeddings = words_embeddings + position_embeddings + token_type_embeddings + embeddings = self.LayerNorm(embeddings) + embeddings = self.dropout(embeddings, training=training) + return embeddings + + def _linear(self, inputs): + """Computes logits by running inputs through a linear layer. + Args: + inputs: A float32 tensor with shape [batch_size, length, embedding_size] + Returns: + float32 tensor with shape [batch_size, length, vocab_size]. + """ + batch_size = tf.shape(inputs)[0] + length = tf.shape(inputs)[1] + + print(inputs.shape) + + x = tf.reshape(inputs, [-1, self.config.embedding_size]) + + print(x.shape, self.word_embeddings) + + logits = tf.matmul(x, self.word_embeddings, transpose_b=True) + + print([batch_size, length, self.config.vocab_size]) + return tf.reshape(logits, [batch_size, length, self.config.vocab_size]) + + +class TFAlbertSelfAttention(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFAlbertSelfAttention, self).__init__(**kwargs) + if config.hidden_size % config.num_attention_heads != 0: + raise ValueError( + "The hidden size (%d) is not a multiple of the number of attention " + "heads (%d)" % (config.hidden_size, config.num_attention_heads)) + self.output_attentions = config.output_attentions + + self.num_attention_heads = config.num_attention_heads + assert config.hidden_size % config.num_attention_heads == 0 + self.attention_head_size = int( + config.hidden_size / config.num_attention_heads) + self.all_head_size = self.num_attention_heads * self.attention_head_size + + self.query = tf.keras.layers.Dense(self.all_head_size, + kernel_initializer=get_initializer( + config.initializer_range), + name='query') + self.key = tf.keras.layers.Dense(self.all_head_size, + kernel_initializer=get_initializer( + config.initializer_range), + name='key') + self.value = tf.keras.layers.Dense(self.all_head_size, + kernel_initializer=get_initializer( + config.initializer_range), + name='value') + + self.dropout = tf.keras.layers.Dropout( + config.attention_probs_dropout_prob) + + def transpose_for_scores(self, x, batch_size): + x = tf.reshape( + x, (batch_size, -1, self.num_attention_heads, self.attention_head_size)) + return tf.transpose(x, perm=[0, 2, 1, 3]) + + def call(self, inputs, training=False): + hidden_states, attention_mask, head_mask = inputs + + batch_size = tf.shape(hidden_states)[0] + mixed_query_layer = self.query(hidden_states) + mixed_key_layer = self.key(hidden_states) + mixed_value_layer = self.value(hidden_states) + + query_layer = self.transpose_for_scores(mixed_query_layer, batch_size) + key_layer = self.transpose_for_scores(mixed_key_layer, batch_size) + value_layer = self.transpose_for_scores(mixed_value_layer, batch_size) + + # Take the dot product between "query" and "key" to get the raw attention scores. + # (batch size, num_heads, seq_len_q, seq_len_k) + attention_scores = tf.matmul(query_layer, key_layer, transpose_b=True) + # scale attention_scores + dk = tf.cast(tf.shape(key_layer)[-1], tf.float32) + attention_scores = attention_scores / tf.math.sqrt(dk) + + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in TFAlbertModel call() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = tf.nn.softmax(attention_scores, axis=-1) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs, training=training) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = tf.matmul(attention_probs, value_layer) + + context_layer = tf.transpose(context_layer, perm=[0, 2, 1, 3]) + context_layer = tf.reshape(context_layer, + (batch_size, -1, self.all_head_size)) # (batch_size, seq_len_q, all_head_size) + + outputs = (context_layer, attention_probs) if self.output_attentions else ( + context_layer,) + return outputs + + +class TFAlbertSelfOutput(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFAlbertSelfOutput, self).__init__(**kwargs) + self.dense = tf.keras.layers.Dense(config.hidden_size, + kernel_initializer=get_initializer( + config.initializer_range), + name='dense') + self.LayerNorm = tf.keras.layers.LayerNormalization( + epsilon=config.layer_norm_eps, name='LayerNorm') + self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + + def call(self, inputs, training=False): + hidden_states, input_tensor = inputs + + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states, training=training) + hidden_states = self.LayerNorm(hidden_states + input_tensor) + return hidden_states + + +class TFAlbertAttention(TFBertSelfAttention): + def __init__(self, config, **kwargs): + super(TFAlbertAttention, self).__init__(config, **kwargs) + + self.hidden_size = config.hidden_size + self.dense = tf.keras.layers.Dense(config.hidden_size, + kernel_initializer=get_initializer( + config.initializer_range), + name='dense') + self.LayerNorm = tf.keras.layers.LayerNormalization( + epsilon=config.layer_norm_eps, name='LayerNorm') + self.pruned_heads = set() + + def prune_heads(self, heads): + raise NotImplementedError + + def call(self, inputs, training=False): + input_tensor, attention_mask, head_mask = inputs + + batch_size = tf.shape(input_tensor)[0] + mixed_query_layer = self.query(input_tensor) + mixed_key_layer = self.key(input_tensor) + mixed_value_layer = self.value(input_tensor) + + query_layer = self.transpose_for_scores(mixed_query_layer, batch_size) + key_layer = self.transpose_for_scores(mixed_key_layer, batch_size) + value_layer = self.transpose_for_scores(mixed_value_layer, batch_size) + + # Take the dot product between "query" and "key" to get the raw attention scores. + # (batch size, num_heads, seq_len_q, seq_len_k) + attention_scores = tf.matmul(query_layer, key_layer, transpose_b=True) + # scale attention_scores + dk = tf.cast(tf.shape(key_layer)[-1], tf.float32) + attention_scores = attention_scores / tf.math.sqrt(dk) + + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in TFBertModel call() function) + attention_scores = attention_scores + attention_mask + + # Normalize the attention scores to probabilities. + attention_probs = tf.nn.softmax(attention_scores, axis=-1) + + # This is actually dropping out entire tokens to attend to, which might + # seem a bit unusual, but is taken from the original Transformer paper. + attention_probs = self.dropout(attention_probs, training=training) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = tf.matmul(attention_probs, value_layer) + + context_layer = tf.transpose(context_layer, perm=[0, 2, 1, 3]) + context_layer = tf.reshape(context_layer, + (batch_size, -1, self.all_head_size)) # (batch_size, seq_len_q, all_head_size) + + self_outputs = (context_layer, attention_probs) if self.output_attentions else ( + context_layer,) + + hidden_states = self_outputs[0] + + hidden_states = self.dense(hidden_states) + hidden_states = self.dropout(hidden_states, training=training) + attention_output = self.LayerNorm(hidden_states + input_tensor) + + # add attentions if we output them + outputs = (attention_output,) + self_outputs[1:] + return outputs + + +class TFAlbertLayer(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFAlbertLayer, self).__init__(**kwargs) + self.attention = TFAlbertAttention(config, name='attention') + + self.ffn = tf.keras.layers.Dense(config.intermediate_size, kernel_initializer=get_initializer( + config.initializer_range), name='ffn') + + if isinstance(config.hidden_act, str) or (sys.version_info[0] == 2 and isinstance(config.hidden_act, unicode)): + self.activation = ACT2FN[config.hidden_act] + else: + self.activation = config.hidden_act + + self.ffn_output = tf.keras.layers.Dense(config.hidden_size, kernel_initializer=get_initializer( + config.initializer_range), name='ffn_output') + self.full_layer_layer_norm = tf.keras.layers.LayerNormalization( + epsilon=config.layer_norm_eps, name='full_layer_layer_norm') + self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + + def call(self, inputs, training=False): + hidden_states, attention_mask, head_mask = inputs + + attention_outputs = self.attention( + [hidden_states, attention_mask, head_mask], training=training) + ffn_output = self.ffn(attention_outputs[0]) + ffn_output = self.activation(ffn_output) + ffn_output = self.ffn_output(ffn_output) + + hidden_states = self.dropout(hidden_states, training=training) + hidden_states = self.full_layer_layer_norm( + ffn_output + attention_outputs[0]) + + # add attentions if we output them + outputs = (hidden_states,) + attention_outputs[1:] + return outputs + + +class TFAlbertLayerGroup(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFAlbertLayerGroup, self).__init__(**kwargs) + + self.output_attentions = config.output_attentions + self.output_hidden_states = config.output_hidden_states + self.albert_layers = [TFAlbertLayer(config, name="albert_layers_._{}".format( + i)) for i in range(config.inner_group_num)] + + def call(self, inputs, training=False): + hidden_states, attention_mask, head_mask = inputs + + layer_hidden_states = () + layer_attentions = () + + for layer_index, albert_layer in enumerate(self.albert_layers): + layer_output = albert_layer( + [hidden_states, attention_mask, head_mask[layer_index]], training=training) + hidden_states = layer_output[0] + + if self.output_attentions: + layer_attentions = layer_attentions + (layer_output[1],) + + if self.output_hidden_states: + layer_hidden_states = layer_hidden_states + (hidden_states,) + + outputs = (hidden_states,) + if self.output_hidden_states: + outputs = outputs + (layer_hidden_states,) + if self.output_attentions: + outputs = outputs + (layer_attentions,) + # last-layer hidden state, (layer hidden states), (layer attentions) + return outputs + + +class TFAlbertTransformer(tf.keras.layers.Layer): + def __init__(self, config, **kwargs): + super(TFAlbertTransformer, self).__init__(**kwargs) + + self.config = config + self.output_attentions = config.output_attentions + self.output_hidden_states = config.output_hidden_states + self.embedding_hidden_mapping_in = tf.keras.layers.Dense(config.hidden_size, kernel_initializer=get_initializer( + config.initializer_range), name='embedding_hidden_mapping_in') + self.albert_layer_groups = [TFAlbertLayerGroup( + config, name="albert_layer_groups_._{}".format(i)) for i in range(config.num_hidden_groups)] + + def call(self, inputs, training=False): + hidden_states, attention_mask, head_mask = inputs + + hidden_states = self.embedding_hidden_mapping_in(hidden_states) + all_attentions = () + + if self.output_hidden_states: + all_hidden_states = (hidden_states,) + + for i in range(self.config.num_hidden_layers): + # Number of layers in a hidden group + layers_per_group = int( + self.config.num_hidden_layers / self.config.num_hidden_groups) + + # Index of the hidden group + group_idx = int( + i / (self.config.num_hidden_layers / self.config.num_hidden_groups)) + + layer_group_output = self.albert_layer_groups[group_idx]( + [hidden_states, attention_mask, head_mask[group_idx*layers_per_group:(group_idx+1)*layers_per_group]], training=training) + hidden_states = layer_group_output[0] + + if self.output_attentions: + all_attentions = all_attentions + layer_group_output[-1] + + if self.output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + outputs = (hidden_states,) + if self.output_hidden_states: + outputs = outputs + (all_hidden_states,) + if self.output_attentions: + outputs = outputs + (all_attentions,) + + # last-layer hidden state, (all hidden states), (all attentions) + return outputs + + +class TFAlbertPreTrainedModel(TFPreTrainedModel): + """ An abstract class to handle weights initialization and + a simple interface for dowloading and loading pretrained models. + """ + config_class = AlbertConfig + pretrained_model_archive_map = TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP + base_model_prefix = "albert" + + +class TFAlbertMLMHead(tf.keras.layers.Layer): + def __init__(self, config, input_embeddings, **kwargs): + super(TFAlbertMLMHead, self).__init__(**kwargs) + self.vocab_size = config.vocab_size + + self.dense = tf.keras.layers.Dense(config.embedding_size, + kernel_initializer=get_initializer( + config.initializer_range), + name='dense') + if isinstance(config.hidden_act, str) or (sys.version_info[0] == 2 and isinstance(config.hidden_act, unicode)): + self.activation = ACT2FN[config.hidden_act] + else: + self.activation = config.hidden_act + + self.LayerNorm = tf.keras.layers.LayerNormalization( + epsilon=config.layer_norm_eps, name='LayerNorm') + + # The output weights are the same as the input embeddings, but there is + # an output-only bias for each token. + self.input_embeddings = input_embeddings + + def build(self, input_shape): + self.bias = self.add_weight(shape=(self.vocab_size,), + initializer='zeros', + trainable=True, + name='bias') + super(TFAlbertMLMHead, self).build(input_shape) + + def call(self, hidden_states): + hidden_states = self.dense(hidden_states) + hidden_states = self.activation(hidden_states) + hidden_states = self.LayerNorm(hidden_states) + hidden_states = self.input_embeddings(hidden_states, mode="linear") + hidden_states = hidden_states + self.bias + return hidden_states + + +ALBERT_START_DOCSTRING = r""" The ALBERT model was proposed in + `ALBERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ + by Jacob Devlin, Ming-Wei Chang, Kenton Lee and Kristina Toutanova. It's a bidirectional transformer + pre-trained using a combination of masked language modeling objective and next sentence prediction + on a large corpus comprising the Toronto Book Corpus and Wikipedia. + + This model is a tf.keras.Model `tf.keras.Model`_ sub-class. Use it as a regular TF 2.0 Keras Model and + refer to the TF 2.0 documentation for all matter related to general usage and behavior. + + .. _`ALBERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`: + https://arxiv.org/abs/1810.04805 + + .. _`tf.keras.Model`: + https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/Model + + Note on the model inputs: + TF 2.0 models accepts two formats as inputs: + + - having all inputs as keyword arguments (like PyTorch models), or + - having all inputs as a list, tuple or dict in the first positional arguments. + + This second option is usefull when using `tf.keras.Model.fit()` method which currently requires having all the tensors in the first argument of the model call function: `model(inputs)`. + + If you choose this second option, there are three possibilities you can use to gather all the input Tensors in the first positional argument : + + - a single Tensor with input_ids only and nothing else: `model(inputs_ids) + - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring: + `model([input_ids, attention_mask])` or `model([input_ids, attention_mask, token_type_ids])` + - a dictionary with one or several input Tensors associaed to the input names given in the docstring: + `model({'input_ids': input_ids, 'token_type_ids': token_type_ids})` + + Parameters: + config (:class:`~transformers.AlbertConfig`): Model configuration class with all the parameters of the model. + Initializing with a config file does not load the weights associated with the model, only the configuration. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. +""" + +ALBERT_INPUTS_DOCSTRING = r""" + Inputs: + **input_ids**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + To match pre-training, ALBERT input sequence should be formatted with [CLS] and [SEP] tokens as follows: + + (a) For sequence pairs: + + ``tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1`` + + (b) For single sequences: + + ``tokens: [CLS] the dog is hairy . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0`` + + Albert is a model with absolute position embeddings so it's usually advised to pad the inputs on + the right rather than the left. + + Indices can be obtained using :class:`transformers.AlbertTokenizer`. + See :func:`transformers.PreTrainedTokenizer.encode` and + :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + **attention_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: + Mask to avoid performing attention on padding token indices. + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. + **token_type_ids**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: + Segment token indices to indicate first and second portions of the inputs. + Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` + corresponds to a `sentence B` token + (see `ALBERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ for more details). + **position_ids**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length)``: + Indices of positions of each input sequence tokens in the position embeddings. + Selected in the range ``[0, config.max_position_embeddings - 1]``. + **head_mask**: (`optional`) ``Numpy array`` or ``tf.Tensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. +""" + +@add_start_docstrings("The bare Albert Model transformer outputing raw hidden-states without any specific head on top.", + ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) +class TFAlbertModel(TFAlbertPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **last_hidden_state**: ``tf.Tensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + Sequence of hidden-states at the output of the last layer of the model. + **pooler_output**: ``tf.Tensor`` of shape ``(batch_size, hidden_size)`` + Last layer hidden-state of the first token of the sequence (classification token) + further processed by a Linear layer and a Tanh activation function. The Linear + layer weights are trained from the next sentence prediction (classification) + objective during Albert pretraining. This output is usually *not* a good summary + of the semantic content of the input, you're often better with averaging or pooling + the sequence of hidden-states for the whole input sequence. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + import tensorflow as tf + from transformers import AlbertTokenizer, TFAlbertModel + + tokenizer = AlbertTokenizer.from_pretrained('bert-base-uncased') + model = TFAlbertModel.from_pretrained('bert-base-uncased') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + outputs = model(input_ids) + last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple + + """ + + def __init__(self, config, **kwargs): + super(TFAlbertModel, self).__init__(config, **kwargs) + self.num_hidden_layers = config.num_hidden_layers + + self.embeddings = TFAlbertEmbeddings(config, name="embeddings") + self.encoder = TFAlbertTransformer(config, name="encoder") + self.pooler = tf.keras.layers.Dense(config.hidden_size, kernel_initializer=get_initializer( + config.initializer_range), activation='tanh', name='pooler') + + def _resize_token_embeddings(self, new_num_tokens): + raise NotImplementedError + + def _prune_heads(self, heads_to_prune): + """ Prunes heads of the model. + heads_to_prune: dict of {layer_num: list of heads to prune in this layer} + See base class PreTrainedModel + """ + raise NotImplementedError + + def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, training=False): + if isinstance(inputs, (tuple, list)): + input_ids = inputs[0] + attention_mask = inputs[1] if len(inputs) > 1 else attention_mask + token_type_ids = inputs[2] if len(inputs) > 2 else token_type_ids + position_ids = inputs[3] if len(inputs) > 3 else position_ids + head_mask = inputs[4] if len(inputs) > 4 else head_mask + assert len(inputs) <= 5, "Too many inputs." + elif isinstance(inputs, dict): + input_ids = inputs.get('input_ids') + attention_mask = inputs.get('attention_mask', attention_mask) + token_type_ids = inputs.get('token_type_ids', token_type_ids) + position_ids = inputs.get('position_ids', position_ids) + head_mask = inputs.get('head_mask', head_mask) + assert len(inputs) <= 5, "Too many inputs." + else: + input_ids = inputs + + if attention_mask is None: + attention_mask = tf.fill(tf.shape(input_ids), 1) + if token_type_ids is None: + token_type_ids = tf.fill(tf.shape(input_ids), 0) + + # We create a 3D attention mask from a 2D tensor mask. + # Sizes are [batch_size, 1, 1, to_seq_length] + # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length] + # this attention mask is more simple than the triangular masking of causal attention + # used in OpenAI GPT, we just need to prepare the broadcast dimension here. + extended_attention_mask = attention_mask[:, tf.newaxis, tf.newaxis, :] + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and -10000.0 for masked positions. + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + + extended_attention_mask = tf.cast(extended_attention_mask, tf.float32) + extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + if not head_mask is None: + raise NotImplementedError + else: + head_mask = [None] * self.num_hidden_layers + # head_mask = tf.constant([0] * self.num_hidden_layers) + + embedding_output = self.embeddings( + [input_ids, position_ids, token_type_ids], training=training) + encoder_outputs = self.encoder( + [embedding_output, extended_attention_mask, head_mask], training=training) + + sequence_output = encoder_outputs[0] + pooled_output = self.pooler(sequence_output) + + # add hidden_states and attentions if they are here + outputs = (sequence_output, pooled_output,) + encoder_outputs[1:] + # sequence_output, pooled_output, (hidden_states), (attentions) + return outputs + + +@add_start_docstrings("""Albert Model with a `language modeling` head on top. """, + ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) +class TFAlbertForMaskedLM(TFAlbertPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **prediction_scores**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``Numpy array`` or ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``Numpy array`` or ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + import tensorflow as tf + from transformers import AlbertTokenizer, TFAlbertForMaskedLM + + tokenizer = AlbertTokenizer.from_pretrained('bert-base-uncased') + model = TFAlbertForMaskedLM.from_pretrained('bert-base-uncased') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + outputs = model(input_ids) + prediction_scores = outputs[0] + + """ + + def __init__(self, config, *inputs, **kwargs): + super(TFAlbertForMaskedLM, self).__init__(config, *inputs, **kwargs) + + self.albert = TFAlbertModel(config, name='albert') + self.predictions = TFAlbertMLMHead( + config, self.albert.embeddings, name='predictions') + + def call(self, inputs, **kwargs): + outputs = self.albert(inputs, **kwargs) + + sequence_output = outputs[0] + prediction_scores = self.predictions( + sequence_output, training=kwargs.get('training', False)) + + # Add hidden states and attention if they are here + outputs = (prediction_scores,) + outputs[2:] + + return outputs # prediction_scores, (hidden_states), (attentions) From 7bddbf5961f35d03db5151a4747085f7a57d7ff7 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 7 Nov 2019 23:50:05 +0000 Subject: [PATCH 114/505] TFAlbertForSequenceClassification --- transformers/modeling_tf_albert.py | 66 ++++++++++++++++++++++++++---- 1 file changed, 58 insertions(+), 8 deletions(-) diff --git a/transformers/modeling_tf_albert.py b/transformers/modeling_tf_albert.py index 8861b7add8..410d83ff78 100644 --- a/transformers/modeling_tf_albert.py +++ b/transformers/modeling_tf_albert.py @@ -479,16 +479,15 @@ class TFAlbertMLMHead(tf.keras.layers.Layer): ALBERT_START_DOCSTRING = r""" The ALBERT model was proposed in - `ALBERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ - by Jacob Devlin, Ming-Wei Chang, Kenton Lee and Kristina Toutanova. It's a bidirectional transformer - pre-trained using a combination of masked language modeling objective and next sentence prediction - on a large corpus comprising the Toronto Book Corpus and Wikipedia. + `ALBERT: A Lite BERT for Self-supervised Learning of Language Representations`_ + by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. It presents + two parameter-reduction techniques to lower memory consumption and increase the trainig speed of BERT. This model is a tf.keras.Model `tf.keras.Model`_ sub-class. Use it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and behavior. - .. _`ALBERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`: - https://arxiv.org/abs/1810.04805 + .. _`ALBERT: A Lite BERT for Self-supervised Learning of Language Representations`: + https://arxiv.org/abs/1909.11942 .. _`tf.keras.Model`: https://www.tensorflow.org/versions/r2.0/api_docs/python/tf/keras/Model @@ -695,8 +694,8 @@ class TFAlbertForMaskedLM(TFAlbertPreTrainedModel): import tensorflow as tf from transformers import AlbertTokenizer, TFAlbertForMaskedLM - tokenizer = AlbertTokenizer.from_pretrained('bert-base-uncased') - model = TFAlbertForMaskedLM.from_pretrained('bert-base-uncased') + tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2') + model = TFAlbertForMaskedLM.from_pretrained('albert-base-v2') input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 outputs = model(input_ids) prediction_scores = outputs[0] @@ -721,3 +720,54 @@ class TFAlbertForMaskedLM(TFAlbertPreTrainedModel): outputs = (prediction_scores,) + outputs[2:] return outputs # prediction_scores, (hidden_states), (attentions) + + +@add_start_docstrings("""Albert Model transformer with a sequence classification/regression head on top (a linear layer on top of + the pooled output) e.g. for GLUE tasks. """, + ALBERT_START_DOCSTRING, ALBERT_INPUTS_DOCSTRING) +class TFAlbertForSequenceClassification(TFAlbertPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **logits**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, config.num_labels)`` + Classification (or regression if config.num_labels==1) scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``Numpy array`` or ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``Numpy array`` or ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + import tensorflow as tf + from transformers import AlbertTokenizer, TFAlbertForSequenceClassification + + tokenizer = AlbertTokenizer.from_pretrained('albert-base-v2') + model = TFAlbertForSequenceClassification.from_pretrained('albert-base-v2') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + outputs = model(input_ids) + logits = outputs[0] + + """ + def __init__(self, config, *inputs, **kwargs): + super(TFAlbertForSequenceClassification, self).__init__(config, *inputs, **kwargs) + self.num_labels = config.num_labels + + self.albert = TFAlbertModel(config, name='albert') + self.dropout = tf.keras.layers.Dropout(config.hidden_dropout_prob) + self.classifier = tf.keras.layers.Dense(config.num_labels, + kernel_initializer=get_initializer(config.initializer_range), + name='classifier') + + def call(self, inputs, **kwargs): + outputs = self.albert(inputs, **kwargs) + + pooled_output = outputs[1] + + pooled_output = self.dropout(pooled_output, training=kwargs.get('training', False)) + logits = self.classifier(pooled_output) + + outputs = (logits,) + outputs[2:] # add hidden states and attention if they are here + + return outputs # logits, (hidden_states), (attentions) \ No newline at end of file From b18509c2085dce16e655dd86b7ea0129335da4e1 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Fri, 8 Nov 2019 00:12:21 +0000 Subject: [PATCH 115/505] Tests for ALBERT in TF2 + fixes --- transformers/__init__.py | 1 + transformers/modeling_tf_albert.py | 18 +- transformers/tests/modeling_tf_albert_test.py | 229 ++++++++++++++++++ 3 files changed, 237 insertions(+), 11 deletions(-) create mode 100644 transformers/tests/modeling_tf_albert_test.py diff --git a/transformers/__init__.py b/transformers/__init__.py index a409ef772e..baf430c17b 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -169,6 +169,7 @@ if is_tf_available(): TF_CTRL_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_tf_albert import (TFAlbertPreTrainedModel, TFAlbertModel, TFAlbertForMaskedLM, + TFAlbertForSequenceClassification, TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) # TF 2.0 <=> PyTorch conversion utilities diff --git a/transformers/modeling_tf_albert.py b/transformers/modeling_tf_albert.py index 410d83ff78..a3f183b192 100644 --- a/transformers/modeling_tf_albert.py +++ b/transformers/modeling_tf_albert.py @@ -126,16 +126,8 @@ class TFAlbertEmbeddings(tf.keras.layers.Layer): """ batch_size = tf.shape(inputs)[0] length = tf.shape(inputs)[1] - - print(inputs.shape) - x = tf.reshape(inputs, [-1, self.config.embedding_size]) - - print(x.shape, self.word_embeddings) - logits = tf.matmul(x, self.word_embeddings, transpose_b=True) - - print([batch_size, length, self.config.vocab_size]) return tf.reshape(logits, [batch_size, length, self.config.vocab_size]) @@ -460,20 +452,24 @@ class TFAlbertMLMHead(tf.keras.layers.Layer): # The output weights are the same as the input embeddings, but there is # an output-only bias for each token. - self.input_embeddings = input_embeddings + self.decoder = input_embeddings def build(self, input_shape): self.bias = self.add_weight(shape=(self.vocab_size,), initializer='zeros', trainable=True, name='bias') + self.decoder_bias = self.add_weight(shape=(self.vocab_size,), + initializer='zeros', + trainable=True, + name='decoder/bias') super(TFAlbertMLMHead, self).build(input_shape) def call(self, hidden_states): hidden_states = self.dense(hidden_states) hidden_states = self.activation(hidden_states) hidden_states = self.LayerNorm(hidden_states) - hidden_states = self.input_embeddings(hidden_states, mode="linear") + hidden_states = self.decoder(hidden_states, mode="linear") + self.decoder_bias hidden_states = hidden_states + self.bias return hidden_states @@ -666,7 +662,7 @@ class TFAlbertModel(TFAlbertPreTrainedModel): [embedding_output, extended_attention_mask, head_mask], training=training) sequence_output = encoder_outputs[0] - pooled_output = self.pooler(sequence_output) + pooled_output = self.pooler(sequence_output[:, 0]) # add hidden_states and attentions if they are here outputs = (sequence_output, pooled_output,) + encoder_outputs[1:] diff --git a/transformers/tests/modeling_tf_albert_test.py b/transformers/tests/modeling_tf_albert_test.py new file mode 100644 index 0000000000..85fc62f34f --- /dev/null +++ b/transformers/tests/modeling_tf_albert_test.py @@ -0,0 +1,229 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import unittest +import shutil +import pytest +import sys + +from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) +from .configuration_common_test import ConfigTester + +from transformers import AlbertConfig, is_tf_available + +if is_tf_available(): + import tensorflow as tf + from transformers.modeling_tf_albert import (TFAlbertModel, TFAlbertForMaskedLM, + TFAlbertForSequenceClassification, + TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) +else: + pytestmark = pytest.mark.skip("Require TensorFlow") + + +class TFAlbertModelTest(TFCommonTestCases.TFCommonModelTester): + + all_model_classes = ( + TFAlbertModel, + TFAlbertForMaskedLM, + TFAlbertForSequenceClassification + ) if is_tf_available() else () + + class TFAlbertModelTester(object): + + def __init__(self, + parent, + batch_size=13, + seq_length=7, + is_training=True, + use_input_mask=True, + use_token_type_ids=True, + use_labels=True, + vocab_size=99, + hidden_size=32, + num_hidden_layers=5, + num_attention_heads=4, + intermediate_size=37, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + max_position_embeddings=512, + type_vocab_size=16, + type_sequence_label_size=2, + initializer_range=0.02, + num_labels=3, + num_choices=4, + scope=None, + ): + self.parent = parent + self.batch_size = batch_size + self.seq_length = seq_length + self.is_training = is_training + self.use_input_mask = use_input_mask + self.use_token_type_ids = use_token_type_ids + self.use_labels = use_labels + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.intermediate_size = intermediate_size + self.hidden_act = hidden_act + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.type_sequence_label_size = type_sequence_label_size + self.initializer_range = initializer_range + self.num_labels = num_labels + self.num_choices = num_choices + self.scope = scope + + def prepare_config_and_inputs(self): + input_ids = ids_tensor( + [self.batch_size, self.seq_length], self.vocab_size) + + input_mask = None + if self.use_input_mask: + input_mask = ids_tensor( + [self.batch_size, self.seq_length], vocab_size=2) + + token_type_ids = None + if self.use_token_type_ids: + token_type_ids = ids_tensor( + [self.batch_size, self.seq_length], self.type_vocab_size) + + sequence_labels = None + token_labels = None + choice_labels = None + if self.use_labels: + sequence_labels = ids_tensor( + [self.batch_size], self.type_sequence_label_size) + token_labels = ids_tensor( + [self.batch_size, self.seq_length], self.num_labels) + choice_labels = ids_tensor([self.batch_size], self.num_choices) + + config = AlbertConfig( + vocab_size_or_config_json_file=self.vocab_size, + hidden_size=self.hidden_size, + num_hidden_layers=self.num_hidden_layers, + num_attention_heads=self.num_attention_heads, + intermediate_size=self.intermediate_size, + hidden_act=self.hidden_act, + hidden_dropout_prob=self.hidden_dropout_prob, + attention_probs_dropout_prob=self.attention_probs_dropout_prob, + max_position_embeddings=self.max_position_embeddings, + type_vocab_size=self.type_vocab_size, + initializer_range=self.initializer_range) + + return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels + + def create_and_check_albert_model(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + model = TFAlbertModel(config=config) + # inputs = {'input_ids': input_ids, + # 'attention_mask': input_mask, + # 'token_type_ids': token_type_ids} + # sequence_output, pooled_output = model(**inputs) + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask, + 'token_type_ids': token_type_ids} + sequence_output, pooled_output = model(inputs) + + inputs = [input_ids, input_mask] + sequence_output, pooled_output = model(inputs) + + sequence_output, pooled_output = model(input_ids) + + result = { + "sequence_output": sequence_output.numpy(), + "pooled_output": pooled_output.numpy(), + } + self.parent.assertListEqual( + list(result["sequence_output"].shape), + [self.batch_size, self.seq_length, self.hidden_size]) + self.parent.assertListEqual(list(result["pooled_output"].shape), [ + self.batch_size, self.hidden_size]) + + def create_and_check_albert_for_masked_lm(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + model = TFAlbertForMaskedLM(config=config) + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask, + 'token_type_ids': token_type_ids} + prediction_scores, = model(inputs) + result = { + "prediction_scores": prediction_scores.numpy(), + } + self.parent.assertListEqual( + list(result["prediction_scores"].shape), + [self.batch_size, self.seq_length, self.vocab_size]) + + def create_and_check_albert_for_sequence_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): + config.num_labels = self.num_labels + model = TFAlbertForSequenceClassification(config=config) + inputs = {'input_ids': input_ids, + 'attention_mask': input_mask, + 'token_type_ids': token_type_ids} + logits, = model(inputs) + result = { + "logits": logits.numpy(), + } + self.parent.assertListEqual( + list(result["logits"].shape), + [self.batch_size, self.num_labels]) + + def prepare_config_and_inputs_for_common(self): + config_and_inputs = self.prepare_config_and_inputs() + (config, input_ids, token_type_ids, input_mask, + sequence_labels, token_labels, choice_labels) = config_and_inputs + inputs_dict = {'input_ids': input_ids, + 'token_type_ids': token_type_ids, 'attention_mask': input_mask} + return config, inputs_dict + + def setUp(self): + self.model_tester = TFAlbertModelTest.TFAlbertModelTester(self) + self.config_tester = ConfigTester( + self, config_class=AlbertConfig, hidden_size=37) + + def test_config(self): + self.config_tester.run_common_tests() + + def test_albert_model(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_albert_model(*config_and_inputs) + + def test_for_masked_lm(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_albert_for_masked_lm( + *config_and_inputs) + + def test_for_sequence_classification(self): + config_and_inputs = self.model_tester.prepare_config_and_inputs() + self.model_tester.create_and_check_albert_for_sequence_classification( + *config_and_inputs) + + @pytest.mark.slow + def test_model_from_pretrained(self): + cache_dir = "/tmp/transformers_test/" + # for model_name in list(TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: + for model_name in ['albert-base-uncased']: + model = TFAlbertModel.from_pretrained( + model_name, cache_dir=cache_dir) + shutil.rmtree(cache_dir) + self.assertIsNotNone(model) + + +if __name__ == "__main__": + unittest.main() From c9cb7f8a0fbe784665b00bfdca6bfc54ad10d5f5 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Mon, 11 Nov 2019 15:12:54 -0500 Subject: [PATCH 116/505] Torch 1.1.0 compatibility + FP16 O1 + TF checkpoints Co-authored-by: wassname --- transformers/modeling_albert.py | 4 ++-- transformers/modeling_tf_albert.py | 9 ++++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 640af537d0..ff20ca78dc 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -203,8 +203,8 @@ class AlbertAttention(BertSelfAttention): # Should find a better way to do this - w = self.dense.weight.T.view(self.num_attention_heads, self.attention_head_size, self.hidden_size) - b = self.dense.bias + w = self.dense.weight.t().view(self.num_attention_heads, self.attention_head_size, self.hidden_size).to(context_layer.dtype) + b = self.dense.bias.to(context_layer.dtype) projected_context_layer = torch.einsum("bfnd,ndh->bfh", context_layer, w) + b projected_context_layer_dropout = self.dropout(projected_context_layer) diff --git a/transformers/modeling_tf_albert.py b/transformers/modeling_tf_albert.py index a3f183b192..ee8712eb28 100644 --- a/transformers/modeling_tf_albert.py +++ b/transformers/modeling_tf_albert.py @@ -36,7 +36,14 @@ import logging logger = logging.getLogger(__name__) TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP = { - # TODO FILL THAT UP + 'albert-base-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-tf_model.h5", + 'albert-large-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-tf_model.h5", + 'albert-xlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-tf_model.h5", + 'albert-xxlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-tf_model.h5", + 'albert-base-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v2-tf_model.h5", + 'albert-large-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-v2-tf_model.h5", + 'albert-xlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-v2-tf_model.h5", + 'albert-xxlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-v2-tf_model.h5", } From f873b55e433e1c9ef364e597ad1d4662db4bc038 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Tue, 26 Nov 2019 10:28:41 -0500 Subject: [PATCH 117/505] Warning for ALBERT-v2 models --- transformers/modeling_utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/transformers/modeling_utils.py b/transformers/modeling_utils.py index d51eefab58..5d51caa951 100644 --- a/transformers/modeling_utils.py +++ b/transformers/modeling_utils.py @@ -315,6 +315,10 @@ class PreTrainedModel(nn.Module): model = BertModel.from_pretrained('./tf_model/my_tf_checkpoint.ckpt.index', from_tf=True, config=config) """ + if "albert" in pretrained_model_name_or_path and "v2" in pretrained_model_name_or_path: + logger.warning("There is currently an upstream reproducibility issue with ALBERT v2 models. Please see " + + "https://github.com/google-research/google-research/issues/119 for more information.") + config = kwargs.pop('config', None) state_dict = kwargs.pop('state_dict', None) cache_dir = kwargs.pop('cache_dir', None) From c536c2a4809a51356fd50ff62a80955fea79f790 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Tue, 26 Nov 2019 11:22:52 -0500 Subject: [PATCH 118/505] ALBERT Input Embeds --- transformers/modeling_albert.py | 77 ++++++++++++++++++++++-------- transformers/modeling_tf_albert.py | 45 ++++++++++++----- 2 files changed, 90 insertions(+), 32 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index ff20ca78dc..7882356d24 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -433,6 +433,12 @@ class AlbertModel(AlbertPreTrainedModel): self.init_weights() + def get_input_embeddings(self): + return self.embeddings.word_embeddings + + def set_input_embeddings(self, value): + self.embeddings.word_embeddings = value + def _resize_token_embeddings(self, new_num_tokens): old_embeddings = self.embeddings.word_embeddings new_embeddings = self._get_resized_embeddings(old_embeddings, new_num_tokens) @@ -457,12 +463,24 @@ class AlbertModel(AlbertPreTrainedModel): inner_group_idx = int(layer - group_idx * self.config.inner_group_num) self.encoder.albert_layer_groups[group_idx].albert_layers[inner_group_idx].attention.prune_heads(heads) + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + inputs_embeds=None): + + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") + elif input_ids is not None: + input_shape = input_ids.size() + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") + + device = input_ids.device if input_ids is not None else inputs_embeds.device - def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None): if attention_mask is None: - attention_mask = torch.ones_like(input_ids) + attention_mask = torch.ones(input_shape, device=device) if token_type_ids is None: - token_type_ids = torch.zeros_like(input_ids) + token_type_ids = torch.zeros(input_shape, dtype=torch.long, device=device) extended_attention_mask = attention_mask.unsqueeze(1).unsqueeze(2) extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility @@ -477,7 +495,8 @@ class AlbertModel(AlbertPreTrainedModel): else: head_mask = [None] * self.config.num_hidden_layers - embedding_output = self.embeddings(input_ids, position_ids=position_ids, token_type_ids=token_type_ids) + embedding_output = self.embeddings(input_ids, position_ids=position_ids, token_type_ids=token_type_ids, + inputs_embeds=inputs_embeds) encoder_outputs = self.encoder(embedding_output, extended_attention_mask, head_mask=head_mask) @@ -549,9 +568,19 @@ class AlbertForMaskedLM(AlbertPreTrainedModel): self._tie_or_clone_weights(self.predictions.decoder, self.albert.embeddings.word_embeddings) - def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, - masked_lm_labels=None): - outputs = self.albert(input_ids, attention_mask, token_type_ids, position_ids, head_mask) + def get_output_embeddings(self): + return self.predictions.decoder + + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + masked_lm_labels=None, inputs_embeds=None): + outputs = self.albert( + input_ids=input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds + ) sequence_outputs = outputs[0] prediction_scores = self.predictions(sequence_outputs) @@ -609,14 +638,17 @@ class AlbertForSequenceClassification(AlbertPreTrainedModel): self.init_weights() - def forward(self, input_ids, attention_mask=None, token_type_ids=None, - position_ids=None, head_mask=None, labels=None): + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, + position_ids=None, head_mask=None, inputs_embeds=None, labels=None): - outputs = self.albert(input_ids, - attention_mask=attention_mask, - token_type_ids=token_type_ids, - position_ids=position_ids, - head_mask=head_mask) + outputs = self.albert( + input_ids=input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds + ) pooled_output = outputs[1] @@ -692,14 +724,17 @@ class AlbertForQuestionAnswering(AlbertPreTrainedModel): self.init_weights() - def forward(self, input_ids, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, - start_positions=None, end_positions=None): + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, + inputs_embeds=None, start_positions=None, end_positions=None): - outputs = self.albert(input_ids, - attention_mask=attention_mask, - token_type_ids=token_type_ids, - position_ids=position_ids, - head_mask=head_mask) + outputs = self.albert( + input_ids=input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds + ) sequence_output = outputs[0] diff --git a/transformers/modeling_tf_albert.py b/transformers/modeling_tf_albert.py index ee8712eb28..ee122205b9 100644 --- a/transformers/modeling_tf_albert.py +++ b/transformers/modeling_tf_albert.py @@ -107,19 +107,25 @@ class TFAlbertEmbeddings(tf.keras.layers.Layer): def _embedding(self, inputs, training=False): """Applies embedding based on inputs tensor.""" - input_ids, position_ids, token_type_ids = inputs + input_ids, position_ids, token_type_ids, inputs_embeds = inputs - seq_length = tf.shape(input_ids)[1] + if input_ids is not None: + input_shape = tf.shape(input_ids) + else: + input_shape = tf.shape(inputs_embeds)[:-1] + + seq_length = input_shape[1] if position_ids is None: position_ids = tf.range(seq_length, dtype=tf.int32)[tf.newaxis, :] if token_type_ids is None: - token_type_ids = tf.fill(tf.shape(input_ids), 0) + token_type_ids = tf.fill(input_shape, 0) - words_embeddings = tf.gather(self.word_embeddings, input_ids) + if inputs_embeds is None: + inputs_embeds = tf.gather(self.word_embeddings, input_ids) position_embeddings = self.position_embeddings(position_ids) token_type_embeddings = self.token_type_embeddings(token_type_ids) - embeddings = words_embeddings + position_embeddings + token_type_embeddings + embeddings = inputs_embeds + position_embeddings + token_type_embeddings embeddings = self.LayerNorm(embeddings) embeddings = self.dropout(embeddings, training=training) return embeddings @@ -603,6 +609,9 @@ class TFAlbertModel(TFAlbertPreTrainedModel): self.pooler = tf.keras.layers.Dense(config.hidden_size, kernel_initializer=get_initializer( config.initializer_range), activation='tanh', name='pooler') + def get_input_embeddings(self): + return self.embeddings + def _resize_token_embeddings(self, new_num_tokens): raise NotImplementedError @@ -613,28 +622,39 @@ class TFAlbertModel(TFAlbertPreTrainedModel): """ raise NotImplementedError - def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, training=False): + def call(self, inputs, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, training=False): if isinstance(inputs, (tuple, list)): input_ids = inputs[0] attention_mask = inputs[1] if len(inputs) > 1 else attention_mask token_type_ids = inputs[2] if len(inputs) > 2 else token_type_ids position_ids = inputs[3] if len(inputs) > 3 else position_ids head_mask = inputs[4] if len(inputs) > 4 else head_mask - assert len(inputs) <= 5, "Too many inputs." + inputs_embeds = inputs[5] if len(inputs) > 5 else inputs_embeds + assert len(inputs) <= 6, "Too many inputs." elif isinstance(inputs, dict): input_ids = inputs.get('input_ids') attention_mask = inputs.get('attention_mask', attention_mask) token_type_ids = inputs.get('token_type_ids', token_type_ids) position_ids = inputs.get('position_ids', position_ids) head_mask = inputs.get('head_mask', head_mask) - assert len(inputs) <= 5, "Too many inputs." + inputs_embeds = inputs.get('inputs_embeds', inputs_embeds) + assert len(inputs) <= 6, "Too many inputs." else: input_ids = inputs + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") + elif input_ids is not None: + input_shape = input_ids.shape + elif inputs_embeds is not None: + input_shape = inputs_embeds.shape[:-1] + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") + if attention_mask is None: - attention_mask = tf.fill(tf.shape(input_ids), 1) + attention_mask = tf.fill(input_shape, 1) if token_type_ids is None: - token_type_ids = tf.fill(tf.shape(input_ids), 0) + token_type_ids = tf.fill(input_shape, 0) # We create a 3D attention mask from a 2D tensor mask. # Sizes are [batch_size, 1, 1, to_seq_length] @@ -664,7 +684,7 @@ class TFAlbertModel(TFAlbertPreTrainedModel): # head_mask = tf.constant([0] * self.num_hidden_layers) embedding_output = self.embeddings( - [input_ids, position_ids, token_type_ids], training=training) + [input_ids, position_ids, token_type_ids, inputs_embeds], training=training) encoder_outputs = self.encoder( [embedding_output, extended_attention_mask, head_mask], training=training) @@ -712,6 +732,9 @@ class TFAlbertForMaskedLM(TFAlbertPreTrainedModel): self.predictions = TFAlbertMLMHead( config, self.albert.embeddings, name='predictions') + def get_output_embeddings(self): + return self.albert.embeddings + def call(self, inputs, **kwargs): outputs = self.albert(inputs, **kwargs) From bdfe21ab242c2175972f53ca6b4af29ada02a2bb Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Tue, 26 Nov 2019 12:54:36 -0500 Subject: [PATCH 119/505] Change param order for consistency --- transformers/modeling_albert.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 7882356d24..5b7b2d3900 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -571,8 +571,8 @@ class AlbertForMaskedLM(AlbertPreTrainedModel): def get_output_embeddings(self): return self.predictions.decoder - def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, - masked_lm_labels=None, inputs_embeds=None): + def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, head_mask=None, inputs_embeds=None, + masked_lm_labels=None): outputs = self.albert( input_ids=input_ids, attention_mask=attention_mask, From f2f329408db66285fd59e6628ca394381bb7f94e Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 26 Nov 2019 12:59:28 -0500 Subject: [PATCH 120/505] Fix input embeddings --- transformers/tests/modeling_albert_test.py | 2 ++ transformers/tests/modeling_tf_albert_test.py | 2 ++ transformers/tests/modeling_tf_common_test.py | 7 ++++--- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/transformers/tests/modeling_albert_test.py b/transformers/tests/modeling_albert_test.py index da87709df1..976feff9db 100644 --- a/transformers/tests/modeling_albert_test.py +++ b/transformers/tests/modeling_albert_test.py @@ -49,6 +49,7 @@ class AlbertModelTest(CommonTestCases.CommonModelTester): use_token_type_ids=True, use_labels=True, vocab_size=99, + embedding_size=16, hidden_size=36, num_hidden_layers=6, num_hidden_groups=6, @@ -73,6 +74,7 @@ class AlbertModelTest(CommonTestCases.CommonModelTester): self.use_token_type_ids = use_token_type_ids self.use_labels = use_labels self.vocab_size = vocab_size + self.embedding_size = embedding_size self.hidden_size = hidden_size self.num_hidden_layers = num_hidden_layers self.num_attention_heads = num_attention_heads diff --git a/transformers/tests/modeling_tf_albert_test.py b/transformers/tests/modeling_tf_albert_test.py index 85fc62f34f..fbd519b8f6 100644 --- a/transformers/tests/modeling_tf_albert_test.py +++ b/transformers/tests/modeling_tf_albert_test.py @@ -54,6 +54,7 @@ class TFAlbertModelTest(TFCommonTestCases.TFCommonModelTester): use_token_type_ids=True, use_labels=True, vocab_size=99, + embedding_size=16, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, @@ -77,6 +78,7 @@ class TFAlbertModelTest(TFCommonTestCases.TFCommonModelTester): self.use_token_type_ids = use_token_type_ids self.use_labels = use_labels self.vocab_size = vocab_size + self.embedding_size = embedding_size self.hidden_size = hidden_size self.num_hidden_layers = num_hidden_layers self.num_attention_heads = num_attention_heads diff --git a/transformers/tests/modeling_tf_common_test.py b/transformers/tests/modeling_tf_common_test.py index 2bb7cc9c5f..31a30766cf 100644 --- a/transformers/tests/modeling_tf_common_test.py +++ b/transformers/tests/modeling_tf_common_test.py @@ -426,9 +426,10 @@ class TFCommonTestCases: try: x = wte([input_ids], mode="embedding") except: - x = tf.ones(input_ids.shape + [self.model_tester.hidden_size], dtype=tf.dtypes.float32) - # ^^ In our TF models, the input_embeddings can take slightly different forms, - # so we try two of them and fall back to just synthetically creating a dummy tensor of ones. + if hasattr(self.model_tester, "embedding_size"): + x = tf.ones(input_ids.shape + [model.config.embedding_size], dtype=tf.dtypes.float32) + else: + x = tf.ones(input_ids.shape + [self.model_tester.hidden_size], dtype=tf.dtypes.float32) inputs_dict["inputs_embeds"] = x outputs = model(inputs_dict) From ae98d4599179b299563b679dd33f8a86da12980d Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 26 Nov 2019 14:12:44 -0500 Subject: [PATCH 121/505] Release: v2.2.0 --- .circleci/deploy.sh | 1 + README.md | 2 +- deploy_multi_version_doc.sh | 22 ++++++++++++++++++++++ docs/source/conf.py | 2 +- setup.py | 2 +- transformers/__init__.py | 2 +- 6 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 deploy_multi_version_doc.sh diff --git a/.circleci/deploy.sh b/.circleci/deploy.sh index 98151963d8..c4b802c9e9 100755 --- a/.circleci/deploy.sh +++ b/.circleci/deploy.sh @@ -23,3 +23,4 @@ deploy_doc "fe02e45" v1.1.0 deploy_doc "89fd345" v1.2.0 deploy_doc "fc9faa8" v2.0.0 deploy_doc "3ddce1d" v2.1.1 +deploy_doc "f2f3294" v2.2.0 diff --git a/README.md b/README.md index 52e74a6f80..bb6a8eaa1a 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ Choose the right framework for every part of a model's lifetime | [Quick tour: Fine-tuning/usage scripts](#quick-tour-of-the-fine-tuningusage-scripts) | Using provided scripts: GLUE, SQuAD and Text generation | | [Migrating from pytorch-transformers to transformers](#Migrating-from-pytorch-transformers-to-transformers) | Migrating your code from pytorch-transformers to transformers | | [Migrating from pytorch-pretrained-bert to pytorch-transformers](#Migrating-from-pytorch-pretrained-bert-to-transformers) | Migrating your code from pytorch-pretrained-bert to transformers | -| [Documentation](https://huggingface.co/transformers/) [(v2.1.1)](https://huggingface.co/transformers/v2.1.1) [(v2.0.0)](https://huggingface.co/transformers/v2.0.0) [(v1.2.0)](https://huggingface.co/transformers/v1.2.0) [(v1.1.0)](https://huggingface.co/transformers/v1.1.0) [(v1.0.0)](https://huggingface.co/transformers/v1.0.0) | Full API documentation and more | +| [Documentation][(v2.2.0)](https://huggingface.co/transformers/v2.2.0) [(v2.1.1)](https://huggingface.co/transformers/v2.1.1) [(v2.0.0)](https://huggingface.co/transformers/v2.0.0) [(v1.2.0)](https://huggingface.co/transformers/v1.2.0) [(v1.1.0)](https://huggingface.co/transformers/v1.1.0) [(v1.0.0)](https://huggingface.co/transformers/v1.0.0) [(master](https://huggingface.co/transformers) | Full API documentation and more | ## Installation diff --git a/deploy_multi_version_doc.sh b/deploy_multi_version_doc.sh new file mode 100644 index 0000000000..bd567213eb --- /dev/null +++ b/deploy_multi_version_doc.sh @@ -0,0 +1,22 @@ +cd docs + +function deploy_doc(){ + echo "Creating doc at commit $1 and pushing to folder $2" + git checkout $1 + if [ ! -z "$2" ] + then + echo "Pushing version" $2 + make clean && make html && scp -r -oStrictHostKeyChecking=no _build/html $doc:$dir/$2 + else + echo "Pushing master" + make clean && make html && scp -r -oStrictHostKeyChecking=no _build/html/* $doc:$dir + fi +} + +deploy_doc "master" +deploy_doc "b33a385" v1.0.0 +deploy_doc "fe02e45" v1.1.0 +deploy_doc "89fd345" v1.2.0 +deploy_doc "fc9faa8" v2.0.0 +deploy_doc "3ddce1d" v2.1.1 +deploy_doc "f2f3294" v2.2.0 \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index 00c020ab39..f762a89cd2 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -26,7 +26,7 @@ author = u'huggingface' # The short X.Y version version = u'' # The full version, including alpha/beta/rc tags -release = u'2.1.1' +release = u'2.2.0' # -- General configuration --------------------------------------------------- diff --git a/setup.py b/setup.py index f49aee68d4..d8dcf7b898 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ from setuptools import find_packages, setup setup( name="transformers", - version="2.1.1", + version="2.2.0", author="Thomas Wolf, Lysandre Debut, Victor Sanh, Julien Chaumond, Google AI Language Team Authors, Open AI team Authors, Facebook AI Authors, Carnegie Mellon University Authors", author_email="thomas@huggingface.co", description="State-of-the-art Natural Language Processing for TensorFlow 2.0 and PyTorch", diff --git a/transformers/__init__.py b/transformers/__init__.py index baf430c17b..dff2479d4b 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -1,4 +1,4 @@ -__version__ = "2.1.1" +__version__ = "2.2.0" # Work around to update TensorFlow's absl.logging threshold which alters the # default Python logging output behavior when present. From b6321452738925983ba1306fb5396982d64f408a Mon Sep 17 00:00:00 2001 From: Lysandre Debut Date: Tue, 26 Nov 2019 14:27:15 -0500 Subject: [PATCH 122/505] Update master documentation link in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bb6a8eaa1a..67fb80aecf 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ Choose the right framework for every part of a model's lifetime | [Quick tour: Fine-tuning/usage scripts](#quick-tour-of-the-fine-tuningusage-scripts) | Using provided scripts: GLUE, SQuAD and Text generation | | [Migrating from pytorch-transformers to transformers](#Migrating-from-pytorch-transformers-to-transformers) | Migrating your code from pytorch-transformers to transformers | | [Migrating from pytorch-pretrained-bert to pytorch-transformers](#Migrating-from-pytorch-pretrained-bert-to-transformers) | Migrating your code from pytorch-pretrained-bert to transformers | -| [Documentation][(v2.2.0)](https://huggingface.co/transformers/v2.2.0) [(v2.1.1)](https://huggingface.co/transformers/v2.1.1) [(v2.0.0)](https://huggingface.co/transformers/v2.0.0) [(v1.2.0)](https://huggingface.co/transformers/v1.2.0) [(v1.1.0)](https://huggingface.co/transformers/v1.1.0) [(v1.0.0)](https://huggingface.co/transformers/v1.0.0) [(master](https://huggingface.co/transformers) | Full API documentation and more | +| [Documentation][(v2.2.0)](https://huggingface.co/transformers/v2.2.0) [(v2.1.1)](https://huggingface.co/transformers/v2.1.1) [(v2.0.0)](https://huggingface.co/transformers/v2.0.0) [(v1.2.0)](https://huggingface.co/transformers/v1.2.0) [(v1.1.0)](https://huggingface.co/transformers/v1.1.0) [(v1.0.0)](https://huggingface.co/transformers/v1.0.0) [(master)](https://huggingface.co/transformers) | Full API documentation and more | ## Installation From cf62bdc962c53d9fb7a5820217f1bf844bb6da3b Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Tue, 26 Nov 2019 14:37:32 -0500 Subject: [PATCH 123/505] Improve test protocol for inputs_embeds in TF cc @lysandrejik --- transformers/tests/modeling_tf_common_test.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/transformers/tests/modeling_tf_common_test.py b/transformers/tests/modeling_tf_common_test.py index 31a30766cf..232991915c 100644 --- a/transformers/tests/modeling_tf_common_test.py +++ b/transformers/tests/modeling_tf_common_test.py @@ -426,10 +426,15 @@ class TFCommonTestCases: try: x = wte([input_ids], mode="embedding") except: - if hasattr(self.model_tester, "embedding_size"): - x = tf.ones(input_ids.shape + [model.config.embedding_size], dtype=tf.dtypes.float32) - else: - x = tf.ones(input_ids.shape + [self.model_tester.hidden_size], dtype=tf.dtypes.float32) + x = wte([input_ids, None, None, None], mode="embedding") + # ^^ In our TF models, the input_embeddings can take slightly different forms, + # so we try a few of them. + # We used to fall back to just synthetically creating a dummy tensor of ones: + # + # if hasattr(self.model_tester, "embedding_size"): + # x = tf.ones(input_ids.shape + [self.model_tester.embedding_size], dtype=tf.dtypes.float32) + # else: + # x = tf.ones(input_ids.shape + [self.model_tester.hidden_size], dtype=tf.dtypes.float32) inputs_dict["inputs_embeds"] = x outputs = model(inputs_dict) From 8742baa53136b906e392fdae57f1c191d1b2370e Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Tue, 26 Nov 2019 14:39:47 -0500 Subject: [PATCH 124/505] Improve test protocol for inputs_embeds in TF --- transformers/tests/modeling_tf_common_test.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/transformers/tests/modeling_tf_common_test.py b/transformers/tests/modeling_tf_common_test.py index 232991915c..ea8cd1aecd 100644 --- a/transformers/tests/modeling_tf_common_test.py +++ b/transformers/tests/modeling_tf_common_test.py @@ -426,15 +426,17 @@ class TFCommonTestCases: try: x = wte([input_ids], mode="embedding") except: - x = wte([input_ids, None, None, None], mode="embedding") + try: + x = wte([input_ids, None, None, None], mode="embedding") + except: + if hasattr(self.model_tester, "embedding_size"): + x = tf.ones(input_ids.shape + [self.model_tester.embedding_size], dtype=tf.dtypes.float32) + else: + x = tf.ones(input_ids.shape + [self.model_tester.hidden_size], dtype=tf.dtypes.float32) # ^^ In our TF models, the input_embeddings can take slightly different forms, # so we try a few of them. # We used to fall back to just synthetically creating a dummy tensor of ones: # - # if hasattr(self.model_tester, "embedding_size"): - # x = tf.ones(input_ids.shape + [self.model_tester.embedding_size], dtype=tf.dtypes.float32) - # else: - # x = tf.ones(input_ids.shape + [self.model_tester.hidden_size], dtype=tf.dtypes.float32) inputs_dict["inputs_embeds"] = x outputs = model(inputs_dict) From 668aac45d206bc0c8ca95e7a8b565d1ff701da7f Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 26 Nov 2019 14:52:42 -0500 Subject: [PATCH 125/505] Pretrained models --- docs/source/pretrained_models.rst | 33 +++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst index b0a578fd80..d017a3be6c 100644 --- a/docs/source/pretrained_models.rst +++ b/docs/source/pretrained_models.rst @@ -159,5 +159,38 @@ Here is the full list of the currently provided pretrained models together with | | | | CamemBERT using the BERT-base architecture | | | | (see `details `__) | +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| ALBERT | ``albert-base-v1`` | | 12 repeating layers, 128 embedding, 768-hidden, 12-heads, 11M parameters | +| | | | ALBERT base model | +| | | (see `details `__) | ++--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``albert-large-v1`` | | 24 repeating layers, 128 embedding, 1024-hidden, 16-heads, 17M parameters | +| | | | ALBERT large model | +| | | (see `details `__) | ++--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``albert-xlarge-v1`` | | 24 repeating layers, 128 embedding, 2048-hidden, 16-heads, 58M parameters | +| | | | ALBERT xlarge model | +| | | (see `details `__) | ++--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``albert-xxlarge-v1`` | | 12 repeating layer, 128 embedding, 4096-hidden, 64-heads, 223M parameters | +| | | | ALBERT xxlarge model | +| | | (see `details `__) | ++--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``albert-base-v2`` | | 12 repeating layers, 128 embedding, 768-hidden, 12-heads, 11M parameters | +| | | | ALBERT base model with no dropout, additional training data and longer training | +| | | (see `details `__) | ++--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``albert-large-v2`` | | 24 repeating layers, 128 embedding, 1024-hidden, 16-heads, 17M parameters | +| | | | ALBERT large model with no dropout, additional training data and longer training | +| | | (see `details `__) | ++--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``albert-xlarge-v2`` | | 24 repeating layers, 128 embedding, 2048-hidden, 16-heads, 58M parameters | +| | | | ALBERT xlarge model with no dropout, additional training data and longer training | +| | | (see `details `__) | ++--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``albert-xxlarge-v2`` | | 12 repeating layer, 128 embedding, 4096-hidden, 64-heads, 223M parameters | +| | | | ALBERT xxlarge model with no dropout, additional training data and longer training | +| | | (see `details `__) | ++--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ + .. `__ From 7c6000e412288ddce7d0dbc4f4d9a12f49d28690 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 26 Nov 2019 14:55:29 -0500 Subject: [PATCH 126/505] Updated v2.2.0 doc --- .circleci/deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/deploy.sh b/.circleci/deploy.sh index c4b802c9e9..73960cbe99 100755 --- a/.circleci/deploy.sh +++ b/.circleci/deploy.sh @@ -23,4 +23,4 @@ deploy_doc "fe02e45" v1.1.0 deploy_doc "89fd345" v1.2.0 deploy_doc "fc9faa8" v2.0.0 deploy_doc "3ddce1d" v2.1.1 -deploy_doc "f2f3294" v2.2.0 +deploy_doc "668aac4" v2.2.0 From ee4647bd5c1343304f7fdf20f06db1fc2524f892 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 26 Nov 2019 15:10:51 -0500 Subject: [PATCH 127/505] CamemBERT & ALBERT doc --- docs/source/index.rst | 5 ++ docs/source/model_doc/albert.rst | 71 +++++++++++++++++++++++++++++ docs/source/model_doc/camembert.rst | 50 ++++++++++++++++++++ 3 files changed, 126 insertions(+) create mode 100644 docs/source/model_doc/albert.rst create mode 100644 docs/source/model_doc/camembert.rst diff --git a/docs/source/index.rst b/docs/source/index.rst index 4cd1f48ba8..55ead33b4d 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -47,6 +47,9 @@ The library currently contains PyTorch and Tensorflow implementations, pre-train 6. `XLM `_ (from Facebook) released together with the paper `Cross-lingual Language Model Pretraining `_ by Guillaume Lample and Alexis Conneau. 7. `RoBERTa `_ (from Facebook), released together with the paper a `Robustly Optimized BERT Pretraining Approach `_ by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov. 8. `DistilBERT `_ (from HuggingFace) released together with the paper `DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter `_ by Victor Sanh, Lysandre Debut and Thomas Wolf. The same method has been applied to compress GPT2 into `DistilGPT2 `_. +9. `CTRL `_ (from Salesforce), released together with the paper `CTRL: A Conditional Transformer Language Model for Controllable Generation `_ by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher. +10. `CamemBERT `_ (from FAIR, Inria, Sorbonne Université) released together with the paper `CamemBERT: a Tasty French Language Model `_ by Louis Martin, Benjamin Muller, Pedro Javier Ortiz Suarez, Yoann Dupont, Laurent Romary, Eric Villemonte de la Clergerie, Djame Seddah, and Benoît Sagot. +11. `ALBERT `_ (from Google Research), released together with the paper a `ALBERT: A Lite BERT for Self-supervised Learning of Language Representations `_ by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. .. toctree:: :maxdepth: 2 @@ -89,3 +92,5 @@ The library currently contains PyTorch and Tensorflow implementations, pre-train model_doc/roberta model_doc/distilbert model_doc/ctrl + model_doc/camembert + model_doc/albert diff --git a/docs/source/model_doc/albert.rst b/docs/source/model_doc/albert.rst new file mode 100644 index 0000000000..cf52f35eb6 --- /dev/null +++ b/docs/source/model_doc/albert.rst @@ -0,0 +1,71 @@ +ALBERT +---------------------------------------------------- + +``AlbrtConfig`` +~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.AlbertConfig + :members: + + +``AlbertTokenizer`` +~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.AlbertTokenizer + :members: + + +``AlbertModel`` +~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.AlbertModel + :members: + + +``AlbertForMaskedLM`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.AlbertForMaskedLM + :members: + + +``AlbertForSequenceClassification`` +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.AlbertForSequenceClassification + :members: + + +``AlbertForQuestionAnswering`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.AlbertForQuestionAnswering + :members: + + +``TFAlbertModel`` +~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.TFAlbertModel + :members: + + +``TFBertForPreTraining`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.TFBertForPreTraining + :members: + + +``TFAlbertForMaskedLM`` +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.TFAlbertForMaskedLM + :members: + + +``TFAlbertForSequenceClassification`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.TFAlbertForSequenceClassification + :members: diff --git a/docs/source/model_doc/camembert.rst b/docs/source/model_doc/camembert.rst new file mode 100644 index 0000000000..82ca9de945 --- /dev/null +++ b/docs/source/model_doc/camembert.rst @@ -0,0 +1,50 @@ +CamemBERT +---------------------------------------------------- + +``CamembertConfig`` +~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.CamembertConfig + :members: + + +``CamembertTokenizer`` +~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.CamembertTokenizer + :members: + + +``CamembertModel`` +~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.CamembertModel + :members: + + +``CamembertForMaskedLM`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.CamembertForMaskedLM + :members: + + +``CamembertForSequenceClassification`` +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.CamembertForSequenceClassification + :members: + + +``CamembertForMultipleChoice`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.CamembertForMultipleChoice + :members: + + +``CamembertForTokenClassification`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.CamembertForTokenClassification + :members: From 44b82c777f0a4e173fa9c68de630a63fbac1b946 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 26 Nov 2019 15:15:11 -0500 Subject: [PATCH 128/505] Updated v2.2.0 doc --- .circleci/deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/deploy.sh b/.circleci/deploy.sh index 73960cbe99..8081153b9f 100755 --- a/.circleci/deploy.sh +++ b/.circleci/deploy.sh @@ -23,4 +23,4 @@ deploy_doc "fe02e45" v1.1.0 deploy_doc "89fd345" v1.2.0 deploy_doc "fc9faa8" v2.0.0 deploy_doc "3ddce1d" v2.1.1 -deploy_doc "668aac4" v2.2.0 +deploy_doc "ee4647b" v2.2.0 From cf26a0c85e0520d3e3770c9ad8dce94318ea3440 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 26 Nov 2019 15:40:03 -0500 Subject: [PATCH 129/505] Fix pretrained models table --- README.md | 1 + docs/source/pretrained_models.rst | 14 +++++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 67fb80aecf..dd2c865036 100644 --- a/README.md +++ b/README.md @@ -134,6 +134,7 @@ At some point in the future, you'll be able to seamlessly move from pre-training 8. **[DistilBERT](https://github.com/huggingface/transformers/tree/master/examples/distillation)** (from HuggingFace), released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter](https://arxiv.org/abs/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same method has been applied to compress GPT2 into [DistilGPT2](https://github.com/huggingface/transformers/tree/master/examples/distillation). 9. **[CTRL](https://github.com/salesforce/ctrl/)** (from Salesforce) released with the paper [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/abs/1909.05858) by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher. 10. **[CamemBERT](https://camembert-model.fr)** (from Inria/Facebook/Sorbonne) released with the paper [CamemBERT: a Tasty French Language Model](https://arxiv.org/abs/1911.03894) by Louis Martin*, Benjamin Muller*, Pedro Javier Ortiz Suárez*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah and Benoît Sagot. +11. **[ALBERT](https://github.com/google-research/google-research/tree/master/albert)** (from Google Research and the Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. 11. Want to contribute a new model? We have added a **detailed guide and templates** to guide you in the process of adding a new model. You can find them in the [`templates`](./templates) folder of the repository. Be sure to check the [contributing guidelines](./CONTRIBUTING.md) and contact the maintainers or open an issue to collect feedbacks before starting your PR. These implementations have been tested on several datasets (see the example scripts) and should match the performances of the original implementations (e.g. ~93 F1 on SQuAD for BERT Whole-Word-Masking, ~88 F1 on RocStories for OpenAI GPT, ~18.3 perplexity on WikiText 103 for Transformer-XL, ~0.916 Peason R coefficient on STS-B for XLNet). You can find more details on the performances in the Examples section of the [documentation](https://huggingface.co/transformers/examples.html). diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst index d017a3be6c..a6bf508d18 100644 --- a/docs/source/pretrained_models.rst +++ b/docs/source/pretrained_models.rst @@ -162,31 +162,31 @@ Here is the full list of the currently provided pretrained models together with | ALBERT | ``albert-base-v1`` | | 12 repeating layers, 128 embedding, 768-hidden, 12-heads, 11M parameters | | | | | ALBERT base model | | | | (see `details `__) | -+--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``albert-large-v1`` | | 24 repeating layers, 128 embedding, 1024-hidden, 16-heads, 17M parameters | | | | | ALBERT large model | | | | (see `details `__) | -+--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``albert-xlarge-v1`` | | 24 repeating layers, 128 embedding, 2048-hidden, 16-heads, 58M parameters | | | | | ALBERT xlarge model | | | | (see `details `__) | -+--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``albert-xxlarge-v1`` | | 12 repeating layer, 128 embedding, 4096-hidden, 64-heads, 223M parameters | | | | | ALBERT xxlarge model | | | | (see `details `__) | -+--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``albert-base-v2`` | | 12 repeating layers, 128 embedding, 768-hidden, 12-heads, 11M parameters | | | | | ALBERT base model with no dropout, additional training data and longer training | | | | (see `details `__) | -+--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``albert-large-v2`` | | 24 repeating layers, 128 embedding, 1024-hidden, 16-heads, 17M parameters | | | | | ALBERT large model with no dropout, additional training data and longer training | | | | (see `details `__) | -+--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``albert-xlarge-v2`` | | 24 repeating layers, 128 embedding, 2048-hidden, 16-heads, 58M parameters | | | | | ALBERT xlarge model with no dropout, additional training data and longer training | | | | (see `details `__) | -+--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``albert-xxlarge-v2`` | | 12 repeating layer, 128 embedding, 4096-hidden, 64-heads, 223M parameters | | | | | ALBERT xxlarge model with no dropout, additional training data and longer training | | | | (see `details `__) | From ce02550d50f2074d54d58b37cbc9845d9a159818 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 26 Nov 2019 15:47:02 -0500 Subject: [PATCH 130/505] Fix pretrained models table --- docs/source/pretrained_models.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst index a6bf508d18..a312346df7 100644 --- a/docs/source/pretrained_models.rst +++ b/docs/source/pretrained_models.rst @@ -190,7 +190,7 @@ Here is the full list of the currently provided pretrained models together with | | ``albert-xxlarge-v2`` | | 12 repeating layer, 128 embedding, 4096-hidden, 64-heads, 223M parameters | | | | | ALBERT xxlarge model with no dropout, additional training data and longer training | | | | (see `details `__) | -+--------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ ++-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ .. `__ From cc7968227e08858df4a5c618c739e1a3ca050196 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 26 Nov 2019 15:52:25 -0500 Subject: [PATCH 131/505] Updated v2.2.0 doc --- .circleci/deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/deploy.sh b/.circleci/deploy.sh index 8081153b9f..61b9f90909 100755 --- a/.circleci/deploy.sh +++ b/.circleci/deploy.sh @@ -23,4 +23,4 @@ deploy_doc "fe02e45" v1.1.0 deploy_doc "89fd345" v1.2.0 deploy_doc "fc9faa8" v2.0.0 deploy_doc "3ddce1d" v2.1.1 -deploy_doc "ee4647b" v2.2.0 +deploy_doc "ce02550" v2.2.0 From 361620954acf16b27727d763a591257b03f90b5d Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 27 Nov 2019 10:11:37 -0500 Subject: [PATCH 132/505] Remove TFBertForPreTraining from ALBERT doc --- docs/source/model_doc/albert.rst | 7 ------- 1 file changed, 7 deletions(-) diff --git a/docs/source/model_doc/albert.rst b/docs/source/model_doc/albert.rst index cf52f35eb6..92970c9328 100644 --- a/docs/source/model_doc/albert.rst +++ b/docs/source/model_doc/albert.rst @@ -50,13 +50,6 @@ ALBERT :members: -``TFBertForPreTraining`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. autoclass:: transformers.TFBertForPreTraining - :members: - - ``TFAlbertForMaskedLM`` ~~~~~~~~~~~~~~~~~~~~~~~~~~ From 45d767297a78697e4ff75ec3d83bdf35e43e4fff Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 27 Nov 2019 10:12:20 -0500 Subject: [PATCH 133/505] Updated v2.2.0 doc --- .circleci/deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/deploy.sh b/.circleci/deploy.sh index 61b9f90909..a32581baef 100755 --- a/.circleci/deploy.sh +++ b/.circleci/deploy.sh @@ -23,4 +23,4 @@ deploy_doc "fe02e45" v1.1.0 deploy_doc "89fd345" v1.2.0 deploy_doc "fc9faa8" v2.0.0 deploy_doc "3ddce1d" v2.1.1 -deploy_doc "ce02550" v2.2.0 +deploy_doc "3616209" v2.2.0 From 88b317739fe56888528c857fc8e90967148a0051 Mon Sep 17 00:00:00 2001 From: root Date: Wed, 27 Nov 2019 09:58:38 +0000 Subject: [PATCH 134/505] Fix issue: #1962, input's shape seem to cause error in 2.2.0 version tf_albert_model --- transformers/modeling_tf_albert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/modeling_tf_albert.py b/transformers/modeling_tf_albert.py index ee122205b9..b2bf66f750 100644 --- a/transformers/modeling_tf_albert.py +++ b/transformers/modeling_tf_albert.py @@ -645,7 +645,7 @@ class TFAlbertModel(TFAlbertPreTrainedModel): if input_ids is not None and inputs_embeds is not None: raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") elif input_ids is not None: - input_shape = input_ids.shape + input_shape = tf.shape(input_ids) elif inputs_embeds is not None: input_shape = inputs_embeds.shape[:-1] else: From de2696f68e20019fef3a5e1b54de10351abb4145 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Louf?= Date: Tue, 26 Nov 2019 16:30:27 +0100 Subject: [PATCH 135/505] suggest to track repo w/ https rather than ssh --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 136ef8df81..8228dd59d8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -106,7 +106,7 @@ Follow these steps to start contributing: ```bash $ git clone git@github.com:/transformers.git $ cd transformers - $ git remote add upstream git@github.com:huggingface/transformers.git + $ git remote add upstream https://github.com/huggingface/transformers.git ``` 3. Create a new branch to hold your development changes: From b5d884d25c96069962030b9e4c9dfb4b1f2f6fa0 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Wed, 27 Nov 2019 11:05:18 -0500 Subject: [PATCH 136/505] Uniformize #1952 --- README.md | 2 +- examples/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index dd2c865036..dd20b80590 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ Examples are included in the repository but are not shipped with the library. Therefore, in order to run the latest versions of the examples you also need to install from source. To do so, create a new virtual environment and follow these steps: ```bash -git clone git@github.com:huggingface/transformers +git clone https://github.com/huggingface/transformers cd transformers pip install [--editable] . ``` diff --git a/examples/README.md b/examples/README.md index 0c57990ea7..e109a12171 100644 --- a/examples/README.md +++ b/examples/README.md @@ -7,7 +7,7 @@ similar API between the different models. To run the latest versions of the examples, you have to install from source. Execute the following steps in a new virtual environment: ```bash -git clone git@github.com:huggingface/transformers +git clone https://github.com/huggingface/transformers cd transformers pip install [--editable] . ``` From 71f71ddb3e04aa18aaa1e3633a2881493e67b438 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 29 Oct 2019 11:50:42 -0400 Subject: [PATCH 137/505] run_xnli + utils_xnli --- examples/run_xnli.py | 534 +++++++++++++++++++++++++++++++++++++++++ examples/utils_xnli.py | 93 +++++++ 2 files changed, 627 insertions(+) create mode 100644 examples/run_xnli.py create mode 100644 examples/utils_xnli.py diff --git a/examples/run_xnli.py b/examples/run_xnli.py new file mode 100644 index 0000000000..ee37296832 --- /dev/null +++ b/examples/run_xnli.py @@ -0,0 +1,534 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Finetuning multi-lingual models on XNLI (Bert, XLM). + Adapted from `examples/run_glue.py`""" + +from __future__ import absolute_import, division, print_function + +import argparse +import glob +import logging +import os +import random + +import numpy as np +import torch +from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler, + TensorDataset) +from torch.utils.data.distributed import DistributedSampler + +try: + from torch.utils.tensorboard import SummaryWriter +except: + from tensorboardX import SummaryWriter + +from tqdm import tqdm, trange + +from transformers import (WEIGHTS_NAME, + BertConfig, BertForSequenceClassification, BertTokenizer, + XLMConfig, XLMForSequenceClassification, XLMTokenizer, + DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer) + +from transformers import AdamW, WarmupLinearSchedule + +from utils_xnli import xnli_compute_metrics as compute_metrics +from utils_xnli import xnli_output_modes as output_modes +from utils_xnli import xnli_processors as processors + +from transformers import glue_convert_examples_to_features as convert_examples_to_features + +logger = logging.getLogger(__name__) + +ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, XLMConfig)), ()) + +MODEL_CLASSES = { + 'bert': (BertConfig, BertForSequenceClassification, BertTokenizer), + 'xlm': (XLMConfig, XLMForSequenceClassification, XLMTokenizer), + # 'distilbert': (DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer) +} + + +def set_seed(args): + random.seed(args.seed) + np.random.seed(args.seed) + torch.manual_seed(args.seed) + if args.n_gpu > 0: + torch.cuda.manual_seed_all(args.seed) + + +def train(args, train_dataset, model, tokenizer): + """ Train the model """ + if args.local_rank in [-1, 0]: + tb_writer = SummaryWriter() + + args.train_batch_size = args.per_gpu_train_batch_size * max(1, args.n_gpu) + train_sampler = RandomSampler(train_dataset) if args.local_rank == -1 else DistributedSampler(train_dataset) + train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=args.train_batch_size) + + if args.max_steps > 0: + t_total = args.max_steps + args.num_train_epochs = args.max_steps // (len(train_dataloader) // args.gradient_accumulation_steps) + 1 + else: + t_total = len(train_dataloader) // args.gradient_accumulation_steps * args.num_train_epochs + + # Prepare optimizer and schedule (linear warmup and decay) + no_decay = ['bias', 'LayerNorm.weight'] + optimizer_grouped_parameters = [ + {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': args.weight_decay}, + {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} + ] + optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) + scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total) + if args.fp16: + try: + from apex import amp + except ImportError: + raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use fp16 training.") + model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level) + + # multi-gpu training (should be after apex fp16 initialization) + if args.n_gpu > 1: + model = torch.nn.DataParallel(model) + + # Distributed training (should be after apex fp16 initialization) + if args.local_rank != -1: + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank], + output_device=args.local_rank, + find_unused_parameters=True) + + # Train! + logger.info("***** Running training *****") + logger.info(" Num examples = %d", len(train_dataset)) + logger.info(" Num Epochs = %d", args.num_train_epochs) + logger.info(" Instantaneous batch size per GPU = %d", args.per_gpu_train_batch_size) + logger.info(" Total train batch size (w. parallel, distributed & accumulation) = %d", + args.train_batch_size * args.gradient_accumulation_steps * (torch.distributed.get_world_size() if args.local_rank != -1 else 1)) + logger.info(" Gradient Accumulation steps = %d", args.gradient_accumulation_steps) + logger.info(" Total optimization steps = %d", t_total) + + global_step = 0 + tr_loss, logging_loss = 0.0, 0.0 + model.zero_grad() + train_iterator = trange(int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0]) + set_seed(args) # Added here for reproductibility (even between python 2 and 3) + for _ in train_iterator: + epoch_iterator = tqdm(train_dataloader, desc="Iteration", disable=args.local_rank not in [-1, 0]) + for step, batch in enumerate(epoch_iterator): + model.train() + batch = tuple(t.to(args.device) for t in batch) + inputs = {'input_ids': batch[0], + 'attention_mask': batch[1], + 'labels': batch[3]} + if args.model_type != 'distilbert': + inputs['token_type_ids'] = batch[2] if args.model_type in ['bert'] else None # XLM and DistilBERT don't use segment_ids + outputs = model(**inputs) + loss = outputs[0] # model outputs are always tuple in transformers (see doc) + + if args.n_gpu > 1: + loss = loss.mean() # mean() to average on multi-gpu parallel training + if args.gradient_accumulation_steps > 1: + loss = loss / args.gradient_accumulation_steps + + if args.fp16: + with amp.scale_loss(loss, optimizer) as scaled_loss: + scaled_loss.backward() + else: + loss.backward() + + tr_loss += loss.item() + if (step + 1) % args.gradient_accumulation_steps == 0 and not args.tpu: + if args.fp16: + torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm) + else: + torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm) + + optimizer.step() + scheduler.step() # Update learning rate schedule + model.zero_grad() + global_step += 1 + + if args.local_rank in [-1, 0] and args.logging_steps > 0 and global_step % args.logging_steps == 0: + # Log metrics + if args.local_rank == -1 and args.evaluate_during_training: # Only evaluate when single GPU otherwise metrics may not average well + results = evaluate(args, model, tokenizer) + for key, value in results.items(): + tb_writer.add_scalar('eval_{}'.format(key), value, global_step) + tb_writer.add_scalar('lr', scheduler.get_lr()[0], global_step) + tb_writer.add_scalar('loss', (tr_loss - logging_loss)/args.logging_steps, global_step) + logging_loss = tr_loss + + if args.local_rank in [-1, 0] and args.save_steps > 0 and global_step % args.save_steps == 0: + # Save model checkpoint + output_dir = os.path.join(args.output_dir, 'checkpoint-{}'.format(global_step)) + if not os.path.exists(output_dir): + os.makedirs(output_dir) + model_to_save = model.module if hasattr(model, 'module') else model # Take care of distributed/parallel training + model_to_save.save_pretrained(output_dir) + torch.save(args, os.path.join(output_dir, 'training_args.bin')) + logger.info("Saving model checkpoint to %s", output_dir) + + if args.tpu: + args.xla_model.optimizer_step(optimizer, barrier=True) + model.zero_grad() + global_step += 1 + + if args.max_steps > 0 and global_step > args.max_steps: + epoch_iterator.close() + break + if args.max_steps > 0 and global_step > args.max_steps: + train_iterator.close() + break + + if args.local_rank in [-1, 0]: + tb_writer.close() + + return global_step, tr_loss / global_step + + +def evaluate(args, model, tokenizer, prefix=""): + eval_task_names = (args.task_name,) + eval_outputs_dirs = (args.output_dir,) + + results = {} + for eval_task, eval_output_dir in zip(eval_task_names, eval_outputs_dirs): + eval_dataset = load_and_cache_examples(args, eval_task, tokenizer, evaluate=True) + + if not os.path.exists(eval_output_dir) and args.local_rank in [-1, 0]: + os.makedirs(eval_output_dir) + + args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu) + # Note that DistributedSampler samples randomly + eval_sampler = SequentialSampler(eval_dataset) if args.local_rank == -1 else DistributedSampler(eval_dataset) + eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size) + + # Eval! + logger.info("***** Running evaluation {} *****".format(prefix)) + logger.info(" Num examples = %d", len(eval_dataset)) + logger.info(" Batch size = %d", args.eval_batch_size) + eval_loss = 0.0 + nb_eval_steps = 0 + preds = None + out_label_ids = None + for batch in tqdm(eval_dataloader, desc="Evaluating"): + model.eval() + batch = tuple(t.to(args.device) for t in batch) + + with torch.no_grad(): + inputs = {'input_ids': batch[0], + 'attention_mask': batch[1], + 'labels': batch[3]} + if args.model_type != 'distilbert': + inputs['token_type_ids'] = batch[2] if args.model_type in ['bert'] else None # XLM and DistilBERT don't use segment_ids + outputs = model(**inputs) + tmp_eval_loss, logits = outputs[:2] + + eval_loss += tmp_eval_loss.mean().item() + nb_eval_steps += 1 + if preds is None: + preds = logits.detach().cpu().numpy() + out_label_ids = inputs['labels'].detach().cpu().numpy() + else: + preds = np.append(preds, logits.detach().cpu().numpy(), axis=0) + out_label_ids = np.append(out_label_ids, inputs['labels'].detach().cpu().numpy(), axis=0) + + eval_loss = eval_loss / nb_eval_steps + if args.output_mode == "classification": + preds = np.argmax(preds, axis=1) + elif args.output_mode == "regression": + preds = np.squeeze(preds) + result = compute_metrics(eval_task, preds, out_label_ids) + results.update(result) + + output_eval_file = os.path.join(eval_output_dir, prefix, "eval_results.txt") + with open(output_eval_file, "w") as writer: + logger.info("***** Eval results {} *****".format(prefix)) + for key in sorted(result.keys()): + logger.info(" %s = %s", key, str(result[key])) + writer.write("%s = %s\n" % (key, str(result[key]))) + + return results + + +def load_and_cache_examples(args, task, tokenizer, evaluate=False): + if args.local_rank not in [-1, 0] and not evaluate: + torch.distributed.barrier() # Make sure only the first process in distributed training process the dataset, and the others will use the cache + + processor = processors[task](language=args.language, train_language=args.train_language) + output_mode = output_modes[task] + # Load data features from cache or dataset file + cached_features_file = os.path.join(args.data_dir, 'cached_{}_{}_{}_{}_{}'.format( + 'dev' if evaluate else 'train', + list(filter(None, args.model_name_or_path.split('/'))).pop(), + str(args.max_seq_length), + str(task), + str(args.train_language if (not evaluate and args.train_language is not None) else args.language))) + if os.path.exists(cached_features_file) and not args.overwrite_cache: + logger.info("Loading features from cached file %s", cached_features_file) + features = torch.load(cached_features_file) + else: + logger.info("Creating features from dataset file at %s", args.data_dir) + label_list = processor.get_labels() + examples = processor.get_dev_examples(args.data_dir) if evaluate else processor.get_train_examples(args.data_dir) + features = convert_examples_to_features(examples, + tokenizer, + label_list=label_list, + max_length=args.max_seq_length, + output_mode=output_mode, + pad_on_left=False, + pad_token=tokenizer.convert_tokens_to_ids([tokenizer.pad_token])[0], + pad_token_segment_id=0, + ) + if args.local_rank in [-1, 0]: + logger.info("Saving features into cached file %s", cached_features_file) + torch.save(features, cached_features_file) + + if args.local_rank == 0 and not evaluate: + torch.distributed.barrier() # Make sure only the first process in distributed training process the dataset, and the others will use the cache + + # Convert to Tensors and build dataset + all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long) + all_attention_mask = torch.tensor([f.attention_mask for f in features], dtype=torch.long) + all_token_type_ids = torch.tensor([f.token_type_ids for f in features], dtype=torch.long) + if output_mode == "classification": + all_labels = torch.tensor([f.label for f in features], dtype=torch.long) + else: + raise ValueError(f'No other `output_mode` for XNLI.') + + dataset = TensorDataset(all_input_ids, all_attention_mask, all_token_type_ids, all_labels) + return dataset + + +def main(): + parser = argparse.ArgumentParser() + + ## Required parameters + parser.add_argument("--data_dir", default=None, type=str, required=True, + help="The input data dir. Should contain the .tsv files (or other data files) for the task.") + parser.add_argument("--model_type", default=None, type=str, required=True, + help="Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys())) + parser.add_argument("--model_name_or_path", default=None, type=str, required=True, + help="Path to pre-trained model or shortcut name selected in the list: " + ", ".join(ALL_MODELS)) + parser.add_argument("--language", default=None, type=str, required=True, + help="Evaluation language. Also train language if `train_language` is set to None.") + parser.add_argument("--train_language", default=None, type=str, + help="Train language if is different of the evaluation language.") + parser.add_argument("--output_dir", default=None, type=str, required=True, + help="The output directory where the model predictions and checkpoints will be written.") + + ## Other parameters + parser.add_argument("--config_name", default="", type=str, + help="Pretrained config name or path if not the same as model_name") + parser.add_argument("--tokenizer_name", default="", type=str, + help="Pretrained tokenizer name or path if not the same as model_name") + parser.add_argument("--cache_dir", default="", type=str, + help="Where do you want to store the pre-trained models downloaded from s3") + parser.add_argument("--max_seq_length", default=128, type=int, + help="The maximum total input sequence length after tokenization. Sequences longer " + "than this will be truncated, sequences shorter will be padded.") + parser.add_argument("--do_train", action='store_true', + help="Whether to run training.") + parser.add_argument("--do_eval", action='store_true', + help="Whether to run eval on the dev set.") + parser.add_argument("--evaluate_during_training", action='store_true', + help="Rul evaluation during training at each logging step.") + parser.add_argument("--do_lower_case", action='store_true', + help="Set this flag if you are using an uncased model.") + + parser.add_argument("--per_gpu_train_batch_size", default=8, type=int, + help="Batch size per GPU/CPU for training.") + parser.add_argument("--per_gpu_eval_batch_size", default=8, type=int, + help="Batch size per GPU/CPU for evaluation.") + parser.add_argument('--gradient_accumulation_steps', type=int, default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.") + parser.add_argument("--learning_rate", default=5e-5, type=float, + help="The initial learning rate for Adam.") + parser.add_argument("--weight_decay", default=0.0, type=float, + help="Weight deay if we apply some.") + parser.add_argument("--adam_epsilon", default=1e-8, type=float, + help="Epsilon for Adam optimizer.") + parser.add_argument("--max_grad_norm", default=1.0, type=float, + help="Max gradient norm.") + parser.add_argument("--num_train_epochs", default=3.0, type=float, + help="Total number of training epochs to perform.") + parser.add_argument("--max_steps", default=-1, type=int, + help="If > 0: set total number of training steps to perform. Override num_train_epochs.") + parser.add_argument("--warmup_steps", default=0, type=int, + help="Linear warmup over warmup_steps.") + + parser.add_argument('--logging_steps', type=int, default=50, + help="Log every X updates steps.") + parser.add_argument('--save_steps', type=int, default=50, + help="Save checkpoint every X updates steps.") + parser.add_argument("--eval_all_checkpoints", action='store_true', + help="Evaluate all checkpoints starting with the same prefix as model_name ending and ending with step number") + parser.add_argument("--no_cuda", action='store_true', + help="Avoid using CUDA when available") + parser.add_argument('--overwrite_output_dir', action='store_true', + help="Overwrite the content of the output directory") + parser.add_argument('--overwrite_cache', action='store_true', + help="Overwrite the cached training and evaluation sets") + parser.add_argument('--seed', type=int, default=42, + help="random seed for initialization") + + parser.add_argument('--tpu', action='store_true', + help="Whether to run on the TPU defined in the environment variables") + parser.add_argument('--tpu_ip_address', type=str, default='', + help="TPU IP address if none are set in the environment variables") + parser.add_argument('--tpu_name', type=str, default='', + help="TPU name if none are set in the environment variables") + parser.add_argument('--xrt_tpu_config', type=str, default='', + help="XRT TPU config if none are set in the environment variables") + + parser.add_argument('--fp16', action='store_true', + help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit") + parser.add_argument('--fp16_opt_level', type=str, default='O1', + help="For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']." + "See details at https://nvidia.github.io/apex/amp.html") + parser.add_argument("--local_rank", type=int, default=-1, + help="For distributed training: local_rank") + parser.add_argument('--server_ip', type=str, default='', help="For distant debugging.") + parser.add_argument('--server_port', type=str, default='', help="For distant debugging.") + args = parser.parse_args() + + if os.path.exists(args.output_dir) and os.listdir(args.output_dir) and args.do_train and not args.overwrite_output_dir: + raise ValueError("Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.".format(args.output_dir)) + + # Setup distant debugging if needed + if args.server_ip and args.server_port: + # Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script + import ptvsd + print("Waiting for debugger attach") + ptvsd.enable_attach(address=(args.server_ip, args.server_port), redirect_output=True) + ptvsd.wait_for_attach() + + # Setup CUDA, GPU & distributed training + if args.local_rank == -1 or args.no_cuda: + device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") + args.n_gpu = torch.cuda.device_count() + else: # Initializes the distributed backend which will take care of sychronizing nodes/GPUs + torch.cuda.set_device(args.local_rank) + device = torch.device("cuda", args.local_rank) + torch.distributed.init_process_group(backend='nccl') + args.n_gpu = 1 + args.device = device + + if args.tpu: + if args.tpu_ip_address: + os.environ["TPU_IP_ADDRESS"] = args.tpu_ip_address + if args.tpu_name: + os.environ["TPU_NAME"] = args.tpu_name + if args.xrt_tpu_config: + os.environ["XRT_TPU_CONFIG"] = args.xrt_tpu_config + + assert "TPU_IP_ADDRESS" in os.environ + assert "TPU_NAME" in os.environ + assert "XRT_TPU_CONFIG" in os.environ + + import torch_xla + import torch_xla.core.xla_model as xm + args.device = xm.xla_device() + args.xla_model = xm + + # Setup logging + logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', + datefmt = '%m/%d/%Y %H:%M:%S', + level = logging.INFO if args.local_rank in [-1, 0] else logging.WARN) + logger.warning("Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s", + args.local_rank, device, args.n_gpu, bool(args.local_rank != -1), args.fp16) + + # Set seed + set_seed(args) + + # Prepare XNLI task + args.task_name = 'xnli' + if args.task_name not in processors: + raise ValueError("Task not found: %s" % (args.task_name)) + processor = processors[args.task_name](language=args.language, train_language=args.train_language) + args.output_mode = output_modes[args.task_name] + label_list = processor.get_labels() + num_labels = len(label_list) + + # Load pretrained model and tokenizer + if args.local_rank not in [-1, 0]: + torch.distributed.barrier() # Make sure only the first process in distributed training will download model & vocab + + args.model_type = args.model_type.lower() + config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type] + config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path, num_labels=num_labels, finetuning_task=args.task_name) + tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path, do_lower_case=args.do_lower_case) + model = model_class.from_pretrained(args.model_name_or_path, from_tf=bool('.ckpt' in args.model_name_or_path), config=config) + + if args.local_rank == 0: + torch.distributed.barrier() # Make sure only the first process in distributed training will download model & vocab + + model.to(args.device) + + logger.info("Training/evaluation parameters %s", args) + + + # Training + if args.do_train: + train_dataset = load_and_cache_examples(args, args.task_name, tokenizer, evaluate=False) + global_step, tr_loss = train(args, train_dataset, model, tokenizer) + logger.info(" global_step = %s, average loss = %s", global_step, tr_loss) + + + # Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained() + if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0) and not args.tpu: + # Create output directory if needed + if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]: + os.makedirs(args.output_dir) + + logger.info("Saving model checkpoint to %s", args.output_dir) + # Save a trained model, configuration and tokenizer using `save_pretrained()`. + # They can then be reloaded using `from_pretrained()` + model_to_save = model.module if hasattr(model, 'module') else model # Take care of distributed/parallel training + model_to_save.save_pretrained(args.output_dir) + tokenizer.save_pretrained(args.output_dir) + + # Good practice: save your training arguments together with the trained model + torch.save(args, os.path.join(args.output_dir, 'training_args.bin')) + + # Load a trained model and vocabulary that you have fine-tuned + model = model_class.from_pretrained(args.output_dir) + tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case) + model.to(args.device) + + + # Evaluation + results = {} + if args.do_eval and args.local_rank in [-1, 0]: + tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case) + checkpoints = [args.output_dir] + if args.eval_all_checkpoints: + checkpoints = list(os.path.dirname(c) for c in sorted(glob.glob(args.output_dir + '/**/' + WEIGHTS_NAME, recursive=True))) + logging.getLogger("transformers.modeling_utils").setLevel(logging.WARN) # Reduce logging + logger.info("Evaluate the following checkpoints: %s", checkpoints) + for checkpoint in checkpoints: + global_step = checkpoint.split('-')[-1] if len(checkpoints) > 1 else "" + prefix = checkpoint.split('/')[-1] if checkpoint.find('checkpoint') != -1 else "" + + model = model_class.from_pretrained(checkpoint) + model.to(args.device) + result = evaluate(args, model, tokenizer, prefix=prefix) + result = dict((k + '_{}'.format(global_step), v) for k, v in result.items()) + results.update(result) + + return results + + +if __name__ == "__main__": + main() diff --git a/examples/utils_xnli.py b/examples/utils_xnli.py new file mode 100644 index 0000000000..f0238f4664 --- /dev/null +++ b/examples/utils_xnli.py @@ -0,0 +1,93 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" XNLI utils (dataset loading and evaluation) """ + +from __future__ import absolute_import, division, print_function + +import logging +import os + +from transformers.data.processors import DataProcessor, InputExample +from transformers.data.metrics import simple_accuracy + +logger = logging.getLogger(__name__) + +class XnliProcessor(DataProcessor): + """Processor for the XNLI dataset. + Adapted from https://github.com/google-research/bert/blob/f39e881b169b9d53bea03d2d341b31707a6c052b/run_classifier.py#L207""" + + def __init__(self, language, train_language = None): + self.language = language + self.train_language = train_language + + def get_train_examples(self, data_dir): + """See base class.""" + lg = self.language if self.train_language is None else self.train_language + lines = self._read_tsv(os.path.join(data_dir, f"XNLI-MT-1.0/multinli/multinli.train.{lg}.tsv")) + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + guid = "%s-%s" % ('train', i) + text_a = line[0] + text_b = line[1] + label = "contradiction" if line[2] == "contradictory" else line[2] + assert isinstance(text_a, str) and isinstance(text_b, str) and isinstance(label, str) + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + def get_dev_examples(self, data_dir): + """See base class.""" + lines = self._read_tsv(os.path.join(data_dir, "XNLI-1.0/xnli.dev.tsv")) + examples = [] + for (i, line) in enumerate(lines): + if i == 0: + continue + language = line[0] + if language != self.language: + continue + guid = "%s-%s" % ('dev', i) + text_a = line[6] + text_b = line[7] + label = line[1] + assert isinstance(text_a, str) and isinstance(text_b, str) and isinstance(label, str) + examples.append( + InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) + return examples + + def get_labels(self): + """See base class.""" + return ["contradiction", "entailment", "neutral"] + +def xnli_compute_metrics(task_name, preds, labels): + assert len(preds) == len(labels) + if task_name == "xnli": + return {"acc": simple_accuracy(preds, labels)} + else: + raise ValueError(f'{task_name} is not a supported task.') + +xnli_processors = { + "xnli": XnliProcessor, +} + +xnli_output_modes = { + "xnli": "classification", +} + +xnli_tasks_num_labels = { + "xnli": 3, +} From d52e98ff9af4509bb803641f0c0d81d67ce73cc3 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 29 Oct 2019 11:51:15 -0400 Subject: [PATCH 138/505] add xnli examples/README.md --- examples/README.md | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/examples/README.md b/examples/README.md index e109a12171..6f8d6bd26e 100644 --- a/examples/README.md +++ b/examples/README.md @@ -21,6 +21,7 @@ pip install [--editable] . | [SQuAD](#squad) | Using BERT/RoBERTa/XLNet/XLM for question answering, examples with distributed training. | | [Multiple Choice](#multiple-choice) | Examples running BERT/XLNet/RoBERTa on the SWAG/RACE/ARC tasks. | [Named Entity Recognition](#named-entity-recognition) | Using BERT for Named Entity Recognition (NER) on the CoNLL 2003 dataset, examples with distributed training. | +| [XNLI](#xnli) | Examples running BERT/XLM on the XNLI benchmark. | | [Abstractive summarization](#abstractive-summarization) | Fine-tuning the library models for abstractive summarization tasks on the CNN/Daily Mail dataset. | ## TensorFlow 2.0 Bert models on GLUE @@ -600,3 +601,42 @@ python run_summarization_finetuning.py \ --do_train \ --data_path=$DATA_PATH \ ``` + +## XNLI + +Based on the script [`run_xnli.py`](TODO). + +[XNLI](https://www.nyu.edu/projects/bowman/xnli/) is crowd-sourced dataset based on [MultiNLI](http://www.nyu.edu/projects/bowman/multinli/). It is an evaluation benchmark for cross-lingual text representations. Pairs of text are labeled with textual entailment annotations for 15 different languages (including both high-ressource language such as English and low-ressource languages such as Swahili). + +#### Fine-tuning on XNLI + +This example code fine-tunes mBERT (multi-lingual BERT) on the XNLI dataset. It runs in TODO min +on a single tesla V100 16GB. The data for XNLI can be downloaded with the following links and should be both saved (and un-zipped) in a +`$XNLI_DIR` directory. + +* [XNLI 1.0](https://www.nyu.edu/projects/bowman/xnli/XNLI-1.0.zip) +* [XNLI-MT 1.0](https://www.nyu.edu/projects/bowman/xnli/XNLI-MT-1.0.zip) + +```bash +export XNLI_DIR=/path/to/XNLI + +python run_xnli.py \ + --model_type bert \ + --model_name_or_path bert-base-multilingual-cased \ + --language en \ + --train_language en \ + --do_train \ + --do_eval \ + --data_dir $SQUAD_DIR \ + --per_gpu_train_batch_size 32 \ + --learning_rate 5e-5 \ + --num_train_epochs 2.0 \ + --max_seq_length 128 \ + --output_dir /tmp/debug_xnli/ +``` + +Training with the previously defined hyper-parameters yields the following results: + +```bash +TODO +``` From c4336ecbbdbb8bddcbfb31e0fcc4d382b430a9a5 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 29 Oct 2019 12:04:20 -0400 Subject: [PATCH 139/505] xnli - output_mode consistency --- examples/run_xnli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/run_xnli.py b/examples/run_xnli.py index ee37296832..a9b2e46c13 100644 --- a/examples/run_xnli.py +++ b/examples/run_xnli.py @@ -247,8 +247,8 @@ def evaluate(args, model, tokenizer, prefix=""): eval_loss = eval_loss / nb_eval_steps if args.output_mode == "classification": preds = np.argmax(preds, axis=1) - elif args.output_mode == "regression": - preds = np.squeeze(preds) + else: + raise ValueError(f'No other `output_mode` for XNLI.') result = compute_metrics(eval_task, preds, out_label_ids) results.update(result) From 84a0b522cf1b3f69bfbc92eb2309ba5ff652e521 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 29 Oct 2019 18:53:45 -0400 Subject: [PATCH 140/505] mbert reproducibility results --- examples/README.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/examples/README.md b/examples/README.md index 6f8d6bd26e..3e5fd03c45 100644 --- a/examples/README.md +++ b/examples/README.md @@ -604,13 +604,13 @@ python run_summarization_finetuning.py \ ## XNLI -Based on the script [`run_xnli.py`](TODO). +Based on the script [`run_xnli.py`](https://github.com/huggingface/transformers/blob/master/examples/run_xnli.py). [XNLI](https://www.nyu.edu/projects/bowman/xnli/) is crowd-sourced dataset based on [MultiNLI](http://www.nyu.edu/projects/bowman/multinli/). It is an evaluation benchmark for cross-lingual text representations. Pairs of text are labeled with textual entailment annotations for 15 different languages (including both high-ressource language such as English and low-ressource languages such as Swahili). #### Fine-tuning on XNLI -This example code fine-tunes mBERT (multi-lingual BERT) on the XNLI dataset. It runs in TODO min +This example code fine-tunes mBERT (multi-lingual BERT) on the XNLI dataset. It runs in 106 mins on a single tesla V100 16GB. The data for XNLI can be downloaded with the following links and should be both saved (and un-zipped) in a `$XNLI_DIR` directory. @@ -623,20 +623,21 @@ export XNLI_DIR=/path/to/XNLI python run_xnli.py \ --model_type bert \ --model_name_or_path bert-base-multilingual-cased \ - --language en \ + --language es \ --train_language en \ --do_train \ --do_eval \ - --data_dir $SQUAD_DIR \ + --data_dir $XNLI_DIR \ --per_gpu_train_batch_size 32 \ --learning_rate 5e-5 \ --num_train_epochs 2.0 \ --max_seq_length 128 \ - --output_dir /tmp/debug_xnli/ + --output_dir /tmp/debug_xnli/ \ + --save_steps -1 ``` -Training with the previously defined hyper-parameters yields the following results: +Training with the previously defined hyper-parameters yields the following results on the dev set: ```bash -TODO +acc = 0.738152610441767 ``` From cb7b77a8a2ee52812c4358817a6a586f19687cda Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Wed, 30 Oct 2019 18:13:52 -0400 Subject: [PATCH 141/505] fix some typos --- transformers/tokenization_xlm.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/transformers/tokenization_xlm.py b/transformers/tokenization_xlm.py index 01f8721d98..ba994dc356 100644 --- a/transformers/tokenization_xlm.py +++ b/transformers/tokenization_xlm.py @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Tokenization classes for OpenAI GPT.""" +"""Tokenization classes for XLM.""" from __future__ import (absolute_import, division, print_function, unicode_literals) @@ -758,9 +758,9 @@ class XLMTokenizer(PreTrainedTokenizer): """ Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and adding special tokens. - A RoBERTa sequence has the following format: + A XLM sequence has the following format: single sequence: X - pair of sequences: A B + pair of sequences: A B """ if token_ids_1 is None: return [self.cls_token_id] + token_ids_0 + [self.sep_token_id] From 289cf4d2b7cf090942e2e7dc4e6134a81095adf6 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 5 Nov 2019 10:10:02 -0500 Subject: [PATCH 142/505] change default for XNLI: dev --> test --- examples/run_xnli.py | 6 +++--- examples/utils_xnli.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/run_xnli.py b/examples/run_xnli.py index a9b2e46c13..7fbbf7d152 100644 --- a/examples/run_xnli.py +++ b/examples/run_xnli.py @@ -270,7 +270,7 @@ def load_and_cache_examples(args, task, tokenizer, evaluate=False): output_mode = output_modes[task] # Load data features from cache or dataset file cached_features_file = os.path.join(args.data_dir, 'cached_{}_{}_{}_{}_{}'.format( - 'dev' if evaluate else 'train', + 'test' if evaluate else 'train', list(filter(None, args.model_name_or_path.split('/'))).pop(), str(args.max_seq_length), str(task), @@ -281,7 +281,7 @@ def load_and_cache_examples(args, task, tokenizer, evaluate=False): else: logger.info("Creating features from dataset file at %s", args.data_dir) label_list = processor.get_labels() - examples = processor.get_dev_examples(args.data_dir) if evaluate else processor.get_train_examples(args.data_dir) + examples = processor.get_test_examples(args.data_dir) if evaluate else processor.get_train_examples(args.data_dir) features = convert_examples_to_features(examples, tokenizer, label_list=label_list, @@ -341,7 +341,7 @@ def main(): parser.add_argument("--do_train", action='store_true', help="Whether to run training.") parser.add_argument("--do_eval", action='store_true', - help="Whether to run eval on the dev set.") + help="Whether to run eval on the test set.") parser.add_argument("--evaluate_during_training", action='store_true', help="Rul evaluation during training at each logging step.") parser.add_argument("--do_lower_case", action='store_true', diff --git a/examples/utils_xnli.py b/examples/utils_xnli.py index f0238f4664..482e79a81f 100644 --- a/examples/utils_xnli.py +++ b/examples/utils_xnli.py @@ -50,9 +50,9 @@ class XnliProcessor(DataProcessor): InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) return examples - def get_dev_examples(self, data_dir): + def get_test_examples(self, data_dir): """See base class.""" - lines = self._read_tsv(os.path.join(data_dir, "XNLI-1.0/xnli.dev.tsv")) + lines = self._read_tsv(os.path.join(data_dir, "XNLI-1.0/xnli.test.tsv")) examples = [] for (i, line) in enumerate(lines): if i == 0: @@ -60,7 +60,7 @@ class XnliProcessor(DataProcessor): language = line[0] if language != self.language: continue - guid = "%s-%s" % ('dev', i) + guid = "%s-%s" % ('test', i) text_a = line[6] text_b = line[7] label = line[1] From d5910b312fbe02a4b08097646e46399dac179084 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 5 Nov 2019 10:21:25 -0500 Subject: [PATCH 143/505] move xnli processor (and utils) to transformers/data/processors --- examples/run_xnli.py | 6 +++--- transformers/__init__.py | 3 ++- transformers/data/__init__.py | 1 + transformers/data/processors/__init__.py | 2 +- .../utils_xnli.py => transformers/data/processors/xnli.py | 2 +- 5 files changed, 8 insertions(+), 6 deletions(-) rename examples/utils_xnli.py => transformers/data/processors/xnli.py (97%) diff --git a/examples/run_xnli.py b/examples/run_xnli.py index 7fbbf7d152..0c1743e6c4 100644 --- a/examples/run_xnli.py +++ b/examples/run_xnli.py @@ -44,9 +44,9 @@ from transformers import (WEIGHTS_NAME, from transformers import AdamW, WarmupLinearSchedule -from utils_xnli import xnli_compute_metrics as compute_metrics -from utils_xnli import xnli_output_modes as output_modes -from utils_xnli import xnli_processors as processors +from transformers import xnli_compute_metrics as compute_metrics +from transformers import xnli_output_modes as output_modes +from transformers import xnli_processors as processors from transformers import glue_convert_examples_to_features as convert_examples_to_features diff --git a/transformers/__init__.py b/transformers/__init__.py index dff2479d4b..a133425a9c 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -25,7 +25,8 @@ from .file_utils import (TRANSFORMERS_CACHE, PYTORCH_TRANSFORMERS_CACHE, PYTORCH from .data import (is_sklearn_available, InputExample, InputFeatures, DataProcessor, glue_output_modes, glue_convert_examples_to_features, - glue_processors, glue_tasks_num_labels) + glue_processors, glue_tasks_num_labels, + xnli_output_modes, xnli_processors, xnli_tasks_num_labels) if is_sklearn_available(): from .data import glue_compute_metrics diff --git a/transformers/data/__init__.py b/transformers/data/__init__.py index e910d6da2e..46615608a4 100644 --- a/transformers/data/__init__.py +++ b/transformers/data/__init__.py @@ -1,5 +1,6 @@ from .processors import InputExample, InputFeatures, DataProcessor from .processors import glue_output_modes, glue_processors, glue_tasks_num_labels, glue_convert_examples_to_features +from .processors import xnli_output_modes, xnli_processors, xnli_tasks_num_labels from .metrics import is_sklearn_available if is_sklearn_available(): diff --git a/transformers/data/processors/__init__.py b/transformers/data/processors/__init__.py index af38c54beb..1c41553ba4 100644 --- a/transformers/data/processors/__init__.py +++ b/transformers/data/processors/__init__.py @@ -1,3 +1,3 @@ from .utils import InputExample, InputFeatures, DataProcessor from .glue import glue_output_modes, glue_processors, glue_tasks_num_labels, glue_convert_examples_to_features - +from .xnli import xnli_output_modes, xnli_processors, xnli_tasks_num_labels diff --git a/examples/utils_xnli.py b/transformers/data/processors/xnli.py similarity index 97% rename from examples/utils_xnli.py rename to transformers/data/processors/xnli.py index 482e79a81f..96fac761a9 100644 --- a/examples/utils_xnli.py +++ b/transformers/data/processors/xnli.py @@ -20,7 +20,7 @@ from __future__ import absolute_import, division, print_function import logging import os -from transformers.data.processors import DataProcessor, InputExample +from .utils import DataProcessor, InputExample from transformers.data.metrics import simple_accuracy logger = logging.getLogger(__name__) From d75d49a51de28df48b39280b2f06df78d3fa04bf Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 5 Nov 2019 10:32:20 -0500 Subject: [PATCH 144/505] add XnliProcessor to doc --- docs/source/main_classes/processors.rst | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/docs/source/main_classes/processors.rst b/docs/source/main_classes/processors.rst index a85c126956..a093e621ad 100644 --- a/docs/source/main_classes/processors.rst +++ b/docs/source/main_classes/processors.rst @@ -55,4 +55,27 @@ Example usage ^^^^^^^^^^^^^^^^^^^^^^^^^ An example using these processors is given in the -`run_glue.py `__ script. \ No newline at end of file +`run_glue.py `__ script. + + +XNLI +~~~~~~~~~~~~~~~~~~~~~ + +`The Cross-Lingual NLI Corpus (XNLI) `__ is a benchmark that evaluates +the quality of cross-lingual text representations. +XNLI is crowd-sourced dataset based on `MultiNLI `: pairs of text are labeled with textual entailment +annotations for 15 different languages (including both high-ressource language such as English and low-ressource languages such as Swahili). + +It was released together with the paper +`XNLI: Evaluating Cross-lingual Sentence Representations `__ + +This library hosts the processor to load the XNLI data: + - :class:`~transformers.data.processors.utils.XnliProcessor` + +Please note that since the gold labels are available on the test set, evaluation is performed on the test set. + +Example usage +^^^^^^^^^^^^^^^^^^^^^^^^^ + +An example using these processors is given in the +`run_xnli.py `__ script. \ No newline at end of file From abd397e95483cc7486bed47fd271ab417e1a288e Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 5 Nov 2019 10:57:24 -0500 Subject: [PATCH 145/505] uniformize w/ the cache_dir update --- examples/run_xnli.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/examples/run_xnli.py b/examples/run_xnli.py index 0c1743e6c4..e8457e542b 100644 --- a/examples/run_xnli.py +++ b/examples/run_xnli.py @@ -467,9 +467,17 @@ def main(): args.model_type = args.model_type.lower() config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type] - config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path, num_labels=num_labels, finetuning_task=args.task_name) - tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path, do_lower_case=args.do_lower_case) - model = model_class.from_pretrained(args.model_name_or_path, from_tf=bool('.ckpt' in args.model_name_or_path), config=config) + config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path, + num_labels=num_labels, + finetuning_task=args.task_name, + cache_dir=args.cache_dir if args.cache_dir else None) + tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path, + do_lower_case=args.do_lower_case, + cache_dir=args.cache_dir if args.cache_dir else None) + model = model_class.from_pretrained(args.model_name_or_path, + from_tf=bool('.ckpt' in args.model_name_or_path), + config=config, + cache_dir=args.cache_dir if args.cache_dir else None) if args.local_rank == 0: torch.distributed.barrier() # Make sure only the first process in distributed training will download model & vocab From 3e7656f7ac369de08a2ebac3cd1d6870e60605bc Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 5 Nov 2019 11:58:53 -0500 Subject: [PATCH 146/505] update readme --- examples/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/README.md b/examples/README.md index 3e5fd03c45..960b218f11 100644 --- a/examples/README.md +++ b/examples/README.md @@ -623,7 +623,7 @@ export XNLI_DIR=/path/to/XNLI python run_xnli.py \ --model_type bert \ --model_name_or_path bert-base-multilingual-cased \ - --language es \ + --language de \ --train_language en \ --do_train \ --do_eval \ @@ -636,8 +636,8 @@ python run_xnli.py \ --save_steps -1 ``` -Training with the previously defined hyper-parameters yields the following results on the dev set: +Training with the previously defined hyper-parameters yields the following results on the **test** set: ```bash -acc = 0.738152610441767 +acc = 0.7093812375249501 ``` From 73fe2e7385af4c6062f366825af570d44cd22fd8 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 5 Nov 2019 12:51:43 -0500 Subject: [PATCH 147/505] remove fstrings --- examples/run_xnli.py | 4 ++-- transformers/data/processors/xnli.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/run_xnli.py b/examples/run_xnli.py index e8457e542b..952128f4ad 100644 --- a/examples/run_xnli.py +++ b/examples/run_xnli.py @@ -248,7 +248,7 @@ def evaluate(args, model, tokenizer, prefix=""): if args.output_mode == "classification": preds = np.argmax(preds, axis=1) else: - raise ValueError(f'No other `output_mode` for XNLI.') + raise ValueError('No other `output_mode` for XNLI.') result = compute_metrics(eval_task, preds, out_label_ids) results.update(result) @@ -305,7 +305,7 @@ def load_and_cache_examples(args, task, tokenizer, evaluate=False): if output_mode == "classification": all_labels = torch.tensor([f.label for f in features], dtype=torch.long) else: - raise ValueError(f'No other `output_mode` for XNLI.') + raise ValueError('No other `output_mode` for XNLI.') dataset = TensorDataset(all_input_ids, all_attention_mask, all_token_type_ids, all_labels) return dataset diff --git a/transformers/data/processors/xnli.py b/transformers/data/processors/xnli.py index 96fac761a9..a4807dd901 100644 --- a/transformers/data/processors/xnli.py +++ b/transformers/data/processors/xnli.py @@ -36,7 +36,7 @@ class XnliProcessor(DataProcessor): def get_train_examples(self, data_dir): """See base class.""" lg = self.language if self.train_language is None else self.train_language - lines = self._read_tsv(os.path.join(data_dir, f"XNLI-MT-1.0/multinli/multinli.train.{lg}.tsv")) + lines = self._read_tsv(os.path.join(data_dir, "XNLI-MT-1.0/multinli/multinli.train.{lg}.tsv".format(lg))) examples = [] for (i, line) in enumerate(lines): if i == 0: @@ -78,7 +78,7 @@ def xnli_compute_metrics(task_name, preds, labels): if task_name == "xnli": return {"acc": simple_accuracy(preds, labels)} else: - raise ValueError(f'{task_name} is not a supported task.') + raise ValueError('{} is not a supported task.'.format(task_name)) xnli_processors = { "xnli": XnliProcessor, From bcd8dc6b48a335b899f00b59f016f740c5230d41 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 5 Nov 2019 12:53:08 -0500 Subject: [PATCH 148/505] move xnli_compute_metrics to data/metrics --- transformers/__init__.py | 2 +- transformers/data/__init__.py | 2 +- transformers/data/metrics/__init__.py | 8 ++++++++ transformers/data/processors/xnli.py | 7 ------- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/transformers/__init__.py b/transformers/__init__.py index a133425a9c..b29ad38e73 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -29,7 +29,7 @@ from .data import (is_sklearn_available, xnli_output_modes, xnli_processors, xnli_tasks_num_labels) if is_sklearn_available(): - from .data import glue_compute_metrics + from .data import glue_compute_metrics, xnli_compute_metrics # Tokenizers from .tokenization_utils import (PreTrainedTokenizer) diff --git a/transformers/data/__init__.py b/transformers/data/__init__.py index 46615608a4..b811a35807 100644 --- a/transformers/data/__init__.py +++ b/transformers/data/__init__.py @@ -4,4 +4,4 @@ from .processors import xnli_output_modes, xnli_processors, xnli_tasks_num_label from .metrics import is_sklearn_available if is_sklearn_available(): - from .metrics import glue_compute_metrics + from .metrics import glue_compute_metrics, xnli_compute_metrics diff --git a/transformers/data/metrics/__init__.py b/transformers/data/metrics/__init__.py index c9ebaac38d..5a46eb05d3 100644 --- a/transformers/data/metrics/__init__.py +++ b/transformers/data/metrics/__init__.py @@ -81,3 +81,11 @@ if _has_sklearn: return {"acc": simple_accuracy(preds, labels)} else: raise KeyError(task_name) + + + def xnli_compute_metrics(task_name, preds, labels): + assert len(preds) == len(labels) + if task_name == "xnli": + return {"acc": simple_accuracy(preds, labels)} + else: + raise KeyError(task_name) diff --git a/transformers/data/processors/xnli.py b/transformers/data/processors/xnli.py index a4807dd901..ce582f31a6 100644 --- a/transformers/data/processors/xnli.py +++ b/transformers/data/processors/xnli.py @@ -73,13 +73,6 @@ class XnliProcessor(DataProcessor): """See base class.""" return ["contradiction", "entailment", "neutral"] -def xnli_compute_metrics(task_name, preds, labels): - assert len(preds) == len(labels) - if task_name == "xnli": - return {"acc": simple_accuracy(preds, labels)} - else: - raise ValueError('{} is not a supported task.'.format(task_name)) - xnli_processors = { "xnli": XnliProcessor, } From d47402263964e30ee17cbc06811622bf2df50d6d Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 5 Nov 2019 12:56:03 -0500 Subject: [PATCH 149/505] cleaning simple_accuracy since not used anymore --- transformers/data/processors/xnli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/transformers/data/processors/xnli.py b/transformers/data/processors/xnli.py index ce582f31a6..efbe3762f6 100644 --- a/transformers/data/processors/xnli.py +++ b/transformers/data/processors/xnli.py @@ -21,7 +21,6 @@ import logging import os from .utils import DataProcessor, InputExample -from transformers.data.metrics import simple_accuracy logger = logging.getLogger(__name__) From 07ab8d7af6041f0d3562badcb3d6f173062329a1 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 5 Nov 2019 17:33:14 -0500 Subject: [PATCH 150/505] fix bug --- transformers/data/processors/xnli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/data/processors/xnli.py b/transformers/data/processors/xnli.py index efbe3762f6..958bdf62f9 100644 --- a/transformers/data/processors/xnli.py +++ b/transformers/data/processors/xnli.py @@ -35,7 +35,7 @@ class XnliProcessor(DataProcessor): def get_train_examples(self, data_dir): """See base class.""" lg = self.language if self.train_language is None else self.train_language - lines = self._read_tsv(os.path.join(data_dir, "XNLI-MT-1.0/multinli/multinli.train.{lg}.tsv".format(lg))) + lines = self._read_tsv(os.path.join(data_dir, "XNLI-MT-1.0/multinli/multinli.train.{}.tsv".format(lg))) examples = [] for (i, line) in enumerate(lines): if i == 0: From d5478b939d64db58972e46b7218c765c918b76ac Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Mon, 25 Nov 2019 19:40:48 +0000 Subject: [PATCH 151/505] add distilbert + update run_xnli wrt run_glue --- examples/run_xnli.py | 51 +++++++++++--------------------------------- 1 file changed, 12 insertions(+), 39 deletions(-) diff --git a/examples/run_xnli.py b/examples/run_xnli.py index 952128f4ad..a3bc0d4604 100644 --- a/examples/run_xnli.py +++ b/examples/run_xnli.py @@ -13,7 +13,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" Finetuning multi-lingual models on XNLI (Bert, XLM). +""" Finetuning multi-lingual models on XNLI (Bert, DistilBERT, XLM). Adapted from `examples/run_glue.py`""" from __future__ import absolute_import, division, print_function @@ -42,7 +42,7 @@ from transformers import (WEIGHTS_NAME, XLMConfig, XLMForSequenceClassification, XLMTokenizer, DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer) -from transformers import AdamW, WarmupLinearSchedule +from transformers import AdamW, get_linear_schedule_with_warmup from transformers import xnli_compute_metrics as compute_metrics from transformers import xnli_output_modes as output_modes @@ -52,12 +52,12 @@ from transformers import glue_convert_examples_to_features as convert_examples_t logger = logging.getLogger(__name__) -ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, XLMConfig)), ()) +ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, DistilBertConfig, XLMConfig)), ()) MODEL_CLASSES = { 'bert': (BertConfig, BertForSequenceClassification, BertTokenizer), 'xlm': (XLMConfig, XLMForSequenceClassification, XLMTokenizer), - # 'distilbert': (DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer) + 'distilbert': (DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer) } @@ -91,7 +91,7 @@ def train(args, train_dataset, model, tokenizer): {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} ] optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) - scheduler = WarmupLinearSchedule(optimizer, warmup_steps=args.warmup_steps, t_total=t_total) + scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total) if args.fp16: try: from apex import amp @@ -149,7 +149,7 @@ def train(args, train_dataset, model, tokenizer): loss.backward() tr_loss += loss.item() - if (step + 1) % args.gradient_accumulation_steps == 0 and not args.tpu: + if (step + 1) % args.gradient_accumulation_steps == 0: if args.fp16: torch.nn.utils.clip_grad_norm_(amp.master_params(optimizer), args.max_grad_norm) else: @@ -180,11 +180,6 @@ def train(args, train_dataset, model, tokenizer): torch.save(args, os.path.join(output_dir, 'training_args.bin')) logger.info("Saving model checkpoint to %s", output_dir) - if args.tpu: - args.xla_model.optimizer_step(optimizer, barrier=True) - model.zero_grad() - global_step += 1 - if args.max_steps > 0 and global_step > args.max_steps: epoch_iterator.close() break @@ -214,6 +209,10 @@ def evaluate(args, model, tokenizer, prefix=""): eval_sampler = SequentialSampler(eval_dataset) if args.local_rank == -1 else DistributedSampler(eval_dataset) eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size) + # multi-gpu eval + if args.n_gpu > 1: + model = torch.nn.DataParallel(model) + # Eval! logger.info("***** Running evaluation {} *****".format(prefix)) logger.info(" Num examples = %d", len(eval_dataset)) @@ -383,15 +382,6 @@ def main(): parser.add_argument('--seed', type=int, default=42, help="random seed for initialization") - parser.add_argument('--tpu', action='store_true', - help="Whether to run on the TPU defined in the environment variables") - parser.add_argument('--tpu_ip_address', type=str, default='', - help="TPU IP address if none are set in the environment variables") - parser.add_argument('--tpu_name', type=str, default='', - help="TPU name if none are set in the environment variables") - parser.add_argument('--xrt_tpu_config', type=str, default='', - help="XRT TPU config if none are set in the environment variables") - parser.add_argument('--fp16', action='store_true', help="Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit") parser.add_argument('--fp16_opt_level', type=str, default='O1', @@ -425,23 +415,6 @@ def main(): args.n_gpu = 1 args.device = device - if args.tpu: - if args.tpu_ip_address: - os.environ["TPU_IP_ADDRESS"] = args.tpu_ip_address - if args.tpu_name: - os.environ["TPU_NAME"] = args.tpu_name - if args.xrt_tpu_config: - os.environ["XRT_TPU_CONFIG"] = args.xrt_tpu_config - - assert "TPU_IP_ADDRESS" in os.environ - assert "TPU_NAME" in os.environ - assert "XRT_TPU_CONFIG" in os.environ - - import torch_xla - import torch_xla.core.xla_model as xm - args.device = xm.xla_device() - args.xla_model = xm - # Setup logging logging.basicConfig(format = '%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt = '%m/%d/%Y %H:%M:%S', @@ -495,7 +468,7 @@ def main(): # Saving best-practices: if you use defaults names for the model, you can reload it using from_pretrained() - if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0) and not args.tpu: + if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0): # Create output directory if needed if not os.path.exists(args.output_dir) and args.local_rank in [-1, 0]: os.makedirs(args.output_dir) @@ -512,7 +485,7 @@ def main(): # Load a trained model and vocabulary that you have fine-tuned model = model_class.from_pretrained(args.output_dir) - tokenizer = tokenizer_class.from_pretrained(args.output_dir, do_lower_case=args.do_lower_case) + tokenizer = tokenizer_class.from_pretrained(args.output_dir) model.to(args.device) From 10bd1ddb39235b2f58594e48867595e7d38cd619 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Mon, 25 Nov 2019 19:41:00 +0000 Subject: [PATCH 152/505] soft launch distilbert multilingual --- transformers/configuration_distilbert.py | 3 ++- transformers/modeling_distilbert.py | 3 ++- transformers/tokenization_distilbert.py | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/transformers/configuration_distilbert.py b/transformers/configuration_distilbert.py index 2a8a149acf..f929a9bc39 100644 --- a/transformers/configuration_distilbert.py +++ b/transformers/configuration_distilbert.py @@ -27,7 +27,8 @@ logger = logging.getLogger(__name__) DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = { 'distilbert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-config.json", - 'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-config.json" + 'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-config.json", + 'distilbert-base-multilingual-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-multilingual-cased-config.json", } diff --git a/transformers/modeling_distilbert.py b/transformers/modeling_distilbert.py index d30f493c69..62c623ff6c 100644 --- a/transformers/modeling_distilbert.py +++ b/transformers/modeling_distilbert.py @@ -42,7 +42,8 @@ logger = logging.getLogger(__name__) DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP = { 'distilbert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-pytorch_model.bin", - 'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-pytorch_model.bin" + 'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-pytorch_model.bin", + 'distilbert-base-multilingual-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-multilingual-cased-pytorch_model.bin", } diff --git a/transformers/tokenization_distilbert.py b/transformers/tokenization_distilbert.py index dfa02926d8..832f0c3d0b 100644 --- a/transformers/tokenization_distilbert.py +++ b/transformers/tokenization_distilbert.py @@ -33,12 +33,14 @@ PRETRAINED_VOCAB_FILES_MAP = { { 'distilbert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt", 'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt", + 'distilbert-base-multilingual-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-multilingual-cased-vocab.txt", } } PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { 'distilbert-base-uncased': 512, 'distilbert-base-uncased-distilled-squad': 512, + 'distilbert-base-multilingual-cased': 512, } From 3c28a2daac43386dbc63b0bd014a966f22888850 Mon Sep 17 00:00:00 2001 From: Yao Lu <95luyao@gmail.com> Date: Wed, 27 Nov 2019 11:45:22 -0500 Subject: [PATCH 153/505] add add_special_tokens=True for input examples --- transformers/modeling_bert.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py index 81d92d8f1b..34bb8f89ba 100644 --- a/transformers/modeling_bert.py +++ b/transformers/modeling_bert.py @@ -597,7 +597,7 @@ class BertModel(BertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = BertModel.from_pretrained('bert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple @@ -760,7 +760,7 @@ class BertForPreTraining(BertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = BertForPreTraining.from_pretrained('bert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids) prediction_scores, seq_relationship_scores = outputs[:2] @@ -836,7 +836,7 @@ class BertForMaskedLM(BertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = BertForMaskedLM.from_pretrained('bert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids, masked_lm_labels=input_ids) loss, prediction_scores = outputs[:2] @@ -919,7 +919,7 @@ class BertForNextSentencePrediction(BertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = BertForNextSentencePrediction.from_pretrained('bert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 outputs = model(input_ids) seq_relationship_scores = outputs[0] @@ -984,7 +984,7 @@ class BertForSequenceClassification(BertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = BertForSequenceClassification.from_pretrained('bert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 outputs = model(input_ids, labels=labels) loss, logits = outputs[:2] @@ -1060,7 +1060,7 @@ class BertForMultipleChoice(BertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = BertForMultipleChoice.from_pretrained('bert-base-uncased') choices = ["Hello, my dog is cute", "Hello, my cat is amazing"] - input_ids = torch.tensor([tokenizer.encode(s) for s in choices]).unsqueeze(0) # Batch size 1, 2 choices + input_ids = torch.tensor([tokenizer.encode(s, add_special_tokens=True) for s in choices]).unsqueeze(0) # Batch size 1, 2 choices labels = torch.tensor(1).unsqueeze(0) # Batch size 1 outputs = model(input_ids, labels=labels) loss, classification_scores = outputs[:2] @@ -1134,7 +1134,7 @@ class BertForTokenClassification(BertPreTrainedModel): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') model = BertForTokenClassification.from_pretrained('bert-base-uncased') - input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 + input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute", add_special_tokens=True)).unsqueeze(0) # Batch size 1 labels = torch.tensor([1] * input_ids.size(1)).unsqueeze(0) # Batch size 1 outputs = model(input_ids, labels=labels) loss, scores = outputs[:2] From 8da47b078d92bee2de3e5fb50a37483d8cb02f13 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Wed, 27 Nov 2019 23:11:37 +0100 Subject: [PATCH 154/505] fix merge tests --- transformers/modeling_ctrl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/modeling_ctrl.py b/transformers/modeling_ctrl.py index 849487655d..3a252941ac 100644 --- a/transformers/modeling_ctrl.py +++ b/transformers/modeling_ctrl.py @@ -373,7 +373,7 @@ class CTRLModel(CTRLPreTrainedModel): if inputs_embeds is None: inputs_embeds = self.w(input_ids) # inputs_embeds = embedded.unsqueeze(0) if len(input_ids.shape)<2 else embedded - seq_len = input_shape.shape[-1] + seq_len = input_shape[-1] mask = torch.triu(torch.ones(seq_len + past_length, seq_len + past_length), 1).to(inputs_embeds.device) inputs_embeds *= np.sqrt(self.d_model_size) From bd41e8292a4bd7db10eb036112019d93c50adcf5 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 28 Nov 2019 16:03:56 -0500 Subject: [PATCH 155/505] Cleanup & Evaluation now works --- examples/run_squad.py | 44 +++++++++++---------------- transformers/data/processors/squad.py | 14 ++------- 2 files changed, 20 insertions(+), 38 deletions(-) diff --git a/examples/run_squad.py b/examples/run_squad.py index 634b566a46..545c3ad55a 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -16,7 +16,7 @@ """ Finetuning the library models for question-answering on SQuAD (DistilBERT, Bert, XLM, XLNet).""" from __future__ import absolute_import, division, print_function -from transformers.data.processors.squad import SquadV1Processor +from transformers.data.processors.squad import SquadV1Processor, SquadV2Processor import argparse import logging @@ -45,9 +45,9 @@ from transformers import (WEIGHTS_NAME, BertConfig, XLNetTokenizer, DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer) -from transformers import AdamW, get_linear_schedule_with_warmup, squad_convert_examples_to_features, read_squad_examples as sread_squad_examples +from transformers import AdamW, get_linear_schedule_with_warmup, squad_convert_examples_to_features -from utils_squad import (RawResult, write_predictions, +from utils_squad import (convert_examples_to_features as old_convert, read_squad_examples as old_read, RawResult, write_predictions, RawResultExtended, write_predictions_extended) # The follwing import is the official SQuAD evaluation script (2.0). @@ -304,28 +304,20 @@ def load_and_cache_examples(args, tokenizer, evaluate=False, output_examples=Fal features = torch.load(cached_features_file) else: logger.info("Creating features from dataset file at %s", input_file) - examples = read_squad_examples(input_file=input_file, - is_training=not evaluate, - version_2_with_negative=args.version_2_with_negative) - keep_n_examples = 1000 - processor = SquadV1Processor() - values = processor.get_dev_examples("examples/squad") - examples = values[:keep_n_examples] - features = squad_convert_examples_to_features(examples=exampless, - tokenizer=tokenizer, - max_seq_length=args.max_seq_length, - doc_stride=args.doc_stride, - max_query_length=args.max_query_length, - is_training=not evaluate, - cls_token_segment_id=2 if args.model_type in ['xlnet'] else 0, - pad_token_segment_id=3 if args.model_type in ['xlnet'] else 0, - cls_token_at_end=True if args.model_type in ['xlnet'] else False, - sequence_a_is_doc=True if args.model_type in ['xlnet'] else False) - print("DONE") - import sys - sys.exit() - + processor = SquadV2Processor() + examples = processor.get_dev_examples("examples/squad") if evaluate else processor.get_train_examples("examples/squad") + features = squad_convert_examples_to_features( + examples=examples, + tokenizer=tokenizer, + max_seq_length=args.max_seq_length, + doc_stride=args.doc_stride, + max_query_length=args.max_query_length, + is_training=not evaluate, + sequence_a_is_doc=True if args.model_type in ['xlnet'] else False + ) + + if args.local_rank in [-1, 0]: logger.info("Saving features into cached file %s", cached_features_file) torch.save(features, cached_features_file) @@ -335,8 +327,8 @@ def load_and_cache_examples(args, tokenizer, evaluate=False, output_examples=Fal # Convert to Tensors and build dataset all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long) - all_input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long) - all_segment_ids = torch.tensor([f.segment_ids for f in features], dtype=torch.long) + all_input_mask = torch.tensor([f.attention_mask for f in features], dtype=torch.long) + all_segment_ids = torch.tensor([f.token_type_ids for f in features], dtype=torch.long) all_cls_index = torch.tensor([f.cls_index for f in features], dtype=torch.long) all_p_mask = torch.tensor([f.p_mask for f in features], dtype=torch.float) if evaluate: diff --git a/transformers/data/processors/squad.py b/transformers/data/processors/squad.py index 39ee00ae56..3d5a3eca80 100644 --- a/transformers/data/processors/squad.py +++ b/transformers/data/processors/squad.py @@ -74,26 +74,16 @@ def _is_whitespace(c): def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, doc_stride, max_query_length, is_training, - cls_token_at_end=True, - cls_token='[CLS]', sep_token='[SEP]', pad_token=0, - sequence_a_segment_id=0, sequence_b_segment_id=1, - cls_token_segment_id=0, pad_token_segment_id=0, - mask_padding_with_zero=True, sequence_a_is_doc=False): """Loads a data file into a list of `InputBatch`s.""" - cls_token = tokenizer.cls_token - sep_token = tokenizer.sep_token - # Defining helper methods unique_id = 1000000000 features = [] - new_features = [] for (example_index, example) in enumerate(tqdm(examples)): if is_training and not example.is_impossible: # Get start and end position - answer_length = len(example.answer_text) start_position = example.start_position end_position = example.end_position @@ -227,7 +217,7 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, end_position = tok_end_position - doc_start + doc_offset - new_features.append(NewSquadFeatures( + features.append(NewSquadFeatures( span['input_ids'], span['attention_mask'], span['token_type_ids'], @@ -247,7 +237,7 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, unique_id += 1 - return new_features + return features class SquadProcessor(DataProcessor): From f671997ef74199823db83ed7b43340764888e129 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 28 Nov 2019 17:17:20 -0500 Subject: [PATCH 156/505] Interface with TFDS --- transformers/data/processors/squad.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/transformers/data/processors/squad.py b/transformers/data/processors/squad.py index 3d5a3eca80..52c2c28add 100644 --- a/transformers/data/processors/squad.py +++ b/transformers/data/processors/squad.py @@ -246,16 +246,24 @@ class SquadProcessor(DataProcessor): dev_file = None def get_example_from_tensor_dict(self, tensor_dict): - """See base class.""" return NewSquadExample( - tensor_dict['id'].numpy(), + tensor_dict['id'].numpy().decode("utf-8"), tensor_dict['question'].numpy().decode('utf-8'), tensor_dict['context'].numpy().decode('utf-8'), - tensor_dict['answers']['text'].numpy().decode('utf-8'), - tensor_dict['answers']['answers_start'].numpy().decode('utf-8'), + tensor_dict['answers']['text'][0].numpy().decode('utf-8'), + tensor_dict['answers']['answer_start'][0].numpy(), tensor_dict['title'].numpy().decode('utf-8') ) + def get_examples_from_dataset(self, dataset): + """See base class.""" + + examples = [] + for tensor_dict in tqdm(dataset): + examples.append(self.get_example_from_tensor_dict(tensor_dict)) + + return examples + def get_train_examples(self, data_dir, only_first=None): """See base class.""" if self.train_file is None: From 0b84b9fd8a728ca46e4109aa38a11b25f87a09bf Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 28 Nov 2019 17:38:52 -0500 Subject: [PATCH 157/505] Add processors to __init__ --- transformers/__init__.py | 2 +- transformers/data/__init__.py | 2 +- transformers/data/processors/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/transformers/__init__.py b/transformers/__init__.py index f3f81f1dbe..aefa3f1921 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -27,7 +27,7 @@ from .data import (is_sklearn_available, glue_output_modes, glue_convert_examples_to_features, glue_processors, glue_tasks_num_labels, squad_convert_examples_to_features, SquadFeatures, - SquadExample) + SquadExample, SquadV1Processor, SquadV2Processor) if is_sklearn_available(): from .data import glue_compute_metrics diff --git a/transformers/data/__init__.py b/transformers/data/__init__.py index b351bf625e..ea3a4e9fbb 100644 --- a/transformers/data/__init__.py +++ b/transformers/data/__init__.py @@ -1,6 +1,6 @@ from .processors import InputExample, InputFeatures, DataProcessor, SquadFeatures from .processors import glue_output_modes, glue_processors, glue_tasks_num_labels, glue_convert_examples_to_features -from .processors import squad_convert_examples_to_features, SquadExample +from .processors import squad_convert_examples_to_features, SquadExample, SquadV1Processor, SquadV2Processor from .metrics import is_sklearn_available if is_sklearn_available(): diff --git a/transformers/data/processors/__init__.py b/transformers/data/processors/__init__.py index 1e52776629..2470e7a06d 100644 --- a/transformers/data/processors/__init__.py +++ b/transformers/data/processors/__init__.py @@ -1,4 +1,4 @@ from .utils import InputExample, InputFeatures, DataProcessor from .glue import glue_output_modes, glue_processors, glue_tasks_num_labels, glue_convert_examples_to_features -from .squad import squad_convert_examples_to_features, SquadFeatures, SquadExample +from .squad import squad_convert_examples_to_features, SquadFeatures, SquadExample, SquadV1Processor, SquadV2Processor From 1e9ac5a7cfeb48ff6a1cf20e07941fc8c59b391d Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 28 Nov 2019 17:43:47 -0500 Subject: [PATCH 158/505] New -> normal --- transformers/data/processors/squad.py | 106 ++------------------------ 1 file changed, 5 insertions(+), 101 deletions(-) diff --git a/transformers/data/processors/squad.py b/transformers/data/processors/squad.py index 52c2c28add..f414d41925 100644 --- a/transformers/data/processors/squad.py +++ b/transformers/data/processors/squad.py @@ -217,7 +217,7 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, end_position = tok_end_position - doc_start + doc_offset - features.append(NewSquadFeatures( + features.append(SquadFeatures( span['input_ids'], span['attention_mask'], span['token_type_ids'], @@ -246,7 +246,7 @@ class SquadProcessor(DataProcessor): dev_file = None def get_example_from_tensor_dict(self, tensor_dict): - return NewSquadExample( + return SquadExample( tensor_dict['id'].numpy().decode("utf-8"), tensor_dict['question'].numpy().decode('utf-8'), tensor_dict['context'].numpy().decode('utf-8'), @@ -314,7 +314,7 @@ class SquadProcessor(DataProcessor): answer_text = answer['text'] start_position_character = answer['answer_start'] - example = NewSquadExample( + example = SquadExample( qas_id=qas_id, question_text=question_text, context_text=context_text, @@ -340,7 +340,7 @@ class SquadV2Processor(SquadProcessor): dev_file = "dev-v2.0.json" -class NewSquadExample(object): +class SquadExample(object): """ A single training/test example for the Squad dataset, as loaded from disk. """ @@ -387,7 +387,7 @@ class NewSquadExample(object): self.end_position = char_to_word_offset[start_position_character + len(answer_text) - 1] -class NewSquadFeatures(object): +class SquadFeatures(object): """ Single squad example features to be fed to a model. Those features are model-specific. @@ -425,99 +425,3 @@ class NewSquadFeatures(object): self.start_position = start_position self.end_position = end_position - -class SquadExample(object): - """ - A single training/test example for the Squad dataset. - For examples without an answer, the start and end position are -1. - """ - - def __init__(self, - qas_id, - question_text, - doc_tokens, - orig_answer_text=None, - start_position=None, - end_position=None, - is_impossible=None): - self.qas_id = qas_id - self.question_text = question_text - self.doc_tokens = doc_tokens - self.orig_answer_text = orig_answer_text - self.start_position = start_position - self.end_position = end_position - self.is_impossible = is_impossible - - def __str__(self): - return self.__repr__() - - def __repr__(self): - s = "" - s += "qas_id: %s" % (self.qas_id) - s += ", question_text: %s" % ( - self.question_text) - s += ", doc_tokens: [%s]" % (" ".join(self.doc_tokens)) - if self.start_position: - s += ", start_position: %d" % (self.start_position) - if self.end_position: - s += ", end_position: %d" % (self.end_position) - if self.is_impossible: - s += ", is_impossible: %r" % (self.is_impossible) - return s - - -class SquadFeatures(object): - """A single set of features of data.""" - - def __init__(self, - unique_id, - example_index, - doc_span_index, - tokens, - token_to_orig_map, - token_is_max_context, - input_ids, - input_mask, - segment_ids, - cls_index, - p_mask, - paragraph_len, - start_position=None, - end_position=None, - is_impossible=None): - self.unique_id = unique_id - self.example_index = example_index - self.doc_span_index = doc_span_index - self.tokens = tokens - self.token_to_orig_map = token_to_orig_map - self.token_is_max_context = token_is_max_context - self.input_ids = input_ids - self.input_mask = input_mask - self.segment_ids = segment_ids - self.cls_index = cls_index - self.p_mask = p_mask - self.paragraph_len = paragraph_len - self.start_position = start_position - self.end_position = end_position - self.is_impossible = is_impossible - - def __eq__(self, other): - print(self.example_index == other.example_index) - print(self.input_ids == other.input_ids) - print(self.input_mask == other.attention_mask) - print(self.p_mask == other.p_mask) - print(self.paragraph_len == other.paragraph_len) - print(self.segment_ids == other.token_type_ids) - print(self.token_is_max_context == other.token_is_max_context) - print(self.token_to_orig_map == other.token_to_orig_map) - print(self.tokens == other.tokens) - - return self.example_index == other.example_index and \ - self.input_ids == other.input_ids and \ - self.input_mask == other.attention_mask and \ - self.p_mask == other.p_mask and \ - self.paragraph_len == other.paragraph_len and \ - self.segment_ids == other.token_type_ids and \ - self.token_is_max_context == other.token_is_max_context and \ - self.token_to_orig_map == other.token_to_orig_map and \ - self.tokens == other.tokens \ No newline at end of file From 41aa0e80039d3148c55f4fe967247d4f7bbbfec5 Mon Sep 17 00:00:00 2001 From: Juha Kiili Date: Fri, 29 Nov 2019 15:33:25 +0200 Subject: [PATCH 159/505] Refactor logs and fix loss bug --- examples/run_glue.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/run_glue.py b/examples/run_glue.py index ea5ac5bbb7..8749593a1f 100644 --- a/examples/run_glue.py +++ b/examples/run_glue.py @@ -171,22 +171,22 @@ def train(args, train_dataset, model, tokenizer): global_step += 1 if args.local_rank in [-1, 0] and args.logging_steps > 0 and global_step % args.logging_steps == 0: - # Log metrics - logs = {'step': global_step} + logs = {} if args.local_rank == -1 and args.evaluate_during_training: # Only evaluate when single GPU otherwise metrics may not average well results = evaluate(args, model, tokenizer) for key, value in results.items(): eval_key = 'eval_{}'.format(key) - tb_writer.add_scalar(eval_key, value, global_step) - logs[eval_key] = str(value) - logging_loss = tr_loss + logs[eval_key] = value + loss_scalar = (tr_loss - logging_loss) / args.logging_steps learning_rate_scalar = scheduler.get_lr()[0] - tb_writer.add_scalar('lr', learning_rate_scalar, global_step) - tb_writer.add_scalar('loss', loss_scalar, global_step) logs['learning_rate'] = learning_rate_scalar logs['loss'] = loss_scalar - print(json.dumps(logs)) + logging_loss = tr_loss + + for key, value in logs.items(): + tb_writer.add_scalar(key, value, global_step) + print(json.dumps({**logs, **{'step': global_step}})) if args.local_rank in [-1, 0] and args.save_steps > 0 and global_step % args.save_steps == 0: # Save model checkpoint From 2421e54f8c354fc110a7f8819a9161163813f7ad Mon Sep 17 00:00:00 2001 From: Juha Kiili Date: Fri, 29 Nov 2019 15:39:28 +0200 Subject: [PATCH 160/505] Add link to original source and license to download_glue.data.py --- utils/download_glue_data.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils/download_glue_data.py b/utils/download_glue_data.py index 86a4e8951f..f676a71c76 100644 --- a/utils/download_glue_data.py +++ b/utils/download_glue_data.py @@ -1,5 +1,8 @@ ''' Script for downloading all GLUE data. +Original source: https://github.com/kamalkraj/ALBERT-TF2.0/blob/fa90194e5fe729dbb19f32ac29c8d6d6372c0f93/download_glue_data.py +Original license: https://github.com/kamalkraj/ALBERT-TF2.0/blob/fa90194e5fe729dbb19f32ac29c8d6d6372c0f93/LICENSE (Apache-2.0) + Note: for legal reasons, we are unable to host MRPC. You can either use the version hosted by the SentEval team, which is already tokenized, or you can download the original data from (https://download.microsoft.com/download/D/4/6/D46FF87A-F6B9-4252-AA8B-3604ED519838/MSRParaphraseCorpus.msi) and extract the data from it manually. From adb5c79ff2ffcd2e4a43a12f082cca55f7630a96 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 28 Nov 2019 15:51:43 +0100 Subject: [PATCH 161/505] update all tf.shape and tensor.shape to shape_list --- .../adding_a_new_model/modeling_tf_xxx.py | 6 ++--- transformers/__init__.py | 2 +- transformers/modeling_tf_albert.py | 27 ++++++++----------- transformers/modeling_tf_bert.py | 26 +++++++++--------- transformers/modeling_tf_ctrl.py | 2 +- transformers/modeling_tf_distilbert.py | 8 +++--- transformers/modeling_tf_gpt2.py | 2 +- transformers/modeling_tf_openai.py | 2 +- transformers/modeling_tf_roberta.py | 6 ++--- transformers/modeling_tf_transfo_xl.py | 2 +- .../modeling_tf_transfo_xl_utilities.py | 4 +-- transformers/modeling_tf_utils.py | 2 +- transformers/modeling_tf_xlnet.py | 13 +++++---- 13 files changed, 48 insertions(+), 54 deletions(-) diff --git a/templates/adding_a_new_model/modeling_tf_xxx.py b/templates/adding_a_new_model/modeling_tf_xxx.py index f1d898b47a..59f798bdbf 100644 --- a/templates/adding_a_new_model/modeling_tf_xxx.py +++ b/templates/adding_a_new_model/modeling_tf_xxx.py @@ -32,7 +32,7 @@ import numpy as np import tensorflow as tf from .configuration_xxx import XxxConfig -from .modeling_tf_utils import TFPreTrainedModel, get_initializer +from .modeling_tf_utils import TFPreTrainedModel, get_initializer, shape_list from .file_utils import add_start_docstrings logger = logging.getLogger(__name__) @@ -121,9 +121,9 @@ class TFXxxMainLayer(tf.keras.layers.Layer): input_ids = inputs if attention_mask is None: - attention_mask = tf.fill(tf.shape(input_ids), 1) + attention_mask = tf.fill(shape_list(input_ids), 1) if token_type_ids is None: - token_type_ids = tf.fill(tf.shape(input_ids), 0) + token_type_ids = tf.fill(shape_list(input_ids), 0) # We create a 3D attention mask from a 2D tensor mask. # Sizes are [batch_size, 1, 1, to_seq_length] diff --git a/transformers/__init__.py b/transformers/__init__.py index b29ad38e73..de25c24b9e 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -118,7 +118,7 @@ if is_torch_available(): # TensorFlow if is_tf_available(): - from .modeling_tf_utils import TFPreTrainedModel, TFSharedEmbeddings, TFSequenceSummary + from .modeling_tf_utils import TFPreTrainedModel, TFSharedEmbeddings, TFSequenceSummary, shape_list from .modeling_tf_auto import (TFAutoModel, TFAutoModelForSequenceClassification, TFAutoModelForQuestionAnswering, TFAutoModelWithLMHead) diff --git a/transformers/modeling_tf_albert.py b/transformers/modeling_tf_albert.py index b2bf66f750..164dc74320 100644 --- a/transformers/modeling_tf_albert.py +++ b/transformers/modeling_tf_albert.py @@ -16,18 +16,13 @@ """ TF 2.0 ALBERT model. """ from __future__ import absolute_import, division, print_function, unicode_literals -import json import logging -import math -import os import sys -from io import open -import numpy as np import tensorflow as tf from .configuration_albert import AlbertConfig -from .modeling_tf_utils import TFPreTrainedModel, get_initializer +from .modeling_tf_utils import TFPreTrainedModel, get_initializer, shape_list from .modeling_tf_bert import ACT2FN, TFBertSelfAttention from .file_utils import add_start_docstrings @@ -110,9 +105,9 @@ class TFAlbertEmbeddings(tf.keras.layers.Layer): input_ids, position_ids, token_type_ids, inputs_embeds = inputs if input_ids is not None: - input_shape = tf.shape(input_ids) + input_shape = shape_list(input_ids) else: - input_shape = tf.shape(inputs_embeds)[:-1] + input_shape = shape_list(inputs_embeds)[:-1] seq_length = input_shape[1] if position_ids is None: @@ -137,8 +132,8 @@ class TFAlbertEmbeddings(tf.keras.layers.Layer): Returns: float32 tensor with shape [batch_size, length, vocab_size]. """ - batch_size = tf.shape(inputs)[0] - length = tf.shape(inputs)[1] + batch_size = shape_list(inputs)[0] + length = shape_list(inputs)[1] x = tf.reshape(inputs, [-1, self.config.embedding_size]) logits = tf.matmul(x, self.word_embeddings, transpose_b=True) return tf.reshape(logits, [batch_size, length, self.config.vocab_size]) @@ -183,7 +178,7 @@ class TFAlbertSelfAttention(tf.keras.layers.Layer): def call(self, inputs, training=False): hidden_states, attention_mask, head_mask = inputs - batch_size = tf.shape(hidden_states)[0] + batch_size = shape_list(hidden_states)[0] mixed_query_layer = self.query(hidden_states) mixed_key_layer = self.key(hidden_states) mixed_value_layer = self.value(hidden_states) @@ -196,7 +191,7 @@ class TFAlbertSelfAttention(tf.keras.layers.Layer): # (batch size, num_heads, seq_len_q, seq_len_k) attention_scores = tf.matmul(query_layer, key_layer, transpose_b=True) # scale attention_scores - dk = tf.cast(tf.shape(key_layer)[-1], tf.float32) + dk = tf.cast(shape_list(key_layer)[-1], tf.float32) attention_scores = attention_scores / tf.math.sqrt(dk) if attention_mask is not None: @@ -264,7 +259,7 @@ class TFAlbertAttention(TFBertSelfAttention): def call(self, inputs, training=False): input_tensor, attention_mask, head_mask = inputs - batch_size = tf.shape(input_tensor)[0] + batch_size = shape_list(input_tensor)[0] mixed_query_layer = self.query(input_tensor) mixed_key_layer = self.key(input_tensor) mixed_value_layer = self.value(input_tensor) @@ -277,7 +272,7 @@ class TFAlbertAttention(TFBertSelfAttention): # (batch size, num_heads, seq_len_q, seq_len_k) attention_scores = tf.matmul(query_layer, key_layer, transpose_b=True) # scale attention_scores - dk = tf.cast(tf.shape(key_layer)[-1], tf.float32) + dk = tf.cast(shape_list(key_layer)[-1], tf.float32) attention_scores = attention_scores / tf.math.sqrt(dk) if attention_mask is not None: @@ -645,9 +640,9 @@ class TFAlbertModel(TFAlbertPreTrainedModel): if input_ids is not None and inputs_embeds is not None: raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") elif input_ids is not None: - input_shape = tf.shape(input_ids) + input_shape = shape_list(input_ids) elif inputs_embeds is not None: - input_shape = inputs_embeds.shape[:-1] + input_shape = shape_list(inputs_embeds)[:-1] else: raise ValueError("You have to specify either input_ids or inputs_embeds") diff --git a/transformers/modeling_tf_bert.py b/transformers/modeling_tf_bert.py index ad0815e2ca..5aa7bb3da2 100644 --- a/transformers/modeling_tf_bert.py +++ b/transformers/modeling_tf_bert.py @@ -28,7 +28,7 @@ import numpy as np import tensorflow as tf from .configuration_bert import BertConfig -from .modeling_tf_utils import TFPreTrainedModel, get_initializer +from .modeling_tf_utils import TFPreTrainedModel, get_initializer, shape_list from .file_utils import add_start_docstrings logger = logging.getLogger(__name__) @@ -145,9 +145,9 @@ class TFBertEmbeddings(tf.keras.layers.Layer): input_ids, position_ids, token_type_ids, inputs_embeds = inputs if input_ids is not None: - input_shape = tf.shape(input_ids) + input_shape = shape_list(input_ids) else: - input_shape = tf.shape(inputs_embeds)[:-1] + input_shape = shape_list(inputs_embeds)[:-1] seq_length = input_shape[1] if position_ids is None: @@ -172,8 +172,8 @@ class TFBertEmbeddings(tf.keras.layers.Layer): Returns: float32 tensor with shape [batch_size, length, vocab_size]. """ - batch_size = tf.shape(inputs)[0] - length = tf.shape(inputs)[1] + batch_size = shape_list(inputs)[0] + length = shape_list(inputs)[1] x = tf.reshape(inputs, [-1, self.hidden_size]) logits = tf.matmul(x, self.word_embeddings, transpose_b=True) @@ -214,7 +214,7 @@ class TFBertSelfAttention(tf.keras.layers.Layer): def call(self, inputs, training=False): hidden_states, attention_mask, head_mask = inputs - batch_size = tf.shape(hidden_states)[0] + batch_size = shape_list(hidden_states)[0] mixed_query_layer = self.query(hidden_states) mixed_key_layer = self.key(hidden_states) mixed_value_layer = self.value(hidden_states) @@ -225,7 +225,7 @@ class TFBertSelfAttention(tf.keras.layers.Layer): # Take the dot product between "query" and "key" to get the raw attention scores. attention_scores = tf.matmul(query_layer, key_layer, transpose_b=True) # (batch size, num_heads, seq_len_q, seq_len_k) - dk = tf.cast(tf.shape(key_layer)[-1], tf.float32) # scale attention_scores + dk = tf.cast(shape_list(key_layer)[-1], tf.float32) # scale attention_scores attention_scores = attention_scores / tf.math.sqrt(dk) if attention_mask is not None: @@ -502,9 +502,9 @@ class TFBertMainLayer(tf.keras.layers.Layer): if input_ids is not None and inputs_embeds is not None: raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") elif input_ids is not None: - input_shape = input_ids.shape + input_shape = shape_list(input_ids) elif inputs_embeds is not None: - input_shape = inputs_embeds.shape[:-1] + input_shape = shape_list(inputs_embeds)[:-1] else: raise ValueError("You have to specify either input_ids or inputs_embeds") @@ -939,11 +939,11 @@ class TFBertForMultipleChoice(TFBertPreTrainedModel): input_ids = inputs if input_ids is not None: - num_choices = tf.shape(input_ids)[1] - seq_length = tf.shape(input_ids)[2] + num_choices = shape_list(input_ids)[1] + seq_length = shape_list(input_ids)[2] else: - num_choices = tf.shape(inputs_embeds)[1] - seq_length = tf.shape(inputs_embeds)[2] + num_choices = shape_list(inputs_embeds)[1] + seq_length = shape_list(inputs_embeds)[2] flat_input_ids = tf.reshape(input_ids, (-1, seq_length)) if input_ids is not None else None flat_attention_mask = tf.reshape(attention_mask, (-1, seq_length)) if attention_mask is not None else None diff --git a/transformers/modeling_tf_ctrl.py b/transformers/modeling_tf_ctrl.py index ae66dbc82c..6d0d6a57ad 100644 --- a/transformers/modeling_tf_ctrl.py +++ b/transformers/modeling_tf_ctrl.py @@ -95,7 +95,7 @@ class TFMultiHeadAttention(tf.keras.layers.Layer): def call(self, inputs, training=False): v, k, q, mask, layer_past, attention_mask, head_mask = inputs - batch_size = q.shape[0] + batch_size = shape_list(q)[0] q = self.Wq(q) k = self.Wk(k) diff --git a/transformers/modeling_tf_distilbert.py b/transformers/modeling_tf_distilbert.py index 6d393bb95d..b3d4889475 100644 --- a/transformers/modeling_tf_distilbert.py +++ b/transformers/modeling_tf_distilbert.py @@ -137,9 +137,9 @@ class TFEmbeddings(tf.keras.layers.Layer): input_ids, position_ids = inputs if input_ids is not None: - seq_length = tf.shape(input_ids)[1] + seq_length = shape_list(input_ids)[1] else: - seq_length = tf.shape(inputs_embeds)[1] + seq_length = shape_list(inputs_embeds)[1] if position_ids is None: position_ids = tf.range(seq_length, dtype=tf.int32)[tf.newaxis, :] @@ -160,8 +160,8 @@ class TFEmbeddings(tf.keras.layers.Layer): Returns: float32 tensor with shape [batch_size, length, vocab_size]. """ - batch_size = tf.shape(inputs)[0] - length = tf.shape(inputs)[1] + batch_size = shape_list(inputs)[0] + length = shape_list(inputs)[1] x = tf.reshape(inputs, [-1, self.dim]) logits = tf.matmul(x, self.word_embeddings, transpose_b=True) diff --git a/transformers/modeling_tf_gpt2.py b/transformers/modeling_tf_gpt2.py index 5e416a5e3a..aebe790114 100644 --- a/transformers/modeling_tf_gpt2.py +++ b/transformers/modeling_tf_gpt2.py @@ -92,7 +92,7 @@ class TFAttention(tf.keras.layers.Layer): # q, k, v have shape [batch, heads, sequence, features] w = tf.matmul(q, k, transpose_b=True) if self.scale: - dk = tf.cast(tf.shape(k)[-1], tf.float32) # scale attention_scores + dk = tf.cast(shape_list(k)[-1], tf.float32) # scale attention_scores w = w / tf.math.sqrt(dk) # w has shape [batch, heads, dst_sequence, src_sequence], where information flows from src to dst. diff --git a/transformers/modeling_tf_openai.py b/transformers/modeling_tf_openai.py index c553d92317..dac3b17590 100644 --- a/transformers/modeling_tf_openai.py +++ b/transformers/modeling_tf_openai.py @@ -98,7 +98,7 @@ class TFAttention(tf.keras.layers.Layer): # q, k, v have shape [batch, heads, sequence, features] w = tf.matmul(q, k, transpose_b=True) if self.scale: - dk = tf.cast(tf.shape(k)[-1], tf.float32) # scale attention_scores + dk = tf.cast(shape_list(k)[-1], tf.float32) # scale attention_scores w = w / tf.math.sqrt(dk) # w has shape [batch, heads, dst_sequence, src_sequence], where information flows from src to dst. diff --git a/transformers/modeling_tf_roberta.py b/transformers/modeling_tf_roberta.py index 450c0c72f2..954279f873 100644 --- a/transformers/modeling_tf_roberta.py +++ b/transformers/modeling_tf_roberta.py @@ -24,7 +24,7 @@ import numpy as np import tensorflow as tf from .configuration_roberta import RobertaConfig -from .modeling_tf_utils import TFPreTrainedModel, get_initializer +from .modeling_tf_utils import TFPreTrainedModel, get_initializer, shape_list from .file_utils import add_start_docstrings from .modeling_tf_bert import TFBertEmbeddings, TFBertMainLayer, gelu, gelu_new @@ -51,9 +51,9 @@ class TFRobertaEmbeddings(TFBertEmbeddings): input_ids, position_ids, token_type_ids, inputs_embeds = inputs if input_ids is not None: - seq_length = tf.shape(input_ids)[1] + seq_length = shape_list(input_ids)[1] else: - seq_length = tf.shape(inputs_embeds)[1] + seq_length = shape_list(inputs_embeds)[1] if position_ids is None: position_ids = tf.range(self.padding_idx+1, seq_length+self.padding_idx+1, dtype=tf.int32)[tf.newaxis, :] diff --git a/transformers/modeling_tf_transfo_xl.py b/transformers/modeling_tf_transfo_xl.py index 8a8d11cfbc..fd325e218e 100644 --- a/transformers/modeling_tf_transfo_xl.py +++ b/transformers/modeling_tf_transfo_xl.py @@ -337,7 +337,7 @@ class TFAdaptiveEmbedding(tf.keras.layers.Layer): emb_i = tf.einsum('id,de->ie', emb_i, self.emb_projs[i]) mask_idx = tf.cast(tf.where(mask_i), dtype=tf.int64) - emb_flat += tf.scatter_nd(mask_idx, emb_i, tf.cast(tf.shape(emb_flat), dtype=tf.int64)) + emb_flat += tf.scatter_nd(mask_idx, emb_i, tf.cast(shape_list(emb_flat), dtype=tf.int64)) embed_shape = shape_list(inp) + [self.d_proj] embed = tf.reshape(emb_flat, embed_shape) diff --git a/transformers/modeling_tf_transfo_xl_utilities.py b/transformers/modeling_tf_transfo_xl_utilities.py index d7666a650e..e6a6dfe686 100644 --- a/transformers/modeling_tf_transfo_xl_utilities.py +++ b/transformers/modeling_tf_transfo_xl_utilities.py @@ -105,7 +105,7 @@ class TFAdaptiveSoftmaxMask(tf.keras.layers.Layer): @staticmethod def _gather_logprob(logprob, target): - lp_size = tf.shape(logprob) + lp_size = shape_list(logprob) r = tf.range(lp_size[0]) idx = tf.stack([r, target], 1) return tf.gather_nd(logprob, idx) @@ -159,7 +159,7 @@ class TFAdaptiveSoftmaxMask(tf.keras.layers.Layer): cur_logprob = self._gather_logprob(cur_tail_logprob, cur_target) cur_logprob += cur_head_logprob[:, self.cutoff_ends[1] + i - 1] if target is not None: - loss += tf.scatter_nd(mask_idx, -cur_logprob, tf.cast(tf.shape(loss), dtype=tf.int64)) + loss += tf.scatter_nd(mask_idx, -cur_logprob, tf.cast(shape_list(loss), dtype=tf.int64)) out = tf.concat(out, axis=-1) if target is not None: diff --git a/transformers/modeling_tf_utils.py b/transformers/modeling_tf_utils.py index 569b2faa4b..e4ba55e25e 100644 --- a/transformers/modeling_tf_utils.py +++ b/transformers/modeling_tf_utils.py @@ -494,7 +494,7 @@ class TFSequenceSummary(tf.keras.layers.Layer): def shape_list(x): """Deal with dynamic shape in tensorflow cleanly.""" static = x.shape.as_list() - dynamic = tf.shape(x) + dynamic = shape_list(x) return [dynamic[i] if s is None else s for i, s in enumerate(static)] def get_initializer(initializer_range=0.02): diff --git a/transformers/modeling_tf_xlnet.py b/transformers/modeling_tf_xlnet.py index 4733ea8589..215d906f57 100644 --- a/transformers/modeling_tf_xlnet.py +++ b/transformers/modeling_tf_xlnet.py @@ -112,8 +112,7 @@ class TFXLNetRelativeAttention(tf.keras.layers.Layer): def prune_heads(self, heads): raise NotImplementedError - @staticmethod - def rel_shift(x, klen=-1): + def rel_shift(self, x, klen=-1): """perform relative shift to form the relative attention score.""" x_size = shape_list(x) @@ -135,7 +134,7 @@ class TFXLNetRelativeAttention(tf.keras.layers.Layer): # position based attention score bd = tf.einsum('ibnd,jbnd->ijbn', q_head + self.r_r_bias, k_head_r) - bd = self.rel_shift(bd, klen=ac.shape[1]) + bd = self.rel_shift(bd, klen=shape_list(ac)[1]) # segment based attention score if seg_mat is None: @@ -192,7 +191,7 @@ class TFXLNetRelativeAttention(tf.keras.layers.Layer): if g is not None: ###### Two-stream attention with relative positional encoding. # content based attention score - if mems is not None and mems.shape.ndims > 1: + if mems is not None and len(shape_list(mems)) > 1: cat = tf.concat([mems, h], axis=0) else: cat = h @@ -252,7 +251,7 @@ class TFXLNetRelativeAttention(tf.keras.layers.Layer): else: ###### Multi-head attention with relative positional encoding - if mems is not None and mems.shape.ndims > 1: + if mems is not None and len(shape_list(mems)) > 1: cat = tf.concat([mems, h], axis=0) else: cat = h @@ -565,7 +564,7 @@ class TFXLNetMainLayer(tf.keras.layers.Layer): if data_mask is not None: # all mems can be attended to - mems_mask = tf.zeros([tf.shape(data_mask)[0], mlen, bsz], + mems_mask = tf.zeros([shape_list(data_mask)[0], mlen, bsz], dtype=dtype_float) data_mask = tf.concat([mems_mask, data_mask], axis=1) if attn_mask is None: @@ -590,7 +589,7 @@ class TFXLNetMainLayer(tf.keras.layers.Layer): word_emb_k = self.word_embedding(input_ids) output_h = self.dropout(word_emb_k, training=training) if target_mapping is not None: - word_emb_q = tf.tile(self.mask_emb, [tf.shape(target_mapping)[0], bsz, 1]) + word_emb_q = tf.tile(self.mask_emb, [shape_list(target_mapping)[0], bsz, 1]) # else: # We removed the inp_q input which was same as target mapping # inp_q_ext = inp_q[:, :, None] # word_emb_q = inp_q_ext * self.mask_emb + (1 - inp_q_ext) * word_emb_k From 4a666885b501ed6bfd344ec2c4c16d80da8aab79 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 28 Nov 2019 15:56:53 +0100 Subject: [PATCH 162/505] reducing my level of enthousiasm --- transformers/modeling_tf_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/modeling_tf_utils.py b/transformers/modeling_tf_utils.py index e4ba55e25e..569b2faa4b 100644 --- a/transformers/modeling_tf_utils.py +++ b/transformers/modeling_tf_utils.py @@ -494,7 +494,7 @@ class TFSequenceSummary(tf.keras.layers.Layer): def shape_list(x): """Deal with dynamic shape in tensorflow cleanly.""" static = x.shape.as_list() - dynamic = shape_list(x) + dynamic = tf.shape(x) return [dynamic[i] if s is None else s for i, s in enumerate(static)] def get_initializer(initializer_range=0.02): From ecf15ebf3b7f5d2b0144f1a428e2d9f39494c8ba Mon Sep 17 00:00:00 2001 From: Elad Segal Date: Fri, 29 Nov 2019 12:48:41 +0200 Subject: [PATCH 163/505] Add ALBERT to AutoClasses --- transformers/configuration_auto.py | 15 +++++++++----- transformers/modeling_auto.py | 33 +++++++++++++++++++++++------- transformers/tokenization_auto.py | 15 +++++++++----- 3 files changed, 46 insertions(+), 17 deletions(-) diff --git a/transformers/configuration_auto.py b/transformers/configuration_auto.py index 37b8c0e3df..43f251bd0c 100644 --- a/transformers/configuration_auto.py +++ b/transformers/configuration_auto.py @@ -28,6 +28,7 @@ from .configuration_roberta import RobertaConfig from .configuration_distilbert import DistilBertConfig from .configuration_ctrl import CTRLConfig from .configuration_camembert import CamembertConfig +from .configuration_albert import AlbertConfig logger = logging.getLogger(__name__) @@ -44,14 +45,15 @@ class AutoConfig(object): The base model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertConfig (DistilBERT model) + - contains `albert`: AlbertConfig (ALBERT model) + - contains `camembert`: CamembertConfig (CamemBERT model) + - contains `roberta`: RobertaConfig (RoBERTa model) - contains `bert`: BertConfig (Bert model) - contains `openai-gpt`: OpenAIGPTConfig (OpenAI GPT model) - contains `gpt2`: GPT2Config (OpenAI GPT-2 model) - contains `transfo-xl`: TransfoXLConfig (Transformer-XL model) - contains `xlnet`: XLNetConfig (XLNet model) - contains `xlm`: XLMConfig (XLM model) - - contains `roberta`: RobertaConfig (RoBERTa model) - - contains `camembert`: CamembertConfig (CamemBERT model) - contains `ctrl` : CTRLConfig (CTRL model) This class cannot be instantiated using `__init__()` (throw an error). """ @@ -67,14 +69,15 @@ class AutoConfig(object): The configuration class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertConfig (DistilBERT model) + - contains `albert`: AlbertConfig (ALBERT model) + - contains `camembert`: CamembertConfig (CamemBERT model) + - contains `roberta`: RobertaConfig (RoBERTa model) - contains `bert`: BertConfig (Bert model) - contains `openai-gpt`: OpenAIGPTConfig (OpenAI GPT model) - contains `gpt2`: GPT2Config (OpenAI GPT-2 model) - contains `transfo-xl`: TransfoXLConfig (Transformer-XL model) - contains `xlnet`: XLNetConfig (XLNet model) - contains `xlm`: XLMConfig (XLM model) - - contains `roberta`: RobertaConfig (RoBERTa model) - - contains `camembert`: CamembertConfig (CamemBERT model) - contains `ctrl` : CTRLConfig (CTRL model) Params: pretrained_model_name_or_path: either: @@ -122,6 +125,8 @@ class AutoConfig(object): """ if 'distilbert' in pretrained_model_name_or_path: return DistilBertConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) + elif 'albert' in pretrained_model_name_or_path: + return AlbertConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) elif 'camembert' in pretrained_model_name_or_path: return CamembertConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) elif 'roberta' in pretrained_model_name_or_path: @@ -142,4 +147,4 @@ class AutoConfig(object): return CTRLConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', " - "'xlm', 'roberta', 'camembert', 'ctrl'".format(pretrained_model_name_or_path)) + "'xlm', 'roberta', 'distilbert', 'camembert', 'ctrl', 'albert'".format(pretrained_model_name_or_path)) diff --git a/transformers/modeling_auto.py b/transformers/modeling_auto.py index fa33dbc0c8..b63e43d73b 100644 --- a/transformers/modeling_auto.py +++ b/transformers/modeling_auto.py @@ -28,6 +28,8 @@ from .modeling_xlm import XLMModel, XLMWithLMHeadModel, XLMForSequenceClassifica from .modeling_roberta import RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification from .modeling_distilbert import DistilBertModel, DistilBertForQuestionAnswering, DistilBertForMaskedLM, DistilBertForSequenceClassification from .modeling_camembert import CamembertModel, CamembertForMaskedLM, CamembertForSequenceClassification, CamembertForMultipleChoice +from .modeling_camembert import CamembertModel, CamembertForMaskedLM, CamembertForSequenceClassification, CamembertForMultipleChoice +from .modeling_albert import AlbertModel, AlbertForMaskedLM, AlbertForSequenceClassification, AlbertForQuestionAnswering from .modeling_utils import PreTrainedModel, SequenceSummary @@ -49,15 +51,16 @@ class AutoModel(object): The base model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertModel (DistilBERT model) + - contains `albert`: AlbertModel (ALBERT model) - contains `camembert`: CamembertModel (CamemBERT model) - contains `roberta`: RobertaModel (RoBERTa model) - contains `bert`: BertModel (Bert model) - contains `openai-gpt`: OpenAIGPTModel (OpenAI GPT model) - contains `gpt2`: GPT2Model (OpenAI GPT-2 model) - - contains `ctrl`: CTRLModel (Salesforce CTRL model) - contains `transfo-xl`: TransfoXLModel (Transformer-XL model) - contains `xlnet`: XLNetModel (XLNet model) - contains `xlm`: XLMModel (XLM model) + - contains `ctrl`: CTRLModel (Salesforce CTRL model) This class cannot be instantiated using `__init__()` (throws an error). """ @@ -73,15 +76,16 @@ class AutoModel(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertModel (DistilBERT model) + - contains `albert`: AlbertModel (ALBERT model) - contains `camembert`: CamembertModel (CamemBERT model) - contains `roberta`: RobertaModel (RoBERTa model) - contains `bert`: BertModel (Bert model) - contains `openai-gpt`: OpenAIGPTModel (OpenAI GPT model) - contains `gpt2`: GPT2Model (OpenAI GPT-2 model) - - contains `ctrl`: CTRLModel (Salesforce CTRL model) - contains `transfo-xl`: TransfoXLModel (Transformer-XL model) - contains `xlnet`: XLNetModel (XLNet model) - contains `xlm`: XLMModel (XLM model) + - contains `ctrl`: CTRLModel (Salesforce CTRL model) The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated) To train the model, you should first set it back in training mode with `model.train()` @@ -144,6 +148,8 @@ class AutoModel(object): """ if 'distilbert' in pretrained_model_name_or_path: return DistilBertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'albert' in pretrained_model_name_or_path: + return AlbertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'camembert' in pretrained_model_name_or_path: return CamembertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: @@ -164,7 +170,7 @@ class AutoModel(object): return CTRLModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', " - "'xlm', 'roberta, 'ctrl'".format(pretrained_model_name_or_path)) + "'xlm', 'roberta, 'ctrl', 'distilbert', 'camembert', 'albert'".format(pretrained_model_name_or_path)) class AutoModelWithLMHead(object): @@ -180,15 +186,16 @@ class AutoModelWithLMHead(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertForMaskedLM (DistilBERT model) + - contains `albert`: AlbertForMaskedLM (ALBERT model) - contains `camembert`: CamembertForMaskedLM (CamemBERT model) - contains `roberta`: RobertaForMaskedLM (RoBERTa model) - contains `bert`: BertForMaskedLM (Bert model) - contains `openai-gpt`: OpenAIGPTLMHeadModel (OpenAI GPT model) - contains `gpt2`: GPT2LMHeadModel (OpenAI GPT-2 model) - - contains `ctrl`: CTRLLMModel (Salesforce CTRL model) - contains `transfo-xl`: TransfoXLLMHeadModel (Transformer-XL model) - contains `xlnet`: XLNetLMHeadModel (XLNet model) - contains `xlm`: XLMWithLMHeadModel (XLM model) + - contains `ctrl`: CTRLLMHeadModel (Salesforce CTRL model) This class cannot be instantiated using `__init__()` (throws an error). """ @@ -207,6 +214,7 @@ class AutoModelWithLMHead(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertForMaskedLM (DistilBERT model) + - contains `albert`: AlbertForMaskedLM (ALBERT model) - contains `camembert`: CamembertForMaskedLM (CamemBERT model) - contains `roberta`: RobertaForMaskedLM (RoBERTa model) - contains `bert`: BertForMaskedLM (Bert model) @@ -215,6 +223,7 @@ class AutoModelWithLMHead(object): - contains `transfo-xl`: TransfoXLLMHeadModel (Transformer-XL model) - contains `xlnet`: XLNetLMHeadModel (XLNet model) - contains `xlm`: XLMWithLMHeadModel (XLM model) + - contains `ctrl`: CTRLLMHeadModel (Salesforce CTRL model) The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated) To train the model, you should first set it back in training mode with `model.train()` @@ -276,6 +285,8 @@ class AutoModelWithLMHead(object): """ if 'distilbert' in pretrained_model_name_or_path: return DistilBertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'albert' in pretrained_model_name_or_path: + return AlbertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'camembert' in pretrained_model_name_or_path: return CamembertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: @@ -296,7 +307,7 @@ class AutoModelWithLMHead(object): return CTRLLMHeadModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', " - "'xlm', 'roberta','ctrl'".format(pretrained_model_name_or_path)) + "'xlm', 'roberta','ctrl', 'distilbert', 'camembert', 'albert'".format(pretrained_model_name_or_path)) class AutoModelForSequenceClassification(object): @@ -312,6 +323,7 @@ class AutoModelForSequenceClassification(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertForSequenceClassification (DistilBERT model) + - contains `albert`: AlbertForSequenceClassification (ALBERT model) - contains `camembert`: CamembertForSequenceClassification (CamemBERT model) - contains `roberta`: RobertaForSequenceClassification (RoBERTa model) - contains `bert`: BertForSequenceClassification (Bert model) @@ -335,6 +347,7 @@ class AutoModelForSequenceClassification(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertForSequenceClassification (DistilBERT model) + - contains `albert`: AlbertForSequenceClassification (ALBERT model) - contains `camembert`: CamembertForSequenceClassification (CamemBERT model) - contains `roberta`: RobertaForSequenceClassification (RoBERTa model) - contains `bert`: BertForSequenceClassification (Bert model) @@ -402,6 +415,8 @@ class AutoModelForSequenceClassification(object): """ if 'distilbert' in pretrained_model_name_or_path: return DistilBertForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'albert' in pretrained_model_name_or_path: + return AlbertForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'camembert' in pretrained_model_name_or_path: return CamembertForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: @@ -414,7 +429,7 @@ class AutoModelForSequenceClassification(object): return XLMForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " - "'bert', 'xlnet', 'xlm', 'roberta'".format(pretrained_model_name_or_path)) + "'bert', 'xlnet', 'xlm', 'roberta', 'distilbert', 'camembert', 'albert'".format(pretrained_model_name_or_path)) class AutoModelForQuestionAnswering(object): @@ -430,6 +445,7 @@ class AutoModelForQuestionAnswering(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertForQuestionAnswering (DistilBERT model) + - contains `albert`: AlbertForQuestionAnswering (ALBERT model) - contains `bert`: BertForQuestionAnswering (Bert model) - contains `xlnet`: XLNetForQuestionAnswering (XLNet model) - contains `xlm`: XLMForQuestionAnswering (XLM model) @@ -451,6 +467,7 @@ class AutoModelForQuestionAnswering(object): The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertForQuestionAnswering (DistilBERT model) + - contains `albert`: AlbertForQuestionAnswering (ALBERT model) - contains `bert`: BertForQuestionAnswering (Bert model) - contains `xlnet`: XLNetForQuestionAnswering (XLNet model) - contains `xlm`: XLMForQuestionAnswering (XLM model) @@ -513,6 +530,8 @@ class AutoModelForQuestionAnswering(object): """ if 'distilbert' in pretrained_model_name_or_path: return DistilBertForQuestionAnswering.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'albert' in pretrained_model_name_or_path: + return AlbertForQuestionAnswering.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'bert' in pretrained_model_name_or_path: return BertForQuestionAnswering.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'xlnet' in pretrained_model_name_or_path: @@ -521,4 +540,4 @@ class AutoModelForQuestionAnswering(object): return XLMForQuestionAnswering.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " - "'bert', 'xlnet', 'xlm'".format(pretrained_model_name_or_path)) + "'bert', 'xlnet', 'xlm', 'distilbert', 'albert'".format(pretrained_model_name_or_path)) diff --git a/transformers/tokenization_auto.py b/transformers/tokenization_auto.py index 2e15e38073..b7c5046961 100644 --- a/transformers/tokenization_auto.py +++ b/transformers/tokenization_auto.py @@ -28,6 +28,7 @@ from .tokenization_xlm import XLMTokenizer from .tokenization_roberta import RobertaTokenizer from .tokenization_distilbert import DistilBertTokenizer from .tokenization_camembert import CamembertTokenizer +from .tokenization_albert import AlbertTokenizer logger = logging.getLogger(__name__) @@ -42,16 +43,17 @@ class AutoTokenizer(object): The tokenizer class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - - contains `camembert`: CamembertTokenizer (CamemBERT model) - contains `distilbert`: DistilBertTokenizer (DistilBert model) + - contains `albert`: AlbertTokenizer (ALBERT model) + - contains `camembert`: CamembertTokenizer (CamemBERT model) - contains `roberta`: RobertaTokenizer (RoBERTa model) - contains `bert`: BertTokenizer (Bert model) - contains `openai-gpt`: OpenAIGPTTokenizer (OpenAI GPT model) - contains `gpt2`: GPT2Tokenizer (OpenAI GPT-2 model) - - contains `ctrl`: CTRLTokenizer (Salesforce CTRL model) - contains `transfo-xl`: TransfoXLTokenizer (Transformer-XL model) - contains `xlnet`: XLNetTokenizer (XLNet model) - contains `xlm`: XLMTokenizer (XLM model) + - contains `ctrl`: CTRLTokenizer (Salesforce CTRL model) This class cannot be instantiated using `__init__()` (throw an error). """ @@ -66,16 +68,17 @@ class AutoTokenizer(object): The tokenizer class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - - contains `camembert`: CamembertTokenizer (CamemBERT model) - contains `distilbert`: DistilBertTokenizer (DistilBert model) + - contains `albert`: AlbertTokenizer (ALBERT model) + - contains `camembert`: CamembertTokenizer (CamemBERT model) - contains `roberta`: RobertaTokenizer (RoBERTa model) - contains `bert`: BertTokenizer (Bert model) - contains `openai-gpt`: OpenAIGPTTokenizer (OpenAI GPT model) - contains `gpt2`: GPT2Tokenizer (OpenAI GPT-2 model) - - contains `ctrl`: CTRLTokenizer (Salesforce CTRL model) - contains `transfo-xl`: TransfoXLTokenizer (Transformer-XL model) - contains `xlnet`: XLNetTokenizer (XLNet model) - contains `xlm`: XLMTokenizer (XLM model) + - contains `ctrl`: CTRLTokenizer (Salesforce CTRL model) Params: pretrained_model_name_or_path: either: @@ -109,6 +112,8 @@ class AutoTokenizer(object): """ if 'distilbert' in pretrained_model_name_or_path: return DistilBertTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) + elif 'albert' in pretrained_model_name_or_path: + return AlbertTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) elif 'camembert' in pretrained_model_name_or_path: return CamembertTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) elif 'roberta' in pretrained_model_name_or_path: @@ -129,4 +134,4 @@ class AutoTokenizer(object): return CTRLTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', " - "'xlm', 'roberta', 'camembert', 'ctrl'".format(pretrained_model_name_or_path)) + "'xlm', 'roberta', 'distilbert,' 'camembert', 'ctrl', 'albert'".format(pretrained_model_name_or_path)) From b0ee7c7df3d49a819c4d6cef977214bd91f5c075 Mon Sep 17 00:00:00 2001 From: maxvidal <44881831+maxvidal@users.noreply.github.com> Date: Fri, 29 Nov 2019 12:32:37 +0100 Subject: [PATCH 164/505] Added Camembert to available models --- examples/run_lm_finetuning.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py index c33aa94a32..4acea00c55 100644 --- a/examples/run_lm_finetuning.py +++ b/examples/run_lm_finetuning.py @@ -47,7 +47,8 @@ from transformers import (WEIGHTS_NAME, AdamW, get_linear_schedule_with_warmup, GPT2Config, GPT2LMHeadModel, GPT2Tokenizer, OpenAIGPTConfig, OpenAIGPTLMHeadModel, OpenAIGPTTokenizer, RobertaConfig, RobertaForMaskedLM, RobertaTokenizer, - DistilBertConfig, DistilBertForMaskedLM, DistilBertTokenizer) + DistilBertConfig, DistilBertForMaskedLM, DistilBertTokenizer, + CamembertConfig, CamembertForMaskedLM, CamembertTokenizer) logger = logging.getLogger(__name__) @@ -58,7 +59,8 @@ MODEL_CLASSES = { 'openai-gpt': (OpenAIGPTConfig, OpenAIGPTLMHeadModel, OpenAIGPTTokenizer), 'bert': (BertConfig, BertForMaskedLM, BertTokenizer), 'roberta': (RobertaConfig, RobertaForMaskedLM, RobertaTokenizer), - 'distilbert': (DistilBertConfig, DistilBertForMaskedLM, DistilBertTokenizer) + 'distilbert': (DistilBertConfig, DistilBertForMaskedLM, DistilBertTokenizer), + 'camembert': (CamembertConfig, CamembertForMaskedLM, CamembertTokenizer) } @@ -432,7 +434,7 @@ def main(): parser.add_argument('--server_port', type=str, default='', help="For distant debugging.") args = parser.parse_args() - if args.model_type in ["bert", "roberta", "distilbert"] and not args.mlm: + if args.model_type in ["bert", "roberta", "distilbert", "camembert"] and not args.mlm: raise ValueError("BERT and RoBERTa do not have LM heads but masked LM heads. They must be run using the --mlm " "flag (masked language modeling).") if args.eval_data_file is None and args.do_eval: From b90791e95026dfa95a8cea72605257e4b9355956 Mon Sep 17 00:00:00 2001 From: Rostislav Nedelchev Date: Sat, 30 Nov 2019 15:57:51 +0100 Subject: [PATCH 165/505] fixed XLNet attenttion output for both attention streams --- transformers/modeling_xlnet.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/transformers/modeling_xlnet.py b/transformers/modeling_xlnet.py index 658048a660..476d9ab13d 100644 --- a/transformers/modeling_xlnet.py +++ b/transformers/modeling_xlnet.py @@ -581,7 +581,7 @@ class XLNetModel(XLNetPreTrainedModel): of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of 2-tuple of ``torch.FloatTensor`` (one for each layer, one for each attention stream) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: @@ -878,7 +878,7 @@ class XLNetModel(XLNetPreTrainedModel): hidden_states = tuple(hs.permute(1, 0, 2).contiguous() for hs in hidden_states) outputs = outputs + (hidden_states,) if self.output_attentions: - attentions = tuple(t.permute(2, 3, 0, 1).contiguous() for t in attentions) + attentions = tuple(tuple(att_stream.permute(2, 3, 0, 1).contiguous() for att_stream in t) for t in attentions) outputs = outputs + (attentions,) return outputs # outputs, (new_mems), (hidden_states), (attentions) @@ -911,7 +911,7 @@ class XLNetLMHeadModel(XLNetPreTrainedModel): of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of 2-tuple of ``torch.FloatTensor`` (one for each layer, one for each attention stream) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: @@ -993,7 +993,7 @@ class XLNetForSequenceClassification(XLNetPreTrainedModel): of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of 2-tuple of ``torch.FloatTensor`` (one for each layer, one for each attention stream) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: @@ -1093,7 +1093,7 @@ class XLNetForMultipleChoice(XLNetPreTrainedModel): of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of 2-tuple of ``torch.FloatTensor`` (one for each layer, one for each attention stream) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: @@ -1178,7 +1178,7 @@ class XLNetForQuestionAnsweringSimple(XLNetPreTrainedModel): of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of 2-tuple of ``torch.FloatTensor`` (one for each layer, one for each attention stream) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: @@ -1292,7 +1292,7 @@ class XLNetForQuestionAnswering(XLNetPreTrainedModel): of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of 2-tuple of ``torch.FloatTensor`` (one for each layer, one for each attention stream) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Examples:: From 76c0bc06d549e0ecf746fdd3d72eb235a9c4aec2 Mon Sep 17 00:00:00 2001 From: Rostislav Nedelchev Date: Sat, 30 Nov 2019 21:01:04 +0100 Subject: [PATCH 166/505] [XLNet] Changed post-processing of attention w.r.t to target_mapping Whenever target_mapping is provided to the input, XLNet outputs two different attention streams. Based on that the attention output would be on of the two: - a list of tensors (usual case for most transformers) - a list of 2-tuples of tensors, one tesor for each of attention streams Docs and unit-tests have been updated --- transformers/modeling_xlnet.py | 24 ++++++++++++++++------- transformers/tests/modeling_xlnet_test.py | 18 +++++++++++++++++ 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/transformers/modeling_xlnet.py b/transformers/modeling_xlnet.py index 476d9ab13d..56d755c11b 100644 --- a/transformers/modeling_xlnet.py +++ b/transformers/modeling_xlnet.py @@ -581,8 +581,9 @@ class XLNetModel(XLNetPreTrainedModel): of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of 2-tuple of ``torch.FloatTensor`` (one for each layer, one for each attention stream) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + When ``target_mapping is not None``, the attentions outputs are a list of 2-tuple of ``torch.FloatTensor``. Examples:: @@ -878,7 +879,11 @@ class XLNetModel(XLNetPreTrainedModel): hidden_states = tuple(hs.permute(1, 0, 2).contiguous() for hs in hidden_states) outputs = outputs + (hidden_states,) if self.output_attentions: - attentions = tuple(tuple(att_stream.permute(2, 3, 0, 1).contiguous() for att_stream in t) for t in attentions) + if target_mapping is not None: + # when target_mapping is provided, there are 2-tuple of attentions + attentions = tuple(tuple(att_stream.permute(2, 3, 0, 1).contiguous() for att_stream in t) for t in attentions) + else: + attentions = tuple(t.permute(2, 3, 0, 1).contiguous() for t in attentions) outputs = outputs + (attentions,) return outputs # outputs, (new_mems), (hidden_states), (attentions) @@ -911,8 +916,9 @@ class XLNetLMHeadModel(XLNetPreTrainedModel): of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of 2-tuple of ``torch.FloatTensor`` (one for each layer, one for each attention stream) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + When ``target_mapping is not None``, the attentions outputs are a list of 2-tuple of ``torch.FloatTensor``. Examples:: @@ -993,8 +999,9 @@ class XLNetForSequenceClassification(XLNetPreTrainedModel): of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of 2-tuple of ``torch.FloatTensor`` (one for each layer, one for each attention stream) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + When ``target_mapping is not None``, the attentions outputs are a list of 2-tuple of ``torch.FloatTensor``. Examples:: @@ -1093,8 +1100,9 @@ class XLNetForMultipleChoice(XLNetPreTrainedModel): of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of 2-tuple of ``torch.FloatTensor`` (one for each layer, one for each attention stream) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + When ``target_mapping is not None``, the attentions outputs are a list of 2-tuple of ``torch.FloatTensor``. Examples:: @@ -1178,8 +1186,9 @@ class XLNetForQuestionAnsweringSimple(XLNetPreTrainedModel): of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of 2-tuple of ``torch.FloatTensor`` (one for each layer, one for each attention stream) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + When ``target_mapping is not None``, the attentions outputs are a list of 2-tuple of ``torch.FloatTensor``. Examples:: @@ -1292,8 +1301,9 @@ class XLNetForQuestionAnswering(XLNetPreTrainedModel): of shape ``(batch_size, sequence_length, hidden_size)``: Hidden-states of the model at the output of each layer plus the initial embedding outputs. **attentions**: (`optional`, returned when ``config.output_attentions=True``) - list of 2-tuple of ``torch.FloatTensor`` (one for each layer, one for each attention stream) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + When ``target_mapping is not None``, the attentions outputs are a list of 2-tuple of ``torch.FloatTensor``. Examples:: diff --git a/transformers/tests/modeling_xlnet_test.py b/transformers/tests/modeling_xlnet_test.py index d97ea6a425..a5ee9b1e0e 100644 --- a/transformers/tests/modeling_xlnet_test.py +++ b/transformers/tests/modeling_xlnet_test.py @@ -163,6 +163,18 @@ class XLNetModelTest(CommonTestCases.CommonModelTester): list(list(mem.size()) for mem in result["mems_1"]), [[self.seq_length, self.batch_size, self.hidden_size]] * self.num_hidden_layers) + def create_and_check_xlnet_base_model_with_att_output(self, config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, + target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels): + model = XLNetModel(config) + model.eval() + + _, _, attentions = model(input_ids_1, target_mapping=target_mapping) + + self.parent.assertEqual(len(attentions), config.n_layer) + self.parent.assertIsInstance(attentions[0], tuple) + self.parent.assertEqual(len(attentions[0]), 2) + self.parent.assertTrue(attentions[0][0].shape, attentions[0][0].shape) + def create_and_check_xlnet_lm_head(self, config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels): model = XLNetLMHeadModel(config) @@ -306,6 +318,12 @@ class XLNetModelTest(CommonTestCases.CommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_xlnet_base_model(*config_and_inputs) + def test_xlnet_base_model_with_att_output(self): + self.model_tester.set_seed() + config_and_inputs = self.model_tester.prepare_config_and_inputs() + config_and_inputs[0].output_attentions = True + self.model_tester.create_and_check_xlnet_base_model_with_att_output(*config_and_inputs) + def test_xlnet_lm_head(self): self.model_tester.set_seed() config_and_inputs = self.model_tester.prepare_config_and_inputs() From c356290c8ddd9037bd08c854b118673983ef6def Mon Sep 17 00:00:00 2001 From: Aditya Soni Date: Sun, 1 Dec 2019 14:08:14 +0530 Subject: [PATCH 167/505] typo fix as per Pytorch v1.1+ --- docs/source/migration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/migration.md b/docs/source/migration.md index d04b66d5e4..f50d1dff0a 100644 --- a/docs/source/migration.md +++ b/docs/source/migration.md @@ -104,6 +104,6 @@ for batch in train_data: loss = model(batch) loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), max_grad_norm) # Gradient clipping is not in AdamW anymore (so you can use amp without issue) - scheduler.step() optimizer.step() + scheduler.step() ``` From 5ab93083e4d9b610df1dd082f17a455a0b7193ac Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sun, 1 Dec 2019 18:25:15 +0100 Subject: [PATCH 168/505] Mark tests in TFAutoModelTest as slow. Each test forces downloading the same 536MB file, which is slow even with a decent internet connection. --- transformers/tests/modeling_tf_auto_test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/transformers/tests/modeling_tf_auto_test.py b/transformers/tests/modeling_tf_auto_test.py index 2cda3abc1c..fa90906e86 100644 --- a/transformers/tests/modeling_tf_auto_test.py +++ b/transformers/tests/modeling_tf_auto_test.py @@ -38,6 +38,7 @@ else: class TFAutoModelTest(unittest.TestCase): + @pytest.mark.slow def test_model_from_pretrained(self): import h5py self.assertTrue(h5py.version.hdf5_version.startswith("1.10")) @@ -53,6 +54,7 @@ class TFAutoModelTest(unittest.TestCase): self.assertIsNotNone(model) self.assertIsInstance(model, TFBertModel) + @pytest.mark.slow def test_lmhead_model_from_pretrained(self): logging.basicConfig(level=logging.INFO) # for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: @@ -65,6 +67,7 @@ class TFAutoModelTest(unittest.TestCase): self.assertIsNotNone(model) self.assertIsInstance(model, TFBertForMaskedLM) + @pytest.mark.slow def test_sequence_classification_model_from_pretrained(self): logging.basicConfig(level=logging.INFO) # for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: @@ -77,6 +80,7 @@ class TFAutoModelTest(unittest.TestCase): self.assertIsNotNone(model) self.assertIsInstance(model, TFBertForSequenceClassification) + @pytest.mark.slow def test_question_answering_model_from_pretrained(self): logging.basicConfig(level=logging.INFO) # for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: From f3776df0f3daca86634862fe3ba7da6ae2b9a663 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 2 Dec 2019 15:47:00 +0100 Subject: [PATCH 169/505] WIP debugging --- transformers/modeling_t5.py | 61 +++++++++++++++++++++++++++---------- 1 file changed, 45 insertions(+), 16 deletions(-) diff --git a/transformers/modeling_t5.py b/transformers/modeling_t5.py index 2a74333d31..1bf55611a2 100644 --- a/transformers/modeling_t5.py +++ b/transformers/modeling_t5.py @@ -132,6 +132,21 @@ def load_tf_weights_in_t5(model, config, tf_checkpoint_path): # - PreTrainedModel for the models (it-self a sub-class of torch.nn.Module) #################################################### +class T5LayerNorm(nn.Module): + def __init__(self, hidden_size, eps=1e-6): + """ Construct a layernorm module in the T5 style + No bias and no substraction of mean. + """ + super(T5LayerNorm, self).__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, x): + variance = x.pow(2).mean(-1, keepdim=True) + x = x / torch.sqrt(variance + self.variance_epsilon) + return self.weight * x + + class T5DenseReluDense(nn.Module): def __init__(self, config): super(T5DenseReluDense, self).__init__() @@ -151,7 +166,7 @@ class T5LayerFF(nn.Module): def __init__(self, config): super(T5LayerFF, self).__init__() self.DenseReluDense = T5DenseReluDense(config) - self.layer_norm = nn.LayerNorm(config.d_model, eps=config.layer_norm_epsilon) + self.layer_norm = T5LayerNorm(config.d_model, eps=config.layer_norm_epsilon) self.dropout = nn.Dropout(config.dropout_rate) def forward(self, hidden_states): @@ -316,13 +331,14 @@ class T5Attention(nn.Module): cache[self.layer_id] = (k, v) # q = q / math.sqrt(dim_per_head) # No scaling in T5 - scores = torch.matmul(q, k.transpose(2, 3)) # (bs, n_heads, qlen, klen) + scores = torch.einsum('bnqd,bnkd->bnqk', q, k) # (bs, n_heads, qlen, klen) if position_bias is None: if not self.has_relative_attention_bias: raise ValueError("No position_bias provided and no weights to compute position_bias") position_bias = self.compute_bias(qlen, klen) scores += position_bias + special_out = position_bias if mask is not None: scores += mask @@ -346,14 +362,14 @@ class T5Attention(nn.Module): outputs = outputs + (weights,) if self.has_relative_attention_bias: outputs = outputs + (position_bias,) - return outputs + return outputs + (special_out,) class T5LayerSelfAttention(nn.Module): def __init__(self, config, has_relative_attention_bias=False): super(T5LayerSelfAttention, self).__init__() self.SelfAttention = T5Attention(config, has_relative_attention_bias=has_relative_attention_bias) - self.layer_norm = nn.LayerNorm(config.d_model, eps=config.layer_norm_epsilon) + self.layer_norm = T5LayerNorm(config.d_model, eps=config.layer_norm_epsilon) self.dropout = nn.Dropout(config.dropout_rate) def forward(self, hidden_states, attention_mask=None, position_bias=None, head_mask=None): @@ -363,16 +379,18 @@ class T5LayerSelfAttention(nn.Module): position_bias=position_bias, head_mask=head_mask) y = attention_output[0] + special_out = attention_output[-1] + attention_output = attention_output[:-1] layer_output = hidden_states + self.dropout(y) outputs = (layer_output,) + attention_output[1:] # add attentions if we output them - return outputs + return outputs + (special_out,) class T5LayerCrossAttention(nn.Module): def __init__(self, config, has_relative_attention_bias=False): super(T5LayerCrossAttention, self).__init__() self.EncDecAttention = T5Attention(config, has_relative_attention_bias=has_relative_attention_bias) - self.layer_norm = nn.LayerNorm(config.d_model, eps=config.layer_norm_epsilon) + self.layer_norm = T5LayerNorm(config.d_model, eps=config.layer_norm_epsilon) self.dropout = nn.Dropout(config.dropout_rate) def forward(self, hidden_states, kv, attention_mask=None, position_bias=None, head_mask=None): @@ -408,7 +426,8 @@ class T5Block(nn.Module): position_bias=position_bias, head_mask=head_mask) hidden_states = self_attention_outputs[0] - outputs = self_attention_outputs[1:] # Keep self-attention outputs and relative position weights + special_out = self_attention_outputs[-1] + outputs = self_attention_outputs[1:-1] # Keep self-attention outputs and relative position weights if not self.is_decoder: hidden_states = self.layer[1](hidden_states) @@ -423,7 +442,7 @@ class T5Block(nn.Module): hidden_states = self.layer[2](hidden_states) outputs = (hidden_states,) + outputs # add attentions if we output them - return outputs # hidden-states, (self-attention weights), (self-attention position bias), (cross-attention weights), (cross-attention position bias) + return outputs + (special_out,) # hidden-states, (self-attention weights), (self-attention position bias), (cross-attention weights), (cross-attention position bias) class T5PreTrainedModel(PreTrainedModel): @@ -438,8 +457,7 @@ class T5PreTrainedModel(PreTrainedModel): def _init_weights(self, module): """ Initialize the weights """ factor = self.config.initializer_factor # Used for testing weights initialization - if isinstance(module, nn.LayerNorm): - module.bias.data.zero_() + if isinstance(module, T5LayerNorm): module.weight.data.fill_(factor*1.0) elif isinstance(module, (T5Model, T5WithLMHeadModel)): # Mesh TensorFlow embeddings initialization @@ -478,7 +496,7 @@ class T5Stack(T5PreTrainedModel): self.block = nn.ModuleList([T5Block(config, has_relative_attention_bias=bool(i == 0)) for i in range(config.num_layers)]) - self.final_layer_norm = nn.LayerNorm(config.d_model, eps=config.layer_norm_epsilon) + self.final_layer_norm = T5LayerNorm(config.d_model, eps=config.layer_norm_epsilon) self.dropout = nn.Dropout(config.dropout_rate) self.init_weights() @@ -515,11 +533,11 @@ class T5Stack(T5PreTrainedModel): # Since attention_mask is 1.0 for positions we want to attend and 0.0 for # masked positions, this operation will create a tensor which is 0.0 for - # positions we want to attend and -10000.0 for masked positions. + # positions we want to attend and -1e9 for masked positions. # Since we are adding it to the raw scores before the softmax, this is # effectively the same as removing these entirely. extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility - extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + extended_attention_mask = (1.0 - extended_attention_mask) * -1e9 if self.is_decoder: # If a 2D ou 3D attention mask is provided for the cross-attention @@ -530,7 +548,7 @@ class T5Stack(T5PreTrainedModel): encoder_extended_attention_mask = encoder_attention_mask[:, None, None, :] encoder_extended_attention_mask = encoder_extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility - encoder_extended_attention_mask = (1.0 - encoder_extended_attention_mask) * -10000.0 + encoder_extended_attention_mask = (1.0 - encoder_extended_attention_mask) * -1e9 else: encoder_extended_attention_mask = None @@ -553,6 +571,8 @@ class T5Stack(T5PreTrainedModel): all_attentions = () position_bias = None encoder_decoder_position_bias = None + + hidden_states = self.dropout(hidden_states) for i, layer_module in enumerate(self.block): if self.output_hidden_states: all_hidden_states = all_hidden_states + (hidden_states,) @@ -564,6 +584,8 @@ class T5Stack(T5PreTrainedModel): encoder_attention_mask=encoder_extended_attention_mask, encoder_decoder_position_bias=encoder_decoder_position_bias, head_mask=head_mask[i]) + if i == 0: + special_out = layer_outputs[-1] # layer_outputs is a tuple with: # hidden-states, (self-attention weights), (self-attention position bias), (cross-attention weights), (cross-attention position bias) hidden_states = layer_outputs[0] @@ -588,7 +610,7 @@ class T5Stack(T5PreTrainedModel): outputs = outputs + (all_hidden_states,) if self.output_attentions: outputs = outputs + (all_attentions,) - return outputs # last-layer hidden state, (all hidden states), (all attentions) + return outputs + (special_out,) # last-layer hidden state, (all hidden states), (all attentions) T5_START_DOCSTRING = r""" The T5 model was proposed in @@ -707,9 +729,16 @@ class T5Model(T5PreTrainedModel): # Encode if needed (training, first prediction pass) encoder_hidden_states = kwargs_encoder.pop("hidden_states", None) + encoder_attention_mask = kwargs_encoder.get("attention_mask", None) if encoder_hidden_states is None: encoder_inputs_ids = kwargs_encoder.pop("input_ids") hidden_states = self.shared(encoder_inputs_ids) # Convert inputs in embeddings + + if encoder_attention_mask is not None: + # Apply masking + encoder_attention_mask = (encoder_attention_mask != 0).to(hidden_states) + hidden_states = hidden_states * encoder_attention_mask.unsqueeze(-1) + encoder_outputs = self.encoder(hidden_states, **kwargs_encoder) encoder_hidden_states = encoder_outputs[0] else: @@ -719,7 +748,7 @@ class T5Model(T5PreTrainedModel): decoder_inputs_ids = kwargs_decoder.pop("input_ids") hidden_states = self.shared(decoder_inputs_ids) # Convert inputs in embeddings kwargs_decoder["encoder_hidden_states"] = encoder_hidden_states - kwargs_decoder["encoder_attention_mask"] = kwargs_encoder.get("attention_mask", None) + kwargs_decoder["encoder_attention_mask"] = encoder_attention_mask decoder_outputs = self.decoder(hidden_states, **kwargs_decoder) return decoder_outputs + encoder_outputs From b3d834ae11381ca493da97f717d77d185ca7d780 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Mon, 2 Dec 2019 15:01:52 -0500 Subject: [PATCH 170/505] Reorganize ALBERT conversion script --- transformers/modeling_albert.py | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 5b7b2d3900..49d120ffae 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -68,14 +68,36 @@ def load_tf_weights_in_albert(model, config, tf_checkpoint_path): for name, array in zip(names, arrays): original_name = name + + # If saved from the TF HUB module + name = name.replace("module/", "") + + # Renaming and simplifying name = name.replace("ffn_1", "ffn") - name = name.replace("/bert/", "/albert/") - name = name.replace("ffn/intermediate/output", "ffn_output") + name = name.replace("bert/", "albert/") name = name.replace("attention_1", "attention") - name = name.replace("cls/predictions", "predictions") name = name.replace("transform/", "") name = name.replace("LayerNorm_1", "full_layer_layer_norm") - name = name.replace("LayerNorm", "attention/LayerNorm") + name = name.replace("LayerNorm", "attention/LayerNorm") + name = name.replace("transformer/", "") + + # The feed forward layer had an 'intermediate' step which has been abstracted away + name = name.replace("intermediate/dense/", "") + name = name.replace("ffn/intermediate/output/dense/", "ffn_output/") + + # ALBERT attention was split between self and output which have been abstracted away + name = name.replace("/output/", "/") + name = name.replace("/self/", "/") + + # The pooler is a linear layer + name = name.replace("pooler/dense", "pooler") + + # The classifier was simplified to predictions from cls/predictions + name = name.replace("cls/predictions", "predictions") + name = name.replace("predictions/attention", "predictions") + + # Naming was changed to be more explicit + name = name.replace("embeddings/attention", "embeddings") name = name.replace("inner_group_", "albert_layers/") name = name.replace("group_", "albert_layer_groups/") name = name.split('/') From e85855f2c408f65a4aaf5d15baab6ca90fd26050 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Mon, 2 Dec 2019 18:00:19 -0500 Subject: [PATCH 171/505] Fix ALBERT exports with pretraining + sp classifier; Fix naming for ALBERT TF models --- transformers/modeling_albert.py | 17 ++++++++++++++++- transformers/modeling_tf_albert.py | 8 ++++---- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/transformers/modeling_albert.py b/transformers/modeling_albert.py index 49d120ffae..0f67bf8f36 100644 --- a/transformers/modeling_albert.py +++ b/transformers/modeling_albert.py @@ -99,8 +99,23 @@ def load_tf_weights_in_albert(model, config, tf_checkpoint_path): # Naming was changed to be more explicit name = name.replace("embeddings/attention", "embeddings") name = name.replace("inner_group_", "albert_layers/") - name = name.replace("group_", "albert_layer_groups/") + name = name.replace("group_", "albert_layer_groups/") + + # Classifier + if len(name.split("/")) == 1 and ("output_bias" in name or "output_weights" in name): + name = "classifier/" + name + + # No ALBERT model currently handles the next sentence prediction task + if "seq_relationship" in name: + continue + name = name.split('/') + + # Ignore the gradients applied by the LAMB/ADAM optimizers. + if "adam_m" in name or "adam_v" in name or "global_step" in name: + logger.info("Skipping {}".format("/".join(name))) + continue + pointer = model for m_name in name: if re.fullmatch(r'[A-Za-z]+_\d+', m_name): diff --git a/transformers/modeling_tf_albert.py b/transformers/modeling_tf_albert.py index 164dc74320..d1650d41a8 100644 --- a/transformers/modeling_tf_albert.py +++ b/transformers/modeling_tf_albert.py @@ -31,10 +31,10 @@ import logging logger = logging.getLogger(__name__) TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP = { - 'albert-base-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-tf_model.h5", - 'albert-large-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-tf_model.h5", - 'albert-xlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-tf_model.h5", - 'albert-xxlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-tf_model.h5", + 'albert-base-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v1-tf_model.h5", + 'albert-large-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-v1-tf_model.h5", + 'albert-xlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-v1-tf_model.h5", + 'albert-xxlarge-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xxlarge-v1-tf_model.h5", 'albert-base-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-base-v2-tf_model.h5", 'albert-large-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-large-v2-tf_model.h5", 'albert-xlarge-v2': "https://s3.amazonaws.com/models.huggingface.co/bert/albert-xlarge-v2-tf_model.h5", From fbaf05bd92249b6dd961f5f8d60eb0892c541ac8 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Mon, 2 Dec 2019 18:23:00 -0500 Subject: [PATCH 172/505] Remove annoying tokenization message --- transformers/tokenization_utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index 60df822677..5d683629f0 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -981,7 +981,6 @@ class PreTrainedTokenizer(object): return (ids, pair_ids, overflowing_tokens) def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None): - logger.warning("This tokenizer does not make use of special tokens.") if token_ids_1 is None: return len(token_ids_0) * [0] return [0] * len(token_ids_0) + [1] * len(token_ids_1) @@ -994,7 +993,6 @@ class PreTrainedTokenizer(object): single sequence: X pair of sequences: A B """ - logger.warning("This tokenizer does not make use of special tokens. Input is returned with no modification.") if token_ids_1 is None: return token_ids_0 return token_ids_0 + token_ids_1 From 66fc8d25a5de43c08baeea8b22b9bcf57346c325 Mon Sep 17 00:00:00 2001 From: Juha Kiili Date: Tue, 3 Dec 2019 10:49:50 +0200 Subject: [PATCH 173/505] Change ref to original GLUE downloader script --- utils/download_glue_data.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/utils/download_glue_data.py b/utils/download_glue_data.py index f676a71c76..de8cfa9e73 100644 --- a/utils/download_glue_data.py +++ b/utils/download_glue_data.py @@ -1,7 +1,5 @@ ''' Script for downloading all GLUE data. - -Original source: https://github.com/kamalkraj/ALBERT-TF2.0/blob/fa90194e5fe729dbb19f32ac29c8d6d6372c0f93/download_glue_data.py -Original license: https://github.com/kamalkraj/ALBERT-TF2.0/blob/fa90194e5fe729dbb19f32ac29c8d6d6372c0f93/LICENSE (Apache-2.0) +Original source: https://gist.github.com/W4ngatang/60c2bdb54d156a41194446737ce03e2e Note: for legal reasons, we are unable to host MRPC. You can either use the version hosted by the SentEval team, which is already tokenized, From 572c24cfa25c167e48dd08a7d2429afeb69acbe0 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Tue, 5 Nov 2019 15:34:37 +0000 Subject: [PATCH 174/505] PPLM (squashed) Co-authored-by: piero Co-authored-by: Rosanne Liu --- examples/run_pplm.py | 782 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 782 insertions(+) create mode 100644 examples/run_pplm.py diff --git a/examples/run_pplm.py b/examples/run_pplm.py new file mode 100644 index 0000000000..30853c68c3 --- /dev/null +++ b/examples/run_pplm.py @@ -0,0 +1,782 @@ +# coding=utf-8 +# Copyright 2018 The Uber AI Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# TODO: add code for training a custom discriminator + +""" +Example command with bag of words: +python examples/run_pplm.py -B space --cond_text "The president" --length 100 --gamma 1.5 --num_iterations 3 --num_samples 10 --stepsize 0.01 --window_length 5 --kl_scale 0.01 --gm_scale 0.95 + +Example command with discriminator: +python examples/run_pplm.py -D sentiment --label_class 3 --cond_text "The lake" --length 10 --gamma 1.0 --num_iterations 30 --num_samples 10 --stepsize 0.01 --kl_scale 0.01 --gm_scale 0.95 +""" + +import argparse +from operator import add +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch +import torch.nn.functional as F +from torch.autograd import Variable +from tqdm import trange + +from transformers import GPT2Tokenizer +from transformers.file_utils import cached_path +from transformers.modeling_gpt2 import GPT2LMHeadModel + +PPLM_BOW = 1 +PPLM_DISCRIM = 2 +PPLM_BOW_DISCRIM = 3 +SMALL_CONST = 1e-15 +TOKENIZER = GPT2Tokenizer.from_pretrained("gpt2-medium") + +BAG_OF_WORDS_ARCHIVE_MAP = { + 'kitchen': "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/bow/kitchen.txt", + 'legal': "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/bow/legal.txt", + 'military': "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/bow/military.txt", + 'monsters': "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/bow/monsters.txt", + 'politics': "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/bow/politics.txt", + 'positive_words': "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/bow/positive_words.txt", + 'religion': "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/bow/religion.txt", + 'science': "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/bow/science.txt", + 'space': "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/bow/space.txt", + 'technology': "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/bow/technology.txt", +} + +DISCRIMINATOR_MODELS_PARAMS = { + "clickbait": { + "url": "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/discriminators/clickbait_classifierhead.pt", + "class_size": 2, + "embed_size": 1024, + "class_vocab": {"non_clickbait": 0, "clickbait": 1}, + "default_class": 1, + }, + "sentiment": { + "url": "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/discriminators/sentiment_classifierhead.pt", + "class_size": 5, + "embed_size": 1024, + "class_vocab": {"very_positive": 2, "very_negative": 3}, + "default_class": 3, + }, + "toxicity": { + "url": "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/discriminators/toxicity_classifierhead.pt", + "class_size": 2, + "embed_size": 1024, + "class_vocab": {"non_toxic": 0, "toxic": 1}, + "default_class": 0, + }, +} + + +class ClassificationHead(torch.nn.Module): + """ Classification Head for the transformer """ + + def __init__(self, class_size=5, embed_size=2048): + super(ClassificationHead, self).__init__() + self.class_size = class_size + self.embed_size = embed_size + # self.mlp1 = torch.nn.Linear(embed_size, embed_size) + # self.mlp2 = (torch.nn.Linear(embed_size, class_size)) + self.mlp = torch.nn.Linear(embed_size, class_size) + + def forward(self, hidden_state): + # hidden_state = F.relu(self.mlp1(hidden_state)) + # hidden_state = self.mlp2(hidden_state) + logits = self.mlp(hidden_state) + return logits + + +def to_var(x, requires_grad=False, volatile=False): + if torch.cuda.is_available(): + x = x.cuda() + return Variable(x, requires_grad=requires_grad, volatile=volatile) + + +def top_k_filter(logits, k, probs=False): + """ + Masks everything but the k top entries as -infinity (1e10). + Used to mask logits such that e^-infinity -> 0 won't contribute to the + sum of the denominator. + """ + if k <= 0: + return logits + + else: + values = torch.topk(logits, k)[0] + batch_mins = values[:, -1].view(-1, 1).expand_as(logits) + + if probs: + return torch.where( + logits < batch_mins, + torch.ones_like(logits) * 0.0, + logits + ) + + return torch.where( + logits < batch_mins, + torch.ones_like(logits) * -1e10, + logits + ) + + +def perturb_past( + past, + model, + last, + unpert_past=None, + unpert_logits=None, + accumulated_hidden=None, + grad_norms=None, + stepsize=0.01, + classifier=None, + label_class=None, + one_hot_bows_vectors=None, + loss_type=0, + num_iterations=3, + kl_scale=0.01, + window_length=0, + horizon_length=1, + decay=False, + gamma=1.5, +): + # initializie perturbation accumulator + grad_accumulator = [ + (np.zeros(p.shape).astype("float32")) + for p in past + ] + + if accumulated_hidden is None: + accumulated_hidden = 0 + + if decay: + decay_mask = torch.arange( + 0.0, + 1.0 + SMALL_CONST, + 1.0 / (window_length) + )[1:] + else: + decay_mask = 1.0 + + # TODO fix this comment (SUMANTH) + # generate a mask if perturbated gradient is based on a past window + _, _, _, curr_length, _ = past[0].shape + if curr_length > window_length and window_length > 0: + ones_key_val_shape = ( + tuple(past[0].shape[:-2]) + + tuple([window_length]) + + tuple(past[0].shape[-1:]) + ) + + zeros_key_val_shape = ( + tuple(past[0].shape[:-2]) + + tuple([curr_length - window_length]) + + tuple(past[0].shape[-1:]) + ) + + ones_mask = torch.ones(ones_key_val_shape) + ones_mask = decay_mask * ones_mask.permute(0, 1, 2, 4, 3) + ones_mask = ones_mask.permute(0, 1, 2, 4, 3) + + window_mask = torch.cat( + (ones_mask, torch.zeros(zeros_key_val_shape)), + dim=-2 + ).cuda() + + else: + window_mask = torch.ones_like(past[0]).cuda() + + # accumulate perturbations for num_iterations + loss_per_iter = [] + for i in range(num_iterations): + print("Iteration ", i + 1) + + curr_perturbation = [ + to_var(torch.from_numpy(p_), requires_grad=True) + for p_ in grad_accumulator + ] + + # Compute hidden using perturbed past + curr_pert_past = list(map(add, past, curr_perturbation)) + all_logits, _, all_hidden = model(last, past=curr_pert_past) + hidden = all_hidden[-1] + accumulated_hidden += torch.sum(hidden, dim=1).detach() + logits = all_logits[:, -1, :] + probs = F.softmax(logits, dim=-1) + + # compute loss + bow_loss = 0.0 + discrim_loss = 0.0 + kl_loss = 0.0 + + if loss_type == PPLM_BOW or loss_type == PPLM_BOW_DISCRIM: + for one_hot_bow in one_hot_bows_vectors: + bow_logits = torch.mm(probs, torch.t(one_hot_bow)) + bow_loss += -torch.log(torch.sum(bow_logits)) + print(" pplm_bow_loss:", bow_loss.data.cpu().numpy()) + + if loss_type == PPLM_DISCRIM or loss_type == PPLM_BOW_DISCRIM: + ce_loss = torch.nn.CrossEntropyLoss() + # TODO all there are for (SUMANTH) + # TODO why we need to do this assignment and not just using unpert_past? + curr_unpert_past = unpert_past + # Get the model's token embeddings in order to compute our own embeds from curr_probs: + wte = model.resize_token_embeddings() + # TODO i is never used, why do we need to do this i times instead multiplying + # torch.sum(unpert_hidden, dim=1) * horizon_length? + for i in range(horizon_length): + # TODO the next two lines can be done only one time, and why not using probs instead as they do not change at each iteration? + curr_probs = F.softmax(logits, dim=-1) # get softmax + curr_probs = torch.unsqueeze(curr_probs, dim=1) + inputs_embeds = torch.matmul(curr_probs, wte.weight.data) + _, curr_unpert_past, curr_all_hidden = model( + past=curr_unpert_past, + inputs_embeds=inputs_embeds + ) + # get expected hidden states + unpert_hidden = curr_all_hidden[1] + accumulated_hidden += torch.sum(unpert_hidden, dim=1) + + prediction = classifier( + accumulated_hidden / (curr_length + 1 + horizon_length) + ) + + label = torch.tensor([label_class], device="cuda", dtype=torch.long) + discrim_loss += ce_loss(prediction, label) + print(" pplm_discrim_loss:", discrim_loss.data.cpu().numpy()) + + if kl_scale > 0.0: + unpert_probs = F.softmax(unpert_logits[:, -1, :], dim=-1) + unpert_probs = ( + unpert_probs + SMALL_CONST * + (unpert_probs <= SMALL_CONST).type( + torch.FloatTensor + ).cuda().detach() + ) + + correction = SMALL_CONST * (probs <= SMALL_CONST).type( + torch.FloatTensor + ).cuda().detach() + corrected_probs = probs + correction.detach() + kl_loss += kl_scale * ( + (corrected_probs * (corrected_probs / unpert_probs).log()).sum() + ) + print(' kl_loss', (kl_loss).data.cpu().numpy()) + + loss = bow_loss + discrim_loss + kl_loss + loss_per_iter.append(loss.data.cpu().numpy()) + print(' pplm_loss', (loss - kl_loss).data.cpu().numpy()) + + # compute gradients + loss.backward(retain_graph=True) + + # calculate gradient norms + if grad_norms is not None and loss_type == PPLM_BOW: + grad_norms = [ + torch.max(grad_norms[index], torch.norm(p_.grad * window_mask)) + for index, p_ in enumerate(curr_perturbation) + ] + else: + grad_norms = [ + (torch.norm(p_.grad * window_mask) + SMALL_CONST) + for index, p_ in enumerate(curr_perturbation) + ] + + # normalize gradients + grad = [ + -stepsize + * (p_.grad * window_mask / grad_norms[ + index] ** gamma).data.cpu().numpy() + for index, p_ in enumerate(curr_perturbation) + ] + + # accumulate gradients + grad_accumulator = list(map(add, grad, grad_accumulator)) + + # reset gradients, just to make sure + for p_ in curr_perturbation: + p_.grad.data.zero_() + + # removing past from the graph + new_past = [] + for p_ in past: + new_past.append(p_.detach()) + past = new_past + + # apply the accumulated perturbations to the past + grad_accumulator = [ + to_var(torch.from_numpy(p_), requires_grad=True) + for p_ in grad_accumulator + ] + pert_past = list(map(add, past, grad_accumulator)) + + return pert_past, accumulated_hidden, grad_norms, loss_per_iter + + +def get_classifier( + name: Optional[str], label_class: Union[str, int], device: Union[str, torch.device] +) -> Tuple[Optional[ClassificationHead], Optional[int]]: + if name is None: + return None, None + + params = DISCRIMINATOR_MODELS_PARAMS[name] + classifier = ClassificationHead( + class_size=params['class_size'], + embed_size=params['embed_size'] + ).to(device) + resolved_archive_file = cached_path(params["url"]) + classifier.load_state_dict(torch.load(resolved_archive_file, map_location=device)) + classifier.eval() + + if isinstance(label_class, str): + if label_class in params["class_vocab"]: + label_id = params["class_vocab"][label_class] + else: + label_id = params["default_class"] + print("label_class {} not in class_vocab".format(label_class)) + print("available values are: {}".format(params["class_vocab"])) + print("using default class {}".format(label_id)) + + elif isinstance(label_class, int): + if label_class in set(params["class_vocab"].values()): + label_id = label_class + else: + label_id = params["default_class"] + print("label_class {} not in class_vocab".format(label_class)) + print("available values are: {}".format(params["class_vocab"])) + print("using default class {}".format(label_id)) + + else: + label_id = params["default_class"] + + return classifier, label_id + + +def get_bag_of_words_indices(bag_of_words_ids_or_paths: List[str]) -> List[List[List[int]]]: + bow_indices = [] + for id_or_path in bag_of_words_ids_or_paths: + if id_or_path in BAG_OF_WORDS_ARCHIVE_MAP: + filepath = cached_path(BAG_OF_WORDS_ARCHIVE_MAP[id_or_path]) + else: + filepath = id_or_path + with open(filepath, "r") as f: + words = f.read().split("\n") + bow_indices.append([TOKENIZER.encode(word) for word in words]) + return bow_indices + + +def build_bows_one_hot_vectors(bow_indices): + if bow_indices is None: + return None + + one_hot_bows_vectors = [] + for single_bow in bow_indices: + single_bow = list(filter(lambda x: len(x) <= 1, single_bow)) + single_bow = torch.tensor(single_bow).cuda() + num_words = single_bow.shape[0] + one_hot_bow = torch.zeros(num_words, TOKENIZER.vocab_size).cuda() + one_hot_bow.scatter_(1, single_bow, 1) + one_hot_bows_vectors.append(one_hot_bow) + return one_hot_bows_vectors + + +def full_text_generation( + model, + context=None, + num_samples=1, + device="cuda", + sample=True, + discrim=None, + label_class=None, + bag_of_words=None, + length=100, + grad_length=10000, + stepsize=0.02, + num_iterations=3, + temperature=1.0, + gm_scale=0.9, + kl_scale=0.01, + top_k=10, + window_length=0, + horizon_length=1, + decay=False, + gamma=1.5, + **kwargs +): + classifier, class_id = get_classifier( + discrim, + label_class, + device + ) + + bow_indices = [] + if bag_of_words: + bow_indices = get_bag_of_words_indices(bag_of_words.split(";")) + + if bag_of_words and classifier: + print("Both PPLM-BoW and PPLM-Discrim are on. This is not optimized.") + loss_type = PPLM_BOW_DISCRIM + + elif bag_of_words: + loss_type = PPLM_BOW + print("Using PPLM-BoW") + + elif classifier is not None: + loss_type = PPLM_DISCRIM + print("Using PPLM-Discrim") + + else: + raise Exception("Specify either --bag_of_words (-B) or --discrim (-D)") + + unpert_gen_tok_text, _, _ = generate_text_pplm( + model=model, + context=context, + device=device, + length=length, + perturb=False + ) + torch.cuda.empty_cache() + + pert_gen_tok_texts = [] + discrim_losses = [] + losses_in_time = [] + + for i in range(num_samples): + pert_gen_tok_text, discrim_loss, loss_in_time = generate_text_pplm( + model=model, + context=context, + device=device, + sample=sample, + perturb=True, + bow_indices=bow_indices, + classifier=classifier, + label_class=class_id, + loss_type=loss_type, + length=length, + grad_length=grad_length, + stepsize=stepsize, + num_iterations=num_iterations, + temperature=temperature, + gm_scale=gm_scale, + kl_scale=kl_scale, + top_k=top_k, + window_length=window_length, + horizon_length=horizon_length, + decay=decay, + gamma=gamma, + ) + pert_gen_tok_texts.append(pert_gen_tok_text) + if classifier is not None: + discrim_losses.append(discrim_loss.data.cpu().numpy()) + losses_in_time.append(loss_in_time) + + torch.cuda.empty_cache() + + return unpert_gen_tok_text, pert_gen_tok_texts, discrim_losses, losses_in_time + + +def generate_text_pplm( + model, + context=None, + past=None, + device="cuda", + sample=True, + perturb=True, + classifier=None, + label_class=None, + bow_indices=None, + loss_type=0, + length=100, + grad_length=10000, + stepsize=0.02, + num_iterations=3, + temperature=1.0, + gm_scale=0.9, + kl_scale=0.01, + top_k=10, + window_length=0, + horizon_length=1, + decay=False, + gamma=1.5, +): + output_so_far = ( + torch.tensor(context, device=device, dtype=torch.long).unsqueeze(0) + if context + else None + ) + + # collect one hot vectors for bags of words + one_hot_bows_vectors = build_bows_one_hot_vectors(bow_indices) + + grad_norms = None + last = None + unpert_discrim_loss = 0 + loss_in_time = [] + for i in trange(length, ascii=True): + + # Get past/probs for current output, except for last word + # Note that GPT takes 2 inputs: past + current_token + + # run model forward to obtain unperturbed + if past is None and output_so_far is not None: + last = output_so_far[:, -1:] + if output_so_far.shape[1] > 1: + _, past, _ = model(output_so_far[:, :-1]) + + unpert_logits, unpert_past, unpert_all_hidden = model(output_so_far) + unpert_last_hidden = unpert_all_hidden[-1] + + else: + unpert_logits, unpert_past, unpert_all_hidden = model(output_so_far) + unpert_last_hidden = unpert_all_hidden[-1] + + # check if we are abowe grad max length + if i >= grad_length: + current_stepsize = stepsize * 0 + else: + current_stepsize = stepsize + + # modify the past if necessary + if not perturb or num_iterations == 0: + pert_past = past + + else: + accumulated_hidden = unpert_last_hidden[:, :-1, :] + accumulated_hidden = torch.sum(accumulated_hidden, dim=1) + + if past is not None: + pert_past, _, grad_norms, loss_this_iter = perturb_past( + past, + model, + last, + unpert_past=unpert_past, + unpert_logits=unpert_logits, + accumulated_hidden=accumulated_hidden, + grad_norms=grad_norms, + stepsize=current_stepsize, + classifier=classifier, + label_class=label_class, + one_hot_bows_vectors=one_hot_bows_vectors, + loss_type=loss_type, + num_iterations=num_iterations, + kl_scale=kl_scale, + window_length=window_length, + horizon_length=horizon_length, + decay=decay, + gamma=gamma, + ) + loss_in_time.append(loss_this_iter) + else: + pert_past = past + + pert_logits, past, pert_all_hidden = model(last, past=pert_past) + pert_logits = pert_logits[:, -1, :] / temperature + pert_probs = F.softmax(pert_logits, dim=-1) + + # compute the discriminator loss using unperturbed hidden + if classifier is not None: + prediction = classifier(torch.mean(unpert_last_hidden, dim=1)) + label = torch.tensor([label_class], device="cuda", dtype=torch.long) + unpert_discrim_loss = torch.nn.CrossEntropyLoss()(prediction, label) + print( + "unperturbed discrim loss", + unpert_discrim_loss.data.cpu().numpy() + ) + else: + unpert_discrim_loss = 0 + + # Fuse the modified model and original model probabilities + if perturb: + unpert_probs = F.softmax(unpert_logits[:, -1, :], dim=-1) + + pert_probs = (pert_probs ** gm_scale) * ( + unpert_probs ** (1 - gm_scale) + ) + + pert_probs = top_k_filter(pert_probs, k=top_k, probs=True) + + # rescale + if torch.sum(pert_probs) <= 1: + pert_probs = pert_probs / torch.sum(pert_probs) + + else: + pert_logits = top_k_filter(pert_logits, k=top_k) + pert_probs = F.softmax(pert_logits, dim=-1) + + # sample or greedy + if sample: + last = torch.multinomial(pert_probs, num_samples=1) + + else: + _, last = torch.topk(pert_probs, k=1, dim=-1) + + # update context/output_so_far appending the new token + output_so_far = ( + last if output_so_far is None + else torch.cat((output_so_far, last), dim=1) + ) + print(TOKENIZER.decode(output_so_far.tolist()[0])) + + return output_so_far, unpert_discrim_loss, loss_in_time + + +def run_model(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--model_path", + "-M", + type=str, + default="gpt2-medium", + help="pretrained model name or path to local checkpoint", + ) + parser.add_argument( + "--bag_of_words", + "-B", + type=str, + default=None, + help="Bags of words used for PPLM-BoW. Either a BOW id (see list in code) or a filepath. Multiple BoWs separated by ;", + ) + parser.add_argument( + "--discrim", + "-D", + type=str, + default=None, + choices=("clickbait", "sentiment", "toxicity"), + help="Discriminator to use for loss-type 2", + ) + parser.add_argument( + "--label_class", + type=int, + default=-1, + help="Class label used for the discriminator", + ) + parser.add_argument("--stepsize", type=float, default=0.02) + parser.add_argument("--length", type=int, default=100) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--temperature", type=float, default=1.0) + parser.add_argument("--top_k", type=int, default=10) + parser.add_argument("--gm_scale", type=float, default=0.9) + parser.add_argument("--kl_scale", type=float, default=0.01) + parser.add_argument("--no_cuda", action="store_true", help="no cuda") + parser.add_argument( + "--uncond", action="store_true", + help="Generate from end-of-text as prefix" + ) + parser.add_argument( + "--cond_text", type=str, default="The lake", + help="Prefix texts to condition on" + ) + parser.add_argument("--num_iterations", type=int, default=3) + parser.add_argument("--grad_length", type=int, default=10000) + parser.add_argument( + "--num_samples", + type=int, + default=1, + help="Number of samples to generate from the modified latents", + ) + parser.add_argument( + "--horizon_length", + type=int, + default=1, + help="Length of future to optimize over", + ) + parser.add_argument( + "--window_length", + type=int, + default=0, + help="Length of past which is being optimized; " + "0 corresponds to infinite window length", + ) + parser.add_argument("--decay", action="store_true", + help="whether to decay or not") + parser.add_argument("--gamma", type=float, default=1.5) + + args = parser.parse_args() + + # set Random seed + torch.manual_seed(args.seed) + np.random.seed(args.seed) + + # set the device + device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") + + # load pretrained model + model = GPT2LMHeadModel.from_pretrained( + args.model_path, + output_hidden_states=True + ) + model.to(device) + model.eval() + + # freeze GPT-2 weights + for param in model.parameters(): + param.requires_grad = False + + # figure out conditioning text + if args.uncond: + tokenized_cond_text = TOKENIZER.encode( + [TOKENIZER.bos_token] + ) + else: + raw_text = args.cond_text + while not raw_text: + print("Did you forget to add `--cond_text`? ") + raw_text = input("Model prompt >>> ") + tokenized_cond_text = TOKENIZER.encode(TOKENIZER.bos_token + raw_text) + + print("= Prefix of sentence =") + print(TOKENIZER.decode(tokenized_cond_text)) + print() + + # generate unperturbed and perturbed texts + + # full_text_generation returns: + # unpert_gen_tok_text, pert_gen_tok_texts, discrim_losses, losses_in_time + unpert_gen_tok_text, pert_gen_tok_texts, _, _ = full_text_generation( + model=model, context=tokenized_cond_text, device=device, **vars(args) + ) + + # untokenize unperturbed text + unpert_gen_text = TOKENIZER.decode(unpert_gen_tok_text.tolist()[0]) + + print("=" * 80) + print("= Unperturbed generated text =") + print(unpert_gen_text) + print() + + generated_texts = [] + + # iterate through the perturbed texts + for i, pert_gen_tok_text in enumerate(pert_gen_tok_texts): + try: + # untokenize unperturbed text + unpert_gen_text = TOKENIZER.decode(pert_gen_tok_text.tolist()[0]) + + print("= Perturbed generated text {} =".format(i + 1)) + print(unpert_gen_text) + print() + except: + pass + + # keep the prefix, perturbed seq, original seq for each index + generated_texts.append( + (tokenized_cond_text, pert_gen_tok_text, unpert_gen_tok_text) + ) + + return generated_texts + + +if __name__ == "__main__": + run_model() From 83b1e6ac9e81cbb053ee272a4a4fcb0b6fac06ab Mon Sep 17 00:00:00 2001 From: Rosanne Liu Date: Sun, 3 Nov 2019 04:51:57 +0000 Subject: [PATCH 175/505] fix the loss backward issue (cherry picked from commit 566468cc984c6ec7e10dfc62b5b4191781a99cd2) --- examples/run_pplm.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/examples/run_pplm.py b/examples/run_pplm.py index 30853c68c3..59ae8a9299 100644 --- a/examples/run_pplm.py +++ b/examples/run_pplm.py @@ -36,6 +36,7 @@ from tqdm import trange from transformers import GPT2Tokenizer from transformers.file_utils import cached_path from transformers.modeling_gpt2 import GPT2LMHeadModel +from IPython import embed PPLM_BOW = 1 PPLM_DISCRIM = 2 @@ -246,8 +247,8 @@ def perturb_past( inputs_embeds=inputs_embeds ) # get expected hidden states - unpert_hidden = curr_all_hidden[1] - accumulated_hidden += torch.sum(unpert_hidden, dim=1) + unpert_hidden = curr_all_hidden[-1] + accumulated_hidden += torch.sum(unpert_hidden, dim=1).detach() prediction = classifier( accumulated_hidden / (curr_length + 1 + horizon_length) @@ -257,7 +258,7 @@ def perturb_past( discrim_loss += ce_loss(prediction, label) print(" pplm_discrim_loss:", discrim_loss.data.cpu().numpy()) - if kl_scale > 0.0: + if kl_scale >= 0.0: unpert_probs = F.softmax(unpert_logits[:, -1, :], dim=-1) unpert_probs = ( unpert_probs + SMALL_CONST * @@ -270,7 +271,7 @@ def perturb_past( torch.FloatTensor ).cuda().detach() corrected_probs = probs + correction.detach() - kl_loss += kl_scale * ( + kl_loss = kl_scale * ( (corrected_probs * (corrected_probs / unpert_probs).log()).sum() ) print(' kl_loss', (kl_loss).data.cpu().numpy()) @@ -280,7 +281,7 @@ def perturb_past( print(' pplm_loss', (loss - kl_loss).data.cpu().numpy()) # compute gradients - loss.backward(retain_graph=True) + loss.backward() # calculate gradient norms if grad_norms is not None and loss_type == PPLM_BOW: From 0b77d66a6dac359c3473d6d4b7e799af5196ae4d Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Tue, 5 Nov 2019 15:35:51 +0000 Subject: [PATCH 176/505] rm extraneous import --- examples/run_pplm.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/run_pplm.py b/examples/run_pplm.py index 59ae8a9299..2d4dee72a3 100644 --- a/examples/run_pplm.py +++ b/examples/run_pplm.py @@ -36,7 +36,6 @@ from tqdm import trange from transformers import GPT2Tokenizer from transformers.file_utils import cached_path from transformers.modeling_gpt2 import GPT2LMHeadModel -from IPython import embed PPLM_BOW = 1 PPLM_DISCRIM = 2 From d5faa74cd6d7de66a058a9b3368e5cbc6dcaf4d6 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Tue, 5 Nov 2019 15:48:00 +0000 Subject: [PATCH 177/505] tokenizer white space: revert to previous behavior --- examples/run_pplm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/run_pplm.py b/examples/run_pplm.py index 2d4dee72a3..4b1a6a2b6f 100644 --- a/examples/run_pplm.py +++ b/examples/run_pplm.py @@ -373,7 +373,7 @@ def get_bag_of_words_indices(bag_of_words_ids_or_paths: List[str]) -> List[List[ filepath = id_or_path with open(filepath, "r") as f: words = f.read().split("\n") - bow_indices.append([TOKENIZER.encode(word) for word in words]) + bow_indices.append([TOKENIZER.encode(word, add_prefix_space=True) for word in words]) return bow_indices From 34a83faabeb8f1f0e487a70f31a4e6b1cc0185e1 Mon Sep 17 00:00:00 2001 From: Piero Molino Date: Mon, 25 Nov 2019 19:15:25 -0800 Subject: [PATCH 178/505] Let's make PPLM great again --- examples/run_pplm.py | 908 +++++++++++++++++++++---------------------- 1 file changed, 440 insertions(+), 468 deletions(-) diff --git a/examples/run_pplm.py b/examples/run_pplm.py index 4b1a6a2b6f..2f853d15c1 100644 --- a/examples/run_pplm.py +++ b/examples/run_pplm.py @@ -1,3 +1,4 @@ +#! /usr/bin/env python3 # coding=utf-8 # Copyright 2018 The Uber AI Team Authors. # @@ -37,10 +38,12 @@ from transformers import GPT2Tokenizer from transformers.file_utils import cached_path from transformers.modeling_gpt2 import GPT2LMHeadModel + PPLM_BOW = 1 PPLM_DISCRIM = 2 PPLM_BOW_DISCRIM = 3 SMALL_CONST = 1e-15 +SmallConst = 1e-15 TOKENIZER = GPT2Tokenizer.from_pretrained("gpt2-medium") BAG_OF_WORDS_ARCHIVE_MAP = { @@ -65,7 +68,7 @@ DISCRIMINATOR_MODELS_PARAMS = { "default_class": 1, }, "sentiment": { - "url": "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/discriminators/sentiment_classifierhead.pt", + "url": "http://s.yosinski.com/SST_classifier_head.pt", "class_size": 5, "embed_size": 1024, "class_vocab": {"very_positive": 2, "very_negative": 3}, @@ -81,6 +84,30 @@ DISCRIMINATOR_MODELS_PARAMS = { } +def to_var(x, requires_grad=False, volatile=False): + if torch.cuda.is_available(): + x = x.cuda() + return Variable(x, requires_grad=requires_grad, volatile=volatile) + + +def top_k_filter(logits, k, probs=False): + """ + Masks everything but the k top entries as -infinity (1e10). + Used to mask logits such that e^-infinity -> 0 won't contribute to the + sum of the denominator. + """ + if k == 0: + return logits + else: + values = torch.topk(logits, k)[0] + batch_mins = values[:, -1].view(-1, 1).expand_as(logits) + if probs: + return torch.where(logits < batch_mins, + torch.ones_like(logits) * 0.0, logits) + return torch.where(logits < batch_mins, torch.ones_like(logits) * -1e10, + logits) + + class ClassificationHead(torch.nn.Module): """ Classification Head for the transformer """ @@ -99,234 +126,175 @@ class ClassificationHead(torch.nn.Module): return logits -def to_var(x, requires_grad=False, volatile=False): - if torch.cuda.is_available(): - x = x.cuda() - return Variable(x, requires_grad=requires_grad, volatile=volatile) +def perturb_past(past, model, prev, args, classifier, good_index=None, + stepsize=0.01, vocab_size=50257, + original_probs=None, accumulated_hidden=None, true_past=None, + grad_norms=None): + window_length = args.window_length + gm_scale, kl_scale = args.gm_scale, args.kl_scale + one_hot_vectors = [] + for good_list in good_index: + good_list = list(filter(lambda x: len(x) <= 1, good_list)) + good_list = torch.tensor(good_list).cuda() + num_good = good_list.shape[0] + one_hot_good = torch.zeros(num_good, vocab_size).cuda() + one_hot_good.scatter_(1, good_list, 1) + one_hot_vectors.append(one_hot_good) - -def top_k_filter(logits, k, probs=False): - """ - Masks everything but the k top entries as -infinity (1e10). - Used to mask logits such that e^-infinity -> 0 won't contribute to the - sum of the denominator. - """ - if k <= 0: - return logits - - else: - values = torch.topk(logits, k)[0] - batch_mins = values[:, -1].view(-1, 1).expand_as(logits) - - if probs: - return torch.where( - logits < batch_mins, - torch.ones_like(logits) * 0.0, - logits - ) - - return torch.where( - logits < batch_mins, - torch.ones_like(logits) * -1e10, - logits - ) - - -def perturb_past( - past, - model, - last, - unpert_past=None, - unpert_logits=None, - accumulated_hidden=None, - grad_norms=None, - stepsize=0.01, - classifier=None, - label_class=None, - one_hot_bows_vectors=None, - loss_type=0, - num_iterations=3, - kl_scale=0.01, - window_length=0, - horizon_length=1, - decay=False, - gamma=1.5, -): - # initializie perturbation accumulator - grad_accumulator = [ - (np.zeros(p.shape).astype("float32")) - for p in past - ] + # Generate inital perturbed past + past_perturb_orig = [ + (np.random.uniform(0.0, 0.0, p.shape).astype('float32')) + for p in past] if accumulated_hidden is None: accumulated_hidden = 0 - if decay: - decay_mask = torch.arange( - 0.0, - 1.0 + SMALL_CONST, - 1.0 / (window_length) - )[1:] + if args.decay: + decay_mask = torch.arange(0., 1.0 + SmallConst, 1.0 / (window_length))[ + 1:] else: decay_mask = 1.0 - # TODO fix this comment (SUMANTH) - # generate a mask if perturbated gradient is based on a past window - _, _, _, curr_length, _ = past[0].shape - if curr_length > window_length and window_length > 0: - ones_key_val_shape = ( - tuple(past[0].shape[:-2]) - + tuple([window_length]) - + tuple(past[0].shape[-1:]) - ) + # Generate a mask is gradient perturbated is based on a past window + _, _, _, current_length, _ = past[0].shape - zeros_key_val_shape = ( - tuple(past[0].shape[:-2]) - + tuple([curr_length - window_length]) - + tuple(past[0].shape[-1:]) - ) + if current_length > window_length and window_length > 0: + ones_key_val_shape = tuple(past[0].shape[:-2]) + tuple( + [window_length]) + tuple( + past[0].shape[-1:]) + + zeros_key_val_shape = tuple(past[0].shape[:-2]) + tuple( + [current_length - window_length]) + tuple( + past[0].shape[-1:]) ones_mask = torch.ones(ones_key_val_shape) ones_mask = decay_mask * ones_mask.permute(0, 1, 2, 4, 3) ones_mask = ones_mask.permute(0, 1, 2, 4, 3) - window_mask = torch.cat( - (ones_mask, torch.zeros(zeros_key_val_shape)), - dim=-2 - ).cuda() - + window_mask = torch.cat((ones_mask, torch.zeros(zeros_key_val_shape)), + dim=-2).cuda() else: window_mask = torch.ones_like(past[0]).cuda() - # accumulate perturbations for num_iterations loss_per_iter = [] - for i in range(num_iterations): + for i in range(args.num_iterations): print("Iteration ", i + 1) + past_perturb = [torch.from_numpy(p_) for p_ in past_perturb_orig] + past_perturb = [to_var(p_, requires_grad=True) for p_ in past_perturb] - curr_perturbation = [ - to_var(torch.from_numpy(p_), requires_grad=True) - for p_ in grad_accumulator - ] + perturbed_past = list(map(add, past, past_perturb)) - # Compute hidden using perturbed past - curr_pert_past = list(map(add, past, curr_perturbation)) - all_logits, _, all_hidden = model(last, past=curr_pert_past) + _, _, _, current_length, _ = past_perturb[0].shape + + # _, future_past = model(prev, past=perturbed_past) + # hidden = model.hidden_states + + # Piero modified model call + logits, _, all_hidden = model(prev, past=perturbed_past) hidden = all_hidden[-1] - accumulated_hidden += torch.sum(hidden, dim=1).detach() - logits = all_logits[:, -1, :] - probs = F.softmax(logits, dim=-1) + new_accumulated_hidden = accumulated_hidden + torch.sum(hidden, + dim=1).detach() - # compute loss - bow_loss = 0.0 - discrim_loss = 0.0 - kl_loss = 0.0 + # TODO: Check the layer-norm consistency of this with trained discriminator + logits = logits[:, -1, :] + probabs = F.softmax(logits, dim=-1) + loss = 0.0 + loss_list = [] + if args.loss_type == 1 or args.loss_type == 3: + for one_hot_good in one_hot_vectors: + good_logits = torch.mm(probabs, torch.t(one_hot_good)) + loss_word = good_logits + loss_word = torch.sum(loss_word) + loss_word = -torch.log(loss_word) + # loss_word = torch.sum(loss_word) /torch.sum(one_hot_good) + loss += loss_word + loss_list.append(loss_word) + print(" pplm_bow_loss:", loss.data.cpu().numpy()) - if loss_type == PPLM_BOW or loss_type == PPLM_BOW_DISCRIM: - for one_hot_bow in one_hot_bows_vectors: - bow_logits = torch.mm(probs, torch.t(one_hot_bow)) - bow_loss += -torch.log(torch.sum(bow_logits)) - print(" pplm_bow_loss:", bow_loss.data.cpu().numpy()) - - if loss_type == PPLM_DISCRIM or loss_type == PPLM_BOW_DISCRIM: + if args.loss_type == 2 or args.loss_type == 3: ce_loss = torch.nn.CrossEntropyLoss() - # TODO all there are for (SUMANTH) - # TODO why we need to do this assignment and not just using unpert_past? - curr_unpert_past = unpert_past - # Get the model's token embeddings in order to compute our own embeds from curr_probs: - wte = model.resize_token_embeddings() - # TODO i is never used, why do we need to do this i times instead multiplying - # torch.sum(unpert_hidden, dim=1) * horizon_length? - for i in range(horizon_length): - # TODO the next two lines can be done only one time, and why not using probs instead as they do not change at each iteration? - curr_probs = F.softmax(logits, dim=-1) # get softmax - curr_probs = torch.unsqueeze(curr_probs, dim=1) - inputs_embeds = torch.matmul(curr_probs, wte.weight.data) - _, curr_unpert_past, curr_all_hidden = model( - past=curr_unpert_past, + new_true_past = true_past + for i in range(args.horizon_length): + future_probabs = F.softmax(logits, dim=-1) # Get softmax + future_probabs = torch.unsqueeze(future_probabs, dim=1) + + # _, new_true_past = model(future_probabs, past=new_true_past) + # future_hidden = model.hidden_states # Get expected hidden states + + # Piero modified model call + wte = model.resize_token_embeddings() + inputs_embeds = torch.matmul(future_probabs, wte.weight.data) + _, new_true_past, future_hidden = model( + past=new_true_past, inputs_embeds=inputs_embeds ) - # get expected hidden states - unpert_hidden = curr_all_hidden[-1] - accumulated_hidden += torch.sum(unpert_hidden, dim=1).detach() + future_hidden = future_hidden[-1] - prediction = classifier( - accumulated_hidden / (curr_length + 1 + horizon_length) - ) + new_accumulated_hidden = new_accumulated_hidden + torch.sum( + future_hidden, dim=1) - label = torch.tensor([label_class], device="cuda", dtype=torch.long) - discrim_loss += ce_loss(prediction, label) + predicted_sentiment = classifier(new_accumulated_hidden / ( + current_length + 1 + args.horizon_length)) + + label = torch.tensor([args.label_class], device='cuda', + dtype=torch.long) + discrim_loss = ce_loss(predicted_sentiment, label) print(" pplm_discrim_loss:", discrim_loss.data.cpu().numpy()) + loss += discrim_loss + loss_list.append(discrim_loss) - if kl_scale >= 0.0: - unpert_probs = F.softmax(unpert_logits[:, -1, :], dim=-1) - unpert_probs = ( - unpert_probs + SMALL_CONST * - (unpert_probs <= SMALL_CONST).type( - torch.FloatTensor - ).cuda().detach() - ) - - correction = SMALL_CONST * (probs <= SMALL_CONST).type( - torch.FloatTensor - ).cuda().detach() - corrected_probs = probs + correction.detach() + kl_loss = 0.0 + if kl_scale > 0.0: + p = (F.softmax(original_probs[:, -1, :], dim=-1)) + p = p + SmallConst * (p <= SmallConst).type( + torch.FloatTensor).cuda().detach() + correction = SmallConst * (probabs <= SmallConst).type( + torch.FloatTensor).cuda().detach() + corrected_probabs = probabs + correction.detach() kl_loss = kl_scale * ( - (corrected_probs * (corrected_probs / unpert_probs).log()).sum() - ) + (corrected_probabs * (corrected_probabs / p).log()).sum()) print(' kl_loss', (kl_loss).data.cpu().numpy()) + loss += kl_loss # + discrim_loss - loss = bow_loss + discrim_loss + kl_loss loss_per_iter.append(loss.data.cpu().numpy()) + print(' pplm_loss', (loss - kl_loss).data.cpu().numpy()) - # compute gradients loss.backward() - - # calculate gradient norms - if grad_norms is not None and loss_type == PPLM_BOW: + if grad_norms is not None and args.loss_type == 1: grad_norms = [ torch.max(grad_norms[index], torch.norm(p_.grad * window_mask)) - for index, p_ in enumerate(curr_perturbation) - ] + for index, p_ in + enumerate(past_perturb)] else: - grad_norms = [ - (torch.norm(p_.grad * window_mask) + SMALL_CONST) - for index, p_ in enumerate(curr_perturbation) - ] + grad_norms = [(torch.norm(p_.grad * window_mask) + SmallConst) for + index, p_ in enumerate(past_perturb)] - # normalize gradients grad = [ - -stepsize - * (p_.grad * window_mask / grad_norms[ - index] ** gamma).data.cpu().numpy() - for index, p_ in enumerate(curr_perturbation) - ] + -stepsize * (p_.grad * window_mask / grad_norms[ + index] ** args.gamma).data.cpu().numpy() + for index, p_ in enumerate(past_perturb)] + past_perturb_orig = list(map(add, grad, past_perturb_orig)) - # accumulate gradients - grad_accumulator = list(map(add, grad, grad_accumulator)) - - # reset gradients, just to make sure - for p_ in curr_perturbation: + for p_ in past_perturb: p_.grad.data.zero_() - # removing past from the graph new_past = [] - for p_ in past: - new_past.append(p_.detach()) + for p in past: + new_past.append(p.detach()) + past = new_past - # apply the accumulated perturbations to the past - grad_accumulator = [ - to_var(torch.from_numpy(p_), requires_grad=True) - for p_ in grad_accumulator - ] - pert_past = list(map(add, past, grad_accumulator)) + past_perturb = [torch.from_numpy(p_) for p_ in past_perturb_orig] + past_perturb = [to_var(p_, requires_grad=True) for p_ in past_perturb] + perturbed_past = list(map(add, past, past_perturb)) - return pert_past, accumulated_hidden, grad_norms, loss_per_iter + return perturbed_past, new_accumulated_hidden, grad_norms, loss_per_iter def get_classifier( - name: Optional[str], label_class: Union[str, int], device: Union[str, torch.device] + name: Optional[str], label_class: Union[str, int], + device: Union[str, torch.device] ) -> Tuple[Optional[ClassificationHead], Optional[int]]: if name is None: return None, None @@ -337,7 +305,8 @@ def get_classifier( embed_size=params['embed_size'] ).to(device) resolved_archive_file = cached_path(params["url"]) - classifier.load_state_dict(torch.load(resolved_archive_file, map_location=device)) + classifier.load_state_dict( + torch.load(resolved_archive_file, map_location=device)) classifier.eval() if isinstance(label_class, str): @@ -364,7 +333,8 @@ def get_classifier( return classifier, label_id -def get_bag_of_words_indices(bag_of_words_ids_or_paths: List[str]) -> List[List[List[int]]]: +def get_bag_of_words_indices(bag_of_words_ids_or_paths: List[str]) -> List[ + List[List[int]]]: bow_indices = [] for id_or_path in bag_of_words_ids_or_paths: if id_or_path in BAG_OF_WORDS_ARCHIVE_MAP: @@ -372,8 +342,10 @@ def get_bag_of_words_indices(bag_of_words_ids_or_paths: List[str]) -> List[List[ else: filepath = id_or_path with open(filepath, "r") as f: - words = f.read().split("\n") - bow_indices.append([TOKENIZER.encode(word, add_prefix_space=True) for word in words]) + words = f.read().strip().split("\n") + bow_indices.append( + [TOKENIZER.encode(word.strip(), add_prefix_space=True) for word in + words]) return bow_indices @@ -392,327 +364,308 @@ def build_bows_one_hot_vectors(bow_indices): return one_hot_bows_vectors -def full_text_generation( - model, - context=None, - num_samples=1, - device="cuda", - sample=True, - discrim=None, - label_class=None, - bag_of_words=None, - length=100, - grad_length=10000, - stepsize=0.02, - num_iterations=3, - temperature=1.0, - gm_scale=0.9, - kl_scale=0.01, - top_k=10, - window_length=0, - horizon_length=1, - decay=False, - gamma=1.5, - **kwargs -): +def latent_perturb(model, args, context=None, sample=True, device='cuda'): classifier, class_id = get_classifier( - discrim, - label_class, + args.discrim, + args.label_class, device ) - bow_indices = [] - if bag_of_words: - bow_indices = get_bag_of_words_indices(bag_of_words.split(";")) + # if args.discrim == 'clickbait': + # classifier = ClassificationHead(class_size=2, embed_size=1024).to(device) + # classifier.load_state_dict(torch.load("discrim_models/clickbait_classifierhead.pt")) + # classifier.eval() + # args.label_class = 1 # clickbaity + # + # elif args.discrim == 'sentiment': + # classifier = ClassificationHead(class_size=5, embed_size=1024).to(device) + # #classifier.load_state_dict(torch.load("discrim_models/sentiment_classifierhead.pt")) + # classifier.load_state_dict(torch.load("discrim_models/SST_classifier_head_epoch_16.pt")) + # classifier.eval() + # if args.label_class < 0: + # raise Exception('Wrong class for sentiment, use --label-class 2 for *very positive*, 3 for *very negative*') + # #args.label_class = 2 # very pos + # #args.label_class = 3 # very neg + # + # elif args.discrim == 'toxicity': + # classifier = ClassificationHead(class_size=2, embed_size=1024).to(device) + # classifier.load_state_dict(torch.load("discrim_models/toxicity_classifierhead.pt")) + # classifier.eval() + # args.label_class = 0 # not toxic + # + # elif args.discrim == 'generic': + # if args.discrim_weights is None: + # raise ValueError('When using a generic discriminator, ' + # 'discrim_weights need to be specified') + # if args.discrim_meta is None: + # raise ValueError('When using a generic discriminator, ' + # 'discrim_meta need to be specified') + # + # with open(args.discrim_meta, 'r') as discrim_meta_file: + # meta = json.load(discrim_meta_file) + # + # classifier = ClassificationHead( + # class_size=meta['class_size'], + # embed_size=meta['embed_size'], + # # todo add tokenizer from meta + # ).to(device) + # classifier.load_state_dict(torch.load(args.discrim_weights)) + # classifier.eval() + # if args.label_class == -1: + # args.label_class = meta['default_class'] + # + # else: + # classifier = None - if bag_of_words and classifier: + # Get tokens for the list of positive words + def list_tokens(word_list): + token_list = [TOKENIZER.encode(word, add_prefix_space=True) for word in + word_list] + # token_list = [] + # for word in word_list: + # token_list.append(TOKENIZER.encode(" " + word)) + return token_list + + # good_index = [] + # if args.bag_of_words: + # bags_of_words = args.bag_of_words.split(";") + # for wordlist in bags_of_words: + # with open(wordlist, "r") as f: + # words = f.read().strip() + # words = words.split('\n') + # good_index.append(list_tokens(words)) + # + # for good_list in good_index: + # good_list = list(filter(lambda x: len(x) <= 1, good_list)) + # actual_words = [(TOKENIZER.decode(ww).strip(),ww) for ww in good_list] + + good_index = [] + actual_words = None + if args.bag_of_words: + good_index = get_bag_of_words_indices(args.bag_of_words.split(";")) + + for good_list in good_index: + good_list = list(filter(lambda x: len(x) <= 1, good_list)) + actual_words = [(TOKENIZER.decode(ww).strip(), ww) for ww in + good_list] + + if args.bag_of_words and classifier: print("Both PPLM-BoW and PPLM-Discrim are on. This is not optimized.") - loss_type = PPLM_BOW_DISCRIM + args.loss_type = PPLM_BOW_DISCRIM - elif bag_of_words: - loss_type = PPLM_BOW + elif args.bag_of_words: + args.loss_type = PPLM_BOW print("Using PPLM-BoW") elif classifier is not None: - loss_type = PPLM_DISCRIM + args.loss_type = PPLM_DISCRIM print("Using PPLM-Discrim") else: raise Exception("Specify either --bag_of_words (-B) or --discrim (-D)") - unpert_gen_tok_text, _, _ = generate_text_pplm( - model=model, - context=context, - device=device, - length=length, - perturb=False - ) + original, _, _ = sample_from_hidden(model=model, args=args, context=context, + device=device, + perturb=False, good_index=good_index, + classifier=classifier) torch.cuda.empty_cache() - pert_gen_tok_texts = [] - discrim_losses = [] - losses_in_time = [] + perturbed_list = [] + discrim_loss_list = [] + loss_in_time_list = [] - for i in range(num_samples): - pert_gen_tok_text, discrim_loss, loss_in_time = generate_text_pplm( - model=model, - context=context, - device=device, - sample=sample, - perturb=True, - bow_indices=bow_indices, - classifier=classifier, - label_class=class_id, - loss_type=loss_type, - length=length, - grad_length=grad_length, - stepsize=stepsize, - num_iterations=num_iterations, - temperature=temperature, - gm_scale=gm_scale, - kl_scale=kl_scale, - top_k=top_k, - window_length=window_length, - horizon_length=horizon_length, - decay=decay, - gamma=gamma, - ) - pert_gen_tok_texts.append(pert_gen_tok_text) + for i in range(args.num_samples): + perturbed, discrim_loss, loss_in_time = sample_from_hidden(model=model, + args=args, + context=context, + device=device, + perturb=True, + good_index=good_index, + classifier=classifier) + perturbed_list.append(perturbed) if classifier is not None: - discrim_losses.append(discrim_loss.data.cpu().numpy()) - losses_in_time.append(loss_in_time) + discrim_loss_list.append(discrim_loss.data.cpu().numpy()) + loss_in_time_list.append(loss_in_time) torch.cuda.empty_cache() - return unpert_gen_tok_text, pert_gen_tok_texts, discrim_losses, losses_in_time + return original, perturbed_list, discrim_loss_list, loss_in_time_list, actual_words -def generate_text_pplm( - model, - context=None, - past=None, - device="cuda", - sample=True, - perturb=True, - classifier=None, - label_class=None, - bow_indices=None, - loss_type=0, - length=100, - grad_length=10000, - stepsize=0.02, - num_iterations=3, - temperature=1.0, - gm_scale=0.9, - kl_scale=0.01, - top_k=10, - window_length=0, - horizon_length=1, - decay=False, - gamma=1.5, -): - output_so_far = ( - torch.tensor(context, device=device, dtype=torch.long).unsqueeze(0) - if context - else None - ) - - # collect one hot vectors for bags of words - one_hot_bows_vectors = build_bows_one_hot_vectors(bow_indices) +def sample_from_hidden(model, args, classifier, context=None, past=None, + device='cuda', + sample=True, perturb=True, good_index=None): + output = torch.tensor(context, device=device, dtype=torch.long).unsqueeze( + 0) if context else None grad_norms = None - last = None - unpert_discrim_loss = 0 loss_in_time = [] - for i in trange(length, ascii=True): + for i in trange(args.length, ascii=True): # Get past/probs for current output, except for last word - # Note that GPT takes 2 inputs: past + current_token + # Note that GPT takes 2 inputs: past + current-token + # Therefore, use everything from before current i/p token to generate relevant past - # run model forward to obtain unperturbed - if past is None and output_so_far is not None: - last = output_so_far[:, -1:] - if output_so_far.shape[1] > 1: - _, past, _ = model(output_so_far[:, :-1]) + if past is None and output is not None: + prev = output[:, -1:] + # _, past = model(output[:, :-1]) + # original_probs, true_past = model(output) + # true_hidden = model.hidden_states - unpert_logits, unpert_past, unpert_all_hidden = model(output_so_far) - unpert_last_hidden = unpert_all_hidden[-1] + # Piero modified model call + _, past, _ = model(output[:, :-1]) + original_probs, true_past, unpert_all_hidden = model(output) + true_hidden = unpert_all_hidden[-1] else: - unpert_logits, unpert_past, unpert_all_hidden = model(output_so_far) - unpert_last_hidden = unpert_all_hidden[-1] + # original_probs, true_past = model(output) + # true_hidden = model.hidden_states - # check if we are abowe grad max length - if i >= grad_length: - current_stepsize = stepsize * 0 + # Piero modified model call + original_probs, true_past, unpert_all_hidden = model(output) + true_hidden = unpert_all_hidden[-1] + + # Modify the past if necessary + + if i >= args.grad_length: + current_stepsize = args.stepsize * 0 else: - current_stepsize = stepsize + current_stepsize = args.stepsize - # modify the past if necessary - if not perturb or num_iterations == 0: - pert_past = past + if not perturb or args.num_iterations == 0: + perturbed_past = past else: - accumulated_hidden = unpert_last_hidden[:, :-1, :] + # Piero modified model call + # accumulated_hidden = model.hidden_states[:, :-1, :] + accumulated_hidden = true_hidden[:, :-1, :] accumulated_hidden = torch.sum(accumulated_hidden, dim=1) - if past is not None: - pert_past, _, grad_norms, loss_this_iter = perturb_past( - past, - model, - last, - unpert_past=unpert_past, - unpert_logits=unpert_logits, - accumulated_hidden=accumulated_hidden, - grad_norms=grad_norms, - stepsize=current_stepsize, - classifier=classifier, - label_class=label_class, - one_hot_bows_vectors=one_hot_bows_vectors, - loss_type=loss_type, - num_iterations=num_iterations, - kl_scale=kl_scale, - window_length=window_length, - horizon_length=horizon_length, - decay=decay, - gamma=gamma, - ) - loss_in_time.append(loss_this_iter) - else: - pert_past = past + perturbed_past, _, grad_norms, loss_per_iter = perturb_past(past, + model, + prev, + args, + good_index=good_index, + stepsize=current_stepsize, + original_probs=original_probs, + true_past=true_past, + accumulated_hidden=accumulated_hidden, + classifier=classifier, + grad_norms=grad_norms) + loss_in_time.append(loss_per_iter) - pert_logits, past, pert_all_hidden = model(last, past=pert_past) - pert_logits = pert_logits[:, -1, :] / temperature - pert_probs = F.softmax(pert_logits, dim=-1) + # Piero modified model call + logits, past, pert_all_hidden = model(prev, past=perturbed_past) + # test_logits = F.softmax(test_logits[:, -1, :], dim=-1) + # likelywords = torch.topk(test_logits, k=10, dim=-1) + # print(TOKENIZER.decode(likelywords[1].tolist()[0])) - # compute the discriminator loss using unperturbed hidden if classifier is not None: - prediction = classifier(torch.mean(unpert_last_hidden, dim=1)) - label = torch.tensor([label_class], device="cuda", dtype=torch.long) - unpert_discrim_loss = torch.nn.CrossEntropyLoss()(prediction, label) - print( - "unperturbed discrim loss", - unpert_discrim_loss.data.cpu().numpy() - ) + ce_loss = torch.nn.CrossEntropyLoss() + predicted_sentiment = classifier(torch.mean(true_hidden, dim=1)) + label = torch.tensor([args.label_class], device='cuda', + dtype=torch.long) + true_discrim_loss = ce_loss(predicted_sentiment, label) + print("true discrim loss", true_discrim_loss.data.cpu().numpy()) else: - unpert_discrim_loss = 0 + true_discrim_loss = 0 - # Fuse the modified model and original model probabilities + # Piero modified model call + # hidden = model.hidden_states # update hidden + # logits = model.forward_hidden(hidden) + logits = logits[:, -1, :] / args.temperature # + SmallConst + + # logits = top_k_filter(logits, k=args.top_k) # + SmallConst + + log_probs = F.softmax(logits, dim=-1) + + # Fuse the modified model and original model if perturb: - unpert_probs = F.softmax(unpert_logits[:, -1, :], dim=-1) - pert_probs = (pert_probs ** gm_scale) * ( - unpert_probs ** (1 - gm_scale) - ) + # original_probs = top_k_filter(original_probs[:, -1, :]) #+ SmallConst + original_probs = F.softmax(original_probs[:, -1, :], dim=-1) + # likelywords = torch.topk(original_probs, k=10, dim=-1) + # print(TOKENIZER.decode(likelywords[1].tolist()[0])) - pert_probs = top_k_filter(pert_probs, k=top_k, probs=True) + gm_scale = args.gm_scale + log_probs = ((log_probs ** gm_scale) * ( + original_probs ** (1 - gm_scale))) # + SmallConst - # rescale - if torch.sum(pert_probs) <= 1: - pert_probs = pert_probs / torch.sum(pert_probs) + log_probs = top_k_filter(log_probs, k=args.top_k, + probs=True) # + SmallConst + + if torch.sum(log_probs) <= 1: + log_probs = log_probs / torch.sum(log_probs) else: - pert_logits = top_k_filter(pert_logits, k=top_k) - pert_probs = F.softmax(pert_logits, dim=-1) + logits = top_k_filter(logits, k=args.top_k) # + SmallConst + log_probs = F.softmax(logits, dim=-1) - # sample or greedy if sample: - last = torch.multinomial(pert_probs, num_samples=1) - + # likelywords = torch.topk(log_probs, k=args.top_k, dim=-1) + # print(TOKENIZER.decode(likelywords[1].tolist()[0])) + # print(likelywords[0].tolist()) + prev = torch.multinomial(log_probs, num_samples=1) else: - _, last = torch.topk(pert_probs, k=1, dim=-1) + _, prev = torch.topk(log_probs, k=1, dim=-1) + # if perturb: + # prev = future + output = prev if output is None else torch.cat((output, prev), + dim=1) # update output + print(TOKENIZER.decode(output.tolist()[0])) - # update context/output_so_far appending the new token - output_so_far = ( - last if output_so_far is None - else torch.cat((output_so_far, last), dim=1) - ) - print(TOKENIZER.decode(output_so_far.tolist()[0])) - - return output_so_far, unpert_discrim_loss, loss_in_time + return output, true_discrim_loss, loss_in_time def run_model(): parser = argparse.ArgumentParser() - parser.add_argument( - "--model_path", - "-M", - type=str, - default="gpt2-medium", - help="pretrained model name or path to local checkpoint", - ) - parser.add_argument( - "--bag_of_words", - "-B", - type=str, - default=None, - help="Bags of words used for PPLM-BoW. Either a BOW id (see list in code) or a filepath. Multiple BoWs separated by ;", - ) - parser.add_argument( - "--discrim", - "-D", - type=str, - default=None, - choices=("clickbait", "sentiment", "toxicity"), - help="Discriminator to use for loss-type 2", - ) - parser.add_argument( - "--label_class", - type=int, - default=-1, - help="Class label used for the discriminator", - ) - parser.add_argument("--stepsize", type=float, default=0.02) + parser.add_argument('--model_path', '-M', type=str, default='gpt2-medium', + help='pretrained model name or path to local checkpoint') + parser.add_argument('--bag-of-words', '-B', type=str, default=None, + help='Bags of words used for PPLM-BoW. Multiple BoWs separated by ;') + parser.add_argument('--discrim', '-D', type=str, default=None, + choices=( + 'clickbait', 'sentiment', 'toxicity', 'generic'), + help='Discriminator to use for loss-type 2') + parser.add_argument('--discrim_weights', type=str, default=None, + help='Weights for the generic discriminator') + parser.add_argument('--discrim_meta', type=str, default=None, + help='Meta information for the generic discriminator') + parser.add_argument('--label_class', type=int, default=-1, + help='Class label used for the discriminator') + parser.add_argument('--stepsize', type=float, default=0.02) parser.add_argument("--length", type=int, default=100) parser.add_argument("--seed", type=int, default=0) parser.add_argument("--temperature", type=float, default=1.0) parser.add_argument("--top_k", type=int, default=10) parser.add_argument("--gm_scale", type=float, default=0.9) parser.add_argument("--kl_scale", type=float, default=0.01) - parser.add_argument("--no_cuda", action="store_true", help="no cuda") - parser.add_argument( - "--uncond", action="store_true", - help="Generate from end-of-text as prefix" - ) - parser.add_argument( - "--cond_text", type=str, default="The lake", - help="Prefix texts to condition on" - ) - parser.add_argument("--num_iterations", type=int, default=3) - parser.add_argument("--grad_length", type=int, default=10000) - parser.add_argument( - "--num_samples", - type=int, - default=1, - help="Number of samples to generate from the modified latents", - ) - parser.add_argument( - "--horizon_length", - type=int, - default=1, - help="Length of future to optimize over", - ) - parser.add_argument( - "--window_length", - type=int, - default=0, - help="Length of past which is being optimized; " - "0 corresponds to infinite window length", - ) - parser.add_argument("--decay", action="store_true", - help="whether to decay or not") - parser.add_argument("--gamma", type=float, default=1.5) + parser.add_argument('--nocuda', action='store_true', help='no cuda') + parser.add_argument('--uncond', action='store_true', + help='Generate from end-of-text as prefix') + parser.add_argument("--cond_text", type=str, default='The lake', + help='Prefix texts to condition on') + parser.add_argument('--num_iterations', type=int, default=3) + parser.add_argument('--grad_length', type=int, default=10000) + parser.add_argument('--num_samples', type=int, default=1, + help='Number of samples to generate from the modified latents') + parser.add_argument('--horizon_length', type=int, default=1, + help='Length of future to optimize over') + # parser.add_argument('--force-token', action='store_true', help='no cuda') + parser.add_argument('--window_length', type=int, default=0, + help='Length of past which is being optimizer; 0 corresponds to infinite window length') + parser.add_argument('--decay', action='store_true', + help='whether to decay or not') + parser.add_argument('--gamma', type=float, default=1.5) + parser.add_argument('--colorama', action='store_true', help='no cuda') args = parser.parse_args() - # set Random seed torch.manual_seed(args.seed) np.random.seed(args.seed) - # set the device - device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") + device = 'cpu' if args.nocuda else 'cuda' - # load pretrained model model = GPT2LMHeadModel.from_pretrained( args.model_path, output_hidden_states=True @@ -720,63 +673,82 @@ def run_model(): model.to(device) model.eval() - # freeze GPT-2 weights + # Freeze GPT-2 weights for param in model.parameters(): param.requires_grad = False + pass - # figure out conditioning text if args.uncond: - tokenized_cond_text = TOKENIZER.encode( - [TOKENIZER.bos_token] - ) + seq = [[50256, 50256]] + else: raw_text = args.cond_text while not raw_text: - print("Did you forget to add `--cond_text`? ") + print('Did you forget to add `--cond-text`? ') raw_text = input("Model prompt >>> ") - tokenized_cond_text = TOKENIZER.encode(TOKENIZER.bos_token + raw_text) + seq = [[50256] + TOKENIZER.encode(raw_text)] - print("= Prefix of sentence =") - print(TOKENIZER.decode(tokenized_cond_text)) - print() + collect_gen = dict() + current_index = 0 + for out in seq: - # generate unperturbed and perturbed texts + text = TOKENIZER.decode(out) + print("=" * 40 + " Prefix of sentence " + "=" * 40) + print(text) + print("=" * 80) - # full_text_generation returns: - # unpert_gen_tok_text, pert_gen_tok_texts, discrim_losses, losses_in_time - unpert_gen_tok_text, pert_gen_tok_texts, _, _ = full_text_generation( - model=model, context=tokenized_cond_text, device=device, **vars(args) - ) + out1, out_perturb, discrim_loss_list, loss_in_time_list, actual_words = latent_perturb( + model=model, args=args, context=out, + device=device) - # untokenize unperturbed text - unpert_gen_text = TOKENIZER.decode(unpert_gen_tok_text.tolist()[0]) + text_whole = TOKENIZER.decode(out1.tolist()[0]) - print("=" * 80) - print("= Unperturbed generated text =") - print(unpert_gen_text) - print() + print("=" * 80) + print("=" * 40 + " Whole sentence (Original)" + "=" * 40) + print(text_whole) + print("=" * 80) - generated_texts = [] + out_perturb_copy = out_perturb - # iterate through the perturbed texts - for i, pert_gen_tok_text in enumerate(pert_gen_tok_texts): - try: - # untokenize unperturbed text - unpert_gen_text = TOKENIZER.decode(pert_gen_tok_text.tolist()[0]) + for out_perturb in out_perturb_copy: + # try: + # print("=" * 40 + " Whole sentence (Perturbed)" + "=" * 40) + # text_whole = TOKENIZER.decode(out_perturb.tolist()[0]) + # print(text_whole) + # print("=" * 80) + # except: + # pass + # collect_gen[current_index] = [out, out_perturb, out1] + ## Save the prefix, perturbed seq, original seq for each index + print("=" * 40 + " Whole sentence (Perturbed)" + "=" * 40) + keyword_tokens = [aa[-1][0] for aa in + actual_words] if actual_words else [] + output_tokens = out_perturb.tolist()[0] - print("= Perturbed generated text {} =".format(i + 1)) - print(unpert_gen_text) - print() - except: - pass + if args.colorama: + import colorama - # keep the prefix, perturbed seq, original seq for each index - generated_texts.append( - (tokenized_cond_text, pert_gen_tok_text, unpert_gen_tok_text) - ) + text_whole = '' + for out in output_tokens: + if out in keyword_tokens: + text_whole += '%s%s%s' % ( + colorama.Fore.GREEN, TOKENIZER.decode([out]), + colorama.Style.RESET_ALL) + else: + text_whole += TOKENIZER.decode([out]) + else: + text_whole = TOKENIZER.decode(out_perturb.tolist()[0]) - return generated_texts + print(text_whole) + print("=" * 80) + + collect_gen[current_index] = [out, out_perturb, out1] + + current_index = current_index + 1 -if __name__ == "__main__": + return + + +if __name__ == '__main__': run_model() From 0b51fba20bd88c4cc4acbb3e9dce82980719895c Mon Sep 17 00:00:00 2001 From: piero Date: Tue, 26 Nov 2019 13:15:56 -0800 Subject: [PATCH 179/505] Added script for training a discriminator for pplm to use --- examples/run_pplm.py | 19 +- examples/run_pplm_discrim_train.py | 582 +++++++++++++++++++++++++++++ 2 files changed, 583 insertions(+), 18 deletions(-) create mode 100644 examples/run_pplm_discrim_train.py diff --git a/examples/run_pplm.py b/examples/run_pplm.py index 2f853d15c1..217c131b8f 100644 --- a/examples/run_pplm.py +++ b/examples/run_pplm.py @@ -34,6 +34,7 @@ import torch.nn.functional as F from torch.autograd import Variable from tqdm import trange +from examples.run_pplm_discrim_train import ClassificationHead from transformers import GPT2Tokenizer from transformers.file_utils import cached_path from transformers.modeling_gpt2 import GPT2LMHeadModel @@ -108,24 +109,6 @@ def top_k_filter(logits, k, probs=False): logits) -class ClassificationHead(torch.nn.Module): - """ Classification Head for the transformer """ - - def __init__(self, class_size=5, embed_size=2048): - super(ClassificationHead, self).__init__() - self.class_size = class_size - self.embed_size = embed_size - # self.mlp1 = torch.nn.Linear(embed_size, embed_size) - # self.mlp2 = (torch.nn.Linear(embed_size, class_size)) - self.mlp = torch.nn.Linear(embed_size, class_size) - - def forward(self, hidden_state): - # hidden_state = F.relu(self.mlp1(hidden_state)) - # hidden_state = self.mlp2(hidden_state) - logits = self.mlp(hidden_state) - return logits - - def perturb_past(past, model, prev, args, classifier, good_index=None, stepsize=0.01, vocab_size=50257, original_probs=None, accumulated_hidden=None, true_past=None, diff --git a/examples/run_pplm_discrim_train.py b/examples/run_pplm_discrim_train.py new file mode 100644 index 0000000000..cc52234281 --- /dev/null +++ b/examples/run_pplm_discrim_train.py @@ -0,0 +1,582 @@ +#! /usr/bin/env python3 +# coding=utf-8 + +# This code is licensed under a non-commercial license. + +import argparse +import csv +import json +import math +import time + +import numpy as np +import torch +import torch.nn.functional as F +import torch.optim +import torch.optim as optim +import torch.utils.data as data +from nltk.tokenize.treebank import TreebankWordDetokenizer +from torchtext import data as torchtext_data +from torchtext import datasets +from transformers import GPT2Tokenizer, GPT2LMHeadModel + +torch.manual_seed(0) +np.random.seed(0) +EPSILON = 1e-10 +device = 'cpu' +example_sentence = "This is incredible! I love it, this is the best chicken I have ever had." +max_length_seq = 100 + + +class ClassificationHead(torch.nn.Module): + """Classification Head for transformer encoders""" + + def __init__(self, class_size, embed_size): + super(ClassificationHead, self).__init__() + self.class_size = class_size + self.embed_size = embed_size + # self.mlp1 = torch.nn.Linear(embed_size, embed_size) + # self.mlp2 = (torch.nn.Linear(embed_size, class_size)) + self.mlp = torch.nn.Linear(embed_size, class_size) + + def forward(self, hidden_state): + # hidden_state = F.relu(self.mlp1(hidden_state)) + # hidden_state = self.mlp2(hidden_state) + logits = self.mlp(hidden_state) + return logits + + +class Discriminator(torch.nn.Module): + """Transformer encoder followed by a Classification Head""" + + def __init__( + self, + class_size, + pretrained_model="gpt2-medium", + cached_mode=False + ): + super(Discriminator, self).__init__() + self.tokenizer = GPT2Tokenizer.from_pretrained(pretrained_model) + self.encoder = GPT2LMHeadModel.from_pretrained(pretrained_model) + self.embed_size = self.encoder.transformer.config.hidden_size + self.classifier_head = ClassificationHead( + class_size=class_size, + embed_size=self.embed_size + ) + self.cached_mode = cached_mode + + def get_classifier(self): + return self.classifier_head + + def train_custom(self): + for param in self.encoder.parameters(): + param.requires_grad = False + pass + self.classifier_head.train() + + def avg_representation(self, x): + mask = x.ne(0).unsqueeze(2).repeat( + 1, 1, self.embed_size + ).float().to(device).detach() + hidden, _ = self.encoder.transformer(x) + masked_hidden = hidden * mask + avg_hidden = torch.sum(masked_hidden, dim=1) / ( + torch.sum(mask, dim=1).detach() + EPSILON + ) + return avg_hidden + + def forward(self, x): + if self.cached_mode: + avg_hidden = x.to(device) + else: + avg_hidden = self.avg_representation(x) + + logits = self.classifier_head(avg_hidden) + probs = F.log_softmax(logits, dim=-1) + + return probs + + +class Dataset(data.Dataset): + def __init__(self, X, y): + """Reads source and target sequences from txt files.""" + self.X = X + self.y = y + + def __len__(self): + return len(self.X) + + def __getitem__(self, index): + """Returns one data pair (source and target).""" + data = {} + data['X'] = self.X[index] + data['y'] = self.y[index] + return data + + +def collate_fn(data): + def pad_sequences(sequences): + lengths = [len(seq) for seq in sequences] + + padded_sequences = torch.zeros( + len(sequences), + max(lengths) + ).long() # padding index 0 + + for i, seq in enumerate(sequences): + end = lengths[i] + padded_sequences[i, :end] = seq[:end] + + return padded_sequences, lengths + + item_info = {} + for key in data[0].keys(): + item_info[key] = [d[key] for d in data] + + x_batch, _ = pad_sequences(item_info['X']) + y_batch = torch.tensor(item_info['y'], dtype=torch.long) + + return x_batch, y_batch + + +def cached_collate_fn(data): + item_info = {} + for key in data[0].keys(): + item_info[key] = [d[key] for d in data] + + x_batch = torch.cat(item_info['X'], 0) + y_batch = torch.tensor(item_info['y'], dtype=torch.long) + + return x_batch, y_batch + + +def train_epoch(data_loader, discriminator, optimizer, + epoch=0, log_interval=10): + samples_so_far = 0 + discriminator.train_custom() + for batch_idx, (input_t, target_t) in enumerate(data_loader): + input_t, target_t = input_t.to(device), target_t.to(device) + + optimizer.zero_grad() + + output_t = discriminator(input_t) + loss = F.nll_loss(output_t, target_t) + loss.backward(retain_graph=True) + optimizer.step() + + samples_so_far += len(input_t) + + if batch_idx % log_interval == 0: + print( + 'Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( + epoch + 1, + samples_so_far, len(data_loader.dataset), + 100 * samples_so_far / len(data_loader.dataset), loss.item() + ) + ) + + +def evaluate_performance(data_loader, discriminator): + discriminator.eval() + test_loss = 0 + correct = 0 + with torch.no_grad(): + for input_t, target_t in data_loader: + input_t, target_t = input_t.to(device), target_t.to(device) + output_t = discriminator(input_t) + # sum up batch loss + test_loss += F.nll_loss(output_t, target_t, reduction='sum').item() + # get the index of the max log-probability + pred_t = output_t.argmax(dim=1, keepdim=True) + correct += pred_t.eq(target_t.view_as(pred_t)).sum().item() + + test_loss /= len(data_loader.dataset) + + print( + 'Performance on test set: ' + 'Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)'.format( + test_loss, correct, len(data_loader.dataset), + 100. * correct / len(data_loader.dataset) + ) + ) + + +def predict(input_sentence, model, classes, cached=False): + input_t = model.tokenizer.encode(input_sentence) + input_t = torch.tensor([input_t], dtype=torch.long) + if cached: + input_t = model.avg_representation(input_t) + + log_probs = model(input_t).data.cpu().numpy().flatten().tolist() + print('Input sentence:', input_sentence) + print('Predictions:', ", ".join( + "{}: {:.4f}".format(c, math.exp(log_prob)) for c, log_prob in + zip(classes, log_probs) + )) + + +def get_cached_data_loader(dataset, batch_size, discriminator, shuffle=False): + data_loader = torch.utils.data.DataLoader(dataset=dataset, + batch_size=batch_size, + collate_fn=collate_fn) + + xs = [] + ys = [] + for batch_idx, (x, y) in enumerate(data_loader): + with torch.no_grad(): + x = x.to(device) + avg_rep = discriminator.avg_representation(x).cpu().detach() + avg_rep_list = torch.unbind(avg_rep.unsqueeze(1)) + xs += avg_rep_list + ys += y.cpu().numpy().tolist() + + data_loader = torch.utils.data.DataLoader( + dataset=Dataset(xs, ys), + batch_size=batch_size, + shuffle=shuffle, + collate_fn=cached_collate_fn) + + return data_loader + + +def train_discriminator( + dataset, dataset_fp=None, pretrained_model='gpt2-medium', + epochs=10, batch_size=64, log_interval=10, + save_model=False, cached=False, use_cuda=False): + if use_cuda: + global device + device = 'cuda' + + print('Preprocessing {} dataset...'.format(dataset)) + start = time.time() + + if dataset == 'SST': + idx2class = ["positive", "negative", "very positive", "very negative", + "neutral"] + class2idx = {c: i for i, c in enumerate(idx2class)} + + discriminator = Discriminator( + class_size=len(idx2class), + pretrained_model=pretrained_model, + cached_mode=cached + ).to(device) + + text = torchtext_data.Field() + label = torchtext_data.Field(sequential=False) + train_data, val_data, test_data = datasets.SST.splits( + text, + label, + fine_grained=True, + train_subtrees=True, + ) + + x = [] + y = [] + for i in range(len(train_data)): + seq = TreebankWordDetokenizer().detokenize( + vars(train_data[i])["text"] + ) + seq = discriminator.tokenizer.encode(seq) + seq = torch.tensor([50256] + seq, device=device, dtype=torch.long) + x.append(seq) + y.append(class2idx[vars(train_data[i])["label"]]) + train_dataset = Dataset(x, y) + + test_x = [] + test_y = [] + for i in range(len(test_data)): + seq = TreebankWordDetokenizer().detokenize( + vars(test_data[i])["text"] + ) + seq = discriminator.tokenizer.encode(seq) + seq = torch.tensor([50256] + seq, device=device, dtype=torch.long) + test_x.append(seq) + test_y.append(class2idx[vars(test_data[i])["label"]]) + test_dataset = Dataset(test_x, test_y) + + discriminator_meta = { + "class_size": len(idx2class), + "embed_size": discriminator.embed_size, + "pretrained_model": pretrained_model, + "class_vocab": class2idx, + "default_class": 2, + } + + elif dataset == 'clickbait': + idx2class = ["non_clickbait", "clickbait"] + class2idx = {c: i for i, c in enumerate(idx2class)} + + discriminator = Discriminator( + class_size=len(idx2class), + pretrained_model=pretrained_model, + cached_mode=cached + ).to(device) + + with open("datasets/clickbait/clickbait_train_prefix.txt") as f: + data = [] + for i, line in enumerate(f): + try: + data.append(eval(line)) + except: + print('Error evaluating line {}: {}'.format( + i, line + )) + continue + x = [] + y = [] + y = [] + for i, d in enumerate(data): + try: + seq = discriminator.tokenizer.encode(d["text"]) + + if len(seq) < max_length_seq: + seq = torch.tensor( + [50256] + seq, device=device, dtype=torch.long + ) + else: + print("Line {} is longer than maximum length {}".format( + i, max_length_seq + )) + continue + x.append(seq) + y.append(d['label']) + except: + print("Error tokenizing line {}, skipping it".format(i)) + pass + + full_dataset = Dataset(x, y) + train_size = int(0.9 * len(full_dataset)) + test_size = len(full_dataset) - train_size + train_dataset, test_dataset = torch.utils.data.random_split( + full_dataset, [train_size, test_size] + ) + + discriminator_meta = { + "class_size": len(idx2class), + "embed_size": discriminator.embed_size, + "pretrained_model": pretrained_model, + "class_vocab": class2idx, + "default_class": 1, + } + + elif dataset == 'toxic': + idx2class = ["non_toxic", "toxic"] + class2idx = {c: i for i, c in enumerate(idx2class)} + + discriminator = Discriminator( + class_size=len(idx2class), + pretrained_model=pretrained_model, + cached_mode=cached + ).to(device) + + with open("datasets/toxic/toxic_train.txt") as f: + data = [] + for i, line in enumerate(f): + try: + data.append(eval(line)) + except: + print('Error evaluating line {}: {}'.format( + i, line + )) + continue + + x = [] + y = [] + for i, d in enumerate(data): + try: + seq = discriminator.tokenizer.encode(d["text"]) + + if len(seq) < max_length_seq: + seq = torch.tensor( + [50256] + seq, device=device, dtype=torch.long + ) + else: + print("Line {} is longer than maximum length {}".format( + i, max_length_seq + )) + continue + x.append(seq) + y.append(int(np.sum(d['label']) > 0)) + except: + print("Error tokenizing line {}, skipping it".format(i)) + pass + + full_dataset = Dataset(x, y) + train_size = int(0.9 * len(full_dataset)) + test_size = len(full_dataset) - train_size + train_dataset, test_dataset = torch.utils.data.random_split( + full_dataset, [train_size, test_size] + ) + + discriminator_meta = { + "class_size": len(idx2class), + "embed_size": discriminator.embed_size, + "pretrained_model": pretrained_model, + "class_vocab": class2idx, + "default_class": 0, + } + + else: # if dataset == 'generic': + # This assumes the input dataset is a TSV with the following structure: + # class \t text + + if dataset_fp is None: + raise ValueError('When generic dataset is selected, ' + 'dataset_fp needs to be specified aswell.') + + classes = set() + with open(dataset_fp) as f: + csv_reader = csv.reader(f, delimiter='\t') + for row in csv_reader: + classes.add(row[0]) + + idx2class = sorted(classes) + class2idx = {c: i for i, c in enumerate(idx2class)} + + discriminator = Discriminator( + class_size=len(idx2class), + pretrained_model=pretrained_model, + cached_mode=cached + ).to(device) + + x = [] + y = [] + with open(dataset_fp) as f: + csv_reader = csv.reader(f, delimiter='\t') + for i, row in enumerate(csv_reader): + label = row[0] + text = row[1] + + try: + seq = discriminator.tokenizer.encode(text) + if (len(seq) < max_length_seq): + seq = torch.tensor( + [50256] + seq, + device=device, + dtype=torch.long + ) + + else: + print("Line {} is longer than maximum length {}".format( + i, max_length_seq + )) + continue + + x.append(seq) + y.append(class2idx[label]) + + except: + print("Error tokenizing line {}, skipping it".format(i)) + pass + + full_dataset = Dataset(x, y) + train_size = int(0.9 * len(full_dataset)) + test_size = len(full_dataset) - train_size + train_dataset, test_dataset = torch.utils.data.random_split( + full_dataset, + [train_size, test_size] + ) + + discriminator_meta = { + "class_size": len(idx2class), + "embed_size": discriminator.embed_size, + "pretrained_model": pretrained_model, + "class_vocab": class2idx, + "default_class": 0, + } + + end = time.time() + print('Preprocessed {} data points'.format( + len(train_dataset) + len(test_dataset)) + ) + print("Data preprocessing took: {:.3f}s".format(end - start)) + + if cached: + start = time.time() + + train_loader = get_cached_data_loader( + train_dataset, batch_size, discriminator, shuffle=True + ) + + test_loader = get_cached_data_loader( + test_dataset, batch_size, discriminator + ) + + end = time.time() + print("Building representation cache took: {:.3f}s".format(end - start)) + + else: + train_loader = torch.utils.data.DataLoader(dataset=train_dataset, + batch_size=batch_size, + shuffle=True, + collate_fn=collate_fn) + test_loader = torch.utils.data.DataLoader(dataset=test_dataset, + batch_size=batch_size, + collate_fn=collate_fn) + + if save_model: + with open("{}_classifier_head_meta.json".format(dataset), + "w") as meta_file: + json.dump(discriminator_meta, meta_file) + + optimizer = optim.Adam(discriminator.parameters(), lr=0.0001) + + for epoch in range(epochs): + start = time.time() + print('\nEpoch', epoch + 1) + + train_epoch( + discriminator=discriminator, + data_loader=train_loader, + optimizer=optimizer, + epoch=epoch, + log_interval=log_interval + ) + evaluate_performance( + data_loader=test_loader, + discriminator=discriminator + ) + + end = time.time() + print("Epoch took: {:.3f}s".format(end - start)) + + print("\nExample prediction") + predict(example_sentence, discriminator, idx2class, cached) + + if save_model: + # torch.save(discriminator.state_dict(), + # "{}_discriminator_{}.pt".format( + # args.dataset, epoch + # )) + torch.save(discriminator.get_classifier().state_dict(), + "{}_classifier_head_epoch_{}.pt".format(dataset, epoch)) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Train a discriminator on top of GPT-2 representations') + parser.add_argument('--dataset', type=str, default='SST', + choices=('SST', 'clickbait', 'toxic', 'generic'), + help='dataset to train the discriminator on.' + 'In case of generic, the dataset is expected' + 'to be a TSBV file with structure: class \\t text') + parser.add_argument('--dataset_fp', type=str, default='', + help='File path of the dataset to use. ' + 'Needed only in case of generic datadset') + parser.add_argument('--pretrained_model', type=str, default='gpt2-medium', + help='Pretrained model to use as encoder') + parser.add_argument('--epochs', type=int, default=10, metavar='N', + help='Number of training epochs') + parser.add_argument('--batch_size', type=int, default=64, metavar='N', + help='input batch size for training (default: 64)') + parser.add_argument('--log_interval', type=int, default=10, metavar='N', + help='how many batches to wait before logging training status') + parser.add_argument('--save_model', action='store_true', + help='whether to save the model') + parser.add_argument('--cached', action='store_true', + help='whether to cache the input representations') + parser.add_argument('--use_cuda', action='store_true', + help='use to turn on cuda') + args = parser.parse_args() + + train_discriminator(**(vars(args))) From 7469d03b1ca3c1c920e4cadc8a007609d17aff50 Mon Sep 17 00:00:00 2001 From: w4nderlust Date: Tue, 26 Nov 2019 21:30:57 -0800 Subject: [PATCH 180/505] Fixed minor bug when running training on cuda --- examples/run_pplm_discrim_train.py | 49 ++++++++++++++++-------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/examples/run_pplm_discrim_train.py b/examples/run_pplm_discrim_train.py index cc52234281..7f10e861a8 100644 --- a/examples/run_pplm_discrim_train.py +++ b/examples/run_pplm_discrim_train.py @@ -18,6 +18,7 @@ import torch.utils.data as data from nltk.tokenize.treebank import TreebankWordDetokenizer from torchtext import data as torchtext_data from torchtext import datasets + from transformers import GPT2Tokenizer, GPT2LMHeadModel torch.manual_seed(0) @@ -89,7 +90,7 @@ class Discriminator(torch.nn.Module): if self.cached_mode: avg_hidden = x.to(device) else: - avg_hidden = self.avg_representation(x) + avg_hidden = self.avg_representation(x.to(device)) logits = self.classifier_head(avg_hidden) probs = F.log_softmax(logits, dim=-1) @@ -203,7 +204,7 @@ def evaluate_performance(data_loader, discriminator): def predict(input_sentence, model, classes, cached=False): input_t = model.tokenizer.encode(input_sentence) - input_t = torch.tensor([input_t], dtype=torch.long) + input_t = torch.tensor([input_t], dtype=torch.long, device=device) if cached: input_t = model.avg_representation(input_t) @@ -428,7 +429,8 @@ def train_discriminator( with open(dataset_fp) as f: csv_reader = csv.reader(f, delimiter='\t') for row in csv_reader: - classes.add(row[0]) + if row: + classes.add(row[0]) idx2class = sorted(classes) class2idx = {c: i for i, c in enumerate(idx2class)} @@ -444,30 +446,31 @@ def train_discriminator( with open(dataset_fp) as f: csv_reader = csv.reader(f, delimiter='\t') for i, row in enumerate(csv_reader): - label = row[0] - text = row[1] + if row: + label = row[0] + text = row[1] - try: - seq = discriminator.tokenizer.encode(text) - if (len(seq) < max_length_seq): - seq = torch.tensor( - [50256] + seq, - device=device, - dtype=torch.long - ) + try: + seq = discriminator.tokenizer.encode(text) + if (len(seq) < max_length_seq): + seq = torch.tensor( + [50256] + seq, + device=device, + dtype=torch.long + ) - else: - print("Line {} is longer than maximum length {}".format( - i, max_length_seq - )) - continue + else: + print("Line {} is longer than maximum length {}".format( + i, max_length_seq + )) + continue - x.append(seq) - y.append(class2idx[label]) + x.append(seq) + y.append(class2idx[label]) - except: - print("Error tokenizing line {}, skipping it".format(i)) - pass + except: + print("Error tokenizing line {}, skipping it".format(i)) + pass full_dataset = Dataset(x, y) train_size = int(0.9 * len(full_dataset)) From 821de121e86574504ec648f76ccb924e38125b52 Mon Sep 17 00:00:00 2001 From: piero Date: Wed, 27 Nov 2019 15:27:49 -0800 Subject: [PATCH 181/505] Minor changes --- examples/run_pplm_discrim_train.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/run_pplm_discrim_train.py b/examples/run_pplm_discrim_train.py index 7f10e861a8..9438cbbac1 100644 --- a/examples/run_pplm_discrim_train.py +++ b/examples/run_pplm_discrim_train.py @@ -72,7 +72,6 @@ class Discriminator(torch.nn.Module): def train_custom(self): for param in self.encoder.parameters(): param.requires_grad = False - pass self.classifier_head.train() def avg_representation(self, x): @@ -122,7 +121,7 @@ def collate_fn(data): padded_sequences = torch.zeros( len(sequences), max(lengths) - ).long() # padding index 0 + ).long() # padding value = 0 for i, seq in enumerate(sequences): end = lengths[i] From 4f2164e40e803677869865b140b2b3f5a96d4bcd Mon Sep 17 00:00:00 2001 From: piero Date: Wed, 27 Nov 2019 16:32:45 -0800 Subject: [PATCH 182/505] First cleanup step, changing function names and passing parameters all the way through without using args. Identical output as before. --- examples/run_pplm.py | 270 +++++++++++++++++++++++++++++-------------- 1 file changed, 182 insertions(+), 88 deletions(-) diff --git a/examples/run_pplm.py b/examples/run_pplm.py index 217c131b8f..0d9ed86f45 100644 --- a/examples/run_pplm.py +++ b/examples/run_pplm.py @@ -109,20 +109,40 @@ def top_k_filter(logits, k, probs=False): logits) -def perturb_past(past, model, prev, args, classifier, good_index=None, - stepsize=0.01, vocab_size=50257, - original_probs=None, accumulated_hidden=None, true_past=None, - grad_norms=None): - window_length = args.window_length - gm_scale, kl_scale = args.gm_scale, args.kl_scale - one_hot_vectors = [] - for good_list in good_index: - good_list = list(filter(lambda x: len(x) <= 1, good_list)) - good_list = torch.tensor(good_list).cuda() - num_good = good_list.shape[0] - one_hot_good = torch.zeros(num_good, vocab_size).cuda() - one_hot_good.scatter_(1, good_list, 1) - one_hot_vectors.append(one_hot_good) +def perturb_past( + past, + model, + prev, + unpert_past=None, + unpert_logits=None, + accumulated_hidden=None, + grad_norms=None, + stepsize=0.01, + classifier=None, + label_class=None, + one_hot_bows_vectors=None, + loss_type=0, + num_iterations=3, + kl_scale=0.01, + window_length=0, + horizon_length=1, + decay=False, + gamma=1.5, +): + + #def perturb_past(past, model, prev, classifier, good_index=None, + # stepsize=0.01, vocab_size=50257, + # original_probs=None, accumulated_hidden=None, true_past=None, + # grad_norms=None): + + # one_hot_bows_vectors = [] + # for good_list in good_index: + # good_list = list(filter(lambda x: len(x) <= 1, good_list)) + # good_list = torch.tensor(good_list).cuda() + # num_good = good_list.shape[0] + # one_hot_good = torch.zeros(num_good, vocab_size).cuda() + # one_hot_good.scatter_(1, good_list, 1) + # one_hot_bows_vectors.append(one_hot_good) # Generate inital perturbed past past_perturb_orig = [ @@ -132,7 +152,7 @@ def perturb_past(past, model, prev, args, classifier, good_index=None, if accumulated_hidden is None: accumulated_hidden = 0 - if args.decay: + if decay: decay_mask = torch.arange(0., 1.0 + SmallConst, 1.0 / (window_length))[ 1:] else: @@ -160,7 +180,7 @@ def perturb_past(past, model, prev, args, classifier, good_index=None, window_mask = torch.ones_like(past[0]).cuda() loss_per_iter = [] - for i in range(args.num_iterations): + for i in range(num_iterations): print("Iteration ", i + 1) past_perturb = [torch.from_numpy(p_) for p_ in past_perturb_orig] past_perturb = [to_var(p_, requires_grad=True) for p_ in past_perturb] @@ -183,8 +203,8 @@ def perturb_past(past, model, prev, args, classifier, good_index=None, probabs = F.softmax(logits, dim=-1) loss = 0.0 loss_list = [] - if args.loss_type == 1 or args.loss_type == 3: - for one_hot_good in one_hot_vectors: + if loss_type == 1 or loss_type == 3: + for one_hot_good in one_hot_bows_vectors: good_logits = torch.mm(probabs, torch.t(one_hot_good)) loss_word = good_logits loss_word = torch.sum(loss_word) @@ -194,10 +214,10 @@ def perturb_past(past, model, prev, args, classifier, good_index=None, loss_list.append(loss_word) print(" pplm_bow_loss:", loss.data.cpu().numpy()) - if args.loss_type == 2 or args.loss_type == 3: + if loss_type == 2 or loss_type == 3: ce_loss = torch.nn.CrossEntropyLoss() - new_true_past = true_past - for i in range(args.horizon_length): + new_true_past = unpert_past + for i in range(horizon_length): future_probabs = F.softmax(logits, dim=-1) # Get softmax future_probabs = torch.unsqueeze(future_probabs, dim=1) @@ -217,9 +237,9 @@ def perturb_past(past, model, prev, args, classifier, good_index=None, future_hidden, dim=1) predicted_sentiment = classifier(new_accumulated_hidden / ( - current_length + 1 + args.horizon_length)) + current_length + 1 + horizon_length)) - label = torch.tensor([args.label_class], device='cuda', + label = torch.tensor([label_class], device='cuda', dtype=torch.long) discrim_loss = ce_loss(predicted_sentiment, label) print(" pplm_discrim_loss:", discrim_loss.data.cpu().numpy()) @@ -228,7 +248,7 @@ def perturb_past(past, model, prev, args, classifier, good_index=None, kl_loss = 0.0 if kl_scale > 0.0: - p = (F.softmax(original_probs[:, -1, :], dim=-1)) + p = (F.softmax(unpert_logits[:, -1, :], dim=-1)) p = p + SmallConst * (p <= SmallConst).type( torch.FloatTensor).cuda().detach() correction = SmallConst * (probabs <= SmallConst).type( @@ -244,7 +264,7 @@ def perturb_past(past, model, prev, args, classifier, good_index=None, print(' pplm_loss', (loss - kl_loss).data.cpu().numpy()) loss.backward() - if grad_norms is not None and args.loss_type == 1: + if grad_norms is not None and loss_type == 1: grad_norms = [ torch.max(grad_norms[index], torch.norm(p_.grad * window_mask)) for index, p_ in @@ -255,7 +275,7 @@ def perturb_past(past, model, prev, args, classifier, good_index=None, grad = [ -stepsize * (p_.grad * window_mask / grad_norms[ - index] ** args.gamma).data.cpu().numpy() + index] ** gamma).data.cpu().numpy() for index, p_ in enumerate(past_perturb)] past_perturb_orig = list(map(add, grad, past_perturb_orig)) @@ -347,10 +367,32 @@ def build_bows_one_hot_vectors(bow_indices): return one_hot_bows_vectors -def latent_perturb(model, args, context=None, sample=True, device='cuda'): +def full_text_generation( + model, + context=None, + num_samples=1, + device="cuda", + sample=True, + discrim=None, + label_class=None, + bag_of_words=None, + length=100, + grad_length=10000, + stepsize=0.02, + num_iterations=3, + temperature=1.0, + gm_scale=0.9, + kl_scale=0.01, + top_k=10, + window_length=0, + horizon_length=1, + decay=False, + gamma=1.5, + **kwargs + ): classifier, class_id = get_classifier( - args.discrim, - args.label_class, + discrim, + label_class, device ) @@ -422,49 +464,68 @@ def latent_perturb(model, args, context=None, sample=True, device='cuda'): # good_list = list(filter(lambda x: len(x) <= 1, good_list)) # actual_words = [(TOKENIZER.decode(ww).strip(),ww) for ww in good_list] - good_index = [] + bow_indices = [] actual_words = None - if args.bag_of_words: - good_index = get_bag_of_words_indices(args.bag_of_words.split(";")) + if bag_of_words: + bow_indices = get_bag_of_words_indices(bag_of_words.split(";")) - for good_list in good_index: + for good_list in bow_indices: good_list = list(filter(lambda x: len(x) <= 1, good_list)) actual_words = [(TOKENIZER.decode(ww).strip(), ww) for ww in good_list] - if args.bag_of_words and classifier: + if bag_of_words and classifier: print("Both PPLM-BoW and PPLM-Discrim are on. This is not optimized.") - args.loss_type = PPLM_BOW_DISCRIM + loss_type = PPLM_BOW_DISCRIM - elif args.bag_of_words: - args.loss_type = PPLM_BOW + elif bag_of_words: + loss_type = PPLM_BOW print("Using PPLM-BoW") elif classifier is not None: - args.loss_type = PPLM_DISCRIM + loss_type = PPLM_DISCRIM print("Using PPLM-Discrim") else: raise Exception("Specify either --bag_of_words (-B) or --discrim (-D)") - original, _, _ = sample_from_hidden(model=model, args=args, context=context, - device=device, - perturb=False, good_index=good_index, - classifier=classifier) + original, _, _ = generate_text_pplm( + model=model, + context=context, + device=device, + length=length, + perturb=False + ) torch.cuda.empty_cache() perturbed_list = [] discrim_loss_list = [] loss_in_time_list = [] - for i in range(args.num_samples): - perturbed, discrim_loss, loss_in_time = sample_from_hidden(model=model, - args=args, - context=context, - device=device, - perturb=True, - good_index=good_index, - classifier=classifier) + for i in range(num_samples): + perturbed, discrim_loss, loss_in_time = generate_text_pplm( + model=model, + context=context, + device=device, + sample=sample, + perturb=True, + bow_indices=bow_indices, + classifier=classifier, + label_class=class_id, + loss_type=loss_type, + length=length, + grad_length=grad_length, + stepsize=stepsize, + num_iterations=num_iterations, + temperature=temperature, + gm_scale=gm_scale, + kl_scale=kl_scale, + top_k=top_k, + window_length=window_length, + horizon_length=horizon_length, + decay=decay, + gamma=gamma, + ) perturbed_list.append(perturbed) if classifier is not None: discrim_loss_list.append(discrim_loss.data.cpu().numpy()) @@ -475,15 +536,40 @@ def latent_perturb(model, args, context=None, sample=True, device='cuda'): return original, perturbed_list, discrim_loss_list, loss_in_time_list, actual_words -def sample_from_hidden(model, args, classifier, context=None, past=None, - device='cuda', - sample=True, perturb=True, good_index=None): + +def generate_text_pplm( + model, + context=None, + past=None, + device="cuda", + sample=True, + perturb=True, + classifier=None, + label_class=None, + bow_indices=None, + loss_type=0, + length=100, + grad_length=10000, + stepsize=0.02, + num_iterations=3, + temperature=1.0, + gm_scale=0.9, + kl_scale=0.01, + top_k=10, + window_length=0, + horizon_length=1, + decay=False, + gamma=1.5, +): output = torch.tensor(context, device=device, dtype=torch.long).unsqueeze( 0) if context else None + # collect one hot vectors for bags of words + one_hot_bows_vectors = build_bows_one_hot_vectors(bow_indices) + grad_norms = None loss_in_time = [] - for i in trange(args.length, ascii=True): + for i in trange(length, ascii=True): # Get past/probs for current output, except for last word # Note that GPT takes 2 inputs: past + current-token @@ -497,7 +583,7 @@ def sample_from_hidden(model, args, classifier, context=None, past=None, # Piero modified model call _, past, _ = model(output[:, :-1]) - original_probs, true_past, unpert_all_hidden = model(output) + unpert_logits, unpert_past, unpert_all_hidden = model(output) true_hidden = unpert_all_hidden[-1] else: @@ -505,17 +591,17 @@ def sample_from_hidden(model, args, classifier, context=None, past=None, # true_hidden = model.hidden_states # Piero modified model call - original_probs, true_past, unpert_all_hidden = model(output) + unpert_logits, unpert_past, unpert_all_hidden = model(output) true_hidden = unpert_all_hidden[-1] # Modify the past if necessary - if i >= args.grad_length: - current_stepsize = args.stepsize * 0 + if i >= grad_length: + current_stepsize = stepsize * 0 else: - current_stepsize = args.stepsize + current_stepsize = stepsize - if not perturb or args.num_iterations == 0: + if not perturb or num_iterations == 0: perturbed_past = past else: @@ -524,17 +610,26 @@ def sample_from_hidden(model, args, classifier, context=None, past=None, accumulated_hidden = true_hidden[:, :-1, :] accumulated_hidden = torch.sum(accumulated_hidden, dim=1) - perturbed_past, _, grad_norms, loss_per_iter = perturb_past(past, - model, - prev, - args, - good_index=good_index, - stepsize=current_stepsize, - original_probs=original_probs, - true_past=true_past, - accumulated_hidden=accumulated_hidden, - classifier=classifier, - grad_norms=grad_norms) + perturbed_past, _, grad_norms, loss_per_iter = perturb_past( + past, + model, + prev, + unpert_past=unpert_past, + unpert_logits=unpert_logits, + accumulated_hidden=accumulated_hidden, + grad_norms=grad_norms, + stepsize=current_stepsize, + classifier=classifier, + label_class=label_class, + one_hot_bows_vectors=one_hot_bows_vectors, + loss_type=loss_type, + num_iterations=num_iterations, + kl_scale=kl_scale, + window_length=window_length, + horizon_length=horizon_length, + decay=decay, + gamma=gamma, + ) loss_in_time.append(loss_per_iter) # Piero modified model call @@ -546,7 +641,7 @@ def sample_from_hidden(model, args, classifier, context=None, past=None, if classifier is not None: ce_loss = torch.nn.CrossEntropyLoss() predicted_sentiment = classifier(torch.mean(true_hidden, dim=1)) - label = torch.tensor([args.label_class], device='cuda', + label = torch.tensor([label_class], device='cuda', dtype=torch.long) true_discrim_loss = ce_loss(predicted_sentiment, label) print("true discrim loss", true_discrim_loss.data.cpu().numpy()) @@ -556,7 +651,7 @@ def sample_from_hidden(model, args, classifier, context=None, past=None, # Piero modified model call # hidden = model.hidden_states # update hidden # logits = model.forward_hidden(hidden) - logits = logits[:, -1, :] / args.temperature # + SmallConst + logits = logits[:, -1, :] / temperature # + SmallConst # logits = top_k_filter(logits, k=args.top_k) # + SmallConst @@ -566,22 +661,21 @@ def sample_from_hidden(model, args, classifier, context=None, past=None, if perturb: # original_probs = top_k_filter(original_probs[:, -1, :]) #+ SmallConst - original_probs = F.softmax(original_probs[:, -1, :], dim=-1) + unpert_logits = F.softmax(unpert_logits[:, -1, :], dim=-1) # likelywords = torch.topk(original_probs, k=10, dim=-1) # print(TOKENIZER.decode(likelywords[1].tolist()[0])) - gm_scale = args.gm_scale log_probs = ((log_probs ** gm_scale) * ( - original_probs ** (1 - gm_scale))) # + SmallConst + unpert_logits ** (1 - gm_scale))) # + SmallConst - log_probs = top_k_filter(log_probs, k=args.top_k, + log_probs = top_k_filter(log_probs, k=top_k, probs=True) # + SmallConst if torch.sum(log_probs) <= 1: log_probs = log_probs / torch.sum(log_probs) else: - logits = top_k_filter(logits, k=args.top_k) # + SmallConst + logits = top_k_filter(logits, k=top_k) # + SmallConst log_probs = F.softmax(logits, dim=-1) if sample: @@ -673,16 +767,16 @@ def run_model(): collect_gen = dict() current_index = 0 - for out in seq: + for tokenized_cond_text in seq: - text = TOKENIZER.decode(out) + text = TOKENIZER.decode(tokenized_cond_text) print("=" * 40 + " Prefix of sentence " + "=" * 40) print(text) print("=" * 80) - out1, out_perturb, discrim_loss_list, loss_in_time_list, actual_words = latent_perturb( - model=model, args=args, context=out, - device=device) + out1, out_perturb, discrim_loss_list, loss_in_time_list, actual_words = full_text_generation( + model=model, context=tokenized_cond_text, device=device, **vars(args) + ) text_whole = TOKENIZER.decode(out1.tolist()[0]) @@ -712,20 +806,20 @@ def run_model(): import colorama text_whole = '' - for out in output_tokens: - if out in keyword_tokens: + for tokenized_cond_text in output_tokens: + if tokenized_cond_text in keyword_tokens: text_whole += '%s%s%s' % ( - colorama.Fore.GREEN, TOKENIZER.decode([out]), + colorama.Fore.GREEN, TOKENIZER.decode([tokenized_cond_text]), colorama.Style.RESET_ALL) else: - text_whole += TOKENIZER.decode([out]) + text_whole += TOKENIZER.decode([tokenized_cond_text]) else: text_whole = TOKENIZER.decode(out_perturb.tolist()[0]) print(text_whole) print("=" * 80) - collect_gen[current_index] = [out, out_perturb, out1] + collect_gen[current_index] = [tokenized_cond_text, out_perturb, out1] current_index = current_index + 1 From 7ffe47c88861ccea7e4b28538ea3dcb504c497a2 Mon Sep 17 00:00:00 2001 From: piero Date: Wed, 27 Nov 2019 16:39:49 -0800 Subject: [PATCH 183/505] Improved device specification --- examples/run_pplm_discrim_train.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/examples/run_pplm_discrim_train.py b/examples/run_pplm_discrim_train.py index 9438cbbac1..519e2de29a 100644 --- a/examples/run_pplm_discrim_train.py +++ b/examples/run_pplm_discrim_train.py @@ -242,10 +242,9 @@ def get_cached_data_loader(dataset, batch_size, discriminator, shuffle=False): def train_discriminator( dataset, dataset_fp=None, pretrained_model='gpt2-medium', epochs=10, batch_size=64, log_interval=10, - save_model=False, cached=False, use_cuda=False): - if use_cuda: - global device - device = 'cuda' + save_model=False, cached=False, no_cuda=False): + global device + device = "cuda" if torch.cuda.is_available() and not no_cuda else "cpu" print('Preprocessing {} dataset...'.format(dataset)) start = time.time() @@ -577,8 +576,8 @@ if __name__ == '__main__': help='whether to save the model') parser.add_argument('--cached', action='store_true', help='whether to cache the input representations') - parser.add_argument('--use_cuda', action='store_true', - help='use to turn on cuda') + parser.add_argument('--no_cuda', action='store_true', + help='use to turn off cuda') args = parser.parse_args() train_discriminator(**(vars(args))) From 6c9c1317800499bd5aaf3f1a7492a72961bbfa91 Mon Sep 17 00:00:00 2001 From: piero Date: Wed, 27 Nov 2019 17:27:39 -0800 Subject: [PATCH 184/505] More cleanup for run_model. Identical output as before. --- examples/run_pplm.py | 308 ++++++++++++++++++++++++------------------- 1 file changed, 171 insertions(+), 137 deletions(-) diff --git a/examples/run_pplm.py b/examples/run_pplm.py index 0d9ed86f45..27ead3c3c5 100644 --- a/examples/run_pplm.py +++ b/examples/run_pplm.py @@ -39,7 +39,6 @@ from transformers import GPT2Tokenizer from transformers.file_utils import cached_path from transformers.modeling_gpt2 import GPT2LMHeadModel - PPLM_BOW = 1 PPLM_DISCRIM = 2 PPLM_BOW_DISCRIM = 3 @@ -129,8 +128,7 @@ def perturb_past( decay=False, gamma=1.5, ): - - #def perturb_past(past, model, prev, classifier, good_index=None, + # def perturb_past(past, model, prev, classifier, good_index=None, # stepsize=0.01, vocab_size=50257, # original_probs=None, accumulated_hidden=None, true_past=None, # grad_norms=None): @@ -237,7 +235,7 @@ def perturb_past( future_hidden, dim=1) predicted_sentiment = classifier(new_accumulated_hidden / ( - current_length + 1 + horizon_length)) + current_length + 1 + horizon_length)) label = torch.tensor([label_class], device='cuda', dtype=torch.long) @@ -349,6 +347,13 @@ def get_bag_of_words_indices(bag_of_words_ids_or_paths: List[str]) -> List[ bow_indices.append( [TOKENIZER.encode(word.strip(), add_prefix_space=True) for word in words]) + + #bow_words = set() + #for bow_list in bow_indices: + # bow_list = list(filter(lambda x: len(x) <= 1, bow_list)) + # bow_words.update( + # (TOKENIZER.decode(word).strip(), word) for word in bow_list) + return bow_indices @@ -368,28 +373,28 @@ def build_bows_one_hot_vectors(bow_indices): def full_text_generation( - model, - context=None, - num_samples=1, - device="cuda", - sample=True, - discrim=None, - label_class=None, - bag_of_words=None, - length=100, - grad_length=10000, - stepsize=0.02, - num_iterations=3, - temperature=1.0, - gm_scale=0.9, - kl_scale=0.01, - top_k=10, - window_length=0, - horizon_length=1, - decay=False, - gamma=1.5, - **kwargs - ): + model, + context=None, + num_samples=1, + device="cuda", + sample=True, + discrim=None, + label_class=None, + bag_of_words=None, + length=100, + grad_length=10000, + stepsize=0.02, + num_iterations=3, + temperature=1.0, + gm_scale=0.9, + kl_scale=0.01, + top_k=10, + window_length=0, + horizon_length=1, + decay=False, + gamma=1.5, + **kwargs +): classifier, class_id = get_classifier( discrim, label_class, @@ -465,15 +470,9 @@ def full_text_generation( # actual_words = [(TOKENIZER.decode(ww).strip(),ww) for ww in good_list] bow_indices = [] - actual_words = None if bag_of_words: bow_indices = get_bag_of_words_indices(bag_of_words.split(";")) - for good_list in bow_indices: - good_list = list(filter(lambda x: len(x) <= 1, good_list)) - actual_words = [(TOKENIZER.decode(ww).strip(), ww) for ww in - good_list] - if bag_of_words and classifier: print("Both PPLM-BoW and PPLM-Discrim are on. This is not optimized.") loss_type = PPLM_BOW_DISCRIM @@ -533,8 +532,7 @@ def full_text_generation( torch.cuda.empty_cache() - return original, perturbed_list, discrim_loss_list, loss_in_time_list, actual_words - + return original, perturbed_list, discrim_loss_list, loss_in_time_list def generate_text_pplm( @@ -611,25 +609,25 @@ def generate_text_pplm( accumulated_hidden = torch.sum(accumulated_hidden, dim=1) perturbed_past, _, grad_norms, loss_per_iter = perturb_past( - past, - model, - prev, - unpert_past=unpert_past, - unpert_logits=unpert_logits, - accumulated_hidden=accumulated_hidden, - grad_norms=grad_norms, - stepsize=current_stepsize, - classifier=classifier, - label_class=label_class, - one_hot_bows_vectors=one_hot_bows_vectors, - loss_type=loss_type, - num_iterations=num_iterations, - kl_scale=kl_scale, - window_length=window_length, - horizon_length=horizon_length, - decay=decay, - gamma=gamma, - ) + past, + model, + prev, + unpert_past=unpert_past, + unpert_logits=unpert_logits, + accumulated_hidden=accumulated_hidden, + grad_norms=grad_norms, + stepsize=current_stepsize, + classifier=classifier, + label_class=label_class, + one_hot_bows_vectors=one_hot_bows_vectors, + loss_type=loss_type, + num_iterations=num_iterations, + kl_scale=kl_scale, + window_length=window_length, + horizon_length=horizon_length, + decay=decay, + gamma=gamma, + ) loss_in_time.append(loss_per_iter) # Piero modified model call @@ -666,7 +664,7 @@ def generate_text_pplm( # print(TOKENIZER.decode(likelywords[1].tolist()[0])) log_probs = ((log_probs ** gm_scale) * ( - unpert_logits ** (1 - gm_scale))) # + SmallConst + unpert_logits ** (1 - gm_scale))) # + SmallConst log_probs = top_k_filter(log_probs, k=top_k, probs=True) # + SmallConst @@ -696,53 +694,88 @@ def generate_text_pplm( def run_model(): parser = argparse.ArgumentParser() - parser.add_argument('--model_path', '-M', type=str, default='gpt2-medium', - help='pretrained model name or path to local checkpoint') - parser.add_argument('--bag-of-words', '-B', type=str, default=None, - help='Bags of words used for PPLM-BoW. Multiple BoWs separated by ;') - parser.add_argument('--discrim', '-D', type=str, default=None, - choices=( - 'clickbait', 'sentiment', 'toxicity', 'generic'), - help='Discriminator to use for loss-type 2') - parser.add_argument('--discrim_weights', type=str, default=None, - help='Weights for the generic discriminator') - parser.add_argument('--discrim_meta', type=str, default=None, - help='Meta information for the generic discriminator') - parser.add_argument('--label_class', type=int, default=-1, - help='Class label used for the discriminator') - parser.add_argument('--stepsize', type=float, default=0.02) + parser.add_argument( + "--model_path", + "-M", + type=str, + default="gpt2-medium", + help="pretrained model name or path to local checkpoint", + ) + parser.add_argument( + "--bag_of_words", + "-B", + type=str, + default=None, + help="Bags of words used for PPLM-BoW. " + "Either a BOW id (see list in code) or a filepath. " + "Multiple BoWs separated by ;", + ) + parser.add_argument( + "--discrim", + "-D", + type=str, + default=None, + choices=("clickbait", "sentiment", "toxicity"), + help="Discriminator to use for loss-type 2", + ) + parser.add_argument( + "--label_class", + type=int, + default=-1, + help="Class label used for the discriminator", + ) + parser.add_argument("--stepsize", type=float, default=0.02) parser.add_argument("--length", type=int, default=100) parser.add_argument("--seed", type=int, default=0) parser.add_argument("--temperature", type=float, default=1.0) parser.add_argument("--top_k", type=int, default=10) parser.add_argument("--gm_scale", type=float, default=0.9) parser.add_argument("--kl_scale", type=float, default=0.01) - parser.add_argument('--nocuda', action='store_true', help='no cuda') - parser.add_argument('--uncond', action='store_true', - help='Generate from end-of-text as prefix') - parser.add_argument("--cond_text", type=str, default='The lake', - help='Prefix texts to condition on') - parser.add_argument('--num_iterations', type=int, default=3) - parser.add_argument('--grad_length', type=int, default=10000) - parser.add_argument('--num_samples', type=int, default=1, - help='Number of samples to generate from the modified latents') - parser.add_argument('--horizon_length', type=int, default=1, - help='Length of future to optimize over') - # parser.add_argument('--force-token', action='store_true', help='no cuda') - parser.add_argument('--window_length', type=int, default=0, - help='Length of past which is being optimizer; 0 corresponds to infinite window length') - parser.add_argument('--decay', action='store_true', - help='whether to decay or not') - parser.add_argument('--gamma', type=float, default=1.5) - parser.add_argument('--colorama', action='store_true', help='no cuda') + parser.add_argument("--no_cuda", action="store_true", help="no cuda") + parser.add_argument( + "--uncond", action="store_true", + help="Generate from end-of-text as prefix" + ) + parser.add_argument( + "--cond_text", type=str, default="The lake", + help="Prefix texts to condition on" + ) + parser.add_argument("--num_iterations", type=int, default=3) + parser.add_argument("--grad_length", type=int, default=10000) + parser.add_argument( + "--num_samples", + type=int, + default=1, + help="Number of samples to generate from the modified latents", + ) + parser.add_argument( + "--horizon_length", + type=int, + default=1, + help="Length of future to optimize over", + ) + parser.add_argument( + "--window_length", + type=int, + default=0, + help="Length of past which is being optimized; " + "0 corresponds to infinite window length", + ) + parser.add_argument("--decay", action="store_true", + help="whether to decay or not") + parser.add_argument("--gamma", type=float, default=1.5) + parser.add_argument("--colorama", action="store_true", help="colors keywords") args = parser.parse_args() + # set Random seed torch.manual_seed(args.seed) np.random.seed(args.seed) - device = 'cpu' if args.nocuda else 'cuda' + # set the device + device = "cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu" + # load pretrained model model = GPT2LMHeadModel.from_pretrained( args.model_path, output_hidden_states=True @@ -753,76 +786,77 @@ def run_model(): # Freeze GPT-2 weights for param in model.parameters(): param.requires_grad = False - pass + # figure out conditioning text if args.uncond: - seq = [[50256, 50256]] - + tokenized_cond_text = TOKENIZER.encode( + [TOKENIZER.bos_token] + ) else: raw_text = args.cond_text while not raw_text: - print('Did you forget to add `--cond-text`? ') + print("Did you forget to add `--cond_text`? ") raw_text = input("Model prompt >>> ") - seq = [[50256] + TOKENIZER.encode(raw_text)] + tokenized_cond_text = TOKENIZER.encode(TOKENIZER.bos_token + raw_text) - collect_gen = dict() - current_index = 0 - for tokenized_cond_text in seq: + print("= Prefix of sentence =") + print(TOKENIZER.decode(tokenized_cond_text)) + print() - text = TOKENIZER.decode(tokenized_cond_text) - print("=" * 40 + " Prefix of sentence " + "=" * 40) - print(text) - print("=" * 80) + # generate unperturbed and perturbed texts - out1, out_perturb, discrim_loss_list, loss_in_time_list, actual_words = full_text_generation( - model=model, context=tokenized_cond_text, device=device, **vars(args) - ) + # full_text_generation returns: + # unpert_gen_tok_text, pert_gen_tok_texts, discrim_losses, losses_in_time + unpert_gen_tok_text, pert_gen_tok_texts, _, _ = full_text_generation( + model=model, context=tokenized_cond_text, device=device, **vars(args) + ) - text_whole = TOKENIZER.decode(out1.tolist()[0]) + # untokenize unperturbed text + unpert_gen_text = TOKENIZER.decode(unpert_gen_tok_text.tolist()[0]) - print("=" * 80) - print("=" * 40 + " Whole sentence (Original)" + "=" * 40) - print(text_whole) - print("=" * 80) + print("=" * 80) + print("= Unperturbed generated text =") + print(unpert_gen_text) + print() - out_perturb_copy = out_perturb + generated_texts = [] - for out_perturb in out_perturb_copy: - # try: - # print("=" * 40 + " Whole sentence (Perturbed)" + "=" * 40) - # text_whole = TOKENIZER.decode(out_perturb.tolist()[0]) - # print(text_whole) - # print("=" * 80) - # except: - # pass - # collect_gen[current_index] = [out, out_perturb, out1] - ## Save the prefix, perturbed seq, original seq for each index - print("=" * 40 + " Whole sentence (Perturbed)" + "=" * 40) - keyword_tokens = [aa[-1][0] for aa in - actual_words] if actual_words else [] - output_tokens = out_perturb.tolist()[0] + bow_words = set() + bow_indices = get_bag_of_words_indices(args.bag_of_words.split(";")) + for bow_list in bow_indices: + filtered = list(filter(lambda x: len(x) <= 1, bow_list)) + bow_words.update(w[0] for w in filtered) + # iterate through the perturbed texts + for i, pert_gen_tok_text in enumerate(pert_gen_tok_texts): + try: + # untokenize unperturbed text if args.colorama: import colorama - text_whole = '' - for tokenized_cond_text in output_tokens: - if tokenized_cond_text in keyword_tokens: - text_whole += '%s%s%s' % ( - colorama.Fore.GREEN, TOKENIZER.decode([tokenized_cond_text]), - colorama.Style.RESET_ALL) + pert_gen_text = '' + for word_id in pert_gen_tok_text.tolist()[0]: + if word_id in bow_words: + pert_gen_text += '{}{}{}'.format( + colorama.Fore.RED, + TOKENIZER.decode([word_id]), + colorama.Style.RESET_ALL + ) else: - text_whole += TOKENIZER.decode([tokenized_cond_text]) + pert_gen_text += TOKENIZER.decode([word_id]) else: - text_whole = TOKENIZER.decode(out_perturb.tolist()[0]) + pert_gen_text = TOKENIZER.decode(pert_gen_tok_text.tolist()[0]) - print(text_whole) - print("=" * 80) - - collect_gen[current_index] = [tokenized_cond_text, out_perturb, out1] - - current_index = current_index + 1 + print("= Perturbed generated text {} =".format(i + 1)) + print(pert_gen_text) + print() + except: + pass + # keep the prefix, perturbed seq, original seq for each index + generated_texts.append( + (tokenized_cond_text, pert_gen_tok_text, unpert_gen_tok_text) + ) return From 08c6e456a391cd463b0a52d125166aa766a2c7e4 Mon Sep 17 00:00:00 2001 From: piero Date: Wed, 27 Nov 2019 17:48:46 -0800 Subject: [PATCH 185/505] Cleaned full_text_generation. Identical output as before. --- examples/run_pplm.py | 103 ++++++++----------------------------------- 1 file changed, 19 insertions(+), 84 deletions(-) diff --git a/examples/run_pplm.py b/examples/run_pplm.py index 27ead3c3c5..b85998d706 100644 --- a/examples/run_pplm.py +++ b/examples/run_pplm.py @@ -401,74 +401,6 @@ def full_text_generation( device ) - # if args.discrim == 'clickbait': - # classifier = ClassificationHead(class_size=2, embed_size=1024).to(device) - # classifier.load_state_dict(torch.load("discrim_models/clickbait_classifierhead.pt")) - # classifier.eval() - # args.label_class = 1 # clickbaity - # - # elif args.discrim == 'sentiment': - # classifier = ClassificationHead(class_size=5, embed_size=1024).to(device) - # #classifier.load_state_dict(torch.load("discrim_models/sentiment_classifierhead.pt")) - # classifier.load_state_dict(torch.load("discrim_models/SST_classifier_head_epoch_16.pt")) - # classifier.eval() - # if args.label_class < 0: - # raise Exception('Wrong class for sentiment, use --label-class 2 for *very positive*, 3 for *very negative*') - # #args.label_class = 2 # very pos - # #args.label_class = 3 # very neg - # - # elif args.discrim == 'toxicity': - # classifier = ClassificationHead(class_size=2, embed_size=1024).to(device) - # classifier.load_state_dict(torch.load("discrim_models/toxicity_classifierhead.pt")) - # classifier.eval() - # args.label_class = 0 # not toxic - # - # elif args.discrim == 'generic': - # if args.discrim_weights is None: - # raise ValueError('When using a generic discriminator, ' - # 'discrim_weights need to be specified') - # if args.discrim_meta is None: - # raise ValueError('When using a generic discriminator, ' - # 'discrim_meta need to be specified') - # - # with open(args.discrim_meta, 'r') as discrim_meta_file: - # meta = json.load(discrim_meta_file) - # - # classifier = ClassificationHead( - # class_size=meta['class_size'], - # embed_size=meta['embed_size'], - # # todo add tokenizer from meta - # ).to(device) - # classifier.load_state_dict(torch.load(args.discrim_weights)) - # classifier.eval() - # if args.label_class == -1: - # args.label_class = meta['default_class'] - # - # else: - # classifier = None - - # Get tokens for the list of positive words - def list_tokens(word_list): - token_list = [TOKENIZER.encode(word, add_prefix_space=True) for word in - word_list] - # token_list = [] - # for word in word_list: - # token_list.append(TOKENIZER.encode(" " + word)) - return token_list - - # good_index = [] - # if args.bag_of_words: - # bags_of_words = args.bag_of_words.split(";") - # for wordlist in bags_of_words: - # with open(wordlist, "r") as f: - # words = f.read().strip() - # words = words.split('\n') - # good_index.append(list_tokens(words)) - # - # for good_list in good_index: - # good_list = list(filter(lambda x: len(x) <= 1, good_list)) - # actual_words = [(TOKENIZER.decode(ww).strip(),ww) for ww in good_list] - bow_indices = [] if bag_of_words: bow_indices = get_bag_of_words_indices(bag_of_words.split(";")) @@ -486,9 +418,9 @@ def full_text_generation( print("Using PPLM-Discrim") else: - raise Exception("Specify either --bag_of_words (-B) or --discrim (-D)") + raise Exception("Specify either a bag of words or a discriminator") - original, _, _ = generate_text_pplm( + unpert_gen_tok_text, _, _ = generate_text_pplm( model=model, context=context, device=device, @@ -497,12 +429,12 @@ def full_text_generation( ) torch.cuda.empty_cache() - perturbed_list = [] - discrim_loss_list = [] - loss_in_time_list = [] + pert_gen_tok_texts = [] + discrim_losses = [] + losses_in_time = [] for i in range(num_samples): - perturbed, discrim_loss, loss_in_time = generate_text_pplm( + pert_gen_tok_text, discrim_loss, loss_in_time = generate_text_pplm( model=model, context=context, device=device, @@ -525,14 +457,14 @@ def full_text_generation( decay=decay, gamma=gamma, ) - perturbed_list.append(perturbed) + pert_gen_tok_texts.append(pert_gen_tok_text) if classifier is not None: - discrim_loss_list.append(discrim_loss.data.cpu().numpy()) - loss_in_time_list.append(loss_in_time) + discrim_losses.append(discrim_loss.data.cpu().numpy()) + losses_in_time.append(loss_in_time) torch.cuda.empty_cache() - return original, perturbed_list, discrim_loss_list, loss_in_time_list + return unpert_gen_tok_text, pert_gen_tok_texts, discrim_losses, losses_in_time def generate_text_pplm( @@ -821,11 +753,14 @@ def run_model(): generated_texts = [] - bow_words = set() - bow_indices = get_bag_of_words_indices(args.bag_of_words.split(";")) - for bow_list in bow_indices: - filtered = list(filter(lambda x: len(x) <= 1, bow_list)) - bow_words.update(w[0] for w in filtered) + bow_word_ids = set() + if args.bag_of_words and args.colorama: + bow_indices = get_bag_of_words_indices(args.bag_of_words.split(";")) + for single_bow_list in bow_indices: + # filtering all words in the list composed of more than 1 token + filtered = list(filter(lambda x: len(x) <= 1, single_bow_list)) + # w[0] because we are sure w has only 1 item because previous fitler + bow_word_ids.update(w[0] for w in filtered) # iterate through the perturbed texts for i, pert_gen_tok_text in enumerate(pert_gen_tok_texts): @@ -836,7 +771,7 @@ def run_model(): pert_gen_text = '' for word_id in pert_gen_tok_text.tolist()[0]: - if word_id in bow_words: + if word_id in bow_word_ids: pert_gen_text += '{}{}{}'.format( colorama.Fore.RED, TOKENIZER.decode([word_id]), From 7ea12db3f539af414579c0e27b87e399645c58ed Mon Sep 17 00:00:00 2001 From: piero Date: Wed, 27 Nov 2019 17:49:39 -0800 Subject: [PATCH 186/505] Removed commented code. Identical output as before. --- examples/run_pplm.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/examples/run_pplm.py b/examples/run_pplm.py index b85998d706..f4d697de3b 100644 --- a/examples/run_pplm.py +++ b/examples/run_pplm.py @@ -347,13 +347,6 @@ def get_bag_of_words_indices(bag_of_words_ids_or_paths: List[str]) -> List[ bow_indices.append( [TOKENIZER.encode(word.strip(), add_prefix_space=True) for word in words]) - - #bow_words = set() - #for bow_list in bow_indices: - # bow_list = list(filter(lambda x: len(x) <= 1, bow_list)) - # bow_words.update( - # (TOKENIZER.decode(word).strip(), word) for word in bow_list) - return bow_indices From ef47b2c03ad3d901f4b7454b004eb44cb01cc3e3 Mon Sep 17 00:00:00 2001 From: piero Date: Wed, 27 Nov 2019 17:50:21 -0800 Subject: [PATCH 187/505] Removed commented code. Identical output as before. --- examples/run_pplm.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/examples/run_pplm.py b/examples/run_pplm.py index f4d697de3b..b18e97b5a8 100644 --- a/examples/run_pplm.py +++ b/examples/run_pplm.py @@ -128,20 +128,6 @@ def perturb_past( decay=False, gamma=1.5, ): - # def perturb_past(past, model, prev, classifier, good_index=None, - # stepsize=0.01, vocab_size=50257, - # original_probs=None, accumulated_hidden=None, true_past=None, - # grad_norms=None): - - # one_hot_bows_vectors = [] - # for good_list in good_index: - # good_list = list(filter(lambda x: len(x) <= 1, good_list)) - # good_list = torch.tensor(good_list).cuda() - # num_good = good_list.shape[0] - # one_hot_good = torch.zeros(num_good, vocab_size).cuda() - # one_hot_good.scatter_(1, good_list, 1) - # one_hot_bows_vectors.append(one_hot_good) - # Generate inital perturbed past past_perturb_orig = [ (np.random.uniform(0.0, 0.0, p.shape).astype('float32')) From 61a12f790de9795af1c3dff5fad7e2c4f6808d05 Mon Sep 17 00:00:00 2001 From: piero Date: Wed, 27 Nov 2019 17:54:49 -0800 Subject: [PATCH 188/505] Renamed SmallConst to SMALL_CONST and introduced BIG_CONST. Identical output as before. --- examples/run_pplm.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/examples/run_pplm.py b/examples/run_pplm.py index b18e97b5a8..4d335a9241 100644 --- a/examples/run_pplm.py +++ b/examples/run_pplm.py @@ -43,7 +43,7 @@ PPLM_BOW = 1 PPLM_DISCRIM = 2 PPLM_BOW_DISCRIM = 3 SMALL_CONST = 1e-15 -SmallConst = 1e-15 +BIG_CONST = 1e10 TOKENIZER = GPT2Tokenizer.from_pretrained("gpt2-medium") BAG_OF_WORDS_ARCHIVE_MAP = { @@ -104,7 +104,8 @@ def top_k_filter(logits, k, probs=False): if probs: return torch.where(logits < batch_mins, torch.ones_like(logits) * 0.0, logits) - return torch.where(logits < batch_mins, torch.ones_like(logits) * -1e10, + return torch.where(logits < batch_mins, + torch.ones_like(logits) * -BIG_CONST, logits) @@ -137,7 +138,7 @@ def perturb_past( accumulated_hidden = 0 if decay: - decay_mask = torch.arange(0., 1.0 + SmallConst, 1.0 / (window_length))[ + decay_mask = torch.arange(0., 1.0 + SMALL_CONST, 1.0 / (window_length))[ 1:] else: decay_mask = 1.0 @@ -233,9 +234,9 @@ def perturb_past( kl_loss = 0.0 if kl_scale > 0.0: p = (F.softmax(unpert_logits[:, -1, :], dim=-1)) - p = p + SmallConst * (p <= SmallConst).type( + p = p + SMALL_CONST * (p <= SMALL_CONST).type( torch.FloatTensor).cuda().detach() - correction = SmallConst * (probabs <= SmallConst).type( + correction = SMALL_CONST * (probabs <= SMALL_CONST).type( torch.FloatTensor).cuda().detach() corrected_probabs = probabs + correction.detach() kl_loss = kl_scale * ( @@ -254,7 +255,7 @@ def perturb_past( for index, p_ in enumerate(past_perturb)] else: - grad_norms = [(torch.norm(p_.grad * window_mask) + SmallConst) for + grad_norms = [(torch.norm(p_.grad * window_mask) + SMALL_CONST) for index, p_ in enumerate(past_perturb)] grad = [ @@ -560,31 +561,31 @@ def generate_text_pplm( # Piero modified model call # hidden = model.hidden_states # update hidden # logits = model.forward_hidden(hidden) - logits = logits[:, -1, :] / temperature # + SmallConst + logits = logits[:, -1, :] / temperature # + SMALL_CONST - # logits = top_k_filter(logits, k=args.top_k) # + SmallConst + # logits = top_k_filter(logits, k=args.top_k) # + SMALL_CONST log_probs = F.softmax(logits, dim=-1) # Fuse the modified model and original model if perturb: - # original_probs = top_k_filter(original_probs[:, -1, :]) #+ SmallConst + # original_probs = top_k_filter(original_probs[:, -1, :]) #+ SMALL_CONST unpert_logits = F.softmax(unpert_logits[:, -1, :], dim=-1) # likelywords = torch.topk(original_probs, k=10, dim=-1) # print(TOKENIZER.decode(likelywords[1].tolist()[0])) log_probs = ((log_probs ** gm_scale) * ( - unpert_logits ** (1 - gm_scale))) # + SmallConst + unpert_logits ** (1 - gm_scale))) # + SMALL_CONST log_probs = top_k_filter(log_probs, k=top_k, - probs=True) # + SmallConst + probs=True) # + SMALL_CONST if torch.sum(log_probs) <= 1: log_probs = log_probs / torch.sum(log_probs) else: - logits = top_k_filter(logits, k=top_k) # + SmallConst + logits = top_k_filter(logits, k=top_k) # + SMALL_CONST log_probs = F.softmax(logits, dim=-1) if sample: From 9f693a0c4831d09ba2c177128452984091fda619 Mon Sep 17 00:00:00 2001 From: piero Date: Wed, 27 Nov 2019 18:16:30 -0800 Subject: [PATCH 189/505] Cleaned generate_text_pplm. Identical output as before. --- examples/run_pplm.py | 125 ++++++++++++++++++------------------------- 1 file changed, 53 insertions(+), 72 deletions(-) diff --git a/examples/run_pplm.py b/examples/run_pplm.py index 4d335a9241..bd03bbe5e0 100644 --- a/examples/run_pplm.py +++ b/examples/run_pplm.py @@ -471,59 +471,49 @@ def generate_text_pplm( decay=False, gamma=1.5, ): - output = torch.tensor(context, device=device, dtype=torch.long).unsqueeze( - 0) if context else None + output_so_far = ( + torch.tensor(context, device=device, dtype=torch.long).unsqueeze(0) + if context + else None + ) # collect one hot vectors for bags of words one_hot_bows_vectors = build_bows_one_hot_vectors(bow_indices) grad_norms = None + unpert_discrim_loss = 0 loss_in_time = [] for i in trange(length, ascii=True): # Get past/probs for current output, except for last word - # Note that GPT takes 2 inputs: past + current-token - # Therefore, use everything from before current i/p token to generate relevant past + # Note that GPT takes 2 inputs: past + current_token - if past is None and output is not None: - prev = output[:, -1:] - # _, past = model(output[:, :-1]) - # original_probs, true_past = model(output) - # true_hidden = model.hidden_states + # run model forward to obtain unperturbed + if past is None and output_so_far is not None: + last = output_so_far[:, -1:] + _, past, _ = model(output_so_far[:, :-1]) - # Piero modified model call - _, past, _ = model(output[:, :-1]) - unpert_logits, unpert_past, unpert_all_hidden = model(output) - true_hidden = unpert_all_hidden[-1] - - else: - # original_probs, true_past = model(output) - # true_hidden = model.hidden_states - - # Piero modified model call - unpert_logits, unpert_past, unpert_all_hidden = model(output) - true_hidden = unpert_all_hidden[-1] - - # Modify the past if necessary + unpert_logits, unpert_past, unpert_all_hidden = model(output_so_far) + unpert_last_hidden = unpert_all_hidden[-1] + # check if we are abowe grad max length if i >= grad_length: current_stepsize = stepsize * 0 else: current_stepsize = stepsize + # modify the past if necessary if not perturb or num_iterations == 0: - perturbed_past = past + pert_past = past else: - # Piero modified model call - # accumulated_hidden = model.hidden_states[:, :-1, :] - accumulated_hidden = true_hidden[:, :-1, :] + accumulated_hidden = unpert_last_hidden[:, :-1, :] accumulated_hidden = torch.sum(accumulated_hidden, dim=1) - perturbed_past, _, grad_norms, loss_per_iter = perturb_past( + pert_past, _, grad_norms, loss_this_iter = perturb_past( past, model, - prev, + last, unpert_past=unpert_past, unpert_logits=unpert_logits, accumulated_hidden=accumulated_hidden, @@ -540,68 +530,59 @@ def generate_text_pplm( decay=decay, gamma=gamma, ) - loss_in_time.append(loss_per_iter) + loss_in_time.append(loss_this_iter) - # Piero modified model call - logits, past, pert_all_hidden = model(prev, past=perturbed_past) - # test_logits = F.softmax(test_logits[:, -1, :], dim=-1) - # likelywords = torch.topk(test_logits, k=10, dim=-1) - # print(TOKENIZER.decode(likelywords[1].tolist()[0])) + pert_logits, past, pert_all_hidden = model(last, past=pert_past) + pert_logits = pert_logits[:, -1, :] / temperature # + SMALL_CONST + pert_probs = F.softmax(pert_logits, dim=-1) if classifier is not None: ce_loss = torch.nn.CrossEntropyLoss() - predicted_sentiment = classifier(torch.mean(true_hidden, dim=1)) + prediction = classifier(torch.mean(unpert_last_hidden, dim=1)) label = torch.tensor([label_class], device='cuda', dtype=torch.long) - true_discrim_loss = ce_loss(predicted_sentiment, label) - print("true discrim loss", true_discrim_loss.data.cpu().numpy()) + unpert_discrim_loss = ce_loss(prediction, label) + print( + "unperturbed discrim loss", + unpert_discrim_loss.data.cpu().numpy() + ) else: - true_discrim_loss = 0 - - # Piero modified model call - # hidden = model.hidden_states # update hidden - # logits = model.forward_hidden(hidden) - logits = logits[:, -1, :] / temperature # + SMALL_CONST - - # logits = top_k_filter(logits, k=args.top_k) # + SMALL_CONST - - log_probs = F.softmax(logits, dim=-1) + unpert_discrim_loss = 0 # Fuse the modified model and original model if perturb: - # original_probs = top_k_filter(original_probs[:, -1, :]) #+ SMALL_CONST - unpert_logits = F.softmax(unpert_logits[:, -1, :], dim=-1) - # likelywords = torch.topk(original_probs, k=10, dim=-1) - # print(TOKENIZER.decode(likelywords[1].tolist()[0])) + unpert_probs = F.softmax(unpert_logits[:, -1, :], dim=-1) - log_probs = ((log_probs ** gm_scale) * ( - unpert_logits ** (1 - gm_scale))) # + SMALL_CONST - - log_probs = top_k_filter(log_probs, k=top_k, + pert_probs = ((pert_probs ** gm_scale) * ( + unpert_probs ** (1 - gm_scale))) # + SMALL_CONST + pert_probs = top_k_filter(pert_probs, k=top_k, probs=True) # + SMALL_CONST - if torch.sum(log_probs) <= 1: - log_probs = log_probs / torch.sum(log_probs) + # rescale + if torch.sum(pert_probs) <= 1: + pert_probs = pert_probs / torch.sum(pert_probs) else: - logits = top_k_filter(logits, k=top_k) # + SMALL_CONST - log_probs = F.softmax(logits, dim=-1) + pert_logits = top_k_filter(pert_logits, k=top_k) # + SMALL_CONST + pert_probs = F.softmax(pert_logits, dim=-1) + # sample or greedy if sample: - # likelywords = torch.topk(log_probs, k=args.top_k, dim=-1) - # print(TOKENIZER.decode(likelywords[1].tolist()[0])) - # print(likelywords[0].tolist()) - prev = torch.multinomial(log_probs, num_samples=1) - else: - _, prev = torch.topk(log_probs, k=1, dim=-1) - # if perturb: - # prev = future - output = prev if output is None else torch.cat((output, prev), - dim=1) # update output - print(TOKENIZER.decode(output.tolist()[0])) + last = torch.multinomial(pert_probs, num_samples=1) - return output, true_discrim_loss, loss_in_time + else: + _, last = torch.topk(pert_probs, k=1, dim=-1) + + # update context/output_so_far appending the new token + output_so_far = ( + last if output_so_far is None + else torch.cat((output_so_far, last), dim=1) + ) + + print(TOKENIZER.decode(output_so_far.tolist()[0])) + + return output_so_far, unpert_discrim_loss, loss_in_time def run_model(): From ffc29354051ccd4fa3fd12010abacb0ff2e0733e Mon Sep 17 00:00:00 2001 From: piero Date: Wed, 27 Nov 2019 18:30:42 -0800 Subject: [PATCH 190/505] Fix for making unditioned generation work. Identical output as before. --- examples/run_pplm.py | 49 ++++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/examples/run_pplm.py b/examples/run_pplm.py index bd03bbe5e0..e337add46d 100644 --- a/examples/run_pplm.py +++ b/examples/run_pplm.py @@ -481,6 +481,7 @@ def generate_text_pplm( one_hot_bows_vectors = build_bows_one_hot_vectors(bow_indices) grad_norms = None + last = None unpert_discrim_loss = 0 loss_in_time = [] for i in trange(length, ascii=True): @@ -491,7 +492,8 @@ def generate_text_pplm( # run model forward to obtain unperturbed if past is None and output_so_far is not None: last = output_so_far[:, -1:] - _, past, _ = model(output_so_far[:, :-1]) + if output_so_far.shape[1] > 1: + _, past, _ = model(output_so_far[:, :-1]) unpert_logits, unpert_past, unpert_all_hidden = model(output_so_far) unpert_last_hidden = unpert_all_hidden[-1] @@ -510,27 +512,30 @@ def generate_text_pplm( accumulated_hidden = unpert_last_hidden[:, :-1, :] accumulated_hidden = torch.sum(accumulated_hidden, dim=1) - pert_past, _, grad_norms, loss_this_iter = perturb_past( - past, - model, - last, - unpert_past=unpert_past, - unpert_logits=unpert_logits, - accumulated_hidden=accumulated_hidden, - grad_norms=grad_norms, - stepsize=current_stepsize, - classifier=classifier, - label_class=label_class, - one_hot_bows_vectors=one_hot_bows_vectors, - loss_type=loss_type, - num_iterations=num_iterations, - kl_scale=kl_scale, - window_length=window_length, - horizon_length=horizon_length, - decay=decay, - gamma=gamma, - ) - loss_in_time.append(loss_this_iter) + if past is not None: + pert_past, _, grad_norms, loss_this_iter = perturb_past( + past, + model, + last, + unpert_past=unpert_past, + unpert_logits=unpert_logits, + accumulated_hidden=accumulated_hidden, + grad_norms=grad_norms, + stepsize=current_stepsize, + classifier=classifier, + label_class=label_class, + one_hot_bows_vectors=one_hot_bows_vectors, + loss_type=loss_type, + num_iterations=num_iterations, + kl_scale=kl_scale, + window_length=window_length, + horizon_length=horizon_length, + decay=decay, + gamma=gamma, + ) + loss_in_time.append(loss_this_iter) + else: + pert_past = past pert_logits, past, pert_all_hidden = model(last, past=pert_past) pert_logits = pert_logits[:, -1, :] / temperature # + SMALL_CONST From 61399e5afe15915f5e074fd25faf85a5346e093a Mon Sep 17 00:00:00 2001 From: piero Date: Wed, 27 Nov 2019 19:51:42 -0800 Subject: [PATCH 191/505] Cleaned perturb_past. Identical output as before. --- examples/run_pplm.py | 202 ++++++++++++++++++++++++------------------- 1 file changed, 111 insertions(+), 91 deletions(-) diff --git a/examples/run_pplm.py b/examples/run_pplm.py index e337add46d..77758759d9 100644 --- a/examples/run_pplm.py +++ b/examples/run_pplm.py @@ -112,7 +112,7 @@ def top_k_filter(logits, k, probs=False): def perturb_past( past, model, - prev, + last, unpert_past=None, unpert_logits=None, accumulated_hidden=None, @@ -128,156 +128,174 @@ def perturb_past( horizon_length=1, decay=False, gamma=1.5, + device='cuda' ): # Generate inital perturbed past - past_perturb_orig = [ - (np.random.uniform(0.0, 0.0, p.shape).astype('float32')) - for p in past] + grad_accumulator = [ + (np.zeros(p.shape).astype("float32")) + for p in past + ] if accumulated_hidden is None: accumulated_hidden = 0 if decay: - decay_mask = torch.arange(0., 1.0 + SMALL_CONST, 1.0 / (window_length))[ - 1:] + decay_mask = torch.arange( + 0., + 1.0 + SMALL_CONST, + 1.0 / (window_length) + )[1:] else: decay_mask = 1.0 + # TODO fix this comment (SUMANTH) # Generate a mask is gradient perturbated is based on a past window - _, _, _, current_length, _ = past[0].shape + _, _, _, curr_length, _ = past[0].shape - if current_length > window_length and window_length > 0: - ones_key_val_shape = tuple(past[0].shape[:-2]) + tuple( - [window_length]) + tuple( - past[0].shape[-1:]) + if curr_length > window_length and window_length > 0: + ones_key_val_shape = ( + tuple(past[0].shape[:-2]) + + tuple([window_length]) + + tuple(past[0].shape[-1:]) + ) - zeros_key_val_shape = tuple(past[0].shape[:-2]) + tuple( - [current_length - window_length]) + tuple( - past[0].shape[-1:]) + zeros_key_val_shape = ( + tuple(past[0].shape[:-2]) + + tuple([curr_length - window_length]) + + tuple(past[0].shape[-1:]) + ) ones_mask = torch.ones(ones_key_val_shape) ones_mask = decay_mask * ones_mask.permute(0, 1, 2, 4, 3) ones_mask = ones_mask.permute(0, 1, 2, 4, 3) - window_mask = torch.cat((ones_mask, torch.zeros(zeros_key_val_shape)), - dim=-2).cuda() + window_mask = torch.cat( + (ones_mask, torch.zeros(zeros_key_val_shape)), + dim=-2 + ).to(device) else: - window_mask = torch.ones_like(past[0]).cuda() + window_mask = torch.ones_like(past[0]).to(device) + # accumulate perturbations for num_iterations loss_per_iter = [] + new_accumulated_hidden = None for i in range(num_iterations): print("Iteration ", i + 1) - past_perturb = [torch.from_numpy(p_) for p_ in past_perturb_orig] - past_perturb = [to_var(p_, requires_grad=True) for p_ in past_perturb] + curr_perturbation = [ + to_var(torch.from_numpy(p_), requires_grad=True) + for p_ in grad_accumulator + ] - perturbed_past = list(map(add, past, past_perturb)) - - _, _, _, current_length, _ = past_perturb[0].shape - - # _, future_past = model(prev, past=perturbed_past) - # hidden = model.hidden_states - - # Piero modified model call - logits, _, all_hidden = model(prev, past=perturbed_past) + # Compute hidden using perturbed past + perturbed_past = list(map(add, past, curr_perturbation)) + _, _, _, curr_length, _ = curr_perturbation[0].shape + all_logits, _, all_hidden = model(last, past=perturbed_past) hidden = all_hidden[-1] - new_accumulated_hidden = accumulated_hidden + torch.sum(hidden, - dim=1).detach() + new_accumulated_hidden = accumulated_hidden + torch.sum( + hidden, + dim=1 + ).detach() + # TODO: Check the layer-norm consistency of this with trained discriminator (Sumanth) + logits = all_logits[:, -1, :] + probs = F.softmax(logits, dim=-1) - # TODO: Check the layer-norm consistency of this with trained discriminator - logits = logits[:, -1, :] - probabs = F.softmax(logits, dim=-1) loss = 0.0 loss_list = [] - if loss_type == 1 or loss_type == 3: - for one_hot_good in one_hot_bows_vectors: - good_logits = torch.mm(probabs, torch.t(one_hot_good)) - loss_word = good_logits - loss_word = torch.sum(loss_word) - loss_word = -torch.log(loss_word) - # loss_word = torch.sum(loss_word) /torch.sum(one_hot_good) - loss += loss_word - loss_list.append(loss_word) + if loss_type == PPLM_BOW or loss_type == PPLM_BOW_DISCRIM: + for one_hot_bow in one_hot_bows_vectors: + bow_logits = torch.mm(probs, torch.t(one_hot_bow)) + bow_loss = -torch.log(torch.sum(bow_logits)) + loss += bow_loss + loss_list.append(bow_loss) print(" pplm_bow_loss:", loss.data.cpu().numpy()) if loss_type == 2 or loss_type == 3: ce_loss = torch.nn.CrossEntropyLoss() - new_true_past = unpert_past - for i in range(horizon_length): - future_probabs = F.softmax(logits, dim=-1) # Get softmax - future_probabs = torch.unsqueeze(future_probabs, dim=1) - - # _, new_true_past = model(future_probabs, past=new_true_past) - # future_hidden = model.hidden_states # Get expected hidden states - - # Piero modified model call - wte = model.resize_token_embeddings() - inputs_embeds = torch.matmul(future_probabs, wte.weight.data) - _, new_true_past, future_hidden = model( - past=new_true_past, + # TODO why we need to do this assignment and not just using unpert_past? (Sumanth) + curr_unpert_past = unpert_past + curr_probs = torch.unsqueeze(probs, dim=1) + wte = model.resize_token_embeddings() + for _ in range(horizon_length): + inputs_embeds = torch.matmul(curr_probs, wte.weight.data) + _, curr_unpert_past, curr_all_hidden = model( + past=curr_unpert_past, inputs_embeds=inputs_embeds ) - future_hidden = future_hidden[-1] - + curr_hidden = curr_all_hidden[-1] new_accumulated_hidden = new_accumulated_hidden + torch.sum( - future_hidden, dim=1) + curr_hidden, dim=1) - predicted_sentiment = classifier(new_accumulated_hidden / ( - current_length + 1 + horizon_length)) + prediction = classifier(new_accumulated_hidden / + (curr_length + 1 + horizon_length)) - label = torch.tensor([label_class], device='cuda', + label = torch.tensor([label_class], device=device, dtype=torch.long) - discrim_loss = ce_loss(predicted_sentiment, label) + discrim_loss = ce_loss(prediction, label) print(" pplm_discrim_loss:", discrim_loss.data.cpu().numpy()) loss += discrim_loss loss_list.append(discrim_loss) kl_loss = 0.0 if kl_scale > 0.0: - p = (F.softmax(unpert_logits[:, -1, :], dim=-1)) - p = p + SMALL_CONST * (p <= SMALL_CONST).type( - torch.FloatTensor).cuda().detach() - correction = SMALL_CONST * (probabs <= SMALL_CONST).type( - torch.FloatTensor).cuda().detach() - corrected_probabs = probabs + correction.detach() + unpert_probs = F.softmax(unpert_logits[:, -1, :], dim=-1) + unpert_probs = ( + unpert_probs + SMALL_CONST * + (unpert_probs <= SMALL_CONST).float().to(device).detach() + ) + correction = SMALL_CONST * (probs <= SMALL_CONST).float().to(device).detach() + corrected_probs = probs + correction.detach() kl_loss = kl_scale * ( - (corrected_probabs * (corrected_probabs / p).log()).sum()) - print(' kl_loss', (kl_loss).data.cpu().numpy()) - loss += kl_loss # + discrim_loss + (corrected_probs * (corrected_probs / unpert_probs).log()).sum() + ) + print(' kl_loss', kl_loss.data.cpu().numpy()) + loss += kl_loss loss_per_iter.append(loss.data.cpu().numpy()) - print(' pplm_loss', (loss - kl_loss).data.cpu().numpy()) + # compute gradients loss.backward() - if grad_norms is not None and loss_type == 1: + + # calculate gradient norms + if grad_norms is not None and loss_type == PPLM_BOW: grad_norms = [ torch.max(grad_norms[index], torch.norm(p_.grad * window_mask)) - for index, p_ in - enumerate(past_perturb)] + for index, p_ in enumerate(curr_perturbation) + ] else: - grad_norms = [(torch.norm(p_.grad * window_mask) + SMALL_CONST) for - index, p_ in enumerate(past_perturb)] + grad_norms = [ + (torch.norm(p_.grad * window_mask) + SMALL_CONST) + for index, p_ in enumerate(curr_perturbation) + ] + # normalize gradients grad = [ - -stepsize * (p_.grad * window_mask / grad_norms[ - index] ** gamma).data.cpu().numpy() - for index, p_ in enumerate(past_perturb)] - past_perturb_orig = list(map(add, grad, past_perturb_orig)) + -stepsize * + (p_.grad * window_mask / grad_norms[index] ** gamma).data.cpu().numpy() + for index, p_ in enumerate(curr_perturbation) + ] - for p_ in past_perturb: + # accumulate gradient + grad_accumulator = list(map(add, grad, grad_accumulator)) + + # reset gradients, just to make sure + for p_ in curr_perturbation: p_.grad.data.zero_() + # removing past from the graph new_past = [] - for p in past: - new_past.append(p.detach()) - + for p_ in past: + new_past.append(p_.detach()) past = new_past - past_perturb = [torch.from_numpy(p_) for p_ in past_perturb_orig] - past_perturb = [to_var(p_, requires_grad=True) for p_ in past_perturb] - perturbed_past = list(map(add, past, past_perturb)) + # apply the accumulated perturbations to the past + grad_accumulator = [ + to_var(torch.from_numpy(p_), requires_grad=True) + for p_ in grad_accumulator + ] + pert_past = list(map(add, past, grad_accumulator)) - return perturbed_past, new_accumulated_hidden, grad_norms, loss_per_iter + return pert_past, new_accumulated_hidden, grad_norms, loss_per_iter def get_classifier( @@ -532,6 +550,7 @@ def generate_text_pplm( horizon_length=horizon_length, decay=decay, gamma=gamma, + device=device ) loss_in_time.append(loss_this_iter) else: @@ -562,7 +581,7 @@ def generate_text_pplm( pert_probs = ((pert_probs ** gm_scale) * ( unpert_probs ** (1 - gm_scale))) # + SMALL_CONST pert_probs = top_k_filter(pert_probs, k=top_k, - probs=True) # + SMALL_CONST + probs=True) # + SMALL_CONST # rescale if torch.sum(pert_probs) <= 1: @@ -662,7 +681,8 @@ def run_model(): parser.add_argument("--decay", action="store_true", help="whether to decay or not") parser.add_argument("--gamma", type=float, default=1.5) - parser.add_argument("--colorama", action="store_true", help="colors keywords") + parser.add_argument("--colorama", action="store_true", + help="colors keywords") args = parser.parse_args() From afc7dcd94d2480e1fa6ef91ec8e9029142566612 Mon Sep 17 00:00:00 2001 From: piero Date: Wed, 27 Nov 2019 20:08:53 -0800 Subject: [PATCH 192/505] Now run_pplm works on cpu. Identical output as before (when using gpu). --- examples/run_pplm.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/examples/run_pplm.py b/examples/run_pplm.py index 77758759d9..0d6b0d635d 100644 --- a/examples/run_pplm.py +++ b/examples/run_pplm.py @@ -84,9 +84,11 @@ DISCRIMINATOR_MODELS_PARAMS = { } -def to_var(x, requires_grad=False, volatile=False): - if torch.cuda.is_available(): +def to_var(x, requires_grad=False, volatile=False, device='cuda'): + if torch.cuda.is_available() and device == 'cuda': x = x.cuda() + elif device != 'cuda': + x = x.to(device) return Variable(x, requires_grad=requires_grad, volatile=volatile) @@ -182,7 +184,7 @@ def perturb_past( for i in range(num_iterations): print("Iteration ", i + 1) curr_perturbation = [ - to_var(torch.from_numpy(p_), requires_grad=True) + to_var(torch.from_numpy(p_), requires_grad=True, device=device) for p_ in grad_accumulator ] @@ -290,7 +292,7 @@ def perturb_past( # apply the accumulated perturbations to the past grad_accumulator = [ - to_var(torch.from_numpy(p_), requires_grad=True) + to_var(torch.from_numpy(p_), requires_grad=True, device=device) for p_ in grad_accumulator ] pert_past = list(map(add, past, grad_accumulator)) @@ -300,7 +302,7 @@ def perturb_past( def get_classifier( name: Optional[str], label_class: Union[str, int], - device: Union[str, torch.device] + device: str ) -> Tuple[Optional[ClassificationHead], Optional[int]]: if name is None: return None, None @@ -355,16 +357,16 @@ def get_bag_of_words_indices(bag_of_words_ids_or_paths: List[str]) -> List[ return bow_indices -def build_bows_one_hot_vectors(bow_indices): +def build_bows_one_hot_vectors(bow_indices, device='cuda'): if bow_indices is None: return None one_hot_bows_vectors = [] for single_bow in bow_indices: single_bow = list(filter(lambda x: len(x) <= 1, single_bow)) - single_bow = torch.tensor(single_bow).cuda() + single_bow = torch.tensor(single_bow).to(device) num_words = single_bow.shape[0] - one_hot_bow = torch.zeros(num_words, TOKENIZER.vocab_size).cuda() + one_hot_bow = torch.zeros(num_words, TOKENIZER.vocab_size).to(device) one_hot_bow.scatter_(1, single_bow, 1) one_hot_bows_vectors.append(one_hot_bow) return one_hot_bows_vectors @@ -425,7 +427,8 @@ def full_text_generation( length=length, perturb=False ) - torch.cuda.empty_cache() + if device == 'cuda': + torch.cuda.empty_cache() pert_gen_tok_texts = [] discrim_losses = [] @@ -460,7 +463,8 @@ def full_text_generation( discrim_losses.append(discrim_loss.data.cpu().numpy()) losses_in_time.append(loss_in_time) - torch.cuda.empty_cache() + if device == 'cuda': + torch.cuda.empty_cache() return unpert_gen_tok_text, pert_gen_tok_texts, discrim_losses, losses_in_time @@ -496,7 +500,7 @@ def generate_text_pplm( ) # collect one hot vectors for bags of words - one_hot_bows_vectors = build_bows_one_hot_vectors(bow_indices) + one_hot_bows_vectors = build_bows_one_hot_vectors(bow_indices, device) grad_norms = None last = None @@ -563,7 +567,7 @@ def generate_text_pplm( if classifier is not None: ce_loss = torch.nn.CrossEntropyLoss() prediction = classifier(torch.mean(unpert_last_hidden, dim=1)) - label = torch.tensor([label_class], device='cuda', + label = torch.tensor([label_class], device=device, dtype=torch.long) unpert_discrim_loss = ce_loss(prediction, label) print( From 611961ade71042ff759712e6f680544ec5ff68b9 Mon Sep 17 00:00:00 2001 From: piero Date: Wed, 27 Nov 2019 21:34:49 -0800 Subject: [PATCH 193/505] Added tqdm to preprocessing --- examples/run_pplm_discrim_train.py | 206 ++++++++++++++--------------- 1 file changed, 102 insertions(+), 104 deletions(-) diff --git a/examples/run_pplm_discrim_train.py b/examples/run_pplm_discrim_train.py index 519e2de29a..5291ad4b51 100644 --- a/examples/run_pplm_discrim_train.py +++ b/examples/run_pplm_discrim_train.py @@ -18,13 +18,14 @@ import torch.utils.data as data from nltk.tokenize.treebank import TreebankWordDetokenizer from torchtext import data as torchtext_data from torchtext import datasets +from tqdm import tqdm, trange from transformers import GPT2Tokenizer, GPT2LMHeadModel torch.manual_seed(0) np.random.seed(0) EPSILON = 1e-10 -device = 'cpu' +device = "cpu" example_sentence = "This is incredible! I love it, this is the best chicken I have ever had." max_length_seq = 100 @@ -109,8 +110,8 @@ class Dataset(data.Dataset): def __getitem__(self, index): """Returns one data pair (source and target).""" data = {} - data['X'] = self.X[index] - data['y'] = self.y[index] + data["X"] = self.X[index] + data["y"] = self.y[index] return data @@ -133,8 +134,8 @@ def collate_fn(data): for key in data[0].keys(): item_info[key] = [d[key] for d in data] - x_batch, _ = pad_sequences(item_info['X']) - y_batch = torch.tensor(item_info['y'], dtype=torch.long) + x_batch, _ = pad_sequences(item_info["X"]) + y_batch = torch.tensor(item_info["y"], dtype=torch.long) return x_batch, y_batch @@ -144,8 +145,8 @@ def cached_collate_fn(data): for key in data[0].keys(): item_info[key] = [d[key] for d in data] - x_batch = torch.cat(item_info['X'], 0) - y_batch = torch.tensor(item_info['y'], dtype=torch.long) + x_batch = torch.cat(item_info["X"], 0) + y_batch = torch.tensor(item_info["y"], dtype=torch.long) return x_batch, y_batch @@ -168,7 +169,7 @@ def train_epoch(data_loader, discriminator, optimizer, if batch_idx % log_interval == 0: print( - 'Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( + "Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}".format( epoch + 1, samples_so_far, len(data_loader.dataset), 100 * samples_so_far / len(data_loader.dataset), loss.item() @@ -185,7 +186,7 @@ def evaluate_performance(data_loader, discriminator): input_t, target_t = input_t.to(device), target_t.to(device) output_t = discriminator(input_t) # sum up batch loss - test_loss += F.nll_loss(output_t, target_t, reduction='sum').item() + test_loss += F.nll_loss(output_t, target_t, reduction="sum").item() # get the index of the max log-probability pred_t = output_t.argmax(dim=1, keepdim=True) correct += pred_t.eq(target_t.view_as(pred_t)).sum().item() @@ -193,8 +194,8 @@ def evaluate_performance(data_loader, discriminator): test_loss /= len(data_loader.dataset) print( - 'Performance on test set: ' - 'Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)'.format( + "Performance on test set: " + "Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)".format( test_loss, correct, len(data_loader.dataset), 100. * correct / len(data_loader.dataset) ) @@ -208,8 +209,8 @@ def predict(input_sentence, model, classes, cached=False): input_t = model.avg_representation(input_t) log_probs = model(input_t).data.cpu().numpy().flatten().tolist() - print('Input sentence:', input_sentence) - print('Predictions:', ", ".join( + print("Input sentence:", input_sentence) + print("Predictions:", ", ".join( "{}: {:.4f}".format(c, math.exp(log_prob)) for c, log_prob in zip(classes, log_probs) )) @@ -222,7 +223,7 @@ def get_cached_data_loader(dataset, batch_size, discriminator, shuffle=False): xs = [] ys = [] - for batch_idx, (x, y) in enumerate(data_loader): + for batch_idx, (x, y) in enumerate(tqdm(data_loader, ascii=True)): with torch.no_grad(): x = x.to(device) avg_rep = discriminator.avg_representation(x).cpu().detach() @@ -240,16 +241,16 @@ def get_cached_data_loader(dataset, batch_size, discriminator, shuffle=False): def train_discriminator( - dataset, dataset_fp=None, pretrained_model='gpt2-medium', + dataset, dataset_fp=None, pretrained_model="gpt2-medium", epochs=10, batch_size=64, log_interval=10, save_model=False, cached=False, no_cuda=False): global device device = "cuda" if torch.cuda.is_available() and not no_cuda else "cpu" - print('Preprocessing {} dataset...'.format(dataset)) + print("Preprocessing {} dataset...".format(dataset)) start = time.time() - if dataset == 'SST': + if dataset == "SST": idx2class = ["positive", "negative", "very positive", "very negative", "neutral"] class2idx = {c: i for i, c in enumerate(idx2class)} @@ -271,7 +272,7 @@ def train_discriminator( x = [] y = [] - for i in range(len(train_data)): + for i in trange(len(train_data), ascii=True): seq = TreebankWordDetokenizer().detokenize( vars(train_data[i])["text"] ) @@ -283,7 +284,7 @@ def train_discriminator( test_x = [] test_y = [] - for i in range(len(test_data)): + for i in trange(len(test_data), ascii=True): seq = TreebankWordDetokenizer().detokenize( vars(test_data[i])["text"] ) @@ -301,7 +302,7 @@ def train_discriminator( "default_class": 2, } - elif dataset == 'clickbait': + elif dataset == "clickbait": idx2class = ["non_clickbait", "clickbait"] class2idx = {c: i for i, c in enumerate(idx2class)} @@ -317,31 +318,33 @@ def train_discriminator( try: data.append(eval(line)) except: - print('Error evaluating line {}: {}'.format( + print("Error evaluating line {}: {}".format( i, line )) continue x = [] y = [] - y = [] - for i, d in enumerate(data): - try: - seq = discriminator.tokenizer.encode(d["text"]) + with open("datasets/clickbait/clickbait_train_prefix.txt") as f: + for i, line in enumerate(tqdm(f, ascii=True)): + try: + d = eval(line) + seq = discriminator.tokenizer.encode(d["text"]) - if len(seq) < max_length_seq: - seq = torch.tensor( - [50256] + seq, device=device, dtype=torch.long - ) - else: - print("Line {} is longer than maximum length {}".format( - i, max_length_seq - )) - continue - x.append(seq) - y.append(d['label']) - except: - print("Error tokenizing line {}, skipping it".format(i)) - pass + if len(seq) < max_length_seq: + seq = torch.tensor( + [50256] + seq, device=device, dtype=torch.long + ) + else: + print("Line {} is longer than maximum length {}".format( + i, max_length_seq + )) + continue + x.append(seq) + y.append(d["label"]) + except: + print("Error evaluating / tokenizing" + " line {}, skipping it".format(i)) + pass full_dataset = Dataset(x, y) train_size = int(0.9 * len(full_dataset)) @@ -358,7 +361,7 @@ def train_discriminator( "default_class": 1, } - elif dataset == 'toxic': + elif dataset == "toxic": idx2class = ["non_toxic", "toxic"] class2idx = {c: i for i, c in enumerate(idx2class)} @@ -368,37 +371,29 @@ def train_discriminator( cached_mode=cached ).to(device) - with open("datasets/toxic/toxic_train.txt") as f: - data = [] - for i, line in enumerate(f): - try: - data.append(eval(line)) - except: - print('Error evaluating line {}: {}'.format( - i, line - )) - continue - x = [] y = [] - for i, d in enumerate(data): - try: - seq = discriminator.tokenizer.encode(d["text"]) + with open("datasets/toxic/toxic_train.txt") as f: + for i, line in enumerate(tqdm(f, ascii=True)): + try: + d = eval(line) + seq = discriminator.tokenizer.encode(d["text"]) - if len(seq) < max_length_seq: - seq = torch.tensor( - [50256] + seq, device=device, dtype=torch.long - ) - else: - print("Line {} is longer than maximum length {}".format( - i, max_length_seq - )) - continue - x.append(seq) - y.append(int(np.sum(d['label']) > 0)) - except: - print("Error tokenizing line {}, skipping it".format(i)) - pass + if len(seq) < max_length_seq: + seq = torch.tensor( + [50256] + seq, device=device, dtype=torch.long + ) + else: + print("Line {} is longer than maximum length {}".format( + i, max_length_seq + )) + continue + x.append(seq) + y.append(int(np.sum(d["label"]) > 0)) + except: + print("Error evaluating / tokenizing" + " line {}, skipping it".format(i)) + pass full_dataset = Dataset(x, y) train_size = int(0.9 * len(full_dataset)) @@ -415,18 +410,18 @@ def train_discriminator( "default_class": 0, } - else: # if dataset == 'generic': + else: # if dataset == "generic": # This assumes the input dataset is a TSV with the following structure: # class \t text if dataset_fp is None: - raise ValueError('When generic dataset is selected, ' - 'dataset_fp needs to be specified aswell.') + raise ValueError("When generic dataset is selected, " + "dataset_fp needs to be specified aswell.") classes = set() with open(dataset_fp) as f: - csv_reader = csv.reader(f, delimiter='\t') - for row in csv_reader: + csv_reader = csv.reader(f, delimiter="\t") + for row in tqdm(csv_reader, ascii=True): if row: classes.add(row[0]) @@ -442,8 +437,8 @@ def train_discriminator( x = [] y = [] with open(dataset_fp) as f: - csv_reader = csv.reader(f, delimiter='\t') - for i, row in enumerate(csv_reader): + csv_reader = csv.reader(f, delimiter="\t") + for i, row in enumerate(tqdm(csv_reader, ascii=True)): if row: label = row[0] text = row[1] @@ -458,9 +453,10 @@ def train_discriminator( ) else: - print("Line {} is longer than maximum length {}".format( - i, max_length_seq - )) + print( + "Line {} is longer than maximum length {}".format( + i, max_length_seq + )) continue x.append(seq) @@ -487,12 +483,14 @@ def train_discriminator( } end = time.time() - print('Preprocessed {} data points'.format( + print("Preprocessed {} data points".format( len(train_dataset) + len(test_dataset)) ) print("Data preprocessing took: {:.3f}s".format(end - start)) if cached: + print("Building representation cache...") + start = time.time() train_loader = get_cached_data_loader( @@ -524,7 +522,7 @@ def train_discriminator( for epoch in range(epochs): start = time.time() - print('\nEpoch', epoch + 1) + print("\nEpoch", epoch + 1) train_epoch( discriminator=discriminator, @@ -553,31 +551,31 @@ def train_discriminator( "{}_classifier_head_epoch_{}.pt".format(dataset, epoch)) -if __name__ == '__main__': +if __name__ == "__main__": parser = argparse.ArgumentParser( - description='Train a discriminator on top of GPT-2 representations') - parser.add_argument('--dataset', type=str, default='SST', - choices=('SST', 'clickbait', 'toxic', 'generic'), - help='dataset to train the discriminator on.' - 'In case of generic, the dataset is expected' - 'to be a TSBV file with structure: class \\t text') - parser.add_argument('--dataset_fp', type=str, default='', - help='File path of the dataset to use. ' - 'Needed only in case of generic datadset') - parser.add_argument('--pretrained_model', type=str, default='gpt2-medium', - help='Pretrained model to use as encoder') - parser.add_argument('--epochs', type=int, default=10, metavar='N', - help='Number of training epochs') - parser.add_argument('--batch_size', type=int, default=64, metavar='N', - help='input batch size for training (default: 64)') - parser.add_argument('--log_interval', type=int, default=10, metavar='N', - help='how many batches to wait before logging training status') - parser.add_argument('--save_model', action='store_true', - help='whether to save the model') - parser.add_argument('--cached', action='store_true', - help='whether to cache the input representations') - parser.add_argument('--no_cuda', action='store_true', - help='use to turn off cuda') + description="Train a discriminator on top of GPT-2 representations") + parser.add_argument("--dataset", type=str, default="SST", + choices=("SST", "clickbait", "toxic", "generic"), + help="dataset to train the discriminator on." + "In case of generic, the dataset is expected" + "to be a TSBV file with structure: class \\t text") + parser.add_argument("--dataset_fp", type=str, default="", + help="File path of the dataset to use. " + "Needed only in case of generic datadset") + parser.add_argument("--pretrained_model", type=str, default="gpt2-medium", + help="Pretrained model to use as encoder") + parser.add_argument("--epochs", type=int, default=10, metavar="N", + help="Number of training epochs") + parser.add_argument("--batch_size", type=int, default=64, metavar="N", + help="input batch size for training (default: 64)") + parser.add_argument("--log_interval", type=int, default=10, metavar="N", + help="how many batches to wait before logging training status") + parser.add_argument("--save_model", action="store_true", + help="whether to save the model") + parser.add_argument("--cached", action="store_true", + help="whether to cache the input representations") + parser.add_argument("--no_cuda", action="store_true", + help="use to turn off cuda") args = parser.parse_args() train_discriminator(**(vars(args))) From b0eaff36e6aefd45c0fe89bb2ab86a495e4e735f Mon Sep 17 00:00:00 2001 From: piero Date: Wed, 27 Nov 2019 21:43:43 -0800 Subject: [PATCH 194/505] Added a +1 to epoch when saving weights --- examples/run_pplm_discrim_train.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/run_pplm_discrim_train.py b/examples/run_pplm_discrim_train.py index 5291ad4b51..fccfb14426 100644 --- a/examples/run_pplm_discrim_train.py +++ b/examples/run_pplm_discrim_train.py @@ -545,10 +545,11 @@ def train_discriminator( if save_model: # torch.save(discriminator.state_dict(), # "{}_discriminator_{}.pt".format( - # args.dataset, epoch + # args.dataset, epoch + 1 # )) torch.save(discriminator.get_classifier().state_dict(), - "{}_classifier_head_epoch_{}.pt".format(dataset, epoch)) + "{}_classifier_head_epoch_{}.pt".format(dataset, + epoch + 1)) if __name__ == "__main__": From 7fd54b55a3f7c3134f8cc5a62f4cc447a5cd34de Mon Sep 17 00:00:00 2001 From: piero Date: Wed, 27 Nov 2019 21:45:19 -0800 Subject: [PATCH 195/505] Added support for generic discriminators --- examples/run_pplm.py | 77 +++++++++++++++++++++++++++++--------------- 1 file changed, 51 insertions(+), 26 deletions(-) diff --git a/examples/run_pplm.py b/examples/run_pplm.py index 0d6b0d635d..28aa66cc7d 100644 --- a/examples/run_pplm.py +++ b/examples/run_pplm.py @@ -14,17 +14,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -# TODO: add code for training a custom discriminator - """ Example command with bag of words: python examples/run_pplm.py -B space --cond_text "The president" --length 100 --gamma 1.5 --num_iterations 3 --num_samples 10 --stepsize 0.01 --window_length 5 --kl_scale 0.01 --gm_scale 0.95 Example command with discriminator: -python examples/run_pplm.py -D sentiment --label_class 3 --cond_text "The lake" --length 10 --gamma 1.0 --num_iterations 30 --num_samples 10 --stepsize 0.01 --kl_scale 0.01 --gm_scale 0.95 +python examples/run_pplm.py -D sentiment --class_label 3 --cond_text "The lake" --length 10 --gamma 1.0 --num_iterations 30 --num_samples 10 --stepsize 0.01 --kl_scale 0.01 --gm_scale 0.95 """ import argparse +import json from operator import add from typing import List, Optional, Tuple, Union @@ -121,7 +120,7 @@ def perturb_past( grad_norms=None, stepsize=0.01, classifier=None, - label_class=None, + class_label=None, one_hot_bows_vectors=None, loss_type=0, num_iterations=3, @@ -230,7 +229,7 @@ def perturb_past( prediction = classifier(new_accumulated_hidden / (curr_length + 1 + horizon_length)) - label = torch.tensor([label_class], device=device, + label = torch.tensor([class_label], device=device, dtype=torch.long) discrim_loss = ce_loss(prediction, label) print(" pplm_discrim_loss:", discrim_loss.data.cpu().numpy()) @@ -244,7 +243,8 @@ def perturb_past( unpert_probs + SMALL_CONST * (unpert_probs <= SMALL_CONST).float().to(device).detach() ) - correction = SMALL_CONST * (probs <= SMALL_CONST).float().to(device).detach() + correction = SMALL_CONST * (probs <= SMALL_CONST).float().to( + device).detach() corrected_probs = probs + correction.detach() kl_loss = kl_scale * ( (corrected_probs * (corrected_probs / unpert_probs).log()).sum() @@ -273,7 +273,8 @@ def perturb_past( # normalize gradients grad = [ -stepsize * - (p_.grad * window_mask / grad_norms[index] ** gamma).data.cpu().numpy() + (p_.grad * window_mask / grad_norms[ + index] ** gamma).data.cpu().numpy() for index, p_ in enumerate(curr_perturbation) ] @@ -301,7 +302,7 @@ def perturb_past( def get_classifier( - name: Optional[str], label_class: Union[str, int], + name: Optional[str], class_label: Union[str, int], device: str ) -> Tuple[Optional[ClassificationHead], Optional[int]]: if name is None: @@ -312,26 +313,29 @@ def get_classifier( class_size=params['class_size'], embed_size=params['embed_size'] ).to(device) - resolved_archive_file = cached_path(params["url"]) + if "url" in params: + resolved_archive_file = cached_path(params["url"]) + else: + resolved_archive_file = params["path"] classifier.load_state_dict( torch.load(resolved_archive_file, map_location=device)) classifier.eval() - if isinstance(label_class, str): - if label_class in params["class_vocab"]: - label_id = params["class_vocab"][label_class] + if isinstance(class_label, str): + if class_label in params["class_vocab"]: + label_id = params["class_vocab"][class_label] else: label_id = params["default_class"] - print("label_class {} not in class_vocab".format(label_class)) + print("class_label {} not in class_vocab".format(class_label)) print("available values are: {}".format(params["class_vocab"])) print("using default class {}".format(label_id)) - elif isinstance(label_class, int): - if label_class in set(params["class_vocab"].values()): - label_id = label_class + elif isinstance(class_label, int): + if class_label in set(params["class_vocab"].values()): + label_id = class_label else: label_id = params["default_class"] - print("label_class {} not in class_vocab".format(label_class)) + print("class_label {} not in class_vocab".format(class_label)) print("available values are: {}".format(params["class_vocab"])) print("using default class {}".format(label_id)) @@ -379,7 +383,7 @@ def full_text_generation( device="cuda", sample=True, discrim=None, - label_class=None, + class_label=None, bag_of_words=None, length=100, grad_length=10000, @@ -397,7 +401,7 @@ def full_text_generation( ): classifier, class_id = get_classifier( discrim, - label_class, + class_label, device ) @@ -443,7 +447,7 @@ def full_text_generation( perturb=True, bow_indices=bow_indices, classifier=classifier, - label_class=class_id, + class_label=class_id, loss_type=loss_type, length=length, grad_length=grad_length, @@ -477,7 +481,7 @@ def generate_text_pplm( sample=True, perturb=True, classifier=None, - label_class=None, + class_label=None, bow_indices=None, loss_type=0, length=100, @@ -545,7 +549,7 @@ def generate_text_pplm( grad_norms=grad_norms, stepsize=current_stepsize, classifier=classifier, - label_class=label_class, + class_label=class_label, one_hot_bows_vectors=one_hot_bows_vectors, loss_type=loss_type, num_iterations=num_iterations, @@ -567,7 +571,7 @@ def generate_text_pplm( if classifier is not None: ce_loss = torch.nn.CrossEntropyLoss() prediction = classifier(torch.mean(unpert_last_hidden, dim=1)) - label = torch.tensor([label_class], device=device, + label = torch.tensor([class_label], device=device, dtype=torch.long) unpert_discrim_loss = ce_loss(prediction, label) print( @@ -613,6 +617,20 @@ def generate_text_pplm( return output_so_far, unpert_discrim_loss, loss_in_time +def set_generic_model_params(discrim_weights, discrim_meta): + if discrim_weights is None: + raise ValueError('When using a generic discriminator, ' + 'discrim_weights need to be specified') + if discrim_meta is None: + raise ValueError('When using a generic discriminator, ' + 'discrim_meta need to be specified') + + with open(discrim_meta, 'r') as discrim_meta_file: + meta = json.load(discrim_meta_file) + meta['path'] = discrim_weights + DISCRIMINATOR_MODELS_PARAMS['generic'] = meta + + def run_model(): parser = argparse.ArgumentParser() parser.add_argument( @@ -636,11 +654,15 @@ def run_model(): "-D", type=str, default=None, - choices=("clickbait", "sentiment", "toxicity"), - help="Discriminator to use for loss-type 2", + choices=("clickbait", "sentiment", "toxicity", "generic"), + help="Discriminator to use", ) + parser.add_argument('--discrim_weights', type=str, default=None, + help='Weights for the generic discriminator') + parser.add_argument('--discrim_meta', type=str, default=None, + help='Meta information for the generic discriminator') parser.add_argument( - "--label_class", + "--class_label", type=int, default=-1, help="Class label used for the discriminator", @@ -697,6 +719,9 @@ def run_model(): # set the device device = "cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu" + if args.discrim == 'generic': + set_generic_model_params(args.discrim_weights, args.discrim_meta) + # load pretrained model model = GPT2LMHeadModel.from_pretrained( args.model_path, From 75904dae669249c9f5d4d4d57890fb6c537d1639 Mon Sep 17 00:00:00 2001 From: w4nderlust Date: Fri, 29 Nov 2019 18:51:27 -0800 Subject: [PATCH 196/505] Removed global variable device --- examples/run_pplm_discrim_train.py | 47 ++++++++++++++++++------------ 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/examples/run_pplm_discrim_train.py b/examples/run_pplm_discrim_train.py index fccfb14426..db081e1a17 100644 --- a/examples/run_pplm_discrim_train.py +++ b/examples/run_pplm_discrim_train.py @@ -25,7 +25,6 @@ from transformers import GPT2Tokenizer, GPT2LMHeadModel torch.manual_seed(0) np.random.seed(0) EPSILON = 1e-10 -device = "cpu" example_sentence = "This is incredible! I love it, this is the best chicken I have ever had." max_length_seq = 100 @@ -55,7 +54,8 @@ class Discriminator(torch.nn.Module): self, class_size, pretrained_model="gpt2-medium", - cached_mode=False + cached_mode=False, + device='cpu' ): super(Discriminator, self).__init__() self.tokenizer = GPT2Tokenizer.from_pretrained(pretrained_model) @@ -66,6 +66,7 @@ class Discriminator(torch.nn.Module): embed_size=self.embed_size ) self.cached_mode = cached_mode + self.device = device def get_classifier(self): return self.classifier_head @@ -78,7 +79,7 @@ class Discriminator(torch.nn.Module): def avg_representation(self, x): mask = x.ne(0).unsqueeze(2).repeat( 1, 1, self.embed_size - ).float().to(device).detach() + ).float().to(self.device).detach() hidden, _ = self.encoder.transformer(x) masked_hidden = hidden * mask avg_hidden = torch.sum(masked_hidden, dim=1) / ( @@ -88,9 +89,9 @@ class Discriminator(torch.nn.Module): def forward(self, x): if self.cached_mode: - avg_hidden = x.to(device) + avg_hidden = x.to(self.device) else: - avg_hidden = self.avg_representation(x.to(device)) + avg_hidden = self.avg_representation(x.to(self.device)) logits = self.classifier_head(avg_hidden) probs = F.log_softmax(logits, dim=-1) @@ -152,7 +153,7 @@ def cached_collate_fn(data): def train_epoch(data_loader, discriminator, optimizer, - epoch=0, log_interval=10): + epoch=0, log_interval=10, device='cpu'): samples_so_far = 0 discriminator.train_custom() for batch_idx, (input_t, target_t) in enumerate(data_loader): @@ -177,7 +178,7 @@ def train_epoch(data_loader, discriminator, optimizer, ) -def evaluate_performance(data_loader, discriminator): +def evaluate_performance(data_loader, discriminator, device='cpu'): discriminator.eval() test_loss = 0 correct = 0 @@ -202,7 +203,7 @@ def evaluate_performance(data_loader, discriminator): ) -def predict(input_sentence, model, classes, cached=False): +def predict(input_sentence, model, classes, cached=False, device='cpu'): input_t = model.tokenizer.encode(input_sentence) input_t = torch.tensor([input_t], dtype=torch.long, device=device) if cached: @@ -216,7 +217,8 @@ def predict(input_sentence, model, classes, cached=False): )) -def get_cached_data_loader(dataset, batch_size, discriminator, shuffle=False): +def get_cached_data_loader(dataset, batch_size, discriminator, + shuffle=False, device='cpu'): data_loader = torch.utils.data.DataLoader(dataset=dataset, batch_size=batch_size, collate_fn=collate_fn) @@ -244,7 +246,6 @@ def train_discriminator( dataset, dataset_fp=None, pretrained_model="gpt2-medium", epochs=10, batch_size=64, log_interval=10, save_model=False, cached=False, no_cuda=False): - global device device = "cuda" if torch.cuda.is_available() and not no_cuda else "cpu" print("Preprocessing {} dataset...".format(dataset)) @@ -258,7 +259,8 @@ def train_discriminator( discriminator = Discriminator( class_size=len(idx2class), pretrained_model=pretrained_model, - cached_mode=cached + cached_mode=cached, + device=device ).to(device) text = torchtext_data.Field() @@ -309,7 +311,8 @@ def train_discriminator( discriminator = Discriminator( class_size=len(idx2class), pretrained_model=pretrained_model, - cached_mode=cached + cached_mode=cached, + device=device ).to(device) with open("datasets/clickbait/clickbait_train_prefix.txt") as f: @@ -368,7 +371,8 @@ def train_discriminator( discriminator = Discriminator( class_size=len(idx2class), pretrained_model=pretrained_model, - cached_mode=cached + cached_mode=cached, + device=device ).to(device) x = [] @@ -431,7 +435,8 @@ def train_discriminator( discriminator = Discriminator( class_size=len(idx2class), pretrained_model=pretrained_model, - cached_mode=cached + cached_mode=cached, + device=device ).to(device) x = [] @@ -494,11 +499,12 @@ def train_discriminator( start = time.time() train_loader = get_cached_data_loader( - train_dataset, batch_size, discriminator, shuffle=True + train_dataset, batch_size, discriminator, + shuffle=True, device=device ) test_loader = get_cached_data_loader( - test_dataset, batch_size, discriminator + test_dataset, batch_size, discriminator, device=device ) end = time.time() @@ -529,18 +535,21 @@ def train_discriminator( data_loader=train_loader, optimizer=optimizer, epoch=epoch, - log_interval=log_interval + log_interval=log_interval, + device=device ) evaluate_performance( data_loader=test_loader, - discriminator=discriminator + discriminator=discriminator, + device=device ) end = time.time() print("Epoch took: {:.3f}s".format(end - start)) print("\nExample prediction") - predict(example_sentence, discriminator, idx2class, cached) + predict(example_sentence, discriminator, idx2class, + cached=cached, device=device) if save_model: # torch.save(discriminator.state_dict(), From f10b925015b03612877fc2213e118d9507dd3ff2 Mon Sep 17 00:00:00 2001 From: w4nderlust Date: Fri, 29 Nov 2019 19:59:02 -0800 Subject: [PATCH 197/505] Imrpovements: model_path renamed pretrained_model, tokenizer loaded from pretrained_model, pretrained_model set to discriminator's when discrim is specified, sample = False by default but cli parameter introduced. To obtain identical samples call the cli with --sample --- examples/run_pplm.py | 300 ++++++++++++++++++++++++++----------------- 1 file changed, 185 insertions(+), 115 deletions(-) diff --git a/examples/run_pplm.py b/examples/run_pplm.py index 28aa66cc7d..8516454f86 100644 --- a/examples/run_pplm.py +++ b/examples/run_pplm.py @@ -43,7 +43,6 @@ PPLM_DISCRIM = 2 PPLM_BOW_DISCRIM = 3 SMALL_CONST = 1e-15 BIG_CONST = 1e10 -TOKENIZER = GPT2Tokenizer.from_pretrained("gpt2-medium") BAG_OF_WORDS_ARCHIVE_MAP = { 'kitchen': "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/bow/kitchen.txt", @@ -65,6 +64,7 @@ DISCRIMINATOR_MODELS_PARAMS = { "embed_size": 1024, "class_vocab": {"non_clickbait": 0, "clickbait": 1}, "default_class": 1, + "pretrained_model": "gpt2-medium", }, "sentiment": { "url": "http://s.yosinski.com/SST_classifier_head.pt", @@ -72,6 +72,7 @@ DISCRIMINATOR_MODELS_PARAMS = { "embed_size": 1024, "class_vocab": {"very_positive": 2, "very_negative": 3}, "default_class": 3, + "pretrained_model": "gpt2-medium", }, "toxicity": { "url": "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/discriminators/toxicity_classifierhead.pt", @@ -79,6 +80,7 @@ DISCRIMINATOR_MODELS_PARAMS = { "embed_size": 1024, "class_vocab": {"non_toxic": 0, "toxic": 1}, "default_class": 0, + "pretrained_model": "gpt2-medium", }, } @@ -345,8 +347,9 @@ def get_classifier( return classifier, label_id -def get_bag_of_words_indices(bag_of_words_ids_or_paths: List[str]) -> List[ - List[List[int]]]: +def get_bag_of_words_indices(bag_of_words_ids_or_paths: List[str], tokenizer) -> \ + List[ + List[List[int]]]: bow_indices = [] for id_or_path in bag_of_words_ids_or_paths: if id_or_path in BAG_OF_WORDS_ARCHIVE_MAP: @@ -356,12 +359,12 @@ def get_bag_of_words_indices(bag_of_words_ids_or_paths: List[str]) -> List[ with open(filepath, "r") as f: words = f.read().strip().split("\n") bow_indices.append( - [TOKENIZER.encode(word.strip(), add_prefix_space=True) for word in + [tokenizer.encode(word.strip(), add_prefix_space=True) for word in words]) return bow_indices -def build_bows_one_hot_vectors(bow_indices, device='cuda'): +def build_bows_one_hot_vectors(bow_indices, tokenizer, device='cuda'): if bow_indices is None: return None @@ -370,7 +373,7 @@ def build_bows_one_hot_vectors(bow_indices, device='cuda'): single_bow = list(filter(lambda x: len(x) <= 1, single_bow)) single_bow = torch.tensor(single_bow).to(device) num_words = single_bow.shape[0] - one_hot_bow = torch.zeros(num_words, TOKENIZER.vocab_size).to(device) + one_hot_bow = torch.zeros(num_words, tokenizer.vocab_size).to(device) one_hot_bow.scatter_(1, single_bow, 1) one_hot_bows_vectors.append(one_hot_bow) return one_hot_bows_vectors @@ -378,10 +381,11 @@ def build_bows_one_hot_vectors(bow_indices, device='cuda'): def full_text_generation( model, + tokenizer, context=None, num_samples=1, device="cuda", - sample=True, + sample=False, discrim=None, class_label=None, bag_of_words=None, @@ -407,7 +411,8 @@ def full_text_generation( bow_indices = [] if bag_of_words: - bow_indices = get_bag_of_words_indices(bag_of_words.split(";")) + bow_indices = get_bag_of_words_indices(bag_of_words.split(";"), + tokenizer) if bag_of_words and classifier: print("Both PPLM-BoW and PPLM-Discrim are on. This is not optimized.") @@ -426,9 +431,11 @@ def full_text_generation( unpert_gen_tok_text, _, _ = generate_text_pplm( model=model, + tokenizer=tokenizer, context=context, device=device, length=length, + sample=sample, perturb=False ) if device == 'cuda': @@ -441,6 +448,7 @@ def full_text_generation( for i in range(num_samples): pert_gen_tok_text, discrim_loss, loss_in_time = generate_text_pplm( model=model, + tokenizer=tokenizer, context=context, device=device, sample=sample, @@ -475,10 +483,11 @@ def full_text_generation( def generate_text_pplm( model, + tokenizer, context=None, past=None, device="cuda", - sample=True, + sample=False, perturb=True, classifier=None, class_label=None, @@ -504,7 +513,8 @@ def generate_text_pplm( ) # collect one hot vectors for bags of words - one_hot_bows_vectors = build_bows_one_hot_vectors(bow_indices, device) + one_hot_bows_vectors = build_bows_one_hot_vectors(bow_indices, tokenizer, + device) grad_norms = None last = None @@ -612,7 +622,7 @@ def generate_text_pplm( else torch.cat((output_so_far, last), dim=1) ) - print(TOKENIZER.decode(output_so_far.tolist()[0])) + print(tokenizer.decode(output_so_far.tolist()[0])) return output_so_far, unpert_discrim_loss, loss_in_time @@ -631,10 +641,167 @@ def set_generic_model_params(discrim_weights, discrim_meta): DISCRIMINATOR_MODELS_PARAMS['generic'] = meta -def run_model(): +def run_pplm_example( + pretrained_model="gpt2-medium", + cond_text="", + uncond=False, + num_samples=1, + bag_of_words=None, + discrim=None, + discrim_weights=None, + discrim_meta=None, + class_label=-1, + length=100, + stepsize=0.02, + temperature=1.0, + top_k=10, + sample=False, + num_iterations=3, + grad_length=10000, + horizon_length=1, + window_length=0, + decay=False, + gamma=1.5, + gm_scale=0.9, + kl_scale=0.01, + seed=0, + no_cuda=False, + colorama=False +): + # set Random seed + torch.manual_seed(seed) + np.random.seed(seed) + + # set the device + device = "cuda" if torch.cuda.is_available() and not no_cuda else "cpu" + + if discrim == 'generic': + set_generic_model_params(discrim_weights, discrim_meta) + + if discrim is not None: + pretrained_model = DISCRIMINATOR_MODELS_PARAMS[discrim][ + "pretrained_model" + ] + print("discrim = {}, setting pretrained_model " + "to discriminator's = {}".format(discrim, pretrained_model)) + + # load pretrained model + model = GPT2LMHeadModel.from_pretrained( + pretrained_model, + output_hidden_states=True + ) + model.to(device) + model.eval() + + # load tokenizer + tokenizer = GPT2Tokenizer.from_pretrained(pretrained_model) + + # Freeze GPT-2 weights + for param in model.parameters(): + param.requires_grad = False + + # figure out conditioning text + if uncond: + tokenized_cond_text = tokenizer.encode( + [tokenizer.bos_token] + ) + else: + raw_text = cond_text + while not raw_text: + print("Did you forget to add `--cond_text`? ") + raw_text = input("Model prompt >>> ") + tokenized_cond_text = tokenizer.encode(tokenizer.bos_token + raw_text) + + print("= Prefix of sentence =") + print(tokenizer.decode(tokenized_cond_text)) + print() + + # generate unperturbed and perturbed texts + + # full_text_generation returns: + # unpert_gen_tok_text, pert_gen_tok_texts, discrim_losses, losses_in_time + unpert_gen_tok_text, pert_gen_tok_texts, _, _ = full_text_generation( + model=model, + tokenizer=tokenizer, + context=tokenized_cond_text, + device=device, + num_samples=num_samples, + bag_of_words=bag_of_words, + discrim=discrim, + class_label=class_label, + length=length, + stepsize=stepsize, + temperature=temperature, + top_k=top_k, + sample=sample, + num_iterations=num_iterations, + grad_length=grad_length, + horizon_length=horizon_length, + window_length=window_length, + decay=decay, + gamma=gamma, + gm_scale=gm_scale, + kl_scale=kl_scale, + ) + + # untokenize unperturbed text + unpert_gen_text = tokenizer.decode(unpert_gen_tok_text.tolist()[0]) + + print("=" * 80) + print("= Unperturbed generated text =") + print(unpert_gen_text) + print() + + generated_texts = [] + + bow_word_ids = set() + if bag_of_words and colorama: + bow_indices = get_bag_of_words_indices(bag_of_words.split(";"), + tokenizer) + for single_bow_list in bow_indices: + # filtering all words in the list composed of more than 1 token + filtered = list(filter(lambda x: len(x) <= 1, single_bow_list)) + # w[0] because we are sure w has only 1 item because previous fitler + bow_word_ids.update(w[0] for w in filtered) + + # iterate through the perturbed texts + for i, pert_gen_tok_text in enumerate(pert_gen_tok_texts): + try: + # untokenize unperturbed text + if colorama: + import colorama + + pert_gen_text = '' + for word_id in pert_gen_tok_text.tolist()[0]: + if word_id in bow_word_ids: + pert_gen_text += '{}{}{}'.format( + colorama.Fore.RED, + tokenizer.decode([word_id]), + colorama.Style.RESET_ALL + ) + else: + pert_gen_text += tokenizer.decode([word_id]) + else: + pert_gen_text = tokenizer.decode(pert_gen_tok_text.tolist()[0]) + + print("= Perturbed generated text {} =".format(i + 1)) + print(pert_gen_text) + print() + except: + pass + + # keep the prefix, perturbed seq, original seq for each index + generated_texts.append( + (tokenized_cond_text, pert_gen_tok_text, unpert_gen_tok_text) + ) + + return + + +if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument( - "--model_path", + "--pretrained_model", "-M", type=str, default="gpt2-medium", @@ -675,6 +842,10 @@ def run_model(): parser.add_argument("--gm_scale", type=float, default=0.9) parser.add_argument("--kl_scale", type=float, default=0.01) parser.add_argument("--no_cuda", action="store_true", help="no cuda") + parser.add_argument( + "--sample", action="store_true", + help="Generate from end-of-text as prefix" + ) parser.add_argument( "--uncond", action="store_true", help="Generate from end-of-text as prefix" @@ -711,105 +882,4 @@ def run_model(): help="colors keywords") args = parser.parse_args() - - # set Random seed - torch.manual_seed(args.seed) - np.random.seed(args.seed) - - # set the device - device = "cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu" - - if args.discrim == 'generic': - set_generic_model_params(args.discrim_weights, args.discrim_meta) - - # load pretrained model - model = GPT2LMHeadModel.from_pretrained( - args.model_path, - output_hidden_states=True - ) - model.to(device) - model.eval() - - # Freeze GPT-2 weights - for param in model.parameters(): - param.requires_grad = False - - # figure out conditioning text - if args.uncond: - tokenized_cond_text = TOKENIZER.encode( - [TOKENIZER.bos_token] - ) - else: - raw_text = args.cond_text - while not raw_text: - print("Did you forget to add `--cond_text`? ") - raw_text = input("Model prompt >>> ") - tokenized_cond_text = TOKENIZER.encode(TOKENIZER.bos_token + raw_text) - - print("= Prefix of sentence =") - print(TOKENIZER.decode(tokenized_cond_text)) - print() - - # generate unperturbed and perturbed texts - - # full_text_generation returns: - # unpert_gen_tok_text, pert_gen_tok_texts, discrim_losses, losses_in_time - unpert_gen_tok_text, pert_gen_tok_texts, _, _ = full_text_generation( - model=model, context=tokenized_cond_text, device=device, **vars(args) - ) - - # untokenize unperturbed text - unpert_gen_text = TOKENIZER.decode(unpert_gen_tok_text.tolist()[0]) - - print("=" * 80) - print("= Unperturbed generated text =") - print(unpert_gen_text) - print() - - generated_texts = [] - - bow_word_ids = set() - if args.bag_of_words and args.colorama: - bow_indices = get_bag_of_words_indices(args.bag_of_words.split(";")) - for single_bow_list in bow_indices: - # filtering all words in the list composed of more than 1 token - filtered = list(filter(lambda x: len(x) <= 1, single_bow_list)) - # w[0] because we are sure w has only 1 item because previous fitler - bow_word_ids.update(w[0] for w in filtered) - - # iterate through the perturbed texts - for i, pert_gen_tok_text in enumerate(pert_gen_tok_texts): - try: - # untokenize unperturbed text - if args.colorama: - import colorama - - pert_gen_text = '' - for word_id in pert_gen_tok_text.tolist()[0]: - if word_id in bow_word_ids: - pert_gen_text += '{}{}{}'.format( - colorama.Fore.RED, - TOKENIZER.decode([word_id]), - colorama.Style.RESET_ALL - ) - else: - pert_gen_text += TOKENIZER.decode([word_id]) - else: - pert_gen_text = TOKENIZER.decode(pert_gen_tok_text.tolist()[0]) - - print("= Perturbed generated text {} =".format(i + 1)) - print(pert_gen_text) - print() - except: - pass - - # keep the prefix, perturbed seq, original seq for each index - generated_texts.append( - (tokenized_cond_text, pert_gen_tok_text, unpert_gen_tok_text) - ) - - return - - -if __name__ == '__main__': - run_model() + run_pplm_example(**vars(args)) From f42816e7fca8280927790f74c6e280c37d49b280 Mon Sep 17 00:00:00 2001 From: w4nderlust Date: Fri, 29 Nov 2019 20:00:43 -0800 Subject: [PATCH 198/505] Added additional check for url and path in discriminator model params --- examples/run_pplm.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/run_pplm.py b/examples/run_pplm.py index 8516454f86..9ddd42681e 100644 --- a/examples/run_pplm.py +++ b/examples/run_pplm.py @@ -317,8 +317,11 @@ def get_classifier( ).to(device) if "url" in params: resolved_archive_file = cached_path(params["url"]) - else: + elif "path" in params: resolved_archive_file = params["path"] + else: + raise ValueError("Either url or path have to be specified " + "in the discriminator model parameters") classifier.load_state_dict( torch.load(resolved_archive_file, map_location=device)) classifier.eval() From 893d0d64fe008a63eca89b8750502fa5cb684439 Mon Sep 17 00:00:00 2001 From: w4nderlust Date: Fri, 29 Nov 2019 20:19:33 -0800 Subject: [PATCH 199/505] Changed order of some parameters to be more consistent. Identical results. --- examples/run_pplm.py | 111 +++++++++++++++++++++---------------------- 1 file changed, 55 insertions(+), 56 deletions(-) diff --git a/examples/run_pplm.py b/examples/run_pplm.py index 9ddd42681e..57bed3890f 100644 --- a/examples/run_pplm.py +++ b/examples/run_pplm.py @@ -121,17 +121,17 @@ def perturb_past( accumulated_hidden=None, grad_norms=None, stepsize=0.01, + one_hot_bows_vectors=None, classifier=None, class_label=None, - one_hot_bows_vectors=None, loss_type=0, num_iterations=3, - kl_scale=0.01, - window_length=0, horizon_length=1, + window_length=0, decay=False, gamma=1.5, - device='cuda' + kl_scale=0.01, + device='cuda', ): # Generate inital perturbed past grad_accumulator = [ @@ -351,8 +351,7 @@ def get_classifier( def get_bag_of_words_indices(bag_of_words_ids_or_paths: List[str], tokenizer) -> \ - List[ - List[List[int]]]: + List[List[List[int]]]: bow_indices = [] for id_or_path in bag_of_words_ids_or_paths: if id_or_path in BAG_OF_WORDS_ARCHIVE_MAP: @@ -388,22 +387,22 @@ def full_text_generation( context=None, num_samples=1, device="cuda", - sample=False, + bag_of_words=None, discrim=None, class_label=None, - bag_of_words=None, length=100, - grad_length=10000, stepsize=0.02, - num_iterations=3, temperature=1.0, - gm_scale=0.9, - kl_scale=0.01, top_k=10, - window_length=0, + sample=False, + num_iterations=3, + grad_length=10000, horizon_length=1, + window_length=0, decay=False, gamma=1.5, + gm_scale=0.9, + kl_scale=0.01, **kwargs ): classifier, class_id = get_classifier( @@ -454,24 +453,24 @@ def full_text_generation( tokenizer=tokenizer, context=context, device=device, - sample=sample, perturb=True, bow_indices=bow_indices, classifier=classifier, class_label=class_id, loss_type=loss_type, length=length, - grad_length=grad_length, stepsize=stepsize, - num_iterations=num_iterations, temperature=temperature, - gm_scale=gm_scale, - kl_scale=kl_scale, top_k=top_k, - window_length=window_length, + sample=sample, + num_iterations=num_iterations, + grad_length=grad_length, horizon_length=horizon_length, + window_length=window_length, decay=decay, gamma=gamma, + gm_scale=gm_scale, + kl_scale=kl_scale, ) pert_gen_tok_texts.append(pert_gen_tok_text) if classifier is not None: @@ -490,24 +489,24 @@ def generate_text_pplm( context=None, past=None, device="cuda", - sample=False, perturb=True, + bow_indices=None, classifier=None, class_label=None, - bow_indices=None, loss_type=0, length=100, - grad_length=10000, stepsize=0.02, - num_iterations=3, temperature=1.0, - gm_scale=0.9, - kl_scale=0.01, top_k=10, - window_length=0, + sample=False, + num_iterations=3, + grad_length=10000, horizon_length=1, + window_length=0, decay=False, gamma=1.5, + gm_scale=0.9, + kl_scale=0.01, ): output_so_far = ( torch.tensor(context, device=device, dtype=torch.long).unsqueeze(0) @@ -561,17 +560,17 @@ def generate_text_pplm( accumulated_hidden=accumulated_hidden, grad_norms=grad_norms, stepsize=current_stepsize, + one_hot_bows_vectors=one_hot_bows_vectors, classifier=classifier, class_label=class_label, - one_hot_bows_vectors=one_hot_bows_vectors, loss_type=loss_type, num_iterations=num_iterations, - kl_scale=kl_scale, - window_length=window_length, horizon_length=horizon_length, + window_length=window_length, decay=decay, gamma=gamma, - device=device + kl_scale=kl_scale, + device=device, ) loss_in_time.append(loss_this_iter) else: @@ -685,7 +684,7 @@ def run_pplm_example( pretrained_model = DISCRIMINATOR_MODELS_PARAMS[discrim][ "pretrained_model" ] - print("discrim = {}, setting pretrained_model " + print("discrim = {}, pretrained_model set " "to discriminator's = {}".format(discrim, pretrained_model)) # load pretrained model @@ -810,6 +809,20 @@ if __name__ == '__main__': default="gpt2-medium", help="pretrained model name or path to local checkpoint", ) + parser.add_argument( + "--cond_text", type=str, default="The lake", + help="Prefix texts to condition on" + ) + parser.add_argument( + "--uncond", action="store_true", + help="Generate from end-of-text as prefix" + ) + parser.add_argument( + "--num_samples", + type=int, + default=1, + help="Number of samples to generate from the modified latents", + ) parser.add_argument( "--bag_of_words", "-B", @@ -837,40 +850,16 @@ if __name__ == '__main__': default=-1, help="Class label used for the discriminator", ) - parser.add_argument("--stepsize", type=float, default=0.02) parser.add_argument("--length", type=int, default=100) - parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--stepsize", type=float, default=0.02) parser.add_argument("--temperature", type=float, default=1.0) parser.add_argument("--top_k", type=int, default=10) - parser.add_argument("--gm_scale", type=float, default=0.9) - parser.add_argument("--kl_scale", type=float, default=0.01) - parser.add_argument("--no_cuda", action="store_true", help="no cuda") parser.add_argument( "--sample", action="store_true", help="Generate from end-of-text as prefix" ) - parser.add_argument( - "--uncond", action="store_true", - help="Generate from end-of-text as prefix" - ) - parser.add_argument( - "--cond_text", type=str, default="The lake", - help="Prefix texts to condition on" - ) parser.add_argument("--num_iterations", type=int, default=3) parser.add_argument("--grad_length", type=int, default=10000) - parser.add_argument( - "--num_samples", - type=int, - default=1, - help="Number of samples to generate from the modified latents", - ) - parser.add_argument( - "--horizon_length", - type=int, - default=1, - help="Length of future to optimize over", - ) parser.add_argument( "--window_length", type=int, @@ -878,9 +867,19 @@ if __name__ == '__main__': help="Length of past which is being optimized; " "0 corresponds to infinite window length", ) + parser.add_argument( + "--horizon_length", + type=int, + default=1, + help="Length of future to optimize over", + ) parser.add_argument("--decay", action="store_true", help="whether to decay or not") parser.add_argument("--gamma", type=float, default=1.5) + parser.add_argument("--gm_scale", type=float, default=0.9) + parser.add_argument("--kl_scale", type=float, default=0.01) + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--no_cuda", action="store_true", help="no cuda") parser.add_argument("--colorama", action="store_true", help="colors keywords") From a59fdd162703009e5774683ec65887a2ce419c1f Mon Sep 17 00:00:00 2001 From: Piero Molino Date: Sun, 1 Dec 2019 15:48:33 -0800 Subject: [PATCH 200/505] generate_text_pplm now works with batch_size > 1 --- examples/run_pplm.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/examples/run_pplm.py b/examples/run_pplm.py index 57bed3890f..5e09427879 100644 --- a/examples/run_pplm.py +++ b/examples/run_pplm.py @@ -231,7 +231,8 @@ def perturb_past( prediction = classifier(new_accumulated_hidden / (curr_length + 1 + horizon_length)) - label = torch.tensor([class_label], device=device, + label = torch.tensor(prediction.shape[0] * [class_label], + device=device, dtype=torch.long) discrim_loss = ce_loss(prediction, label) print(" pplm_discrim_loss:", discrim_loss.data.cpu().numpy()) @@ -508,11 +509,12 @@ def generate_text_pplm( gm_scale=0.9, kl_scale=0.01, ): - output_so_far = ( - torch.tensor(context, device=device, dtype=torch.long).unsqueeze(0) - if context - else None - ) + output_so_far = None + if context: + context_t = torch.tensor(context, device=device, dtype=torch.long) + while len(context_t.shape) < 2: + context_t = context_t.unsqueeze(0) + output_so_far = context_t # collect one hot vectors for bags of words one_hot_bows_vectors = build_bows_one_hot_vectors(bow_indices, tokenizer, From 1efb2ae7fc4b1350b2bd46bd9898b9e5a31f8381 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Mon, 2 Dec 2019 16:01:57 -0500 Subject: [PATCH 201/505] [pplm] move scripts under examples/pplm/ --- examples/{ => pplm}/run_pplm.py | 0 examples/{ => pplm}/run_pplm_discrim_train.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename examples/{ => pplm}/run_pplm.py (100%) rename examples/{ => pplm}/run_pplm_discrim_train.py (100%) diff --git a/examples/run_pplm.py b/examples/pplm/run_pplm.py similarity index 100% rename from examples/run_pplm.py rename to examples/pplm/run_pplm.py diff --git a/examples/run_pplm_discrim_train.py b/examples/pplm/run_pplm_discrim_train.py similarity index 100% rename from examples/run_pplm_discrim_train.py rename to examples/pplm/run_pplm_discrim_train.py From 0cb2c9089074cb1bf158da701c9e9c027ed08258 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Mon, 2 Dec 2019 18:17:28 -0500 Subject: [PATCH 202/505] readme Co-Authored-By: Rosanne Liu --- examples/pplm/README.md | 48 ++++++++++++++++++++++++++++++ examples/pplm/imgs/headfigure.png | Bin 0 -> 668261 bytes examples/pplm/imgs/wooly.png | Bin 0 -> 679776 bytes 3 files changed, 48 insertions(+) create mode 100644 examples/pplm/README.md create mode 100644 examples/pplm/imgs/headfigure.png create mode 100644 examples/pplm/imgs/wooly.png diff --git a/examples/pplm/README.md b/examples/pplm/README.md new file mode 100644 index 0000000000..6eb040a442 --- /dev/null +++ b/examples/pplm/README.md @@ -0,0 +1,48 @@ +# PPLM + +This folder contains the original code used to run the Plug and Play Language Model (PPLM). +![header image](./imgs/headfigure.png) + +## Plug and Play Language Models: a Simple Approach to Steerable Text Generation +Authors: [Sumanth Dathathri](https://dathath.github.io/), Andrea Madotto, Janice Lan, Jane Hung, Eric Frank, [Piero Molino](), [Jason Yosinski](http://yosinski.com/), and [Rosanne Liu](http://www.rosanneliu.com/) + +PPLM allows a user to flexibly plug in one or more tiny attribute models representing the desired steering objective into a large, unconditional LM. The method has the key property that it uses the LM _as is_---no training or fine-tuning is required---which enables researchers to leverage best-in-class LMs even if they do not have the extensive hardware required to train them. + +Paper link: + +Blog link: https://eng.uber.com/pplm + + +## Setup +TODO + +## PPLM-BoW + +### Example command for bag-of-words control +``` +python run_pplm.py -B space --cond_text "The president" --length 100 --gamma 1.5 --num_iterations 3 --num_samples 1 --stepsize 0.01 --window_length 5 --kl_scale 0.01 --gm_scale 0.95 +``` + +### Tuning hyperparameters for bag-of-words control +1. Increase `--stepsize` to intensify topic control, and decrease its value to soften the control. `--stepsize 0` recovers the original uncontrolled GPT-2 model. + +2. If the language being generated is repetitive (For e.g. "science science experiment experiment"), there are several options to consider:
+ a) Reduce the `--stepsize`
+ b) Increase `--kl_scale` (the KL-loss coefficient) or decrease `--gm_scale` (the gm-scaling term)
+ c) Add `--grad-length xx` where xx is an (integer <= length, e.g. `--grad-length 30`).
+ + +## PPLM-Discrim +### Example command for discriminator based sentiment control +``` +python run_pplm.py -D sentiment --class_label 3 --cond_text "The lake" --length 10 --gamma 1.0 --num_iterations 10 --num_samples 1 --stepsize 0.03 --kl_scale 0.01 --gm_scale 0.95 +``` + +### Tuning hyperparameters for discriminator control +1. Increase `--stepsize` to intensify topic control, and decrease its value to soften the control. `--stepsize 0` recovers the original uncontrolled GPT-2 model. + +2. Use `--class_label 3` for negative, and `--class_label 2` for positive + +### Example command for detoxificiation: +python run_pplm.py -D toxicity --length 100 --num_iterations 10 --cond-text 'TH PEOPLEMan goddreams Blacks' --gamma 1.0 --num_samples 10 --stepsize 0.02 + diff --git a/examples/pplm/imgs/headfigure.png b/examples/pplm/imgs/headfigure.png new file mode 100644 index 0000000000000000000000000000000000000000..f4c11ad54d10b300e2051ef6ba2d209447bc92e4 GIT binary patch literal 668261 zcmeFZXH=70*DeeQiiiRV(xeC*q$3?^fsKNKfRuoAq>41@Jrvy{T{gX=5Rfjt7dO2t z2%!f-h!7wkkRT!CtnBxB&v?%`$8S8(-*0Sxp@U^|ueIi!*SxN4-qDYAH80b!(U6gm zU4C%?t^pYt%^n#URRHyQ@Gl`ZsVT|G$e%c?t3Ps4*Hrg#_3$!${>0AVuCw=Z*C&CG zq{zqwlI(133?7IG3_f{cV>37=EJ)+!Z}9qcoPkZ)VE2`m?X}Y1*pqP?5W*kFG}M^lJSNIlr-RNQ5RVqyz0k}}W1|ND; zfQ6MMOX_y`UpL;Jcyr5(u=b5}_hw+}HoclFw0yQ{qoj*SlOeBUGgN@1` z?cVeMi%9HZCJ>4K5|0iz&c8YrNXB&2VOM70GoFl0mF&S?HKPFX?WK#0zSE1h&zQza z?>nbVa_g`KF!Iq{Of5CIFQ~p>z{!*<`O@kyyBNu$M@5fff*wWlCvF55-1nBAkS7up zF~PgUmI+g|1)dhsQibm9LLan15O>hq9oTuKavuH^>oGR8p}q zSZYLL{{3wLOT)>m@c+>QvIq$p>J<*&#(NC^{mK8<-qUxxaQ#OMxNmEalegRTHVXW^ zr}|(2Jw?FIx&Lgh`vthe{o`h@-}?`45Vxu-S#{`tv>?2jg^F@7fnnj@e|E6cD-Cr2 z))xQo?PVzpCtH@VHD~;{1LE(cwhW{`9GEDzaY;)aE^lVA2|14Xz?F7NBvJv^iNOpuO986 zp6I_?u78Mye~5*DHQ)ab3;z%c|H_X45DWhh3;%y57P{N1{|DikAw{*@yq4gz^tleht-r@QAG@0 zJs#9lnWX)+?b_v|vy(H9zhk&^IHji-#RUscIQn9kt=85tlWc~wBa$(uTU z?nR}mxKn>P<^ij7xd~S31mPEu%TT1%escclZ-kW#kf+tpI+veBaZF+tJdn_8+Y{=$ zssw#aj?~Vxqv{fN_vW`w0~^^Moj!d`CPngAOurHfY=SP0z9w4^TRLevJkBIerGIOm zuxzW|ypY+vzw#m!=kjJfE-3GkSTlDYo&Qwjs3N9)6OSaqOq$MqPJ|U#hwiutO8(xE zJ}7Xogn>1647*TFX>muIt!d6sznl%mbd*4m8q=8~6LnG8E^?ZMD?$4nq^6Mzsn`MnukHL9 z3K4^Ggff#3p{0<6u?EGY;M=X(QU2e4#n1Qi+dS;3(bvnTy7y+tM}*<9@zd{@e0@b> z6B9n=5K#$PsUoGuGTNOw>i{l=|!nrS%_LoP!v-}-I;h` zefA?gnTrrLEe#>3kkVh_A|7jc8@-5^ofla0M}``9(h$KdnnGLQT00~C3#T}hGJfdD z*(pB%e2-a%YbE-#WJ>0(=e-P?MNOSYl?^Za?JnnvJ1+aptYwPQBZcMW?32N|HAOS~ zuh9RI0{*v7Li!bca(PUTd>4aYeX>rmf$uy<47Ca1WAW3QhvbAjKxZfc*wKc)>n zfR&G)Rg-j7a$L5r$(pHia<8ms*d;tD$qRahw2VGz{=I-Jg2i&B0~6{IgWtC}zXzL_ z-1$Pi2r|dtA*YP6`n8xJecKb}}gyni`5WEqXcsTu;wGg7J3+e197? zc}rzKnv@Sb8y~tQ^kud~k8;T(U{qqA$E)?Iyx`)42}l>x^p@|*0B5iW$MbIThBS!@ zrGp~lPs<>MzR5fJQZkb@ahKAz>(0sd!YLSjCw;g5K$lzU@k+sqE5iw1lPy|1sMzC& zHZ~=1aFqOdD=)9$p7 zvOcwHO>nOn!mBg~YY|VK_76&rxg|wK8)t3D!d{3FlQcOWG$CQZelrf~ma?MFE0J>( zTSonRQ!PB>1 z!M~li@A=GB<>`K6*u$pfAeTn-OXBU%=fQ=YYFu#7Ha2sHZ-lOZ+&tZ05`4NpPQ_CH;c0;tJ-$6@b{N!ugU-XmK>`LTVqFNsO&}16+w+n`9$kRB{$FT7>_*h)!-Ec zbcJEg#D%{r3&N9aV08d;wqL*7iY{g{v8XwzE(se~&|T~J{TC&LdXzSTm7^P91W$i( z;88!6@?fHFkEbJ@SaeIr6sdJ7;bB48*+ITGm)>I{DW4q|67HaKl8vnb*Do%7cZ#HD z={)^0(&m^McJ^}$l$iFaASTLUtpe$gL$EmtausVvJCHQe^?pL7h5G&!S5Y^vl}nc* z(s4K`cqT|mJsL`}AG){Oh8eF4u>Ya*?Txx4UZmD%pi2hih{hlfYWfU|h{*=!PP@F| z-ImQEJ=l+mE2DDAkTGH+>k)Bi)GVC&S2AvQ5%uVIvgHR1pP%YQ7|1U2rCWG?``Gq5 zY4AC7roDo;eWs~R1eHR0$F18A6dr<(X3`Mmw(dX`I+ zLBgmpvsJ*8nWAz7DnO|7o32!c5nTK(0&w7Kvghvidn$Ms84CEpy|^zw-qM9Wt@6w$ z6>cS97fRygmYd7*oWtLFGWVI_c1J5O=<-AZ{QIllGyNYq=6|TF-Zem)Z3$STtKC|= zVjx0AJ0Yl+^uqPr!9z%dMq!fe_aEbuAdp5S)C`%eLp)_OV1y%t9>_TUR$8k!BQML& z7I@bM>w$e2WC&P^xKKN?XLg)ur=`fj0cOWOR-~ zPig;d(00v0t?=f2o`e2_U)&nUqV8lz`)c1HEy~q_!V$4Hk+$t2n)e+3-Ov5jjjm>> zu7I+kja%}!Mi`Y>HqPI;MyaoG-R{`<4-$ARwklwJ15(L|PAKXSd_!lxl@N(a*{E^) zL7J3=se8fAjy9j5DMtDfh-%7d%B=7(TWoLe2vGOp}?qh0l4UnXgM<6QF9+ zo$-`lDufPL9SiCOYMFVD$+T@j=`(~PCTk>E z2mbKQ5h5Sq1DRlyG2lmXzLJz;6#|5)W%p0p?Qhrb7kH2lI3w*He|7p(X5bU+mzAc< zz2UCS!d1<-LH$yB51uH`uV9(L)*vm>_xYfOP8z6hs~y@zyD@@inbO2iK*(u|4xaZn6 zo8TQBc=%Zp^)ynGGw`>*7B5{fPvz|7DykY4I>_l>{gV9?)VY>KJ&00<-{HNE%bnW{;dT(<|u&D zmgqa0I57V!D^7Qg*lkQsXXK2B4+WgNk5pI-&F`CQ!^6K${#ezlH`G&>A$-?_Xdx`T zbxy!DXYs^hrt;4Y>rb}I<%oUlm=KiW)ak(lva#rMNBhx?`P9W*IiE&vg!EjND}n&H z3lgmRv-57sN2Pi?+M*3cK>!-3Tm2k1DF*dT^4{Kok1)d<-{nA*uP?e~BGlc?y=C38 zybIHb&sqk#HPNsF1AcMx^rjSrV*XKu6C7M{p^p7{kB8{$t~1wz_!AvUVyn1?C zfvu}C!jl~Si=R(%0KpXvK4K(9a~}Zc^4^*FBfYELRPH`R3{;szkQBzVWGL>7wS4>2 z6tiG`3XV{(>*tuqeEB3#tjOf{fNQGEa_>G4pv}$}2w`13%=s)n6iuoGyCxGbY@H## zi@D?bJ?D6tnfIT6Y$xCS;3`tmVG*+5x8vc|0o#zupKE~aKiNi*Lb76WtF!sXuyFRm zUQ-vrEyhl;(TClJ`A~oyKP$mzOr3IDb@YaPYLK}g{aU(dT3a9g#q2kY zC~XFXZ%RDu=ZMcv141^jR6eS=t8KScc#M>r34TL-=?{e~KKvfEscoGFife5B*H`Q_ zu=j6ux%geO_F}w8z<)h`%Dy0%kaWz9Ujn%-`MRag=fnCE<;tJp#M?T-t!Y@OT$)%3N-e26yI1*} zoSQ1*IpJ6kIkN;kTV126g*ES|=ip#?iouC}w` zHG-qD2s4*TY}p1niZe)n@a-1l0qNdMfiym=6xJ(oZGk@_9iW58I|kjPN-m2GIjCQW zyyVxQU-lY(A4SmnvaXdw=S8RZa_ILs=|aPN;=$3|D9$Vxt_=wXeafA`5s8bgYrdu59G_e55P#(Z7h?Pl%Yb%; zc|ALh(ev_aZ%vDlH8y%;-a!tk$fRQnIuv5|79C!DqI%&@Bd{a>pVBufOiBwKCy8RtJu` z6raMnd_5LBub5c+Lh%cx)pq&gZUY(I3DNShTWQT27D1cvF|?*@)3N(Z>u%F79Mt2l z8%MlVg#Aa4b6i&0t|A81>f-wS|B{dLNo%%_ae+ET2jk_d?ArPzSnvUh$-TI|$4Gh9 zHaN90v^lCBjzi^m>nd!$FU&$}+jiK6#p8QsBsjv>lvvS<_oELq7KQ!N)G!|v7o5Yw z023gCJHWOEa3C6*JLAiGJsEMVxqMY$4ey-jhR{VH=vid1S_$lqtj#kbKiWc~L*+W1 zUWJAX#I3Jft#9S~QUy9TS&nTl?^+=0rr_eViaziKcYCg(OzonH0`P%oq* z%akF$w=#nKkS!zos&TXyRtc*=ff800kE@9%)fz|Yp4OJ7ux>Y_2!T-1@`lkFq)FGw zbVE8TDkfP^rKJX>WNgRiY?t4>b4e$zS}33qBQ-|!l|8Bd}u zFzdzWvfAiP%u_G8#5#BXaA8#L_h1(@Kgj>=!tE9u?B4qJvx3 zGTzbzJ$0=clZ_Y35JgXSiop?3RiCvKLtR!)|6s8-sr;W05ei0hFGpqdn3@+566RFb zWUk1sgm+G8?p(T@pJU0?Jf~v1gMb!y_k(n~qroJd;U?EqG2YtDx!Z}~jTia^$9gT? zs-QS(OS7cfXwaK{Pw|uu6EL92HvNLZa>@@|a@0v9?H00SnE%X3ExoP*8g%VdW#9#Z zIaO$S7FRPYW34n)x$J_PBU=52(@uVqRg<;&n*967wA!hL;=}{*rQk{-mbw}w3@=j_ zj>&UtoOdUhQ+0#Cl+De&Maq3fU}e{iK*QEKm(>DlA-kuw8C8Y9&cYp!_UgzMw*2GX zTWnCj{<(SQuxqFokSAO9sRU=9)djsN(^s~cTJC*p#RZ^Z!%4dL>sDWcrQE{ARz#a? zn@1?x+$p=0X57Q*~&PRK|#1?eZ|_8@oap z8;-_H=O(yt%$1$A2e>XGzRTt4ElsD#DJtyDX$Rxjq5`FBlkWVHTKz@Eq{+X6=H+OW z2dagq)6kgil0(VZQ7Ic$F#|F=D0Q|P-{RK^s-U|Oo%GQ&NxbQz3+jYadz?xqdmL!*7W+^!X?+89@S83HhkK_V>0obo z>UK!X*Tr<1s*7H`NtWLMWdy6cv>$kM&QI#Viq^%l%sHnsfZuW3 zwNCO?kSv=PY}v7P>cl#4RoMAncHJc=O;^xd$r5d|4mR(w()6nv3}G|9$m_lKP!vk$ zEhXi@9qXG|iuH}(wtyBq_71C}xI7QZnnWb+i%ZgJZ1f4ycVs&Ff4XwduU&|{JTq0i zHctm(KxHj0#eo$gp}K4InMmp{6HX)d5&p)gR(4h)E7i0ECTGA;_Z~M6Z+{> zdXSDl%jk+|!VO}QL^Vx@yQA9nXocdI%6Rk+l+F#F81ju*aQthBR!)BTO0!k6G)B1G zX2pm~6aG}L_5^3nV;5r2n}}~6-<~BjLbFs7(9IW4<|bgq!Oe%wdZnBd_ZQrnEV1pJ z2-H(4jVrIW6MVIp?&g$T2$}@Mh{VEuGT)8d=&8HsH@4oE7%5Yxjk#Kym9R{9(%kiM z{0p-BnpZ3N2+GBKuD8R(K0RSOn~*j*&XH-%tLSa7n`@|T?^qp)%*OFT_uRmJzLWU8 zj6xyXG*C0ND#4|n)V`Ok%sIYVS^VMcyC2$2)2atmbw&($60y7fU2U16ZTT)vIk(o@ zY%GFHT%s{N5?K$NoI?@cE4aReE=t|-U-Y!B(?~-$PjLB1dwW}C+lIMMs9H5~CABDF zYmU#@Gh2U2V1}!T13z?!Pl&8`!C{YcYM_&`4XPqqbB_mBpBR?Q_v&R_!3c{x!ObJs zysY-^^p%I(uIRdaf~z;n==e!1hW8CGEKdBgwP{l>we<0)D3z45hkrDjQ{QS-9`tE( z<51t|fApQ^&*eG%;MQyaI*ppR}C=B|2Yf2ZRng0o`&De+EwW>{Q?G{Yxio-1+@GHe^CA26x)3@@pKpcHDyPT6n4Cvm^=h8` ztQRO$GUsIN0uJ=ksF&F`n$G}@J|FVtE#YQ!Gpby}hDCr{E@8?eWQ~37g0)6V-c)5J zH7piCCKP+nZ5bOr959PM$93G@?baLTNykKoo^FZCdaAa+bd%GI?VkBHQJ2O10SONB z4Nq&S7`xVK;!`=!88%1jrwvSDB(77AOQft;u*n1g<3K9&Jd&HKgYrBNT#J_*KaXuJ z-aL}qMe@kE=SPJFHOOsEl*M66*`|3HT$SP!Vy-U8;lhnW5Q&(AH6hRCS?>wuY^33; zTHF)u5T^uuDhbl}$U4A&!L6y#az6yePx_qkg$g&%Z{iNe$S%ZLDb4 zToy@N$crs~C9KELd}o@ksgn1FqkTav?R`>@Z^E2z>X_*$+e7_*7s}ISKzT^}XNSmq zyhz^khb=+acd^uGvsg##)q%-e|K%1|0R}aA$M7x4&ml%rvRguTvZFVgiAs{JQ4ZBV zKq9_f>ta;rQ?f>QgJw4hHEg)4qzfX@4U18_r_zH6)I974+NqTdoakRvj(5YvivK_^|7P zj(4DZ2~&fwSTl9eDT7By|rClR2jto=!^ z<=$#+Xmec>8P&P>hxnIgWo++M`i|V*dBcM3JTLzQr5<4=^9e2Q z-y7(~$S2vU;Wt4oX&iK^_XRTpt!a}I+H~TXWB-^A7vgPt=OJ9MHcQvYmdo60h|eXL zclG2dB4W@65_1(7w4)aEqA{?a)k#`vb>K9~A(*jH>~2?8%SM)3n{*{A);lCe%A<9& zIO$M|0(JiKSsVj}E#nIX)azpvMqJOeTxO4sqnVA9%XqSt`_DDL+^4h{P&Qv0tq9}e=%YJ{!#6iJT)^X& zx_=v|M`n8S&9v?MO^hMSgmatLmhCP0wn1VXI6SHj>zLtLBeGpLcdfZt5?_D`ctc{v zF^@4j{M#89e+j3Zd2eItH=`3CxV4iku_)g&FaYixXXwDg>I!BKM{vn#BYbeYq9e#CDhdMB1!w|u;A#k zoS?1JF|K_t$nRnMvDOd8gFeS*LByISi}L}m*+k=^Lqxo-Zq*+gaPjxTwuTLUBkiqdjCVul@Ukb z*t0MaGIosLR=Um~ykyI+*`+;)uw-ZKZHg?Fw{ zx2K3#{W|zfX_{|bJV~d{+r0mr7(*O;#6eBM4&SS<391AZ1w|7cdGS~7VpLS94c1~Y z$?vI=PVbgv5EgZr9P$Zx>SB+bWY68`QAARWT-7xOvKtFEMz*1=ESOt1Z5@^J{@)U^ zsn5qg86%MbjEI#%HZH`%Y(CE;09D{*?ehNp(lvS;T1p{Pfmcpv2UOC=IAK5<@v{tj zH%3x*TZ*H~Vnr%)@fM5O-y#PAXOX}0LbQcyCiFIUtpF}L^UnB_7MBpp)xL(qSM2VC zz9pv2h`9i=TaC^+=XhoKy}n z$M9n;*OhnWc)hXf>DJ3WiG<&3z`bbl!(%~!JZ5EK~CMVM29Z=f79A#X6S9{XLFI} zdinZ5FXtLh&pP1L-c&WO#w@yywH_zXPt9CxbdkA=GxQV{1*tM%*7n6^*{AMpw_`$8 zb`1;lJjJi#@8;MkXe+rgMih}4Yi};hX~-80ubnH3utQ3o8$E%!BQa4Gt4i6zo|gwx z1K#uR>Ui;S{vH!5P(|<@?UgC=B?bX$Saw3#vnm0fRhX!v5Nx5@R)g$2lX=KL)f`59 zFg_VH$iMr<4w8U2yMbT`u~z7}W#L?z<bY+iam-=0Y)z<3^VhKNIVKvsM;jd9#d8>Im*dB{$j7lz!{j>3|)$Oyyw?42K z;)VZ@PM`k~)!0M)kG+WmkHwa+LYBl`;$g&8E5_gbbg@chAjk;PtA~d9`&FHMtkK_@ znG1vR*NhKo{^%U8&9by|f|PM#*^ocS{n?;o_a@ejAt$nBT&=Zk#^lDzC^zDV;jA<_ z-V-Plm2?@|q6U-Fq>47ySLTLmVSNI$K(FxR=50l!y@|1!MBkCioC%~?u53H+n%_F9 zkuXw&_F>D&+Al6V)p5cIc;vjk&igBio)eNDW7q(dPD@8O^<7hZ-->$Ch*YJvAbC1JAHhGVd1u z>@Oiz_-y{n9m%s->bVC2sVz^PFtFleLNAyO8 zG#V~eg{vp?3UUKKB}~fV2Kun&WC#tWcI0~sZtuErV-u>Rqs8JHJ*3})TiuvpkRg>0 z6aI@CMrh@OK2-~@eW09k^r?|rPG4i$XhmJ<^;gsCf^c^kba_-R8?^h>^`PN|F)R#_ zj7K?Pk}7e8_71vAgr~)!VHx+ft70YAEUbL=EMBOf)oBWP+oO<1(!+?CjB%lYlM+3r zfsabJ3UV%_83$)FV3zuzDKu>xE#I{Z#H4tk4{{5}Ic?dgr(;_m)WP)3zZjQF|BaS<=g=u z5#cJC>2nT!vTJSk7uD0T(Z`);C?d@}>lWW~Z;S~Q%n?;dQ4~u^pj!=gch7`AdRD2D z&Q{HNf2h%1S>REY!?z=7|B)+~ZK$213Ef}Ps{wiMjp+W8ExT|z#;p(c9ac324{Zs$ z@A|IlIOku6rvJ;zkfBDAbmTtIOx|wT79B+84a`1Nycmnqr=|`KLT7u^Ctq(z2Cr(? zw!()Wrn}X-(?3qVT*-2m4f1aKGNbbm0Oi9)SvynqR2ofbmM3jMKybuQNmke1mnE55 zCbun*7y=AVA`~L;pi`(LW5WlHx#o!)1sbUzYjvgSbgo52?CjKcRn_v2$+}g`LC=r5 zD*D2dJUp6)EfyaGF0{@Knz`|eD!RUjxte#cdYC88t!_AF`cB|JVZ;hs>&nhhKdmQ7 z_|Ao8pq|pRaT1Zk#tI2PZS-JJHNJKHHvbG&%pn~vH*Vg-T_{xUwMG(QMav#DN4QtR zj>_wZBj0+B<(%`vN-YLWgXj%kFokxd&YAPb;kR9rd>Gc z%IU46Zj;=V0fSF=c-bZ3ka|XCuaO_L)YI7X6A1h5Gv}y3J5r7`PAr@Sy^AuiJu9VN z5nbxiK(3vtJT2A8A2RFhxw$3)J-^bw8Y`Z+qrO;;yLO7)@=)5I@6=ha9yqGn4Xmse zzqI1^01ssH|0ff*K^t)k3gH)^45B&xN*6W=@R7ctvrs{uinV0VkPNTj>^gijl)l z?aG_gicS@^KmCmZ^BHCV&1{e=BMW7HCzXGOEE7r&z}6+g4h!c_gu<1fYJtwVzB5BG ztFpKQD+@@ONdDXd+xu9dV;Q{Z(y+WoZpe?JQkwZHnGr4F$`o5725M^^N2naGSa7K| z&F|~L0&YnG9g2??%4j6)+&EI$oi6KcVZ>`LUB$hG6RO#U3XG*{!b~Dw%sJBm0=pjA zjT|6oC@b=2*~<9#Z~K-7Y!80J&-qlxp$$Vk&?26TMsqXx&JyPaL#mSEt(&#ZAgS z#~Xx1{nB(rP9Yi*)OEX|80%lfC_p7>te#y${X2_qrod}yiXxuD+FiP$%CpqsMAd=l zVUoAY2lTK$Ix{ppW^N7Btt#b)`VKdS&r|d!er@6EMc3c?E)BOCGY)3B-v-*7rsbJ4 z^UG6Nm&Gb1ql%*&k4Vmz1o_q1+qFFOt0aC5Tc?n^Z|9Z@Ep?gfI$am<3;#MEtxX^% z5|0zdAH-tTF02V9dteWMDJ8jdcr=?(q3a+c{oJqi*(}O6)X71H@_Cu$nxAnAb;#@P z;Z{Bhld_xd+V~T>Hh8=3J{{brOg3Tt(mEd>L%o<)#fPo2V3mG!lamLirJ}7_ajJdP z*Evn1-7#PIchLf zsL-#<%rZ)q@(Z2U_}O$dES021`qpE^kp0S*sr%CSxx1-Q$N3HAC(fQ9(^g`v0sbNXF~DgNJAS^=MqH38c;~Z}7$scxnf|=h3zCV6B?A)UGUx=s z&u#mM8$_+$tg}Mi;jw8s{GeYt)~p$Ofyc{1feQhTx~MDv^`()+y@^WQiB`Y|1kX*Z zDL@jg4p?tS%r_H5(B?>m=WxcEEI7aV_SV%!h`Ose+`HYcS)kZKQ` z?NINI(i(~V$y;YC^UY$sf%lCR0UQ+}vIc{YX@bi4BDxy_GE zhqn#@?b&#H;G2QE?sY&{`<1;Cofucp=btrL(V?75k^JevoKhNR zGqlr$R5^GseBQ-hmj&OalewLrq$V@d;vT&{#&C?tq;egTK|F+$HMhE@7srNAD8PM0 zu-@C6{cQ&H&h`yu3+V=7ybGpP-zavyr*!UFZeGC8GM^XaI$gPt9dYjo8!BP2lby9k z8-H9dA~?p3N_gz(l(s}&#s1q~SLhvvhis@biy^8ER581zrM(&ZZ{~i7qKhg&k`nFP z?qq)a4t&pQvMi7)7Y0?ChtZ=WEs`;A*nHy8{FY*u>3eCt7SRNgP0>A)zrl}xempHf z89O^(wt=T!y!R?y__qpe zUe&|8gk;~#(GQWpi&cGgq)^4%+~gC7eO4zu;iFQu_OWHL=MNwtc_8T~`d;5^AzP+) z=d^#BYBd9&;OPl+znjq<#^HB2Ct$Y5r!$cuoaa*gFWqzW)Zm6-U*u89EeF`O;&!_w#v%=WS2s$ zhj+7b11M8F79j_j6O2){m37}I6*eer%y%q3`h>@ zoODmYgy#Tr7yfNp@41TzucYF=;!5i95I{W`-0)_Yy#d)utJTo^(lmntRx54vx@#Ev z{8$?HBP@YnK<`I+OY_t82e|aNAlji|V3)YEa9T8KJ=`7uh;62*oxJzF48Mg9-i(rt zG7Td+6h?m_SRX3tv-gB>&TaCr+>Crh>ggCi4kpF#9^?~4vcBJpRu~bb`teOt5#j9d zW$JFhfS9^tY^+B=*kXpmtF;D?XyQ9espwKxI{ zD*w{6IKS0a#LsTsT&#BCyoX1*puxV37vwsjUy=3Mdk_^L*ozToSOxy&|CoiDJ zyX(mAAp;8d()Cq33u&)CRe2uuDrXf zwZ3Wx*oou4N6Yq5<*V~c)=sOQ9EB4pwG1S-+au}A3I1<)4#r3K2Kmej7!u>^@(u@g znF0dFylek<^gQ}hiSv=5GW}6D^0e673v=-oZ)}NZF#;Aai`K8`DnG&d5*rw~M>XS; zyi)7H9hZ_TUAu8v(%!GU23YSZj=UPXBz|+R{&R$uaS8*h4d*WNF11t)&SYKs5=UFT zCIU*{puEiJ-rJfh@VNR8$+!3X)&@tvnDX#W^|N~{nks+78`yZ}a>q>8dZCn<}fwIauB*wD!-_+Zr*`?Q=c`nXKj&*yn~^g9sl(M{IH<=nx2w` z(G)QBe|>b2ej;Oug}*g+|I>-{aGss6^e%UhQ*`%|0rz86{k4w7;8eH9Iex{t0$#S1 zS{WziNn=(tH0b$TAIKNkV64L;*SB3JuF%8Jhk$@mRGfHN8MlIZGNC5SRNIJa#n4t$ z3yh|!;Rxf6JONFIEBd)n0}2b-S1@byOv_0v1WQbiNlet)s+G;Gq$z@+mWZ|@Z-2pw zfZLn}6nH44PoC2T)*^BKQLm*=tE6cK6JTX7=4h;9>{i_V&QJ*x{m@Mzs4+B{s#dMj z()i^U_Cw%!HwRZ?bQzYsj0gV1)0M|Gzq1z`CPfHI-dP|1pe}-X)f>NF0env_eKKgC zOWVNaI(ZaGsSkXdoR&u~jC$#UTf&FChy=_FX}LWmxLXHERCKu<)hYLc)92)kxobb4 z?L|RGt47+8gk4txcPgB0Spf?VM-5TCa@jBOplxWWa)se5Nmysz4ACZxrj1UuU2#Xe z2cWC}ww8^#SV{d^VO)7WYA&Htw*RNxE2>1*bc^V2LayI@Y37k9m`>2yUpxCip4hTJ zTGYy-)heCiqjV_2F)UE~U?3=kQ{WDMyRM|?+03qbPMB7N;J%GUh{{tzhfkFY%K#0f zO6vF%DVcCNl#;8ZYsuuf+#W03 z1?8$>hnF0O&0S6N0^w z1t|TSGv*#+vdY7Dojz_c48r|{Zg)Qi=+$32JPm=%F$Jf3p}0X6gtz3%x@5%VHc9w; z-`;k0@DlpG`x!`<_eq%>aVpX4ZqF%_YJgoD6f>o{BG52ht};_yw+JNFTxc>@CGxr$ za8p-dc^`YvZnu4YYM(HYYB1AGtu#hZ<(jL0H8wo6oH$9hd+l<~4Yw4Eyr`k>h2^H! z>K;Lgw|2zNpBveRrz|(r=zt}s+Q3ohLM@+HvF0%wK4H-P zWsy?=@;RYlRC0u|di~m%IZZd0^qZ>s_DLD#@sbxQBt!R`xvYb`*OviwHgu|IQv2eq zF6c0{>$Iee6YSvRe2vd*<{D28kQes1&dT&i?q8tf$+8Y2ELT zs#QoVOsUdgxU01No@R$2ZkcVHHTVd!9jb$`ZRh{L2&I2r2FK#=#eMDP4Q`zBhXr2* zP6ZYwip}?As_tC-rFAQ-(xkYgDmtn{fXby493|&aVhjnEG)7GX^h?zq!CgZY zw?FjUKR2K~mv^f+aVK>ogHde>?mJxdG;tWrF-hH%ZFH&bXk^V{SNoonByy&Fyz*j9 zqz{J-3p?;lbZk|L;VME?il-!GwvrD6nb_LEy-gE4GFz=-dVVRPa9h30?cViOTsum6 zlO1{7|FTX)V=Gx#q4cLXe*!>R8}SFLBw{Lw7OMQ@mWI3<<*R{C1~p%XW_ohQeR1qj z0K`;Hk=Npns$w4YB>^ol_TYN<0iscQz$5NP(^D6>x>+kQMRW8RZ!%n9j&QPwQ1iu0 z85XnizpqW1h5_3v%jQ;kWD$?N;&@?86g;E4s5uE9z4cc3 zb&bDcZc}w}CG_x1KKyX7N8MxW=S*2G{}>;6J44E~h{zVnU9#mT=gzbuK6cEk4GUMh zp3UZ%6F)?UC7Y&M1#J$w!WtX3RSMbus!#d)o^jiZW@QQFN=G#AgNN2fwxrD_6T!fv z`TU>;#ED2G8wt^rkc>B(W0E*^50`m0B1u-RkU@VyKc0Uj4%?5_I25!NKdSzIFjW zsJAns)ml4@gN?|Y;Nc$wY`8gvGUG;Z@S~Uu0i^!S@m0rQ9E+)6iMk-GPE_aEP$)nF zeCMq?48kV?MW*sRyMUQruR^M6^HDjm1=@<9e69(|xct01?61Mk(tLWSn zd7H))tehv36X^@YsNaqkQ7TP!L6ggg=y^ugkG$m`?$QMYT>NfunTNk1dhQToZyzq*o~sgFkgo{D~+SxJM}scp@&hGisHxn^|??(DiIUDUI5Z24%aiZq-=@Rsrx=iVxO3ROUK*61Gqs)F zTs+L3IyyK5W09k}D&w^6r201rB-q)}M_CKRI*w7J{<9nE^(24i7_j+^r20|>@V-{4 zh5M#ai&bQ^mNSw;w#b$Tufl+B#MiGO)7g`s@>#Fn0vNi;TE=6#!py+TN$U*U{T}7` z-A|*Wfz||IHg2tp4`~I1j-qaWR3C$d9F<#h2nG0)9P^mKiD3Tqw@Lby0rqN~P>rrx zE-kmq-mv#|RSeB-;|geLDFr(Ac05S2;-O%$q@wQe)F^$#B+Rq~5C44Qy4lmW3U!D) zehSE%LaJ~Qc@0t$8c{^RP8}99Gy?%bW9n0QWlBOac{V%R)iX3EuH3V_b3z4A%hb+3 z@XX$C$+;v~qw`9_=I1|2&8CO@!ML$qO4g)*MFy7SWX$2~^9v)9>QG1~m}GMQf+&6* zW#G84R;82d{Lbx{o}p)y_A9w;IM$jk2It4E)T($t)0Z5< zIEqe()(aPsg}<1Mqhg6xxQE)}0;c0MqH zNdQc(&LIGFE31o%lMIWD5zT~@wM0lKYr&h5>$7vZlzjL6K4QZ@Pmo6X`j8|hNs!`V zA<5t@*y!RE{RD=l{M3K49{Jvmuyx)u-bIl#YbJ<7xUl=RK8(Ui)bEz49F;nLTJs3Q zFF4X;1xs@N)Iz!K zs?vE|@QePH9K6kT=bUK-b$iQp?MQ9Djxm{spgj8TQBf{mVeN>I`|6(61~jU84Dk%} zV2EbATs-;b9I3m#JQLUbNk4_Upc8zNwycYf^k5oBiBD=e9wF z!Nx~@C#Cqc+fB&6KoaqBav$-)&O5LBsMFrC9ID-x90~%kOoNDyV-Hq^YM$QNiZ<~ZB8L`q?7&HT}7ah zwaWGdfr!(_b*wdhQ2KQQQ*C6CE#Jf8A3rnd3BF>!VP4%uo82SkQMS=_IH^4a+VUjJ zQ+uktw8-&Gss^xjq`RuS2m49Cz@MQeA|OuuIXnAwz*I#dDnxh)q75>x1R zVujAnee$OeFTE!tKxdS>!ldz`s-W+!0goKOs~$ueuSK+YR>h14kj9p*=>$_6O+{Q= zU7J>8@HRyh@b1CI@6XT|{L9yT_0h<0EbfR}0)Yd4E#Gq5P2>ldTvZFB`zywZb+)&-9)Wk80Rj#ZrQ zO_k6)#W2kykQ-09>qm<349Ns~tp&M!Yej$Z#|FY>b6;#zmxipxN9#B&YTBeq-- zs=dAn!g-;Y#}+Z%34o~YbeW`1i|0BYB?t$?%Q*}<(sWB#!Mdk@tE`VqX>!D0r8&u&3r1Yoqs;x>|g*n_$CRCjp2?y#sN82X$g==o0^aleFg9`4nzCa@zg;U0^ldxS;I5& zAx6Fo(y$*xx16$Lx6{FF*HbA0Wkt34VS@ss&qYe`stK1VFiUS@rxT>-1;*T+Ze>)E zEraNrDrNkw0mu|3pqUX=r1Wb{=17IRT~F3Y-CpU2J@<1+Z4mZD@^5 z=vH`DF$pLOV8IANk^F#WTEIGynL7Oioysw>`g;Hq?ujPgox16oQmSrB9{8#1y=VfN z#l?io!v{?&Pp;(c`%5TLZOly)kw4O!9zF;U*_@}x*?Q&!nr~_PCmYlLBRlsFyBPzn zXVNZLe;Kw{t_Dors24MKcu4^#>m1CI|6h!K1yq%7(={L+k`mG-4FVz{DG1U?Nq2Xb zbVzrHgoJ>kba#VvcSuQh!+)LUdB3;gUyJ2JL^$V)nb~{x>>H>;_7{J`v3G9`gSpMX zo8!FB5s>@@TB-g0{P?UVL-!W$s+9er?`+67SGX?taHTbntb z16Q8L`*MiqP-=Btb*;8Em4kzgVXHh^bfk+mG6OKv2A|TxP8IX|6k8&_N5P&jh(sMg zB%^xHLtuJo%{cpJ1u~;nU4Zx|qIC%r#x%FCi~aICj}Az`=-{4CX|_x`rXPp5X}`po!xT?&0@=PXu699j$6f43sqt zwPAKk8mvMPWbAS)tv9}5{oI1w% z;6}%{T?H7~O4--d)47kvwT}O?G{T}OgcXg0$#>igRl|2AXK{eXgn;vF&XHTtUM(Up z!1w8d=r$Bk1-u~DKj6pgGcVwiA(^Td-dt22bAiKx4(_`s*$#bSQH-VMA7pMMn>Xi8 zJPMSG#gDGBLck~Non*6>L-l;9l|IOxbBwOr1Ju!wjDRLvofeST%s!l}UxS|6YaU71%gFjK0F%)9 zD%w$($aWarGr>{zoO~N_8Yph4A4O^07Gdds{rr*)G7NlC@s=J;48}ujL6nOlS}kB~ z(ecq)en}kYwk|k0{?^NgwD;g;e0B-VUKLkzYg_6H`WvX{P)13A?B%$1Bv31-$4~^- zRCG3}-}*g=bcV7tL0)}hOgwOln69ZG&QU%93>PZO0jru&B7OWwlp6>S4}$rBL@mh- zz%^vkszH3$751$2!;d;P3qi}0pZN@MuPz8*&CGfhEWX*5LtY^kdAR^?NdiC6(C0yl zj4q5nI?m&eX|prndcYPR5Vo&TBi9&-8k);yk$D0N&3IxNjs!fP{@jy6V}ABqxjV$W zPHb5Nf!(|l;DRH&e7DJ2QSmNH19m+@0v7w%O&I_XIlZscYTg~08V56i{^8(|spKK$ z$T1{jUaSYR$d)QSbd+S0W6hfD6W)0)os5bJcQ_A&gUoMWVGu8I-A56LeUf`YFpx}CeHQo zXZuo}r!}2_zFtB;i#X)3%iJsIZ=V0X_afC|>=XC=vK*lLy!e1;iLrUZd3RL-^(xE1 zmF|Fp0de3Sp~gtIlb8(;K+E4(n|Un6SDG3gx(GC_iTlCiP66edgw@fv1p^IXbt-?p zo<|*b&_3Pt?NUS2@Hn5G-RmkIrnzYV;l!B6brnT>hJC;i)^pQWmr|vIFI1uW7>PbR z9L$)rie#D_|KY^(j=)B;4VKWN?5(kV*Cpom)pmBIlLv5R3C-K?cuDQsZg2TMHMhtv zZGf=rjc95v>Oj*)-B5Gf5~B++KhDDwxN^wc^LvjSmTy(>Z2h_y5bf-Z=h7wtZc25LH{U}M zF6KznLsy!6I|(nLauG6a)LQXrm^Fn(Z5JbVO?Mi$rCL10Y_Y=~3gY{s>_r)5GDGSkxhqfe#2PjQ1Xbf;!D3nWHSDy>s^ttIlixuQ zxO(e!(+ij(t)DWyzWrJPQle$%rLA>Zp^sq9BN3v+*`vaS6J%^WYVVTD!iJkM!khU> zknAP-))0ZnMiAmaG5_trQ}ww}UB+M?Wz0X%)ojnalZRY1g#gRmU4bd;f*sPnvOlBi6?TM0v0K3>sXj!AY9v|-F* z*gPo1D`*HyEJ7w!^6ulHt_1AynO(As53j9Ns70q=9Pp{ga=DZFD7qux>THng1wuc( zYPjlYC+iL5M15Mm)_Y@FGt@lks$DO0skZfIf2eP~5wxXb>sp6_#D&LWTbT5@+2p!b zDNgGxK&z(t)jt=QUeX*smbz~+q!OGoOuxy6%qQ)X4=@jSC^nRw-IE`nQ4l^#(dU}? zmI34gPYgy_MM9E2?`cbBP#?(-p4bDeQ{zd$1H}G81In5=GwxRAl}yNX2e4{O(fq{& zMANe*)dMmU*=z?$0#|M5MtMPxp|r%pvjmF&v3+A-3G7+8Kb=0Uw#&^N0#-GYkSQoX zjye633n|6@ShUprO91U9j59che`)RLdwPnSK*NF&)Ww(4m_@E4+%m|b3Cn9P4mAh+~=GkfM(odIn_Qe65jC5iqcl=At zx)?WnI?lxf6+GyWT%EZYSf`Ym@ zJYOUt3NgaPy{}&2yIf#LOEKjU{YJaE)wK;>2X(n)p;{mC#d*oKYp)KwSazlvSRJq| zZFSwT-`}uCw@E$l4t3cs?KAce*-KqO_iMCTZ#Bl=^WQW~Yq=TX*#n1|ZxFLQhyu!1 zIs|HE;qrSDXA#bSD!aV}B{OgF5~gS#hN@t|0nvS8CYc#q)!0jwg0dY%`epY)jKqhu zZZPJU((`WH1?VplJ{A@}plDwSvG=k~=jHM3o6@nxZhD^WnVGzM7QIRjqx~r!Q8dJR z9pHw_X@d&>P{x>4i*oVTD24_EA%l&M60p~x15rLJehCIlaNbhf#{ryA^E0fhZq=oO z7p7bve>zx#nT&U{ph2K&js;G!yZ)T;CGlI?=uyY4!3@Y8p80WC%a40ILmxQYUJ}VV zQ*{t{HIA<#U0#O?x%X8siqAPCs+1eoo)t{=T#@s#ZT<9@2ejKdYkBb$-;&zy^Q}0S z88SR7jO0PdxAw)??~iNZKHq>+&M8DE-E#oEu*HQ^h;>eSMS^dg+t90E(r#{Mm>+gq zDGPd}Ab#TrS8u@>f#q1j@d=N0)KmeD)zGaHl|awS4#b69AoZC-wHYWjcUyFrx7zn8 zJbhC#TZi>lLPC{{Z85Q=%JwNkxXcGL)&sfMfUv?zp0(egB)t}+*R-}DFD#_NK)3vL z;6_n)0x=Pr?tPVRmY3!vaXg||0!SimQ!qd(0s2dh2kUx^z)A3!4<=ixdWHfcW#4IwoNbbA1v2@)1T#*=w+?;R>daAL2L|SV#Cb<}~cjV;;4F}D` zZk|0nHFfO6?5BgDj1TVb1v2WKH+g}f9ryAU@79+sSI;f4m&ddV*jG~QzdPV7JV~*M zv|U!=Naf$)KGRPs(QN1kE#58LWs^ezUR>sW&NZ!4^%^UyTznp8npoGRE`yU_DWC28Ct;wwq z_qWn~g6-Hkw5XD#gsPc6?2^-XDXxnC#^OOfO|D6I!09p9a=vHE(Y*R2t)!cnsjT-JC$o#l^i%ImI#;q_w}f6L2DoHIJ>Kz4KC>hYZu+HH`>tb*vF~v7I?W-|y38(5zXw~8( zw#y<43OwG~PfOo$HWCYa6PvLYjvsvs6o;ZseAdAcO5NSyd}uY3Q2WjH>PUAwlkM>y z2x?4Z*Lj@wj*{?#;)dVsGPZe9HUJJOS332zXsRz3{P3KKWH{Z_)vBJPzBLa2o-H1o#-xQX>V*rRloQxx4_$U*rx* zjZX_ftZXm#-^J<<-hf#p)i> zMfzp(OI_deOtWgNPtMKO>22(|oEF+h?B7`r6@{WV5*w2 z%h9myh8|6ttsd!3_nwkn1a5O+uwx(H(4&&D8-+1y)=!MgCv>P!ICM5PqJ_1fe z&f1TL|K{R#;}9PDU?GM9J<(w@)<90VcTaZ_cq1 zv)(xV&PmR4$GAudHv7Q>OGk@-;loLs5RC+R=B5U%Ag6;qDQ5(U)y|h(DDJfmc%HLe9b>tIdE>+Hw1WSAr0UZzX!( z#12a3UDn$V;jgi`(6!=O-#+np(iqqg`pQ4O_TguzG`Q{KmV)^M@X*hB+X!3(0yB{} zT2DrkI(^=3V7ilU2QvYCc0H+s0TZh?>DO)hP)J@Qqp z^68Jb!qrdSc>0JLFQ>N0^CFv_RthkezK^zi^?Jtxt(%jq>+X9&CEuxQbRD*7qwz#k zYkAaDxN*ng=p{;N0Y0fnXqDBX+Dn2b`g)WDeSM?YcPnYw_k=g4yN)Y<;L@-pS8)G$O{H!6a39vxq#k7O^|f(BT3kQsjJ zXav3qXMOsSRL~=!jMO7`_P&Zzx=6~0^<_A$nD_tq_=5DIbyv!7Sz7x|o1Z>y0(rnT zHie+FpWp;Jj@_csc}+6@tTMTsynM_B1;ow_No%Iz`MXFAg|+oia!o`PvLkww(@q2y z)>Yd{)v!Rk0}R63HA`GW{~&mlH}|ZKSMT|!i`9*5d9-fUpLE0wVLYm+s^RHj#XVUg zD}=YU$G{{AMvdy;{F!>*6HVezPm|$ZzHVIbh=DGf!LO;UfI(e12cl>8=Lp0=PQ6C5 z0FzmVQ3JLIX2y=9Z z+(htCG0N80L*K?dEgd(>6SBm}60`_JdLkO|Kmfz;u9IK0jS@D;v~{nbpm0tFceXC- zLG=5@3!Qg`=Xh6C;g4tDI=PYmbO1#i){KeO_0a$v2p}c z(4+ddDxN+T;sYyx@C!d0xgs0yc%m=!PFrj`N{H?wkJn~^C{4ZixVP&WjYGQ56NVE0 zf|5Mrs5vs8LI?j=j{{K(>X^z*RHMkoSDZ?Px}z$swOQN!_%rS`B{Newq7HA6xJlB2 zi4j+e5d&1xK6Zr7Lg7jZ!1O(6J>N_aUKUJRY{1I8b|zV!{OW^n+-@P=mQ;c;|NY}x zoS+mO6KANUP^;7g(s0s8UY8Uq^XC?ux!uYvbtMSiGo!scgjz%$*jc6Uwo~7g3h(Bo zw^fSOs>VoA!u_R1kNt?@SHnhQE{=@g(Dg7ZgrgQ}ZS`Bw(IWGYH1g(_%QmE*MSi z3cFb%p_MOTmX^a%(}x-fw7zB7_0uh7#aq<;rY3~Z3{oe1I5nVU;EhxJ0gkAAXj!&g z<1Q`f90j7niY|8&{9$qJ#b5##`sR->hjc#BvIsFW&NOqg6jRIQL7XT1J5M<;u#HT;R^26gp+TvgCF+sio(~C;ltNyks<%sOThg!X=5y|2QrF2YzQ{lGLVj zqZ?>JZM!i~uv7%2ClQH-YQSK$xRr~YHt&8>K9k(e@a2Yndil|%+jr8?@*xOnY_Wi+xC z>0@5NcrVxE_98ti&cbhdJ{8iyQN#Cbyw=C??ht0sdd-vY&1$whS63c`!<8LT{D(sw z=htFNVg^HpPGT;b%p&cUrScCWKQd0eexb9ptnN5W<`sPr;Fz;~VoWt@$+agFu8VzW@E$LV}pXj9!>Gg9$PdB`;OJ`k$7IJqX zs-*B|TU)Wle*rCcbPvj16yB{H^CTi4>Md4Q1Z;dAu~Pk>2rF?R7e4DH-kKv5WPDJI ze0A2%W=nF-dljNC*FIwWbtwDbdMpwDxIiH*uYo;h-2N5CebXE+v)9Vg9kz6z)3iCXML8xRqu^JQxgJ_!Mc>$!iZp1d|%w zKiplP!ll=nMvFt~t5w`$eo-&V#O#|Y)fV6*po$XD2Pa%SER>a*Z5^|ZD2elbmRm=d znckf4MU)R>M=GHkgW-%HB5OrWDF&rX2=jGj1}m&HcZ6nw61$I9LN;2C^@c*)Qr9}r z1sFlC8wF6)h#V;gVX?OZ!bJpUf9Y z@T)Hp=pcr*9$2@6m3+oSL`^ISZv)6?v^yk?8J~q2>Ow4EBKuUHg$f@*4Lg9YS*G__ zPT@4a#5U{ZYtc|I&CfALLLVq22Sj_!MzcO!%vVbvv^;w52hY+*P!5_A5FeADy=^9+ z@x|yS$91;gyhVSDTavLqUt>;Q(*grr=Lg;p#XccZ^TGcPk`=$(ug}J%s~v&6pZp)6 zu1c!L7b=%(sj2XT@-Kt8PY4RR^cw(PlLM#?pJHEQpS0@lFEzOhe6P)o2;~(LydM!p z7YD&k!jk=FG)EkJ@tdxzt1)@A+`@-|ubMtIT4gOjW6OjxKxQP|@*Z-X%G<{HV>#7CRLAG* zb_?wKjiY`F)_DsTVTv} z*ezRw_JkGym zt*lBsS&>GWwH_P~mo(kx9bLr2{jrunDXK=cm4b(;5Qs{!_PoeTzt63)ccDzz54U42 zm#QBzni$sqNFHYhuly(n#r7Cr=}H%b$JM%W)e56es>XND?wpJtynT+ykXDi3bzQA+ zcDevRw-~`~pTm!^bwjf=EnY3Sk8ejphe774NfeG>?V@swAf zImb{fM-!OTD`}@~4?e&ns@Nd4_z+7*-W>feOlqj)G@W)|&c(D!TeLr`ek75)5|*YaO47c|Q* z6p|tr%hf$=F zC)OgR(2QQ6Da(w0KJQ@50MuB(Dk{~#p^ju>k&%%N0I0N*d*gh-2+*_xiZK*Y)c^eO z79%2$V}T6KWizT(=DR-&$wi{~2@6B*I0Kzz%VaQ1y80SEMz9jg1t?-RKnFW$YL9*S zA6`GkGgupzI=4%!nnpU}chFP-vhz;)F#!8#tMv)?;3>Tt-kr6Wg zD=b=BSUWqreg#_<+<(}wATAh?5E8%*TV~Qe-O@O|)o|MLse zM#OpctZ%I=H1~ZZiQTVW%o5OioUYN^n2dKkWU#1uP$Qk=|8#*R-c| zpu<6`%?&U^dx7DXO3&s4?$<#3XOu;Z!7tnjH)lKRxI<7_K~6Az0RC&iZf5+$t?mbC zM=I=#O*_&FFe^~&@5y}Y_v>VUr3NQ4`tRSu0us*t65VGWXR+gSO?mgTd@;{-F{z@N z2?<0M54P&#V65r<;F%bPy2C+?e_kiZiSWs%5QMf?Ok+MwE>;Xiw`f%OUduWK5Vy(| zNLLb*-<8m6Sfl>or9pmurXIElD50sLt@C8k4Awe>pAq6p%Kj}7dScM>JAANP${?@H z7)o68x^R^xXO_Vl+Mx|@x)EE0YwE(qJMLDtM8VeGpJ4t-KAFe4#O4{Qbx!Ty1<40$otk-S<0vWx4WPX@%x? zIm#q@;~-}7I<*xuW!)Y?T1h~6Zsf$uj6J4uS~G+7Vo22JSO0o(xU6lLUSlzvHT+{; z0}lsBOhQ7U&$UFUFoC~?Yx?Sy`IHohD77a6{u|_fL^WCj*esz|!s*6b#rkWR44a;b zk}U;yFN&Ut{2lw|JMO2K)zt63$Rl&l>J~!|M zsA^aINzyCd>O9Q`Cp!?KC8eaK;HbJO+$ePvD%|#$C~W_4EB`qc2zI^jelG?}+jvmk zN%+B|#_&k&gC+0xgQOBvxxkm2bIAN zyY64U5s?Ghh5ZirlNyX>iOhBA=70W@-X9wmN3~e*$bjjpC(q8r1$(hSPXpQ~lUvtd zH{$Q^?pCPw-qRn2AfYE?oUV=6b0~8hJ)v|a_puY#IsZM$X0sx=KSO~J_fCM#@wZ}$ z86IlV1{E5z%RpbS0qfrfvH$crMMi5h;f>^K_xU%1Dy0JYV`Jy2+85GY32Lf)*RJx7 z){!_?HzLTK)@+NzuEHmogw*Fgt3z+^Wd3piFr~?Oc+^9k@}yJ0@$HM68O>LkW;dN) z94x92ac>a?hlYku6p0!%hlGUusu=CVsR9B(FMnn4IMQE$n~GgBYSP+!>85$-{iBo}HcV3XYHEgNvOTZ(sXixH%;l zNV{s8>mjt4eoM5S|X+#WI2*}&4h-;*{fn7mPe<%#@;=gk7~qB?wQZBGn08k(Bd>s#jk~|VJX`UN z!0MD{Bt3(sYv+CLRWG>@-pYxS=bdi5$DePp5nIrU)$751XXZyXWP+8=da2QlZV4_J zK>j`g<1geufK<4*+_KE`#ST{=L6TiQ_eutOySoY;cE$FP!l_DJC4;TUKd#MJ zCninwzo29Pb00D(FllLNqmtQ;`)7XWgVkVqR zpgV3^$5Lnxy}rJ_(MOXJ2P77XdiwgBoTr>fQG)*1t({MTj>pHx2YB-!nP4!TPu=3R zdh*t10z^`1WS5Sfl=l?>ABo_|$GfTQ1Dhy6Rq!HnTqjAB`waU@Ru&@yPbJny^%Rr+ zNNzTDhYatS(hh1;|$o>)Q8VL$_?0zxarBqZRn zx{`npd>}yiLU>TD7X&6kyH4K!h4tcyeX{%z;%E;U==f7!*fPmPzoRb7juw1{Njo$c zMd^1VCVx1PGiG@S|0}#VJ(c10Mszh}k;$?ax4XD;hw${>$@|~}`AM79NkO-_%|y-R zi8zN31D1H|X5T07EOP1H%IB-silqAPZ4!PBrcrkx_%=*9OQn5^;k_~vZ$5dB?7G41Le21X`I8kT9i1axX=Z<)jXZsSBz zt)Brgit3c=>M_VIjTl$p+jBr7wmVvR%fi~x!2RNdDCkZxSk6@<9s0OdvQc92NT?0J(%>jq@GF+lM)92}a*b6&yQO;0_MxU;VH^c3T>*dYy7jMi%a(yQ29~&mN9L)X z8fq#Di+k)l>7l&)@^7*Um4OFQcasb`CWj=!d?n4DiY03b&z~`|`MU-pT*NF4<4HWE zJd0^?X`Y!FyFkiImmB?%lz`X1>Gy;==8LO+uFg1obURG2<=88-==`0w+BOy#6XP{m zZcVmyPpK*ob*dlc=s&LOcx?~7_+ygPsye2O-r&`44_n4GtO##q>MzGlQXjZ&Zb@3KV@tYTR~6Q$v%A&jpc8)QJDjViShz%a*;z7|>8rF%SK6HM zdzW70Uc`HL^We{1=7})_cso?k64&y)Hsp4{DkG(fJJe)jV}qy#?6yb9cb1wOI=i|= z41F#Smsv3KmVhj#+j)Mhr)=G#~Bo+7Hg0`KM;?Q)l&}x>tJ7Ai z53n<#p&Mc?>GGB4dGiOsi5lceqD`3{D4wb`1hRY-B0Mz>T_}cvCoAeyhTDGqgw_rr z2||4VrjWV@FfFw7^jM6S_hXVQ;<~!JmAI5e*8$31b@p5RASMz`t;2sRBVae`1>;l< zfU7`zRe=1%3zVo;CJo@Pa>`4Mjl4`F0Y|(vH1CgiK1$qTOZb!e_4Cn|F;M89a0+L=BgR^$9W9s_r4SUHq$$AIqY`h8#W zc|vIeJkfG`tYuPwfRI6lzarVYKJi5KmkSmO>7L|AKjmZJj#-i5I~_B##h)I$YO7-y z8hklO`Q*=1G%FN}cmLw|q$Ef*!p9}(c?SRi;s#Z04hBbvJ7O3u1FF^zC{sT)?sm8a z)U1IgM1Ft-<1S23ujtYj{X&_1pp-Wq$@HP3p^>aG`uS^+eK-!l7V#RZ#Tdu52N!4l9S@>)Zgr$$p&t{{sV;hAig*PgKlQW5cxZ1TV2>f zk)e@MKZN%IoZs|PD5wl5*(w5`632iiH_JDJr|Sd_YUdauw=Wu}yNZ4+)nckJ9-eBS z78_tMSq0Aq7Ob|uv^0cwVZ8@KR+KGwC!rQgjjC>2;g#jA0#qwfP}sjl4?F=!HPnWA zdk?y3ak?D=A{GxxKUOd^5tded7)boLTY#Ymqssp6ifYM`h}Z3cryu(ewr4o=epvIO z&P@#o3Crwy^0y$`wvvhskE)3n zm+6m90aV%^eu}9a+Ga!AFXA&?WMXc$DUmGaP0@QNGR_zN=(Moxb*9L6MLci*Jt6>)0EzC&D8t2yRD+ae~1WtwaL$t~s;3GkV^ zO4$=+PgUb21B<2`K(*f`dM^sLGQhz-isr8@BfrL6<;NlpFvu+`9kYT^{o(e+^w2g|zCfDKH2Z5i{iSnWCy>N4axj2zR7 zw5;s(vnqf@6MP;)HUYRSqqgf15IO4jPyP!3-D$jYJ=Ll$=s=lJRcS9L7p=Rj0W^Y= zx9nrAFm?UvND0e3VJU9T01&V~5yucPJlH+g>)Sms&0O0L&oW6|;C zy`0YRs_;Rb(&gH#=qwMn9G6EeP8`<$!h9z3QE;H5g~=W&*{L4BM97yJhO(e@S!Tn=ur$b@ zwA^W&D7))dUN66-rk82<49cNBE@9&4TJm0(6&Nq{#v|yxb&vj4$t@XE+IX>icSQIn zV(7&3rVON0h)Z-<8$lAxf!iPW!fl@eNWQnS%ZaeI5r?(wPwAMMFAO=ms|2Ca6B?ZM z>0@bRUpv?skOyaq`gq$fH+w+m)!A+Quyt)kV)co_Q~`YB3U9G$g{YU!fi9AnQf|f2 z8%bG;TCnxGuBV&i1H!7*sMg0oXm6<~!<)=uo&cfR3_nNopkezYy(FgD4txtIlI^1^ z@Yg1qPdN<~`WIl16`Ic7KU=%XM+zwb0d+o5X*%BSrbF^iOf{JSY`Kw4Y?daE^mwHC zl+brxj*s*LY!YiJDJ>dw3+3W={9g=HB3KAdri)&Nk<4dCLo(axBg>e*G&@m`6Q5D# zJ@ZY^y^FbLTWnl&-Y%NP@Y#Pi7*3*?Jsgf&5ILnf+-0G9 z%;S)+G_ILJL0KG=al%9q9wAh3C{b_BgzI!kZ^g`x&~zMYi1{=v{1*F=;?Pz)MU7c# z)0{<{`4LmozDKWS|-iXs6>9wb5DGB{YM`^py;I6-~{alWNnI&Y;N zktQG@ume7#y=A%`Fw*oZ_g-Kw$a`E6#)jGUY;6RD96P_?{^F2GB(|Scdcoh~w{1 z_C1C0*U)|ZPwSC7^CB+3+MJfgup8Ga)?XM!_5sSKc;cd6ds@==Mh-wzrIvi=xjN0N zyq4|PRiF>Ymz?F&(ty#dNMR%pcWLk95TUehpb@G-)of@Utn{d#n$9RP?>X&8{kiv+ z8EZJ5U-hKrc8t*(EAK?X@lG8}p*X;l$cwXcf@kQa#LTc@&~_N)-lfWBbp4n=z_x#U z>tV|y38tR#v`|k^&j5&bMQL9qzAH%{&2Q@~uWP%c`bVKZ#D zrqnKQl7yVG>vwXr!RsnD!PpnlXRcPDxLoD#c8yfhF9n==+!y07yG2DsV)POPjbVhG zGLX(L#Pz@6$mU_D-qC#IY2amo1{$*o*Atoi%uGSx6$$4G>@wh{tDkkaJqBab^n;^9 zhR$~u1u$J}zZE9&7U(aIZ3J&ied=8Rt*u|6)=B`w3$8(3(xWnDX0VM(KE9#9fn5W3 z{<`U33m-$-6_9KvDr{JT@;*mGk2A=s95sRzxc1l1F|9yMG-y4eeO~}cG zi9z*5zr5u_4%RXAnqaTG;~9fpzL%_=759Qu1*vrG7wXg8v5hB-)v9PCOEfiBufdJK zqUt!(fy1@9nte&9K50gdnMOcT6!v7bui_|jIaei3lF^#ftKSq${m@0SuAHzbKigZm zf?>u)qee(Gk14Iz4j*aBHWZy0LtWzgjA8>ExhR>D0!(=4QlfaoiRW%vQD3RJUZfUD zX{0aOF41V@z4Em1RHv?hoXzN zH);hA$es^!kK`D#mlt}?xNK#>eJX$`)~bFqFDhqU9v+iW+fDO40D)md zs+I8fIy}SU!?!1+4hJ!CF??PnI93|kZ<3Sn#Sg}T4_=eH-tMOhJ5zyAIcLpRhR*2U zO~|_>1XU{6Wjpi|FY66ewqL>u*%U6+i5~JT6FeN>(Z>DnEVGi!c zwiXH;(hEhonKZPzx0k>j&w_jarG63ObGwg$`eSZ_+oCiQI@MEFE>{lNd65#^VV*`HvMp4D4NBE@o7Sv z0xeJ(L1ca15yXTxMi8D>#m9M!iX(cV$F0g6okR!5qy<#vM7WV_5K>lf8c}woHht6z zVVwWrO5fvYOsufWBkmw7J5vVI)df}O)~~uJ{}3+zK6F4n_f~?Lt;h?tvoyr#JHAVg zaXGjSn8!%68(5@x(awS>82WUNb8&h;z&CssHGAg!^btK7$HHi!Ag6Wm<1(IPOQQ0? zwZ&JW#>$GENRk8QC6TZ3JuidG(J%IylgF2if}pkB7Ey)zk*$oDoPDU9MXrn04L#=c zBMVvLt!kye4?jfaESyK-94^z$VrwT<65b4aJuo&OHGPmZXEbInaU=w+c!T0y<0Er| z(38m3B(iAkoo93^73GR|Y>#qLEUwy&(Py{TvBG$cF(BL^pF9sTl=V^waeg#Rf^`9m zyPio`16R3P`Si~glz7JSuP_Y{gMgfxeJxU@NGqR9v z@OPL8l2B@h`=nUps@NR_;WoKlzVQsWdlaWhqk_~D7Xc0STQaUdgPsdiv~QmsraYsN zS>&v-Ue+ST&Kd`ly*OYm+U+c|dB5I)|C}&5D3$$_91UfYJI|XlC$B%R6O0m)RF%aZ z_sj58=TOKdvx!)NvH!A*C(WCqv=5rji_}0yz&VFAQKoL)qy*;1in|)buA<*hyz3_; zoC^SE1h0Av*y_X}It;2bw9U`|nGMWr$12;}62EwVn4HiuMmEf1wf_87+%JZb*NXyP zlg{7f8!c(C9Q}0T{KtpcP|-hTY9h@8YL``td7|evuL(X(B>!nrP9%6JH9F~u=c|n- zcJmVk`N~V{j@qMkq{zGyBS|R-wC^@8KYu$w+8W-0d+) z_^ZF)>u0cA%98!H9cBK@Mteh5Ie23KGd#tIyg`bztLptUp$L!9h*P?7Ek&_stCw`P zdv!6VQ}5o}yn6^VUb3L>7jO{A>vK7#qnq1m2>J$CXHbTQ*j~rEwDiw#QdK)?uJ6A^ zWg2q0RKcIm|8Y_N<>cXHWg4j2Ap4=PE zQJHc*HUAWrlS9?ye+Ptym93|~D9kUz3H#A70{8%g|H|txL%2lQp$m~h6sgz9tdQf7 zcn!#tg`cz+ngCYI01W5Bl=i52;J_@~5O)4P!ZqVmeM_!*&F#?J?vdB!N6PCmq9-Ma zLJd8!0fypW&oPoFyv-@U#B#&XK{c8OW=s2R9reZ1Bw{@}14#FKq5xa$0`2l1NO;gX z2vqnHK}VH;h*I+H-?SS4kSIW|z}SGbYtf*;ILi*m4N@R^i#=D#v9L*$Wb>`gIsfk5 zJT3d?tm&vGJm7eJ-v48(dhNb5CN{LTdr((L0Nhhsogt7S09C#m5feqm%3)4$pg%ms zhFU`nesk??j!dF$Q+eTrJ4)Q;%$LuklHKO|RQGk5c4s^9sm(8VVjc%%_0VJRs5?UB zM16pmi(*pXK?93rUpdAPKp1NKvvrAHHy;;lhwh9|qJ5qbEZ577v3ux7ebK215_8f| zsH_W4J{pWAa9#X3S^k*&z4NsNqitSww7-ld)mM8?@$ETb59X+=i!BMzc-+~qj#Fgl zOYV_dyVZz)?6_x+Dk@lK%5X+ZXBVZ5l$BVpHt6%S#-!8|wS9ud@L-&wj%E8(edi_O z#?Qv1O(v8By9%phoj*%4Z&_G~BBKrGDw&vQ>HV_%skG@teza;F8(Lti{mxjfTJL-? zV#8P^DxnTX`$OWLcfx>jrUdJivZs5c;MI;9#`ts9gcF_J(%J7yi9nG@t%YQ_p7J^o ze`8g{&G_PNlDhjV>5z-PgsimrTmW?Bujb*;y<)dccT;q!XNEja9hDwrserg6Cf~o^ z=ws1bc33pJcw0RwYIY_nUCn}FEfbE9$ak5Bl+(`89R7RiWx*o###@%3Ocz)1B@K`k zDx}OHz__|Q9qzE3&}#bhd4tP`1Xf7PMdEXh2qF_Y5K)-KBwPe4PXXU0nQE)W=UBrL zV5&((h1a~D-O$diJeMueFb9`O%>@0MN;%N!98NEE#AkrwQ8JQ*54NaXx2>oH=O|!3 z%#|OW_g9I^k9PYUsOolXYj^iXsbsd?xanD_7}Vb-v;uSelX~n;$9I4}ywEL+({bnfT4Wp&cj7t4qn!7$_n6wL3!pNR?OGhpk zG1QtP^ERpGI$yAk=HilY2Fe=LMk54rR9vsZ#y|-ZSgdr}V>f`{Pl4#C|6fk1Hg#x;1*;O_2n zPiDTETQ${xq3AC9oU`A()_Rt7kZkJ3fI?!KaPDWRhDfy=B3D61ojJhkSY`m|WqAzp z9c)R>IK;s6rhBdM#hZ33WsmkOy8n;58r4~L{AyD61Nr;n*I*jg{a{mtN^|=)hKQ7} z@>5K-@(sdVXcli51AZEW#=3x6P%gJlv2u=>iKImjEU9=dx*=b00;x==G#jt>t)Tz?oTx1&V3+vE7z@-s&ySk}^~k zR?2yp{Ytai^z^i(*QCu%i9BH3H3wpKlp=9rk?~|gnt)8fA zjPFvz)rml;jifRO@+{XkvDbSk3UEn^81$jdyZ^H_`#)Bkd}gF==KG=R^WhFw96g_) z1GhaiOdUGcF4Zrhxm%?AmmT7sfr=lY*{SQccK)_$04j5s8I>|EVV#u~kqiqGlW29< zcH0yI3%Nn9c(jpi`%9o2E}3R&U-F_$36n~SKUk;{dpHWaHA7dT6PIbDhz|xKB7@1m zwWF!jVL3W^Z{j%K>ZqN>Gm*I=`0`S{S;kXJEie1d>K&zXHcfU}UvJuN()TO2dgjb% z+ev`=ASPXFr8qS__|C+1vt{q7elsO)D*ceB*y&DrXV|3(_hgOUIX3fsJ2rY&hR9hu zfyVp^M~VW92jz#~Ubcmoua9L5TwMA^(Mpk+Ola-O>JF_lR+Nsn2xlJ#-b4xpcx`@8 z6EP3_RteHIy9p(F2Hdhf_Am?^2zSw_2X3m`zNYEY=wR<(!b#2x(@M_wY|T|D#=De-5@4pL zAJqI`FKkF2NSNpqn6pd-&VfgF&zEdt@ONO*quU1*9y{8w{URbd-P$(>W7sI=2!0|j z2eLR4O2mety(Uv0xkOgp7MaSSlN}^V8yc56>ulwuNj9)%Z91G4Uwr zQ7sJCU;~VAmNeSAx>U@Gl`X2A5<0=YV3=XHBOi+E=d~uPInOu7mNO!o7g=Rx6zEP{Cbv&uV_3GVC|_B2x*0Ww ziczfC814b#aq@i`M#3LnI2@is4VpV|2s6_RRiHIxz!9fbr2aVb!-(xTFHzWF-j`Mr zerS}Y=g+Mt`(}6VcS*Kr&dQ6`o9&~uE-Gy#;Ylcv8?z#gD3V3KjEWDd1r7PoZogBf zQi$I)2)-79!?Qadw@nw}ljEfCq5|r+XTdBTQSPzmmATZrObG^$I2a>=(Mt-C3M#e} z9kOZ$z3uRBDOnJIzi>qC z%nXXM9H^jQsk9$`RrSCY{c+$@Tk^8Z0AZJ1GXD6?ccc2@K(kcC!Ek=h7!dn6= z?V;$G8(Psw_bp{1sdr)mvY-7r)qlo-=_3tV!)4l!bkU!~?$(oq$iESzbS92WbW+vO zH*rVpf=f>IC>C%-=zU@m37dyjkrKNO|qFwP8;*kwdKh z0{FvN-O|A(`5et@w>X%MFHH`)dA$^(nt}glh1u?CBX@wT#>8;Io-jm>Sp)X)zuyQ~ z|M0bRDAo=*yVcq)ayBWcKIVO$mUr0bgk87z3!Kha0WrJ=$k!VFiva){f;vu?*>_-k zQ}6Qg6}afb`L`wd0+5kDn3@){4L{)$#G+b$@B!T5wO@8VECTttSHMCl4@_XQKzyIQ z_XtS!r75tsNPFELGo~BGz2ngw8$U;4|Z?*|<4RMX0V=bn1G zCd)tXgR%H*ksOA=Rt(p*0HTZ^WgJ8u7AS0QsL-yF%)9Ch$2hu*BN%3P3T67*EaM2c zA-)3VK7VLujW{zx7KOligEW-D{C_s%BtRcl0S3%>*7?p?$=|+F&}TaZz2<{g03&3o zmEk&^z~P_c#X8r_be^zQ^jjbjGnoe@=nlO?Im3I-3z4}@zhN}j58i}*F-gU*6xfDy zCC%nwIQ`q!G+U-YkE?Sbq=Xu#A8MVhGgECM)x@pV2sqC6U0Q7CSswtw`9>YSIalpB z-~~3E_Z-_k0(ds>kDd?b$`uZufk5d^i(C$>MULDHXXbF^5$Wh}ev-rIL|TQfIdVdQ zk4S#$ClQTMH=V$g+WH%qT!(+RW%|kG41VE_6O2DX^pw;)EWNt-2hh3zQ46-SzDP}0 zbRRf~$a-D~Pzwlu+?%Fv0HKY*L<4qKj#@yPC5nq?cR>1Nz&jGIL&)z7Xte*bHHWrD zd)U6BgeEJ0>TNV$$nC8F`LOu&+%&jgZ2E?lbYAE;ktnd*o!&#)LmM5=?~nkBllJ~u ziX?LSJ&yp->-SO%j~AZ#8kT8^CyQB!g0tS^aX1wGeLt`-EPsKobX$NArPx08qyyE& zl=;359GZj@rpvh0n=bLDBMU}5d~an)Vgk2XSg1pou6!}4?~e6thfPJbU5eAEce zFB&i!)Ztg+{r>IHPURushl-D7iIr*1x$LY`dI3+txGsG+oz*!^w z;I)$EQmg7)(TA_P_G3qtS*ZTKPru3|FtAQOk8PDND&}_WF+XdopLsU-k(9f%wv39p zyb@iXg)YBgA}aZEAm-12JM{huYH5Fk2SQbaP9V$018rBdC6^q|je}%`=G2wJ@7Q9r zO#fzmDy~TzafM)%Z=cf|)FUjZlZ^S5UVPn@N&7%c>k}U>?%01mmi4`TdlRN=M2Ag` zzEeNTvOlg&L*-|jKhyC|&o41Gd&B8|-@mm)<(F3G{Ua`d|2;zIA0w?VS_4tSVWD2t zUBt8e^9p|5+?M9rZXtBxJz{R_Z+?IFTSd~lF5(`5zKO}R)8G_`e`1`=l5z1KBUyV^ zWQe=S3wGIjTXAyU*gKxxGga32A`=LFxOH3sl4tjda5 z=8~WRbux0zsAs*v{hIiacjANRzgRrS44&@d+K24&%GMG3_}PmS5&AwiQv z0vhzWDOPX|(09pOYKnu%|8GN&L$AzCYt1_(Sw?{c#|lZ?}_T#yOAY{!mX#Y6ag4pW(*;RBuXnK|4l!|P``qQR&e?d zrD0I%20-abm^bn_i1yUqYJClJLX~uwwLtWWMYS;v8v-H=KOG}tHu=LPxX*zk&7qEO zvkpp_#JM04;LlNsIIkccL2$;X#HhQ}QDV}*px}p$I$22xT>`=V6j+lnDYmvIHR`_VnT=IcKqyatt&fUF( z96kkePk^M4L$>FAgggbS^8!$?U9$Z4_B#KFVu9!f_PgIm6kScwq2vh>i3(p#$`KgN_QEC2)CI4c_m+00Xh8Uqkbb zD=NGOJ~gGM0iz~gER1&@n8W8hgVJZTY?m8<<(6)?lRJE-SG=Fsz!84rQOb0gx=beaxvXcx=;*G}el=bnBJXC9)Mg9?nA|pfOt( z*aLe(1OA?Gq&sTUL+_ZXB2ylX%<6}^QK2FJzcp}(=LpBji*#X4-Ry|GtD{a4&Gv5-5z(gV8S@!F71h;A!@3~P89iwx_wq2Z> zCi}0%&koK{cAvj|DP`HK3r!T=^lR!BpVfMfzuCjCMEaP=sfYbrc#vn|mVkBk$k9cbDUkxh;2*@VXQHed{R!F?MUk@vu(6~Gkh5~Xa^z}S!;SfRy@MLqx)9V zWL#d-y7VX{a=j?MMHN-nZ37D#&&gO*I{X{9jmc9BwxP7_#hH}w13u|5wJ(C&&pLbW zQ}zs#(lZ;Lm)6#DqEhmoozAZM7GXR#X?=@F7~B&SP776N%x0+_!lP{e%KlV&oBZDt z{suh=DltJLS9KfqckX&VZ6?|Tgp#J0t5cPj6r&s9-#qd7$9XT?Hw`Bx$IcVRttkyhO}OQq`BA)TSj2Lbk-p z1(S`PGlV^B^U)ZYdWF8WD_|5uHYl?t9yguf?LSwyh$BQ;z13gJb@Qc|JTP_XMXtx9 zv2C8KgALOvhz%P%F9t}K+Q6(!#)PnB>1A+%g;iXiC>SyYTuN>a!w4K+kz<0?(C4`+ zmb%5sb9a33&Zt--kiNlN26jA@u& zXMcR=mtZS?53italZP?L)>j)6#JS*DOV{pTMnkXOUmc&Y;8V8)_A$XMvD5V*34grb ze?K)P5pxplK_H*+g9+KBI|Vp|%(Ry-g3{8y{o{z2DDmZ;sG0wk?Z8d6quzG@_hkzw z6TfhzggFR`L74oJX;tR2^y)pz=1=2!&sq6d&smjn8=*tk_E5-ZdbtE5ObrceVCSr7 zJ!C!fl~OgB0V8+WY3mgDi-Y|EB&(EDeyd++l$P|Y^B*EpU#4!2>6rQ+J0slHM1?;F zdElLdWHcFH`ay(cnlZV?6sd$OsMjlpV8sy*rs&3{$fqx-h9ny~ds+r!@B zmo=^JHK4lfQ1ZWI47LxT$cFgr2wd;uV1yL=y5dg??1^AruuczqDMTW7Xm6Z$q4u{e zuylr=mHYHJ*)Vaq#&ZdOUL{aK?WIgbPocft-%0Wyco_28q6L-*fVt(P-#${=%i5g! z%bZQ3kByjb%=c3Q`6%rCKj{VSXlvztadb(~bz&Y`QFXCe3dnfjPT#W%D>wL!I@Aa0 z-%oc#)LmZF61~>U)M!`XG_pqgi-CMoY`myOZg9%N-|cLYi>6Oil#Cd$-+tm)g~=qO zM*R;9z@R<)d7I_=d$NZX%M(S)M{;9x%a;TtC5EE^_5n%5L-AuGrolae>GdD3zmoF% z25@2V(<2p|%*hHPAg4mq#G5OGv#qLIB?K^h0LHB3;XcenKmiA03f$2{BelOr7~-H( zeJv%3C8g_317?^bJE|K?ONVe(=r7O;LMg&6-R2R01bpL(db~@GNxmA^U&;r#cPsDV zv56IEVr*MbSqebP!P7n1ZLfrJ7|!^HoA9481G5X;l!0lCb4}DL7wwTn^5A=tosfmGjSlo2 zdPD7}cbTrFfzeih&CC}$A-J=^K;lE;!^EQBB%bZ4*Mtf6Vvo`hX3!U+(U$!nM|;sF>KXMx0(dJ&F% zj;lam|5wvj=in$3-s~r1Vmk5-2npS6_ox>;Godflqq!V8R!k#5;m()ufsurO5p}Ld zOD?_YjZ(SK1K-z}!dGOv8E}Qs67n@`M}fyE;X~6tWC}sW2u%4uGSz-n#(G(+(5bt= z$Fb*Z|N8XVnqV>)b}Nwnk-pw}Kfb0gFrmq`{2>@YTXFVEivkIYGu0t#7NH2pRm>Si z2frADed%E#_h->oU$52DLvDeAMpr}zyj8%`&F=zi)LVaD5eoE?)KH;no+6fPv_cML zJ^ln{yjK^v=i#{hZmCT9=ihc}Ok22Ysi#n+jTW#U zHTC_B@z9L3w0sW)eaMzegquniqa76Hh1^1#nJZ0j68dPSOqp(Km)@J>19nik0yD+A z&+7ISx`B#F*3qx4uFReuKR!+H#xj>^lKs;NYl=1TBT{JTZWo;Gd7JjnyJtkp*G-Rd3Gtsb$Bwk^nX>`5gFW~yz10_cTeNm zHfh!_&X=&HZGuVVQiQdH&=S*1w( z?}xGRVSWr3&nPr|FBuNr_;#;zET~ZD@1q~lCmIT+m!Q=s@M%B#W+%|zbn54<)up&^ zZ=1O;l^k*(HbnQAwi2v0b|{qY<$12LCLL#JC{aT+zT)jT3;mJ)9z$mT0WpK~UNf@H zP-9w7#vqA#tIgZn@DKeez}@9B%{L+mM;j9V2OB>rl?6#&uPV{NI)@RO&Q! z1yGeOLrbP~M)yKEgI&r(LX!<}VSf=71Y3pNJ#y%Lo z2{TY0A7ZB-zSaF7sn|FJM0dOSphZzmuP35S5vNfBrq;ip*z9*J)72wPqz3n&anBz=RG1E8_@R&&3r`&Y ziz)_&IxP0}?sa&uXZ6=WHn;12H3&0rc`g_bqsrk1SBXs#LnxTT72y>LKrVQ_LSybDkcB zjyy_yZ;P2Ge!X7xZOU0cB9=lkfSr?tMJesC(M2V-5^B3s?`}B*9*j1#goCRXGa7oB z@w)shC=k(bU8s9}H0)7&5g%^G}1N_av=bXW@ z5};aGBAOaTN0}777xZb5vN|{)`UJ607kIdmWltG)Gv!wCRW=KRM__`-D$?f!yKH9Y z4E88ZgTmJ@t&PJU13H-q!;rAfyHOo0I|)v&W6a2v}C0UQO|N;<#;W^a0CRJDWd(hv=!z;prn$l;ng8>+lmAMqAnV zskGr9I7#Nv9KyyZ10nd^P+RaGN!N4B&J+)Kjcfyr>-AEezj!$Hx#BK3={|XG#&frR zJt&lEKmU1LzCp=FccgQ$T-}E`P-Gm%k)L5g(%#ckC$vPc)CD_vw)uGOmKP+#e%if_ zd>mX;NFR(yvZ|%^OLr&93MSiK5fLJihbPt1*Guk*;r9*0!!rDR0}$QTOk|A=etZRr z7PK=_1Kg1BXofH@cRs&IF_RwLrky{rxFNX8f9u%Gt67Xg#Alagg;`Z zbWa2E*r_D3Hm{PzF0U5zj=M(O3ZA=J^bN<-@mzb~6oGBpmN%Y%PWqr$kV1GcrTlr7mm@Sp~)8eMT%0H32zemq{&-(s&T$qCm z1l$?@m6|9)cjxJ!=V&$Ufk-g%3GtumR{vPx1oCursi?%oQdl#t^uGWdW2Yo^hwlBau*Y(9GJ+5UJVVuG9DUj*; z847XAH@(r9WP(Q^FW}w>h}O4*>mIO6L`Cj`uiwLcgEsPq*34YGCb1*5q0WG{zd{Fd zzf5>wLAim?MZa$T2Jz{U0#^}%U-LJL#~x$cQCc&3jrNZN6evB29?L=NeM3%DD?s&| zSEN_biKR(8*}8mt^cM-5@Hg!bD;oz$Z2UFCEbU3-*#a}F+^`$ihPq|}gb}5)%xAeh z9QgswkoU=fi+q@Eo55+T3`tU;2!aAkTX83-iI0|SlN&AQ`V}6VJrsR2mu;9B#zMp< z*)E8=7nk^9S5eQ;0Lk(a$Rd1>ud>_;`ym&>3*$T=cNMxNK=PYRdeuWB#tX@L7>J+1 z?GfA%8tTTP1`hy&{6TR`g1=R;IMhLba)ASIGHBnC_{Q#eX|AzB4I7D>(_p{+M?fHc z&Dq`?yRHzYSHWW&E)O1!c>kJxGT7ZL0z{F|jqSem*mn~}>-B3EE}jT&S~eS}Xr}=} zXlPoqEg;O(8~yGQ<80u(^W;DloywxG_o2xYlQdsBE7G0J6bh@F~60=KcFq1kirK{cWXNK+R@2elE3Gr7L(-?V3;46)fp9>#0wI zc@wfkk?w-W-aC^u6#`oMPHYg4*`&)et3!Rk`kS0?B>zMrcijMt5j!a+#T_NuJH|fK z0*FbeaLT19{HcrMbR!WbrA5|Vs%aF@QvSe>zIOvcWY4(tkxR;8z?{V6oQrc-C`O(% z%+2BX{vr;rb8MiIK(lIH0e6$ku(>tN32*GP!rY>q0Dh7eQP}KtxcEi}0^yq?3?5Eq zNRS(7OH2hmp^^TAwbCSJ=A8V5V*osaFrZ-_JZYJ44;g`yozLA`Xtt3ekMVussL$HZ zGC;dQwvc`c@Pp>!=172hkQ76NDqI`~3&#~~*W2UeAF$dl$yUjR3<=WI&SKcy=%U zRWbxA(33wciE9&w(WE(5HQeCotK93l4f*`~Kb*(8-B~FN=30?h7$1hmGq){@)9cyv1U)y*V-~5)_-~tGT)})jR)1pvJmW(~qF2D_x1V8y(Xk?S2{Ks4KtBB|2 zK+GwSXH>f5AS~Kgf})NIW{1M7>&$sFw(E^TOfXGP-zmfBUw~VN{I0<0PbctY7_hbs z^CO2o%oXv@lwm}= zEd<8BxZ>wae77{OK_e!Hdm>A{x**kvuW9!JQ1J}+A)@BKje zM6}4s`vU_!vxw9V9f%_doTN5l_=YRq<~$+oapPwB;hrze#0wTm-}pO?=JHC^ z{!n(h<2_R6RQwX~p1bUgVrp;6!rSvH5{c9q+;?U1X|<=z?fq>`SuwiNL(Pwd?&go{ zZ~hB&+tOSG{wtA6v2G@W|8d@7b=Pr4)cuR;bUaqg4bOL461@0(atgZ-e76=x2<7^2 zOhT$CuD5wMGFkZ$h)FoHf8l@^x2T_Y(l&5+=k6&qu^zCu=;44S_oaZnoeeECU_NUL z+JD6K_H`t$zB%3zcI=v>vv||rPI&ZesiFK&@k3rzgNUd( zX1$&!y}ZZU%kjifw8ba{Q#HrzV>-5}asBSC8d%(SqzgN;wb&%N5_@`SPJubzrd%S` z0O^O~j0QoGfq^i1#%Zh zon%)ok_IqJrxi;vou#?aUb8DA?5TDr)+fhdf6FW>kdAYs#coVSr(y&YN}i>X1J~1) zMRnZ|xHg$#E-xPoG25`$GD{*KodFhOW=XN(bgR7vl}XznyW-48K4Uk1jNcDVua(lJ zB~Psms%(EwK6n^srg{ZBY886S|6TE;?&8<__Y>&K7A8Nhc*c{c{hL$gYA@YptyD^+ z4cus{*p*Zw7lcm?Ka*5?4qh0%ZLK^`E0<-7djpz3@Nr>k)dzW>3WO1}N;b0PX!p-` z_fiNU+$=?cRecX`v>{xhnYueH^L{4{^O+@OA$|Y#K31JY{&>=!H``8***swK?Rp)5i)QZ}*~cMN)kx2CdN?OHBN6JD&`^Wp8zUN9HG}u6SYJDaH)DmV zLb@6Zn5Jy3w!{y;jbRZC%>`OUF@?ChV^L0`18-Tj5>6zpMgF;xygneC6?dNY%$E`{ znSkURwBa97YJ!zCA)|S9y^)k-IXyN)l< zi#Tx2+N^&EaxN3SO%fP0ZyrSG=v4pBU4Ud`3h6^)5uBIF2+ZdY^vp4MY^Fo9$l)0< zYGANX39#x@NH;V2olPSG%UdoAe2$G9el;i47ef_8SrIRI$v#b{{uwWPTo&1pSk4&p z-+^;jb<;O{TOM&Ug^w44&mC0qo>&op&pUMt?)R2@%U#XP`h5%KD`nC4t84j}I@{8YuaJ|PrS$j{g4^AJtfy}6qWR{wL7^nBY+v#w(Ur?I zm(fYpgGT?Y%9u|qy^x-u$=}fvKUIx9lTP_q=!Ck}AzhuF3Vu4a#F`=>iBu1txG^FQWvo(PJL(352qqun5SxQDm^uj*s)Vb{gI5#O}cZ+|8o1)F7SEhNp z4XvgLYq@>b7UNx3xh&*r^FCL04G_U#d)N$Z=+O}vFS>Lzf{9sq@8x@WxG;w1ZM`sf ztwD&T7?HsBpsU#z|Fu18(J&|jJe~`sSMdGNCV)&xt8gBqC?7P1K2~Ulqxi-EFt3G; z$IJqQ|r!0a8qxqNZ`Ndr^B2DVt{5JmqZrAf5D=5+DRoIp(z?0r+)209|pB9mGQC@ zU%*xj)hcVS8?%n*2&JJ59kUy%XOb)-DCz}_r_(>qDD$5{=#ijByS*OE9X21T7}hd$1+!Y2#Eg1VLfiPmUxbrOP;f zQ0c2hag!9H-<*7v-W5D&Ef^0zn5yjlDxmxY40P15`~&^qb6U8-r1SUmK@$@GS`r?c z2rvQO0&n~)*+pOt27Qin5HCiUJ^S)ypF`H*0euXH-619x+LFMd4f|ffiyo48Z2~xLKH5pk}T1r!P0DiXV!ZVgG$BHj&+X?XCd=Z)zs`G94D;nc*Y0ep13fkal%u%6lz zOt7(V7Mf%p?A2dk2{D*36L$HaiwVVZ(wG>+E6<>rBI8_Kqy;<^>VxiH1kRx6Ckx5)&Ql2p68JW4HhK_LNo1@TiH(^ot zHfqN??3F3|))H0>y)pC7a4g?yX-!9?%}m%`a}ddb3dAY4aV)E=R>~$&3vqoV&LCW$|a!LP4FV2a%7Th1QDQrfZH6&Jcx!yKmy$u~HExD~}PG zC)mY><{#Dg+bf*eNb?@6xbwNY|6nw|C_qm1iYk$XO7?p*kZXl6X=*6_D+jKCg_T_! z#YymXvB4POH<__g9lo`Vh!f3^zhSGF<46^5x^$&(#PsBO&b(t1?vGYZR~vD_?7XL34&2vzXZw0Jj`8~W9$SCt3UH0^IphUdiDC%OpS0I1jZVJo{mA^ zd!0kyVBW*O^T-vS2mZ}lz!d?1&b(CecbGZgohCFUo;a2p-0XRKtPyHV%=ic=@jjyE zS5Ii19XG{Q;@&s_xyj{f_jOQ2v3UV%|Dxw{BaAof7C~dk4X(ughhj$04f?llIpZ#B zSqST(l6=oIY(9w`Y7i?B^sE1E@kYDqN@Ncf50+1&b6#F~oxL?$w7CM|QyI2VeZFER zFdKY^U1}eyM_sj=GJ&mohSKjzzil>TkLy#<+OwQ4rUq4kP9o41&I?23-KHRfpps)y z4F~J_*AYTc=(@V0;kCm)iuCR@$Kq{oNTR5(Sk)9fR5BOx06NG@##p21?OX(8$luzJd_%Y=V_`2%}yG(IJigYd(4(yzkOb5dVo^mXH@9Qu8f zU$6mR0(?D#dIk9_Hl@dQ`;<2m4z#$#Oj%I_YDpkI;$PX(5cmk6qHXHi-6BDSz4Vp| zz>pzrEjX&&#vQ$Uz%_Q@8RQxZKzP&dhNfe6vM2E+Ji!(*-t8VKS<~XKf5e9;#56x# z1PdKIeuZ^EUEk%n$OpvIjJTe9k`aYAX#{?*-Hxv?ZxrcMd4HckR!TF>cEc*=E%7<^ zS?iTKEBrli!}JYB*U`WB$RoqKkr2wxyRNI0@J@UAio0HjJ1fq`4kC))tq64>RXaw9 zJfhu0qV6vLqE$i+rfg?uyDEzkQ%2DQ7TmLVyxN%pJiZ0iUV*&>Na#tE(8VP0ng5XE z`aV*g5PzYBd*cwR&R8iCne%&*`Z@KKt<9$1Fl6+)jUbP)q+&OnTubg3H1;S=D$8J4 z`>i~0HrJQY*cp^fqWb@@=}#~qn_>J_ucl+}cTA!c8jotCAR$j})Qp{EgL%Jw2}{q8 zK39OcVcUmgxVawMzsmg{$2{8lWJN&`x%)`Qd$s2gF4Hx$9I*VxIRK2+9}v3-1gL&M zP_uh*JvsnbveXp`qnO5i=F!mSBY&*z`hI}gTfIQmra)JkAc1!phzf921sxHQC8Zb_ zsr5|}lAfCj-asI?H?FOW3X(y?8qLx11gc^zO@tb_M59;9Nduga?`QUKjEPqR z5$`V^U-xGsZyWaOMRqkE0_!{+JuA|IzA5_E1;8k|=Hwc|4}$*6R9Xd)LY;y~OY4Ac z7@YL^9AtEl0H`u7#gEhvCb=8TF> zq=Te_J)!~!gsYze7arpg-X?uTB-FY|CfEz%WulW za=&{>y*6mU%+C&Y-x>f2%UbE(ff}Paslvk|xH{s>cyx}rsdJyzGX z2wut=9WPu%Z7(RKGL1oat|uYwb9LXchPT9riun6|i!P?rm*2!5eK(V<&OEmH@1J)5 z8YushnITM4R;xw)*NseRr1c|E*2EP}A3E1k0_Nanlpb1M{sm*RKotjuGv5ZPqZ zapBJKgMuvz_0Fr=e&PA}VlrPQ*>-;$8+PMfg7XW@$E#t2;h0z*T>X~OInZiL`$bWi z)1T|h2%IeLqD+-8MWlwK)HWhENmRSMJK!YX^57fm!phL4N$>&Yzh5=wP`R|iRV}Vk zjXON!)}a74SKtQV#(RD7gA(c`f)gN)k6?XX8Grk!Khe!{52vpWV2nsyv%)T5x{t!d z$GDRY3+&N`{Iu-p$1d)-lT<##s$|Pm=7^J0QFXhJl~GbS*->TZK+1pVvGH@OLDyn@ zikMbAotg`PK0mxh3&^qNQr(O7W`r&=o>}++y4Jh#B;<~D z_UPCPq{F07m-ve;uCuV>2Y9==HDv?xIa=0Jc(UO~+WlU<9LS;xS$a)Qo)H$}=?#S@ zqFGztBT{V_6S%=heSdFksb$_{fD3<0<&=PzjsKtTX2O2ra;@mMntcG3qqxRr#)Tt-)5~(^~F#x{}9Wh_lY<~qbVG={Bn&1ZP z<@w&~6x9`ZvyVm*hMN_S3%&amY86llboyCuM86S=|KqHFA=izaFeX6IsKQ!7$d5yu z>`{u?LZq7x#@Ka4_9ElrAC~Dy3a8_>Tfj`f7oEO%ZNaCp;X&R- zFjNwbe59>%4TI6*A3o5OHiLCs0sa;9PWrau77uI#v2D|6PMYY@Hr3X}@aw)h%NVstaF0j7N(P0Q*R0(|_>sUOCY zqdC%=K#M?_h|>F~j?bf&wi7|Dislwada)<88+;6w1MEw&-g#Bl^;pk8JW(Cc86d?5 zR3Fskr%te=N2j;{T1qRm&%B|wBpUhsn~s!D+9;p}Xj&L!V-G+7qWYUzX^V?_>1w)X zVg0t-jvuXSpu_3;uxmPCy4w#e0+jjWd+&nx1+RQCBYgHCJh5aDf82YOe`|!`!BZ|XQ+L2)+?H2=BFb)Q8 z_1+h=4I%i%#0QLY9&RD3&w7z)J5as74-0MgSLYU?Zv`R`$=(wu$u6S+6z%+>{?}Pq zeZxyl{jcTWdKYn{5h9Kdw|H_H@7zr3(5g;F8S=5B&8p7t)2?C(I$26WM?N1)Keb8f zUN~vA`ylOznfu*~1~%mAPD~VBOU%D>`$JedYy6vaX?)2M3nnw1wq-~DqwQmP>iVJ2 z0EI!Li7&r)=o@2Bn+7%v6qDM2PTdi{XxhK}n;D_bs_U~nOnFd;gr70R78yQ?2vu<$ z_g*0mWeK0uO%GsZ64>9*{%NfBks-7kNPg3V$_7 zH3Ms^L~Q>spXPr5q&?+wOg)4z^rz6`N7?3;5sdsRJr+4*K*lePAIgD*B*}GB6XaT!!!z=t&F9urws%R zL?cM^y88^9Du-`{zS6a(mBEaxR>%)nmEMgW`YRv2+k zUFhCq>Vn_kT%9{?j4Vac1(>6$FH!Olc@LSMk_q;2Iuxh><=N0G9GLUxx?{e;tOxN% z2ugY*ED%?CSQ=}Y2^MZS-xliVmwO_7*OB&YRDjJ%%}spc*qsbYy4d}LeJgfYsgmo?Hj<@P0ONr z6vNWv;{y$1fm&pu@omJv7sZRVKKn=GCuEI*dN!L}fx3D@sh7aCAe{os zUZ4jdgy4W--c1;BuK-EFU7L0UJy!7rfRY(EbPC}7-U)YydJ;K0V{K7auzBFs%lU}q zpUt7{f}MIPD-Icq6%oFz@~c2;^a4P){l;YnmkRvrws?`-^ev}C0fux|4`cryNXEhg zfs#e(zE*)9*Fd_44}NN4U_$7u9nqa0-P#2tc!eG=7&%yOM4H({NC1}V-D1FZ;OO+i zyB-9bPFB*~nlCl`ZMX<}|2nE!x4fmP;1RH-nju+^O?%lc=o{g#;ph6~Z!!q!+2-E? zJ#R^ZjveX{j_%xvzwup`;B1s>w+F2nI_Sbm(e-wcvC;SE{1 zMoRlaA7o#p;a7Ragsvm$n`~4Tall*_u5(X!Ut;e>`)*NV@^1>`*M4-6cPbr7ITmgg zWeF)$h4_)YaKUuKUwyeFBeq)mk(@3axQ#2BN6fZ<)PX{vx*EcPyVO#ok71LBS`xF- zqz1C!Ob8NSQ=cM9Si953#OF0z7^f?kqyD#c1YUvK(;&x_IosEtg6A0Fm3obK!v^Hn zg>C)O1USq_50pQ1b8PWwEGb|vKOCAX65TNCJ%i3Qd3mK8<2D&PM^;JL58s9RzJcz- zd=1%7puN!+<}ZrGruQBI!4^M}`0z!bWf#kdA8F$1XwZ99L9Rtc3uf17dTLyuxa3cb znVc4h+lC^H2UHm4)-HZGjwML)tG3w7wFZAC@*6;pG1^!?Q38&tEfOY6cn>fPJt|O1 z>?+lf7?UuHB)!a)2?db^vSB33rx4*^1YDi%J`oEdzjzZpoe%y>k9m2CwcB!ms;sXv z3F23kV82!qQ+RY+_A!;ml7AVZgsnzg;Ji^+RTT%!`(#k0pE9Vt5KeaG`-p9t?19x+ z;3yMH8b{Z~^zKyC#|GYb4g}*8GND_x#3H|Q)=I;U%=ShBGXt$%Ux1n-Ov?s+uFG*A zh6CrqppAc##OT){#(A|*Qwb=I;YT@Ex4B1CeOMI6KXzN4*I~tUwoq&Lv~oOsjXL76 z=zgZ%z7K0*@JYV{wHx`~q_F1|J`Dcrv^cqDKEyCKD55Z$!|O&d0Ii_%suu+)5gmUA z#9xF56W;fE;9Yru^AWpR_N=S929zezTT{pe5Zwd?YZyoDl6`#IFN(i59IXP*;9psj zKDQu{5bXbO5=Td%HIp#4N5%6^3+e|H&{No5^I02VSk)u&Cuqrpt>|8o@)&IFJv#&Zshn<^?{Ee1AltU|pZ#-o!~ z%bA7AJ7azd(fTxj^{!?oz$+DPJ#^Mkw()U;I@^zn;sY!_%2PAJ|^>tf!bxE$<&rcL^`=C_kv-?KPr(yZ>4wsZ8%Wh1f>?LyAPu|D6Ae%TU#Xrws8EChn`KZ#EW5#mxzA2G zJL3+zRJy)xAj7r;{)=$lE{JLxH$^Ib)Bi|(xu9FOtDiAG_jrTA^H4v&z_6RWE3F%Hjx?B3a`JZ*pDkE?UZYEO=RE?DRT1mVP%|SLThp~(vq@kEcv`veT8q+-f#U`{) zMJ4`6C}qLzh(VX#MmK%A*cN`v-6|#ppTG zH4l0n$Ou`x|A0bcdcK$b=_FnoUqh>CMaw|x<-`8(LC<P$y^5c~|9g z@xE=X(JW8mb@DqQ5{mEY^PeaUy)`JiF{S|Gl({#;=^I8=NFH4>IOm$HeR1?b5lDe=)XPg68Ee-7nN)vPiy)+Nk&rS`Y##% zGma&Cmkh+CFb=LlBu$%>l}QSnjr6nJrNM|C8vK(*glF`1;aa~#x^Bfcvh0T#A$E<9 z@XGOXe2Ii1TC&#jx85Gz9$9jPS5NVN{o0CXUF#!i=~6T`EZ=Yg@zYW8o1{-(ZujiG z0(dp{-J~A`i@Lr85~^Te4_Pz-=ELwF&Mr%C1xmRN7ET+bt^?ik ziskZ`%oKxABwh@VHh3^P)5nUl7NE zoBf?3R_%*o63^6r7NNQ~YEdirC;IHzF&qvVP*C z9vQBe3iS;NXFALmm^y+mu24+s3$-!>M{`3G#Sy~mpcnLa**{StdB#{;2V%E}4&v4C zq`U04e3fK+ly_WdSnE`hu8%4;1Ugh7z1t~qgPz0^ZWzaDv8;Y`OPJV~`Qdy?UXS7I zMvmjmmp-&a_i}eQ^s2v@LEi})IoNo7`)Q}1O>bz=sFb1D?3(N7x|>T*LC| zjVm=;tz+Tmodq?iCvbfzq9(4RD^x@Cc%|S2M`c%!!E?foZ=<+X9bTLN zHR{0`qA{mjo5q(^bz81{sV1TK->Vk_{Tf0Le)b_-@9CWuR;hpLs~2;B>4k(QRk-`O ze;_z?I!-^w#(p)pS~mExf?l@C(tx;`y3&N0rX_9$2~FR8>Squ9hbAn{{MW>sjaJYu zx!=xdB~{#$7s%U@SN(+yFDR|B2W=2L{MtUllkoHLijDQ@q(@CC!E@F&V9c7!nmUQ; zLVk>@fBT}v@8$vZ5vT*3gXT_{PCSieCuD4&b;qQItXt z$q@H2qsyD~#cEo?$iw*RKaPovQmU4aDgqV4YG=D1%jKIitAg(oJha z(4DwbVf;VKc`js$ICP1Z;)ayVUd#btu!p)lBTp~Au7F)IBAVIqV1+2HunF>qn7=$)jGyW>Uu+Z|2DM?>h+I3fNQ-*L z&WcJ$F-geY3y3aWnMgb}W-9jq6SZ%8221Y^s>_A$8VVJ+vpOmjNkX1DAth;Rx(T6l zuTz<{y!F44asL#1(H_WAD@Pi`QS67QKM4|-rccw)pYEE}BnRHfUltIA1$-ddn#jH1 zyb-p@?--@s#NkP$#>Gf};c-c|N>e%+wb4M$a{KW85#dtc}am)%s)0 zC$M%5lz*0$EkZ-8&PCtnb(8pQDDOIj$(IE75yRTY)6?3GK8l@h?&gWh|7p&+WsPYy zyw41!)*?qzNWtOf{T?>74W*b@?!v|w{xx3^LVIpldvnh=lgxDX?9Ne;o@HpHd|B_E z`lKjwxaLsG#ze}%n7#+l$t%uXmG>l@RpU2+7Gu37^6f4&-RvXG(s0b58Ub{ji^caLezVGt{+ zz3h19mC$Nd5Z-%QeF9B4i9YlL^r^?oTL(1QKk`O0iq9;c@&$oj?hku)>oy&Vu3KZ} z{aE^)q|3kkrBBiYsV$h$hr$7$}mrc@w=T4-TatG>=u$ zjlBJS58$@_PU*p35V-U1_tfi-{cvW)5F~S|AK;*ltAA5Oe&I#6W<;W(3!0<{w#i#p zN!MFnV8EZ%$im9923`1PlO^X5T&?nAd^+eoFyfsdw%TdAE0tw?;!GTD0q_z1(XF$vkyph$-$2&|Yzd8*ULvQ!MPGO?!0&r+u0AvJ)dOpM))Me8C7Nf{jz{303pb9 zzaztKjNr%mE|pt;g~Ffea~+;AQjSf`bv>3Wp2+RAzYw$VHO}#ioH(}FUnzc7OBwkl z14Wr87|V7|18eT{H!~8SpkdTBFQ(e{1LC745V`B=xzzSgsqpN%dS*;FX?K<`*8EvI zCO#edF^&CD3Evgf*dq63Jw$#?nEJ_jMZFd#LRrswX--2dekBacH==q3^GM>^M?o!? z1>1@HFS=X)EO^G=HqJM+3TUG+%YJCSms_b$?cYb%ti2^~rXi*V|ET$>^WRwDnj>iY zP>$G^nn*x=;SWUCvs04r`mm@!qV1RP}|4xeJ=^1HSlarFzEBX!F2zUNVr< z7QCCls3d{Ci_qVm$12baOY7a*wtHfg zPyEhELW-m6yI6(4HSO4T_jIT@dL=~DUkN(A?IyhwPux$(^|S!cu6ndj>ZU`vQ}hwL zU@>UIxte#IwZAXQslU6u+Uy`wtBNeBn1$?%VbiJ8Sn^{r6b>d3$K(c5dm+j0ixKQ` zitUb7xkIV}=DU9Ni_u01si~8GobQ73DPx_USxoj8L+U$xye%z|#!wY+LNVI|& z4XWfl&RW(46GK@k^l@1@^xs1~;UPa+Yssj{IMEh2mBe;hP-}bR&eE%4r6?r&u}}C2vP@bVi@=rHcwaG8_O~Z2kVgaE`~9Y4 zi_%vLcf`Szf&>!CS&yTGNHObC}GQ32XS=<0$G z?zop9o~Ou^y0EQi;!VAS|FxY1UTqN%5t)-{Yz1wya-ERLb4IG~^AcLLUuR1Fr5UAs z)F6+NZYtdT#dolLJppn}vn%mTrQy4qaFmTohN{RU{=(n~qsO|31Be15*45lP3w%LD z)-Gd(TWBUb>+H1<@ql;Krv+cFo;Z`o#Z@108?jAT{wHOt;fAqi)p4k}TI?0|v9tG( zw1io|p5*J@#bC&_^X(1eqS-PKl>c5oRvdZuM}evwZ%Hntd>uAnWZtbg?G>-#^meJJ z@;AMH@wryn7b#GLezYPnzHS8dIGs_Sw5G_vo@cC_(+PT0fgzjs)CHF9n)-1;O0*TM zD{ZYs6{0BQFlaXX_~-$XYZg`VsE46CS|>AlT%Zk)Mm!n7wPiqK#|i2Jn$jL{7jA&J ze(lSNLtyfkaH!=TFOHfWSE{;!$_Bb&(T#%&=c1HTw=|3@&qgUgXLY6w;WpNRR1^qu z6)pVqg}7{UZ#n}!ge`_-W4cYDS&Jkt;2guMI*Vm+$kzkLmU1gp{x*UFtPf)v3c~;b zwocwS`(1^y9^fc&8TD%2_~}MdA^Xp1JeQ)V6X!`8%!Uzf7zXqoac`6uR=*VM^chiP zuKP_-lC?gVspdfo_E3fuhQcK~g2+PuzXnyGISU6nzA73x*OjoFNJC79@hx#O1nnt2 zq5B`Zhrk|rQ`5;E^ZPPjm<<7d&}Mc7=Q8J`nJDt5*YI)BJXOduKZDCWy9jEDREJc6 zrB5Qr9k4!C0i=oc)r#`}T`gKDSZ2?-gaW`+k|O6K(pSTs!ube2wUR+GRA9x6&2nw` z=8;Cdxgah{kL8_8PtPNgAPi#?uP@iq+o5Pn%XCJwRSb`zaG~kTcQ1!mwc~bfz_pmMFz3zh^RBg+KNT|gk^}`$DL1Ij0?E-JQ^#m!B5K_TC z#8QwC)e!sL(?ge%h4@sk*8(Y z6?l2**f-mZ^C_#IZ;pl{r}pH{_o(-kfonS-Agw8!NU_z!h@AD%p85YKVsoPAiewjp zM$sYKyXIB6MzCl{dea^l_a=@1&SzB~&pKp>)M zx_Zym8^jfUK~T_-X4a@zmL5Qhj0VDk=zV$42{0)`?!=3xJm~pqLP&R&Q=d#xGEee%|8(XK$v}g$Y!B|MItaIO*N1xEqE6 zSGD`_U{Z>*i2uM!Ci4u*gZ;^%uUM2TkBX{y4X@8${TdNKC) z2Te{JU-tB87kSfuo%@mVbHr-c}5o~2Dx+Jq!o%gRC6T3tGerz<= zvh23?_0I1Ye{UIAp&KYWw=OSHLolR*|! z*Me=A-y#Z?C`>NJCe{)`C#p5tN%e`vwOeGgPE5r{wco)j&(tV>0pP|@w;Dv0k z8?gz<`tx%ZqQ&-p9y!w#HHm+p{Gt8EZ^PtKUVo>=u%eSG<)hQifB82c>Hh*%oHUVv zX1ZE|b%mX{^fs^ueD5C8 z!rP)=R9FJK{l=ga!!(xM?(DlGkM%jYaK?AbRexaBnGr(F^NjtQC*I|Gbi8!meRKsi zZvsV#xIGd3n%pmDsbR>M{$1p6-iChgpmShrs`JEK;+d(Ye%Ho6w9zfGLB7+|7~Xgb zb+dGv5yhjEr0q3#m;CF6fafy1^*)eNc+)hQs}m*d6Po^kuj+@U6{F~9jn`C(B{#pw zs@*c(to=*G$o%9`q~EuF-GelGx0t@P0zwP(uVU3@5tsR3`_yVS0~98EH?N3cQc}IJ z_xo{N5Bh&YBA$YGEu?KUbU_vk1u2Xt@z+=cUa{|;peg2FN;dVE6HxOY^CXpLu%Qyu z-rLJcDkS!KaB6k_+P8NU`aj#Q2Di%1yR;GntOL7nl3mxEP^}j==l^_c6S5XVKcF9o z+*Z*1gHism@}McUEi4(dba_*wGtE@csqoXj3^DhYee`F*elUF+DXyDT-(^VU z#7&3unUQzf!1@Y1skDB~{%7M>ix4h+iEw*!N~o14oX976HsRR;fT5QXguTgr?AJ&% z-0RJMk~1<@3PoF*FO7Ivw+zMmOlS}(lc>ppA8x@%*b>X2VAHP+hWbRFD$2K$vU zI~ZZw4+u&!)WfmyP+seRlk6b|4w*D031w{M@87_U;5Y+>ME+BUz#oF9bvgnT9S`om zr`ueR%>MSE9p62{V0`UjOVb`vn&jakwN3z-G&{r5~*%K^d@>RlBM#NmL;~V57MS2vJw)Y_# zvbztD<*3No|LKi~bRTVf>(3V4f9+?NNK+JG?6sLYgYO4meWe38?OCRUovG86Z$`mI zzJwaV{hiSUmUxrF2>gnM?1^Zof?K1%B)W!LgD}#fzs@(f=K;cq6jQf)*^_Pjlxzni zI%Mc4bs-N}41Z*h!ZH{J|!72|Mi zQO~@?TyN&Rvqlm(x!j2<1x}l>#r3+Ye-9pzl*Fm;86?uc@>Tbn94f+J$blZRzaWRr zax)h(CreI<9+^syN&Gj0My^xhtE!2EXe;&E+BE%lQ+T$!sUN4+;xRWPtt-0_uJ9Tk za+wC2%&T6A2U|XTmJp6hsX@n)Arpc|=p6J+Xmf~(a8DQS89IHxzcP1oMNPVo9d;p0 zZ)OCf@oRd9cK?)|RhgLOLb{K(OH^@{_$xOk_v_m#)&$oTmZr-3TQwND9?rOQj$G+6 z{uUujYM;iFdR(|r%4&{3^&y#3Ej2WZ6sJBUQc;iV>zv7@(XJ@i>Lj?{J(=+x6(F91 zM=bcKjg(TThDVx+d`_2{K9>SVQ@Q~j7J8cFuPvPfpSEy9=Ts5Y zWz@rvWjArr6RpPgXuHiT=vcddE8WE+RgYai}>Q=mPL>rX_~U3&WX~!p@~%Hi)G=`t_zM`-Vs1!i>s> zYGS+mQT0@7FPtxzLH_Fdca=0O81yrb|E~GqNL37>Vbr{O zM~6={!ep;3q2_cp_&Tk=07!HK`V${S@B;UkVa01j&ool6N8;+>Q6bYv3;pFpnD$tT zY1J@=0{KpVI9%U{3e=>poPay=sE(e=DjCR~>ww4R^$|5rydV2cay7O-7eQ`#xo_tu z;$VC}NOw9XLHJfbCtHp|bKz0teaHICL#JM|*U$cxwc|y+$N^_XLWkfpPtEbQ>1>fg zGl(fd)v7K4>J32HsPY={mV0mp?$SFFsb+i+dWsmZU-Cwr=hs1zees!o%at>S*Qs^C zyw>|i%Dm7g3?pO3TggO0usq6B#I#Gj>?^Y9518DF%b8bZA1y`-7jQlN7uD71xZDvw z)6mw3K?aA4BP4xA|15nXI^bL;it_f*oo{je3S)c;DMw~y(3Ix z3tuDbg8WjTRp_w9=HU36eX&o`q|~G)QjROsyqp`6AIVeKOTfqdE}!lp{3(M9-^C1>1J04&*UAO-=!+1_<9MSZ za_4iuQ%0`B?+#WtW63V<1S4e9B9fzC=cACsbOmE(xFm3B$6Xa|%-0#a3q>oUs5`f^ z@#Vjsrx4Q6jo?nH<^3{2|8UK=bZ{ggQGAw^zQ(bL~~$fw);Z9ljAwl*9B<8K(ha@t~Cev!1MTMtDB0~tXJ-Q zyUnYH@0R9)sGN+apq_C~qS-ah@W?~&y>G$Be^gtGXRwMC11lJ{Vf{N@5nu8johOe7#HB#IU=K+HR*Byotzax3r#gvqvT#@BIix;@F8BD_>f}$G z<0Dr?a-L5Mw7|ptEEb3c-o3gy6ny4hX|+?DU4lceIh5fN1d(Wm?*#@qpRBM)W-Ui=*Tr6vO%2hlx!OS z;C`uCgN3K@xrgf`-+XVhgHI&;*=TdmTTq6cusb(iAN93~M7-VOqKIi_zXq2XJww-a zI|{Hz@6(6mr$v3uAEb2PhCKGb;&CqAx%~-jXB6$6(vzVyy+ZiWZXero{je6C8!5#_ z){-qZ(j1!)I|MmL_(lr%7JrIounFZt^(Y>MC0P;Subp!_j9~8l;p;)?-7VLpAAnrp zp(IB!qm?%ITN#XBaah`wGj97;oaEsXHBXGJLT8e#LIRUAalMIdxo&nzpIQ|qKVCC? zf{BY~(LjrG#zvxwdO+04JStJ{{P&eWhuPWS;#r(xnfV`!ZI=+2^`SFU>bDrM<2?#z zv|X~HB)4>6BVY7WNk%7=AH6hT@rlM@wg0Efb>?@-lgNEgMhy*eY^AJx+l?k6V6Ejq zrt40whqofh>!Q3nk#_ZSW92;A@2=aI#I3_y=V;UhYwc6OzXPKjr5LBrpY9AAdZ`7{ zha?kaZ6J#-1}n7R$M&v8u|gm468JAn8NFNgl*s=Kee=B2yFKRTU3@C7TEc7Yc$+ru zqkR><-a5|h&>V}b z?-dDhtx_UJ?b~Pb{xWhoG)G8TUe)#s%xM9(pM{kF^``vD325>j=r+w`T;1gpj$$yq z_P@wIer~U83zyci?K;F1QUO- zRZ`tW{ryz^6U9l%N|BXs8iT}`+vXMDaB&u&7x?VfeSc4w7v{PJW|K9h=|a8I_mxe? z_<~gBo>-Oh;^8Dc0ZDG`KVKc|cfgq{qZl~3S?<7w(dPL@%O}$63((m;3WR~30?#E& zdmt~&3IFg;lRBHS#8kEy*RV(AB~H1(SmaPqLlh70d6&KXmj(8-HO?zx-RBtWdG<3^ zf4+40dC*1^MAm~#K@RY6+JxfYQ%~ywg0aeLDD(P$lq&!x8z z86B1xI_lAbLL29=%hW%S@Xyw+2cf954~rXFo8f$BOP1&9CN7t{DzbQ~qO!^qo0Qb; zTD$!d*bN#?LvE4uP;~YF`KBJLBole_sWnWJXNMWD9#!EIQU>%3mVq|-z!G&`v72XL zUl?UDzW0jw*~gv@_olxvQG%$i!qa%@AGj(!kF6hau+I)ljVx5Y!H053!R$?&V}M9q z#Df{3k1M;FYua0iA0fzqGPB~e^*EF&`+V=8A^+ZE`i6S9^n9!=e)R!sz~BpGIjr~?<9xzLPA z$Sbh~3KR$zGO5mep~If-MfOPyZk6qn^o+=c_{_xVlNsk}5$%eP{!Ywwu0|+RDa=-P zSMi<0nP>f;v4IUPo~B@jvIt?47h60&a!C-ncs3*P3THLCF6FG>r{rbKFG#+%O0x2% zAaeTI_r7cI{VeLDrk&<1-I<2hk4Zgf&CMMdj^9fkNQ7WP<3F7JQ(*awC~xu8&v?`< z>3t{B(A#+9fHijM$niPgH+0#RjkS12eK*BXYs8t^Dd)xJ2hm*o`j1!r5thHs+1~AU z?|X;S)7u@ksjj1)eEV3);CZ%3UWQ zK`@)Yd75+ADp|kdr2$!ZTbs4*;1lM}3MpwK9ol)VJ_tN{f(R>XC`=w6r5Vt#UrArPj zs*iz4pHvROUg*$qGZ@UziZhvK3*Lc&^2NL3fBNW->6Y*ko#4ycXEI&M z(6eW;9ottfpqbw(;shXm-p^+Uqhx)<*!tLh-+HVKESv`%fagmfmqRH}j)ASfJDTE#Sxwd+-%k!=3LnueG5!_tyL=Y3A4>zc+1T38K)|uJ@lRvT>D~`RIUAfkPDAC)Of#cA*j#XY0SltK9>tQAB(^mhTlc)qW!>v3 z?-u1^z4#yBq@(LPS1@|&H*SyS#mv6#YY*>|@KCojt4kdV8G#GoCUfU7(M%zY=I%l! zkU#7Of|(UV4<{Ks{r;d6*k>`(;uxMR7)RArgE%g0a%IHJySW!YlK2te>Tcr8gI0T8 z$z>?$K+0t#doid?$k^_xw%wkIfpq!Z^EKh_7 zs;8~<$D`x?rQA0mbQMpaqZ}xqsN1xL9g=s7BoL2Qal`%(c8tH6YpRcycSIn-CjC+| zssq066#OfcRu1G6u@`m!;eS5BgA@*K74h7mpf(h{E6mK?lFdvsaqLofkrOxFePOh9 z8R6;HSSvGn+F^grRG_FeLS`TW1!A8nYy$;O(?6?{i$iC4-ToB)X>(#Z^gLU$tC{l4 zNVU}P9>yLP_z^Jz**iQ$^af^k$7mn_Tr2G7*h|cgMn_x@;i0=NI%_!2__>jydpIf@ ziSg&{Dl3REd{n<(!PkJoa!H#{pArxFCmrR9%Z5Z|{CtXilH|M_mCy-UiTplLkq`0O zzge=M@m&TxeVU@RO#s$ak+p5ZLVR3 zV6^9+x9#WhKl|RZ7}N#655>(k3HqJ4oY-e2SGfGY6&e)RqZ(q(E@K5LjPwlak|?6N zSHMT&QKSd(f!WWrkbc`>&+=;|;VgZN7+lxrf%lgEPkC!h?Stfe_> zHU`SaI>I-b9-1DU+<#zAQy!MOKGwp};VJnEJ@_;gFRN^OsBJ(`_f(qeUc_9_76*d% zw@m$4&t1Wl5fx$z&S;5lM-nI#f%yPTE3df(@|OT=Iy(t8DR)N2yq0jYod(hEYgMk2B>wR-5U|Gl9TABjUj~3Yu)|c z?8};IG^M;Xj+hZAy92KU*1_=K$vi;-X`FfFaRQ>tac^JhP-V7$W{cDJ^tVvihZG*7 zVi?Op7!dx5Ss=c)xiB##t7$v(ZrR;yt;jrz^}2Db*5Pe^d~`kTISMq4`pffu#}Aw> z`@nO0&d-5YheIto#%C$DmIT;~Y-;!zU1dwv7M)lQ^{X_DW>ii{=p5Ow!#vgkvk-f= zIShcEqrL~t&CMPCg;ofNHLC9o|CQlC?+#3OJFLl1Gngs0$mc|4kRk5*9_`~h$XjSf zXKnO`7&f+mut!(?8-(Y_b-P*IMUyE=zB1uyXHR_O?bNP2DRMQ_?~)ZdRq`e#vD|>( zCarN_GsTr*|?YvO$ zM2uiFd^fC7n6BP~TKoqqdxtqJKU3~ZNaqIB#0nJo{I2^2dZfQ3GKOOaWCx30pQ$&_ zr6xa)3b+fdchS#B{gN~@*eoMJ_Iaq}HBO`9PT~MzP^1Y-e+#4)z1E?Xm9X5uJs!2I z-e|jgy$1{a?t$(fLCc|P;=NO$9#Y80Y<>C*vEL#YZYIu981W#CxM?BH&3@uL^vhuL z)ie2Zim(TId&n8*&5x(ry~#dM8Lq3e0Rv_I(DpxA3qFiK2Dx{W8!o>kb!T$^VdVV` zd1GI2vQCR>J|m$f{<1+KD(ikU|7J~m6({|@I((6$s72Xf_NzEKcN!^|fxcf%#F#@< z6>W)|3^fuQ) zJya!fxN^J22&69$Hvbv5L6cIj0inYVqHo`kA@JBy*<=mQBAdXfO-~Hcp zuji5g_epp^CZvBy4pM=htoo#axjOrU#Wc0O3MyxS$ZHzg)cGUGH>@);1Uzy7TfFMB z#E&Bf0M{c53~j9OgwrAs><`D+!Z};6-^)-Ryav(T)D-&#J~4+Ou73Js)W}9=M32xw z&oNhf@zdx*dOEz5)zh2S09a+ah}&YO{-_u{4iveDz14&$Mp23Iw#}Eokcy*uYU^G? zyds~r25d8+JD8RGUdSZL=T7+k$gTPDQ$JkuIpkmzJkB8=X>+}36+l%Ls=wTL>`Fk zT=LD0XWbwS8TZc2Lhz%^x`$8{N?A4O5MLJ@Y%OQVirogUI9I(sY_g09;CZgDuGU4m znsHOTpK9{1<;LSP!G;xJ-0O*i6c5IhV$6V&D);Yn83MOl8*q=K`S!(aPcDJpTwXwl z?It@QVQA>;*}re!e^2=jhrXHTpJBJH$qY{pB}xm&hX*y)@-i0RF%te_9xGIPw~ia; zBqR!%GrZbNNtnsO&vn}^Fq!mre{8iNoQgu8BELU(C0#s|z;nX%@9(;TA^W~#I56O=!Oi`uVM-cC2-tLQO_dK06RW`@YV z#@S7bMpxpbmSGDv>*-iYLIEFEhshadEp)q3TRu5}EUqrP65xF@g&eRNz|z5{d?#VO_=cHn7EG=Uw) z`prQ+E$Q367FKI@ij%5S_I_H z+hYiBo&E999l*2wvY$yDUMpPw7a8A+LH&V1uiLZ>hdanN@ zaT|z(sVTUvRR}Hm@P(7G@BLiC)AN)ug+%-QjaBfNrY~!zah8?SWiC-rjLtr!**yYa z71wku<;Q>4b!C^v03Ia`qVCE@zow&c5sP=}LiOVhW|7cOv__Q12U!&eEB(xmI%bPt zmt!+8`tbT8V%DO0Roh8_`FzpeokF?ZpK8W-EDT?~a!YA|_qdzZ|MR$*;{+*#8d&|u zya1yuCkQW3!dXM+#0jAbyWe8Y$86>VN2@dV6KL<3`Mwk{ z?2s63->W3#etbf)Q(<3OZbcokKAXqaC}uXxE*d_sf5{NQqcf4(L#D^B{5wtl_mNhP zk)sw@V<;rh+IV&Mvo^h@jEd+ri|I)MyHn`;Qb~{FLx(4lBGx#~Uq&!Yo+}Q-sy<#-*sD&Jnh$$(25t$fhXr1odLg^m`n4`fHWpvP zcD&YhTT*fz*9}1V-;RftpU(*VtyI7LdwCQFeSgwjXq`%XsS{|^k0PHir>P4|dLI$i z+}KgjrrV5O)PiJ_iZR4PW_8Ac;4oIJxA1 z-hpdtu(gS&=fN_Q#h%Z9R8QrLnZ@2}p2|jAIb`*km za4NZOWtr;M06W6NADug=r|}RRLeGTBAJBymL4{>dQCpds9+2%niSjsOvpI&0*6~<2 z*^ECe9xxh@w>iEt#FgHrU)uzct0=FNjXz|BRr$@^nS=cvOR5(vpxK zA$M|%6o0UQmWQGx0>LwAqSYO2-~4pa4Af~e<)h$Hfh&CY+N)6Nu2U2yx5!#mROQhj zN;QZ2&F>H1Xk@gm2uvD3S0zp1PLdIIgZQ~^bI^~D_q+mLu@4glZ(`%y-E$OTzJr~l zQ+4n~F7=liu-18p)7$t0nS}+J1Kja$Z}tyHjUMHKqyn7A!7%**q4wL?H&Vc98nS{r zvkz2mM={niEVB>8ay2){;)0;8?=v$f#_gb*DS&er@R#TZh(b|F4VI;^WQ^WdsAD=k zxfB~lVV_|vl1|3NGlUus@TFC~YDX6ZY%R?FNE1L6e$;wmS`?NqB~R0Gb7s^%jq2ii z-ziD{im*V(jQx@`#3~u2<3)axN(152hewDCzT(HFOY73lgHzdtQJh8l-sTe_Za% zQarKt`Z#|6g14FGjg^$v{Peyih2`H5*mJmV2rf!*rYaNsHWQ7roTf52{OS0#ED1^k ztVG9skBNS?CI?$#!I9OODRSYFf5xNWk41`Onog7XyX1}ccL!5AKXRKqX;6$tF_@;% zdb-t(htHM8lsk=s^(cA#2!9nwPs)LFD;>cFji#GoWI6=lS+sK zZ;E-9WN)K>SkX!Ns`W$0Off0Q^jHp(%8ADaV2II~h-!+5I!?}7tmgE56;9QBg;)Q&;L{qs#An<;F@PBe-E1wdHr1@#rc&4`F3dq8oY_` zxA1efs{cLvv5Y#igb&2ew#JnXIx;HbU7Z_P|Jg(593+1fO0_^6_0>`ZY)LZQX5R4> zTkL2AXv+t;Mr`)a_N1d6%&2Qpd6nM=M#sv8Cm(RU;ld8_n^FW8elIk!3a_3$2XnLtST?71W6}QVM5}JRahxSt@hzj zu6`nBdl0F^?b=JEUK1f&SYv;Y*AmMqsNZsZq-2^v=)heq23D8$mJ4`&iL^wSValBW z&%t$j6U-EZ6-!^|mn8_w3pXRe&aZYV2T4*nx;p-rchgwfmS3xo`x@(9%oGj4l;2Fs z@(FG*tE7Ib_Jj}@$yQ(ZQvA>>MWELZt0X{`?S`|_p#8Rt5G#~_=m#unZT1KZ)a8q$ zO(u#=XbFb*mXTwV9QY4Kpie=iyCUk|r(D#0QaZVX7vP7!P-k{SE%Jd3y(+Q5h_pJt zA>5?S(kk;uNdPQi-Y8F!{raB9+6>;QQy9^i%Ub`R=yu^ESQC`M);f;z9kdbMH-I{k zncKAuru>YVenKJg@WGqf8(+{fG}F!8f2MG9@d~X1lSW}Y#&4ulaRcR?|z0wl@RgW9EX5< z=>e@zdmspCDP&|Bn>&vO&Cf@(2|KkBZS0O|spU58)hqRw7UGBc5kZVKe1yq{bCpPa zKZn`+L@1{Ta!|4Ln9dPqiZX`*R}kae|gQDDtWw-GD6pU>y@7C2MpU<~v}SvfXax$&8C3sLX+VOa1V{#Wd~_Z<*bDiOxC(q&ueTmJNKU<0A%< zG`Kf_oq~Z-^vH~KRP#+$RsS*u9p5g!8{Jw#(k2{j@hwhU0k!Hes-%z-zFimOMP7zW zfIZyfp{*Ktp{PvX@sy-?minVJZ5L50R0>`YS{X~oSjv%<8^VgxfY}j*Eh;CXZKdvh ze2+Cy+g`Lu_eqdE$ZNo_B@7K2n#Z`B7I1hKDdR~Bt5g!N2vANZJuR-LM zPx}njnab-=w6;E2NVdw4>D`QNjKVSR%pUBBQKDV7f=;p$`TAErB-rP`%Eu0khZq|d zg;BGo$2aPp9*%@_{)k)r`K*>2pbROV6FTx?TE{|rhp>}P%E>9!P-^5M%CeMgkl%lv z!1{{=#~(<+%|=H?_{*>%T;_3TY{@T%QkK63yvIvN_&+iq;RhECq%JKZ8P;<;X_Nkkl(<=te0hBtbo3k^BVy zRPkWD5W&5XqJ5UB=c3l|aeUfsaY3(0IeB$5Rf=VT$i={Nn%xu(<2EWhxc&xlD7Vv3 z3~A*uOv~s>kp0+&kaQV9K*doRf;^w(MrwHt^YVFr07SJdIi^m2AXPx1bc2XzQQH3k zU>Gr>Gq@D|SsX?+va&H{5XjT;F$jt6Nrd#<%w&9XPSLP{>*3HW>ZP&=GtdR{vG@mc z*hq>64(9$Z=4PGO6#D@^J7QbZn{8?yw@^1FIQlSYj_G@NnyxADy%v=7{$bHo9q;~U@{y0pZ``q$P+%TMc2+-oL@rO>k42|pk7#Oo6Y zVz5K}r01{H%5gQ>$F(9DT!tex|CggX5mw0>45Q24T!8b7}(jYCVf>xQQb=bSAUGwhlnHx8h zSx71%E=DRAL@r<3U&*Su_`^T)k;6$JS}gW`Z9}nOeph9dU!*+>g%7?g5`a4DCXB{(4i2>ptvMlSKQ0+SG5he|TE8!a#= z#V*s?(RUVn*Hs?%PrqxxUzR87jt-Ba(3(iTV)dmH;})~$-MyJ=6UDJ~X7|}N0R63D z75lyk0h2M2DJKok2QHGg)jzYLS-NoDgMy9ZH65^RjxV|d>igUrR zg9SfY81r?(gnZPDFkwaV!2(^YXEevEkwL?cx{!7a;>m9gKD=S)g7t(>wI&h3ATBG` zfQm(;k(+xzQz=lefU5Sy-MRnA*;|HHxwc)SFqy~{q`Q%p2I)q+I|Kxz87)~eD8D>B zHcuV0s$K|5c0Ajz6p#OEBLB*In6z5*E9N3@e=4tl7W{NOt4i3(PB6PB-sg>VCW{!m zR^8iLo5{M-q4@5w2O}sB4ld@_cH3Pveiy?c*!?Y6I)lDbF&_-^8Sj#x{5eRxK1k(= znp3BF=3}W$C|d4_`F`paFKEc?1ZR12Z(6ORgk6^pzv%F9Ipps22NmVkhMMr#`{r!| z3-LWH0`e2udczS|jNKT;SH)49Dgx5B3&h`p%P-1)e7oru=Xi!#ByC2ErLoO3M^B6$ zH2$XeTbd2R6@0ld=77=%L#GpFnVY}KyErYIc$1-*Fu&)rWAl}jkev*UsZgBp07uWF zFEW4!^@of*ltsQueaC(5WD7+}!vy1jVxkfvgz|@JLeI#%s{1*3%2x!Ag70`7^~-q3 z)A^;)M@}Y7)il;(CgrB(Eh9oIkf0OTBRzg)XxyUAOVS1s#kkRy#)Y@oh?Z$qKax6l zs`sSX>*w85M}9Yc@>~^QQ6c`t^VG3m@NEyNy|;-R)GpulXtE56VV+(kt+4X;yC^!3 z?Kop+ITx}eV|%8RZhIjSJH+zrAuj!NKN}Im%d+Mq@D&O6P6Sa{`*pt=b+3&s2f_?? z(v>9@nUE2WyC%o3wB*Qe&Y;-IxMtnDgU$uSfHA4CW6YW^iG9YR3purT0=O6ROW*GR z_3of)Y6%AlY35HrOI*)zYQlzBrr#vAr*?Sxblrp zt`q&zwBMQS0yvJezWd+BfKc zG%Do2p~IYG{RlaZuNRg1D_}`&BpNtY94FbDZtl*VX~}GkIfEO zPitO*>PhlFndcWxpT-&wpW4i?bzM~4(%I`0=R=;Dn2+)%aWb~By!*S^MCX1 zeobag6Or6zhI1?jTpEh$MA@ukP?_E!=LT0sf1o`&Z7TBo{%myBc0J>bt4s08HCcZO zM=}nEy@}CntZBc$r|>R?%Tu#WHtw^MjQ@y&YrTJ9f-ZNO(y${^qoDqXLA{i)Lokmr zh)2Hd#IiPQrN+>2eagnq`~pPi{qT=j-`{_ECOkbGKqr)(U7yDIbV1KdG~s!j=R3vI znVNLJ4D20jBn-PmTV_Ss3hqchol_s`9O3CZgw-%kc)JKmpb0INYT-;08yPZ1jhc7b~3Jj%!sd1 zDI77`F9ZOhZ!?-LopBft>7BG<@x$D;3!N{XffaK4BSWNVF1m5L0I^2a{?Xa)>{PXa z37aoqhjvdF2={5@(=@<4(RivyHzem#XN1l0a)4M>C4^tD*6WIdoXh;(e#jxtEbUja`H3cYGqt^X%l)QyiPS+Z!7&&a0EK+2JlsWE0v{a$k$cZtf@* zGMk%Ndo;oWMOayl5@MrM29w|m(R$^%8&Dw7Kd^d|GrXxQ6Zn~TFg z!MEa}oAhKdQpTZP4F(+@gm2JXx5mFa;ORHO=U^zB$BRgg)IA>P0R{ZsoB1zztw+R> z8hl1Sq`ol-`3iPxYhN!@6&Of}VLUY~JVQhxQd9N`)Q>d1>B_Yh}lT2l|T7$^_$kF_7c0uXD8HC$_vI zDSQiuNRNtoLq1^(-B{|Mwj3?yym))5x%rt?`KBoB??nOyBp?Z6ug0Jd;P$SVetIBI zqLFG0?Udq%;FsGA{B2^C`O3SOC_5YZtx4Kkd>u+%0ZTG z=2wPdR?)CuRb9HK8@`P(-5kpczN=*X`t|Erbsj?Vm^KM6b}xPaAqgqBHI^^f6qrG< zJ~>3HAsHSkGvGttL0H1#W$YNT(}i~iVPh+Gn_SwhAS~JPq1WHG_d>DQd9M)`TpPbX zOne%^2JjeJg!P8aw2GIQRNsMm)WZVK9n_o`3>>_iaR>T(s}C_Mo*W(`En&B}DlpkD z1>>rW@TPKRLNc{WbjsdSDD53n$u!g4JMZ={PH(P605NI>1`V?2aj&KPfqN^dm5A6tC=NPQO>}KFg#Z?v(D99$T>}ePtg(J=76-q`m^^u2=_XP@U1p3m1r_x(?rZI_t+Mg zSe^GZorC$<=-gMACyxrxVr47dL%Kg4u9%=B)VL!%;QF#5&vW&AwLfxmexzcBj4Q2t><*#Uk=PSkHFCkqZWKrq zh?23M`P)uq;4!f*iIxmS1s*@@Z**gs@LqOPupj%Og>OOXmz6efHM+nq_VlEH;y;N& zN*BrJbmwCRl~eQ!Z))`jIGAN-RSyXcjG<=Evb?;R8|d(A+U)~cs)y|z;VvFagL`RFe?QHss|5WY=p8k)L(_r% zud1Fey0f>%`eL?COWJd0RNb^j1Meh|NjpHsVRx8s)JAGB&0)V@P48SPdFNEDa^Tmt8Zk zTLhe|0uv6oFmiW2DN+wqgmA0*Z$Fw^2g7EC$_J6+oq>QF5aH0g2yG~~O+_Jk;DBo8 zqM{P@fC?>o@Z2-b_i{K>$JpuB7?Bx%KcSyLTpc6xQ7_sXIixuhDs658mE#cU1&$2U zC=eA*Ubd`?1xzNe0-2o}lMcu(73K_zAAfR0GvaiJBoDuHz*}CfbxtBp=L$t*C!iiM zi4m2#z%do@%ZtTR0C7^cPIOhkm&_4j_8TlNZ2+P>WtKyxe|wlOV3zUZzasvTSJ?V&5@D~RSg2}?#XzD>qR8##Hv6+C*{_*)ET9rc(95`a|gAUxRe4n{>4rwx0r@qr4Mx-N+2w+6}^@rLCg_sNaJ=~q6T>*z-x(`$%ZlV#D$U>Lo%%86%n>KSKD0ANSRa zbO-7pZVP}Vn#F>%PjC}?n3_DNgIEhXUB!j=JSIWv zgNF%;8UelrjLUXZ-KQ;w1uD7OQwDi&PlxFMZpf!6?%|;RfOr)Ny6_xkMHAy75N^}JC4X#QmYh`MiMwrG)*q504JF%N9$Ejz=IW%c2cBE&FBA_=KR~wcbw?J z2PjRU^?C0wKGKAx*fBIOV>01xu9L^(w;GetvA~2uqYR=etEuXW=em?|OhC+&}Cf7He@#x16z%Jn~gp6dtl(8Gd6^+SKas(~$BwmXZomQ)CrEu&&qK^OeQ_BP0JXJV?=s#Kj^EXt;wv#I*ft;&*ttWy^ zZ%!<3UC^fD-O0muWi%Q2`Z^1|?PZa`h(LW8kY^6V8B)C$ufC%I0Syz>we{pn&3{fP z{=xMG0t@T+M1EefvLXmoeU@aH?T;qEAdioLMAl_q@|l#9CA=THPyNxe+$sQ@5|~y( zJcm@_Y_7;u4U}MRi;h-Yg*>?$ul41@bp^dqe`{(QYN=in0Fp&-&&_NI&^!JzXtv&; zpU^cXCtP~5fv!Voq4@fB)efWf8_rCy9vm5{p!Wayvcwbwc0WM5=LKQU&i|3^MTO~jL3`WiroB7P*%e)iR^R!idnahM6Oi#v#sb2R~~xv47dIFdSlI@1VZ&+ldSo- z>%E8S6D?~AFyWxrXQ^j7Z9-Oj{qU=HzPZiVg=Q4@V z;lJeOiNPD}*3$n^$@uR-M@JBnUO(M$jy~LPkxViiofFWy+-|(9_8>RWz%w`-TKiT7 zmCk^m!l;=bYr&|d-l&Wn0j3b7b2O$)EhWq!*qqGS?+}lRh9eM>U=JeRJy-v3{(;A)(zFZS_5 z(B?Lu`CnaMi4kQ2uuY&u|K>v#>zjxl?L$7B#`+ftoxSC4nX#Ht1-w6<3QB!3HR$Xx zP)50Zuh^tkKK1xOUq|tWjy1XF9DeB4bUu=?LF_=^E;=q9)#A3}wK&~nQ0@u@$JGu_ z?Uo9mGhr%LiNbR^-0kvs^B7EzFZW*Y;aPcBlZn3Zw2mq7D{D!YXl?=mM4N!vf^GNy z_lNh_)&YQOe6{6E$a}RM_~~J@pCS6G>!rjJ^4CeUz%a|}bid0@qe4XzawDIm!QtTo zz=h5&5+k)GN9<{pn8bHh`1ikDhDsg#8@US!z9A0{)3OdTYgU!KmHI@^*>mdEfOgVv zKNQB97+PK=w|L~$xe%ybM%nJW=vyEsyl{TN^X9~BZ0Zy9{V$CW=X$&pW;IDtIZgi& zdG;G1Vc*+GO97vsJixf_9rNrc5zPA_EeUU(jU6MIB|dKj1bS|OC~eKuGS7Q&m$51= zfYd(cjD(Mew=}|cFoVSYE=Opm;;qwZm8l)f+|p8C(V9kEF?Px!DFIJD@P&m%UO`>e z*$dTWB}NOSCFR036SIHyu>U%oW*I7vX~yNU6VB{TUfYQjnTfMS`9GgBR3#)ZBI#{< zcAt#Y@$2J6xrB>*7YvP`j#@ERFN){Umlekcal#IWJHzp;T69Q(wAF z&64O%)d6x(3Xpxe7Xy%qrcefo)2$~zDDue!Uef|)14jzmh_YgyS{E-b0T5w6xX-)k z3IdkDO-B$ibj)25t$8{FP$XIjw`)-rPIqTNL_YDfLR}*B$)tA4XI$qj{wGVh5DP4Y$TxXQ$f=3Ce9ZN_49#xpilvD#Rl0 zQF5Ou&7$9NV8BrV1E^|hzS_`hFNmhweokj*&#epXXA?|)zvmfKb-G}j5i0plkP(&u z2eE#yJOg|jASXbVX(^93SnV-Ul@Wy(5(f=;} ze#4?A9SF`S3kV3XK6olu0=AkaX1xwpdV=kSq6Gzu)b#aJR2Jp|gCX6#KE2P*^q&3Y zFxyVtEKBwD=}(tDr6o6vqNy`tmhisnb^%D*PYj6CQtexU0*ika{J_nTvxP#uE(9Of ztR*uP9)2N}uW-#h8LhL=AA4A!HhTpPuh;!t!M!=}6PkV$-Ufq0EA!xEB~N1F6nUb> zXxU5a68q_+3$l}=H6{amf7|fK{~dVSc_47L?b-l60iX=>f>roq=g}_Ey^))>uSN%E zTv6iYAbL&R9UdOSY^tiNV8J}sL2$I;JgR3ax<9$ogVCtjZf$LCU90GYZ%hr2fHKyW zk=rMBV-?`4QZhAhAVs-#1qwkmup%llYNZ=ax!df%t1tK`Uei@`L<>NE!sBsXDAJLA9 z8I}VMt5`bl6JWJWoa%ly_nGBDwjC3*+0D)Epjfe81-(#N{#n;(xB8;p3(HDGjV%z;-~n8BWNOPa~$ai19dE z6!SCsbK__gW20N9pLa+1O7knP@7j-KZog~KDeoP|06tUp5D~a{c)rVvoxmvpcfa4q zDU6t;zmUo)Srdo^@APA1iRvow|NV;=A^zcU%vV`|ri?Qji&N4<7R}klkbbrlas9zX z3KS&b9U9q(NyFF|>#AAJ2XXOv))SrE8B{O$Zm)B=&}ILsH`-zRvh7Z$1ug>`G**s| zfF?Yq$Zb86*`V`o_Lu+pKHxC}AK~mfGI zt*kjhft^1+<@WkTawyF|87S>>d$)>J!}ISJo(tr{o)rndTl1IQPw|Z!ulAa34$V*W zddtTY6%|Ev2cbs8kKJ-P7#RB0Tdk9qu+w{pA%1gq;A=lOpp|5!hto?kS4RH(qi=nT z1!e)fe1o}ZFLUB8##>HtCj5^BxiV<<2?DRHhYEz3BX#L!5(l-s7i`9B{L+~ohCTf| zhVqMN=I8fHunpm$1qKB{yn(m(#=TldLXGsw*Yuk}0W8=ESbS3?Gy)cM$lW7bVY+Jc z@Lq(Lkn9^O_~hhdUCc?5ZUiON=(Ii6dn#O=h8|8L2WS7c;WaT;%UUhoS7gWo&1bv?Eb)Jzy4)*h(#RiQ zBtk;MpZ8q4FtXS0&icu1f4jTG4CC!LH#fO`PVEK3P#`DwknjD&=Gzrg)m;F}W)Kcj zc&1D0YiO(pS5q_iENLUeK{`N55h(MU>7Tp!FFqP%5sXGr3+Ba%ea~W;>@JTyihXX{ z#`rOSO%5_Hdsf=V0!B0^{S)Sd7p@pKI}%Fa*ni)26BJNc&qy%4&w%P6YQbk;WCj=? znK3wa;LQy!1flO2-ZwyiGV4_`ozE-Q90Y85d-L(r<}jht;0-MloZ9Sb=>ZF^SKme#2XEGpJWn~Wmw8|Q(1+kw|D|Gw@dqlxk9Czdj8GD{w3H=;#b ztKg@`!ZIqFne!yHz|eA%FuLlj);bya!LR>a`mfhA<{HLP`SS(oUd>OH<6csydr`!~ z%*^(5E=m^{p(dny0Q9(mq3LKWo#MR$7;y@0LLvX7avb!^oJ~CnWj}9C;rnf*`fPw z+W&LAND%2=iz?N7$lky^j)8FBSOoFYZ_y5W+J71!uDjPo8v!xbO(2+=3&c0y3%YHY z8%v=5Hi0;y1(5YoH`8l=N=WFbUk6g$$g? zW8C(?u2blLeN9IW1PLdf_CC&=%6+DlRZ+o#!Lv+l0rON|x&ev`7Cp{-kD$dRB}J{R zqB2ZAqadwO7=2GndD#4Y($xN6WQKqO7*w%NZMEO2)AfPyf;X4aS4;*AUE^m4GLI(j zc*%JK>LSQxly@GFqvSrfu#*%oDT>uiCU+fpC7(GG;W%Hk5rq<7Rh6rQJXMx;$jlHo zIZhk7{?DZkl!I1CN~h~VJ8;MYGemsG?gd8y<2d_eSE#gVT4p792J;+XQ>g&R`6T;Y z{2{zAIf!0=>m#e=sWp5LD2Sf=dSEfp!qU>aI4(gzw%v~FmrIFwE6nC1 zUkQs`_mc+q|C+*UjG#dY_*yr=X~xf2&&F%qYgJ5xbURCLFtg^CbDqTXic&bcabH(( ztx(I;L!2r~<82q;2$W((E|z6JIQZ0BHs@olXDWJMg3>`EnOkFf^V~KVK``%rMpeCt zVC!<@pyhWFqR~YnYV=I|Y(WK?P$i+)h3fAJDb5d3T0rmXdNT>=*!S%#D7T}yxMEYI zx}Zib+qb?L>;-WQs`tw#F;f((cWgbMY0ASxzU%n=t0|L}8P)dz5+QEEYkBi<(2Nwf z4)V^@VnghUUqEAhx=rfq(lY+rOBtvWBz{BqS>qSaCk?Fbc`Tb;r^3CP`l7uqkD^`W zr=Z;+#hDevB=x^|lD%RR^4?*JMx$3O^LDjI3I{V*xx}k!;JhBgU_=83EnzCMnJCXq z`1DanQ#n3e(k8@#5db*xlV4hBo(AFn(Yli17g$h>AGIS~jq%4N3*70DRbL23E4u7R z)=;#|@Js@gb$S)1m%=uM4 z01;h*#Qz4=pW_Qm@)StDGu7W#-wgRAkWJvSdM9UYuYh^M|~Wt6zQeE3N>fu{TJ z%!H|eN@B!)9wau1bg=wy{R0&$Bq4f1QTuF>m=0O}W<^@-^{9Z>o3k9eZ$nWjyc0C3 zw}$xYJ>sdqAH#WXv2DbdawiT?d$G|65TAsIcbzS2?D~Jsxr`RgT-p+gck8-I`+AiT zFW@ZJKXZoAzcH$qJo#>fgtTyFrYJX|pTW5LRA5A|zfC?seBl>vkvz*}q0g0DQ7!&m zYY-PSoUcXG%0?K4>+kyUIs_q3@U9i-wTsIpK=Ja!aTzp#gUW4`+(#P6?KToFQN$X$rM(dF9{v!{?G& z#>@|^?5ERp;)Q9=yDV=lpruqJHEXKH&3^=ojSpKBemCkaT=wjThoJ4NCw#-UUJR@{ zo%zglMphWd`?S%GaN*nCAyeK%OwE&Qr&2$-qwTe0h?f5hZ#tvP{!QL1vZN$ffW4;Y zN#ywXQFN!~5j(bwEqJ+KP4s9Tj|ii}vN6#9sZ7WM>A!!$Jd7_Q6?9$_7u?K^yH`Ok zINeS6x=?-Ks_$d44^lrj&n9KUim34M@yAC-mTa_>+P9yieM!J4By5^|se{19 zgzGt2C?$6Gwtc_>i~Ptu+5+NKRZlg0pQ!)W@Y8;WRp`5pUfg_TY2Z2De8Gk5IaLK3r~q*?*k{v&XTS$XpS`br`TL_E~Ma{G27xw*HjM zwTd12txUx+-##eTV=~wxSKGTmQU6KR!7*K$&P*0-Vp57|2L8uNOJrT5pKy1e^Y8CQ zB}+xLO8jIv;W`UmAq0TI^pRvH^C7)5fEG3aOk{BPQMP+=_|$?&U^FF#u_pRiVrew! zG{nB7CU(O&&Qed}u zW@V-N9gMl!^7{OM+jT?jIbImRpaRgK92YN7Hv-ExffvNmGiIw3oWTBmh53?*=M17S zF`%-sW+G|@UT;zux5I)|ty6QZ57Li~U#W*G!Xc1fzj(QW862=Os0Y5(?aT8qHrG%3 z-w0UjE{GYvWEMno{Dr4JTZ2ahfAIP>%~lU009Y#4<%W4h-`c+JsJ+5XoNk{Q9LPvd}Ghbju?8;(fe;(q$eiZW>GRx4p_ zOk=nNG#w&g;|ZN0jD78AANT<^B4!%xYCdfTsLO1=>mnZc;3W0{bYXII=!y0KAdyDi z9(}%(8ZQFAv)C`!5K3|4{`}FFu*^N5Lx%uUuE&r3JOCQXXUp#tO9Ka;5Ks;9#Z$kx z;S;5fK*FI61Qw3*jUM|de$B_Ue!IV($z-}c&Lm-ngaGO1IUXVawTkWn8esfxP}1rQ zOs5@2O9b(heX0rzXdj5>hmB87DK6&K&pI}BbH)O&CKf9&l(H=+_JH9dz1(4_mY7J` z%))DSi_>3&u&V2l2`$ZE_;<2ex&=StYWD-R+UzU2TTg30>9YpvmXx8}p&(_NAGVhy zBobMM-MxQkLm)$?jWr^5rvpSJTVp)ROygT6b(tD>MN#3N6HMk0liAicUTWQqhP|Xn zN=0|D&wRhlUQid!YQjD+m3a|wcL*nE^pcsYy`D>Qu}P@xy!4F z07{sN^fJD#p#-MbHOI~M(Xi6e36lBbcUTP$6n=0JtH_q+b+oognU&l=t3Z-b^8G)- zNlGH~aEO#zE~n$fuIP7jrPiB$vi_Y}c7?rfh3{tzG=F|>la;w1eIr@ z<$`^`&@Pz^-siu{@sTZT4))GdNDP=Ls+SwJ#21UxA`_$14i^_Qb*f05mlYK~?zTzE zzgy-P5lI)H>85P$Sm9$}2wS4#KE}+@eV$FT39#ogNfkC7QB7EDK1{bv2M9FL85f)>}oBU=XHyO*g>%ojgz^-6eYd+1N8aLSuBp{+wCd!itMy8)LF)pTwAH1iOOTe$lY~czf+|UxuAYB z*UivQ(=07Umw<7|Sy{j9mfw8ipud$i6#XOZg==#tgE%~cHFFS4|byG3w#?m4Ke#k;I!^P-rdF{~<@KXFXx;0vxs0RMgS zr_TzZQulsBHP?x2%iW=(97;H0y~#gh0Bv@(^W=%512BBB7=b?)0_Tj~OGx?DyT+e= zP$Q3XuLM#w>}EWAfQ4F`Zka-yOzY4L?y(8^AcW@47jF)Fj=>5eU3iZ@Jviy^XG{0X zsetA6Ig!ev8>{aR6JLLm;NQwEN;#yAEdj>WaURG&}7lei^k!Y|^)lhia2Qd~)gsl35 zu!M2=FD~2jEtebq2FFNlcHuos@;Cf{GMsVp_D8f+LXqM~14|LF)P}+tG`yqnrmP~Ki_naW!P3B1(|yZa z=UQn<0S+~n3UW_b1!h|XCW8$M|5LUs5d@lipG;yRX)~5cIGdLMRzdknXwM9O)qF@; zphfb!&~jaUf%ol)?c~RWBA27e3o1R5SvHppEq+WUVR-(ZKXlS3HI0IK=}Jr5&$_VX zLj&l)#gD78*9uzwelq6#oC?;G+;lu~x8X0uZ`aA}{WCu@H^Kg~f?|QVBU@g0YZZ?- z?WdQCTQA|IhHakM9k?yew-$ZlwnEm|z?7I<_CVmT_+3VTSl&`afQMJ3&w|^sGB+8% z@a%Ng472%^Q?ZWhLUe~#OG6{po}+u-sQh6wK+8V(q)#LGV=v_?-L4GG{MWHH479jq z=t8>bT4Dzr6Rdu!_{!Rtb9p`=wdfeHr%vK4CF?6Tjva(T(!(;K21UB~_qYfR=I?xng75B5XJI__h&IJ0ECU3n=OSd~5M_+7+S?{-*|psfg|1=_i>d@sJ} zMQqe^fIdIFVc3?{@2t8w`C+rpM}-AfFunAejbil(5Klb+R{!SW@Ld`}WwBq9{qHd4 zc~HQ6=O<4}cV-leZhn@ldG|SVnM&-GdGFj-RE)%>naQf{srQ9F*z61J!5O!;rZ= z``d;CcGAnx``f z&uy(v<-KK{DFKxhtTl%V^lcWcO zPepXwV%GIqLE#;Z{&%;cpE|ex_Xp1d(XWsvw|PIRpjddk;!-p(sj9i;zfhDi`6a)q zgq=6<`ilF7XRN);uUdbUU^3*_N6&^jF>3|wG6K6?=RfgbGtJe&2_aWK`eAyfoc6Bom(|CQ;lROFhabFI>rsYtVIKQ$s0Ji z_Pjp>zEb13#stZop0u)kaQ?(qvP z3>L==30#eC^|_yM=@=l$pNy30a-!&M?cWuOPuzH@onp&sQZWnt-=lQ^(9za7cUi4O z68FiUI+c|CDtl{-yt5DPXlNHLKVKu&h(F`!Io4M&*mP7@d3F5uuOwKo=G;mvai0Xo z)cy+1|69M43xi1b#EnNXw`En?XclgSHpzil92>qW$E@oq{(UGvJJssr`$E!o6|JO^ zqVi}2&62Wkm>oJ|2P+e1vu#y1xgiPkWxe-am)6w~X%NvQ(Z|~RF7xR6r?+Q5)#2Lb zr{Ps0eCj0-oC>8ePipN4Xkm1mkgp(v4dP@)qeBg8)+$s>(H4pmVUdkc(Mtey6Y0(}p!$^Y~ry@O-&lWmgRis!0L$aQ{vl z-^;L4sklt|YnC|iFG85jz};T0$Kfu&@oX`%=u;?9>3OWe-9AhIT#Hz3vin=Q=;n^t zGL093*`_x90BrdFUVPQ5ZQ}zj_wt}Vz4lY5to&NLLX-Zs6p=jMaKFMgGd<(ygP*Ec z>NGSgntq?WU~ z2>(;#>0`lD#=i$IbvM6mcC;FgofW)w*hLBCV;@cpDL;^9;9)VrY(8-616|Fa3k~SR z1#%x2!~lR|iq;Pm7g&wZ;vOxjwYAzG8@k4R-a15EBg|a zGFg^SFVU`OXP^Y;$5}N&ZFQHyx@JYNv^0f5J9do=bX{zLW9M=B$$HJ7->bmG;-}k> z>|a@jRF9l;St7)lsIc(PtpAivY0j6h_m-;cx*QcQSLelx9cWZqo3Jy9oM}~VynONF zjXUmm+Z{&2o5l##@}`g~Op#7Uca_hXunnWMKIP2f=k?NhL~}KPEpVC4PQ?3WfdsqV zRD?Y*E2~4%t#p6}3iuqa4SdG=*~USpK=zc6oxp@Rxw-Ky-wyuxp@f6CVwXP!HDU<{ z(7xmw=AOK(sDLB(s@BxC)&TEi>CWh(?_VrDVVzMmHQY^LPAVd(Kb;u0d$htB*@=s? zBd3)%_w20%n@)n)g&LBjb-kw#o5Z{``NeY7BJ18a%&P!x>SGk6@jiBPYe|!Y!rgbR z++YA(zGX0Jz(9YEgHJP%P!WSQg>;$th%m06;C*E}^oB8G|0`Py)i1UNZ9-L4L4YAmDv*D;pq7Q&@Ne{P z8RjX%9=UJB5*`*nnTGDTeO~{xb<~R$3<&I`<~JoNBL!67pq**-_xC4B*`MK-Dh?Ry%|PPuhTZ!Ay-!oXS|xz^+QV;3xns zl!1pQ89IgVbp3ahM?)E2YsB|s()&)>y9tr~5md-yPwAf77By;9kS+sgD&d7;KM`VZ zM@20WC3a&ZF*s3V)4*cEl^`}Zx9O)B?xPME=v@b3CM+8*ddk+t~@8}5vjENQ3S+`X0uh!ca&{i*bS^*;_y}beu3^n%`T29&mkx<5fc8wix zU-c1i(v36n+2y%cq2C0}N*=IF+?b-QU}a*;cBr3EtXuT+xw#A^ui75@!r7psu2X1| zeG-6#0Rx8+cyIYuW^8PYWVi~5UQ@GWU_V0e7Kpo)_(cL*@@O!J&>=XC&0Xu?h!k8M zk{9V73*qm;6bTdjYj^BrR{VhQt$P=HHB7_d4d#0LU(W!}oA>^GTk~K5(Q`vXB_bG* zif##P6FHmv$@%;N)HdEKQRH-5QgB2vbvTtzwas`m^t=QQ_mj29oMc09=p*818^c*$ z>>a*BRzqZ$;2xx2F&)_zX;RvUH7owc9M!t>&ia z=;*u`gRKbn>f=<*! zIP?r@&!^eUBrQNpn3SqvU2?I+`DP{KK!0R7bGYXw_zYS2QvmzkljDyKQE(RC3?!l) zqO-VpAHMNZz!n?@>tB*@kE<}iTT#K0wv^G=vUABu-s%nh@a_F_D6vm- zP9}Rt5}Nz+MX<*-BGQB0-!A=5Y0ukyp%BELXa27oEL|6qXBKX$5>zNusluLrJpLl2 zG3!H-6EF}`?k)o7eW==z7eqUCzUra#!(Ii6J=6jaQqc>d3p(Zfv9(fZUXYEq-tkVr z*xOGA2+X8Cl$6np*%+>$NFR^7OW_93RxvYZ&Nn&Dd?$G7a1l`ggYN!V@XfP%YtY$- zP8z;6m>hyFrvr5s)1G%Boe(}rszD8$6CT@wB0ipJ*`x8Q|HY{8*W>m{U1r!Q9*`gb zUM*(?b$0lZ?#`<{*i4s=1CKFovwm!WDa!PJ>vdKzxvF=oshQdZhB@t|cXqNoR?EEr zd~cK({@e@}uSVN;eTvc_HcyUp&q3FwTJV05SJXhD-Q8wm@2Og?gQ$ugb{5g6^dEkv zO1B1NfDn2ZR>t@XrPp>-hAxrxaw>@q96R{I=*mj8U#lczim3ZVmVCN^DiT~H(7-3pqEithWG1?Q*IAH8Ts{oSp?`;-lEmm?cnubQ2-Uw(NrY|Unm@ep z#{d29T}>$Knz2Ob)R5pdi*gK4r>nFOHj>)F<=~%X!x)L2qBB+ux8{xP58M)Z?)GAP z1`_Tr;|#FQpqe>g)%>J%NHLLR6v}Ckq7%x{34J{4CPrcz%#SUFq|8D!0)0Lf%NhjD zzPVbwn{*~8$!@gj2^f^anKI_|9H#)=A<8k8EEAk~{>QJZnaibP$Z7%lVjVR8a+bbJ zR3c^Uese|{tBf_7;ayDB09QX=E|sDMaX3^!K?u+&xg{nmwRtnN3rU?{GnuM~*CGYsS2wAnm^7<55UAsRmRSx_O*XmCw5FW92Tb{D32@E zy7s0RDFTVTX@^U>Sy(!rwD{>vC6)c=DvgAr2L4YnHn|0oURg`u6|zBQ&$4gacpjHh}mt+H4SH+(b^Vm4=x1t|p%^%MrfsvGTB z$yBkS#R^HQ0KC0tXIE=8JXP^jZ3J{fo=WE-^_G$S6u0d1YG%*HW?-ntQ)$BX?DO-s z3$oSLd&fLe&d_<&oP0tl>s`R^pX`2yX26SxJ7*lu<*pUE9$5RhL2R$>&JW$p>}hMU zO9WrqLbc4-;>CL*gRUmdJqhiLD(494gLw(8H`>JlY1nXR;MS+lQz41;zA4=BrDwEp(sy%q;o9kUd@T0o4V7cD720aWje(U4ppn3hr{GDE# z!~rcjd&4%JS;2!xQ*}hV%N|`FK85e&LUSltlDQt|qmLMP|1mGK9DO$GNLqWj-!@^p z;G@WVb%w8g|#azII@u8SN*+>+{EiDPblrmD45 zrZcE%RQ+U2dbX&YB9kx$TjVZm)8SgiD|>U9p#R6$!+K&^IBViv>(|Q=tJ*DORa@<7 zk<5fe4&_xsjCr(@z~E#&oThsQq)pF40}d(N)BMYzR3{Y}?NvqmWg z1aaAX_@{gCO|*bz;xr7#@HWE?)A51g7EPw}36*0T^b_BLr{viWCi+fl;q@M_7fNbl z=zMdTK000HsoL~FlxnPH12rfs2mcXV>49o9Fw#6s3vpUPV9$gHp$?b9Pp!0LPJs^I zq{vFUWQ;-Jl(A<9J}u$APy)X1N*7-Xu#;*e%m#iF`rmQtx%K29zte79{{?0uBUR*Z zJWpU_5kr+A`&;4$Ib8n?W>J6+4cBuzN_A|MKa0oDnwCM>c{`=-nTLa$$@}^?0dL{U zJ#lTm`l#TupRG_^!XtP#wA{GCXxU+^ICBuHE_p`skiDdcQSaf2|J|9tw?B$%9NcJK zYe`!rCctgj2Ohef*M5b+N9lQp+pT6$ynKLj!zeuWYjf2+q}^@-xbGG%irw1-{3)Yb zN^k@1^i+#M*V8ic4i{8k z&R$FFpHg@xY;vxb)6^-xo}#c7CTSl>MCM9Qxwgsf725d^PTc9ED@$I)t2CY%JvYS4 z75zB;snX4L8x-k-%Q{2I2U^KWYsyB+{IrlJ^>5A3(YW;Gm_(JJ(4`lkVm_M?nFY%z zi|&iT^CEC10U$m?E}|KL*nDHEi)uJq-hBApf%MrcDdXT*h`nDNdquJYBh7@5D&qQc z<>Txp@Xr_)LMm{(SnH*{ow<(dkJ%S^?nRYb_B)Pj5~jcR7GY$H|;q>rLU>KoCkWvFLYcwn(6zJ zBP;%1L@D(K)*8*>3iRoZ>j>GyE519554xzTPney}4o;@EF3*pP42hDb?hb`@_hWaXqGEe65&R>-t+Y zo;C9#Yw>Mdbm6kk1#bkv%xrpIzhEK-(M=8t-6CMmzTd9u-KTCL0HcS5_Y^4%dwKqv zDGj<+5B3y$j*O`1jCJS$)6Q1t4B7T>s9^jwkf8|PzZvhpVBSwIlK47`T;gk&jQU=9 zf!adDL#^}j?IO=1$j4J$rsh2zh5auJzQ6^9Ppf)KzQoeG5ekU8o-V4w&uuMM-!Qt$TDSC`j@zO3ggM|Od;RARR~ zP6LbRN+Eg>L6&wmijm=+-Dvc`D?;jLA3j%N3S-}&P#J)`ieAE`f`q}z0eAVW_S#IL%Nq>%-x}*OWd=Ic0*pO!g`3nR(Q2Thm5o zU>4$B!o(u>En}LO_%a<;?lYCcrSv-&+fTD^sGcGnB|o)l)GsutLkfo3)7>oI-AHN{Qx|hT5ySp%@aj987z#lKj54^SueFTa2$okRASS@tBFUQvt3PB4g7Z$cN6{ z0zjucchsqM(>~Q`*-N_^Gf*X5ktr>?y(CzW4I5Md8LisQ@Ra4^Hxd4o91^9Fx|Ry0 zsYNg{ms2S>PFQYmKky6jAZC~HRG$csAKm+;_b2lX<+-uvZgj%aj+2NGL&?xbmC{n^tVFrTpqR&6=-c(PgH60On||vd z81a8-I_r2k|No5-#~e&gO&!hA#+aIJrn_UhX6#_jd^KU&J87hLw#q)T~17Ta7<|ZxolVcaJauAuJ||7~$8R*F}J&tG4uzmE-Q3H7A#2 znrfE8tWX15T1+Cd;a;PSE)_@mDGat-+s*wm7#x`QfXMl zsM4!C9e|*0PHz)c_3sY>KBZ9l_!!y({M*vs7CvUDUJ?uo%k=#8@w$7g9Cvtz?Ay0v zrnzWWR2=BS4kB9kFhYy{^~N3AVJ&1sxV@2to~QD=)JpXXSiNZd zYwy_v)coSj;@;20qu;8X_d{jXVJmD2EeggM^AM`SZDN%lTsUnU|Sn$ZFZmlMdn~Y zk%=J5jsId6OZS?aTh@MC)~szmWiY@Gl$Wn5HoyZrHF+xTHviC-Qv`@S3CY9{TYQ7k z5Ux-*UGsy6xM%U}1dPg;7M++lww(Zjmr%N3^nDoDyHZTc!^K^n6)xBGnps3isZeg~JNw z<9_gy*)}`k{t@`y1sSN$XnMay!zqvK z;b)RURTR43r%}ODY?C2Pu<zL8e&ii|ZkP3rvp$siHM(xUbs}*nxyFy{)*oeoDKRtmlm| z=fJ3BiEYj~Qt)H>cp&-Xj;&dMLm=?^lE7D>IxV~^PjTZ;n?XlWML3!G-(a;|e>Kkr zy|C9Fm)f-5+Ls`}%nYPo{xh0EX7H-d4g1)i-+u3mgMncyv&%ouoXp$S1k80aD)lnR zS!f3y!k+DHA&3?#(~Xj2G45Y{Pfm6Nycx;X43(7;y}WEw+_+*jK?U;Fg*-x2{y+UH zUPmIKGWa{kn|PVSQsdA!W6+HF^U-AqA+^%9eWu}x_ATgwfLCl)h!l;Z79uNjv+UX$ z3?!<7`%O3ON)Me*k#hXZ`_U^uqU_%-S?y!1|5UEdd?O5)v#U zQI7oHV9xM!Cb!3Umhe>tv%gI;yW!=V9uO%5BUkR92??Aak1=GC$wm(?qF?5{-Zv|Z z^uea(>IP`-UN;P%N+2?w3iLvxPv|?faFus4Lt?mmUIBgUXavWP4>xn|zG57M4JjmF zWgvvHV1mV`+MvITY#R8@Ku&h$b%QPDCq;L-K|LS|AL=t~33UH@{rt=P^~)P(M9T1Q zf}eOsv@RX~zxFgZI|qc#800p;2j2j-xcB>W{Ebj)VH0A8HlPhq0($ee8ta&J%$#vk z;)KOui~A(-ROtDaQ5wwG#b=x37KAaC7RV9nxi22+@5d(kuse5v@(Jgo29^PNlEF`ql{C~jGz9Dz=;iekH0!1!IBQ-tRp$|B*erfetiL7~@_FZD zgewd)=*A%NSrrgp+W!SIHw6#MHD_l7{YmZbar)$f+U0!}0t1dn9hMw%&1+`2F1zr6(QKS!PtsZ=#6 zhoI#;LTgEm23$3_@vz4!U46sPzD5XnOe^v2V=9ea!(sXOh|!!Xcm-4}8?~yS(;E<> z?HEqrA%Rwuh?=tgci?ORH|Po=-$1XDx;K!F+&S_%Q1PBS_&wlBYFrkRXm2bKI835| z5ql@*v2LuSd)#=HC^1grC(}C6vlq!jQpAP@be6f55U&l=P@flb4yvlLmCxspD^dQm zKVC20lgPXwk~E7U6>HXxBtBgZ@`)jkKpp>5k5x%u4)>C<49-Ovf6Ka94TZRvvRV-( z?wYu^S_luM^ekr^d>o>m>QjM?4>u;o1>O0ht;#w3mWj-6kcQQao^JG9oxK2O;Idlu z){)FH^h?%y9RQZ3xpwD(3_C>h(1BkK*J``Q92F4}=^H(9%8hzP5AMN!ti%D1pA(mC9z zjm5q?pC`mzri$~*FzD*bPO24a2Z}l3Rk&gV7yD{yEcTZ3GGiy^941kGuF#CQas-yM zaQAPD&&#SGK86Q(F_3z^GACQ^fi>}l+%&J9DStf^IAA^uIAwj^pD4X%UZ@7UenvKv zg3G_PAR-%6elu^5)BTxD;$43eN5`Mq-}^cZ7q|%dt%Y!k3>^}raa>?_StmrbzH$S_ zr5NLHEN~a(=md@zDUp>N)I3Oh45T7bY>7L6|JUnC2UzuMxFO3a+pIsde?1 zCJAAnDKuf6|K5MOT&-OC2C&v92;Y(n6ImF-*xz)d|3sWqK|9>bxP5b&-lUSIj#wF# z*3<50GpcORPbXNG&}bY#i_6y)y`2RP{|wODd{#EVOME7tWAi8gO&7{a^9eKPjrZ>5 z(s1tUV6MpFEb?f{S1wGmqAbptAcnq$OGYikN@@>Jo*!ZgATBoO%DlFNWW8+dN*p)J6Ig`G>;CgeOf2GtT#t=26Tz~Z zHgU&FeSYk|K&o2OyISc_b;}q*N3K0_QNAScNa?hL#Oiba%?KCEkQoENT~zuOaYp4> zGtIZEJIH1WLuc%GxD=n%LLhcFX5tzIGm*Ch7UD`VwmdcTHlwZJ^KT4bQ9OYhV$a^d zPZ|{PQ--ndKW2IQ1o0_pkWT3te_65SiC&95wd9oQS>Q_)g=HZKG3KklO?sgr8#A-l zWy)L|CFprkYS|Df=`pHT%!7<+r;c(e-o0q+4J>64DwMxSML}iOIb3Wg*TA_%|Cr6dusOCs^8-~^o`5pw>Y=-u z_EH;nm51vsS?qfvh^&vim$=*ShB>3OQ)g!NN%F6U+>723?qaXXX+qG|mUR_5ZiadEPgMyNKch?m?6b6$kNIiHb+%}D^XE)tvmMeQCp;sj7T-g zq?V5s>UhOU)DK=3ytVY{paEpzyn4p^AMluOp+VyJ7lb$TS!3OH?F@)H!ROXl1=@#r&0J5(zc z_a!-65lS046MKUKYf9MN4?Mjj9ogdC3XKy;NtKQUZkm(_tH`DFeBkFi9r&&_Q)NF_ ze}Y-8%D1DjGq_?IxlD*uOHw}{2|+mOOEoVO;?)|?uMIS2xxoxXZBVyRsmwHphQ8{y z?yK@G53&1M5YpQZE%3o|bF=!?J8hUtRB4awGuAxQXs!gEOG~=w`v>3v&eA~|N`Y@L ze3(zv8yn##ngb}g4v&7nr#gz_D!!X(T=b7jeKdW)^>uH}cDTs6bX4E-aa*nJP{gpz zf#2RQtIFaWR&hQglPCbCY*r}iA=mLd)+mgMlsU@R?BF-X40BSiu^0(dPw*W6U_i>! zrqyvyX8E^*II9$u?K^EcCbZ?&pwwvIE~%&@z0l_B+JNFwzgm|VaL%wriazog+ong& zYcKKV$lR_Wwb=l!glY%ntJm)X5Ai&kB19G&DOpG-#wF(;J3KX(-Qo!eGZ^G>=ec)u zZ1kOk?LNi*^c(wcYV6bpof=O3p**=>hH`W-rByfTfOR&c=WV<;su{Fm$xmJ*uV>EN z)alp+O@p%8K3pl|;zrR3IanE>x$jbC&SzqH2mqeDx7W)27^QxecI7BSS#$`$7j!~o zavUEOMZVu3ls6vu{g8hn7NU$+pq2=B%6k=3TpOHlDr;p|OJJE})SaHHK{Z@0pA;#z zjyj|SJ5|?Yp&^eS>z*#3VlOW!W2r)@mG-dTiNU1Q4!9R-@xTa3lOBB`2F^RK!@ySb zDybw!E3r-9Q_g?zPZm;7s>4qtd^jQ8)B0_9e*V``Y)9fVf8&oV|NvSKzGnJo9j9GgWa_UH8O4dK@K7i{!e96sruay6L z$GFXW{^|NTwubM~qqkKV8hA9SLW*FC!t>rn>(sSRf8%LZ;U09i959V zFU%O9k6^}{5`ee6z5MErFYF+|!cEBBmmm7mZ;!h-=<2dxN*pG5AjZ4cH?vt~@ zc!+5sF@~k3ZIk}|DtbcqU*c8JX6}6+h55$GtP7)b0;<|cQNV+ZQsRMB%jD%pgH39c z$0Gw9!Tbs`k6g)3UK0IYz#9CW*6+q`<_4p9W8JixjuQE{@}yqL@{SS^%X-T#Sn8P4 zrl0Ai_hHlt`u-eVkJ5MR`_YrN+SZKy&F3&$zmOouWK^Pb=eya8LK$o{o^IriEi#Xo zT6zovJ3Ns13{pHU@}OwT*vLR=p>C4_Jq@9a${}-X7JC(WY$6pdztw9|!Go<|7=|w1 zt(ET%`u)qa1F|k(XR3b&C?|4T-=62a5c`n_L=Sbw@|kll-6}31K3Jj z5kc>(Cp<)lCc+taqyE zju3}qWC7$66nMj98ic7wyJR!B_eF%G+=D@(Hwsv|7k2V!@v_KC78hOo(PnW18AlI1#B z;vcuJ0IsfryTD*E?{u$J5m%B%PrhVoBi6AY@II$>HH{e;O4su`6NKaE7pAjayT9>p zC0)0@aI?_N_+mpU3iA`6k_g?m;xX@Na{3F%8n zS|h-sXzqAw<-x~9ipZ%p79#`f75mG~$!8Z^Q6M zn^{pK5g;&A=mP!*SXlE+R9+eLU%kSxD8DHBe4q%#7#x12^X**jY)u3d&2*?wFYb*W zmjzqN9fRu~WcuoMR6NlR*F1v_Rowkti$;r*VB=D8b^TVbJ&w2#SQ_2RP&j#Ec$1WX3| z3Yf3n*pr}hxJ)(c_(NicL+l||th%*yG=kt>Gp~aXXAn5rmriI)7PwP4{oO~{nq(Qj z#%RN`A`5A3g}|B03>;iTKt_^L-tDRKXXNOBEdf1HRB1oYxn$>3 z))h4HDC-A*1mHSEqd!{1?IeIxAlo?LX)T;IJH)erSwk3pTjIRLrCG-9i4r(qq7Nmv zY{kQU7X(z4Lr=S@5xepix-I>Mqy!KZso`f`%F?oRwzWFKGTr%kJa>x?c7+@6$WJH+ zWTe?jKiT+-6iEG4l#V1d!G?cK?`&^vNr@?dLk4k6(S#mQqR8#mjgThW|o#qvIR{AJMqXPx3IRanRctQn~Eh7s79iQ=}Ax>YD^TB zS$(cQB$8xapy6EBH`D*9uoJ?zNIPY^%yg(-5tr!fJ5nhTv81~h-3cfc?;(AusNQje zc^Pl1esFzPX^$_>LMuDN04LNQzSpW`RXm~-Yv!}SKlX_$bMqe!w>?Xsk;LfZ_zDGsOC-!&G=j0 zZ(Tv5?AnV5bUKGrqY-F-y${+GXAnjxrnn45dj8Tjku!kf)bE=hVBO zhgMiSfV>;35Rj0@*b@^{!4vVf1CNh}glmiNLjBT>;*-+f`8TD~G51ia_T*uw?@98- zMxyk)k%erG|Ki_Jv`{$+e1Xig-y|&?oEZ#>?45=*o3y|?(=!QTlYmh`QZ(m&|E&hs zaa_m)BfwUIwm#s=+o}H=9tC1GlE12ej%phWvoDfnWJsyr?)gXNzfRhC$A!eq!U@P0 z`EBxOVi<@dEslMwe|EX1@_YimucW&Kk0RYHG0TO6tbuUw>vJz_MB{Y7jRB7!bq(LUwIQ zis>L)w00;{%4HUhO`OAYo{}@*$N4)PdTNb|Mb5ACiedzXgR=R_`M`7XgwLOAC2QF) zN-v;Xaqbuj0cnkVgtz7DyG8b7CDEyqT*J;-8od(ZPJReoD9#^-WuZ5Y$&!f=Vsf_IolG(kM%J%%z|8@7|$0lE5*$_F7iA4?c=SY z>EmS#`{>t41jXrBt=(7E$;tQh4{_t@pE|Y_$|6-B@;Afeuvvv&R}D}Q?`wi#Kt0d| zzJ2Ydy&j9lGEL+Y;LL)r`bJhmhSI}Ak{_*J2zmZdl%bhVBP+*pf6TP(GbuvCe~=0a zcwzw%?tO?0byf#J4Z>2geIOCgSgl@((iUZJY^(lJ;%7S=dYg1a3o!ZEy@`A3~qen6(?Hy8+odhy)}JZPjE*OP6T8I{~zcO_6B%)3+r9FzYnmZ5{ji23pOJ zuC~dK*y&vq^N{;%!f&A$$QhIQ#M|Vr4N^Ju0>e7V*l;!vM83$Rfo_wO$pIh`cD~X8 zr6SqQp+x%g6D9rk0!i{5LgB;z9y!Wd|0$R0p3=>-(h*JaZgyirBMGj!4+e&%=9Q7W9d7@DM zi;T%?f=($q?c`=XbgNusdAyC9c$-}R=<~CTEE(!qU`uQoD>Ocfnnp0#1+)#|Qy?*% zN)H5^=hMcVf{YX+czAiGej=}`WH{D1Frlb|SBtO#EAW4y34Idtkx!{I;)FIx1z1fM zwDTxtW%g!Z7l_H*Siq8u2Ur53C-F5}(*EFk-~*O-(K$(T^r>Fn_S~e;{x_f z3mMHa;KQs$_?jE;LcKOAM1T=`1qCQ)DZl)Vk`=$fthu8!T3G=#`?}=clKw`LBI8;U z_I(hP#kmrbk~1AiESRdK+@lfBe7*A7crNhy{3pO|C3}(!XuxrKSes&BI~4?kX1q_k z(0F8EG68%KPdnTSTUU{e%IO_sR`Q&GDjuS!9~@D~$q03ocT{I=J)q+cS{H)t#1#iO zG?`@hghKAx+S=2A?^i6Gh;6E!FyP1+C|zlNGq&$mex8KM(Od%x(FuG^44rO>*mw}>@M=rUABm4A&#|y zM!7gV2Iog%>2T(|9P!y`F}`*T`TPpj>qgW3+(p=|7@O7J|MR@gi3cP&fSiqhqyLSL zbI^*lMJ*7SxCc%a7OHHo+<|UsyzlS$3BO>fw!50C*2#<9{UTVbms55}Ia{bO@t*=3 z?rDDJ7^{2JhRKcsFp8I;DU3xXb4`%G<@0nrNbL|^g*l^cHD%E`M*4Se?cv3qg_{Au z?kqQAw+Ok>$pgR&$88_q2Ll8mH|0O-u(BDtRX3~vx9C=5U(<}QkI#c10c%2T|B050 z^P}E|Wj!_P>JT~ti8$k(R2pwgJPTIwy?_UQFgLP`c+SCwSBYNCoCG~&l`#B+h&)j9 zoV$NSX+?#<&zb|kgO1B;5=7cBt`lC{C-AU^+_RSYi(dN&$RfyPS=5^b7sYLDKKyS@ ziH-+^zRq|$msGKbY>HYQZSngr@bG9| z4zOSy1Py)oL*YJenw6TnXxqi6b#_p$0b0eQm2U2TAs{;M93H7Jp>cBsT*a282-}-hE+&ji1&@;VVpe$Fimh8~^YH+q3f$gQC-brEX*r-3Rjf z!FT^6E*VIoYs8C{Gr_*fYo->7S16x#-|le2be^I)V_iHz!Exq%=*q`x{jhBKjoZQl z2d8^0)3p5wuwrxLJ_9Vf)=^W$Sf-|CLfgPu(cMr;_MCgut-dDG_8aD=+r#y-;Svay zglY&e(!-GGC4J~98=(r)) z$8Nh787gp7Q*3;IO=e^M(IZZcT7gg`$;hB&d3hr0p{bj~JGvS*Inon1$@D|V!Uzg4 z*_r3PZBdGF{Wt2>tzlE=8OL84riXw5R2@`C*_>WRl}eZ-4B(tKKHcOeOq%ajJ98*= z-w3AIW_i#kY6{=uX43(To|5ar-H=xYJ@VE%jVVvDUSR$V3c8Rxr}hhasrx`7d37Qq zuv}`42s!UR2R;S8M7AeJRWZ0%@90R<>6?c=qrlr}um4VERyPitS^;yQRoIs`_i!=0 zq3E}g926nVE8Eq)>%Zw&pBqi`TS~wBtAqub!f$^Cc{U`uy{(iq>p;TrYcwlAhk!sl zh2&gbBYy5wc~5fL3}lwCXeJ!Ek*mbuDEIoSa@FAbuOb{imikKilVlFux{boEgqcue zr*DX|P}QmsA0}frrLy5D%Q|`EZ(nlHl&daOC@MBc%7Zkr`YbFUf+@`oGK>x(B8J=4 zGvVJc-uTbIdhcI6!SVf8+*X`sZB8wK?RocBmhJNTwLpV4JWFw=cq1)j_GF1!!fmZH zy)b^p8)k~;G9Ag^LcEe-#mA8~V^l@dWEa3H+~^UECYGwx;utOXn_pK;ubH#;hB#G5 z!hABgEIMS{K}Fgbd0*Eu|EDwdB^HwAzmkLZUtagh^Z|w6m=*YKxP!gJk8~r!oH3tQ zuBk#-Wy%g5>B)_VG$iHun)R8MIMJWnV>tXFhGL(l^pyGW|@pWc(#Yq zjz4pQJd-k8{?UBDOEPM@uQ<0=(|Zz#SIGn6EF7i9FW+9dmS&lckvfo<9H}KbNboXR zf|&e{XX*7TQC=km8&sKKIz}3NLR|s=p40tTKQ9m8Pi?vy;KG;(V(Oaa_@h-W^IG6~|PqZP;Gr=k%%WTya_Q<6(*hVDQM zLh{)Uymo-!(Jvj3j;P_(OK0DC7*IU9L!lP0&2@BVe1D7LMjHY)v@50VX3acZ!r;m!&A> zMO=_fjPMxzK}ITyvpCW%>dZad1#t4tX1R$aX}*zVs8~8)zvKHj15{-97+WnkFXc zdbkLt@{#2}DZriywsaNL@~@RVq3wZ^Z-Gv{*1lXsu14qLpj@Bb-vS=!;%T73qd?F$ z=ar?U<4~5gybP&_pb0 z33K_n3yU}zg`>)!TH6!Nq?FUIn)=@FGm5|aHBeC4lJzBP5g^&t$=caRloagbqZviW zi47*o>A48%a`JvTOXU%YF%~6lzn9(m273PT1cDUAxPHGqQea!Rf)7)oe4UP|isvN- z2Y66+%0!;e)Xd^f8~3%4gB$|2Hv*V-g%28&ZKQcM@Lkk*@ zr^2SFg+RB4!wG;IJkIo1$5AihM+c8j@2`9=Vwi4UJe~O{fAZ-YO>)~h!iAvZI_M6* zK_%j=_OQ0gbb<(HQo2lK9f}l9+Dk5^$=b%sW-mMvlPH)3KsismjWTM#lV=blGu0+- z{3<5}qlN4;&lG^L%y!7@PYmu-jU4uuI-pJw;yq9640k$aoy0rc(goOD`Q};jHRwv~ zE47y4348FN_Z$qW6>sQ{V$d58FV!>%J~E-aDPi!|eADbLE08&=5sQML`O$oZThcwE z)U5w)%rZbS#NnI1Jdi+yw$G{jMQ6-dZzI*I2yn1EHgYvucWhEz>$1*8<}_^rVzAx4 zufci{%5|JwNKgGsbbQLTFD<1J2bvr4%xSXhhQCqX% zsyk=CnuGdiR~nSYe?&0#*ik@_tj4pHZZGsROx9lAtH&hhi%)dGmp>RZ?diLWZIVBh z&v7{IHHqRs0NKo*vK!zuK|o|ND66@*Wx2|Yh0O61Xa|>V64YRJ4|;}f*r@Z=CNhEU zoxATMWIxa8ikm%vjxgM`e39zqhmm1k}*93vk=YEQL)Fd&7 zIMmsP{9iVyR>H%iLjcHdW+Bq~;T9RzyAbUA!vy;z$H#~)9%p%75p zp?s96UwrqI4)aT{)bvmD$o_2K5GE$bcikqhOWiV$E2%Q$#*~A1gn}c?vYn83Xkm+L zo^9BH0tIf%050iYz*N&onHODOm8Lxk>bc?1O97?>B2ytWPGk{?%EjiFvev_X&M{K& z2i5=3w@Ruy%~Xt}!d_CiFtQFjW!A6v66;E9?t&|rPRFkAgEVEzD`_%&y{4M0okiqC zKvRnibJ^Yioiv2=yU%_wz)}?rdIG3G67dEJ!Uqv_r?plCy$HRDA4V8Wb;;HO)7!kv zWLoQ9HNi8b=x!ONzJqCHLZoRt&@5Tv7BKaiqo5c^%mZ5^Goy7 zyllnteL$3zP2;soQl77OZC`&x*)spLv4ZKn6v~(-n=c)i2b=}ExHHcF8s$LOKB5{h zT*h^lx6WCF$2&(Z+BhZ8Bzzc12}eh$S@TakN8YDs%@VaX46a!haE?+*1uni&&RucmX=?gTI2ns`;yl+>N(`Asv3|@rvJs8QQoz9jgwBGtY zs4n*7F-M9XcWak^?AJ7P8=d!lZ!37n1^>L96cdrvrb5w9qNR~PMC+tefX3)yl~o+? zHik)WyCW>!Li--?#-|9ausRGz^Na(bhI}%huxUe-Joeu^HJtVteolM~I5BsKpSYk* z0UG3`bp6Xc>fg-9z#rawBmtCdL!^x3=K7l`vx>0V>ubwxB z6KJ0_jPejPa6MjKUg{lQ8v;td)8J6axa@MHrr(6$csuO&C`m}52tQsAk`smr^g)d7;?TWN@?k6C(syiU;ao|{$Z@~T zqlKkEFS`VPeE7sM5?*xpl-g*D;VVX>rd$e1->zs{3(6GI@5iYaCR zS_J~}ygaEsFW&zGDw_c6rQag^ z+0dj2=RVCM$5CC_?j@S>YOuGb$8=RIN*8kPZ*v^pK$J#`F1}g9mqSc+>*pI!0Wf4W zw|n=0G5w`NKR+BDefeCQrfnRP6x4Z1L6b?skKJF56{QSnCB8g;dTJX}V2OU(cOf&FL%#S{OGs)eNZmwgW#@lZGyz{a` zwo=(M+#w*E!c+_KU~wMxS>wb~7$Ac0IBmGsZ7GR#|z^*tK4u84WPS zWVll7o>?txv_Si=NEH`G)|g`>HF@*~?<@%WR)lgS1)>9~{7V|O3D`XQO4n!2Wd=X~ zVM;CYS|yoaCQvp(R`f#;#6xNi$~+o?=E7U#|7UgsG}! zl9$=8(a(b-y4d~7uk14S6drmKX2ptburPJ38(?Qm;U3xj{!Hnl4gOU$F)7pvTEDVyrmN%e`ETAAgvaERzkTxzT!oSF}MI zTVQ(>Qdgz7Mwvz^@SgjSU4ZQiggTUMa9>NzACeq>Ka{0|Z8dW+McQRI@e29;;lDaw zN(PpGvQ^*j%XVA?8Sl)p+W}-{iKW87)`@Ei!&uI7V4f|R80v~tX2k^k`v!LzzWhLVB*IKekRSgj1G)Gw` zMy1cmCBL?b1T^G7@MyJ%`v@5V%$Mq-ofQO>!1YDFYS@Q(X48E{uHVtHhwV zY(|A09tK=aWK=X9mHTF|(4!w@I}Bq?Hh`ay14h7lDLN3aA9}()US?Fg? zYqW$=tM)r+S86&f0$tK@Hbb_#bc80_5N4^vABoQC52nv);*7}Dm2j2WI3`b9>dRKM zX*a)Qe9snu;(Pwf@{uv;<}4iltYz`aeX=cB3vq+9r&@KO+ou3foz#Kj0r&x&!7y3jMkfh~J%uwN1s4gkE?8W05Xv9q z{;hUc&$$*PaD*kFT)ta*l#yE*lL!x2w|7C##1BxtNXNDBhHe?xO!3dgK-ab2=yT^& zxbu6*JY*0>(^NKc?8y|zrDBc!QuA1~4aH+d=b=C2q6 zQQAPDHh3CXhH_+gDM6#WLmpfxd#-L4?W*XPwI6||qRVbTWY!fpjh{Sy!H&44;CB{v zBQXMs5Fdd-Ryk_(w>%2<2@r{ve&XAB1D%H5^D6vg^2p;IwZyKT9UK#$+I|n&Cf|Ff zZ97@Sw%^Ewryi|p9vTs|FjqMC)vxNdR4|Qqf{wr>JgpQO>=yH5g(o>5RyU2ha4V?4 zHn>jKb3`8evCUrJI1NfRQZdA^tQ#;|Qc64*C8`&rX7i*gU zU0n2m3HNt+7e+``1-FYkB_=GOL1}o?6IfK{Uf`^Xg~9l2FkO546E4q}ELgQylQeAq zMV4-+V+7t}?{f0T+J&xOF=cQy(DX&W#Om4^XN}F&{KmH%VEK}d^P?iZyhe3I1j(rC zOK&N%;icNH(teLM2R!W2fQ+fP{^*=Y47j17%Qs7LS1RP8%iR^qh|C^`@D>gy2suom zhS=nWj>Z?e9g9I!ar|>vY%}WRY)#EqDE5VMekP7iq^&&=YOrYw)4;RAN1 z!{)E)%Y18Jwj9s=-apk-3IpH%#}(LdmYC%NCB9`3%-1tO+n(&WQf_rNy!}W3wW;U# zN*h`RTNT6^1OZ=2A~0i#uhm1A?RT{!=BkX12}uN!RH7e=@}y*tI4ECkMA&f@JQXlz zO4{R!6crT-5GfE!{Yc|2mlUA<(eZe#fAF(ZT~#2^A5gq} zdU4%`ENr}JJEMB7EDhYNOW?@&^{rxuMGgK}pFMVHDm8^WmJ}++gFS_z zex;{zy0nTP&*uAA`R?IHH3643$(;mRAgeuujZkCFAbEx?uf>q*YyDC;C}fVPD6N6n zCd+wshTq5G0`;InYpN4_q?^GuK`xB4J_z9aMnc=@{j zAi#PtiMy)VW51>_qTngsIU~sXJ}G%jT?-Y#SuWstSyC1WKSRB?ln8A>@*#Y)%Q-EP>r~XYlOzb6L7*$9nlw2BJ?}6B0alI%A=Z1Vk z>>0c16Ab5rDIltv%?O`=C395i<6LcavuW@NDiK*m-euO^bITH6)k6j?c%sA>xfw`I z_&G2T*r4T+*9rmM&a$tql0H$_WaEe@04*GNC!RMO}0@xHyHYE*dL9Ji#0s513BZbl5IzL1sy=Z!GKCS4j91nRCPqfbU zwK|Wx0Vpo7R=uofbWoxnwbpOS@(`Rfe+;S1A_!){UZf1m3)PRr=kNeDrq&rvz3{h% z`r}!-mw?b-xP-syL?O5Hs=*N1s^4H@ng>KL~X=U^3*SKVbx7ihwMx#M~ zd}S``j7S{Z!!T2B`i`2spV=*uLqD+2RVcNQEfsf7kUeqzOt>CPYb@^CZ`1v1)ClE0 zw<)-b25*#vXAk8l`@<45K?ou+429g8hlB50PM*r^eJC&U?zn1^Gd0J*%X<8MA$%iH z-co(BqFn68s8O{OjudaZ0e512$}<8Jx8C5r_Dof%{h=rr z^QzS4*REQ-r`rk(^8-WKgFHNix?T<#%rv~3mg}r=j#t`}f$VB=(a@zK6O@OcElPu>d4X0Q;sZ+#RY;q0gY|*!zsShXfgTv{C(?9J^P>+B3)gr`@gZ z{@Um!IQ&{IWFcHNa0UjpT$j~KDrTPv=dtZ0#+T#*vb&Vu)4y5$!v0g#O03?}MpbKE z(CZw5t3bL*0A|mo7YE0Uzq`ZLg!zcrBu@MMh12?~{>{ovXmg|8c36>QUy0()2#z(@ zh#z5o=-Iy3E$_K&1MsqFBTV1daEopH6*)DlsK-`6cDv4b*gVe|1PMW6?uNq?Zj{dI zim16lX%j#=W4w-3kTC4nPza6ZK zH0pY4=mMC`UJkC#tb@LN9`xLu_M+Ix$X>-pP(eE(ZZc*^0M0q0#QR^z*);Rsdk6z<#+#0ssrvsnvYDN)5KID&L)&#Q zV)_|R7`BXzJ-@uIF!@2lINF233^00+^;3DyOePDH;Jk(D#G*>>r1Lpl>mtI1fcrh@ zp^OziWT-`{e38*fZ2X1VO;Z~CNtgmqaCfY$Qk0&@P5fWUQ8<(*xJk}{%PCvAn;#Ew z^sfAVZ~IVy-^1;40+f)~{s-D3+>XYAawhp;#z?g)Z^_lF$!z}!>F+d0w=6sbM^_@a$b zig9sZbF3M(jk{;doAn6TA`1QhFYiBq;5>Ebt@N`s8hrhLyu{l-=edw1G0ss-yisEO zcz@+kmv~#{W;AW4X=X}kicBs346b1l^1M~v-<88~%A;NBW#qJlDQnenr(LTod-uqL zO&gPyKWiJlO-p)5Q624l?DqTy{=O7#_Xgj7%K~UXzQT{6Z>RP5)DZ4w*3nej58<ZD8}8iwbRK^J@)(cXz7tWWsP|qq(pjm;2)T@sx$j93pOPtAut_%OYla ziZYX6WgdnYss5u~Ca+`Lg#y3A)UvsR9S1zPB{l=DEb?13<8dD^u`}qXY6AZSA@`h} zN(NCcbwF?C64X-uIk8Zz;dA_g?E-SLD`u%_iw$4q|S9C^7$lbM}wOlG3?3OIEU}iMLmlvB|s2o%F3&NGDLj*BBFE-8W=4t3*@|-1JKy zF|zu6!`Ju;)A1GQUtR$5`)cWybVf#F>~j0sFQE!ry_0@@MuHzh$OUrfiNYOc?Trj% zA6&c0^?td68IO?t1E4!NP;p$U9Psb(D6|*!cfv&Nb9vciVDOS(5MgiYUv0nTnDEMfv#x_+O-9#i8}Hs`q=O5b)3N}$tDnj_`4WIF=pK~_ zvv)WeI&M;(GVk$W>}LJ2w zWGY7J?eCgbPqQ&z?*~!DgT667C6C#vbUT~cP3Nmu`PXg9{27gx{;e^fLETzo;E%gj zg9LS0U*Xt@JxC>gO(L#iV``1h8L$q&oXaZycH6LsDWTBBu?KtL$L*(#A6Ly>wA(Q% zu~$pDBC{9Eeylv=whjjVhHHZSB7!+c4*6kunA_;+`{&~CBEd1KjvsKC+B%5tI_?wYcazZt zLCG|b2J_eP_-*nw&wrIl@-LX8UEq=UE~xfAJ(^M;j9;7ath$#klgsW1o*Xeu!K^Ur z0#nZEMGS%I^*4H;m@x$kB)?;?%4I4T4Qe-YSKtF{`lm1ICmjA`@yZIAn%Hmu`O_E8 zb0`FzYC_rM-HqC`Jof6zxl^@m|C($rLC*UK--vn*yh#%|3An8sJ@37lnvF6xNhy|itMNGQ2+7`ZL?@&D`lE(@LStsu2m`DeUO0l(_cS*fkwU!kAFo97 z#N-ipp%|f<8@wm>YL=5kItw&$P3kQ}K|-FHt|#R2FBjw?F8EVaS#57js}W-LX7z}H z+)s`SIFx%wNnbB5T<`S=JpboTw7pmXP zv__UZ856X+yCr|@ygTemUmHteJ<}MNzZQ?eDkB$M+*%HcdKEIr1(j^|-Us8_ zd-vGbd;VPxc|+(&^zu|bncc`G1;bj;?#QNuFRpkiPj@k`mV+UVDiQj_+{Zt#$4+-Z z0D)!KlKg(3M#Yc6jyyuje8aEx5kc3kAr6B!Sl}sc;QveCH6p~45$_}#GB$cACA5)- zuyC(UMAZgM9wgiyGJ<;csH$*z)Vad%?EHjq;&2(Mm(Nr(y$~m~InCyDLaiVW?@#;< zD81}%HWP{j5Cp;Krf|0p^uHOBmkocsb=OWxwT=<|yb@a9H!e%Nc%tb;m6H1s(Do|NI7n*XNn+aL)!zQ(HW;_hUgQB0GxRvjc`YM7dnQw2xFIZordae#@` ze%7|nONS-X8%24$A|yOD)Ca^ieTA*x@U?j$ReJOohnUYH>_BK>U6eVw5`^;dwGy@7 zGbNMY|B$8Homf@PAqq4Yl*eIY+ZXpAzdb15h1A7 z$ol$3-{n%{SP*XDuXj?K`|N%l$PPpxe~;i|gwgkaErL=44{vp;>=|iZHRo3@ifhuB zt|BpiuW1^wfOZ6dZ1@i{i^&q+Vj}$U2vu#?EInU7sI4JbaoPy4Y0rFupd@JnN?ilCI8-SFsEp-|9^pw_CVSZl>~6{y?TJa~M#%lQG_O|(rAGzuzd z-Hu>l#>Hxavkyo#QpGAo@o(ck)n0)0z#}KBK}sCo4H)t3cB!$$+e|1oeb6XUhT#l` zXK-v5kM}noMrgI`04eCFMN1ZiM0kUf|4)(J96L zGW|97oy`lQrxi?cu4AOk1;=a~G(oCiHTsJCIm8_Dbxu`~k0=vzY+$&_+SQ}vKabQ; zO88F4U#>W}!6Rh*s{;A^*Hjz2^=n}<^`vIh(er^tO_%MGLEa0{BE;dc9A z(HHgDhMc4Uj={WHPF5sROH?hD@ z@ZC>a?rIP>P5LsugnQ>SdpBng8F!spvfXe#fFyv|5i2|jmJy!7UmFvwu?26u+#o`u zK;b@53oqq7RkgxM!Q1>%l1|7(D!Xxnoy=fY9V-oPVNE zvzYnpTua_5peDo3sME%Tu^ta__Jy32^i<&jwQ zF?Mjp$SBkJPhyJB?UJEsFv|U5CG^q!vu1>fZr))!na7Xn^MWZ)e8z&6UL$`%kR9Nv zlPM{42-g3JmdQ3s>;tG;N`zkIKcfb?v_H1(}~ zBAYmG(uQr^OogBSL$h9cbK$Usrs`|3jx+?m9A`4G@ujUxY_~O}a!#;&$TA zu{0^h!4fN;!fT%`-}pmqzX45Y4|Z8EEt;lYko2k}dL7 zSr*k2rj(M^G9m2-QYz#|q!Rq>!NOn>n!;omiZ#V_fykxUAE>iR6IBU)t2wWiNtY=o z1>Xfne{B)z2{oJB+*$ZMWyDAWu!WF)NBZ|(uWfZ4Lz)YOCtXZl)dk zzvmz39x$QJ+bPEtmO9Jevdtz4JsBfWe@3NBr}iGYVW)ixvAl`Q6r9l~En#ZG?7D8$?X>IpdKeC340^$<^DGv;M$Xhkd!&nOb@yW)ENF zb(aK|TMTKP2Tqm66D-QsD-4w%S5%bpp23)DDN>ug3S>da8}O0obHrRq=PI*`Q#o?e zXX#ww_|i!_c3SmFR?7~!C-Ijxs~4T?>mL%tOf%OmaS#MkyUCgUPkadj?S zWp@J{Y<}~dz+XT2GM&@r&j%5OXiYdH0u5Ns`MCGN%r85+gquaJqAHd*ksLf+j;$i7 zq}xkL%7n^cMReOj_e4KXhL&j6^44ecZN=n-6AS!Y72bIbA&o}FU-jer(uL4FKW)ve z+;ZCUX2SFclk88sIIND>0Q6)B>*^F77ompm=grX2BR@o}d)%b{flkwISWs4}@vpP^ znYK9S_!x){au!;xw0z16cp}jz4AC;nx;9}z5+HnMbkiUWaIPFXHZg>jWrkH>-Vt}m z)zm8l>Q2rvbXXlUJ~(aJ>QaG-v)_?f)_uSbXw7y5^RDUqbpZYA2IwgNVV#B9QbLIQ zcb*)59pGl~l}nL02t5jjd+ouPB44-JZ)g4I>Gx6|?W#c&CfkW&N zdxX?fhA4a_K}bL7vrHBN$4)ub#H~k3RyguZ%G&Y~Eo0Cz@vf-^!;kqNmLv_1ZB|tT zv@XB0DWVieZQrtex%MK=JhJFy)5#`IuTE}~H_|&r=&A&CRd3anvetOG6qyr2_$Kob z@*MSw>#@KthXQ7`ZtTEFNzMJL-$Jnl`+`aElL^9eraPu-YDC+-x(rPSUb$xYIRCKT z=ncekT}ITaKZe@;?|5OOd;PeRl!H*DGvx3rM^+t|`MR2*$o^n13VQ96&xK8_0CW%b zLt~1)Bx{2>{SCwV&B|U!+{D|iO2p31OlRkS-vy@>M;OqHYW=U%(gk9YXe&}0K6b0< zDNWeg`i&L&F1pW%I@qnGz3JHqa>|AE#DMI@x z{;}0&^>zf*Y*1E+e=Hw0aSVUQvx8ZAw63;r!0Y2nwS~Xlk7Yr|R(U7o8QzmN9#lW* z)N;CL*;8!#bsteTE$4S)i6ao}wYZdid!@#*-dK~vD6JRI(riu+f70b>3fi?3XkS`P zVxfvX+qplIOLZA}v$SPq>JaY>-}lo&f;Sus02~0k77?D2iE6bsFqZU0+-v1uwlZ(t zxU{!uN^z&mTd~nR4tdb%d{zbl%=(-fDJ@=3MkWg|w3PN|N(unRRAtzYDR&sBKu^={ zec!Nbgxe*zPHxMn{ ziL4+7QzL%r^&(1(g;@TZ(f5Zyb)<;8%DFmgifZ%Ok(?8innf^72aBR3m?D-yn0+jpc+RQQt5;IN3^GTwURA1`P`hxpSR;!w^ykt3jmlOB5|@4N zcSJzCX?FrxBSQG%;!6UQzL`Tao+iciLGoU`{5t6dx<=~Bsu5}@Vnm5mqoR0a8bZM} zlf7^QI)bLvMT$`K(t3$oOn?t1#{R+*s&Td809 zCq5Y&qOT8R*IRv;;j;Er+dKY?303y-`>|E2AT!BQDNJ{G-Gb>&m@zT+{;sn8?ppMV z*`=`E7@CI+zJ8ZW8t6G7%h7}SpCmOMw5ZD9l_vZ<{xs`Oefynt`dWvM#&7r1Ujb*j{EL}VY5y5>4t zc>h(S)i&?Kr0j24on+F31?XoRFd6acuQ%&}r=!YgmMrO7onYN#&r02|M{-Sq~1<>D14jFqvgeixk#ohDJSk)%Y?HCZpqjAXg_&kM6)*{mej7kLo z!6)aJ^ZHw$BQU>=tuULWtJa5K%+Yx{qR+jOb&5wo$twZSE(FVs%77@%EBZP;F;TVy zv{cv#G=7S7eP4QNb1hx43Dj(MKC~7kRL)(U5FQ*oF892!>^FIF4oEu-(b|t_Z(SL; zjiz3b{{>F|zf0=g`1dyYfs_&1U#GY(3D&c9xW`zat{0?5I%o5;Wmyh^{Tgoxf{>a7p?Eg)Nh6#2u~5t zC8!Rm9)0T7LI3E{rloNIWeDG^*zHXm)j z!?L%dXG*dJTe^lrV2Fh`d-&fM9G7KDlUQZ`jpP@H@Lf`VPjiAqF`3=m{oqK!O7zNu z#XjXvV%28=gJvAYkB^CXecE~+sPsQ+=U`N&I!&1gl;Jb45w8Sgl{!!s1!Jpz#XkOd z{jtk+9)d*47do=Smu#cTL>a@qP04)s^ju->Y)U_G78K|lU~c5?a&=Vl3f+1=V1eH9^28TG zebBi85F-k?dzKKbPgdIt)8z(t(K3k<*h#N_oHF10!udriTzes$Zp@TusjQ(jbWFVp zDkjC8z~nY%#0%L)cYcrHh2|UqWer$y?kX5ae#zIX0dIuAQ3NGI+PfzIhIl3!?Qy@z;5PF21k4q7EO>*7cM{JF6`5%0(i^3DH_2DS)2$u#)4A^|tuS{=%j zaC0?uFQ3OH_yJ9wir=H#gC{u&y3$#ak*fZN5rZ3F@ll9$U3Mu|I1n(5KpROTQTOnf zpoVC%67uM?^SU?BARXi7NL0kO5&{w_hp_J2YZ@+C8uYw=P1bgM0+7_y&Qk;&3CxPb zSZ;x@_4HYF@tYe(UbVH?DWLPG!;P_^u%kz9?*}G?DhX^0);d`dD7P})+#7X|U`ZiB zYpF$4KvE|$OXp@n%?XLitG0YUzht`MR$UzmhI@&Kh*)?{ct=3c--bhg+Fz;KP{%4S zB^9Mh4N+JPR!%9}AkZmWH6{Uy*Tmm-?SjMa!d&qX&Z@eCxFyraQT_T>BS;!{dRSi` z;=F$uRGh)&vG;#^?f-(?wzaboJIhdVlr@a{q6e(Jcv@)28P7ycf)-+EZT z2ewu+4%O(Fx{Dk2$ag;&#?X=Pq7R%AW(lV}$I{=MQOIfAT1RvuiF3EC6L`U4b2Dd> zr5Cru4}{zi^tM^6$HO3x&%GGcscaK?QF`X&(Fkgo>HV_9mn5mpZD=TskcXvOt2AZY zI!Qk-k`CG4wn|X=Impf!5oXGANQf*E!i+}M9S3|SwtR5PQ!LXJsJ3)E^|x6Id?2`c zJ5pcSW_tqy_Gps+#0K1~#**$E!H=qfSyYqD3L+E@!5zR>WSTuT)Vm(T_jyvijR-C1 zMFkY|6cHC4=UeN#AE6VR5$;d#y#t!_sjt_0NKo(~_hyKR6=mzrT}g0cbph))*xP6& z#)^x2r8^Lm0s9O`zaV7+7af#(^ka%I<;h)IgQhh}@)C%k#~I7Ao2-ZkMnfKc ze-agPRG0kum26o~rzPH|Qvb;=6a zYs?s$27A5`oQP&t9hbfJI&*r_{HMTQY{}lI^uuj6VI=%MmCLr4uo-y^5Yw69SsY&2 zO57H)o5m0#@r{5Lan_Vlv=nHwI5Dc3YFM?mp*`rqlUoz@YwrCvf66kWw~wz=+4Nkz z2g2PeX&9&AW1F;jg)WKhS6H|#tp%fYF3$67o$Apm2g6^CCJXyFfml3%s%# zGrxJm!U6JOu-bWd`t4I2T`T%B=t z&k_1AgdikC^=@FXf@3Y3j}X@I;722eK|&_5MJm`to9PN!nz7Quv(73CLW=4&Y7Y}Z zpk*8c?0_{D!3g$HwDk$M?m+Ec4V{Qkv@anLwCF7(Jx7HAe^Ox#JQGvh4%U7n9u_6H z>jb8xA3t^+G=@mU3@0`#6s`#X5B_~d**xv~dnrwcQ>f`8et%LqC_IgudpWGk7vgSh z7j^s{0;-y*WanqxXeIdmPU1gnuf2+@KLp!ZcV?~DUBpTO%s`r<>?+8FA^o$K>qp{b z8UMad7+jc^+wjz6Q%YhKaJx;Z<8z1Wc!uPiSLu3X zDSl2vmZF9b;YN1_;ua}}M^%Z(dFPAV#6>w0Zpicq`IZA_Dj3C7mcQmK+-qL7uf``?B&|LH_WZuwrN?k+<(gm{U~VFZ_J~*xRvYx9Qu6Zt&ayWU1eBv#xuBOg zYvr`ff!*wB=*SHW3z_1f+57FMf2>Rdeyu%bafosR#my-7;LW}!3$x>EMc1V3P|;50 zb1b#fUAsM~2@yfxtzrPdIEZ9DDqTp%d7DP|f8U{8^^24rNwJ1(VW4nfkB_mEZepN# zV3XM~YpR)|nk*=|dRgtU1qmNrEyTu@5YSMU)u}(^EV&nb!MXaphQ2!*!Aw_Ma-YXrjU1f0 zO>!&xUgO(l<-@3MBk$XRZ7DLguL>%Io(+zY-8Pq1)S+UF{5)Fkcr8TjwQ4N$)w{1~ zQ2$x3Z?Vw7DmDoDZX}Ta$|8ER&|XUl0FiMt;~0E%+OG(Euk>8PHnd7iR*!L%cHa1{ zMUl%=Ve21dkfI3;U|KpOOv0DdIkWxBTTyM&_rE$Nw1IZ<%tIHTA5iNmz>pGN&Wq>f z73CbOrcPE_Ezgup2SP>LwbC8aHkDVXW{}bM`)tk7BVTvea4prMus`2#Cg>HS>ORgO z$%{ty^y*Q0YZB}D0w0OmLeaR?ePOOY5Q|XQKvPINPn)tN(-5;QgJ^@Uw{2D)>hgMF zZbNLtujN2xx<$3MPZ19yxR4H{t2Q2|v#sI6!E_;ej7+F$4dl{_3|tVAq)aq=Ξ_ zB_dL_pilhq&W0zu-??I^?gTb^+uH#E7XQX!&WU z_`h#gHTMQEu$@9ne9TFv8J4^&*#? zy&g&7W`?6%4G3|Bty7z3hqS^|dtvwS2A)I>;V(E)vOSr-Y za7`rbd^l>e2;t%~Z+*F3U`_+yjF>|p-gjlG-tICS-!Jp2;RR*~o#Tk0w{8;@*R!{2 zOP>^_>mGFJgw{MiS)OOK?+8WeK5*kH8>_wD#&L@O zQmPyo{WVlqBQr~e;XCW5)WE`xXmFJxVgHjewNdL;yk=F`Q}c%&!($h;SKYifA9M2u ze-7ed9gV(hwN^W^;Z6q3IpJ({VP=R6ybJ#y1V-ql6j5{69TkEm3e+xFiF1+QQGqK$ z#98vPhTwDKl$>UUuAkTWT&5FmBmI~oHaq=aW-5#m3|eDG!he6!fGvJiHaEMmvGJgZ zCHD8)2`vNR;Ff~+B+7ZcE0`GZ7LAyHM|EtI`OP;j1ls$<0m5}W1WmH@a?YQ^js%r$ zy$&TuW=3a$W&{mbfsm!yQh_$vx>touuz_YoWP842I6iEiZ!%VSL)>A*HJgT!W(AY= zPFU6zxVq2QHQ9uo4KpGR3 zu5gx@WBOEqoLA4jwE<+Y8O1E+t~qJee8WdnvuJ%IHUsny7G9hn61cS{aY5vc6(;o< zXmun&{?7$%0?eWlF-ert$4h6^Md>$!rk0d%e^*n#6n3WE3O3 zh~iFrdfNZ#=uwj95%;th7fbowdq{~r*=H64+CqJUkbP(NZiii|o~}L@v^=D@BV82G zCVp{`=iDDhl9`hA5cIJ8X%>HA3W~^4@+}mU!o{>K{p2la3-!z2DnVFRv&PbD`s6{J zBV@yjLn!zu8P`WN@yxie(g86`|w^J}L*T8vrweKAfM0c=&v-JJ~l@s?QR&|c#gjw-z7*yx>c^(w8< zTp*w;yk!_q%Do-NUI|%BT)Wmvg{Tv+kho8m)EAN7 z*8PmLgjZNBMI zT$RN$3V3?Is=$LyO?qdM2~>a5&Gj|RFDy%~AMKIHtNrO1AhKw^*8D*t$RTjV=oW4E zlRCoX(D0f;32ih}FeP->+9G{O8U8zhokirLU4iB#ut9aGw*;Ak7SHif)3*Gj!>Mq@ zC!`Kyo#;~;z$KCrx0^MelB3HWP}lM?<-^g-MduFb#JJc?sqo}P>sM9XR@+^@duQc=^{w1yf(hr41!Z(O^!#vT*7 ze~V?f(Ew#^Z$%*yBv?g!>29G>Vy~U?4Lp)%k-CJA8ymnv`6FMvw|gVzYO$=)-V#{3 zwg`KgB^S0UC13aWelY+}I^wR1dN8XU+0PoBuxERkehF@n+in-@R0p@*Nld8B_UxUO z!I(_(bAFRQ(5yQ^O1C~;fz;0IBI=D!FgS2@SUu+olEUHnOGE3*gv;K|2{kphw%up4=MpgtZE&iEBQskLmYBD@B{$zHS<5wpcffojx$FarPYAp5T{~C%;$20(wy z_C5uas*=TfQIH7kbsQpRY9xv3krpCjyz#SwycXes!~~NFhKZi&-?bC5D01<6=c|$a z?RqCl&+GQRtJm?qw;z7t6_f6cCf*lxh3$xc8(u9uO zLLy;3B>6IyMEu%qDd=~69;<<8m_PJgMV#ocmg0set9r{Lg%6oM;v_ z^k>pR=B-CnRU9e|k+``vXS*P$;S}E_6MvHvYIo*5ytiws}4@6aEdXSm$R^oqpN{-$&uE_ zUdH%!{QEcjz6pYmiEdWICm1wBnd|p%5j`J|{KlFRoF!je@z&TT9^QsNV2S8`vpPrs z1G^mg4k^uJJjLa>smA)XMK3swxwO;h*Y+I`%pTk%8!gcGon<B&WbxUtQe$`8u2euO>y3IdV+rr$DxB1F@>@j>a`S%T) zsyb!_NJiLeT$##ydFD=2ks|QCm+_>FmYay5YyanMz6F!!`hiQjQI6vWQ=i(epxxi0 zn0X_ooQ>&>3GdFS=&mjFf-l~fPi0O`>&1OQ@{c-*liBS!m!b^$Qmh+8*VGa#8z*zz zQXmG!ApzLqd}ld$ed-0Ta!ToOoy~`cPy01OSJH>-NBO2F7qjsxThF|ed8z+dXXO4b zs_>m8^BvRMTZd#n9A7{@QZf2~p~3t@nsGh%Xtl{WIRTGN_8F`^Ehn_ocBEBgAgaH^ zC|VBO+Yj_U?F2R6wE@_v*mwf3JI+N$+Zr9tzYng-eg;g{XWO-HM2~D1E1^~Y`AiIo zRTTCkZS8+u^S@U5KR4k2{NRTTMJc3!bF7Jaq&8?&sFM(#P)sGN?e-xK#KI-lSFme5 z395ckRrZyLtvuMkG|ga(et5)eIO$-g3^6EF2?UytB-!c0gb!$2A8xM$X&fIkLo8RK zqPJQwhBE!S8fP(Yk_}w{?u{fVYuuR(z!qrOb5Cwh?$9}|uDM94 ze%fKRIo}y?P`&d-6NK!AtIy&J>#P^2p@NmX${xI`8-o2o^R#4R8APa(tSAHrq zN0U@?BrQ+eH@71LWA~XS^_O3old1o?&^g$#LHRFVGLO22nf(bf#7WTwGEW^)!!6uE zmlvuvy6!qUMx!!M`%>GnH~bA=`78L$1g@fKg7Uusu01>L8ppzsHiSmLFOZxDdmOej z*`7D;e|JT~F~X0fX*73Bv!S`v{45cf1e~-4Xn9cLZtAK`hTJ$jQ{s6Q0#>v@L+gqwL8~r9M(e%~|7=jx2a=D5?SA|KDk$8OfGvl$kAR@_@2@ZY69P>KBZvE6 zgN~Y-6W?!kji}gMe=a|2IFhQ8+pcuGh|GJ>_K@YeOG@ij`Gie}5G7-iedT>>BZ3Vy z42to^59oSpfAO+|=!fWmG8{}IomyOG`Ms;i&ynEg1>*gIY^>5qW~nGTu^&auE^yB{ z0MN!^U$2J>0oKYv#~VKj4&Z}C!1_f@cySA#T#!v2lz#Pn;- z0jWt*#_`2+R!U}YD=mKG;mHLR{aXH^uzmbSK7K(^%dP#rsk&jouA$gEt zkxG#S5G&@hwa(7oMMmSKxHBAqc|V*2#P8M@O%|E9d{>k>_950&fm~%*)Lzd zd>Sv2i#5N~0|0!Xa$ba1uS(I501OJ&i4YT0sb4zRShCq{X~`Yk*d1)(J8y9R#(@_@ zwegAm04NPb08i{vUrP6FG6J^DLubL1w(5WMim*VnBB)P}PWl`7;%#DZ*}WA9o{i&9 zb*So`uuVNI)~nUlTnK7q*9*rPe<+Lk7q8j0*p%c^jats3b5NRtN!yr)gDkb0JpZ^g z6mqB$RqnHO4W^{v9Ki`1*5c!ZfvV6Q0c2kIGPT~@xU=0cCkP{q*fD;_9|q_}YQqd+ zhh)SU+4tC3F)&bY+rg#@qLh-7%6Oi$Yvj!3?=oAZ?-MX57bju@V&+ z>~Y?e^m^HJTk$9P-|cz-)U)XCV40e@vjLa8_bLe(PH==b&!qIsK>7db+9)G3YI?*>+Cdp)AOP?8E&K}M5R`zRl+=W(f5 z3B5MkJB0TE46#EWW$;>6qFn^teQS@Xsf~_cc{qWN?S%SBu@VGU%(K5(ZR(nyNB`Yc|=J~y&r|M zox$$Sq|<;-m~eqm*X*z}_CB(h8!ZK~>HhCvLkOBZQ>x{>j&(N1SJH_S5 zg$rGA#lM3CA9r??=nq${aC9iOc6&(}t1OcITb`EJ8~9(ooY;j+X&j9!_4_ZF;XRdN zkt*ZfLTwpS&TCGySf(2Y)h(9Shr5hG;t9$P;3%y4=~FC|vo+N%CQC4!!C(S&-g7|K zN|KE8;m_~QqYXiiD8jLf=+1Y5~Ba37fH5F4;p_mH8&Y8zF_`GQg!+(D+X#` z`6K7Yc)7?;_S~?%WKQ~&N@*8U4Ds4fhF&g8>Oh;B(%ge9yqL{ZZ%q4lTxPP?qQ~h! z_8az(N|RaL**1&a6m&>tl=V;d${`GcL!V6U9Ow_1w65~3J3|JDcN$x$_ zZetIOP@U4>M^<)_%I|y&@fiEm+l}89Nh8>DA{1a@TC?`}AU9PIqyF9Z`3|IGd4on& z8K|7Pc%+l_-NN2jHAM;D?of~xW-_2*@tA1x)}GN=m!O>>mO;F-wU^JceJJ6zZG?_X>1!k+}cSo}D; z!d?ZLpHr9Cy=6@_pFX86Q`FhHr?^sF&Miw0QPUrP71Gd@qwJFX_Rm9Aip}G(z<~JD z*;z=TQT}(43a+cXIzit)yC}gWJ*^0moL}yx)Rou`lkuKk;R;5Y#LZE{p{`%MU!}we z-VKT9d0MD+H;Oo}DxDvgW?wCbX1v1DcFb|p-+H2Pz2TF0?pbF0KA>=|7kqwhCytQt zs=L$mlBRI-xU5yv->a_c8x(&1`A3-HL@YU-#!4qHQZF85D#aeeCeX!mh;SFV(65IF z2WJ3U&re|D&C2%6*17LkywTzCWB2!G)@4I6g=9oFc;|!n&0dxzY`&CEYXcG>Il!*M zLP0#?v~?2JMmvsKCYSV1 zm&?V=$q}^AZgS_ywp~Nr(91#dvDz5_(MfEArD#*Rdyu|tDBvKLLaV>1j4fnRcL}Y} zO9=#7pUPbWm3L#*iYaX$;N*9s;y(*`M%c`jWF$ulrTaWMjF@&*;-Gd_JOn7(-N}12 zGUW+fAZ&WnNOZ$TK?Vj!`_##J-a?Lk8{y~(;{~OO+s|F(Dne@9Z%x_deqwJ%b*kS^ zZqPWE{4#&L7#Zo{8rJeKL+62e@l@7IXISl`Oz^ZVhnoazU;pIs3mv)+m*TGV-3YMp z@vfA8VVNQtfUHEMx@4WN@r&yHm^1KKL=WKU2z&wSHfL!*gS`w>N5`YBVAXt~O2BD` z&1to~I9<%o2#ZL_4e|O78t}@@;RgHt!Y?`q4rR@kn$ohe$nQhi88Hcdz2`_vO+9ja z|6D)#1f_M_LIe0aThRS>46tbgGXdlF62 zKgWAzLQ9US&|b()?D=oZNZ63I9;A`-c%at7;WIk?%BE7f7fala_%aYp>F8F0_9|4N z|L=BtW+6e5YS!GF+h9GCG#zK8$6MbCxJx>lweVeQEJ{JXUnWF30^eh@@_m*P^qA5~ zJnc$AQm~#`q4(ppD6P`awG(D(<{{-x*#^)*wQnVA+E`txjHyT?DG$*cGTW=z0LGw8tWrKqdqpPVs z!p{%ZzSZY*t?Sj1-Dx)-eB%fGibrttj;ZPKT*Sv=uwu<=?zj{U4LAd8%&kY5f*->}!~O&Si^j(B+?u{_3Lacg5-1N31R=&aY9+$1>2Kd3Zf1YSo_ zR3bd15_Tu6@9p_Sv)!jv)Rcf*8bwjG_8XOK0Z_C&1JF7^S@&7k zucDouT|uc3rs_LT@)Ce7mKBvmZfkOVq1BV{Ho%8OT=)W#&kMg_Wri|k-d^l07pa!) zfZ%U~wNfi}im$Y|ClW)se?tD39S41Z(L1#dUn)#&+Xn$sQ1SoKAN_@P0sm_aPl zQpfi*Cgt8?y)PsPJ0-OJf1wzD+}>a^hLN1GJ)9(oZUs-iI)p-Y4XoNlxR$1?PfPyg z1z?BOc(C`v83?CgjlX#sfabC===X}7lQVdr$%y4%vx$g#|L*>I!9We3X2m}2(vfF` zNWZq`!q1$$>4Azm%DQoN#OMlAN_c0q+%Uiryg(eG%;L9*M1r!@@^?U{Ux#5f&N&Xy3MV`{@3sQf7axRa0tT439QDvo5D>+G-lxjR(lFS!V>5G8@?<=V{j!S zEOE5(B}-Sga@9_>VCm@R5R49Pz-UMD_7R1rwKxeE>+nzLYKg5}S7y`s)S1N4VYUor z5ZP58OUVP@id9oS3N7!%jv!JZ($v>Km+P^$M!t}pho8lH3QBg35rsdvZhwv#vPi~h z3x^e;j@Dj~4Ia1R$c2(zyr5is_&i=m3}b5gN-iyxxxtDly|_4Q@Lgw%yQ+OkC>~yrWtK(3@>7|{q%flvfxmF~ zhXh@-bQvbrkG^o3O0pvTe~J0N=8%d0H97m*DAJ&=vppgKCSfLKbBavr4iz*CU;cM1 zSr#nNl^KV_`pXz|cU0AQa%_E`U{@PlZ+jvPAGZCJzXp#ErPt}!a_87WV|~X^^xJt{O!3YC ze1&RnQ{PLir2UjpsX;--=sBxkmO?~tXf_t~G2Z>)$Iyl)0yg2bO5K^rP@^{kb%fHl zV?k`kL1DJ(e3dW#_bVP5ww-sF8Vvrs9cw(+J@nX<1v0@mZutHEN}r$@wH^JbvT0V6 zK?fB~>^>!{rfW(kJxjoCEt0V{{L9}}{glubF41JZ^2fU0Qvl?kSi>MAL6QqL+?A0gDSL~ntddI0t*4Y-m-)k|3H+5zGI zC2Zx2o(P_bB!Oeh&q0MRO6|PG>K!GGESDJwiJ%89w>kEVG)Gc*^w@a z=jX-HGq(Pq^Pp~T(S2%Z@-~}gx)+xWJi7FLV|CPXu{Zje4N+~`1AD0=6tc;P{rzjp zUvi`78#C6Ar%|mPK36n_Qoq(Ym;;Jc#qEZ_W#pv}{aIoO+?(PsExCDZ`dvJI6 zAR8yR1$TD{E`i|g!9BP`aQ3%yZufcn-qWMMzrYxqvDdSnIcL?ZD)E|59C>>B?3%uU zXfK-o*^CEKZpJ( z!prZBWNzEqnTKlxGcy36WK`?!if8Fwl|l>sIzBPV`wR2H$)sQAPZ57?FfT;ZT(3Fo z9RtVLIb%n5=Arh-63DmeawkZ%{4OWQ8r>g^&Np@H;!v+SRKFAguTr!fju))8M@ck_ zaWbu%%fE9nP8u1=K-%!G$PMwtJ?oTB5yy@2AYb)+Iu+YGBMrc+nVsiT%AUxInd|OO z_rB#&xjo8`ia5P~xmILECQ6eOe$`@P`IciZoS7uv8`E;5@6phJ2y@`&1$r{y3ZP&^ z>okqZ5q*gr)$|oej&O;2ao;lfACD-6FA$&}BYh6@lEMgiQ7H^o=Zx@i&b{A6Sr|e3 zV|)VKydbW&|L^%WP>NE?QR#ipy7vbs%>xwdcWP33bNv-VYE^B0SS0gNA`3^@NtfO5 zUzhq(yQ4;gFVcRwX7@moWsYGQ%ZT^3l>AXHyS<+Qn|3zDaqN#fmYd7PX3$+OfuYA0 zRAp%acR6f6-=F0Ou8f-8N;?C~L5grPTW4a%yr7Z$$`k)65TONP%3oQ)HD{C~DwK52F9$)6HuAA@+eBRwWb$|JAbGz7Y ztPyUv)dEB;D7y|*FOM~H2)d8S$Y&t1Up{Q}lK4)^x$GnCm4&NUAcSN%bfGD?r`1uL_VRuHm^{k_%$GB9Lma_9&;T>S7C>V@^|H2dGcQQpT z;l>l%$L>G)Wy?Z+G%=+(9E?y@6p!XTWIYop_q*mAo^%>gr&s-0`{*RKo&_!@=47+BiP=XRt7tR{&E zw?5|ArL@%T09PW}CQw)i9e;E{{*OcO&kHY2IFp+4UUx33s&Q*3i>L$gGQ7va3=lo;s@fut4`) zisJqphdkN-wIZEKZyy7c%Qa(Z@A2qJH03W)>!j-MZJx|;0k&Y>vge(|teN2CH{|E2 z!dmn^sUc*GQd~A|xMpW77S!)&{VzchZolmoC#3Y-zp9B^+1gr$lT}$+&{?bzjQBcP z!#9?Pdwj>>P%qPMmll z*^}D7(E&}dLJWo(*OM!eZk3~_hUFj%!yunJqcE8PAFkZ$ACTHeV@^t?6e*h};f7oXsf)sPUpbZjGvs zWuuD;b$VVvjbh4x1~AnFTku(7A-t^ObdKLuz2cBwg+h8@{OcHI($giZ!fj0V3yDk% z7`^1?6Rn6|t39!JvWE3kk4UFaMu_8lCms){Dv&eSIQ5ka=W>Gqy87+W!npWadxjq0 zQ;LS4nV5BYI4;GqMb3B$s&Hf6RSpD8NITS9EWR@X@3H{nrMq=TG<}%V##q0XTg&B+ zC9(4^T_&7`ZOG;|c9SvJofw)KSpkIol`ztY60d87Mh)F0gI12|NNo0AzeSh321?1w ze18z+WsZOS%tV&PR;@;o*-w<7K<}Sym@!0F#mh}EE3CaZkhq!u(MSb`QvzIH_)4K3 z`TzYUe0G4ox;rawXud0;z)tq#7K2N>T9Aos+f{j*izArOY0Bit_9-__55cG$xr3D{ z22H0K*G;ubo)rU-bWzclQ$Iv%`E2s6fe2uu?y*pk{%&^OdJmI?MDeou3zI6@gfP>W z)Nw=pdETu+NqYZ#eFfkyuC+-`UJvLNaiavy+K2`Dkh6}UqF;Qo)6nT$&gc8tJKf~t zQNZx4^NbIZn%XNRHk%2nB6)^1;PVx?uQG+M$zFNVyF0gK*FuQ#^u!vzon=Iir#n`x zGj98nZjtLbdWrD#hQph~ZHk^#F21h-mA3tEkh~$i)KC~RhS;Wi_!s0Z%<93fOP`k7G^x6bFM>q!mup>}I^h?lD-d^5-XH_ z2_hAf?gfjAs6`v~N4R-|hh9I~IAaJca9Dmkp4PlP?IS=``lRaXiju8@`i@l<}S zF}W`V+zn>vrREKDKX+%X?K~Ia9M2}E{xLAI!XRvz$E;f^_H3&h05}l~3G3pEy#zRO zlqt-ho&G*B%AZr)e+=^`Zt?wRiFSYkhHOF|0oMPUcOesl-}QZk&&px@sMB7WQbljJ zrE=J)BqK-2Df52b-jPA2s3ri&;Yv{Teou`>BqVO}_LgFVbqQPG=##xKeo)& zU%8nLu{Xtr2r+j$(=I245wpA&E9(I0Q6mV-4uFq<&cEQXeH(#NLc%Mjrew7qEMG9+ zJeIrHtuLLEb1dE}BR8EtUqN^QANJ`jzUe_%U<8J5X(CAIReW4Ss(eYqD(s%i!7OVb_fNj~nJcTH4L8e_$Ys zM|Wxf(wN{x;8JD?`FzXjl#OFdZ}IV_C15V#>wRvfD?a6#vC7oqhST$OVlZR%ldZDt4UScQu2nWG*hXm* z@Uvr7naP@u`7e-;_!mm-bTT8N<$4F?e2hoW8zanPlfokUM*A!Y?4W8xhrC2uY3!SC zmPUms%rVgH4%fdvZUxz(9d<>~xwv32H!J1=-4Li4c;o5$+gp0jO5IPbu^`nsZ_n%y z!U-=fcQo2g>WO4z84YR{l$z~7e&u5P`t==rv?K2E%W1RN{;BAwOHNdK{ z2U>tSj;8Bo!wa}E8i4e!bjQxke~p+FftiY$qy6pq^_Rh+VIWVzdlv!B`4rAk-LoZ3GLwp97~OO2e8NYw+aD}iWx+Wq8g=1-m{^9 z@d^0H+Mi?#g%d_Ib}T2`p&#v^jF>cqY~yc$Rx?|03oS!23TF>Mrl;phohQc~{qQ*|N4fH~7X~zu zNq(t%@B<$7`>bgu5DEbsD@oiA$B?q7b3y9mDdX^0hP+yx@#ebIhZ{am$86opv!YZ2 zwJrNtl6(#bq!v1ltOfnRm< zAeFuO;^)gJ3CcC7)oYQb-BVd;WTfC$g>n@mcya$xe?<<_K0!DfI-*20o&2l|2Bf6Y z0Go^uapg!3)RHNhTJKxyX=;rxzmd#5M?i?U2Y_j+D+MsOs+PQSzz_mIyIfK%Wvti9rClT;e z@(q6n7;@jrSb#ckl0U2aVm|{~_WrOA77|Qe-}~B|mMX2f{6VjaXgQIOWV%`*!)Ls} z0zG9$*1wo`BgHoq)nn9Ag|}=Pm2&VNf^=Cp8WH(lvIoEo9>hv2So(MJKbNEwBEzMW zf}mj_(5aPS8|L^+Cgn7%SLWyE^_%VKS!lJ12Ae&32o~D2`Qd5C^GBmvD)vBJH^5vi8Xx8HjGq-#Eel%ZyinS`ex-m?OpGpHosmeR z4H$SxLCMD0k6W;$c0V_FJOW7VR>SY!2?zZOsHofYy8flnwh76o@0k}f!9S^E9WuUR z?@|$BS};>wgf5?+Am#xWd%AzLvRPZKx}K51ZJrHplpn)@;ZlMD&ZvMPbWs}F}#v~AHsXPiSBe3XvW8(hJBDI-nXm7Ip zEvB1N`$1RM)sAbSE#>YWf%BK2&nx*#SxNF;`N*R*G!-NUM8s!t5ehQ&I5;Wj5o*YU z9$gkz8^@wAB!V)42?;8o6?Eyan|4U(Aw-H5NCf^!!4Vp;S&+u0k;?_h`wIGNHN53a*FArPH1$LX&EYoq* z6vcWxDsl;P6|ve+hp?_ETRwC}m)aq1{5s!oip)pGQ(dB+v%TeTHF%xVoO7}i)U$n+ z_bmpKL6=qivz zqSRs#FYZBw=dV2A)F|fybCg>p3`Er~f zYvu*KpfS&*G70hFx5!M-6YyLsL+~-D%S)EyU*EO8xbt9|WW5XM3jWH?b?}5VJ>~Ox z3X#Jik><-;0Q&5}wQzFaL3pL-S(JvSHpTo46 zSw7V<%Z+tcHOWA&vW{A=)MEjWc>Nd;leC-F3Eot3Glj5Hh`czkU4!^&S zSDYH$`z(dzT$p?uI9)O%=|nj3s2;$1z6xWK+kO?#4ud0GoGcF=$S@PO1TMgvC5s$t zS6&l(BDnUQ!B|?RlQ<1;IHK)N4S=g1)9vXhEEjY;n9951Nag+e5R*xxU7W3aSl8zt zPR;lrW55!aX)Y~WlGLCR_cCVgL_UDkab1jtY;($&v`|qq#cD(xqbB*i643-?$W&qi z_Gl*%OVfk@nd!PZF{C_f-yi-3wRvX^nHGIL3iSz z9i*e?JoBG-0M`higm=XdXs5|(i^0v=#P&1*3ex88vZ~IvZN{XKx16&o9LMYTIW3Cv3|RiiCp-?JF>R1B7zV0IOL2 zm|tnk-|fVg$$j{*n~MrkI^<8c2{S63-;26Vem9VVBx_$~5KKhOr|f+`I_#vN+vk`k z(9yA1?{d4}OP0PEO%{btu7Ou@LL4(vAwt3#y*K8mg-dbXzqGsd#H#+hVPeXzayQ>^ ztUiIdkr&jD2qPq+kX|rA_?Uh<)N;f(()EO+uQW;8jV`tEO^2zOu49Wu3@o7zK%6HLg)}w(`3GXr&uV=;*Y7YIBSUO7AATQOXu|gZV`kx`@ZFV9N-zg! zOkWtQx^t-Koj$6*Uz2Pp0iy8@p~HJMv(1^(wQ7B#b?t#+>;9s6U1gGFMofL*x)|@f zw^oZEX>)A5kl$w^tob!(z)z5H&})#Y)!DCA5(!I;^9+5+PNIWk@VW~e-d*a5U|7BL zjQB;3nF`$f|8W3ha3G{NnYDdKnQr4LC1(JmB}4Ov(dd6YdJ4)Z!1&!+R$SI}b<||( z57i^R&K!%Yu~3E=*u3r?m8)iuZ3k#4sfOE&0zftiDVJSqMI9Fl*RB&zq2U703H z-Ht+g-46)#&%&7xC?vUJG?CkmnLZ?D@}IE34OoDpaCQwZzEc>}Ka=rfYB8u4-Uw+P ztYWTzH@_HOfeu_WsDWrrtE_11jm5?ET2;GzwJIKpKilA*sgnek$QR!WxcnI3wRBRr z^m-Sd(*B#mW;}7I*>Dog#s8MtOwURE-SEhJ=}?ia{KetxQSYWX{KXCN-ydTVE8kwS zvZ#$n6q}k(pDd9)?qb%+tTQuVfjHbNl!HtC{TQ?S+d%Vj%xV=H)5=v}LD(+Rf+CGD z5V(3qL}W1gd-OkSOb&mtgFIxY_#8#@{GPDZUnKH~`F(Hpa}dIKvsvMi$gpP0wF(JwT9yCw58g-)k)K1%bQdF)i?ax zu@DdZ{C#nZP9;J4-+ug$!}DLiC1Zd_5PlM;_G4I*EQkcUE@tAcUmPO@E`(+r@srV_ zoy_1hnBH6~+&jk0(=fH+-P3iAReI5(jYT^AURAvmD5z^Bg6+z=(d})7G-% zYO+CP&??Up1xy9pGJl^=pAV@51OJ zmD*b&L};nf)h&}iWaM(NSH`wyRHNgCJlF_$vfTLy;UDp>cWuhYk zmR@SS0btR`gHhk7N}u|sxP%mFEj>2;S45jP&dkNk$UC3x8T383B0f}R@-Kcbv@DUC z_=p)qae7=+0vNWZki1oLKU-NGZts%#7#dSJ94sZ|n42{pYUjG{r9l%(>HrmA7+f2L z;{|Z_xQU~bQSfdDhqJ_U~BXnQ2%-((a0fz`Ec*FT+3e6q|P^PmMXci1c>w~0=y8FHD6tal%@VOi2vJT z7r~&|d^X26+u#x{4Ixx^%N-gB5khpp%wfM@u2mM)^{3n$GBRpuJWGXJ+u1r89nV|U zRxR7~(*WP%*b&CM>Kg_~drb<$3@vI3Jy}qB)i9$P+JUaTXXKj{m|>A-IQ?+ z%yE#(8usWz84q;y4@_$#r)9lhL75aR$<`_xD(q_lx(kx)db#SJ3u*MDqP6ywQWMDv znRU1K`Lq%U^>D|oM`ap(w`%#aEW8TbYfYn)K#p4hHe*MlZ)@uB6Z#{TROajHdNY=C zt#~zx^A;y}>$%bcgJvD(cP_=jM+;3#li3P1wS1O6pD|4c&=zekx7GN-&tj0%51n@^ z@FPEp%nrac?Gyd{CP&?egF0{LV2?&AC6~)2V-J$tp++~I>0L*e?Z0vkVjd9v4%ivM z@!p1qy*=~m$O?_2g8D%mxLS6ny0s0mZqweO9T4JmMm%Ck6M z$m{2AM9NP2-?kiq+X5C?#1nzV|1Wsh`bXJU-$DF>O9jmTCYA|oQlr~Ro*Kxpt7M(@kFZaNX?;M)HH-{ zAp+Z!DaA8PX38#9A`B9nJg3gI5npB>if4bqNa6aR@{%I zw!WNsiHj64rU2AI8w_8s}Yd*f5Svc9HEI33|M1E`uFvFDomO{W+hi zaIVr^ogC4kLBLUHNpPNO=qsmmTWejO>JQdeR#&wcznTJSz|s7u3?^-{(*A_Rj}cu5 zd2+fd6q1vC?FK9YL7i3BBNuu5R0ee>IZp-Z=ls_qtGl&*jT(7@Yk^xe#27rI{l1S) zHiKc&zPsUKfPbA{*-tqyPR9fu-s4N-|7k!GIU(iFw0|OMLW_JWiTHBA;D@ z^4GJ-xbn=i@eEhxtxv)Y^xCztzM11quReU<$8;)J&V9vt_gnl)%VmT`m+0|N^=dHA$tu44ceV?pZ zq?dxrT65Ecoy>30Ds>BbmA#Dc9Xe_wgRQ;RGp(^3H5vmdtqf*gTC0E?dqn4CqXxiA z?7d&TP|YkbJEt$;;y?H3ej3fsaQFu7QE*!?P7v@=pU_1SJ2j>-SIyNI1)uH9b~_cs z8u*NVxZd}%1pDU~$@=-eKkP-U0&rLQxuU3;k1_R}?CHR?q!)6FxHFK2N43u9w*A31 zW*4&ap*W1hlgyJ`g6gu#XjJDUqD03HKbGZ zcFB5inmM0cyQmO3X-tN<)8H&nv(4GD?ZW3?7t_i|Aq_x zd2E1=KNEc@;K@y$UOl6-$(jqOS_t)xTP*6z$;Ky6~cCyPSC@<3a*9Cjha$OW#mKL$< zaw9H==bCU0W!u-w6pb@9I!_mpJb+cSPZwW^OdioE9#;dR;6T})y;TfTzxIe@mHXXq z?xlOWNmA8|MNtWFP$9Mwwg8Y=<1631D5`L#7^&}Okwy09)JU=r0Ov~TxAPt5M^*m0 z)vG%f6ui6;Vu*DMMf~&QJY}y5{O__fsig}74&Y^F4SG$a1ozs+!LwxkXXN}9Urh?|wMYH53`zYzOOr%vP<(FBcePhaq z`1;6i+j8U4UmGT_DkJ`U{dKX{J5o!d;5(JIxfr$NzHp=;2D+WI!v-}%O9~Z=ze6n& zI)}AQtMnwW1j^Q&Ed)hjYP$9G!W!386FWr!sq)UVlKchHH=+>z`yKKI7aAzE)g4*(P=Me z#Fu@l8*(&9Vo2da1Xk_1IKs<*kY) zCm823R@YVxq6&MaPo3oTM|A>3i4fn@i5LNUuL7!Il9(iZ!sX7IT3!&Sx?Mp5aiD^n zh&O*KS+gxf?B4wPjT1mR5Q_4)YIABdF+=Efb%b{iet*QjijP_6o6{7O$(6EBP@qnR z|4s(8h?Fj&{)BFw^evd6_*5-mQf@_IoReF1ydZ4&n)3^3@V2fZZ?cN#>ZaxF7nSAA z!JRUSqUT%@Y1afE3mWXPfLG55ud#Tio%@~rtDJv6sJI)rq+=d8o(cx7`_siQRhk$> z9R{qedL8m|-rkVO3|$iRbaYY&kJwkKk&W*z2cPdLl)cOHI!g+ZR=KrEV#6GhcvfZ3 z5vUav6xO|Gzmo<|1;|8^LhjXVpywq@-Ku$xYJsn%zFADqM|NjS?yF)p88dcb)IUG_ z-fF8QcqJ)K%Tj4Kx)~(yJp5VahL6$d)hT| zG0oxJHaI>A>fV-bi3w&hIs(AhSn|U{G82lzc{i`YL90&Nqb9v&D@870!7=kE+hz#D zHVx-pAgo;tNP}Ts!Hzwf+P3itmBbsE^#xxVgMsQZI~31Su01Q3s)rE(THF<9{-(0w z7##KbD7%U?UWnAc21_E9r%@$y#=ksMWB@FfTgXwFaJtbCvxNz&AGGTceUFM#P*{>= z=JYQ}N$*y!h&DW2tu!pnXHNA6Kg%l+k+Nz;nelg$^G7-B~0#m?tIRI zyE63VrdzuCzL*d*h&%JoQV@6*EMm;AYtjQq=mLuVZ17s-_Efpjg>M*!P^-z7y8Rp2|M;yZ1eE!C_sn5!`q<^vShZax<$A3e@i)4LN>SHl$apyDtZ-KQmRt+HJAf&If68Fz#m5#65KndR@nuvUP< z{0-n*Q0hl}hJKUPg2QFAK-EaEhq+2h%;!os&1W-e?uwF+d1g^%W@e@gh?^QG8a>Vp zdO0bko~b@mtpN5SBnO4Buz|&Z(;A-#C9FWepNw8W%(OQgVi)p+59=ra)&$~RdtKoD z6al5*ADxryg0=-Uv=PB1{SB?o2SPM-Q$TG~^1G%b=jDa*G2o{Ez;OEPVhmVab%+r1 zK)qOZ0LBlMCzd4A(?=j%`zY`U?vCM+naFa{lV-~+YTq8vL^aJLdvyWCSZ@$~H=wW@ z1@jzu0LC|!?)eUiKoXboc|~rIFGZUJ`T(4doo|85=I8x}1z`9R1-t;J_>Lze+$UQFMv<0`|n~CPc2n_|0 z=J;L>y8`F0Vv%r?X8L}Tj`aq9h<0kqAqwg4OSpK1lSZK*?2IHhybEFH7ocNSAm%YS ziQXs3Mm@-3vlrUWbeVJFtTytZVA*NK`)Of4*!F7k`3_HcrB^W4M-ZovA8$Qu-D{t? zDfYV$L^Z@R%o*_wsVD34QbWnI$5u82ecuz@**-uUtGx6Jg4|I(1e%pd8Y4c&5R{!W zcp83x!LBeu0&$mc31FIdDfYdG-7&AqBVpG4HpeIF`o+frP!xykL#!)Ko#(xL_Kexb zuO`es^<2+&m||HL%?Y~vl)JUw%kLqZaVD=-h?g-E%WBlsm z%)U*v`qO((lW(ro$&<$CXf7a(i9}5?0eg>JMjU1vPKhk$-)B%mpbe-c$@Cu#ZEcE1 z+FbM9hBAax1$4O)3oO>0am`SO4Ng+I{3&=f0~8qOe+w@L-EA=Z9?a>T6v`tQT}Ytj z;u=~DFs{;A1M}Q3V)F*#!eb?R(jw6|f81kz4>gXDS-~hQl+k?k<)l+JQRb(r4fbUk zOo$Z-7gl>DJzP{vWW?#s;+@UV!;(r<=-~$dDA}H*8*D-442+8NOBemYgtX`YY;qECL^DUs^CB}eTKR6hF~6S1???r509R28*i=M^)83}LJ6USBrMR#HzB z$4*kv&U`~@x0HsYu7^d#N)HUptt@h%h_dfBt2?e_U_3#bH0|^r8-C5bRDp3;Z;NYQ zCKInmC}Rck*`|5)+z}FMu`~v7e)UuJ)Lu$Jq|H&uq3&f02XxWNXz|d%(8;MS6cKtd z_tIdPy`hmgzLL;S`+>zRK?8fREd)d^iC(5@m9v%2T<}@euuh~y7k2<94B>IT{+$6@ zUIppMAeEL;U0M$-5^H(EP|Pq>uWQqnjyN%+dSmgl$7>UguN;1lt{`}>WmDN(qL<=s zxxj8Ydjy8TikhzSiy&B|pFN&|V4s@23m~Meo+^4Ojw~j$1S!?97bMF;4sb`o$0yl{_^7}Yjo><&D<NXi%(ham; z{B>E#+27!O2`5qI^phGk2pzlyQpu-LM5JV(GY8y&0yAG1-6K7~#T@((ONRj?|S( z+JOAc$oX|om9a)qFE|;yRDJh!rT7I`F))m&GYq?vrh1F%RZ)2hrWL{Kg;Lf|gbY^| z&$?&dc_v!-Pe@lFN2A;rEm2#Ix=soqv<-r`lmqC&coIsB`^1_eBZ$h63Z!5d)WnoN z1Y2V+tLOF_m%cAL4RdB^*Hq_2O6FPK)ph7BQnf&dhPq{@iK?)R8E!7`HOdX30_S-S zQ+?uMrh^LOD7_QglQm`3s6^upb3?>dEZoN77Df5&-&?awj7MVKIBYA|yJy#`GaW|i zU&@f0UWKT3l(t%2zQL|MV%z9U`h|4-o%@NFdVh?xGrG|cEG&ZEU#XxxV(7fe$Oxr? z1FTx|&MUxuhet;EV4d0nl$co2i4=y<-I|9wNSS6AZr*IX803#JsZB?=;WS8GK2oyO z!l9>38j*$kjBxrfO$3BC=^vj^CX9j=p8mePd_YW|c#%fugKi(EIS?!kJ^yOTT`K#b z>N1qj97>D02aqr# z-Y{H)l~DYjF9+wlUV!%bON*fK>dUIjWSLvvd*nmuC?bex?ObypS#OgJX48@0Xlxu> z))+(JvKQp|z3{&>x=5Qd7LZMt@3PLNkfAd4SeL)i3Zvu zH3WIo9&)FmJe~m!aarU*UhlwB*lXDE!_iMX>ydc8xsY_S>M!K+U)}*^#O4o)&zi09 zj5XqASW!P_yl-%dwJ)aI`8+_~c@bj(HiQ$VwWfh3l%zX>iHg>Gdn;!(A6h@4i1Bg! zAb`3Em5AZyn%|q-;RZbaLO_2_uvI17(*x0>_LVzq1{ReQ&S@1NAppOl-_0tGnlZ$< zpn~Q&Dpc+IQ{W&pP__!ZG_eoJDk>51Z3i)!ipOA%byB{DZj>H4%H}HCGPrHfF7Uz!6S*VLhBJ7foSnzJK7SS;&)) zf`(YQ*m}UH#TV!WNsE#{=7p);^A%pUTMauzKQ37qyoMe}mPYpm(RLPq{3nE?vVXc@ z-roB`WHoIe5_Dff=s4YoIX|2f0!ym^9xS>CY&g=f@KDn&?$vO+ z(E`k=`8SMxv7#d#ko9l=7qIvT92Iro9iSjnqh$$rp#F_ZFEH53E#W#5 zS;ZjDjV|4RDtLodSDsY%Sa_XEE@u#hq^n|bMOs+Cx~iufSGp>4(U)>a<{8GJ5rE|# zMhQlstlI%h7p!-S?B#6_@Oz<5<>HO%1+|fPe*=~zKU~M_iab+U^o?~&<0|hXMv%83 z0pm?fPI_#sU@Bx4h8fv+CfI!II_I`-HV2lYp|7tIwAo6-4AaHmt!D(jJHiB9yb3N; zxn7DyLX{yL4qT@fj+PSf=GMT9j+|UW?cm$OvvM$Tf4_&MTK8Pu?vE-1+;ElScek~_ zSkCqaQ3)t6R_Wr}VTfKXRO?r4qaSV;=ZbGT20bC0^Z=F}?A4R#(Mx9?*9w5{T8~Hu z-v^_`zeTdPmX&B&P|v<)QmangZ=+|M(drlan_=vz>wNt&Q09e#JK)w)aqj)g1qB5j zlA$8FB1*#8FIH@%vbUnEFO&fMHkGV(!h9lA8hC}6jV>)3e*?2`#bkQfl8_)%8s9nf#@7bDqc6O}2yk%ndupPglARz# z12HEv9QJLdRh zVU(9EEE+X0lfD>hRNUzF3RgkIE7v+HH>e@=T1x$!5J8giS*DVWV+%vHr`-W{uT$%$ zimnK-KRb1Iir7t*ig>+>iq&am*##{=!gQ5&bZyoRs#v}}zRO00dtKNZ6|&Zf!A$N! z6k$-WpF(6Utj`SwmL!>iEN`tzfz-)>XoC-Pg<&{>@R)i(GFGgzkSBd^YG;b&t)2~X z#H_5SyGxBvby!((T5j*thd913b)%dmo_! z@sL2-^^L$3Tyc~@77)UT7@)OZ)kDw7$f&2Ju8w3nprTq>UiH3BW5yGwjYiO>cN%5wt#yi^8za$j5Ajy0eRO)Ss@as3b+3~h<%1N}slqLWw8YDlV?lgbrcgE=UqF1DhZ zw=|ymfdr{A#Bz+lyn_rWgcHw&z`}Zf60Y6j!F*$#PGN^*l5Mj>`)qs(WcPAa50sD@M<6j&l2x_Kp<%Ig0fCKtywNl}F!;|v zB`iN`=wZyu5}_jHFbyxeG&DD-nXl4KWg9@UAgd2#($`=(^6WCfg=lDJ%RT?WHQ2;s z(-Jn$;>wX>Ubw|J0ZEQ3b7y|Co2d9}c2g=CNPn8S|GcIJrOi za*I0vf3NVw-rHWcK0KbxhbP|X&OU8-cW!;>VQK+*uvkm3^d4&cBapX=<8G9c1#h^O zeGM#2E8#L3PfH6Ri+lS*h2sVijEn%iV!&kgVa0!Y0SH9V&Hh0t6W}0xf;mHq)SodT z#5ShB02q8+lQ*(H+2jVloBit6(b`Xmh}rzS4zEf0*~t$7cR=pb9L(FTh1}?GiN^;) zxcqpXocA~4J=^bT!1rHB@(Oqw9`%jxLF!&LV|kQ)HB^&l_Z_ksOvaA2a04^n(f70| z$ls_-HH{a&r?c5k5@*nfNZPAv7I=PXiu-0eeW$Fn_OccBZk?J9bmIQfp5J>{8n3OW z1hXz8@OV?lnf@+|rERF3Px!GkY{SL)oDk}oZW6o^u<@a$!x)A2l}Wm$`z&{Fbhgz0 zg`z{2fDmniwe`m^qOvRtZMOR!F&}e>*20LF_QO+h46fD6!&639v0UA3FlzAyN&!97 zcAOZed10-k7=qxHROAtlSACX!=lq0A2e(0?tw}r@W0S4b*0~N24Nt0Gx&%GxwMHRK z*yC)nZjOoB`sCqRH|@-?pipZifHf-nw64d;*CRfJ__=m`4^-8Z+i(+{?O58C>XW&X(xm31t0%}MX1l8XSz(}0lTVGK$lak za32f}u}%4HbztGL-Q#?L8p4f~?p##69talY3!sEeqQ!DTi&W&<@Z|*DXEND%&uP!a z>MVe~`XdMo>TmO0Ds@mx5V|hL#)_suIF80K_5p0N2D%r<8C}?dxcw`e|Mmjl!}zSw zW=Pj+;Qz44a8}AVG|MK(WMb?Bz$Ogokh+7`$W+pDGeJ|>QAWD%(+fqGh-4MUQ8$1! zWtq5I+!QcDiz~UHG`%p5gxH;auzMnwO{a{Aa({ExhM$lqtWVjCb=2ro;8`Jx2Ahla z)~o}liiuNQ&xIdWX)T*Ab;&w*vfnLVVPmc+_4@2*c|7;SKUM)oQr_~8#E{5GV@j~L zKnIxXG6Zf3>!qJDk_DEY2tabQQ0o9g>)J%!HNWq#xPr3-pe zvMS4C-HRb_ZKK=V)U=e!aJ7#ND^S3}lx!(*$YMe0VHeXIaG6F5=CPa-QD6TNL?ke4 zU7raj)uFT}&g-aH(4Ee1@==Q73jDGGp5|KAGikAeknst8x#$HUf}lyzR+Vw8jP&A8 zLVbiYN4!4~v$K34oF>505MiFToPJwPo-L6Ii25FxdLa>3OWhGoAi_OpGeb$H^gr#y@+ zA!gVg72@ihNKHCEnF4FsO1Q+IH%qu~%9br42YjS7ZCX(9_^svVvR{lY*4Z65MY5G% zzn>RA^`DWFRTp&Om24Bg4ZVXx6TkHBp+I2@pCMR>1)ScCzoCL%BnWAgC#Uyex8aSe zES29rx|FWiAfJ+BV0Pk?2@Rwj>^rd<2}H6}4RyV*ZTTU!MSO>u4vwpDVOnobX%gFcLL6xoFf!gtJ9JKS8$1{NJw^2 zlQ10~`4e~rTj`(oA;O>Vd~#8d-c8@byym~Mvl*0v4=ACYqyw~=jQj6V`EfRN^0RUU zw;qFgI*t1g0+iNRaY96!+JUtsKH1$v`Q7Sc3`m_G5f9OYT3%(C3{3W%{E$*_zOE<>WLCeLQpB-W5QJQIOmQ> z6Z9kZg#>2uc~pO}L!}8Cq#hy-xc4+DiMvc3VV_hzZ;lrgjW{HKwy`P8v;yG}*x462#eJAZ2$(Lt z#ldgn2$Z!><jL;ze4W5Gj4-zXjA00GZ1RUo=0iw+f9q9h&&5tOIrD}_=1XsRqytp;K+aL z4zR@n+3L#WPWLlf!O~2om74(0d{{I@3^ogu%4uoEwHz6}4D$$y9N0AL3<6~}j0}nT z$ScB$3|QHp7W&zpKT*W={t)WB(V|v1x(ZZw1r6i(dULR^vwax~9$x8ng&6%7_w zMqW!(W0i9mPM{8T+CDUz+W=WlzCKTbp%?UTa)!wfD$*I`cCREcUpeR%j39%IyiEq8 z(FYi%C2v6aeC&B6uHP&(mfwB$q8fna)F+Tigny5~I>eTYRbj*+w}4h0+?ZBq&XDB^ zBlfhgO~Gf;qzD=gByW%a_mxi~Bw8ZRia&9f$gQiUeP)$|os_4^xE82flv6=SEK}3d zR@X}-IeG#F`WZ5afoYqxa$&gPX$nvfF=x8bsK)Fx=ZK?W?gSH48{yU63&=0l)sPOQeV&QyZ7YZ;f<1bU*k0 z;GP{O-bq)d@n@(MVge9ellsiV8&C8cwk)ZU-ASZ9j4c=*4R!Tm`3&>&?%vxzp9p20 zq8HzbMb_+Yy!ObW#k>z;;q~9!!|8~t2-joT%u&odKVF>CB0O^0`GR@RJ1`g`$b@OE z35$c>CwHAxPXnd<5Ll(&$wdv1Bd}_~kE|U0C^cleX0c16S#_FxU`KeCFmxCYyG8f( zurM>g31)&z#{xFGfLL=QQUWcOXoo)h$DN+{AHjiO0P!6*PCtv2Htq=WP%Cit+;SMY zpHT&XHM$%)PZyV@&MLwKltrBD4Ri4!qRxN>7Wv(sYCIj^&IYUO(uLxqpe4S{~qnmLj;+D88`Eb_4OzBf7yqZXk!YS;ZK$)=iqw@xMd zY-ng`=k9gNggM|NY<7CVHq2O~86R>&9$53cb3YJ)aTo9oX?OJgvOL__cN3 zhl1K&|33Tqe_3}wb%O8e)k?FUyQQqJUOK%&?60|?~4S| z1LC}B_B$RmY{1JK+Q??#6EA5pw=(LT#Ntv8>~kmzaXCYT*u0`?L|fJ);B~Ertq;T+ z!(c|5%bD&oN0O+iALZI2-AY#hbf|RcdzhjR{~ud#8I@(!f9pzjcXxLy-HkkScPbsy zA>Aq6-67o|C@mr-0@97r-DmOcvHxeEv)@mQfevx--0NPynDd(7%#vI9qu1Zyyqdj& z;?qcyC{0ERqUM-G+s0cs+a8zs5$%ubxUfk5#F(4h7?1YgJ_6hh$49&jCHI+$pH`Q`%ly%$QDIJ zn%jl62$x-u@F#?WRIhl#%!MbsXdj3^9#P&d1d2~x@z>CLm_WDIQZ1F1`1po*HJeRi z4o{|mKfVMXmcJke>pI)T4;?M$sf40h@qKkWup6eXV2(jO3`cUWsrbZv_N!xE51mir z;#{^WW7J(`s$OzX%MLPWGi^WOqGC!~@y%TLKxYr_Lh25($EM;7hM8fT!m-;^^Ac0@ zHHB|~JBrR8p;@_1aQzr`B%uC!xdsEGY?}RHXnX^o&(Vk)#dJqJG^M~L@J`Wi}(7bZ?fcb94W8F_4 z1yXOUqR$DhM7tBkd)f*L$i}>a((dBpMd6;6zH(lHC3&uo!_WLM?pU3l?6G-H)=SzI z!X+pwh&O3-KGClUVJ`^??vAO#G%?>zj{Qs5zd+cN%Dk#6zwat%!s?>ccGjyje7FJky6V zUu(B+G&p_JVW3B@=-%w6O8>8&;rTY(KZnl=T-_oEB}mXl`9X9`i}hq$@n zRVX!Mj06g!ZZ|0^D=WN{BQ&_~dk9q@N2k}YIEMduTsov}`g4EC;lokpSeKHww@%FO z*;S?DZmZN`6b>)1BiiPNzqh*vw>@HUKytOJtex#%o!+;4)G2YAZ5(|0j(0{0O0S5#gcXO!$3zva)n|5R@hgM+ERxaQ`+y?B|P?AcHz8a8LwIqhpq~XynfSr0AgEE zfAJm0#?Qa?eU9yq9gJ$qI~HA5NkOEYARaPFC$nQ2>=qu>wneEV2*L(RIO#vNgtHCn z1?l{)$qO-kiE%A7c`avuI|i;u)51kI`Iz6>Ms{HQL#I9dOAuvJFITosj*5BPzl9>hnbsJA(}bpus{#WV5<~6TN=fzIVf(C5dWR2(7A05 zkf0o-Z5Oo{-m$6O;LM@_cy6o3VHlP z6l%p8WI+)qeu9(kldlB^-R{dhT(jmE>g%4ddc5pBY0SLO)juMw*014cG=IjaSz_)9 zw~Vwyf8ilmAg~^p(z)cCFd`)4HOOPltJ&$&4l?iR+jz5!f5c0R((<7p{<`iRFQZJ& z)AbWIMnoq3>f?2x+mAPLTMm2hu8b=#ogRnGs3E)(=qj=>9b_o3(&2*N6VsFVyjYLBZMm7%|KTflYP@+2t;w_L7GzP@?g@O3y{|3&o?hLq~D^;TE}x8 zM#oI@XBFNoOrkLYuEkG)l3b9*cMj(6lEfm5gBu)<`n{wdE@Wuc&B}U#v-ZY_I;D~D|WlLp_+n^7e%f5`kX*{b`rcN8IXW!(q=uu z${Wy~dyIkbX<){ioG!QTQ1Hf}HWxtXmK(GBLj|1Fk59)rI64u4uev1L$>UrAk0VBi z@nK?6nfVm0I59Pi^ia0v*(=p?XNYo7}W#hCI}-ss)2EQBkSy{ec+!|ivgy?{l$K9%<$4=Rh6Y6Mi9)1MZ0Veym^Q` z)EuiG(sf&;=I_41=P&ljXJR9?Xe9ix?f+~kfY=IiXsOXE=4X$v(9>nQ7dDpVpe^dD zG%Q=9pZI$PSeT^m@;#!4x3PEg4-XHYQD{$;pMcsbOfQy1;O(B{uTx+Z#fL|mb3&rY zDTs%J#+8Xd8Fs_`wiw8 zUXrd6>@5ERKIX!jb5(G})HK%%%$CK?@)Y97=3*Cd8TzuOU=!g7ng%8FTEH@ERY|Lg zJvO_%f}yyOl5KwmDdZ`+xel03Bey`6VGdWV8fGSb!B4D#L<9S}$WV8SQoknLG(q&v zR(x;kHs&-e@*&<6mDM?J=x5tUR)gel&kg4;y~wj+0n}!dB(rqXcVOA}B`WOWaYjGS zAlXtI*J}TxBqduGTsgiRv!2~NR``cn0uDnw_+PJmTF*V9`*mlgT0`h9_cX4ZNW!4h zf7nvHUkeo(OS;L%+fmJ4<^>pLkdl&-v6s}i$V{@>$>IXRrZOhQZ~H#RNL8=1Zj@@+ z>|U9dI$!xIb+uvIs||(%2$+g^CRqzKRN1x$jo4QT`^SZz7$v91s`y>{^rcr?Qq)w6 z$MNfZt99;gd~+f^w_MKA*&syhTe%*{Uf08tD|$Z_FvR|q4aSHxTg8xnm3I|6!@V?a_ml_jnc8H@?T-QI`SO zKe9@k*ygzPfR38msk2n`JSFqLI2x=%#0PM^xNdKO75(66f2de1;gC}^{G1d`DL+k% zLZI>A`Fa9oOh8oTWj?DAGkV~+1>2q4W-UUSA2~$2H5<EWs zhlv;zvy*aA;DsX(!K+2*79#uI?UxheBRnb4!mR-8fppEuemzR$N6KK^L$IR%DJbJd z?$8C^`SqL}m?IMzjHD^9K$=<|iDOQXVeXCmM1!GSg+UB!(h-K* zt>VB$rRe?&(gcD&W2BE=A zve0jHB;$h`GeIa0Kw-Yum8YB_gfdhWP5^%mCHe$T4aFi>7j@FLxK(>`v%>Z-)q%<9 zif@sqgzOZUfm3b5K)G;0h_c0L+F&=&j@-8F+ul5J_YCw}!u`4u1l^v;D>aO?+7UBw z{LY-62}<`qVz*nLI^kp(U4ga~F$Ug(i!~W+kUy~_U1)LiTqOT65+s#-6$33T6iA9; z_D~Pl%i+=sG;-gnjGtydH-(Ly)R{F`$lw}ph1*mmHHzT>jlzeDoK~O}C_cqGA`>Pl z;)-3C$m3wLn<+zc;#%UU+A36bDmfvobC5|bjHLxLC0btS=pgKB?9Kbg>-^;bA3IDokF|56 zH;!aOWhKKwhHA!|A5)#b|=Z*k2g#!@^PwEaT5jiFnlspF*XT+=xp#aw z%WbBVg`5miTmw*@xvP7dqc*;=yp37P=Hy4jdJ-lGIzq^!sC{@GK-byEZH07-~A_r5& z9>?yA+rX-)r}=Kju(8BO8p}X7>Hj~p1A8#Epl8uATE~WTRy}G~Qu^=I3(&?Hh(hLn zjgK6F`Tw=C1}@(fG@rf)&g>5TBnuFd=)7}n=% z!A5IPso9`e?_onUa<0gstwL2+7G5HX=jV4vRaRK8F2EkBHLT+ z!HgnJ`?PqBA?*o|mRPt+Vq-?X{mKnM@Xzs`x+fSvl|tXwHcg5eWNUugrR3l6rlnTi z$Rru3E{sI^WQp|y5LYVhaVoCNW5aS$q3hoPLAahSBO5(!fg;kNOzdW}cGf1N!%9+4 zabS>n?JuwldSSdH-lEP7o#vCU>7L)GUmOF?w0F$RdX>KY8lRlMd5i5ZMZ->Y90#!w zCF+oF;SkxPd+ZMo)KVF5(fmXxpqeMF_m_3@S&A}^Km4fzf1IhXZ<;ty3HR z;M#@|tb8qjm^h;AHC>=IynL>`QV4wPGUdPC^MxK!4};DsGDoFDPn~qY8JQ!N$8H7Y zes7PgII7JSTQ^wRpAqTr&CZok+92SKjX2B`4@Pi_Ytgk)~Sfd7qlW zOT<1?<Z$A?+DRvUkiHtBjh%u&C@y!lQk_>5DwZXlqbU&j3&5(sNxbiC;W!sIxh`)uX{IL&Yr%1>INzxz=6}I*QFxu)~e&7>Q zGy=3P7~^QV`X?gfn|4`L*o1glTkfGqOUD6y0EvtO@wM*K+oDy9Qg5m(UM#11ZxDKL z>HP}NlaW4+f#vU2$W_Fk(CZB=%4119%@U7@0E())>2HYNELdLw%!OaG!Dx(On_UTgMvpiAiMGu7)?S@mB#HzH}FKmdR-Ng+Ty`hZ^ z!_?Jna$X-OxNr%Evzw;_dWx!kYLeH^lvWKJ5LWoc3lO|h!u~ssj!f;3F1{8uP`oFF zy*DU2nSSw9106LX=7*l=Bl^Kz8kB3Rq%5}yX4mJNTy}lMPdl{gWhxGVP}_TwO)$I1 z#?K<9#$HJy0wvkzl!XY5l|*jOtw~_yAb@&cvAqG=(v)fuY#ZmC5XZpuPgrrm0niLY zclIZzO2e?K?>=Gnb5c0TO|q+s$T84c962VWZ8sP^xCtVD7+T)mYRX$IyVrIZhZPJN zR205FGciq{0+SJf9&}7-k5>aL;eKTKiRt}g8rAUnMCX6+fFW{**yO%xDLnXY zb5gPVpCxoX<^IfMl{`mx!D=SEfR38hM*`LpPVe-4k<&xm@?>&V`23s)Sfl6q{0f7X zs%=l3W|< z?c1>3>iZqYP-H{OhjF8=g-lt?kyS@z>-tG`q6zai=>FLbwd)_yT!%jcmI9h!| z`Y+WUUAVX;j#q=x2}lJ6f$6=M<{G47NkmKSiFY<^ig2rR)l z#P8_*HukfBFOSEdM7mGx`!5>ubzp<~f&&u=3U3Wa$LulxSZVbZ4ZZsKTSbke^mwEQ}NAs-hR!C>^mim7{n1{{rBYvkysVfd-{WCMG$5*-82hQ;rBpX+)O0$)CaXHJHM>m^LEW!lvf6|FZF;6@%`5ShMNslLcaq zq6fHhMvI@2F+%6m{licG!;SmDVLWd?A>D;d*NuR}iN3R%%Uao2Mn} zv?H0O1Q{XhVw<%1UzctVe<)M8c($alHd}Y1oTW9MSAxjvOyNF9Ogf1Nv-m3ETj&K< z9=Z65&O`FEhjW3B-aE$I|2@MUs{wwk`zr$vVjxA(C%9T{2r&EPPa^m3U>%-HW5qJWsEd{;$REY8gk*|QRE z@5bzadp0lQLQf}_iR|>TbBknCveDFsJsJ#PAk6k$M;_uy=9LVy@`uOQ3$~35tI}zM z3`WaTEoSnQTO2OF%@b=&a(Wg$OG2y5N_TF4g&)g+M66Xx+)&)4WU zF3t$11bwp}D3UOXhaj-focfKF8s?TP<^iRhse15WBOLQR&lJ+7G8bKUj$M?Oh zXG(#5S{9#$JvF%ymR#-@ri$9lexGNajXi3y9ERH-7KsVKyXG3AXiq4t zGx=rB>o{}+wOQuX+hqE^l&@P!9=aYMZ-dQcP{?jiinyzU`&7oV9U~4^NIsG^5BW@f zHF7OF0Mh}T`FzxLahdnQs!pdq@7YPY?^IFFAX0Zphu1;L0N(q>)}Mj_n_|c{h1zn? zmBev9I?U*_a-Q<$0$%a(PcEJo2_nnUraSVAf&8u04~se7eqZ!JH-%#X2q8iIJ0l#J3%=DXK`|cmA5PjxA#e0s+0(~r2gcH9vf^)YbTT(B|Ff)2#OS{W14)YCkxo1j0y`CLFC)X_r4 z8t=^hoPnI)r?lc=5IlX!gfF;ODI19Yt8^8GWSRmF&DKpq3NRj1&irIhg4ydH=&Tl^ zGoE^3l;L-1R(9GyvuL&Nnl)-ylOLB@`TUn!4x2LZXOmB1Bp)3g7J=6UI)x~y7x87_ ztOXbtbPu?9`IxE`JWB|^@tM6TR-YvrqAJ3=dU3X{Gro&~_oewTEI)>fw<_JL%MeXd zxdf`UYP&YuVajikhirN9u9UN+Eyx~}OT<{11<|bPgy%S3YQ4Z@@fWe1a+Ib{J_)TV z4opPmwa}QWw)@R{MS4Fi)Emm*b_b_{aFyjD9tfDo@8R5(aSYGw$ zXfK!p9ehgt$?cXTyp@U{w?1d#+%Y}#%s+RBsdL|f1YN0u<~%t#){v0DgkCBx@pj0` zS@Na)0WGXuDLX%Ml|b|i9gVqx>1eCt@)%Bn!Mm$}3-3)_fm>3e-ngeiHD9z!-^uj1 z$p0EMs@Csnm6IAO0#dGD@mhbEAaEOSt_^dD1Ne|zo3prVX#ypx4BzKCg0 z*l4(+&)eoxv;$G6lJ#Ca$5Q*+%`6d|8lU1JU)ke)mn>Z@Il;IV%x4j0)-H8a-uPrF z~qLpNkjHdl}c$D`%Lg-hT&%hiXJw;>0(C!P+)m`PppRWk1DVf<$}9un;Cucf62 zMrP8u#^8gB1a(gDFq#L$P{bw|o`6r~VSN`sJ6S@%c)MZrt z{XM0)+_|t9j8fx==EbG%<9?l9>p?cw9?HJW@%ycIjAV;CdeyKu zUi>7QHy+EQ_Rq+G5eJ1@9)T<-S532mo$NO|{EA&Ae%~cz;DY}%>G_{Y`u}2nT|gp{kM24AT1u zmi3x^V)jhW;KoVo#O7^5Bmv7htCEx1T>oj7_P znA0T^h2A=^Mvs^#f2j^p6y$`4#06}|3C!01n<}H``eX6h`g?{11Os8paqUacOHLjG zEfuZWy^d)+2)VfQ1}Jv1nO^@>9kIx*&OCWDr2}u+fdjkxp0cYMUC1e@15xg`&*DFT zyL4`s8^2VVn|>f33gj|rCX9Y!s8$?>fky?pA$Ya!IkC+TXg>qljk|Sw5(00x2^3|y z-_y=B8n?rhY5clHNTW-7VjvD8n7sGO0-cNZN9i0?SmW5s28fg;;r#H|0(alDQHJ3U zRH}L{wO(sfGcUp{Yz!#IU0L(qN8iywIa+Hgt9e#>nDE9Fo~*=Nj1TuVDqjFKk(&NQ zlIgdHkJQ=GZ~T-jn{)cHOukg+*lgW@b-4KI{r664KmIX9-t-ZL9I$?Fvex26b*$9X zlUoZLs88~K`lC2pwqmoT0{Cym-~Y;z+(n$rZ?d(XFEukj&XSo*2zLrV^gTz#~|okU^{I7Fprs45WEZsJgmsq;x+FSD-GxA=hCrk6V} z>wmvk;q7L30}DsxM}_OvcT2}(ea4O}F2Q)TFeAI%Rh*-4q~lDKya?#fM%@;cSfhRb z>j4IpA$`lTa>Od{DT`Piz^-_5Ed2WuI2{5QVRAh^(cPGm2Gw zZ`jE{&wyTKk>;^({+NU;n}Ky%OfZB~9W(RLVTt>eylQBPPR;zkp2f2+oa zoYb8C^J~>GvOc2{3oD(#Nb1dcAcc?3b)WtzlcG_+vp`35bhx>s~*H; z(^3@ud@XOPuJ%~}+69@6c;K6UecrbWKt)qLoSnG1`bX)9-+i6-nG`6hM~9dvre7ND zd1K1eBcu00O-G&itVq2|h9x78fA)%za4ytyY;vvEJvD_W_}~Ae6r6T6)x*`M6KEuL zN=wp_zmCRJpu-E%Mcr^5j6P++4QKs$!O91y%V$5coqVF^Up4MY0{{7^-$G@n#GLdz zcPCN_RmR^*2BhsNzhY20mKRDLYS!xSFI!yQDer94xNW^YJ9ggY zuC%qu^Da|No1<^^yLGMoq&0alXg z7563>q82kB#8ZgnnX6d|Ij?`}=wRK(IhQfgRi5J-Eg1g(XMu$DrNY)1 zhFo{-p7S3_>D^zQ?jD$Czult@;^^6{f z#$B2ykZw_6V^^po6ywVESn;(6O;W5Ig_l0_gpX|WydH)kVtCmSg8D)N>*HO`kDH~0 zp7l(@U3oI`jhef&C1$QcWkbgA>e20!-%U6tm9Ztvra3HRI*`Bn;bfdQsWKj?z5&qG z0Qs{z(9Mkz`$ka0QKukP^~g@5*|iYgd65fvW01{u?Do7is1S|)2pOs7b$Dwydn%4S z*6y0Y9{8G((#;OP00ri0H{*JmXKNWe4)Tk1X9j7Q8L%HN8ml~>x_9TMOD`$pDPz+a z#>X|hbO>!4zFaYt>w;*C!)7xnVuwE&KUQbeHzZf!tmQn(_(P9$VPTDH^vJot$sYF| z7d+n2c$XJwuDls%AMFa?qwgE;d%p9^`!;xvgTI2utdi_W(_O-2)2S!1G}t9CgWn*B z!BT6OIrbj%v45y_IswL<>9BU#1UHI=^j; zL&bJoQUUcUh2Inra7$UWo;MJxc-`1v257X$;c}|s|Chu#2ZBi33|*hpNsfB8;j$;Mx@u0gp@2l=8Ejh(6HaJHA8ZhMSeF)~c2TZ`4nnwVnH+ijPw{YY zaBybafwlXSpa77&V^Y1EUeiz&DQz{w{9c%Wl)Mj&7}EVxorfzu*)NgDS_}T5)S&vB z$Cw9UA{X_0jY7STXE0#)YJdqb1Q2J`@Oq|nZraEL|1}H`$f&*#tcCWb)`H(sNqk+b z6f&t_?w%U=>x93h!j0S<_lV_7-XR2%2iW4qvNmq{^O4avodL*B`P{a%r zU(j`7llcPatouYMd=J*S`G+Zjf&#C-S{>;PT2O)?kVW-L-HrVQP()c&1LN&$EJ-=W zyIr91&ODZT{e6RzVH@Ua4ytVK@yC3%9?*US|9!nyRnWYxU=Bc0zQkBK?QBGuDQ|YJ zzOmcmmUi7*6FiM4$HnB1Ag$i~zd=FcGtXRZqdM5I58l$VH-;Una2| z&rf0yQTx^2o@dP`MObC3SB<0KGOHrLdG!F)0oaH=wrwx*jzQ8+LqCV=d+*){SJ_6O zlYN(FYBPJmo&)t5kIY@QL0jFT6EKN&x35aBh7BT6&s@$g^HX`WV0<9P>cf7_g8#)f{j9sk2fFIZSKikE$f22^v}aQA z+3u>-oW)z1LcspFsh5+qeyj7kiPt(@)@v8@cGH8jeqGEU-`6L=s3Tp%oP{{Qw3oa6 zbbnmlU@;tC+k^_~#pFE5-l&~L4X(zD5+jDLUwHbu_g?0_o2%0^5kbCf?|j1yppY?e zt%$*OB(6YJ&<=-!`+UE%VY`s!tl56)=<2jmlI(3HRzr8(@4qGF^*NGr0IsRgJm>Ku zyry}4I8z0*w;ZX`*eaP#vT?-&bjL}_`2?iPn}s>=Fw|2Q>S4Pg-oPZ`!dhN;j`VJp zUH2t3D?gYt960PmW;5N#a}E6QL%2d&KD?a7?DTDYl&CxmLF$foTU1|f#Q3nzvMi9Q zuKR`~&wAo}kz#_VjGPmN${3b5iq(hS3#bq#x8L#_3u~GFARyxQE@o6cQ!3(9YE;2= zG7QvpJ;}(e!W6nPk-#5m%C&TcrQYS?t|Qze>X{j(aNLgfsXAJ3m%H@}?+*x_yq!wE zWT%3FQ9h5(TWg^}nEAkScGY>QUox>@y1k8wg@6B`T?x%&$0O)PHIm5b+dwApzbTw| zXUgJlImxLC>GV`p-l%J({(WmbU4kokEX;D*`)BZA9Swl@NuI@g=b^}D=2|Ax{?Svp zqsZe)NHYPc7c*mU@b4SUQl{u9YLA8jM(XJN9!lL)96TX)trBKvW@btmIEgIWeKy*q zYq8^RXa-0BezaQR{P50Gv3Be`{?jqDa-Vl%*~o#22p#;9lJfufB0i01d4W^8GOU`2 z1~`C0ga-QX*xIj|INRAE1JDbI(M8}Yh*I*mdk%OcQ0f0r;%0#1TLDw7YHo!C)xJ(H z7SQdu2`2l+aZ6wckpk??rL-A;t$vZTohCvR6h*q(2#89gZoMgei1K>9~cv?8856-Z;fdTI3fQHXM_dbS*t&m|GNunZE88Yz06 zzLVq7Z&tSRlL%|lCfh9|*1eC&d^_)0%OZy_?sAxhdGmUum169GB4@doVj9HQYZx$c zf~a|^gH{3>MqAb0xHnptE)d3-7;aslz79_t(euxx>JE=V(~6q5OJf>L~kz z5F6GuK9q=h)q599ZpzSA4ur9R+t6DoW0ttTDj=Jk&%F1&3vf|pt4prIjRxLdLBhUz zH6&g9=L0Fhg9?klax*Sbq?IpOmsVd5dQ^69d57Fw`M$V=x%%4pI?(;5g@7j7{jvQc z>FST7fAMnoQgXD&C?AXL=cb%y`WYyuBKsc-@Y1N==l%RtsxE%wSURmz(L#UlfnS!1 zFbQO$B^6r8?UD(ldR1ar+MsCMuH|#V#LaaSh9kxpKh|Z(nC`D|8o(@IKVdM%rLmLb zyUh*8|BYqb_RS9i3KXAwWlD6Iw-9c5%5?&ZWm3o8DZh25v09gI&%xqZ8+4(k%+-{( zLO5}Ra-IhmYxQ=rS{TbGKPuast2+Qx9?yC@x{%k;$;nLiyVd9TM1*R+>wWDHBjd`; zl+{|}(f~1EO5)Vq=-DytD~CNxyTA$uD@-odr^>7Gc(+efUiz~XyApe>%m411+)7yM zz7GjhVDg^KF@Bq@Ubuoikl_R)Y53o zzW>G55`Tba>YGEBq!9L0OIT>HJ~ZV1JduE>_=DZ&Z}$NKlW!kT9w+nDnhl)(1e{kV zZ8$vT6y*lKMmY5N)QW~+4#}%G%xpeS%B_3<>=Iz^@PR%zfyjlj1nd7iS0zwEvGgYg z9U*X7P9WquE7^gVlS^|e1VgkUDN-v6s0!UDTDUY{;c)dNbkS9EmPLplcE21b;otoZ zEqOsVK%AhBN2f2hK;4KAAyO=NDHqB2iIrMnFJ#V6@=iOO~$^;*R6e*`y7%inzfKq#+% zh&%rapcJb2MXwpB$VExZHm;oHfFN7?u$Sn4SClS2sZ)p3+pUq*Rbiilw()ch)pzAD z?olgmpGyR6o5f4`l@9?xD<^&){ahTV3(2UFI5l#5QfRdw{m)xXoEh_^3vALesx^5u zn>$?br1m#dGz*W>G;bCVYR*V1?NRWIm-cH%4NI%m%~kk?Bt}~Y&_XKLzCI_p^$Tn* zogQx>8|&Iu24Y*~{>rhp`TS}?W;sU7^DE0p*D1e;j8Ou*=+JTNlNNic=f9e56xA4i zyGM9S3+m5yQx7y#4=V~X|M(h^9ZL%n-dze=2geueHg@4X0<8R$Yx&qkr-U>4@d;e+qXER1|PudY^ggyQE zb#tgu@q^0P(QN8J0*}&~vT_^L%z3WAN@uJb)D}AJn*qxW6w_1-Q3;=3D9;uIMTtme-5PvW&=_%Coc(T0sI==uHA^;2{U6>-m2%4SpeM;MrBTl+27o8RwOUuxSjLDq|Sh5knOz_WM zY4V$Xc@@WLHJF_Q%cix7Ulp?sy8?{YKacyK>vGS3KY^x9Qa{SWa}TrX2X@Y@trcfz zjv?+v)VXdDMuWgG-&%-RR(?Q|qzerj1dH{`<#0WM!g?^lmltmW1m>|Q-5qrut4o@z z;pg=SLEg`4pgH~iTZNu?kX#ft2g38_j8Boyt`diYq(i42j19QSP3^2BLo4&CW}yvuj+R- zA6R{>GR7dENh`{8xhCbV71KO*EH8}~XZO)$2(~J(J?&deOjy{UY4I=hkTt&TMrk9a zEGD6dGu^|hH5?3?=7H-;55be4xtiOlo`?sXrx`7d{`8~qw8UfILPWYoE%}ow&FQ%H z<|ZYp8r|4?QyPf{3WHTwX3n(QZ{KP)ZhRgqHZB6dMe5_a+${Bm+^7yc6$9Mi2&dor zt@IDvQx9Qq&Bt{lqxt#u_T(d#3j~HGu`Dq`lx^UH;&)|xC~Lnh(|MA+YN8=H+36E$ zmCwyQZqW>PF*xaeZqC1LCsy9+$#7*qmiZOVP@SAEpOG&DO-BA0tDWBBtz<;EkNr-W zs0#LdC5Gr;*>3)|tzz)HNj;HUMnul}Cvo)d|WM%nDW>hvQ{w@N__i-^8cydq^T2M8*nt%q(o?%Q=mb% z)cm`pbo&Ax)zx>wwCWw$u60khih|xI#z;as!U||>$9$5UK&+NRlBe3|yexHvWQVn0 zT`NkTkEohtqCD!R&QvB%)+%WCUVrk+3l56wBAGPG81D@;{foLm;x^ZC54Rh{v}%ZL zR-y0<;gO=}&i72PE0ErDhEanKH4xFLJP-bT4wPNfsXE3ACZI8s{uRT~t7|7G9SVyG zErIwEJTA+E0WJ+$c*1YXKrsYANWA)eVqz|Gt31~uOSS@K5~*~hcuIV4%FS2g{Ve0* zViGF5&pG}W>!=<3!PoBcMyUw&%3!pJM*xQjpGlJ*0IoV)xB4n}#(sLK6`4%1^O&VG zNx$IIy5!3H3`WTB-B5PY$who1#>VD~`WZUT7_azzQq2Ms7=j4qog%&#R_S*=|J110 zvWCDO{O9ys9%Z}4y^7LFSgYQ9J4@+B`NiJ=WFjV0i^$bVQ-6|O0~;f4cFDmF=!%m8 z10`nV?Ih^!p)5I`L!X&CBOxEKy`iDZ6!vc9m&=$KWg-UScBUeGByg;tBq)n9A&e(k zYhPPa*zQ4jbyMs)azDQqnbsa~mxnHLPpIX$)(}cirdvgUqgFTd>TeWUOyn1M?u>J0 z^%b^jLix-Eg$Hs0GQxoRGp~YedTr@C_vgs(N7AIV=S>4=PxtT2TF7JCy zeBU?P~aHxu0rg)%z3+Be;TWWsy^+0@HZH+iXK63;nOi4uR zhx;Ul&+h5zFa^aYTicR8pW#2_Pg{!7-E| zOO{&)=`6+h`0^Zg9`DdU z-V`T*IEb&t#JLXvWr_JQxx8k<5h9$jIw&geOG@kwKj%%B_NU~QL5IkWk1KCtG}^3j zmWB%5<#DUWI5NK3Ke)=4qgnozildjqQoZBTQP(8k_8tQm`j~*X0RD7$U2eclznoLw z;oMGgol!1p)y~>=4&1|!840QIc0vY>TK#PcI`cRA=IczS|8J)OdBPm=f_4JUYf8mt z$Vzd7XjNxDg0W1&*mf_DMBCwjp;~;TXlT6GfXt zKfuxfCLCkZN6~>sE?ya_{|Y=D81IlYZ-IM20yG5W>{nDomJIB$({CKhRr7yT%g&9< zZW6Txp;tzF14(Rd*;+X@%U%*W%yv*LdXa1UM6QsI+dO)*>gWCzPDC#AZ33t{(s%9w zuu~q3&F>Zs@s!U(Np38SS^+3e2gxUnm@j30TLj{?0!?L-@X-<%$I#6gqrgUXmwirG zGE;U~qaXnDOcOqAR~qeGxijCn^thsyXpcqonda%+OF=Fb*89V;PY z&}byupjlDF1VP9!=xH~5Q4V^&JYH|7XrQJfdMhLkik$nnm&o{BCjqDqrD0%l%L{1F zQW$pZE7OLN{->zorFm64 z<;-LCDu}?LF@Gkh;9Ip#e0}-U6IDp)w`v4P^ne89!{bS=<2&z=UniV@kv1jq#9$)tv;;0kH75V$4&ZKUkRuYB0WZsD3UU5xbamyyPu%#mv zh0N^^%tUYTkLC!`Y7>&=RrYGWT-*)B0(ncpXj=ur&{T+U7Sv0@AIYI9$*&GklB zKDPLm?X&ZHc@AG3R~*d@5?;pHp~cd4!FOBo7#y@$jg&F5VXurEo{m>kcs=V9jJ^7$ z=8gYu$eg0%=Q5YLO@6T5y1@{;IB9#Gebf4jQ5EwD{5^h=C6;Ei4g&K)$F&aN7(07_|eiAQMTHt|V06rrmSk+J`2s z=w{RXOi3-A_e!Kiv*M9Fu1vt`aHmc1hR!0R?doW`+8}hQLy0d9VkwdV|dwx28=Ea~M z8$rcThC9k!90aJ_)LX*V$ep0>?0;^vKmw)++*az+3y2@(K02FAdJSlrI5{8lT8<>1 zyx&+HepAGad2xAHrAR>U-YOn~St^XM!5=DJn#p-x9km4U&pt~4Xcd;fvCisZD zY%NaLVmh$Yd5WDx3JumA6(Ew?9i_qKcO0`}eQ*4M_1H<|*Ba%&2pw8xJDcwzKdhda z;$7Gu%DjX!(VlyInxBJ~5@~I4gGgL$g`}Q|XV}^*w z@89y6%rJ$?$a&PKKBe&J4q9U$wI5(?Y-r8Tv2yipth$ktQcs$PLu_P~$Hr$rNzM=$ zk3)J!@y|BiG4!FhpWh2|1UBI|I#wkR&tUS-J*FF<*~TCc3wYQ@4=r~I0hspkzsgh* zrV;8K&9Cg+NdHr_i!;$Ap28aYeVnB!5WjUkS?do+)M3t{@Y$m>nxB3NM&7TKYyp66 zEz(*LHz97VJWKAi1{LgUC<5d1t0HK&qS0AKKr2g}P!n<~Po`0(O!a`x)eaLOWSgp0 zsUAjARIeqYpqq@i7iYi-A)#vz6@c2x1hSv0tHV2Hywlhv@mmN+NX(wI@5@pzgtycd zPl8&wF#1ajFY~}Qo{k|_p3&7f5%lQyq;~epe+wE>i1=pzmfBQ_2v{)PHj89pQbqxV ztq_&tI0)1zcVsFbc<|Zt{jZ2vIJyb=GN(FNG8=IJyy~$4JViz0AW?zWm4$}@Frtf;32XcXxL;f^^-4fOI!VcekW;BPHG4-Susr_dM~Q z@B9Uba&fuVUVE-N$M_BSn;M^{SH8Ezn|jT%`5Ji#5ff`NtVT1-Ll+?K3jKT4;VZj7FNGpm9weOvH~pg)!c9O4nD+3cz?v-130z)7tpV?tgE(%$Na0bi9`Rr{i)z{Hr6kM9Q5c9g!+Tk3h`#TAEcJ< zKWNt|!$N6Gc~3xh&hKItbFW)0-B#nT-EYAz<*T{xIw0`!aTB~K=CfqVr$(e-d{%j| z=iI95v{7}uYJOZD&QKZp^z!wJv}o07z?Q_@8}m#CA-PyNXYp+a-^Pc92N&{k>cO{A_V)4n(zN;;)laLG}D~& z9(qQnoSjtpkMY?<>{Gg zHIW;#cFYZAkzcO7JgX(Q2?io4#M_^NbxmB)P_SsC$k!R85c!3B`xeb%*kU2 zHvrt>X}CPm=@tNqB>PGcFGdK5V)c3U6uRMPRO{gBd3nC!GttN+3Vwh_s{>~7gyV5Q z!iXdpRWV#CdWb*7N4bpBPgT8|VTPAUFjzWNsY!YnK*mV$PY2YQpI@yyNo!oK0rC92 zA%O~UVLZ6R-Q$N$dCid?Vf6DpaEdQmXb+1UE_cn ze)pQk!nW9x&gEh`rtlgSv;FJqS!vCvVlOnTq4XZ!v4OqpzFP5BnWNVvkwKJ^M~YD$HxIcHEk zI`LTE&*)oA-5G1uH7es0K^!~7pquJ!mvk(cC5g4%iCk(*3TW5X)B5%G6Q17w!bhsa z6&o!J;R*;&ZgFJ4qV;GOvAa8OnURy~A)5Ve6J|`w06D&+tby3cm5MSgFXfVivzrVP z%!x2_=VoV9@5t>{k4+ly6*HDg9eNeyzNXod#Ml*yLJ5b*?~yoxskf=25)XO}<;j+R zn8&O7-tjdg-=>#^oSz4`-y{dmkX$@acsg$Oe4=}N#D_lzT^tK!FE!7Mo$NnJV_r}G zDf5;YcY1Q?vsu{gO9_f22(EWo-pkJtT53s}61-VT6e-9^Sxn;H%O!nh7leZ1o#8N% z#^H?YAIJ7gN%i~ay2e~}92bm0*DJN^W00jsAoYmvyHc^bV@9tdd}z$L*%wh&>*ZP8 z^z4q<&K8nJ#zV{(8hCq~9+0>_a&hJ&LE;lSM)R`dp<}4+H3;b8)Iu4IL{bZsgSSaRIK^^QLJ$1Gy|6xL<|s8QsX%xKns%qq6YJuS z0A^tF31rVSbmcH7_JDo@LN~RRFIP{6P$+14KbWDf$O$3i!P-Ry^`(;Z250Zzh-gJQ zwYd5r*8mTN2w6jCo+k|{YQz>0^Cbg%vR-UtEY0oLK?^ur{Li;)9H;#9bj%zUI(*oWYuW3h-f(889370bJ{^( zc#JuHQ{ozX(vX5A*wcl_pB>V_DKcWKvjAO8%g_AfuUy?)55>ikK!Pbb0jh8lcO$}0 zb+kr^T!BRbqcHzkV(IxZ)v^@l$Y4icF>=Fu?AX7F_TK`Iyvg~n2Q#v7M&+CF)C zszkT-=e2asccnmsafDN)xmVNTFALPJns*sZ%7a(x{K3x~?x7D%s}XMH%)z}O5N~@N zDCv#BhK+09kFO|--Y&qZUP37M%Z+~(Fo!(UInuCmSB9GzV4d-ykvx**+HtGBTq@;e zle7%1dgCGLFdk=}6WW5tHTA<*&s0gD1R?tbwtqky=zy0gmxd4`%` zEYi@gW8BZTUA}J;yRFeIDY-y`ksBZYb{@c_iLTnnle2{<&XN8_rm7 zHa@MU4YcFp7xF!zwa^FGU;mYO3jxo*_XS{ixzfR5xDXy-J0!Zv?y|b z*ji#p`tnm()&o{orY-$NSb1ZNPZ*__Yo|H`-C+RcR!4S_M5h*R{EMN`VB7h=aVSmV zoxq>{rG=goHqsTn5>BM`36GOG^~&zSoe+0!kRAj0CW1@5<#u4~CT<}5O8My1NG03K zpDpZ#YSG$^jQlu)kU*ga|zof-!4_JlWO^|f@OdV?M{U5j%iX6 zf;TRbh)eaxpE$S`#NzwLH`2{sAPn{BmLGl#Hwxwl?*-7@vg%f974@UU3LSp)fq;^} zn721u+jAI71B)bGAPiaqHWvn$msdyxGob8sG`FNBREIt&C{AFT%4WwZtlSA}!~+R) z3k@_^CrRNf!LCurrvl26#z-vWQD86^Lmw)M+F3MFqklIPbd5QQlGUjaLU5<3xH{N* z;N=$WO)se1??)jMn*9&eKnKKk{J^TQ1hS*a^51vNLOS3Wb65wZrt>if@NlGopsNwj zXG%3bb4yJ<0i9?In_1jIa-?&F(XTnC0Ud}~%2nr)KyBIN80q;2@e+4~XcPU>57~%U zeG83VI$t_Dd&C0-3Lt=^1;ENV&PVeW!b!dm%%eiXN0g|1G`{MU`U0-k>t11V89Yia zYlV$|OqFj<#L;*KNa5L5Ve9@hpR{Q;e|u{-jL3jC;mg?nUOgaN?6zw$m-Ab{Rd==y z5ERi(Kg-|I>E$>0gQ2j1J%m*ksO<0!PW%w*ev60nRp&y_hXgW$po{O{GI7Gitu4Qm zyvQRd{|wqIo^J*a%{Ez>!8Y=R-HbhM{?~%xm^bqF(-DE{^1iUqG>b7j`DA)@)+ z*#9KG#C-S-JxD$B(m~wEw#d)fmjuVkO=2J*q)`o=vX(O>X!a`K%tQ(8Xl{LP;L~1` zE`GlNw;2l~3n1WUzEOTl47V3@JJ7X1h5bdKLk;hEp(^T)MWrhgx$7`ik{-kiK`rg# zC?K^hI|76k7UIg>GXztJALbNtX6tG@<>@ojAP|~G;kww{m4y{Hu`vF+h1Q~lWEU@G=qSVl$!zoxSCQkH&Cjm@+8WE*LmTf!?2|QIIL1 zQBVg`;%JSyL)oWa?^g_Cc`>M)p;sc7n%qmuGo>x&+$NfzOl{?%6h=Y#=oN%cBO~YtoIa zn-1`7_w&XT0XE9&Oj)W1*?GNH;_PRD%xrp1WxTUvLE*9Jc+N%7)viX(gM=YO&eHT_ z{-m@X=#^~y7#>nKRvpO#s~d2+Xx0gN@l!_5Z% zk#1<-Hb{?WV(1oJhhH1K1tfY`Sy2ZEZ+V^@#gw4hS0O^3BV3Lw$Ac#&RNv1*|7XS} z{t05wYf#EUX>EDkT?P+bSSa2~Q#lCL`!*0mIh=WGJJs&blR9n&35gLvq$)zE?A)k~ zL`|cl*9T#)36(;6Q)c7B!Pz4tV7-qIJ@KtJiX*Kp1RR}K#|u~X2xF*B(v6+QAmqhI zzeeB4L8$~9c&MO9STO!3C6Ax7_n_b~Zx8qZl8W!(j0@!hM?{fm>Po=77kg+VYJN7E6!^k`Xz;B?rb3?LLnb4SEm_ z!tPDT`4dSl77YTUWfB>n@9c*a=>S-N@8GE29l7F|&ZN5eb76}pG8i+&d6F|=5d~Kz z@#v~lVB!^{!cP5FO4Sv(# ztsx3`E`;5A^+&ybG}z#k8DY|PAil9_q^CmM!Bp1E%Eyz{N4mtrEy6a2UN?H`Yi0Hp zMM6jeoq-uFyOoW&e@qdyv6IRZEA#ocGORA6A^P&qN0`b#1XlA#uwXWIA@49MryJqL z0T&60Jmu2Cl6@l(Q7;Z$A~1WNz!ZX9`}N}Rs0{V8%Iv<7)LiKcg^;`Ti-A=cN>^pY z`DD2a-8CF-Oyy3B8p#pwI8UHj1@HL1z4Te=%>JLUk|qB>#Z44?oD2G6x14GCYC7O| zE8dfDjI2ZZBd&3(@9X#iBs_&Bb9-i>j`;@C%Xjv15ZkgRcK^>f)duP@=ptT*!Q_+> zdfhybcZPa_7Q83W5oeOKNIa8UaEe5&lGrgc=I**2%21P{4ZZHjT*GQEJeQ~~5n+Kr z7UQN$OM!|pRdyx!Jy?plqrr>~>+ytz)tG>bM0*fIf$vj5Si+M>1n5u2lmr3@uACOAD_n1N zP%JhZK97p~c9)GRA#9n3h^OTWVdX z_$2DT8bSis7#Qd+gO0Ni$`a@2{n49VJOzL6tS?m^s=qrx8Y2|WI~PM6Of`)B7aK5RbzvKhNX zC7&`t6Rg!!w0qN82d)7A_}h~>kS?bwlnaX?hyA0HCDp?)w2GD5lZR3VY8p?xrnc5yi7M@QXP9f(*s zWd(O&j_^}{@B6WcEmMWF3E8-gMg0P&|4Wd2L(DEBpqecBU=US8q65xEG-^xHt4u0e zA}<9C;lX{db`6=U+pgLO_<802+-$C=`ZDksGuie)ftHa^OiWxp&nC!(usKN}1$L$R z)?w&xjO>Wl8x?)WJNNV50P0tZLUaPgMZO-=6GQL5Cn)W45|>4X6i#Q?IY0z#q7rep z_6x6>&uRDfx(G{=HF@uo;}{Q)S~O^!K;+5sQ%k$~pdvy**=KIrYoqkOBqjUjw+G7j{fF|_;XC*7gZGmP*uk5i zEc`n4SpwPu@<}k|_YGFV?%+7pP!SS#*!g;!RQVI5vQhN1hAIm{%AaCPit#hPXpS5t58OKSsG?iWYa2!U#JmB1E#|E0C~{hv-Fx z9Vww_19;odLkKWvaHwe>y5OLM()td{pxM;^}a1a)_9Sr^S4s2SPyh zqcV2&OdrJcxnloHEa4+>pM;v;(M12%Kkc5Sk-a6F?bYoYDqG#Sz3etGoyW(0W4RwF zA5XE*xrzm!O*Qz5xHxN>5pmpECN?QDXVs10r}EU9jyLV3Br_A3<@m<{49Lm(;5&;M zI#aAZz+G!QJ^xnmEb9F@^p|y-pHs1R%ZEmO)j@rEp&^kd#oA5s1d)(1jk6wQK_UT) zV#_`EMAP`hSa7sjDYxln*wm$(>Y$;r@c8E^ot9IMt4$qt!q!73M4oGP?9dj>@a9$z zM~yAERL07)#=2_r;J_6txKGTt>Ra0>g(}pauMW(%Zf;qz`TFyTxW3$RNuz}3&uRQ7 z4Q-v!P?x57YJ9p*&Fl*e;=q%A&XDgvSxawW?+1FGJw9!)HwyLts{+6TlN0N`D`UFz zyj8yg6pZIWS}VCKQ-D^m-&VD4S{n;MnoU|UPW@V4#vXK*r;wU@4M^}SQSiJ)qLh4% zDB2y+*@ ztk0(wf@k5^Nsr${7<Ee@NKSmN>v|_I1C5TmV-AQVzUO4o6*1)?W+}T#ky$_9 zo-vya<2p+fok09CZwB}=`Lw;Agcv7RBsaz8h%HRrOf+vaffRFLrbUuf?>EbD!s0L# zI@;d??V=fF7aUqmW$s>q)z=(yVQe%oAiWq0wpZx@L~ycz#Yj;^BtWG3Bh))1+qZw@ zEn%+*aD$uG``d-Mai4qEuUZ#58sTkYLvRC;HI+XX3xLkbBU_3vL zdatTVR7b6Hp8e_6`@)g9#F6-YTD=8EF`RnB+S92Jy^ql+Qi3Y7)Y^Hj6IVEdZ_v+^B_ z*PCz8l*NF8zLRds{&=#l2g(GMR&&!RQnfddbSjR3jg`@jfLs=_9kO9UNzR0Ee4+QG z`{B zWC-hZN9xG;*`~+QG1;ylGsftd!Nxd=pCYxkiROfHbvV2R{QZQH{(xq3Au79mg5_FwZ^L^-p{=0{S1`Ll3>a2mX;xJAmX(PbdkP&3~Z7 zssUOaK(J!7C`?hfC>VGIj8?M%Y;E^j4Rb|Le&{O0t-<(j({B|Zcw_ty;qrJc-|idy zGDfgI_(Te3u;}KG@O`)fdb&DFV&Ms9fDdLoJ9OhW-D<1p2)_GK-OMf^hx~|!8HSHJ z@C}tj7Q_HPj4K-S6@5@rot!RG665mWf2Z6OH+_K9s|>Vk=Z=PvMwT2J*Xel68vsx& zDVKJ_K@)O>I3ebnbNqcJJUG3xpNOIyfbh)z1C8?~=#7a3TE~xIoX}lds|ct$!d&?% zKgQm59W)5iGr)aUK2iqK>rH(zXSZy$^_%($=u08SM=6`IW@w|nW)1%|=z>w@#(gJ1 zJVoY#-?|LpLnE2YVwI3DOnme+I5A^ah`CD}vHBb^9_G{5Gh}xodGa5oa=Da?;!A%6 z;g#7Gg>_W{67WdYhG%KZjBM`2kXDNeh6+T-;S9V$epI;*)R#(hKpZ$! z8>l3&-*X|xEi6IE=Srf{?#<(a>85cJY>+!zod8pOft%U*3>av6EH8|7B6CbfeZ|Ax zF8aJOKAZIzBuRT*yA3gDXntk0YY%0IE9B_(;C5Z1&!oO`7d2^o59qPWhckGzJVv`a zW1aA@8w5jeb_f{=81#KvBiHcWT)=zFPX~SBfoo18ISpF@dSdGz_h?fTicWCMVvzq5 zV@krk^$`G9e^ipa-&QpVd`Agq3_x zVv+QYvm)cGYVs2e_JmL7*qH5ll~6 zp(?HTAsG8@WTK6CI{qQ>Zl>O0*=hlND3dth(Qy z$BfS|TEbTzj4x`uP-49t-KuO++LikAZ&n&Exa2&EGmFygJa z7?C`tlf8%$T@~%7HEdfn;g32E<OF>9!v+3rsyf zOtKD8|9i}A5xm8~kQnSCg8ljS98)=*)o!`eYqKkeH;Q?Gra}(%2;(geR~)kCYQfNm z+#6&3HRJgVi0B8OihoZE6s3Kp4D&HLA9@jSS4;9+_T_;sTS*P93A zm|yvKyjje;h8;dzX!U5wkHPKWJ|&q=gjQ6jeNO0O4-4;bMScuw{^H5phq?ioOM<2u z_6~WFOm?E+Jp4`hd!Lslw&ugRtR~wae0}|h?nfe21(*vE=AQh=sST7*>tKWiVp9foA8!FB;ru6tZN`RG*lF{ytBywq2mG*dl`7}c1m}=9mtb4CqLsS zaqwdJ84U7FBR17|hlb<>B;BlYKwRL7M}Z%X&9H%v-+?sd{_JSPt2^;Uf|0?_{Ld(M?N338aOR2=+pbf&o<;){8xHTz5;56=H7} z5NK489?Fi=2xBn3aUaMSav*b?9pbbRCw-^h2=1v~gE7&I^6&YdMuH3!LV>3wY^Ez^ zHoS^1zSy-Rr)YqbFUX!5RCy5NIMCp)!M@hJd=in&6gmA>4%FT!)ZARBMs=!bIWs*G zXr^|=wLSa;QpiqCWSmitmVC&8CC)(x!NUx}E41z+^Lih7pD!=xIZP5OUvn(3!6=$(A$PnUH`}L2Y> z5X=z+dJEn1~{^cS=e^<2u6iPl58cxA3}v$V1gKggI)0 zGnXmrYr|L#D0xampPTf^R&}?!Ooc$3XD9KI)YQ2P%>MCPyY(9xQ*U~6q{LJ7DxfQg zv-5MWW~{~C%Biy^*SOuFi?q`r2n8Gak?G&Jyc;k{$PlD$`-%Sg-cDw_D+*WaX1eANE%5@+${0Xg@(3r}GfBB~Ru zp5OzFE)j9*k=lzBj0OR-VlN*2WwTE0O>z0=LUrro%(0Ic>PCk65cYlE$4w}@ zVfX}5K_vCn6p6c=I&swvH+h>n&6*mkc~;9=J$GfbjlG9IiS{G6egOd<#%I1{_uS50 z!6AF^Po4MH^`X1Tif3AEGjQJL7nGmdTTe^}5!gAM3wG~Pl_TRP%|2v@2L^pGYtKerr`^gk@cV zX!8*M)*6c8KztE%a3ZqXEBkL*_?N_rum|pSIG=m7cP{--G2{>qLILfx^1JR&q`_sX z5&-N$rQZ2JPpbmdVLIJMs~jc4{aRL=_nS*-!p1x1LZF>u@1Ht!X}52Q@Gl6r+Br1Y zWJdE|#=}E{-#iG8W%ulD!6VQ=I{Aq%i^vCR1;M$o!KKn_d8V9~qn{t2uJ!geQ!e}H z`^eK~rNVM!g+lT-N)-H3+KeC)6R+DVMwCqcwW%|OA(PCK?EKmqe^1qjDHF>xr=PN*3dr|;uvlbArQdh1 zgsl(+o@tTTk3|E`SNq9@Kqs;FS`!=K&Xj#h9Lq>AE6TdrHIRXKbD3izp-$$U3(fHV zC@W3k#zr4m=;89Y_3;b#)3XPwhrw``%W{=cCa>x>ue(#&3^Phe7!|(oJQ%X<@~qGK zS~M+ZL_&Uu_r>$Ghmw?z(%Qm(ABh(?MRSq27cBJ!=kB!Hk~D4v4^D15mSW7miy6lT zB@Aw8AyU|j{Pz$4ssB0QoKBY0bx6I7dWeUfQC!xCN|HeG-{F)C<-q{$LWCmI^UGNU zK&vms{pvhX_6qCX0~8{`lxBMqc~jLUz#aB~3`;+9(NDP$SXdNp?5IOphzDD@Ijg6N zl_o}_?xDb8g8C;Nx|_Of%*U+YH+o)=JKrvvyr03>OA~s@ZTDt+u~*&S1{>AE0JYuX zWD6(iW%X?}tA(V>a%y)#Gj3S3dSF7VYr&-4PTGbs&UuvtxMA5ckI~4G`UK|3+;rQg z=t;a;?r3`LA_CAp@(StA*IFeMFDUh~mB$8Hb_se@Cq zW51=$kt+fBY!7)`zle)eg0WJGs-U;j9+*Xb7E6uYVt;Cvss(H~$~jb8bjE6lx#IAp z`DrRIGcDc9HPN~W^FmjYQGwdtx4 zQl89!bXn<*Y7Yg0seytqC6vpaVs7<9J#Bw#Q94enP0l908x#uBc@1YpqUmtQn3z_5 zuD5sf`6sV&5z~+DT6^b1v~j7_8|9mkwNsRTS7LEGPyroU&XMW0+JB$w|L636{m=zx zJy#uw&uq?b+n_b%2nbOCse7w2dpkKM)C9QW>@klaO*|tuRtR8i88l>vG_z%}SYi9H zH&JE}IA7Cx20!_m<|%rA$j1JkJ@J>#3M`Cn@5vR4vdG1Ld>EgJFEfYZ{5g&O<*4eF zR1m)x`Q(h3d=XJtPld4uwcUP+i6!f*6YThyFG{HLdiyu!|VcFGma& z(d#-neAv{e8;1|W#l^p)#E63K0gUqSrEWIUr~Y%*P|}b``nr=p4&7z-U}kJcsOtTR zPk3zPSH!81B(|}VPXK=Q;=-{|5Zb7*=sYNhh<$aZ$2qW+0iRf?)ZRR&q(YYOEo1d| z4wvybX@zJ2+joeH#N6>58>cShGoli}f9CW@1H~`fu9C=wkP&`bQF`BIc`=t;O#lMs z%%dBQ?Pvi<+pp>Q6;Ud!=L0oDxC1p2X$l;85_TXgwG(q7Gx3acRk z_tJakfSLPAIbHWdMmDc2mCbek)7rCRf8WKr5|IPc;C_|jTwCI>2HKc5YfzmqI!@}T8hN^$E|^xCC3+?7t@>9^E8TQ5#i z#Z4YzSI+4+mj&$k2NGTWP3Pz`BeUYi#gbxUF%#Uy`V}W7>xMH>+(?E^!J+lWCiTgv zkGWlgV|?TNC+5gV-lBIA3*uvANG^Pp-22r*1|w zN!}_{3jO?M#cQg*%pZsUTLM;#S{?$U!;&DV@awU4);DCKkRSG+_^gvs2OfyeduEI+ z=cl^wtbstO6em4-Aa>j85peTHs+OHw4A%2KZRR{a{`uJG^*0M(6+7YZ`sqe%Z-Anv z-*B7{6_|g^+^n4B1K1!>x`lYCU17~u@772_#*5l|5s_GL6Kc$=Rw6o8riDIml^rS> zv3cKxsw?Agb5OBC_(p->w~`05gT4uh4-qIVu6bo5@P zd4e-OMX1ww?oJO$KHY0L?bNV;kInsi(R@MjvgmMLF;oTVakl^37XBf|*w>Pt0$t{6 z!Oy~6`JjB4s-pNO;C&kEW(*Q_R_Q6}l;xS0Ch>yuE265WwC2n7j-~LhIMruk zvE2M|e}!0gK_od&9GMxx(JO4%Vqnziz*=X8lLr#PDp9@Th>QhHyAP>YUN)teHt6TJ`t}1a0ceFPL(={*CTHv;rl<-W zDJjl3$EJp`vY)Ebxg?r1>TM2k0aE&2ve{y-SMxJrbc|#k+JB6;j&R*KwaVK+DUh*W zbtnHgaQ}M!tnP~H4<|7`RvqpbqP>O|402UqoeX^g$)~{W799%Gx&!v5(hOs5Uig3& z;DS66N?*KWAQX5%NL@iQaxA&@@6Oam150*^D#(uU3jf(f$izDF3aFN~HLRSH3J(uj$0!-es9{JdPH>s<}Nkd{uXqR1sa z00vx*X{X_ky4O}O6?GP|8Dj=xya(_O@a?ub_u?F6qJ`7;kX}wrifT82!o#8q=IQ+T zUW$+IcZT#MpG$cn@5`bVsf<7AMWchRH;jXVfx-SM1S8gtbPJ52HDH6dzct#(RVYm5 zeR8^VV{ar%WxY%0KFE?yPgBkjeXrxw8aswc&thkr9f){z36v4V8lTz4E^P|0BZ!a2 zG0So2BfJ)WczWFLnSR7HW2~AU6`m*jI5@9y_D8ZH9h=W|T&o}=P!LdKqwxAnk-OBM z9W~gLPDn9(Hm0)LsTL?D>SF#h3d$t*S(E`T>`d=xdJxsUx~<@uOSF^CSNRcx=Jqmu zzU-DY97Hb`WfTbsQu{`?zCsN(v>wZ!an(Z`Pf3wRDgT~@$fH8Gs zR?MDM>9vICozR=j>x1%|YEnD&biz}6o_aUq$r(T!uxw}uoWDOl-gFoA%$uEvV$d&uf~+5-!b=c=>G zpRJ%&`JHJJD+dxU;~cB<5WOa4jnj6peD~4sUjvdkSw;LQT+g>t?}h=K>jcM6VaN`6 z{^jv=s*1dlkaF9pAS`vtK1s7?opdQ@8mXw#4@isWNXEIJbxLGWerfTxkPcTdQr{!y z=FC#}Lg^0y%&TQ@Pk*$n`{npL4{l1t_6l!zwJ2w{=FGP^9 zrn8z+bD`26ft=GQ8;cYDsw&=-h@y0EyoK@b& z_fA|T4ZAzjoa*(Um18qMFZhG z)bxh*t&W^?-s0Z>9y!4zZ<`P!NInnz$At0s_TWzzTm|&3Xy>-coQ7g*Mu3y7=U9gC zL+S{29ngQ(skg?HNoES2y)07Q*hMNYE zD)B|+LkV}gYe7b_E&}UP-&zo}_rm0Eb`2dnv?N0PHLa<{W5M4&dHJYyIO8{Uay1(0 zVx|cCQ~O^65nCAtnkm^-$g{_Dy2t~u!cCvaZ06(Uy{U>NG@=%c3Jkiw_iknheV9Bh z(-PX@Xp{wP-UDNdS!*Fq)y_7bsmnj!gxnN7c7V>86LQ^?g{_ziBtn+zpB z>?LR|PSusfYSuALBD1Ljcdrdf-_wQSc>ifDqcKMJyp$v`O3YKL-8{<&KzPGuo#eUF z3A}iH3#D)7WF8=2cWw@H6THedHBOD z!-0ZlCVnLc#PgSK`gNfPVIhjV5I&J4iPQgIJK&%C_D2A^uj1ukJT*(;5)f}luI&Mk z9rpl`cn*orV>F*rTADpajKaS+?eA!(TmTzfzuwk610JnsNc{I*{#{R?l;?t@nmP9 zvudl%5`~j)duMSIo$4V{I#;PG1~Q4RFtA%C*un{RvXtb${;r$(X?`@HZR9Y@VS0b~ znZRZ?N(PlXN@TkD1FIeWug~MmW@!fb^)_T@#k%R{9d8h2GHqwXn@N1EcLH%#2Zx4Q zc**Bo1}Na5(0rNhpKi>?&$l=pZK+egq1091`>2fK?vvMK;# ziR#H##VzhvL z!wXj2%Y7)6if+#KTy@C#3MFYX%+A@-yg;B51q|$m7Mp6#qKQh1;A*6pp;8UywO*~F z!J&=*G~X|_znbl0d`^hVTASFGML%j0o=3yqFx7naTfCpQ%7Usp#okCbo|HQRM!oB~ z39uwiHSf&P4JwLN^uRa11wh#B~o{h%^Ps<9|JP-Y+2zS8p+syWHT^_l?J*P;!(Nv1%S?FT~fg2gg1?=(!AP zk2KiX8N?r1-|tayeLYA>k=rBYtt)Tp?b-Sk6AlDz$mfWD1O!F6YBL~n;hyJ zqDD0o&*K;wSu6eT3Gly965O|JK4r7jCWx;TDkU+_wEzsb_LdgzUt<6Mi-`dKwZz!H z)tsA8QVkh^={M3dqZHYQdijKeDLCz7hjM6u;G98amxmuOQT&HXpEFh3BRq$xm%wtF z4~D=!T5qTa4aHOzjcr4ngwkMJCACI`S-t|qw#oY0GBrK9;mU5p&#R(+ph+)bR;2*Z zlQ=+QZ?saXVF0I~I2|oUQa2Hd4kPjBI|R@N=?ZO$U{EPwA?CNaNcg1qj;GO$Qikp7 zkGQcqWkMJh=BtH&_ggq5>0b339=c<`A0;qe2Oq1y-}BMypEDIL3587 zV(lfR&Z#a7t#J!G?mC?jp(-9XORnM7NJ@EwPQ$f!18HixamCVc!ap)UptinPlPSs4 zX%rW9%sjT+a4??>e37PBL@8(A7a43T^mtcBL!trH3rxwMl{J`J%#IdPfvyslhpKp( zc7u7en$kRyYKo5QK9pl?y9@`#aR80jyJI8G@W;-BR75Dt z9Z<+26MPHI8$fr99F(#$e@R2POscqXQ$JV|?fH#+Fw36w3hHD1xp|-CHl335_2HC2Zlk1$9;$mEQ z4*o$1Uwhn!mzaD=yn;@5yKBt2VAo0|fN@_u@UX54d`~-Y5T%wM;E9ldq z70u})o6*hYUc12Hh7-==nbf}_5fODUF1Fm*-&{CTMBn{=x3Jzao}TWU4q>4>4@x6h zL8~G{yZu!?i4OE?_x)jGxp?>SBLlK+b6^^Ffu7m!K}9%vxBpOJwNoVVc;9WUg>JD{ zLom0`(%11xn89y`4mZcL?l(Ft-r{QQZOOWxjP)r%s&2~e5dD!RR~B}%Eo4)@kr)Nf z3pZq{;OA7CQe*2p7OQHF@WcvQiA$52a#BxJ!vBx4w*bm|>)M9}MLLu&De0E(lJ3qM z1f;vWLAs>7ySqcWySr1k>5lJro^#IgzW=w*H^Vq`<2X8d_S$P*>k0~-K((1S2DU`t z{4;NHd0+nbBol?vN$KuWBEah5a>WYpT&a)Z8LSgAn}0AFKw%pv39x^@mH`Zb?y+ZxQdig#hC55p?&g>2tk1a3P8AvNksW(! zcEj1sOVN@S$cz5iCJmCxKpxVr6l+}X@*#im0l1DmM+h@aZuP1pJ@0P9Un27)aLVw2 zIz<2#95=N_9VIKj{?Ns;1%&5$ASbH+(PQ;S7g55dpAXPzM07~nhkWk zTGqc^udJt#`FdmeB%r_2ZT!G3 zb+;6uH=m{Avsl{EkJIG>LRqA{0|DF9QMEd9%<-*bscHws*B2RE_AM@qD$}u1mdTp& zCFbixR*URg9u0Lsl~?3sMP7nhQ13B$6w3~jtpS>Ml?)r%p)|<_-Y3g;2YCtA2es2b z6~`UegZ5qN>rA$DnZ_t`ZML~vE5-JbMmR5~aQ?1#gdcr+GH8g0fBq*d`9Eze|NdkD zE{YVQFo}tW%7>E$gPxuVg>4rnt3r>KP4RFA(ljy&z^18pc@r1?Vg2jg3LK!|Xaj4s z2CeYX%`6#PxNOL#Ga-K6^hzP4l_=N`bsg8((M5Zk$DN zw%A93`1SQ7G|~~wYn%E5VQ>n~CM(3Ehu{Ab^MJ56IQon4W!rT;kLk16;{b^zkGo~xy$?XyPnj4a#g&Xkf1xUn1k z_L{V5iL~KAu1}92|CSZqR*~@3e{6C7-=6pX{fPo?E0d3EEqVsE`8 z$Js+w_lL*z<)egwt74pKtebs$9(pY zOyeZO;aQ4D`6`zUP%6lupK6lbRY#loOq+~Aa-^p2wFCHEAxd7T%2QC-OdtG~&SGSb zsMAHLQJqQkgs3oJ$<$&c7M3%Wh{?|yrM@-OQq{us>%H=3ZklOFoEG z&Cb|9m_OST?Q>9x4=!(;lb}}FnFzynQdN}WwLlE?qwT3=Q0!Zh= z!|?_+O9px#&cK>Sm+nusE}=Jniz9|KqsMYRo<%f&C`RAj5&z5CD>M?y`1B{!sUCqe zE-vFCw8+!!u(Q_x+-{Qk@$1Q=0<6p;;lC^3|8vMp}BvkXVr%13=wZ z9hqytjt{3&3CnPXY`7%^kEImI`j_hQVAZ|A)60f+YN{O+4Aa?><@(~*3A`s3jj%@z zVhvU!hHK@`$k6s+@w{H;f2_9*|7`z$OXB#o*5p$%pN`i`CMc;8&@00q4FfX*%=Sc~ z!wHUuOgaN3V@mlR3`Wf91^?l3b zvf5-*kPGCTRaO7)h~>fS(J?y$Kdtm3l?-mOYb*eE(446g=Nsv!2?q>TUJx%p2|QC=AT7^$j4T*5k;Vz0!c=9Wqfmvmf-oMnaQ zgv)Jjh<&UQd^?PyiR= z;&dcw64qi8&Dp~fHO)pwTdt^&HLG?7uyO><`dwH1Btq$D0X71_*zbY;A!FozO>_-B z=1Yx?4>n;cruiS70S}y2uh$QGs%`ZIoVl1W?ss!#yi0YG9eA4dRT{J;E9p93ZLUbl zalm79b&23l3wX}y>0PUG#hiy%$zamo0E#P4H5ZQR)&!k;VbJoX0n&L+ zH^T#Hxl1C$d;af3_#6M@$AY}6-y{RU!S7viALG^t;I{RpL>HN*h43!h!zn&2rgHiK=}UEz%8n zhO@~Gh5p-Y5kyoLi(;=*R`WAJkaFaxPDAGU?10Pj;;b^l_~RYms+`nw{C&&G*kv?1 z34m|2QGLi|Z>BDGv+ld71CE_hT=qw*rjv~NypRGDFxy|ZLO&f%cB3bB**IfQ-YnPG zP5dNX{GHF+Uhj1oE^B8)P-Y(dFW~HrHsX84p{*unQN@?j<#|Czx2?vFHC><+2L`8@ zG4`COHJARF`g_K66^VbZ`!b(MP}Zk~?)!yY|9{>ZMwH6ABH7pRwWLmUo@cA4rpLA# zrD@`iacChJUl$dA5>r+y<%SP*d=}g^!u&vMEGzakPcRx|aX$8^LhOyw^q1vT@M|iU z;oe%(Bj}WM^dfcs*s)_+op=8kRPdO(uav=ptT0i`1*@F1{X@ICMl@y8=^J0fiWBq` z^yLYCTM8>-jTN#pt5sNiao3YitJaQ`SD7`>^{H;aaGZ?*rz#T8&JX)(B}oMoiUQ9F zvO$y7wv{DqhVNcrm|tj5fzE+wE`y9(g_jz zv6@!%OrUV+Ojf-~&Pz=Ca2~A|BBBA{hLvk9M;rdBjIL{~hP;_|b}7UugWC%T+RWHs zNrGInYL_*-IXG)Nca~L9L5eje!{fIBr)%IVnBw^p%$-Oz4S=XMkhSIn)Q;& z`6_kZqucAtX(lyT2)RiTVP*mq^)l)W?4MsfBK(gP-3vzslphHp{?9KTIH}IW_y|*a zS%D*bp{r|!!iI~zQIXCP+MhG3y3XaPl-rx*Xm-(ox%@J$)#MSdiyv3@PPwcItq;Cc z0<54CT}yWrdvvABl^8OYbAavxS)*yi0*eW9VAQptS+2jg0ebe)W}#tO`0AG8HC zAu^p@R_UsZfpEB-;h#Y=>t5?WRgTVgqanbx$FyFm#5BEL+oZ0YLQw|$&2AB z>6-_F#?rzWN{r)%e|p~n^_o4~UBW*L$QCwd$)eVP)TV5#UPMr9oXF>ZYDR0lIo$3V zOFNoR{ms8zf5dlh3hQiu->`1k*H5q<6;C!l6O-!sTs@o$P(^BfT*$;H$J-Nf#8l;x)8n>jZjkt9K0)&pR53G18el9jEw}DASHLQ<{jy$uy@OmD&agqxCwoIN99yhzRV5xlh{^ z_{F=x=s;cfIY5bR{r8Xgl24bym_;RZS1`^7c=%k&XS)fGqmAF!o*#)){nPGt6S8^N zZ33|YV>9=1k?k((<)}eZmTz~~q1FWs-_E|-X?4{;;atA1R+AFuhbO#+zRKXr+P$W6 zIyUUX)LD>gsZU>l;fxwjib)lqdm#Qs_mR-&LV~|N#_8z$$A}kJ^JT;SoXBK{bvNQ` z^>$3aeo=OWd!sAKh?7y*7kbdXG7{}=L-e0_kKcfZAHvFjW9m2gUw84}m-4^-=5w%j z*)?e)tL~5TC7=dw$bpZeF6Wrv)hfj_n~scsFU`tizJjqaw3I<(*)nXFoH#$i<-K|>`;uUs!dr?Jx@o}+j6v(5Yu-?- z+6e%My!PNz-*~sOZU`D}b5!D`r;J%7JUI+Si3aH!bppRK&1Wng4G`LDWfmHJvR`?b zX>9ESiiYEZJ$>$Sfx=J75nRgwO#fnhSr#M-2Xfbzb%IWilL>sK(6V&=TkudEFHtR` z1*QA_R%o|Ze3|Ba5HeqFus&{j#odzA`aPpuRh(*mnjdhl6&~V7tj$giqJW!>sW6rUJ}J6&;;QzN%`%G0=UeT&7(oDrbVKiXw&ck!Ki=W0|#9L{K*Z#|f8T?vx} zV@9!|oQUFcYO7)#ev6 zNraLDwhMOd3$aO#tb(wS7#ircab88ff{JFUi}7lYhs1NQSJfVF1wP2FsOyts5taCh zs+JAkfoseez=YOz(U&_5sKmJb#tRXbV~Q23@q!Oe6v((m_VkLRKRR+8L^J7auCHuprxK-$EF?OYW5h6a9>18VWWTevU*?{+tW+dv}&PJfgwE05aFwy$}HS34N^HuhayDq@*WJRwt5EY;d{_1!YxeeQt zC;IysDTp(lF$Lwd>Hey-e|I|nrKtY#Ek7*e8i;FSv9i9$5rpf1!SFbnV13XhV@>yn z1yQ6FO2onU$gW3e*WJfpTtpxw;o9&5JO5MoUeo`?M53cb@J6M*cV7EaulxPA{ulQ~ z^~S2A*PYJ+oeCQ^zo2A@q!cSEyf2cBaC~J_q?5&NB?#t9DX9{4G3qnm2BTDjb&)IG z8VFRi54+kuLe$}$!mxDv-Xg2P&F}i!ug6*DONd8$JH5Vm7pj8LPORT-_o3I7ophF< z$=}_;qAK}fs=M7V6U{iNN_SG5@j6!?B>FeRKGnH_9r{1_DWm@C!@2GY_cUM5AK5Cc zL5NaJrUqhDtBv6d^3Gfr7%muy4XRx5sM z_2vY$&<;{JR-fp@cS>Y146FNAcjmS&a$Ws1Pz8s1AiO(p){f^UQMhpl{W;923QGwj z0;0Qa{BCwU{}_-{+#Fc(M1gn{h1168w_jfdbgj^&}kW2G53Ll+xbHL`ca#kc0ixUt6iC0 zwKxMc!($IqZ@q^xgxoNm6yjr{^Sj)bCU;h#slnH5SnBO0W-3;VY|EI6^SUu7&}eYX zQl=(19KGl|GWCEg z67GaSl4qn-NQLrR^6T~%1JBJ64N1$6vbsRV+v#!r!!g~$0hjX|b$T99rN-9sT9H|> zg#_+O$IH9h`H~AKcyu+1bpZ)=>fW0`0>|yb>Pd^JU&Hj8Q9nPb*ETy@y#b>4e9K#o z=MFJ#u1?IlhT>Q^HBk-?3S&Q z{(~`+Ax;K)Qzpu^;tk*G0J5t#jfT=pf|yGR3lo3yD~7BoU8@3nP8^w(944u@ZSV(L z%^&>)M}4?OFMiM@7SgBnD=XgC60?)K0SpoN)4b4VAyk&>Y_cHZ#ip8OHANNoB@+>; zq>uNl@D^$}JoFXc334au1^fo)vuR#|zi8jm7`Hb;Q4#TO2Nk!xiR|f#B3-up^^xAX z7oBd?-a^5_A-3y9$19vCf6F3e^gZ`g6x8wDBGLG+Gr&V}{tBwf(ATh#6nJq1s}q4) zOik{rIS#URs#K`EJ^podX?@-!hB==Lmn0f=`W#M|ZGrgj)w2suGrVqcboiVm;#Auu zUoK_0POn{;j6VG}JyKY`+j>0CYrT#?|M`E^-hch*bGo;$>x7M6#y8FPDTs%+d?80Q zd)@9LsGt*QBqL%`7-!_`gCM`v3jR6jaU$?Gl=br+Eq4lA*M0Ao@3JXDLTnDc`}<&$ zITI64x)BH2@J9NnzSLlP3`@rqd)zQUA>&OyqRxm3SlG2vbk8rM@Kow-)BI- zkzk!TjnM^ih?8d@P!Bk(EPS3AVl|fohb4PE?aT0;wgI);_@gT?WqPz;&z88BNQn6G zB0Ce4x6_(DU%JqbSnha0XB0-l9OMUGm-J&#w&y}o)`b}q8&H+0H#DTMf<7I=_j(2I zq;G&UHCYMb^57d>bFaZ>TQ8+MdU^!EN%!Qe#(Ml+6So{)K7Niz-?Hirw9lK($1~e8 zD7U|(w>;RAndLc9h}Tz|h%gBf za)bc=Gsgfx>S@*U4nYOAH2Wf53^o^tpkp!p-4SDx$X(E%f59N^{`brLzX1#s%kQS% zBll0rccTo@+6vd{IFXc&jv>OYOx0o+ceJ6nT<#M-UMGAQ;KZocG3jl%B-m+ZQZ#AU za(tX!MZZN~-+a*H09xGSj>Tx-Sl+jxMn%lW3R;Zzg`zO?u`M)E_}l$~Edde*E2`S( zaG3J++34cw_8On1Dcpwr>Xi-5oHd+r#TWJ+$SDqj zn;j4Z!$8#_S_trF_mY-mJ;r0a!|@XqmkMZt0%V+LeH9~1OhBD6!6=ETL%M%poa45S zf9>4tz$A?Jox1Mw;aG);f3$n8O@bqLVl{NFZ4{tbXjLvZ+U#6bWRg*`maEt7*Z>Yv z*R*N+05lR}xvHhs%=~5D7jU4O;AI1n5;yPn;UzwwYRVw;eGh_~*EZ|IcWSJ1Z*nb_ zpzU}tecA94kLdzCNZjYF(gi%&?Q2Zt1mcEbiefDA-KIhVD2QMh=Vm|WQhj2^y9hRj z3br>KiC>W}DDMW`Y>a<)-I&f^@L^-WSE{fSTWI5~#iuhvZM@S%-7#8kf0HjW>eUeA z$D}82OE@n|aPOP(0}-+D(Xz&w{0nrX70q=O!e3*r5;G*)nMy6m(98eOqW$Lz!cU1> z37SG;!7TH)o83RNTzn^46JM$0V9`T?Ah?@oyIoePGl{Lfoe_8$gG{m)>XcJt$+N(v zol}zREqArs%39B0B{9CNTS{imL~Y{wXhAUJU;Hr?Eg7Iv$g^+1_s>bzY20)5>q3zI zt}mzjGmR7b98m7e6e}yZQ-JgVGmBQ|AMtOkG}_;S>n%KqU>t*sQtF@po|^ct0%>Im z``>DBynx`~d37UfLXZSW9;VI}pm%Wiuz6U3dp5!}q~gSyQp#of2uHT?OU}LhjkXsE z^qs1gIa zG{cU2!|$600@6ILUvKS7(MqM$Oi{NKRRz#V+jkut*F7w0G!2oL$ybeLr{ZZLwCtlb zkWM_N@HOF9TkLI2(x)}*#C6;3AKTBm7KD3EEoLgLdiFDx15g_=Z`UhmGb5D}?B6)C z6Q)|nkZi?eA0YEa$+XhZs4HB2icUl1zRMJ4_bXAe7KKY)7TaL|u+Y8=&9Si6u` zl7!72Sk!dTH(~!UaElQlLH32=h&8=ntg>L0%=6EZ>!9hatqr7eBb|af=0GacIYruU zze}Zu%aNFzOh)w8POAy^xK|;FkPg+YXANF%_q-=vL#Wt=jJFv_CfBJ3j%b z;7lD5HFYgZD=uGSAFU`+rYVXcE8&xkR$NGWxc0c0qM{5@mHf71IB#WKIMiLvPN)tZ z@gU7f$3?~5g+L(TgdM0{?RBXZ=i)U3SwP6fVjftONAr*?o$ zt!3+@?V#zp8@&tUns4Q9?3ryQpb@ilzfr$J$gOEaI|uYWwqIMJvW+=F-Qt zhbT^M_A)b7O$e2Y)#?6zXM{;mo^`^|!{0m#{SzE&y#k{)X6X~AYDA>}prxi+7tDn; zZ4w^EcnwMfJlT|0JGVu~W`YA;@Y^1KspXk&x*BLkP1k`}OV-Q|wU-JaaTW7>=+-RO z*MhE(Pq0`=KFYUJlSFm#9%5;51E@9HI$C|Zi4hK6%Lo+D)@r^qqFUWh#8TaCKxDNK zI(1jCkZyQL++3Mt$Z@V*t&uESZE7%qb7v{#m?G*~F~^xqV&kChgw!dx;D}8Bv7Kpn#Z}fzoxLN>un6n+y71^ znFFb{ah`9jehd#ZKMPr|K)Y^#s+MH;-4EJdkhedM59hF30n!)iKKfE^hVG~8Bdkd} zfgNF9FpPQJY9a3q$@w0x?Y4JfE*ab#aZ}h5@CGuKp>O{_Ru?jqbseSMrv0$Af4bh8 zBPf;i?8w+LYa-X`m(f%ENQp-o7!KxhtO;2UGfv-%lHi{3iCh54>M46oxS&u0h6lYR2fCq zJMX93U4=Rsqz=ay!6VW)7w(K(0zgKUMT#qpQHDjQqO5wX9GYdq@7SNi=q{0|eeTz$ z4n!jpSwj|PTxTMdY|)x|EoUbUE&g)KZF7@3v;chW2u%NFQz9`=Tn^<>ijV`*1q6bKpgi%_y(dt>K}mP)Wwb&9zw|NY8@(7TAeY zD!4R(`aZAXNm7e99IkuT0KF=@9NaJNn?%tSk{I%>xYMjq%w^)fdPC_qOooN)erR2L zh984cI-S;C7{80r9nEr4;oa4oE$GX*zqZLA_4)3K;j(_5lcWng zjCJ)c%EcOsibvAyHi}66!P^fR_Vwotil);g|N<%&cz!8A1oMkFZbS;YMF?ZX)L)I za4VDo=AO~p(T}p6u*OrLpNtxSXZ_oYyTh-vor^IpR@vfRC$;?LGe?D8OU=fgaa zTH_C&oQ&~`isaC@zZ+08!IqJy@AebiZrrZVE=NalSJJ92+>#FH&qP zo)5Km0&FaIc&L*dcVoAtlM8QmlZ?LO-9JYwre-*2+rEHWw!lv3cX{4si(gt$v?v=i zTcysgcZX*yVz%zLK~@u;6sZjsf)s{6-5)>}fSUbRuK3ZsvOUX>eHrmVC)W0zpFih! zNg^A2pJ~tg8Mq6xz^KzT)*k-KQo??3Y_(>m-Ph+ZP@sM4o3o|9T1cbMcn`^1Tcq8MC40ECBy zeyt#NgA$XM(a^}{xnGXJS6oX7;+fh!&r6uWN1N^?%HWvB39HK-rNZHSa^402VdrOu z>&Ay$RU=K5lBI^h? zq9RNneYyLGxHm$-v&S8F4e+D9(O{(2;|*( zkjmRsrVokoH?MnLMMJBC;=)D3tIu{~=z#v@9z(|Su?(4QB`P#ux`x569=AGDYLEoj zANOt8ncZ3jS|c{VeQ>z*pjnFF^{@iwM<8)=Ylu2c&pQG06LO?(EnZT_zY2AxIcgBt zM7{p8@8Ev}dH(%56$*skA|&NeU8(~sF!2R6ofhe4w;4AdjgJYQYEHw2vb$b|4Yf5? zeuGkr(u>#aUV%LcMpNcJ^TuATUpznS#vp>Z2sVkohQ;A~nh3l;j_bvq9T_T~)Yo1g!1il&ibaUYAP7y{;Dc4#f z-(6=Qk6Y~cWGq^y4xh(@}*V`Aqt0{*~3%@7J>jN~VeGv}Xy4^JPC&?sQb#29dDnAM2w>0mzk@+lJ zXjKuC2fMO`(Ynaft)0V>0tHscNAK>9+^lZ`cs-<|YLI%8cRZZ#ZTSF1D_l`_w7XE= zem6Xhbofv7JeCJ?)9T59o>%1G`{%!RApaeig(Z(-b9)cjAF(%2t)xlWfF$LNyPGFS zdG&O{X27V&{A@fs6a-8&K_>P10IEbS!#U^;@pc7^Z4B;^tT-;W=CDxys4MwBODw)~ zWHI}h1kuXHE-PbMjU!%`%_m+wm=F=ch5)*h0GZ3;l8b;!b#(|Ww=(!CMj6^P+Yq7y zse<@|yLMwdXAfSBz=dCaqYNo^>V>dhwjHWz*ZL=}A5Sm8T^e3rUKE#vHEu`z6jZNE zG{CiWy;h5{4~q9yud_ccvwg6(s5f=+KXu!LI%L3WCUtiKYx8UOf0-@(&|Yz;VQ6?x z8?2dNIR%fol&K1%IbkbJcB&ZN*5%Z9Mfr#r!Ck|lh*S!wWe44b2D7JsZnR_tlnPnbP!)$`gTb6wx~1wR)@@&C@py1L z^&^WyKh!xhTbOnVAtzPo`teYB$COvk7Vm~K<;z@Wi=&bxv(4Ye%*0Y^OHUehqyBM( z55->*R5=sNwM+mWt{z!H_>{$Lj&C7|OMhZgHN?>E#@-|(E3eUJjoWMnQno}{{fu}! zrfbT#);bHB%L$_l)13Y;p3d26O;T4t1-$06y9>Z1Vp9c=|N7|2V|+x3mp)`3 zdl<+fBLhIn<_&g*wyA(yXL6T}Ka@=(5zL_w&`Xo-X)tg%Ue7EpRl{zF)dsA!#`5-B zdGNbk0Y~$Jj#ne{Nw5QM8#$4CPysEGe5YTl!SDvrb!h7uUY@L{&D5Bp5ud?0bfF%6 z+KjV>x}8BEFw7VA`T>V@>roaFtn0QkZ8N8N^-)LkTZ{CmUy zyjljy{CcmhG}9mh>;Hb&{>vxlC}$^&P~&~;8;e(5kYn6fVS<~Vy<9Nky?>ZuGt8Dj z+xh+Y75U3Fw4ewGobu5B0@>7cAx~BVMd@)mpi}Pu%KKa^I!#;89{+HSD0rpU6Q`OC zOqL%CQa`GDRC#sSQ(eD@vgthzs-^4gk>$?Hjztexs%7jtRdL+l!T9CVph}NwI9lDM zNBOZ+*i^@4`RedUunE^{n$*BD-uOr}<_c~UrK2G7|5q|l?e;m)b+>pJo? z7;Wl6+*DD!Ma$JRh;AGx(g=8kCk#}ewBxyO_fr#&v%YP-r{6V4<`s=yLHW`o{pk3? zXDhPYy1xPm9?%NvULV2y*01LnXG4Wn?TcY;Zf~RLhFqtS`NYUIL)U(b50eL6jPn=? z&zI+CQkV~8IQ**NPT+K~-8!x~F`l6~?*}<5eVX1KJ~5sj16=Jcx1(=UWYTtGuO7;` zb0PG5cSF4UQ9E}APcx5CfB7BlJ2oAuh5dTyC2<=yF;HJ(cumj+?Aw?8BKn&-I=UujlFym0k0X33ea*E8)Ezg zor`foz&Xc6BqIJ35Oz-%bvx<4Gvba9WTG7$2UXHBXDp)COD=c2{t~Wa=74@r6iW*~ zcnjv;@i>#wrMNMx2rRl04>*o9(H+L%3xVeu-AW)iG*6D!MX@w|pXy!>;c$fIFE&q@ zAJ3L;Aem#izwmf!2hYaL1ZZ%`8Qe}ekdc7w6iA@S_ci+^Phm0_fq{C4glpIbY*W55 z9E4^$9|If%OvvvB^5r|TZ7s0+{jL&~G};MT;Pv&~<@I7fdFt|bHPts$j-cxoR&HlRD&vZl193THM?Bt3w9z8F68-S<_S zDX;P6&bwc2t`paibljvn{bg3+f*vsaCEx2c0Bd5S3wU^Yc$hed0cMy^RTmNm=&3Ip z!h9K{QJ1S+SFd>e+Km;CT z@CP>c%W&j2*QWFR495lYC5M;GNCKnOP50}#n7NQ1QE)qS#qSoTMxd{e zT+th90XxDs2pT}>HCU2qIi-oi*sUh(#i*2p396ZeE;815c_mmEDq>iD;y!?BK0?CU z=o(2Hx(5Ifiu_az28Vn)P-@6nmjdR=ZNWjQIM*_&Y~%S(%uuTVB6;eYQqBF&U~;Mq z)hUB1(@{lpxlawqd|_jxD`4bI;ZS1e8L>DW=d2IwUKJS@xtufg&j6y{$X2ra2^m|j zuP1m2;SW?Ca^ASE{$rKwmzEn1%y^nfa;%lQ;dEvQbM9yLrl~;B=aq#jp=YCOGHHan zBWgDVe1csb%@Aic3%H<1mP(fz&}1-YI&Y16FHxmO3$oCUoz3mYI50eVLBsj&XBKql z7Z>Q$rhc5ujf}gT!WEVQ2*as3r#YI6y-vEeT$zvh*SRECz~C_e!6@}t&6bOYIU7`p zk9e>D51SF6FJB3b#&FL8$id5>YWni>b|C4?r$F%UfVXM%B6E%Vo2xHQMK^N|X3(jj z`{?y1$L^p!bg%@>AEztz2dLfn8wZtaEzUu5e&_?qNOMoim)C*lPs-=zBP6TZ_}Yv* zZ4#Vhc+LTIxJ-s>+Zf*EA$iRUU*!DiK~a+-BtHm4+Kz5Yl5clZ(PD6Nu@FkmvZ?lJ$K(tI|i_U@>N1lwh) zfh}0y3YpWmWR^b72JjjPNX5Ip{91-o<^ga|DsA)d!*Q^qQbIGkb5*cUD^lOJu6bLt zI)oh4hi$oQnoL!i%?to%EGZ2KSQw#Tk)Os0#Zo47hY68}N z2O3Ucu?t!Jd~r5JoxVX>?|f|F@^?*s=W~}~9~|WC@DD}#4QwnWWIInw)bo<0crc|k zJ8Dx+!M0pv1pAYh$L7?^`~b6nPkx$94SXlwnxnO5?1-U#dL`!!%!YevXh108p%@}}$Lf$)O zL0L&Z6j_r{nOVK)$5NGRJWCr`@qSJgpM12BlDCV-KN%)eq)ktXFc21#lCV3=8k$+u zfyyniKlZRyp+K7JqZisg4vch!yT$!O35Z|QXL zwUs>Wvv4bGs$(()xNRS`yaR`gDDPWSTi3^h#>qfdi)u#GSj9^Y6eqoJJJ4 zu8pj9)OTJVZgu^xr>^*MKl2c9tR;p3kwK4QKaJ13LX0O@2V%=(u$>@$-bzG3ga8*B zm25AW%|b=Ex>DgYlglV!nh&|cU2RA>NU&|(uGM6z*W7vioc=f^G0#uQsUqnIof05a zbz=itCNw&zvWNP9*@m=(k1)T^QFmX);e4cNAE&2w4`bCr{8yer$ehn@FJ{}_KEd@} zx|@uUZW)?X>~clh+~<{=xW|0uwAMWdE*5j+9%zDvZ*z^zY?d@FkZX{JsyY^3;0`YT zLMio9AZz9&lMs}?JSgFCi~|^2BVHfl`j^z>4w_S|dwgpPCMO*COVGC6TCuA+yBtSz z?3cV7h<*bMl9)WMJA_O%zt=OiPOQ}F=#4_1chRYDe%pLI)uQHfWQI$)y(5l)TNzJx zons{1V0|;*4JaUKbm*WBNx^IQIz4|OL2~l2J@;ZIg=!1*f8DVEZ7p25@~(CHMd95y zF=;qA0V&`)6SIB2w7j5fYC+^|3Nz&JO&f@+x{BspTVAs~hM1`4hS zQoS|Qg>4`&_wM~0iM?CEb8<(|s@A|X&T2)+N!NQ!-LAdrUVE2cdEK|K041waYf=oFyKcvxs-Z}w z{M$N7r1RNf!BJP?j1o+OVTv`8N@QN9%zfEjXG&Zi-yV?N>U_%K!05DGFS0t;nZ?*w zZO)A2KS1%Ytu~=DU{U!^vB(|pR_OAFJ%ao#opMExpa83p6%DY>6X#ZcxLlS3$wjvL z$h>F4kC_iHs7<*qWk3Qq{nqHnyL@RyOz?3<9LF%VkH_OvgiOsP{0}|J`3IQq!8+9>pB4S4<;`Ys zW>Q}{Ovf~TpvJ!jY3P&-UElHw%qEZ#oxF;4x|+CQ6)T0yY4nrn2JSxH3E?#tp+@lD zqqslb98lz@_tx0(tmwFFCk`&XP$lBBQfciAZo!0MZ-20KFwD&}38iGUxX=fbI$w=u zrH&xAg*4(Cxky&kb=(;}qsN%W)vV`1N4~wrzOx^vOq;+G{S!DWwULtNiM6R-s}G2= zp{lf6>TO-hs*iZi*UHp@N7kv)=#~9$yF^yR+UJ7Bv0=sW=PvkQuCXK9UGT~LcZ+~v1_Tfy z`7D2}do2fK&Xf7Kkugq?zYM%Qvw4a50ph;RSjAMJ42BbUX686NeOG_`uj$_VqtC(1 zlyY&v@;_3os4yS`bULN$WPP&|Ax9E_!VMo9b$Z^%iJfr4U{^>9m!b3J^sSJJ5!^GI ztJ?b#@kr?Att0P?UfNW;L4R+1AcmAN8gaMN_TnftKYG&){sAFAF$Bc;l&Yh4W(q8w zW=>6-PY|+Yqk9<|{~-{V-aZ|Lr+4mAT0B=~GEc;gB9U(iLWj&+bF6cG9W^9NEm zZMt59TX1u49*&|$$6BQ4b8T-pk6ZFYL&Xh;GItP_ax7FU2gd}9@Xb$B-jR$7x~0R( zZhv_CFq#dTPm0-d$1{w5zkioHaHFEKucdV}aW_Q?a8TMb^Q1Ir? zj~3KlzDV3?ASqyRvA#e+kebmgO;u*e2W?s;awlz#5>v(Li}OlmcrIbvudQ};xP!QM z&tb%w<}h{avmB1i8Qr`eYO=U*Dq8z!(hj*CnRQRKQWBxMd~CL9x){~(@d=YXB{pw> zP{dwFC&0{roBOE}*KEG;ZVOeqd*siaUakn{>&tPW0?{|4e>|%YR0j5*)awsFUkl~b z4#nR42{f*-iVPw0$e#qL%A4KG4kX_5D!ASRWNEyMi&04X#Aa(G{VPke+{bA8eV$gx zGLM|5elbDgi$Ts{6VC{8|{64+5cMlmXZo5X6k4>nT0pi%@QMD#vx86#$1#f zf@8X~?N+1GGEE4^;-rgbJ`7cMMqQ0$`Q`+o^~@I&a7LOelux(TcidjK^4NOV_E3F` z@AT(F(ikPkv$0qtVCC<1#(%+I{^MJ@7udw3RJD5i+%g48bNADQUP* zxriPp<@CIwrz;H`>W`8=jv-c8ClAm`Z!JaQ`qKk*rt5m0sLS`ITuIG!hC1E4=9+$D zL25QURGB9cShW|?)~_^81B+fcOc5$!`FU+Wdv^!E zU1(d51Oh(>My0t+zRyzV<1QVo?89)kb}43=py$!;RT^F|2cx z&no4<>3G!T%wjFw*Un5@f4;mud%Doq49rF{_WPx1A_)OA&2_M-MFNpTuE5~FV&(h& zW?8&^O-q~YX<5X^r;CM({t3rMEe=>*BJ-Jx9Fq+d zOHyuxvxu|JwVc9XeiTuhKWQLWZKc)^p7ps+cN>Y?ic?c0hH9P8-))&S>HT6Fi_PQO zLmn|}yAem6E+*bA0B95G?1RH%MJ?T~l^Ib48pS`2+fPeR!V64KMOckwwF1W0G^r-t zj{rJuNe1A^x0((4`pflHi7eFULV-)xgQSL=03ib*&(VUUys?Y;z32iU{VD27YM=sm z2%w;6eB*f;-r=FW$vEQe2z)yRb2Yluv4QdUYHXL?P>FV21gvj=h~;{v-Z*kU*FqVk zCpGS8+iNF4Zfdu!0IjTSD=t^>{qe7O+E%b?drdgczcf3~@ATa4rgQ-Js{c&+`=5Rv zD+IasfT+uTxDyn;@6q}OqJU|uU5!m+%-A;!mMNO?-Xd|FQ?m}|HCoEJ=olF55W!12 zaL0@p(~xtQnwaHwL^kho?M~L(I>XtDsm)(-eC zHC?HtOhL}ZcNGQA<lXH0W!M zjnX}1qCEnzI7G{~547$rBTk)}?&-wD#bH(Qy6MNA0es8Pyk3pEhvH0rIqG=K`T#Qc zQ-IHyQFt*m=8^|J?(UNA?!CC5Z-0A_vES!*42M7T7mIbS zIp;BtFlRMBxhnaz)rcQJmO*paxa%w2kGD{nUlt_FHF|!q@1w5X^cq{eFaG(8($}ua zsXw8g?nNW~_Frj&4@?0qDClSy8Rq)!JYE+=RSwS z+O7cm-QUaghoe@f1K7t9F@^o$Uz?HqcQr??v{EV5wAG2dDBgbt!ohe+^KjGWP3&2r zdu7GEGZ}t(|I2Cf@qycGO8auPZg)QKGuf;CIc}hM7fl{=!^Sl3uz#Aq^UPp-r+1|c z+lC(g^4ST@kn4Q4Zq&1yk?(9{@PqV-jaD*l+^B!jKR}nEwX+?DN2V!lA?=4UlIIkD)Z1d>&F=Di^Ndwqlb$m?V4;8ntpB90leyWTyl_7 zi~2u!hAVDaT&9y@73WtL`9OaPqWxNE?QZxo(iY2v*Rr@8S31Y*Fthou(#bhUR_1L{ zrc24ZfcebIa!bh6)c0qlQE$AJwXA=7K0=l&Grm(HaSs?w(414d{L+l-X9E znTzbtx=8N!QNmXMCg-ccXCpuTG3OYYsN93s}9ZydhM6!@CxQ)b>IA%EIPO!$17IU%+^ zA}KHMji$XLgDQJTGb3L^h6Ioo@1g{|UlPU&`w@HYVaQvUsuTybD`X=bUCb#-rCInc zXuZWoLh8(}lO_FH`yPkk&m}x~@94$Y`@hJUP?zXlGo4TFFxd@?~AwGkSBa+wEn3W zWJ0J?+8O|e2OQVy?fjyb2mTz9?yC*sbjhA6tB^sA!$b>Gxl6r6tyF%jGfe&JCn8LG zl)&l(5`DopE8!64vXui?ht^p?OCx4nkAswd*r*HLgv%fNPaM}AaDXQ3+yq(&? zI(jJ0U`zlExf1&=jhaFKcJ!;GHaA_1z?adUHs9piMc1rck2^$2@yd{=n!deB_Ck%z zC-;rtFez;%6Ll$ZbG8#~k(Xs%PO{w7yIK2ojFr!~lFz1Yr}&ca$9G&_qH_m2Brc;| zW)hCtr_SSO-5-Z`Ymt}f82clQ|FX%iW}q=81Inn%$nCC)sOBk9o9ECv?=M8 zI}U7T_}&;Y*EufPBbVxE{d(?r+(F@Xsrkc7NVaXpIKh0y!kQ77s`xd(MKc?an#>)Q-<5)ZX_U@Y=C38U_IG%G}vQDfx+d% z4%u;$C)vYA>&V^Z#;(|^<6>-v^FgQ=@1I|k6k5<-6O|$L$LD`#8h`o$3-GJv(_PuR zA3PWR@7CQR0?La9#@_@X$$AI%5skh&QKtgTneq(cdjOcOeyWu?kAe9!N4wI2QDM%acaztoW+50x7Jh)-&h7)FAVC zn27)jz7JEvb#_VxYxuI3d=JxP&#~79`=EGJ*#!HH3~>EahP{g$C4_b-g9)Y3W@Wz8 zg19!oTiUMKh;Qdt)(iI>earr*f5ho~ry6Hp92eAfKkzL_9O9)MVhzO2GYBLI0#QDn z`_aE8TV?6rCRyHCbrD)Hse5BonrsAL^5BR>UIaNBr+xq6%Flm>2OUc+AF>v&`pTryKLo?tnh&Al;HUIRibJ|>W=9u0`GH7}z%aJ#EPXf2#m@9*j zKRc37{Nm4g5KJ51?<|^pAa;itFAvT zZGO%IvP!HWw$N<1_Sdq+p$X<=oqnW3{dy)2Aw?nOzod47;`$SCxK&D$%x`OZ@mih@~IZ8Rpj+JQ z?zaoFRJZyhvDVq+UL@YqM3ByA{Syal@JG_UpIV010DO%P9}cbkTRZJhEdA zpeB0%l~K=vCKS*C1H(Dfop{alQfm%dbxowL+A{K>?AXZ8y%Dyt zsR~m`AJw8DV%L)X37toDyEbeTE3f&di?e8$@b?NiC|9D>X@RE6k*y|WRZPnxwOS7{ zpFG(TEiF9wMLx}u^CKVuvydy}c6pzP& zDt3^iHhSsIn`{%(E3VE|KcICl`s5)D;3xQ*nI6&6qLkOWli8V)2_gL5+w-;qAltxh z#l~FTo#Jg87ue3`vi#7+H}vy@wRj^4w&RNfb>W37)y87NoYmY;@4HdW4(>uZs@GA7dox5`KT_P7pJk0 z-@(yGjxaHE($3^6q8>w2)7zX1gU^0fB&l%CI7AT0;JBTE^JLq+YTU5)L zUe9H-HHp5erf>WuD9<2Fo|_UsjT`qyN|s$9HGhPHYSm*6g$8(Or*LG3Gsrz)(}e-b z@44+=#aP8qhM2uAKB$N`i1VJnqs+l2W;V!fEt#DGZhwx_Axc!vSH^0?eD-^XK&lgl1G> zqIj{WOV-hE{-1LehSj?z{`T`k&U8Cuc_*H2zZ_C z1!LOuInr#Gal6&jjz1SwOwH^{{ZD6u_DTfh`tW+<@-LxY!739MRbFUEu zeHTTQA8GKCW41aaDp1XgMLHnS4p*=CL%r z$R5(N?gU|r;={Jd37wP+PXN8%UUwWQUm4ARU?pg4RryP^C8-^H1A^(+EbFm*ch6@_ z?1w#48@uOb5O zV8?>vCVAs(6kc}boe$ivywASS52&R#4UJkz7Rzap)4dNgEuDfQS%*V zbm^oM!qq+0-qNd;oZUcz<%`=xsN){(!TzvZqm44kG&A5)mDq9LMq6agHy0;v{p#uY zj(r8~CiITDEBN!@;E`5@n#Kp{8TqcZdTaM~E)>PzZ-8$_*8DuYNNGW=SwdCd`8Nlv zLbU(_)ihLC(KsH9b-HBa&9cbP8=jfEhgS9q9WZ(aYsuTi{&A(6BZxV5xdG;D65z+x zPGbm(gjBe%Xhz zjEb>;Rk;kFh;i+>i>_p@ukq@GWV7oKuTe&utQ&s?EJSyhtf0FFASNLAuRS!Bj0LAk zoa;y=X121=F(e{KO0_4SeA#^TEz^{)>Ndj~usM@y-hPn|gkUuqJ-a<1Z-~)Y2B~1O zoge=jj{Ww8+56W|a3gZvaRhF3;7mDh%M9E%tPf2dcx2Y!CKz`o-h%K-53_}xt`=V? z0uHZbM;RFsCwHzC)Ftob+brR*!|16Pr+pCX3bi-?8lmx#y1~?hjleLV$~=5EnKe_L zJ;o_=!=8|w&xjHzfW6;K0#^Hb*DtuZe*yR3#(No$9dcaUJpwpNrc#e8wjYq_YHgQ9 zCUUj>_J3A|TyH2eUtB1+Ocs%8(C`UgR3IMRR|+$EW)Xt=OW?lF2Qke-m4`0k}*4z)4*cZJGY_2lDehC?cG6RwrzIzr=&8k0E&xZIdTcSw+_{m`H^SgX3 zWexx{Ut%JZ9bgja!i_DS4))=gRDJpE@iAO3*%TdbT1=jyk17`ah1lSb$IQ%1t67IH zs%{_;4&U8yKz~ zC^k6>?HOr5`83DKy0mk96}MlMm1L-~F2}3)N5NDM@~PjeYGbb&zWC2mKZ3Em!0Tf4 zR~gqvguvk5cK4pEZ;lgtg}yggBFB5XXEuw;cy*ksHlMoZGMG&u)v$_bY0rQL_st*q z`F9GzwZxFjn)<56uja}DvA-kS9%!YQJW&)dhxgt4m&)$vOm z3JHY*)zGOI;~EeM6hEznxMx6XMN*=Ioqbt$){z2|L|0hpav-sp-p$_0C3E680}Or% z2X%en42|!$O`JNgL<+PmyMoZ-ZFa@paf(We?sKS8VZ$$i{^r^SpT2(YisKNwBu`FQ zeO#ld@yT?E1Wsv#7wB}!Ap&H5GMXxygjaT*zHd&iiXfXEJ`f`1G?Nzhm zR{DVpk{@JZ)6MU<2lZpx$bF`KnTQ3g-U(FKuqP+b-TnQR zq8`;|Uz>;c=YR~Y2cz?O1mWj>6>pfs?@UCbZU|}6sv+XpxO&$%+<0uXOZ?%&=tmN| zHe)$Gy9pr{T&qmKKAJ{SWpD!doaNG zG86q&0%s}BL!!-jliV^&`n8afMVq)J4ZhFilbk$Gna61b?iKJOWO1*}aO)~Uqh802zEaCV^$ z>-6I+8sp#{ zIcx$By-^dDYvUJ2~COB{E9;m(M^aa6BKqV9Jj5!N?EUQQYb?d%8$5Mp*w^%*}Xj-{@vkIr+XnmF4E@&wz}?H?1=qWmR*EmM_PKKl3B>E5iJzF$17Q z>3a9Y!u}j@_hA3h1hmUG-wv_WG-{xgzF7;V%(#i=WK40BSn=@f2#FB>?RvmOIbnW0U20Xm zRMIJ=gR(2psbkQUE)vF{* zZwcY$>;#Rg)1jYGjKMp1h?KY`2I4<(d0KXPK1->}@&Q(|7nvxuRXtQOPSoxR=b_az zDyruyg@*EHczwq-U{@(|{CI8r3~1F5J?I;!gS=q|YPKolk5t~p{eMrU_sNGRsi1W- zF8N>1^P_NctzER<19c=H4#gtMz!%O_v#bSshO#bR;<{R^nbWR-8aRl5oNz5_kNXsf zhU>nj>qk|L@-7Jya=5Me8Nbam9Ek`yDLI~^cn4zfuGhx6|4a#5YSta|880DnW^V|i zViH-xBK@6k^+Jr1i9m{~3`S}y5k0-h;=Gq(-(HkYb;x(1GNdu`?3-hXU9F~5hn|Ma zMDn^qPuH_Q+Aco>+#jz#|7B66jjWLd$dW{KGGtPgXXM#7Ej>RVKeyix2H~+fY4r7R zmgol(11m`>ry8SU1gFP)1#6j4kR+>`xOBlLelMVqUjyDY(DMGOdpbD7AX*PqApR;3zuM z3|_1_IjZ}girKWmX2QrlMPhz$vdS<+XunqS-|n6A8{ZKRy?3lKKImue+k|MH{%~#D zoG*tAHEaT#b=z5u)kEZbHW0^^ZnR1h6`nK{d=2p91c(wr-JFfArMjMdnT=f& zu$GvAa@X;`B1N6iK0j{rW(Br?|80CceEujQ7h$P>IaX$Y1yx!vQk~$+HhjKz#P6rZa?RfWVeW zr_Iv&m_Ww0hPN>8MSXlCW47b%bD1C=}qT z8o_PEbyPDzG4LM63tKjmt*VuagcKxC*N*uw)Gw6e<*gKG5dv)YJZPf`B8R*k{W)X{ zS1+=8IOCd#;n-x#Y%lLieewVH0#IRAtIk;$GA|1Lkv8xLdo3DfcZ&OXfc>S}?7D!3 z#kP7<49xm^jUNIoP%o&`lGW-plP7Eq|MY)pKD))c`E{eWl$%d{r1Z`YZ=Jd0dh?cV zryGsE%G_4>_F^neI_WLL(vVW4tz1O(n_Vh-f>hbQc(oU=e~LyD9h0XjS-dWMD#KA4 zuvaVz8utT}|2}lSg@T&uS+lw;6DrN7BCMyL4aGB8XTx5gt14{D^C-RlvR|V~zb>7m z0O25=m)@Q!t_gQVqa@*d!161gJ^zRR=S=Nh&n!!wI5f?5`Rn`9s^-k}Pa-E`mpn1i zWYUa-0EPv-1{XZQ5)^E3$q3aPFU)wH&#LBmT~sR)y3r?1#NhG}8FZ*Z4rSR9(#*Np z?!e!{e%?Ae2iR-qj(RQ7lv!4nk^FDC`5rEbkCr^?WmVxSqljFxqpF4A3Ar4`op~;? z+mGwb5CGK$XDwUv9Pm*VyfTD1y~XWI5Wlaib~yAW>H`LggMqPVuJ>pr$i$EtGH!*% zXIXV{yWZfk+cyCsmFJ}K;R`+dCg|UY9xZD=Y+hro(+D@n9YL(uwvTUf(=<@|ZChWy zZwkjZciLlGez-*;VzPWAtA>P+%lywR#(jf&t$o2XuA=1sKXp6*SO4?P+spm1c+s@^ z!L!ZYe(*H3nQpO0oliUKyGYD8B%wA*%(XY>XHHAJcIhfiqe&%IKA&=#KE`>S{Z>Mi zxtW^4ev5oN7w6n-N%SM^O_9=lPS{~L^u_+{*OBxO>%xf=L?F;}3X}zXjUe4@_{EF; zN|lOkzwz<*bBMZ?QlDXCUjlpxQZYjV%gLtrhwGl3`~7Me(g@qtF2hSG!wo7E9eq%(IOcH~ zNvfyaGiZNMD4V@QJ&PdtdK@O|IIsKh@j+z#cbvv071hrEtRe$nuIuWRHM>%=gWd!+ zWjwuoNbu{G^I`tSXFy)tHC^gJx<0WsB8rly#e1CQ-J02%`LuaRH$!|ol~Q7+T5k8R zuL)o&AVE1+smLmlevunX&zF-X%^5bhwt4MFXbJg^#qwNVclDh)mCMq^cGRq;Rjf&~ zUI_X-)M$yho|x!!M%7OiBv8d{;;;}naEU!QxcFRTQmFT|t$Pref}l<}yCNyox5MqW zN7qpgu{CJO8;;n%wobLcnT_8$a4i4}&A_8~i3~hE>b!jkX&f0wh%1}+YK)t$r$cj} zn;PK;ey9SngM-*%XHw#P;2v%Ibv zPB&kXfl&&2%nO?*8~U1!^fZ6JOmGbT}`a(`Pwz6h+M|OgV)rvnJ z>7YXO;(IX7YvOW^(?c}>gx0AaFF5WXNlp}7yKTPf2+9=sb?AQCLn-}b<~>&o$7(&i zimSNJhJ>q>uN9*7_Q}ReyR9f1CG76b0#zlk;+7t~3N_#Zx^*f15 zgw>{E)&3?RLp+9Hby2US*<0!EhQ=S#ZSrdLG@xN;E+cRT-xb6zSzg@W6 zz`=3Yw+XgyYMM%WMKW&;PI~x*BCXLJ3j9a;xqT-i@#>`jL zptN}n9OIkI3}pc08}+9dPIC5lPtvXj)$-(4-)ecMyNMML#iREgDQ#b`^iY)&v<&NH zAobwnoKyYZ(<`EuVOg+fsH%qT*ZGBAy_q~mAU@K?mZvBw=}hq0;WaWTUcyfO_L~Mm zOC^X>1L`gtSRpMp6F62x)5?3OZHk5I#p+Gp!?{PuKrC%W_wMBq4D!r)o+`;J$ZXj9 zQM3B5Z_j<73m=}Ro}FfMG-r1VHGLh1{h^;mFR;*{0cN zy>rEB(g?LIP8X`aNBxqdmfK=3#H>bAxYVYHdw$J$f{=vPV-;@{>0Ves;3MxudR&V? zi8eG{Xj&pqwLkI4QKMOOB7@tX?MS$M6dR7g;J0W{JiNZU&P8T6PQ^Y2g54Y2nll$# zN^4+ef5s!vcbS|TQ}JfxYk$mBEfXY3N0VgtLbw-W&F{+QskHYCKcc<$06 zm|rya++3|rIeG3|WG10nb4t=079~pDqV_YPw$2c1IFV{AczZ|}iWL!`f{tLvFXEQz zitEJtPi*TnVKEKdt+h=xUe$N&uXM~N-^3PtS2Bc-quE?(M_k@3XT91j#*DMQ96b1v zFZrzYd5PSx$!Gh%VhJWhU` zyhOd^U4YDP5w0-_(dOrU{%SI_079rvYXMdR=zYW7r8?dio{yS0Q;#^>1;;@bu4h4u zAE6A(j?M3PXt-yqhZHN(gaUAxbOnF^c79*pPQmB_QFKBXio6xM*wSqt|3#Ij5wF{m*k6BufKe2zb!HlKDJ;qma@sE+VQR~fXxmb zh8gYKAI@O2J1C%CEY#Naohc6&q)mF$F5mec<4|I9bgcMnm|NEeE$~$m5P@pAXi-8g z0~Rrw9(F{AQrUo#ACB7rfj#_1_M4+l{2iC2LM3V6kxZCkpHIJgPtyI!f!;Bf(ze|d zW^<@Il!*<0$zwO@GAB>Xvrx&GUh_<)VKz+LgK6!c-kuRph(VA~uj!!D! zIthPf^Jdtzo9qc%A~^=K!lWzre(Zn0o^M}6g^R4oTbW+#1-}Wu>qw)X&1TA0=1;R0 zCbQvB{%ki@zJGPk&Y)frFu#51L@rHe6`2qou!WJQdRpP7Ty$Dvs%)TrR{vi1#$4w; zqSL*P&RgKUBMb*g(CSrg$>yYrAh6D~H>Fwhm{X@!sxEqN`v(`q6d6TsYEq?!XOqBb z?$2%mE6N@+`c#%4Fzg8yXP#}%Rs4c`#iJu+WMt5FrLvkUq6wZ}y$y*&TgqgJwksVR z7WI+K%Rt>S!rLK!db9a0sqaR_e0w5EY@7gqL(bY%4OoT6V;M$b=%CnJvg|C;b;*iv zRdqC?INB?Bh>UVPG2l*?v7whqbbXo)P&uL9dP51+|NQ&w33%2b&C_=#9c6{4$}Q$p zjmbr8k-jC5XAOET+db?Xo$8otA$a@so&jr{`@+(x=F%p033Rx=_6sH)mj_hw3a$=x zxW)Au-~lxh5w!L-QHts;69d4uCMUlhg4Q({XDIztjKF+^u0<=v z%>ZgdxJ*ic;75~=o|ulASX!Ey%Z<^~a^pFL4iGxm`BiMD^PvOQ8OvfwiRsffDFPI!^0M0b|6H^1F^KBYcVn55`*5x!UMu~oVD z&jCfLzzVu@$reh}r0J zxAVUUlv_=%T@(@fwl&=sZbBS9Mdmust@dVBX@k2kj_7zpW@_XeV0Xr;FfrU8-3_#N zz+ie~{tgb}am@H05plez(ECo|w}Mj&1!+LkkGlbO8M`f=j$jP`z}J3bW5;VKS0|Fp zgWL>ETKUx$g>6&XNy14|t>qS9HoNv_WFkd72a>jRO(o!z$iG}SwiXj%JjhAi7ANlrg;0bvY+Z9wtRc!>} zhsAzJUh7GF?H2pJJUuSkuA<=-O-ZJrype}fHjc|fIuovPO*|;eBA-i5+e^OTteljd z1bjmw5{9AhQvLm!bEVxxq`k#`5hJ5@t*!d9_d~JS6&VBXat(6AVZX%-?9ZyLq%g8r z8;!Q2qa>s{aywMKxjchlzY6)m%BoU5=od&NA$4qUd%Fa%yTEYp(_X@qm(LH*~z!{6Ntk=zO(l~8{~JZ`QS7v z-NHPy6^*?rOP%ku=XtIOzcV?|)t~sXW+B@3;Bx9^=M(5H@Dn}Vi~X}}{Qkd_M#Nm8 zb0+sQ>X+XLzXXmNEf2f&>+U>~OhhinHjcyhz1HLf&d8hbC3lZSoN%3g=;GGgRLs@c zTT1V{R!e`zzgJW0#gOK3?9%<__X{GgbOd`AU3AMYc%-3{kTS(!EHQ8fdkCuI(@6E*c zEH?OB=4*a&BzE09`xr5=5mJLB6YhSSJYDmd7fmE(-4F2i8H+pM~n#jzF-LCvBz{P`oD zR0_}%rU%-Nx(@W792y=@Kd2m&fvh#D*KgdpR)?e_}{ z>vq1iOW{&f$!m?LNFj2F_$6X^;V zUJ-VS8<5?6XhziTK{Y7ARGt>eVoQ{YJP@ul~8gh0mgKhT8A7On0^|vd`Hh zt6d&Zy<={=a&fmSyd83MO;h?ny4MsjaVI%36cr$C_!WEKpWk^t)>^NHD-rN(eFxEH z=X!!ZXhyatL5|4kuU0taBhx%((3V>eLi~-_^_pWYs z@|BpF$Z&FHtn~l`oW_LzkcW=%s@X5iCF@ZH~(supt$$x33#YiF$rmG%FAo~|^1m4<3I}E=AL&i3rZ*WeTmoBR$uvroE z{}C2aU!zq}ts@#mwklZ}kMb_Dys1*L5ju^}I~u(0Fq=nAZ9Z(OFENQ58#_M{uIGVU z7e%ruFJA!6+O;EHcJKh|FY&Rjk8U;kU>;oT=VR!-fEncHuSmFSJbBTXdOhu40&*Lf zzjh+)>mE!`81s;ck<1{N&HX`#){+)mvwY9g+h_aKQC3;dVLGAU=X51m2GqnmK?^0Z z4Xhj&d{4sHAd{DqaWy!c=RAqw^2z^fC%-?x{c){RSc5ZA^{znm7cQgMrv7)_n1iEm ze8N^52bqs^bZKx)?(A?d@bQB}6=#z-K!L-wsOI0zW+RskDB^m@t1RQi#CLvvK|Y}| z{W$b?%g0^&(@p$mjnUA{;xS=@!}twL2-ZGv6Hf)o+w19T3<^U0AJ)A87w{=Y@_@Kb zcLzJe5OFq#YMX1(jeCPxfkk~St&Ey@x(K^2<6N&(J?(f?k{dng_=zd0iSU+z+seeN zx@+-dEw^p@M9nksJ^eFpD*3(E-9$oXXDIZWpdv*{GQW>pJV#zc#%YB22`7|?bto@a zQf8Fjv0ETXO;wy@NpnBpF_>h-_0?FXf3RG_;)puB!ZTIMaNm}mf|e8TTsk5MTM%o{ z1nX}XT}9@djZcD|M8kW#kQp$o6}8cg5N=sbzO-lkuwlgWO?!qVcp>$p8TQH0#>B;k zK3D?j9EV;?DDdTqYhXN)jc9loLQ5`+jgRVELC#VE-vZ0Is+B2r^z+aOo{bQKwNB2X zQAr1;8HNfq3Gd6JmC@5O(yt9Ofg)kQo}kWU7VoXcKEac}bW8%!q-Z6F*RNIeQ@tzw z9A`hh7vFclmvdQKxkT+#)^0BQBiZy7>CNdM&!G8VYse0v?xy)^{^O~wYnBM4mN##U z$LmfRvYe(uCAI=U?7(st@l!U46Vs=GA9&px>rXv4Y4mITPTana5E}JeKKowF%*xk- z87-hj2ta>8b-FfmWja0(di5Eyi>$s-r)D;5(7D{w;#?BDxB&mMVzQ*QKQ6+gU zffg-Xnd$|eL6(y;d3;aofHw>X&LgfhWuQX=WElJQ`Z8^5g<&LF$gkCF78pA9=j@$+ zvuUcT&C=`}t80iX{@d z*?<4`^wqBhJcEN+k0}>V^zAK)NA?2w!$NCz~;NJ6XgMU7 zyKRAUOb%wG#-7)df(x_Lnh{`{Oce zeZt0T3QdH^fmZ$eoAIpl3d<#)=qBO&ehePbZ4|f87Z_D$Oc~C$#QeZRX1acgj)z#? z|JiiNT34JW!wpBQ(NoD1lYqvnMi8H|Vf*oyCsyyf*X}rCGQoRW%2PtM!3QGU%G@0TNFp-0Ik2_2__>Etce=gmPb(s|a{#A?lK3F3y| zjhh4tAu&}FeN#e|5I*K7UoS5PX!w^kemFMBNCYm9dwCtJ9TZ#VhTI=ol->158Vy4+ z7V2h>E|(TBUuNG#;2VPt-wyIaY%jIs9MCV#yE`^gD}EUf3=GF-R_F;$eEZ5fQ&clg zo)Ya9`Lz(RfZN)$W{9iX$4hy+A<_^Mt91|aXJz^lzd+j^YIj!A3Sp?;yS@I$I7_Uj zy`I>rX4gZ57I5QZQf>MKxq5E)0wbB!YQdgsZ(0|?IE#|l44OM1+XApxtR)*Znujno zLQs@tsfT5R?@$LdtH!>TVC9l2?8kX@JrS=-Y;TyBrf1F=568ipTru{b&_TJK=1m;^a z_@_@k57-rJDO$g0{!5F6kmj|2d#E#4vuG?bdxGWG35P+d-5T$zhuIsBDX5bvA<1f5 z`R;Laj#0zOno*G>hQhM5-neSWEQr-kzV4E&5WAS^4(klJ+ii4z!k z;p_*#jN1=&a8xon?v0zr*g=<;FcGJ@{O5gt;T=d+h8P)<+t8`yl<{0L{(IC9JSh9= zM05wS?ekm3;J5es7?b4IYu) zB#<9L!SWSd?Xf^@LoibW8*=eOllA~Tnex&=rY`y`u|eIKNRD~-dqtX@u=;m@voJ4 z{6eE}RyLN-=goaMu0O}dTm%`tjr-gZ>ahLcGkuCWc1s-9c)N!Wh3}%{{jx45;#*$t zSA14FvlU}~rOHJwKoAY0oLBoN7icOfJX#fVNS}!#hxen@wu$i?MB$rG!$ULt* zg2_b3YxzMCyQn|X;-%ikx3A>hW3Y>EV^;ep86z8H{%!dN(Xz!PV z(2>j^jGQTE@;+meqUQu%zKD`Dt1$c3_KN>IlSMMr>(?#DQ49nmfYVQ?tf31`C~^-6 zT8IZ?|3NZpV6R}m*SPn^|2r$IofL{Hw!(ZdUBh;p({a6Tc_U6DoCuWA7b7Bc%u0S4_iSveqN1W7~JJJeb|-m}SW zNC^g3IhBG8_4cVXle2iGd=U8{5=KNg&7F&Ryy#wyLhO+jk*mhKQ>;X^G9Dib|Hrii70T2o72*_Elm3(uDUlJuu(DV%UQyA}fe zh|HfJJoPi^aO!XrHk%%s%iJ=QIXfr!)AI{c1!nS};b9_fo#T~em7wiTCR!cTE-q3v zU3Zel_f{^SPwiWAWnTVVh>QEBvIL!(gSn~IQu>q@ET#cnTWbN@NMWtTZMMPhy6qFn zffS4q8eBV-X6s@wVEs~Zq%b(~nqxR1js~M|Z8b-Fr!v=evzcIGm-A070<4zj0(E=j zPSr=5ogbXciVIS0-O^+f7bwIQ*h9J79J{oKn`5I!bX{4E?zW}w^)6FZUw6?DyCw>L zVQI}sg9U$(TBXk+*$n(z7x!?wH>P|(`|CH0&SekaO7@5gkQtqb9%{$08}uvvXH0sk zhX3OqfnnIaTaS+5ISVBz?Uns)M$H8^CK>b0(<`RwDl@OO1VSsK>nXC)38i01?)Q>9 zb!AvAMz$)#Mv?AN@wMd*C+{8?MH#gw`}<^s&JYX?Ydc1_M%90oVOf140PWH z_oO|N+Gsg18n2-!R|@(EGrM;(`v4f96A>h;V3ixE5w$&B^24OMMyz&=r6R^q&K)oN zI>5>iW#z&EwlR8NE@i;?I*)tWy%~%~#~TXiis`CCjkon;l@Mm;h3V3^;3bb3VjPdh z1%FVWIjmdvuf>ibn16aAd5HUsO3QqPeRHzEMnz|Q*yzPm_+@hg^sCs$X@vlRz$3jI zTZNoffJPtZ+3(3bj{dDQPYqy}QNSF7IsV1Rt>W>Z-VVf0^&VJEI!t2x|>g z1N2DHss>Nxn*F=KkN^QjiTNWGCH&=ug||%LZ|z3}bCVyA8p)~}lo4xA#-tysu(dCg zwVTkW2QMyq@r?&0zwdaQMYoZy4=7mqn6d~$(%E&|5qMt&bT@n%N_lvBLJ@h|Gd|u( z*4uDbHRL1$Zo^w*kk3h2vrC_pyK;ZC?H4r_>xSB!-XYh1+YIBKkX5WC9RfSR@pyNI z-7lV@R%??8CEV9y*AXN816bZtwY&{?`ElpQu2g7)uu}O%dh#2deqF}MFfC!)LG?K| z0(%s>0P{>jxA^RUinkb83*W{2XrrwEhUy_YXySi$*y!q#EFaoY@s$dfhoh>i4{W2k z90$>eoF_1j9rp6NPB!!$pnOe$h}wQN)zu87KV_`GB2~zHwlc90^ubq8{v(l~{P)=zx_YSyM{>|~q&N*_$wPgdng5Z8 z(0uL>NU^a02mx~E=2$AFc~?ub(1eM-L$OWrh62yWhS!!28%K9q-F?VK?eeRoo<}

u+0s$%k9>kC4?UF=IVFAJ2M@VcSywh7%%SSgF%?6R;A97MX_ZRcm|GP}VDu@0g@y zIwdz+xP!&~NS!rd#CZ)A<@1Y=7wQac4|4}TdEajw42@e(wga{7N0ka+%GW1B_E=)+ zps1o$L4P!oiPE~3;;8AcXO$_8jGR0;%wy)fnPp~e2#0!P)ZX=Nt+y~U4pvM+8d_D= z;%rM-*n`gxUN#(+=!eexj$#1?X||Ex-QzA4Tvmw{qt8EqcV_)ac`nvVQG)|Q@{rK( zg9yT%Ab8eQO!FDJJURV$kh5=aQ<@2(z&t$FiGBJHH{3;A;DVlBGzPQLG^5#fAd1}l zX|qAgVwoDAn>N_h86Kfi9@=Lo%WsZ072%=?J|17k#bD`I!xDv3(i1v8-&nvxV)5!> zTK{^wS~kU0?bl$>m$7XYKre&wxb6shNhlbTQ>e9%zUqbyx)q|*CET8AN;BrCp#UAh zJT`2>p~UDf7{Mna*m(=~_i3&#;x~|?NY@9Fcu~6Qv?3vJY0Y#C!TzWKeGwFIH-+3u zq}A5=C5-kP6Z@VndrUh%_(3?wtXD4vIhgWr3YLn_oi$WHt`DW4I9fy~j{rRULngb^ z%-46W&?j(r-Oa)Liv85`XUC`m#UYx9M@^)?_SIc$S@K3(Ei zj2}n+fi}eBp1;93}eFuI= zi(Nh^D_tUPniD>5RukNyAJ}8Lx$!IsF4``yh#5&_ohD=oPP4}a6nKv2Z7YH2N*JeY zSe_c|Rao}aVi!^Q|03)gqw8waZeumJ+1R!lv$1WXvD3zG8Z}0nG`4NCvDMhv@xH6? z`OZCm?sv}^`Pb33W3Rm)%x5AdDpU(lk*!=DZ7G<_|1*@JKwWv@#iLl2!~QSL^WPm6 zn`&#R0c(MqF=lzKSeg9!M-m_Vmex|xh87dcwDMEp5TyBb?dJDe>E}tBi~d6w4c@YV*_qY9%e0Oob}Wj zEZlb3{7D94Xzlx>L)cVW;eVNO2<=a$lHO|`lu~?2294uiVpv(&Ra2B2zBc|INsbfu zF~|~Hrf4rIJ(TB)?hkraA`xHaUWB>VAs(>W29$mh{rD4@RO$-=uKDF*D)eS*s`ue) z;mhR?{lJgDa74UahVZvnKz`Y9$r}-^D4_Ag$Qc7)c9wM{NJ~Y5H`d^ z)>i!Q*p{rS@#~_y{n^5ak1sBK2_z_`@7>$tOB$Vs7$(yrkg%-sj!P$|pD-BoI(7DV zjh`FITnxPTCP={y20pz58U$f4Q(t7vVDoE$4W#RgvGBN72Np`LT4VOtr`^FMzx5Aj z?MT^TSioDx4#lV~5H_URb_s(D+>Ws~o+Pl~amHc=OVOuKxF2rSvA%O53ia%Uz&k9p zYLYlNmTHI$E8d@}%XIra?$@|?zC9E|uQm=C8$X=Xnh&jc?$ba(U0yefmt@Y`H8! z+|@whChCpm2+h*{~{=02YY0!T0 zs+H7R)=~7G;t{9VjI#MD25=dEZg07r9qw>)v&^}DC!YLq(piY=?|75hU1`X2d+vCr z$doQHA32$AK9*@~HV4{ljYURDW#!GcQUX&lm*E_4ti#dQ$EIz$G`k)q3P5dyqnOKd zw|P4H`%0qzX@A^?FTUK#h?^35_kv8S7&xPxoOv=Bi2<-mNCbIcaTuk~30MeYzjMv} z4Y4KyC6IZUUt6%#v48j`*5)a2x@L=ORC|{*!rOz}=+{I9B#;b9qAkP9X-lqO|wWroH%PLsQVrcF*%6VJOL zJwA-UJ`Suj4M0&mO!e&y^k!RUg#jUw4>u}zm4FS}%yzusiwb%y-UqMsA5MaU32Dg^ zzs*Tli3<&+C+0XKDC1X~0L<(TM4J z0ooEt1Mo}On%B^_PO-h~by81tr7=pAKNVrfR_aF3W^QS^yZmNIv?rl?La@P(|5VOS zC|rw>@7-}v*C(_enw4wBVp)Q3p*)EUo{6hrm=L~^J7`a;GTKI2)r3ETt+7;n4vY8@#uG<)kU3tTu z9$3n^9k74T0XGjd)r|HIZu|^E*ZgW1%|zl=G7KNu8c|MMj%WN_%gq@rxi8;nnH=DI_>Pd{})mm=cM3^@|u?N+$tI zHxvef{XGSEH=QJL?vgZDZ2K(D zrcVreKgcD}nwZDfzor>dE_j?GT#NXS=}w7+tD~Wc{YjRTFG2`dWL`<*wU6Ej`E}b_ zHcX8h=l1IOpjaXrPS^ssWLkw&lt|l5le3|5k0SCU491z4 ziS#VAaUE%uy8tGSX2zFICP4gk;4dR#M(tDyIh^(RRJcHxc-wjT^@ExDvUzjTkUumI zsdvY=fFy-RG0R4bF2;eF9|{y60Q*8Zr`4C*hKsNr+=+o_e7ZSi zuZ25ZoPdHFAwa2nr>B}e7ln2FyCU#)cwO?HJ<*HSYNPMr{Jc1vu{S|c7Q3WWQzJ%6 zP+%uff{5PEHsiRPbR0*aMA(wRMX$rx1DA$e8PKUK1*dxUi9y^wwL0mw5dcoWyrVVr)Bobs#DDAL!sS%#xemE0l{Cz4$`5n>y1^~B_?5;!V=)*&~aF>^!^c@x(;xx^n52`_RjDRsu}cwb`}w%~z&qxe zVrJv~FJWy!Es2ud0<srh*R2R~<^9K`m**-XL9g(p9Vx!@SCTg8TrBe;y;o{P z{_}=KwJ#C-OM(XyPLaKknx!Q{v!s0JnFrw#H&`pbo2vejHn#3pCEGw#Kx8t}#ahco zGCCF^K9LMiO49y*dIQvOTL0}ZS0LHFmXB5H--z^wMffwrbfYOrJm@$wYs9(Nu?EH? zw(G;sGQIrXljM(}kB=H9^uQreEXMy_=DDiU@Gl z`^$AVNQX)NRTJs~17tA+-Hbp;o6!^y(UdH}5O|;kTpqEMGWA?$qzHj!Q94_|zk>wa zf%NI3a_DmVt%FWssO0ropQ59CNUvI_w<7i`KArC$NB~8i8WftE*OKMc*^Jb zk4Cni=BFf{cBg!oZ7)>-c9V^+&{R+XX3f*P<7hk*Nd=w4wl?28yY`z&!5L<*$XN~! zj{e-|x^}fcdwV9`!H`Dx-@akjAP*9=3ZlG6w};vLDnIM~kut)<#(d;Wzr~IEr*gL5 zY^-v&0Ahj!bjK;z-?Adsj)5=Y4=jzF7n)~H8@>uJSGubJ1Z#m1OT>%13FPu>|DlqJ z@3%}d{9~E$t!L3W67b~0@Y)RUDvCx)zo1GG3Ka#v%rUOl+s>)hm`VNp$N#dyreNwN zymj7|(~@Ru7;B9P5fAdA-!jedNL7owKEJT0lJCwGn>x9F2H%SJAD3!KJ{h9U2F4-Wv)+L_j}DE2*9@r-cs7@ka$bC;Y<{%WxA6vZfD>VlTv|TMuYbVq!IH zH`@<$y5EkVVufEB9sxC?)!CLe0>9=PkrmDLIWp~UK?TJ z2R&cHV6uWp)%JALd;P{1y?3%X+){riq?z@E!X`21edBL(TGzW8&JBE$tLbFcb0OA% zkv1PkPu#fy1D2A9Yb~6W9Xt;;0#=0RAEfTjRwHP0{V{L;OkgqolJ-u{|Ky|B2wy^-X^UHv+%iQY-Z9L~g0rIC6Id|}^D za#$cLS(C9;=4eo68vxsc{_wR9xZDyS@cmu#u}{zyo7p(hsd08o-^11Uyl!wiM^$0Q%NdrEtcJi=56=JP>9+>Ux2<{ z;D(yUQ3zew`xGe2_mB%goJ@7e(nmMqQvrd~o;4N~P2Sr_#VCn6)s!R##V_x?k3x87 zg}Yzm00`veD&ierIe8PyDd<`B!Lpq`GnK~cmv09gS&##EvM%%R?XF9}021DY?^uXw zAEl`4`z6wkl1w_k)b=h4dm4)^1OV-t<=AZh*Si*-+V@bS!(U-jw*x&JLGH=~TJXin zEtiLd40Qc5T!2O5tW~4imnxb)6V89jT!EUu4Wk=LhIy33@1(#XA z54a5g=KqZ+gp*N~(#GNHfg|R*{Ppff9q?MN1)V3;*! z>k<6(WU&tj&5l|%x9jh-4woStJ&dJ_kgdKi!4R8$ulh>cpkw0!yi_=46dm#Out?r1 zCDU39FoCak+5oS{h-qelDCiX+hIKxeeM>v}K%<&y)K0|XK=(#WhEg_~4eyO-O2rt@yAjgHL#5psa(rH2nNN7M1SGWicY7KsbFzTA)} z3JTqd!2+8f1YJYMug?1H;x_!Uuakw_g;_0VZ`-+=uAF|E^f#LnynKIl{-*GqjL5VzIzeh>xSPAHm%BISx{axZH}z}McdaN713W8c~t;# zXRbzm4|@fVCcy6RE#lNl4^Jh%7h^vRcy0d1BgXrd7Q5-Vl65~QXd!E};f=@Y8J!|` zQ{&CvaYeK1)5nu6#)J`h@{4CsYms8V5i(kc`{*e8-XyLC2nX<1(m!M^Av&c{l!2na z`U?w*y|hfC3&JQmt~}qm?eJBjwK&H{WsklEDHRpB0;5#b?>hmCx2QA;ydUVwoj42{ zV%Y*7?X@YSm-iQ-z^M1a-$(Z=0AY(3g3#k$ayp-DfwqS~;#>#jU`W1Un`iW*Ib{Po zYHNrlaJNH;!DZ43W;g%sGx=S>qg3aQ7gWoCgk@rD7{kUjFps$b#6}%qGDc$uF(!{@ z?BgT?RA=WAc~$C9;5d2xjP~U@h%}IbEqx=`=2}Dj@^v#Ex=Y(Ssk4UY>)u&ZsjtUY zMH3XOd!-e(p>FYu6|iLV{0lTSO9-AnzE?!&^O#(uofs6b5W|NdcFNFuILkU^w43KqmB!rT(^Cn}s}E7@_Qq+5~vT zekMwYCk?yox5vua%FfvoARyRIxL^+tpZ{CxprKp?nijOGAa^w@C_gVdN#5V6E6<(`y^WfB%DANy5E6>FS6rVD^en)G9<2iv7*uZtKA&9} z-MgUZH#upJOZbiyf^ej`xKc4fCNg;$Q?uxHw2aN3fY(*h2>d^$bAQL#fB(!-ZH`g`e0L2>PCj{H*GV1MRy6Irv$`--9$tOu=e6mc9?7#TmV-iQsvVS7WZm*JEdeE044$y zw%{Q$(3z!nw9zxD$rOODiVmB~CG_+js8<7cIbX&L6D7lt^gf+fVdwVtqLO)<0>r`l zGk4PqqqNJVlYRg~Cl)$XTd#94{qSwG*y5VXbSEzFk4TcjD*LmDSdG5;#0;bPR+WP=XY(7ccb^yKSN7DeIVNNT)fg+fghY9n3?!|Af&3P<1f24uM z-~@*6`1y!p=sY(G^oCu8;@hpl0Lb~4F{ic4LXg~XeKaM^vAg*2WyuksHi3(>S*g{; zclwn+Zt1pmmy@yGfI#7uQZ6^^Up6Z2CJk0F?IO=ZV=L zz)ofk;nHca^57`|W>wIO_KTi6p<|%OrB@f!nO?Y_;753iDRwsVgrX9l;mNp;l6RBC7*DKE?|Hal-jEMT+>0is6AG#yDTzu z5TA;CpiQk`pMwjq(x&037K99~vS(LY3=|?SQ~QCm7x8c3aKHuSwv|=%f8Na+@$TyZ z%z@}byG8LI1UGpVI($|N&K->xo#Gq4Dkht)a#wt;g)Z7@T{bM&Vd=aLUO`1D?PN*l zBCT$CB;py-HZj4ki3^A7a4YNoTAr!}VsRzD!c_Vaqbclax zH^5b5Co&kIx|QpyRa=RBHeN)lar!nkAJ(@&6-8N;#r}BaB5NUw*gptA!c7W3?iy>D- zFMK5L+zt>2h90g^+3#VT8WnF=>`}{gwy`&olt`zBLlVr8{tkl(2@o(a-lRu zFfPHiQ&`U^h9bp8r10&Q$oSWRh`a!1QW)^|$U3Y-f3!QPY;yp-eFKN~FLbzfO0`go zttf=2pL}JCrAf^B(dd6WZEgf7h$>^nhjUDqY2VT!WlUTmP>y|_8%btCCi+=>vJQ4u z4Gdg1Ae#Bp;$D2aRM835u8?*)IpT5h8_V)yiDC-to!!DXfDF0USjPDJ$h38h76=_# z`mOO%EmZL$R{ib)|6H!=i?ffmsRGy2M*^R^ zpYPQamT~_+kxTT;{SbB9R4#{=`d`Q2P{~C8o^PiZHwn4zBB?#>{FZIW|C&%LtP#?i z`svnl60GRp8tBseXL|hC0`^6|yEU+3hhn94-qcRr>~$NH9_5ZGe@Fi!zuH)y|>PGA-y>1aSy$@;@G>~4dB^;y;IHdpA=Za@;_n?@DYxsb`OyWx`5Vs2KEEg z)kCCW6oDA02fTH=Mw&c$qt6(G?$O_+IQ|u*9D+{)4Ll{3aLf;W&@WkPmEFB*>+7bEGm`iWIqZ0w+Ozk; z=rcZFjRP7`R~s{2!LOsOY(IqdRV;mSZC4kfWp96}Wz+*m#d}&nK^JrDSE|D^u05Q| zgApIl^%VW{pOe|n*I-siJqtCXN(Uf8EbzJ6n(UFS ze-A+Lj*r(#bg4f7nkV}aLI6%xiQasWB>*s)Oc366I6txClxQlDWDJl6m9SpF(*Wtj zU92q!qY%xSa!D75gEpO`mBmHmb-Ok~2spn)3x=BR%SXtZF1NU&20qbWwKqDg9d=vi z{$kxB6XJVGo+j8fwSD!a1z5X)nWTmJ3KwI!mNFc|A2oTU`K@fPSilw^Yfql42e3{x z04}?spiPzb6FMC@VDx^Et_yW6q$}MR|8Z=6x*T>~_?bTKpqlFWk#@_o|04Jl*Dtyw>V|>E`fba8Far8P% z8f;W{V)J`?Qd^^&o9k=Lz&~D0aS3i&p3;|O;u&3<)Kw04hLYp7PcUXzk1`8#fo9bGYKCTE zax=dD*L;*iw>thJN#=fBIYBY^SbrNga6f%BwqI?PtJxCU@Jd1b=S*-=_2G$1MIt$; z@>~DdpORBVR5TO^f9N}dYR5M6r&dlvIRUHC9D~BtVF#&?>pN^8)G(rmn$FM zt5$r6Q~&vZHyhPO38;8c`1xl#0#AN_l(D)xsj9hoqsE>5`gG^f{OSn~sFW54x>>&E zSgqe=&GK3^$mo%*m9PRaYpB)|)gaIZwbg5CU`kX*N0+c!ptPuZm8wtm?aT4%BDyEZ zlz>7uJEf`=5VamP&vos3-lDtnxX$p#7&j}2Q)0$W8U&ndU73sWtf#hU8LccKy*6z7 zDw_1dlHa? zrpdD>vx7cQ_uZzbOEw>B3)6Sx??^cL6&eG(Q9Xbm0=~W>ykT73JGA)jRBX2;p7PIz zzlRl$hG!Gvd37y@w1JQS$&(g+E z@N~MY2OGlGuhj(iWKG?7Q^qcqIiXq07n5VchB+fJj?z9h9HZjeC&a@XC`wgdI*t1B zgiVHH4TciL?-gqD-bcH)eduyzN*%8&obUFQddJ#^UA^M$&uFCd|+BgcZMii4rJUPyBIg(kTl|kTFUch~!En$znhH^_d@P+TsL?&(Ogf7$( z`z-+b1xx|4RecagPumE z!SZfJz2#VH^L9AVbfux-w*>Jivf56+`_ro(@{?@J!RP{Uw;W-=#&n+tSX5G>$iKa4 zPU|Q|Gg`at(-CFhmKR>#m_3fL;@w%HOoG|t^?r`;u)v5fF~1v(+N}-tTvb`LEr{EG z0b8NJUOEDm#ASke(fc%C>jRQw=i{GDvxwK1o2lsDzLf@hV7%#KN!2=+Z4)AsL zjRIbOywjY=&f8f9;e%%aNpvDLm^#xwxSB6hY7XnI^K%ikWGXpAh;hML0)21222vmuk!-4&La8=s)+6zu^nw z)lTin#LEFs^d>fQyIv)po0?YTca?G15;rA(*X#Xix@b`6!OPeEksI0gc@jZS>>!SN z=J{s2Tr?5zi~H0Ai<;vKMt<_CE8)Z0w zKyXWzfX67K;L(wxGCutY`P3tTXQ7e->@;hpwB14>v^|f2R2L=kA$uPs(BX_^`>74V zJ^w{j$F*MKK}pc9QF;*Hl?w;-{a z%sOgG!xN$ctgyOrT8D3Lfa}-I<chozQ2J_A!?Jghuj*e2UO+mOpB8 z7pGI_qt!4D5+%udR-gE>1Pd*f_%T-rqlB+XLS6elEV&{39U3{i*8)tBLS#p+96`X` zc83VS+J+CJ7Bi>Zgj3)IDWx8#F|gL$(`MYa$@L^o52$NAf2KFdo3nJvANExyb@v&Q zt!@Mc;I|6=+}UT;D>4>vvHBNv=|O<909=bUu_AN*)mT$gnP0)@cHE3W+Xu9rl-cQz z%?3_DOr^rhphxrp6JDdUR}>icwMmmjezCnp>qWE>3;g+by4v(pKku9UfDWdt6V!U^ zk?X3Xn)ikNC-2L=Rd?fLBb-8<(X_?L&F0`scuXz8rqrvgca$$4Efp&g2xvY1a)v=Q zZYh7bpH{KcVfblab>{*W5J(O6hI!r|TXwroWrTsW3^Uviih-JQGXQow=4=NUB|o6i zygY+bw58L|y1=b~Bl)BEtH(Ql{9xVVka6A^?9Wqlp~KDQGjK=T^)QTAwlbBwgS zhgsM>pcA2Gj;8nSfUJk~k6baHLirEGtJ1xPbhOAw#M=fZk-{*@S^<43{qF9V90@V!dW5 z#Xjnl%tJ*0;1U-&+=PI-Oceo}+~7udRoi}S<(!bHM(0;>a%VJjIN64tB4W60Ic3a+ z$8Vl=4^ zXMEgsHAs|`c4$orH(|c3?KcI z-Au2a+lLvhkj72_8zr;^u_KH2TX_NVLF8fcA~ybSsw+1qEeAP$hcg?XY1PqGs7t7P zgS_3MrSnI?%sB{#nU5+Q@l$MUS^*n9A2GMxH<@PQadx}n3?Uy5QtvIuDu;a!VW%}d z);7`4*enSof?Dl#Pdr=PnjB;!dO1^HBeIy#6~yEB)!3d^?5o6;T@bj8=;jR@<&ajS z0ap>DFWegPnB19J-XGPb=?AK=NmbGJ-hY)L^?`v8TM|hpzmRfU>)4GKl&8*1v6MA+ zkyA-jFIL9kJi55o5RW%h)A{)xg1MxS7!RnBAE%a#*cR~m=>x{x?~aOd@4x@f`_@>Z zgk-$$!MI8^`p5v=t;VFk;<6b45;v;jkaew_{g7gE69~8&0=8UJzF!qSW?6jXxIJB- z))Adw)9mTsf38jW{X`R>9hm1 zLByp_f0%OcB8esp)A7(zu9cXqXXLYwlMt!nJH)D#cyJJjLB-~>)vacnKs}ZMa`quh z1G)PcqhS{{KjiqG_b9M-e3G_VE>+Ils-u{vt0IZ!7l|(?Jw0ys=`IPnXSyH+AuWC3_1qz@bN-xwU_-Gn=O}3QO zZZWd@*&_SA69GmvDjc-=Mf)%Eb$B@h5ja?|Y@@FC78mg@=i2z@K*>t!deGvNrNbk~ zla`4AO4@Qi7zQi5ZlWca>OGvjn4%rh;>#?^V#n(ZT#)(Wq83fMoIot}(wg4FilK|3%5S zB7=^VFDI3AkIF@Eigi}Yy*^UjP;00pw>mWv>mViuUe4oX$IWwWW$0QeEvIA}y(_Se zbcXQB>FU9ULg{6*fx1X;_LA8BvWW9_l${{ox^x(_juV=aQCQjA?FgRR6TF9@7KmkU zsj(;_KTs+bSTqc5B<%|;dQz^q9o$`(UV?wTFuxN7<}CDbmdwaL`Dj}rA4V*48hTsV zW~Fpl;Nr7MAfUne#>*86&Uwr#QPXB4!j#p6+TQfRM9;p8E;G5OF2cz{s?7<4QcEXP z5cf?KUIyV>#I%M1=_h9{2&QH~(Q&Os0!53O^tnS{(@6McepD1I_e6VHwm5Cua)Zva zU5_Wv6F1D}k!NGdu%XBhWmeI)8r(7fvFhUg)KuBafMu%0%E6R0%+pRhkG?83QZ`F=Mkcwyrh z4`ZE6MJQzP-;FxaJ1UdlR`E<;9ywXqx#{j>Zo8zOFYaE1XB^Hba0jE*JcRaRba2@w; zN_Md`z^~AP83h6bVy$}*DOkOH%Mc$ift9Tna?b&l>xNVw2f=beZHoK!?!t zd{c?^FXw*}HqJI|ETA6Fvq_+7*Vp_7 z@MjK#SW*50@ht(QC8Tc-dr=^~#1--kHRVaW5_v$*vj{w4Md+?o;13*^&>_0Ha`=I= z7cq*n2we$pmk+N|-azn7c4*)n8tvs&l@?vuLVu3d`ZjPevSoNK?o`ETcv$pPS92^s3uGq&Tod>2eb3IH&D_ArB|tuJ-iywgwo(dO6ZbCPIx zgX>)?B8Zp1w%04fUZ0aq$YptG|r9=?Q%csaqTJ?(aI4 zWU#{BH6b?H>TWVtzlwZwZK7UF*fQ2HdFTy+;W4J&jFzkxbi3R{uGt=)eW=ogo*6Ay z?rfQaFw_iQ&Pw*vG3%?aVq>oi*e|y>hdVzuM9cF)tV{nUaxJV^uD_#VPI z_;JeiBP1?517T>w_7Jgq?4ST5gjo3w=$RXi{Yz#8m8mM6#3PgtUIIiUkeX9Gi=`rj zkw73b3MB-Ymu<9VKRmB(ToFQG2CHAdBxZ+@S%IV(FQf%Su?1NwHvHfX>j+s`sma!4 zXQ1Z@e~!c1Xa~XA{S$tWw{4fRRv3W*l}8*R0`izU+50)N2GgoveIAw#S!~F>D7h;|PT!xqh7j(E-OE>_(Kw9tarm6p zKro+%{1|kw4n!$zHbfSh^ha8lOq}-wJPzqa%F(DG@;|KAl`F6Z=FlkVDb`atj%^bV z9HhY74l`)mzRFKs{VuW9aKZY)`fxgb1&*eyTO(lgYU9g<{FO-Bo#)&(@kLVlMfbFm zcC{%ia#Ols_P633UwLSFh4HzX1!!{7CGXvuFfSq@<8HkUpPN5AnR(-Ad3-$NYk|*a zqe4!hCyBrHpluC#4%hG^;DTjQLZzk-6yQ>Q&nB#22^eBK*W=n*sx z6)EU(grZjH?L&1@2EcBa7VLe&;wX_pJLn40bG#_m4fF-A-$JQFec`iKPJz2UU2FN$ zC8M!yL{@(c-}aajurO8LV@cdGSqbaW0*!X-_G!rmlgk6~<{J>CynhFJI6%Dl{+quW zGw|i=$GDEcU}h$!m-8=uRX?>$=G#%mx`#GjmInKSnzc*TF667G9r;1n_a-KKPGOld zcfltiE9a1S-DS<8Iqm)otv&^mDUv2CnTjD7+0blsj2v=wnf_!c8-Y~p9vO1Vlq!Lc zK7<#8@PlYX5TuYfw?R)yzm(`dJNFo@1P&k+M7z^R^(i z8w8W)V=6M#$Qkt_(K5cl$&P721>~#`lOJeF!2&zJK594mNS$C<@cTD1sax3N2mW9J z00?SX8e^+gSB#bX1%+HF0-*>=4)q{IEhQx-*oRtn{c<%#M(Cc!VEAJw;?oxnxU%x3 ztpSo#Et5?sfKDqN_iY(c;zP5A|Dnnc@*+NnB40?8djOa!_PglN&iz0@Kme1fT|Mv} zQ4H4vx&B=(q$(CgGGwYcVkK~tJ>yCwtHkPMAPTdEFToL$WK>MN3bldI<2F-(89x7} zu3XbE8+jF0J#Et7KxrKG%8f!QwB$kotJrlF8a{y^?h1x-Xb61uwXAdPKovM7M_Gb} zz=jaGJ0^%wf#u{n&tG|Of(}#nGs9&3@A3wNcsvHhh0tw9nQn0B@oRD@mY^`CRP!jw zcbu&4qO*-B;lpuUJiE#U02WH?e$9ci+5)!)BPDEPtag2L4gSF{$L!@01BfC89W=CCKT?oZk-i<*-%y-y9o?}j46a6Bykhx zpPCRPqR$YY8Yr8!tfs!o3VM_EKGY)xqotv24tcX=$OQ&=Za-73Oc%~wxKi9-WbqvqTX&}7@i4{MI(tw1<4sQ3rfq`jr;mwE@(e(h0ak)^sK z^|E&nR-J=J6U|ccJN>9$EZPRzfC#t>d-D4%5+rosxPlK3uebBtXx|HouV|!Cd4wQW z`T{}e3R?GxVsjj^l9P+<36b;G(Y1@qvc1}|J6^7fEj&nJ!=FjhkhX7;3bL;uIQwqT zkpA!^YcVg*ZAGDP$@}F>XmrZ!-jI)WND?rwbNfB0=!fE7{d+t^^LN1j>PDi*k<$l` zEBi7nBOu5(&8e9pc@as81Drgb8_LH~m`nBTv7~uv{m^p4-ihV!d_yUm_(UsZVLyAy z_gr%S&2JLD)c>Q2><(J8;9y*S_*AES&iL=s*;4OE9~9Ygx6I-a(*11CbPjz|>5K_| zyVza`c*t&@*Ij|@4iKtlCr@VRxMyjast>^#K4&zc2nf!SYgs$DT&U{loM^?hF9;mp zLcRw@#{(*dgrQj&|HJJ0gnHWihKBcX|AZ9-H-L(|-?Ww?!%_#^0kMw!2bS~Ct-;*= ztkdLEEaif=xv8JAKcADmnuLd+(Y$$e^dx$WgYu6zdEK5E0EpAZ<@Z6)IBX9b6(S9hC&dx@N?lF(?IJ(cBf6CBwT3!RfNchE-Peq4N1ZX!)O zPLD6s3K;2pqc2q)44j}VJNx+YNmeQma&;B#*U3lI_vlhlE9j9w{#jq`k9Y)n_kALH z9s7f*kwmAL5iJ-#hAaC`bW;$23OSvSUoW?QEE^0^ltwS_qo6KJgTF!q?@;k|V*LnI^xt<>AX@W|4?a6wR+IQGeSGeWsg^%fEZ97+qM+a>VrMAEf& z0h%H$vfghe1k2M#)sawJ2x1NlA7HZp*fzo*;wSsi?eTmJ9$IBlJc-~r%pt^5n0>_C zFMk-^T7bqVHHM`?207^ul+F|>h@BpM2qH=dIC53jLt)?5+ZUC>EZ)s3u+EDuAFh(NhoW9$iTS6KK~lmq9t42aWgx76 zS-d=4aXBopp}drhCNl}^kW(+12Uk`Cc)w$Oti!t}0p|^&)}x=9AGD+mL?H;#L>^4k z2Z77Qleqy{2coD#ze#H$q6HQ?ks;OIKys}OVt;!%!rBeL8L^u54{5BnEhqZ9BSah; z{~kJuf8~YD1M!q5TW2^dxyP%27*}xY6cD5C(wH4I^RqhG74x*;)aUuENs30*;k>&} znbnuZV?07d^tE1uh*_^bayke?_YA4hbkCjEVFhXYm)sZ29}xFCk8suoK%%igQApIjOY)Jai$ z>`~FJPYApLe_?%2#wu_f=F-QEiN4zsv4tq8OQ46VMsw;2rZF^v&T<_ z+F%8D;cwy} zj638Z!AkjzIB#8+yi`)QkOJ|F!9enLS=H~8L&G*-A5J?| z=cSwXO|H0heNB*5S6r&jmAniGKj#C{9$qO31skJB5Yc@_9$%NMDf(fc0^d6HCc5Hk zeJ1!`&vW?lepHUe3mCMWWhNDgWagP7>d1&pEI&78Fugww6S=&tMu1A_-lYS(lgNi-w zHitLdvGT8~?#FO4j=tmO+Ws(|e)WZZ zC)21=bvsl2zNlgU3!VDY;p<1saqLy;Tf8#>d5hJZ$v)Dw??iDtb}6xeKVtx7+H}>A zQ`wK2@IAG&sv{jujAsrczE*TP``S2F`3DJ3`4xT7^o}^joE#b*mhN#JvvLEgRbd*7 zvZAP^O-@OQzo#`>3`-p57AUO0KoJF1CT$)T+4Vh5-`lfluB`g+VC3g*T3ZxIfA>aN zZ7NS1I(>gR23I2-bSFGtZF97X7PHUhxsybK<%D~ds+8$qhwE`t<@(_7-efM3a&z!~ zMGS6pNI^p1HpwyQEI}HV#>-=z)fOz%` zq;dTKgo$R&mlFbSsz& z{KRiZH#13_#fAROw2WwN29aYEL$KDX%LKMuP@2K@pAzdv|18P0+RJCbR&L`Aeg!o- zz!PKy=+^g{T%@0}J|jS{@mDKi~NGbPPbU?&zB_G9^s7@W&w{d&P{aO~C@h%X-hho-ZPit3Hl_zWp6 zAe~A{H%NydlG0t0(%m(Hf4WOLrMsIMq+7Z}y1Vlp?p^mId|=HQ&UxRxpZz?)4Wv$v znFv@Yp2K(2)cpOrwp9IyOh}Ai#=AiOjKS{C(hVAZ3y=6Y*(>>qzkAKSnjdNOKSJHo zI={K^!T*&FBfBeH1f{&?%noZvmKW@Xa8_f)#rc=#lC2cIH|3pThp+k(?%xr#^(S=q zHMV90a}}Hbj-ijOP5S8QQGI1wHRywpB?cnBgk|NC6iDmD3q9%6^;R1BLP(Ibjqd$C^?_pGBPm^Y7FJ zqwI?7=hFL;PLei44fCU$muQh^;IY>CI>IwDtg!I}Zn3g&8GH3tmef4PEN)emy44h<8w)>%={{iG%+qBNzWA23YIGf?9QaSYcEXJEJCIP^^74V+#Xz4M#Od{h zre{+VYqY25)_Rv2_rI8;pZU z9r|KhlX)p`Xj(fO_48peDeu`DSDRH&Dx~@sHOY)VS#vgeQCObNU2Imz@ap#N+RX>L zg$}H=rz$=@kmqS8)4n|~U+E4`$+_klrzNy56@Ji^ljSu9RKd_a@7&a_7yIg7Gm=C@ zG3=7__>*PcJSWfggvpDDA3i^kA|{@D&^~azkefMuf_|ypp?`X|8-+&KL?20D6pZ9X z0CW#}PMo7A3$FhdCIuuyJH9CZD19;P8T<$j1+?V4+l_`CttwgnYOF(7z*N9#nqdYD z=}l)OGF=P$9W~*PcU0cE5;pM}Sn9R&1Kr6`rS^m2FkCz#yjETfm0WXJ3KK(Z7wU99 zrUty3A=}44J73_o1=#Wf>J@O<);U5x%Y;po>$j(}K-05~1K0=?-DrJFs!ve{lo;5U zseW@O2hw=#<^Kv1+f$A!QenprMC8HOYOp3cDe}={3aCcB9{hU!mEHI!&W1gZ030TK z+b)m}R=w<_@RQh7rwG7SY!QJ?p9MoYL{Y;be2(lIJ%;?C2}Dm1=XHLF5o)eic^1udQviu|*9V8iDS%G*fT z47H6LlOhbQ15wntS!GOt!vC`!fMI4o)sUIdOc+^-{)5=r5q`Y~OdE(V8oDvd$Iyai zfMBl19-Ytgf+HrrUR9iIfbhmW)nHJH;qjlxH#7Eh2Dvv5}B`fLTA$I&}Hgtj9H zCs*#sV1cH~wl0A>%hB1zFLemT^5|SM3zNVU!3TSb?*l1h{<&e%R%Khd&%C6j`v0`I z&4H1;^!S$W%p!M-f}&{2y_Z8Oc7fam)9&VH?dM_KT$_&<$O9*NgK)M1dE~c`7nV^q zDg)RCgcsQ{Kui5dI+)Yew##iK8T-2gTbivKdoiMWGCN19CQTh#2!KzE@!#*t_y$M# ze&ao0LB?3`Cnq8mql?cQG?@%%HRF76=8@7E+TJxvnaT_PmEt^7CQ-N{d@wiOcTk}A z_#oypCi%;my8>AkYwik06>W0inc%Q@wo&>CPWRC1zv}5-RXRewPU6QUr$25lyZiff z+i7+sd&jMjP?`l7n4a5~|%%DTqtV{8$;!RT-eDB&##o}ANNxCl3VOk#Yo?O|B**gr*n~wF44GM@hJ7;Z%w;CD1`Iy8$EOyNP%SA6S;^Tbl>lx_ zljj}k+Pn9}P3kh?k_4L2q!K?}I{ad{F%wycQL%mJG;eI-+DT0Lz4Me733VWNSgX%D z#Fim=`eG!Dq`Wcv%&+OIuIh;8YAna~oU8>n`|`YyHhgbo`Q~@706C>yM0ho*iZ_`f zUhw7TOe7ciy%m~T$C=}A%U=TO{_j#!I19X{OE!i@aX_yBs4jpUuzcr-SeM-PxJN*i zi)#tO%n1h%#vmAu>`fNp)*{Ic$dExmI5A+6(G(A5Efysdkktpd78#JR2T)nm#Q+Nk zJ)d^R^H3mU(tyQDQd;jb@*1FjZEbCFUw#6E&_yW)K$vFzTNEvAsFk=7TO+VvNpA@r zn~ur!@E;u3q4 zqa*;J*x9;-74mThvNen4L@bu5!a!!sf96agi5H${4CNXG8DT z#t*rr7W?RYITFQ&R*r+7RuYN%Ud2Bn)WTGk{uO}Xc8RN z22NqEC+aG{ScDhqr1f+v_4QLZ>4F`d@>V^j^W)NR7L=^Da~eBPJcp(N(sFn_3}2PA zrRKO#it$@!fmYwAisV*4WqyXfWcEP~9Cv4QYAtsE%JDxO%JFYR$gQNON^kU>Sy9EyaE#U#~279X$8)(qFsi zJ1eb2SF_xWxc0u7=#-Ox%gVfM(6*6fICrR$BxcKwkfTItNOj#WPT?|=uwH$zR{Q;t z%cUes_^w+kyEM1tcM(|5`(_y=V!-H5*77rzXIbIt;n&%MBTc1_#<(365oJ}fyuT8P zba$uIlLknC*||S?bx*QoS8}waq5rvwUUg3B7VP<%_LL&j^Ds( zDzoMs^{C!$GkUs2I=wuc;E2Y?y^Cm(QBK>h!4ycZ!8^n)`0r(4O^3^1IMt&&Heq@~ zxC;xG1EQ71S+cI;V8HvD!_xRor22ZWaVNx+({xM6v3W`OTq>kW$ouP?=QFR{y57~? zG`g6^B7KaEY1|XrR+W_)w4Ot}F(FCPT2Z;u0~yR7@Op z7Xcqd<30QL_wsO#K|?=Z0#I-$_1hs}yU}#MM=+H}mdV~@Nohqc5I&RK%HXxrPhc5G zg+x$SnKkJ{zQL>s!a-AIaBPOUDeTNnyAF}_ng(=)FQ9d#kjW@eM48chm1?GggS&sh z6cRDJU}RF5K&{)Pxke_dcHLL_oLx>(8@yoxQC?1m?q)M;-RS##s#^MJwHqU*8{vI8 z(3d!Wcw~mY-~rI!l#R9wAKmpQnzl^r2KMJ_)OOSq9QI`ifOPAG=@Y-%- z`Ex-2SV|fWME=()Fahl35bCDD%hN(*TPcuFqlHta8Y+$ItU>9ukcQ!CkswB#G9YOP zFtwewcEix^88j0n#bAS2>RXzKW4w5u_kekWJcaLVPL6b&oAd-A=)=XA>VaeQuDokG zeg*u~#sOWbm31jUS7G}82}*EZf&bH>{6s+Uoz47ffH|KxMd{kpa_F77=?0*L7N*&C z@XeSV4P#xO1E1ws5Q_q0vlL-*57c&af*Mx=J3sO8&3&3r`FMV2_*~EbnHfo~XafsPje-4t;JDR_{&Q&d3rV;W|BSQS%)ny?Hnhd(Olm zqxooxAtTyD!W)!4-4RyNQWD2oli={Xnk*p;nnBfZ{8zMD$Sdi5!GS1VKjF;FlHRVD z{-2${Kv7jya-Qvxp&pT+#`$<3g$7+@KH+fVNr#T8*O94UFf~k%TaL%}FrSDM^|sX+ zeAHZc=Oq*Q2Qv%uy;(jD$MaGt%M{emC0~TMoyv2?LBh@R)AWqLvYz(kS|1hZ+vvF z;S*8}44G;8z{UUR=CYPvU^E)#4n7zc(&kzGyhT@XZ&Iw^E}I@0DUG@sHo9o)$KjM7 zHvMi36wn0t&PU`z_6}w%39iiEJAexkMY%u-fn;dt$*zbn1C$W>nJx?xo;hx-(sa}^ z6l3#Dj{@dXltDOejgM72@NK1V!Fi-}10c(IgzS*faT5l&|7QVY2*^q&xlnzHc-!^< z1@Jq{&^CPCC~+V@@B!0`SB-*;aOpvk(rd^4g09gd+7a{X5)_>UAHnrE3s$VD9HwD5 zAj*48N)p&r9CezHx@DjY9r!~m0s5>ff~d#OO(acnC%oe==<|Iz2!|tW^>B(Vu~Rcv zwA=7L{Fj(AVt*io+8AANWa!Lyr_gSgWxoCruvRx=6Yv})OeHxdYaGHA$=;y}!vfNS*!?J3c=uZ zML<6j^16N(E_^*_4E$FS`ZbISG?qXAb5wsW*(R|LVZ&Ks4JTm5uU&E+V$NL&tFc*Z zZ~*g^PsK(E0s~+^&U;Ba-5w|SvMN_ZLv;znDn<&wM|No?6t)f+q}-1;JfFO6W7YGZ z+z1=b$@kO6`PNV|=nYN=ID_+j;!aF)AA$OZ7FP+|x|ZbsYm&BFk>Az|{mI*(6W z`;g+9(QL@7h3J7hZ#WPW`=B-#Jt;|Xt2FGRpw@hX+l(lq*^(fp*IIQS+_3J>&d8Ew zmmyw;kU&~icroIeBAr`1nIiW^C|wv;nNX|pjS!okFL$NsJyR{m^+}eP)y(EOQU%kB zu9aWeNp18ma?g)CwJxK4_}IkWd@~go7L@wGEXt3gX6$BnFvZQtk?@TCN>EYLL;OjEE=%b;tbA80!Uf3+HWdXcb zy;k|xiOBIBJhh+%0XJI+xJ0eZzbtIt6`mM(&uta)F73PJef>!XxprMqOelPr$Rh6Y z#LFXH_&{{ntxw;|LYc}@7dH`KO@IPleuVGyyoYGJ%8^_>nKf$bUICMK4RtWIR5R^` zuS#IqacFg%`1z63VeGf+LvQ&i(71pf$iFTxhCT5VmYMKyzTdj-&`ly;X zcZ&avwL)+(CRm4I%F_tlCFV(8ygK&kLt^~dGTrQBE8+5WP+C?R&8#iIo>BleFDo#7U#Jobr5dct?CHG|y+Al8d_K~f z6)1~};jG|D-&7^(f&UUQ`NR?WCpeUY#i!69wuV;W8h=^TzIy!J;lZ803ccz8g>3vX z@EsSe0On`C2#-nvp-2vRm2kUoUz|>^A?ZJk^ijMG|u>o63iD0N4Wou`ZD1 z`vk!X_U9f(L;gNN>d@7!IX8=91!?4L$Fen__KfdcL3?Toe43r*L2oBXJ^^ycz_BLK za$qGF{|H|Bjg(%8tE_MLrVM7tJc9$pV#tuo^|7cmisQHtO zY!g>?$aGph15q6hgy|$4~gM|SVQOq+m!l%4N72;-{_}6sF;=}3+(xtK)-+6 zA+w7lJLBOIrJx@8@!zf@A8BIe!c(maBRg77bRIV~QjB_IWy_+#Ec}en-?rvoy^)+q zZQoz^>Swi+CCW5(sOWf%GCWH-$EYAAznB+Ywj;BAQI-3m^5qOW`x_AY5eZBtVgBdrKX3fHq!Jm9!t^a&a8q$7=jUqlC!@2{;AFcjbOW(B` zg6G`b{>Fbh_Dn-HayWJ9GySV!u%*mI(hfsktYktWuU?s*-v_` z1%u!is%Vof)j+zX+iF{2&BWhfyu6SRWt^%A8M!QFea?)*8<1Rb=>AFw9*jf_OPq?r zvHv}B(Fn^f%LF+jAx^W|4ZFny&;gZt0P=_w$W|*>I-XWZT3T9F9fAsk;E3}Z^Az>E zoXt)JFBR{6%z+T`RI`(Lq}Xf}>qHKEFB=aJgyyp4phAVix_3eya`asT2ZfDpOSX=M z9HiA`0|LuB(P(z6-8eRt?b`u=P*u9X6Vgi!reqgcO<{8^k!RaOhyC)#nE@9ezip@H zG#ieY%Ee5hp#EYjND(D){Ha>5Ch?W@CW*AAxiH@9W*7VK-Pjs%UylEMH<#uBHN1cU z@iz%W=Oj%JjL0{kCFYg$x_NL#9|FYM0(RhdSf|aTfBUK4q~rM&Z#TPdkM?`A({9ZF z_;~=B(fX_?kIy#xR7%y$Ka4E$Hwg6FsBPmrzTT|=$WPBbO?_yoVsb-s&P%q2jNs0a zM!c1mtfPOJdpKC9BKu4BpnYR}%AtlSCfVbzU%^~v@9u6=!AC3_Y}T4coD=330$JRr z-`i{)bKB@4jo>_Pz2Zr7kdr~BzOXF1kIjPnk2^=u$8_+xbkxxTcc6M%e}3&~4bnY$ z7HD?wP(WT9x>!x*U(=DlM34S%u!+pF`zEL)PvMjBsY-P$vzKyR)6=gDLr$*+t>0}` zjnlnR!q`Gy!mTsnaV?PcayYc;mLiRY<~=d@;Qe+h0#6V3Q#A5I{ZMwkESkm3(adKr1r2aN6oV*VPFbZSv`0II}}W&Tk=!w9-UO^J4yXTL+|YFFwq1WnGws z5%5@1mJU}z6h-?w9&LM^8g`GNaE#EurSySXIknpLp+MzZ(hI6 z8xI~VGh4wtvuM5au;M*RL*8c~7?Ya{_qPQP!~zP$IiLf%fmF4z*#>0-*bxiBbO<1p zff@Azme_?8iHx&_-Py`4oS&Fbsygq!C^A6AJ;CQC@7D3NVTiU089|(QjP(6_;lyyi z^R5V|EIJO7xER$9FRtSMW23tGB+#-$b-J82D{7}J2m?3m>U@aRs;t^5R{;gdD(iG~ zZ>kqr&!s^@jRf0>9m&jQnwEi4?zVc$l!kSs_0^5fUJAS;Mv=CQ4dp*r@o6%f&p5pM zl(Qy1I+-dxp>BUf2O$v<#*RvL+ga#tIEBE_(<2#nlcv3eQs}Kd1>rJMp{bL#Q|?*X*y0Q1hqhOrM;~p!B|$&aqPOW2z|zWOW8*v7t&N#a_)c(V%GE1IbNZBJ-R_$GqEwkJ(5 zRVzJw^<%34b7h+{OvzdK<`^J+ z#sEODVBWR-hVQIU-JXbQnzm!J$26+?ayg%)+Yul!)jU;`U+)nNPcSj9p>fvBMj@pj zH>{iNS=q#F|9aTqV@*ZTsWJ)4h_942NX5rVL%^am8@+E~4GOY`%<$Fk4l7d7|Dc~S zopeKoKWm4{W0gHAOd<2Wyx->}g*)}cMlCJM)Rf{#cyD%xAUfS>Po60JDHd?$@H|Z) z5tbx(GU2;OI6hqBd98ara4S`6sE;OIY+duqW^wNeo{oflBzvb-9&K+TxRKfaqXtVU zvnmz8=mRLxji$J$NHL&q{W}r&dCvZ4e!`jc?-V*6x@L;%{`r4wysziYyMno&qe%Ib z5in-@P0l2GXwejHY7h)nQ4ZhOPh(4wV!LswT1OgwNizBO=fAjA1(7Fb_IK6E^8x1& zc+9s>;71E;AVG(b_)F>}2#K;So6>j2HLvB$0%ZIMq+p5Pm;&RqNBAdN%uW}Em)-cR zF_50tcY0NSo3fRM*VYvb%-i8%`w5%p_-bqc2Z)qbp-&oFtav+y>_D#g0;3cOat0*i zwyJ<8+7Ra)e(IN;A@n;uZo(Ty5t!u7svop>J=_XSloMpQ#{fDO0E>TqQ5~aZEL)57 zqyAz*pY@wF6&?hD*s{V>aIB~5)uM#1H8G3vM`|L?Te?dauT(s5Kp{@2Yn{s13A;d` zX8UVm-vz7LaFx4K*NajT*P80$X-e^_FY=lSa;Csx6@#h z5?QDWJXObKRA><~Pzpk+rWekzdI^0GzQV!kaGgl4i+XPX!dHF?+4Oha4j^fUaJ@3A zAP`B+Yb)0kjOA8`Ih*{y=JJ330sdX|X<`iw#rNl&3y!_gs?QU0Hb-(+r!__=PsCya1?Qcas!UmBaGI(M1CWL z?s_0QjtX(h{h|uYGwt|jEuxW5ba-U5zonJiir;g9UYm^lgNiYrdc3u|()VjeTbJKQ zbYxuI68ziYgCupyZnmah&(Lj^&^spI8f7)rFI$*yr0d7jP}!85I||pL@80>1E>lxR zry+UltooKIoj`P7tMbWzn7 z3|nh}CiFC33oE5;tBU^!Mi)eTu9qzPU~GpZ&rePCz(rG)Pb*V(OS*I!>?#DFDPOro zLj+_z0W51xDY?qTqLRr!wzkoe#fSe|LvOp>eyfgp1)^3oMG4aCv@-0BhB95>u)l~0 z=y57V9)uBO{95Go>^bs$ktfqxZ+ug|nC^>dS9s*v|TjiaShVTNkAwo_Nj zZmaZNNl@T4yaZmGzRBMirZh=H)FtA*CrrC*{(~Y4)lbFZ-{G0427m{Qwb%l{FXygQ zC?hVhU9=AxsCSmDFQ9!fnY!r05_t>rO5?RN!;dW|^)zcml0&4Tk76|T-qu8H?p~Cd z=0EhjFeW3Q=!E(Gx)R4gT*i zpG+3LJ9caGh!D0909Y-INj69f?H+s59{@8+fsT{>B0OcDWqg3ecfBBBI-^l|_i#BT z^78J=Y%mCeB-Ipq5}d5=UvQ>tzdui@J}LuQakU?2hAdm7rc{psvGDu%v&uiqUO^TL>@{* z>cqK?OefK!3M9%U78OSfO5{~t7vNvou4Z)D@c>YCJO`9PJg^25iBLOYq%Oby^wya8 zg{`-1!~1G~y3S^ir_^l8`*kif1;(19aO1ThbHKC{G+@1^t+QcFlxzEM(Br7=G%lVuUcgMTI?I!4lsCi zSSx{UjFMGntG$xFS9MOI;1vmBL2GFJEWBZvUejOF?$>%#=+0z+E?=kNa)K^v({kA2 z1ZuaU4{?7FZ1Pg`VvSmKoQ1bI1Qw7E8^!2DB9fMSqbt|#aERJ4*9H6Lfj*4Ot%4Ku zZ`BuVyhv&5^00P*hCq6AtEchSVS|AvW>*kJ+oYAkoibX1pWfwyNi&nRDLeh0E^n;7 zOEm7Ge{ZDF<&x7Kdd0V8cl+G%41UvzFg!`Y3Wns2hZ8pxy2D<}z(50bV{01tF!;;M z$-76rzjq#7Gd%gm&jD~)LKkwFE=R;`9!q1JO55e#1@(YwpPT3I{b$`bS2v$qgeZ@K>MP@$+#~*G8;2x%iTI!}GIxz@M-z2I$Rar_Qu6@3f0J)!mwGl57 ztW*Va;gCqKj91z`1{u`;nI|GzQy&8PZyuR>#@ur(SdzjU7((1vkUof5y)49R5sC7m z8#iMak_%^dSQ~v-fRhM~=#8K9YquSUqq&??`8Ez1fIsO@eDSHF+A-v;|8RD#gt#WO z_dC8!)s7EK+)x&T$L0!um|6i>U{>Yex<3`l1F+_sQ$WAJ+x9|r?Aq0gX+<_7IJp|M zmnin$Y$GbZ2G3{3i;Rum%n=^M%^?oR2rGiskM7Hq^1di@7x3?8axib&Bh`s*VFi~u zE|DfX+BN+{S~*a6swGKg1IuwQbW>KJ>#}!ssaN`c9#;qhry^G%tO8 zId#HCYNO0~vd?U>d436|#!Dk|8N^L0qCdM>rPk_-8wHQVPTzsZ{a~By?0g8^vP13! z(p~K!y6na+hCc~vU#-ds-X0ms31_{&_;QpACF=6{@B-I8ltr)yeJNiy%OBAVnhQ&noT^<8lvJ8`eq zZLmReL`ZS;JmS8%6UwZJB8u;V@aD6}8?UijSSdDW+R~-aYme}~T zlHb|B3^cL7I~QmSXWA0&!@+~EL|=1B@COD@xoO5~L4!j@-rHOYo}L5@|9XxFH4JXk z=G_R8PJ(=r7Yprft9n-ln=7Y3&1R`~T$SkOXY5=H%T666zBeVI#6?<> zBoT1lg@^>f^%|gvqAOZY7RX}^1>u6fBq1DDE1*JN;*#|~7t7VSfI8p*yzY;svhkxv ztO5{*6#0M$;cYS#g~w*>$G?U(4|o-JdlUIn043m0J6aXH=FkzGQWwwU8S5XI01X3M zURdWunSQHHEqo@Im2BCbYo|^1Onk_ryE$04K_A4H!EKeKueYHKcWR^yD#~J6r*ObOc3LKrs|oP`uyg<{6tc`dnIkW3q_ZOW;>tm!ofqJ+n5t#LKL`QLw*YkEy~83JA@N~~$T9}1AGz@Ts-h?UcNJh*<9ZO=IE zCly^`^fv0ur;O#Opo!dG$Z*pi49~H*86Wu%tJX}7bEuJTZKa5uzP#%1=g&c(&>Gq| z;~bHkrjPH|H=-NuUBfurEpYS_<}^e`ZGM)s2QFA9%g!p{floHi@hGo{wZbRL1&631X12=h^LiY}e0;%n;od7s?hY zaVWR5(&N(ryKk5hsohDb834gHHA$wuP6e9(q=f;$34Jh;sp>h9voDQ{%l(EAAuyEx znLHD&V%_+Lxg73D+D8>!n1SS*3my8!I_st*zzgT6e%jDm35~*JO&xM3v{|UbJ|d#0 zQ%EoJXJ0pU`SR5XG*Q`pt&P!5KaDb@5#hzDv>dK};!~v{12SY=CkGySv+2GB+a2Ow zRj~k0_ylgi2No%f1*-*s;fve>12+J2HV=@i$7@W7tQSKY*MQu4x4?gklR$PYyBa|T z;$7-e_hUTZbABXM#4<4Fx8|o|nE3%|G{UP&nK$Kpc zOAMMUf2e59*6rmF-0%DXrXsPf6VoEanLC5Yy#s?601iB>02p9!Qca28uZOc*8!!lB zMTtDSA7xhCbTjY1iWTLjkjC%EmY3ZZ36ob-O$%wWZg014D2RVWo)=PRB#jAJf!-Ts zFos^igl<&Fm)%HSl0IDwqkBG{EnuzuNB4O#9D{71y=%&(lfAOCVR z^arG6)oa*|Yux4gNxQ?pnFk_2?BzwLBfll)ACq?$stR{EJm0jA+*>GW{iU~EC<^ni zPnbNcG&DJ!ER5i>F7mmMba4sc{h@K5>PQdl0lnymYU}X1H#z?j=DlcD7~8PU*Rh*B z#gW%LdEKJeU;t&h-%k7B+slvk`y1~nsjDVZpm_LhZn*N|w?m4aWZ5cm7dyfiVD9}K9;)Z=nc$$OAOBzob)>24p}xVqj^WEjm7!(zvl%U zQ)bQWC8>KwQi{jVcw2QF_d^3Oow-Fk+Dh)Is408vY=eS__JhLrnuC^Z=pN_i#pfd$ z{e^~E^+vqQ=kB+8p0?4PMMk~2+-bef7tGqC&C6c1Qx`S7fK;2fN`V~qVz$OH;4J-FP4J9Nt6PUF+E3$>Beg&@wf-!jYm^$mQk)G&)b zjp4VF^DsLf%tQhp_H3|4{Cp!Y28ivtMz6kv$<(&e$uG)p#k|CX5J_kRTp6NQ;O-@51`eJ^D{U*{eK%pPMuaj_T`W5mc$B z$J!vB1#r(D)i#sLJ+`#3mEFW2_{=3iOZyw@l4OBkndA25cT+`nT)DP*dQP?aWRo6{ zP=-fYf?D?j%yUR{fSinqQAkdcc%~VXwAt)QsxmW{UZJas=hCS@?E=t7HGyJK@YL z+rx=9k|=&au4rGLwNxW`w-XuOd0Gv>!?gGB@6T8xYabO-E=L9STa4KXO%c&HED96H zfei@fCyNX024&i6@;ezC3R!-c?x*t>g{J2V-^2!aT?H>9vR1stnh(eiKFudyG}>i} z-7No->LY7B5Y0M27kp&q$bvSTOdt}dDmplDDbQ;fub=fjkqVWk>iagsdX}=UH2sx#6jIy~T3gw})Lk#5ipp&h+hIRF z8~9<>NtqbYnnXP6ogs)+7Bh~R$ITF9yZt^`xoeMAir{M;(oqRMsO>!+xXod1!C@nD z2G$BV0Xz(y20t|+z6kx~!2A@#ahK(m^|XoQ;B9k|x_QmVNDLdwxNH}aLO4}t01Y>{ zkJW4vW5Ig~N}_`ZL}6EVT@vbxdCc<;oLUzU)Tz)3Gnj@Ix5SRJKN7ekI@0Oezl zFM2fP$-G1={FY_=W(9GPI2i=gc%p!(lj*4A;n)**@B#~{9pnU#n>Hlc^@;2#;$1Mt zZSf!)aD96ZDN#FysYk0&=rtCES$F8}V#mr*cX$Ytse=vb^Ss>6slEa)WuSWij=CR! z2=c*Q$3R4M$1o7d>MAFgyyYSbb&X7J98OVuOL4x6w#@AM z#C}iqE}GSd-U~Rxbl?`?-bBc|xfe9pE_?0}ifqOJlpT}HU{e8O!6esK553n>5i8c_ zS>Tm*+8Ar|nI9lafnB9%3#L1{Cb8-`pGh5t;YWl-!|Ze}xp1kvm?5`waz* zA^h&5nZEpKtBDg7ZDx4g4?9`i(ElFW@YUUpmw#g5%vK%v6AIZcyJ%ewb50NImLf?< zQW*4%D{*)@wbqV}rrC{}f-0S;HFoa-7YFy-6M>U=r?f$r8OS~zP%Sqg#wn%NJLwsr zp*re2NS`u1s1GYk8rA>Y%v0|=ltdXxend+DvxSzIW&$3Wd^caM1FuaDk1fXZq$6=(d)bf_dnH|M9mXE9{9L+?%MU|%0;eCO(|1PlDFa8ZFhNfh(++lIWr^#p~{8HXWQB#%U z!as}^;ehcRT%aQjX9cBV`r&c=`5Oa*z;gbq|Jn(Ze;sH=0Bv5Tvit9`#7-koG$aSKxU`Bs!bKax2LhU{Z!ErBgy|Qfre}s z>gXBh=Dw&=Jm+yRzGE<~!hTEiJthA-I}8b9es_PHt^lv2mFU+cJ6DA7b2MPW zxv-*Rzp5s!SQEWhyO{mg!Sff&(*qz{TMLoZ%NpskeYt}FjBXjm110Q4xf5>E` zj>E{VIAaE?w0!=}F2$b&7zH!FfV61qI8C-q%FVAQz!y(GZVGvjcTB;VHj;$k}A=HKsQDkiN8pAT`za#_@A5)R_!9nX@v18^;i1je;L?D z7(N`NI;Z$3DWaQiaXH8Vt$`2=I85HoHE zAPS;<`uHY)rP+yIZ z3YP>tsobBt1z2%s-0xZoq|h97l(5&Y7}f#nn06z}S9doqPdn!l=Hj|ZAZ23rIFJLJ z1?${xEf>#LvxN^QC;2q8sC!8|3>JVX@GQGF0*tV$@!{RaSC@pJid2!; zwkl^gz5Il?GUojXWxX$UaH^xP;P6*95kRK^@F6cu^^AKge4-`HT`(K2$3Ia9&yffF zss}DCVILW7Ykzv&NnA-OwpWoR(H~P39VC2di2fVgWYqmPOt4;BmB|>y+Ra$i?tSMt zK5S8#LAwt|Na!wn>Hlb2Qv_dR2BfZxj-5vTnb-Ok!OE-IPKBRuH;9X*3@~8ShB}g( zcCEm;7;671F{^vs@SHmyP~&5rF<@}TMx~RB34>J8@w0EQ$pPx)b2V{$%PVX-G7{5c zKMr8|8o$x*`iOigk=LJ&)>VKh*x{GYlc)F!`DwJdJ9dX)nVTEuCwmgq%!4 zLafSRzb5K=K0rs0FKYao8Lw#SM@8qu2hldaN9^{{cZPr6S53z|DEuN;Zb{~korTp$ z3`8DJ1C2rE0?$e|=v}RYjjO(;>YP3eVOlb&0UK3>XsXh>A{}gUlokqU*_@;@Utmcqkt;KQ0W33 zp}NQ`{f3hS)wa`!CZsnHT2jvxruBlu#U;7|{8*qEhwT($q@TMu9~7{ckk^)bBM)YR zm)3YLM#NUWgg)t!{xiE_^Cx#dtOhab;J4$y)1vu<@h$l+StKRLH-2&hS1o+_V)5Z* z{3jj3nszLqciAcm1KF8EM>dbqbz9`Ez)^Hsi43|wSrLL+H|_J8Z4X-#bC;yf_NKSL z9Jg`)TJfq>4M2Zf6eW>j)Ij1x@It8^DZm=qD$GLKEh%N#4aA~uVI|3^Q`}r~s@*Lv zJojklygCuN*wHvk^?h7j`E5(U@$+86$gu@Vmt&P9TE!FX_RZZ-zW7oo9S>1Kxq0MB)x>@wiGKxmys}#);*-+I&wzj z*&Tu!bx&Zz=1@V%UjC)aZ5-E|t_d)0>YHD4;I>6W|D_5T?M+ZVt-h#D0KP%{DVB?7 z*ivp<*K1$vd;#$?7jT!X!4!!*N6KK;F=3M#xV55xhpb5tUsm8QfQ?|CUmQ0~WumCO z=4HO1+I~idnwZqRB=nMt_>Lio;Qky5J3$&81;)QLe5wV|k z^5kZ<>uq@Z7cF>uXm3TTl3Q@hdR#{C+g36@i7IjeLrg9%CBlGEXvs-9YsXu#uOT*% zkqJD+4M1ijq-vLny}QQ*yB8Qi85tQT&7<$U2NURxh<;Weea@t25JiNLD-58qwozV_ zCk|1;GFXbKz3mJ`VXLVID7g5o5p2)0(Sx)@NNRa5j?D4PIc``6Erx`6a@MX z|C=V1=#3783W>{`Ul>7ppvbqvIyhA#CEFVGP%^UKI@GsV5bWLry;GKjJh-1!@^JQ_ zv*b_$+tZ6wurART1uy@8?h2N^3rBnp`bN{~G>vya4nnJ>2KbKi#l2?0@#|@|<8hlU z`-|o0SA;#e=gc4sckL*U`x3Is!zC_ zBp<=)mE{3$SQZx)z3r$lGAo0lTVzp9Yt@+U;se@wp2>g9-hf-cs6j=6??2)DU8+TZ z1z9i9+k@0B^IDPN9Qy;d+QmvDNq?7BmFYp&B0y63(Kp9vaqzeH4NPAk;HE8N`8vC7 z#)f{U370-?wlD+X?`WlnIQCzolGXA?bOx0uHc8=o7L$XulcH|#SK3HIl8*so_QKTG)F?S>+z5s8M`sQvUI;OYV>*)$lt}3?0!+383l;LSf zUvFoSs(O0tGQ0JcbL=;=dt;lR1c8C>zgyE48(u8C=l+cyai3!=@YPY84S@3ug2Xnc zm7=$tir*GPcx=n0018e2VSoKa2KGtwM&Hi4+mh$yaqE99?z1L!E=Q?72Tqdc_erz; za$y2VKr&01ym5>4`+FCs1mg_p?409e&RZ`77I;Tf^yH3x3=(nYC5Obz?X-bKwj8@a zuj5zUY+casxs=_(Wmb9!@t3YN(Fo?d2le*vLWx42SwLiJ4>1s~mhr1maE^Knyq;kf z)}}#6M2X)^qBxxFuMSekP#i+XQ)-Fd;zydE-xthfD5Uk;bB#YblBZi2k5%pO2+vPH z7zpBM;*Nky=1aFg`VqB2+(DwbV81q+{uV2yB~?p^dS)vfw>7p=|EZ|vOBfK52^p9c zsmN1I6tn(Rolu6({&t|zIp^A+dI_1&e0e%RQVDVRD-VPh|8KUlq^h9=4cby~m{>U!)?5dj!Z&)~dJPt`g2^eOlJ(gQTLl_J@rITms|$F+OX|^ zJ_LwrhTj-(V#}YJ$1VN@I=3W^S=?V>O(-e3M;4s zxVYsj6>dF%^1Yk?5VR%nxvU#04q$Y!4|U!A4^?j&6;-(Q55v$P-Q6V((w$P$9m7Zn zO83wujdV(bfOJZCmvl;32&KDh4Lv#aH^*{&$(`mHp zqqQ%;EgDUi654;kGY7blKt8vmgGI8Ig6)*fkR}RI&oF%94N4KEj{9BG$@aLKoE-no z1&;89%Vj|FVs*aR2fzCzeAdY1^U-fyts@ItWE4iUG=%mKf-M;Z2ji?3wF8)2`XKA9=iCR zVrUn$k=S(#;jL-3gQY0hEIN#v?w1CniuOPD!vb!5nfk3mV(s>V#aB4~TI1r`J#mCl z1Bbi6d$A3@H(Si!E}a6UPC4~n>+3;nqZsL*7ru>)p?w!~Mn`ADVN@QZeJTEqwK0x3 zYPWBe^9dzX%YE0dBq;~==eV{|Yph3^!` zY(VGYR+hJ?cD7x1*ZR*)kN;n(=FJE)Fb}7%n@qAM`Ib+y&!aL0hsRY9rtH&7u0P;& zG+ipeDW2!I-Yrn#_+2O)iTaCg&D)v%xWn(*hfu!$eb>y>AM4+-j5-E?1fR_wsu zD(szI-<;Lo<{6&Q6b9h2>5gU;!kYKVpbeL~y^n12K5dtU$T&zT3Pdg4L?TYktujs} zZ(-(lcxFoU5n}wf*eid;t@ddE`%=l-V@xw6RF~z9e0(yFjZB+&`8Yvi$$G+f9Jgcd40qjo+53Yy3*^rY`x4JG}WtXo%m*?t6% zNPM$oz8qoiJJ)~KRGLoD%%vwGc(0i#I7OG~e z6lJ=R{3n2S14abipHh!@3wxO#CSC~#Q$Vk#XdX84M?AoR8Yh|Si@To~S%6s}Oco!J zaDmM7rp9vELF3?5bs?Z>PZ)52@7INkYr{feU*fYC-3;CVp~M&;7BUU$ZHl>EPJYdU zzy3!2r&uL-{;ZD}wY^Z4pP~u~TN$C(z-QBpxLHY-nfQ+_PKmkj%&R{0tA#^@yCT;! znsG(KF-r~hl~M6I-|Zi+_VZb5r>llwZGt9^aB7?6T+6`#PW>l0qL)US?k~`nZGo2a zHdQfAafea_cPj~Sf9PijmoI4lcCNJ?kA7?pKBVL}M|M3~*7xEc`u%%0>lMiAr&)2& zZq+yB?e8KYI}IVKQ@B>DKC0hFqBZ%jiY_6k{){&6<^l7Ulq>{&2WgF7jtL5}Q=R6+dcq;ToQ^O)P}M<&8;mJGbGk1x?jp0W?}Vl#+eRAmitB1iHVnst8w5*J4xM7wx$I`bVIh zqHIK&w5mSI_lIFtB>V8k?lhVg_UzPi^Ma0SKk$dwTR-|y)Qn}p9>t|8W>v|6&`-aI zH@v8n5O_)_jY2-2wHfv~=W%?z^!)ByHe zTf?1qXa@eO^e7jm4vRBL1wT?<+A#@nQ=Q#W?KjW@ZLcF$Q6!9$wyzV?2wA}1A)^FK zT70>c-g~Yf=W22kyNFMIf#jzrUVpfB^P}v$mcq`BqzT1PP$a%N7|fVdo^>Z8!mO8> z3B|fsS`})nJ^J}A+_v4Sxacla>K_Zs?f=FjHd5`EM2cJ-rg))*vpHVl;6!!MzfVZ` zvVaL>A6Drgz5bB`f3jY{mq9py_eZ9T$t2*cN`j!Js&p0-aHML_60+Fjz3tb4vHzXA z;9^|thtyk|5UUz@E&Zr|Ojw zrJX1+Ra#oTgka>+2I)`?d;?h$|Is&{rKrW;+i~12ty+_S{eZd{g}QJHo6_&BZ710< zks(@O6WPK*02bmIo`#Obs{`9vg>YuagZNX?^xf`{r}z_K>DCQQi}I#gRIJAk^WDpI z{|nfCO1NTLfxZeyehZ)?2iFz`FHJs2yiDd+w(!*51jO1B}>A z><&U%%&H>iUl&+8>x!gc1QPsvhyCSi0T8*pxwdk`GqW z&a>e@C&0whVb!&ly(}vjdM*i2ig`m5NnX(4+^dVo%u4+B^l8orLLi@?s95)S z0TR$7`>Km2=}YcE0euMS_|;{8K%GOP2Viybp8!g%@wF*P)2ecrDbo0C4Mh5;^K8bm zMEgD{LdKo}fN<-JnaGPe7aUN-bQ;YmG@dPPl0Mx{F7pFka*?PqZc)?`&|-FSVQO6pY3}1DSShV(T4@&X)deeH*Jrqi*zW$`PQSOGJxD5a zP5RItQy)YAfjSgun$fX}g+0vHBTVk?Ni=rjyHKh(GX8)GA4E+evM_%5h4(tn5^^CU zf7Z+@nPAVf8gk(Jv-v2cd5S;kQawM)@70pq&V4xXNC<8j)_J||^X~2O2YzkAv0Xjg z{%=c|0E{FG1DP>)w;(2XJ#N6_NtCni4Oyl1{aNUBQv72phGWAXPWW&Ak~i5tk61c^5kXRcg%Zar0f%}2+STF&5IEwo zmsFYqQunFK6FB1JU^6z52dwN!?FpI3x$MR@cdLW>_$&wKLUy7I%x`lXx10d@Qu&XU zz1c)dl9+-Al;fRW$_+P)B7x|W0|l-U-qg+Ub((EMSVaqaN+{&j!nE#3Hw=ib-Dz!J%{}qDaof2is&^aBTehM-;%)2&;(Tt@h$qQC zGy}vn?rFzGY4^y_I!V45;(Bf1m!E0;ke}K-zr?apAxJPLh3_!7j;c}3t%K?30nRU- zFRs@VA?F9G5z*&Lzt@+46A;$M2MBv|%Np=>xSrL$jGmy(kFcC8cj{0f~e*{2hqN1>loNl6e(n{D6cyo0pH>T)QrT z9!CJbOxQ&qr>{+&q)nYlI}#Fu3GldQAHQ{!>Q+`bQXCEeVK1_0whPhtFdW??N@YVp z=gj;9RZo4{Q*b3cF~40#e-2B5{v<4Nur>XVKP((FRjvyG=hP;6egsIH8b$klF<|Ed zo)sh#rSJL_CKG`w@MccIKW7uW%GHcrSD_gu8b*xQltA~Fc$puNlhXIdFD#r+DB%@c zOMh)I4Ze=^FS3Fsf&yCviLMuCc1lu`>|&?4_o79L?EW?CzL*q+FsW!pU4cF0SQj@d z?0#4KD*VOpO^RBV=ubp35S~RKq^soJ zM~e#BzXtIDC9Ho`ny6bT-9i&^@^A$tKFRHCs)_;uZa>xYZD3D7zFBv-$*E#t%yhvi zUX5Y+@Li_lR;={8xF4^x<)8iBib36s%uJ8BD4$xVzi|Pg`!B}C-xd>ot~|Fq$WwDM z@$r>`cEUCqbT8$gbY-twEQLj4L+(a^;JZpi&-g)?OL{dn1aUjd7sKNMEq*T9xHzv^ z6Am~QZb$ehe+UJzCrCFng~Mw9h^+ho+sY1S-)!{O@EB%c{jl;|0HUr@+z?e(^EFDX zYPK?opvD~o-iE9Lxsgj8SEC?Ji`PG~iN7r78=_cAav`E|=G$f5^$*_9Kt&TCzGDe&Uz&ivJxQt`YQ{ShP1zKzVck8sfXe4}X4 z_NS@ZsS_6LdAlOT5X?o9AMWx*;RU@nR3@mz+~v0RruJLj1q+GZ~4-?z9y;NZw(q9PN5eX9H%r8R@$S4v#lmj(7*VH4S z&y8ZW)pJ|OAqX}cp^G1R?uA`fd@Da(n7NhzUlzc&D^D%ZCx<`!}2#)JGmziE3%mQ zg+Gm?=<{mRn)olKD={qpyY|51I;+f|Y)lJ16H?J>TiLLb{c_952fZ0E%tj@$nLYUz zvBFV(+{eBG-TKx_D@avXoCsNx)^mUX(ZT-3e`7a<10zQdq__B15k(X8FnE8t=ji-h z5<3X5oAG! z5iX8M;xk&c!a&>V4U_c?F@wp;kKl2v@aqL}$$?9U&;XoOD4&BW@SHdlXV ziZv6(R`8%FqgkTDyPtrPD6gwdILxpe_SQ2g#FXub&uS&R>Cjyg7(ygW3LE5?dhYos zGQoIpH`!@&pjW-&ACc`%C7~t#OirSKU3nl%A#s?xE#_HHC4sE?2nW1_&7_bkM z-d#St??4+$Gqz==xO=|EqW;n88Mb~Sd(ySG>@qHP3y1Qx4JqZ0&iN*1=cJn zF?nF*?2{|n+FlxVZdcR$4u$&tqkzrkO?{+o=EEAvGVMRKln#>>Og5uORZv%j0gHtd zzx?_0Bu!8U_0S<$Wgh=?3|#Ky(>-kS76#?@sfo$}ndWd=i7+(tlQ6CZ-lcoG>5H++ zsCCj3Z`H{VW6C*05GuIi+PSWE2Gn(zfV1LG>Hh;^S1g91l&~feeruH?Iw`q zfoW0%`lPB->Dyg|PU`7lOq;;ZX2}z$%FeU9Yu#xGto!sq^xz*@UMv2UR6 z38&3Av+Wdj);nsYOt-lTP zSyhK`CtF-2&F|5or`UV9E>%oI$2qLyI6p+=C$!Xx|MbXN!CZpCdESnmNEc8S_o?V1 zv_}M^C!T|BwF|Y~%s$>ADD#z;bqWS-^=n+8OoRk%+;DVD-J6#W%3%srSiaT@1YDqg`xc?Bc=D^(3@aT$zamf zl}0-INhjFKj6j818u)~xM%wv-r7R)tItL76N(a$17vm%`hyw~o4Mq!5IXFJOyhnk_ zLiOhZ0E<<}io3>APE`Ld4R$MuInMrpur~Q&`GZVsZTP`57#`SLA^Nn0@Z~Dnd6)Q4hw*e z$re3pzK?976o)i7aHMcBFiHhvE)$aPT>h$6qd;|GCXRr{K&)x);R2({ z5%LlWiH85OJtL1wCZ=qvnly`+!}Gh)#vvv!^au!$Vj=YgJ4pdAzh%64?aNH zMX_qO7#TO`Ll4I8TOI~${4G)c@YnHG4}U@RI>)CE9lq~R+h{=`kg0UFzq}?Z9;_rh zUO{-6R$QKjHeOsFm7;nCYFOr5)WB}b0@@D;l+?eAyp*e3 zpa|pCuc7&PWwYAa#1xiP0&=?S%i}Syw_B_!Sc_~%or8@*W3PyJ63AL7%j8E&R1ETE zQrr7|xRpjMV>B_~+$t7*&Xrk7V|b(lwcTXdt0+eZNjm3+F z!(hi!J9zAxGpvfz;^sTVQZp03k^Ql^gkXj`N+h*g#_Vv?Zn%DN6(0LKe;O}`GNEC= z8`!XZaa2r|b0Gtl(gvmACtS+B1=;WM*wLV>$wIFFiqvDJQvULAruA~8w{J_x7OIgl z%CKxc(fKJDy6m{GX#&3r?dfv6k2kS;=Uj*oA5XJIF|6BcCF-@vU6;p}?R0nMUC*SR zmEetmrwAVggEKfj(8pyJH&-EW19`${$vE4JIaou7knmPms|qD}%=%gCb{GWP-jt8vKlgUpIPkxmJJ@L@Jktj<2CJ4>30 zrX6{PebE^pILcl7azOAF@ym#U0#}tgFCQO4Nz1DlA7kpQtMF2JFhM}Eh4Oir|0;5< zKmXu+y6QD5hWhft&&*72${rScRX$zxm#$TzP5Udt5*R>BOWPYKidQ7|Z*Yvdy8gHg( zXtl#%Bk^9<+^gza4TVq)~q zFn#pFZZFvh)+wL3NG|F*E+gf$`MZBSZ4Wl|#-_=q7s#S#trH3DW)ZaZV;V~n_2O&E ztw6L&(;3c%SO7T`HmS)JMFmq<8w0qdP&@i4j!kvFVO%#e;|V8L16ApfCXGB+ygW8= ztKKzS;=?t-g_#YW7pB{3&`(H>Bm;W+4b(4H*}1{0%YXETfE$xCuuw)!UKDafTQ1%Q zVTwUm`Yh#9(?u=^Y%|p-A~wwm&cQ}Rma=RXAssE(?wIbBtWBb#?)XQDW^P%@4nMGHnM$t-^sDXGdGaCVS!}0xLiqjUB zfdYE)mN6w@pr(nKn=u&cm!)HJki%v)fYo(+xl$?+fkPQM$7Ey1B;|{;J?%635j=2` z>Anh!S9kuv3pP{xi*I)eq`)e&3w3&C3OdW-nQi{ksAwZ1ti%Pn=Wud@jq9$Mc7$9H z=%TGPK4$rfd=#348>eo)*%X1Pgky?cYx8QbB+&J{yb$tvs7sw+ZgPelFv2^}g_+gm zq9o8NQXy9;eG%x#cVSxAHb_+G^lj|eg02EJf#*Ujmc@AEdq93hq)35G;DTKSBa?9@ zbY69v)BDb07UWsR0(N$c9$50oqcZ$uhK~B(zU8<+P@&S-1St5fwV9BbLwzf^`pY_n z=pRAe_?^$7T{%l6&FeXx^CM`Th>E{b4nttCyQ#GjEdct)Q5U{7nZfCXv2O*%VJtAu zsNkO9WX9hIraa}64ApDrLGZ*g_WRps5sTwLzc!^PSBVbvcY%4)D|iW5=|qC){7R4a z1nf3anXW_UeVkI*}b!6 z=ZmexhYZvK@4Za;8v`5sVX_@)0Sqd2JqfCxc+3Dm?8 z37irQb{?D_P7W1j!@M~}!0EwDb#{XF{`zW--d~vwdVBoP72cva0u9lc++65h2)J9C zVX=E~Sg88k(pK?(DM?)3bHFqCboGf~0Ph4NK8q6R98 zwGzqsLw%e3aWjNf8_}pp#s6|a9ZNRXq#VJ`VtY64><&iW+mndj=xt~FIZNZaP2xrG z%EBHtZ~Vp_dgFn$DV^sCpGO`d&FP9yd-wFtf8ZI+!AJW(*A2yTw({o5CLd<4gNkhg zIrVJw=+WxF2V@)>wj(|MNt!ZT=Njo{GLW+-%Br}lU+c<|YIOHFfFCp-tIW1+mla=w zFXY}q^$?Ds3`Qe2#J}jFEPOkg`*(n}^~vWP%gK5Dsf}Nu{``xWO)<2Z!w=rfbV{4U zuVX#ge}5$vp%QoCPM`CHfMKlPm-=P43{DCoeZ5-j_-T)GyW$nk$mUQgi-+6yGn%e{ zu?#a9ufp#I{AuGmN|0z~fg_K6`eSYpU@*j_5YclY^<3^IH97cVi;W?Rs}cAIXtXND z(Habpw$1=dMVAqQfwU&9u6YtLhg-}wj+REPP)CWw8YW1OINq@-3Eo`}y<8Jpzxunc zh6+mh=B|5`WvY1P>yhrNq21Wah$Q9F_%6)M%oCL+;_CidZJw^`=FRqNV(L~GKtxHz zW?A6*%GxA1WBsJABp<4`+RvU^)JVgud{k}J8H_lx=SXwc$%b~=f;m_Oj}!)D34eSk`j`P)Zp(n7?=JD$5%bO*05UrM9MB_`XtSt)4Y42-_sy{F z@yOhUs&m5CMX@lNO9t>JN80vX!jwOo3ZD~gQX07KqC{4cEr*cER=((=uU)2t*twJ43T_gG~joRhH^)WjD}K*UvE##09bd@1pqT8W6x|5&X6BnmcV8V+fTBHT}Khk zEeB<(kwYeiazC*T%8m?!+bTUqU$%%=Vmk&`tHYGS>6kumY z30=AFF$*PdEg&-p@4r+kIaTPpy}oO_l%RHaeR*CHrx%|rVT$;Az4>Do)vB6^@EUs2 zEf$k3R!yl@VMKiPCv1zq9_g`l+50cx7e+F=%I)9yMFebiq1{k2!ZVU!g6?viQUStF zYeKNeT$w&?L~y}wn7~@!eb!zE2Q#w*S>?=VkovpmH&7j~xDlWoh?ZUFr+#gr07Vv% z-)T#ES0rFB8QGuZOeoU98C@hj(hem@Pm-o)qA*hpo$m&OMI+OO7yJgt{noJd-ra`? zs2(fxXd>f|*vJuQfB=F4^-`2)_q@w2dLEM(17SsR^B$$qjY}EpRO+1`K12H_YSpTdjB=8z2PL2G>IX zxmWcYUxQ``?WH94^m*&~{hJbJDBI69xKodA=Mj)FRsTDc82@T3M~TRmLXCnht>U-q z(YkjclgBlSG*_d<34NA^b{!Jpn*AM!=eOeP?tCfz;^SGN70Jr>?d{Zf9;aqQoELAR zB<3ru*uq~kd^=%yS}4sQ6$ZkD-Bq2j*faL(`7HjBRsCv>BvpZZpyM=^$tq(?TDhsQ zeD7%Uk;2O!Z)ePoaG|!>i0^j0BhR9N&8Fcily5Ck#kyDBH>}v>BLk;>R*-1RsM&b? zQ|VxdwvHQz^$kb0GN8+Gt+srVvUrLA(!`M<@+Bw&kE<%U{`^P`+~J)0{FphB(*@K| z8m0G~b))gs|D*?}k1`R`r~#UeYC#)D=EChLnA`Or0yoUW&edNNM(_6>Y*&xWh28%w zwtOi_)$PuPcK5D|8ZPqqn3hHp_0fPf7$rcfpaCj95SW!F!S-WvPZ`yqpb0{2DxrbN z;CC)&CNHC2(K9Bx4gvLw;pM7j^z5D;u2IDg{tpg3xRga>zLaA;lfM#K+ATQN;T*GY zFOFhYFC`sLULy$$VmqkInUGV`8V%)UqEM7)t+`kR?ImuW8ams%mK4o~>lzl}dc9D0 z*skH&j=-imJTuLS^MwA#yWQP&5L9S2IE67faG7_cNxN=k9$SM!c`M|R#9t?ugPUQD z7Rl#B3DH_7D-WK26`3;rTbBN$2&`fd7*9H{ad0}_Y&TL$)StWkKCZM zJLM<~1A(4sw^Ulm*D6>?Db!W6&?=LE21C}3Y494)JJ$+?;L0Q;;RsTVDyna1vXM9} ze?d)jBz3_}xT{TYUSs`G{_GV!OyL@4t-5{8252Kzm$ogI$PYG1q ztFs>KbBwPo=o|&@jbEBHH%;e22xV&wvQti;X1y=^Ie0%$Uyq?@J+=C1MCYRa2dZrwd~>K7q)jLpG$=?3>uz*`B4!uvHYT#y)dpGRouS;LK1{!X766t zKe-*HJUkpfmt*N=Vn}SM&zooh3Wu^}Lyj8~{R#KJGBwJLJQ_3t`O*hDEB)3ySwR|e_UB~l`tz=P8!x>G8dkR7Ek7=itNykG za6MT^=M?r;(@J@~Gt+dy1i&^HAQC+F1q$u&>f>x&-w7scC1iP|S?}|9Y>%0DC}`i* zN8;^Jzl+K<_=OQ%#pJEK<)BW&UZu9FU!ujVZWI~$HslTQW)klhJ)?@2Z=o~@%;B;v z3GYo4+xCUn=u@(TXy(oXY0gi@+|tXn1c%SVM1xE9Lp-D%zWEkYn>N*Lf2WhOIxxpLad{WE%NsvwUWSwouhIL~u5QW`l^ySG#BN zfL|xO#Fcv^gBcJ_etfoH^c2jc$X2dD-APb2fFOH!)$uBrYcZmHV-qCP3MHfP90UO+ zIG6>B`|jt6V;{~^bNyA&=4%Slh?C@_(lYVuAaWi*X{$abzyU&H%^}~c=20oUQd>iw zrE@;)%}vuSFe6r3C`MB%EcZL1!jlKb(>A&OIYb48UOudo#AM{J5&piGQn@r(c}_Ul z-a2(*C9aMr#-5VHqwZa?u|Gf0)z2MM?+2s!?h(&7lmBS1{i>h*bJLw-PN$su+O}TQ z%L!E+Gx8}2t?p+!w+BHLiAR9-Yh-eEB&z|!?(l4<=X5!OhRNgi$KZya zw+4TI+1DGMz$n$GS9&mcJ5#JyVo5))H)3R3@O-m=&9P~$3}E=gD|Gy<>b;p)_y*6i zh>NSSW7C+|;x89Qgy1Ap04;d88MPoX;H5Ri^vUrQ?G99&ER$zhw-zWO&P zq3nCWd77`fX>71Fa#ZJVZx(0BCsDO|X<0rC+I6x^-8&to!uGZ9a#8b*ga4>?&E$9ixW=eK3jJ=|_ zuj?$k5ek+P!<)DutI;LNU;aGVT_WE7I5VTQ=nNlxoGCmqUa;bde9MUX?0J2F@4!3p z=Luj6_Y)w z5raiIP~0>&@(*m$S_Uja%VN=cN#bVd9LIT$!0hMrp&pR>dD+@f;7Y-vxnC2@XDvo2 z#*4>c-fC6yT&*JImV9uBK)cIo+>3uoqLc!ANIr~+hPwNHZ5A8)Xd8M&yYKrv$^{(0 z>E=l|YRe5VT!#BkVQxGnB~b0BoskIl#eRaLw1^AQfMi!xcW}93dMf=dYn{b{10Kuh zpyRQfcvKp38VPuFW^RaxKn%{;Zud@sfh`G?p@EPr^Y49xK)x^sl;Wrdwaln z-{{jUU306?ZyNu28WZ;T9_}7Vz+2V#7KLv0+tEqs$e(QS(?GkG8XCpcNUHgA=g#L_ z%j&+P1DWFHuatR;+T~x&PZu^*yCi>oRJ%l4T?f;8qy^t}7|HPayU-aAC2EiD z5lEG>K7yU?t}Q;c^U%@-c_TN+h~=G$)rFE~mr+qxN#j`0uuid-#kMqE9g%5?kv4a5 zwT(yBJ3Dq_f-pvk+Zld>2pT7|@J_HDsm3(b`Tj?>tLf8DqgoZJ*@(e9tB;A6_+us; zr8vl02rUvLz$}BqM$BKwxkQ(VjR3t_mLg?~L3nGRX39cX5h5HBrEV0EeCX}S1d}P^ zQNwhl(PCR|^fE{)YxXaPQI^liai@EthQffZk8Q^B(cXj{^+4=?8+Ue~9B8(TkX5FeRpKZ2T2eZzR22q`4i?U8HbO0 zJhA!C?(wLy#~D%9?D6IpZ9jSAWlr})bG6w(O=vWWmi;+f3Zubi8S3eOf+B+%rj%^) zk3dd@FfrSj{Q6Fe>m9XCVvef;X?kg-BAv@lqh{5p9Y%bOlU^#*(gL`%q$nI(H z!Cfl(ja$CGNd`AScIkH;0W1k@@45wLz~HPuWw3n(zz$}H+1+ygD+eUo2@rL)Oo%$h zR`9aD#^Mffhz2e<1AuTyg)Bxfk!eHK$&MsLb3>{Dum>w&dxxbE8`#JlZbCCJTt6bB z&Yrh#0cIs_UiR|?f3%6Jr%E9IvIL9lc{qjRGzs3FIc`a`n+4Vb3zlY`5LQ!Re@~}k zx<~>NSu!hRR! zn#E|>F5=p!l32{}_UE~-EVTT%)WNb2%n#@SU%r$Cuz{+>oh^COju$op4efthoQhjX zrjmg1LXp2F!2HILl3&xPgzdeBax>BA&1&GS06l0JHnze6xncIn_A-tKEtHZZxTkq% zIxL~x7Ii>)evg~H#r;+gc;p*@_*@t|vvO>2mU0gA%yJHPue%S~)WV#Ttgatiy`cfP zJkVd2i7o@Suk9_lf$@h-zV~H8P4z<6WvQ;-AU*pIB|P3DuE_2=Qae_yylN;4r*@_G zpz2c>7=Ec^W&N0A!|W{>XNrI|^s8@Br(Q*%Oq4Zv%rDyL5xd*o@t886mF?sO352tz z@aZ9&u^lVybQwRQ;vNGhG`g9_slP=iX)IO;9tTMbsZVST9wi3z?65C*IM0Rr8>45R zA`*@F4lhZAL^95F4iv$J6=@S{i=stj)L>`4(BfXn)F_Rn^Q9qGCJq}l2taT_s$#?$ z)hmQx5wWZR;y!#UW+>nH4g{mY4M;w!(_M`_3P^^YLpSGETDJzQDR1_G_>1m;QvEE-g|(jjD%JeHWjG{^ zFjFmF0`8UTa-iFs=}Hb3+o0zPkNZN#mFeqzP|rCcAxTcKv`FC>nQx8P!l>?!%dLRw z?|R}*pHPm4V%^=h0Tedt$1iaWa^FZciTr)qfxEq~Y(fNrGtGKqG)URVJRvr=Gu*Y0 zwC~jkV?d|wU^!PBsc)+&k+71C>`~2zW5@F|WK!qt8svV17wJ*t;{C{)9c-DAhXCb2 zTi`k%@1#5%``mPe&~q?bIzS<2hs7pf0_Y(Sg3LR%yb;FL>-szFR)i`YQyA@(RaFVu zR!2UaV27cpIz)9h!HpHn=tcD(vKTP*KTua%xh;qpEeD~&ojn9^{fw6`U1@}D){cu^ z8a`EO$tAtj20EGgOvU>yr*zQ?rqXi2uHcE0+5P z-!ESJ%$AL+2pMAh9H9iZ;MmXcPY={*?B5Tq8c&bJ}d%#8y z8`dC+dOa+bAyo$DJO74d(~xhZ2@5>;3r%DE_I9SHuJH@bm+R*Z5qy%BL%O+^J~=tCE>eHXwwG(JaVuYkM#D z#QWt*(?8w5zVMX6eV>|Yf~xp^B5GrZ%3H2KRjJkK2>i@Eg80zSW1mFQK}m34-ccEceT+mpD9t;_o(Xjr9eo@Z-hg_iqk<5o6-xQS*CdeEPmBVy(yhz1k4*;onqf;J{|2@iE?Mm zwI&f|iu-}_7|kfVU%q8SmnS`0qT?2_5Yb7(3_afma*~cq)+wLtt6hbgE(c;Xyc^)c z9sev+xVK>F20)388z#W52{zbja{Y`XY>ylD%F(8`1L#eO(CJz+L-gG=*%?j>;tR>O zFfuB0*c~Mn02nH!?mlZRfcJ)xjceCKm#b!oK2OEm4GnY}MY`;1o%qw*fe~D^z=0}+ z_h$1Rd&t>+`>PMdBW(2E9q9mL9Y)Lmx$MEh4MczuSM+1kVAVP_At{MInnz%6{XP_p z!eI<&&@@ZpAV%3i5__!R40AxX3=O$Iejr$$DoHKgFDBpz6tGahM-nx?IpNTk+V>)DoJrAaw zvlF=%hyd*QIxfH*Y>bkBgSlucZ)WT`!=ZCI=~o>^=@JLGf-o;qlSXVl2#MU}ioeKzI)x+> z3R`PXS>W)%-<-JC?B%BftjNg)TTgTjW-ZcJubA%}sXI#$vN=htwR?0@U@{3U3phS| zlJfXt8{VcwK$t+-6DY7aX9wF7YP@&w7q1WG+|6EwMUqPGpPj|l8Gm5We|eKw zGaUsJ5g&oi!oa@Dl*$~zrEWtww3!=toO=qacv!IID0ejLv&}Zc974!BoRoyO&}M+s zti?au5^3A#$EE`P=dWn6g5jsKPf$F>Z7;v16?@Y&)gM@M68ZR@r;fPYGz*yK=w|(m z>vZE%`qZNRdEaTiGGN3kRLSx)XBGK$JE9Gype@S6F{oA|8&0HWpi%#KIPSR(m)e<( z^q+e9^kpX_1bYVn_JjZS&WmwMXVlztAT2S>h_qV z6Op`)%QVZ2R9VILoz@x)bQ{J-0f|WK?JyW@HC33(U)rU+-9rg4OFUi&%sfjU*c){; zdAN;~m%smZqu-2yJ1{KIPPO?>y!6Z&{k*uKVi?Ekn<=jgj+q#hwtkrjg!A%01{DIxi?)t&^>A8Zpl8=#=Dw_mUS2<)fPdo z?N9$3<+w22G=e@VP%b^zl6!Bi1^H|N3I@KC&=cLd&`XTDB5p06ax`%Cz-B8^r@~~R z)&bN`(5)6=NCAS4eQlMwxS?DLZfkuS3i|J#10RQ}fa;eF6!_k_zt;FY z2AikG7afmzr9tZR%M$2oT1&2Y7S|PyB_8ZxHo}oLHi?6;GvyvfhD~8kjM(zSh&zRJB7cAuVam|e9gjGbr-o8?Ez#&t6rN0 z*Jg!ez^zRU%oGs$eyRRfPl@*AFNh2xCIWUq7&U;VhY4yuRQ&CO6n?BZ(W zHccRmLGa@dkFWUqxIcI2$zfJXRV@1TWpVmziQ;?eGV=ePh&7a}Q~Qt^t+nYU&Gq_i zvwryT*rXP+x<>JjQX)318St|uvV~uJKEXk6^jUC4iaP5)pR47`nAe-_42#@tc9lIBnB+*pKJasghpx$0F3L%)+j>t(E*n zGBwVCsuS8a2Z{W9wu1PZ4b2I7IZ!RbgAAPrPX1`#H&a98Rbc1$H8)LO#*TFllWtZA zAWbhJr5!%cfu+aCq+%(`pDI3pzzUq(#IEp1Zo=~Vy5E{ZG=iQOs(c=QIW|BTfNY$v zM(8F7_@n9VxB(8>RniEO~VSpV&tC3duJi)*VDDGKRMOzsgz=iN(2s? zxv&V<`Gw04sK`;Bw`(J$9gWvI9$Y4nfz*5P|L-a)`NT4kxq9YvmwX5ao92@mUV1)nV6;d+;m-;PIXIqS8WdJ#e#~tK#bMz5Y@$j(iyVgIv z&D6iMw6gtRc9`2@SOUOY+WvnpAUjZa3}!4qA(V2cW>Q5!K3I%A!gpzuV(_;I>5tAz zbsh>dQvTC|B;&5NVe&_TK(y;lC^pu5?f!&pjQA`B=^VvLtY;*V1ZZ!>3NVNREJp53 zXTHMwr+{e=TM#hi5*Qe)rh1XZUs*-@kd;HLt>S_lYYCL))_^;Mnq-cNIFEAn38ddi zOC~uia3Ac=g*jL8EdhCUm4$1v^MG_*Mq7vI9TQUf%OAh<>YHCJ$+Yau>eY5Jy4G7{1tl_HpxW(!HrX1-^wo)QH5Qc6D; zsdT=ys*ln*bo8J1v7YcaisAr=*?>I1b9dld5 z{YgM^ZSaa8(3n-^k@dd6OJQZSyykXz@zUT{9^y&SU;XWCA8r25@DKyi`J4p&T!CZ% z@5}$+PzL@XB@g2+2i;AxTzGr40xl2(%&L&;i5_+iPJm$f--OxGh{Zm z=($RBL^%`t`->ZQs%bb#e55EC1rONw_o|R7I$@hdoIllruYbNI;0&xr zlh=0I#h_0*<;+YNoD-TXGTVzFjW_X*cG*W1@cy#BDDkHeXS?fITF8&TYn$+HE^9JZjeSs5Boh+i=Nn-2du z?*E;r06aH0caHn%DyP?tO(h5i7xyCoVt`-%zccqo?nMmv5ikxLVOL=ElC=_hf8$b2 zr7xbPHtG71Xd}cMP~mLZHTH(-%*Z-%+=CEHIx?ls-rei7Z@S+w$%kQ-SBZ_A=pumU&R++f<^d&Rt8*>3?aD`ipg*sV`egNyzGqO= z3)G*e6?V9c?v%oDGrT+$Jss@EMh)OkR4>(xokqnGx@1>aM{ZSqzX)nlMv)ko+Q03^ z4|E{**U_{>fNug8yORz1((RcpE%b2PQNi!Oqx=89oG^A!zTCIKoskr2K;V`Ol>GkN zi58li8IRUFvdg&=edPtLCaCIvbJ^~Z8jA)PNdNCgA|VQkTbrDI2K5{C^a?T0JCZ#G zX$neQbZ*3NXkWXK~_b?I+9q( z9eU75;TZ&hU$ZyOL`5cwh9)dK_=;IScd(KCzWn*$=E{Eq)81wLLkRjv050S9o+U+?xt~fl z1*JB~a#!)_wp(~+7&@h;JES`WL`rEyx}-a$W9V+A1SAxc z4(S|_ZbZ5pq`Tp}dEV#uyx)HJ-rx5p2gfl3*L~e*taF`f8DYM0YGj4>G%vRNT)Dr* z+rCqC3RYl17|eDsIXw_f6B^G=Tg^`;iyt^!DQ}iMRgl)g4?uwiNSLB~q;EiwZSEQk!@eh~{%|fYQ`^0DWa+7 z>dJ#NDL)hCWs$m_5QG6W>e2a_Ir{0fT*eM?-so>{4?^y01LQuK%>!tb#=gq3bbm$p z&wDHJ5l_yv8+|XM72I>^O;7sy{y`6?s*1{UJIcstL^?Th_cwhXQ}-%8{r!X&n$S#q zll5}};YjzGH+@f)L8gDA#5dcSX?%ef+DpPXu;Dw;uJ&^hAhC`4`l`Qv$Enx*6&>2zhk>P%QEn z6*ur#wmt}FpGU!odK+xU3zMWxw;vRVxkL^0X zsDtrb`7)mHxKZ91J3K4wA_`{D(WPvjMYD1YjvsZ1H&%}X%HJ4(=G~K5`sNQs3`!27 zMDcKJ`F%% zul0`jNSV#s2hp&xy>O^OKE9ZXQAsQvHzIA9YkcVq6?7X!BzPfJ$EVQOoy7PIB_2*2 zmxe%iWIwYSSl;H=vEaV@dinDwQ~!&7w$*IXk(rh4 z+!^~<^CEb|O#QjdFjN|<5?U-G7mlHbqrSK1O=F06R+XUU8*4rIwF;6{4pl%`)Tz-7 z_s+sQwCmfawqbg6t}`qnI(#d?SYN%)Nmrol;uVvz|h z(=T>~Y7}e4bT+Aab{4As*SmuA0z$x|q5d8}(j73@Wdn@7LW-{NBmR^A!u=oxzs;y! zf3x-yPow*@H77d{2i>knjv&Lm2FT;#Ze&0Cni^DiiSQ20TaT@&+ls}f?|#j8!ocNl zH_v{9n}>-#(Y9%w8bWvF)3fhOv$*vi-!wYZ%fE?IjVY4z46Yf98v_c&0FB$ty8e%TTML{+V|2ez#oT7JX-$V1DQNN_F8(aXD5bdT#->;_^kn zgv)%r&jN}^zK&X6@Ux@1OzwbZw9dRW*0bf=1lE6YY_hmLBXd*W?LkV!f+`{j4YSVY z6o(tB`>U%#uYzp7%8Ts(Uam$Fng>ITwpGlY0W)3^xG6Wh-pTeo3VPxl7w-h6g!i0GI@F2>N zSvu`qT)(JJv(g<``pO(HzHBawdTDXz=YPP1XN}&udM^2*mf_tkd|7QezYwr_yM+)i zuXo*BX`QRC4cC>VAPe~$2rp1%@AGQD_#A>6W$LSm8yjZP=PP9(#Qs92+DeLp_ladOD*Nv)C4sOa0v4?i zO^y?d3YSn;4-TOz1p+|<+MlRop^|Ul&icMfJsDL5lG@TQqEPhcRR9ta@k_6>iKPP$ zK-$z9mahyvL*T39sYhG9-RpN49CppDJ2SXv)Ku_s?yCJF4@fshxa*4 z0Llob$B_|}W?AN3#?wO2KaWTyT*9Xb89YtVnA$CdY|E|6G!gzH^nPEX~ou0sG?CKlN zcV1Wn;1i&Uf6702R9psVyvnA;-P6cMctE@3MnV8rS)b!~hz=qR2H@ z#OvfDzb$+RxqE2B1_FNBSrS9?<<9kB@kF#(?Q z4Fw`|V_#3fhl;Y%RPxRC3}9y(ju|JLFX_XpLeRk}u2%de_yjV%vqpc~y8Y_;_y7>yd|E*ZHZ?Az(k& z`(I}skYV2+LJ3<(KT`SnZV4hH2oXGCn?{ojY_u5FTj@>D%N%91p!|4|@)gAO= z-{n=Q<sB`(8zRj6mY(P%7=8YAbeK^?pjmSc;gSNP<3;SPlhd_Gc21ogE0C_9jzg z3ebMEEy~as@a23Yw%8se1FEMo?VIuw;BEvr6`z?+1<1P;g>>thAxH(}=HhvjgM6zA@jB);VOmCd17671(Alstv zeQjrX(J@GY?=rkn)1z5t0XP0UO(yQbP6OE|n7&X&W5py1s<~z}DlSW%hNy#D2@1ossGLP`1wt8X+U)0uR3HzIyf?pz~#^yuHJ2=4(F3qYZ2dOCeJ zu$!RS!p1U5LBA#49|Rp;7`+C(b@AE1v3D^ABrEfGsa7f(L4&QjrCfw;NLD`y{Ttr8=I7O~ z#KK}F7MITWbpFu-C~#<4LDi>34*D$MK2OF3CxTqz>JFQkb{6?xHj@!L35d_2;1jz+ zbZhY!H`8Za0#NaMC@(t}amdvH)qda@Mcm@$q>;lSihw^%rzTOn?T=D~-&H%}lCP%6 ze3S|u*E_o=eSe_j=Od7rEHp9o)G-OA=HtDBB4P9b7}J@zxc_O>&iIlz09}iTUaIPnPn4yGGXk;BC*0W1Mbkr;p)o@ZR5Sv@>SM1E4lO0BS!x zi_gxwNoCBZ`5(RW|Ka$YpqY$uzd{VM+8-u_{TA+VR!CM?sXRx~ReXl(aHdwMRn9}B zC*XSJV0`@(W-U_w%_&)Fx#2K$Fmwly*jvXBx4HC>)MJX&fJPFfj_mwl-Z7!iesS_T z1RV`xQ2Uco$QN(AwWY#xwH;Y;GzZ)gdA9X~SsP`csgBbI)}O;GMWul{@0M}Lzsqz~ zs-+t8IRfl+ZC2Gm`sUtpzFzNYm&2}SMAIMwIq8i)EP_r$ed-7Y*!maMs#R!Pi;Td` zK5R(^y$yE$uUG0 zxO!E!UP|PLrxC7xQzR+JK|4FZ+g}cS=YIk1Pe~^7%oKce$qGYU!(5S?06#&WFd_>{GhxC9-pCDxj`9Ua6ZbC-0yiEmsuedyJFSxVbjQ?i(Y02og9*&2EH8^R< z<20{VNdxw}z?fe>g9q?yHSs@FQ>(T+5Lgm4TJP1fFwcZcxws26Cpp4jQbTi_139C^LpzJ8nGo~j#*b^NY#}4_pMW)}A z^dD^Ge}85FeVAf6j%2ggJ0lkw_=lhSH3Ml}#GyR5(fEvIv8SOxSBAzMPp#(x~MqK!z+ON@!*8Tl( z5o$e^LbOF8sFK#;jAr(Bxi#cya`1?rf2w*}&C&Lei>o5EU&SE5 zZ^;2t-fSH9(z}|R$NRiW@~ce${jbBf?>f?GfGtAuTD$QrLF1p`5`)rGk11cr-ATT7 zY;n5REnatAGvrgLBS2XEaA}|Jjc&*tm0OR1th1b~`DyEjXu5I0`=Ihy0_|*UF<0>C z&u^j4D3&5Uel?%5l-=fxSpF?%GkXRLOi;`Rh(B%3bsWcQFq<;R`u zCO=G+9iQR?)P>vas>LybLY3mg_}v`su5?eOVW@(BuFVIhkn5O zkdJ%7;tY*jW!o}9bC_I|*xMK;L;>6jA8IN_fSNxag=;=k1s@U^$c`e0txDlWQX{`? zT!Oh*ttAy(O{l4X7?fkjd6oNF-B9`agx-e;>#HdQkpYs3;9W@m$|1>mST+<6ScIU(qtm>!bG}WFsK7wnS4g za-KQ~QL?X3PXE3mJAO=xgpll!GAa1kuKHrB1JiptRVB?cdl9Yd) zQAfWk^RI7M>0S(Q^t>)qqpKN9v9{ezNv4;$Jea8>RSk}co$2n`?*%lZ&0!wuPu1b}yGjox^pz zf_uI!hpI|Aq7ug-{m@NyP@L>@8(C-)sFji^J4GY(h)Tscc$GVkK;w9j3F?W}_jFpy z3wdVW^8PvmCV|*3w_5c6dJRe=F<`L{Jd~q2xa-t&DzPuwwaaNCO&NN&%~;ewUk}4E>`{Jd*OnqtYV$ z3=}CufRP~6_2HY$|L?`ofq1q(U^I}5ZOI5swNOM3`UtF+91z;73?rkXJ+PiFsQ<*$ z%2?3rjq?jmkyz_KDYux{bq}l6SYa7*ReA;24TLtYhzR%7Kb{Qmt@P!*>eoGZu_eb9 zFVUAsx19e);;FAO<(BvrdS`Py_7$Sn_)+m_A}<8h0H$xyTkF$+heh7VPk~f?@TpuF zf{ZQvVK7b4HQ{|XcNIGifU_Gt0>Bo8zQSH#49NT9D_X$!tcHt2VlDDv@m@U-iJ6guY(VfQq|iB(A!Yuwr7Hw{txEC;6=2ww0ahrT_If*&ks-q>89AiYm%yn8{1gh#-Jra;&2aJ4f& z)fsZ9h5tlEh?Dp#bD|ajOyPXkC^+Ge7VlH#x79h1e~YmUoYpBNArPL3yG}fz_^eX9 z+#sYGpr+UPk;gVMJ-EN8O$11(Iw4Ph@&l~KiCE?Qfee+MY=wVZrkjc~rOzlXbNhRK zXMA75$KjG5waW}@%a2f^Z4y+Vm>k!2b$ora67~`ETkLD-9es{Z%5?@$KJEYiiSR&% zVakENYg6+&`O#k>1yvgjD+xI9r|mxWd@=b0{`&{DPlzujn7&voL%^nK!NpQ%1IAo- zdP3--Xc{mDtEZORsfFL3jhl03^u5T{=*3i!e%CB0UWZO32Os$Unp#RCU1Fi}WNm^H z=8EFB^^Zugw78u<<5#sp_9GWpHGoe@g*Bh7rp}_W652<^DFAbbjlK; z)%B!B{Z~1J!*aCLr-hPT_0Jz;0-<78{1YhXmUZ0?liEIV}pq)p%-!-YWCSXGabt_$q>+>^Hh%ios z2O_#%oxQt{!TTyR15mSeyOP;#OjA6PsD!jLNOF_U!YGFWUwX?K{bd<5?izhjg&9eZ zq#Q8?SIXMsa;0QZ4wj7Y_kx`KR6D;`QK0#EH3P)lD`|M&LE6RM%rA|@(lEdj>(_c; z(h49Y5dcu>J2=|zAQr=!;Q52_fAJCi=Wzx|prT+Qua{`EXT=O5^##ek4U>*{82&8{ zx^h$Nc%GO@UAFcMmwKJR8Y1%*tWzBilm6-H!iws#!3K=EQ_;?;^G?)pdk68S<-Nd- zAGs=1Z|H2*A{1SDg$hzRiibmK0PL3P6`T^xVg4>R@L38+T|ggQ0`uu-kty)j$hZim zm2msz{*pK;yZ-OmOu$Flc2Oouy*i&Hqk&v_J-9unh9J%6Rh6MQi$Z3vhRM#LKCA^* zqeN~!wysq}fCn$fZcahaiN@{Vr;k}@l;PLVXBy?Tl_9Ln#F}>_*~r_Y?O3TCng(<(RtPH$0fuHU^Sc6%>m=l-+$IqL2dwm#0WVpDu|;$9AZM(>FwPa2xGuIXSl_bLDYiK=5JP7kPKNhjC%_{0xY1Ov|v z0%eCA#sBRJkJ1O+wKTQ62DGRqI!5Sw2Nf!qieo#Q8HCRBvC!$bcd0wF(TdbDgopfC2B)WVXlD z&<|G|^zops_S2EA#umtpsFS3>+kREr1uH5puH|t$2d6{J1QeZ6E7$u{bjB1A)_tOP zkR~LZgmFGq3g&nN%OnWS= zGnQHRtscjD>!Kw9!yY_w2CS+CKdJKPh%#|KJ??+K3^>9N!MNi0#e~au(f>u+!9^9s zx?|I~yHI0(2$0SHMbj(bzM+Owm2NH7pU#H!Xo+2jbApZ+&~zv>?z`AQ>F(Nk)6abH z7Jr}PWyri9&iW?|+YfbI$(s}hez|ochA`Jx8~sR644tUoUd8U;{vg-cZ3*$##u`&q zZ!7-mNulHr0K&5I1_~}$6(F}JcnRj)Y?IfaT1Hz+M6CLMTTa^n1~{?Ud`GQcs?x5Z z9dj~S!1#Cl&$|3<+P^`RerXbGC1f6G?-Y%?F+}iUyyA$8cu*ueH-{9D+L3P#*M5cd zRO;(y**0%1iChOFpL*?8+Fq?ELBlV+*$nG{M9h_DJjbHIH<}l`HmNj^!87oiX(yEE zEs=Tmj;f0q!X+uOS>WsC@Z06sOqHq5zqKa$=M%Q6{n5*B zfGsG2g^zf1keG|VF<)rq*{C6vvT!HMHq)ttK z1>II(VJeEoKu7@AN^s1c$dS5F0NM*ANB*fojGOO?nV}<7j`W zPI7sZN#6Upz%1-YXG(7jqObi;$7`o6rz1N$Ov(|G5pV*?Y>fRgIef$>#m~F}7TLap zqeXzw&_79msvdfBdb-lbm>s;gXPb={Ne2wFE->`!9jX76XyYwBw;A0A=dR{KLcQp*%U7t#`_>aw~SrY&ctWpy0 zq@n}2d0|Ms`fR#Q54r5;_lAAB?Zq%a9|stMOny0NQhw+HNCac$!!B7OEZ!i6d|>2` z&NLX{eSxxv=;tAD_fZ%zO<)8}z_Qt)?>_UP8w1(rz6SkirvTW5R7GiX&c~C9=?^PV z+*c{pVFxCTV%`EjvPN07!rJM?r8tY@+h3%g&_gO;<>>HTCvA^XUpqnS3=M1gk)$I% zPCpCrP&&J8jC+5lp@gCA&F>f)>w4P6w{Aumx9d7!9-a?P3BU94#dQ>fZjaGs#30`k z8tQxoPnIj2Mxk!>NZkJ+rs5i-iu{X*S5pkgK)GpE(kM4!xF+JUR}A!`9GoK)RRH*# z&*O*k!5PUmQ2=oB(kx3Vdo?l21-}0IriTFgn>DZy6C&n&19Y@Y7;O(XHdNZGI{$e} zaAXM6!!%$hYK%6r|CUA?9(9VAX}B!921qqU?nl?40P5V&fBL8q0qT6cwG=;lbWRIf z{wrolKxty(W%^B0QB8nao{5GnyOiF2VaLCI$qW%iZ|<%(RvV-3=kV@>*!z|WEMaY0 z^8vUvju>QrZlL&dvNQ@2eE56@!H88`hCdVXOBRI4%#`;-o5!b|G>+Wm)#;|2IWHBq zQl<){xz<|$E#V(Pj!(N+BayWDeHZ^iQmgmHj$@Y!``JhFBXmfZdZ2Wso_O4DT z(~sIp#E&OaWAdpWVYg}kQ<|N6g|6`l`PnFXz|n$gSOQ=)%Of@5VM!hL-?a}A?wOX? z8vvF~d!y^_bc%WyH3tVaHxJJWKpd+8v_kkR4}0YcSYY@Ec_A?8MQEn2cpDbEFg$@) zwi%%>>_5-|kjo8PyxL>QMLuc^6yZOGF}NW_QX|Dw%_Jc&@gZR~{c@cDNzp=g;i~7x z8R%G=p$DyVP=3-jED`s{<-M70ztk0Bjy9B80Ft{VryHcUTmf{gjv~ZI8?}-c8jem! z`XM6B>h%u~GZv>S7jyf2>B%rpxsBp{LpT1%yHrw<>Jo$vX=q-$f0chK42y96cR>s) zgMv|!Ahv*O5rShSLc=ny<@_|%QdJ%Oay>~Sjh9&|s)Pxf2JQxSi~_ zx`uUl{6Y# z9nXHaxh&$+V^nz6V6Luhc#kZ>(DUdGesA}=QzZfxwXD`~|cQ zIKAS9lwWB#IGb^13=Mk8(##E)eYPB{m5RV!DdbIl68HOeR`_=$qJpSO!{a~E%}`K_ zSJe#-J{Yx{HjR$!-3nLmTXh4_$`P*ITkZ;??K@3A_6tAP{n_m*l#Ms@u1brL@HlLm@M4;sseqsx-W$1WI>jouvAxnwBCuGhc2hhWq(!L4^Qpg^@pcE z(O($2)Fq8R{jENLpU$+> zjrq=}8rDjLplK5FFTdPKVPhxerps;#8zCTqE2bx61E^Yk8c|orVQcaHcd^HL)bWCy zZPU^FXW!7eNx=Iu)mpT&`}F^=XPy{Yj@Kt=63rW@H6=?QGUztqSFyQ*~{+V`_JM)maaK;^;(};7H9TC4Fj;m^#15MGt;^cSDd5P> zns@tESU9Vd@h6sMLkdb0CjrcKv^lP1ZaKD8WK%a}d3+ zj)@pIt>shsA-Oe7SR3U6X(?7)mTLct8vQ&8z3s8g4?{KkCycAf^(S1OgE{T(Imd4N z_eg-J7D*5R*UsM6^wlWsw~$nCGPF8RLm`SB{=#Nw?&ahRe`f?tgfz3 zzinesd$sYlb94WR*z2goiTOhWc_@=kSN=Z6S)t|} z+VIl!N(CWc$N|8EAYUBIrNz!pn!84y(-?GiLk$OTe0Gx9YrQQ3Xmfrqq+BeHOroBD zh(v%r8`Zd4oY8p(YYO|N=~OcPVieK!K9J-MMlgyrzdie*76N-FgWlL$|LGIUTm&Pd zq5Sz)Hl##HIxycxD)7>-JK965iVplXWu5X};coy>j$GAG0^GLLJQFB^@$g)&kkQS@ z`|wu|4PA5>$CkpvHP-2AU3*Z)3|>ZfMQ}1A~4HKrOM;`D@!>Pbh_o+RBP=O&&>0L@tj!1DX}O{{P(Pl zCJv+w2h>_;dTlT!(*6@?fw=nbrj;hqztEW8zU8NBaK*gT2Q0@gFEKALvt4WwMoR^V z6;3$z>LZw`Oria$0+OejcT;?B7Y8sP6nG}=p=$I&$|G2hylBRU91IcFbkxa8KMZF+ z7cKr{`M%ynR)kFappAf=ZFXmPZB_bG;80npICrt~U*Vo2;Klo^DWhun#4iUKqd%0$ zX<2nl*{TIC`adnl=r+4`6~XH4H3S{05X}d}gS^hKdI+!6(%X9%TS^sd)1iHRfI`EB z*jLx4pHG?|eU929|S*JeDEh8v}L#L(4~8#0_HvyJ-aS z!IdpXONz=FwRc2|rcPho<-f131@z?Pgw962kyl$a0s8vOKgat23br{y0pA#rD+SJf z&&h52-Ed_#6f^6Q36PN?UsxHqadHo8H=<0>@%EKq7Fyv<9a966SAQr0Z@fez9oe57 zVq593Yvkap^3c4ls}+1)eh8yd{d7uGE^J89?5)`P7INLp*2q^6+Bj5)4wgj#GyoxX ziOLdsc$)TRqg=`4sL^{?j)^~Bi4g+mb7%r>xgBHdTr%w1?2NtpIL711xBQR?FG3$Qb;)`Z(MKOF zGSMH{=j?M(0*!JdQ;ou}b#ZO~ipAynw|>`MPa(C(%Q@%1UQdWtA(QDILyG_AL z%>J{71IQ`H_(!mU5o6fQ6#fr+Ed(`+A9@Ma_@#|XlFep0rp333y z0Frxi^S8f0c>cY7LvkeSii@+e%jFj{5QBu>`3b82Y-u-fv^q)x<$zhC#hu(W+xk{O zykCH>zDeQl=&0t3$c!{dpc1Gq5<9=i@s&Te7)lJtet?h&&o!igmEzY|*^QCwDzn!E z*_z4#Q8_^=$5{H>WhWbUhqu~tSA}@ca`~Ce9&RO*1j&aCpS2=k$6@l_hH*tVNYhRd<%QsAt5QMJz2Gr4SeKlRsl^?Gw z2BLxZT=YIdA)G@za;&V4KN&O;96sN{L* zw4t{T03!dEGDKBy2JN#m+r8~B7 zmwv?TqB8`0a{?^g*sj3<@*~kZ_=!Cwy*UCF%%Ps5B-Oe?rvhMQpZV6>s`~mNT5V!C zn*wD&Elevj-Xhgkz?{W=V*=@Yk>0apK-VqMZZ7Xohhj!H+gce~P3w4^2Ws(N6CvF{!Jzxz z9%MKAHHr`bj_I|tVB_xeZ-1jC|Rys zhn2#v->-2`<_~+gMg|lv5qc?ZtpItll~7W9|A+hADBlj2cc|FzkIq{W_vmppeTnqB zfX|HPqfVVYrt9B73BA;g9DB;sbw`JQ&^qow&qH!T0<}_fi3W=WK=R$$I4noLNYHO`^Q7m{C?V#x zogN)9>{|mol!Ff29u4k(aUBBvsqJbS`^B$MpYRi0)49z$AHuev7wr;ky{n$hut&f0 zq7^|MfEmXDGZgg$)?<1*1Vkwg05xO69!zY9A-`3@Koj*>?%Sief#T(UJtj=9_gyHx zjrI$TnrlVZfg7t@fiM7IY7+heh@2k(^$47LLH6#2`VU7}QLD+4bY+Djy`Iy`92Y}u z^qZASO1~@cQRdM{z}?dS>Z-RQly1VnR@vJ6g>KDE=TEEE^Gm=Av-$4`Uh^sqRpDBk z3F$Cx$NtfsUX>{-AZ$aaxn(F=2lpiWl6`PZPyQbFxIB+T9Zo`{fMwFUZjXW; zhx2M)7kJzqW5-3^Uai%KVr>N+cmmw>2e9?lz$i5JHlsL}!0Cby`b5Qw;hr-c3lR?TARK|HAT5boJkQ!K^2`{z`0o`v}O$aXH*T|*& zER)si$jf^VG8o{gfYDUm~y3IsLXU{F5wl!&FG?nfs95BS}oYFc{DyernY^1=_|5SA2O0YDG zKlB=MMsmCrai6?sfYraLAI?lE4XmtwysW8u{>}&vwXOC4c_b8+%<`JW$E5BvK)FKz zAoRn<*)1e5Izi`PaSEe9{mC^ydAC$~@97|1(oe#e!CKYZ$@`tU)xBmZXW&t^!$#MW zyn?hqkZgZqsDf!^v_VV?fI;X9ogyT8@b<-e87B$U>7#I#M7^ zY5AJlNP@zmWgvqMc`MHxAX_bPjHC=8>P!AGRd`tc1LsSu_VkU_r_#@WRlZUL&}H$E|u;plEu9iU%OklJ+WG zA$#{%Cqm)|cY=Lx^K_I&SF7Vt5Ru1UHRsdLwfzdKlhsb7la zJ;tGs&Mj?3qu*D5z~3OAfPmMFq7d!Dhdsh^AUbiu!2o(D6Gy>@Uy^F^rS;A{Px1qx ziX#c=M!c`-Tc{1p-ZLu$M6yRv6-NKck7=%PuY{V;C`kG6#m;h!7bqqTqVEm_^Fi>E ze+>P#7G)5_0mK|>I1`Ws4g*E!m(sPgw&&`~;Zo+Br)cM?fG?NJ?vsv{rA8L8uKp&4 z2e7o5Fk9BT1`t#f1xM=i)QsXkjCcj8c#KMCHa;$zj(`6YCT^@=QT{NIB<^cIzQI+v z{99c9Hie~2Ww=~6j-sfj56aj9B#-xT32J8Cl`)*wf0zPL%hZ$mI_bf638+Kr$@yV& z%@^G5>89s^5AK5h6(?7oX>0D0X`U9id`MxPp7S{MuNx;<8-NkSp(}_Myx~hfQa0)k z1D_`U{b_yY$PN!S@|&<2Oye@x;0EjEHm5l%5sUjZ`@6j4J8u_MxMMZ=de`DcR3s7a z$N3?b(|%qsy9YmWycw|6QIKCjsW%ryhI^)R*uIPdHHg`&X0CIYb+~4q zo+$YHcK<8`^fjCe7XX+m7)VqR_itY)3yCuz;UAFu-OfFWK7NwA>4;e2-7hNqPz8P}~)qqiNdr7}u)FO1b|0nYNl{^mv>LY*I=IPQ*tUH=ECC4*8q1j0?iP6PCkq6qKAJ;xPDFX+igU3+XU(tfNgW-_|Xga<;b7N&Hc?m2$1lvnLjwEnYv6 zr%Z!009DF%>-%C8C+fWKTiTW>cE|Z>{k-iC_v+Rgw)<)aXt_|Z0$g`briEe)q<%eW z7OI6hUfAx%@&L;S)^231DIWniay#8&rNwljXhySK*73YSjxNoF6sy_=jEQ$gya!8!iHd?jVGR0b!ZE*CFV9aqZy)Pt!kTBEE6Neuy4&N@9er|pAI<@)z!&5SR)RIXpF)vu1PPHxa z8{InNJ8p~q%x}TrOlOChOs~UfUcVtxNZ9oBn#dT2`C?x-9M0E_yxdxJ8ELs}X%Wl5 zJZSM=+}-g!3fF3KiRVsaI-UwnQSvD0ap+kXPJ(YaV^grlQHw5 zm$+yts}O2Q)qGOI^ba8|0a#A%0yCmqlp+MvbQp_tVpLL7`tDhM)jPZnj>4RBfeWM< zL07TPv;@--Bl(uSs)22g&w~e((V(oz^JuGCFQuq%UsA2F1xi_ec{4b6jKmj!$gwHv zul(IjGsmgIinvqJh}7KCh-!L*alY&H1LP-WMU#=N>*D+4yXCYUE4|vZ*g{iu|{`MkbpRd1Wys%;@edWYTme zUGgm{JBCYG@&|c!*D68G~^s8$Io>yO{%N82htn+8=_CNMHF26q)2`_PJ zRME-uN+2vf?NUWJ-~C<{Q@mIUc3H|4N)V^*uC(bdLa&>ib1CHGMJQMEbc&Uemp`sS z1Wb;559NGHf0yjViDO)Z<1KjMSrx<`Z=ouv>g>dx$9o~>0Am>y*c9` zdmu-CP!oXJqQs_Yxq-76OF$KB0jZ>2;6&!U28^SaH-S}_ z(m`%8TBH=n$GqTYxWs~^6S||2e-lW^gC0bWNy6I;UGR8cFz%d|4@j75Dqv~WfgC}s z`|6{q!1yEGKhZK^qaK$2Y7|=J?a3jsFAr$ZcYOoNdR6ufvwo-s2Ep;dX)$vN==Pv; z@EzEQLkGMasvUO?-v9e!^Q+~Y>gSU+18^^{gaD|p1s5dAsl1XCkn%e};7KnY>e620 z%^O&QW%C*cM^-?`RTc@!A-D#l9$7T6JLtQc(dV!vTseZZCkv&c2=^W~o5G`l>?@R- z&wk54OIv2rkVZF;afnPid%dRS&t6P z|D}ndITI6(j&hJ7&)yJ>gh9OA=M&mM7&d&?p5-5Uf8D8N4Kyu`IgZh&WPV6gA`~Mc z50(m3d3-s1DiwAt`}K|3T?e5#sABYD zu!HmmVvvvZ(CiBO7_-k5NM!q{W!RxLVyVEd`Jl;c;Eyl6X6z3ySJeSBn-@2!PW+uTq*!y;wxxoKO^ zVeRfsHB$3$e*vBp&C;=rG%9Y}wBc-V@e=<2Pt3ot@y?lOY3~NiY#oIW{f!Yw6$C-u zLeDCqG|Kd5-l*|l`qzSV8qIi+ckgu)7$ZUg zLP?7NH+R=#Fm7DgF*Q+Rrw&FV7t?9+v_K7nit0bnLwX6r@HlI;ATY#w}>$^86?NXgNt08nd)ef|;X?yZtq#!-`)W&`+s zN01X+ZULE35$?V0{w*6s1YttFrO^S8IxWKkpbXs}XjqI{&i%{`NjCLvraa`xwvN-x zE2coR9UNq=1hwH6Wf!b5Ibv1Ge@nn9onhs8v_ET8POsNyN>ox~#E8XMFAh|Eg;|PD6zGlM7Wp?`tKKUwLbLJ+ zVCrN9BcP3;OltPwA(RndjZC9%W&$7#4wh~(-e5w-0-(=I$<7>bPQtojv@(V3joEy) z8pU9=7*QJSiDg4xW+9jkxV>G^6JXkS)(d}Z2JV}L4{vhACQ~I?3Jzh4$ zn$br$E3~#cCUHyR_gP}8+;eU?yQrkSiy*9F-&8j}Kkz=8Ga~uT>Uf&HD0y0u`O=W! z`Nw|Cc{%@2ih0`B%;@i=-U35kWbTU;T!d(TV8dHtsvxP29#(}TUF z*z_xD{EX^6I1LX?4oBQDni;hQ>H4dngP?;@fJDNp!qG}9K%y{@dq11;<%=Y+ z0xex*^L=!>k{%tWAS?k$AQSLrPfuP?=Z}X9Ex#Aerz}pss9*V_m;aStcYdL%l#eaO zG!kt1w_(BRR@yF;tax8Y-lciYxI6&xK>L1%d3Mx zuZEDnx{WwLV)BliipP8qN541Fb*JR#Fd7}pP4L_YacD-a5nkY~eliF@G)9s049PzR zS--7hQYzbAH8kZq!R)8ZpJ-k56s% z@%3D#6DEk8xJSLD9%KM!qP1Imd;azV9)kkst!<3RNiVJ9W+PAsr5!|#$@n9J2R$O5 z0~>>VMi9tpC832YBuoy;AO0jP#H52i=*M2ONw~}S`Ix+kr}8ae}^;_q-2yB%w7`c*7$*; zb`zK{)bfJ(;9i!DAT0lMEqc%%#3$MtXAM`OA7^m9e(8x@1BuGDZhZi{Hbn`NGFhwS zW!D!uL4e3Z`mXx2`||1b*Es&EVEEzk2rw~3d}O`B0@W_G<5%o_@Pi~#O$NYQ5`VQa zAlML`1huFlPD7_L!ze+6iO@Cgll8u?yaQl-9xp_QN&%QZn0Nj#U}f|B^Fx)EuH+ju z({nSu>kPp+Fn9Jv{gNYj$tsl0Y6x%VaLN#tMd7tgZRR$uSK(_UAK*#|5=Ft%FVSmA zTo_W|Y$2EXg10%GWdk|-CWfS0Z`_5}cr4=v#%SXXG_q^m{A@FLPGUmRHilI~Ug2T{ zYo7!M{U9&Cx;kDPayDX=rl~h`@58W{C-MP5?cm+_Ive4%v4A?` zYwWMGzzuj_taLQbO757{0r9M4Fs68iTK2q~($;g@IlvRkg&E5W^bI}!<)p%!qd|0K zhBR~p8=#E{vjWOZx;Ug0je9wfu=}uR{lBjKU#-yJ_PN#ChwXZL1=0RES;J$#Y$~@} zy6Rza*r{8o3O_dL=`z&$MO(N>uc>m;K&0~b13_1H0(A57hG)a?I-T2Q#xHk7zrRzM zhMG^M+~7rxeO7;M9EQev5BFj;yHV43Q`^H}grkDVAL;Q^h1CLR7Pk+cFIO{O_}#o| zWxTV58^WI7&FlA$U_1nv!>+(a&s+~b^PiWq{ngkeE)7Wh8gO$y0iyR~Yrm~QB6kWF zj(Ml(F9dXX3X7B?od(kA*W@X!sab!OzK@g0@nY^R*c>gYDoOGdyYBKXnjCM=2vQ5L zzoO-A1BAUi{YL8J`0c8=iaP#u>As2SQKn*aqt6J^+2&eR5rs9_jo z&J)N;v7w&=oS%;_7m=0Y@uVr(63up#Mj-t^G<{`QR9)Nl9tH+!1?g1jMp8mh>6UH~ z1VLK5gaMHbk(4g!?jF*c?vM@z=?v*XAKC8LcZPoc1-6I?;ElDFSRxbk!0pa!q|0dUKw0QT zV7rFrZN9jC4}nfuFWr8P<^4`Ixjp%?z1Oyl_f@LnxRN0GKt4-+(1!$0!}t2qbTP<> zrsTM$1oRKYnZRz!eNStkKjZxmAXyty{9x+whv1*E>yA!U+(D6mH8k>j#gaKOLR3xcXpGj6k# z%3?sH1ae?vkySIsJI>kGJ?pbi2u2#2{>1e!?2t;`LV^UyKbL?{^M0*;f_v zU33QUZPg61DHX)vZ0T2|>lnM@S(a-E4O zXXO4*>8E%&9)gawe+4ct--LF2p(1{rGhiLzH;7?awDYhSNp`-1ZvE!K1>bw4^TSHz z=ZGKw$eFn=j**x&_mUF`v_n1qMrR@&*CkCZO`MJ`0(M5zzWxMdeqYcPMWkmC>2 zD1RcjCd0)io$K;j3bs*TIxQT_-|_v!L4iyqDc(l8&A7OBpqzMUk2;mt1D-~%%ZVyf z2ojp>x*O@XdN&xVhpuXz3DA;p$P@LFiUy&d5MG6B#Jk_4{Y*F)(9jdHOdd$E8*HAL zu7lYr7LfUThR9o|yOtf1(PZ^axd?0?(HRs%CkxX(>%7Sn$$B)oS94>V8c*B?`w~x= zmq~HLZLc}CrtGI{O&#+D1Y8$|=l@%(ZvR@lN8=P$k!$|sFpAtytl$+Xp)X!IWkUb< z`p4hKORbn|OR`aV&bQwESKo-wf)VZ(_y$pkJ>x4|$y zAOk9ykYpPI(Sj;U2vJ1NLmrNTJ}7`f$6XUw#pFA@Q#}i-6mLcv6u@NIP>_X3PHb37K?1+$n}zRT&!}%Pa$d{5!_VOsgnwmsr}q)n&qH8XZ(q zpS5Hn8xiN%OQsku$`GUhla@pTf5UBf^hE%s5YPVlX@|d66w=ALpQG+RSz;$v;sQz- zgOOJb#cA}?=5t&+Q`&?bm|IY%zPUO4W zSY9TvhWz%MQG^}9_`gVdc41% z#K65s8~Eu7REsxDW)ky|2m`|(ixhVimxA5&FK(znn?Nqdpjx;-eOvJT+ zelr0Kxb3%^0-jsEb-jV#F@C2H6&DIm`Gg3j(77D~7PJV5vI z%{vwrKUy(K3oO+PFVTVXXftLnC&?XYKoK%j z_>zHl#kC0PXBE!Hg6y9rIb+Jr*XDuxuhM(rQh8Iw)E9-~M2hexw@Z`uyxt;~P@2b?wEp(Gj1?4^-IV(K9U-5iv_u;r+i4+LF&|4OA!#aK^#MR(%fw z_l{?t)QEW%Jgo|P(5&>TtVJvSyp|s{;7&0rjghszP7{-m>B;)ofps=)QRw;gk$1-@ zMOOW(*4(=J7zIkfn$j4cWQ*bHo7RRiOF4lNa^uc*;@2EPGOwh ztVYO(BTm@U=n@DiRUHQHi0iLDCxghQ_qy3OzMUL4Ko-!&2MW)>kqy~McI{Z1QVCR~ zLbEvokY3@uWubww{O|k@=wS~dJMUmY%%@MC{Q1~dA$X>7ICIhd-q9qC?z!xHZ@W%WKAQ>9#3vqhtP}DO z=nn%j%E3Mz4E24FVYQmbA)YjiWB*EtNN5?o)SCfW>bU>@P#F$QLHSRx{60aBQjeiH z)Iu$N#W$6lM(4yf3O@xGs=jD+2}P!DD~Cm}ZX`L?RvUGu&4n&7zHo4mh^ZP94ogfm z48^8I9I&9;Vz|RE|th>Qa<@o2O66S{Pr4_FA0^es&*|Z{gn2$}H?52Nr%~yT!z6D#BsE zu!pNkI5`MEg-Qe0?g4Z$b}T`sl(YSguI*+59LwNz#D})aZ?2JKCw0E^QoL%cn&11@ z^3*&#vmdfdv=SL4713+3ReH52EQa(f*a`6<{=--GBG69caOKOyWyh|3PaQ%^ZR; zW71JWfGK#n#3ZApB{$?W?`Rdi#;kI>^>g?%2EWl&DZyVsQm6BCf%CA~*&3!k!<~Dz zO0yv5<@IIWanlm}BAnyZc;36GKTRkON%S?Iy4pN96P~Db&PJ-Pe56vqz6^gBN+8Bza zhkoRfeS1ZS1DpA!S%K%pQ(LTwCOg;W>w9=(HB+96wWtY$O3X2CZ+<5jD}LTyE=Re! z8keH8V2Qwuh_DmD+(#jO$5;Y zu&TkV)3Af_rR7|CsH{eX)1~0xd-9UcbF|-e=QjR|Q7CSY^sTzI-5SjfN_t4`U>USd37RHf+xwC9pcX-W$V$!AnoPnmCPR59ee}N& zrZic>CGV>3$K?Wdtm(i;-8BZuXoJ&6sIAAZFEWx%GKy3W@6AcNkfBT1ZlY);Y-4Qk zx!-LJeSve|jp__u4wXaJprT18y7u;Z>r>6~(!}jYk*)mfd&9z7u&U>x2{TwaHL%qh zc~8$qkMbW_l*Rm9A3GTv{%sZIGXF*j^Ek#LVFLCAiLwMHQmbOHWBB1H%43E(Pm%cE)RNmJl>_4O0RXXRd~kx{mUb& zV*FD2s^0|~hpj(04+u!q1LyX#!?nEWsPC}`5y~Y8gb92&8-HP!C*OBQEx}?F^`7M! z#l0ULmwQdU1Q#sxrnfeQzQqV2hKbLL0*}6_y)~ZVt&K*y6 z>)RJxh}WbTqxEs7@OoY*q4Z72H}c2!+KT}B+DMsj**Y=N&HUwd>)C9pTrn{)f>Q(iUat*2k_HoFMtPD$ACEqhypDqS~f zOhGzk^rWdX!CEk`6y{Q&%d^L990EHGZZQD3B3g*VC*wxRxzt*fKjRFk-sH$t4@BBl z=?hx421}H)l%UN6Lf!a5ELk$Uwt=IRBt$4eqEqj#g-qD>90gs2s`Q)!TD{W7EmoGWZ0=$Y=#3sm zP!b5`6tm{$Fl^MIvJbHd2y*VogNh4c@+A(`^sLGD-28Cbzqf%JXIG4x(YGOSeiz8* z@;QeQHNsd-N=}LwUU#sxS{J|{mBp_PUdg%_qd7Pz5ze%1PnCSlihbjbxCTtxtJM{y zY#@z#w1R}FyrXyQMf>ydixEekFS}N{tifmlfTa!4$i~pKDd9%pw}DGF`}HjlXgr|q z!Q;e9Q`E3t7*`RePSNn4z8mTs@E+PMNjH^V5|E?mdu41gx}P^_Gq268!Cyp*J1u&~ zx#H1E^e9_gR?DtT4Fdkta@Nt@t?!oqD)UAI@gSk4UQjdXfR*~@=5ZnS7EPaMgNqKg z{TW6JJAc!qhsl(Fae9mC_gHyN=B8xt{1*cm@UJi7D^Ro32BWxN9UZQpV2Q&KT^>(%tIdV9X=~3-gR#n5z#ukWWF*b9oNAk5V?#;`7_K&3L$~D#^ zv}NKqmLrKgm6(qOrw#HZDP&3TVt0}lbsxJA>L#~0@DoBrOLBJ zVSJFO%wr(Jd~Ko5^O%kIiU{^uL9T=6z(12f7-MZ4sPs~P|#A`^-IBquGZ6JI>YM|&hkhb{FW4ZR5LfW_|%<$*Hv zohwVV5dFUa-eG#bf13UtN`wX?17Ak5xAw zUk47?r(dWQMET0zLen&m%>e1^r&kyNEq9;Da4vpC==Gf9K?xp7L?kq$UXgog6;NQ~ zK!phDY5nx^O+Y{@ZTJ9ykHsQr#TTo~K!0D@AJZf7$yJq=Be8-%r9SsKdW&}Np}8a* zgRPNz$#MDQV=+I>Gi55fP8ZDj`H*uWDr~h1SkVr(;54DYIR>c#oyFn!M>x(2`MZ<{ zWB63h1A}0K{&3teB>yBbNa82!C7x4}a|ToN@Fvq9|G;I8<)zYC=p{E5O*}q3JZCXa zh4vgn?*j@Rx`hg8{Y+xMnGS}LV%PwYSI<=fHhEW3%tYa2{!_RGFGYHWd=~DK;9uR@ z%Mr5GZrLQngTF<-S-U=5S=VLCwMwK^NqV`gal1c*Sj7e7@YCxO@tclc^t~U(EZ_+G z#H(y`Blw@7P*vDIg7&TO!R$lx$fqr384ndw@1RmEK42STMVhNKY)_G&tVNxz>BR}Y z?to>~eVw4#YayDA%Uf$6g?iGK&_*1ZyN3nXbxeHi>+EU&w(2yW=5J3^$ZFudeVfBA ztI)D{bK`ZjktsQuYL@`$qEQK>v=?y9b#4o~Rxyir^C#=czw`C@Z3!elFx=5a9>;yW z`s{3D(v5Dz86tbkKX;mtdxG|9@`8)ZL}(aeD9Y~q*L$#<%}w8`LGeo)#{IRoyf`07 zBB+vvBz?B{1ywgX2{y)K7#v$y&>}{`e<074-!(PaTA$0|TQrM};dHJ-G|r#d#5U4v z!v#<=#7%j#qm!2sPb#Yow5X_Q)AUh)89fleSS(s6&I)3(EJA7xO*0=L3U#(Ri1W}6 zI%=G9Ow>cLaCc|GbXLEfcnE5<=&sr)3$zJ~Du^wYME09sB^bJr%Z1xBr8xNa(tmy~ z#R@$tghopAdA-6gvl><6*>z3P>L0rt?$laAdf^oOtda@P+Ff_m9pND(CBM`39~u6i z95g>Dg_=W^ZN%Y3<--sBDILFKAQrwaTPU7A0{QvYyq|n}OK?^N?yoVPhLU zZtE%b3KQrs8zT_u@)1CVrhf}1iYrVqtIe`Be%IjqsJ{WP(^oN)X4?>GP&IzZN{9xr z_jof`Kr-W;nXKmbr(CRBr4d{<2e&upZJ)&R{5?v5etF%v$#ply0w-D7CG~2|X$H}# zNXJ8&eUIDQR16{mH{r0GZsFxGldU*(Ya+inqkB7oSoC3?4uXsB*`%66tZ=`#1#Qr; zI~8|ipTy(dk8r2fg*T(XGe43g71kxZ)65x6{tyXVW@e<80U+ql@evD(57)0O{tsV9 zWxFHUG}|1RK>aU6^zd1sD@^W2k`h=6vTF*3+$>Yw%NmZ2P!wxMXnwrmmoHmVO_C;d zv|M4TzHRuSIhwiR(NMh~g301LC@;KI8pIs9l>IDD!=jzT=?&NMuu$}?h9$1dX4(WE zhkC!hlbs^buJxR&SF}U99ZHnHa(NWoX~#B<*LCS9b}o5noyto$I&;pLIt zJn82*%Sw~D%~;Pq!gE@XwXSSt?f%E<=;l5malQ zvG?!v^Jbq2FR$ower$n~o3?evgynK(5Pn4v-{mWTkIZW;K~)7V)_A9(($hbGvYhl# zn*Y_^y1!O{PXaeTCLw_>FOc(r8SN*}kodX5b3;!nid2v9$8HSm z(^y__O7!!RdpiDR^4lTYl&WMf)kSQXFvh!S1~xthcjY zY;Jh?TNu|#4m@-(kEli6qa#xHUv`)gLxu&3b_{h#Y9gq3p?3f6#IKM+YI$XJC zH2LhEwWx#zcmEb$my%q8P3~KMC^~wJ!h3>Jo}IGTsmAY9*b}=t zyM;epL@InqKVes~%}nm2A16H>XRY!)UN`H0-g>S+xvSe9?+;l9rJ55DaP`OY(Dfph zlSqrZwc3$bqWk&i)A!F>H67AD&6%=o;;uitJl5SOE*e_>OF66<<8BqOW+Df?1v&3y zhapo_>U;oN-zUX;xHqS8ndYme>(=*%Sjlxx7S$yEyvdEqt*z3gDLBAsDOc<~oKUxG z_XuNEtuHpWSYS8*YO=H73fy^m{HNbY|1&#JI_{%4cTU<%Pge_Hp`hq_Nc$DDowx>j8|Fzw)zP+E^R>Ld!az1bv?bykJxg53qxx`Rk zmQA~G8^-GvK|z69u32>P(1VXaP%|r%H>k#X^87n%I{w|H$4pM4Q&Z>%OT}DUOv2$( z%zN;pa&0j(23${a`^~(`EoNF7nXh=se?$+xeXFZ@PZdeyU9MPww!i8b&+_%QCG9$ytr=S>S59?J09{0DuEe2il4ZKqjH$fsh2DEx|C{G!UWRtU^+=!`gW zW#FuoxX0#Cv8Osg=KBa(oy{UQ0rnJr11{G@X9m4XoCvkSpM@^d+mZRD;;5#>TDQlW zY&H%C_A|@nUrl`B-pokaCoJM$j_v?0Mzg{~B`9=tAF%4YNm(MUJF=<5E>CZPRz0t8 zoz=j+4Q8?vw~L^wq$g`Ce^mt38B$&}h$e(D;4$GPh`9 zSGfsm1uBmCWjH;jEQNi>#n^ycl#JxIZ!uh z;ZZqCvbq`i5ctO(B$~+=xjMJiAmqys2RA+=8B`yaXpE`(ETdnle~*o@e8 zyL95Q?5K71^F1r)t%6y6*CK5&U-I2I<}*6_!(zdqiNwr;>Gl*?+&;vGn*%WAd%sux z+a6kh-{m}?@v5#b&wn&xN&POGnF}TS^fzX`1J5Wi%aoAX_BKb_1H&M?@9eP+W!rT? z@;Iskt*}!jy^79ewlieBVAH!l8w}T@2wiZGk=#gU%LHQScIZN8~j z78>}p9VI@q1f!&KRV&8lZ=j~FP!=~qL9Rri(_ioqA{&5y=DQu?bXOGux;3mbSum78 zsjbmCn*%E8i%&QCIz=ea#9yR-q^a#N_FjY+Kn6B5gJ_--q!tL{NgP*OjlawSfkzfb zLD9h(P~7fBz2jPj#;C6*%>DJ#S%`p9@9{Xah?kE~*tIGw4;f7gg%-;QC)^4DrYX4b zf2j^tlg4$&w^^W@*9u9I+mpi17OuHeG#Gm=r$Q@dT{C>RM5VfDLNzLVkE&ulw#Z|D z7U#0c4RksgdA!h-nQB;IF8N-!&mSAN8C-8k_vA_Uyrjt8Q#LkLx;}RcQ zCs|%-zo_}Ujltc)#*Z|21}$%VzBk@qC-ER;N(xn3XjFa5&dyG-;KGyAiH&e@SDiFV zi%VqMQQ&Fma948bR(7F_F%sqPfuy!a5GiNV&Qn1{&iE-6)ejY+w|`gMuE~7ww0Cz% z`J+KtSTeOADXIpjSJ>!jXKH7`8bL@yl`&f%)h+fN*4+N>{QR=n<5*FZbe*VtTVC&Ig}$oCrPrxF|!-=?Bq5FFVFfh zqoBk&?lRaUIK5AlQ0@*@(`Cjg05QT!^*Q_5-rn76b|vrk7X;#b z@`*q0|JVtWp<<10w$7hbE&lp1NDa<<-98>18e*uclX#E#zW(HA?(R)XFu=mn`J&Iq zKRiAAZ}9^(&o(!gRXDUk4Nv15U^!^JBB#TRB4+95!^p_G^cOhskQyf z+m@rk(uJ6tv^%9_c9v$`x%i`X1I z3ySlRvdKx790qYc`p}rqU5y&?e>~skCa{PY6QM;(X#b^P@1Mn)U49NMh`eBUhAREJ zk8mo$?VP7{-|@8E2#x#rJVm;}uN(Iow-q6|<+?eQHgboO-D66~|6ltmBUaC+)Hy3E z`e=w(+sK0fHekk77;&isbI+~qy3|v!WyN~qp~w%XBb~Q$+nY~H^FERz8p&;MVp!Sg zDJ4j}1hGq)!;DF9Vfo6b!p=Xa z-GoeeB7=6{?xru<`lU{mrQ>FFJc5gSN%Z>iJmnlc=MbE)lCv@BR5ayO78`7WhB`4xkti!V=2m-ex`R3=E*` zZH>j-qZAnmTuOO+3%djnC3@xnE88KI4!iq@m?+A9kJsdg8}EzYbIhr zX)Py@R8sUKTKOiZ2Ai`0x*HcIIR(1nK~sf&zsqGQ74D|0~o z`)}X(meT4$IAKW)v22G{G_4MT^q~QVZ zN(a~Drt&m^m9}jp%zd%w;Df`^vkg_iz$lqyE@XaopeQqLd%oa(Jorh4Ba=ck0_i@x zZ28+PN=zC!7BWDq*7!Gm>z9;!+~|H?;Jl3TPaCXp%N#MhqP1^1)%u#`-WB}du5CJ4 zq{4Pi;Lh<{0Ax%9HfNEx&kk!BMug%F&E^i6pTL&mpadXT5@=WOryn84H_^qYrCV=s z$lA)QUFxvnbX+~HR%+e`3OJ7UtuT61m&{g6Gz#+)<4B-+`F9nE551R?lW|OfFJ7Lao3; z$t69WOK?UyoGV{q0kjaVe8=08jg&LwOg!p?srz;7>9V$|{t^meWU_c_O4aUx-&s1|I&DZ>64pzE|o zE8%Rr`oJUww84mHau8YshEThnHUM^tU$kB!aJc5%^)VFP!OmKPNM`@x&~8KQSJbXb zuB@ThRAFKjr{h2RJbc>y9=>@j-;4Cpwj0l{49{a)VHwlKy1Y)$t(IC_deQNhNew1C zOB&;2%3yKVfoYxNeumz~)o{TR;M&TQZAT{JR7j_ROX<7Fu2yU z8^M22CzuX7sts}I2fOa^<$o~e|4+Lwuk(Lwq6%E7oXB zc3dM(3Gy-CL~+tf711xfl6sXg>hzyBf^V-j?gzM6u{?QI79rI)Zd{FcJmLp#;0!*l zs{BL>GZ?GWn4RBNik}-VFRxNvE2JS{HTDT2e70A=rpPGCO|F+qZ~Id3^1H9dIBKTC zj2h|IE{yxxD7=J7S*V>#n@s-)l@sopd0mEhGaQLA;BI`_VDWJ^gh}zU4tKYn`NhEs zV?GRpCzxd#7C^Z{Y2Q4m^ja>iRwey8AJ{LzL07GCg|kQe9p38rI9qLH1AVqr6LM?| z$JTJN&5>v@ji|j+g}H+j(@u{3hRJV#G&J<@Kv-qAt$xm5N3!LlcWGG3)H~scB52mc z>oF*Af1#qT56>&6a(#bj!wa9mN;;@=(A;_oC)u=$xy49iB?PTg`IB6+%>DtGhF$N; zn;=cKowu8gwv)?CZJK6uMgkWjnXkatTvbT&X*h{=Q7QoGqxzQqRMx3J=n%-J*@&{W)Z0>zzB$HL8uYm zjHq8+M|EB8ReuIqQGIf}PYc-L?wT=My(LgPwb)V++{lR8#CNUIzyT zVzJM+gyA<|#y{0e`l!ve<}BGe0V>xD-G>d`Y(D}<5$X%|1?-tWU% z!Zk3pUBO}nY{(k-f7KBdYKCzzuS{y=Xlw`!l-61&^g(Gewe9w)Tfm}3_!?w`<{cSj zP$&|REDG3OC0!D}7H?G2gXc=Y0iF>(RjN6R@$t_}npfp!IPc4I-s^n*=}dkI5Ad+U zUWG$>4NjBiXrdkSZpJhVK5oqMGS;u}zH$TN#Ae?>n@6r-c4W53T3d0eitXV*f_?Mh z=cjUDx%KdG^#?v1i>Kal%S@UUn!e2tln+u!fmsN=k>bJd#%Ue^eL~GNF8kwv#bv5A z@tvlbf^Jf0FpZe!K}hZN-yBux*O36q8BhYUpj3;K*S&cHyJ5JTM&R=iA@K$BMfKV0YPEYaNDnLK8%gY*zGb&1X&+Rd`B5`m1 z{=hS>b|FZ1imTlx@)e%z@&)AY-H8IC1~J@>A2r?#n%iaa#vfP~Kl%6ij^)Vp zb!<(Q^fRTD@!1~4bLf*)Da9t;gs7e}ee>WTpJFU_93j531Ml}I(YUZM@jq2^p+3T&ILHwXNSV0SqA`HD7@rOuoaeITb_>Ep zBQ}#55>tSpC`W{~X`WH;*l{t4i!a%AipFUzcd9yrWw9PTSU(%E^304yLRnF<2)}>SioNpil}A`|znP z_oKuM1vTJSguA?=aQkaX>q@&}aSF(fw9@j6FJ#1R+M|zpNuuDfnd?2bTGp&)zd6e) zEAt-DJqPJ4LUVy-w{1_}jIJlLexUZMZ}RJ-DUf{mn=aXUV|0 z#?p^zf=)RG(1@tbZ_{}0q=AJ3y4f%SX3C-`gvxM}e)Rh<3Au$jZN+A6SK>+i?g?g5 z=#M@HZSuF*k0tJ0Zj64?@1OYPspl!j0kXYNX*&jgIq)S4>>_@O?MGnft(Xh64+t38 zhrZ|}dA=gNCdxoFhPdl21}QXagFC?d%DS#z0}Ijg3%7isR=-d0moy$1vY5O`NYmuD+p!tcS$OhsiPmUqOpkjf1t_=8wzM4hDjF3RDSc(&izJ6 zy;yC9?S-Z8c9@qQLx9r%Pr@fRmW7hUp$sUmO-qr*#cjx9TJQT?JRDkaf|k)8wdTe0 zZd!KP(iL-RAld;Q(3K zloC9@@PlH$zCBoOR}6VxN0uEEJH&~*BMg)uI%;~<@hKF*h$U`$bM@DB4Zy@UPVCa1 z8YEE_{2#B}zxpU{O#q%xg;HRq*}L|w+8_)>QV=FBu>bt}j|m?!QA*vYi%C*v!q^7_ zpTe#|A9EWSCx-vE0{;-bL%GK1uiOwA=MC8yYs?05Ej;mus5t!G0Q277O?=6Td>_gn zi=x$`EZNr1NHTFoG-Xkfu|;d0l1KfOoFt*9@xserHzrP;vJC?Xzm}ftQ1&HAp5OfW zE}eguWt$??)CdE00jzp?3S2D{wv$qu1cyCs*j>i6ws(;##G5P^V6D8}`T5W5sF!J6 z_g$(K9D$nw1j`A_yQ{!b#qN3!R=ipmKyN~XX;zHQPBCvcd2YF0qf??}S@hoL$lBKb z-IqDLrojoZw67Ze+fPE?-vA_bKj#+%>%b5{YJ5#c`#-^VM7@3Vp_TIU)nu37I7VT)3@3kSnX0I zo_tbJzygNDe>4ZLjG59d1Rb7{xF{u>47cJO*fo5pC=w^}88ZjCOc8>6I%if9FbLVO z=3Y-*m!=v-ilxcXFN4Lfll|zJ8l^;EBcB>}qQ)6T$tlHubyE`I{Ms2N#z8NQ%{fwb=VO3 z-P-WF933h!ZfnH~WtIQ6+z-mUIGiFhv zpf({NM2(7Czhp6 zI%eM$lV0B0;dEm!Pc}O#q1)&FKz>q8sUmh7Yu#hGEfHb?R3%CPW+$~3v0f-#`uq3q z|k z9~Q-Vl%_v{$?ngpAJt+dR(F7F-$1o^0c{FqtvXPm-o0JiYgLg&uLwo%U4ToYTjNhr zYX05>+cx6qZ?pnJQ+8Mr5r5BO)s8=&-;wv)wqGRHu$eH>ui$1hnh2sx+{BZGt zjg?%?pv#MdwztDp&IyP=FHl_&pYbw0`C2)=S*YmtXPh9;aa5YDd5GHGrLr=yQX4-j zmU(HM6=(VZRvE1Ppzzb6%yW%Bf@rl#MBPeYvH)vU&SXJe`1pgvX%G_Z8zq%iX*pGl zwnQJ$`sm#Q%fEW<^{$EG z`HXZc%Lpr5OXMPFL~RS-kFPwUfd0sKgLB?H+6RE=uE&j_5S87an+)so00^lgQP@7i z6iCseES^D4t5x!6Q0N29j*8l8fakRv4GJM=Dpm`m%J2R4t>u7#;&$}zzLE<31V*r6 z5SKts7zq)xWha6ZX&ZkB%7_MpqUrCSm1uNm5>Lzg-V035NuO$$|3N@Xkc4aoHsl)s zQ>~4=BJ$`Ve;3@#JuOdpW4g+X6k>jaHM6ckN01W~W+69KqR)gOf`3eyvQs}s#;7sK z1lH`(P!GY8Umy~R>!4=lUGc_O)3{CARx1gNZ?Ed1 z@d;6F+WXS8K4tJ{1P4=|Rpk#YbG3M$V&9M-8M;JSUY67sYJjsmXX!$mFJ~T*Z8uq^ zMOXI!Wbti?zO%hNo32(Mdheh*$JDKybIEx>zLZ^P-=acH>m5_ra{K@oOf+<(K=7Xh zCd&+N|MjZ6xk2@V2Ap;E_D1n8Q#DrY%pE(iZXs$NVLpeQinFUP?~nA99uNc9UMut!*xfeWcU!`@8&Hhy?A-&^0S3_DQ7dN=xv}L`kuR z44@!C;8nlRZQWRAyb7Y_6Do)$c(8CyScufOd-mVr+Uh&?I_rd86 zU`EgYerGR-XEw7i_=hBP#*gQ(2@5m?HLWue5ffAID&8_|2Zbt;B+nm452cEPO?C`& z>Y%b7fW+6+?dh;d5TGQ3>qc}jB(i_dkNI~upt@rFyG^RVA^-jU3XC59@-jKX?Fx+L zVH~dJ40NMNG9G^opSs#Q1=63Vj|8$vIrQEt?fpB46NswXaKiFwZ0wA^4>4e})GDxh zGH4clc)-CtGm}ti-u7_|@42QgII_p$5*=Uw)--pbx&cvw-*)4keuDeu8}MZh$YN{< z0dC$Z>2N5oF|S#th=kKXmNg@u524u@-9t9k)8C_7!%MRg@|_Y|0dg#=ENp{)OIV8W z$G^G%CH~BOg(=Lw{4T~?E5eGe5D+wvY@LXp=-(h7KUQ}>+}iw+98D%r4_#+Zsyv=q zhGGg6B~niaFjX|t2!S~|A-L>0zOd9|6VpJSyUD{8a)u_eeJs$b+UQxAjl)Pd@&Q(? zMsmj|+*{Ph`U0HVpP{cSQn2NB^K*7OHrZA|toy{;29IVn-1h>Og>`>_B2o?x))c)M zdF7CFaCCSVF^DO3LXG=#?6l*NR$K&4zN~qhl#{Pt%Vm`K+Yzk{qc-tu!#0?E!aCv1 zKd4ZK4KVeprE@Db2bSAYe`|pHym?e&O82T5*kmipJ8bo>ZgEt?9F@_=qb%P(-TXdT zx=rs)k~f_NdSBMhwX!ND^}=2dOaU+*UE+m{RlPAITqW%aD`9DboI7 zVIc1!MX6Qq!h6^+Zq8%nvdsbjhAaM&jP^D&$I+~pA7#o~h!M~3E8&yJD9=QV+y%ZYrz5zRW_kseE_nBU*1Y&}XXkb=LUOeY za z4DM~6gTWe$GedI~T{NSwsPpS9IFPbJoj)tw|MJwL=tM0B$PQob%+`b<2x!=Qbm8{( zD+uf8zm$7}0PQCvX8wJ?aC}w|343>8zHv%xzHgA^;J>+1>ul4R|9EOay;RKPH7s@X+ z#t_}_pSS;u+Rh^sJeJG)Nmls0n@v=Nta;CZdN{k5-FRqH|peJ2J`ouUR$d2Ybl{!xJ#v@aJvZTct_)g49} z#v^1&kG1XzxiGw-b#!N?Xn({*+;~i zLeb>IqoeoyD#5Tr5+=0Ye0*T)d?x z_U})2H3-Aa|M$u3wQw=y*V#>KOr`l#xayAD5vP!qQ^>{^T*@tGf44<1d~t}`ccCMd zVMrd49Ne9^+`x#CD4gFlwAfRSzzYw zO13UatCvB*-A5B0*LGm}Mgs0LQMHj`1r;-pa1zVX6K0Hq-AqwOT>e}BIuWu)Bljw#e`G*rQ_8B-@Xu zk4+nCUN&m?8_1_?F&G@_XQDB2$>`3OZ!Y(lireil=$@8;l^6IA$gJ;o=oNmOzEgV# zPv@XubRwVM(6F00>;mbGL-g8y zc~oh`It7ndVkr!YkaTeW)yG!$5Sh+Q;J6~m{~Q%hh4mk`O*K#i$ko61iAtEVPf3o^)&~I5A~R72IOo9cO394ty8wirusJhr7goI+wizk@6W+z`6V~@Ra2Z~x zQ@xq}Pi#VR{?E=+)2Z0vokZ?OH0~Z*I<#n2UIx+y*wQBEjMT4J`BaPI#VDtzN;17* zA%&kh9H2$vQ=*#9M#Ewxkuq~@%|#y@##QN3I^Mq*0b>P5VwYa++hNW(e+zbcOWo63 z@}Gz=EyFX;mft}^Y??=PZAK?b37P!SI~10SqHz~4rE1SF2hyLZ{Di18?=JWPM4AU< z!a01^%{YoGse2Limhh#DNFlJp9!S7tsN>|%7v6~*t>l#Z-Wn{jDWtvAAkfGT19Q4O zT;TgVjo*yU79s7JcnCAq9Lly)SF+_t^#wK90`bX%3}j51>VN?Mursz@|7+auy-g;Q$@5r=b=MYj6rtG+7Vl!K&*e_& z5j{TT=vO8c62Y9;uo;(BC~m)TyY#-QN(S=Ih=xVz2zN(r{|0L2QJ@wfW&S?HBq1pS zBw!Wg?$?l?yy*|L;C5lA1Qzwr&_#p>YBBFa&=oLlKbYrhq85aD7d~F@ntlby?e;w# z|G%_!aJl98^LS*WCh-7;*2I$ivLzeOJ?ZTae=;^pQfdrVJHZ!E)_z3QDTO-ccZTHK z;6jS?*eG%6T+SCzXK8b*gpXU~#X*zd2ZPxlFO6>O5CP=m@Srfpa&ezsBS=ZS8$Y`L zg?Qb9t&NL>`zQEtJneodMsRJ6`-Kui3}SN72hn9u-}A=HCR6>7NwBTB0dt_>m8^Nu%< zn9$0osD+&~K-;2-T&ZGqCG7tQ`|7ADytUn7fT2OUL!_h|qzx1)Nu?b?O1ev6Xh}gp zK)MA<=^k1-r5gc}Zjibgf9I@wzO(N7?qd0eh08rV-uHQ*Kzf9E_HF3Veu112;fu&$ zL6CITFR|7Heh9Umfis6!&Qg;!bL)jYs@zg{?>S@XE^HcIyd2mU`KHOH5nb@6;#dL- zQEcZn`BA-m9J`V5WVfi?>QmlpK5yxZEdf62P9m#SJHJ}8s(dCbX$_>ErAMJa=`pVl zQ>I&)i@&v=GaMSsyf@IcZI}v7Y@^Ybg7Np??E_LIYo?fU((OdeN>F7fXof9N+mxB_ zWH&42@r8Oz=dqeoU#nVxYvq?hyAlEF?@Db4KH86Kamc@G^YiETt=>ehIGN{W$Sna|=PG_9_R5#m&JB?v^{~a|TJNjaOqacTcjkOjP)(lw+Y6vs+Qy;$ zo7$PzS4nvK7el&tC>wuJ3Ep_9sF@S0=aTRWBvOq*gqxPy0F3ix&FTHP%$AT!u^x7c z(@67Sa%Ja&w}2b{{8Xy1P|0->I9^d&RFmIwqIT-ShJ)-sjw&k{i?fR>S8l$ogfWhS zkk35z&GG!IL~<_n>}PcQ>y#~3cg55!8J-KmeV#EtXPp%;cdD7LRp;QIUu@rbTUPr?{c&?Bk$U8f1MRe-BJLuuY ztjiN;t!@~YuZc>Z=j&BqCAuG|AX@n5!qUCemJSZX`nyhtr z^lP|{O5=#y7!^YR^t`V~Os~EHeI4-`3c`W+kXCC~J~Y{FIfPMH;Lj;AWgh{C2ulfN zM4QYz;(564En=0Uc|Dyn7sx7U<4)X|ZSt+pHLtcQ*6rTI79Va5mYnw&mqfScA2|YZ ziG=W&A*`LElBPfMZx$o72khLD31OA;M3V-9Mf=Uf{HYY=rVJmx6PM6{f~J8I!i&!g z;*=WpHcJK)I_>Om7E3^%DlYaus?b<&L{rcZk6Jhz#M!;%gkp>gNE|ATKE)87P=;q! zSWT%t$ddr}jG0|HKcltotCEj ze<>R!KQ+Mo?G9$NeJ)=69ppY3*N4&{kFf?h3p`;MTInxB8k@B`ZZ6LNN~BbJ0j|G$ zvPq6avk#}RtRS$W+b<38uHZ^VC_=&JG$}3c4tAS*#n_iuQ4Kj54F1&vQE@ zUhwy1Ok_Pf4^>Sr1i3OF@8oxYG7J1ti{0s4#&M*0iMp?rviqa6Rg31}%bk^h6X75t z^Ntnu(KBZm$R^kLA!~_%nQne zH7W8>nAB_B?`_xo=I^=LB$#n%2o*e|HJd-kF|xmJG4G%6ur=vgZ}7z8M?skNuy2k6 zDu7Wh4HW0rB|XPvWSoo7+DRsPZ!Mr??{84RogG^>;9Sy!IFO3^pabWo)%A%nRf z{-fLA{WeF_X0gOVD&dxHrZGTaWl$TQ#|k$#<+|UfsCOUrMD^eW@Cn3! zJ;-|AhpSR$y`seCuaBh0OD| z{tt~{wsN0M0Q%m7AT$Q-CRjO)vhNM+r5Il9jjRQZd2ASZsxab@ju8#nitc(;pVyAs2t?IM4}~iG3i<$HJRSaR=_S zGdl&ZKCRj|;m!YA#qU{XKWpDL^{E?9=|2^U=Tc>ht7}83b-zo#SO10 zIgK?+4D>naEsKc_ktqq%LsFZpsQR%T1;w;I@WnFz*M*FLS;&48kyu3gsvGWAhWph3 z+Ds|4QL|-EWK`60MfzppPsl~eL!GsT-gF-0&1f!-%sk2 zVOjy2-s++x-&z>z6FjLgO%41`5i5QefLs|mODgBiB7|OIKTBLft4fKD*WPP0!J{tG znV>|{3fpoL@gfbjEYokmcNf@GL@r8?3j6LXc%AB_RuOKwPTupa7=ID&DQ>u!%Fol| z=hd<(YYhHj7`+F7SZhO2S-IQ3k$>W11PqVry$M|J@F5Z34rmxEEMKb9j0;$%0l^T# zqPBla-I|y0dMM~esQPR;*E$YyQ!NWuk)N=CTE-WrSBP@-&9(TNn zd8=akIfNXb#-L96>sIydq`R}F1IKti5tY8N#Hy{TmuIiy`V!BFQdjxG_~ghp)n5zU zT@>e=;~qL`+?=&Sw>hsRffpUkpSW(k&C+lsXNBIuf4SYP(@nbflXedUsx5SyrS;=c z25-;M^X@hL4&Iu1WiDu$+eEK%8j`fft&hLw9yximUe@*K*vxvy4}Bn2;cBS5__1ZW z;9GcScteys_nB1VfxPX!YwXjx-%^%%Orfx*KkOeCws(Je&fQ3liF>-(WPW*6{n(O$ z5igvUdOkO>KQ)P(I!P?^=_YU!1(q764NMVk=!>$!W%o%Y8L&^g`z)I_Fc)R5#DXki zX537zmY)yLLm+OrNxRBDFgX->j2%Qc7F~u)SUBDj?4Th)W<1UoaqqqzDWMH>(U~AIg$U!_rY^Xx9}s z@7ql5^2v}1z9D)*>Xj2 zN{gUv4$xMROe40VANVZ9VMPv6!_Etnkr;pGVn!z{xyERkBM1!3dF-vlfXirG)mK3j zH23w4yD0dK;~35mZZ2?ZaGQF(v;V+l=ufpP74ag`Ip@7W=?pU4(}2+R+y-g-vgya z{dx#S#3S!`N}T(XSvSxn) zi*Al!{bJs@-^=Te6IFa#C0yfw=o}uMoes# zq^5L_z9CG^w&{Vp)#=9vP^XfHAsKJ)yTpC8chYb`o|O2G=ucVaI@}^(q9R|lO;7Y< zy%g;7SurIl=Q?as_JraY3_L0lXKi1oviP>3k&ANfe6_1M`e|dI+~#*l$pz910MHo? z*``zN$0P<}+E^@|B>Q$7QPs?ksT~MGM1Hc9ua3{R6^YCaOg{&b5+duOu*tw|vpdp! zqi^KgEb!p)iVaw5i*IUvIED_}}xJ5#@-MaoE6i%QyCVrl9J#{=LH86v5p zlI_Aj&&3<&I@WJ!D=cU>K-22{g2`gGrtE<7*;})k$|M5vq*EOFqquL+NP^r$35p5J zd5BXzD{kvLrfQfP$E%#4mDd?#>?c*K17puCvUbs9>yLC_i1)Fgm`~unQf;x74qMHQ z<$7k~;dWYh>y`w((LZUCLwm2Ec@JjyrQefpfNdE#=Q2gF&-vkP#!avnMV7lt^+25h zc&J2|-)p>`6?GcAI&F3baqrhPaV0Og0?D}ha&dSnTpf3x-gO(RIUI2EewNlTc$jhJ zsQ!t4`-`f3`-hV%e$((P@6>;hrWZqC_|g)AIu$Cr&#kAQTuQYxOUk_v%fY#&Hn5E# z6?j%hCSH@1T%8;$Bnevpox;fl%)9E@M2^s%eAvi{8pTZ1^M(#n4&$yhptIkvU9h6g zy;lh%l0$v_Rlt{uGAtKiuRR~CvTN>*+yfm_$=nIP&QM*Mjw zJ6X^LnheDB4U?%a2iZE9-2*2AOZ6Y{6YMYnq9sIUE5pUjs&7@}kXnvM^lhFGCk zk>^9+qeap6KruzAHb(P4V9ncMyH$^O6p!E9`Z}RAR(^{9)tu-Ro8T3Fo5EAfp*-`U zjTzWgGng7MoB9cq8!DnTVZ@{IV#_osqn)5bmX!mvx09JT_$=&7An-O$zsNs7tG$7h zn~5JC20jhNk4Y$-#r|?fdtXfCpO7bjN>GC(7rv%~g73?_$!*x+vNc)u#pXxDPm%-! z4Hc{#zkMvb#lA!VEJurMKj+KbBpd!N#Aq9=W+%bePAf2x`(DxK$s|~Rig@*8h6Igr z6vSGz9jeGAiZVQZvQN1B~`bJ__*)W}jfVM7ktk%aK{@X*Rh9pw<+Kv3US|*AxHkQ~I*& z709DK3G(L)1i;Cu4q&xPh`JtcS2gaPa{z;ONPuwnRJrWi*a8Df#UU@IbK{hCkl=K; z;julV^0-ZdE3*9V{HrGg1lt|Ez>J37#h7F7hGOi#_AN+Sx+1S zR7V^`UtT=;Bx{y=m|j`nz`F#L+o@0 zHn_^&zHD=HN^WR5$#&kUT)@88(D+bX$8SVsx z#bNOI$bq&jmCK(%co?Vvh-HMtBeM!uDAd@(j*E!hXnB5oN>qS?*a1+I88mFt>)VO} zlw_#NS~sT58WjI%h^)9;C3IzIZ&DhR0qHm>nO8`Jpv!|(T7C4F+y$~uCdeZ5UPJ+7 zHaZU58Vaw>Q&f4CAB16cq+z1ru|TR7M?4#H+U-NgScxp~KgOTcIQb zRrcm1wn>EhLBERyw-fQRcUz5n_1STF&UHbl^lLIUGIxk|jj?p+WxiUD*2^DMAyuvT zqiTAnkbEP2Sj;`EiNyz*ZavG*cDtw*5yw9PdhG1Qi)M)}gMdUsoi;U)bhv9OyspqY zW+}vk1mZpHdukAwM)ijm5Swf8YW20Tvk#ftxtHE7r9do4fh7+)&1&~4#F2e%-ErUn zYr7QgE3DKqyJi$NociZtRuskIi*ArO+op3kiF-{9N#B{i^iGd`~2Am{xC&pG``P zqhXNu7dRcCj*fQvMl4|_OQI>q9*+&fc83b96 z<4RGm3d*ZW3>>|7)}#{HGuan-DlBCETcRXW&cseR^5NCR`nL%g6C!VJW+@7)2g3n% zZ)me_xzHi70m*nwXsVEIQg8p{P=-nu{I(3X`yYKbovhhr)MuQDRrUE+(g_ z`QFxXzqCZYL~RSsQ@=PlS@Bqv_dn%7wCWuO=RHRo^?rl>=1os5f@-18`774g zo^jnTMjfFnU%)_v*NnM!ZG+{c10}v#`uZ$u>S=}JV5;QWQ+nZ2DxtUSF-D$Z058-> zg>7+T3EwTt50Wl*TXvi%)@ONa=qxpYTS_eho0Rzw)G|_xCszFCEqt!dd5y(&?`*ok zz5GEgU)3Tv1@#tW#?ZgILIF$`ySHGA)JFmCI6jayLEU2rvHl`Wy0u40v%+HR`Nc!a8p*>YtWMN9@$_IBvULe>F^y46*0DWAWkTEo}`?i+R-nTES4>0TcvcSziIGi(n5X+-!O;h2 zYGL%$CEiba)gW=RoEGJR>uhv;NkOyGJBWaS7co_wOY)V-lG@5ETwVGn5Us4;9SOuC zC!3NV5<=twL?TU8&$m`!+^-TfSOJxZ-n zuV7Zxn}!N`UIG1RCCmw@c2~R$%hQj{d^g`;pgR_OpDt>A6-ybHIk04~?U$bxT4raT zZIueum+K7azyI|yYmeAh;+%=AQ*HJdO#k)G2KuHf5J%$BzNouDydOK)q;5y3>NS`s z9C-4t2R0Ujm%J;Rk9UAbJMDd(99HQnuOH`~T=3LEq&wVAS;}eGck;j`j=lULZ z>4&(oLGZ=d`4F#fIneo)>Zd*edKF%*M^w`HvUxG5(%ko_Egcgt+pZaoqwa1fVyB}t z(BY!!2i&Sd+1Pl`<~mWlpK6i=(X#O1{gZTNXPdvrd9fPieO-(AEI6g$ZjU64q>AfM z%rUv#`BwbG5=o9%sIU9|CjrJ(ePf*%&9!bRL)dB67Qy!~3PXG#nB)~I6;jusq-ZDc zmZ~r=Gh&hcs>FZ=Roku&MWqu3R!R*zPqrc|5dFfV7-f)t^V;XC^7~u*_Y=U2>fc}9 zx0+N`77F*0)gnDW@mG9lEt-OHcODTFrENA!Kf;%aqHdK@4UGU{-)Sf>)wM-F>txQsTLhCtL6*q}p?cUkr z1&=k&c|C!|@c!*8TkfqWH zqgB~a!cb zNR9;W*9ttuf!Rs?b`Ai@8MQD_89#(CR?2^%lN5BDC?L72AG3hqwzVviXBpb1{G&bo z`+*@eQEyeRX7 zZl1rMruBAnutJa(-YDc>SHPaJ0hX1~+jRYU!4tWx`L_*!e(QY}`{5I%$Y_DXiA2Si zMx`8@@1pA+Tbz4RXQx`l%-Xf+ERrVXD)rq!7N1{B5f2jajXg!Olj_&}-1VJS%MyzV z_Re@2@)DuNL>L+8_b?o{>X$_%9*6x7R1KR`abxJr1uz(%d>*&{*dWaO)V7&Y(BTaY zTPd-oMV-D5>Ww+_dSLkN$iKE8kYYba1v#U}I;MdG32L^0HZUr5GzkJPwqJC7NYNqU?#MS;% z%OvqZgsxr-1hCJa!z+|{KG8#_ljC2rk&#_yZOQB=c>DKuoB|DI;@mJD+S~ptAJ$0N zcyVR3%kl4X$JbB3M*AIa#>xWOkD3q`t^IBzWzPvI8PV}>ngZ}>QNFX=83A+N_IQ7? z`>No3c+H}pVBMvIXH>cumY6W1AVP?)a10fLZuztt6Lv(I>i7Qa9s6^B2_z(wEumYE7ZUmftsC<_ zlWRPl!019_zsP1l!A|O|q~m6?q`1Dm{?fz`%^&(Vb+Uskz$7(!5J9=4P~&dCFps~U z#`7WY&w3okK1;bVR1rCbHfw`U=fNhu=C0VXb9HrEzT}6a^_kT1EE%qcshIK^X3}HM zDrpi*EOnDiE@Gj+T;BZ@;TiI#enroK1&Nk`c#fr*B+SKN2A;w$uE}Y%vvLe#ITmQw z2l~@8<+(;rSG}BG`lFb9d^_z#2r(RL%CrYi?lw^2qJA3(Q$RU|m7;s7$WjdOdx{9G zCjp4YC(lK9H{usBPo;yyb!|}XL>Y-}Wti^gs~qH$XS6qdKugA+$g(?jVEBq;jCa^r zjdJOT%TzbKASGijwM~O236~+6tP<=);e&MBp0BckWYHRe3FNVIHH>i@)I?dNhKo~B z>ztyiyO<1Uc+>B6Mk;YD=;poPup3o6e>?Nl`N9s5DQd_!r4oxqvOuOt=Ug;R6(V3- zV^>YJa8Rhd613Z7oDs@HYDtrmJDqO!VtaR=nDv{?43%q)V`3kgX+L`am#)zQK44bB z0KaxFi0RO4=^HDZZ%^Dduc6=dWf5i-{N|ZxlH+Z%AIq1683*r)QMx z{?nG#{?D8dLY_ljBaTa{la&gpT#%2X1}@`FdxwM5F#yW_{2KYSMsHEdLCXV`=7*)+ z;69ckF%Ywp^7B_5p0yLp?tQ=KD_&~x+N8gn^#wa{fI>g(*S)A^_CF9X%Y7V(z2@Ux zypbAi3l>^WE%8T zZ?ThcS8-?gma-26kmnpL?;^3ZVr6#c?}Ma-X|MH+$JnzzJ`N4kT1UbTRmO@4^aeiI zKsO)C-@VD>3z)j-T)en@$L(1}%<`C5sL*d#n8dddLLE>V|k5{kt z-}U|974=5~Xk%%+dI4YV518f$0VyfQ?cV;o-e}JdEaX#^_ZGkCp$|M(s9lx5fXPe! z!Oa%7f!3IWl!Y}uUSf=`_%tDtUuOGo%3XosI8&iv@adN8(wc3{`6!v;yZd-NK^0$% zNmIiD)(2A7pD6$n>nrX=O&CO!*>U}gQH+^+%T*euz|#MH;vgSYz}EF{Ak+QH%B68LNW zzNi?P=*`IIT|iJMxaPi$_V=;~k%M2lr8Z!mG)*hS1r-`xrEeXbrnmNRshCcSs~EM* z$<^24P1^aEexDlT$xG!q|Jj@|`^{Npttc^Sc;{(pMuLdW&Q{_5kuT+$=3encb7d>D z0mtJ-Hf)^X+dnHShTgUEqWp<`daAru@UpVSeZ4zS_S>e>_O6(dO6nUOs!0ybPigA{ zxvYDO{`YGr^qcWgq~_35#7L2e0wEJ*S7%w&pG5|P`;(4)$20tCQ+*6xA6fmO9;D=C z8LBO>Lc~8w1s1+54MuRX3qGra6UMR-;uNf@V(l$qdzVBRi0{2;vuBd!3`ptEbVu?J zYz>5CxxPwshnh->+OXVt0bT8x8p3>kZS(weRvoh|a^pkFiZ+>Y5>~q(0uT5p$K1gg zNrwGh$>&%kq@)i48j=Ctz@hYqsBD0)4w#qdfC?jinPC4^F0gQSXSSA@U9*5mVlY*a zffDoQ+FC21zTyI9oj>sQXal$+u1Qbq^Po&e%D>C6$S{CL#$$jB0t*p;jyJ!qh_DXu z0~%U&DRB2lFH8BZ4?^VkfG2#YavA-`} zX<5pF;#YG-;h`7}ytfbP8S;^@KgMyw*6!N`SF0ecGqp_OxdQU3Xa2KpJUTtdXm0J; zq==2;GvCCj?`-l@i{_7+N{xf5yG#$PeZ*ZV+D3BL7E(ogx*wfg1hNK_&{S?dnsSZD z8Z66m#-tHl49ZnHTpE-_*-pj~)C^mvxvwU{^vj|1&8J8zRpq^jlj56Vw> z<~PZLP)Uf{*i`JUS2%zEB=JPQWYeb_!nS33WA?0MkU%b0lJU{vZZqi@J?)F@rb`EQ ziEB}u!+X%=dxlU#MqVo|I>l5yij2u{*~318jlEW!#7?Lq8%xw8+Mi<%!~@3xudpc6 zI2S&xZ%q0wpRW>S)b0!!uzw=Ktckk4rBPV!C_(e{^FjXtn8Vg;H{sbrAf=FOmkOoB zkm$3Fi;Hkyia0szwt5B_Jz(j-i;F)n_cMR)h&9r;B*BM_@o*Wv-)|TPC6XyV6hd-Pir~OMYJ!%&pmtR@v|}(p>}h z9Mn`VFl5?%w3P4MA4$u-wSfHto&AG!D!z+2kH+wFoZ`B9Ve>n8DQ@`38E`s{$rTgT z&>jaLoUtYThQ-2e`QuHFw4-y}t6&e10w9@!{Jsis^VQ(f%rNJ&4zPC4~eqM(Y%9!Z! zeD&4L^DG-(7191=~*`HV{PgqTTAvWaoZ|XY|=I{8P&O zeJh#n`+}YM*fh7ydLHN=a8Ku4Uw{0#Y$R~|Ml&iPSS}=Wxkt(Rz#7WT7gnRpE{A+6 z^li($rVXfiC*|;yKJ|@nd`cxIkRLRg36Eh2+S)|q(@hxGqR+>^6)JRA7p?w`_Py@c z&+dG4FC_u3{q7f1_K&lkp7VOYBP1dF+)OZA)sH7K7fVn*_YyOhfV$WA__wfDqZsY) zF+R+k1pJ#)BJggy{2ISV^LYvHz}^m|e97*}zgp?xEPk|+#Wiw0=#xCj{rzJ5&2oo{ z33Q*S)FOtMj61w2-Mck_^NY56eZ+g#81w9_9*%Q!dr^8lgSqMTH1Uk*G0jW$yKXJ= zf(g2O1JpKlU&ww}9w_5n>@<|$kv;9~9M`emf4~BlgKJ`ATh`_Z-Kt(7N#|Oa()LGQU)g|~ z&%2pQ=-b0uhw;*bRWT;=J+Xo^bOXyVxs}>%F1z)?7yg1?{AGQx_bScdT$4cj4T?-8pg6KPN(> zzd|eqGUF+>Q)QO*qk&x7_iM?3Z@#*_=Uu|IH%FewKQz9)dU4Sh=Yr#L?Y{Fkrp((e z5#?xsA%WfC@Q8wb0f>JvJ<>ZboT2WkK;=}(dmj|6+*1m zu3m-iL5x+NSzWzl8X+wt$3tU_-Bd~Sp%HU2jY9JVisqZ*^uXRnUtUYo=PEfG-B7%~ zTjCe7Hu1VX-Eso@Lw+y$0XIE|qX+ZmST~C0geiCtB$&T;v8xWa?i4O?^Jp(abpSoV z1C2_sv~AASa$)6k-mzsp&GK|%oa4cI0+v=Vay^ZFsljn8_2IW~(iUo;oFTGodF7_0 zQ2hOz$RCRgT(pr9?FNhd^GM9T}rYwI=y_m9yG}B zJ4sl&6vkQ|Tei}F|8E`sF%=1jRzq*UgHgR7&d{=q()WP@We*nIzf1kM=K*aYvx=4$ z=51L{%&PpwSmaw@Qs!w~e_`O#yT9K=QGRjgu82)u%C5kErHKMe(5CMZeat+6+mu6^ z7>du_2-Q2AotxLzC3@&R-h0fGMnfeZ(UaCw4@n6>(4}C%c^H{;j#Xh9PtY6RD{Bx& zp7$`GTdX{tg)P@{kXv->ip5abiW&Gbax{^r8V|NJ*zKiLqn8HCysY$desh>%yFgO$ zE(xlX)LCqs-9fP1wCPvtm{RI!wx^!&()U|nFdNx?9f$2wUsYKKPqcG$B%W-VD++Bq zb~tY*zRznKzMHAtXnEG~0CVW9(Pr*l8oJx{gRh#sFx_fXnGmAO%Zu_QW0pJC(Eg;G z2;;~}Oh|q+a1}$-bBs9eT8Unpud>GH9^*Gv1vc%pd&AGB8q)8Nh_v|aH=Y?0Mn-!z zzc?fJ|D*DU`{g&L(~N7YGf{i2_8lb=drF7ye;zvqOeoWI<5CdyBESYs_igUXin9Id zblhGp%MTHpdmov>zcW(+H6YvVT;aC+3~nPPG2h{=6qG*$ety6?BxImFbX8p3=aWi< zf=PFsr@3N#ee9=Zbb|vN*(c8?%EGvOV^~SG8g;nc`ufn!7Yt^w;lLW=MZYsV?I=6T z~FoB1y zA{Xo3QlN|5ue;V|blSu|XG@Ppeld^hN$bm4 zt&|l0qU!0r7c_Z%(6o2UP%}8O>breB5m7`HW2K+t_5y2vZq`%l+r<+_Q&YFzJLt98 z`|)Zcs68U{8`cyEd}nK%qUlAZ2L8O&Z%foOs=_i9iJ{> zU!ElC&dmc4{xCbsb*TBsPtI6&XR_kvCP-(5W9tiPi>@Www3}TNRGLz2U08*MF`j)R z2-XngM-y`$Z@%gJfXb;yL5&j~eV?>zSqVP&MdOR=<9dQru^;GT@wb2ygD{%;;XWYz zeNZw=kUw0**DH)6EI;b|6xB;Q>$U-!)vjj`O5rFt0{t#L#)Ekr9r=8^J zmzOUR8*zNcxQ|(3?Ww#7LIq?CC|)UK7i)U_mKL#b`s#DNBp(Z5)1QXbWMGQ!@!2nB z#l~bU(xWvZxU-&e8H@;8m5zeRuC&Vo*=@5qo!67 zco{7}7``7Rbl`R2x03SV&u=NOgT?sQdV80Llr;j64-N!YBqKCa-~Ra428!^9H`jgy zMh1x#{MmQNx2DeP1IdZ66Kl}ivJ+}fKPhzn=S%nB+ZNz?TT?5rZ)}VXoQtHYWszxq z6cwW)**B#xt-L=D6*vSv>MJ5*Qh1Mo?oc?@{4ziT031goTF}wVJLz|Iq zd1G*22Ls0~$BO9qiRsz{%LeJ|1eB+oaj;(BfI18Uc9|K_jk@pLlo<~b?tbDSV4GEv z{n|*$IgfYpl-cggvbgMVN9Z4Py~gPxoq%zI?z??*i=T99hKiD0UcXyLlbI++k4mFY z&XE`R6o1%`)~ATvjx0K!3n4B~SEk)wU+8OKjIQP=!SzDvX}-~iNP7HSYGt>fzGu%< zNYORtn)mA@!{-XU;CpNt~kSB*k#()1)%*sr*=OLGHcvHVC_E4d3p&*X7C$2oQgIx)y zL1+qE^1#yY$ni&M2CdW|HrkgZ$^eckACF}Y5$K9k3Y@DKFg)8E%dSF_Uwt-eB-wJI zcG@sfoD_5}Xv#6zB$P{~d6Ak7i(|KlcRe%>M3f5u6SnS>PIC22X?-YH$-BWj^_ z&3?4dR5qj9A(y&L7$h37LdMPMXIat@PzNK&h1PD4*VMF7_1wS5&vlxo$u6tIla51n z9pA3dlo4K3Z72}E((qrW?Z5B2M+^YYwMh_vWuvXGP(l94;Zx(!_9&?+fqfynG%<@r zezpO|?@8jC39oe%zeTxw+Ah_ayYEciCHF5!i0F*nBi`MbIuQ0#1KJufvSt#ivDs(W zcl;kdtZ#H6-G6YRL zi($%zvCi=a&8B%yOI*^kiWnPb^~(9kvFCokV+7f#BJiga-Ar-S<9!?Z=GZ`#a?uh% z94c6xx@1Y~UtO%lBT>I4h!NFX9+3Q+5|AL=Jzc0@?BQ^Pcj0fl(!){{&<3OzU;xYX zRWcL3I$Mzi$SCKLa-R+|=im+;r6d^b#_TWP`u}Bzhb)Sz zPp%6}(tiajMTr|T((H~=U3(jI*1TA}4qE&_seWfmOIPf0KQL$v*LK0>VU4bxcD8F4%#8h9Y2W(Ntn;OZP@0?| zluJhxzQ625(q)P)AIt^i;o28!lO6~z%`b%W=7{0uxMIzEs>b*4ywB!|itw4#+@~wb z+RoQE?U~KfjH8IHNO5ep z7WI*_(4U;i@QdCQg#y3&v(BW7IeARqTpA{3st6ObE#?^~5WkBft5R11?g4JBwxApo zn?k~G;@*0`Ro0FSFF(aT9$)5ZXKU` zY>dGFXkq^QVh4i-ao;lI6qRg&^{6c}H=OYz=sM5LyPY^tmdQ0|WPM1R!pCm9#?`wx z#V(KgVT-A=&<`&A-1St+=}%vp&Yxa@#p|>W!xBxwwz*ee9kO3sU&zcDD@p!@A?GD6 zALfY(R1Q%nQ2lll_{p>K*|J=o=)!9|M|DKzNU~yd6abN!nLp#M8|5u{)6t1;?h2RRoZK81c-GJ5J+MBW5cC#XR)IAV4Dq?j0a!G4jh%PewF(CA z)RZurDm2~6f?Cfl1CRYx)kA=Fm`@Jj<55jA2*k+N?Kd@tB!B{NrfdKuSu96i3_a

Ra#l?=gxP%}Mr$zbe<_bXW8H8KB1?D4^(8e^8Q>DP&G-UMz2Mh#Z z*Kw^*yXR}6+aW#ftGANn$&b`74MlMOR^EyZ1^hVhZxwbTTfh*ib_}>nf0qmDd^}`D9wdV;(5u@4r!qdfvk1dRfW|owC60 zYbU&?NkSICU?Qf+Q4D#PiMJxf8t z{Ky>9SX3)!*1Z^;+wnfRT;QK;F!(sCmG^}7w=^5N^N3ZBt+@P(gf!U}YdTo-h7u`Z z_-rMeC+S3=k(Q9u;^jN;Glgh4I`?k^xY12@+!^5Y8Ji9TA@-s6}1q~?FAhxA~mW?#M$SYf7@NiJ>s+AR@`qdEgrXL zd}qoY(aTFJzRNU!v;fb!4E(+d9%k>y?`f`rWffb%Dp5&JXGa1fnHDHuhYV`eq@$Pl z=k|G?l2R|PGcLFX3-><-!<{XkxY(#reOlT9atx(vzSXc;@l0_*a~nL}uyTpKWJTLTn2$;|lshr1_8s_iTpodoOPb)(336sj=vUG{@aZd95-(yD_S@_Yl?q zAr?6awc%UK(60Hub!m4i;zn*dt4{3W3o!9o2B`G5$i(^1?aWT+W$lp%= zvk(7<*^ei%04Iw0jC3nMW}cpGEFtONTR~&IRQs+=ePEtUVsbI5%$BF*Fp+{}WFwgY zyynbp!1$8!eQ8z``ClMSx!0-r=xn*>w$5{gL}1IY%)a`h(Nmyth~NPGSzP(L>lq;~ z$`F0D<#O=o|H2ffYdA+91*WDf8BCK2)HqLI%Tj9fS;+g(L!7Sp0fiI>DCGixQ*}2@ zow>K$4s3zcKY&lF`y@A}g$$OqdUNYy**f#!u@rqv5kIt`F1#xsdFfBoY%Z}a7Ho}Z}O z!F#KPeN~wponu0Fp8);ro)i@zt%6Ac5=Vq?nF48Oxy8`Nb;<9~1l%NLo}*kMWbCay zZafm3@OEtS&yw!g_M3T~hbKKnF87Y16i?MSkZHFxm7~9@m@^vHy=#@p%z~9GcINq% zT3*vG;pO>&8r7?%Vogd7DD?dmPobow-q5Un_?)Tmcz+X7vqp@sZOI1JueW&|J8EJ& zOD!qzr_ZMj9xD|mZUyAMip)v2epkWcQX0}6gKDLzw{VZ&1WwG5yBZk*AK74yxZ2L& z!?Q->xfKR#gFbMkwF;lVo*IO5>8cwW-C{Q5qZX}4$m2pF+Tizo!n1rr*a#-IFa#Ae zehtN0?eHD@xn?tFA`a0%UqWtMG2rEM{UMtN7NXKl5MM2Dd7)->+B9KM8T=3M*B3t7 z3z8gE@T@$O#I2|<-fO%Xi#snFH8yLGnw=|6p*MWTflV$S-Pk*xnEOA zkOMEbI;2da;NZNnm~|iyPc0 zOC^M$N|jP3rUtCqSs9+~WT#omq<@uWgY=SR3Dr|h)2$Ck&0yleU9@{wDa}^vy8@(3 zy+sAuS9Ae8wwSqJ4rrh6N@b(TXRV8e!}X}dWyTg-G5A1*v*(&yxnvB^TGaps&+u5G zjntQRCnNiR`lNYXDsk{AO#hsna?{IM0bvW9 zVFZdxKY79l%%bxiVbo??BI(w6l~0~ODLE9=eYGU8YNVKHAJ{<_sJJ?Q60oSiYmowP zB|G&m9JAp1aly8y{6e2#Xv$2V*S9ihb!zAhAG!m*S8a=Bi3woRr5$N8z3b8x-rrv| zyvEzSzi@wZSs9iuwqNXqI5f8w#SXbm>l3)&E@zQsHyHwkH43YcHliXLz);d;nnDeG z0&wOPj^+$!x{lYT+B{`C1slxsx#ceH>payT_k2Zv;=kTa%m6zlb-f1P`COT4&sj6U zXLj1B7;9@QCwJ}Q0^)>87M}|SFz~k6`E5?L{gN{1m@N)QulIn-qn2b83J}X58i5g| z!de;ziEb`SbjODt9H}LJ63EbtndO9Y0BRDk9N3%TTmbk}=GJkCN7fHT$NSV(FUJNm zGQzzr)7|b#fgAB$pnl8zmvrCIv3`CN67>MCz}+hDisp&| zz(1QlM9EyL=~u3N9qFK(XY-LTCvqaM+3i&7?nWmkY0MZ>*e$E#1&5DvnFR$p}Swe5^6iu&1-XXv|6W;WI?)!5D+_DroV{UnF+D zqie8w5!3snnWU28!8i3Fwz6il2iC^Bf=*7JFbR=XhWu0+2>dy_VJf0P-pF-kbfKAG zFr5z2kB=89Goifg;r^6zf!ASySg8N;dX#~v^q>tz*f`dfJHzB@o*hPW)J7r{3PfIi z#V2ns_=S^l2@v|@rT<1Rj1r_LC0a>BF)n3z_w9*#JccvLStD~craOFO&W(tUYir(i zyQ*eci6+gL36YizufpCgKbWH1t+UdZ@+9QYhMUHp4Vc zFI3svleDMwbSt5J8XHQz^VG`oU}aK!3K4_+aC7Sw&t1WznnFEyH!9I&6YDLOGq~I? zujgTa?efM%q-*a2H8(ARbA|7Pst21BYZ_)Hrg-KIj2 z;gS_-i-olbyCLDJ87-ZRE-|Z;(Ajjdf58x=0Ko8W(|v=S-!H2Y9~gwpD8@zM^I@i8 zjn6m9<4cbI)OEGN=ISXPTw|Wf>{kfRR%fu6B0Wy+Kh!yiz~KxH>sZ^G^ba38)Hxk! zuE%3)kEdNE&D%ng~gOitxMvVj)TB| zq+)m}qm#x21ftgqw{9$!O<0S}qK^`+1-xlbW8!}rN>(J(A6t3UDU0sGWi}BaG>WC+ zu3&&#;WG0IZl5V?$J?~vdMnP2)3C_#tGV22AwIrM;nCane{O7&yX(|Hqg|m6KBk19 zmJA%^ge++eOXK+pc%PSNau;508lUDyr6`T{ac4qv`ra^__{x8S45P9Ql*mV z8kxp6@9A~aX~M}~Yeja0*?;`3TT)aal@2WgLXip!5$UPVsA>vnBOOHxI(n47Km|XJ z4%;=o@*%T^9slS3VO>Mn(rpMPI_`Aur<2HIPvZ<+t5Lo_$)~DMFv(h;vi%jb!Wj1; zeNY8i)E8$e4LcQK0Bdl8!5dQ)mvaD=pZUXM@R7E2c;fp(>HPwYJE3|o9w z@Klp%wJJ@I#$(K4nlD#jTvQ-tG!B9TirJwc!*CL4>wy7}6Ah$JL!={(#m+B#NdTv@*xc}|K83fYyOy(Y?q|$iS7uS0vzo^ zWeClo(X(gC&gSwQPFRslURRNWFQt)0XZds-ZdmKf`yJNy6wElYDR@>DN9*W6P<9Jv zP5RidUG_0iC-eb1ht&Cbh0&UTwB0h6sbVT}*lg2PwS8kFooBTJw)VA8yt%OXO1}~o zU@lvA{fKY&6`sIw0=9l>e;-RbfDJ*=RIsbZXtlRpR83~@UnzI3RuIS+{5}F`9yg2H z5iy=^p`Bb6)qY}asefr2=U9#oO}cQf8}N_9i@1&si*M(McvkR%=e3|S!<;c6prV)~ zzEP1;*XX^19f`z97tXKTV;2|=IRYNd$5ns34j|B=W}AvA_SHsyllNzi&_U{g6Q~3S$9y7_rj9+XGj%C2?hRW@=^n!aEVFDf7MWq&g*kR9N%ms63 zYqYytVeCXMoXRJ9>5m6_d68dpX(@)_<=lC7uVW$8{tY1kwJ|A8V(W;$;jFlVA4G&d z@NGl&->ci@igKo39SzVZc=Ap;ZTEi5s=nB55@QED&T3&1s=xrP`n*93!qCI$ijVLM zwXbGxCvT!d4d2=WddE8aDgBX{LT0L!kcC4LIHO6?&C~77mU^mb$Lg0mNmlDb$@nXJ z@*4WC0g@+QnlYjbxcXaG!5ACK&{wTI+!58~ViMvE6>I=kSb^)Xqgu#jb`g$60N;}? zi$Y=$%8|tw!Ps#EH_8@X38#GrHQUCF&ED({N>DESDR&xcI%yDb57NY(ikKa3;kGEN zIHUUB>eITfG9Ebdx=JN2l_kI7L+|w8&{*_+-E{vW`j~nto2O4Edylst;~kH=y_zp{CvfNx?`L^3{IR( zm+d(b;>J=cegjbNP;26xZ_rUTQ*2PlV1f*M){hc6xsnOTvY@3*xdpBL=Qq}g0CCxc zLDkkdUc798Pw`+JTu zlTkKT)^kICQqIAM2KZ)6s{|#xCs6c(9_Cl3KaNn|Fn(`vrpjr&iWa3*jQOI$t0S@$ zHO{$^bh1)ehsOi1P8I+G%L7K#^-yRYg~lBxRL zLf?&P1}guvFotzu%tRP6YSKM2#r`u&IfT_)0G!3f;%DoNjQoG!fNQFAhP+?cKcut` zHbrG_I)F+&OCwX+_^}sOwAsTbw)vQJMF3Fv=k2l(H@gB->SItK#w!=7FcE3x&Ao4{hFC%bpNX0w7Iz9t#@i?wBj&(r1H@ca?j}R;BC?IVwe0M)nlqO zPrV51h_3C6ZCcXb`(5;r`awC{+Nkd0b;PR*CL+J=r@POj+)&^kHQ!ETPs(l?eZ`im zQW5yB#Wz+{E0hSIyp4vL@I=UTUK^5J-{LiKdTHH|n}$4pa@P}|SFt*hGfjU+N}ms} z?CMTc(lgQ-d9m#i$hiaVf=BDH2}%4CCR8mx&Aa~l*hkMq+VZ^7Adn495m_Ml zMX4pw=l|{E!Q9lEMyEbUHov2!G;2t9-i5<;;03nA5@i!-v&0=Nk8u!A)8sC}Bl_Jd zA?N?X&I_La#*~|qcgsA(x4>C!Pod+c3@O9oqThch6gnm%*o{`AZXSBrYuoHV=?GfC z_@w)g8oW(}(86jk(F*vR+v0Kog5x~yl>Untc6Zm+FREC&&3T2`%iy78N9?cmE!HQ(4xk`#T zI7Xgk8h1~6w%tj$G6=nX0h)0_&1G35Hc7tu$C;sTU{Rp1Ar80oM1xZY$30HZMHqVO z^}|6g3C%?vmS;|ISx5A)bcOG8(j43zsh$NNpi1pCI0h+tSQm??8E)oo$twb*hurD8MwH!c8Ng;cQ6b3_w zy1(GFj}Kw?18LG0;rB^gqD2WK`A`+0J$t9|L;e+^3+ksF-pci8feemb!{te7Ov-t1S71v=~MbX>>8E`upAZ)mI~uRt+i5T$iG@CAjU}i{^UT=DsvQoS?M+Y z$rL*C0gsLKVtXf6sa#lbX{GT#i$;e!?LQ)_A{Vjuj@m}L&A^B+HdBAv>F)s9CLl&9XK2= zc{)?I3w|w4o~SPM-*GtqgWJQb2s7CCZxFVG@-tr}q5s5e?jV=t(baASQ3uE&D6Qz# zR<}#WFpt*QNy+FJ_vOk?eL5#}u3YB5fAob$ltiOK<1JT#15VYk_Jct;Pp24gp0Q4o z0SfWfTM`|08wj^oO^w?i($9?Bfijm)vB;wn>M??l>1scYWB2Cd+7h1ucW!@x$MG~H z?sDP{5a;cNeliawhhz(Y2d7z|1j=$;Q|`HO}Sm$Km%MK0sJEu#kXTwWp+;#RxMzo|-LuU$MV5LOJ07&atan&}z}nHoJmwsj zYM5)j1lQ!z9Dqoi+pkvgGd#ic2$1BqQqv|&vZp|T9nPf9QCni60t?DiOcgax8)P4E zAklf;zQM!zGrGFE51(`E?);U$E-gJmigCLR)oHHw(!a;ym;C`G8azu}%`2C^`2RZ& z+ZVRI&^{-zoA7=YH+@Z|?btEOMIY9_QK1}WKG| zzfoSKAuo*+_DVShFfux1(ygXgdZK;;7O8}qADhuC)IYfH-R5U@*qY|w7f}Cd&3vmg zl%TBnT(dM)shVrKO0_EOzt9ot?ZdLFPWX`kroa@eiZ88q-?5FiAKx(R4)Y5F31EV( zoTQu15;B5&B4!3ztGot4!NMycNY+u}tLY6NFo-l_+kA6JB6CM@<|3#Z*D07{=D*tj zSvbn$h(q4=0?UWzWF}C*J07!y49kj|cpdQpFUVOgedS|?%z24HLiFPcdv_9yzJQ~k zbp9Ri>4q6_IBihINe(E4<&0fSxW|F|2euVl33irx>#G3a>r0HK9vPps8{_0#KE%7b z&xp`c?{^IEK8_Y%aeJY*_U_AsJLN-}haUb5J0PFfRfJoSeo^X=De}EN_n`BM$+`5Y z5DZP9G3ugOe!t?#O}Ici021#yV?ha1k(%uu?YSR3QHve%+;|y;vMwe^(LBparai_? zHbXi;{AkP?h1#v_zOgx7=TZiUQGxgw3>goy6#lYJ{dE;Y{v)x1V^D=#?F~8q5p-0k zb8LSI<)s@V*j=Uzj?Ago!#LyeA^ZH;+hsO%m7xo&+*$2?Piu=kY4xlSEU2!q7IR zkq?1hH+A=l%(uykVph37uhFEmb)Det1bp&D*IaNRV!?13Y zb1U4&`bAXO?J&3(XrWY;b-)(st!LQjMfb%SZ>81f2*i}Pd8z?9Ab5G6UsK_#- zHFi7KLUPv~V8R7xRZ7vG(3BU?259w?cGf^%&N zlKpccjh=PlDf!Fn*JQuCDa6}~>~Hl?)OrGKM_RD<%H^!(z7is^G?FiR_M=4%_!xei zTk$Gmjiw;wY45&6JYvRxHZ_5QU@}pvlfAvYxt9-U&Ct=84v9H6oqi?NXI7fqBG+etpHlM)j z9s~vg4gi^5I%&rZ%}?lEPyz;31}- zJVa)QP3eR6cYJj{SgbS@k~Gon(P0}b@@KBnsi!TZ&@f?>)T5%vjqfJVbs#yp<9`cqt8@)uYLE3wi(@qzLy0(&0`I9zEpMkS=`9#Dl1=w~? zd{eV52RCVaqasZeLF*{MoI(O9Uf`!aT>2LvHNav~R8P|Ak6kVcLiD-bakd}tjeX}M z1&sp{U22`(BfwtQM%lfV@bAt-uwfA8!Ug!@Xn zpd41@f@T+k!GNEK6aG?)zngj}pnBfWzP5hH@KU{Fg$*N1EjqWttp(Iv2y_XBqLOxm zi3-@7+Yx%ZpDGTI!dB*m z?(gJlo`k=bmXwqn%{AWZ5U-~_SS4$JS~Qj*lI!s;dILGE5;qCcJb0*QHi~AR#02`O zU614pFhn|lwHoImzxlwnPjROm!HF0$qO{ofe7W$A37pjkBFvc((4P`|q6;lNQ6_C! zJA^z8?8`%YbEa3W4b!!pA1sUFbYuax_$VGjB z)t5U;630d=O~9gs#f)1N)fcTN2caan903yh-tOlc+(*}Kh$}E@G2z)w zffnZc1=5D+dOZX(tV(<_vthNcuQNn4HmoC&7E$9;Ua)?3g?TKO){ZTn1kV|7MceQn z#ut0tzMkKZH8g)j6sj=> zrcn|UmGKYiON8aS>oBq zLuA+4CB_20YCqQCdFHjFnc3Jq*?VR3CT5^j4+ts4}@+MO9Tb zh%;)dlaP;jA3#H1@dV_0&?cDH1;o;D=JatnhL?>VhY~aoj&Rxx4-^|%h68JVkzcu; zNBfFNO_l=LnMaKuhH{3rbnsQc3`_d-m|ZF~6(`QKUFL4%@Q({)ytty#^hSY_)r_A^ z#ioRpwyeojU9%~jBq2CdJ!y*^x`<7Msn8q^#**Rv2f$(Zx#MLRDsoeu-nF#@x69Fc& zT;ScpzDG~*pM~kwc?#Lm)_Cds@V+MXgEBj)R=(dbHa@9MM?E6nRgf9ZWo~26tjqR) z{;)`iaGnnN=-bLZiz|d31Y-t&_(R3PES@Th!vs}xkzAp$4sxW#&u+4ZM6^m}rmkP; zb?p{t*XEszAMSk6=SVA(EgqK(L5yhJ{z`(YxnL51- z(W5)Wp?^FV`{`bDl~b~v8GC71Ug%_56PN+9X?ztJpBaYx3asP~@nDL0KHIZ4VJ0xn z_#Qi zB=6h==l|b(-&)REx`st^zkAO;XP>?IkzMk(^|G+zm_Pz8g8%t~VXd6UYT<_*lG*O;*rM+^bJS-Q4CYb4%96fq9rN_b=<|8Yk zq~OTLVjrqiCk^2B(ojx+{0X??eP$opuYSx|>F||>iDFtP0Kg-NGK3Acd<8erI__k0#XW8D_S(3K!*v^=8 zMji|Pw74_50+{J27>Mb4jH-EZV8@q2@hZB5y1Y_a=45uxK7p&JPFc*kSS6noLrRmV z*XNx}3jwSj5)%4{QePeRb;H4c+D4+8F-v$pUHoU{LD348=QZClQ_GX0dRxS^zStU5 zUv!Y5CAJ@k!s|>lW^ON)x8C{9p!hF5dP7vyenK@;`UI{{z7vaT3)FaXXwFwbAy>SF z{RnO2&qY98;qxDbU~t@%Z1oYjyu}f;y4or;QC)YiZdJU*Zx~65ZZCe*_Z9sE4Pg z=dL50QM;7OC{;pNH+09XkYa{6 z+Rfcz{8EuyZJs~*Pd9O-l`_!RclGB6OyWrPP=Q1kA?nnUN4=R&pK&et!aF|LhPA)$ zEg-VVGlEwuoZU*ME;*~)+pk3;lXW`0Tg~oO0G96TjoBq}N=~ysG0=o>rgMR&*>0)2 zoeebA-iHy|(&>U3f>OE~GW{m|_vUJ+g^g3kiZWp5IA*+Kf7^&-f(NEICBui7ow(^?_@qC>25sSkCKyH{VU?TUre-qiop>7v z$zwzY6kxwCVkJL>37DWAGKKzm#d=J^eQn3&nN&rSZZI=2vVI9IC~w5CIX$*f7O0nz zuBJuvQhzp$5CbXbb=>yoBu%uaH@D&QEBgz&iQMv_Qp))pZegX?pxBHsXoio+&BW=x zX^QcV$1Ipe4}E8$P2vR=xu^Pqu3d2R<)3ZSP+1{MEVkkO%dc4Geq2pSyKo{YDY%|get4boQv6AP#sQJ!t(iAX{hA|HQ(Uf% zuj>7}<-RWHsh4_rd#c!ZMf%o(VXqhU$n>7@PMsO0(9Jr^z_fZiV*(rI4HIm%9~_;G zX7n`)v{a)USwzy<{TLmoR*~=nQJE}+U+hR;Y@ob+XPo(FIPlj{f zsc;okm`GwT=#|+NHu)akA1Y>y56$;CwF4_~FOWQNpJPkSQ;Mwl>3FC6xV$tDp-?>S zLZi*ieXo{A>lVrZnI3(&uAxN)I>g0{4zcuFkojmKy$gVO)bDzD2Rd_Wu&(ikEuKa-h#LPT@k5&~djp23KZLGZY#Gq%vnqT~ZvLmUEUi0uYEQ>7 zx};8pIWTn~H;Y4i81G2>7*nEOK5P+TdvFcs&p(jO%}00c^I1n805a0 z6Y~R63wME5^#5M3V zrn(7&C7*03m`~L7NwFVbRXrp3WbU_6UV2$5tHM9C_m$LEcf zw#E}%eAT;x{sbhQJR3JafMG%+ZnznL=D?Tz&``rh2J{bejrooH7+>t}CS3_9C-i0ZI9f#IuWrJ>re0HaP0e-`A5r zH~jx>7mvj%-0wCGx{@$0heV>2>6a-rVeB$GY!uV+Ey(CVW6;MW@|qwwBI)lt73Oi_PjBS^+d(F`qq(ANGZ@2NI)Rz|82-@p zbBdPeSyZ8OhKp5)YaXUFyx$+V!}Ec|`UrO=o7|8hdA*c@}|= zxqdafXYRxy-32Q9LGpGi&GP%1WTm_T-HF8@UFk0-Jyi&-cqsZtDRz?1-JJ6-HV-%d z!eUYpr}zdo)&T~iVvS4zebZ{dJ?{c~LTz?VZ8ZcKuih;~f$oMHLp%(vWEZ(eeJY~~ zJA+8beJaHNrd+EgFs^siO;-MgWos6#bcN@}@)3$0GCQIh=JFPLBWhPx+J>FUejrem@zxW1?)>XemJjl|}doGcFN=1dfdG>4{B=qpE zeRE3Aj*gD%TAOm)95Q73BG|ucCKSdTcGy6M`?U8E^*UH%_I?$(f2@&a#`{fjRJk4R zCr@^R5d_<3zu##>-00HK9njdDWD>wQwP*`Ax+)Xa29(R|R8KNHWU}L%B zh;fVpk&dziSd$S~yc?pZS`0S=Ht|2Gw@%Lg#NZ*wZ9LavaLn75h5hM{&UxzMIKRw% zM3xHS3Q^AS6CfsCjZjoML1J7Ncha|eAN}7-6e4x%*Gdnlx9jVtX{JyuV;Umv6@zis zkSQ+d-nd9sR@c%OqPg7gosVQ9^2|h3VsIaJUy#RLp}fnK?7mo$=ida^o#98>pHHmE z!7IKr^tv!H%kla0oCQqMPz`UT$x?3f+Xz99Hv1AwzFM;%Xmt zwc{3w9xwYzubdbStiUuoJmZC?3*iQ8T?7{@Rk3rlXZNDxdb8{N>=%NzvjFO>*upZL zKeeX-SgHpU7+8drob%EGe9+AxIz#@9Hm&J*-t5H5@zwS-noPRS$YlyDQ8^{FXYO@N zb0)+n()}#&KdWnz$lO_G zShfD`H&Il=cjpf5^s_5JAQUuHfBo+_QJd#Ar~I@=pEFV-y1FheVw>&rR!S;IOTW5* z-D-r30A9(!ttT(pfMXWy`0J)!edSM&uN&>zQfGL4@&fOR-=x=0j#Z&wx1>DYS(2S> zg2mOlt6%Ng36hi0GM0#D_{Lv2nCB7gFhcpiAUG>7HXQR4!VIl11*2_pr>rU#F~G^C zQ?FPsdOhRtl$qH@_n^Gsx&Z&5@Lub=#_u?+r$02gKUd*W#^K`$^NjMbB}u=Hi*ID* zSC6NnfQDjYlD97CqB86oFWHniwz*jCYGv~eJ-T5fvtUCqe%Is5!H^gnbUu;x(Z9Cg zRKEwi;mG~pc)Wcd?|#a)KUVeq8by6j5bV;^=eR7Zgn8p=9dwNC0Enp`M_HDgv6x%k zz+!xu+0gZG!2bkuunl12VnUd2Alh6F>YX~2BL!=8zHD=^vwQ!`aZS>C)~nd6w|aNv zJj!KjTSYP=AH5-8>Jx~meGlP#Kf2lF<#`zW+E%uMtg7K4dum&>l5<*`bWWdF zcdla8y1G%Mz>Eog9wq|p_SkWtwb2smiz z*fab(z(bE{#;Y21@OhKW_CtMQERKfv6$xyWr>R!d!$CrE4aL$08hEKZfA{v!kAdGR z=)o>Z^m#-CEBXU_=Q?+m7znPl!Tp^Uu=zmo=B%^jHB2A2fHBDoF& zMUGS24W_?I=2JWVC!F0DWE8W>7h`!x$_Ac=Jm-21cJ_DOyD|bA95jA&Rx_fd8!15$LnKG#Ig#qb6gdDGkS=%E$YU#B70y7ZaTd@=n|LJ9S@%N{jqOGmBjjcf84S7Zn?I~o;> z6|g;jvd}SpW3e%Bx}$wfFTNly;O#GYy^QSCoQvby)>Y@An03|np98@R*aQF~nlEE; zB#8H@Cl5xn72*5Mxn9{gYQDYc$L5PD`<6yZ&K*Y*v@_COv{<7ii|lp=H2k^h*X(Hg z-;^FQSbE5)R&17fGw^raD$_OhyO&-Rn>9?G-7kt=xm@Tb9uzTbb#~*NEHXb=FL(a$ z^*1KJDJzhxJ9~YW(%1h;^p$0GZ*T0mfRuXG;}x$*8v-Rci>t43+~2i1=&b`GN?_>3 zg}?LN?Jfuz_&xbbjWkr1HoyM7fM_u<>*jO%woHL}i>q!%R_xJfxjo5T(=vNNey;4e z2dLc1S}469)^E4?#AU>WY4AXCPWb`yl=3B~8VSxWM2F9B({K~y$5da-Rw2PD3<-g% zw|vz;!3a>Sw0MvRcd`EgBnz07aDVl_DMZ49HZ+$GEttU>vdPKFGHI2$y;LA}K&4Se zIRk%cfGgu?(5y}Pl@uHNIi)71B8X_U=6YRydY~xRop+1vLoOd(o$|vl*I`G$Nvy=) z8kZ&xVa(O+O)(7%j+JR`EEv^xHnJ+$buQR{vgBsD0j;j0n9bu0IZ8Dko`o8X1>DC`4e zp1nsv($_sXmfvOY3m2!(`-%G5F)ke^&UFpBUU~d34B`&INBV_@qrF|NHkB?kp;moz zBIs`0;T$04SdKF)$H85O+2037vv$Vp+amKWU%*Dep!Orf_N~OJNBN$6=^-5jbMRpm z{8K6Y(Sct7ObiSs-q4A>^ea=}xcCI(9eZ_UXdZIsaEoSf#LuQZ=r4ngmS*kg0~ZLB zD=QBt`*O&6YX;XtFYU&{LC=mJ=Zh8EbdhuJerI?$jyEwY?#@jE)BJ!b zta*Bdxwz_cYTa!3hyn+6<`D2udwIvB#j!WKxKU;&B>6n#p+l3c8E#t6en)%W%m8PY z1yfuVzq7?H64{&aAQBXo+M4Bj>rKB+9CZxt2&o&pVHElTx^LVB|Lf|h>}LQtMrrv2 zVOfeA=x%cpJoC_3SN*6P1k3dW4*olB|kpYaFjp3$?7>6YV*otB3>&WSZNppGif zbQyw6p;L`%Sb&9bSVg&Qr*9@XY+ak7Z6ejs_@U@4(=t3>@8g{)vFV+L8v;MoC6DwZ zvs#Gvi3EPS4i*>q%XM6-4H%`~w;hIw_=1P|{z*Lov9?TqNDL@Y-jq_k6`pWO8*eSk zUdoH_mZ|zg9r4uc`}{BT8M>JrPxenf)cgrz+;tQXWqY7GyVRUvNBz#sk}YJ0L}Fz!|}`O$VXbpu4SA$Q@;|?fRB%~bamhOB-}eC?CPo_ zp@>PQZfQwcEuL!KfU#=W{0+U2X{v|Y`fE*8^4RCQz4or;l=tzj(G}&~>{(=>gng&6 zjn@=ZUwK=T4;;(u#>;;fT;LFm8kB5Fmha{R_GQJ_9QmQyye|%TkLZCZ-RkWzkr|_T zGj8~+b8d!|$9uWY_0E@HNX)@tX11dN-SSnB8HhTx9;W$>+gQ}U0j}!P702d2sj|8{ z^{gfh<59oxfhp4I^c*8srHut7(r?UC#4!Hqjq@=}u_RNMD(mc%t6yjr8zM+-pwk~M z_sb57SJG};vx$OJd)7wrG9>^7Ao5Wdw;}gg%BJXc$8Zk`L6keVUOT4MV~?0+Xa;d! z$-Y|(8R0FS1Z|N6*YwAH1_o*E1Pv|L2;%6^pU|uRGlzsl^?9D0im`=n7t)rWS=y5a z^vD+hwI$_`7L#aiLY(AZ_DoV|7bl^i^1|0da9k(n<5@~QZ#t+Ae=a9-<;>0(_F--r?jOPnr6S5j4*sT!E z6e3D5HQqkC9~yAe%1*-lrQtpxsdlkVdrJ;{U8|TaTGUIbnrvGF0r7`b5+ZVICJL;ur zqWgAFz*$CvsF`IOlrt({ax??SmSm>nl^$5VtK6j1QawG1ce)M)6s}8&(ab3@(b4qD zS~uKTjHofk#s0#Y%EL8QDQx%U1{h8a0~&X#_U(vQRLM^Iv@NE*S94>gTepGX%0 zPPWXEX)&C$92D0@$uPvWSo-g;-cc*LIuBJngZIW z=$RH_V%oObe7wnqM>VHB7BaE~2<&ZN9r#|CUv>PxyQ=B1Nk+;l(GjVs=F0c-`_jaP z*XW|KMpvEET8Bo;EQPKEh29#b-8tbdv9l(oF4fdC-FQt*SCi;B2WqcWmYdBj*OUHT z+G?Qj52m=b;qvx-=3}RCyYTf6X1ke6ofWI8U-EXI5XFiShJ!bVzOkh6*Ba?2zRG-Q zwf>p)?L4B$j84y!9s+?-wMu?D;wxTzxd4>q{@nFLZOkM(HQ;qMH9l3Yrl(y__l$p+E*d@DgNNzH??=s>$4;H{bKbh-N`WAu1z=AQT1g)K0 zW`^P8pbj*;MFlzF{YkOX7?w%?FqEZ@i%AK|p<1MZ4xg~F>rvdfGozrIkeYR*nIsitbhNtsTDE+zP);pgi90FXj-qXlN`2z0(wV-X z;lE_u<(Z5Q@6c(mMP_11Z2zA6$Q|?)W_Zab%^y^uJGWa{HbhaXxE{17kFH?h1%HO) zttoWF%N!^zt3-N=m+wL_YOEwHmt@O9Q>o(7{N0Sh-Rmv=U9z1Uy01{!{{RDuU29eV z2L4Wk>M!#NxnCcgEa?n>sTG)gvLgL14hFPf7CZvp4@5cFV-7x_Z^0kdqSZ@mM~*$c1C)se&2>d%~g%y<>UahFreDH_!$-*Uzh z5|uK#=9TCfA*c)yT)i{DtbsFGU3yrRK4zFi|0G+z!yu6a&+O%o2pNx)+Q)UHt(pr1sX!y-TCSq3KkNe0FHi*=Ah z$siZ_l}?7$^pkY!zqF9-NqWQ#_oxLy7*=q8ID?6*jfpXvo5=WML6P9$IQa_fYXI;e zbTYPOk5S9!HV-?!Lv)&q&ZQwzyXA|25Xu)4Lhqwb;}!*fKt1;9<*4u9tUqG71L|k) z8Wz{OeH5=J-suC=&aL7}p%Qv;S4&H+uIJ*Lj+G4B{zZ|q(Bu85nP^H=bmqLzLXKJ; zmat49%aEgnX@(n?PLqRxr|4~EF~{tGUmNBhMm4a69>*}EBUCPalHKY76L`&6eCLU+ zxXaBdcg?ze3uM*dw)w2zc_PIIcVx3(;^r>iG_FRE|F279;8sF-ncjf9!%IK=gWsjO z$;ldL2e+(lL@GZX1WoMZdD?lgH_r*=DvXCYL8!pMM&*LSu@8&3+Ya1Gz3Z3(BQfwN zD$+-0^QF?E%x26nDdm0wGYWIVhKNwfFb%h>Zw`2{y1vcI?61z6p?V?FCZE zLfjCL7>5&k;kF<=H`R7^b938qdEQ)@O+2U6L?ev45#SGxeZq+AxzM_>=T91$vWCTU zeHrSVweEX)U@?*F3XN=ks|!ZS)ICmR2fpsRTHOCkpTkxDic8C@S@r6Lv7o&0!8pbDdB`oy`lOwiDaiV@Yo z6OXE52PsFvARUu?f?188+`N5zjMg5@t&SY+V4z_}YZKInhoe!ks9R`8>M_?EO97(> zBfX0p;)1X-mFenW@oMYB_@hjR9A#VYIFp9R!0P*wbcd6N8QJ<5lak7_$$)vl?%aK3 zps0+YF_h~zC06Cr{yWN?jH8^(a^m799Y3+-*c5P283*^2@h>(N_8EF_jJu*TKbvY$ z(4Z#XS?Hj8C!rj29Uy+2>jbYD%{;2^uB|^sHT0x^$Qo<2?Z~30Jui|t7j}=+Grru9 z?cj$MvEmQn$AJJGYCBCn9%OuUapT~d>_9`qFNbn!zgm@T@69Zj%}}59xmRX&@~&C_ zw8ECnxe^KjpmAaBVR0dQf$1bl;V*8u8z-Lz)T3o)6s^_mmS28>4_E!eo$W%sT&Zx0 z4xp>`A4(sn6DG>D$L=K2b$6Uw(7>4{)Jpjs+03>+(TQ=lK#8G}^;v}gwJ!kq5yO=M z4Vie9@T7hId!H}^#}{U_orD=JuvDsVR@tA87sZtRWcs%x8y-lneRcj{Ul=4go?%bY zB~ln*VuG{G?(F9$`oh1aQd1(2t1vxR8D0Ve`JiWTF2J+-EXFpN$|2Hv=-iDqwo2tD zhumAdy*7nnsxz`!Q!9&mEJD(_>_fd0NmYiTX^t+kU2gqcWX>-8xbiLerY5E3Q<`yB z{uljI_fgNqTLIRhNGYL@SpugmWH? zikdYt)*K{@4y0Rp+EhHa@4aBR!>-fLK=qD*ITop8ga!tF=|G<^0K#Si!~x+>@&+g#;1cCt(^8 z8DCyyXN;C116$YKv6s5S-9GdU=(T@#*eGl;*kOrg>rwrK``z76+RyqqAowKg^D69) zwT8VOmHYDtM$%%U;G96+&LC8>iIe?G{JhU!Talmtt))f z-1}ZeIW8jG_qnmCfnAWe*Bi1InhAe`07?^ zXH;Wtd*K5w*gf9hQ;{U0C|zZNsv5I_qzPq&vE`Ls<=wT_XXZGTU$^3XKz8oAl!6OG zfA(l<*^$(6MCYDRiov0_B4UgrI{u5wNSytXxso~gSXZrlRys$3lx!4UPxe28A+#jP5uRqnR@PA2TQ#2WrpHo=Y z`P}KB+CcNF6|U1;Yd9aIXr#t!-34ZlDK6s$}*68wdzj-9*yeStxV7BakGLfZ~DrynW4&w(d^Sy?kppG*4E!UuL7$6fh*^Lk~x;`10+i znM#xQrzw=CoDD(BE#~N;n>^D{0>Y5qN0={L_+(dJSl}HZWqog!RCjwc4(fOI`uGg3 zpR^vP4>!TY-rk3=dgAp7MH4k11y^~;>`1Q1S;ON7lPn5H=c6Zoz;rboI(KUK>7zDp z0uD9?W^_8xR!56Zs!4?dqGq;|zDx%1ztAnc zVnGo{!%Sc|nc;KR^4eee$J2q$h=G=olEYN!_OzzcBed~M;Q|N^{5);Y_bkHUtNw5v z^n%xxA}Ba2$~1c#JdBmM|6MEI=s#?eq{m~Ju%07OC2oa&GMtX%I2!FNy-Q-h5u-Ha z;Tf3HmSnC20-Y#J9v8Vqn>N=QO%tLkvL!a;E_RKge=BeG~}sCJpW7Bi>FBEX=1 z{-CY<#c9KmQtx+qaC&b3#p=IYI_`pO82{0~VzmCU63R%nv`y?0T1&R<71Q#e8db|r zfGJ*s{nu*jo&EMfkV}?iiG2RXgeYXdH1Y+F!=F{S$F?mrIQ-d@R}u-_)6yGRrhP6u zrT~~Lyb%mvE*mARP@U7W^ji5AnSmoU-S~JF zP)8e{My`g^N?K~jx$NfVXo!n1{2Vuj7#IBdWVMw)SCP*^67%dBpqP04O|nvfUlfAQ5Qs2iUhw z`k%JF?Y1z*%JitkP^y*WEL{>)B(i(s6#m_84(;z7w3_;pCvooEK$|sJhSNtZMEAPt z`zWSSSE&8F68F@Sj)Clvk?wm?$(iZ>X-O54%g3fXPPBsIx?_K>=gNX3nrX=;S%+~M zgwVnj_z%oXDv)^nE-!N{Xk=L2Han&T$7rP{(}dr?e?=VV4jnE1ccrIPX6@2XYh5eBV_sd>-F?UBiu#er-7)c> zr9@H$$vcWi843V07%-rGl7UT_PVI?^Rja?aQR@=lV(A2;L>u91mcTjV%#R)WSb>yX zWzKQJawfjoGOg7iV5=yS@5=);PBjY|HzQ_-6!?s;GtZ6t?ZFcM zq8aW=d2k<7s(WazOa3ZlrWMp4@D{;?ao-CnI54VwE(C{c)`s~R1`x`FBrH4(9T0Q` zMjPxel=vxiUk0K@%Vd%(!KSD4`PvdCb1}QA>M-i@!a}VLq&naQ)70908}3pS;U^I; z|3Hy?3XsQlsEdBpzHX!mE*?fXXSK??v3`doE-433m*#*bk~=Jq{&aX9R;B!wiO_U#xUR=kC_vpCm{9kQm?oEIjswW> zkM=V(!*7UeuC*KhmD&4wnSwB8IF+DI}1oj;!T)p$_M z_)vj0L;8t*5X^CisC=0uwYD%r%)xf5alb~cP~CTf$vj-d&m-M$&y1U&J%fl-5;GR=HN&M`KqCpImxWaLesi+4<2~%eyKodJw~-`O=L*G@5AQ=U`DTF#U?dfdln zrCxXt_ov%hKElyM!jR$)_X1@NEjBHR_E43c5 zSWMx(cO#3o8)fH^a-LFu6zK@{9+sblo4CXdb1g+ik2~2@()nN-FyIzRr-_1H8;3K|JvoAxt5`(|_jL`5#+D z1#b68eI-)><{3>uL;(n5ypC&g$`Xs1GEg2&ub6zE-#Ol6IVaORI$6xg)RxCHC2EyR z;73zLQs%H8<)Cy8kYG6>gv`(PzMy(c-3$#CSz>~~vfU?}MFyl+D0Rr_e zhnZcsaoE)krKv(&mY3I;@{;Y8bJ#^#GbC$yASK1mxGqfR(Sdnzjl6qUA7a)Mu49Ot zxdJsmB4jIGooDw9WTJXqR#588N&bFyeyg{}){}=O7+z%8CLel|!J7;^`jQIfiPUc{ zbXzz*Nt>>LKU~#?>*_gI|s@vhXR0y)X!* zl`#+9g1*HGIHJjLBVrqqQ68!dB?OGG2yo22EseQb+-J-b? z8#hn8LBd}My%rXpe(Wl9MySNaXEg@cgpEG0_h;~WMs>eYJ*Ro)F=tVckd|4rVc15P z$Q3vBj6HU)@UYBnht6L1+;iC?7U01hpW`?wg6Y?9@g80DS#ZA6VuKPsi%6y>gJs?T zo#gyrjnE!4G$bCEF;y&d~I zUcU-4bC`YwAxOsvymq@P#3-9_<=0>oVpBEPi9X(CE%UEa zhhZEL2tWKlBq?(hZn-^mLNR+|wR07;-~DoT;0?69^&*-pqG>^VQcqvywps7)g2+kz zno((-d6W8NTk1*2(DP9+J%W#=UJ*!Vf1txxhmbve5LcZGJA$4e5XO0Pwy7U9X;xOQ%kB2 zQeId?*Crn4Y%bn-8<0KzBc?@3R09#+$d~QocR@MY2+c5(+BMTPC?xqjlY+O$;n0#- zU$MEmb-xdz;y6+nD={fJ8if&s`~tZM<%LkD|F{TEz)+$?n1ZVu2MI+xV%OO3osjZx0?sy#&A)U?DBUX;h*t&Ka6jHv!De&U z`{u1}uiUlEpM8z$nc&MKGA%<(GwM~L{iW6r0q=$}QW1qJoB<=V^!|Cj&8le3HRzVH z@ux}d+x(Bu?t$GjQNMY_k$Z1dPLD;2{aqH(^Sn+02XA>>xF~C*I?0L7bunVK7QY~p zZ`xN$+b64GX71=aSb_uF`uaPo@!E|`&G;uD0FR9QuQvlYLYpn}OnZ43YL;6GtM-25 zpB~z6wPRr4aCcP?Fq5Kj-&?TvrWuDXcQ3O~nBI+?B_V|4>wFbJP|hHrBm}pr6ybtT zjF!?FtS!50guA<+IrFA{yq^XHO)?=Aq`m`xx?lct{|&*AR^j7{52z_OesfZ+9>&nX zfX_3;xbp=@MOi7OVy8I{fbYr*rB9``vA0*Fn+Lp9=tMfK7`wlFqtM&_U0%^#*XlR7 zJ&f9;*VkoSH#Cm!**=w`?uCUUnzHZ@L`8wY0luJvlw2JD85FC3!=XhU~273jQ$3l1Z@5YNG^9YwwSYY|5^6lvigS={Y21G zuW%9xp=}@(0t&t*PKwCmy7!%tDewrb8h|X-5XA6L@qz>aq=TSKfZSnpmO4Z1%?6^WH6wTrBr(oAuo#bxA z9L+KnZzUw9OYptWNU(x*K*_rL_o6oP!Ah-Zzhix2qZ@LYINPKmtpGb5xiqo)citup zMOzhikv)5PVOY3+#?IqF5~~PX_)!jQs4XPr`)~e=F{vkI?RMN6!*!sK5khwjNjgFj z&&YW*I{Z>Lqn~X(9Z;X^HEQIIfg2`)yEfXhVKX_CSvMBlrcjt*u zJFB+Ea205yWPp0wEs5Hela6KNNIN7xBN{6roF{wG|>W#ii#pYsYIS#18OMW2yzCXmjb>A*khmN@nQZ5RM->0o1rsV=w5 zwFiCL0Fhv`b^K*uY+h$l?bYS89S)C)pQk$ye;XWE#F>FbUTsegVFgIoU0t;{AXWL0 zrFnbUG%xwR!wYtU156t|3C8r@f8L8-)Nfdl&ndLYnCovQ+RIIFMgGdM!ae)c;5Ulw zB0U98@#k6e0xJYuB4o6?M$k>Iy2|Vf7&og_ETY0_0tt1`Wkjj>uV zTgD&{xe;fK#EC!4VxnXrE%JPApQ(?`^)B=OnOPYez9BUK>pr3_lEK@3Twx7wha*S? zKcaOOvo5b@eGi{llxO{^>0DtweIO`HPhR@D6>P4ckcIW~gH}*A9)+k4G`5Ah_uE;7 z?Vh3RX(6-Pa`2NcR$S*W)YIaoXV8r?MBfYE?tg80GRNIJ;6S6_V)hnm>JY$u?HM!u$F$p_a>8Za6EL5=1R)C7NM z3%qhOXT#$hY1#`%o4wS?1y=nUYRAz>HmF4Z+woYAg8-44hc-9e(Pm=JJP0F^^3!Gi ztVlf#HHFR-Cg_vaVjJA?puFI5E(|lJ6{keO?eA$wBYqBM2 za&2R*jgPD5ZstH%$Bk@agjPszMuEy&m-R;rt{?uh^wp!08--RnbF;IvA`e%le^Y(i zY%sao`EExC+!mCX9(uomcyLo)tEP${9d6&^XxtcI(l|d~=xpsYSK0$dsdCk|ZxV9< zGg|Tm#MWx=!UJ{9X=aAKyzz?Ea!2YEN+A%A$ek_6_J1`;Yd) z&-;<%?ET!(ZT(_9Go-dxkDxH|Bw5>EqmlZPi6M{s@@g{P6&dn6V@TPq zgzFG_2{exgF-qj9a% z(D-dTj4rde9w=cFrZ&pS3sY~sKe$Gn$TVVX0M*Z~QNde_6#(1X2w(nI<7%rbLzovu zWa0{)E5psX@YTmr|LO#_)9dX{DX=`?=1!vDkjX9sUpheL z`C?Eah`lm-@L;Di%4?nI*glFmUOn5U*)38d)OytYw$I_6h$$5EtwSc(w5zP(xW1A_bAk2%+_m z&H;G={j7$uhSs=%tUzNyFuyMian3|UMu-6A_yrnbLM8QtOSo2RQMZVy}yhCxV) zjzoLHDPNc)VH*0y2km+{!5}o&1+of6-~t!WLbIvKRsiT?C0WCnFu6Dy2u^%HA&$9_ zUs6)iGp_wvPYrc7x6NGcNB{u|>|+?<3(#>m2)vfz;f=1McCu@zcgSF_xhs`^LyHT-(oyp_@? zYK5EeHu`eX3Q0vQf4%VQ+2XErN}<=KV!{K(%IY5=+Xvl#XK50mEX&H8atJZn=+NG* zNx*cF0OXokl1itNffoZ;c2`$^H^j;e^o9>Fkd(p6BqkKKfS_ouM=>{qF+`(Jg&Vdv z*doi?4^3$+@9)mAmUfxyYjOKs(#pQJ7P03T-j~ui=*n1RsQ?;k3!8`%`u!y-dh)=mggb(@!#oJRibjrUv8R3+zdI7I3%Za*3VGO-HA4 zrq3}^CzwmKT241 zRVg#QGA1eq?hvf&3!uJ)@@;PtbPmpH1wHi!zOyi+4*b&-7ay^yM&6L;ApJ%F(O$ z(TfNfuZ;95G)E>*@c18N4q=cW?3XqSo4fItigFnq`49HDnihxaqjsq^TsQ`kF?o3I zAX*Bf6p*c5Vx$zYQ9EMdvx*s5m5M8Hf&l0sVmSlrlpGwSc4aQ1^A+Dn{Ow9JB0E)Ls3S};Y~e4q*i2MZb!Obq%NW`&oP zyrtUOrlS&QCr&rZ+=uoMYBm2ij17=(ltlia&w~_w0N0V+Pr!ELcPvCfL+pmBGN_0e zRv(uLE<)rLx=~7ahY$f_KqaRa2t?%{{W)r7Rzh1bAi96`?ss6?E?qTN91Tct^y~tr>os5PLSrb9m z4lZ9-D1i3l+Yc<~t0We1)1Gq>tgY$MZK_e+=P11yp&flz^W=2Wa<{^t%C$vF|Sl?;PgEW%fo$VnigW*ICI?aGRYk!m(*|p z;hyvOvwsO)bPC;JdvcGOv-OeGr$Sn$q`UU*eeQW(3gRJGI^7#CFRcG5G52AB&&OX@kJ@_y$A^z-{_(a;MU=dcOVvyjso9Do5|2Pi-5qad{ zg0_mGQF2LMWOvAFos@1e&2PE4*; zRO6`3*-w6-I`!36Xr088jF%4S@EgA#UOtQivUinL)DDRClwoWCnTt7b&%&3Z2zSLB zNr4Xt*NoPO2iiDq^VE9&y8D)qVB&JWIc51zvT#gb41h-rM)Nvic%0AJh zR9()kW*S+#)*9Ph$Ful$U*HrEj3^SWsJx-q&zrn*0vsgN@xBF*aaBgXHhhT3L7QCG zfX1OvM%k2OrQ{ds!$X5^7zd-UxP>W+FQa}P_Jxg;tEBLH%Wdy)0rlgnk%9Gb&DKXf zE>`5FW z&kVMTsq8J4O%aKlBntGuQRRPsOW_o;#}HFaUxXWi5#=|d6P z%VuQQ9EDOrBPMPLNAoW;mW*!PwZm6MpYA$kFS;q99^-JDd7w~^skq`et3;k@{Bgp3 z)NpFzTcLfSQP$mul@aUd5oigLqFA1_|BgGoS?j)xrtdEVO7QS}>UeoEiCX!)WDPJ^ zI+TM1nn^ar1JOJ6T;TZ|{9raPqg+{YH`I5{1Ou=5k zV@~n6SB2VCEhpwozj^hY-Z-yzp|L(q?<%okH|bs9GQJ&ZAD=cZxm71pK`_eFBX#y! zMFu{tAox3;PdeJ-R5O{fPR`E%v4x&!=x@IQ0#^svDZ(lH;KwMc`%U3QnEUGft~YIU z(Le13vE*CIr8n&gOT`NLX(gImAu9oHH&IfxSsCe@z^>-EgH+Uy7i= z^Wmo(Ps?KW{`YbP!L{&2f{6KtuhG8D+V1*RH2R*FSLA)i1`JmcRX2@#;gQ>nFSN9r|33O7~#)G)PpKg{m(rUtmqe zEZke&$oP;I^_DUvm6FRN*~+b^n@eDGBqK0wsD>IC!EFiz?IhV_Rn8cH-#ZSgA9F8` zuSP6)0i4#l@4;|QNVAvD^hw5-#bCZEGs?rgscvB;H%={Oyv2~QQC5(Xep2?nv|h2& z$+~{7D}L7`9Md`^9E%_7`u07=S%L31wX8Vl(wluJSTLHS*#Yjbn-(Sr&?7SD%ZQi~ zVgF!Qd_kbFw=CM3o`bBR&zvy~ow%xVh4Y}c+9;;)StSuhqMr0P?9l2`VltNfeQQ6L zRyJ7t^vl&~D@1kMEek-#(;uWV3S3epV;VFAHgn5o{R2Wc4%D%;f>BJUc9&4r#U9h&^~Bd$FpWOi{} zqpv*4njGX-&^T;3Rm+7bf4n^{Z;(_;Q7zy8EtbXPYVx7r8}AI_$Fi?`e7)@3^W;+d zPn>unYFPC#!(hXV+E)E^&SYCDZK zRJUo?grgn3UOp(oCco#G4PD|KmhhTj4lh}McS#29Rs7#sNtb-b#e@{HsN2J!ea(1A z_aW7i|7sLVxQZjJR7mH$^|Zlok{f2eZM|qt(6d9Kuwzui=Cln>C3qjIPl>zR9p5{( zKe?it&qVm|0m^xO z%E&B|w_&HoJ>C}9vgk?zyT}I;Ag6Ph)3v{8cl$ZZFZHMh{xV0OtXexx>tHyE)EqSo zZ057vRsC!`S*n%MP34JSQZ7S1<(?civ1r}p3Pe4ruVsA8snZjETBJAa%Pk$ziCu<~ z74$Wz2g4{^sw+$RViZerejYKKmr?B@8*hER@=E*zBoj@hpClz1m;0kyZI#}!RoOT} zlk!q_%))>meX&+$u(m3k#TeFBLimTl!uxMK(Z2Pd8;_^C`fiUMXttO`CH96*KLdQe zQmR`2UaXYeATlYqN%o&PXeRXQ_Egn2nlXT#kpdIR=p*9GxNPMnIF+xfEd%ah_P&Hz z`*?~H6w%IzKe~yd*>j8Niv@+sS3#L-L4Q3T|LJ$)MO7Cgk!OC1v_CU|b8(Elnj6h_ za9Yo+@~Va0t-y$?Vb8#DSa94<3e+%_b*l#J`~}ef95|2#`p*N6*^93@6&g9-p{gW2 zndsI?=t*J!9atamT7pnCPTu&fn6J(rvn6;)$EMv`q6;iq8@U;Vyi7NPcCy_xkSK;F@`Gqu&EkgTh8N0&J;tW8!96;(aaYkoCPoD_*EU3 zBNZA|2N|0WgV`(02rR!MJ2T;^vmc!9wdap1XWkK0_L{$}!{!8&AOI`=czpzR0Z(LC zfK>DG#i5b^>y#Ny4m5^@xKrhcC<_a_3Xaqo4!kR=eS0&OdnK@3;R$)|-G+*>3;i+bk7%6D1)kQ=yV6*}PGP3@IdIN@bpsjN9-T@IbDh(5qP?H{UiVs`^;w^_?##cbVZ9;y zf((ZqekvAPve@tDv9Q|MyXDM6RqNhwi(THY=m>Of6XuCAvdB{%QHRtJPRS8LFk;H0 z*n!xlp%qnl`-@zE@jcpt+ueG#(88~L(|izZZF#O{sp5sv@{iR9pT=00@y+D7$m=XQ z{0daRh4%$f%VOIKZ&V#@?J2Dyh-39vpN?HQ@>pxg(jAx@>~VqnG2jmi|BU-V&^rAZ zCZZ2|PB}$&y(+Qs?_cy#YJ)}*i5Ii0VzX2KhQ-;cM@>DC<`Ut)vu;f?%In%a6#N=j z8*KsS)pe~xi6w*8!1DtXN7OS9`t=X92=Iwa6={&caj5>E2MEDlPdS5_IVt4TtD-~KJM#Nb^rOFpR-E< zamm{UP%2Os4^PM>rh3*W%L(^4pHAby*=%Pjki2GQBal{^SNAR!Se#GR^@zHv}d z1U;p7L)8e({1KSBH+i!&!hi>kx#K+Wg*8JMQhV^Z7}d&36%Qr&1?i-7HpI{xglQ%>FS;49Spq~&+v zrcMG*XefMVEv^F?Xv+^X3F+K%rD7nR-ni_}Kvwi|j zEn*<>hE_dNQfp~J8Cr|;{O=-X5H_$m#Ovb0#1@D8abLvumqq@_=)4X-cF-aJn7b5^ zvxi>b4b`Q@*rfyh(6M@)`San)jZkZoCsDo?7xgDsf?i#Yl${<<2=As&oNh?Tbb4F9 zMlV{kePCLV+$eS77CAf~T97hvV|{}Wvht38(2Qm<~(d)#g}^G z@tFg>*+m9SUmu^RAt1;5S@p}=ACV18nY;jyI`D3D!LK@f6L@(iZ?2Ma3G>=hjoi4k zM@qe$1J;&)4My18UHPka$COjqT8Ho{=gh8viy_snu>X4sHl0~eXIU`ddcl|c@i5$b z(#Ii`dgJZ>txnRrLz`-I<0j`Cma&v3;sOIKYryqt>+c}E*!P&&d2zYCHp}k(Ig-nT zi#oR=jA*pj3yTJsc0bISZ;ztn!qS~yz{zFVsZ`Pt+#9G)v~9a-P5@IeedZeW42-OB z_Qv`x+SGdyesCSQ%FrU(>eYO&aeV`dQHL2p60JTw6H%BCXdZ?g6F&X=D!B0{6^|;i zZ$z3cH(%)$BNi)#JDi(w&AXMHO&#o0%Az&B%H^id6zN!}T3fA0 zygW8{fACFjq5rUi{H|twyCeC|4m~0ZWl-bC;k4;JopN1u0n@{|w_USgP4r4iqgMOz zd2Kz4UgmI-*(nTrBu)CJX$+9vg6ExLX2(y5#X(jzztUtxCUBiHMES)L+q~3U#k3!J zUU+@5yTg7kmHf%W8%K@HT!LPs4}!;oQ#$)C>2LI>A%5-o?(r_HHj&+sbn3yRnZq7F zD+`YA)zG8kzc&3k^@~Z!e6<=++g-^>J#lTzn9f#>VZOBFu=44pa{pbN>b9p7)Q->p ze8`9)9UV1mgNoAoIp{!8t67&<6=?(Y(AhtG&~~zKJ!7%o%+%FGEw`TG6ru zd4K)+MjGTXW!zboN)(a=UfaoHQccrywg$ zbeQO0lfx1_I99N(j&Q^}5m^tGkAi=!rRfA1(T8i+66p^s_@kNv)s4a>=zDz+;U{6v z;X0n)_C1lZ*h2Rw(A~JM?}l&5$uv~^Lzn2xECsga^DOKF+kunIX^ReAiumGeR*hhX zZ*sEwd(b<_oP#4}v))G8T}bN8cLh`n|7Wu%U3TawIE6g7j@4gRW8V1WqMwDwBLK_9 zKFF|@(8@Xc?rTqlRcAkBzE5}B)2g7GL$SOmu2XhIddipT2yrW3+0K7YN-}eRR93F! zLy_VOf!@I12;4pE$RthRWQ{olld)u10iTyX{~Ld`xtBvVw2#YKIC(xgp|z+qzBXcu!V zPao{%i8Zf1kJccZqaVOEi&G17Go`CYU*Rm&AJ9?B;nBxq=L`N)(a;N!AFlSnYAbl- zI62PJH{1z1YEQw3bX(iaQqug@byLS%96M!$iTT9a8*oo;SxoGOS-|3v#mnqCw^XPG zh>K-(j46l>nAfay=6Q9o-jQbeBPaU(Wu*9HEoNGFS)Cgl3E2|W1L??6?xo#NAbS3} zO@?$TH^eZsNyB28TUN1@8@@@%Wd?Q)i_HsuTp?JT17Sco(3;gh889OJJ+Aa^5%+b} z!6jyBWMm{R1~E{^etZ7!Y(q=!qc!@m#vB*Tig>fbE2XnHP6$?;z4<&t!41dy-u?NK zGAi{?j*waEa-e$jORLwGtCpv93wi#VI1@+>>o7L+^9r6g6g&YaM_04SF)7TdJ%jV|LJDk&l1;A9R9t1R76Y2#>2}$ z2vk|0duW!mH{lhQJj=q9kiCiCfkxh%l~-_0Y+1W+{&$EnLuo&|{AYr(eY0^dvy#2c zhqg8QTQ99|oSTq`UiIk<+q7Bpk{Tk!%shp!>%P6DHhxIel)zt&8Drtc!adb!2U-KP znv8J&+N|o!z*BRFJuz0xLjXI~ngt#OcA$Sl?!~O*ueJDLzd$KA`#j2DUc0&PNQJDW zy7n0rF4t*mX;smdx)A2|*bR|K7-E|t_o08er#kF;&t|XB6P*>4S|bc%>~n4=)v$Re zyRkRE+p9~SwjmqaKtdVxKt0dPdaF_lH?CW0Y-I$J?7oqy5Pi+ueYu}rx!i_fKPuq~)SGG*>((rBrsdpg z<-x=Ab6q)dfxTpU7NvJ=W!YfNe}zKByPWjzz27+aE4ycY#gO7|lK+Ml%veht_X8|m z&g#^a*R)vcJb%a>T(Nr0KX3N+^&ne^-8v_(a9-7r>Gi5l!k#F7!IyE&pO}^|;g3b|;`*ye z6DyX0j?GJ&T0>j&R=p9~amRgb(Vbn6tVX}#ZG7{4bRnE$4A{I5lrarX z=cD%H5IvrG;=aq1*Ot9`khB0- zLmJd+Ez&r?)oI%{F5~Ww!Ur>Z9A}zgu9DwhqJpJHW9vS)<{c15R;9aAlvqY^AxiOK zF}6zbeq`q8mH|-mwf5kLN9?AHYJBHL7f)Q+jkB}82ir~^&D(Tizz%U(;fqR|7r{0D zdIHC~GuCI@EIv0phLWT)G%X}TyHpia&{tj3m+_4~O1wmt&0K<9W;olqNDo9fjFzuL zgIMskxo|-BLsJKJa(QLtLm?@ed+G3)Y6exJ!n4qQrnoEGwoAn0Xvp$N+mAbGU5x<4 zYRV1r+KOaFb9&y|Ajb?($OdFr`;Pc%UID+BmlMu*Uf(*4zMi}KKe#$IK!cR+?u#Wc z6kY7-U^^Y8W2B7ch>Na;|EjXRNoVWE@wNB!F%_=Gta9n}7SFFgtDb(k_4cZ-?X(O4 z)9RMDL^OF=$Bp}lVffFjihXiQ9H#X1GqaSicy}5pSR9u>jj><6P)ol>1+G8;5F-g6 zDF3{{jucDn8BOmW1m9}gs>swt-lRo?Y|TZVtJVg-kttVcuzS!Ny?MF{IWt1xrVE%^ z*}5_(V7OIbF_lTOCaEi)Anmd26U3Rf`Qq(}R?2foti;7I`EsF+sA~?;YuJ(mz1Ogx?b){RPAe|B2=&{utt;Aa&vyUR~8n_T^$H6d1om-p$XSmG#<%J*t=F-Bc#d82*sPHlHJm(>3!$r&M{1|v zk;i9p^STFB9cBUWNn!(jeMh9{ zSpFlq+`6)yTt%(9A*UGnrA#JNsdK%xz%v$hN{%Vfeupi9-Jggc&Fdd zmSHcRQ(44Z+N|6lRN^k)CnQw}hRo{LIkOw)8?|dZ+tkuhDmblBZVop9Bdv$k)$a*Mj|HNSH+uL`S; zgTd4V3O1>7C@VO6^yGfZ`$lI9S&v=V4qVMX`=d)ry|5p`qdIhIjWbUcmkVH}sEO?n<@6x-l2U1xgTg;*c zR!$A`ka%-M*1>p4sl@*!QT-q=|5v#`)o5i zmc?K-07W&Xm}Oh%(&xI$oS7t}5oNe7`Qv0-=+UyYIteLxm8#&-UVZa|;BoK0RklHG zxI1pv5IB7PR&WrD7FVrwc=)mpcO(j`Wk%)t~9&Bz96Cz)gA(= zcSxX=hNA&>f+SHCp#rg@S>$yv2mxX*rw&cc*T`p;?I|S4 zW2 zBf%|R=8krM;Vw42AVe#MB*Eg9HkU*I(M7f9E*NpACgE`0`r{9=ao`cRfO&#GG)Ud= zZ+rC7Q>V8CJmyYbpy>p8$pm~_?Xbenow)uCG|T0e>6CP-I}oRH{e$)avlwg#CKWP~ zt}Z1QviBjai;m7rkc|~b;I9{@Q@=)Qq;dnSf=c~raQT*wPR19n9YM?zZyWHK!UYz- z?TF1-k>$}IZ*znNH7{B5+E{{8BP^Baz7Gpq@WEP3tEf+5AnnR`$9ij(^G=krop1{> zgkxAf{C9+cjjAIeUYd~)`~QJkV%o1FZSVeZv`w(g87J!3QFzyJ8)uv@mvALyj1e=c ze+P*HxnovQ-dvL%auwIVYNBMt6%u4EdNBV1^Fvz!`Dg6AGHV^WvYGmC8Nity|4;4n zljl&zbWJ;?DSQ#tOmL+4KxgF94*TvfcEj_%T$?0op*anm=9IgaKd^vSENXrGXs`!QI~`YGl$i7D81ePW#**LWGSlspIoY>Tc? zUl0y1;}#xcM1uQ(Xsd!BR&aI8VaML}lRKKHQc@#XEEjxsZEdBB>hl5UAU0(iD5g*= zGHHu^FO!)Qka+AL|LmaH6&@3U(Kucw=yx9Hu^Zc1W!bl1oN9<>(JkMO*o;m1i5}xP zz_uT_-RPU11PFO@Vq#8jeE~+@#MucftiEI}PB+Uw4pc>YsKpLouIDXFz@}Z&U*1$bG)|akcA@ z5LVh=XLTn7!YAZy$x1nDt=5R9+YRW&E9hwjm-0dI@xdn|H%QN}z4Wy^1OAPjcarAB zaB(Nc+k^mpMPO67%uAT#7h5PRq0K5gHtM1wZx~e;Psb_8$TjWHN2DN+mbSiBB*KQPEc?=n+D7;<$xBbf&Z`?tllfkvSz^4}IFMB^g;b0IW1CHv z@!dg6#K=p&Ix(I;sDQ$Y3Sc2@IF~u<9d>MmKZ6q`2lJ!D&)D5LQYVwgxp8vku9s@) z;KED(E$XmwMBQKnG5oPH{Vg$PCVLzr$Vl$Ms&uRm>zP(g$7a$0ZfENVhd`bi6E4_F zkdLS2D6wrfoXh@PR;!2SR9AyMVb1ywDkn}T3ceqCq^5>w&5ASTMiBDs7GZ)G$cgooxm1wJ}N3E%?wq_XUg!MIP+ zj3SLVNJQp;_JBk3whDU7?GHq}9?nfKC>7$qdv4<4$(PAMtS7JV%tH+jdxRt~gkpR( z(kFY-y;()?%@J_vtt3;8BmX}S4CICA*6-WS>vN;TmiI7PtR5bIR$TX6SDxZRIxDV3 z2{K3Cq|gk+WApN26sDq`QhUgt!oN$OQ%^^WDY8VvyQqP6dQ8AwA~WMqEb6aD0RP=$ zcQ^Rl=Mdp<9Oj`p1_e!7Oi44{ zdV0fJ)%9GyS9$OW2v@ufG>fT=-{w=(g45;$^n%zNtKxk8H+K0B`gZVbAA}J^MrE(m zydKYbTXBUJTy;JrqapbS+$>Mja$$}u0eUB0@`;=ma!4ZSnuE+1OP+Jfl@BUzSG?@( ztNLioqM^<#a%*?V))d`qmm%u5i7VEL!}0hN{(To)LB1sR*o+>Mp(?@Kn?B%E;!ax* z8hfulTG;N>)K(UWF8+f7^y&gAD)=u?W@rg5;rzVNB zwmd&#b=vFtszf#xJL!O_i7s41aa>GCk8m$ZaVHN>vW9OahM8oy+r-0= z>&${b3ofd&cp-ekmdWFziH9qy#_ePcIQQL1XY^Y8q|YMBIs74ywLfXhuT<)(QdQ2E z{f~1~GwSM!U2ynS-ETy&VBO)0gx;5d}~M+ouZ${rg52r~fyr$(=qz#=eSStx^{IKq)j zUS%dsdSw-T6(2j`yT_HYNB+M@wxUpD`S@FcGj4eCdze7Vj@8aYDlbHUx|pJetcLQlS^bo<$d>My9F% zL1~;V-v#b#vkCp&3Z1bWxH!h_+NhR>XZK73tUmQ$32llyi5+Q?$pIRpXatI0ESm}$ zgsf*lOt1jarEsWwe2-oiXUpciIH*n%`u73zrFMQ?$Ui|G{hL%-`yZv*4JjxqZ2jZJ z`LQ)?L8~O@o9??K4&EvuwFg^`Up-S(lT_CwWv8-EpZC|v<7w87fvbx8z;1J&aSna= z1;XLt7=PCt5yYGV&Q(tvJl1~;%7IN(AJ3q{HiBfBP+s-L#s5$^Il3*gqW9%Y>#lQV z@o3UDh4VI>sLuy)l!iirut~EagGUZ6|19#cn|1O$DxE4MJ*S{Lqj4Xgx|%(;60yE! zjO%YM1bv5(=FPS^945C;{|V^Maq30%=ie!oE6f~>j4*&(Up*%J6phonDSr{)05ATe z;&#u=PA3QTPwd?f!O68)v)qwjD}!qJ1&B5`$(oJKyiCehvQ}UJ4F2xY@-2R0q&?)j zLX6^5Tiv=3jvlUD-PCo0RXgpnDly51XN>xQT`&^|Qp|SBU9W9vTX1E3A*hS>YIXvGeKT zk!{OciHmxuekCRy{Z(99!tu+2MH#mJ&=No^F?VFm{eA_!otK+IV<~3}C zOddMz%h@;7oyp-3JC;kn+>fs=>G-;=QY=+PXb-;r-4HAyn@+4WQSuNxV|~M$x%N^L z#&Hd(eJdeCu}#SoDispN``2QY^c8 z+@OmD)UOg(xs{>JS-V#!+ze|R6f;?DuVydLm~+Qm2O}#wyM;R%lDp&hMp@F9DZ{*X z^X17<jiw&W<7UUk0k@@2}a_I#PI!-{OlnC=$3etwTG3UNp?K6APfYpDqs$_ zEF0JC0w>!nS{y2;}YmXbhs8BB}dyx70y`%QR;T2Gtw%c zKdVq=tjlkHzV~Osm3^d_$-EBP{(`+`vhjSk)+h2WPMIUc-;V<`OMSFnA5m}8T#f#Q zpP3B~sOl3%RA2LiYqZzIvDtSS9pn1zGo4L&^EqMPholZ{9}8tga4jP(P4#|Zi-5D7n{_ybBY?S(h1{h3g`qt?Sg&h zA!SorQhxExMN!6xk*i5GI@emKw*h+mytF{x(G_Z5NHS&0KLiWAt6)q*XZ=Xlv?#SB z-#Tj=iK=XRxlA_BQ8%L*Y~uWW-w-d|@Yq!A_RsMgU=7Uo&w0!izy5@F(hJ{x@_(4H zVm)atBI4qtHO!i+<=Nib^9dN zEVA^t`r0{p7r~#>lKjD{I)R9_n8oJk1sDdkH4JWjX3U*AU7w&5E0QoE2)1H_D6CN-fxZa{nLV2Pnv zv&x>0*R&&GbGPj>rCT4~yFp3kAFed3ksAH*w%MciPqK8=G{rTCrO^C1_Ys@7Ez7ID zBG_p+2$?NDwjqh@&Q*A{tJ+zes07I=JjOQe0Xr=1#Br;3pi>bSgL5SQqO$H8h?IqY zt*2ZGzxZRJo#qN|OWURIH0X^N#@l$(k_r{tH3s?45ZQ$jQ!94js-00v9&9n6l*2kS z@syFdt@=pVjv^=5efz&CI0CAf^DDYYK{HlDnDJxrcx_Ts+e%CRVw8fvR?%Mc3WW_$ z&1P&@aX5E92HZV%u#t0MBD7fQgOcx_yt9un6Of*28-EJR_|clJBoSlXPFGZtZ&n07 zTPNIv$-}Z?o zVGl!BV;3yp$}&;l9hXo&&weg=C^)Tlh%bearfN7$eaC5tehNmDM>*)~zEBVZBsTaK zQa-EuUgj)9Z{}6JA;n!7(DvLjIGzm$H%5Ov)nJ{K!6I!kGt_KxX>6s*uCr~tWC=sa z**ngkN|JtCvd3ZVCZmWN5qTzczepzkPcOga0Kc1dsRzB;&S!S*^8FvDma{PB!01E3 zOjyGGc*OTUu{+O!YaGc%&|3L``*+tEnoXY@8*b^6W5`utdq;AIO<~Y$hZ%4U9l)jW zCM{m(I&WxAT=1Q4gv1ziNkCVf3%dwQtt=|qQAc@E66c>>AvlFD>I4Ep2yglfBVD?0 zV`W6Z_a65cs8B5bSA*PLpS#%>5WPMeqHax7XVz@)i4S{7-6*}_NDPV0?9PNMH|#Ss zdQEdh$waT~p-q(j0n^-&2_IwS68L%eXtbp^c}gE3g9{0oFJ{T?uyMM$6Jur9ij=Cj zDC384h;!}0wUXLX&TBU1!nM zW0?x2e~8eG$c21^-z-Z1Bb1_3UzY~|8aO%{i(8V_Y_9MrtPy3CGD7g{`^KgpmLBF`C<=K{VOB~KU7BTbia%6escfaY%Y!^s&hCyUQw)ib{Loaf}_Vlq~ zHC0g}CUQjifQBE4^OVY;T!@lo!$nyO#-KSqybb;1$<(yzPQLsn?CD&3x7XwQx-h!j z&lY?I@$@Z1O^(99%3GDu7<$q%CJ(r~19VJ1tmJ%K1`SMEf8&N_;;uTOO&BRQoXc)E z=Z9wc%c9-b^w0) zX+=atXG3{>Y2vBJuFJHWd%gtf$Iq``N1lDkY3i&lEC8I&(hlCRr-rBD?oSV%K1B}) zrPJd1hu}Q^fz$+cclzc7lflJ$m8)Hk)qE*sw?o+~%UNlwy5Tf^-mf-k7brI) zA|Mt<$oKCISw--n3&Dp96#JSB?djt~gP3>i`7eaHRmC@h1RG>p#lXtEQvwugn-8!Z z$Yj{z$R^CZ5dj*kRS)NOBo|BoJ$3?&`e~zTN{PwCls;2b?J2&+5$O{->s}m)cYirM zQG1GTPuQtdEtZ&ha~+4=k-b~SjNSR4)+3aAf;pR zduENtPsW>dZgYraFezZTKgAYgK zO2i|8ES%wILc6x71>)OBgHRUkQq58P<>{8%&|Pq9C~c9`u}F?gG6P6q`fSIlfrJPn zYGnM58nsEFop_hMc$Xp|r{2T*4@)hNpLNU|^{`c+!nPEk|HLqnjIS$NWq1~Kv7ljL zs>k15@d|$R2S2U%rfyiE`^$qJ*FU~{GajExt#E<8+x7qKT{v>0%KO}RCnvM-8x?$H z^zC3G;T6M?_Is2%ZN}W8r&c{`gsGp(qo+~K9F51IISdq9*{0?qA;)>)MNO2P1zjqi z$~xe<8!)Q!K?T>86UJ3PuHZsuhPH2I4tBG$3Ztz5KvyZROp4s?$(8`tBxS+`pQ|1= z`2=Jc5fPB!-0hx!p6-YGB+#?Uh>Cs&kb570VItFb{e9;k^=ETrfYS~e`f-O%vj#Tm zWP+_4o#XP^dxD1yjD6-@#pk(uJ(MS=XKCLj$;qcTx>t~?mgQAd7sc={gs%f*2eJ_e z-!-j7?@s;@Yw#H$HAotyb>P|ophk9Yk!CBE3yUKge(n{-F0M_gdp|Y`?Q?u*pk07J z8_4w^#UMhO8*(hzV5svAAa?LZNBZEqKo*805wTEt*Y3SLUMBalRZu*f&V|3#!+Lej z6=BjN#UPHAjRP4H#PB#?k`^J;uT&xHT%4Ky(xLP5pP z{j^J8_)0col5)F1D+!i*lKvz6wFi8+OpD+62Zrnb06@4N+WX4OGygY~s557BgXE#z3 z`ui+TC7zHqclP9(&soc|YFiAlc-z98EW}8pv>wCMbaW(b2x^fL*A?-lSV%m1!#3bK z`(BZX2K&ALprS?pSGU~}6A%=8Vr1l1g6QcXCgIwlZ(t+G=5$k5CqXQrU4J#jQ_y2e zttZA><*<~Z9;1>@Ad@7FB3&WrM9eMrPR@<4wY-nP3Mm$z>jOOm^Vh`6>QB}?aH}Ie zn$1}?AtkXZTTSQ}QE+`^j=db@*@;-CsQI~Rkt$KaB->jB>nBs%bjupN4L<~-$E#;6 z2fq2AzdBia>a#Up4k^22Yk9R?Sgh@3wcwSrTJ`q7TE}FkOFG8letNUvx(SYTb{92n zq0)f}-A#7^n~-BCM^aS&7IF+XbuPDd2$-oT%-uwA_UdYUOSfB8SVB@~udX5Msdc}I*=e{#3H zdUonz!byjRr{ZU=0h1eqeIQ@>f1uknbV>>=L?7{Qag+IZA>_#F{{eAVAw;ZPrqhe- znxWzY(c|2|KanV>-iSbF0^uSF5t1Mmdn4N-l2{&dj_~K_)-AQq86MW#yv`LWZzD|| z_LhE65QAkWU;-}kZ$=7&kd6SwCo$UFoDHPI9Sy8f?x)9nkD~@i$?8V$t_`;%q=Xe1 z;!K7u%9w-Ar(atV!7cTP(gwkdn@$wr-+!?I_q^~02LJ=YThc~I`9lx0iBj?FLT zL_{FLu63V#X2-5!6du*fcf}?1vu3b`^_bOFDvpG+dr9Y$5*HE2dQ!(gb%;R#C?)0j$x!zPNg|;1^lg< zX>W5tzZ3sJbbk+%Fls4ja~EOreofZMd_9@_Y$z8hsJ=&aRH!?IeiprcY5{ujhzL~s z@17TVbZujIC!R2N*RdTyavtmFgnFliL2-F=s1$0D&bcDr=mmPZ2JM>OZ0q7Br_)hW71Q-^q#jwUWj1T;}k0+^yZ{-Ma38wCFN zF2|2s02iT!dTB3B*^8N>&~*Q0Wnu295rx;4(%;v~G0bHcyu5Zo+8jp>vRKdHG?b0h zyLRHAd5bB%W_#83Am^&}JFv}Muw#(Ik>eM{I{E?%5{X8Ftgdwj(6W-Oxt>(|Ju~Xx zhjX`w3_a_(thEHQ^3Cu%yKM!#LXyM6yluPWlIDZr*#&K(u~kb;=xT|4?GGrCK}#9Q z)WedIQ$r37``i9IymA|0fWCoR7aL8ck(0n>b>iLJ0J4L|g%thpGJ~*O^&Wqzz3rDY4nYry3KQ!{-J zF=Tq$Ic9}qbm-!0TQ#l5UggwY!MR-qyQAdFg6zOIE;kR9trqnoE64wyUyrYDYwqma zqVbTqw3`RBGq5=L<%*%w`>P6wtt69K-%m-)tLQSll`8cE_D8r<1Zpduzh+8!AqY|h z2nAVkdpO_-ujq;|7B)WDsq5W1lB#W#MeIoG2EEIYql~Bqcck>48PAr<$ks-w$X^-= ze!9@OwCOg>qIHeknGjD%EmONbdW5eK%pHFWp`kB-F)fk4dX*_O zqft2hpWNVDa6|U~#tGQTt^I1~_8<9M+wCmdsY`rm6Sw*+Ovu_?CbE_Uw%P)nI!lx` zs}50xIbI;{W9=LR^9cgx%a-#pud7CsNbGx1pAjQ_pzi&u&jz-<0VJxUO^s0MzQdyw z+RU(~fi~0~nhL@zPU8v50!55=|GQVL)=#t0rx#hwt!Fwjzo%|b#D$TF-U9}T; zZO`|Gl=t&hak;zSBFeRKi^mpdqZ0IZ-Lq6-LO8V|e9)N=A%>yjg<9Y_^cwBGtbc-6 zx+BNt6NOPnW!aa=z}GD_>3Ukv{BT#&e0N9o#3w9D2lJ+_723Ln#dpjxmtdV1h4v?a z7?gC&y9#L##6w!x7jk^@N;LT+RO3!*A)defs`_8Q2SxU0{PxWr0@)V4ImQ0Tr~LM0 z!e%_2{Ff!Lt7ep9arcYNomes&2mii~!;D_U03>D<{MtK!n5cXxHZVJDOL(@bg2REV zVWdNmZ66sfZIHX6A3y+|5VSh?WStjQgKkvo*JkGCj8*DRzN3a9Qa5XQdfiprHr@y2 zQN^9E3ArNJ1d%0SX35AenH~UH`3qEnm2bk*fPx7(8L-)|;?x5v{>IPtR#`hzo27}bPsjU2Uq1hByaVN! z7pSP$#P=9){3YJ|i?fR!^QdXgxG(jGpGub2J@?{g*L5c71V1Q6+!-Y4LF+|y-W@M&$*C`${zGhY?M zzDMLZ;DnxInEr(^BGG0j_cn1B6{-opd@0k74D!kvjpJBf;Atp-*$HHiu^Fu9#WAmo;r=tKZNNZ2_w`j9(6#z>~#$}_I)|q>bHW^BmZZ93?!Efbt`mFuGTbx z>#3atIjv*;0)Ks#?9wA8ammNMPuhsb#DGw*S;-_p2OI_^Q>-w4Wcw0Zt*3tIqh2>( z8Bdrc%Cunl`Dyv`vwz*s>ag6U>f%VzfHqdph39JnxxCjwmv;~-%0HelhTG2A}?yR&lM0G0O&xlxN#}zswSIx_t19I#;unwoN zw5J{`&H5ffUtxEaW$p5bkU>Rg$>HU_f9kT#V^jwRk}v|5r0(#r zJzR3O;5*rI$^Nv%aAb2^$HG<~m-^=%qNu00D z3|(*xxDwDia~n+dKT#d%gZ>y@#6?+-A3>qjb!`<4fcu(n4H+^^?PY#&c?160!9c zPROn>+XR}inlbX7%kd*S(t^Y7R~}ctsVM)ph^D(iH3uO`NcZ2N4h8rhW0{M$Wv*cr z-uUmqZ?|Ro8wo-^n(yo~1vKU}A70F(1b$Zm;?*EaP_(Ij7Imn0Ca?2ftMD*-)m5R6HwgAB$%GWJE!^`r8Kd#mN( z1*7!G$rpTo39pX^kB=Atej!EGCaj}kBo8nTP&3Kb$D~v+&q5N~euv|Nc~HH)zhz)? zt`M44ZG}oOOnNt@0SG2_avr`7fN0~?h+6oKqzaTqFv#TYdCdCY-B*neUs#))6MiWb zA>y(~6^$G*!JJv$!>_z-nd8hUT@6&16Bj3*4!V4^K7q}hWI&SKksZvjUH|QH6B#b0 zLnF7zA0aI9{MB#@KwN`6UJ|>o(qUsMXhr1b&!~GZrL(KqL7li86lv&jPy4BO_YtHk zt82c<>Prxj@o(b;5SDrVg>wk$LDAK@t|=Y$?#Ex_MgQvWBpzdj!;UQjl3m@urxCSi zy(36{-K-v1vs7ie))!@i1(JMDv&vvOI3z+9U#=yL>`to=hoAgt-J@<{G#2npq#MkT zcabc?iG87s_T>;)OOAo+Eq|?3QMY?NHB(z=WITMJSadkx>!DL|rZmK7dI@;@c8yVI zE6~i^ZdNyumz}Ee{#51rW-@p9Dy^5#-94gq6FNkQgTX!jhjiq|*;TZs^cLYh*?jQr z-J)X{unJ+ex7cG_ah6&3H#x*(;Y168p19Y|s?vuz=tv zI2n!Q;1e=poePQ@e(S1BHnj_TY;C`#<9_iyQbUvkNju0k{cY5EK*EcqC?KjUyUrGt znC9W1X=|ZzE3 zcFWRt>OJP;-d@{;xLlYk;13h3G?sgXt^~d&i)3vEj!okdD#+9IL6Yzk%E55N_9{bd zG3p;|go40mj#+@RZOH0_C=Rv*B8aDXt#;@WnLa^`6#WNLhc79(z9cYCI-xit@wQd(cAgyR=c4h^JfdyCMifa3Xhfx`_8^a4H$!uBi;Ej8dGg# zLaZ;~P1zz;!B=_&bvwhh&!~c*@tak^%GT6HdqOQ zLA21&7df|tUv>4gseuV1J`)#J-4mJPU~rZg=>E8H)iP9(w6ZN(Jaub-S4d^N(dO&; zba(I{s8%w*5F17-x?evA)1`VNDC8aDyGnMjRtWA$O=FcL7oOZ?Y*Ql)<0cN3ci{=B z|D%<1^pqptw*)lRsNq2EG7PvbkLp~ZCt5pnL!S9-mRbjej?q#wW4MgrXp;bnMv^6b z%2r=PryMU?*k2MB3cHw7%+bgoa)oZc*zLIe!!6tPkft}*wk&pd zQ4Q$8_+28l$wVNoS4^I=QFyG7$7+%Sb&q7~9-Wq8JAfLK!(qWrS|~tFA9h63-qyir zd>*41wxc3}ISy(HgTSYgY+BL81-#1%HW`ekQ}o_%LSV+rQf1O;@4htVr#iYYmZ_HK z<`%9%Lvh804w0>jJ;r!Ro-1rkbYlohqs$S)$l@ki_UsYe^mi67NIX+wX z(5>rh&gZe4Txy1)y;7oneIqiLCRiNU?C=aZ^YMlSK-!@)P|CL0-J*+FY)w;#i%4)64 z_B7+L=3>%@kdvl2ytPTbO6KK;rZ;{t)!s~Zn|rakwiNc)xsuuP#fO36=yc^^Gh1Tpx}-*s4+R$Q8U z7G$-NGKOvXGeO5;WQppN#~2<*J9&+Y&WqL= zllh&C_Ipw){cVcaAwg1nF`;ZqJ709fT-eb;c7Bz(mKhQ8V+J4ere!q;E9G1L9ysAH zy|v$fWw_(dQc}ApsVL>3>knm9X&&LB+#j-7oB+UbBI|hG=j$YjvOOHSp}1VX>lgbHC4zOxm`P zlUURJ>C~bAM`t`kQzg_v&t6u7XU} zA0$(iBDjh0q`7@(K9?sckj1D+({STo;BH6kAu;Y9%Pa7=(Yh6XEaeszbEYrSsFPK0 zhhLltnZ1;)fCzE1wcsv9ry-FBOl4JNKgCpQmR9u?Q?Nx)ZH(g_7de0WHO3Lf%X#R4s*Bvq^hSVE4Jeyj0xVTn8!;h(Z4$QYMB_{O45lHG757^vH?AFVVmQyDn_8gRi4nVg0JI-jULic>x)ikc*A~NmKc32}q8lI*tivV;C&5v8~&MPltU!>ADt$VLjole z5p6;`R=H_)lJWAy8`jrB9WgaX*6-DT-R|aawDVl)(+j`HV?Evux2#Q6pl0Ru%P6*@ zq&&tU2r5bW&KNGUnUd}7Vsi|>Ey_bvdCzV+f@;2u zkCC(HD9@wd;j0NlZpMjQO#}A}bWvn<`f;m}Bl*(|X&Y3Avj89&J&@TRdYZjZ?wILcpJu;hp8P}o0-RvweU29|T8!$&S&#c-yi$Z@2H!>n z!WZ*jqHwBeQ2s({Acf+{y%#x5&1vO33oMpn`TK*6$M{pvc1+*j$LM47A`yO%ndoLd zjFbEu__m1dEdvI1&1E^!Uu|SX^);LSFTLm{XT7KbVxWp>f*hNEMsCXGO;1?+<#+Vv zCRN|RvHI^OV|JLlH1Z-D1Ce|fi2UD9Vsp~cYp*o#e@Z}_58}G_uN5`;598IcYGhmE zbu^pdZyr|^2Z8wX4sTMK05oAU7WdW({c8Br!^VaW>DU%F7lOt7C&J9)#mA^~!)HG; z2R;mob3Wys(9{Ax{HyrTTCm&iUKih-jJ_U)C~A1dO>7zG!pQ?&VE6sIg131PLp=5i8 z(NQ^AcruYW_V@JLyOM*;u&WoaNu&FGci<1P&6y>@Kap4=X!C^ctXF)dc99j=#Qtc#l_1VD z!S@+WqX6<4-At1U?w`Ez0%&?lz*&@~=ee~M`=SvRyh)GeQ#ttl@5aL}eu%&AbHS@#EjB^AFmacYu=5 z9X`fe;Fo{A&kRMED=g^d!PHzRQJ1UhYBn>(9GyRIEz_2UKzBM2Z|4~d*@jAs=LV(t z8?N@d!nEXzT3Q7H{4iJ<3c@RU9dUG&jB6d0SGbg|aTWJ+}DerEa&|4J3LFZR= zy0apLSUIod+JSRyp z`+}tr1q43!0ABKlY5%9#-raZmYb$<7c>;V{Z5Q@2sMmuMSNZDip0+MZAqf-oz?5u6 zyzEaCduLfvvya_X1wGHISE1# z3~3NPYvKNG&gV3uO6F+LfRKq27Q3YO&-z@(hT#(}2e!7OUgrpV551`;jGgD!6#S_K z>6-i8bQ;AF0x--&q}{s=(GyMwCsT9n6qG<4#ywF$!th5c;E!-cOrHdeJF7S0b7?cr zYp+C|1NUPt2nYy8X`l~ZmCB6;I79Y{sI#p&7xO2yx6jxYyewhu%ba^S96vQ+oKOb+ zkBzVLV80y@XrC>q%{v~L`V|RNyW3a%_)Lr!Ru*M-%4UvetevX8aR#Z?e0^50|5xJU z7_~Zm<5?gt@7bK{!jJ1#P-u-=uZCCl8Mg8a2_8pJuLK!DZ1P&v{Gd)7NCz_yh| zR^a7WUJ!SPqtS#yjvn*t$4{)c;7<*0Hmt5(>mp(by_Ge{e# za)A5n)cSnI`2XSFtr=_M@F16-rWr(W%RH`c{il?C8CbqIUJ=*ro7GbaF-xp(Eueg2 z;xw+O7*&;&2j<~vgV9@Cb-A|rqYgWvh($@q$ zRT4nO3zX!sO;6?A5S9s|p*Aqkq)6ksbH3EXjGPhrI$t!Y&-BI7SsImG;aSBlr17Yf zz5C#P7h*LML{lW$8q6@J9jX~sREpAf3L5uEl6f&|7U>nOn7t+PXIV2U`HBxrcUfvS zExKT~qgAr{xkG4?>_#6^Lw6+l(x*vq@s05*03@s`d+&na3QPSE=k|IkCBN<{CqOp(pUa$8! zpZmTazu)_h^QcaldB2uxd0x-!d2#Z-!XO}hG#I)53b1YNvTw)7o&~J1v?9lS7j$LZ zZVj$dPQ0*zwcRm5j^)A4_hW5?0%N5OOejp$6<{Vt_I*F(|5FMt9J9YBKw%{8`sIW*ff z(TXI*AGGnw`$HUY^nMOUJuL9b#MJW>TtOD$&u^ZqOAFg>gcw9oylt0PE2LTe?Nv1TZNKU_>n%~jokozBZhV&#pT(Q!j&B+bKvZ2ka$XL&zE zpS^;n=aDTA|7uk+1i@q0ee+^D&*MDCPNX%PuZ&G){TpNeBPpQ`eLWelD?AixcVu%@ zf0*(E4PQu|xV4qY45p89;(8@LcKhroW>wsnuWUPZaFE@^0JH^@9j6dEasCL)nCGN% zk1Z~IK&1jl{p`8dY2y`H#QzvBjYM%JXjAOCi#R~6XwucNA*ncwX7SZ1&W}gc34B`D zm4K_x*JBo}5EY|_!Y5O=y>7MU=;)7y&_T}P&8YCf4A%L#JY0uy-4+Qtu;*!^&`hQ* zr>Veq9BU}6`j&EK#tp9WLlJYLE01{t)v_!rdDH=f`Umsv+3O?P|(-u_kT0op8F(#AIb^ z1$B+X(|Ev;YZGG24!0HcXVaumfLbl~D-<4(UxJGfY2>`I(a}mKu_c3*TFY}6T$QGZ zbd0PR($f|}p|_*P=bu`0cc0UqivYfC0IVH|uW}|jZ#`XjnOmM~<~&M$NzCJZju#oq zaKEIr9~J|H0mO|2$CjK!ty9OGReo*2cLWz+9^8!U{h@#w{qpU-b0H~{I1=^?ENoIO z#ze!hb;U)BFgCoOh9SR0u;tz+skbf8JGp9V2&zY?CedtepAn`#j^1T~ie03GPUgEQrUB!O_6C*1@5 zhYu0i6~u)2?>+Rqey7@meX0X$!9d6o%c~t>9TML!5CPwBKRX`b%;B3cFlG9-Q|pzl(ySEWRx>hF7UeI(tnRn$=Hk=ip$BW zg7fjLqmfk>C+DRSghfA){ECf-yh{oCoI~4#tZRwr%w}9d2bhrDp%H=XEiz z;xhKs?nbpwWxE{kc7B!{>I$C!RHQ*%qp*xn??a5WKh9-iN8=!}+l(MvOvL8=@AqB8 z&mP9Y)y#h8HxIfU{r@fjyU2)YVoFOZAfto&%U*_Sfc?m0Wq!OMa=GqroZC1O#xn6FSQSF@Ha?1zd_g%ToE4&r7WFIrh}+41Eky|*<>4J49G~a(DjgyE6P!N%1i+A znC|kLSOH`Po^UUW5g}3p(JDjrsEY85=Ei=*YnP8+4oy*b=my9T4~6CL^HF*LuO)flhu~-c|HE^%5bH97pfLbLP=_}DK>Qfj!}gVf z^%nDQb2Sd50$(WJd7Qp{f`}aE*@k49&ib7aM?u)93y;IZK z4YvdF>Wzk|u-`WpsI~DiW*HJ|d8CpBlzYJl*of(=BVqOo*5hv`Vc!!QVQny%WByZ; zbC@WMX+ez%Z?;ni)<0ceTjFb4nd>Kg{;3-kK)Y;G!smVvHYf{Wkd7fo`pQ!TkPHmh zeR&0S46fCC0#$0@i}~uD7v4kj%jam}7-8Iye23)=#CF{jcB?XR)FFHAZ^ezTE(3Ug z9(p{Yyhz;$#7#|fX1Yy=)H`ZGo#bmIZA^FbXEwp)%JygP>@UKa+v(&aNuRt#xgPJ+ zV4MNAQSbFcPZRARGnBDjTqeZcin~4q&0x&b1&SxZ7qW)@`m6U>s=>%y01Fb?cD(}f zhG!jS1iu8gZIFw?Q+(Cq!$68tt^wBu)M~MCmIFuc!2+0}t!vy)8}6VI668o14cI*^ zbeV@D9b=X}E=Jc1jt1wrd=MxDc_@o*c08I$NlKX1`;#o<6i)!RpM&0xIFeu6#&)kr za~Fe+x_B9uimV6WaYpJUbr=Y>dQMdnfLR42!Bb(pFmi_`Z41qDee*9uLR_BZIw7I< zk$+Da)5f-dDi8NC#+*kl9mY;7o!(}-UC7Kd@t>N*MLBz>jq0|n4*2b6G)vN~Th2uN zkjc|MusrT|bauE+Pur@47GSQEWYyhYzd7zNbGB5fGD;{$pFyZbT2sYT=uEvmu37gb z1%8-f*OP@t{4UrPux9v>#1ER#Q+>OPlrhhyhAaP}D}45KO5g#Z7gqc!FRtSnzg+ES zMm#qhnDSuhv0(P1fGOe*PzXuj#$Xedqe*WaxkmyI?GJ{+e33g)_=2Rl2O&3DK5;gn z7FK8&W)tBEc5rQ+!cM0;*C28e*=l)EU2m z2YnYH!lc61b%Ifd2iVh5hObbYZ1@p1lel&qHI%`fN@4Mt9>FjO@rMD<2$;#S|LLs8 zDhzRG4Eqfr>5a}pJxmjr3iG9c9d!o|eTC*ZTyaK+NW`b<{fI$CO;a;F0=0)<%u8sNwe9eyc+i}L1A29h4zgRvQcG%7pE-b@x{*WZ4zf<0P?fs!;K48`BrbJ z*ivnj2t1o3|4(g@d(Mk%AZ|FY0T*L3Q3()m#NL8`0Cjxhj*T`weId? zkNL9aZ5*&k9F5_+6|pDwk7(HE-7Ywasb9-VV zwS=CRcmYx&i-ADlr$DsSapR{Pn&fy$OH3^t7yb(pRF8bM)zsLL0b2wL>@G^?xSy$} z5Wxd;SY~%D=G;((?-T|T8(cC%B~7R5#SNC)iT^P@GS|+oSDZ7{dNlF##+b$NtN@wz zF-PRlU-Y08(m`kd(S|A_cBFqcmtQJt>@euu*V8F4(f;EYwokYgV_9+Nqd3MB?lrbO zv2+JCKL-QZ2VzFSlI4MrA77dr^~FS=pb`pkf@UC71A_>LB)V6b#U+(0G7N44YEd0-@9vZ$$sxt#|taO0FYJzm(^HCE(jDQ~7q0imGc z?YfomdHPLAZbK)jnPTD?xMvDW0;HghzzzR95Fa+4?(oAO_xo z*QqUsF17Qu z7eKnLqGQ2xd5{}dA;~ZSt~556CC!-rU|#z{!+pS&L;KixMB_(n&P#1E@94giY3>h& zHxjbWX}7D*J-eW*zC;t>k?-G5f^s9ALxq+VkW;$RM7Zs1hmD~6uahOvm9D|NcxJsC zSsH_2Z+?EPTRP)c$e;VW%w`{L-7;zl_0~6Ri216dBjsCAt&igS(1EG*jbb=ocj=?m z-&!bSO#$77WgyZki(tkR1R(DMrl$@(Ii9zeS|kWJ@eKWzvzv-otsp(8ObTUu+52G^ za01$+|7AgukvfmYVvVwBzOv|ztb{?<`_{?iAN7|=yFi+fp#{8aXidaoR2`LEo+Oh4 zRNr5-)d5lbx7hB2UmteSc;GY>>-w3Oymy3~*|24`U{~MUOna;?@U;y0Ns|0&Ri;zt zTp6!HAqI?!V-Wbti@FASR}rMRCJSSXB?}oEn1>Dsf3Ts457uvH}qD%3y>%Ygn)V*VKc+H7^eUJC2`NW2XSJb(P{|fd)!n}wEeuS4WNityBDv`xNVuc7 z00_An%kw@_%*Tm`6p#Ll^ow%rL$$OP4mP#*=HjzNRpwQ?J`tG4T64*KGE?qL+mDo-@sI~ zf-W#T4otXrA&Dh&7kRK1*QZZ2(`@g!pu_u8m!G>LqyzYD5x=H}TDa|UaP)YlA5wUu z;1SH3)3U;b9I*QHEAjidvvM9P0yuOKVpGstNCJU=Fqkj_RL@`Qe1!)zwaJQSb}jR< z;%Z^+Xewsei%qZ^LsVl??WzXUC>kqraMa98%$aPaRQ^5}B0HvT+C7&K9Y3$A=N zcA{B1iMfEF4j{T`*Rx*d<#||A4iGCyRY$QkNuc;=9gN%?gVEjR#NZ!0qHY4M(5j`- zjRvq|v?h1mxLIgv>+-cnI&Q@8`H}F)(GY8KfPCb#L`Q1WXx>lMHtx-(2|z6(+)M2TGp0=I z{Llfm8s?4FKl@BL<5l$jB}KqQO3k8mEaxMh^h@?eeS|JK!<-@>a147kpW5J*3wgRI z;9Af;4=4nhTfqY(YxA^R5y+28h$Cx#K5egc5LPQFN%vnmc%?7lDEpYadT0e%Rgedw z!|+30Rn|VIHshm#0@sBej+XquhyD)PNr&b?Hf*>`%Ii`?*l`JVWca}#h(Zd@-Hd2t zb*%rIQ3g;)cqn~`+|1*cuTr;A^INjv_WUlq+#W1}mMZwfu)KVHz-ht>iTA!`{sspN zjRWM@9XbXGfyItbmwXi_i@+qG)&MdyLivJ|kdZ<}Ru;T^cs~*k>Uy&2;L?2Qa43T@ z#tx!$QPxq8W7L~ra1KA2F~fxL$E_PXRPTExQ;-!~4&9~xEmRVhaBLzLVyz~{lZ2BK zvCujJZ7Kt+Y?2PH0ty$ZUoQF>^&nJ(3B4%?K2(FCt3Hu=j|9~)dIkHgJ=W?at`IMw(F`Y&&< z*W-Mpu&nvlcX)dxuK3t2!!0@jf`@oL54^GLI%j( zhTNL1}Zs-NUoHkF~5PdZXzE1`BDScRjuU=_+p z{!&QDJag0Dw~m{A)u3Kny#wlRu=Dww=)-R@04tQ2BjPM7xM>1!CgK?=02Vbwp*lHg zcJZ|zzj6udG=yQcE*aU4f%C0cZNlfh8V*Or1bo<@pm>X23SeL1?z1z%7*ApR)q6tA!n~XmIH|QPQU)qQIAyF^Y2W1fr^*XiaK98aTuy7eV>xe zPW|kVL1J6MhnLGlIiTWE79~`ktX{7!WQ~Zd;Gz zRwmFKqRdSl)omH1?MJ`W{DdI%J27fCbhb3>eJj0;q(T>Xo~`Hp#(;F-l_03%uEaPdV>Bqid!4V~ESg>E1fg z2ru8=>M|3!dvT%%mE*Q)!i-q+nAUT%3-0_N1Eplk_36Lo-x|X`II;OrP!2eJ7$Xwo zQSBi5$-o_3@}M&VVV7yN-;hTLOP+U*>Jo^MD7cG)c2HVOu!TF&1k!dOY4ZWQ>9;V? z2xYTIM>L&a;sk#}Y*Ai#kD zIK0mMgiuSKU4g0-H+u26d$)SpW4XdRJY+=OOsSJgdQ zdl()dg}`K&9&o`IR5)S0$#dRQg`wuRjyzi(-*4Y-J~YXg%PjX~yihf$uFu~jOCQ17 z{VmOl7m5L44G^|+$~XIXELa2au#HR?FNUdMxCuVdlX7`^1$3YsplaBi=B3OaIvfFz7BgUl08)1%>OLJ)Vk0)Cj2{_UsB&Mcwm|Edy zZ6H!igrEid^BJr>82vRfAFlO=#RKdj7ytgtUb|)o(Xe;lrP*7Wa&C#ta4i#eX98MTFOf=W@}% zG~d>0Kcs#1YLnLAyBEWDUL}#3Xdr-~4QC?<8id-qfcm*Tp@6nys0KqEs=fV|tmum5 zPmm9auzJYS;bADW&^PhV9eUiM`VAbwv(ajxuvSV>Wf`oTuz2)*oLH1J?~%<(EhD}v z)56|6yw(grOkM!z$k#0!m#3-UURxp3RX{||)x=y#lzug{orlCQ)PATn$)Kp!oto5W zx$Rv_-j*K0O9RQVtIHKnmYay0mTeWg&X-a$kPzM4D>47kCtZXPQGP$dT836L6VC|& zD113xOr~ydjJWW*6ei4wD5|L^wtV-aZ&CV4W6qbNuEbKx=zI7!aQR6Bd%1ArQ#(wr z`oqIqD!l3!R}!|zmxy9e1#p~_wdTFaXjq&2m2F5_USGeu^Qb7avCdYKsKy+=Ec5S! zD3G{888PP4@#KlOc%f2@0+1N3zY#R|RUsgkgI60ookN|C(0fPe;eqYo)$pr2 z7DG$261a07^exc4YwVt&24MgJzHdkd3z4X!_V*?^LNvMHT83Khag9NC5O8d|36@fc zF&y}@Ve*whPlWMtTm)7MC<=_&vVuZJ8Iyoq0F~hQ1TmCdv7n_rZ38bFK=*8vCmwH9 zlVKkIGr1CxYE}*D0_0EV|Ey`?d5~~2=TI3*T##eBRWrKwE>CVx=SsuC8*tu6&g?a~ zvN;M$^)4AkttTJ1h+*SEWI8QBmStG$R_9Fii+|=bF*Y_9&kUqkov!OZc1uFW1UNqG z!xtQ%)J=0!s&kG}f6KXJM=r`-Y(1g#+H-!TGgSpT647iTr1VL|OxKN*$joH02AB=9 zV-fp=OmnUb8(7i<@9(^MIPi!@k&HKi{;M=WyZXHS(xoi)`y{(bw)zS+RqX zr|bS%6#KQrlch1$tfI*g!twC_18~rxOWdOI3=JUZR#uuCw^A3_eEp1P(M2_Ii6UNy zO?rdzF9o?Q3jngg02%A(s3dl*M)z15m*0Z4^>*ZJhzy=xrw(__M)gVF7d3Gevout# zQnu5gUp1SWc=qgN&2kTz>zLDF6{3efmr_0R)%R7J6YOW0Mi356v1o^qyRuh>3yiL9c!iRS6q1p}? z%8x1qY)DkVuAj+iU@$=HGFRCF45oGP12t|>Y|yN^GPg*vTiN zO@fh>WcKb~S%nrITcni|bIlmn$YHoKhF+_@w4Yvs?Q(c>aOcYrRH)r_;~Ng=gj)ST zBr-S1vBujWBLvrLk6zJ?AIc+G_D*b07q53;UkC{{w^5Ilv z_WgVLOU%`7nMLR8yIFO2MM%QKpYM+1H2$X8+mA<_QqkZ^ogA~6V%244%{Tiy2i4q^ zMdkzCQEVzoL-%RBbD>4SKR0|Tjt8a?!^}TL{EqNYM01~LVeBABK!0Z4YVgur z#q3|4aTxyxx|5v~kx+ehWgiQ?3PY=fnsU`I;H|L!ex z$@~HD0x#J$4XF!djHt8=XQ;bT-GLoItD%9c;BQ5?M<%DfPrXU9^umsTj>x|4@9m|HgfseAjz}YXtK0Rt(5TwDX=`B zA+wZr(SkA~ol0y?yPk5g=WqC9+u*xuTO0LYv-OnlJl;sDx^NWPsM?3AC%2;joIgbz zlMhPsN)gmG)A5`a%#}`>zlzxpTttf*7^w$-$GP^mRxi~ zmJ(Qp>dx=`bX%0Nl$T=Uk|~xF;{SXF&Byn^HM&^7xB3tcHxkeL3&_s;#jps$O6>a8 zWa$r;*EFHMxtZsXhgy)>N*_oL6B!RFMQMjSN7YtzQ~esa8?vwv@%grCgWpAc8?nP0q@{L{LCF zz6nr&MLdkIw42HWZG*J;Ulsh_IdE?!cZn>QT)*C*L|5gdd4pc8K+_2bTg-~I4uTni zXGlNi

0jw->*(OSCxR`KjKuZDY{gL9w|8a5UlyFx;0LYfEto*aZdVhKCOfx_V8wxi${$);!Mm0nkLNX z0>vrpA=@{D9Aq!>cw()@zcaEA6bixrg+L zVe^xpOD`3D?HrI7S(_i5@qh$>#JH5t+zdLM*Oc)|Na&PN#Gk?mblg{1ixAZ%S%ej4 z^SO=;Y7u#ixQpcz1Z*va`nAu}9G#IAaMAKYXLzOMz0lLOPt<-^TWz^&X5L){oeUXi~~+yTN^lCDZM}7rS+Q zYA7QGp^1-^rOX2mJMZB4!_o&8bP7Q-!v7E0{Hm*Yk@I|r>&TyoCRI~;16kIzmm~Rs z*Jdv|J`vH~QPON3QewKaK&34{!SI*FGY}Yz@-8jQ&^Rx68_JwA=5DEGglJHlaYMMA z+NYMOp_C5of>(9@)Hm$ll&n)=t+wvPc4xqqkBr2^#7O_1o`sxiVDeKO;ak<_kp)GO z1Wa`)`7cE&r5?>4;XR2_jp02?nAwx+$&rbAbB*gcoNpU&axRtZTRY)g!7X<{^L$GMQ zWrs)kn%^%dY_WW?9p@Vky}?T2lo#oqhl1WJ`xdW=cgRGq2Jdl{ZuXH}(E( z!f{)jdupv-2D!f?vIj&xLqVv4Mt$GrgjlV&X&Z*tP(Q>;4!JsWGXY(78Lb7xKH1^@ z6w7HaHKlLy>}%ki!`$vc(hDIMj)(PwG^MeGcz35PDU28U7R6JT^;>!t?I5cl^uBxE z?b8REt)N?ZPgg0*?@w!Zqxk(Ba;NmmbrpeT{?sp!R(m@2=+iQ1PCkA)+_L#5m7Mf_ zvH1-@zwYfN>MA}6uM;z>OkJ_3loa&WQ07*-P@t(eT z)O9_aH9O`;9djFlwC2V4jmXR6j*5c)5adUVy8J0&wRH}C8`*n4#m5fODiYD2yIcyj zS6;~`KexXBc%Ns4+hLSCM&`GpTw21Kc8n_JsoQ^o*5e1k=?B$i8l>mxx*`P`xjjo> z!hhMA-E?I zdPtO0f(2vo_y4a6s3_J2PtdEGH^b;bq#M67d4iE{am0Csw@T&(Z~)e2Nd*_w_6h!` zwNKw8rzd!ipiEKptJsbnl5~Er%s5310>V)~Ezx?@qoMN+lKvd8vh!pLjiqmkH zbr4-fPv;$&`PQ|1Mbmu+tp1zaxT8Scg*eu;@YkKU|8e@X?J@1E<)*1+;kP;PX;~}t zEUm`Tv2iqfM;il@oD^HGf0X6U!WE;I@R`{$>X>W(0*aOpPW$b|(&^?9?_EPv0=j0e zvl#JOqsCk~(92y>Moo;ar7WeX8wM+XEfp$e?3v0kUUBK}+{(T)^fog(+94&Y;M3l= z`KyoQ%nMKP1<<;l+IVZ%F3*gNjQru6$dkjDZLovhg8&Ppmpg8KOC2Vy$CDwVrkkif zOo)mPII%IeL!zz^p{)=j!Q*?SyY@*Y*^0q@IU>Ay2CGU7{KJ8p@y~Nn-a0rCI9{f` zLu5-FR$r+Y4&5*rYxz(eZa3eWL?(t^HK+05sy3q-@jQjdM~g zU&Fh!4Y(>JWHy&50AnJB*zS*JwlkJkl7kgm6XTO{xO3*lS+~GD!f;Ig9d57B7-xls zVHdq5-EL_r0Q(x@wV4Qm=@okgr|i~~)dmqS+oi`MMwWRw`@AB~SS47w+F&Q%mGLFS zurCWRg7;~>L2YO^X|dITs<9+&;%l4T_Dwint(MvcNVVwgeTl6Nf&1AyKC`lb4oYo& zsgIpTy-?QNpO&{$qwaPdEqm2xv;o)Xg^|aY)rF^;nyVMaa5y}KP?uvgrF)v!=*e!{ zlsnfoalI$7M28S8MuZ#vm(qW-oq4d)O)b-q&{XKC-1aXeDz|O^LT$-QKT=Z=d%%kcV^2VNAS7Xk#kJX7fF`u`QBwge#sxC!x0^n&EZbnAeOyK(yb@za! zIkLwU3qaliR+~p_5KE)#Dpj$iZEYB0+IE!21S@!uf&=ZN!0T!lU&YlT`7Vk&k}jbEjp1pccQ!l75tos32285w7)F!I~v z`ZAKTpwx6LA-4AJd;plkdB&R`f!H?+Pw}?|8{O-yTU{H`iIo{aTW$43Hq89eh)j`zx# zR!fnXMsphPZp62fO(RPW5m2jrSJ&vB0Tj3C>(kl~XMl~xUicUShSc^>Eg0yw<(t+2XYj@9NB4`hVY4JgZuPBGjQRF!TlSi0z`;J ze~Zls#WGE{W3i@*NmiG_5~f-H8Mi`5?RpYNblKgMjZ?CCNnIfBdcR(XzH7nnZ&zfK zum#sBZ_tzP!D=qN1PdLxDR*OdKY|55wxw}i*#-OLn`ULg>v%~}C-KeiRCg)9=H%XxhjNW8r8RZW+Q#BB2M@G&H zK;_)_ohEIVMv8l(*M_vb@QwGV~sQXs%lIQ)QO9ist*YFMC7kVU95Wf!{*s`ypP0C)_ci zn`QC4|BgQsv#*bU$*+TdkDE6`&5)BO-)Eo>1e(zyqVw7^_Yj)f3UPvp2wd*q%LP{i zd&Mxc#ouveVs7^P?B<)XjZ?uW{UxnrpkXh-T=p39&K|CXj-4xOPOy9h?2EDL@Z*~l zMC$$c`nQZ>0F|#;AZh36)yEE%`RdJSf6-_%HW|r15jIN7OR3$?aI65!LRbp53UjU40K&#o7`$4mM}5-PX=D@``@OgH{d4o(e4|anXMo<0U`yZgUaLRhX6{OVSZpXTDRHb zp_PpJ)$c&G;yuwPY>^(+>!3CL^zx3~Q>6z!B!xXxR^e-E_Pt`xF#=(^lcSfYpbVE1 zcw%ef0v;~J0Fn`mUhsB`dlTeG;S#~BrmIinuG9mMP zbsrHw#+PPzMpp#g^JKPGy!bB&sx@%ER@OLm0`G-+#$az)Ug1XolcT55u=Z-pfL02F z`QzS(&>RNCGBrsQG&py^bdiNcu$K`szTu3ozo0xInf-zUn4zp7ot+E7J3*NCab_bs& zwF2a!rB&1+H132H!`~aBnP}{JhU$J!+~gg_{ekxM;STL=OERwVQpuG*1ib@squ`0+ zM)Wy!dl>^`^GEh}CJOi^_`Q#~qfM@Mb*&~Y2<2c}NDPU73fd_U7<0wWNpe>KDM|Dz zv2b(MD9tsU)gV8g=Mz;eVEFQj>9>D0QLv=%_WJ;aXL^B!d;eY1?aNhaE2Qvr<7Fnb z!Lt&6klmiNMd$}mj6f$uLCa29n3$;6wg8>m)Qgq`-m)4Qu_C8;OO>;8kFv66YsS7c zj|QWxL1_V+P3Sfc?C;K-c=m@TdSO%`>X;Tkn$9TzE}Wf!Oe(PBUMZx-DNZOC?Xn%= zeciwKLYx{w+ZSH0cwLZwOa_}u-M(xHVUhrba?@CO5Ki`TjUP4Hk?_F2rqy7h`;a$q z!P4q7;MQ&cb_bYxZSz}T>hTJKlF$X#|2$*VC?`yt9YG{>;e72;P|$H&i@~5Kg_J;1T(5iHA|AEE9N~U3 z?x&#}UtUIhjEwe?m5b*5htXHZ6%(-lOYJP+po~jaz02jX=6C!*bxS-c;m_S4`$RVHZN&4dhfCh0EZ z1n|j~Few}!D5eD6aNRWoBMw1o@5B;GR$=WGUju78H5bl!O&B=a|x` z)($|PGGAd7pwv}c zCZzICdPZN4L%Li3?q9Z@%4K!OL|&*QGKpjF{eN|LXVGl<`P)V#^tQ84C$2FK$;U<- zeV-{r0}CVDhMar3^OWb<7Q{bSMIB5K15Zp=<8Gu`RBc(R@&<+>WhW{&wzIx1G!>%2 zyrnY|c$&nqb~g?;&#u2^!fU_Je0deo&T5`6aYPy<)2sSAaDsrr*EyMie7yi1haK(2 z(J^WS#@(uwwlpq!^MlNQ$w5+rKjrFM9CnsBx_@fLbW$HnlEVjbvHXe<+cHs*zf@u6 zbLTj<3?gH3nO25SMc|`@b?I!pY=Hj8#NYkz$p5B;mz2q!gq#BdFf?zcd-&fN}ZUx=Z$a zvwh;&M2Qrx{k~d3<;ub3HFfXUya@PYUc)}0PS{>(39PD>nd&^T66BI19nuWN+eUm2 z#W2sMS}Mwwabb?JP89fag`I>(LhD$4R#8CiwaTw+XGv12Kk+7t_cG(6++lj|$cr1W zC{U`J*D1lvmRdC5VNZ>MAOqqpo+FyPb}i%~R~Qe@u8Hwe%P9{cp*MXY(zK>*TGS>P@TO zwRG%u92VID z@Wmlu{o7is-)-j%@2N~bxgCt-&XUh2;O~b=EVCi5X5X=?EbNXsO+&^x9{4U zd2nB)o)tj@lJ12?F(+Z!q6riu@fX11y?r|8rJK`4=SiCuCJ|4tYTW_d3hyk$;HgyJq=yLxhwW@r3(o zTkFygOLFyTT{|gM3vm2a2=kXH%;%w`D7ZfmHerPLIyvv!MF^<}lLD)28EmU{;>BWx z!9p^I6?}he`Z-VpSNq45*a(V+WGeWJ^#Eoxga)9IM;tTMGVxAQ`4A5&cH)hUoah=gk0$(D?ve4P$XIj zaoLH)Gbn3GYE56vK5+V0V$dx`Ksq<`P?OQ*bvD@V3Uu)pRP-1;B~yVKm&df`)@l{l z9F4*N^~G2eB_WRrT(v=W2ix%hTNEsf>#1LbJVlLl2>*Z?PbWdlv@$G?-$6JfW9kh@ zEKAa@WcH(EwFO6GKFY4)vH%qaX-iPwm$g9K7`T# z`k`cdtgt2C|IZH}b1@sUUJ0wLx2E;xt4pM!#*}FK-FZ?u|i9GJjYAk8<)a z%Qy!o4Ik$-{suWIrd*TBu(|LRaqtSjDW{xPt|o45!^ zXN+Y@NDo{}oR$PE9Jwa(7LR!9GVxCpS*>%>paNAQi@j$JA3wnMu2H)*5aBMG^^gy@ z?h?;@E`PExv=dUc>dv~q0?+unIX--50F!3Biz>Um!q<SVsCOh4>AxyWvOB<%)qcw`*Fhx$lXq`?91`lwyz)YWMg+! zTZo8q`{Ne!!A@kYFvY20Dnn*+#N8^6V`UCo;c)dfI3(|&l7b`xFgFJ*MwseHC@K*{ z47J%m@KH>x7*1?YyRMYjosW2ivrIIP6;juwE8ho4nZD3W@T%qr&R%><60rd`1=tV_ z`f{JAAKd=<6t);gH4(Wfyr8u2?(oQ!KCmqOCGw`w_HCI>E-#;Qv(~lUif5q42MOzp z0@>mB73JbFHc$irUWl(F8>zuSVRUQ28W9W?$RGZ|Bi zvsVPca$&Z3Jvt(*27aQ&BtYGU&1?li@|1&pP*eAv;Ssg7^#NL_sMu9kw+2>b7=Zkd zB{3zawJ?9d`J+4rB}y`uqtq7Z_>^kZ^~GppVi1mJ#y)A|Lb>^8*JTBzwWKR6b{&RA zmCE#gP*LaSC^O^4t|MLmSj>ZNxmv|$JzZY*nIhX3!Wp=M5OP9qp}^i;j$-P-7GJfe zT@we{tS(rEAuH5&;_B#I`KqHKTQL%SMjQE8D9xyhRpc% zL?s~A4B!->#PJS5y;H|CDxot~s6p09&C%rk7K^=@MIEAlmVvm-GuG15KD(t*U1oqG zR1~HlK(G@!ZbKlV(vjbfV)BG_-qf;GPJg0PtPQIVwgISbdWv2^XsxoMWU$E+xpRxr z$ecF#*zm-6B2M>bTjY+|k1wTT?LJ(C)->u7G|}$$Jl$rvI6G65eGo80D!~B0v2}Vv z6A#~7RvoVflu2dg6JMSSvX#*@Z1NCEvP?yDhDoJrFAuu0iktu3J^T(C?3+zW+CzmV zyr9HRSaQtzK9=_iY!9{Y_F$KWvd5wlNRT(FWS+P-TOo}>(a>!LKozydI5!Ka6dbEH zzn4nagAE>84(K(GZtB#dL#Bv5jhcoxJdrbonh5Hus6KQQ5|F=a#Bp^Uh}=yyrPByA z4_BGC-&f-8JK!L_0m=*K;3*W3h5}Qs{}^mF?Z37<3<>}3y05x<0kK^RMU$JNg@sdS zFOh>aKgD-+FNWVqn;SQ^gksZGurWO>W8=Y7F+(J5yhoLVhMV=0k?3V@X1cs_Poux;& zoP3HXQfueJoUX6pb%4M^0#2uu1H!~-)-uI;ezWbo4{ zA7np`6(?>sdt|f~Vq?{&C_S$U|ME7WItDYnE`C?Uq5;+=y5i(h%aIMW*}zZ~^@{TS zvKOG~{->zc%fbUsyW(Hk*pM6HTTL;Bm-LbGI8dIzYGcS`L^77ODDm^<5K`#<$=rv1 zhnc;ey8z*trNN8X(zo#^=AA6;c`kpiFx1Z5PYoE2yk)*2fk`soP%Ab8iq@@{VQVd5 zYvXwBCR<)$>YP%1=inWcAnMKkj}?#jBkg)`a_2S(XH(R;d`TD*s0;*^K~$!Pd%Hr4 z74-l90ISb7k21Jwv$0KTNxL5Q-SvzU#5d|JTf`G^4og?t<+}+T>ItTQ*l5f0JFK-N zOK<742e6P>mDU_`-19eK~` z^jO2%4q2)&*0VBuNfRn>)rT;f-dOPzMiXp=U-R~`QH|e9WM(act&E|7?Pamry3dxV zWz*iW&k=)PQ`J6AtyD-YKZ>ec>6#`kxATJrmKN~=oOqaQ87l#7?KjTw`Z$qHN7+-} zV7(&Sw72(q^@n84p0&8ESy+ER_0K!jVS5K$QEn(13g{Knf z@Jfhds-sE$`w^u@tB&{Q$~^Nt%eDDRvNW%WjIjtD?XP@5Ods;fx&Fp-byE+DgJEI= z$|lrEP(pTUN1TWYb8Bc$Fu+HaB2Ry4_F-l-3QgUk>e!Xbgy48;FVUwh>H zI|dwl5LA6RAXim{9?{J175VHShG^RmcfkbO#L*tCx^$k%LrHw!2K#FE`g|c9mUxJA zrnerJF~bcZWUzlru9oQtDb>ljQ}LUhWC%5K9&!!+%EM6_MOh?Gh}&&-KHa)Pa+Z`N zvU(yTQMf>tLy`(6?SPcEv*xfmSHy)LaF)9}(QV#z^Tgz448I&$d?Y?nCOUJLySaa~ zNt`My28%nlOu~>`vV)BAAVa&{V_?6a)GJ3-XeX{;bgY&qg0Fw@3K8S=O%zg3Yl{b*d>4F$ol=k@}I&)dM}C>;jg9+WY3-G#-Vw&?|HS^u?20*Z%2 zivR-fny`_PwNyt!y4u!=lXqZlv-Fhy3Eh@{&9S}7{w*HCX`)~1*bsljloTE|dnlaI z5jkBalcsnwLhZ1eeyUA+f<$yow{BM=W^{MaN6o*Ws?ia|u5rD}6CV%0^vW*W+oNX= zdd5@z#|42v!zzNz&U$$ygfe3K~ytzab`I~ zjXC#Y0VqN+L%hnra*aOSeu8$LoqYHzp&IS`K#s}VV6reCeE(j-t1>^%B_oo`M9Ir@ z750df`SCH1u^u!wJ;60&0V+@Yy82^ErsXW+^hTw zc_DlHc+_!45#Gv+BUa_h^n>C+3JJZVwi)MZfEDraa8_{LNNQzGWZuwBK(hJ}S*L#O zLOLWSpn9wLZMWd=jMUUr7%C%{JsfK(75c4%a(wyf38G2fX%(i%j^8i?K}nt;gGMn& z6io4Vb5+~Jjyo;s(gh+U1;eDhalL%VcbKKV+bmF3V%QOw=XxUOj=0;9?)ZWocCki` zUpab!A;`*NjB!HsjMoM^fVDk%p2b3%Zc97sG`*Fy+7hCUtCTgmtqvNE(UHh&Z3=L6 zLUhmX%z!`9R>6C*$$L`9qcy@&36)lEX@`Qchk}VH!iqu2n5PxLEyl+Ldz zE!i1FQqMe6b;Y=u800S{2Dx2lBOa?_lr>O-Avnk35fM&Y9Q8tF)@in+dF;%Y^TCQe z?_Xe>;QO1WH1g9bt+l$jnfveW1RSHKvgH*9XpL?IqE>;3A5^40S2yD5+A^Gdi7zq~ zg!b01KnCZMk+U*yNlu9h?d-AmsDPIZTP9;7cGLQ>Kl(bT=a_yAEpVJ+AGb4Dz3S4s z+Ym^>%z?O@{iuSF=saTVSB3Ix#xx{hzV=`R3!cACyG{n9Huf!^v2;C6`R5nFy@0|_ ziui$E{iuX5zS-jMAUdpvp!%;`ELL?c>R|#c;CKi8bN$DwQge8oD}5-cAA1ha&+ z0;79V^)Ii%bp|gHcHR(@@VIt4A3y(3=83{}e_=R;lhT6skrgKNNxOb8y917=pj*or zD`sJ#)?_I-9%Gb^9XoR{V($QHGU{UaQzRGZ{fFT0T-QNVVY}5!u5@4f5ib_EK@Sf1 zk1#`uep5I4RFUgtn>D9sp9AxhKi9}rB9&iR`#|q6#fX8^1)W2~45vqz|)WjWA%=`kD1b<7JKGTHI(}ZUR z!G@Y3X9j1>K5;`k^5t|npR@3y;zwv#eRs0Cf?(O)0Ti*xV$`f=0oHcKK1z_!0@aMt zWj38v%3jFBz)KnYj$Y%u{YA)SHuM3Am$8#H{kzu9=7>FxK4`QS&~iIx&LUuK(ZubJ zzwV2}Y7i4bPmY`4*ExzV8mrrMlM#v=G=4Clj3AE`akoxUlB^Nc6GHS&D`(LT;d;Fg z`hx3~cke$CQa3vuve~f`&LmJ4bUMe&($*t#WsY$Bgs5N?FTtsweB3nbA%W0@A+PL} z@50e{RZbza1~N+is03tCdigv)c=oYMX&oWy3^niLGP{3|h1*vf0^l0XM^y%9g~C z?8{h^B9x_3)?^Z69gS^_`8{9nsr&Q!d>=o5-FN34&Ai{Q*K@g^*L6K<4Wyi^(*3t8 z5`%5Au97t+Dl6X_%1nc$b8yi2!W-J!NB6#u6tvfSkVCtmOf*UG)kG{ry=)a3`N`diJ=)L1y;=A@tJ$u^ZQRQRc;I#|!t0N-N?5M2J z+B~(EA+0%TJZ$^}2IS--28Mq8ZfqA|IT=C5mju?7h)PR=@adJs8!)gDuofmrFxL zL-8B1HXJdn6Cs@Pfgs;%+R=fYXTy>^b&Ft>Yi0`TbUe00*|GyGm>vD5nm;Q5A8a#N zv)<#(59~!@nt&Qubk7gv1AzAM;JONV;d^%*p#-!6#?Pt{6)Is_Nfvi5@wc{O)tr^w zE<>FrM5OR6kX7JuQxN_)%TbRX7Lb(=cln=u|L#TdHoSkaNqa1wGK4S_Zaen8g&xmk>J>5t0% z=rb!LD@IoB2NdOnZWB~~ehG=-%bH4U1%lJdtMN&ZSA8ekjymkCYyxi7EgvYo!9w;B z`zk(9CbZ)2no}6SeH0o=&>~@-y}f9W7+fz7?atDj*-O1^WVQBOj z2Su<-2Jc8d2PUEy0ED{Lzlbxe2udxdYjMwCh7(81Yg&~$Lhcdz0hxCOcGh;(`dWgD zXLCGLP_e&9=wI)%oO@3Vya3(8tsXDrFg~b~#FXtuK$|b4beS->Tt%O)XpT`PkKhJ( zK`h$qXw1ojAkO@!`;mK3(c&@e$&9OD-KuhYFLzOxpsLjcoBNe?g4*eCGo?vb0Qf~v zWq?mfzqY!z2Wm(Qu7#=!@2aN(Yy3{{mVRLRVf4R{9#D%37N~Wn@-NjB0zEa4AHvF= zigdUx(!RtR7t0s~qxM>KYi38J!;6-;3TXYRl~&2h6w!BOE;)OElHE zUkSaX{K(9lt=!x%MY;1J(6NHqL-JFG`7IdbQyEWE6dBD}b+ubJTSlX*AG8*Rscg4d z?^*{mjnGqZgb*PS1f#L}3-G_9$^Re^Z61N9I&xD-Z}y;D1{1ARtqdZ1vk>t^9?Yyf zjyPXFFL&C2X(N*npbIZx8@9KZLgw_K^CwV6aks5lsMGC)!_*V2J0-)+yJa7k(}1WN z0{^3_2a{~L&<2UJd#CezrI(Gk}x5a|r za?y?p{6{cBTp4*~ajHpe1K9Es=73PU+TJ{!pn$MU!M|X-L?Ig>XFBO7=En&7%7I@{ zCOM*LAzt} z%n!~ZeL032*w)2NN+ot58)ZD;tLdPr3hQ`v8#%b4p%?+AzhxQB+^cLK&cv*Fh|D9P z9M%N++0xVQO$wrt7GYlBl;b%Hh@Z&SE9P|?N3kuL706y zY@>;Lth9yyOqFfw2n1}LO6+6>0DK1>HMRNYjs|LGQA#Lk=r8M$nNS2BmsJsynQ7!n zhdz3^wQdeIL;!<*Mmbjx$O)e?>yy+GgIXKpSev#xvbTG~DB*;dsv2E~*VQsX60pjlYawlS(S1lR+}pV>%n%d+F_pJ~FPCoAZEZL&JJ|r#?OD zx}-_Jc>brp$gg?f4)vZh!vs|XhMTQ}fOiNxO;G%Ds?(&?Fc8JnD4*n^fh<7U(R*5g zzZT%AV=lIAE4ni_YT!{wUg}O{9dl-jR_eh4Y3QIc>}De0UBuFlR~Bwi2hrz`&-6S& z&GI)ksD~>p1MN5#F)#@9(xdD~@3oCsBKtz0yLgnf$_-Sg5^$t*=9^i>He&H&Ra~T` zlw)DPD%=?Xi5HZCT)fwjQ6KNB<=)4lHG8(GSs0ieF z6cwRFpjWi36{A}~<18Z|RPJtv2Q)nI4xc&x;L+F5t8?O=DqR2W|F>?8gXRov{BOSw zOcpO24tCLXnRNvQKsCyN1^LeAQxZbE>TT|~1Ou`NXvKK=&+IemXURKLcN`LduPsQ$ zwMBU48*|R0tG7-TdT~Zl37%B%}?@SB zzTpNg9%t5{nwi9yihI@Wm}NJ=YI8DDRE5;XMux+q0ybO8y-$>OoOP*!a6!f)<}d?M z$cXcJ3T^lxt95n0mC?udoRUfIRu9@aN+0}1ET$T)RvaO$Ep^90rt(D=GQ!j~RBnhH z4>yTR{uJ7NPVUPs6+}{NOg)NV+L=VCseZIGt+8WmB2sS$RXyJ%@`Hd{{ZZIA0U^q2 zkNu;))?WnnL!Sl}_@mP0iEoU+>ybwrMIsdYY7y zac?0&rm0#HQ!=_ofus){e*y95_rl={WPhrmB4Htz>qFjg^Ac*8f`~owJ$Qmwk8QzbqptCBOnghSCrNRH`+D9#|jUkf|EYii_NC?77bc33fet6+49G~yUCooD2g`CbTMhPvDCz=82Q=6 zZez$i0YB4&?1HDMOT7(HgyFh{sh#gzzU8$_m&>;nVYuA1ItEGhd^@AIm32AgSjLFy zXhPa+d9XDq3*B4dW)=A|hBFu!%yPb|QIZ47(EorA9jJUQ$@2o8LbU3gd9rrC#*qEJdBg_V%zm2?_njE=b=zDbv z!~LAfWwv_L$~OKt#o_qIndy*7*mc|HwLa#B%A;849eKI=lfH4IV(ek*C# z?7Raov+BlY+fV+c@_x-jrjVgLRIRWyfSd-qIL2lrvNeBnivo=jfso|0q6&V>Z5=Gq zs!#1`to5unvHYQXup)vi#s*j^+XcejjmMfiLNHIFWc*ZxeCqxDtaj!+BnNS}KT#S!iS0R7jkr+|CsYt4MYp&1wa06w3PdT zOfsLX?7GTVFlBjWnfm7rpvz1_#nIf1cFIKm7nHdhT6cbM;tmGK|M0~J{|iVQYSRWM zWlsfbjs|OKE(y`Q;6UsdVum$UT5dgXPfNHc8ckwq2v`p5vy;?9rG-q?6)z_#vWd_~ z;E#xkmUVo9EP8^*t__nVDi6mkrP$HfU>c!e+3w zOJsa3jk`I1a#Bbki}t}J!Wt!{SHUmb`Rbbt+fPuS|0Z~x8BJl`b`;RBC1PxQ@K(ZfO9|f_ujexeU}++ zigr29?~nE~OIrMQl*+fUU^}}QxR0*PW+5aCc{?G?dOY!8fU@o$LDD4f2CsL3M12Hi zTA{90Q;UC%2FvtG!V#Q6Y+{*pt9SU4H-0P5U@lz!fOLmmPG2sJG7>%o2+5KqLU*Cd zT`J&A@UJik>$SFQ)WqXUns2zT+kwrz+}ZG0-f?J_UaZ+AWCsQAQH%NMI~q_Q;1`_T zeP<#eubwkw{>a=^sGzMd`eEmlGCe-{nArOn0vS=(_X#XXFgfzRKnp>(s)5 zcIGm6&SDro?FK~p+8P3syQ4u=4!=;p$vICDCCRTrJJAQNdBUMl<#M!4f;s)Zu8Y|k zdvzb0Z_R%G55>zYT7k?_SKIy5R6bo!r+^KCCgNYjaHjEqYO~uIBS5udPB;k)hSoZf zTavj!YJeKy0wy;AWe!Q>mp$p1eF7e!I*GYU*gh_G(v^QfvK*dXWVcYK`dea&knF&z zKtd)XlWuH+Jwh8S@mrMin8HyIEwGrP=7TKxO~W%SQgdn>HhS*X*lHM zk?J^q^rfMfFuVU=0d0)#Iu{0BsdLcPY{T$iFY+?PvXu!QJGWp1dr7p5n94)~C(oJw zi-Zk0+np0hnDbEI0N&fGU9A&NQS|9OSQPb;adCi2S1~1DY_IK>Ou=tG)XUX*_S?2d zvGf`Oue2u!>ATZt&M@=i^y+56eP`_Ls?K)55;uS1ZU8r^@TrUeq8Wu~2<=L=-7}^i z_D<{07k-3-LlP@1bdCM#Tmo)2@H2TBg2geHPoutlOG0aT45XTTPw1oqRut$H4>@h_ zq)s|~U05aq0|2+k7xtonD}rDP=3qFmn*)eV0G7`Km7wWx4Qw`>%)*0rns6@~pp`yM zxmpTNn*(Kwo#VJt|Be@@>Mwjvja&z16?f z_}xB={Y1&jwU^87RGtB;Y6lg05q1mNWRWDxMTynTh5;zF4f-Nx#_LH^RwlujgAy;w z1a%uAT{@_=W#PSGj2zowi3Mv1dM)&gfKu(ikURvDXVhh5;4h@_kRxXBC3NrT3%;m3 zzu*baScDS!2E=xQwLDbTr3RMFHRKGhIG_lRr(eG9Gvar0<>!9tP$DPG$F-Jaz$d$E z`?s)64~G+yZ^wzc?q{RlX0Pvf_GsdzIvO~hB_td>?%fA>L%N6!az}JGSg394T59|s z;&wxXEXK+0p#f9dWb?=O7Fy?v11EoqSzI0qu{d+vJ6&F z`x^ZfRX1GEk%A}Pkn85|C*eYRp{4J`h}~(uS*$^P6SM0Z$vf*t1=@g~IF+c=Bs!&o z8FZj$fmB=~1!|Op*0M8tOz-wb*{@)$1pI{baoillv#1FvO)5QA@lNr4p#qiK0j7P? zg4*#&@Tq$=a5w?Dx4x6>6^%)F%a~Ve%$sIVdBVzpj<}{r0|xi|?Xn+$rXv8+FUKGL zbf#dr%1g4HwFCptqe!sxjzRL?-2V9XE4Nr%f$$Y|89cW+Sa3pcoX&i&a4}FV5IaPB%isqTDX6+hjw1Y=@Nta8r-4x!1+`!>DkF4(nIF0=omhS2 z!e;{kr4Q-7z6mt3zDzA&^v19IeO#{7XH(;8zUEUS2HZ8pCr3WFFBTucQxeiyYG}wy z`E^gWi8VmJ^EHq&$a4u0oU9d*$->WDAP%~cV7v~%g~J8~G039Vme{NOOEK$_WV`1S z+G5)Kp#Nsw`EDKvEYYAF&mU>evdWh68K^oNXBE74ZN|?)RLAnt)J6$U5M9OBFB#>H zkO?JdDFiFUK#N!ZplEXg)3@Cqq`><%@;?yMV@MZ!ZAG zCZTcq=Nq@<6z5SEtfQ<3u`>K-^_6{oTUAd%da(#Fn(PQ3X%mDH>~mgk-?huwc$V(n z5Jf^0FXm9)$YxxnCB!{29R`~=J7VyU_JHC8;V8g(b38f+8$bUCKIeFfrP_b8;Qn|O z_tXl>h*CwKb`?%+UNp8K8w?E~2*mFp0Z27q9`W_UFX!1`&VO!D+}bmt+Q#_09x6ss zV^4rv!t@R+t&BFwPqtMIx6Q9or1`>MgByURpqCBWAf|-eoLZqYCmOW)Y=esU%>DH}uwKY4Z5sl$1teS~0Lv+Q z**06QdI`%CS z#J>e8S%@o=YYGZ77iB-qBY%R|nK(+;iOSm;x#0yalv?$Zo<3~%L8HNi5?RxOj^ zt2n^V;cO2Z)CwSbgB*w$iGbd7`CrRc!kCLge$){ksIs!(X23U%j&7ojVxlujY~*91 z>eaY;G)a&M0%Y0uQ29VI)=^pK)k|80I~ih(nyUW!WxoqE->Bv&nM~MfP@vYkjG`&r zuREM+a7!7d%qk8YIuG=i5mSi-D?hmopEHvQ%wp}unZT1R#X=j%*=Ndj${+<$la~r# zkss`*8Xsw$mVGX2#0@C<~9jA1wEcs*d%kvIlz`(>)w?5?f-<;S!IR( zo)_y@)ikGjyCBCF!GdiaXQyl-9=!hk$m*dJqDX)PBh~;`omtdT8>(#YyQBBgR8DLK zWUcQ`9OZZdK37@reviRgk#}G~A4WqGPp_=cyZ`r;9)@ke>__emHqzZ(_PaASeDCq` z3Gn8N{kjqObHH1H?>><}gyv=>?kvUrED&)ROr9DiiQ2hC7(`nUvll|J?ZJg(i*wSIP}4!`%a-#)ZEIk*-hPJmtFqx1V(G>P(+Q zsqWxVGiu!fwf)gJiat6*&@#6a>czBHVA**1>kn|<$;V%&CM4^fvw3WN(#pTM=ebNt zt2MzaI@(qf8>431O3$l1$%XWb#AhArhqBGTvWw@(wW^f8fw=rwDA#)v!5Z-g#Lz3* zA6}u7JE3BFh!80EC4?Eb=2SZ6t6NYY;hK;Pf{uZs;7QLTBr_0MTze|*X7F41DstV+pQ$5>T%D&9c7NH}A z>uJ(uLS|eUO%T2iZ`i>**7q{d50$xQk|~I1c+?$Q3GgENex{7No7yW9-@!M0M|mbNl-gHFHuR^Q<(sp9}}qFgLlsZ zhmJLFnTUeUw!*AKe=%E;PNO!Iu|zjWwO(Uuz7$}HXOFIwp-j=w(`}3u#=(eWU|m_V zqCp3aG0r9DYV|&0nWT0Ly^%fyw1Jiqu~aZMmFgeS{cwDkS8N^!b#}L$QG>zS(bg^} zD(@&qK?m;GAJyhvOx$KHI46jBaKIdM-k%g=UIG^WRu~2JEB_8dXqGfyjkzVmuXWqi z7n9XU_@6%=hJE$+;K;aS=gN)Uo!f!7tM$L!JR2o1SYes{50-LV5 zjKpES2_dZCTL?TxB;wNsN#84MKbk#X-~16D@BS&Qw*KH&?l`2+@h-5Dej!i0G2Rrr z7&JLJF0QJ!$RxuC!Uj8N{L1o>vB?0~I0JhSl&uApHnzx% zt%h)tdp$VxD>kyyne?mn9B>6}mBVDimCs7CmNR{dUrl1;2}NVzJRl#@m!=h8+~ty& zj?)~+vku&^hC*FY0fc9b=w47qGmlFP+pmVKvYcG4u??3tp&Zd62?zMceTwb7kIB*$u z2M?wj`ZO8dn`?`j9|idA0zyhW{X>P??eF~5DoRd0hJsc8-gb2lOT!*hs_fmCYV|p^ z#R42tXjSpa{b~W5FN8 z()`P{K{VG|z|Mr`=AVIN)}9@XN(s8(;fW!yY^DGM{28--s+u3@JofCu&X;3ez%u4F2 zK&O=@2>3&4<-!u0>h(UDtgn8tA0-8k5=6sw+#hk(Ji~_t07|b#DIvQ%1L`ZZvE!>x z##1uF7?$0Wc4&uC5P>UZ2OOzyLz-dl0B(GChPP4SB=-zl(U2tMiK|yqsSrGUsLYOD z>uxQ%dLCXOBf2`8Gl8ft*JAF?HBW&b+vV1g?a(=?w>@YrnOM?F?!h)GQl~n2nZfYE zz``*R)yN@7uhY$P)>G2%7Mm@*^avn!#YGhLLDS>nt<8$E1M#B+Hw%W z$}rmeFDHBS)g?Jy^92WJgkAe}2QQ!syEgGcZ4H*8l91m<*Ejksi-W!To`7^Gp=lu+ zw-R7{mhY~WoSWg4A-o6g@#G%a{rZo} zYJhw@qM*f=bJ6lEQ5Bbam9&qLb8mRh)3p;?H_ zWXHUY+)aI+k?WEP!3OP1pQ?!uq0E;C;5-JdzPSy zTZ1`Ooq;+7ZBW;OH>!K&Q4V*xH@n*xz5V4AWr{;+zB4$%DGOB^fDVsG3`htRI+jqq zST|}efuJ}f=_HMry#UVv)&To%$dcqx4*i~_R37qk|JCgnuJ&E-O1sF z0vM_rcS-?SX>4$U+6P=dto9D9>hW2Hy4 zVcI@k9tym`p;kk1d1)hA^Dr{jk3g1yld|xUEpx_lt4@TCLP>XR z=Hqt)&Q`}^AM@G&v*1l6#SN+BLVyAk%{%6lbbF)d#lmZf(g6Dc6uEyaCymnp~Kvj@z ziRZP2aW$<79*AJ6;gb2RJ6KEABIR|;s&TOEEci3<1UdTae9iX2ZcM35S8gBqgEQ(V z*)1%Seq)QTmE6WtI$SlACmKf3J@hOHIK@3Wt&Xt#e?keTQTlBnoJ_<6l!trE;Z1o} zF=}~|96!8j<<>@jl2&0M$(a-a!rc@9NgpaPwSXh&T6!Y&)WxK0h*6y2>D8mM1IvQCd1@3S1}i83G}Z&<1A53k(i<%M zOh;#Lyjp3%7Imf>3hf+EGTHc8drPsQ-1sFpR#8zcX@^fEKz!~qHuw4;pB3DsF(Td) zjYM{``@oc*t>XGMDelww3_hRsfs4FGfjoKa)0aX)JC)( z>$g%^@&A@$Al|utyridZ^jx#EbWlLi1`koF4-EPmRlubP$tP;`GeX}pje_oK344+G zd%m6Ki|Yt<;s`!#9N-ccMMOZ#!_*>e0!H`tF`u9c5;z3hCHmL*xx^qYdLYrb`}exE zf*he;g>6Q)K^Wr}#-=zAWI+SY>j4C2tNRXAr!6!y0>@C8m@~B0{Y7v07D3y5_}nN! zXvsTv;VaC;n%Yk0!6rADmd+0alpN3m9pg*&U9Ur{!CZ>t_hitMTH z8#ywt8v)?V2rOY(^r^ zvOSG!G59t6rc$uLe9bV`4FWf)YJ)Y3i;ROOSY~jfh zrGP64>A%z6UpI_|bf6pqi~a)(o|LH5Mqf_^{Efa{1={Q_?OXErs}9EB82o$u20>B8 zpy9(BXbf<;jRh8Lz12tkx*+4}#D!ffEA+T#APfe zqv^epFL_2*F?>rUQ1wPW+e5L6G!P)je=JtlWQF5FDHK=I@_)=5Yl5$#sCR4pAwkiL ztJosL3;aFm_5Ys+clwA?!J%@y0dPS8r%V1ZP#*g42hn|%B9(P@3FbYI4ydBqETlf+ zQ*C=7EAK>>D*Be*PG9E$8-1j~KBqS5%Rob9JvoJk7+fplS(k*Q_(0cgY)(x~y^<^B z>Twbd2|{a?MhFiHaeR>|t}eD&6TQgojA?-rq&8H|!8uu%f3Tf-1@%xw8!eJX0F7;h z+s7HYaAr*MG`sOc_+`M;3o`hZ?={z;X}ekpq8oJgv9wBdw)E+Fm$q9gpqCutiuT~) z>?tqS#h){Yeqmd+u$&MTFW=|2?$m_>2~V=ZspyB12jdZ|MMci-L9g5lfl_W-L1IEBcZGPz$_y@tAGEr(V z>~Ke%e>^m8RpnoV8e|rR6`&jf8!6g6RT05#@%@HK)`}Q=-NM=3a>Cx3j*I9y?NMfH zoS>BW*=Uo26rTe!rR|!A-0u~c<#lTIlIb)Gf~J;Um#?a+M!Q0FsnwHFQS|aTBz)?P zhVCHustR`#lqP6Vn=q~dZR(J2@|oc}6KT-jnU*kT{;QO%{AZOSu2j+aoZpup zRae^kbmaDV#`E=JqK@8bg4$i1V9U*%H8D(od&H#iL~A&z>Sv1P+nA#0g;OS$$_bVO zQ0O|dqWa<@t2oC@C{!ZSWc}Spb=ccL1Gj6rxAMZ<<`Bt|9gT=UXeVX?Nobe%aX5Q$ z6Euj5ep`$6p?#A4`;(3-`=#vQDLWFR7yqR>cCssqggK1^ie?5B@yW&z*L$Bfv=Ts<^pXkq6lpwm2{a$wR0M(MaSp>`3FE-0P>endJTUva zocvzngH5!iz+)%JZ@_K%hyUtzcLqs^Ygoq{X0jJ9 zU~#l9Ig*eC@v;*Ris)4QNHlErga5#2=AztVBsxpmopfUuj@TWy$+#YAn^fTU&>EU@ z*pnl?unM5s4p#3?b>OU#z#yj)+Zo|`#+IBHbog~jz8g^?DCv))GP2_JgPLq-T!#Lek53jvOouyE%jX6VMxbhpit|d%O#=<|io~ ztb-n^b4A(#sD)HsErm{{>M7?w`P|Tn=h!2DiIeve>FVrNR zOt$C%NqFb#X>|-*p^jLIO1kFhI>1GL`cv&TO&a#KJuHczW5*f$3YfzQ*L7)ai#NWG zCE>|bbb+@Iy?}Y1Bcl}YB+MFECOM|%RFO!%bI=v;c$!pff&A0osZ`L~}4g%4yFp|VM2K?tj92L__?{X!?<_96$INskO>8I+?H(b}jUs`jqz9=3b??w;+jz)lKnB->5a6Q$$#Dly z8jLPme=$72q=??~uMLF-OE1JmUyf15M>)<1JLCkAQ~Ue?-}pmUMfzoI-Pm6bmTByG zI6mW9HUp1y{nvKe_5_m|M^ZGIM0N~ki2%wF;qt=&D8t!!9k_9n-$;h9@{--UIAM(0 z1qIBM9*tRn%LFIIspvHE#XGRm=SQ!y9!S8YLtc#PIDVb-$#KYmSS`2j z-F5NAyh5YvUT}EKeU~=b;3hjD6&;JC=n5vydWJVu??8`Zu=;ZoFgL=-KSBpGqIk-{ zY$`$6J^QU%rIv<$>Gl;CGJ9Yl7nUjkg`qS^03i7|V z-~%X2HlNpz`cvuqHv4G~wn`@~8S$u$+r^ z`hF`flmg5%97!fY1Q>6Cr6*;8!j>VE8dFQzp*HPMJR%>Ws{+-%%hs@&1k@~06uyDG37FPY$5fhf3Gu7 z^O2evGe^*dV9wsXGI@^@(kSv1XapHC`NVzi^~5c(Rzytj<3w;}JWs|LLw7bJVsUwZ zh|tOL#H+$~wp~+`_Vq}}m48pH`y!^uh=}+M@(hUguYVDYYrbNO)X;1KxoKcL%J17% zYXm+g&R2cV_!s3@!r_7I0A_f%Gy+&zeK-$o4Ti@~-7%)0ftlAfNPM9(8v z!o5F6$ou66qbEN&{NifGH%JQ{j9Kf_XPG1T9wEMho_3~{oCY%OIQ{IJ5*iD!ou&ke zdrIjVXZ%9by}O$~5JHUk=Sr<)^Ly14R*OE%=j8T$eT(8h|J_k09Ws@)T`G$zDJ!V`x}V9eSRNaj zO>k5?zPnli5ngCym&jH;qij>i#oMX7Q3*AaO*>Xx52sc#$T~M_l8~yt0jDoG&(*N% zD+G>{$+8(|_aNtDDVND3s5Zv+O6fj=DpQn2DB78p7xwPs(yKkq`))QnC4Fo%M4m=I za?g^U${fqyU)f@>l>e6mZn9x!X2z4ZMf~+&ki!ZyN)E+{L&hqJ+~z_ zHT(ngVI#y!xzeznvk?E3vibg1lnnRq_xE3*ux(kaC7Pu0E@J_bXrx@;xkM;TJ}&x2 zl8PGuR2nB{Dg{tB022u_9O7lKY`+Je(pKTgJ-~+)w>rY z9JO}OnPE&xX75_<{SEVFD8&f``!U2G?&%k}9X5#b5cjnEfao~9U995H2S$+%A0cpv z-XI)9->9bVO0J3#4+MOP)+IyJhA-ukCA$usq@j;<^Mm@A;(N^=n`ReFsx}_}!C09k zr?=9te3=k1m%2Gkr?OZ4vLW60-flPvl}vVF?=gmRFq6+D#C7h+NA?JNgVfEgu}0sq zJruYjURw|I!VQ?1GID~khGv2P<58WY0-Pg7AaHews2$HYnqx-;r9M-17=5nk;34}D zaB~d7$#Z<5N1=dMMI^D#2N5ItXLD{8!L-O#(=a)*{5DAuz!Tp9i`;qZpm=rRTo!i1L+2m4uWeH^K1oR3=za4nv#Rdjwkbv?G%Y_qz(HM zUa0;j#0KmMJQqs?D;HdO5u#E2^yra>wg40ccmSz;e=POqszIp+*{Yc!1l?z}#?{dp z+9C2Msc%A``~xP5vFFJpzc^O(Vm34Sa`vmp%e14k`J`MSi(X5vitQr18dp)v-^~&i z_TAk0=9?m0%0~p(%@0v(%VAz!_(6{nP@l_P|E^=MEZ+3-<_qmmX0N=ZZH5P)< zX&Rf&yv>t{9081-xE}7U=6|!}fj_ZubcmLA2m(La3p&bN*fC#LjA5`ErY=nrsAcIt zWBR^i{&Safg#%>ry7WL+>GI`Sb-EAl1&ajQ?(B}*o`zKmzy<1OtNLs`E?=D@DIp=T z!XV2RUsex$*BkzFPxExToKbaUTp?rDBGy~9Ti_i)GLrbw_7@Y0$r2CB@TH^;Q#oS;vwy&tJu zL)2wdq(fD7-q7A%vDz_&$M6PGmC&!ZseRU^z(VNZ}r9Io2K*dv!M9 z!UOdVeLz!ud8N<>r>Z5f$7&DF9z7aiPha4b`lr!u1Ta142^l2~#PrWi+tkukV!Qdi*&Wep=b}T_Z;?J@qHfp!#BPHmB_l-2530 z>t02aQL^`E2t~!71IsY@slhg9g@kX6ib*w*KJzKL<=bezNlKC9N4ov~I=|TdiLOSP-@L>84xCB2U_sOIQTRHj{bX%r(+GLuvC@NG5e-c!`>#lnSFEo~`0Wkm|A&%h zk#HgR0;7^)7o_kF8UqzLbtA?tjbE_iEF)-9qrnOs>dR@=Z==wUO6K-Q2_cu?KGSrQ%FcMOl8X zhwfVJV$s9ARsF=3#7+x0wAifXNF-%qM!nh@4)tMP2N>=g_5c{3=(+u<*v5}0GmL%J z!EY!a6@lY)vSZ56Vt2PeLCtoNtu=&5ZZndoypu&jm}qu~E1vtEOW{KKf7Znc7uCl@ zyyA<6CoUksUK*&HQ~_V(;xo(ETi3-mzuZF+=WbpuUkE@x*p|d=rZ>fieIt0>@hnJ| z8f(A)fFBb3qtEUhG=Af+&d%$Y*E$J{u`_RvM8?3+k?vmdz#-2sQ&3h^E{UPfHi$LL zg9obg<`8&d@Gw*zf^G zIw=?agujn3cZ&1Kt~y|D92NV`$D$H=CxcqASGN?XmSrPm&*5il?q>nan&8ha9IcQa zqB#X=qI_(xA`X%1)`v<~@R^j60CZcwd5-O=#Onlh9)kEe?#f59Wb1H!a&5TrFC-Ik z(A;bzV?TbtB}=1imNk{rwv#EDWw4ro=zO9BVKAZboPn6<24c?Tm-c8W(5DVN5rxDH z7I~@Z2F(?(X=MLo(WtwHdzvkqTfRl*&<;~Vai@vsn~boZECu;GJN|L-@b!JTDd|a3 zh;RCMML(;qp933}JBY#<6%$9sOg{U=>xPdlWNR;eym+%Y3V^5M$$WWNik1rCT7`#) zvuRx;K56&Y;U=4)a}PVcLFohlTrSpYircO(Eq81@yqOh|u#B(UK*FFg(a=rrY84ohf@Yn`u1>?lu z?dJ(!I`93c$Yz{N8CjIqX*vy{ktlofVP2h?7&}y=yc|y}+;~{^ATyO1tG*6*z^d|j zM8iq_O?Vsg)MBpULw%&i{-fv`5|F{k#yoxnl{-VLAG-m$3=W$s3Qa?_Q6%`_B&BrZ z;V)$pb`fP}Xk)@R)}fMWY)@}88fo}j=6c&e^|7J%J&hZRD?iAjMF~lbqUPj&_~N-E z^NI2z+Zt~ar?QsV&w>e>RdSc3*yOW!*FzD9IId!1GX0??x2L4hNg}dhBDIsCZoN7e zeOMI~@|p|F&pm|vRo$1CSHKaWX=H&?W&3`47m3&(gJ;h(ye095RVz%$5WZKYn`yhl z8gVsAXqdEQQPj$y;4|Tf!*nX7Vq;r#!>o5jlQotHwi}`OvC5e|P^j_JoX}s*7n_v# z_c&Au=6rEc*a_mZ_48ktV(-1Vv56jTX(nT^%MQOz>SpcWX9{+Y|Mszedgyk|++~nV z1`<5)sl)byxooqwIb7*dCab57oXYR`=c`H_So16jRbr4s;M7494X0SZ4G!rOq6iKV z@-in@2lXK!hu3uG65L?&*=giBF+zO7_e4Q78`z&YAaEeMljOwI7LHw)v2KKGVH0F7 zEN`u004wqk#}C(OV~tnU(9l?>5lJbe-Py*6syOxH<&>t{>$Gb)H?-ILNU5ArH1i){ z=ma+3Je|8K6HJHUCmq{Yk|!;)i)dl`Aa^$+e4xLA7yB%o2VW)PhIUIfDW+r zZJvL@cHGFR>`v*JFzFlD*e?Dc^pS%Y0UfWI>4*tc*G0qNj2>B$KO5_YGy5rlHmDZ# z@JWrD$miV1>n~UR0jU&gk;D$e9q6CW?$F-PdF7172@>(t?}u_g$6681%$uFy->3rX zwLyi~vTwRfzD;~~x@rtn+qT2JSkk_4g-ysR&MW%B_w2YwWUMGT0)bqR=>$kqMP|>_!)jHN~I3$~5$l z=%aIpajht_(|Sf-;&I*(;Y}vlwqgEGiyM|yi{I0jE5$x{oi#MOeli4u^H6HiHT9&f zG-h|;@+aV#TPhv~cq&R@1*9Zy2m(`BN$r#^&4b75wn111Iu{0zIXoB7I_&Fx4b{D^ zLLOfuWV`y<&TQNzbiE-XbKb(#Lr4MwUoQ{DJHwJbs4os8I!R5>aM!;}<#<#k)lYv2 zK9jzj#W||RY^}JYGa7#pjmgouDyhVZ&vmo)4r6=VV@DPfv-d+5gO>W{hsx`>0dHoH zN#lG_pYssA0?uQMYeJ*}_>d|-A|bR0l2t$o9GJ=&)tcJp8t|9b_(tf|dD=T!@!@Ps@Z0Tj~N4ASz|O z(vqedBVj{jkJ9{$Kg$^@`wX*TXTeZZ=a&A9O8#>`yi%`vB$vQLw&Xu}AJAbqkL^(r zRIx_$cPyNG3_LeY_%5>QH4U{H2Fcum%8s619e%rAd^i1(g9(6IgeYr$eYBmd6>ra% zbt}LK(Q8`YN3xi)!)Gh*p&L|;0$TiE5nFRm+f%0Z*q$#Sb}#jRq`rY7Y&S22gls+t zr*!xar>+hdJCH99SjT0}wP+;W%6GkmPXfF@qo)%>Ge(h- ze9<_h0;Wxi8b0@bT3QrkJY|%0B}9Ij1+0$u2^J{#CLv(P0f= z-(94^9*Hg9hC>5AXXC`8l2a=Qv_|!nYj$qMkus+Z&X)yw3LWM}T{Q&4U2WdN>u}Sy zL)#lN31gAla=zWV5;9>vPumZ&Tw~*Pv1xEHc%LkWrPWLZ5~hzBH`uJwC)y#1wp*Aa zFl4gX`gWGl!{VuXB~f7sBoXUkA?sBh`p||I`Uye*cs6J{ajdcyB82gwLZr8BkTl8~ zEO&GWf_zb&0_6{M+ZPeCz4Gb{r84+m^$jkGlyz|sbAHlu&>eaXFNy!_m_}b-%VH$+ zUuEMUPAO9(a+!l2gs59Uvzzq#aIX!+*nvKUyZ|51Ds5IX%!wO5IdS z)?y#9sTEi2FPy)4l{f#0WiE@=xy5Zq8~bZBR7x@A~9F3~#}*E(K59Y*m0{%o?nc)dmhsV#B>J|Mo$>*|6ri3M`C(T1GKP!0N zy|(8xF2pW&B%qqj`aaW^bbX?}GyCypKyH2vRcwTkK|vzRpuPf0f>xYNJsVb3Aa;SV z_ygw3-BzfT#e$6cbi0{*(8-lc@k895jlO>P4wQgAOl2_n`T>kDyC6(6M8*45UPQ68 z<8^o1$BS|AXx!EAFJO`Ii{<;`%CME za5uioIVuON$x3RGVa-WI{7Ur!<2}{UI^DfsdumAwEwGsTXu%5+b8v2`OQ(Y`_T2Sw z5Z=wd9~qc40_pqUlZL0mnIgN)bFT8mUY`)=*>Zp&dbrZCw{sq|a?FehJ@Re)nXq>y z-{Ri+MuqmkG{K%-2p{=)1_s7KnR)Oaa}aFPziJ^`=we5wtbgL!?Ig53_EIFO4;MfJ z`YaR;0g!j&sMbbYiBMaYYvvzQ<7z{IX|98uWWFa3OTZ7)iu#X>1L`|AOn(XeoKY6< z6XB?+3e#H8ZXt2L%ch3uIe@tqSVZq0vO)8I>ugy0Qkp3+-;qxy?z9tH-0yKI)~hMu@1`3_)rqNM&=qDz70h3PGj@a*|2-fjYbCSyi5Db8@{a>u{?Xp4Zc z{TSx|e;QWj61;WT#yV&L*fV3^xWV%0IZ+$iZEbq3EBQWK$K%Ggqk^vDPm!ICL&KS) zFh77UVs6!VBz6gt^a>z>;A&XU3`jRU0Mrnx>k0&K`F=_yq%#A^L&Ced0>3OuLL_xb zf@{@x%~se)EU0jc(h9(1-3D;~zMd#*Wl`8?39XifMp;ApL$o1;5fc&s4u|@l)4UZa z_y(t_Z}1QY&SvRJG;FOiz30`dQY4Q&op0u_PIAbckE`d5K!gxt`JnM~;p|-JSHY!*J-+=Ki*;>Z}95s=IIy&Enih&Z!R(R8cBtu%_Fl znfmf$JYZeDM-lS2e(D^C;NXr+O+LT0#Co4e7C`Z|(?;K5t!wVt<7kU>cA*CmeWdj& zp*`#?jLN@H0ZFMRlJpHZT26cZt-UXhRWlp(#^UL*y&uJ}D zSrAsy+$u}dZgG)&&<)8OT#N<@5sJX*c%Kd+Qy?H#9X&*m+ zqaqcWEUl!bvV|#2B*aLYr4U(Lrb56*^ajTV9=DI6w_?<;=Qo0ACnEH5;eV zvzj~JtTMXV3#%msLROVidc3|f0{39QT!#`UaJ;K@7;w~gN1vTyf?OmmKJ}at3>ntq z@L)ilK2^--Sf6N}(uS2@1Z(C$VUeSFW&#pK$78bVOF1&Zi7-oqm*@+(Uv1F}vaZdu zvRNFw10aQJ`04A5@>30r;mLnOmAB4?!;NEhi~C9;zaNH|z@=V-F8LCUY*WILeNBkz zt99&E*DcmMSx2C-I;6xM_(sgq2}0EaO$FC$hZAMOtRvT!A$QcE%3GV_hgtks?q>kg&u@^I#T0O$kpm2@fMkYVpJ z0bnhBW(?s?7RUN_paqsnNPD!r_yV}zY+qVp8Fq2yiljYIhGv=}bfejjbdt?}ZAeXN zwRSMK+4w_Jm0me|XMfxon1F@J2@yTWpZ7FtS{KgaTXCGHs|0T}=wGOQz0WU3lTWwg z7T}2kYEUH`+O|9P%Nc6LtrS+|Grg1%ktLAm6ge{kL7Fi2F@BrinOq>YJQ(nBzNP6@ZfvIa!RWz{$Lk)?*aT{c+eu)TNs9mwerb6 zOdqdr)z~7?jb&TcSZ!Y{V3RN0P*7b{lWN#68+g@1Sr|itf-#zajNw@p-VQ(+TiST6 zy)n6z3SIDkefU-ht(6#FM^Xw!Ul>R3&-qbq52yVsZ*PM&ZrBSV*;^jP=0(+&4VFw= zS%P|apkWyi_YpRV0?;H$+I9|NN^pA0)llbPDHZjKrd7_Ww39O0Snf1AB?cd$+g>%b zBWvgVsm0+pp<6UN(cO&&T*nEBmy)}orJiAvd;ooqMp%jJ=4@FF~A;S>;jF*X8=BKEs!>f4_fD>q5a>H!tc) zrQwzI4>XN#!ruM_ZPWT1ZB0N`0=dH`2&f+q3+D7(Xf6!5+-9^b%+R+UY*NlS|->nn&hgDOv0INfUE8xD~rG@{PuklZTaeTaMIdBVV?*;v<_PM zu%K_m?IieT=$n#0CWJ6)6J6}w44;;)A0qN4%l}GVi>ORqpi1@^ZI1c9>8j&<_Sgj) zJ7*{aZCNNC^7WsIMCgBwV*0`|;jH0=9-Os(^stVSD{lyEv)5+<8Q45GRyUuK5X~NcS0PBC-bO{U3?^W&)ZTmm7cza>5 zkNk=Rm7?)oa!P?Uk?vcf`%!xXd^&JiNMdwZv;&XTAgyN=D`#AKsVN>Cq=yzu4|?6T z285Do5Cw0#gN$ci|3deAx1fAMSllvhBJ><_A0w@?yCjO@K;=Wb(0cKFEC~$|DhJQ3 za&_jRn{`&>XGK=f$3tOS(3xI%Zc1#yE$pgGe)H5B6pZ6gq>h-0ugmGfTcOsBiuN~f`^N{3Q_7k{5WtwS#)a>#Sy+U zR6uoZ-$Y9YeXI+WE7WwP8p+opL$d0%&V4JnvP(@=Yv*{(XqDfC1Jk8BflqgEr_Jx# z=Ea7$*rWxz04@qsesq9H;4;oqp>CLpA@Y;qor*+0dyxb9q`tt5#H|c<*%~P!vlIDO zyo&}4g@`VR9@-S}LWyrrt$^ZMM9(W+o2P$G>D5=v#)M15M~&z@*rp^EtgxHz>&3Y<+HP50 zn$}7wX);+d1uU#G%@UonAIX#ozylYgLm2s$HV^0@c>PAv-Rl9l)S=B|_(LHh#13|Q zTK{!Rxd1V;06Fu`#SuV7K=8`lb;i!sS(;wo$;5Zx|CT<1iw~NGYxdBJKfemGq)5JW z2rmwJQT;PCiNI0FV`k^a@T+Jzuz2@w`sTB(W+0!U zqexgJg;m>dQ1ZjT+&7SCV<_fYCAtR~u5s01GLNhYp}9Ft9|(iubkRdr{Cm}mu=8D` zQ`zi6gvABY;%g7wDipn--tlj(NOFQds|FXS8t!Pmq*boDMzR|ir-p1`r39qOG^>!e zIHNPl`rCX{OQSNAkxsF5_JeseJ*0 zgN%*h3$%{zJ4GR%W;`&GPeMH;+iVnlLH%{M!pV{(!IeLBjH#}IY*G6)$0}&+J@_$P z$M&4=w0c}D-xuegWSTpSRSn^0f(6rbSW_nKcxmwscRiCIL^*E5NtERrKkWu4$J+Cs z4hsGe3%{Y>DPi#1$5oR)18Mf@0i7oQ1$Bp%?>#2rP=2=Rlemi2hKvEZn(xYR%>T@O zx_EdeK?uixhW>c@ZZ_0n5 z1iGXz>fCsltr)n440!E|s;03!K6Zs;PX@9`EgGCZ|D`P7j{qb|+^R`;ga-DL15e)YI(pN-j68HiP>QZP?25w5Gwn%ll9GM!B(X0xR|R>6qqnL1OrcQ|#MM zJB|_^PZVk(A;Ysryd?vBM1ozO;ppVHn(Q5ca&sNSwp@Ie%r^R@d%o7~7yK?mR>dZr z4O50jK;|wx$Ma~~R^b;r06T)q%ymMiVgUyApyO#bT0G4k%PkfueIg>ce4}^`*ng2k zfcon(txj&KT)I1mL;kCs6oJz*{T<-Vay9Y{gqgXeI^|o|`4}jeDuzAUNfKp5&X}QhsQfqPmS9W`W0SH!gW{IPHh1-rl3(^Ld|I?>V zZYRnNUcZi=se?Dc=Fa>|a=`s^-Oe5kb|s}*OBp`Pu9S{fi|EOO1x>G^-HEcwP*TMA zvJ>vyl7b%$fFu9|^K8D3Mg^Tyg~)?#KZ^?mjWj#FhqHFb&{pEo)JoU`TWRb2kaz^$=&fb&w)N>n5taf^f;o zl?I;(!9R1<(6Z*q-iLC$HS0prA5$NSHHQ0Zi%I;9;-kH zmKT1KEAj}=B6zs+eDH`0I`; zhVS-q(aRNRo;C1yN)k%ae_Y)UcTqx&1^JjaYZC3Lr=;Zn4sjJ{ph48tZbW z+~kvht=q8!8G;3@KcF5o0~`rzM$iAJ&N;MIrPs`+ZHsM0Dh_GpJ7g&rIPn`sSvSnj zOpCN;EaAsgOnB(@(5K;R89}(OqQ&6)9!x5;GB1QanLnsNPGxSsdW9KyzFO9bVb2qKw^jR34kSSs259EhQofyclsBBhTm4^V>8+Y z_s9*Le^=}^LVmCmi5SsDS<5}M-B!%~u5DejC9D*tNYwXIxN7~^F_2dIMZGvQLKCor za=3g7d$fjS>G1&knVldU6S5UnSR8~df2GzO!tpQGqF>n{UglMQ9v}XL0lG zUg1#OlpbbT%Gt7!)^HaB=q^szSL#uvp<&zI6G0uEuqhQV0GFYONA@-L)C&h%a*Br_ z@(L!!k5A%lLX$80EUN~>ZXEWxeeB?c4T5R?$ngERn$q*RWS9at$opx=Us^1d)ir~X zt4q3aEFeo44foAX_zO_z6$xb6)+HRuQ#2@gUuY+4a9z{Dt1(SvtBh%S0cs4%s)AZA zV5SkkmWqA?%x>#Lr`Qh*kI!iq8}x2nEI=Fzg?*o}YGLh|mPs@ZLp?lLB2vu*a=rQ> z9Y$w~I46v-mc^YGC=6RrTMt|Fu$9;&$up7_lp>YmfAXH@s%v@*2OT8-dqi$;wV zWLUVBR%j_Nu@!u=j>6a(ln=1&Iutq_(muMK^9keog=~#6U@=_9hgTGIa+M)C>^~7V zfg;w@pBRM*h27KSy0lS-Is-CP(8O`R_{_>tWp>XC1ZxL4V>!ZKF`q&&WyjWp_}_qo ze4uM%-j{%Cm?wWLteh_byGzdT9)q(3Kh#}q9!r$qH?O8$bg(X42`9Nhc6pNJl@upd z|8LpHDX8ik3RWOkk`Tt%)Ci*7&T04i-`j|VKeAT-w#|-b8mdA>$eRci6AT7u||BgWGiq{L=Vb`zGVFo0Bpxh z^dU+FZ2?@y72XSzOc72kiO=jl{=Go=*0KBXLPWVx*!gRafNyKo&?+setm5F2{{a0L z0Q3OffwFG2G?$00VA=99dY)YF;f104Ik{~GR^@pRMx~}tvb;)8OwMbtR(N;0sxR!u z*;WdJvOKTUU`7TQb*5c9SWhsPYih%fjt_nQHz8Oxtq|YzVG3%yp9)Z01aNKkbWHa| zYvucm5Z2E($<>W%(ZU?80S7l*>*4-L^=b8ii?nGk`KV3(K|sy_wmZ7pl@jFqtcscy zjas)@N0{UgM&@J^gk}IV$liMtLY@}+D*KRdG7vW8M!C>RN=9b0zQW$CbKkUOl%eL5@lXKsAF88Feg@!La?T8u~SJnI*lg66D{jdCK=Pu~O42M>G zZ*;=;d8)0vzh$XDl>9R09v~fJdPk?BP*=Xr>o!y|gql(>1*w>0|4Kq1pcQ@TPH;rG)lF!fP?ZAtYFoXMK znO_SD153bhjX7?ZWwesDP3))Dp!dVcFE-bVYy--klf;_<0<9#6_mxZ=#rAQw8M5!# zh5*39Pb+zeXmMy$O3Yg0CaAkWp{cF%%V+B&i@nA3e*e`Q*#G;QbEHKf6#akXdDUa_ zj?kh)-&3yNe>#SQaC~v(U-F-)slL&r^8D^R-cV?}h2;;#$+8RZF(Xqnuk05~OL&{T zKrQh}pnpgx0yk`q#n}O}lDVTP<`OzShSvy5O+WG18sEbj2nc-3*b2E3Y5cVOC%HXS zxc_G#!~XC(cM&^&=}IBV_->2h$XcO!T~z)4kvNh8N#!Y?>NGcaQ?~E%LgtNQ2b=u& zrR9AKlB%e=tK=~AB{m%x9f)7FTbk=KuaJ1$_x|htHQ3pBQLFk>uQ&@p>e9(ol{Hm6 zwj4f^@r6Xeed1V2%g=Va!Ox^cI#|c9gx3=)@+QAwQSryqnyV7tdaMe5QGUUc$d`cd zmkhT&li>t3OsS127ua4Mc%fuwRKNX$l^Jz*iF_`J`70hf1&flY1Q`Z~oyhQt#xe8v zKQ0C9zz$HMiF;cDFXH=f=dwVfX$=Y#P6`B1AiG%{(qgd&^fBiZP;W4*0=sqHEK*?! zzmJB4opq?I9eyG%L^d1rY~AN#d#1;%FZNQlVpT0X1fpWK6?u1yri2A5fIWXA+`QR z&t>KUu<_e__MWl+$k&J&D6aD`^@XZce z!k!7@y-{Oml>>WNe(=Pf&VICF5an){hWSYRL{Ih4ja2ht_wG3qMH0h78mkPJR8^GE zHuvXRI;V$H@^i^9szLUn8gt{$A=nsA`}&{%M37QBI|dLxMTTzWnVXuhY~{CMO+f|v z5DG!$s)$w-cpPE8#zsHs$1#YvUab`KVQNTbP01WdfGs`yMTdWwojqmGd_U$9n^5nR z#((jTkA|Q&c}J?|U})|_f~X;UzAfQ) zV#DQ%7x~ovsJvM|1!&&_rS!0q4y9)%z6S>t$}Rl{*u*w7z5E2I=YebI&^5=cwr|R$ z{~KPzhxfp1nC*^raq>rOy~V@kf*xZAzr=!D*mt2+ygWC23|J0QHKTd;cerJRzWl(R zPiaz7ziHArly0zM$)1*P=aBCpOM#UMFbVD#qaMdtMdZFFM}|CB6=jc#GnllZ*ary! z7kY5}tsfvmDxI&$ANS!o?}NfplF)RC%D+mk9fZ|o=eEbXBtoMPQlb!Knu3Exng&>t z>SYD~+HmBjVl6Us(hm1O*SZDlL{oUG4tJd0-3v)2Xw0Osyf3&;|4+yvQ}*!O+w3>y z7Mi*=F8wda&=N*g4&LcSw45kH+sL30QZOsS46+|yEGY9e|g0;9av1jUlSta|~_zWhjat@Y~APk2Z z*EssvT`T5U5eSc`N|NkDF^Y*XRQ5m~`?kteFoOFdWjr4$EOv~${jmLGq@#E|?Tsnlq7L()v(VdM_mSg%30>t^KrOsv3g}Mig^Bw=H5FWP7Vbkc%+x)e)yCM@F~W4{`V*t&pTwP zaJRwmA{0Yeu1S-!}8b$C9sa4BEl6GP%Ed593VKNnZ=bO72P=!VnVD z^PTwKThlphHp{N1K6-5>0_u=v3*Zn+0~3@BWP7IU$aaEk?UUZ7KK++m>oOvh9Zoc< zkq>lTV(1{eXP9rpDg4yBJUdm_mKkz?Nj7cohwqzdrglC*?QWwZixrB+uzpFai3W%a z8}kB#Q27%m7t$d<3&VmIv(4r9IKBT@aM2D^Qsa&}pc)PWX4`aK5T*%l{dU3o9mk5a zukeky9lUcpiaVhWoH)4F6pCpaNDXKt`hl5C$`}vam&eXhPxdB}%@NZhXS2886qb)^ z$$racd8B&?OVg4WVQ%(ElWoaivP!XbVwXTAkRChzT|;MfE<{Ofs>DHd=hgE~kV96@ zd|g%cQJH1QXDT(ykrhvPw-5-fO!mhwW7XB9O3<8nTOueaeKQjTeT45_5-@OkR{A~9 zasW*0r0kU_COMb8(q6&z>bGYobLi40)8?F69mmL1xqds7$Slzrvn&s97!6#iH{iYj zhw7BW*V-wIGCqh+ygGAgg+ z;sO|{rS8k2q!0v)`P^d0Y6LY7(k_BnV-or5qchigK8FHMvaG`5Y&K2GH_z&yR`3=I zIBYgOdoEi$=w;CFhkq?k18KGGwD(x)cjU9ym72ib9qMgNK`O`kWXyv4-XEM)&mHA_0}-@L2yL0+2emyX&HicC zY!Q{EwEcyp{|*$cJbz=y$uas#I{F0PnJmh_a^a;3bVU_7eFLIhxWFxF#z;UK^4E+c zQ68M4EoQ&y&(y8#A3PnF8SGf@I$*@17fgO;YZ$|Q3DaWAl0g%~c}0L^zkXoMv966S-Lc^~}G97>#0SM6eapgH^`52kfR}#_4>VSM zQa;1&6oz`LQ`jHuyeY9il2pMFssG|?-uDM~15^YlZ6o&anfY}mVWuO#_OufP#lrq0Im>NT2xON@y$b&rgUD2FuFKoKZnt#cqkI`!+$>sIC*9{}%` z>&YzXlkE`*9BOiA9o-rtl2G5rbqB_`ewfSIMZOa52?*DjQIVT2W+ud@!>9bD`!iae zp5iILQpwwsuuwP2JAkz9BYb5>wg37ex7THvAFsXfv2eT(3`*3WG?{|4ZF&l}*(2(4>dh6gF^Le>Ye==esDk}#|B3ud-bHr7XokanqR9XIrx!-4YDC+P`VO7` z?=#?5LS)KEdUG9HD=Yf_AFSaex{MH`544FMJE*e@W;WjZr^~Z=!JMcE?HG}}=L}s6 zzJIumOH)uu2cJhBM1_vq8Xj4?8bd`G5EyK7dZquffofO@*&p;P4=aiP!hN*UFYDl> zHoeQa@ZO2BP6syJBCo*m`6?E6bGZ5E@_||ibQy_be=OD$Xdl`D{|3ME+YJg?jDk*q z%CdDNLocGITD}<`KT=*u>(6Y3Js>AK;+g#jA$~^adpQ1}$oTO2aMjeIM%fLJv&XI?&f0E5nNoxgDC^Z0312&Rj zH|l@(ySGt9;AZLw7LzkUrbpA_5r{11?PaEoInNpHePqpBHXQ{e%{Kohe~*(x7^z2k zYq{LS>KS}Z9Bxw(!gNB1OD5<&!$sOg+fH$r9Pa}in16q7_cCXRr#gu?}G$#P;p`YwX?0{c8_yfROwe0@)W_+73- zC_x4Fg}6+p0qYFs5A{12m^ZgX=J~@&{)EyGhk&O=2g^F$cQ@_8L1XX>I5wf5Cfou> zQ8#P3l!%}3h6>s_5m+)(XaHZ=M~SxYs1{>6DBoeUsps?f%rD_q{c^0S-{7;#(=R3N z>>+l2xS041XN@!M=MW?B$da#FbOVS3E<|DA{T3;wIa_M43ef=n8)9+ zaX>9rg7GsW{}S%B666B1wdrNTluWvwKsQ_l%JpSw5ZBNq7YM*rSspJFcY-*U89q9h zCHrrg+d)#&(ok?{e-2U?D%C*~Rrq^tPn>-W1<nA2!jzXMX7nXoag-I9`^t1s0EC zcj`3!%t(pas^Zie*i3jpHN5i#c{IV-!M~(z<~LAUuAe=gfH~ApwhnLu$S$aH;u-5^ zKdlSk{)`I=D8vqSPnog~05%@1v@TUuwA#n5M2nXY5j~Iw!k{jE30=<0#|RU*+5Rj= zp6>WFgPN)nQ*VrPz4N_nXY?O7l|88w)2-GL4nQ|#=);{91aa-n8SP=$nGD& zAWQrX2<+!aM!?$x9v}pYPR3LtV5ZV4iQ^by^poc8N4f{rZ~?|wo2nO&vvfaBE{2jH zL^Sc?!=mfeEoMto_*35d6pXk1JecFE@W9V}%m)qKId@YZeJ1lx=X9)gS+D!ds8Uce zz^ldkAFq^x;0f9|mQQdYm5Lg<+T**i^*yXHhj5Exj#4t-9!8pcf4qscZJIlc^tJg9$aXHr3g z34f(XM%)2rJR4i4*C(AP8yAHTQcR=N`00fM49H*;;2^)Zlz&v)K%rvWtdw)MbF|pk z{X$(D3FH{rZ{P_QK}{J#@ctHQ6<0iHcJY(;Q*wIBT52zKkM(5cAMpG}Km5;Tujx2G z(s#Q0{|}4dTcOX)VC?aAbF$ zFE1G2V^h&1oNVS{2Gw@^a<{OXT#~iP49C1G6^cfYckhDDty}oqO4I_BP1sow`COx% zp^q?eFob%E&~7ELCme!hZ!fCWJJCNl4nNbynmSp~_Q4hh(lbkuY;NkZX_NgUzIx)!IvWmAi-g&<5*;Ctc{2OJW$b4lSeLRX{ zL^LNkGrP!e8v;X}Cr^)jZ_ifPihxJgP%X@y>Cdo_bTceei2W%20nYg)L1ucD$Wu^0 zRMk6SR|JXL!I-S6c`w<8*@HAiIzLx`|~Eg#&Yc}lAi7?c%)kp(-XVE=NL8A4o8QL zWd?|nkp5iy}WX&=EiulT9pEhRu8uU(|-;oR$}+UzhnxKTFU;p&d~eFAqK4(z!yP!FHj z(4GENYy;O82lu3(lOU`>KxYh4`gUS7r?zHYv_AZuK3g+V#mHghrDvicrFbl9O#J-C z)RxD??AQd%JTJ_9|5fWCN*g_VB%K@6t#f!7{$Hucw03ARnK>kd>pRH2HDy?I9M(x+ z`Q8$u%Z>tlSOrdl4``nN;Rg4n{wxrz0)}wtZhlNis6=>>M0izCaFE5lGj_Q1?PVh+ z$(b*hc`C*0XsRrMc-&_TfE|?3Ir2cw#>9}1&KN}aoY8vUd{5Xb7C*d1m>rayop&N z?z&yYG5BX{Y9IuwS5|zsnvMe60dh*CRu<_``2+k6iwCjhoQA%00;X-9h-M3_T5g_) z(0Ts-;^j`IGuhU9wz12v7v5GoU!En9&r-r|5q2xSDRcR;HgZ;0C1m7CYo1U;6{W^S z3JBv83@Vh8oEJDM6^)rzy>4_CCmPJ^!`vnNIpla_?8z4TV~QZb*NiJ{x&dBZ_b!&k z*B+2VKv-%{M3GA!6>QHA?DF-^kXT>c<1loAx3?Yd@094YIDd7(0BeF2J9)C@fvIlX z5JmC0L<4>pZKQ1>wc`-p{?NkG1Mi<-msTAErpf5+NV=n|;Ef=;y&YK@I??|Z|B7-R znOt7R7v~m>;0i%T6kMfYh-BrRV`RJH1NJomoYPA(2J$#@8;IHigQWr}`oOLQ7AMsy z(E^9dPj#Bx4yPD@l(e5LqSU$>Jrz3TPj!pnyyS@=mdq)5qKV!=&`MGkAsADqd9 z;7n#SKwow@|9gUT}~_0C3_ykFiZl%Iygg}J2s4Nc^!uJfY+pY2p$U9cy|x~e6Z1Kqf01~bFIsHx)P3zlAA!m zeXK=M2=}6-RdEDrdQp%(FwsN2&9BQEV!v>c543C5_7Ejj&yJvJ_Dli_({?~%FYNZg z5fifLXmx=jTJgEx|Cf9n!J7l%_wiGTerVQAVaz$2^?{!I3ZWZcTaHFvX-c3ZdPNB` z>|jtO-ZffLvMjy!^jBq|v?%s^1S)eLLO|LDs0?}WUQqS|_-}F8bFpEbi%Sduzg+H2 zUqqVE41Y&gXUAw=D09NdcS~wx#2|9`iQJQ8dvX+v--3=DM3tS~sWLEMBbp5a<%wLo zqV^xVg$b9lQ*y>vQ;jmxG_jZd76E!Q@s zOEhu@@{`9|uj+jv=2ExQ-n|=U8>v{6{gm|x$Q_u_{O_Z~`G>_-hqGfWcr)q=u!S!@ z%;HwUtY&y(qyHzAER2@+Op((XGu~sqw=}`5+Zl~?z*H!|CF3|3Ba&_jDVNq=b4>Nd z5fY4coD#el-}K?eIu(zOe8Rt*jkRl!APc($@sB zX>iRmdn&n1jmS?+5Zh4AOWuVdc|gI~ElfRa0ojqiE1o*S6yx{cC0wEa8x_2n$}15` z+IxwjbBgdGlD-!^TS4D{uXt5YY$vI4gbr#LuZ(KCDS~iDU0udd2CwdJNLsClUcxwH z+G-<0mgpN%i}5Y#C+uO8`^-7j!X@Rcz01@G1($d@Iu8Jj`dL~~v*!f7>~weVz_>*w zTC9+Zra3*j^i&4n5}Y8~Nuk0!k9ujQ{MPY2HVfV~nD&@nt;;hg4mv)vp2&11%ZjfS z-Tbwbu~cex6IeXL$#y*ed_;^n^Cn95&JI3S_S@PE!~xxbQy@9+L)^B?Vi> z;Y|o&=#*#`rv+^y340PFdvK$A|K-1LV|UtEk>oymVL-uHQ(JcPIbQu=PB-|4?%_>@ zOHP8Q#qYQgEfVXg3Ms&S`DJb?D%OKs2@gy7OaO+jPWd}PR}7QtM_zcC{ZK6((B40- ziG#SbCUVB8S)2?teyp9(WZPhIB;Sw-<05ovYyz3~p(2^R9g8UFbhZabVf8dA)8yPv zLS?HIYcU)_3KSW;L2LCU@Odjt3%@nMuCXB;YDC7WQ84GYIk^1AyEiF|SEos(F8-RP zM5@{a4z66pt5SVh)3tJ5?X1wCE9}aBVtD@@9pJA$RscnmiuHusaC(O2(X7va!aSa7 zA-d94o@@c1w1PeGg^6pXX7eoUedsdjY?szTlj%v)I|TqJ$_aJf=d6vWHZALXx(C1? zXu+C~lM}%07TbUI@DW`@XKG_z5izW+Nu=U{e~}wg@D`fkQhq(8JJlf@*RUH(5}r=!ACK(& z-ceQp;)Y{bMmS!!H`@_JxF4Vx3d5K_!4|kfS!Ga8|1jLl_&WQWvHUz9;zNGwK%;N8 zeVRCLxwoedL^i0*99s^(iNac9%O~Bp3DmeO57DnEPNT-ILHdzlb*~Nb!{G&A_QGFb zAy#mBznF>Lv^2y*MYMM4l;8WM50p$)FFKn!)6hg|Y|Pn?UIL5Fu*<{JUB?Dj4a~^k zH?0=K$%)qE`!`mpmMBjoKKkrl2Q&Kivb*2r9-A|d9M1-OlUbjf1c>@KMf9jqBFKPx z^vvduH*j~oCyf_?nlmciUK7EvyqpSpbn2P(_N;W7h=9Z^uZ=SyL@f?{E4ZTs^r z6D*zwtuSc-lU8saw_ncR{nJkPoD{B5)`)Ie@W)$mnxH4I(ZBpjg<<{@N-9C=*g*Bt zV1dxl;hc18r@~7~Wqm3ARwb1_R1d+@liPHB(xUQYG0@PQ9aMyd@_5jRdKkm}b$n33O(mVR_IOazud^=e@NPj%es^WQIB*CT? zX3QfREEEb!yJDc5*kTLrz#kl{A=wwV^bII+X_dCG1EXmeg-))2CkpqPyd+(KK$$okny8jbu|U0@Hxm54wlt)9C>9+I@IF^&bA6c$_s@ z^?IM}hL$_5N3)9Au37QH-_ABTzYU6Y{2ZA2MRG|ER|OoHpC|(SPCgiHRJx35hY*0n zQwjtv8~3CG6NFv~Rn+apMugiLz76Kgs5?w+$YM);{@`^Gy8`pW5Kq z9AbX}Y(HBb!%sLJdvYlb8O#dLGi!TjcsD~@?a_1WUl!a1M*k1sAe@@TaZON&az;## z)SW=)i~A1TQZC#dBGBy|3+yBCk2uz+tDi<;0t34?@3@hh?OZuQ%`Zw9EBCMTuKQmK z@DBa7xE_#m&oaP{mDun6hVde4nPn(ZY*W-8m~kE29H)|DmXoc)WgxIJ7ZeZUF7sGr zBrj5$O20z7$fr<(alS4fJ|`NB$XntPlMpxZ&1AXI^;(fk7l)@K%&u%)IBFO9)>_GEV2p8V6#jgx4ytV=Skh z{s4O$%xXKZ^?3b7^EA$9n2Rj{CnngT>x5cPhLN*= z@Og~(vC`mSGUYCUJ!BT<2EUC+3QcG;DIDneER!y;%4)FITIU$I9%3CLCp|^|ULUO= z?)ZddUXX*G@N>~N_$%zy_J(NPhM-K`ebM!1hOm4pt|<&`R8Akn}BV>+B@U?!Dp{9pb2(_ zBO>GWe9j?o_`M68iZt8U40S+*wdX; z%RZIYCJc^UCWmogsinMq7{>fYk;5>kSeI&)~!w3x=41fA`pA(5X!ck;$1Cn-dK|4Oxg4y zPmB*tYk1zX9-i0;;8}7Ok^KJ9!Ryex@plo7^~uJbs@3$tlb7dyz=NKl?H=aq>;dD5 zs&k08WDqvd^g<~3uwt(B0T%~nc#M<=)M0|vwnb-RYDtw5p zjnJtBTM&ijmRP&N5X4J#eXGfSMir_Fe8Z_0uf8n$!Oxg#xW%rbYCQOA-UsV{sCKY9 zkMoBbp4Yo(DSupT1eNS8KB65{13*@B24G^HtwsLeMv{LLzMVuFZPk1*yg7b!R6*ta zz}z<^-;`ing$9R!QChOk^TZh+IeW_N?Vb+2Zs)v*#bc5;Vm1G=h%mJbLW4=^r(n=) zlyg;m?YeA;F92yKg;J6%oFfKNo%zT%O4&z^c##UA?!SQFy~wGH22)s7aW zTgAC_V9{O9!lFAVnO20ARG9inPX03K-`bj<7VG`_SaQ`_(hf-V88O zKa%irDT|W)9L_gAyT6j6AiUy|0a$r8;XywXCujcTc0eKwefNNsGz-tVIRmnXJ zt`90K0Xi*)rD-c6yV}_{6*|7-`|kavh~oC9>(D;T!W@2eq-|tUWdSeVv@ET1qO6rx zMRx$Egbm(a@DqsE2aLkksm`gaulf-o`7uoiVC^7t^%`+;R(}qcjl}{kgD3nyisg5| z8nt@()%VCT2F+%tc?Z4i5_yS5r|SpmfNK@#?vGnbcszmWG*(+9d}AUrATm4Sw?@um zoa}z6(YXQEt=tGWY6RzIqW2gqy~2dPG$6ndSu&rE;&RB{`h(AmioU{C3wXRsbKl?* z3CVozAwCzK91I5GU; zj+b!EY6H%IJmw1%ZF*+&`V1hmq4^O=5mK6%iBJT0Cj<{2g;4U&R%i%gdxO5&yq!Qd z9}DBU)z5Q#MT5)9i1H#p=q_w%?7M)SS~tClH{mB{3ee+&lV)^ai z%#xwY%*F#Nc)DtPUIf915cyXV+D*c$%7r%u#=>!K2UrzB@rG8ivn!D+kNb#53)~fR zp&Kf1C*8^PmXaES{jmRhOOy+sQnWS%yG`raW}R%2z#@`{>Y&~=A&~O7a=|tmLZGJ( z6E-@`{Y&9=0@OkH6*3TEHC-}bY_#BJc&)$~%s7rLphg7`1Do1(-W=w3csz)%ZREWL zwJTJTwk$(ew(uRRey^SNTF5WOqtQLdH-y(8(T;c#F`DYT ztt@trYY#g)CUFrDJh^c#JggnsbAxl$L_|)(9*>RGCKSBc&7>@7 z8Z};U@UIj?Cy0zJ4!973if^SrGh*Y{xSSpIksoIivmhN~I=5qdZ?wViZY1`1$yJWk4%9FM~Rr2zv~VuqR-XShP$i3A2*@e=f}YBfUwd2R?+nt zHSpe{auZWh+jF60%bS&5wEuM*Lna6m1tWp82`s55ZaX9NnQf_K15eRU!He5aAv0<; zZVi#S4NEE5e|rEkYWnt%Ee=)9e<+s@$~$J|lm|Hhv4FVNGz&bKAn-6!N|B;^Frqf7 z45to?Qs`4o71*O>gtT0DykK=t3mQyP##1XYd@`C61f=EVA{ards+*7ErwQ;&f^7;+ z>&l8BZ#SS;NaR5s#xZYnEM-(Sas00+Is!0ZL;5Tl^EE(-MDsxGUWLx_!N8I%-@N$z z?H_jRI^+Wu58jvqT89gR2lH5m*5I21_@*IeoK2&(RyQM2X)7jE-A5x!M<}J7Nb*nU zFbQ!zF@8BkG8?6ucN-|j$mSj7ELmuSY}_)a<;J^?f{mayY(a9wemm(j7&b7Q2{?T@ zQ*}hcUI55iipzX1jV(if(}A35&6LMC{rHsi6@M$J93c$;b-T&g?B0V$S_9v;4(d7) zO4JM22m$&1A21?!*GlE)Tib*8m85I)%e~1QxDT%Apm>e821+H%(l|2cHHT1L z%zlXrUOIWkrcz@O{N7tY`ocHUM^E@`yW0hC21LEdkbnSoO>>bcbm?U8``ORF<~sCA zxfkb}8jYsw$ukB)BN>2fTv59{iAZ=27rw>2${LpO#5v^-5<0lR!tqW6o1iI5-aH-Y zo1%!LuXh~4p9wV*BgzyueuPtsd3IQ_gt(Goy%`#dRDxoPESHUx)I* zO3NDxf(xRUV#E4?OhXn)>3czUBAk!3k`~4)bEYcb3ozPIAiMfH>$&}Hi!=*6LrsFU z51`Ep|0=o@S~;IpLPT&*nG`iN!MTU$sw73P@Edf;N(Lr3Np86u+>5yg@`MBQllbH? zc@<9@fCZj!@Qi*NXi7BO|BK{SReXk*0XMZc!IyCTWdRy6k09D)>JCAsh;4g#lh$K$ zbZ$Il=l$cP>7RiBV2=$ zWkvoDxIIy==a5@Wp978*eLGjUs50R07RSsXK7O8#PD(Yo1a1>oLDXv?fJr+>s`Y?! zgrCekaOaI}gc)tvn>1GCl$ODoqm{Wwp8dw+LQ~&HUowqZ_Fc86nBd%fv+hHy(0xdE z8<>*7-9kAXUOECX_aUJEY#TrL%Nw2@Yt_T0<0r!GBvP$_YR)t|RQxIJlWnT4bRRK4QMgv3G}XUIO^fkraiR%gS?! z<)4k1w_&$I-AV3Bg!EyBALZMd`53iUFJ+d2vwE6w+AM!x981pU70JcL-9OS@^WP_l z#KTCp8zn@M#nB8~8o|(bKkVb&bz87{kA?nXm!(fwCWd ze9--`TtBMe=yQGflaORYD&GBls>=>u96G`LUkV|+hUbfJl8K71oD&JAjVKfvEfgo> zmSIQ83@m{dX?7yGp{EWw2-JJ6&H;b{q_N~2f(QZ->mx=x!^=#j>7R}kcbdUG`eXop z5kglps{Tk0g>JOHf+$!#dC%H5B2yb&i^rV18w)B@U+w=B_J!6hUrs*$CYWzCHKA%3 z%eiT~oLQyK&mzr;^hgK6uP4IRMBorPl+o|Mns%p1lpRWzgSY@55~Q5DS#`kAXvbnv zqtqX~k{u+tp(j%#1#?))Gpu&|n~Mf8;J}@ly2T??mXCAf;ngGZ40E_dZN=zS9>Ua* zDR;tdUgb;ui);(~pkk%bJ|*qa~YrdR94X$O` z)dmNXm6w-!@lY2afYqH;`6K|-ZA~PG`1IO*2vRz;kw{?r1KIAGMaMce`Wr77shV=e zf?(z-&A1?$XfNxCp3hC0vhm7{(>UF!{;29)GJ5CWmZ@#sdC%We3R7I53&^*Z{8Y%X zO1^4tO4o#%?#WvL=ypU&eo

|DV722{ghbRYPUU6S+NgClz*I^uz0e8~cXk4q+Kd z#2xG*{!B=!mMv=C3YVlTlIF1KP*dqU&ajsd-Y-ZQnQPchz_eewlIW0B1!hUgKsoaY zpv>S}E;UtE4@o74Djk@8@BW~+Mx+19)TS{ZKMQgslA1=ShA7_E*HavvpK-Faa;*Eu zC47Duon8hm;zrK<@qhWogPz%5Kw9Ch($)WXvJ)SitvyfKh0_T)_X69dNln=MA3L7E z(SRZ;`PmD*OF?x3Fv`gN-4UB(Q?Io`>LfDR)Kpz~iH-VTFO!R}bqqH-fCErEBo|5@hbEd|smZw+G`Af$_T zvw?HJ09JNpC-#g|2~N-V9~h3ZEK^eb*o%uyl&em>`+~jFU*Oo<&g5rI zVp&jg0=2(@GJ<}4(Sc4bh;*Rgj3Tu#-W-9l1o%w$58K$6YW)ZbKzi~Y!82lxx$Bd2 z$s~%dKp>A(b0Fw^N{0#s+ji*?)Gi(z+W%lmr0zWCT$S@!?}l#jtCs;e?3>6X)2|e!Pik5CE-!y(y=Jfx^+oTE-oF!O0|w-eX%&jjnh&8@s=7vXYp7-eJihAo%fkX3lz|{ zodEreOv;^D`MmS_fP=T0R%RQHBgssb$Y)6Gs}An*Hh>R66a}H-h0Lt9De+JKEtsQI z>!fy1m%if3Aav<6la3c~D2SV|{9eawpL3k5{&R$<$ckT|d?SsTH80Lq{LySM+NWJA z^iBba?12r91Ww>cl#=%mHe?|ClxKu(|-!!RMzDFAXyyj>ok zgtv2se-S<{_YZyP=+ue@DPq5s4}P@6a;)6WakuC9+|IIyAYk1l9+LOL0CP0r1uNPL zGemRU0nb~HrV9vrC7gno%624~a_3GKS@GK++2;%(>+|I_x0U%PM`?#(R9+a+(hE_P z!F0;Ke|n1G7`uwsl|Y~?s~a9m*Xtc0Xf5q@vaHlN3&Itf%CThN{QNB78^yV~yvL6C z;J@;se-}4Ho0m}0C+lL5CGb`xtzj&VRB{)AIY#NEttVvjVfV`5KZQuO(U48NP`Qba|B<&ac0D+lWOz(2We(=!OtPhLD>PCM?;YtuK==O2)ZkscJCwn-6mJt@dvv+t!-4sMQxE3l< zLtI|o+%0#@h7o#r-oW}9I?{9SLpz)dapcemK>s>JNm&smkc-Z=jHr{=*BgNIg^q0` z@J+T$<)4-Wwl3Q)x+b#Uv@Y^<%ur_qeGUegEO^cGc+R7!>ENA z=?jmxvQ3a^2tC)~PfqS=J*Q?7S}f^st?Imf5EC!_q2omtDz+((g-i6B{GnAE2zyH)o=H{{nYEFd z%12qWWLF9H-B;VAMknZA4w6Lw3Dl;G@@>0E&I)}?$%hEtRSM9A#Xa+>L{bvU!bT*( zFwEp^>j0ToX|coQ^wL)c(MHR_qRSUu{maz?{-$dT|5CTTEY(zF#tY0)E(SRuGG$LM z-T0se_tk!cIx|y6WnKzFk_55fPy7$YV`5SJ=o@4te+Q2_1S7ZOvozZu2f^O~(oRuH zX=F2sxHfOc>*A!RxgV!u2Xo)5n7pHJmveSWQjDuPbfw7k>{p9VD$SPH4qYxPp3^M< zrKDS4*>ucWT6e>v>iLyo|A(wEkB55u|DWngSGQdEW9n9-xVIQvi=_}PB*|{bPMe6a z#aM@Om8+yKQdvgXcam+aEmDM=z09DH7+bc%7=Eww9^L!>e17xCeOxi`_xrrhIj`mU zdcK}V?vs$I#xDfZJN%zi6nA;I%SnI!NkP?M_bow*>q~R%6YQ@ij@}LY<=8Lt%N5nV zs(KkQ@Mc0wX@74TwK)c87Oa9Ro$D3H9~nEvuP;)41~Jv`((UW!de z;T!m1{fMbkN#v2}Fs~eo(fcgft0?}#|9%^Gpe0y&;WKPWY! z5g`!Ofa+m9p*`fED`_kYFDEps4CiF}NaR6+X1VB*yq0OSR{9FC_){uJ&ah&8hn3i7 z+Z9NJTs-VcwFswW!am=mL*cVH4U&G*5wdGX`N>38ShFD=3{_poR`vOZuV2{Sw)t?% ziyyY`Bgd}2m>xmPA)}$`I|Jp~;So&P!ryEUfc}=p#c=vNGmiQ|TtxMcR@MoC_+}hA zT6l_YDKDy_IJhz!OLs`zZur;%?6KkR67Sb2vy~6{Ec#yZO_hw272G) zQO9p%kbJ1C3qkXVhC0f!G{ud3ih+d`>aM0pzVCdxnR+z>)z$hQBXtLFDUM_KPgW6% z7KZ07jPlv{8>kVYqL&J* z%86H+m&c998b<-0&V1NPrdSd$p0=?B?tsJjqjT}zLPIc}#I``-?zc(yRfnSloc7b`f~-r z6Ygni=FrKja@N)XE03+Qdauud@5t0GIz-t~O$I4z=~mHAb^+P?AAEB3BKbFbklwh0>K#DSv z*y)Zs06tdCwrtu-*iP#JQ>N3p2<;KX!A;sQTPOerVhlznqmL6A8-o>BOq5)dz#a21^cIM$!?nkR1wqg7{G=YXajV(E zpCiCDInAuhO(Yf61;XmA=|>p#uQ5aMz5bEPn+8xn7jGRLoHc^FGWY7`b_is|!0rf~ zrsi3@tZ~w&DXIxa1d#vCdO^QR`AR!J2w`M%e9`F^mFLEZ6_PC#$A)0aA+bVo!+iy_ zS!VZ&RP~Z>Tw{RxeN{6oQQyi0F)>_?FzPmKM7VM96Mywn?>|m?%^eOUerA$*l{EQld{76t4s{&;1$Q$39$j-M{k!PkI7P zaIfDA80QndB46KGkf!!8{QPnP_WlDiKHsRaH|{3!IX@c+I{45LpSf zj0kc{gM;m2qlM3e-$U2}R$zNjFUq1{H@H{NzJ+bzY6>aI%$HkJD$f^v#^q{cP%r%k z)*wle!PFVLjm*4QuB>rSj9(2{6@kHJr*Wz7DAj8Aln=@bgcDFkt=aJaC6DS!#761& zvuZBIze3-oJzQ8&pjpAsLH7Jf>2aTR>|9uQeQBYvKaT8+uJ!_Lz?DvyTK!TkIb^fwrspf~DR02l^c#B( zG()UIUh+56k|C7vW`D&?U%$b*dUZx*MPBq=+0ueI`98lxk!p2lXld#E$};8*r_kwX zp^JhQ$-xhX;8RwJTgo}vKe>;5qgy;|g?%$L^wZ?c6=gDTw0lo)?QCJUee)h&JQRU? z>cvD+sOE0}m|}PgDMNl~TFYt&VoTXU`fGFqgt|Hy`T;DItf^}stT<=xa+;|}cY@Q7 zM04j%i#?V1Z@OM&vjN9G57}J5*}~%IsS1BfYD{$%#fzwp#5H1?)i~9WVULhaDd-ZE zm3b15J3^|Za_^P&F8CjY&+&^<^ z`w@84HSs$KHiNw=(FJEGDQcZ!uy*$8a<))(!7>{fo|8gnnT;F5AK{O_Bt1N1P1#|< zcrlPH9%hG@(?WaBo8Na1=kdEFZs$X?8-)&Jt~;M>ShbT;4Kz|gb6 znGR%4P}ts}Ml@Hhrd~neEY4QLiFzw68Te3Gpcz^{(>n~+*$$JN0=76!#Jr@;0JUQx<5Pfy7Q`WJpMu5&EPZ1+Xfl6C=qoNX^+1gRf>^BZ}~oF*;((fUb{ zy-53|CBz~xBPWTmnS~}v*1<> z-t$)Dfbt_c7d1NMk<0QUG>?Ywb39Dj-K`$G`<|&zSgy?Mnk&ntw^)Wzswb>O;p(n? z5zIq)Qjl&z>=+X_yivFB;hCyNIH0ESi+xsT=o4|u8)&LZ^hJCb@{RISyvJLgYXte< zZ@4H4KVB{}uku)n!*@U>adsq(ad-XrONFw_+B=D3qRXCMgXf^)f~%>@=%(pfbu?s- zfU1eq=jC9F)|~1olU?vTT+K2nTr{ZNA7aSLqRosj_es4$RU_~@+k8g_9Twnj)hvVhm22{o8~LOJ8`Nx>*~F2T?tlYclqH4`7R^HxsiN+3v$EL@qAGe zsFhpP-x`fDh*1In=(lW@n@eBvz$XZU+qnt*}m) zpvlv19D2E(D3>mXC77SMp&jI8E@N1j@n zcn(ho^d5_Kh(EIFXCCyME+RKt8l-*wSn=E38Nz8KvGpi&ngz?4N}4F2*t6Z|iWXD03G9DZv7|u0A3NjQY@@FJ%$#b2 zoRWTkr=+fK@Q)}*Aa4nBgd*h-iX1FuE2|M?K{x9X;rBEU>`8fZO(~;6!!>v(8rDdT z3{%McVb1zg)2TTUd~Ot`5J@ICfU^L#~)Qq(&JQnIu2(-Y38lwUf_#p)y-UlX()Xhr@b(m7(krZ=B=&kRz6{O}g~CW+aJ?zWUe zqZ<}ROzwkCc<6I$=R%LjPkiVX`edjJZfmapJfLXvMR#397I?889QT#J$>vx;4Guj6 z5+^LQc`80VSx$GDe*m*>9?@{F!6uL~>Ji}qr;)tP#WUQMhv~+&y-U zWjtivT^4?~7g@WWG&I~y^Fb40|1&zx@tM~}rlWF)J-WM_k2MLOyKiKGe$jpBKN!|6sAgN|!Ay!To4Nh4HmJMvBG2KTml8p<-;*Vc#zdLfYEr z257R-vT;+15}M`uH0W0MSM&21?nmaw^By6{kZcD_Y;#SyrfT z1(AfXzr>j&N^2kHnL*eY5KM2AT$iea1LW=AmKov* z1LCi)2d(ZZ@W=-j1yB}0ptazZL{-L%w8HYPrbH_9Ksp%Z%9<#>b+vf`vfK-%38ur8 zlIcrus~|{oy3R9o1g5%=4-Af-bjwu?dF!(AvSRHsl{>Fn-35=GH%fj7CT?x7N+4ds zh`;cIjjos|cXrFPzz*G0Mv@dr2nZ73<6s-1jW~IuCj^o4${Wj@Nm9hmQ5%VdF`_Q3 zgFC@kFVQnzEXINn_RJQaBa{}8W6QivgYXVT+q2+ty@lWc0j@&(0h!cWj#A!KGmhZu z*9FHHvDYwLIF-=PP3*LW3SRdYkRQ@5XfBWVZji^8EEhf62d&Y{&JTdN!j|e3b`RQ1 z7ZXD2GA2D3EW&A{(u)ezkmMUn2w@w&b>) zLx*>&p9!AC2{-wxssSz~^Konr+!2VZM9b8-J0De@9at+y{CssIJ-!q1)NtjGF~3#1 zk-aej9cCsrzO16PP8uyTT&5#HrZUyyx29z9g9UrBI7)USee+7iVSajZJfAIV#<#NP zIhk*Sc@6_t?_twdQA%qJ53Y;)V&U`=?u-jMf(-K9^HL#!yL@&N77zU6QFS7$u_;<$XWRUq@M@i8l1vy@rn+F@M=h^qc}L>6TLj0m+$kUEKt@?zs^ zN1d1Zdjtmp0De%(MJ4{F%sA8chX6&jWu;)PM9NDtqAj01MCDP` zVmTrrFr<0_Mz}#}8)Q{1;alm?^{SM<Qp0Og&%fLoJ4=mD+96{T zW8uK81{tMZ)#m1w%e>ye4jPsFXC9H?MIJZ5d^K>?KeSGz0+qVK8$Ii8ZbEouX#&Gi z_{2w@qO8++3D-ov5rlg|NJv7RG2oV7Dv^X?S>oM@iT)OY&L-$p1dSjKW1DJ;3BByb zi|6%#-XyeqmDu-VO+Nw}UaCpx=h84<`MdbF5L?iZ3c|RQzUOeoLyzllfqXTvxivCc zwY4kWsHK=Vv)E;LY8Xr_XZS?0IHY9f0B50o(geE z)ANQ8fk`*e=vds-S#hwIXU7j!YC$D>K zg;ShPeB+GC$5L}dMPYl~LSVX=4f17e6F4V%lf6L<z4a0Xbk`kifmioF;t&SZ@dD~oh=OV#Div2PbuJ4rr($kerK zf&+D5B9;L0*cD=BqiN}g^^ZvqQvG>($=l7%P2l}}3SG(BiSNG8S*HD9N4HB@l%%p6gMRr2&xS3(R=$? zhMCHRDMnLi+3eErv(m$Jb_D(9F6URP>$UIkKj>gEDsoSyJq^WTC7ehr1{HpW>~Hj~ zh@iM>Ek?t_#%CNyhhLTzI-LOV?r1gs_5&ztm@T`BEIJ*5f$31vii`n#K(}^Bn2%(} zx2wpsZ->)83@K>8mKlvjlxWmrU|3#N6+V0->+OXp=kcE)l}iBR>#~AcR%^2eNFChU z9Gbc=$I)ba zxe9T3A@{c{Pq#kJR*~%(4UW{(?){)&T?@q?GPs=Prid69Lf7Ua-|g3o7z= z8l>BTMc_2{LP!D|caj3LAwq~5W(F9g3pKZ-Z}4hG#emRIC=8t5`fjVn{&z5rAX=q1 zZ28eVXh}@+w8Ys4mR^2yK3#jactsXH=VqNj%#)w7LLkn|h)|WYk-fP+s;MybT3RPc zWawtdrvx$DumHwr%Sbl)i7#I+oGAf#z5t@s&lAVxaP$~BQkoN$TG;7)ht6){yum7; zN)hOIQGw2_gw8SgcApf#-XqH)AxiPJe(?4~RNGuS5d1SJVgsscp~Hupc) zR?t>0b2jx5X9{uS#X43#4FmEd){&=Z@zUONTS`MCfKub8@f~i5MF#BtFeb&tnqq1oN0u6)B(K3{gS>JwC%sGWX$tM)v zBS^WKygqRpgJV!4kj3UXJ@S}ToA~o5pN9VQZimfj$m!v{EUcP#cQ{))+h9O<`{7Pj z%CQXe8ypq(U(PupB-89t^)`OJuqL6!389WqorMxGI2HcQ^%HD8ke3^+xZ10R7w@zp z6j$b4tx5;Wbo}vw5~E^ybRq2ElFWh17XviM3V77Y(AN!(_l+99j>RXRDl_Z=A&!R5 zL6dKTsi%A@B}^z5>6Wdca}4=azzK!#0s>=mh11B(c-ue>@TJLh>-w#0**W2Aly__b;@I4M77Zm!?zV$^VB|ghScZTR8w}k5#_8( z7W|wvX}pxqCnUI8qLNdZo;)k4f7nll4n6@lt3;lkNFS;|ZTEcL=2{zyO zRkoAr%C`YVT_*I@-9)ois>|3+N+s`J|La;Rso<;-!iDv1*593eAo#bT{qS`Kl|N`p zDzMie_Ce$F_ICLDLi=IzSy_hwH?VnXGtEZO91Wiu=YLG?Q?U1kz0Gn9MJQ;#%Ea|4 zLa?n`y&o#`)DU;0Rv<`>MKTWgbhE@;d$kX9R8cKjOpq{G-)w>-Hy?tX{g8dQ+gpyu-j12Q{qdmw%ulQFP0H-R6CrR5}ODh>+a3! zG;({&`3?Q1@BIH5RXcP>b&}1Npd&ti!O)10k1p6HQ_FG0d*joUflaBT7N11-k>!`= zx%xhUg7$I_ai(c;G5ub`Odk{mI7Ptbd$S9_Cy$yU*#orphsp-YmRih5TT42d-4~_} z{!ck|)*p0u(!*($S()1VH|RshBXSS0fjhw!tC#*=*MF*6F!+4U#d$}p9UKF? z+}LZkaY0CE!Xml#TlOgrbaJYc{T+n-i3d_nVz|ENAIAr+?v0N<%0(}67O|d$@4dL- zeN%Ms^jYe*ol~X2!cQqNw8&|eMUmQ&EQ%BQid&w)qn+Y4Ywc4GV!)}DTo!=z#7i8h zPDY6z?8IA7iFe2-_EAQjZoa_qQcS*Y`tY$1L!Wz4`y6v~gRwD7=XY5BoB$ z)%t#E7Ao+i_pWJxNNmy#Y$ArQ7k z35Lr7hxtDB)wyR)j!hR3OWq6NJM}C+tK5bXYrreenr)X`jm=kQ+z6FT$A2Oh^iMcm z%K=ryA|(RIWZ%9F_-uL(O^|LV>uJXCi8|PEw$*LUM^3p-kxn(2U$yYWG861=WFpy( z9)Gi3-aIP20De$nF5$m}w_2MI^p}Mt_>fnAc@K_qQ}i`})XZ~UXH5;cVR(InHcyqO z?1)b50@J;DDqy&x6-?=HC(}#Vr8AlR$x~tkwnk6@N}`2dq>7u_O(iHxYVL>7`wg=h zt(9|?mJs>(E;Bnhi`{_Rngq_$=KUqbIE*gSep&xA^c)^vpc|t_(WA? z9}JUT)Ag6}5B}Vp%@>u|3g8K^`L>|#AESpSQ645xAPHT|0q+o{CrfnDI6GVaa71I& zG6bwJ14lj2KH&{=J{?j#Ik z45;zfifLU1i{R=gi(tNJwHLiF*Fo|U4A?-+KZLdXjU!$^w&DFk<#`xC}o8sSzpQ@K2}Q{v3>GOC415;ZMust1-V+u-^VIJr6BwG+B za~r0ygl7IK=tmc*!ttR!{`UY!x^1j~7%RNN_OGcjOtgfKghNXrWkxuz>SmAMS-<7DbXYQj^S^qSPf_SeV8hZHh!?0kXse5 z{p@LQ=R)O~1sl+nIyP>+EhR5FZScNJe;qQT3|V7co0Z|_=Hh>LH!s5k(cG%Y*BF5L&+%;SDdFH}Xj}_~jn>5h0 z+xh{R#bVUaP7ghw^H*9d>Vc=^sn3AWUrN%rYNrMR?r;}{{PREXMY4xdEQ+og|Qqe{++|)rkKVkbV-=VzR{{aY9p3!z*j)x$RzAY#dS@B;R z`GAq=6|$!98@DJ&Hc0pjTfsK%DTm4uN)DqRaHmiuf9lrLUx%ku07Utfo3i<+mK5-Y zL_xbaLH|3a=l$nTU5)bFl6*=N%r z{XDp_E=AfG%ymp78-$r zSM%~zZ6qp>=Lhpd;YL#dlu%tVQpYlYvR1kccgpPsBKJbhu2LaqRfUk2`|MM#JxGOw zeBWx4@^}i!JsRrV)V+N=g*3FjxaTezJ8`fs-T|pr@y&1ZPESC$Ua+eU%l_ZPbgP_tQ z3nvz)%b+ZVuj$u@Q~z+*`)G;!;!6hMIG?ZiKMP>lN`a(4=salbhXn zaY3go?nfVI295)yOSc=({ia=3l(cW*jzUN=+xkIVh4doFu5i}JWcFSz-s9=E{jq%S z6l-gFR~Es_b_(#u<~6@ww{=@JhlAgvc6YT3l##FH_8(l0Dpc-kXd=vhKCTmuGV*4ky*m?2)Z* zgu@Tv0X0y8WWccG^Tu_XU{;GItu{O>=f&~ok(jaNUIwV19mLMiM&%;-v=^i5fy}Kt zJ&3&sX8-Rl45|Z|xhofiycJR#qo_-dmwYNYB@bd4fuYs_Pv(XQMyj z1V4U>PS6{?I>Uhao(AI!8eq-K!YF>6h+&kNR3`&bo_2njGqeL<rZZ9 zToM(1(#^REYz;Kjs&e3L<8R^kVWJ_YMzW*;AwUMbOLps;HK#q`YD+2;%&WvakN9;@`+C2UnVbMO6*+IL`>A)$`W$ zevCfeEjLS_nRYD8)n7pLKrvAT=^u+?1J4%!2VI!|?G|O6f6ZcrwRT&EIMHruZ>EER z7YSnLDG^ln6#M)E_R)ZD5ig2uV)IxfXsC;n$-lX~I`ZuRBKgAEV*8-Fi*rXb>(E6Z zTj^eBrF1@Zb;wpwC<76=Dc3rdZRCppjN~R9>4cCGeKpU8g;<%?G)?q0*e3|8Oh|P; zOS9{DtiWfv$F{UXWvh4sC`+{W z&-_fh*Kv?JRe=cBTzDiMF}a$63yd;I^q%08%T44&lN5U)R_~7H+iheZhWi;xKyH&p zued}_BeT2S^9)X4uW=MQT~%-S>MR@jtH!S>@B}Gq4?PF18+ZG%2w3ZUQ5iAdsypF4 zRYpXsn%)XUCL^atNF8p}rK-)r{VY%*RiZj#nArUwny48LkRWi8QeP)P$fEU%wvS*J z+krHTTD{qxJmp}HHvu6VvXtp*h}l?eIUCMnDHqNiC$oz$@+Znl z_u5$D31qB<)T_D$70U$OC=|+adEi2~=$Bw=!Tsy>PuISMoD1AYK(b$)XL7)U!xQ8!|6)xEbu? zyEl_J`xfd+j}SLf;Qd4(`v791DyUg#s9RcEF1bulz)YFelV8qC{7r+s6`H@}+Shu60xVDr+}zy_hfOCYCH#8_ z6p(6*m?*HBz=?O~k0boA=4P_!XGFmU>!_em(t#2?ZUqI`rHRoUtrNsnN>+d@=VVH7 z2dBklD*;$nZB@pfkQ$ff_g8AdM9kF)7DsgK=A*0!VRZn3>|;@pB^?>xMTWfk50d%E z+|<{pt|n{7?yG$j=SO6M zlY_JjC^+!1c`_k)aQeK%GX_?as#GI;J}knSze{77pV7c@I-E)_Mru^&;!5Lk`RH(r z0N3n&@cphM4$GFnhlIpHWSD-ox;Jko>ay@jfCHe^vn_4y7{Vn&v!3)jGm>_~GTJbW z&~V!wuwV0g3D${O`xfl7V;!=t)#dm2$Jg2~_hb|rw>>*s0sT0GCkzeTv0BkuU|_Z9 z94eCu5vx%JyL2*tEl&PikO~e1&IbXn^|TN{dvR9;EnDm`5Pjiuw+=jN<-&n;P`3u; z4J5eH3E}@X#lYY9eM)}ufu>ik$5Li`HAmib`-ievei~c~XbEs@nKjlHC5j=r_;zoa z+3AVE5ksZGq|X{3w9cc0RRwsf5l#c;Jg86(>Ust;jpHhw6`erC1sthVtE8J#2WAH{ zfl}**+(K2MQy~G|5h8Md{SanN?;raH;R$9`Zj`5MbG2IOCyQKu#KMEo1jv-mQeDC# z!DOv3zJl6LR|LP4;ZZOV$%h@X2JJigmm73iuCAcuAA^NL%eNBJro&<-*l0X^p)gWI zA=KZCX1d91YNUOpWA*2<*3molTQ9n(PK4{&xhF>iN|oquuTIX1_BD6mi|oXb#Xr_w zbj0}GqUQi){te2LaDWuI{*$47&E1!Ewvi~*XZMezpj-;rh)dtSZeUH{atrMuw8=1^ z%D@i99kmwOfjf&QrC|}EN-joY>h(wq8pATa3>AAn+tXm0@-at6TIm^yg%4c zA+d)l2nkRr3~zQNLo7u~OGrE5E!D0P;8iJCny7+Wb#OSvtOljOJVMa_;v&v-2_T&p zWvN5p8{T7i{l7&mASEYfeDyr6=oOE$AlKa6Y%LpJNFTNepdz#YkeB9E02v7?hx%HL zwYi$M#jp5j$3nrLO!gLoBICK~%J+wQoIL4s(>=0TIomUimf4hKX0>!TYunwrNihE} zt^i0OegUZbs8!07+ca*8Jg5#1xNs~`{kZDYEt}cmL2QG#cRC|pA5%!o=V%ZE#FB0# z(eV(tKk%uMf!q+7Nn7?-BW6<99dPANGYy}U_UZxU5-0?+j5I*m-^5=;sjeDeL*@@R z*LGCcq^k&R*5ToW(I_L%1^(|Y!7mDw579Y~!wyMm3TKzYb?z&_bCEYRnEU!4&B)Xt z9fH1YwM}Dov!cbH+7^Er!#f&=U@MaJUyM6_Q-^#zD?Zy16ZN%pO{=g-i=6wcLEO<0 zs}lAlR~0}H7d1A>Wz zW`>-7hw*eSh=!iHO_M=73~I7itr31MzW_E3ofmGm72~7`zDHPK>V6oWWmcTKb+yNZ z3GLRVcnn=jUvDvs8;$t}mSkY_H;ZtM$}rNEB#TCa6v!Aw(#4hW;vzz6(yt$R7faBr zx3}x+B912(F%4g>QuSj&`Z+F(w{Htsm!8iLu#a@6KlBh|!9RH@0FU#h(Kte=v3WIC zl4WY6hsLolV#Pj0&BG|-PMY8#M1(Z|X<~!21;+1Chd~T9J>7&r%mOnO`tGhB0O^dz zJTXs_+_5)SI+4O4gb_$a4=Pj<2a;}=Tfy+c@AZwM%cgg$)oM zrQDW_Q1=X3t2M;0A5X*hsG(Sf?*SERsF6g1F;Yky4{#vGXAGgo_krqzZC>)Zj`|1N zbQXT@SkZqkkW!}F3t{;%@S(meYnO(QG}Hn|OWzI*mIw%BwaYfr%~V(O4b+H*%#fGP zb|)lA2f5yBNRy%9r4p4{C0niQTfG^luebiqx|*?+Rw=x4fL9JJhmBl@z7B-mThe?+ z{V*d-Z-!ZieZ<9+yZ1xT0R;CUhaHfEz+bl^?MKbFA(aoFhHg&}dtr}`I0Cn~2P8?9 zI2zrq%)R`LblzagulCd05ZS8t6Tb+F6kwA8Z7G;(zQKV;IReKY4rAF|3!`r{w=$7v z33^ErjKYzj7st`iOg<|c^G0NV2iNF7P%xP zl34uStGPd;$LVWD@<_2OamqYSnhTSO{bU=Q3qCWil7e0y_GswV<|+|%ZBUS}hlmNtJB&SHVY2o8+%dB7LdKUyr- z^GOD4=7Y?2mHjt*i?&le9&~B|+;g^&+ENEuEyQUG@Pu7!*4M48sUuFywam6iBswOT zpaqmj0(#Eq999&r8AC_%sz`2xgOea-r|V%TmH?It3^n?Eq>LNGH_B01;$h!)!#K_< z0Y$1c{fLEe`09wlfyB=WAvC62(MdnW#n}1`1JtmkJG~}$?ZoTe_VZb8j0B+d(j~ff zq8jLgs)fdB>yp{UN$%$= zI#a>mM4u#XbSjI<+%lrD1>A9ra;@~c`gDQxFU^Sw8ix4|wkI2XoDcOxd=gN>U=HVnN0Za|2Ch9v^Y_U z!$N^ck9F#GnR;$>P4Cf+k1c0Ck^Bj>Unzgs&QZ$d(TmR*L zErd{z)RYms%GNKlsQWYnkm@z0m(xDg?PFI@1Ke{`Jf9cdV4oevT>PoTt{0>6_Ps5Z88Qh=uznm zEkLKZIPxd$uBLGT8rmDO1LTDZnqeBxWKG|oPB7$TWt z=Fvl)N90YQ>-HjXY+A4Od-8&M=l(5#=w~)OZ@pS0m|lpn5XIqjVxU)XGn-JX^n8@JQie4-o@BuH_8#6of{Y6GC_afr=BGiU80YIiW_~qP;Yc!8n(3O zAN=1{qJl!|AZPT@lo8XUMf;rj+19w|q>UZ+?c4427oL17_r>_L;Rf=W7W0dryuQEY z|3b6oE!zDR;Xmd(ajKy7oT>3U@q2^Zr>=nWJ2Kt2uN$wM_ErN(f(Q%^w9MVN4h|y? zCLBf!jRp$wAyX`~94*HZmd>YLL#W^}FNd*4iCaiV7L2g9LN}qMl}omM!~lJ409M#y zF^@p!kBHuPnAuZuAg|-=^^)2bdp);tT!n2&Ov!r9!!jt1P(tZ3fiOKg|E!+{;0A$FhU|VgEW* zjzP?<5gY3btEs{Q_$~XPl>+OA8n=fzqN;qL8|fRsuGCjX2@>dF*mr6Ty@ygIm#Q74 zgmMs=h3v&NB&12-?f4rWZ>oO2xL@p)VqkB0=9Uv=rV{Ob9bA|Y&{x*U3`dqBh(GIV zdCiq=S2v&szw&J-s2y-QJ8o23tYc@mb7nVfZG|L0sL*C)&SPIZC96vU!=bz^tvZtE z9T+EMt{oQ5(qd;Q>hX!aqK=6pM&{KA{0U?h6noJjgz5Q{7>0UZ_DvwhN}mx5zGUUi z+qb*XOyc|%@e6EUi{F+q0b6rV^Z7$<0;ut#5Odm>|czgIb@c7NxE9SngG z*4Wr;z0jV`e;tp9ndZysw9Z>KjoybKEm{~CqiZrc@O@SZ)$HFn^sLto$=ysaecaXF zr54+Lwdj-OZnTv05%a{Kqw-H1u^v{u=Y^U%Xk8#c4S+`opmo(3D6$aDfLDULGy3Mp zDL~KJ88!t1X)E~>6suel$=V$vof)q~Lo&DgsP;hT*i83}uVH5+6vd>PZo3XyCB08? zv3T=6^rh<-_3&6XXYwKOeU*9uFu(UT@VgZHZDJV~7;SUi3oMQE%BT;Rd_&J5;WHMA z)e%c(152=Y3QkT!$o(BI86HS0s;GxfVG#ua&~T1Ll9T!^ax7YJKMFNML;p#d5_J*e zLp=#eZKU$6Xy_>O;XKT_WAQlKT^p1?5N@HpGz)6St zR`Xd&-Eo+VQ1_#j7+8!OrP6Y!A6Xc#;>GwMkVx+!JfU8KQ&Zg>79#XLPWHP8!;4dH zP)Js|26{U0up}^XGa66l_CmSSPq+7M*{hb-eX8jU1>~_pFFOf~XrG@gU8$_kxNJ9L z?sh;xU{yh44}(O0gtBP5j@9~|+g)KtU{DvgjR={4l>~U9w@L|7f9Tzl$?dpkwMJBs zG7(E#uE^?eP(%x1j%d|k6fcQw3a@so^r+)DwH1QNG`Kr7ozuyeE1<;@HeTIdP1N>) zdOsQ!ozKVZ?Q6RRG4Y)FdE-XOMP?`Xg`WjbquWDeomu$=mU}uMx$+$c41Yl28^%fx;5i&)wwhTv*EgoJk9ioTrW48 zxhiTS(bo!l;lKJ2pfI5Y{IQ1~_!^vy@X+!J6V3Lpy4KF{wSxhWEiR%`&uYLt z=`iAz*p>*LUaPznh!-$tH$Gl9F*Hgm*yJgp8?>RhGvFSj3KMmZBELt>aF|6%Nep8= zLSnlw6F!)bf>f%|LUi39 z!PY9TAO1`*He+-=pO=emJzHedXoU3`jAnrv$lfMAwv#-$rU4l&#lS{n8vf}QhxLAx zn9Qymd^W!bYXlm+Q6so4jIo9n<(Q$Dr83Rp;<95wY9mUycC!|AL?B{wU?5POyk^~u zpXrw$F#I;3el95(zjxx{YRwgTisR5;a?*solp36LbSH$%k^hC!b^D5#c3#)Y)cCZQ zh)$js;Hx5|YW+!dq}mL;qW8B-H!m{J*7G=tbm5#wV- zvp0rAfgk=@p9mBjnd$M|$ke1Oc=Yvo3z)23qEBq@+|RaLGx~b*!zIDlE_?tX-6rRK zHgm@Fsd02sH#amv5+Y@V{-;(m9A+gdLg9*&WO%i55^&wXZM%=w5S>xXca&+ePfH#W zv-MFbYOI%EW{6|r2`G9^tk3k!8uk7D?u)=0GN)I~0OSA5DRIP@FJb_;*n#NDT_jeN zGK|tF9Ab1XpX`r;=!-6qTY@#XmKoLbW;%ZI76d<)p!@?rh<*i#K&Wjgp8-%5NOc|M zv|p_Gse3p}(AK~_{z(NrnCIONvn-h$ekE>2p{*7pBJ4+J zCvkW2K`vf6Du#$U@LV+14F_)DrN3cYKdork{| zm%t)UTK6-)C??ip?)S!EEpGm9r28LN`(qp<5-nFd&`=Z2knYPKrjy^yB1i5LP&Hze zdAb$1heP`OukGCT)1e=VT`&iL*xTF=O6vGyuZa=!qgwLGLn#c)%$jM}5>&NRe+63- z_~Ki^_5SCZt6Ckh>9(;gz9v?y5w|hdO+S_T+iXJp_n+B`7(+x1DTGgEVOc|${IIGk zV}d8`#Q->uX3R(L;)P7!H5i(G4rP21ub+KNAAu4|vC@49=DGvvA3L?GNzvAHrvp9< z&hQ98%^sH$`EQLYoXTym6T`l(EVJLQ5&n@?Q7f*zW!fovIx_`DlF(Quk;>r5jMvMw zNe;3nK}wko(lT&+_#7pohPlOs)I;DX9s0cfza?(liBxXYIbS|iA{o0LOgp{{R1((o z?a6W9n2hxvdE6}P0lvRhC3ly$9@SXgq1^{p#ZC+a@yi$?s~utbZZj3%OBI3q^66E% zAyLLB2)Q~DlMk?3Gt|CXMF$-+7f)&QV3UYt!E^{N*hX$suwVw8FuOkaZ*``88`kUS z-1NMPyWFi}_#%zqU>bD7_8-$uFxHx7v@=WgOKKXz))dw$qbNX?KzmK@1Bm8) zSOtzWM&N^!|HvcYB9dg*ZHBkY6)piz+|^_uz@#!l-MSUZTV+1G$?`e*oZ%~Hov_l= z0j8uiBAbAJ%n)`|8+ptnrlEhp2@ICiHOQh`>}QmhmBKMD|15K3w9JLg9DhvrvZ$#` zIlKvV7;qyZmGzoG;rcdj4^)KRED&x2A?+7kkIDvaaFlUsoA9yo7oP3o2Yzw@6`B#S zmPCljgZQ~@&mbm%W9GL{j{!#B0JuZnvas*gyj-4W<6uz>;m9ZBjQe4_BPQN|>0)wv zV7!|~ikl|_&E6;X}1^miFuyEPI{PGsoa~huPs_0qya}O$U&Q?F* zhkMshkT6i_Vzx)hu}JtCf|gY#>(FGU`~&+s3Z~6hEtRaaG<``e5!%w>Z`d;NrdgWk zt@}Mq%|zCz9Qj%QFtfu?pYGRAlFZV%b|Av;NM6T%^Y$rM)5y2UExbv7sPPltGL&?G zQurYg4y+Z1LT@Bgo)b_^IhV|GbFk$Ur$FD&78Aow8C}I#$D&unIe6cEx+CUSKeMsy zy&T%`q5NJN4i(hU(vL&fs`i$`sHYJ~21XGHf{^T?*M+~D(9FRE6k&F&qs`ZjxKqZI zhE=*#5#och9YWEH-{mC+0?XyhXVZTI*jN>0EiXHdT;xJ($e18A0?I8Ql{dEhQV#jM zH~o*pih}}RUlJ-4pw-%I5*O%fhM}gwmq!1_kKnvsUNGDtQ#nOsA1r zrr}r!bl>u15|fR&W6ZSY+dxX**BICwzm}8+2557&qzHc%6eQqJNhZZS>TOBVAp2u! zVKvc=28&6U(Cbo>`|b=`dr?>&!;f!zYwGYd`t(95J^(NFiTDjHaxHnBzA@9Oz}piP zJaoQSce+_;Yi|cXPi!ea(sqKNe?#w{?nZCnvEr*E8M(E45I_gcAEmBv@`f%n0(^?Y z7Wd>?$R!7+BGA4>Xm=yFx?!;gU>$u3qIF1kkxTDAQ~S2546W{RBq`wZ65-ln+hgl! z`ibvM1P3>KoJ9o2FPl#Ua9cD`uKuNIYB!2I1OzI3rZMFWRM}*$Aia%jk2N^uuLbMd z)7CaX7~8p z-^#$y=#A|ql3O2^nZfb42v##w)qtl*#a;ODj}@>|REg&H*-T$C^bh(0+;H~k*GtY6 z0sN}C5pyZroQ_Vx3Sxqd5%8oMRzVag@C8dUXOSdIv4c_DBM1mCam)^zU*({tUoG{x zvZ>}x3e6F-g{bNYnwZ_6rzln@okq)7wIY$Q!gtmUENE@-ThjZruo;;G-?r63Z z*1taOZ`Rq6Ck`bacvOn*PWv05cMQOB@T04w zE87yRBGE0lHxQ`f^z8rH2gPCxrWCgZ-NN~3j2(ZduZS$??srHV;K?`bBPHq`8b0l- z&Dx>_liS+z*_b&chw;yAFU)FM!8wc*Vz5}UFhK4iSZ*1i(LV^F_P-aQ^%yN2d=dXp zL0Dwuv+mJ=?W}uY)rh>wi1^@=;eEnxigIcz4Y|m+X+aFe@qbj;6G4;ZMg}#-=t#3X zV2!O&yaE##+&`r7u~Kr!l<@I<$oNhgpvPY$pKw3PYzId!v&g!m!M*M4 zVCpV12kVy;D9C1RFf4w8DgdTD-Ne>k$LhBXRDm{U8bE=k{uQPLVBkAO-uLOq9Po&& zyw*fGLtQH;6cc*ez*EmA#J~8a(#4Ny;1J_16^Gd-7|#bYpAji=Vo;`!HbU8;BOJC^ zd4*KxPAn)E>ENcen2@25hj4c|dU`K9R$yGQ{vYZdj*PXH9qUJ+m)Q0kUzb0#jqJv! z(TUxmzotq5GWQ~NpC6~WUC3d1j(G(UZpvCG-6G#MxO~?oS-6Yp>7PN|7O3bW3V;8c z{uei6*8vR#02gfelV7~A>#0OzG>j{!`uDJm7=a|L)Qp}VSJpOo-_+;IxT6}k5@56m zYtMiVGev}Fj~<>*)`w+b%|}PXT4Dx9?y7o1Z{@0*4H68^ga1e6;d_ z=k={VrD%2bVhQFMFh2MnF9rgTV4;8j*rWT(4QZ!(BU+}nqH)X_nAhdqqUCC0bF=`i z5gx-5m8w<$cYjQss)gbdKyeKUad7z7Np<6^jwt1+L3wQ7{_y&oyTO2ZsWk1p#O0ia}O5jnc@H7j(vMzK)Y_5UVP4cj!_RRja6w zJI4%VE3u_=wInJ>Ora$=qeCi-klbgE#xNsdVwm|opYPG-l^< z9?!>9;n#?=x~7TmHiB7A+dZIV%k}rIYqE<0kg~!IG-4wsX&Q5iPy(b4S~7gYYg$fP zJ~nOpqi`%JN2l6+8M%jx9@(fd{D!B9wMAw+GM*0A^DJzdHWJ59r5gM;owox-GEY7Z z$Q<4V^>54PSVPmRL7jK=*WMqoE)!BPc4$#u<^PT>lUlM1}L((#Ss<_|K;Qh&CGEb znW80tY9Gu0fi$^K@lJJVTg;YjxE3oG{TH8F*t>M@oqHT}r2b!7IH2+WV9l#f2QUB1 z7~Ewu@=n+fE{W8fN{F?3wk!e(3w(6ZytApc%5yXpJ0q@QA7>A_e`TA_&X$vhj8>{B-z~v)M=KIS36d`Bg2;h#6heiv-VwGDwEzNrQjy!4|n#K z=5zV&MgWwgC9~>3$~CGAYIcGj5xzH1N0^pDMY{6uRaOye2(mS4(?wLqE&$6}Z+h zie+D_!uMMmpqjdjdxx5E^%jrXaJh#N6SufAdc8YL7HU)wQK5-u?LGzp+ORo`_-`th zS))VILZ^?Yk)#V;l>aSIGgWr(+*KrD<6G_u<3rMW*cQwYL~y+oxs1s z0?lrDREtrR+{QU2Jg$+0cXe3Ciug-kE=2DsFmsvfy_u5>LOu-X(q`P>Zv?8W&`^}4 zg@y_gejTy*I7{FoNnTx&rFKbyf7?Cu>ylAO1RV+W)wG^AW%{f0EB|qKuj{J|q8$s@Kcm{*kO{IrRQh-j z4C6q>WA0@=AN53%l6Wk60y2)6cxWi%tsE=RyHy)o80?JMTN z)6`oYjb*wSa?&=*^!zce@CU!Up7xwhn_Th)CWj&i4;NVUsLemXub9X%*9fqjXWEs@Vq?N17;3+vq9n zJ!!}Lgv&_$^1B_IMLOfEyBen15ik!)?qedbz=v6;iJ4!jPJI|2U|!6a>w`QZby>(2 zmV$Gq&aP;9;>ToXMM%vOfFpqJ8iG(^PKlCDIjBtF{+sohq=j&lmcc}s?dk(X@8io% zlEcfyQm3SleK^d2am)NpiE*YvbG=c+&~&H!}q{Wu7Z_4l!X2#6qEXz=i# zPDywu9k*Fz_5me*O^Lzd*zkDsprASGylA?*y0oz8!`C#ADEE`O%j&dfZK0PV_6gVV zO=Yp?ymhHgHQkmPlr-9jQt4j^xOy z6;DUUel)9ZB2OP(`&(N!rrd;29k%gI`$dJlY^cNsQ9c6h{HqL)KglB7EDtO^ng27f z+O2Ws>eK{1X7G}7wbBDQK|xhw_XS$~#~%+!!ec8B$rxIBrP#v+kYJI2Van+;#w*84|`GUwA z4aRb6O9T0Ka&mhf^2aDp+CEs|yd-5_o7m5guk4-LqSA})v!Df*H@>1MXbgfK!{(F0 z)l@@|Kd8J8E)qn}IiMqr^;tz&iNb?*myFOnkfzb8eAL*^jh#5Tk879cFSceRJ|!z6 zw~fcEWOU`!zCO9{@>|A~4H@}pNdnv?PSFn*9U}X%0o33XoJ@lPpi27&__^EZDThb$ zN9bS|nL@}?v&7Kjpcvc4;BgFJ^sUyak{r$)C{2 zKFBDp1r2Zx?UDYnqz4(i@xHuzM)X%tVnclMif{996W#Z*f?#@@V{u~yQ_h3c+gCUK zlTGDTnBN@$hy$Det3SZec=Jz8gF(ATJYsGL}JkXkj^T zM^XQynhCwwhkKjSLu0M2(q8n+x$KONHoZO=U)L5-XMavR+|G##yEW*zcC770Ur->s zt1a>=O|&xp0F5jb$#g_^;4ql40Ba_z$_M1IW?xrILpkil3$f4tFck&|rFy$--}wP^ zGQ30s@9-@RymAOR&VR@q49?iEr;%iACPQ$qIoJ88k^Ch3<8py^BMoRN}M5AxV>X<%Zc;enOPth@HmZ+&Bsg- z97aj1!X-eLTpVkYvww7t_Tu-4>xLMNh9Ev&E`xlxoe@Zpk#vV{NX0{)8qVE^6b(i@ zWx%NpcfNsvIVF1SDtQYhAq{bnE`BWJnxE-`i69s}3YBxg_1wjYANy&FBfq)&teW@c zSLwDi?^i`q_l{R;t^@q+K-Nj=PxDCHk5$gQ6TVu7*HuQxh^B1Zt|WWHN&hwBU&){B zdKb}ToEm8*P|M;V485sX0`3m>O$iVs9D5`kSLjnTWO1=*=Z3Tc%^~WAax%&5?5ZbM z%n!9WfuYF(>G;2Er8YEm5avxo(i=7kbO93U#oNnRC!utRZ1#HK&d z=(t2#{DaB-$$1x!GcIxPkSLaIvhW)k%77Ft7+|*B4Mi`|s=JKqZWA^GOwI=JV~@jO zA4WXl2Oq>6>dqI)s>CAGuMit@oxo6Ba2FE$Jz4PI5I$V?4Spg>415gsY9QPIxJ)6{ zWu}2>h24F4^D>1)ozehg+|+t`)l1VZv6TB}i+|AjJlMIx;4pPD2)22G)_kz(0YfJE z!)5j5(?yFi$w`ZD?o~XLSvB2Li4@qY#tNtjeRgY}M!#FI@_);i81@!^3x^}P6XOB; z!EF^9T?>v~QXXL$-y#6-le`3`3zC?0qQNj+tn2LooXn20W$?Qpvxu>{jab!A9;U5a z4@61QTOsh#>K_|CAhSl=@XK!2BJEs`+Iv(@X)hLCcCifr z@n~DOVK2?o2o8brd&`i5smO(u>E~UE>Zs)kWZTyY6wg58)BC#NT1 z$rn|#V{10D-Z5=S!5Qgo{>NLd5@+g=8_+=1*|RJuQ=CM-(JuP(!Rwy{ucMf76d@#I zoK63r&XAO61P91Z+Z6XwRc-IV0^F-YO>cX1HskSAg(1s3C1-)YS)ae9?YlouScFd} zq-(^(wCu2TgZn+u^y~!{BpkXRI@wTtYTWYS=sZ$rf#U5$5#C@6(e04T&PqR@$$HnY zSu(x>axLnd54J{8ZF1XR$N#$gg@PtW^;)+J}kZd=Yt+62Phy34y8iy{( z5Wp1eUxFG^du3+38R}6MaOB~ZpoV{}<{=6Or=u#76 zRzab7*21-E=63?;Qi=G-O>UhC=9%g_H>pei)JWLK57%7Y9b5^k?UW?>vRCm5&!Y7R zPa$TU85_oc1}J|@T#dADjKJk(EEkkcZMRw1v?rOxCXQVMy*I=Jx4~~s{L=QTWss&^ zq0Hs*sNyi;{e+e&b>KA$ciwyj6710Cgx(&aEz3h^9YMw>-NUwMlH$1!kQqu|m#wNZ z7B#_svpHY=o<|w4Q`Z!J@yLlz+3Ht`sF>&JZ-o3~!R5Eywc@%Z#)@C(L+~0J)2y;T z^XO8)hZP1A?q53rUrgLo%7(t4Hwpb_$tP0GzgfjT=YraS(n_V%%|10BISk{Lp@heQ zU7@s{I|)0o{_der8i!NZ*~EU=Ee9)jzu_xy99}6o8IQ{{$yk6ai$%T_xIzn5_&M-R zaGQqxD*O(G<(<#C74Vwc{V9Z` z&>S6Y{zcJFb_k@(G8=a&Qe?60D#-GRc$Vq8?6z0e1=zuYe!?Lfeth0HepmZz_riXU z(NAD?p-|&4%`lQg8kSpFe!weN32ZWXFO!w9OIcg6bk^7O3L7=xCmL5JM2Ji4e%*X*V! z%zq#9QQG$8;m7{F>+w#8BN4_s9pBrh>;X4Ij9ziaj1va55b4I6t2V`5gtQ8t1K@x- zIH{fcILwHuGA6(9KtD(n3IF2vSBuI;k%mWyY=Y23slCx{lqe z7l}mQwMA&x9KFoCdtAb9uP~`P+;ZLM*4X#gW1F#F`nW7{uK8H}vFb^YTl%Eknez}` zN&5|Un!9WLcR~C>-si+u-9&`-+{W1>t!vy!!N-Rti+~6;e}3XA*vj^703s5>{osyB zu^U60Z+Mhq+cjc0fug}YhfReEO$xC#h5HsmyLY{=8-YNLDM-OkfpcVk%ALu?Arw`D zx|Xon$9rz}1=6gHYReidn>xu-8Kx0#wr6r!vx00fNvwXV0C5+qJoKKDbJbHOl1Mu^ zzJdb*;Gmhm3CLD3hdDQ{EhbV}d&6CB! z6-=s|KV_s{j#Ta=aMntEL4|S)DkxYpd)&(2BwV7P==o|_bcARZp&BKb9|mt4rcM|K z$@o`|fyf31a}+?>>f=pXO_KH%B!y}HZ|Yyu%V27Y_bb7>41G-lM9b_^pdEv)ENxDo z<#uSl9$^1Pi4MCCK{g*bWWoyVO@AiaY{+s{6!cHt8o@_16-j>8gL?OnJ^ViK%g3&@ z+6YJgmFgKUnOsyNIy5v34DDlnhegp=mee9}3;*zJsq$0MA!pk#CLB6ke@Sh+TwKJ% zV_=0u;3NwQ~P$6COu#HZM3{&7*^qE1lUbe!_)~bn@^3&{&n7or_HKj8L|K!!@NT=mz@LQSb)ISVU8vW7 zC@W|`?0oFTfzbu6vS{zT?31b$=E;wk0dAp5(G7Z7Ps@8+czJPF8J&fmpF&N>w)19ovaa8O*~hwxgP(*zV3< z<9X3(rvUscE=u&bgxCqO1{7ZP)0^IskWUpxbiqzB)glG@Gmu|5bmb*J*V z&c3i93Y$O5Z4Yu7P6ef8-cNSGfoDQ<614TO+pXS=m~~|(vp=5chc`M9=UW;L^Jg;b z1rj}&U}zDpcmbkolt>E*+cA>ZK%UWng#YbTuAgJ^Dbw{T8(0fTi z81Oo|U*qFOzUO%OGwmMySLQ}>XA)Vpz&PC-sPU2oT<>T19d0iIo#1E9cGoBNu-5e5 z5^-M%&==q-$J8~!c)Kb=k`#PZ1$C=(54!kyM__m?F(n96V*Usat*2odeNo`+vE3k9 ztA_n=@FDHZJ6lon=U={b%|?WVLa)mDg;7mRH8-s3*6go%k?wSXTpF~k&QRCBt)5fo z^~I0qV)9=EQU*z}&k&<+{1b>ys+_%O$zD-ERZIBWOiTX}m|-se06ZG(^#qMhoLjbOCQz=S2);NO z8wXE+lXAAfs{`K&(A4^m!*s6IfY5+fEnXCq)u?ZjoJh>)@Y<1wvggYicbYkmURf9e zvYs<&0`S29@}BP$=6%Kb*OQcv{9yQ_T_5@<%!mXF5P7KywMROb7 z1>wQ`Tm7D`{QsX7&0tEGv+SuB=PU{bc9~#KGrX?iZNSjUg*%;}FMxs9I|Q83ZWMK) zNkeldH?l~IKabRBnLp8Eo-5p!dvwzcv~ifDW`mi@LditZu-dFOAMBY$`sO$7XL=K5 z3zu9#$c&3U@2`NK{r%kM3~vXUl%|QAZ9mRL6p|B3!~bJ3sZJOBq7 zQ#}YcK7Pvt{nE!eA5KpMMO>H$JFXxuJ(H&a0oEO*l25ffmmzKSLI)p5GYr+@`~!sV zo~5}zlCtTj((j}^4l|EBj-v{5AZy~TMTJm4lr0i(u5@G{dlHBF@?0-V_elicdSTMF z9dalJ==TAnA9Yo;9)qc8^GyekXFpC8$M6^jcBNLhyl8EI4fG}HI+vSt6xIsk5wMEH zC?n*f?H#asvl3&OXWV#qw8XiB6Fdwh2H0A!KLNVWxBar~fM&SC8=)H_3C){6`wEiO zS(u!Bl9iXa!|228K4)_C%lG1%g**M8eo~ArpL|rX=&NjjsP-3)GNIw8<=BcX{K3g9iMLP8O^5A~^cYjaa z{(0Wx!hxC2ro<;=b(vc%2iIk$<@0r=mEeqJ$4*uY@SMfYWq1m|r^16hWMN@Jm|U4# zG^!PDFR4DSKC`wrPk)&!P&*ZxJUt^N(hsCJ>&5%dhUG1KbLEtv#danJjs!xI%h>_1 zOfvuSU)&AiZ}fC^b#I&xnm_p##}~djqi_0($Ko79EJ`2SNF^nS4gFFxnaS(-aL(UE zDqo)bJi>@VSH~UMDT~)tPDy@Y*n9;N$H1=>LtjRj0OI(dbMSGycA51N5RYlfWY2?{ ziJg3Yr0pP%ZM2)Mu5OPMo#8+LGU?R5+UDiV@@h4K;-8cY6BTrBg>9)1hLda3$cFDh zIO4$k+5SveLNWc22xikhTD0e93Y{Va=bb&UHuA;q&FeuRlBV=$*aIi?_1XY%n%yb9 z&5D3dZK?qQKmYN3>iq`BpvG()xg)jZl1iFCiQFW$_^) zh&34*Ai(;+cM5JLVUfS6mCqw}ZYgD-Ij}9fSE_lvw+;ATj#`KHHdJ96f8mi=hcB>3 ztDn+^-x2T8;1xRQ{&8u&)8Dy?+h?`Y64{jgsnu|L?F#`E$6JR~?w`WB25<3Br>v?Y ze*;_v>0ASn>pVJSfp)kHbg6+kiQdEroc)*IqHt@ds&V!SlQFGlGR?mS5Oj&Z_c*m% zqLA(no8AaSQky-e;AP9dplZVRIYE5e92>S}(Fgb`*?YFC`t#R8yZ)LnRYxmoyw?2U zGiJh-@SJXO&&~&=B;KF@OhF*v+i%mPHT6pilj4pFOOout`vZvwMQ7?S;*9Btf{u+M z!}oGoF=4mC+(P}uMVneS$jn1iWqL{#Y%0-_4;91jH#n1iZ%Cf#&B(|gxK9KOPP~RR ztN`W_UddCc`a2Lj%z~KDv1_k*5TNhjp`pC3er@mzwn2HCdnbm9Rg{TF5`TU*o~{hBCTuY?FSNsY|*IhdnJF_;>6jXBXi$n%W${Tq;u? z&gBuplj+J;rz-t(PFIU~{UDIfhuYyIbfzEsRDG%X>z6XKH)6#bpX8*R@Dv^lGRB=J zdlJAExz=%Ak2wg~DKa-JJDqc@{y2$A$hps(-TVe7mhXKq+q--@VPF|7#bk=7U@5-DETLmBytAY!sg3#Onm;)Q zi$J1!;Id&(%XpT@X}|ctH-XSBo2hEi9xDclif^7qeu?H_E5?z0<4j%{38GD+_u9Tg zN}ivmV#=?UbF}xYr&ePR5YoiNr*oL?~PT%Pd$noWMBTwzrbYUiRe3%E^WG znpGHQ2HOOW2HFlG;%ss51XNK(z+4J*;c!B)y}-MtpL>z~rkgTrTA z3U`g0j6yAHQqH~ia+6ulotkzbvZp`tI@Ej?FM=QfBkK-JX_Xc;Y_J}a!{Hx365*3$hTQkvRyR_hcfxMFnVjA$xgRggHBy}Rq@oHpEAn)!2^0)I;s_r3w zUE|tXBfX>IFgyaM=e95VsL+vbG9L<&qY+^5_1WwYNVHb=P~?c0xK?_DGK?6UQ1M;f ziVZc3CCJ?45_Fu%X(Q+?_h3^M7^Zpe zE7fqegH1wuuMJx*H6ue2Sve0_Bk`eaLj5+|R_b!s!Zd4Li=(e}T#B02J4S*0_l>7^ zJ#!_}wD~tdo0)Jd=z>`mr-E>*IoAe~gFU%}q^?x?OP75QG99V9aDzOf+Df?nA2FG! zyT`3eUr_5U3PBR|cRLL{yVO#-8UuJXk=~y|9b!Y{Y(1{F`%j32;PGU>E(*eeJEFs` z!0VeCf?h3Im~pTPPITdSxa*ocJG!&r8+7+*a@synyxb0O8F8w5sG%^u+rHe{;dSsL z2qZ9}7G_%f`@N}8@c%!lpEI#U~_et$@g zB9T;{gFMZ^fY8#l+mgq8*osGtMMc#pT_PR_xLe>j)H|_b=^!i1Q!ia!Lx&HjqiU|> zd6QnvA4vAw>$@vc4{*-j4Z&@bbRCD0PRLqvqqB(Zn7*|Tu(}h(k$?qJanZYk6O?*x zICe>X@}+TI$zYfc`MF=N3u{+gxcy>`lPb`x8M71w$aq%0Nzce|lXG1d!p%%Y5gWMNa@7eCx9oF*~SO*1pG$jChLk}Ey{rfNM~d<*7<5@1KOG-@>@(Zp}H zfQUEyCHXnm+3pzJr|gPaUw`cx}0$nUL3Ob3V?_#=;_+s0_~Qg+;E|BK~6QPe4j^ z!IFnydO;9O->bAO&!s~kYnus#I%tE-x3giX5}W?;IYyxSCD#G#A@Pm~5*08FY@fZu z-OsV8fBxJ&xgDBlC21EO3%BpO%_~c#$1>Q?(*g0k)oERoFOvsrtKhSM=>sqwOhc8d zD`N&WLHLTV`+70L`CjnO2)J8P?oJDfMvOq0@`hk3z5EmkGHSoCeh_A=>n3~q(9g*> zh_2~2HdNd=6Mc3ZD4ynDh=+eT#_=z$D$V3suW2~>N%jElxvGneHxOT0%BL3 zUVi_;@oDJvBE>;_jSE5dJsLi4KANS*zn8qLF9bftvP%0pAAgy}0vq)gVxu-v-ZzCUu*v}@QC<7^6Ygpr4kjIkpV|3(rZumc z%UC}#m9c+5xcb20wz#syEx7o?v$L=Q_qo;EcV)lmQt{9q;MlOR0FwyZoZuyVF|}=S z;s#R*Hf^#v#pwPcF`qgCIgsyFY@&=vKvH^r(hR{~POcnnvv8B?pCoDxh&=ZU+GYAv zgXplT3b&8<9H}E8O9Z(5AskvGYT>!@c{dup7*ktx_(YQmq$Q)KF2(ej0RSaP%;=AX z@nx&yV7~8O48r!Y#F2^bG#=KC#!|yW0A_rWg?mbJfir%1fc(dgH!v%&UTDq&BJ@I%eM@BYXn($|=DHW zXzao*94OOCq@S(wexvGXr0sx%fWAL*nc9>{aj{&42wMWhAS=U?h*gjV?g1 zEs>)qjU4$>k6BQ=%x@`jR-6?+RmEW`ozG;IrQf9L4XQ14?0RPVl*U||KKiYiT^WD5 z&W-2K;`%%$PjmmemtTJoF3lS^SWt)FHZ` zqftM*39cIC*uc}$sd@e-(;!uDeE{2AB7)2;jNoYieL0EfnXL(#Bh6n*n>`*uj}R#8 zJ=5mHnuGoTM!R1_?dfESx3gYHyeE_&u$#3C%5G^baEldLg4^y!ZK6za{Z13e4jCMN zhn``(j66GRe4PJHLK4FIXb&;bxg%sqZF5I9nv%bnErn&6mfMr95HfRQTU}MGGGN$3 zB~{O#KLqZ!q?*A9`W9pe&dz)siw$&TG;b+V(8Aie#VMTuNxn(hv zL8*J;CO4+rUx0Mo!%-fyHglJ{yS2*o}O~ZB@8rwB25)!7} z4G;%vo;yA3ochM0P1Jx=*UK8Xpf?RYrI>oF?CpUu z*q>RIM$t91X;Td~@MfSPapl0`a<{Y77Y{(XX{mUi^BD|SjC#EFuZjJasod$Qc_h_- zq8hn!74`xo^TEC?&nB*^NgLk(4RX*!^%n~u6S!8OBM%|nzcA;PiK6N~2=d;#iiZYV z>Whx~^A5wH8SuYlg3{dnkuw+mYTSk=S*nyZC;a*95b1pNkXs2h!qSroPQF%I#BpTk z@LPFm@+BxIF!iW)UHAwmk*A^-y1a5f_p>oE6xr&km$97$I_;fH?Co1MF_5$y3O9&? zU10b%haI@&D9t~5{OUZCOGEC(w_+*wu^`1sSS{Q-q6;Pl5Kt_MI-(b5^=S9znHDnU zPhRZ!GBAEf^Zaq_Oozg$FD8@Tb}UI73K1-Qa;^OJG!l%hx4jO8vru@{oe-DfFH9sZ~Fpym%oT=Cu3#ow>YiOcOoz7jrj1+m=m!|~hx~^qwD7ncyL91#v zh*@p^vw<;Jh9~>Yg+c1>hfhbuJeHEQT-t*hsLENubcUnEx^Kf# z21q=u{?Nd8LOO@Gjdpt0I<`7ce5RtgVMyJXu3_RtX_keu$@h`Ip9{7w56ShSo+l!x zc>3G4945Kn)SZoY5t5O=7bABB7KLEA4)hF3zYQ>~ZH)3B_;!@R>ccDk*KwNH^6WXq zNJ-`g5M!o3*`D_N5>#4|S_?KQPq}_x zuX)t)6eM!2tj)r{?AtZ>=aI6lP#slwZx(&WSJstKQt|GF^B}@D!JgV_{T;&Eq6KuV zeGt?StmkmDKcE0DKj!ammNcVAqu7zVh{hJ#4I>W^a);u!z(fZCwve|akzCT${VE}& z>aT(OGX?+)CZ=J2`|mgA`V(6$#14|WgX}CE4-QyKD3I9DeSIOZC^G`H{qMMI;2i+# zL)$_w4h{~g`Su^*__FLk#hdh_I?_+_Q*J{3mZwk};d=I>N5dqPP%0Mwc2xxkEGyJm~Amn(<*7F0Uz;#qbGuDJCde%0TkoqJoIbatF?oloq!ZrV=Uo@G%bNn zA>~&YkmQF>4$}^fKbjDR9a2636biO($w(e1mYNM>aoEvJP0u(3J-2PJfZ$k zmj)-HZM&{ASRhb7fSC9#jjYCD3L?i#z)$uY1k|`B_x`S<17DO4serG+G1C9-GXaWK zNR!ts{WEkapmbnCcP@6t2LMaJ5dnY!iCZEI6fa%e0P4Vww}~I-PD;o;o2uPpGV=Pn z*9=GI@}k3bbj}o7-h|vSz9HsBv9F+zfxnZO?A?(!9B^pxG)V5C7QmDyNdv0cndz3H zF9-*ySU}pg@=rO8ak6@Vg#`ajXbEZ(P6P=6VncKlrLkeoBF-H5is}f8AsJ-!(Hzp9;dmmsci3j3w+^s9MA!jk_cs^H#!YaQ2 zov@_+ipE>O25NMsWCwwUyIUt=oz=922#~YT^$2OQyqZm1tPY@hf+%7wZ(Q?Ofb%Tx zW?*;F@x-h>%WNbprT3A$@T7k=QT3|c4s z$P~{NE*kV(lXcEP@O!}#r^EMcV3a4xZ6UEf6^*y;{U^&!EK*uw_b(O}6GpFGE{(F7x8HYVsoVGc{w-Z|}^j9-<4+%oGu>hRgfOnCt`#)fDFoyvgddMJ+>kvV$ zNjSwP3dFB@`hK90!ufX)i#U-3s|0??MAU!kpLma0k}A=?eTS&Uyq`&~+BT~c=(<5i zQ(VvO%j@$|Lz{%aLO}QPJbj6evb;ZYqK`e)PZbxOR6IcI?Q!Mdv zOuz+34$rq{7si?%gt6LX6w*SjxUA@wh#wuOC6cr~?YxyKFFX)+Na1i?N>>|mS4+2{ z^a}QuL3`$kd?Todb14~kx92|MVW+_QbP=`Qb#%HFc@>WIqNKdwf(erj%&R!XZ{G|o zjW2E1VeTxTKXKs>)dvm{=8tgqq__j_V=*bue+kQn31&>pU6%9o!lE2R|M&gLiGObM z7Ac0;z?w_2_N`bZy>WNs2X6`tes7a9?70D%hxWdrEwT`*1rrmCBN$B$>=4S>Ky4*4 zA=#zp0+a+D*oLG+Zex?xT!5z4w}u1E)gVAbr;IURp5dvAw^ts5+X@-=+@#tq&&~5G>w+RW{wwS zs+rDwSz`E(-_zIE_bX?KIQNulJOzv_bZQ*TV+>4xTS>V)fy_0N;W+?E(xh8E{ zp5!!PIaSZ)5Mns{nYE0kn^7#ci)hv*oeu((>HLKP#n&1$SRd%3KJwK**e5`8V^u#5 z1|@JGgk{gGk>eq$c)tq-H@6mU?rjFI{)I4!Sqv zj>yKwtm|nfmnVS^5QI_2#svCmvs{~!BQ{Uv1r(8bsAloyBN*@zGGY5BfKc|fXPPBb zJdCXJlUk1S7$}ZWaZ+Bl4Ww6dxp9n9HXl+gc5w*Vg(9p9z}>UZw)rVY_XKm#a4EyNe0L~<@yu^N%_!SfWLK}s zC@})f!rt7hrBJ7lg7d+LTpi9t;o?((&8!Z}+`7!(E@hC9=usuzE`jr{Locglh0b&f z_o~bO%Z=z3johPe0}^%JwJC)^xM(c7Cze!=etf~cW_L>*tTX99B&EN@4wnL5lg7RA z-sjDt0!Pml?I4}AwXlH9jHL*`>pHnBFHgge`xj;V=bOf2N-*yz}bC^*J;yh(O~=#d6XLAoT&3oIDsx<#1@8Ya_~P5;AB1x4vk- z3wP#oNK^0Lz|Sh^v1Qn-hZ(QiLv_F1_4+@CtjE}E0?RM&+mCkbf8N~O>Y+sPfk8@$ zHxry{O!2$FA#*P;J9B29yP4;F(ugPYy2NO`+?_nHA?#ezwV|TU^oc*ggs1suK*gTi z{J8LlBrCq+UHVpzpibPkMtb;|TeFKZ`CYu>Pi)2a#-i2@?6U#x5CgKc>b*}{r%*&h zfNaL0HoskSVslM}-{FSm-bX}(=faZev&30TC!Xnj`vPsE0RIS*qDD$iCAgI}PyvT- z0;Gd&OG$%059LkG!?f{1qu9w>mqKqWI+pizPlNc(C35lDvlR7~F3DhV1#*@Yxhs`? zqgt~-;ehV9wD?;&VUZ3CL9kLyHyAbsrZLceyZnp*xRJp+TU{&shX7RKRA;NraM z*F#KsxLh|bN91eJo!qKlrI)91Pa`~SM?MEwmx^H#Ra;ZD^;$<+{B5!6xeeMqNi}R; z63053Yg7~j?4fHu>bms9=W<*qu0S>b!ka>%G>*@(V31611Y()#^;|T}^`Gq@(rxut zQf%-+(bmd8T^BxvA+X(}Te1E;x_>;QVP+^*3mD<+M@!i?33xtxzw)Ny{7g zrX`fUtV!+i+a|~5EyY%%^vpdgS84BG7ZF_jSM|LhX>dTLOlU*WnA#wF4*(wEC}5Tg z1({_@NvJ?TWgH4%P}!dat4I>0?9D_=1#P}wVUG469FzCIi;6~o|I?_dvK^0)FlyTP zwG4=w-RnvUtv^*qbXk~YC)liJ#)kcPTP-}I3((M}AXv%oEg9NJtrs%uCkTQSA*J>V zK*S3udU0aWZKp5`x18TB=mPEj)QAjz=tKDMrxHQ)-v*ng@5`VhoFZ5CfVR*FRZn=u zgdw4gnl`>TmM4IP$(-@U$g~2N2x}^3*2_7_#w=x=%UHQrvr7V(Doi|%gT;ZhtZICJ z&ELkn8>GiAMu*25K{J@TN{W{9B_yb*q@?7}&|Mai=Q_W9q0hn@6J7z;o)vQDm}*y5 z@h6)=hQp4dc6dQ&#&TkQ3Fh$JwJyzC(1!}i-`Yh5AsejWur)By_8a4WlLZn12u?G| zPJu|?&r~m5Y_F&e`~I6cW~h>fku|H@kaAHAtj=I>5G~2HouiTheWTMAd>nebv_@VF z$^){U(P;cLvvIofApwU!fsS(QE#P&LW$kSGj2o%=i_Pn4GFRJXy^LZ9NZ7b2XwoeN zLy@G-^+_*NWrF6xDUx{3FRDH~{$Za9oFW4tCZE7jT1$vS5!fA9-WN=Od<#V?vJNgc zKx2sJnAA)yrJ;)(P+z6SFEC@SfI?}N(+uHV56m6N7YR*8-1?D)=K|cvP+;{(F9^78 zB0CdPcJ&~z*Z!AkPc`MK;0EJdU=hetq)~jG#e??Q_)o0F6`P?Qt;Pp&+Yig+PwO60lyc>kFQ~95G2LhPAG_UodkN8^ttR}gd;`8lwk|7- z5U-Rcp7nSCdqov^gC%68L?c%D6Mo4GI;&DPhJep;x)!%^$97Dl~hR zh*-YQTgYmhUsX@+T$nuTVHCgpNBC2+yWPk3wtCF2CmC5}D#~P4jaY78J#!3DA8zpN===QuJkqul^6rg!kam1O%=DR)O0gEI1o* zyV9^20Ng$gN>^=j`{w0DIyyNhR_=N__sAe!Q28~az@6PfJAn8g^}UmI#4r#3e2f^Y9@`>QE78HrvF!8k#C*^ zqV!S|=Rv5hFLgW9%1u+bsqC`phd6LNVD%L}fbBqhRMCB7U270uDe0~6Z;kfyiFA-hS0zX)BS;g5YEPvJ+!f@LHz?WYpk~wv&p$<3VLU$^tT%bKn zg_F&wFT4R12z-S;2)6W0HO~{s>Ozv%5E9_l6N9PFRU*6A;HiliicLbz#;Rh1ONaEpkLw{@lFkJNZ_)@WQ3H%MdxuRf zdJvf)`}ogz1=i7yR=U#~zGADv@V&R&a4y+}tzG_-m_gK(pLLSh6z>u;N)ZamUWkgH zULPzJdiy}VMtJm+RN+mC{`M2`CVt(#!>#gpIK0_EL#aT0v$9sVW@ZLygf*AS-3Lqt z27g>A&g-}!n7Kj<_SrwbBQB43*K+NoqrFOQo5vK>*mWN0NtPfGj-PC|$$s5$7_d-j z+768f@@eB*a~t430exq!mVkujB{%uR+hW7W`%XUxUlF{ZMoMlaot#wS>?R`*G9br8 zsOzz}OQv{%O(dhftN>2=A}%V=bFcOGi5?JTwpxA@E{}_*k^f|R6i0`N0&ZT_Zl^sQ zPKY51-kHjgY|izX-!ie5l)c4B>5%cHqp8a9Re`%!8<-&ymu=5IG7q_b)~5KV$(^kd zKa$>Wa$8J2B%5#HPbXZtpdLmf;7f$qLSGyqqCFkx9W-OVu{Kty{AEhh4&irgx5T0l zlWg=Q4v$O~(Wje77C9##htz;@ZO;bMM)py9g*540(?r8a*@=}UM6>eee9EuC3}3=j zRPF>atRR4hb_5T`MWb!4nx<|$V_Z2+Dm9AywB;<0WiN4YhT(oOpvP(a3WPtI{W2HS2 zewi5y*jwX==P$XxEnPl&(54A_JwjCjUrrj^FYM*g^(;|@(YyrK3HY-MYc@sK4Cr7;gxdM$K#Nj;Fa~fLR_U*AJ>jtJxCM*$nJEl+x)|D z0K{Fkd)abHuUW6+tfit=d7Dai|6*>j16m6+5mG3Q$S!WxJ^w3(;z>22zTIe0fo*_T85vX_DYs;j>)SisOL{#M`dTQ0x z89%%~AdtEcd=>#NZxM^CzJ~v|c|Fmh6liS@Y}597&JIc<7!36q z!uY0aYslje)BF@PKT7tp<3#4tC~%u!0L{%8NA>JG0NWZxtc2&OVEKUotVMB9Hb1P_@I; zp7Gwe*Fnb6eQvBEkQH!ro7L^%;9z+Eu@fBJ{e`^rYPcnic zz<6~a+e&=p=g~B}ZR_B8VfVNhP*en0~8lq>Sut zI5n?3MBcRX20TD!8jX7)tIP}nPWdSShHVWDAd7?~;HH1no`<&@&EefGe`y)xt4?UR z<^`|vDLeLa84SzrF}T~ztV=;!11$rH zzJXvLxVJfAFtxOElW8VYgam>7-g1uK^uW@{K@YR9;%b>Bwd7tg(goVT*LphyUTafG z7E$R?D2{>!0|P?ux~ZH}uyYacAb55yl6{f!!*IylkttP?Mks#q7RuX3s%F0uo%Dcd zRhuOZYa{z!z6By-!cw9mMweQmC_Lc5g?dV@hs)<;s|3<9 zjFlfktG+|Ect73wDBEQ%YP(s}mNE{)qg>dnvznst#{rNbD(Jle^nspPqbCcJ`7>N3 z%12E>ci*Q%3?x)C?aU=w1&%@mGLe8QQFUFyLWQco|4DFMRnN=+4?EDAtUh&R-eP(+ zN4Pm)kap%FGYv@9yi${Z(X-Yu@BXzPke?EOw5&~|?voS#h@ z$4QbTpKC(>BbYIit0otQyJ*NBv9OTtS&3V%7_vt3CBvkd7^d;D-i{j~IDkk_PDf}( z)OX=AMjXi9yH8?LHB^+jE-s&USENl~`dFTdx&=5!;kdS%Gx-I2NT8`Y7ecuOCBpuT ze+v?Jn~xwL5v_F#k1}(2fMWQ0csoP)_`Afidl-=_`GGmQYVQ&jlL`)A9P^RKj6D*v zLxxvthVNe}E^seaMvc-(~&K z7h~1^X_)IB^IRTW?}^pF%i*`-$f8UT5qG9GF%w{NrQpNKQRcJ|<{OLt0knWD7M1V| zhEia$j$Z&x;#4u+PI@t)l}M)Z0Bk7w_=HQP4>WGo_CGadY8?r5L!%^iX{h|Sg!Nc+^l_?)QaLjRXR+NC~vk-`P)Ol{ch zc}e0@@@`j&MV}5*{0BON2K9G`zc2aV*$H*uU<^B>mJ`*72Z9RxQkswKuY8{6tgy1V zBqO8j8tnNR7i9em3=Rk46=WUm)xjLFTf5;i?l2)iY)#!rxXy+{+s11P=?h8a8&ZV&LqjXO z_JWTLbf`f(ny9B_s%r~7ekG0LxD6L)a4M=UL))h6jk;uG2Yyf0XqlwL^<^80Hi30d zOP={1?d8|o%Zje56s%j-J>RLA8H*7c!H5?fk3-H(910k{e)?GRz5n5AVq^IuS?i?c z*KMGY8`@Sp;AX&smODXMaRFZYiBPb|JjrU;)WUjwqWN*K5>0QGkV*_n=H8Y}O0N zk35dE1l1@ytbCvv88+l!h+qc@OKJUe7gEw}{h3`2BBhv6{BhAGI}y?r7$xE>ZRkLZ zn1EyS%7Ks*MCKA>bbu>&w{|{oGPx2vq<_q(+&lrmw1`7$uM@s+sEgte#F)`UD;VV6 zjRJ72MPrV|&WW?ajgYo6Z!%L|!zNfM(|;*gCzKKJg<@|} zr*TEyPVbx#39Bz|enhBVAiV|sMsT&$F^BHdj#0YpuFr@?v#)8}`<)08)A& zf1H?b0o(07RQ{b)+EB3UnG1bzjh*1>=7Vw#0n;ZCN4giAWHRs>%2~2urX9Srq}j!p zZhedY-i&Leu3ad+eUiZz%RcQIb~}rY;y$Nn6-d$w9HC~6WD0f0-ddLq~ zfFRl-j1B#6@wY^%OaILO4Zs6Qnm7&bni<{6PUh$t!HblXbcay(m)62;7tGON?h(Ih z790S56B7BAL80Iyg>n!)AmO81GpAure`cgu9xyf558)I}Bb)Sn>LWQWOzx*&OHJUx zxh%Vhg&t%Ir|?c2jh?fWgy}1`!vPxMDn^r4cr4Pq45c3w_ngGv{kcq{fnszSVyvIf1`)iMW@14kGE#uzyhQx` z)tPL_ivegbOr_igm{1i4DIvR5h>t(;rJ(=B*SL4YLKi5CX-348++CFXzBO1V1 zQq4lq+*r5@4s%gu*M%`=+r~SD2A1|4i=TUe;LEF#vVzou+;-Gl6MH;~GMTDg=(13x zaQJ1Hw0-RdY({dQl@Q2))MVXS9TX3h0s5ut+!Pj!%%6e5p)@`1ssv%xvlBzDyqy~) z`isO!t?No__p<}y&Isq#bM z@Xw>-w*>(u*7yq{#&NmtVcn8X^uU9sjO#{h2|nrD#d>>iKwh_(HWY`-1{H47(My4m* zP|<6Auh{%IS0S60zT(58O(JsBvq2n_?{Z8us8X&$NffYR0wo~4bf!02z|BeJHYE4v zq*U`gmIR|^F@wErsXsaY?rpk#Svz?n%c$nAO} zsP!QPOfB{lZO&COg#?8Zlv-xB3yP^T4?_$yHVCP@A&~TnCAA^8Uyr)t{?=tqRoBl# zMep2E=zRI@ek{!v>AQN{0CLyfIO4PFPr)BgN)Z%+MhE6o?}9eFd+ibT^UELK3A@~; zn6U9c$hHRVf5p-gg|C#y?|o0gE{1*rWWlhJYksy4(bi1CX82OxZ@qXX@xDQiJVD|; z^S2G0{p&@aOFU>2X{YnM_<}aD{2*OMt!4Fp>8kt z&n;7LDZe}cW2|#4CN{!flx&*ti8&@QjVbRJ7``zFN6E=)CYZ!X0`WzS2hrYD=9R7i zO=(hGR{G_)&KA9=qJ^5z68Z-`O%DGjtBc1LrdPbf5Y3>BoWEXhKwiiq89$%_AvtuK#*EHSodOpGzV^Lh{WbH9(r@BPQUkDIQU z_xtsJo!2?f^L#!}x)zk@NS-G!@FkPCkINz7xT~D2rHDp)Dr-{3ehXvLqyrUT)4fOS zhILy3V|bC9rj2dCguGH9-ly3FuL_TfqMKNq%!T#2wNCoG*RicN2=MH6)Luqtfu{9K zJVHA6Q4|hrUx!QBQxm4@p>zgh4}HW{b1*Pv95J{1Ce6WK4ZBM0?F9O(0h06oeHO%T zp-DtgeqkMJ?$5i3SlirSrMp#U-`7S~5#@6!vRLg%^(V4HT3WUrqnjdu*!&mHu)jzd z0S#{OWWGFvO~p}Dg)E+Oo#1i#8LvF9e+;n+J*GM&_aRy~>0OcHf5lDu&|qnhM$DMA zBreoqG$5deQwB&BRnF-Xk?|4NA&F{)eU=GQ7<7fRxkHMe?#Y0pqXB{B^FMR(Auuu# zg)a{z6VNHckU<)Wk$lcDEeNol0v+=H)@xBc)N-!S@CC4azDaY2=NPRt37fhS(aB|Y zr3Tns{Ar_Ih=m|MU#Q2NbbB}McBJ;4`Az#Pa?W`XP2B9Uikz043&|Ad$C|jCq^Q}x zoLA5HiIunvyj{M)NKWkhvu1Pk+Fk<;1jQo7T5h+_$s~jIgJt#u_9VT?i~lu&HiL%O zcEE{-u5pNBiT8zeFW2>Qe3TuM@Hh%nk1g|TkMm+<;vG4k%6>@d=m+R-&({7Ta*uH$YEXkGv{ z1zw&1t;*nif&R<7wT`$VGU5eKo;S~^_x6x3SsMzrKv*^m2O~x|LpS9_o*N2VTIs-) zw86^-J{qU>XXg%b7w*n=Kp-hZc%~HexxTuW)c>Nnkr}Zhd)x!->m2p zrTSq=QY)eW*5#uc+4}fxiLH^~Sa??DB+%SFre>EnM<*H%oVpMw?F&;!bP-FJuh*ob z;5?60sxrf0BVV)JdFQqw5wg{Tr2iA5$dDGJ(-ZLCIHLp8KteF;hO-expVd>v+z?!m z%59he%j1)AYobdTH2H*f^7w$&Mlc)P42l#t$~82e965m{0&)KTPSN1@jCwm001a?@ zSF&b+OYBX%pVA8Ih8K(=Je4u5+jp3A@y~he?BuquA<~{uK#<-UCsp^6&T{Q`X7j*$ zm>n?62r8|5t7Ww(yv~B^e;4|B1npn6xgaWKiNLCS=GPB|2^E5)P;AGxo5*>>JB*v@ zgwsAiRr#xJpFzlC2yBo8++MZ#3pgYLw>U{>UhV!Kx=Ot|?>K;1Ed`4;ztduekCi!x zT}UYWm+M>0Yg?e@4^432N}Km z_D;0-z?Clz$`e(aaNSQv;SlWGJn-S%)2Dn(`3z*)&%K+As z=!XFLA-vbZMgvg|I6yThGC_4oL4i*;0Gc;qp;wRe16(3nXKx=J`7|&AqvM zYE~;zgSr7iV2exWWgm@K{ut4!`AD^TSNi@n>nI|cbva7=yyWL!$63v4xq&AU3^PdH$-vR~+W1+ZnF&&BXdFJXd_HVj#V`g18x)mgO z#XFFK4l=zGG$qIXD4CjAj+Z2jJTrdQqy2+J;p&KRNp*cK&j9f7O+MO-!&Gi9k> z_~8_*Y02V(763CR#t2x4$&Rkcf z`a)qm83texCCn8TutQ)EAVG&;OdQq>HzH3K$dd}_<%;|c@v3bZsqW?8S0+;WLz72( za<*v6dZsivVgpu^2slv@4+D&= zK*WKugwg#1Z`AxJ4{a}yHf<7{d&rBs(+G>2ZT=a+EGYKiq69!b_ab_woh%2@YjHs| zkLU*if)1R==b&pUC^pbX#~3r6(vZTVo~;B}a9DOOxf?h}0%ca1;oa27#sx*VSkUQ`bsxfE#HDTG-)SZf!1gBXxGum|NB&fyF* zXkPWi3;YhWtdZ84btp&ukQ_lny+_0aLs-m8?|j7~LnVhW0umILc#AYGD9QtMJrk8} zX2P!pJt07F7Z_rM0v(Cdc|VW7w)z0Oor6H7g;HxD=Zq$rr=^d^* z#HIm83_EVF9Kg3SOxHhh3jUWlP`B(y`lYBj$t1olP76i9)Y0o(4vwd7ZY#lhWc$OO zOQ^Pey>`j3tC~wQ>Bab-V2*pSdMjV}MO226ZyVjU`lY?aYWkD` zCj^C3(8Ovbg}0||7!HQVdx z3k`7#1)SLyvrmFc%&km>pve@pb-e3aCU}J!q`!N~Baa zfW%x<3ma|Rs5@-jp@YeDp32_{VSveu-Ubi_ckRI%YB$l4k06F?sq`>I+9&(!1a<=S z5WS4zTjRTEkXjW(G~%0ljGGPY_cTfRyArx1&gEW-D>_6G_Y%FRo5Qn~lWlmmt^c;#`T~4| z|1KTB4NfHznTPK|jgSC*LD(Q5_mx{&J}!+0nz2aaIzHwKRtL7wXVE&2X-;L3(;qS) z3$2GfTYUi6CuMHOA$-D`?)UR)@CSVt#bYr<_k$bpgNgC_hhO#(0E6xr9T zw~y}$I;w^daV>wrNE4t-U5EK1{7cj7x+JiQm$ESNO1?2Nv;LLG$20-=h63?D5=Wo@ z;qS&>CzoQ3a{jrjjLdrMUbTZ~doeMS5!oQXL4Xb%1T= z636XPiOc@4<(zoGH5s))p=Ya%IIVf9+}LMR2d|=PzEZBo%O(*@{Zds`g}RXDE7qw{ zwjn&4O}d|=6Hlc72Dj9YgyoaxJ)O^#jm3$+wZQJ>P^nra7S7Y;;+h>bAvdJ^>6|&H zk2MkjY($sd!QP~cF3HN2EUrQg#Gc>7sy%?m&YHHsdK2)MM7cgyj(sK}(Arm?ILM_h zwn$fp#er%}xfv6K@8rhwyr4)-m6LdXuCjD!(Qxd@30BYdov^b?PM(KVvU&id7qR8B z98b)*oZ=a;rO$+khX_6b?hKK_Vd~QMi5r(M{^dQ(=w;FuYy?%0HVM|1O&mWs{tP-5 z5DPz-Q7&->N|fE`7z8ySyp(5gUCu}kvNa9n0NDbAs*c+^#s4?Sa*4~D2n>BZkt701 z)@{s^c4G6shZ(acTK99q{PYe?d2}1SkAMyicII_z{cxMrzzrN;W~(lkMI2!5nD7m; zp}(3uv0oO>xg@M-MDKX;c`!@(Hy4(ZP}cIFpgrgUZepU~J+k{rknx_%LrAxj-`%`N)8IQXwUs^B?*fTNBT;uOeNS=)N7 z-o`0URBwkKo(HO=WvsHF^`IdqGM;UU+`FM1f#8T3{_bx|Sbc%z)881JFkfBBro?f| zF}%)mhc!s|<|+?4PtLJ2=KnTI-YEnJd8*r+{ERGdeeDvpN}&1qUJ|g)s-=l_M!p-{ z{B;1zfrq_ei#jBE)7lF4^&xF-Os65&jjjHb$#>PbtChR!>O>BhKJ{*1;U9CQez>0L zxAd}J=n996mP5b@ltu`NOgfS8L%0WQA>Tcfq~v-l1is12F_a4COQlr-+je+W)_^4kmDs6aCm9VtIZ+5eVpcZCP-bGWU3VbP*Udr810fo|;;bv7#kR z$j=}7z+GQdh+0L`Kz_d`avA&k0nR$IfSPQpewCaH+T`;P^g5xe!RZytnO>AeAyZ#} zS=&Ck*t9aviP8XTiE+14T!>q<#QgORZ-Arxfw0uPOUSp!Yo4Kq%tK{FYR^c1Bzi~q zEO7b{HaaenbTIXq>iA0|@psyF#slZyV0`3Zm1N^opm0sReIN0&(bipECG-opSA^!g zHExfTHvhQ7MKhO)0?UW`TM>^wlYt~q6B)$~pG2HqM9FErp2?-iVAgbc%bc&yM!`wF z{1#;9xX|FL@Tt=h4s{WfZBBdnW`_rurkzU6{BX0RE`R_2Z3y6Hia{lJvFjtyWvA0n zKs{MW*Nrp$r(+;(@5FF9uS?NkfpS8rajcJe%v&^LPI4B!Q%m zfOcSA$w27{#Jlt7I^}|cfj~IRZH%5}DKyZqQVR4;sLY&z&qY^&32HTZ{Ay?C&P4VY z?#H@0J#j<|LAs1d=l+$@zmQy0?biQZ%Qzt^Zl@Ot?nHnbKsh$K5%J3i#eO2r+)5`K z(cWDq?SYmPPAo14$YXDbjrCcd4k2>5q;d_dz4tJq=u(nJTIdy&qBA*D1$0oN{#CIc zim5)RiXH1eYld+UJl(?zkgTpyzuIfE^Y`!;!lol;sXe6 zsH|xdWaVS5e{M?7!jC`6ZcJiV$QFgLiSXTkwZ zlfkZu_77}+BTW-n#X4tyAv0O$Shd^Bh%Wu~`5n8DPM6W~4ofNnZi?l>Y* z)ZAg9GQN<7Ko0!qN#(c3Ft8dS^_(4nTWksuBxh5ji`qrdLza1taJg`Nxi4VV0~S?9 z?l*{_aU>a2L^SF~Ok9A#cw6@E%5ncnsR{PTK8DW$>iHgB*0Co}*}|_C51G`Os)@7@}K8&Dp=A#UI{SL@pR%F8;Lt z!|WS>fAi2q@L?D`E3QjbJqlA#9jmW=I~PLBjvNYzPB>e)vQ@e5>xb zc0BFS(S=&o35_i{6^LE(fq#g;D?9v;o-q^y*^@Tn|KW`3X=;yu|7Q}_T!@n5)Ys0M z%Sv@MzSd;aSGZijd;W*qrp;|F3B{h321k>L%>eczfpQie^J(GI9?oF<6&t{b<-QfA z=tj4C7ftePU|CVt4*C{lK`uU4mDU7xG$wPU2KyINkS>JE(|g|9K6hbk8;vjZ|s z6@!=jXg|CceWyEwViN--g>i{@1TLd)9C*;K8)Y|o7Np}^N^GiO6aW;&V4<`;(HoU* zCA6pevv;=ZC6tIlsolLxIL@9Kdc))7%O($X6E37o@q7S>hCyoK@U+%~Mqt7mn3o2# zA6|u07YkMn6MJ-!5yH42|Lc;|3D-ZyRX!NCe+|8~K2#`&SHr~c41WpNQfE{CJMWZl z-UzyV8qOb2g?|_wNW-V5h5Yb)X|Z<`+S$;OXz$GOR9T4guowiJ9aVb6Sy}zC8Hhr&brc^5;g|IR;_74#7s#u z4syxrXIG52hn1sqNl!}C9BSPN0gtV+-%Le{>O~T1vwQsfXk!#cMQ{Kb+i)2_oO3|K zMuollS7oTGu*wipIIQ0%P?4$}o z7`oa(QiCk=ONKu)Yr+i`wsR&LlF$0gWfITp?t=V$dvY*pqxYWCIfgZ;u*d_#dMt7x z)1$?2ATTe41{;IX0UeYyZAgedzXVPSv%Oq;dSo*CD{Z$9YDzlsI&e?44&*O z=Sxsnsk}`i@Y~X1+Ht4o^IKIqb&i*3IvsU`W(Ex>OA-U8Ig8euF-LY@)X%*ZvPS!l zJ|A9AZtmT<*Zgczg~?nscf`&SpQv&L8A;067xEN2CP2dg!w%Q#^>D=x_!}Ujc1bA# zK|W9?&K<&SPuIS71a3!ByuqMFZ2>D53@B_DBi~OHL*vevF%cNbqF-y9Fm+IV2^C=pAxS zJG|xhGaHXS{gXp77cJq@Osl1}cOQQ0*rrV7q;kILE;VwZDuNMyC1%}*4GSpny78#6 zZUWwxD0(7E+Ih>hwOXJN`M)c!lAo{oBSAy2D2M+ZZhA+1T7K4X`fDsNTJu%~-o!m0 zh|wUZTulddzVhL2$)fH<-XTR?&YfU6++;ZWT^VC-wkFVT1PP?eRVE)TjC!ct0IOI0 zFH~hME&Z2Iwrefc^~oQWZ~$P6@5vF!tEz1#_J*NCyC`)f1QX)X=B=1;bFF%R+rPsY z^-1H99+JSfRt4_@NiK1F$Kp7N=A;-l}UI!Z1LFshc!-}x=*JdrA_gACzDRAM4S zaFk`qp4&@F4Wk{2&hc96EuihIH72EC&%GVN(kNa@RXYUOe z?SmW;Hv`RPq+R2AOF_7<@T4)B2N7drq{4JjKtQT?+81&L`%a=x8xVEK+b0ehGk;Ai zGtn3_cD&9iK<^ABm&m_LYMTK{1$38)n0|K4QOxv2Ik0LNz=%gw#{wa1(J&H=D3Fl1 zuX*MjsZ#D3Q|qTQ*wH*b><@rg8wV+pe<%@jyC6s$d(wa1;E98y&n4E+1e%?z*RO3L znRhCF2Azbf(OF6^JWh4?*}Pwv)#vb>>!ORRrsS&>XjiJ$KZk1D+*Hues>AfQn0RYy zJ%|=sT68b&H&$r&@tM7?JIKC;Baa?&Jk;UuWG=8eD{1tfr-ygqN=|n>z2E31vRl9= zm1`tw&5c=3TD063vP3N*Xkw4Sne^xBNk7VUOpKJrVqPaz z$a%L&3eCO8Xd*rg-=Jw$5A)Zz`Z-Q&6Ud7s|4KQU77$ohi^3=+SaMk8Z3)lX8!^^v zQ&1v4Vlw==@1%zyX||BVs9OTjr2AoY)EPTM;X+XgD>fvj&K@Y@A($F;2t_3Z)Cl7M zsi9UX->D!>?Fruv$&_Yke+bIda0d(p?_pKh(laG>@n80yLT7e&?ta$Bqy$(Q=}hLV zQL-BnnXcgWMY_P58JPC=cbG=fePtR}{y!0#b|5~4;$MVIb+_ah{5#$1)j2@63$w*g zl~CxAuOkv8;!>epsHv^ehw$ItO^34)`fF3@4n4U3VmXP>(eTZCjpltLi9UE%(_c{) zOCy{m8iz%?t+t36dVHQnIEenXr2Ld7(HmK}mnu&mV!S#yabU~Mlo5azf92MUay#Rl%78z?0a_ya< zz0AYPN95WZs&)4-6TZl*3zI>SQ2QLq+qz}#P)})-JN-8@QV>yvl;3Md$A|{AzMb-A zymZ|))7jkF{+Z!qDfvJsOElFH$oTr(q~P=-yr_@vJf0T0^x)qG6BNutv~*`GLU)Hm6IouMw|lF2^6R05@G z_M#@#FL`}DEp{X5Vf?T`M^9R1JG(WjwlEdCW-HX{E?rC_P&iqvjncYdtTX-5pVY!c6|TT>LO`G`uaR4Fh15vWTHIx=o1wmU^L>S zEwFyWF{qF`!i>CgOzZ^wN@6t<|jZBI}jfUy$6yON{b zx`S#5hf4s67|-;}RhxL^*&VbyI~|2wNEXhbIFWhot)kZ&p?M{)IZx+&O9LmkSoa&K zxhU7UUY^!0Uuoc4ZI|q-{l3r*>vx?*Iw)~AR#QCmf%3@C2?-P?e)K>V@^6^9uKtGc zXW{{u9l}xGJVGVM&hDBf+OruLH`Shqkh1dI)YK=(lfq?PvT+9ddU%JjTx{!i;i_9O3xw$%33IhPF6mqvhM3q2 zY}UOyUl94z8&uaR1;OsvDTAFi8+W6OY?l$O4jB}n@S&$LT6?KVx_cvZAI18Cx{R(n z*7ByvqrD;dLEF1i|IIpG8JCeLR_7ns@gP7gmHvcSTa>WiS%!G`_6c9N?y&MeYzS$O z%vWqilk`P80x#UTEx3R)=0Zhhr8Cm|nU%qD$ApRvuY=p@P;q3BUfj*_}V zi8CQS0OK?t(SHH8b{rJ4Z3!xRN;PYKg|MKT5*x2~olAQ*^TuSOx01jS;?NP4J-|1ysgl49w&> zM9ef|X;(BFh8o7c-b)DgKh8fZ6uTo8I^1=^o1jte&j44vA}M4V`!tvXR+kzXK?wZb zV+$x3F2NPMV^`PaYs$Euzm*;aB>ZO~|26)vdBfJ`nL|voqsoP=-+}riK^AYR>Va3U zal~~cRTUH$bX4aoBNTe(nURMPWG9vvNo3AXap%24j>X~*HPAWMF$Z#PibkgIDWjh3 zP#Oy?57AX*2ef=+0^ZktGM*z5 z86anc)OQlteK63|PSM!2yFSaFZ^uRzK4yRe)t|jY3s9PQ^UP2vhER@4oe>3D9pV2WPa*Q~@g8)_fjZMRo~k97y7_=lqqHbU1y9f{%Lf zXeq8d$STZMp0Jle`A$baHM`>jQ&V^up#VNtEb`@&)^#9;DuK?ta--I7EZ7uq0tKPi zUzCdMBN-ei1VH`Tk}RNO>}mjYLcXg626FeXFT-c_wtct}gZx|5)g?Sfc{PN4d&}%o zC0x?xwQt|plyXMp(=>9XJNkm5`{bC$-cwol{}JvWZXrG^*GwF$KCg~gdbPis?T&L* z3SPE6zd#291DdWxeP5!;uPq7!O$Q8Eff>K>xdyhW#)4qY6sG%mDe`HD{~R`gZ&n}z zy`bYa{2UatRc@vOz+KT-h?%`MAK_B^;Ky*?!JI(9jrD&1LfdP)WWznf<*xj$0^%;M z#UO^(JegKFCx>uip}7Wb69CKn$QT}zbf@gi8beK^-#m3)9T9g5=V@Q0_#1sgG02vZUT7A{_iLj zcO5d)ewl_+AJ0P*L^gcwe?d7fz+o^xWn(WPArBMA`&KJkn0w3vi44d*eCnA zh>UbWnQYZ@BL+B#|AIGH;!%f6%U)bo*D7tX9u?UI1u{r~>ri9%eYHnB#@!Nk&;jkg z$YNk!9dT)u$|dM?p+t}WiGfBV5>WtOqY7aOHsX+f!458JMylP(tI=ilX?!Kuo?n;) zRppnY)H=*lR2DJJXlufmZry2jclDX_hvV-Rlp!2LK#0%1pKBdou;Q~-nMwbF*pgAA zj-;?+rH@N;a8qD5N>?!In@aPB>duo!9Z8A4?1&DOa@I@kVe2FTD!fmhsEBHSmMNR?S$89hB{5yojuF6dZDlIWAfes|G z|K^^lgHQL4e3i=0VV#5&eM3S(aJ8JT0em00y1cOb+m}C0+Krog1MpF(8~qi7Yw~Xl z7Qkd=z7IK&Z^B$Iz(F2K??kqPGlffSJ?Dljj?<8pj^-iVpAybs@d;Lb_j}l5wh=e0F5OiW^~L%FLNOmzk*jgo(tlhnr}*C< zOH{CR=@$NgayQm$Xa_9EF27EsE(XnwB2u?myz&P&jUF$qOOv2IkMYUA6n^Qzd7+?& zTm%^pvnE2o?c5k4TC-r-a2{o0aH<-b@jOm%V?eSIofi#<5IRDkS1!&SSMw+ed)=$| zXVg_xR9snXD}VVK9EL~gSnP=}D8pDuI)_fj~xkCra$RvW_I4p5`Mk~5yv+}Gsb3Lz~}8SecTd= zRYj_pJA0YZdjgFps!UFNFB5LOezvC(cqI*~UeR$K=Yyqc;abY8tJ-gC_2i8X zfF{B0k1aY$U@Aok&Zyj(7k6jw%??8%p5O^D1*Nv<3e+ATg}& zQg~qZCyBC z39C6n(W-${d6z0T^Mb~9y32b>Jp{{6{D$7@sLuLI|3y;ALXgH|fS|$s!=<~nF(Zd% zgf>Tjv++MHKrQ~I{Y;zY49s#(AD{s!`oQsx7jnh4c09&43PZ%6jP#*E81yrPGVE>aLYnmhf) ziBR>xlP+2>ZEVTp%@_Z`i6#{5js~+ybUSb=B;634Rh&7V-HQxNEr)rR+oG-3Fh!FMAYO3;nLQSI!HRaO`xUX($ z;WC@}mdM4H;`g3k=*|S8L~EPwHkcarxy(f{E!pf>{f-QttB~Lm-gZ_+KU6U~d*Du+ zPi60){!MULr}`71p=VYl0vnsz>+cP=&FCcV{+p0(+OD&aC5w`>5gJD$NPpKJ|o$3W=go8o(**A_IXIH}Xl@7W;&T)ZrC=ObI~sf3a>bwltCO{B{P zmrZ(z z47f++nE0|AiYV6?5!2(_?J8DSegc_!#=|q>L0cMPR{Dy_w^LLt@}192BfQ1F8csbz{P41i%Biw|DYj- z@M>jeP2NTZx62oF+3sGHZ@r++O(%V_kv-XiBnLqSMMXu<*pxMY^X;%(UjYk-TaPV) z0bt9!(G92wG6^9|Lq-JSH^{jjprafZt(ipR%JfmkcD`dw`r=7H21M( zPyJEO^{d3Niyw%uYZ>JZ z%)T4RlNqA1*<)WCQwx{aVB;W1NW`pS(DZ-+28>E8+DvF=f}j>zI4QKaNy5RyUB9z% zo1?$w%E8GA9zeAA?}sFx>@_AF5ls1ES%9n}XRPuPp$5B@-bh1(gwo-P~kSz+kJ3^7OpEWf5aG@0#n2k&kKDCt}#fqdK21j2%0e^z!<=dM~BJ^yF zRW3i&g7&^kbg9JhJ{*I6=-wfyK7Ir8J~+weTEBg&R z9Hob4dAXb~iCm-F#u1Zi8qV{#X(-+%LPejfd1QlfzM6Va7kMAd_AmklLm>@HkQhiE z3%N>pL|U$x0r&mQIn#(oxJw2l{HpWw?Tn{nj90)yGqYfj_Rna6i*C2G(P4vbK5#i3 zUgpRsx0GU}1GxPsG;n#63d=wKkO5Al;NjD4O~O|53_;;_E-^#9UF%`i_i(uF>bCCa zm_)O*9_)-O`IFDDh)89GEe*1do}JNNlfn5;qiy^}S_^9?ItshL_L+CvBa@DpB*U`J z?QNUKuR)ya-JGx;e_dsQf&dMnXYg8R20pVEs{)c5=1CBL_n8n}DAAS&s zJ^2nM@M~~G-2rzxj<`YT8a2~j9-#r4+x;%N^^j8EG4Gbt1sB$;C3aaxSSBiD zyX-N8&LYPXy7$xn6WHVl@c)1H;Em;X>TgBdTWw;W^bh60Lpbv6#XHEoPy=$*DTh}D>4S&K+Qm5T;PGaGu}QWC#6pp?^fFE zxs(4FK&?%DK0qT}w}owkv#`!d<;uH+aj|(Nj4x)iLiq9ub(*j7z<1B^6=JkMVkTC)RS3_592Q+&m-4Oh{Jpm!|J2({ z56B5KsI6zYH&7f$rb2J(T{Oa?4eFv1i;UCBfl0l78rMbRG>+hrkz`(>k2MeZ?r>(t zyqy2t;VKct1-6CXJNdajuE_J}FAwomdLE^=Wrw6p3H5u5Bodr&D{XqPHt^RiH+7=B zR>@A;lG*RZ7p!spyM@>9^Pm`P*=F;ap^NLh!<&3!t6mR$TBRE{x2wf)a*K@2mYbqh zVqz`B*Ci_5Ad`?rP^cI(YT5FXO&NTT_eL!kb_lw)IXnE-n2zAYrR78s{wQ}BKOQvr zOONUX3k1U3(d71Oo3@{>EMAdZw~(j%gtG|rsjPTFtl6l#tasFs^VM(pgxUeKQF$a^(7x}2rQE%;l<%gP zx^1wOv5(&C{iCe64(FL(ax9@F;hf~=S6-F&QbPs6sOfY~9w`6|SQ0WzWfb!i!rX;&owdE9^Hb#87;q>orm^v?Fs~Idc$f4slvG-ZR4IwTkz8 zI2{4U0QgQ$5=)yNEY8DeZ1|$n2S&leU~)~D0t5AIP95dKT!)MD!oG7G(yHB(TXn%B zx}-Tbt(nqMxf5$BtF{-mZ*h4dkLv`GG_lexY8@(M@dWEdlzg3I*6&BB zjMxyv9NkyZ)=Y+K?QOJrV^3lSF07nP1AS)UK6#z4l2RiDSyd?4f8Asg&V;!tSOve( z*6(P8{AZe*doA^(kg@IQRQ0_CPa`4I+A~!Mu+U+O7%S}mk**+-W@-NVu*TzSA6u4Q zN`8NRd0s1w}`dykKN ze}UTGd%x;?ab|B3Dvu$=;V6n9$peGXX?ue|BKec?ACgsPi@cTdYTiR=k$Uc;{JLsY z--HeetR6ltPRlUQ^Gt_XO5)6Hy?xHXDO0zrW~$#?d?&fY)BQfr<6A=fdn8IXtj3vo zA{{fe zq|+`~e@SGYBrl>TEgM+6^*Z=YUfBl=_YZ{Q!D_ZA^CMO4sxMB;DktJ5JfVAJ0bFi& z36Fhdttg*=33?IL`aMa?&3HbX-n@_C?(9AW!xPPT!u7T+SQxOs!qJ>2f~E!s3YUM1 zRYO04pE*DnkX~}u-ZMJgpTmorHpi9;Gw5jK1hH#pzbc|TLd9q&wK6Zf8ljZwKyb#! zKaf)c;a!+kVD^8O9olZ$r=XPgnqP4`YvY5^pnk}J%2%HXle`l;jqoR}RlDKH+80lg zJgR?ccbjh0W!BAq1YsmWSe@=02$1hhgxsN5)Y4npdyG*306b$+G~?L%6N(Bg1=BQg z*C+M5mbUs*VGeXbiD!bQ5|Rv~U;|fdVF6M6D&(EAXS%N|OxT;?3l-6yLslmNklBxK z(o~k`_4^M@?ioN_lh$g(^Xm~2M*N88(I5C2&6uyLeuL+?IFitDZ#Xtiq2-zp4^lf+ z_Nl7{!9MqqAZ^X$H|fj|8{qlNYwmT(^!R8;wu!F>N`gv2{W~WL8O7HpR_~s*M{$La! zZFde0LDl2rbv5Rs=#2a?DkUW&{2QHv8t%U4^pMzybTYhmww^a8(O)v3FYwn?lJ9yb zm9;p#8Yfh@NI#JQb?MSbrSzKc1&r=+sP`!0@I#!t0bxo|z?-KN&`kNudfA<>1SbJp z#Z%=w3jyRoQANpWxf&A?z~I7Nb6COVga}_4EBvRp_vQG-!fIQo(PiPaasM`?2zBzeTR**SA3A0bl#r^JMoiw9-Wmm><; zGD)5859Yy%?#y4KfQH1|dN$w`(t^PNU;HQIFnKr%25NHGKdA1{4+L#5T!D8N@MhAx za5#%I3%$83xAYYuAGAWgJ}LGufronHuWItV+F2ul_#KN7NNCc*_yXbjrfFa7BWg(+-4s6`Zh{ub;3ylL0zctYAF}GZp|+y#?T!Du>gl=)qJ1r zz0FTqlJh^}2XBcF6-Su_gBil)7Ekasvb~AReIy^zG%;sT+zVYm3&(EBHc9l|9i`pd z*bmUxHzoP2XEPNt8PMP@NY#4B6bWv2ed#F>F@(do4$W&4;m>Xef@=;`T z^+oqRhkeNrC6}EY{vz)>gCgs9AzYMJekG`jdnXN(s^i z5z`5)L+9}k9%?^mOA~eMa*oR9vB)0Dr`it+9{ss$cY_fqN85b*VWFjmL7zQmE#$Y) zNcwDO$QtlU1fIY&rXwfNLH>N{_Ub}XN|HLzNbbW;t;zo8!LUJ+kd!nv?sw4aJ)1NRC&v1Nq zyuTcF1BMHs8dxz?dGmc?A66%D59V-U$A1sz1hN~P4v`Y{pueuywE;bPYNuYM6v`XN zMm6HUzScT$B~Mwg`hDf6sRKyrJT!CN~cfs?TT10pvLDA2`ioDTH9^jm1l zMwVBqd(=Ja`j%V**n4r7cM1eg_N>MOKR9ZJ_YhA-@-wTwAB5RZwp=0i);DpFHY6;p_vn@Cp)%3Z(of8X%PJuk(P5>kYh;XL>9%c7_Hh*Pw*l(s%*YsF1eeJljRk?acABMngtZz2te(dbmB*1=1C zMYg;@S9a*RK2E??)w=nBLj?cs1|swU6n*-afN2s3zVclu?N=pf7Gsokk$gO3NXZB( z>ekHKR9^ow`>;TmS!&CE`|6BAUgvt~KA2Wg1{ssA;l1e987%P{VW9K~%I9*0FMjiF zA4d=v2=_3Cm=BF4d%-G{s}HlylDX$lZzs;%np<{WhH7o@$K^@aihR@wRE}R8b{gJT z{U?qi1+b5hu;_8?B9NEUbkMtY;;jYb=EDV_c0hJ)g<==}6|d;P9}!K0dx9M2MZ%!I zAx1QLtHw0Vn#iaeRwsY>KY1=-d=ZhE_$lPt^gCTYyvneUo>^U+&1`G`a3+EX+$8yL zuXjP{?((u+bIO=n2toG&8OtlWtVg#wk3|9v6pv=n(;ZKmkInDFZWQ;p7z)Fveht0( zcO)AU$@I?+rC#x4tJ0*pCjwXl^9v02H)&t%1|zQJB9tYY&A^?>%+^f$3$i0EfOf;< z>)ir-RN<7G+%h-66ubTb@6n%W)A!4hM^=x1qNfc;?m4E?&n#)u!nVvq!d6D>=jDwg*9Use40Dp6QxZ$9WT!VhG23|fAy ztIdP^X@aB!yQTw{0RhPHZ->Q`6H@6|&G_;CPkJ?o2J&f&>L|_D2|QHY=2~itzv(#A zs?~xQj@`j%yRV_Uh;O(5whXo4Oi}PFgm-*0-3uz)QXguOus}Mm=+HE0W{ShE<1obp z;a>}g7T@=Y!_fc=7JP)~WC&ko!l@L}dYF|P78v^lN_G?7+fgnmy49~BKlC;y@RgL% z8YND-Yu7lvoN3~~qKsn>PXHA& z&l%WewRniCv@&dBVfy0yS^{l3v63|ql5Z8af9M$qu~Le&gZYSuQUC@BD9of3hnVL4 zw9X!>g{`l~fCCX+P{X`989)3ks}_D?cu&>k<&*4j5+Q%;>VjPM6Xu&9o9h}vWY#^dv)C^w?fI0`$ zdwq@3&)}!KE3R1A-}e&Ojds`SH>E0%=UcK_H{gJ5bgDSJ>X#0`W`UsVTAVRysG+s1hV{|z%QApJulsd zXXm<11M*`>)@(c*(C~F3CbgD#30uI`U-J?LlvJrZ6S#|*fe!Q+yY+&oa8TvPz2Nc8 zmXG*ZH2=Mc8>Au$P?LI^cpsn~$vNTeL+(o%0;p=>Qzl~oHDT#A>!TX4(%XJQ>+|}2 z8f3?Tr^_jWB{ck4d=jW!6Mlm@%0IBPGwnxuVB4-fay*hA<$&DjaU&RxlqI&j^3 zY~ti|0I*ib4!9RKOIVuisXa11J>SA1$$dS*uehB3gN>u{$n-yi)wuCIbi#bsnHlNr zoX$c=<$=PrQZLZN3HRZwI0eQ@6o(u*poJYUx*D@4hWBZ10W4$eKrV@VTAnxN?;G$7 zE{P@LhVYOiq@qN*s|>4+8u0Hurq}4<@2h+*jrF&1N`)x;D5;$09(DyxB3we#O4s2Y z-$gb|j~$$R#x9)@v_{`Ne1@BP!qJsd;+V`ElUGI}I_8+z)h%bt?`hz~_VG(&qTZlt zrqd8YC>O~_ranRr#yk&j-dn+3iu5C%k@N?&a`Sc|>TVu@B;mm}*$w)CLF8dfos9R% zH46uo4<%8gz#)X{cxBefs+bjRTuj~9Y_ z3y2(u>jv*HnF5j&7_;yM8uE(i)xO%QLyOBu*f^S{{8x4%-v&GS>@&f}t9N<23Q*z} zT-NG5|E;;V^DP#S)+6%*?A)C})ya$9Hgj3+N%zcJ_dehRsgBOyh0-3$es_K-+=|7A z+)v+FOV2$NlYH}7*SraWui7JO%{G~(nf+7vh@RRvUbyDeAM&t!Qjd=7D5-;6P-^W` zDz7)rRS_SEykPZ%_Qwxz{aP!c&Q96}d>R-?i(>yx zefp%-PAxq7E8+mW7#cyORVL@MU?JZ7F{VrGvr5@Z?dkzxQ<#SDwWY( z>{*j3Rx!@AAeJ7BA^HmfmKnmWs6mWysqP|UDr2gfBH&d1A#a12XtBmqA|gDCeG6vr zrCrVHAnDwMf|f1Kp(gpixyUIbZ4i5hP&rgg_X4@mMqhs$@Z_h?`6wY`mU=5P(U zq3TZs<2{wfuvvt8h}f~f^mpbgAJe$-Bq&@Cq!f&dFELCi^FBI4xF`MyP1c?LAzGOy zM@6ZR`*6sI8jicQnUq<=rR3J2j49e$_ksoc8D*1fg|^$@E3y}Jb9I+sZRWPsPAO)r z8OiM_|6#&oRwNiFIxDmYk+{u4ksk_BNl>CjM)e88(fL0>hcA3gen^8^(4Jkhaod?2 z-$Q;qE|z9%=ItYA*03IOuuah?^jSYa9rL;b5-qVrK0V8rg&V&kX-&#Ggb+ zwWwlpAR>}SEeP)lm0;1ZcLx4paTxOaf1>|wXpCd;$a?=^FWw8~J6nw4QFlbOgwo$L zj8EY(HA6%~biP5)em5ttoq&4(Jbv~|bsoxCsA_+5=k)X{?Tz;jC2*jSL6=Sfre zda|-ElRr=IzWvRqzzcJiBAQyN_IVt>zyH6;Br5e~`Npk;#xu7d-F2XF;9UX1Zpy01 zf3$yKuG+)b8Lz2U*%=(1g64Bo)R>9rVTX`95rbSQ4TgLHD7}AgkSy>^*5Mnhb4V32 zPFha4ai+q^W$$ZSWnPyfhRG=hiZgcTRc>itOi<;EH-GCRWTgXv>IV3Y2iMR^UUsv# z9kXAMi3#)k_d8}wXM(o@sKqu<9Jp{f4d=!ivm+aDidUD(s7WR!#$L)sC3iS+w^}8v z(gng+nwRI#e0L0jwH**;q_kH*xNbIR)njl0eaN^EFnybkF1(E8s;C(5tT8sCX38Oo zG-|_?)V{zQ@_FS}-TDdAu9LopFi#kjNYRrI%fFez<6%N4Z~1zB6~q4NwQ&YecDF1+ zKArdTNI|;JD%toQ3^ADJV)){si1FA#Ej2Z@s32nE+4&^~Agvl$x zNw6!}rxOPTiR5s_p{2N)#fTFLeUpxNC>LH4Di(g{HkB;* zhh2d}>F^>;_gA73F?Q$OY?NBCPBt-4RjF;1jXzgp)ZF%l_mnZ8YO)zX1=yALnwxnAw@w0WrIXUIKF!{1G|6zxP7ArBMqNQ3Jak$e-TpNs zV+OIRmlla;6hDHmH@t>Dc}CtdwfLfVAJWMXsl)fT?fW1WC|Fa@iPpXHZJ~~H8+G`0 zC~XF;$6=en3i=O1a>$i=D1Vsj>_jf1(4dYcO6&36IVtp5{wTNoI1mrja}4AqE}=y? zqFhQ-gkaW+{60yiih!gqEiD|M1t z%M9ej=_QLxC)vlZd0N#!WtL)KYPbhfaGNi4`x>98iy7^MpH#WZ{l-yQ`8#b+NF)+wP|y z$LU?mZJ80BmZBntY{SfgLhQlKXZXYL^~p)mdbDlHQxP4wq9iD8XW+W!+3B0U&+BSUp zNDpXg+yqb_pzjcHPEy}N0+dLOBP+cXa3beQS=`U?KYi0yj{(Q# zm)_-M3nB3CXa)6OU{7RdH!602OBF%u?;SFllyw+RQq|evuo+C)hX$)U{N84%>a4e1 zJpqL@4P{#Vd#2tXF%J9Wr2Jdye{|H*4wl_x0BP!#^3RbmOyYKe+?sVF+j;+eyx=Hv z`~OjO<$+M<|Nj}|C?(mnZAzqrQPHam#79r*3hL+BVyONVIS02mov*zNkPE+rO)PaF5D2xkld6 z@VEVEKgs(LjrR=#W}LfvKnKlZ?!+y;#f5@VSCZ_t+hN^*fBDY9P0~H=@1#Ai!*@x@ zFmLMhOhvx}oRJ>|%sx`;!}RpsJI}#4)c~#_&Ss?H+s!@pL`A}iR%GQ<@VDu`dBk2Y zGuyo_fc-V0EdvXICHwYLIC%4mA4JH?8Lo8_0{{>3jWw^UKcffgH4}w+KJLN$%@jZE z-yfC*lAsUI%x%Uqf$n{|gT6_sE57kn@j|bdJQO$r^;fTv(3GR}rP}a#ZLRGttPVCh zrSl(mfyFI#P`bd`nx(YE-xIi2_?bpPrmp@Iv!%zmv`f5U0OMl;q2~m=_|~-46AuK7 zHe=2Nq#`X|`jdAEwX1|tjbl`q%c)hGSY|=@;>1W?`La#@&nIxe2j&oGJ?9{q1>h(L zAjU5p@E>~1JOfIOUigtgRKilD5(0hT#y=2SNqxrvN-%nV0H}=s;mkn}o#9eg=;y3p z@jDJ_J#j?YNnD7vJOnH_zD-unNogN=BwmB*-foUVCZT0#%6>VA87SeBDa@k8BohHE z!CVC_P={*$g+_P<-+XdKtGN&kM*M4feF+cFq5P8G1$c15v{l$*M5??~p{DO6jh=&q zQmUxW3>yBcHvgf^x8HdgC^+8%KI8aLQt)*)nV_jay;b0JzQSZ9%LL@}olaT@57djH)x4{%WgQqXW26Gpy#6|V_%5TT`qZ9v#@3r zcKo|b)nC@@9f2C<6tlErC|VN!l?29m#>o{{e`iC@jHB7TXhOsQR0wMj_u9Fo^XHJm znae>rup06JE~8O~sZk)&tKx$eXd=15(-Uc$=ok5RD9%^F+^$$9WHstj*%SBzBvWE9 zr|go;c#KF->;#St8N1uK%CoN)8uzy++jE% z+o*JifFwv3b>UJ#Q7xcufoi8@FSGg@s(xpYtA3^#Q5DuUZ4j$vV;_;>dAy@d5J>nk zoKhQsksh*#g8amT8x{Y2fp>G^b%w%v(m`J-A%- z)9&|Jx51{JV2|Z)0LBGC5BRr8HcRQ=1{(~5+)*9>N2O@+T-6QbFIeDaAam0zeonC) za$r?i6QpM$#rY{Mh7{L-z$0M8K#z@wdPpK(AHCcaxCtA6;}SiL6Nt#w)_dmWbS1;; zJ?)AltI4JgaE`yc7tKCyjBy8@>jXp(uC=*g?!Z5n%0-WP8$9MrRVcJT`)9U~ywMDk z&g#TM!~cKbj;+|+{w3=yo{N8mTzr*ex0DX6Ne>9lv?Sl3o?C=N7 z8$=ZaK~IqZ7#v&%j#LrdjVTG6e~f_n3tu;?<9ZDZ6}aTHWa#CdIwu=)b)@nnNS^UM z&*bax?}S0ZQ#*~}vs5P3H6OYew^|>%w%iXVfq9;96xcHlJHHhjjd(};NCBWWC(DMm z*F0HwJoW)Y#Pj0xBy%nnH4SZ1`$Fg$m{pwYtK-k3!$z(YRLlX$^doR+HKDAK$DD%A zO%kNNLbU^2WYRgXMK1YcZ1w34n0SD(G!?;vcggCsVH+_v=2+9^~zP={h?5!@7jR9`@bX*s=Etg4PN z487g2ef#lBtEDR(61Z&K(6G_Yr+7|M!rCs4O=paIqcP%DdJEtxh|ZY4K$WrYv1me| z7%sOdu4Qi0`C80jWDXGQD#j3Nj(?N6N1HYI5l%4Fwfx~}YrTYXwR;Y62z#-+Kg+v4 zkhUCx{8D6Dz!kEZu`4Aj4|xR6t0`b?@_h(-t1wzg-r+5VAe@FcSUT+ugcN9q9=jmU z<);vk{6pSQVHFUC7Zm9dJ82FSQd-6DPFmtPSqLbX3T7;YE7M+-!O&cMTfh51PunKB zYH6m|M0h4E-TLoPWHNW8sSyrJe>V^0Bg>ApLtZR9bN4DtD%M|48Wn z5r3?So0yR zwzLHb4845M?Am7h*~>zpKST5w$Ou-|dB!QxY1g_-@wr&=Ljy55zL^IKKJ+?F`OCQ9cN6e4&u7|#$a9;_6>JUsq5)XJBV|{cx>ET z>z>E2%{Ge8m!G}g)!Os%fJhLICZzA)HdCQ7*X9RYYo*|i*=>K_wx=a4t}FDDk7Dz% z{kzSX8}ZKql{NAYV)o@299y+lkb!gYD?ypv#v6;CHKS>o#9! zYpT&HRn3BqscX#>H~NHG(=`ypDx=u0MQEik^br9?sHEnp_#2Cf%%Hs>&k^>(Av{tl z3?)oCljkkea8|ZifoO3JjV6J0#^6XO2&y2TV#nzbn|p4jbA}#{5iU}FHEOC2^H;9g zGM})Y;t^KInyof4;rcBS=AE$+NA$~j207cF%4Iv=S=GDw!=T4^$?(&PWGY<2UF`JU zBfaRK(y?kX?KKrUBJpTX{o1|Cx!A1+LEkIyrwExqi@^5fdef|Fq+6D;*b{vgDP!lo zyy2R$N;~%_ zbxvsehLPXJ<34^{7eHY;H2`f4S3j*rJd2S?`-lwueaPIXW0h-zSX+qlZq z_7y7%YV6sI9+xQjBnQ?y5+x2pH}jg|CN)yZkUpw!B#V_xg_;3 z<8^BewvVdOyKXkJx0OwYxI=G|4N)6tP7|ArYIKsLmhN@a)r9d~5nbz!(^HgJWLf~^ zc&53c=NQ&=m?e6tnd)`(ocE0dc>RQ_XE#q*%n}I@tmz#NLn#6p8mctWdk-7LdL$D9 z_V$Tt9-tYkj*NzXmV&~{>d59d#f-^+%0H~56(WR6bVh}0uHNfxBXQ;PiY%+<<^YMa zi}jI+N+6N(?BY8;5UP6q*6yC!4Y=H?_XC{T%G_YOCcO(B_bB}%q8LJ`U=8!>L zCkJmfddBo`7_D(vq1+l57qZ?=%)y4PHRNSayuyc`!iNnvkpSfJqd^!0agP4x=viaTv^bC1if?k=i$3t zHjR|!iTSZ7*!LLoT3pftFor+ExOP50^7%-1?G_v@nb6kF;1qO_wR@PJ<Yk9OPys_dQ#wTlz71|6f}f~Z!U{_pUf z6R8M8ck6T~k?{1$=ER1{g#{Oa6MG#~yYYM|JUN6*{)t>%7pmZJXl^Qtxepw+513MiC=>?cN_{03i> zE}NskC`*|=L|Gw;C7%~HOzK_kb5&uLK3a&k0V~=dkQohls@nCLqEhR~=JWqio3n=}nl+<@zxo*2 zc5GePQg@;GVE{~aiShy&OMuRN~yw{E`6)L{&$=pGlItG1{rtXi6wI$i0X zVSWBPYV=lGx&r}Lai9IR_?;Q7jK0+Q@So&WfrK*dA3EKd-4Wt4MH@7{Ti(oON+d!~ zBeTyvTemCME+IgeVFB|U?CBPZbZPkLg#Ts0H9EUc^4_-fpvT7Hg5+=ZT!XaTa~KJy z^$->2%Pidv!zq_wh=b_-%`!*z+Ykr^69I|3&hFH%QpUt~5!)slHqx}Z7r=-K4vz~n z$Hc`hTzwphj0&1s2`2Wm31kxKIub`yN!aTY<=dpuXjayCt0{2Y*{U zm%IX0PIF|CB;{Ala|ChRs8>oJK0yzHl)-IQ@pL1xPXj@`%2dX@$QZiA;B5y|;T^i6 zxeYx`*1Q=PWz(%MRcoH$)EF*C9(X^F8b$Cj8hlU)shTeQ&-r8QEM8d7vJC3NB(X-O zKiM<-4D{Im&~!E+e~KR~aCO&fi6&Ol1v6vRr)y8$HenSypyzL6Po67ujP>r;Ph+nO zq5UY=>0kk5X#$?JKLd`c&t*H!Xm_twTAfXxeXT4$j`0X|vH9zk2Px4=R(Aeo1>l+Z zMMeyd%Avl3efQ0Xks`e`kwI^zHs7jA3M%{Zngqs<@$XarC3H}mkGV7C&rszV_@!NSNHm`CWmZedNg_;71 ziQiTnS-pv^_?|ixd@(6z1m-~Rn+yX&u6cbb0C5F2ugIP+F1UKYjtRIP=an=Z^jP?# zx!H9|Y-)v~FB4H#M;`F5D_m*Hcw%f%c$pKhl}4%Cwi0R$ z#d4w6;dZTCLBf%hLUbXnqNkTl*s45eriNVXy5gjpDhc zQIir;K=+qh%tH2oQ$Drm?5DFo$K^DbiJT_Zbv3g6FEU(<@ClhInaFymdcncz46_-~ z(4nj67K_#u|7FrjKdcpRFMOg$fjPW&PB4Y9VPR~aS!+MBN=a}>_dREhOLdB7YFgjQ z8--BJ4~;(_n`hHqz#zs76joq^CuZrF8&i{F9onPU{}#Hu>gn66+B+5R-ZW`j?RVBK z`4a4oSeC-d-=4+aAz1br6Nv9IVjc>Di8kBdaJ&`GOwKg?C>;pBFfNJ4XKXO?esM_0 zga)V$;j$=co`55DEZ3|hLpdL-@y?oV8k@daZLZ$Xkrl2Spu9S}koDJg*zUus`G81u zNmxox&lsQd-8MZ_LhaCz!5ZGh+c1bXW!SxyA{kf~Jy|GfREujgBh;g_RCF-9daFgG zEc_3q)Mxe>tvQ7FD)#hqtHD@%X_>}K*ijjb0+77Q-WSC_eUm?W!uoiQB8j)Be1>F$ zh7k0|-XinW<$XrES7;9M|#r2uZ?%;i9m{ z!BwK{*G>=VQf1_pOqdZp;WJ~_-z)S&ifrkOCI;KhZ5h@}?G7cL)3uSem;F8@nL(JP zJ5%EBGEAY8=S4zS<-(5CY8Io&f;Y^8GWG)=ho5CKJc!P}-g~_I!UthjnP%LcA3@9{ zLtj26d?y)wofohQcl?Peo9-O_V4bfrH2p;+j*g!61242RAD?QC>wmSP)rGwe#NiF@ z>_gE;w8_tsiBT?>oxx;xPfc@%W|5B`Z^&)$M=XHLL^5&I#oBgdMLp5X^kPmwX=Zws z+Wc)6%W$=(=I{eoc$kMJN6kQm+PHX8>iVr8Ldh2@$xBlSsDb`EB=YXr04wt&P^IAr zMZH{p)+ET9x^Zs~2t;h)-fK?(|G1pr>ZYV zSs6Q)N9_@vL(?@-sD_a}DSK!J6W+Vmvu64n0dmsWml0zNF)u>LA(l(NK>2W(HM&q> z5p8b$A<`OKgBJ9hFJG}fBD`$!=Eab|%c1bxX5rPCXC7HS*{vIZK6Ruik;`HRi8DwR z)7&^KBa*qd-9aiOl)I8Zh5i%m^-;2{ULbeh|fi?9D>>`>bdb-~2Q2k|KX zg>^nVTKriu8QeQLi?Tet4{{23r(pMe6d+U3(6o|NPrU8g)zUF5(wqohUq*KL$DHVg z&B5jqTW|>Auel)d^)o;BrnsA2mdGoecMcmo4Kpo|>N`Nw3l(Z3*1N8^{HFnO?RD0I zR{K~*hh^B@{*2~1@HA2y=h}qNayvM8hMxl3Rw&p2=>P3p*YyS89YJ+~5v- z*=3fvG%MsV^qSI@RZiLTk7F&q`4LED7ysHyC~tXM;1UltMH>%{&c~!{lsb`a4O~3& zVklac?bwq~Kepmec&FY~3qD4P%@d_BC;=DCT%E&I2WL_2IJs;Jy@XGKJF6keD zfJQvYjtA)46(;I zO48j!Xppe$`z_IPI zIEkbNkGl@!J$y-nYV{iyaS@9I^%43}!YoD$^&PC=}s= z9xJ{V&O5O|>1ei9dpSReWfh2=N5}X5r}kB+BX!BS-=6*17L66$*gG?6q6gb4`;I|R zhZtwn;3Eq&@Y~9)d#>l~#=2LSJTi&-9(^derJ+%Lu=3^N)JVM7Az!_? zkv!KV9NM`!{8j0*#%`b8-nLcg+iNG}y+F-?8d@Q_)?nMf%)V=5ldEZ$akEIrjX_$& z*OaRTg-V&nV8k@vG)Ebo z>^ss5%DvWF9?aafy66Ce0>FB&6~`j8+4wIH;cZ9R!;lPXhUAI(7a~^Vq^QsJqAuxY zE$CPB^+ZbMipOcFKbK44f_W#gpdT&odkP{RHrrkVj3G+4&c=-~QhuyxnREKHBUjdJ z+y-Mqz_Ft^0Wj2S?rrwYU#*_cy#b# zdQH^sINxPf6|2RDQFqAX7yFX=MSn0)I#a}|yOM3em3d!y(WQ0`R!#bHGq4t~q? z$L(nmwDd>iRDKFB9~VmxA;#LbV*zbAzx3SS2kHpq#@$#enk<%&tU{Og^OIXO$LRT_ z-iW8lhgVQHI_GedH{1BL^7H2IRjva~iBnMLEK}AJahm=e7DyCGHmV&kb`*&Ii=KzG zK1+Q2>;;T0j2nQ;s&hhYgy=H}FK4rCKAL;gCRkqxNFci9kq$ARiD+ zpn%U|uM%#~UMpTS8Y-AblOPeOS(qarh_75TkemSzj+a~(95jD3uW0Bi{dcP9w~jM_ zsO{Q$&V1(lHEA`mnJLnUve(*wQ26sv)lAZsc;&kZ9Oag4_~(H93?6rdR#?`f@NgBy z4>aRE76M7bAcp3L^*kbrcE1Y^3~~|rYJ*vOOX9`;1hf*Wu8WtKHx{l}#Y2VA)3v&j zNb7dwLE#OBWDu)#&nKfb4osbS>nVlYcLfc>?z3317|}*dCQzNg zVYZlW1+Rlj6wQBNSo2KMM(t?NE4_ss+CcD%C?Y*X6COvn`;m*Yo*Zk|^?vW|ytJko zJmE5SA&6jLb^Hm?pVvch6c(1)Zlr=Z(l8{^mjk#h@XO<#RMBbHJ$Hq(P$aM4aP0Mo zQ9a+$-76(ggpG2Yy^U1o}sjW*0Sm%DI-Ycxat}85d^*utJO)t{b-f`*>6LvmD z&Ol;o{e}1?svRk@tUtdI8mgU>D##Jin$j`P<(=m9I`Pu`6n(GfZ@Y zfS`dowLjaXn>{~~2dQL&(nzWPlwlATFQ?ql3_GvF$f}Nj3rJqwh2O<_r z+N6u~rn5V=n$s0tA(dOSsl= zBM<}OuI+YX778#cAui#xk_dqJ?*gDJRJyA66;`8`^}_^*5!Hj=?KqVzzuPrix4E+q zOCsHbHj86j(Y;pRsrd*r{9oQpR_7@;LG4WFm0;{$!Nv%F`r;3cz9M8*n!xrMTIOX1 zE;;>Z3q`oNWo?m#Op2o|0Q2|*!&Nx#{q=q?!;3CWlGr(9T(~K@;;@bDx>ykT7XD%I zo+)^XIEAe`3oskY<>rB9I^>Ft6P1c>Fb)3I=K|szP{?TDnF-mhc;$8Xc3_0T7q%sHx=ZJ$!fQUb{=UheDky#Z*KR3cww}CFpzh$U0uypqlOF+s-tR+=H3?E# z!Zf#ABTILX(m?$Nk$BfVVxyjEY);$dlE&-Z(Mqd{HBw|H)9bE*(t!0eby1t@N2JGK zGI6g{W47kkJ(y<5UHS7l+TKBEEw_l_^?JlzOd>U(wjt@_6{lu#HDl z{TsAWmeHMa;&_h0=GiqowS&~n%3Tf-}}hne?P`HPt4 zgAj2;5)1Un9iW{J&1P8DHJ?0~Jl*WVwQSN&8(U1LT=OwTy|SJc#hPJ~h^(O2&tefr z7A4*Te4{(ye=1(Cb-r~eI`E4Fm!XOkT|F{I{Q$h)2OB3Pi{Hz~ZsF1^EupTZIyl1Q z@adc~V?;WWf%;0XzU0D;a&h71i_D}D~qU)*MP6YY{(yN@;Lh6O75w1v7?#>1io?(~> zlUFMsOQemIWIPW=BCD9BxaK`z5#e_8%s}UhCw9RNph4RDM<;)`FklIM3O4p|pl`S+ z8_!`V+|AS6`XI08hFo4Y-^S%3A!O*t#=wVJm!5KnZfDmMJB~PSsCfi=;sb~$ac8Vk zCRQ6g7GqzOSW5cb<7w(pvG0sl#D?#TO=dE5U*?YA&eDC!$+F6FkzBz-*Qi~^E%YBU z?t-Lnm1*ruk^xX%>v}u4WBXqLj^RChLi_sE2j0>-Y9ZIAX~;Xa9uzO~8^gloHc`;F zM#Co^Z161proHSQD}BO_6De|OkTLWujllt*;uPc6Xm7YH!Usoepa$!q{dy=UaQO3K z#`spP5>WZ!5~R+(>=^-I^tc1m&}Ph@y3EQH%VUkz+L2y9@1geGVvI+)2t{T^jX*tI zgNQs^f*3l`Gy!J0QE{yUIS$5;wEhB0kOp!`fCBcYuq4>aO>Hk)JrExxJ#!tY(!og;Iwr5?-2) z3YJ-iUNOh$o|iMB`5F3R<8jH1U-#bskziR7hwtPrxO-s!V?H&lDIY(>HPYZcOwETO<^?YD(%Nn6WGf^ArKjY)_%G~h=%tG4cy4GS(n9d;G! zu<(Pd4F{Expb^WxR{|0}FBfu#D(F&@nwRWruU>=|S+d3+vaA@n6Swa~y>Q{~Zqn7J z(^hVrWL6m)lnICGp_$Q)k3gU8heEX|Y!LL|CX)PJB4 zy{7(=%}cWPLzS*C(2>G`UIwy%+s_V&K>n|$vd{4R*IZ1R*iW)uBe zMJm12O=y#TgSm7=hZvz8k^1*mk)p}MdtwU(qBC@mkP&Y692?&wOxn-_U)2w18=}F` z>oQUsu28S9(4>LZ4(%$P*pTJnPWI80g-;Vo?Oul?$18d3>D}m z>_u6uRQ?5UBGZC-Q{QEZYiIVi^BnMiRh2N8tW1w%DqCbg4_!GGWO;XFWD^`*dk*~z zU`ORsj)Su#ThC#@*WTvw`U?I)Lo1@;FCiJe#$W{JDM;Oos~vl-Ql)MA=L$$l2JW?_`+LiS)1Iv7X4aV>fa`n|qC5@B@jvilyi>xP4iDO*Ou)BpwQk zI`4W%hWYT~wv$MY0JrP0!Ce=t@yrsT^bhch-~OswI-t$6lR_svZ8*?*0MAl?8<7XX z_GBzkvm*#m9MY>SSR;^;l-4jMQA*evH1L}@io-2BGyXDJZ#cZutE`yMq{LY=hV1r<3tgxvBAg($hy7N4#b5HYDeseN=?sK)<_wqkE=^U?C6nb z4*CLWBQZDdofvoFilFV*5i}{&WyzA&4oiQ(s{Dpw9=|p}<(kI~ywP2s3YF@T$GbS9 zchzLs1U`M8Lhz;rE`g6&j*);_5~Z-Pu;4kg0z%!YbfAuF!Tt4XCYr#vrbnez{3=_y zb*U-LEnA1FO%xB&IAtupld!44BRbSv{TwK&Bane$+Zu<37Yx-7-fjOcr-JeA9AXh0 z!6G;p2)iZzVQ?C;&I4K{t|7`qCTr|cuStCU&nt(mSo3ooVK}T8MWJ8YrpsGXIFf52 zMp(~)ynHN0dL)fr4CB%CLGp}1%4i?DZe^yGMP#fMdxHtR&!9(; zoir4$Y>|KLx3}1fuuwz~_|Ph*Ek;H@zy}%vE0UpPPMOszJ70wY?)PlbqpwpsD5mDgXq!Tbc#K#iS;@^KczXFhH%JzReV=-9R`jPGT}Ivxh#03x}JEqTyys2On_B(LQ@Q30#m zB}_8Nr~GmHk_2w}b#kZ-I$#QQXeLCk+dlT33-2fuY2lAFP0Vq26VUuAD(0+eiAi7{ zKFgs94&kUsrY%QiaD`Dx6wJn%6|91nwg(@}EOObTweJI@AM!i4YeyNGqrXNZE zHgzA_X3ZHYaiib-Yirds$0~{4ECx5urB$0?it39i&3KK7YT{E6^O0)`)zU~Q3Vv4a zTkt-WW|xG#Q!8Moho1l*P|7m7NPTljrq~Fc=x>l}O zl4a|S1sL|yau^fga;fG2)c$u%YOcDdL-~L$!|N$!a29r%u_xJH6-;@m{&erRX8u8; z3@U1?-3l=IS>zxntt>2Y3Pb!;@zQ2xtUn!0Gf&I$DoP;{`qkM|F@QP#u|3vL7_wmn89%FP#dnn|jFSl2WzMQ~V3sT$sda3Uv#R4c7uUjP z3&we4RAS9Ea(V#@_lGX2vkUuZ4ssO$kWM3|)`wNq*mXddpabZU9Jv-nF95kKLT&1q z3P~|SL?7oB+m50W3euA%#M7sB1n?f>Vb(>_4CP4p1f?5! zax^OUR!K;4Go9c}t5~pSFB;W==(B;m>|=UsxM+nJfF)8JsQUX8IIS!5Hhnn?nF>_# zsnzXfEPj%$xs6c{dlU2rZo+yb4N7RMgitk!TuJj1xZGU`S0kJ+h0GZtPT_K$fR~zm zi)U~wHoDRK*|bLL{8A=JZeE2%xOPeJIWW`R@fbGA&&jU&h`G*orDYh}6S*7_E;xUR*95LgG39*!YN;~(&-&{=&R zyPKqW9XT>+KGAJXK8(yVP<)(xO6M3kII z=c*0F3H+NwDGj$+QIIRo&q)YgVzVWS@QoYWc}x5}-O}2g2gZ7Opn$gO=o^4>VUTu0 zztFso6x&Z;y6~m3!?Gxre+jGSV~MR%qUN|0Npyr&$tIqK>TCV&oltc3i)W_rv5Nh+ z9&zjaEbliHA*a_=uteWTt-#m@!sgA2pF8YquSoUsOYiVpTN)>;NMbY8jESYhq79^? zsuFH}3c2w1?C8I*qwgjzs^|V5eLVqZH)MFidgcPP=NzJyrv7~OJWNfIf;ykViGYIi zf^pY;N`;~P#Ega`i#Yz+UDvUp8M6S?y#?lL>bUz8pEJqR?G7T*|!DVwk# zLH{2T5|UhzI6=Dg{3>gn2-%51_RZWSymrEzj6=&jZ>OkN+b zoo79@oEfHGxwsbMPv0I6i|?63yw7>=+isvT{-PEJw!l;BPm!i`I@m{uqA$gx^xgh! zU<*F~UOh8CBjwG8idZFT&4m_L2&cp!8jAfqN2y8w1ZG3CMC*Q^ieg-qwu1Kfw>CD` z%aNSnp*^1vH`WU-(BrNu{|<8_16ds>xn4}7jd;$UZgmpyLfYB7A4NQ!74F8TLJ_8y zUbSHVG*is2AAOxM?_i%F{s4zoAs5TH;3qYU98}xIyn=T$CNu*-0BbaQI@4vnKY+K4 z%(!?Api$$n4PjRrE@bI?)m`m#Wre#{tThJpzZAw)uoh>JVJ<`_m^<@j%Yn}&__|@I zV4gb`$||{Z$wgUdq@JUjay?$ z()b3ncV+yS={eZl49HEF42IBG*c4aIV|CArSvbTVgpid(eBON$?>*KMZIS_S1Mp8F zNwii;7XTSBXWp3#Q%f^K>)nyaS7SyvQ?{ai@{78t@pwX1>0pw%#l68E#PnO8udcR^ zi~a>YX>h*@j^@Zv@%n<@(HTAN8R3mO^@aE3QrwzrI4BO*WnS{JkdTS0TRY+0E!*cm z3_i}@D}E4~Im!5!emI&Z_)ma2V9b{c0(@YQqeXY?jrcY%=OM|(z5+-N{bbe4OJTU{ zYznyT;4Zpd({_RLYr09tV+%`qzUekn`)1FwzgU}`7mn|yUcy+Fg+w%vyPlWxwMaCBX7YR98lm#oXsEidsLp{;@|dO}^K`!_hZF8UqJQax0*Ecj;cgt5}P zY-z>{^_V)TAKu8Cg3mX8H6twb+4-uOX<>97OzR7wuGo>YvuH6ZT^a}*j9q3X(KmKa z8Xfv$1=R-}O!xA@^2YJtA3ge^AuEF^APNZuo0Q1n*BSB=Z7>VW=<oKI-{iSpuyq5xaFXaZ47~LIUxR1w!6)rf-Sembx;wF3`?Mz30Uq**964fH zWT5vRq{*KoT$4|_GE06GfR^cYL`U)!XrsL&mXp;vdul~kSaNrxa6CfpP|R*1+N2?l zq-9bP{2~vUfhUezv@Sij4Um-y-wttOZKB0-AsdGarw=dst`qL?vtHQ4mMgF(fSBC$ zfjbLXyn4jS1Zg(9+R(U7gt$W-H3j|D>G4LfZ0re8tja=?n7}lT%PtxHq0;*%F;#)l z6nWRrfv+NJ<*UPh6L9XS3m)C~W+|pX6z_#v#G(WZLB+=l#xk5r!?&X$VI16$-ergu z#x57F@iSmroibF@QWEAdLiQ2QAnE0AAH2JLJZ4TmOh4VDj1`dly55SRHSAAt8PCCB zXij8LsHeWE1=awNUKG&qSo%sl0|AAv(H-z;gUpBgAdjHULf|LVHgUEFH8d7Cp*+$b z=blHTZG_Cda+eXVu@BOM{kWyUd-2d!#I+K z2bLk>-xSaF*gMlDee(o9%Ks>>744YJ89UD~x6`+$U1p8#AWX6vi6Mfdy@ek@1(X>f zS9ED!_i`Ij#ns5W*%xR=Ma+si$!Q>P&KJ+UV+A$SOiQ%S#1(DTy9Jlzho;j_eu$iL zW5a9>1g;37cSGG+g4HYVVvlRq@oEph4*xVW$}h@LT!?d^`i zC(Ovp0?#Z|g|TIPgcrf{_g{A)b074@?cYV>uB5Iqrm&H}W+6g*?giF8K1c@+>B`HJcVLYcMV8uwjDkiNW7CD&^DiMjJ67&Ft3Bcb36Z*ZUjmsoMcd$c%V z_L1cxmB-}g@_;WOvb!mhqluJ8mEweX+o3^V{16ya#Txa%trt?ZxpZ7@P}`Qh0E7c` z_V(#>dVX?I?kaynIDZOKW!|ba|64@6W4V8n5c~^v2uW&L?c>k89%aQ`gg?fp__pUT z!}Jrk$3s4ii-0IU(P6A+i-H3@ms|(w8>~M&_a22c{Y&!_cs4l2F~9FTlOuWQqCZQ`oj(eRH2E)>am3)5*0%wLY6^FR0GBZ|Ypn#@pLw~9Q@sLkQ_P>iicZmCo|jToK}qxr)|Ng@ zdbZ4KaBT6q{NEmWIN`HynPjr(NzH>$MfX+=gxFEJ8(_*PSADe|{}&GSU(LIfL$-B- zi39DKS4*a^D+dnOb9}FIq#+=c!mU^VlTH|mLaP63;7!$&jU=VQ$jwDJPTSd5pcl=S z#EC0YV)cyo#~CJZG*p?anD%lv&rWZcAz(H;pB^R=%K8sbV;2Em)7kPb zG5)$k07AOmXD=SXdidnI&Ze39-5hg!f$c_9#aofA{IKLl=};*_lTrzDfJh!Qu@t-e zhf9IfWyb}Qs1%j#U8b08v|>$^L+o7gmG07UOS^4dyj&Fu`_<(03j0`la&mgML_216 zRYG!<Ys@CnPABJ*IoAZ8vLv(Zd!0w&P&^OPOf|_eq4OWWtoz z({gC%`i*587^DBSZg2B$yqy;hguxiMxf;EA0a&7ir> zV4>pZ0pue=vGSzwLCLL5KMNd&;qw%XaSV5gte$R{OlD-y@G7!0N|6GEJv`CfjGri< zXj=Wwt7B!WWL0LY#@m7+P#(3`{y4ZB<}twQ5J@;co1ppAkG;IEsI-Iu_<@&VyG61p zR71qKr84bh&+yoDr^Ak)lseRAj2lp~!;OZjIqX=*j>Ku`D2phH`lAN~7qc{#g9)9j zM?Sybiu)k({kj|AYaJ8Mr1nC;v){YD=q`vB93Q8QMed2+_&C#obuTP= zWLm9AidXR2A;pcj)^#!CQT0JC2EVqa4?Z`&KPsn#utEK*Y3?o$rURwP_rE4JtJGGL z+_dam$ZY!~HP4>dHGUYI+Dph{+0$<3@ZR#B+H?fG=$t}|Ao!Qe$3|1mY4x1!xMF23V06c~A@;p7?{o7FNeHV}j>X;x92zkGQu z7xhPuOJ>HL0wQS{^;}9u6@RE|h|PZHB#e@zcg(zQoN4kboHs2Qwdq4@pD)skXya4p zsl(iFt2#)qL4yrxbqm<2LNb0I2Jl6}qSQUVxI|AOW$M$u$&|iSp1Wk5pfhn43J_`U z#xTNuPZBZEVcGimi9GP}Ig(<{M%Iu!cUH)*fCZ&L6lw`sC4|^cN7yS7vhhSjx>fUY zy`t^t;YuWP`fr0qt*KV=s{^!Pw)UsSY-Z(mKvYyOZ0<3{2D1lcKT_#0@S1HKO=}3P z(nS_Xnu+x?G_<>ucJVH1Zz31 zBAJsy(uAGrk-ZbvmPUdf&8@7grayD7Y0D{5W#N@gAD9L3Xb=m|iFk()t()0y+weD< z1hq|ot1vvFd@Ou)BPVhkF?x@qd65xOXnYRIZs{`TJ%RVtbHh<>0e!X}c(YkuSL&IT zi|hX4^HR@#;Ah)hpGAIbsGJ{sNM6zXaU~i|(@&t&zG)e7OG1{08QRPQ?m)~Ng&uSdb?%zjA_jh>H# z4LWLW`Zi_l&<=jY_gV2pAtm*^DT{?@dh%0Ybi1M+d+AT93Nhga1C-Zi^XieY%#=8z z)wbR^x@xkD#ao3QB;ziQYoF!a&E;5~fin6zSZ;yH7_>?_grl6hO5BKhcy!Kq9W>`M z5j5YX4nnPL;t*t{T78z91p5kmoHA(_8f2px32F*4%8~4X?`43rrAC!rLCO#K`xa|D7Z+5z+PYNZ`86X^C>LE(I$U{nO6q)JJsHhpTOe)epJR8hnq3b`|~ z-cp4t4p2UV>v9UuQ(}3i=8WxT{K@I&AC*fBg4~}}K$kpy+ZvRvkAE>KeOMB1MfKrS zXPAA+=8S@UA8#V{U_?5o27LylV5KsyeTR5+7;&D-fRQaMQX4kHJuJRfJ)zRuhrkDG zw##6xJ5DSV5;fjk#;rY`(6R_eSxVv>E7AZjyXRP@1XoCQcrLlX{6}j*8i*(?i2^aS z?}g>3l-q#w=W-^qkdhgB-{!<536|gP`8YtPJq`&^ZR%M>t8msKeOJ{S%@v5#bS zonRKVYHDlPVre!!J{_5l?4RGQoX4BUN{|TJ5)AL0#E%1BjzznO-s40T5}F;kg|H$Hvq8*M~j4i zFhu)_dK0ct_U>dX8kQMBd68J5+%rI4s(4T$(k*g=jIvi{tsG2)QA=e_4CC(KaIlx= zH35XRJ(ar|>rAMAoI$27$AR#crIpUYnkyzfy#>ULkJI4!{|=jqqegivoeRdc-8LM` zE=~4v0?S`(7+tAxV$q8=cRu&e^6h{d0K@Li7Uw1COjdCl^KiF(W`&{RR)wkic z4JMVN$M_KO!mjjrFgHb8-oSYRXPTh14a$IBoicpqjYdN$5o>ONSp$O|P>2e8O2D`4 z6*7d=JMvI+c-?J(3Gy{xs7S2YVR+o818NR_K}_Ux&_=w_XnIb`A)4H3ARpoy=D$^L zI|*qt`!+~Z!SR9y8-MOH#7~=bT;L9oQ8bN{HO%`~zJsh*Tu5A2KHFCXW;3l;i zUo(z*fIxQHUU*^8%z~H^Mt7+f(h2N&Ud@2P%%3sH)?@@Koqip(SN6Jdmots@_#UHn4tyHv;HYjftJ440_)$<#-anNZ_8sPDAo7BwTr`& zl#R0py5MwP^QxJ0q^np<$Qk-_2R9cgq&V*qn9JqRK}!U(dg*u2gW%bX7wlea&B6LK zz2N{xSDetIP~-40SQe~Ck5mM8Z|r75Aq0=CQEk_e@$|2 zsK0-O1}xfj_HN0Vp2E9&#})0;EWURRzaPU6E=v6?pAl4JrhV5mb6%Uyzls~vs3_H_ zuSikN~x2kj0-4~l>sV*wN%Ax8nWz9neh4eHt7EbZ)-z}iM>VAP`Mj)lh$hz3nFcyTB z1qrKw^&WmZ1Zspch;9m;irDyt66F3MT5*$t8`CEnvNfo>Mf8t4Gt@=b*X!Oehq5s* z?{h@P$6jV3KLeId;Ga+#OC~PyXS7~1aA;rRH+zm?3twf1OJ9+`yo#!0_Gy6&*tu>2 zjkE2)P@FW<05pAE^`ukD`5MRQx|%;>{Kf17!>Knwu&~i?N@C4;au8@mZivV_iOHr@ zWbKlZ9n0_w-*#gnvTqpHg+ELbj*7pq-i7?6XJBG?+oU^X zuEJq<7XN~(D%8W6uQjyiwRr(tS&6arqrD!vB>~_nA>V%ByM(+wI;kOGYr^v#WPYlD zM7IEWw|s8}1p*bYMx*12XS*M_#7)edtFtuKPAhm3kJeG(!K@mHE9l^E>W)Q`r8U3B?xy=zOQW{6(tQIFOuES zkzGz>dSP3J>+xeXvjIvcm{D7g=0OPkN%D-Z_~#1L0g0VNjlUSv_WqQCIs#zVLZA=< zzCk{EC-2UgqUkDJqdU<8;A|W_x@#zZmDc$8cyY5i;cLdX7I?Lzj9+=ds5ON@@z*{T z%wPu~*ZX%s?%c%)uJsbap4=3moEu9&2QRyXmY2$%y5`hWles#qKv``FqC)pIh(}X6 zKTrg*^_i<#il>t$<;wj@a>4d#@sqti#b3t^53cJqHzxU6ev1yUbbJdVG>up@022(C z$ELS~0JL;+rfYo2eV1^Z#@0jEAO~N*%NyFDWFU(Dsmp%o0&Bb_o8T~{bq@*jwT2Pt z&7|}e+?mD`5T!Ntpt|F#KNiAG(}p`oL%C>{jI;b3-+3H9Tk}iA{|+RcXKN-VO@nP9 zNEwZ(IKkx2{L;rn-<>!b!Xu!#cEhBM77mcRt)y`R<^kl6T@1t zG4^U!%}DOL0ol^F-4OlPiH;9({7X+FS(5WEmYDRon}}iE*$mlwZ19KZH=$i6>g!}x zE>aEGo@2s{+w1X1&KAEn`LbGxMC@q7B@>SmIMQkuF@kTUr?*rj7J#Prv@j2ET9#Gr zd52C`F@LB5=abklOm#O_^QTB?Fh@q}L!3t_Gh6#taX)te+Lod*V&U~u)Dl-Ztnn+#J}&N%>euW>VNv_?Lf~SH}u0U5{ztQ zAiLmqd@0LsZJM>F59aL#@38hngX44`s;xo)5AAkn%N0=Jacb7E&8an?9BoM)2X~0A z-Ms@)gXrzPM#MS}kc*IqE!r@P8T>XqhaLf4-mP;(sthlPt?`cF4eFASUGpjer-Fhc zVF89R>7vZ}>c+APLvcj0k*phr*WiIiiO(*|dJzgS3#c?p=aJ9AV$|SZMI}Ciy&GUv zNBI~M`0vrLX4xd*GU(IViBmAoO3*?71_+0fot@;uVXmn3;!AXreoY#0grYkkdTP7$ zQ-~bq=f>p3abkT7S=($_;w{@Vo*Tikxx+hg)*oyQT+K7|-z>v2Z~p2jF%b8h?P8d2 zEoHM8F_N*fi1u32&ALE8*mtb@DJ2F!37Jq?Zjxl^Fgg-1o7D60;b7Ji6F#fJ{N zq`)Ig4pG3t%gut!7X)%Nh10KA_TEWDHlS0Hmk*d*uQmhx#sen3^aaFd5^=-)O;}wi zu9$|wm<)*PfM~;hi?=W;&=_nU));#NsZ>54(!nwJK%P@Spt9SGzouXU{;$(xo3UXBvS(xNZ*UrZX`8QX z&tv}3a}3asGXh$|P!Mvn{d;s%C)5nj;Imt{uVEV%ED7bGld@F+^3AoD)JrNmYDbYZ zxtwjAuAKQafgUMzG?j+$6}J}ok?bs$!Yx3G@rp7d5c4Z|XBD_Hks!Ss1JN&DHdNOC zzvP)Ot*6kED-0iPfUcglm%duOjj1P2s{gNEoQXB8T+5EO7KGn`AZ7_S;%o-4Rz_>$ ze?4tq>*U5%F8l5nI8g)n4gM_7Q3kWd`^fZpLIdkAC=t0WJ2!RpD}10k??|LDa2kE=HignIw}$6vOgG?q4nC@tD7$(9)0NQIWGNDNa^)C?+1mSJ#j zw4hQH*;=$<3~6XExGhM9#9(YU35}(UA^Z0{$G!LS{{3Em+}y&vUeDKgp7S`5^El^u zp=wP^gDsRy_YOO4TCIXr!7O+Peg+jmWZ^ld1lKh0;8Dm@%Kc$5;>W+y`13Nb^sV&S z^#RMwWajsDDSHxJ+4V|Si9oYQ^HsPH0tMv70w4J`AtorX%%yfbWMYt2CV)De5iZF_fpu8wLh-+s4 zUn3F}nGoG)VF7Iok+HkQX;F5qeo;L8(7mk}HK}#>igJBMN<+D~%yK?;#DTQ`=vM#9 zeSjYqLGM+^{Q`kG2c){d)@~APdDvC=45ehz=RcM;rj%FD+zjpXB<*RRU6skH(8Vli zIwip3!f|VV6Xv&T?rH2`z7TSk!SH@IRFJyU75+Psj%0vzB3%=wx(vB_h~>BcouoCd zRS|;H0vVgxpEZIV{F>9ybL@3eNL&NhfV*LY{Zl`80NT{?G!X)Z^@Z%Qk3ia*&UW(% zyX9lcwf0bJAI<9v&2G-lAjOLn`CG*8P!&-u-8X7{P97S!%wnnaP_jVH8J}=V#Vmlx z)nfj@$53}*%pZp2|9P4+`?=|_KB@+;1?g(pX{35=a{>BMzFA15<)kq98bfU zhZl=~pFFmY2eykn+XePy0#dd>;9%EEsH3xnERJ@*v+epC3f?CNGzjO33l3fifVNjS zGL>MpuAjO^a83-Hc!UwcgSodwldfgl$rt`#AfHJ)cMm0;5qWPh{Wqk9N1!QcN}7KWa{` zNr?*DZr1X04MsDD!FcxwPbw%6du=yWYa;{oBIn=V`L zRBM~w9_wUTJa&-vYHBXJeIz_D=@WMInYDt7=&kVY`^*8J^gV}-hlt{TW>QZr#0u9M z-Zk2%RQt%;p{xdpk&YIIb2kolT5lSDRZf7()aqyoW|q4)^j%2rd)cvFP6d=+Q7{#< zBl8xa5d8nOi{n8}s4yXQu;ccqp_A*n6HCmFwPLAU;bcYI>pBAuMyIs|LE6;RAAus+ zGiVP|T2#6UVa;D5)b_DHsLF(w%vW-G;Z~xzRNLm#O6MHCm5N6$4qN_CZ|SeB3Kn~u z9(Y3fTJhTjz7m^$Tw79UG4ZpFmYa^%@}6R`JG-w}FOzHRF4*;??e0~-j}pocQVGvD zUk;J%I`iykg2v@Aef?iEHXiSEUaXE^J^r?X=djdrTi6NKLeLiy59;KeO`lou&!|JP zgJVL^309If{*`vgMd#t(WB2aRWo+dw4}G5U_nTRadDSr4jpFLBaEF#!s%6a}f}=TZ zZG%KxLXOf|?nh?VQv7}mYU4P^q&;Ry1Z5dq<<&~ z?}g$|UjMP7-+Ip$n0LV&zDr-=VdSJJw?(l-)PtVCb*jd6ma1x6J3doA5?5=hx)Z}Z z)SM#X?-rsqU>^r2u~!tEzD4Kf^@M~r1a2jUJsL5twdHhdXvKbuJRhDlDtOHAX5Mr4 zvaa0{bdRJ+IjJ^mb{ysc z969o$%4_ad@t43<&S04Fidw#LE}xfd;_|pGNs?VXE?*^ym3u+k>fL)=d!1D2#JSl= z{om_!*~3%q4g#aMLr(Bf3`aTf@aR1&F4(*iN*B( zjH}yi`SZim=sl@rleI+S(3|TNzv}2wvq4A{L+Q3J)jrL=xZsq>$-R{Kk7C}Ew?oBI zO*~4>AR{I^IhM1H_HE0RFTN9%3dPeqVlR{}mj5}^Ih)*n5?-jSo0FWlx$x)(42!B} z&gin}-!q%UoGCjyGLOk-;^?0yPEk`4`F^GOIpI5VwB&x}(@F~0=ycbvQI>;pCA?St zjNBUVLAvP3qbGkJU*Te`PqYkncT5ntX!Y8$GP6XIER}9vjjkHDb!Z?cRy`AG#NIb_ z$;?b!9@L@vDQZ8dK!SVQj5NEyv=?d39qddC%MSa#q~S=}oFU;@VcU5s_sz&yW_PhF zeJ(5QePZ=yj}|{-;RWX8q+j@l_hKTLOm%SN&AH>O&AKU}p|HgsJgBvC5KG7Vc3#45 zJUiE$*%8sV;18g(!M=q9C+V++3!DB@in3_3q!g3@x@_0DL4YrFQ zkg}7Q2>Q?WSGx__A$D=K3hJ3zo@wwZ2264)*S2%e%I}SI1=}um)ey5#TJb~-3$XN& zq;a1;dv=o?h};`Sx*n=o|I;QZv%e>@$F*bi#T5bkj)}a9k+F(aPOdbWz4(201x>La z+qjl*R5m+N>niKzPvi{Vsv$%|uQ2V<`@$1S`CDs(auYe5is2*cHn7Sn6S~~EV|i%B zn@xC``&wCt*cxPRb1hVE1J6x=c@(|JF_q#clGmGJV`*{ot2LTGrE$_aJxU||j>iT{ z`spnc-K0#X;DRetkGlMTrM7M6y{&waO65J`9bS*Bk&S z9p@5~is=fz%9ZkYVXG20O-`-#Ssaj&%iR??({IE8I3#zxeceD)LivUE-t3qP!2UZ- zJvYvj`|1sLv*_F%S(U@3R`3u543#Qqo&8knczngB#5hY-Dhc~lsZ-v4wxLmk5sqts zXS?%zg8R!mO`NA%>=P<2{zr}+xk_GEm0dVF#|jMwv_IMHkma_-2_`ZM z!Sf_~+ZS3MT6$GI%Lxu1*1z}e(CtdpUu9@Kt3L`+saBU zw%c%V5*^2g9G(j8tAf*EA=&cORBDq6j9CHHT`*oFl>zXq#ZTiDhglq=6! z4DcLmHH90=?_pS-NUVmhm}bSQ5BWK7*G>I}t)%UMt-MtJ>9gU&Nf~9!TKM{BNqlc+ z9Go-S&Y&ni>6YQXwf4YK+8))ebyDi{-1VZ`?YFkoGN~H)=-U+jTTEuJW5Nt|TDkJ2 z>*eT+9o1!b38yfu&W(9(oSf{6f2b$u$-yetoa4ZK=S7$D82+}AHy7M{ zt)C;A$P4<5*}xw{5wL2rSt}D~>0yAM{Kz*pfWh4shn8N>1~>&)s2GxAj%%iGymG&T z8#McnU*)SuBcd%gINT^}X+rv8-p$juo76Tc8_88p$@YC3sbVAVN3GnCax zt>u?|%!QDK-XHy&stESs;E^Ni9ZN$)2(GE)s+zryf@xC&;`&xJqbfGmK1~`UaSCc* z4#l4_S{0WwEIgFO=OQapx@e2yVcmG#)8@renYC-ViscaA4uxFUtfr$fM`3)Z6qLk6 z47(Y!m!yNG_mHK>HrC|p62%4{#T1jv7+KvzOkI01{o_tzfll8hFlDur)7%%UrDo4e z9tCq{T5GbOuX zJ0@*|qv6HP<$QAn2@7GTECpMixVZB0FYpuPgC0NfJHGRedi%9?0MBeX`#WMILVuZm z6;gy@pToj_HhyPa?pSVlXlv@{nFR`V4OR=V!3=&@5WA6oZpEd)-<i23Vj@0he3yG}Z#FBEf!S>H!GcdDr!U*RPX zz0}>7YheM}@p+S;se8MdDITI7`bOVl*FNejcb`&D{SVow?5#DQeDMayYo@l+b`|87 zy8#ATcG?Tvev$@HhU}<-02V3OU(Y44hoyii8xbF$teL*u+Q?pqKkP<)eV~7s%X|w& zG(F1r^J+fe^5svJMu?@d1IQ!n2^NjNo$7Eg44{rN#gK?Fk&Q!x3fNN0!F4tH1AQ<0 z&(QpT7}xTqTYZ;djQi#)FiCN)JNJ9wtBE$>BUmdK{FCvV{>|Xd*VL7N1&gmidwd`f zcfm`sC_8s;ib36kDBcA}+Xeo0y(85n#q>$-V}-Kp+6Si7C;iwxEIt3~acbGrGncaF zU^kW`7PW_C%e^V>&YRUo_QDM)n3ZtSzqHu6k<46leVLdWR-BmGFD-wgWuUm5LhxE@ z>3Su)Xz1}jEBT2Qml>A&_>eYpQ%%0jLzm_=tc2n^=|;u$>`F1^#ZD2g7JL$|bGAIx zr=Bs_R$0rZhMkzagWj3&q7~<9T(=uvJ9s86nLx*h_XegW6TWRAtx~W#8rHF@m3N49 z1Gum7s_!KaMHqA*EAy;Hum|Zp*X8sis~$oEDOUfxdY8Asd`#{BgZ}RljwS|Z+Blpn z6Z+M}0FOnoZP7SoA>eY6*nBx!YWo=GPlCB$!3Pq%uWSdS!f5(ywO1DHOi}$+ZCqiL zQagPnaB5enX8scTHxnYqZ|CV6!A%D?y|Z_U-5KaU=^@h9t-E^SjL-CR^~mgSBGf%& zLdAW~PKFvGC!cl~W*^v3F_Jpch@L6T9!H^}rlv`#KhIF)rdMp93^@~2_X2m#XO_ilYM_bhMJMR#717KNgxH4*w)&NK*GonlJ?+d&rhu=z^g|+Qa zxT7jo=lr~7;G(K&rJt#ZYsYL_W*k&1bsZZxyRuKJpe)2tS@AUvGbA&!?GCaueK_|w zUa=S;hP@c<6ysTtK4nO5+=6FuoT$|AgLKiJ`u)6$Hd|NkFsI3&qo&a3N1JKb-G~Bl z*|E@{ZV#efFN3YuKJm|{n)1X_2#<0 zpVn4^`yT)q0Br{f`q?h3`9(v?QqH(K5>KQWknh$0@9flG3kx?d)-h!kWVl8~$4XsA zX`W1h{24CGA6ZHv9INE!q<@gxRMdU2->fC!k_0bC0wY~GzT*3zGYu5uRN)rS-x^#r z{plsjr^>HU_4c$aM|fYtQrGV0y318(l+T`h#nWE2Oa2C*e}T{_ia%Lnyp6W&@X|N- z-&zQ7g&EZW8&yyhdFXKG1HW=CF}Luvi7D1>&QWMllZqgznp#Ch_?dKO=ylWy5?L~M z4^h!uVVi9mM!ydq2ZOSd`u%NGzoD&IJU3{Sb*%!tT})dG{(0p1%*&HM5qioH5qC^* zw^AhUe&tTea_0ECuf#r0%p7|~#hULM#&*{j2#h|^ zo|C~YvQH$EEYi{xmxlC>8q>|+fhW}ZjiDR5kMs9;{)2ve?$Bnbh_3G-dC`8n@ydyx zZY%Fiqos`piwm}(VGLQtW*BI4KP_H}MyD&k^B0Q@Vq+Ro5$RlFjF#{Qw$_~AmXqt? z%H9UV*b?2o!(s>hRMste_2Wxk73$xObG~X_WS1|3<+wh&;@+) zn2~1chScFyJoLG^s>za1F@NjI;CzA=+ZI>;EUeL*p(_3Voj{Vg(y#vBu}p)DXM(7V zc$_*EIsrWK*SFRkP+Fy!2NYgz>mj!X0jXC#gASQl zqCQm+p;$9P1i_xlWf}d?Afrs~GlVSg$4J&Bp&}FW>Q*;@%I_JB>>2bkA29vQvobnv zGSolHqc)^G?_u5a2`qD_&U~lt*iQXywzIDnqpmtz9(ZW!n=f&KnU|rOMMpab+C)7c z1GK&|-+qI5olNi9(8``&1PeQ+H95ksf0K=A~)dprEGq zyd|KLd(l{kS=}7Rd?p@UXsDFJ%$|FQm1P=C_e3cBZ`ym)!s08G!F?Mlb@nPv*lt*I zVqv`zvsAWjwSpP{mU~NQWi3zy`{4}d-dS&%w`&slY`NvL2w+3C$$AR(~Y%KTLvom>%$VBIHTiJxdI1V)APKn(%U0RYY^5I&p&z>vy zH!fFlqvUn!<>E)B+>)l<+s(`UR}%gT|+=k8(lc^%HceSrtnJXj8s)D$+EVF z;9rsy?4>L2u71%u=M4c#^QTelOX8BYa*Kg2hdrMVQ=`8^EE>UE?^^&zX6xr(KyeHmkUE~vU-bG{RR5V)bp)#h&xd}=5n6H7>vPJkY{yW|AQAwiqY^dU+u0m2UwlO3ZQ(09J3f`wWs7%d$`zY>$O+WL zUV?X;b$j_yyLf}##FmC5VyEWGo?0#uKu7stKOr1|qCT-EsxK(8^u?nxv{h*zX0{r( zFk?P4VUCZlkr`k__zv4XvsAO~n1Mp&&zc_91BE{?{!1aMr^7rZ*-yLoFvZ`T(aw&I zr%^uPj<>G<&=j~N#Wa1JYO96P5`$IvK@r_r@>u)Nk+Hds%3A3NQo(dL*gQliavgK^ zGW13V^KyS)$>BT_NzX~@Xg-lbx6^L#x}Q@~%}RYgI;_x~DZt!`-c`MyWE%uELB;I- z7fg<>oO(3e$t#T7mlg-fLM9Rgz0h9t>S|@OsZ8iQA>LFe3+RN5*U-jvLD5CDuC@;= zNQf>7J1Q+Ff<+=vLA?RBz=_WCuXO-+vGfmL3}dzC&Ia}-0>Yrp6l-2|ZAE6`JbOsg5eB>7}vsDQPD~0$8%wAC}wu`Fp`(h`;Hh zrH1R!cs&zYS{4>pz9i>3Yy01ux*feI!Cf}d)yRN)4d7NXMI_Eh<$m4@e{-=*&$rfC zSb%V#+}AcB1HTBlQ7=8<}TeAssCd$<9vl{REg zn9oJ2STWiwNcO#0m$)R;X0N_U3;cdu>z?tcexnmat(MUlhHXZ-1hw;M*xXUs!%a?f z1M=>)G+tT2TW>vLSrzLzFlTFR8+>O;Ek~zfz`kFdQ+~WeQ%P%XeB>%neG+BD{7Ez&K0O;Hafu8GdF5Q*dH{4?q7B}uBYvfWmJ6EO>Qlb*QRZXqTwU}t&+A$J} zZ+i!<$yjPrp^$8&eds693}a6=ylDGF<0ZUra8#9(U2gi~TuXy0t`4Sg<-QzD=P92$ z+&CT^y+^rnu6D%QX@S?}`+e5(@|>Al*(4JpVYl4PX>m)f{;gxCS$MWHR^k8yKDXNpK5* z85r=GC1)qJt3Cx9qNT|%-Zo=+OHkJ6c&W;hY*!AyF5}bqaq4V8t3ojKYm;zLWOd^d zxTkwi1C5V&Upp3s$VJ3Y6xIlgy&*@#Fbn&%Fhtp^Xhm9gkO3ewHgWvgKRQxphWq?X zv^rb*3D&UllW+!a!T-#jnRpW)qp}!dNOB}}!hB5Hfz{nI!5A0U3kKZnEuS4iq+MvZ zBKr<2^mOGn#SD7^B5IH2qpGl9Ty3?Q$1UyAR;+G0B24pofvm|(J@hhFN>S7DBrC3 zD-~hLQ{bbTR}st7&`wN{ttXTQWlbUa6f5A=f0GqyYn!|j;3Rqypi^X;h;sh@U*8T} ztgCgM>D5=MJvZc_rskMn;vCxFHo8p|BdK#W$@PXSrqXk-0_D6kuTUiYqpHV@&|8Xn z)g%OJE6R#`#6Fwske9#vfjyN*oi5oT>{u*J$B!V~2v@vs=T(g)rfFq@-zv`K`x_T~ zb__4m-pg6}-Dn^F_fsxmg-uEdPkEvV98OqnuIp6uI8mWIqyd*a8FEzCxM_+rW5V1e z79ThVb>$tpQoFqKh;Rd?VFpAAjGWeO82d+N7Uq@t?Abn@8$$P8Q)}Wa1Qa=v>#Ytg z6%d@*Wdt1}fGa~cz`lvH24Tu@U+D{UwF8BK;~Z7mY^tTn!G8Qr|LjI|&x3vxs3C~m zP4*(~e%`nHl#^)kXy98Mx$C;;LF*NlIyK9DOJXWx{&!F7-xqn!jM*-=|1H5o%#7O6 z(pqlo+G#{6be~-q^?}LZ`o*k+5m{I68xm)K4nv(LH9%(>0vRnuAaOi)Ef5i%T|RQ+ zl33LGTE6pZHb1PP=|hX+z3eh8qAISd7vx(9-SiF+05++Ov4^-{STy%BX;mEFrir5t z8O87!T33&0g6n%5henn8jH&41t{;$D?}q7TKSS7)C3ZmH{3-=ZtE?v6#U`e$Qzw?) znT}6)dkc(jc{-$BZ{jC|X1^f93a(^f9Zh_Ln4z&Pw@|Q+DHuews82vUZ#IIqXy7JA z5*~gtuo`@*G zbgky;S8#BdaS+By<40loE`}+~G_A$TkLUT7S=%*DwJ_~~2EihE)Lnd4nlKDS@l~?W z@o{|EbJhFv!*{FD;c%t3-_QLvu7M6EPY;BuqgI(d*cRZel~A)#HiDJPDzvR+d%$0H z`8X@xBRiSxKD6}Denp=PlhgU`>De&{$&`&^q$TcwO$zdR8xld|a8X!>tIi()QXY;^ zOiB85@aM^*)sjm_1sO4w!`y2G z+L%W&IOrvL-ln&4x6k~@^<=W+JW1#6LgQ$*O_7y>CL+a&?v5MOz-gRdEr;s19lR0a zv|O+Gjbf!sf)Q6~fM3@wLFcEJ{@S+=LF;fZgMzy}mRE1~P-k7RlWTyX(c-}Iq4wh7 zu&2FN!MQwwYzibquo6Wbpc$xMz<&d`#NY;z@&PnH_GWFyv?~(`gRBGlg@v~Q&I>@* zl+h-VQ}dgI@cjIoj72nx)GMDX>DcF6ryfK{^Fo8XCYvFdR)50>7th!Jz@xqHV=s9SwFB;-Qf1DS~umF7Pts)P|xDK8YQjG5iJnWdOaBt1DV zhB6hMv|CJl!bFt&Abbc-)^ivZYrcou04INC^dAGeKH!;70#df;WJ6;?98@pHkId|N zGWFS44`T!eb%@z)uE}?{nXdp%Ta|tcn0<|iOS8xu(3i*eO8d$J3kee{qEE>+&>Jsqsmvq%&DLC4!9^*P?oFYk8_hjppIQ4 zkM7SSAC4def(Y^)tRKIGZnU@mZE2^la6@bV zX!8Dj|W|;4d=WPZz{@XyHQ!9`^s&y<*7Njxd3vP<{1>JG(QrI zZ=oLmHwBC@6zyudkR$-g*ahDJDF!0G8Ebtij4#5()=!mz2yBV+b$EgcaK8X1Ys(Hq z+CjN{+JHraOtpt(?>IM)FV9uf(oSzo(VCGaZ zD{f!Qr!u7ED{rUj8n!IAmHq3lsD7NMlVXuaq!@)rcJc|_O=~X7 z?ui|Nz~{lbz{GMalaOZavRy+s`zvZEDeBNd8!J zL{Pd`?Pg~GRp6ulw^*tf0q6}&E5wW0qvX(t#6Dp`kvxqSBQ zVc4!h)C`|7@R1!EPbt>KL0O%&p9^s|qiVBUk>}V9dt#fSS+$W&`ow5SW-b?K$z_fy z<1~HP5X+t_$y`A%nr4fLoC66J)xZZe5K?;s+e1xc`h=P9KpOJejo;$LN)9Mj{_Ihj zH?7s$E49rd_l$<9f}|;U%T2^Ju~M~4#%5kJ`T@V~^c06e8?MVYL z195)ayZo&RbGpJf|05LsuH3(wdEhc(5FW!U7);F3prAG4t+oHB-6F!*`!dG3_CkXm z)_i!yrLg6W(3E@of9i7!Ff6(`z&!Tn)+8v3*^^aez_~m%ut=s$L{&0-iFQg$#SGV3 z9vWD0*upm^1fo3%m&Y2riZ~`w0i^AHf&MVJ!7qBF4P>;V%%Y<-czmeCRq8jya2LZip7=b!^A4bs3%TxG zqaT?ALT}EXimN|3M3&y))T<c7j{?bM!d%g z)?Bn#TG3Fo!uCEuDDmsAR+WR-I#JYyF;KxB)rT!!pBuWc9;^&qX3}}`E3nqRu-6`h zm)p8eQtiZ!09Pp!ML)e^&q~|-Qx^p@ePH;`fAp@$F#U$~tN&Pmrxrd{jD*UI=tY+w zhYC8XhhNS8J|nmaYAg}|deT?eDG#jtlzK|`tUnp$OnWRml+2S780$Y@O5eTG2P4h7D7_AMjzx4SE3Iev2<4R|Cqpk zNg0mETAyrW?^xCn!X7#anyWhmd$b;KHLs#aikcO}dy4*SE|Pinr8IX;V({uvZA1BC zVwyzPk8H_JpfG*9hacHnFC$btrn{9wVmzSmpU6d7CaUytR{0KW3=J1wlkRsCTu=%d zj3*gh`UFQ!Sl8C#@!GE150G%o+IfMRvaoP!<)=NO?htVU!XnFihZellE`Qjd%YA=` ztwlSI%8!~rRxz}gg0#jNKXlKn-EZM#>z-h|SR)nsj)~heS@L|noTQsz^^V) z;^DD$;wN2OTaJ%HtJ#D87~QdU!(HP&h!qCXCw;W!F60cqOS(Fjc8>BlC++R@)?MM4 z<^ZX+1I)U>nzdrPe~s9-kDoA?>-z{Q76Dt22=b*n&>AJNb}+^>UEODiu#zO0?YLSA zxmJ>8q&axmSe4623m!J&NDUr-9+X-%9weZON5M^zl>W!(${c}rHB2dG&-O(J9>AIv zxMS-X-Uuprq$x?Nb9HEI3}3F&yaxJW52)jRaeQpnF4>rOz$D?qrdKIzs zB~Z8{T|#V+GnM{%^>xsf|N0u=I0IShkP{&QZ5?!L5Y_V^a0INk9cvNi{&&6L8}qB# z4LH&fUT|#;?EaNS8=aK?=z1KnTFM4mWyCc_Y;XN1d$3ihPtU?NQ3~6w~8^u;9v%Am}X52F&D}2$>N9Y;^vTq0vEm*8HJ~x``)}W?Pv1 zb|vfMJLb$GEXi7|S?~HxdvU<9#di{3Ui7wX47;q4ACA^U<^amGRBasUT@-x)Vtt-! zf-H;a2_SqqXcrlvv2{r-cXNIKcsl3nMML=UJKMaRyXte!x>a%kpJY9+6f-YZ}k z48a&q*rJzWJK+x+@e^+d4N4z&Gzn{^o`Lj}rZI7HAQ`PkYwug@Uf*3WG{zdsMT^~? zPRbz$<_XS20q=|wa{a)0krx`mXREpd9_$ZFckBI)3*p^!>fZ)2#c#S=iN))gt!&0| zKXx+9Vnv$Cobc)^kcZA+gKt@Zn1`eZFl(q>r%7I5W6$#Pa2Op2Ikg)In3OLGFe3hN zDzFnb8Yillg3nJpPj1FrG||2a%E_ogjX3Mh&oOIy?H~+63?Ewmz}fTISo*x+@f;JgGs*<6&> zx+dG6l+ zqhYh(sb>Mca?!w`^S{^1YTl_R-d=f5d4a7%cqYimoNXM!tHr%DV(8L)RFM1^QgFxd zO}$p=X5gxuIE=R$0~yjNq-NOB)7C?GHeb1~J)aE@8nOmW>+mmZWqqI&U9^glE(HDw z$#v=LDGyHs)0UDFQCy0Cf2wnp_lO~|4SOuy3`0gh(hEC>X|7e(-^(ZJ5!#l&eZdI- zxWsd6#;wyp7ge*0GYlGP`6p1{Rm1=WF zNi~@qN#{r1F4T_J$>~NjKm5VW8=Ek!_~1Y6&X_5Xp^!!-gsnsK3eaVCwcl?)MoS^% zZS0}b)q{nL*X`1QE%2R+F;Z|--v|Aqj?()leW|K-Qy|AFIRNgV7>WX=At1myW~>c> z7%bTLub{NU!_RvZ7Klq6vy61HwD@E7=_q0*Nc+%Uc=e%=>LR#Y=02*bXS zMH))qKrJ$69%=uV_O$=Rqi$Ojt(tswH8o%Z>+K-F?V^(x?6(JgZtL&f1yK0?v8@$| zA}CKJh5bEcG|6h=GpLg0Rn;3PHSl{#BCfvkN&OK8kJf;^uXOnqu*D`D3;rsCye- zG$res>vjYk9$z^K@ClERoZ$60wEhUhKp1vp*0MNlQLiYK%2o_MQcX6=j%Em{-dZm@`{OsdjO}@Xl>jft7%jDFwu+ zrMmdXOD))T&B2?$u;|t_mAd^xLOB^c3m{@*M7%66yqXa{4`5Pv!X%uzuiH@-d`A#- zkxl3s);{#h;9|tNC;(gE5Im;_577D!VL7}(#!N>41MsR?^Ker|^*g4>qE>UI?B?a^ z*?7u|J!0H>$8dZFyY=fi<=)WsYH$z+*u4#;dg4x^#21Iwb~`mgM6hwwOD(*OQ|B&@ zmIh%)t~I9rDX=u(nCtlfSoCOM%CO+_B2=EHd1kN0u$}!zOP$6OJxGik{>gxuQXV05 z-QL9wmFP|ZZ3#;s?mi=84UNjN{}d1~hBcxDO_S!GYKmzxT@?nAWJnx z@;#ED>_p#@UW)jHwdp>cfF@Z)(J>l8wo}`8Je@l5;c;eGULx-Hrxj)GQZ~Dn5EW2X z{wcViBy?F49VT32jCiOcM8&qlmy*J?$X;@NXYkwUEts-m=QtIpe%!=K_($5zkn%mU zwb)Ud=HNq=>0aF8q|^p_^rfBmu0%iVuTg*phu*T^E!Ig9RPPct0M1yf@QpT8f9WZ+ zbg0R*`=?OxjU|7&^cG{77R&4(`&Y)4a}3S^6<)NIqkszh>jfJM@k5j=!qR0}X*hju zVui^^V8_qAf^JZ_l7teY<0f82Xp8&iBPn27({|P?ImgGd^0dbs67im5W^3Q%nrnyp! z4!iOQ(^!?9%#o$Md8_)e++GGu#WGS+WQi9Yr)b^HBUXpt%Jjr zFGriimz7EF%iW>>y`kVZ+SfL{ z@vsqXIQe*}fvD-Oiy{_OXki0F{ZAv82+|4&qPv7kik^v7eAE;pIgC#K#f}=>)1bR7 zFMxV^h@1CFbvStVPrVr$=BGD9Jr}5$AIAe{zXL%H!qiAx74@2Y(|<+j z;5Kw`XefMJir-t0qj+i?!Ck2daOWCclp8xKHgIQIVx$KkzzLZ6Wi;Fi7Sm9P{2IEZ zUKxSfeetE^8$YF{JKK#K=S*3V0SUUXE>Tq;<1B@i&af78WD7>y4DjC{$m`aHD&D*H z0je1QkxY+4DNm8SPb?lh3@Ux%WhBf|l3{jXf;}8?R~x-yX#K(Aw{OVDDTmf4Tct&xW`q!seU>5w%uA zsz>+k@D#gW(JS(&Mk}zT?|t#y^zPt@4fWr7e9^*+H*h(BAS1Y#fw zjO{+^<`=rgFmU_iRDoWfi`XCB5o6eezv|9G*U}xja8J~Q{#(Y8%*@(hjNUeASwu*Q zLbW#xnn_qK6*2U-y_%#LUfryYcaFQi@q6hqG_osr&h{Xbld(DrfY7UibV_zFY8qPt z>c$5T*8UUZ+9Z8W0Yao{K*^S(kwy6+o`+#!7&4srJ&p1%XeXo2&~o?d05U7CDeS0k=>jg9EjLox1p8x znHiYr2bEBfcZOS;;-|Qa`y|rws4XtHV`yOoGE8*INb~*{A#*2n_ZhPJ+6B?=ii!i= z%jIqZk*TU^ypQRG4H#zjcv3BNYhW~>{^uPd{108&L^u_A@H`M$(AM%D!zjnOvztEz zq7NdeM#R@|ZcTrq4_r17R_`~}JZDGmnMZR#K6G9N>Jwg|S0yNH-xhj+l;>FetUECO zmMex;7ji#VGC%Z+GFo8821b5HcJXgW@x#Gbbl*C{Z-+yznLIlzX$$Owuf4sunG)ZI zuIf88sHM?4Q|Ga;f23HLndF(As+T8BdIlR{K4+ye9X?qHdQFhvwkZ{6^Z zrysbKs(=QZbv&O8s;RFZVSTxa9u3~iU%*K;Ah8Whd&R9k?U=BtbT;sy4#yijs+)C~ z|2*j!Oi{jBdS1Uh4sg9$zR4XOfD4dzkp7|8^!be&2F zTavNKB<`VS5_@=f!G5+9QT@ag5W@&|QXcSbv<8Ks+ozp^%I!@XB#$4->pvrd_@}}H zKP2l;1+k$blD1@Nv7X}fg**HBra*Xp0pVfYa$bcp;+aOX38ub=!qAzs7|?xt(as(4 z8;8EL(!YYO4M1TEG({iWoK#8ZleRb&>v~#uODS5ey0ltK6fln^ipGkxnY|68f1+;K zcFzYHxb^DQi2&&UZ+nR0Zx*eON&(8R?`>tWRK6-ji7;*&KY4+eA9 zF@4nFnHZ-kUI7XSc=YUV%a98oxsqRk?)W8v85_>4DYRQ^xdz(s5Ka10I;Hay={UM6 zjPX*ai4w4HTApGhy|P^EnLZ;lg7fTleN?VZ1AZk=Sab~$ycBD`^uL%UBo(c+3?Thj zP$z>`4O~TKxZ732puZbjWx6A|#bJ*r>UJr2kyHQ^*jwiX9%SPQ*T@r4Lnf_HPk}Nj zARIW8^NZ$Ohfre%^BV#(W+I&(8}f0$k4|cY6Z5%1Za|#qJiEruc}h4Igr>HJL^JuH02F# zDz+T-6hJz`bN9;Hn_tpK($ZaZl67aG+$pl-+)9r+9zv0!1 z;u}uxnq+ZkvBLV{5g)`*(_UqWAQ4KlOsSWNuxj9EHlm)^$j`H?{a)+3iU5@&#kUFU z7o&&JC&TFQit6ttV_fQFTFc+)i*L$E(cK7h3a6>@mG$Lrco;B)HETx1hkvV^$oyv$ z#E#!hJfq;W+@`K;MfvPf%X6%nO4$gK+vEy(^#dnsegO>xtpZ6wDfJ04b>(Xs_?5Qf zn|!Dfr;vyyDL8a?g-KYUJ8>SdYyi^!?!C`^twdQ6b>eL{{E+p?o!2tZM%1WuP>A8gO9_?OVEEZVM%wM2Av?l!uj-P9I~G& z-H#~yDUWKoxpTMbdi0GhPQ~jyolY)|cq9`EO6*oMNsmPBMRQy87anBiYz&%x6dgT0 z=YK*70+T@kq!B}1+4)5d7CO4wV6K1;B$vgyMY~$*1<|u*`Y9U2XR%h$u_F$jU-A@n zOJPUzsBE;vG!ruZn>eY%mb0mkwJGEdeMyn0oS$8Pka81ny>R`r4a8UIzxfBtiI!`k zs$S}6YM>^79@{Co>z{3qJ;>j=_Td)*AJ<0n>hW2SihqWd!_DZI*aF+Ib`27DVg`@W zlIPv!k%RdzxlOgun%?}HNj)`CjOgfr=P#^pK(3rA`%QmExiXG&knAL;N_n`8rqjBn z)|HBO>Kw%;lI;Rp&z2MMq2>JS`YP#p zOSoF;C)9Gwg>+mw0E>{WL00kxIWh9kKQ4r%O8PjlH2==*mF0fhVN+lJ6{Hy3i8aK& zIX*l(I2QOe5N^+_Pt8;L>X}*L^D3~ghW=#W5j9bbVm7gVbE}ou5ojUofrxn!_Q4X= z^U3H)r1~C~3{W=d_CvQ4h9zGWDV){9zJ-|_`H{aB5TW=?kTqEHX#Et_S{_=F&oe= zkM9#_gt27)W=_c~SGH6iNoCgYedg1p8}2Lf=HWE?iLj#6>#C^=vVY(!3SY^rO20i% z-1l}u(Nu462?_;43Oa3#K_s2p;I_AoU9rJwKPS<9hjyrGqzD>>bp#I`2Kdk@qe>_s z7w)9eu$*sB(CifymoPI88wSzApleA8{RlryqcYhK>k~+uC`W%2d-^Ipyo0%jahqUz z73GhjZ5@y=CRepb{*F3=5u=#a1o?J3Z9dlE6u$fH5yp_aQImJj6ofTJRr<`707|mL zf4PIOwF=~QsSyH6;RlWyov`DSE-6^3i^?{{*_5H;@+nWb7dgD0qjQOKh{Ax<7%GuY zLOMe>!aq{dS~gqK{jA?bMSjx>h*>+ADa)&?E@YXev( zPzeWn@lU`6<0~KP6WkD%w$z3}e-JRNC79{pvBE4M4cP~sql`ykIXP3)@I$m+{tUp8 zml#p&k-7~Za08L%xUO$se!a1lzmacT-0)`#%ABL4ds7(UU|TC5D)faG+N^+!0#m? zTTvY5zXUAdJ2`0r{8-ueN)e$8Wgdu(7k1?$aVl1{Q&_)oIS@AbL}t(IS1mQ@xbXEY zm~4Y*u_w0}keysf=reap?H_|VBGiM0^`pCgZ>GaLm!DExj{51UHt$*BkvXs9wsmC} z)s)FXo~5RCXmr|I2(1|>xP}f&exgxkN@L7Nd09Ahz7aE0gRjfnYA8~NGcqlr2-UjL z-EF5mA-s8LJ4O6?6q@i*;=qf-2)iq%SVLV*->^R^Q(MO@#hz015O+KF-W47J&|RP4u$>qM6B!uG?gcdB#R$i{Yx2swww;B zny|7AUsv|~#m(CVl5nPlY(VEoSj{~%<@k2$!1{}^kUOXKP$`oe{fbynE1W?A((trl@U_*?RpMv9< zQ1u4JM_jpBg?sKE{)Yy?03$QApn-mB_O%dc6+~%t3vu_BkXXqd+mdlpi=Rw|vcg_9 zQLR{E!Z{hB8HTG_6X8qQ%&c&1WKbF)DeXn^PR@j zP)Kypj14hR=}$JJ7}$I}MT5v(|75q)9XX2&N6Gm8@D3C*L=-_A3x&;z%>TI5vLIE6 z8jj?Zotq=#&s71hZDAkdN_o^B3Rpz{I8;<&UyFJRu^;D{@7Lycpj%|b z<%FkT{s$td21G3jpOFLo)0byMDS-9Y{UsEdfy0nTt(KXok5dlVIz%Ren&P>{b{E=h zp}gxmcLL~^0)_DEQ)sC&1+(=?K45N#Ea6|g@tuc+7the=Dd!{PRR+W>d@U+et*5N5 z4`wa22o-9ThZpSehgmR2`l1-)2ouh(pF{`(hL)v0D`H@r)IwdT`Lwtnb~c1XfpOD&%_pHF?^ zUaUk*QMbYr!;M#<6x43_% zRBb_`l-ctxe+%3GXlPK1QN|uLUdE|5O2uUHJdLqn0#--yA55v#Rf2)74mvDmpghZ0Ux|3&Epp?2~geG>9hJ184Hr6cUmRTccq z%ITiK>WVO!?t)?M$~Ukvu}@o|k{8z2&qf5jy{{|cV}>3M0z z%g(L3KwKV`wE=8+%q=tPH#?86N)TvtObxTr%;&i{2=+n3_2da&wWo~Uh2Ky48+G5j zn4bROzwFvH2;2k4XHiiBX4(ik13H`1w2VU7up{J~4bb}}IryspNv$2F5%Y*aR17)g zgeB=43R{k(WiI0~pp~Gky{B_()*A|Y|DiXZV5S5E=7HRyi)d>=apHCA#8Y0MF-@Pa zdwa7^T2;gntk`bpn|@8zG*q1@sXKjJLtZINO@QU%&BB}+ntt7k7?)t$%uGAMZOzwQ zd*wb<^3dLfG#sYy{2IYmq)?C)24c%*P)y<8q1HBnE}&OWvn^xRrmoOg_3<&>^_TJ%)nDO<9Z(n5MFNx}%(W-LW^2G3JUl2l}G zqrzCykaeVzWRDppOQ|elNrS-{zWaBk-tYU)AJ1EFbLN~m=eOMVbzj$Yuh9wy9U+@< z6jV{=i-$te!_j$m<$H`vasNHE!3q3zIB35AK8hbk#Vg+nLfIQ;-=+1nXPa;F75f2i{e)g8vRe&P!*`II?&MDdp1E%)By#(F+P(x^WiA(Qi zo538^fO}WSJP22ae+hH}q+u3hvqR^Psu0-d=~I66UfPR$!VaO8S<|6lG@hpxir>QW z?`oZ|M&+89@ELe~+X+la-?1%*xkh+f=#LiqH<&#KwqM4nLt)pt(1t6-20}#L7eIxR z^?Ep>E>fGI@*%Yvd!DrBp8fBD-%=2b)@EaSKu7{i*<9WuBL#4J@gKGM6PxIX9sHpM z;~5>4=ypG3EoLhr^x>*AoC2b<*+)@+i;Q2pJ%QSVf&yYi7Eb#}dE9nrkIh6)L54v2 z+Z8hBDy__2D7S_h78@=&w#=8$P^@OTuU)ZJe0t?~Df!oIgq7+ej;<|tVA~n^n`#_hfvxR;cv;yH zc}vO2V0_2}zd5uMS`%ID8D6nFd%FHu_xL~XJc9C{qz==#R@n9fa4CbiCNONc8Rx(P zO+loJd^PRvcMBUskef(CNDpG4f6#&p`W6TSi2BzmDp$V!Z@`6^P(Grna$g_A zkI5O8*D-hu6A$JXQgE-lsUK!}F8R2OtM1x3bwkb_1FH-+H(;`aZ#g8Oq~q zuA`~-099^xiyD1BdA;4yjdo9>0t)_KnS1{a4YakDry=qG;Ns|!@O~4yIyB9y36Ru} zmE*Uvj@fj7TsRa5`4X+JRyemt01aZM6qf<9-)&Vz&P5ItI44<4s?oFk1Z&aDhcyOA z{dFj0&RW*P3UkZ81LLKMztutTLDJoHXRQ$Zw2=qz0RamkL$Ctdl{oLM_ zG5TbF^NXRQ%GX%aLCUvBjuSkOQExuKvv8u<%ldCqR49cbZ4a!qS)|2-VAu^bS^9dy z@jt_7MZ^`8x8OQy_!!JM(4>N*Bf`$nB2n_F7tfNqx~og{v{woYZ;-yQR#bcL$aeu9 zC*AF?gI}g!Rbun>f$CQc%V3)4#*;&BKPsXt*6sJ$2kJfw55yb>l6OJdG`VqT0muQ{K9YWn`y%pKIIL zt{&}a&JO)VYxdqbbO~Emdqz~5xnS2Y_Qp$%SqO+)t)7x}Wa3V|rb<#soqU>8p&*y$ zo$M2X;#!bktW|EF&s(-_!&8(J;O64++Q?Tc*diw~sYQHo0FV$^dn~)k9S=}pVeexw zW=k$kjOOe{mAlyQ!!oD8o`%96%|FQ-OGGZ}s_o{p7A%jgp9c=mp&~8i*>AzqoSgYh zjZ?}D?WspE5vEW2cu%(yU2fR|MOIkPx2hk=-eSq>7Wwm*E0S)o(7uM1uUOyX+*T(I zLAfHvjF{gAVY=or*hvUS6OV=43S|L`77+o!eoVw6@b-3}jJV+3YpHr0XbRBl@(d(3wTr2aHoP??R`-i%D0R={`d5s& z+lIj7hr=hM@(RR@mxT9KfeLll@~8s9QJlNYoWYr&zL{)8Nj=a zWzoMJT?;k5tK?hi(-#8czy*1ie3;l7WaZrRs~FK>^vtvVxH}O0Qqa4UOHyg(RMf>=b4@sj!B%AdWzJGzX1=!RaPaRZ$+m$@Wd4NTzOj*kx z>r{RFfX@5b3rpi%l8-s|>K@_I{R4=Xifs%Sa>Hz`f)V9LKA!x9vrEg=E)&Qc;?Jac z_y3@1PT^DFeLp`2W)B7@?NM!n+E-HyX^%(&Xl?J>dsn2$%^S$qmm}VFQ`0v6b*u-e z+mAsvN9FRbenC#-6OU6jF4}^UKMD(bn@<@WD?|`Zc#VV%gBgX%ddlG@?yYj`SZSPM zhNub@3O}aK&a$QGS@Uytmt{U9KZ9ZSkhy~)%H(9p&*R*5gSQ%dE?cNyZYE$4Z>kqo zYN7@tUxtJ1Z|SbnD`phY+C8b+!4SI$sgs4;As7F&J8cbeGZ-MZ z6^Su>bn!?X+OIc;t`rV!z8_`R@}fuk@QI@E8eGmQCB^V)&|O+bJ4=f zjgWEYtFa<7&)@Bc$kPd~>vsM;@c1#Q8GMQc7N1}q+feawvHF_?K!YulvQiXrG(}YU zOY5rEN=>Bx_=yFAz<%kt+8}ZDFJ!54SqVs6aOCD;n8S0CvwKhHj8~~UQPert&_6U9 z4{b!#F_#=5c8r$&_m0Y%6&8f|pqXv-Qt|;3O2?u_SC5Ds11=zl(zaJybFS;)?v~V5PG4((zp;Iwz zN7Ia2{$*tQgp$pj>9w5E{XqAkB6Kpyw)pyF&!xAv00e^P%Bb>)&Mu(r!BG&63Sbjc znOp$0oUUX1kdIi!U%!P(6KiS<`VT%u-_Ra_5R=e>@Nxo6>bPwoMr84jl)8}~Me%qF zqJ_;0CxG;-F&&JVHz6*14%$%YycPcn7ev>KFehiqkD<32Bt$o{tsOOy{tJ3I-Envo z+aD>v$Y~m~x#bikgY!FP(Sq@@Wulyx>rg79S_v^>WEPc`_>d7xc0B_i9PedNh2cGi z`->QSG}m!}us@bV!3xt$E%yL0f)zIQOEBNGaz?+)ozB@lWR3uYlB9XP-BwZjaX%eo z%_NEFo=(I+)P^!T2L;oi(F_)aG4*%;BF^NTUNlp5zEyMoF zwd|08Ps4ejl<_r%|MSgQWSoKm@EZrnbDap<|E&mzjRKd@aL<$4eB}WSYyzU3y>-E? zY?(3R{6@va4g@Blm$L*gKgl|&ZZ$Bfa+N6{(pe4Muh5&wRa&6fy&}ri^R8|PIY4$hl!?<+3+k}-4kN|E=P0lb$}~+W^c;?K9APW68W^q*W0AtoBU=e^Q07jb zYp12evHyW`DbPwfn$zR~-hY$Ao??&BZ>ZhXF^#=I52KE)Hqw0}tTcoBQb~(@`jA+u z#Xn{87jObp0PseEd~SDZ--KubyAAPK;Yg^1O`3CL6Iw3Yywp^{9MC1Nbh%fhw>x!{7b%c^RNs{CT|?zPM2wxDI9yMGy5Km|OUl1K$& z0X*ZUR*+2!%jZz1)kTu&5IoKpHa$h-5@xc|RWC@f7z?u|Ih{xrI>wN!|pPN%ya>sOv$| z_C`&=lQX5DVJ)2vA8^UtF4%o3RJHTql@B7)-64D8q>-a zEhZe?puf%>_+vI@FQe3#=VyPBqOQ7PQ@TS%ftW)w_O{3v0}eszdz{OU*Mk_Vo zUp>pj$e!nmr7#ZYQc&Uuz9#MS_`7VR2YigK{tfUMIAqC4$CA3PB(g~X|9%f{H!v8zM)zF zHbxLajDRIlz~LV`2{g!~sSPk%;(!o=X93X}onH{nE~X~6@>2qk7g{<%0ZRG)<-fiH zod*4L&(Hoj43J{p83VUdsVXQcjOos3(}t(BRM%eXD|TqYzfrww<0k; zFc{(7Mr&G>7*=@0F^@>IXy5k*LvFc9=|Y|y>;j3Y zSiSBOC+w22x~Dw0q%FEfmLeFVBEoj+an)L^w7G4?`eDpj27W#;Yk=s5Aa;hYq<6eJ z!e94rCa`bDzI|74jj41fvpM2`^vU)PU(pf){n7+>7{Ku~1U4q{V%*I~LyYh72cUn( zioj#+&9F)(zfdZ=h}d6)cm;6$AU%qze6hpz08d6o%t_=tj{X4A|0JWO7gX9{HU^YE zG*KYDe@2^eL8ubDnJmq~l@Kk7`T4^W(tv;{oLGIp7P%RFZ<2~P?$O5X(g*XtOt-F9 z0TL_zDk@fe_2dVh`x$6>)VT*pGh=V1TUzy<EnKNoRR?u z1YbnFM^=6&QVgNNfivp=k#sq4IfkldGe~|{{P^hpz{~h28Z7cr8 z=S|((QM(Js9A6tnViLAPiFSqfy6AcJq8hHN{9!a220GxjmO}& zmgW?F7Q#dJlybuH2h^-uq^(494PZss6N?$PdE6!UYOumSqw6IyFUlNgS5FaAZ@Ndf zZ~er`q3UTOyCeT@^-i>FM&jyum#mutXey@()#l?&2@NfhAmhCH(ve>f| zYn;H5i2Fz>w5C{=1kNtNAwf4t8OF-6r=y025}|&XxFVjk5U~(!Pz|I6kPEw1P-aet z&v4$(f}cXJ5=Qh|{02LD_RdYo-}Hr0XK0V^;aRDzgT9rNNZxZE{UIUK^Qb;q`X^v$ zVoe7%u^<>$BfKFt5BC_uY@7cJF~F8%PgH#oR)ZwN>jX1#Mb>56qMIpl%N@$;_pVH6 ziHYJ0&7m$C6Wy@C>-35R1rd<)w7+5`AQtYBB&apPGuaNq|_}selS)%o)q`HuKZdT^PHY2udcm{>L?h( zy>b((oZ)=+VD2quPXrnnp@1KGrxJq&s2u!jS{{(eR}jv7u7Ye^VN(4~ll&&ju&^P? zYGrG1Bs{9S1*7y(?=;QLVV9Vc+{~(Qv<&ObdBoP!)WhD+ zACjBzLo4(f;mt~7R>w0m4pfsTu>w#Pr&LK$O`XXI1z>CQ`H)^%+toE8ZrBf`T_{d6y$Hki0ES+H z8KAlS;#*8+Kz-EU=N=&lV{lO!y;d@E%rO1c#DMXM9D)})gL9teB46fCTd-xj<&Tp> zs!g=;SPY{BpKdVJM?w6Ld?n5Hd-`Z`r{ihadllENCO>4nf+$BHTnjmYa(d=AMo`C8 z$gf~GnfUQxS-(Cs$}QVo+_e#HLVspF<~>eeH@UHeK1dOl0?pfz#}A&bwpqE!HS&Ms zV8$=3L1MTE!D$SJgMSW0M(C&KnqUA?S$z(xh{|RbV)=jVMAgS=^wjm)O>$U_U;fVe z^^5G72r?myco3q_h#&*!BO)!n@~)jq9-|@>JIz z?psXCSSkY0xCtZQ1fpE03am^7RC$_I(t1SvMZu41DhS0su0mI3*|D_FQTIO}sH#kZ zPW+U8Nblzf`nRCnGD%qmAn#IaoRGoh-tVyYcQr%u95M~hm4nByAmDtQAfwa2g$Njx zCa{bnACwtL=Uj#!glb4pL`s~0XL(RWgRA1`mj$#>@5bcaEv@uBsN%W>^Kk~S09ylx z5=J(nB(L#Y6yaF9!H$kY5RbXX+XeJK{Lx5+s+^5gSx7SGe{$O)#Q!{Il>ZslaUp`? zOC|l)x!@sjEn@oMTh{G#=WXY09&y>si(M_#TcBC+!>u$(-~zifnrDE(LfMIvihI)-dd1uW61Nrw;mD{B574f{gFXT} z67h+!5M)w|EyL1Km0{#V{*e3J*6nCpc<+LhjWiyx?MRi{8{ms_Uj;49=z^y#(X%Zm zIT;WEm?kqh_+jh9cE^tWH^=HC}^ zd2nRp$1{Lt+T`#coFF2ftfXgkxqt|Pp9P1ytK}sVT~s5 zMZA33coKU~SZDM%CPbff>YauvP$UX|pcj{(U}4D?HUOP&Up3S<#-it+HoUkbtedq8 zxl~!^$RVeOC>>Q5Z4`(@8kPGD9$^>Tj3R-4JPz5|d!4rT1Qbf6BnJS}OIlx!|cY({Z;D~l|U*zNSPx&f#sl3Z8UYT2MYujj|4 zM0~LlExWhc7W$Dvmzev*oIV%28yE*%EjO)h;w@R-#qhS**$Lt21Xm5pPB2A`;*pH6EKBTP}OFj_fy@45dDL29pNayAb=uKcn9 z#Aa|915s_Se_29Cr7hSsCYhRZsMuGd|1hK9Da*Xf{<0D_W62Jcn$uDg`=e=%bC}l! zYKjoj^^GU+pac}!gaMm7#R|*MIPZZXRVgLK&-qq{ZbmlZchagZ>50#`d#-Qwd3Vb8h zFL(bnIw_MxW#7JhASXi%pq5r1t%-sfVHP9u9UP=%(si!Aez-!xew(t8KW09m-TU?0 z!1G*XRLFGZ&xefFwsM3ngNA%S=WM+ZgeQ4dQ|5hTm_xlCwb-$Y-K94C;c!IE5n zYOXnK)->b)iN&2NU=4EZjX_l5m!IF2OS9;19cMmH;_amV{a2v1V$|g8>4=MC8a-DS zG4yh#de`O^qD-A+u@>BS8q%fZW!M5qpG{6R6(+uh%{pA3h!z^69W81!UJ<4e&aL4d z+q7i)IZ^coz5fu5yBO>v)x)4GU@>aSZRSh-W!MvQ;5T)0RrO1y;+=NjzH6vCioM-j zlLgYkEVNBU#`;^2ROblfY7gg%QOio!35Yq`xA~T1{#~1e#>tr6p6ceg7hs;ad57X9 zpYLKUOZ{!=ndrYjh4dK(h|7$CI49ELWlaNScV*xX?1{mD==En_+<{(QdDvn>2_lIZ zHq$*)=C>`Y@CBIuo+Do@s2ib>=I(HR9)}=iYJ{BjZ@58-^mIi@4>`6NcfB32z$(JGQv_Lq zHsY;f?*Kg;C*`nMAIMsZmSRb{NKC=st>scD)J&l<v-=bi6!^0EQqDQ!+D;^yBcSsck1O^7qZZ!B11r1U2d`~o#SO;zv#3o64ELbu?&PQjmnmOjE*$0$; zdJqNirfe(2K`ZF@hW*nPMoigvOHE)p0$;GqbqjwUPYAXz#t%}Uj27Mio1u@A|S>&!sf%$@mV zjLP!xmcp0a>Di$*wCn~f1)$^q6eZ%3*YkU-*ohRNA#I5F8Ep%{4P|H3klE*RGmzO0 z=59wc$-;?JFGj+Mb=`x|8h`6>D=arC2$u0eSReY;U|PDqI_5a&wV^fKATJrZ9H<=V zYkxH-2om5ExmImT*|h9`7Nf;iit&LLW81hf{N>w^sfiwE3M%MsnC z<8a=Yd~#_f;lw(wFb_19+w)!eUH9rl15y_{aF*KP(>qwwak|PCchz27)_vjz8zW`I zI4y`KbN(%sC1Mm7^rKbpDzTj={2x>8zf$1X_^oc@6VH6>}5b zf#zg_u`HIcz-bVpsw2J6HPmHK7b$>X)s-pR@>HmhQ!f8qZtkDa-d9CuL}$*Is2&PW z`&RW0u}C-nCizJ5oaiMfg+xh|JE!*Pq3@%+L9efiryFXy){r`-4|uC27h-P(`^O7e zOED5Zyxm6mHHnP}jT6;Xb}22Zy60Zef@EOU8A3G}(B9)e^QxOBZ)_=i5QLGqCp6W` zR!)Ta>vwgHxw7=SI`Am6DeJsb?~ea7jT&{e^qg2)f}R{G7H(9P(sw!f^C_7@(V&wc*f!3?QmXWMWfn6EDF9jiDUWysDGV(w z@RTC?s4=vosXsMT5f%7~mB`=3q-WMhUN~MfVj{o}7NDQcNNyLO80GoWi zr#p`eg9KQg6!5uaK}RO#;IXPvc8-ni8NRG%JQ9%guM5rd z0WZ8f-`9OhEeHA}kQO_HQ;lVwX@|W&XBvFaG)5TJfJ^y$^c1nf$8LDDhSZy{x12;C zvDA0DNCn9ZWmdNCjQ-XtY3|O*UbqDyH6b?~d=0&m*;aH?`svEFWkbj=AJKHB?o%BS zAu7=aHN6myl!0uJ>o&!|H&OzbP6NRx|w*GWUXT#km3nwjt-{xrlfXbcC|&^3iaL+S(#) zAOF7X_#TLWe~p=nv`Vg7nBu~h3PjPAWW_nr!>+mG*q)c3dEg|<7sS_=b!KjK%rc#-B{T?eeuV&|Bb1U^cQD#S-mGa+pQs|*+nAD*#~*6M-EC8 zk-HgYggSXRd_Q^drnYyFIwDrn=fF%paww;1 zoa(IqsXY8a$XKM!?7qj-m31=#c}j?3E*|0Iqowpmr_RQuK8wZ3y6mV129;dJd~b;vQ9ASw65xOA`W3yuKUz?)>5U^%5S;h zDHRx>GbE?J7aM)+!w?Cqz*;eQ%uJ`dh^Y9z5;h4?_DqXI)Uana*wEp44|vCai*K0~ zV6E`CRh0=dz7p*z>z9BB)Lk;+{7WG)9fMLgF^9Hb!vI^~MzZdS&_;-H<{Mgh?EE29 zQ$xcmRf-|Iu+5Z|5oK4UQs7eZRQ-f{tM3|$4uR?J(#=^&+wmqIe9mKvLvH}{L)8pI z703@EkOPy*7C+PEg|Jo8b^D8PR*H>YTFSo2+F(K#Zp0&k^lHSbg>D99@ZPaA_Al!X zh}`ipk3s@)7<*p{p$+|whvR^)e2TS=T12`s=UJ>7h}z+3AWKRi!4Mm`f^Qu(fRXYiH$uV z;sHefX`i(6f40oVVR_rBx8N#~ug$I)9aY=92V^YH#i!xk3ZjBX3YFHrrQsX(r3a?> zOXCaQQl-hPHu(e)W5^X|3$zFkg18hSb`M{#Sx3uT7JoJn+(}Z2n;XklsPZy3g z?cE%!)C>%JJZqH0s7`$Si5n za7+5t)3ip)mNFCyE{8&VIN36e!bcsYEkrki4tBJ+&xPm)G^B|4*qE80b{U+=;W>}b zXCTkJj#crW$LC?7!_k5Dq_8b34~8;)!eletY zU;b{``C}Ebu7bM7dMr1hn*btpXsbZul*ty-G{%ZGdFMhpuiyLv77~pM zF{uNT&Ez%PSCH2mhMysYA6xDEXse~Oe$$+^Itnr)1))1{t_!O;A=icrAx3a}n&)Uy z0qaJE>mEe?2j%M4T`uI=((HFFC2r$oN;AM-XJFBo_D#$o9qP5934<9-p)So;V(w_* ze8wtO05M!nPSQUSkrh^sGLr*?2dxw`y*o?NE8hb#7zA!V4Oi@W>` zgbaWBHMtlUk4O;rCp$_WMEpAHnwgtq5s!D$}ZgOos^8l#$8m5{Fs@x zmbu}7tv}%6bT_RAgf<3%_i8*)2mTI0EfE5z-=WG7=^(c*cNBYQFi@Q6_Y#KlvkkKF zRn77XSv>$OOr;w>H{zM4u!gO9?5SNNkbqVA#YqUm08`mQW@38Z~ z)TIM};%fM|Cz-G9LVtw6*Dj+;aW||)1;B`J(0z{@Y$b9V zU~ou#!T!l*rVyPzdxxCLdD%o&3Vlr9fZArpOzzNc;nZ142eO*qC*UV>RC9z2$Qtz> zMixNW{mLC_ue)D)PhW}PQBxI1>w?1C=72ZQ3FAf#O^OqbN;fCMr_jL;*!fT3+{?{Q=iH4Gdc z!0H=Py}`fY#}8F~L8~xWV$y;Z8FxkhPH^ZZsPTfVQcm~JawM+B(U0{P?ZvjHj}>y; z%5*b?J^;w@8lYN=0~ms#kNh4!B-j4O20P(ZTqs^sa8-Tv5yaT8Rd@3ERcE7UyLRqX zIsfb3_YL9u=Z3rIqUXg0(WiRT7?r&AAW$^XL*b8?VhjV&KFEJlS>|4#(~~Y#YhueP zRip%M1?o^@bl;!7smBhM0BTpqzP`m480}O&Pf6u2+vIYK=iPy{ZzwwT4{$Q5IZ|DC z<5IV&$jb>q?JzzTsl|>rY^+N~F6UShY=(P~4S&!If)%S^VQ38x050A6;&Qiq#HrBT zg0(B|#NIxovt=4_u@I6#Fa~{MlIa0h1C&9s0<$X7wi^oF=%77#dGvHRW@Ii2b4s#6 z{O)j6z3P+`)brgYl_A4k_~_eQU&vfV$SNQa=7&IMN;oayHSNj^DPtE){l9+uU!>|C z_b-PCeXfzqpKs@N%O4(@E$8q~dpF!Zrxf?TUE;6Z?|T(oC7K7e4iob)yA9+%-mBoM zygt|?V`F~LBAW%_OHI}g&n(&d`s$8a;v03SL3#!9ivk{>z^)=(b_xy z?_H+K<~;@LPc6R?r@P(De)GNjH@)45ooXc9B|EKxkMq6NmbA3YKOc;%_{e1+eh^SB z-94yhG0o%h`vXOAOxKf@H8(k@{!&=EpuxmgPZ54gQXC*#*ZD|V@l?dhF&}z`c&;~k ztjiAd&Zw|Ku(3k&v5buh2-H;g5q^7G=j?g8k5>o0(+5I3;3ORG)L{?^ou?%^Y6Zdy zJ^>Vme6CrG8L8f%l8@gnLQ#UDt2Cl91dDfz)Q*POk#2Y?*qL<*-6}3gD(rCVdY%W_ z(A3YU@1-401cZ$KwD-4O0MwSGN9d8~{`C?lyNgv>!grjPMUr9MisyxdO_x?0DfrFx zoLe}G50@Oh9&w}E$3~Su1Se29d60}iu^A278bCb^-^@*mJc)W^S~ADX7YcvZi_?9+ zqua+2`tN4XZ7GM+oO?p>F*p48Dvg3YPXO6)ooG;LCRJt`pnnW@5~x@vQl+Mv9$rS%g4!%baPW{n=5kfzbq z{jIjB_-UZu+8_1ZzwK-KfQ!dHz!)ZvDgeUq3p4|8QNR}yZ?-@`RIW7|!0Ui08P2N> zs)@u%&Az2p$*#h*x)=%b_~WGu5a_o5q7cF}-GWp3|Bukun#Osh0Xbk#fn8!})j z54c>K=p#@*JO@xTcGEL6%S~jJ`gePA6EO}J?y)uvT>i6xK6uuY$Y#${yk+}7ZZ&g_ zD#Bxu9vc+Wa>+J9Rs+6=xCTC2$we5d#X1&0hpTh`H;G=+=b>D+*k@U*ahy z5PnULs`VA0lYpUjwIQA=>mpH0tWz-Gzp?cOnxw?OWlO ztpz4nr1ZWEn#zJrfMW8LFTR?*EHmmnxoI4)#oS5uFpIl(fuP%b@Vr9E{OEjP$afc; z7@Py#Df2bhaTmMTUOq$9X32Z(*nqlVmH>3kBA%g#%lXr;Dj?&H z2Js%4sI2RI#BQKvX}Bk(w16uYtDX&Cfx{hlgaT1yyC>xf%!E&IAS-gsUD8jIA{TlT}=xBuD~$%v6b4seY>4|x3iq9 zo|*0cZZmz?X4*%6?5>*rp4zUiLpb!SbN!?u&RdF{kl_kH)g|AtitDjXHflWhjr3lG zn4@Zv+Wm$BmcS-04JcDHekoyoHDxd9V3DW;D?Z_v<5aPiyyJDK2jC@s8RPD`SAFr#X9j zz-3VR`a`GD&R#$Bu%$3O>_@FHg+CS=O2IQzmw70v_;F$V7fb!yO0@nf8iOU;(OquF zg?r%EU~YVaxKi;)nV0OcI`GI3R^4mO4XA|CMoDksoLGPvszKRkmDG#7X5o)X`|Zw< zV%7JxygNf{>G?KF2pN;?o*gy`{vKR0e-!%5&84}{SfBwI?PzIdNtVy7+=_edj_Qs> zOI`G$>NoaZDSd8NB~Ot^1_m8h0m=P~bmp`sRK&pqTNN4{(6z0x*n@P$juz~ypQYOYM$(_W2v7CMHyl{q z)mja!YA(e@N6G5U3^O#Q@2XKCM6B@sMTX~+rnSgWD3{!Ee6N5Z*+_CunA8~aKF0QFXjc9ew*i7WMciCx?+iH#XQaNs| zI+_*@5=dRYsKN$52YV7whT6YQdV2iAW)L0+X;aM{aExH_yQ9YZ1S$Ij9oj9D(kmzY zt(tn)#v}WfsT%{^JnjJoP*GQ+o0i)d92-E;X_^WwbE}2ka$Y47x9v z;b8J`B;V!E&0*L-+e>`3l{HilCxqz*buDU;okZXgTp!5Gi+mI!lG(WykS7I>zq{B; z@@xSPe-v7WJk}@8!lH|PAAJj?|1agJKWZC|%dB&JetvJ&+5o>F#4JfD^@fAKMRg}+ zwu{@Z$p1AIrhf2++SJxXhF$)ylJgm=L79l5pHty7V}Zu4+yzVd>*{MRd|t{l>JxJ+ zSdJ?>v3k_|n69-%Wexnh_XT!w1>Uq(jRFV~?(&Oa-PX1sPGUMi3+ji*?Ij`t`7;x2 z1=KM~IiEDXGoMefC$E0U+pD_bKE-}7^RkR)IST= z6Vw`)0TH+ugZZp7MeEgI8v+HnD|D!Z!?dmF(H?fx71D#KcuaQ?Hq(OtfW6{9Vn_rK z(>eFw8_(J25rHMz;>xA%(I|Z z_^IVwxlqH=)=RjCY%snoMJ#|%cT-X~P*%bMTR(t?O)H}aCXlP0B)NaVP?%DsQMvN@ zbWt;la;WQqR)%2`B8h(D=CY7N^5@Ln&ioxm0YofXOMaR~sK~Q0ON|rCJ!)}pLVe6T zOg=t7cqe?CDPb8NJ0WjnCHxHjM${m5C?Tk<){j~2PD~E2K#e{D_WyF+@G*luVQ6?Z z`}9$Ji5uDw$|=G_XlB*gi&{)xHU&#Vn~>IVA2hMf8C%>C(o^&7hR_bS7AcoCM(^j1|KX*qFs*uy{rs-eq#^C}S)% zxQjNRc_cg<&;~=pZ%vPkq7CmE$SPoT#b3SuxR=cES*yv%AN2{1$Gj6nwoFGzu9jP$ zsn1c`7WAvz4N#f@u{cgpb_ta3mJ^O7gL9W__nG=jh0-j-!lxPp55zO#TgolJyy0oN zKOmIfeq1aY?|+Y$Bmf84mA}q8@;0|X$^Zv6Y?_{c%&()w0l+2H?_(i1j+Z1^hz=qbO z|NMW!PC_k>*JLF+MO`S*<~r*vdiDQ~vaC^tzI~^1nvD;uTTVvN`BoNwgYufl|K8{S zQ}Nz~f0Fypi(rOv$4>`VMQY+h_QXuU`YX0h#aqx1#(C_)mT7$Ge<%xKjL0rjtZ*$( z_|e?+3DWEKZkzhQsXvvaa zodqpnu@ip%{N6qWy#3vO0LgGXpv44#vh%wg`$%6D_Wk-+NTSn~-8-Eud5>Q!)~#YCarp_h zE3}uBTV$8y~TYl{0yn6pUZ&mKn zj;UubOpM$9PQ3QHc=K03g=qIL7disxq*KMAiF*ckeM79Ke{$eTELQ8RJ>Z19j?aM? zjhAj;*<-DSR{4D!ebU;gHuIkfk+DTk-!_}8(?{U4JH(13cZlgiSFv4Q<+rc|Xeg{p zgB|46%6}g{`#w4%VwM@rXGUL3lD&1#tXTGk=FLkng`rkY2PgC!V)}ij3wofp5FEr* zur=H}_3PKU`6e>tD~zkcI__ZF>DVOS02P5M$D4BQ@zf>s!`L2D>&NB?EGX`90nAlw z^ipzmF=}NoxhupG`?8)#UbjPo0=LZO(`RlEH@LqRHz^8(9Y5kkFg$&v?=+( z7%NTOEgsW8lY7U3HW zGg|Alub4bnzqve2euYKyMq3*J`Z0k*PA@yZoAW&om=2;gY@HHdUG{?S4m7AZV1m(# zlo0H)P5?m95Oc4or3(w)*`v_6M|uzRX#wZ&6`@Vc$f>V{jqAZ;He?8yS zw8R~chHnphOuRjp>{Zf9eLo^)FkSwhRQY zbL-t$q<>rj*03G=_4pz7E9HjoiC1$WYq>@P2=d9SeQiR`_ z<^=2=8#Lm0!+KNequ7N&l*mv!gWqDqyAlCp5$MPzK|K;Khb4Y{f=g|Sx21Kd7MY1? zKy}j6$He@VA?(Z8L0Zm9PQUp_3YVgd=s z&iRUR<@_YA!UOB8_oD`oxU_)++vjA$ZOm~Q+PNFoZDd0E6{z4i;Hei>T+3N?<%^6U zg*JP{v%|;@7uOY=LX`x~m&C^eS&}U~7?djMISc*uLv&Lc4*Vu_rq!#a-8wkI0#Lbt zyEC-L8!iHI7`8dSBvJf^zkhFd^|oOha}gnhL5#xUv-|XK#X#iL0o+vd*eb{bxGO_z z(1mb4U%hmpieo>T0+7oGENZq9br!+>)P&_%VRG^TYzjuLBAC|?GaQBxkiWNw#oFG+ zPos5l=UfPNJNDBP(i1vlF=x;#3rl5L8g8pNm{H|4#_gARU$2k0a zQ^Q5!4~{YeQH^eKp0>s%FBTw_-!TB-vtm=Q6Z9ZS#J>V&O8ZOq^$RcPsai=875TjT z4SgI}n=>h<aDNj(^M-M6W0mh`ms-(I{;G|Ls-Ep_{y|Fukyw=&wgMmMGon{ zM}gzSV{RPHwv50`foR-nHe`I%7Y&G^&Ti~M;Oe3MM&RG81bAY4<}+nzgV?4ImI}s` zTyRab0;}AZ%3ZkgqivqIercE+bl`N4%K_K*f`SytsS`lL4~ z<@AK5Aki~JrHkOA1Rwnu0CeqeyACAduF5zp>XI5pg*!W>hTr+2ZSFAKs6J>Z1D9Az z*R9y}-!^0s&_4Ty(i?H-Nd)^kv;CLv$xuc36#e@^P%)JB2hQ#c!>vv$r>R8~_m<1S z+HBFr`hU?S8mt*$ZeRuuKphw1hI&4bSXZ+RGF2bq)S;v43Hx=#f~cIFkV0fX`75SE zu-7Emp-2?o@bwvBySJAZ^(fv$sE5_>TxFV<7ve6FOHhr$!pv<%YxFEcCr)FlPM;6j z;t9tyU zoi_f(!mD@WM#7#|(DmN=q_*j|N|Ba^k!`p;#7T~< z(oGbTt+iZm9o`YZJ5pI=>&v?t5dpxY0L0GyuU@i4OSTT=3)4tPaqeH)&o75@IU_y` zbNyISHiJ_EXk<+7#WK{iYmoZ<{5khrd=CvYY-yd=I>ON~{h7hedu)7$msVHa5nFuX zV43sreEe1Rs2-#C0$Mt&_ComR3{`KgwN@9y*}Gyhj;;=dR7!5B<*Z!inK})@FwmQM zDu>ca?Ro(gnV@I*0@X~vNY_?Bg6u~GrxF!Rz)@m9SGguHjiab{s629Fn#z;dds^^sWUFnXZj>8rp%ir4I7#DZE4$Z<}reB9O#86 zdMI86Lj=!hqDmTB3I4M4BQ1zgTyNU5EmtRT$q9ycH!OV;7WjW6n1wdzF5zieVI_@a z+q7WUNSHY*wm`HV`ZpNiYlC(N6hgbe_Mr~gHttgDU@Kwy=tgRHZ*g&WlEWdf_ABtHN`4qO0CBs>8*Aw{YG zJe%sYz!>eHul!fM>W%o+tAVIS;9%hBy74N%t zWE=&{Y#qAYD&_mshjXfHPf5>;#C-vZb+ypv0PfB3I~A{%q=`N^4uOu?1LGaFwblH% z#Yf&kY}=jZzw1D8uW z#dHfl%jL!251xeI_H+r_vwbAeZd@4T^7!E*@0J1+*<8m6@;yaMYq|bz1hb(nX*W}9 zlY{F<_ieIUAups#vigq;3JSd;;~$uC=4WsGgSpicBEx|Ix`Tu9YU=Ly|AZC3BLKz4$$=$sdpEY-}A-oRy)q&ab zC}7cPX(ckev{4*+)kaI#J@4(~|IBXXUGra4ztPE~A44?EVB$9!BQ}HBa(4svluQ{FMm1bwT zQ99YFAUjWT&_tRBk#*An-3IRigA&(HuNFPI6+ZYh0_&1+?w|maBQYGnL+0(3rD@l| z7uo7N$LcrQ&3O9M4?aE=hSzW31|GBDzE+x}(-HMMHky{}Z6%jmW6^75Re@9$F+Z{d zZlM%T3-0m%svoGlFv*RUo`ky&Plx8IcNfYLPP9=h38kVtK@b8cX7j{kUn2V@gZ zxNh0Oqm)Q_l~*0A5sPrgO$7xf{jt7#yj3^d5~+iaRNvpnvcjatoNmA>hvCszQU|6S zu)Gbav~!jG>yI@+7%HFtq2#%iz%T-CToc++DQ`tN03puc*6Qbn=2J-glH%e_{6qLx zT;e0{cbd|;^UuQolmT^?6o*%9E!$8^X4v8qp|kV+wp6(VLRi$vTnPhBoLAio91r=| z+6r*?J*b?5>N~FKGoMBu$y)fFJ(Eo}cO~wMBBC$H(Veky-=hBiF!kl}P_FO)4+X#uXuj1~2CLz$yUoI0>$Z~6crNA_FOH@*fN-Y*j4U*S03S63Gt0??*)f|KO$AP2_X9iF{ z7|R6g#GwZ~jO02!y+PL22l2Oc>!^_q6pSqGy6556vGoh0h{kOMkom!T@|-?6U8D4VQs zT^TjnxMyB|pF{@?r^@FxU+THYkE^S3>$Rs*##7IV47XQ#KkRtK3O>V~g&26YNG&_p zO}<>x{l2JT=hD^=4TRQ>$AT_?z6%fs5#8}0Z)>@9)-qg`{$9WGyXpddOOMS{`h`rq zQWA1e7;BA8O#e0yYqO^unE!$!bl+X#{t=B}Q(2Jj0zO;x^ytqIQ{@80oOEkKH8?Yi zqyM)We^QNK0Fd^`f!H)|RsfA@XaFi^<#DUm!-J|G4GD$d!Y%eU>espAHaON44IcQi zqByr-M62oI^C*AtH&)k0rqI%2eeUqe_p{}%pVlh`z5VZE!&Q)Ww;Av%XeyI~fM3#s zj0NHFBCFzuAj8NR+b-_QZ{NOh_O#P4XJP?wKEwffc_C{=TSPuX5d}%L2%t54d5~1l zJZE93^y=3vd=zPichFJB>!w4x=3wRT#-W_izTL0)29-r#fZ3bjEZ_Rj%gOM!!3N^y zwDlguvt0`uA=jwgn|uHczsa%cz08kC{>iO`x*L$msf}_Aj7ii-5si}CpjPs;(f2we zbK60LI`GF9r`g-#1_~ya{X6i?p$PUy{t49l?-qDeexg>BhhX+g~3`PyHr<*D*_^lo8QYQ(X(c@=+(;`ek89 zl55Q+tiT2XJiT0iNoQ3m*w@!G>w(|VcQYu7#mP0giMSjW?-og)iM)*_oTX?F%YAh& zixC&LzF5yK#i4ERuZMZcG1avb<4nV-wxiwqxN zKZ}AvF_5|K8b9iygHAnMR(Nrwjx{M|Xu~J2KdNErg;yT?uNv?}pj}_jrO_Yuf756i z85p|(6ZM-LlgBiqzrYz~&Hf;#KFsTx(Qj}go7pUKKZftBaA<>7Tmxw10g>zAgPq48 zq2i>CEDo)$lD^Vd6`U`CcE-}-CH_76>7b^C;y#A5T9K+=|HbTM;_!VdduXQxiT;px(w(Tc9;ssNX=T*NJq%+D zcbU8Z?C%i$k_&yT%&l_>_IUSqX+4eha8#`&E8rvi!uH9P76z zz1l&$h?9J_ae7Of5EHCVgE_nTOgn~|j4)OHVIpHhlDZb7{ZDa77Zp*9cJ^3GTDAJn zfUd(W;WG%7@09s1lphh5?$p@Nw@%-)ag@1ry2f|3$T~%jrwG5Y{|!3KpHnyu?Jj*OhK^PcFKRK{Ni}srJuXvhs%IE?2XFz9 zVC22C!O&);iaM0fd{G9qc=%%IdZb}l2(Z54FIb-8gQ52|byg8sE<`sZWFf344hP(B zXZ5oxEkO#M`&jWSC|fMAz7G}=ab17?J*m(6rjVBPmnU#I@+!&$D%?;78lIfFixFMc zeSsa?u#H3CwzS3iB&eoCowhCD31AYa-iTFEFO+o2+9vE(c`b@5pR(XRnpxMx>qqZ} z&?Vtyu>Y++0$&CC6&XZauOMq#oSza)cig{RMR?nSasBY;Np3EJ-L@D48nM z(J9T>(6vy9!%tXHgI!q_Jx<#QbiwkgFGG7Fr8F@)v)y+JaC7v-_Pqr-+SCMIoH5W< zeDZ6gN>eYBrTm>0cuL-@oad9Abe`F}*Ltvb$dB}zS20|fNBQcKB!)7%aSImffYe|C zWjT6Th$} z%YEmQ@VGDel(&H%7!h3W!Hn*`N_VV1RAKd%ll-6iFiczL9#n7fZ$>&6V zfv7q&g{K!%hmk--tN2fK+aW{mnF!=R;cCN4FP|>{+b4;Lig;4hKYRGv0wL_pa>?Yi z>ZK>pZj_1=iK#>&R+aUTzK|j0wIhSYNCvy9X@aTP?+;H}4VbkX`7zU&G`_<~_7P3KYhtE#wgMa?0(Bie{!da5_rT%p4? zd5NW2MUPzN2EzTbo@_91iP1oHFIzeW7FvX!y*xs3(ba49u}C0_{+6UD>$eWQ}j;avA)XyUguGE5JzU_ms>i zdKgtgo{w~G>dj**ceDEXS|6&m8V5U$z+RIxVDUp<<-h*bSRxncLG2nzuRTZCP|E*qrTChxVM;}j=M3s3+1sA{b z!wM8%;4D3B>{M2pD-0K=vWRKCa&N<{2K!cZET~W%42b9T42^%A8-Gc2VspJ5gkTOT z2kR1!(cJhHZ*qdqPu7K3>H8OIDH&>#N1z_x>5ms)T6d!ez@x>Qcn({KA&2nfoTI=I^oO)f83eCE{+t;KDkk)TRmXOb8=Gq02<$rVSFQG@q`46esX zOOfsFwpSPC8^~iy_@r#J=2;jv4)c9K6lD86Y>>J}I`w;jQbIh7qbtcukx^xf`YKi@ z0*Nb3Yf8tsC<_zh!Jvdu9I15@EQPh3I;1|lS z`t-dkV8n`mmE?i4^|2k0t{!dYZ0Klr0t_BqR|q=3b?zQ#QX}PEel2#lNQ~HOXX>;> z=p+1FhHzI+8K#aJ_Op~#x`y-{3= z&YaUjF9gR~MP6qmgl$)W?!v-rR+j%jR}5LODMYFXZ7)!94QZ$!D2UhA`5w3k>_+rSe=J3yPr{TrR!q$rE!edE0*LbSzKN3-9m*C|7@r|8Gq-L^pl21w3 z=-k00Nnh*=Ld)%E#{cmOV;(q_n7VB0t zNxJNR{Am%4SRoy6zYRAe;2($%|LXc{;&hNOyhvq9M!SUcMa-3d7U6)X7F>I&Vy15) zME=Cf&PuRf&_#OOaiZ*OC)VEn{@8*8&lYJ^{ihlF*a8AX$m98{?c@)ZOb`08uC4S1q%qT{Yc)$rl5A zjUh{k4`>ZO7T5zjIn7P%-;;AbFDdEr&bdMiCZ%Szv#6z<{BnR|0)YFA6WQQF!D(RR z-oEA3u^#>JOpIqCvykivJOR1aKMe08&KdFlB%OoIZK~%1UX<$989NQ{2xl6zhDs8B zSe5>8W*lsE*&_rH7V7h_;QT5}3X^ds5NVq(z|)hA$AGDR|MZvb+6IH^|D0K(3Km^-%A{6LsIc-c`$SETXRQeMYgQ)nS{nwiif?Pw6a2VffvPhr{3G?u)&x z&;B}-*|7cI`K0b1IuPsO?lPFtrBbkXbGzHhekYwOcWRX=g4&wRe>ZFvMwp(W~b+-zsXEQMi zFK53@u19Ap>H-o49SFw%rVTOR8)5D?(pznQ zuwJQpC(t0QHKzsI>Lwa_su$GzbY7BPNF)yh8COIltjPGp5)Ix|oI3?Wk|Vs7QAiG# zI+`mdHLgbA83m-9m!Po%RbfP;=R>g{96~zehwt1=D{&sMr98-lW{6!Y)jBwChxf5S z@CXm7(h^jzmRxc7LAB(ji~j&0c{PNB@>oVR1dTDC^wb6d;9`XhCT$nZ72uwHIXe%Z zLt>W2Vh3DKAFH~PB^nJAGHo7XwJZ zOP4D$NOcEFYV zQiG%p5{>^tI5)gS&}=Ye3a}SgVFT#fL@0#tM^uSW1|VyA4?`y37O(P|*&r+11u@+R zlCL6pJS%H{hvBWkbcqLuZC1&W zA$4~u|HPQ(y#}F5<*Q?hs@K6Uz8OfVAGhB#E6(Verk8ESU39(~ko5^O+mwez6|e0F z&`>0PTGvKS-u}p=vo6#XRY5=tR-N)i-03r#(Tm_)X;?VabYqRsr<*->7n*e{RSOm$ zKgVAiJk`v+0Tbl?DpAx37YTKcL|@q25V0CkNHZ?{o`s@yqAZR^h8RZ5g1B_x=j27i zl(z)H5V2X|OsLA^xVZfu08z`fAu1uK1PU|`zdaf)1n=moGGsFRG}Ry@lJTE2>S$*p zT&-$xb#2&}9`P->%&qe^eM+OOOl?7lnymg((4SL?`C*&^pM{JIIczx*qDQqMVH@eh8xA@L31i{T?P zjcdABTY?#Ep55mV4T#Q~du_Z_d@Pf2#s)k3MY2I{JL@u94S2tE@>@~722B%!ICZ>d zLf1&8T?p&gEG(M!4z*a2WLx1y>ecX!5syenW#D6G-u?*URsb9t>aYoBN3O36vITG` zM?wUvzX|o~s+f+Y@i1WOyx{0Pe*4Jot-EJmz(CmBlNg)^a|^@7Xe|;? z&xrppPsd1g^4}X36>(0b0W8}MlOwMQXEz;r*Ixg7E!@{I&e=Xn2v`tI*7a0>ddOAw za%RYS0gXJXfxcoe1@(&s0kns`%u?S|og?H#x9`{M& zO&7UUvg}}MTb`GHkjZ%hVesBFnv`1FhUA^@-AUPE7fO?U^IETX3M$pP+Jf@}t?|W* z+47AdSa5e!u(6d07%EH8-$y0GXv`D7V*Vo35Gakt33DGm_AC)a6i^5lls-RVH2&d; z42P8s;AH(hdEDhf82ne5U=bGaJF}Cs^W5_H3~`gBy@AK~Wp7qW9*F9vJ5_vQxb=mP z8|hW}X(r7dx-iky9=ZPGqsAASTjI;LY&l(zuU(*%@tjV1KFPc1xd~89J++ok=RPyOT zW&W46}&t(^n;h`Do8gcq`$VNyO%28=<3YbP1pmuQ=|7~|h zel}G-2`o*pB<^z_GmfwYP6vksP&CeLZ|J_%ql9^ZBX~L_Umq$sQbF$3IV66YFCzS(31(g*|1}KZ)AKQTfBIgRX6N0PlH9_JuXq(GZ(uy@x9`%b zcTT=3>Vs9;7QW)c5028uY26IEY4THV#VG{VakeKJJuRQzLCC`C2bi2SC_PvQj$Kar z^^ei~O#*+?DTrN`#LfON&h25-*eOQTGCl?A{pw>?k+QeT_dxjC|V z7yFs<>$$+n%-1yYZc3v1K}xS?qZLDYk=DXYD4U)$elpphs)Eq?5<9NY$bXfr?f4MWv-~<^_%Za!_!Qky{WV8tJZ~v$J99 z;Oc=xNAEEbyUwg>OO{a34H5K#tL~Cp=^>%fG_fsmS}6W;{Y{tCeEh_F{0*g}+u7Ir zpGXmRq`J(_Pf&U)J}^qhK#CXzT|3gy3ByPw)TpwopEZ*S>pi)#G{lRYP6{m-KeLy&&>TM zFKCbnJ>+}ZDcF=Zx4NBK>}c^O3jZM>SzLw!j87LycpGQ5=n8-hio~}HV^!k9_L+YQ z(ENoMi9S6yj;jl>*Qr9;qb*w03OcNt5{FK~gDmav#6+!m^h%kMzK-Mhau-0SKMJhP z{5V$|vWiB1zxvRs-5MG&V9;5rp?Q@jjf9}&u?z+N zH`z#jLkikJ(r8zaa#xiqcy+aNb*%U3ApErz)vD-tSrCbx@TT`?Kne!bk2?4W>ZQ2|5ME&cJ5ecZS7zkz_$C==N6XnHvw~m9yA=sAJCDaF}DksUR8* z1~Bj=C~VIn>I9EAaSZL~PAu{>RVJ?V3K`f#9Q<4&t$!7|(^&v`f}jT7jLk+d0cH{< zV&4M|?x>jSAiP>Ac?cmH-lJcskbI|fp?m}*cB2zSt(!W9^cO^qdUxA~778Wjn(LLM zGobMO`CQlRh5F0rpwFC^c=TuC;glSazpkYYn~UrYdu!t)2d#gGoNY4qCVq`Yl>Op+ zZqeCk)^c!R@|#?{Z29k+e-dq>@)05)G6YY!3J9RE4Cj5RZmt;$HzlyLL6*75)nP)z z6r^8DSgV}?QV!sMc?n|~4Gpxp=!XqgPlN7g+#~bG+e(>bB z`&4qKTL}D-q(Fuq2q)w&R^QE-Y7c$H(tWL1WdIdG6~2Q}VwHy;V94R&hz^2JNlpqd z3eIY&RIy{P8fIR1M0qE=U|KI~gR7f;w~+Zn0q6Ith6F<)9VY&yuCKdnw|Bvnl?9nt*Q8iZXS-swn(QoS zym_;Z9Tj=N8c*n354Y<_jrQByDO`meYT%rzir|-lb(Y|;P*!U!-MV=@9-GFqxa)PD zuC|3PW(uXR{;HSs?$j&i64RhQv6&g(U^M*l9j2i@EZsDrG1MbQz^PA!tg5pW=0gE% zGq(W?5vLcxXkEKGUD3Yp?!7h{bpNp(f>jG1Z4qrl(4jWaV~pf~0i4*1dMSVk2v$)* zSG9yh+=E&ughhc^(MazO2h+fdN(hfxCMeptb}#u1u~_Lou)dZ~=?S5KUR?^e7XNdE$|`eD)vO68>P)# zR&bERfvSN^6o7Tyzu>Y}Xpb6xR`Fh=8>pY@_!y?FEfwX3GIbEOiddXOPDy!7jfY8%uTI_Xh$ zK>)N#AdQbAWHH*FGykD_nusvG6TR%f3Yj3HJWSSZy_4F@=Sms=YEqJ&wc!u<^|d5H z5x&vgk908ar~uzQ)?QK~s}mj3L@$CgM*5`FaxcZ=aiyPzOVM|L*GK@oB}Jibs&;P) z$XOUcZ$7Zf=}edTKQLk{awf?1;RX-b=<#Tdbs!y<7RJB^nc$%6l|%H;cpD~ELaH9H zupiJi?qVf&5hKUI-%NU3QxU>czBJ37>=MjW9r?rV^0w5-68AO9F2R|G&o>v0jDji2 zW53J06~07Ntw&a76Hv_3-1KT8S56A}x;!vZUOMn_BEx}{au&8C`PmBq4Injwop|oG zrjD)LRmgx4U326GIQgqb&UPa5^_@Km}`1Y;F=mM}hES4^Sku%h&&nS-F+kNwe(#`H> z?p>0=Q!^^|vBOGS%HT!OMXTY&`hG;Lm*vy`)CxF`X;vI*~b(FkPjlIiJ%K~ZWU*R3d2hYzv+|37NNqRrw|>5yaOJ4Q_6!h5O{-KT`eq} zg~Z>hOmRHuN$&F?RKv3wWFX_~egx;BoNYW)hO{A{3w^}Jqk6&2d1;MCL+Pr&C5(d^8y zQuwQ@{pxIH;a?e|n=U)RBb-+_gcoY6KiI>@w2-og$s>iIFboTd|Il@$?I7U=^E*^E zqvcTxCT`)*!8tPQT>2zHJ|V9)aHqV6E@C}p5x+Z#4Z&Gu940FovsVH7^A7UR-tl^V zPj!ittV>`ct$ptLhu*nfZUSZUCH)ZPvCxxz#f);rm3egSKkj-w8Vhaa-pB1Kmzdy8 zx*D7Ia_B_vuJIIjOLPfx(z?L+Vu)=L^qXz+6m_w}4ox%()h0bca0q7uR<*MfkT#va zDlS6Y9x1$7? zO;=4S)qpGShIqHh{txpINNZ}6*5bq)`fTKupwnDAo~HS+_0~p@jL(0AhbHbAd~<9( z)FZ`sbFnBl0CZ<%x>)@VdPV!hq4{swrn#??SR zH&YrPy6UJz?&fD-tO%-XmvSG}ojpshgRt22m~jH1^NYf93K&oO(Chodh|g5=C(V^D ztP*7kW%yOpoEv00vG)QEIWBk6o2?bSP2@<3rL7Wjr-rErY=tX$6!l3$-I@Fuk#Sy1 zfxH0F8*m^XvY#a`{{|$qTu{k?SKW|vP59({=pm^RP_X)7hA1}f{+EXF>k zaS0Gu%a$#!wL&&&82XuFoAzXNdVM;ELb%DMJ^7Y+AHn*l{%7GIt_HA8TQo=iPT5QO zcR%W(9X=pq{QpU&+J|kHS4_r&EdO-pmM;GpGbzh7V`y3-?ik|cX$_Rz5PgDV2%tTq zHS~ghYh*YXY@Bj|{J2P4@>EV}b%c-Fq;E`ZoY6WYB{-pgd|X(gh%_~^d%9{Mvnly+ zg`A}M5g*EdftNOh6Jkp9My5R9xw3i{T7x$kWS8i22GZ%#W?c=1+n_W{s0&*KomZ+z z6u|V6kP6g(7CSK7$O17Z=otat1MSH+4k6%QIb>A8CHg`8vhPUyjhVN?Ao>CIxVv#k zsBo$xFD^H&6y-qOXM3~oR(gztnvV3#E5E9dadydW3{|2tw?Z{<@ckuwFE21K>K=Fp z&0}t1rw4#IRYQUHJGI5&JTUa%{A_YIT8Fm8%<07y=QNx_V;36SZJvpo&DiQ+rb1|D z22Q}9PpCg%33>!o1M200dj}?=Jz|yy8l$>jtVjiy97;NPKE`}nocz|@NNaL+>hLlb zcw)yAk0qXQtSXQl+$_1Wm_@xRFM0i(p>!K00Ag~pSLuO%s)Cuou^~R!xan_GAfZ*)*|Glw$}TKKH_*qqeq|QuAYwmVwhfbM%C))7UqP z$C)kUmAR+USAi&V=706PAPviKFv{2nXuaUgW}si{8TbxOBj|epYxzzO&mJ*F5LwIM z-c7dOD&V)(^glt0gncO?wYM)rT?oooa*Z`8LC#QmWaWB5VQHuC-IhgdKSdA8zVlym z&)TxU&^7o{i^*XP>L*4HExID_mb;pB=T!L`)qTT%U(-swa3Z*%nfRy2ha3Qn;W22K zB#=m6Rs5wBJly1KWS19_-o#85-6V2uFyIs<0FDDw*OUcway8SLufT???IK-j8}gfb z>oiyNuE;Zdp2O!=`Y@%~!`ew$vLBjeZi0C7XIQ$hsYz5Vg&}5+=*AtC+nZYg1B#nD zw*To1z{pIH z%;KPeKGLhFs02a;xCp8J`Wm5+PP&GV6M&scT$a$Do)jbvO|R~Y%-ih|nF?bWdtEXc zTJS=PQ20>mQVvBqQ}mJ`)Wp;7dp|}0%tD(BFV8p)Oo4PI4doeuHUr;{Rpfs7YD=yr zHdRigiEVVN@7J==eO5S!VR-qz`|L|ARjl3Yo4*qltV+?$lHH z40jIgy+aGVA9`KoY)7*eWD+S#9!Yd1kQkO3^ZcDKdNjXp{LM8<3qm*#4G?R{ za)htq(Pkfn1q)^F6gzx{P#oZHTpThnsk7IK)-4Z9mL{kK1s39!px_O>eOBawW}_5Ez-7up}0%m(|LXd58k)AGI%`EQtW%>oue zA=~RqUvV2w^L~S*wh*4R9LT&&46ri;ZjLknF_0*CKlbNlyu9 z)<+=T;bWT)bkd`}DbdZ9J|0^;TlWMpg$`3s)RB@^OuK;I%R(2Zpn;E-Je?Ha8&YlzY!Q$)et9CK=NP-0j zc`w-H6q&0BsClBeSm!G{^3%$^t~uB~=(04GhXLl#UagYwciMd&3uI+)|EWPDjt8E zRY!A0s{=>>BYfvSZD+)EaLn+V05d#_bKrnjYYH`!?`vADfvcBnCZ`3;bK8Lbe)wAu z1@5ithay~&l4^t6NCmXgJ`q5A7ctwKBm}#;?vm0N)Xfw0DjHxmT>9;b!2_%YO(4E42b|8NsC8uyA zwlt(jbauwiX+9Q=I2vfWRp^8HcvZ&?pbquR>?UJ{a&`|V6 z{lNzA#=h$ulsdshii5Q=Q7{A*46{Ih>BlMx0}#OwV8HxEfHXw-9I|P$2OlZm!nEFi zz`XU;O>51efMw|R*LXhi6#J3ZLY_vkFp9e}QsDp9R3uG-w$#1;*((yxPH25G)^4v$ zy8%gjlKXA5MxRyak>Xj3AN1~^og^y&r9q)BPjvqKrTYW*9w19Zmbll(wib!Wn&TGf zt%O<;@p>2?^;bh#0Jt3HP95N6h?*{^-MR*My%KE&TIA4Gry4%_+0-=w!qs=FS!2{` zpdTtcBLW=`rG)6_K0QbdW!j@Xo-w=TGQ1NXgeqaNdtmq|`4HQ5(r@PE+*}?f<%!KZ z(?P>-L2u!W9(E^{wgi0-*xhGv{X2C4M4kJmfkc2y)xnzWhc%O^PhYv4D2b6`V9GZ_ zUDl1o!974H)<)4pAbIvm#6^?SLsUP2r;jHr$C5YK>uMiRf*3g5VCUB92=ocV5q$$; z&5I6nDFnOhuKX;SAAxI*di@$^A~A1v9a{Q_So;0R=BuNikE?Q_VE?hmOeg$ax+ z!;0n(LcBqY8JtrIslC!nW>!y~+-Owj$8X7L2sk(Kc(j)EgdPGVB7z%t063M7As<31i8MIMeDA3$NsL7Q9|c@r<)uTq9JUaKsY;Vm=mHsxFc!y7Td&x_rCX+ zh2yeUDjRKmdpe0dLn3tgt(R1?cBGj7~}vifyxs?N;DT}l zqv~9Gf>^d)r7>t3WwUc$>i&lN2s*0>kU1MGpYp7F&>Z%vlF#Qg@Lrcu9Df%r@no`S zJ{(y$fIJntC#AtPY&^LVVD7pxum$k}|J-*xud(lG*fV&6wo`8XOeF9kY$W8KBpTAH zxza>r&C0y~T2+|k9=3J|BzIVbj=_gLq)Sj|JUKt?yxwTKiGrf^7ER&c&)$^-xB_0; zOL5Dv53Wx3I0B_*JpT9%A7VRn6{jNsMUa;SgCEHi2*4VSbMY~z!5*Mm=2U(fHW3QD zmbov&I@H(DPnRKZZwHdY0S4?}0?M25M}N?W0RleCHT`Gbyzu~+q4nKIwKzBB%lSCCbMP^u#hDlYq@ z;I3E!LURJwNg58LKkkNxCA51`Oi$9rdo?LLdz^RlI4^Oy*-)_6E?-fqPgqB6F*G9B zIjRiCAf&lfsk-*+R>=cR%4hA)pgxt`1Eu2ka^FzZmyCyl67yQX#JW1wmmk!ffPpCT z8Q_0GC1B4?u~s3Sv#(3xfk^&MQJCQd`~qlUiO;Q;a;Ac`GGkUJL23yqYT+xQgiZ|5 zyKl$SXxT!+^Y%g{5Vgw6%c3R~xQJWkYFCVB)HOWBb1(Gv9F0M|us0r;`#uPj3aT@| zK|vz}WCo_*0UPu)vLZBG=hsVC*9h%&jSz^8;*VRjvP?C0qjIz)Y?FLr;5o+d95~>0 zV{3~q4;0VHAXLFC2#&?A!h26Vs1ElyY*R$EDgm3raaKRDGVB;{;YgU)%6PmVvTCWJ zI|9_4m0?{;w{zGpsX{}A$;+{h6_VJ?u04AtlbbK<#%^d(@-{UAUGjQgL&GI+^$1z` zpvWqHhw~%QU=ib)m)Es)`FXhnvDHgNqYZKu7Z`hi$So->R~xhtp6=uRv}gdNp2=RD z--5L0TF7DTFjaO}9~5yn^<8lOP^^tA6MZ9Yu3nhlH+!_USU)HTHH& zl>ocP=E};^k&{+R;I|}s-d}&m982Ccot!}9;RM6l1r=(qMw8F93^g2N=X{C!|I1V; z1KNA&mv8HU!OXY-8y5T%;_c(oUMKHBt`VVp5cEz!EJa-C9QvZgL`I3Y<%LJ6RrEa2 zB9lMX>oWE$EykR+Mvq#N{VtSW;Ns{@!1BE>y;SAHU75tcqD&oqO$UsAjj0$;)J1(@ z^K6V}W3(X^lg+GQCnektH)-0kz{4q4G{-H}(qrXqsg~q4;Nr@x&4-|j)b+d21ce!P zG?ouQi9LA-*gXOPXohKS)vC~yXUh;nx<*5>U|Acz0j1ZqC^U3DZP0G6t7)@ZztHF4_EKbmGaGO8F;SWY9f?@)HFqAUY8^G|40I}GFE&>6@=4?Ar z{q52;BWe92g|*syRGt)`EE%Oodobj@8SBxJY#q?`mzw6`3Qn!^2oY+Cnuoz*Y+3)KS0fQ zU~ZsWZgVUYc+)j?#*$#%0a6ja48v3={E(?9i2)MYrM8W}7U#-#tq^eeH)B>!F5Icu zOZBoaf~PoMEj=2Q-~ohD>EDIZm}FnLXMvgjk<$N_26KUu8YnRM^tPj~*>&X~X+YgJ z+9v2}qaQJ%5}yklocr*1ZNJO9X#`QoKuKk{j67;UcRzsl_@O>)Arf69Jo!SknQl*e zpZNP;@(#u5hPLD8jL^4UJDt!;8%Y!$f=u7Sug{WBq37gfi9iuOuLbo^!|7Z{ERCI} z?Pk|?nzOet4e|2 zD=h1uS6@Y4@JEvNJc0+N!3^D`fWk=8LFdNB*hp->M5o^xgGfgk4mSuoaynkXfEA;L znj632kR?sCirtmT;)im@4E7atCQbedy@~27s|VmWKTcUg=N=)aPh~II7?;Ya^X|2E z0J6!g55vDF7Qu1DCmg(Xe^P>`fcS%%f!-D_ox=vI5J(vU`}e^KZ`Xntq{9}1@H4Y- z0$D8qq;^^m;kChBFc1ONuPJ21M3#IPYLF91WoGw_=zJ@ry@-D+0!J^$Qgo+rHdv3EtiGUm< zUd^H@kkX~0`h5PqQFapE$RP^(oX|pY7YRs68zx`O?&?_kA8(z}NmU42E9nfMoPT=3 z2JO)L{)8*ZChXo0LjV}A>IXW2k&%%>gYlqjc8xYRWT-L{7Szkss?1^U|0;)03|w95 zl8?YyBPQefAXs4kamDeH4H;5c@E#bwjXDJ(07gp8n&7;WH=!W8sI%JI41C?+r6hPj z-z$WEkI5w4+kDa%u{^)9i$+5DBZ(`fv{tLOKZAClv0dZyJxTJBH-F~ z;g~BnE)f5oz2Wq0xZ!sAd}`&i9Od2g>%aHPJh4C9xzZ~1v3q4MXxi=s>5Hfjncs>< zx5BIU)p{GZw%<5hq13t;>(E?NLMN}z6FLWoJecjXXKdT`8fx(FDo}F<92*@VjT^a4 zc&4XrZfi}8#-GGxSpEy4d~$+~Lpt-gA3t=ok$cLIV*8kTni^Hd{ms(J*F*JsD5uHJ zw2)}1(`PqeI$+KbX+*FjNKoRKsZ7U9m>>%LC3JX*MX($e@N-I-CyU&ji(j_@q=XU0 z9%%Z!1LzN?q~q^wX?hXr5&{e=YZ%0ajj+UnKNGZE|r-rP2x0Dwh<e7-VkBJ{D-Y>l%B>V%S?y8?8JIW6fD!xELph}A1dxI}ksP`@*!d5aE>cv}?#XlQ1%IMk24Sxt+iLGgu=CegvK<3j4&@La2(OA| z_+0m-D3x}GCs?1yEbfN68C~v9G~fya0Y8VByIyTvK_F0=4ZIv9|4AVI5PJu% zHW7#FAV>H_9DhM8f;1tZJKeeiALryCd>=N?USM)=buC?%+=;8L;{BAxYXzVMvei|p z3pUbsBfcgc0}$I_i4av-kYmL{OPRZsgZ!h1XJcvH;t!rY3*K}R?>qk>P#kARN=!`N z_m`=~=Khg>86@=y1W+eUH#ND|S9)tGHh)v6&IyH^JMNJUVS5s?_B@q;;a+zg^0KPj z4n-@NS_<=yF=<-|3w(%R!G8IxZ4myc^zLbU;nMkk>~rxM1LEVVfr+uj5J&}M`%R3k z&fywCuS1o|NDp4cz}gUIip|d(ac3z|T0p!KkgYfHTTj62b#+h~VAjZmd-v;n7zZGGBsC-e4Adyp>L3)e{)hVr5tp{BBfIp{n})Uv zmKlbfEklc&ZD=b>I$A&MM&j(W0;nRMHD{?=?g%$xpLct0*E*mu-?>!hhYqyVEe#|1 zjNkqQDok@0zs2A`Sj+01{JMwC5?T@*k?Nxv$NS|5J-AbSHul+-;>lctnJQo z1J5cc!<}&D>%p$Lzcn-l4FQX+5Wp9-A+En*eNZ%z%#ZwQ!7;!RDPQr(`SIT1f&_@8 z@H{m`c=V=^yg?EK*eh@l2eSgf{Eo$vvCY9PnT^Q5hS8ZT%P1QUt=X-@lYQ_?crK+Kq%7kqKCaWzbyb(yY=CyWNFb!UE@TM558C? zdZpDvB=v$gA)mGxsJ^AoccE?h8)-zr3OW>_k^y{!%S3KtT_Yc*9ya595q0iJh#j|UJE-Zm+-Hh@w$zgmB zjRY2+otq^ueN0>aIOUk-ieexp^L!0qcCoe@K1Q6X<4GzQi_17bT$_82%LsauIT{D{ zGh$oyM`~^==TZI(P7wr#Cb3@q6iGU3doJDI)DYQh0_CZYz~Y7w@1h^SiJ>kDQ>pRo!BV6Q7HLKytqSjx%=B!?U(r_dd)F@3h-u?(U+s|3c^tB9iUg9xIiy5dLdD-Wr~ zbC~x(uV3?BHryX8AuNrj!BvqarnRb3jO}M4Xm8dZK zX^8kRaCKB5fOsCJcXp@=+W#Tc!@T4nFNdiJ;Hyb;{|C@x?FznB1fk-GD)^^Ey3F!t zIozW{Slr2jUIjDhXt&_&ppw}vt<_+JdN+$}?aOh@Jb5$|B0v$J;MuJBWFDK%u-e+0@F)P}LB>jc*bg z`4Jc9walKpixkmhR}ErjuJI{`Wb5B&AHq;4oc#4}Dj;BGm1R!f@ zKGrbthB6T&Cd|JyV!=xN^t>Iw(7SolXnJ=Uhd*4+g&m;5HRwpC)^B0*%HeRL#tr;x z)WA(B!%hl;TaHj*86CUCpF&In_;;|nE3Y6W=6B}i7$y(?Ibb46P-^#UAvXhLjcjcQ zB<>(YhupSRqz9wLV{{6aNV^X?T~sMS!*AiA6<`Rw5@--N>7z6Tf7ghF9ih zk!F_i4^Wf>jY%6%MyYs?huUtD{eoF4FtNhu;-rW0K~kdD8G!#{Vxr}zqq3Ubr*>K< zqCG>}B1wD2GCVw9`9?~_Qd3tIBHwGGSqZX*VG#g5Fm9_#fSMI_DuWdWs~R|Gr&N<* z%B~nR1Z|n&zN1OhE2AY8yMaOHR~=nD96oP8{^r6|7Fp^LjAv;$&E5vL!IkeO*da>M zAo3vaEsBa_<9Nj!$h~N8f*I6=zg7xinN={S7UUzJ^Zt)T`3yKBP8hBnk!#bnQBtL8V6q!Y47!dQmWVmCQLvW&6}Mur(0X3Wh0{*2D^|6SMjzPg_4d3rSF^ZwlL zdwJcj`@UN~A|u)nqx+Ya_@J6YU1)7$A8bEacF8Tqcr~o%k$Zw>;+tIkzeGF;Lg~$) z%0&P~&rwrp_#))d%fNv5723m*!WE9td=h?EY(1sQXuaGl${a$|1$5W(pN)$RzQ%rO z=ty)h&&3y)dHXJDLtdth9?7AFBuPzbMd0wC|7X5YDCJg{mU`>QKRba$@j9#p%@OL3 zqIuD2-gK(O>$RIjK^ydi^_a)&0m$&v<63?vA15&1Ot6)a_SH6;57r@1C-Qr4=MIIU zR}48hpx+#t{OawS6$zK@m;u_%FPFiNMr}rKmt3XVU_~IeovEZNcFQ;W(AF~*k?zks zE*Hf_?o)S$2}814EU`ju>efAHvtb5aq1Za;oM#%K)Igx9{Nf`%Hd!knnn3@p zXJ=^tRRxvuNfKD*&;HaBc4s=(eb=nP*u5P8J*y3R8eWeI7lY}$&>5B z{nAr=m5EJf^M%g|2h|0EY-8K#Qa?Hy^ObOzAzFb0ivUvsYQq}qV)xZ@3NjPi`Uw+(RA#i>k6&s z>t5Fv7u28Jlmn=Df9)=&KQ(G$rrr0U&auC|Gm0goY_rKrof>(Bub_ElqkDlfRKm;f zmf6=WAi_#U6&UmSqsdq?_U(~}_Aptx3TGnHbYV@}uPHX!Fy{mYnHeqOuKa0}f9V`^ zdF>v`M@|qx$=35}+&SMD_eVuCa)ovfa5-om?aNTM3tft3!IFYI4?BPE`L0jNx(qML zyzAnFl)+@8!sGlyO`$5peQE6pX$Ez6c(oAN{RG}%#FqaOW4}RzxHQ7&Kex)tGfMMi z<&`eUg4bN>}L@^u;nPo|-Pav5OC?!`;M41(Typbu!|Wo+n! z1~%bp$!AcPANnn%+Lu26Lub?D`uPQ&<;ftBN1IFx!$3sf^9rlYW|P(Rp~h=EqNH;a z-F!?K;m2iHugj1@a!MJvuvZJ$)z&ViQ5Of<{QG!_F|z?2TJJp2V{d;_Gy#U7of>(5 zU%s0`3?VC`X9Js{o zcK|QOi!sOyM8`Ev!8KUkM{}y)x&ARa6>jfD>WHSDBeK zs9EIF@U?4b3tidef^K##40n>0#4Nqd$}@JcXRqoqh54ko%F?m#kWSm+TnEViQSpC6 zqi^Z{O3)aFS2uWnR#d$&iaGRimtz#Dx?6WEt?hP!Q<+|bJiMwknyz-;}dSG+|50Q%M+$T$A8W+y=a=ziWq~)Cath5n}b#?;cCB z-UMbj*_ZT`mG{Niu~Rfbk7Slh1R}g{39RKrzb<%c$WDb7v##X<0~hu|FeTi0^!ZLj zFv^JO7b0xv&#E?{Ae{X2YAa7S?!HYrD$A;J2|3jSsxn808(6Z<@JX#H`tuv$H1_!> zIwY@hl$rabZvBNdGB=3eAs^bG^MxAY=T8#jz-<9QhvpjtCMiHcg=`!*j1D26K2rF?<$&5OxgsM5N_y2Y@&+vX|#iXK^=$x0#@QKp@5N?u8*%CJF5T z{61R!b0H|f7%U3rDQu)c5w=kwa=oAri2n~j{6Bg872^T(Ah zL*qUyaD4X5)gxK~WhXAeeNT-{$uNgAa*iPTaCvUa3qgiV_SFgJNKPtdRI5&TxBU5c z8?1x1y|)S7?WmzqE13EuoYb1LTE_F>&I+3bdC2qoEX5WlNw(qu&n>2qxjkll7rfO- z!wp+L9#4ZJSfz2)05vIVg{CA-?YS~t>&$o?>Sj;IPYDw;`0fXUda&Qka(;P&)nfZk z45Tg#ZS1dv@FJrFjLWTrE%4KXq@!7Z0M_ zVD>}G)9}Vb#yuPU5uy!?-^6_rVJi(fgY4s~^FY!0Xaf4Ek@x3co95oV8}OT$cT#?G zNj*to$K@kk=rt?@w|a2AqN^LRbzA7>Zbfp8(+oDFczoHFBf(;cKw^VX1%j^Ozx1Vz z8c`tk+@=bEyq5ls?+mkJjo8t&MFDo>=&!Cm0goQbC_1UyaO;!aI-rz%KBVh(pKGi*F3ZPa1zc{?-7AaO;%#&S|trSAxY- z_viQjS}ANKC7MGVUu+Y7^@{Awr@&l^tD@#zKwr3fMX#=YUybVJwN|_$3N&TikvX=R z_VN#C$@+QK2p1!Wkq+jt1F$WFEa@YDp{O(Pg&6={J_1_J!h80~hk~zyx$kO%PrIP- zc@jUbw)8RnE?Odr;jw># zTqbzmylJ+SYgA{&2#q5(8W1xW}Bw5=NNpcx6<+)osg|b}baRLw%q&exZETafyIHpS1X2&}`@qumnKf z0OQV~b0`5_$y9dbhK+eA9E>xJKq(9SMW!b2P!a8mp?wh=W|MUfXFw`qojGjptM9+F z211{{3y6c8LF@RRFF3B*r8;!9{_?+(6>dA>6j{znE~){$(9yr@QEu z^Jl^VQ|MRk;|V@<$)A5^1!JZnbE|wreXf5V>5!0m_iNnUj_LIGg(jOGoiFQk^}hS> zhn4GS7CPde=Wl5kysTgNr%v?j#0ST!wZ~?4e`qAg^;$iMl8@Q;)=cEaNX!evn6z8^ z8nk=2#BCnzrhmS!B6j`Jr@(coURfz+id__iY*Me+t6VU}wM1F<#Jg`b@%C=KS~8hZ z4lR(wZz&3%g+T;nJ;+-%8+QYwE~>JkqGDs$*%#QdQ#E)gq<;`P8{!fR15zG6u%GUfD;3gYDE%^;5c6Aq(U z6$cauRfc9}W^p;wAD&d3C5M^Ke*4yQSx)ZuZQEL2E~Brvckxh%uQ)PPJ7WA>VRruk z{mMz6fRw^_Z6&2YVuc(8u_P-X6?&Cwvy7zd2UBeaWS*-5ODu6<)8Br>!5`uqE@yv|k5hztk_m>It<*9RhdsslXD zed%9C2=;F(D=T{!f;tLvY5{xOT;m)98b#2U`u_d9>B5C|e^-~6m$PS{bNw6ILEeUG z*nqfms}in?^mkc{Jh5*YMTV}H_?bKB2!I!NJa{Db_nhCa*Islc6zSetAS}8m{#|`Z zTz4^l!hBSk)NDRFVUApHMv6H>(H$XM7!-82)=rWyYaHHCW*-uuXyk5YY-~*8RA0(! zNE!v4m^_;+7`%ex>u8+4#-&?Xmdhgq@0QR91t2jgV*f&1@^+4P7RY8ki>$@U#?Vtp zP6D2RpA{^T1+s#aCKjIiUC<(!otO0&(BOH>w|=3BX{O^Xh@>n9IhEF0vx z$A*E`&`7@ta%3SsU-kw>p=y}*PL{f<#~a7n8SA0w59#NJ+X;lRB~wWdS^R=YW5}N@ z7o^kA1z(*zKZk@@mZX-pb5BB3t-=745f@}W<=WcjrJ;vv{2)s zzP!9H4wGSJk(!SC9MOxX_Ho-q-UH6^)OMMcHZ0=%b< zl1GDE(l&kUX+Cd+W+>Y@hC7X(s%IeBDF*ZY>~N{xeH!hkq-FV}rSr!KJRT3R0B+q1 z7@GeYeZAR?`;=$yX1b*PUCF$EhC!?t3Im&v@Uqk}bQfc`+J=Frlgi+~dpz_=^57PC zz+8vuj^P;aw&d!~#N%UhoR5l>-{z4wPTl7NbrsFsSR`D3cI9^o&Q*^K6NM)wUZW97S{Ifve{(^cMK3e+XE(tt=;}tStCO#dopOhyuI-xcOye81E0zJP@=( zleMD?L@pt;)p?ZmOHUJ$VM7}5GdNqKcj1KOP&QsSZZDE?P_h0p{ zE+E8ONBA4Y2uPX7>OH8NZr(}X=~Z)!o|BE($vP7+Mny%{jKAssGk6G&FjRHJ0OFMO z93*CPg|w~WJ&DJT?rOo`^rL%Ol}L3^b4J62c>8^jy!g&rV=Rh@x%^1fE~yJXB5WY+ z-Px#S9v=g1Nbbip8SW(OG2%rNJaikfSAP@pPF6PE>r|0DCbP*U{q5Vgj($x8z1XE_ zU;zM)eBWiypk{tm-N>bsu+0gQ+M8#@GU7^z^Za%)n`cXX8X4sSB8S1S)9S`iXv%)x&>AA9^n_>a>y_4r`d$u(G@EHIg zBcpcHbJ|_Ml{zOCj^>>n3Q%~H7f~!_SFQ7Fy13S^RC^bR^FH+*6SUO;NdCQ<>W%+J zEm&lqTB*7Tdw5}C!SL`}89Q9%!>q~r86#pvPDWx?-{%EI0nXwbsy}*MhL#q2;SY;< zV`C|+8vt?s0QI%^>~qdpuF0)KBhJ45{{B`L=p~cqoklXWJ6Wk@o`4@bJBL@A?Gec0 z&&`dKPxmukPS&=TAMz=Ij zqQ9RW##VnwlK%>K%TGt2padTemut33%d$MvZ225Fydsmyf|-I$7cr#h5o;SAgcw;m z-hGOQLt`tutH2~%FNYTRia2IUdy#p&H^wiq)7MDtr_3lEbFAv%VgUFb==}Wjd#}V? z`3v)X0n`#ttl8k34AZU$>l@KarmZhWaQ4ESHAWUmo?Z?oe~5wKd3Rbg>4Auw(R4xX zw=sDxj~Jq%VVvf_G@s04U*&!uh84uaME=hAwN_xu;_AHIy$7L~P}+O%ajUJ-S`l*< zZ#>!{5jCY-O~HB0peEz2bkzii-snl1i4?r@5V9#ypDFwZMq zRUYgWxYS;6w>TIs6yC*ddMuK2+r&1Fmgi{S&23^$JKt8Ps~>NP=xzXU6qvvnxgdQx3fi}-8lEb_%e_ah#!`J?^DeCMGS;%378GaE44oBd7&7^L$t8vGe5M>08{U(~{_spOZ#^PZTyW@i;Dbv*q*jaqo`p|8RNL6A*>}w_DcCCDmkg~MrUNqcf%J|KO3Z` zc(XNcq+)IgWHOjgLb_b%;=BuUVQM)tqtnT=w(dWuDQy&@rA2BQuZF6_831SH9`@aq z0b*r=Mn&?xO-|{S;+j&QB?!Nf^A?qff~9d=tEGrJEBvR0+X?(rKZ+jhV$iWfpUO$q z$Ei#fMy21kX5I3kRX`g>ERYold%l1~a0V*`0gDxIcc^d>HT;MU@WPOr)R^d%*VfkZ za~UlWgHNnP%=ZucsI`-4K7teApypZo@S5=WxVX5xek~fK?<^J6H`uM-uuW@SslJaw zX6xbn8v&I&#Ef8dsGde9u*fwu#jS%6n~?`04t5lza#iZSd+A%dV^|SSWm zS8(R5*U;ukP&X;!yT&~3Pi6&++FcFpGb~*qjN{0H`;I&|9Ki2wpJAycB-sgj7py&l z%0_8hXxkV&FnXd1BAz9>3D=t@YPt5DK=u(&rq8Ge4#&@8cl=Dc?5yF32q|D~>VGxg z4^L{RQC>iKR(sIg^i#ofSo(T@*jA0|?c2AReGpFSIGmev?U1>y;r+D^E5#~&3U(>( z@ro)W|EaWJ>zxK^&zn_^hjWy?9>p6(sbpLUs^6o?EWBn0C3{D;?YFCK(dd%vYVi)H zwK90_9llQ+nn1dG#OoA>*W6U z?S|Ohn@3WuQJtWS@%Qn5T|N}g4rp2n2x(Yv(CX*w>s#>fga~E_X+MN>2Te^)YfO{` z0!d2)ltDA3P@?+XIyFHo=XT$qVy-SFR*N<*f@D-LyX5>E6Xp;46hd^R#ODYl&4+W9 zu6T40nK%0y+Zhpk>PABayy^P0b-7R0?6J8Kk9|^&aaDW}>9V-k3lt7I-HhC72xjq1f?4sx%NmE<}g5y>BSltSu!2g)$1yru+@Ob@Y^(($^H zs(34y@xiS9vUHz<$IWGH)i>pJQ<}wML=zbwIA0>OF!)8YG49WzVX-zk<##GN2D&(| zEx&J2Z`_Pmbm73xvW2Zw_!^270+o&NF=>0TCQ5OSy^FN%8cpGMI4_MgI=()C_L zn$qDb;fXx}qOn5T7EShIV?Kk6&;#dy)In3*+heCpXWrdBV0RUpmcHgx;g|WRmnyf1 z8(C9{;`(m~1`dpebLa+9YOYB=-zOR@J<40$oy-{MFZ6LFik-6TqOI>75NkPbfd8JX zCbxSt|LcdYB7c88Pg?`rXoJ9^qi796=zD6QZ^TU)lgz!?F}(Hqbz~+J2>ACmD!SHk z=IyKe$3$999yEQ=|3W?I?2Xw6$I{Pt{!bNFXi@c`GcvIkhRfbljc51aIqq;7;!8n# z$js{Y&fDnWfG$K7IONICo{j3i*xn>^$|XJ49Xt*^YMu^dn6?-RwtIe+&7Y2q>g^Dg zZYS-E4Pg`36S@-DS6~Ow7P36bv%7l&u*Tt7PA$%Dh8c=ioj#(_*xqmvt3sZpSWEhX zc1XuQ=xL&h{>9iiL?it(=zZ>xQVuD%UCXA5w{(^)NN8VBS^VIE2MsafPmtHib@-8^ zT<7v&U8*MPYAq=qa6P_TT14 zN!;@p3NI9-r>3S_Y$Prp6T(`b8cDtiqFTGx-3Pz$7LCVB z;TD1PT!wJGFOqmgiEycfGv!RX<9Lz*aY^&=i4GY+0cvV$RCaG~OcDRotShB`o)9Y< zVBR}Azf((-wYw0h*WW8rUSyqyK6`Sz*nC1ydB1+bb#?m$UQPe6p-}{; zr+#CB196p&-vJC>0X;lkiX|DonO{&(-AF2BElgPG9FB&7B{GAK^@e}ShZu{`z!%+(?R}ZjXCAq+$prCbUHm^SA7 z8WlEd@U(3qK~C|xhrpPBfk~wR2BV=#*UQVDqPxUql3nlhvr~9IiFnx?DA5*#P&Elg z#7E5notRu|0CCwos(z+R+Go-1`q|JC z{5zSOUDIR5GM?r>jScZYbqtJ&7#QOSyAZLA;}CYlX*iJUhVn}Yfn8#*$RF+vsMp)7 zDekqmLQiX-yq6?_*~tAd_s_^&lo>kr z(NYp$u;@*#eDfn)_XFb|WMqV{ZTe3fokKjCbF>fOhO4!EPG*4u)a2=Km4a(u7na~t z>B*cH_D-q8$MI{vr=V3akg?E+hzMhjC4gD+LQ4vvI8ql0JuaoQXVAaw1Zhd2(YEY} z`i7$2+Id*|0oac_EyXf2F)_Ip5C>qP!^0u3$A=Wg+mO**`Kj-0E^nn+ct&lB`bCM; zX+$E?>UbeHRBvcCV%a?Fs6e0~BhnV_+^d5PtXj86hi6Nh!A8gu+7Am9My}_!aR2CP zNK(qU%EwR6ukqNl_^PyYtG*KA>AmXsnEeV(u{vH0m+_S?yXKp1?EbheKG8zd%)Lkt@rD;!Nr_xJ7LxpQM*Evb7vyt3EV=XaOr-;^p$=`^ATG|?C zA+OZ?=R;Z~vAKO=eNOCVD~J8+fA`+oxMu6%!q-wCHU6#Ss0N5-Allys;s5le$EiWM z%xTs>`!aBTiNjlo%18w`GrFaP>pdKgzVN)MhsE!Oi)UFLs8Ffu)V0`ganU%qrn5G) z>j7T2axj%i?aSm;q2FF8`H$d%HbR>m{4a+G!M=z^g_=&w^z^jfP2r6chtl4f&g<^> z3Acu%heXUSlO9}_Ftm|09>Y5>Y(O+uY}G>t(jBdK`doEI81?U2BM#PDLYpyVVS;6T z>yee+s)~s-G?`)xJ{^|>7opP$*|q)mo;`2!!QLpgLi_2EB?KcI19_z1dbu9<49O9a z+mB>t>z)EDTD{}XZwe4(FN&>3R*pM-qQIASUA_FRKl;+6ym%*&+}{G2L9m_L?8N!`R`FfX7TPP6ld?4gneIH8lxAvCA0TlMT8b4LuL!=-{$t64L(_?ikNqx zNK?z|8b+_|lp)%4Sya|tcm*@4~ygYRf+b6oV%vi(2` zO94QcpEEra$wsGAy5nnvmk82CcCVxyTwwOpeTZ5yC=yx3V`2%RdCR850)F~~Z>_D( z3iGm2EX_pF8Z;8MkC6v)2M>!Hj^9h{PD|Xhw}z6mg-)UV9`k__iwlE#B@ezsNdDWq zzdxS0XKY0hT@=nzLCNAC_fvwd`@@G8no$IPN7{Iegs+j*mL1W6b0QhK&8j)CYE)(u zh}9b(n{2eG=Kc6Z-^hNO_9i6b%9zaY_RxDJKU}b7-r|}xQZg8o3TIW%YQMq!OT1#& zFr7SCV#|C-E|2%gve3sCuZm_$XMY5uqhW+nXN_8oXMcH|5hl5ccl}TsWBx86yzEk6 z8`E_w()Ek}B5s5`#e>al+@9sP3s3k3T)~Gfn*GD|!%7PmGdFu01%2J4Wm9S%g}&fZ z48ExOt37qUNFDL9W6lLPZxQoru0>LAjfn?{g`Rxl`4uoX|HyOeGE$AxV$=kNnbi0N zPEV15qJ>TELeLa0+hOyCVi z+Ek936@Ys{p$H9)AHuX`>+;hTKZTO#aDwe!Kg%d6D0a|`P~ht^s>W!*C4&6?xj>xiY$e6I)@<|9eJKE^%BkTaxkPk@pXV!7vyJeHhi=7!dM~E*TdDuL7V*2< zS^pzfvb?aUsK_dxAJGlZeD&Sm_$vdsPYY9$Df7ScT$Uf*^XbqCcN-+&JJzOFZ)&Gd zC?0|i7LOo|w*i%U4)!ZH1}#SPMUsz0wDfcyS9jLP3P>hw+98OQcC=jgwU<(`QT!q? zgd2_0lp0Y*;qH3|URK97PiIGG?;)%uTx#SVa;F6^J!D_6*7LIajn9_0arjvEl!^2B zG1MeI(h+~x_flDlC=FdgHAfpe~ ze+8sxri2!|BT=?)z^nLIv-4~H%*q3AIqMj&%+^*~sT(etlhXhBX?<8;kSWIKPB^_F zrmYQf>X_JAfWKqr=VLs0m-s~gT4>!r07^^?KN#&V{*Qf2H8DPAg!Z9&8`AHtno*@| zSgFf82V_4rxenmWS(a}*`>$*}GLS3j$O`KD&ThlUn<*|or}-tY4Yp?6N%CDi;9_Lc z@3D89MU;avsyjPwwc&Dba4_r8W!Hvp9eS~TGVpKgtegc~=+`~8=!RZyl1T209ih)z zsr*d(rh#NMw}|;oB^*+Q*%ewxL`puMQ83GYO4@K`MGh&|DSgc!={OzwihbrMvcw)_ ztBJ=JOJk|+w`NM3@mfl-G}<5!uI!;ErPS1jnixi;R%YM*g+aCgo7-w2ic~*Ep~&9r zO*e)S&#PEWduR&O+9G?|UEe+3Ap762$Yn(dqu9rD2`{sk!bkwR167xNhjE=+kdW58$e>S}x{2w(sQyB?OT;F29D?fR?oYGnub0hCF6!h}PH1T$7 zrNbSeY}Ks4W%Mvb>g5g0by5>QN;hD1K7d1dBA!?}^oFj<-hv@6Vt3|Ab0&aOmf0t2 zcXM;Lw;(fT@jd>#N9;m8|9Ony2l@VjkDxcEr6mTe3V&E&_|1dos3=%IqyFma6}{wE z?2hXmT^bM}IXD&uyM{e5?UA9cgE#~K`w6lEn|vzNL#woD7Al8#rW{?0Ai7zWu`Vdj z-tWwGss9R63S}WQ$d+VW1y2|5M)7CBaGmP=1C>t|o+Sw=2=W#aG>rQSfZ z0LK?@*1!&vISv(Kvp9XS{r;9U$KUW}k+85n98BS5dV0Ez9Qp^%1i1@Qy~HNIOm#Dw zW4w!{=ZTj8jP${9FN`E)x}1nY^L$xVVM)L=;uJuup1Yy=Mw{qdE`#Zi_RkB?M|LY( z#C7NX4)81-&!js2@Cddu#jB<)pDba%AB{3CEFgZ9A9z&NXzZ|@^wY3YCPcRblRHJg zyhA=IV&`KNfcaUX4a(dmr7Q|NAk_D1MF!XJmx_ry`uxPZzhP2{hi+qcfx3QicrUe` z-ZUqA${o*)NR;whoL{O%m22g#`0b$Uc0X?QNUv)Y#}P;*MqL%2=MldkD|o9Yme8r} zZ?QB$R-UEtUEficQd%xsweKC=uMOTmzk~|D&e)MieFW7am#dyZg*}N;?yHcyiQMtR zDqEWIc{jGzi1Ad(arSL-&AqX7PJK5#fKaBbBsPhfCsmLO{uGwHjm(mF8*g8BZz`>) zWiuhTj`iD%#3qZp^-h|P$qIhNthMjvI5;y-rJaPl(4@YpSTtxTU3@WjQ${t^^OU!k zD-j%9MC+Ub_K7|6U%gNMwe);~l!t-%7TWx`C6LJ;7MDq>^t01)4#)M1k0%$PFsqs9 zr>-dRp(t*j+_2o2JYJQ+?_HEdoX)|GYXBzj!f0_#du5;6x8F_*Jz}JnOlN?aw9jt> zBLen#Z6K?*k|en}WEmcTUb12m!UNMhhjs^ut`het)v#?Ga}}d zPy_ke)!?e$zG{4;NQ|weNZM2EZCtTUO4b&yrcbUG0DS9UffdtYaUO^>e5XVw{Z7=i4SuSD4qJ^sDZkXkg@!YzRxr5~Ovx@w~ z@BI1a->LDxGCkNK#Tk$vW#lW!KTaKvq8E4Vp^H>Fdkc)%kQs)B0f@LUFc@!x|6Up- z7|EwQ80kIbJzg>@{=Fb+ugl-UjG#-pa4-IJ$XMS1`<6PMUdywvu4FUYWLaK zO?5~o;5^K?4V1;LM7q$yQ$UqbE$vRCumsMwc{chF z*A-6?J8=VXz3howlhW^s32jkN*CqMy+|!|RIq9zEx?ItSI6b#)&EL)N{@ z*|9f#4j2SZ&sH3_d@D@7d(2~p8Ove#ffi&8--x}-;P>7OE0BB=RAhX}Rx`e3Q;;nl zZ5Z-Z23pq*@6)= zE0H%Y)E**K5mCvB{{JoQH4s@$gsSI)16tyGZwTbkOBCmQrpuA#$ZMFF_CIvi zK9Tls7X6fMOT<%WNm0Tj?Q8sxDAM@tNeorTy8z{HD8hMV zI4Oe}QtZuVPuyaZNP04zI}bHnBnd*?hGqezRA(4C$JFLCIgiAf#Cb?$cRuT$U`Fle z5=#*I4#TEcEV2^_HLjV4o=jPEz#Mmh^ zpd9=?xOnUa#HqSZ^;BzA5Y{e^(#+PQQZAy;EH`|k(Xu>=L-MONr9BIFM@yRM(ZIrA zHuPvC_F0%$PuGB`@&c;Wrj`q$@c;h>M_19tUUg|jPFDV2`#1fB{9t0}-SbKFiD(>C zXr>p7hF&W}d2kA)%z1>a{!6U-i?3Ls)$^F!vNS=IFk{%fd-Kr^m0b<+$xi!)FPjYs zKBjfvzP^0)O^*8a6sKfK!uyu_8Izna70E7tE$*RSk9PhsiET6!wG9(lcs6#%5x>le zL~@UP{0ftJrOlHjZ1b~z;`CG&iqrp@O9JXyO!RDRZ%0zjLlk7{(EA{W3VE@vZftDq z{PX>Tj(wk_7JJUIe~@8LDuKH{%8#ijJl2HZB;?dv4u=zAv3Mg|(O&CWerVA^G8_gD zJ&fr*yP%E@mm~-rbOlIe-`Xw38`T@qDxc*})w6|aDWh*75G~p>abc7BcJT$TVs_mj~tYM%Fa&z;c_L`0hjEoy%dhv-3XH}N#zTp@r zwC?-t%AU1YunR}=XPAsg^uNRtCAaF${LO{47>n*^sgLT}iYDz{i58lM-)2(f;Y2u- zpnx&Ix!p-wd0_=uSgpkQ^cGFJu}>PUs@eD$YGWBWlDOY>qVt2y ziIlvfU(ds*ob@^*(3$gaFe)?w_FNTbGlBAS&DDhzeDLPSOa4pB!^kL>*8wqE6Gmxv zkz$paX6fo`k^hgp^N>0cKn2v`4es{xRf$ zsB~>JT*NIG|IfTb11gjWCWGi4mB4D$DE`VCjRt=OUb%08&PgFX! z%DmFM=4C#{{(w+%9>FRE64-DyA&nF*?V_CuLEeid9Lb*U`|KO?KaW`zm01gfu~GO@U_^+wdk>^nf8 z&>>mFn3R>r$8x2eTCs}2)vF?@g0t$`x@OPyFP=(?z=L-&gbKPNH zhDNaO{vM>|wva5H6)$$wH$5punNgPHZs(CGBSt4c6yE`&SiPNIhjUnc5D?Y=BG-)e ze*`V7nU#7-`$OrodaWB zVGZEJFh8?zP2JKSEA8N04))#Kn=?irSgYhjo~UoS*v%cz4IlzA)r&qWspxQv<;ii- zg&~~xSAn#scHU;&G0|W%Np)Tud9jMDEH~@VPXT+;!rgWM=(|!MI4=4qZ;ROKy+abn z&>M&Y?MMBbDwwm0Yi(n0s{kg8>gC3%tz46&~d1vK)eE@)L66pN@Gy3Bq@s$D-m;S z0gC4IN9yixdH`V;coJ=rO!>pm!jftGZtLbcT0XH@Tj_DCNVE5=eD#aqZT7gV$+(fF zwJvvu){kJ(ty;gsS8EVB2;*V9S%$5>?K~-d6DlqgP z+Je}rp^#EwMcoLn(7gV$j60h6gVvIr!KN>!=sW7B3Y8nYXGn`+ZcxdIL*HIbxEQi? zi`WC0jwN6Qxi$)VB@-H!>5GX3G^#=*4a#iJ>rW%(-4{g6>{ zJFHp+yS`t2=c&)9_W3LFp87vx<)J)QxdiSO#Jt(FA8W9WX3XI8`M8dvJH6IL9zKFV zXJNUbY5peY`zK;Frx}M;{ghVgH?lBGB7l}IOU`ql18n-`0Shg%&BfB{-``Fs@_9h( zVn7vNnap0AonSQ>WZ{~W%P+1%^w1jj!W55I&r~~#lG;p-l^7{kUf=Zt=2hL~)BXA< z2UZd;IbY>HvrMU#^^QWzo6o)ZgC=2ak6d5m?|o4SL3e-pkXMd~NVDbwBzX#`A=#>d zno|wUTQw~^8k(FdOcX~%Ch2Ot=Ylb^GvB|r|DJY_Lg6dv!(fcx#xzOjvSpE+qA4{n zA=4%pEdw((F!8OdVC|!-c8&9su4H!wFnj@cW-YFd?>^lVjF$*94J8#@w(HTt{~!Qq ze_6ceo)Cr;ntHOu;$D(;o=L1ttoyp(qAfFj51wp&qb`22T6z`Q8!3Jz|M_k@Xq4pkrOI--9#3#Jl9#w-1dZvdO8pTM0*44?xW; z2cHf%RW|{O8?IEkld{z zcDtO*gStj5$1}a$UP!C!9!vj2Ld$31WhDBZW8u+ctQGrpTL^duVcW~JRoq?|A-deU zRq-inw_>x%HL!x)RZwIZR~<>sR$4R4btAQxzc97kRO8W%zspEc07LT-Up@)PlUfv_?VNvKqDV+ zqg{WWLgaDAfvmC~?Jf0A9?pZpdrb^i zO8WF89j8HnG39RL5sVip0v3ep3_gHHWO;327W?eCjy@okDQZ1cX(GP-_R@z1384O+xIZQwf4G!M? zw4Y%W$)j76C1VrnjVsS!BUBX-?RZN`O_yw)PbBX-$~J6P4hVHwZ4DuVj#ZpiaHR89rIl&7T19f{9l$=1 zj|u-V)mzHt7AQULpRib3Xn^NzNnphrsa-s`2g!%69VoNO(nK$0-)+=_#Rz>}riZdY zOI-=dO?WI|2;StM6EhEF;>bb(c+nb-E;Tm0ia~uNRI_L+*6rszbnt)?5oTP^bt^vv zEz>UjEFknTNZ&B9$x}X3b7sM7U4SsmY*UPmFOt|6zvix=DX{iX_S%G3ae zsO~#jwbP+8`gfS`_ZrBxm$h^6Vzpt{TwA7uw;oox3~<08iV^3hlQwv^--#8XCd0(UQiD z5fs}*S>JYArGtVnlwz>iL=r|y@UXnckLV9?gP@Hb%Q9pJlmHNyz7(U57NhxoOCT) z94+iF_j}uUi?qy%y$c#*iEUP*b(2o;d{DM@w^%&j-eb)-^%3L)N;S=5_xMJsA`T~1 zO~~IxURUF3#w+XcgQ~;RxT?uN9}m$vCb5MLe2r1%1oaVzysik&``^|w(MqBghPYyj zhay25N$hzKGVa;07LL*eK@E!|OS^Y9vGSl~+MSYGj_ZoLC^T&|JHUXug(#nY7~0AV zhTm}<>IQ}OJ)&(&H@6joUVH8qQaGB)ayz5fZ`dK3OT+MN1>MvgBpJDs6uIGDX7his z9>OJ;@k8vk#RBG-;dLkiIuiyBggHA|uab{LX(97Q;1Elw$*eu)NxYhpqHDMy9FYlAk@L7wFH8JCPuDcltG>r)CA9Hz77YDN?k?hEkgCDfte_2H; zR13-7LRUKFc2`M5gPJ}f{nGaB_xb@yRp4K%7FkF9id7!WH0Qf=8dG@|3Q z#ttW#$Mac`Y-3|)qYuGD$rcvi9_`*&{sk;IV#m<|025& z(l@!`f7~r!ZxvIpg(d)vNE_(UXf>w%CD|rKe*F&GSsSQ)+I7j~zXwX}frFYDx6t1M z?`$~)TL_#3nl&7-PF#isA)-0reO}>}P_xn+XTdX_)b-$-SAlw^b?rA_ggU7{td@nZ z?lvmQLhp2kuQNJoJ64F8AQEwV)!S} zG!%g{+MkKa*kuTNm{M2#;=i0j%;F|($=Vy0z-PV%@qsQw@MMJdPs(hZ2I%=Bi38Q4 z1=B9htXnAcFV^p^4$R*`>TV8sdHRVaHi$IeK=YyV35Ue&do#QvS0+;MOyUyEn=Jo& zzLOQNq<+E?Ei3HS^^`labpz7>!A*==m~f~f)LGFjRB7|0`p+nR%j+6SCD@AQ0y~;; zL;1X$uH+nUQUD=NQ4 z9OmxU?SY%4kOeo#ImX2tRkJI5e${SB@=lWKfO>2(2e;Vb!7mv<(5lN&MgbGc@)y7U z451Rte?>es8hk3#4T`32UjZWnw9Ji)|HQ;xK-uTtWY>&mgFz?JtM71)1E_w+IVr>H z+*8XBMj)Yca8B&np;Z5iegKpy*uCW%D3l2$*@eYUe``KhXs0vAyG0QllniCc{X@v= zuc)d@7vvr*6f?K?cKcSMO^?ck*rw$(-$V<5rBF>nC8r9Kgfe{N@k> z6K!J*-^US3pulVC%-m+t2yg>OO)wSG#eeKM{k4b8pDzNw{L-FA zP1=^dPML*SG%ZE>jjIsvVjz`G`TV5KPG0Ha7o4vWQWnIAN^NX0OU2^{s#_b>1df8P0YOOnL6pAh8s+YYRcl6? zs@2*^&o&{=>SE6o4-T!l48%O7v7q%VaD2Eg4?1Sf-3_lSK*BZIp*?>u#bUkiSp zQAczR7gp`MBzCpXV=N*o+LC=a!Y$PPLq{i3>d;-Jm=wf6*ByPUm(eohMX-M;(%iF5 zAy_GVM7S8q*KvSgvgRg!zP~EWHGH*gM2KAo6m*-jh4vyArDxSmoiAX#h!Z!jME#LB zuP~pTI8SUI%VK@bJBns2=pCyF3zfBN|L40f<#XQ^t=sYjiAvLPb?=suF;7B!ygGa< zFlc+OoDgCK%v64aij&YT`dxhs(t5VzdQ&Ct2~XmB6MJVv6N+*<5*2sRwxMm&8j-q) zXOcCe1xmKcl~l1j&4z6~O^fc5UO$cE5P$P>?}r(2elB+S@Hb+u)@ND7vjY-p-J_Dj ztn<$sgnZw0z%8MVp-J_iy@k;tTkD(5&rAZhxWtZ@xf%_$ zPT6XjFlFn(5@^%L!uDE;EA7)Uw+6BOF zKpHr#p3s^SZ6nob@MbYxefq@uJ4R|EkhDyxwSOUAKnto&m78LdTZ>MjpH{@F-K_W1 z*S<$%&JPf)XOe7eZEdZ{+9df;_7dCv=d)gb8sc^JfNQ3Ln9u(|t(5_#(&?#55f_kA zha}GZMPr8o)lb$;vxrXYl>JsiC4^wSE>(JiA4whCtGQupzuQoa&U$lD3ih^!`}V7J znggksH)@gd?AZo2)F|GCTuXEvW~V`}Hz7zuNwjW9BllA#!&d1*9wq?8Bsl&s05rX5 z{D9^0F%@C_D+GwpFe>06cI{N5aj8wRqti6YHtgRY#WHau6B_o11T)zQh8x^_js+ikFujIO8P972@c zN81dj)eZm1Rh}>eQ~IXkVlR__>W$20o-IwPYSs;$H|CJ zw`>60Z(6Y5qQ}P>zVh`{;UR?euI+hje~0@-!nTzs15{kHU_x-u$9YmEUS$gLT`U@* zj)N0fwN>bk(!~@33ZTNLJlSr%tCg)7TRr`sPu@?^bKZXBe-kKXUv&MPicy2;pK2+p z8UXdSNrZL|O37;vhb$%W>iN}~Op4JuOro{h$g<&izxK&sa?$Bd!NmLx4Nwp}0?|Z z=R;@-k`6?yUcmB(jT+mw8|v$bHy?{1#liRYZ`@=+Y(M3Bc8Uu(YzyhF8&6)&;0tn< zs0Xu%W~NhoeNh7MOt1p19RE20VgQs4xI&XM-;jW)Vhb^g+QynlVriPpa;0Dm!X8ds2@&B}#BamXmGFOfcMSrdVIT!z-rkK0` zKXEaQ-#&C`L zXH9{J2|*~)!A&SJLHQ3zwrP~8!aQLYKIPMbpE~x)WN&paF_;-09k{8YdRr=gv%WE+ zeK(#W-Jwz9bPWUk84$-5-{I0xMTomlmQs>wg3Uw+sKKT0v{VIKrs+N(hlJqfJekfZta#0-WBH z$t>A)3EM-?SGL3u<&M3;FpRIbC0E^dzO;+^xcyCtVrhYOsM|Pxfy=DB$)L>o>$elC z$Lc*~mlbXRQ75!_vww|j$WC94b^d(xYbfBEMR);Sp~PT;(gNWKf$MaiT8=~Gi2u)Z zahWFDpc?~c3$jmToV!-cRrbeuiKtkHn5y_Yw}MgDVAW_|eEB46_u>L)Hzs>dN8KsL)tx=b?XyI{%3HfiK*L(y|V5G;AVGO<4wENKSq;d*>KByjp$hSl3Cj z&?*89yy%7)yaxNE=tyTPMj2VlaY!6vD@C9+Rkq{_tJ97Z=cprQy7b7Cp7xMc!5!sk zQm&a&C*0e&))GS9vf01tZ!!fl0_1;5V7g(1mlZ_; zih2?W$7hWy%~D?4U_$Aqjl-QF6hCr(*|J5yVlYBYzhT_j(BrOrXHi#_pqa^Fgl<#7 z^*aNq+`BWU7FFqPjuC$c_}`+%c!JGPTlST}Ekk~kM=5-JAw?8GpBa!o*?63A_RRc(=2rHh%Bu&nQjFZD8e^Zq_ z50e{kqJd`+JelY%c*5NfJMCDW=M=K)R43zdW_tO1UWotcZQLiI0(9w!(9xe>aiA{E)#iQNG_Lbj z{Z{z_z!j*Z0XJ?0uHM#D$Cf~>t^UVk=i>SJ2TR*Miow4jdWLPZKxWSY^9yjl_{5W- ze+O=3>L z4ma9Z5$C6uq-W*=OQY#mANAK{TKAfQ@Sb3da?TPmtl=W{eMBbPSW_(u{}+^O*;;so z&j_$c5NRM&Jr9YyH;k^2y8^`A*VN}-0riT~LE&?4z%xsJb609Z0dsEfAYR3))nkPz z@M9$Es<$-%<8UE(I6?j)LUT`Wn%;K?G;?gsV==ZSomi+YGKi#BL}0ooG+-7{fcm4Z z*p(T~Yn83P^JNVigtGDaiJnY(%ksxz@t`#EIPVy^x;WYVIJ8qZKVkFV9jqNuQK_BShXvgh7FaB$62c|{YIgM`S|egaD^bQ zvMJ^o=*ZSvmfycY?j4K}Q}^~i-0?(9-0*LUnspDukFF~9!qr+S-4@i%JiRSM+GXvsc&Tm*K`5r2U@gxiyb~-=J60p@8+1u(ptMDsAM18n& zq~Px$I&Uh1T9tkwf$3MJW!7=BdLOBEa^i6mgZ$~iw(1}CDQUgjG{GbhO(EMoOA15J zPCTXzWH)OM_g&q5=?z*BirHI<*sawXQJDj~qnMJSSe~;*Bf5=ULXNq61+aGaT?Jw9 zBGO_}*$2dghgHCf?vynBML-#R+03Sxn^}O9$}N$0Z>Mon;zP41t&iclm@bE1s6N6Ezqd z4vJnwt@%jydxS$b1`1A{P($c-jwBUdOo_U6=cc4amr9WuG*30&nr- zkt)H|-+GUPS*1pPQ3rXhBVfKGz|vywr4syD)03Q1$qJ@9->f=As36B*HcIcl^23wy zBfBp=eWA{o_fn&Y5KVDT@t$R3Y%hW&9DF%WiOnNZl@mJBNA%l7*dyzeA&vWFBdUc1 zqz$|=Nfv|)+0jP%O?QZ`HeTZ%qJ83Q;)`_J85HAMB4}^dfg*Sy(K2APRWh`)wJD$R zb`;%2`83BKkX3+mD7S9(N%yYphPe8d#^Sb)l?!(V@|c4aJ#&zip~uz%HVC@297^(Z zoFrgQF)EqfUJ*=qngTgl#FV)O|H`fk18)Q*!|3ECczVN2PUF-hPnKX9iP86Q6DbS- zddyk_7-z2&Ys5UD#hVLup1jJAxWKzL|Ea*^)}sGo=6A1*3dDeljfJCEHOrB($SM+x zJZd65ltBMjFpxlqAZwj+4*)|Xl@oW4B@;-KVJb}$Q8-9qJ{^*Jj?cF1SbmMjFjEoR zI4PqHHM9ElJP~`Xtxkg32JeT4u^s>}A@7p27AVi!*LX@rtk|JLd2du6mGA@_b;A{f zJ_8k&)Co`jz3(Jram!!*t1TEc+aeS_8P6>3y9GdZRrQas)PY6`zELqOj!ZE`YWe+E zTJPa1i-%sGNWKqX5n6!$0(vQf18j~Lo!9coLUa{sfgs81`=3z40dnS+oC~S`qM-FTgf9{I8Wcskz=@9|2U6aJsdz6(eln_qxjhPGbY(_ ze%Agh=-SjTXkyHMGBD~&OjR`leDYo|h zBk8U=yeoLXLUcV0(`-<>kh5WYw)i{7sPe8pQJ=CzRC)yy~L zy`L0%$1F2v6Ia3RLyuDpv>b0h?>Bf_od7CAFl2UKzWiV3Q{@)2%( zA?TzCuM_-a?L4C3Ra;aroG9=(f2>Y0G2r}$^X<*jIoJGGxR_$G*!tO24;pLr^Kh*H z3MZs@znS`JHO1&9t};L{u>Ony=A4$;U(U9rMMC!!%m&R~zXHU{*1po#Eh&nkP|pyQ zZW!?`?;}-z*DJ{U5w~8h;9Wj0>Xd&7m3N$^81;rS(L~l;Hyq~XZ&6Sk9vQ5tt-oRl z4$()yG)wFT7YPwOBsHw~msR#$bW0ned}ooEGnJoX&}x)~hni$14L4xa zuaGmEGEBW)llupskAk8VOG&m00bH-*23Lrpyjbhy_Nriaa91{CZlS=c7+dE=bfZ+o z%9W(kClL8`*0n^(tQJT}sdrYdHH_NQb@5HkV560k0!8mnNF@W0RR-E8iR7cDS=BI- zM@b4b#Q%NS^JR65Fau4k>-Gd`2kWFG| z|25@3wv`)nS54sD3*(Z|gl;w1-RRNiMbgbt834i!5m|UX2f~C8Dy|VG0!MK23v5{8 zbnCRjmj|J68EIJk4x*06=O29U(BS(R_NF5Rww+t&NF*rfaMXY97O@h|Ddi@SHCvMv zvt7%$Djth9gx%Rvk7JT`J^6y(6Gw(ap}V_Ll{3NSk^Or^R+&uREy9KzEx~uJ)f*Gc zm4U30-n=}cN_IRma}^~T@=!DT!M{47P5ZKa7?&b$wjxs~Kfg%VzqEyS#6P=vX1~0; zm#(7mmc4SGdTr4y*L#~YfHmDg~2>Z7;^W{BDgK2kN9F3cseuUa8WfaQB&OjS6gZ+CfYY;JeAjC zgu~@qi{@(xC3E>)AdVlup}rBx{!~;sQAm^VG^TW2+e-?cB>4<@{Tp^Xjk&uR@sYD# zph5`%Q-9z$XsDUuJ#St50t=M4vFUu}xZPSqQt2%rLRsE1m$GtzuyeX>O)GbN1BT98-0>UcwmevF}Oa`Hy9qf zAOf9u=&Lop=eY%&?p)an=sJ3?m;SGfg8VIGUB--o*2hMuk?&|ctqD7KTESo(Of7kD z%7Z29Q`9c;6s@=Df7&Ai83#sd#yQ1(dMu+pYm^!X%%w2R3*yfSAJ*mShf~rEKVMXwG6;ed7qIRCunC=0l(gXzy2nc~|Hx1E;O-R#^~;z`*8aftW|ajlyJ*brsf8m` z>99H4t~$A9G?VL;oUuo14>&0j6=Cklcp#wcFY=% z)IQ$>L%d*hwm42ws+K&*hPp z&|Uf#gQCm4RN$0p2qmn92edGJ$#|YpNt$_dn|TXvB*22F67T5rXm>v(N=_FZrUr4$qqDJ z=v*oZ0B7eEqn_{F*U4GUw^pAHpe`1~B8N5FeJGYo4CrIB1&0ddk0i||W9%Kvj1HHI zs-$EjgOJQ7%G2M#dd3zTt!&^*fp7X zA+g6QI*&5(irhiOn-u2BkFz7FJWKxqtuE1N=AbC0+jGftU^&J~Q!dpwVBBj;Hkc$7 zRoz1T+78ZeYcX9TX5o7kjtw(7!0A{WtX2!B!+NT+&A ziOg`!o7Dy3^^b+V>joUm)@i`Q-Q;vDy}4QWj)Ltcdzru7?9A-|areySKg3$G?@}tTv=3lnz*=y`L-VvMw^Fu zpnKDKy@U zl(>aS3TbZ-_n3f&%ZBjkeXXktD0#mlubnxZN9?&HA=~nXp&~qi`b|!21t|g;nUhP< zf~oBEc*Z_6Hs85zc91BlE~LGr1leb{oxd>Wet`I7|7g*0v2e0LqO~*L9)ioCENh&k zM5bOq1fziJ+`&}|j;63HMPveIt1<;Ba=GaQk#=rn8rbC)SlxX|nc?KMKN5C&zJ%A> z+8ge;bE&Pg#MSMf*z-ifH#jr7#R|iWR~Ggwu6VC<5dBBc{$GSv4YFvOX5QY=8~Jtq z<@1`G3_)Jbh4|@$bg}o68wK;Da~N+smJrFCzIYYacx}qrK=0XwRQU_}ItyvZksg&K zHFY_NQB&9P0}D0sQ^D!!Vj7f=NMZHs9C%{Wn#v7L-_0CY+CKebI@^(rDDbdkz^DrH zL_^}w6k1F$1l-Fxh>=*j@-44E3xIfE;8}U_7nfb5G|$+#icH$|PFpA%5QondgIoleAOXz5RN;097T z2H?8b8Ht(#mUA`62B9G#A$Grfg>qw@iwBS@S=x&Wff^o9Y3Q**ibE5M?ggtW$TtE3 z1?_=j>#I)T-||)n*Ni}h76KSpZslTF;WuNGo&(x83+I=1VdwDd>yq_A1Y`b?ZHiu& z0u3ZD&MwHt23}9pG3a&hmw_KuGRH+{+{a;Lw_xJmog)(3BgW&9obEX=l4x%!N@{fp z*v7mA|CApXM2>C5uAG@|e<3WR$&z<&+Nh+vJ+yI_E`oJXDU?Wfs#K6U~*3;cE z;I90%PT(B|MN(XD6Ylly?fEsNVWUrEy*s!MZtlenP^+a$?QbonTNafggBLI)$3VB( z4fz&)7FK9OrBVFzT)sI@^J zmzY%5-TnNQe5uVQSgh`pym%(^u2NAIkTJU(V$?Ui$HD@v*s?$(d-MOas7TsQ&ueJ7 zTX<)Ee9}SDeGJjd;X4CX9H=gyXvq^%KNxU}+EmpN=T}*$8oLUUO3zt|FLc5Fk?oHf z>EDKeN(p~0Esaw;QmYumIw42X!88C?jsDkY%0B^oEhY#%A`YR1KoXPKwxo31G_9|s zyJ{NO*LV8hg-g^SAKrtDKgAqw_77GAs{;WfPWE!jccbr=-dQeWD}^MA&t?giN?Si6 zylrEoe}{J0=g*%jW@|>rW=Xh=xPt1#4l`9E0fLs&xyDMI*yudq+LfHZe^V2(!vFbj zjQk)14$7M2e?e90wc5nyy=0QKA|yd ze}iV?d)~sQb~k?~dMMEw=PJ#?#p(5IDnq zgP@v?q)g1AsP^r?(eX&b8QU_wdTF&Ps+Tyd6NZh}yVb`ugys=Dg6}a_vUTaa4$L2- zp#zI4Sx@f!gy_$G#0`Nplg8VpsTX1dWEgydHk}RJNTjPh3X|df^T_NEo6M|ELJRia z70^kzAnO)z!oHgmK{aZVd+4fnmzyyBR>V&*|Adik6{FVLp$OzH+VCI z&1LS%?%kCvG9LVwx*7#Ow!=X!2|GXI6#2EuYimR`j%fXqdC^#1IyoKmktM*@6k4x4 zt*(Eua}@#0UPzyO@*5c_msGMJ}Q&YIk_mxFoVDU-Wc^fdh+J2{>H#U&g1OeDS_oZ5T z7@p$A)8S*^KWaB|v^Ui4TBs9KGGV2dN~cPcs3S7!*T%-~|MGfh5VMLn2ZbF46932C zC$qGj9SvjF<$o*UBvIYHjV4Ltc=?7rKD}d~+Z%~_i%Z)JC|aJjtX*DNLzZ~=+v6n^(U`3Hg4n(^T}Qya zHLxO;&wRrk2yj*OKYM4N7Tq!&o@a*3&8+rstS#(T&S2V8@5uX35(YWj^gT71oFq$8 zmEp^EHGM8vVT{aX-btvsN}IFNcv%XgqE3>;z|k@~4q*l8DKWUPvOEpQ(^bri%Dlo(&xy^Uawq1-=1 zILpc>{oPTS=TaM6&~~n?o*O=oE6%?={FdoXlXQw8_Lv|kRjdIYc{66}u$@71#@St( z`Gjd&vUSNR0RRqj8RzWhXI7u4-Vw(-pWej}>6W5v9dHUM10@op~E6>mN;#P{}UI>-eTTb4dpF_Av>>D>( zwz92%_IqB8@?#KY@79kWCV&%!Gye7YDcVrd%XEI*bLh5!!@<_JDL9^fby&K{K(QnJakOyDNFtVlKC@mKrWFVC(nDJLlxADVu_d))(s3YZhZoGB6dpJf6 zYUZdkBot7~TGMll?XrYEG|d;!^Id6ca-rfKV-M?i?hvZOI2|m_tk&G$;n$35;oTv; z)`{AYmEzVhLoxEzZ-0|Z;oU0MWkwrdr4sGkuas(eca++(V9SMN5w7U=9duLgulqbJ zXsgp1{qAal>v(RpLhRth6Qb5BgdAvbTzkrv9U-RrMB>n8 zCa(Xa|1C;fw|ylx+1~$qbKE@?RCjdVQv1H%!A7)G6DS!BgCPK8+T>gC^Ij-l_2G%A z-ek;gXWb;$uK^{L#<6%FvQgl(TyO9M9NG&84h}#o96e47FGE8USZUzrvsp-0H&7fH zd@*-WY3^}8bZ+{)+x(s=@(!uHv?Z>7cpdH`X*s6kuc5-y@9COmhKY|_d)=7P@3*nM z?>>lt2W?jMc z0g#<&1zPxufWwH@31%fF_we13T|(|S!|~twD^3~1@<)8X4RuGMJR-;yWB^B}=*VVq`zOd&D-25eb$B=f~+`K)tsb%(bd0}yS zas4QF#yRGJ0A(Y~(b-X#Y3#t5tyPyx)YvOmp}o{Hi+a<1^%BaKmnk;?<9yCkDd5;R zP?~_={%1w*xi%bi!l7MndxH-(_f=e+cKH3+zDRWDEG$Mxjn9t9irkeNEgXpeOT@Ze zG0$|)S=!3H)25%tt9&GPMCz32>z_MFWd{3?zqfk1ue610*kq&EDuChoInPrSWQWWJ zczB2&{NBO9TpB-hu4r9i#5HV0X8Ud0@!SDK+_~4@4!7IN%macVU-b~p@0(|h%}86; zP0%Y1EmlGkm!CZ~i6qGvzM1cqc=tY=>#WFU39_mj98iJ?eiIARywF7jOvydAX)VMB z-Y=-C9@o5se=%Q`>Cv;xcAVb=J92Y*RQnxHDSQAyN1^0lNW;`m5Jq_RFIw>($Ejh* z=`E9;MEB}Z@~PT5B47-deHP|+nIRokLb+8n@xu{kpfhvLzin$v)y6iCZo9m|*#1Br zGfSL#Te|YcXA1=yW?8Eb$=rZHiK7O60mT3mMwKh`%Wezi`Gs=3`r=~^jKkTdlCfO1 zg~qJu)zZ>1P!)cJ&eR&d#-l<2R8e*rXqE=?GlK0>Ov)z&+!F^tWewyQhRB-X`X=Mk zD$2xX?D_YEE#@uYWk`;!{623nbV-1+W!Mcu6IZV_`ny5NW$1Ys zVMPWA6IA`T%x^YeuDYv%dg@jnLY&?4c>YB(!8o>%-qu2^J|x<%WwdIhZj>+D@2F6k z7zKN@*9Uc?c>DxbWnL~S4=EsGV}c^?+ti_`iTntcEDOpN3m}k zChxMz5X)fqL*q-w*$0xSsjQU@-G9^Zg~XPZ8G z*}OV%9revt&>81cMSW!o#!d1jY4xMb!`Hq?x9^0gH8FIqo~DM?uG;0QCR9_ZbhIPe zA2a5)M3{d|Se04p0n5o;@(5@e4ngQOOnvc2IQ)*~@9rgOKwH+KEbL`|m`6($PEOf1 zDXOI)GLWRt0@~jp8wC1yPd+Oakz8VMOE<``@zNw z(}Tk(GC;QjnH)*Ep5yz}U`8}(!@Vzx_=~E?A}rLMa?6(>yV*PIYcg}~Ejvu!aTlv* z*W<*$)m_ElAV=l74K);oVZYVoQkQoFZPn~x2GS)QP`$x6%z6RWf$WZ6^`l6!6otYy zGAMd=G}Kl0xkP?}kGy;S)?)q2_~%ee91TeWL)#%l#W?ZD#k(|F28v}c0P6gCP%W%z zn8Tas()9!GFiBEdmyiJPLiq3kbeg8~x&u*#3N-Q7)p7&TvSTuZ>?bjuagHW8;-6* zCF6Bj9>ib#+=tKsh)n2A@QAixBHzcQ@f&l8hD#^@Xr7CWCes%JEfX#T2MYo|3R-wI z_l%mgQwF|`Hs#s~f`lcvGChiV^+A_RO@3PXIYY=b(cA&B4!1v+o9QTejWN9<)v|T) zxl2Ip+8tWaI@GKHXx{GTd#z*Rp&niA9v=HQUc$W>5Q2*DTw(tvQNbcg+#hn`_inZc zP%F(>j{RLkJ7p%FZ)vLoOa*j+rLQZA^-U?uVi^B|@DapSAIfJI$ciPux2R6;dCWJ| z1AL$q&t;mPd-A=*mcUopCQO*-KVdDm9cq6Z6Zb?2EO_)<`|ZrfHC{u$W2s;PIwdeW z2&O;F0m@Ck{qIuGNQ__VF%ss-QJuJir@(O3ya2ge!)=fb={o#)S{iCbds1BDO z`J$jo*0TCXI-%o!Yt<+F+l2L{y?Z;nh5AcBnUg9#13^h5^?t|2LzDq+nEBablKeicy6isxpXPnBEKQT@Q-Lp+2wS5b?SlJ!s>Ls8Tr8eTN5E4 zg7_Ks#_?4AfBTErg7jSXW`Fl!DmL`+X<3;3$9#c<%95WSDusWSg8$lxLAPYGBf+j| z14^S_E_t<;-ySAX9k^P|B3x8prgzb`w2^JzJvR%4j;wVK7(`h@PI#7e6i;>U4C6-H zeG9-|x6P+P^ou{Y zWbvdJs=iK)%>| zG9B6vmZ_{IfIiA{DdyFz9BjiL?NO$q*bgwNxUwmlm0=qSqt8&PG}`ynCXi_FDjOTl znF0Huc}XRi9v7#A8WLQW9Zx)vIbuRI6OG=v*{cbNM8Q2i&wqp1Pd;k6rTBMWx-azi z9i`@cWh?U+Z9irWq?gKF1=xxaN~UHqXZw;AlhBp7QRrf6X6N(p9(!@hyQ=GQ%7q)2 zH~bu&2kRyRZAWw1W+Jw7jI<+$BKx2WB~e>>#aOl4d=AoFkeDgbXdNg|D<*#^)U zeG1AOTg9S#XMWCnp>G_kdolbwsPX~9oUZw?P~;NCii}LoDy?6vWxefr>^+4w_a=PPO#sptd?~d?tP>Y#*{6hZF ze~Tupu(-@bOWdZtf>a;SS#Jm>bmUJfXtG9uzJs^}6=zByh z$0+3nA3hztedX47VZqMj<)iF5Jle_yE8+TYBM6pC?d=eG>f>fT+$u1p%34i~OfS0P zqHh@AI;USLVhaNs1yc7N>c&}B=B?g3Nx7Pvo+|AL+nKY&_s^iW`0hmUxyt^PoSmKj zvOA7Rhh{~$Y_2UWy@`d{6mykw0>oo8!5V9icn~9LaibTGI&Y?nX!NaVtjX;wS0~b| z$FvAX1A|EORn@~GW;&c(@~0<1Rzd2ri)wT$WquxoxO|#GZkh8+1#r3LTL0I{cbvk| zky}i-fz+~dzfS*gZUt^TKT(>DjPzJvoR2*Z4vqF!qDu7|t7b-m`@Ntk9Xj*2qHp{c zFk+5km(Kn78B*7NOA}1Vvi~62oSLNqOt-gqZu|=BVdo|gX)G_dH*nUpW?z5-2comZ zwil~8wFlSO7xtwS7InUdRUNXL+D6VgY(G=^zQLNaAC#dWb6cz^GN%w1(Lkx{*=30- zutUc-sg2|RdRnA7*mZBrEG-r2i6brjFHjRGr|XCp$_}Lm1ebvvY|O)bAE5qIFJe34 ze)!3XWSy6mMkCi-+q3HH7Ak4)*}*z4jDg4M zdvPm!Qs;BBcDt9{eHs3Pkk@rB+1DnU8$-y+AI&By~Td zpCW(5*cNk@rKR*FI_xqdLee_?bvI2YA#!ns)VK1w|Jv@G5_V__sZ9eB?hg_ss(9bu z%Vwp#UxK#>`v`m@Pxyw5-C+~;LT~{{5~Nqc~zwr4K>ARLf>OVp$y+IxLWSP;~kr7Bmu;typhQb`F;@SeNwQS$~4~A*t_tdmGK|mzK05 z^0%I<@)DKX#IxQ5hG*Srr#wHR%sfWy!kw>Lpr(T9Y3#Y*E=h~=E7hWXL<<8Z> z8p%}C7EM^jE6o#RXF$qUL{3SkU zeKYIyH;B2KW-bo5qp>omiF3mns?5!_N#u~;$Dj?*bfX??j)yo#djuMXvHL#s!?22d z?M&>cz`%~S+j2a_e9_KT+UM!Ig0_>z|u* ztiN_H$hYn7TiLo*JI+6BJ^gFc`59}C-NA)z??@JI`Z5?-weveFKk6&-O$W*ZFl+wl zz)+pRCrZ)sbGcx{=K`^MIL5nAkc$8cLr?co8!LQ6LzvF;6B)Zwx5V5=d;K|KBV>7@Y*dv4Q!czjJ`~FI%qv#@+{b{wMDfTre0@UqDATstQ~##ka2T ze8H&fG`6bFBlmV7!YSY&HulAn>BODRj+*bg7S*eNWK;9RE(vK^!!BJo|ZFH-E)PUwn2M&;=^wC>TbT` z=9^GL=vkOsaV|)72=>d)6H|9DpG6|9@cQg3yvjFZ)Me3#3eAgATeIu%Cl42?F$g<~ z6Hxb|>e$qtUkX>&-xLhAkEaK0TRG@LTPpRVzMaYj7nOyD*I>UFvD9N3h^T@_!yh|$ z&ZjQUBaHZk(!quB1>@)z@KOL=PuKLuXW+_*e}G8KSrYUkr};c&oMZ*P{}q^Qsi8vg zVksi9a{sTkbG64`%jox4$zP1rxmbjK?5ee<7w%}Vlhv>&dBE3xBTIU z_3pCHi-|pPvY8Y775jv+Fw^Hz(Le4*f_%EsEwkaLAf z-mw%7p~NToZ9~k$5oX=Kjx;cuZ{=Dm21a$FVP1{bIdR^FDn}fM9!UFD_}ZPxr=1NyY zmgxxC=Khd+COA{92bH_DG#|%bZr6HM6ENNx{Qc)=i*41`Z_B)>uPTeBR zYAY-(Cha;gqk<;V{Y;6dMR>%}%zRo}KwV{b=!%N8?uwMje3VH^CwTrVv2|^@NOwy2 zc%ryv%0%rdK8;k<6tHD{nuS(!jrz6~QkFP7e5DgW2 zdXtaYfa)1zdyrowdm!(5I0Ghq$IAP+M3KSx<#?s$NoSi0ltqk8=2bjx-P@5;;1c+& zhO9-X3qTr2aFA|y#=6F@Y@p(>Dy(AXz6u|1C-XpGB%}W2{8Kw8P%kE})`Xe{@CV0` zG)guI<2Le*CpN0C-VWk-_Ta12opUqT#o@(p3y#bNyaEj5QQxEW&rejFEinnsE7j{S zTCzrsNps{y68-dr^ruh1tQE@B7Ah7_5v1xUc4yM>c8_nWP{eqeRz}aFt4R)v<^5xL zNI0r1m80y}+oZ7UcS4Ja`u3mPHpdh$Gw`{a!=f0h-s+ZLE&x0IY=mqg&lxPGT0L3( ztf|HD=MVbN;0gQ;w6jM*^lQcl_b4s0?s%rvMG)|G)=|?69vE?`H(-gQ7yfktL)`?I zOVXXy+RlMkHBZ!5_+C~>-e=r)PEb|NJ=uOQQ29pG)$+ied_HB7?Xf_WZkKFqzBKHb zeVKP#1^(IM2+fV(#>ZP~Hz?F;Z8t2h>3(5?_5(4iSjZ`PU(KvR^DjQOwB*wsZ2qL- zIZS&CRypc_hD-t?`nt-Xu+GV}*&s}}B_wZfc_tCuK!UdPIGMo; zhT!WuZ)E1ih|puM9X{zcvew(4TZ0{pxUVt4S6v8Dl7i>yscBk`w0uURH41^>_k#9j zu!!r8XV;ik7LG~3i8D|nE!pp>~j~^vz9D%WcjiF2W*UA zY&2a9V_%j#X*;q@py08@t7C+bh}!O@^Bu66d75wC(Nl(NCbfi^{2@T0CUKgc#N-Q% zY-N=%VWjjozaq>P*QTPhZU3KYp7?KDVs>j_dJdg^sJpikkPpa|k=2-}3f7A+2ju`ib6etm1RY!VDP!K9?3igF7Zv}vv(1uFSLV&wxkXwGutT5L zA$I6(Gb>mS^+%Abins|Dl#$cV+l!5s0v&*0^q5udEc+~vlcc*njE|(yMbxz&-L-Z4 ztRj4~$JliV9P83dd$(ClqEd5rObu436llR_TX2C*kUYS2^%|E$qqwgi3nLFSlk-Swl3!=mnsrUDp;V`FrusTy35UaC-uYpKjwAZE>f@7KW! z_O(9g2^w7d+q?$eEEvbbpjrsz$**s78rL!5ri)FBxm{Yf$G#6MKzY;5wCcDDlX^Tg zSX5{3^E__fKhD}3tfn?DuHOp|g!+s0SVR7ph}?eWPbee66&FgZJ&mWn*<^#}_rNT5 z#z{{u^=!hAAp7>WN87~Tz1`h?$lvnwms!9%HlnNhA+l>IYxIZtl1#_?%&cE9(HH{y z4C-pi&aaQwp)CCE@7f#}Blr29yqr#8rI!49&)awvpQ*{x#mvQ4{A-_u-K9ZmF`l~^ zdxmjJ7h6aJ2xIzKEEhFW!3_V~8HN0%Cwo8N-OF|O*4;OJ<`@zv<*IU>p~(&?ncdOO z3kf|oZ-HWgJ;5e*m$~CRyz_k&%OH$3C9-TfB5JmrzSnum#1}fCSxMX<1y{nlpgL*h z1)r@1-xhi z+y-WNQPtuv4i{@{S>eMt4;H%UEa&F({C^5u(^H(jHy_8Bf`uD>W|qmXxxd1H#hy$l zWiCq?lkHlx``q)>ZS(svP7j^wN(peten0B&=#9g-g8z5jCNwd&UlH0ZcFDm-M5U*2 zYxmwP)%sPy(Pc-xGxE2u#d=3Dm?Npc6I8Is#gb3Y%)H}9_AaQHwV$f}RFLJ8LOSU1 zj8;<){4wpSgC%+J7u)V@TYbcE$I8>tL@hqVDaso3mGD z8;v$RycT3nq=OyK?~PvBr`Z1luO+pO$TJCIRsqxxm`{J5janMm7%kL&Nb56*+73k} z;d0>0@;cmM*5i7Mc4?l-F!1SB9RXudRrBI!mpqtLFU- z&}MVb_}9&NeJozvZ&~^EPo7%kgx2tRT~9-ORj4JIV9bHm%n|wA(sk5JUV#~EtJ1#qihy2o`HvdC{sK$ zCAbcg|5Ssxo=|^tuS}>xp14A zA){S~x2-d>Wu}i0ZXki=wbV18sVpkV&YrjXohnwW@}(|JajtAp9+ZGHV$rIMV*)#d z1O6%0%z(@dVgd~j70j9|Lcc{5kn7d(b3HIvvg_kJ$}L@gEfF#pr{@KS#LRXjF^oJ? zFfW;>#sfMckNx89aC0omBe${%S-b+HGy;bmaokky#`1=ITy00eYS$E>?=aURy`22Z z=D6H%^3a+sLXp9Ly=;8dGbnsQW^mU>;Qrnnych)GrgS?|v|Q6VJ`EN18}oKQb1xT3 zm~D;%_soFE-6z;r-3fB3HGJ6jzyjk1G>Kj`_qD+YRnTuvOxQ%a{PGnAsY-cse!$e~ z$sP5k^AQD?C2moP%u#(CtXF;UnEYj!j&Q@+?D=SfT4&_@nRnHqPJ#_?2OPG!=qA~` zk7=x||KDDzh~(pHtB^`+p#}K)2MJ3F;6Q)06$>0WFbw0>A zEf`5ifsRE*H99VI^Q4I9i*AnbnZr8Gddw&7DB3~?T@iu|j{dQkN177KQ)S*693FXW zL`BkmPQ(`L_Fns=^vEX6WZrDVj1iv<%I?XPg=04UhoO10h@Xgi_rXKzURGVgz)i_N zeOSKz8oqCJm)x3Ed2C+UEfeM|)BGYjZ-J`eU&zN9R|Fq%Z=5PVfsR(!R z&BGXaOX4&_5c33@)p+8Fl@{}#?v%Ok!Qn?fUWKLU1nE)|C1l>h{XH=(Y9`S{`&$t? z{&NI3CVLs&y8SXTF9PjD+g8W7y#!*m?L}AREUAKHgw8Bez$>?X<9cWJ@#8S(!aRH4 z1iDsNbNk-jW>aiVh;CMby6{Ly+Sm#2;RzT|8xx~6r@Zk5B6heBFSd$s6GJOU zuk;Mo>8tUM?_!x0EL_s(Z zIhN!B{Q={+{J(VoL;X|&UH8_isCL`r@Y_|H^%+uo-px|DZcntB7Y8eUbal*ZOLG=V zJUVrg-FLKeBvcmW-8Y2iWkv>Oq~AqImG@ZP)(wp^!F|Xng&>^4i8+qFYP3*;FAHcuTP%9E%XjO|D(b-!Gca#c6_qGHE z2E9d=7Kj%ZYN0F?24sV8#b}=B*#qCsg@%IoLAz~R!XahLeNf$zJnz=t=4tbe5i6R)SJG#?t4?~6?p79XZlw>}U#3j&KfMj4U3 zzPg+Ab`(=T7AwLR$~K0!w*CnnkBT#feP5tcf%Ze{S$fQ8slorguh|3wZ2vSM{@|Nv zAyf!{>D*niW3f!dnbC4qoC0jXAZIob>+EgFeA*4Hcz%on zYaPzvx4(1+l2ur{XPh{^kX{>L6Kq2z?Kn6LzN|stS8%ZL9bY@yG-p)W5+8c#7VZ{V z3WQyg%_J{pUXo#5bJREc7cgVNFdlJdZ?Sr>F@)whle)eqQlD37#FCKPg-$y@} zhrbW~`SS|PB0N5hg~shf-zqmHT{hh`>V<}(Z^C3DF#MKsNAyc(Gk!?H0#=_Z`WKse z`x^Dq9`lY5KPvxn*>7D5r|67!dg`_4P4Zb<?VB3b6-!CjoH1}K_PSrSaaVrt zzp^d7%B*2-*Q#)=_U%8ic$L^9EPNDRd13IO*$+ZHt^J}Ox&2siFJd>j%V$c{->3h0 zownO@+g8$6xrcMzSr6Vf{0KkevVE$~z-DF1F~j&~=0T87P8zxSwu)K+FC!|Wd3kC+ zJh8FpT>ZPNT|;eqFh*-H_|l~`i+ui~xBcOlv2FZV{iRw!r9P`WW%SD__JER=g@V%J zRDjhd-s-tOy}hxshbvKM#EjPPilQ6&I`x`L z_f_G_@Vvsmn#a`0_`ilK;{<08rkPWqIk8ImnX6XHBXC>ICgSHRD$U%lyEZG{Y#R9Y zMNK1v!N6IZmz7}GSwykR24x<+>`wsdk5?(oN(nhweG3cn9UJ7*FC7##F*Ce*ymf;DEn0s5Aj^4fUA&I#^zEoKKD7e;PN~{e3Ob}wsgqucAGiweUN=L(t zu;FVSLU6RGjc$25B{JcIh3O=MqFuOaAQ@az)ow;mn|ksa`7I zk1HZ8*)Kz$mUTTm0&G-%eJ!Tvl!Xa=*~$weJgTRAxgYDQoPXEZ*|Z zU?UuH>>R;eP3;3d#~QuZ>cLk-zOs9*TQ1FopUFpc(CL6Eyno`e?YC4DloqGQ*3SIw zi_i7m_R*K|$LFjW-N+%D#m?c~#|`Fwwf&1M#W0B}lg$vI$h1ZMVXH}Lj1M@cqMt|4 z2=aXU|ERk1c&PLL|3R^BB(hvIN3L;?a!g_jCgT`pey{ha_WOJPp~tRcKJU-_{eB(K=kxV?D-okJkWX_P z&8wNInv&*$XE!&r-cTL6;B_t}qCc~>9_!X3y3g4uI%+;$K!M!!OM&2KWCE3FOvg)U z>77i37Z4OMVjQRUec1dIWii}TkFC5upW+BZO$}j$sh%ED6^pyIvN#K~m9T0=IUmTo z$obqHXtChu<_W(5oF$%J^Z$&`(n+yY!X5=F%p_cIlp{IN>!dpn_v!Rq^z?ZKCo+fK zDp*~w7^Z_WFp@DyFpS2}iF&`oK$aAwyQsbqHMsOClrix$Z5+|JtWIa@qqb(EK>NC%8S9RbgZnpjt%hnhJK5kIqks zvYZcf-t@n_EGZ!6`eu@M2A_5|Ir3gEf3Z8XJMEG`3GYOU6W|w|Jc=g~8mlgZq73*_ zRSQ%g3&Du20NM4J>1Kx|Jz!Ri zGDmbOC5sYXm5%%Qr&n!PTpqA=P5fq!;wFY5fbP`I^Y3AI3z_p$cFspW^KbY2ugYOu ziZmiwzQcji8FRQ)ddzqQ;f-h4d4EoKUvLtenms$&vc^IZ)j6J-+~QB&N*g^lhkq+=L9?oQ z2FsGOD;mSrzhzdg!{d~F?ff+gk$C&0E3w{*tk=BoQG94=0#}tFINvNXFQ0rc6liJ) zyRRy)^Tm6rr`XS#ZoCw&k$UN6HsWN$#h0^~yXdjH;Pg|V&c-TTh2gX11#EvCPR;Km z;^IdZmiUlQ(~A=R^5xj1ux3NdSMMojxMD~{ZD>>~;Y!7+p5rEOK0Zv2%|NqplcI=F zPsGkh7H>q&NNm`OSeB=e*~@$2L}hoRe)4aX^pDQc?d=&~cI?iz5Kdz6Ff<9Zijfw>ukoSDn_o(K5v zUOS_W_2#)CVD{Dp^O&6xha|tZ9bNV z2rAd%(L7f357-U;OuxP8S#Ua2X}Mfm)T^8X%506Rli;-bd0FXIoVX|51z&8dI!*Q~ zo<8AfUB=SPhzrP5FK*m$d-gwlMwXj>`FVFb$KLv6~Xd{8{b}jP+ru+ch@es6)`+0 zWSH4)FDbPc*M};af7x(brm^lPRY`}cB*oW`dmM;uTMl~mEDK6XdR?%y;<6k+^Z6L7 z)RSX|Q?){=n{0^J!MZ6W$7S3rl&u@`>fkquMU`vu*2;A&^DiGKa^LTMmy*q+?edbG zOFd?8Rma?$C!fWdQ$0C+EqjPcbgXwQaYLA` zrILHtYoy)?7h}e2ZSsfiF79p)lR!K6(xLsmDr!dVTZHZirv4-`e~?aoV?B;_PyVT? zY(DG6Lw4*W>R;pjO($~sTdtB~1PMb+EOEZwd3Q!t<)fQs zB8Wt~c1yfm8NS-&)pJe!*`(R>`A&-=0!-7w)vwggURN2-(||qxlU!N;i-VJ&7O0Je zmO9mOo0NA-tfimfrmva}X3z#$T#TN$*#OsoIv+d}jX55dU zyBqbVPC%~P6@{H2zU-O>zgz9n^5w}x@6FY|O;zi|DiPQDF;nnFXR9P1+!3=XGa%Q+ zaCBT+c)&R?tfxvJ{+Hn>o?M8I3vOfQo5~v@ao*w7e^%l_3 z`4vz7W$V0bPr`8@U5l{jb&KtGYG6foT;>uXB7#uc%InOFl-xxWZcPl<-R3=c8%1O2 zf@1(@UeB|70*&b+N|LDu@{oL+-UgQwCyAH$2*(S)dSaN}P!08>sPxC0k?=2aZ;Dck z?b&QDnLj+W)^NsWX8-1f*rB#dKRFyvKnY=F*{tWvdn)P1{;Tvw(7$@|Cy8Q8-5-Dt zoyt|ae?a5b=`B9_!;*jap>^X^$)mF}RTZ)YcH)qw2J8nQGA{PFacL z-lO?slv7@)E_&3=Fn#x_NF8fMu~pop8ElVl!@(7TI-TH5IBV60<=y0Q$&2Y-*l~>a zcj%sBt>cP%Y^qp9nhxJi3w#C`#ydzv>S7Oka&v57$x(s8wn{^o@SHnbUthofFIt1$ zjj2rQ`yU?p8bBb#;USk6E(!>%Iy*XT5+5N|WJf8v=uSk)%oTtyErZKDWudA`SgkDD z+{`#_)flZg|7t=2+G={u(oH?Ijc<|@S=*(g0}`3A|Za>@}CD?njF!sY`R}cmoBeYz4y3z zUScATzRr>o)zQT-!`>=8c%55(bZ_^lpNOqUzm%t*lGyp`y)B&wR1{Wla97sx>*P5_ zb~^^F4YPY9idu5Kx!bK^CzQV7Dq)ZI8`61W4-TlWA6+G|pRwFK{(rXKr#H?!^S(q% zhb$x|db(Z!gz9}!*{KaV!Pk#^@PL;2(d}yzG;gkhkZvrGqut6B{BUq(c({?=^yJ8& zQ_q3uoM6@)+=LyPlFO{8+DrZ1xE?C5<-&XUbAnt!22exn+gmqb?p4Op(fPC~SCc~9 zEOcvb56E3H%=3rz)|Ml0DfENl%98PXZ9%Oeq&xKV(#!9(=(G2qo6t&S@73NX6a5cKc^A791K#tnqDzJAEt|{%W}F_&k!)I(>|&-!o_wF{Qx^hK8(9A`CaOoW`LF19&zhvaD^NYBf=@n~ zq*WFy$~3~NK0q($_kSLgnS|S3ayPalfj`%@RlwHUnY@++%{@&h zjx%|Kj;rs9!*02zW>et92Na?rCfQA38AHBCLabFDGHxVw?{#9zBL7Q~=&AYFmFkSw z~^zvPcjm8lK1 zsdpSax5{qVy6j!m1r$qNs-kFwmSU%CkJHI5CAP!Wfj=V)-+p>m;vLikIXlFyeSPj4 z*vcUGK}s++H8{(M@=P##csLaAU0J?>KIDWSNT+W7!}+wKa`uMCW1Awzz=>acbL9$H z!h7w>DFpH7`;q22$WADko{M2URd~^@!*hI0jAoKw(Mn?Lqc3e z`JwLboIjn6P>{}SWJl)w=gsV*sVO5D+Eqi%ES>Ax?fo2ImG+kao)g?6DA!!EN^AsS z_V^p?S8V$zYBBVCVU$X=AamN2r*3{)xT&q7Zzs)S+V9L!h2RqVyY%gAfLBP1QgHP> ze^3-xI$L6x$e?5?BXaA5+TLtmm0&`PeDt}(@R3cQzO4KNpbA_8hm7O?`i^oV_T+P4 zl@_Q@yRWm&O8HVU%_$F1evHm9jc81rqst3@+FAlZR^;~Izb~1?E@U5-ZN9m*!*TpR=&W7!)ssXO3P_|EE9`>`P%p$U~9jg5{KPcw@!48Qm zY-fZ}N$WuCtTu{YgK{%9vYSbFpY9rsNBe1B?(C=U6KZnq-zK}><#X;NUMi&+2JQf) zzxp?r@3QK$cbbhZmKNw5ikGTu3P2*af}n?0@A{U|y&Lj0oLjv!L9UF^=6!X$8hh+t zu##+@!?T`T8QzQwgPTjhi7x>WG&L_8;F`!s-f-J+d>13aQmgp;8e+}I0V``K#$?FT zsqhK8mxag|*xlrvwD_fK)eZ`d`DskjVQGUAFyF!uV4IwJ`m&iNf^eOT4bn>Jz z6o|pBj61|%U_f1?L;ia)%=kWpTJD`0$G_OnhmT7%5JHG8d^f}CMc3;Au%L$7v-c=s zuB#N*c(h#)&>;rlTvO*TW-hmXs0KIH7YAz(LFKWQ_V(W{2Q>LkL?v6Tmu_AF^XWK6 zPI~!udtd9r4!Fappxx7PEwldmk9awdg$mLX(mmxmkRW72)CnnD73ga|zIqAUbkG!v zDynMDV-|Pw8yoPc%ByECHwd%vjNv>hIUkrjAy`k~3pmqzl{d0Gpl*||fzmRz#S5vg z`Il7++52fpC$5uwY`TNbn`>9xpxFs#pwW`(#0eUCIW6VKz@?o8pQS zSRwM%mdXXHVTBA5J+1{S2-KQ#6KJ*o)5ks}I#ert|mk#W(gAP*NFb zA$|&86h>y8(m&+F+-L+xyyRYj%N5%_Rmp5o(Kk6i-0wz4Y3Z8fkb%~bP%^qL>onnq zZ->cxuN51y>Mos+9to59XHwht?w$BWK6~jqGlja7%#up+&nf9G$LGAy4-m9(n{H>+ zI(qc+Hk!rpsoS1=&(ZNkJ|x;TUhr$gH~y4GiZ9sJEp9k~w}|CN`XX^e%BJ(t2}>8y zd_oVe+tH+=gbgmiuVtl2923XRzS}tz=#r=Inz=%*?9Jg~gc||M1h^5`;?m~YUVGY} z>G+FN9=1DGAquAol8%bIhi9X?qfvbT4%Z@`IM|s4XTbiPlSi@YQ%!Lc$5+?K1t%;% zlV+~ZsbEeV%~|^Bmq$eB^B%&9v6{YyqoFLj-D2@bqhH;~vpdhZB7l3WgUuZ9K8J0n zAt&H=AlC{ATk79T$mJPYYP7My zC(63bas7f@)RdfGe{)GAB_CB+o&~XI>S^2LznaDKbN0Jh>ueVEO*E>EQ=&qzHN|ZX z`+MENaRf_#>XR!UA02KTKN<=DNJ~RdM^Dj5;iL~sRgQItM&t2%dU)=Ynq8*n&mLOW z<+m==hZNQMUfJP+GYU=9^>*am2IsX>_Y)FO>we-xtfM!uV9<6$u_7iiDp6oGyX;84 z18sel9|=Hv+BDgEvO@_#C^d_zQ@xq-J*5VxrWaafapPKAA2|h4H;&Hl>Yojeb}kob zU%w5H=ikEYlbL{VS^S2;GFBTZlC;VduGKgbh}P#rqOefR;S^$DuzeH*%tg0CjD+rA<&*;NOeQ9Q>$z05w{pQT z+P+Htu1R_Ku8}pFX{0Ej$vD*-hisIS{dMxX?g#+_E?bpWoVb4PZ5&o@o~w2@He#mE z^R&f;_!SFgFh^KGf~D;*uHRp*pKK+wzT0{;Z}#L8Z6*bh{2B(DR)MS6UxtNq5`|P! z+fT}v-PdM?w$u{nxiCGdw|z&IFv~kXtQ7~T0TW}<^qY=XJM2BC^A1H~0BL!($k{U# ziSqpV*s9dY0v13u%fa~6#fJ^m2@4Tst&OL>Pnl3H)w+!8LS9zI+`IbT`lKf}ds5=j zH-)kPM9u{dyV=0oYFKfj=E?jma`d1LehOTl?!XV{gK>o>mK4)#0LzGrx?GKp za)BAjL8X!_c2Rz^+W$JP&hXmGrE(+=>QD?3n*F#>s@T!gFu0G>GLD-+V6*! zw8hGmjgQ@ej2|u@6kCt^f8&)-ew`A;WgvNaM>kpF3))|$o5-{Ph>CU0l`s*Hq2%SU zx#$N@Hh2p)yllC*IbE(X6>6pPruGBB!~-gEumc4?gB8P*wj#pQZ4H}IAT}bpGI^au zHG}?g2PZt`#+H$0Kdmr5Ut-SoEMNMFQMC;s@*1YV0{eWg|3xx|5Lr63rBbiXM72ge z4u%?kwnODv$#l-txCo_en%cAXn5a1)Fg$qdJ|DtYH}?o*xfJ|x9%0jOoeZ+p!M^PZ z{+&>CU<&8G?9wGQ98L|8;N;+WYqhO|C6r!|Q!Hhhq+K9v{kADxDwS~VTYV}!5Al@f zWY_%4X6D6J-?k+DlidOUr(GgIGE_iBq;3wdeq+a*^m7k1RpG-T0l>ZIYuDo7oVBdT zhhebXz^ozjUaZ!6CvNgXSBsG$j4pbob+Ljb*16evAyE~N%2=v(juD?n+KL=KooU;z zjd~_~cP607F0dF}(}sgy7YmkDDz@epB}I<+sXvi(lal{<*3ft6nvncjoAqb=g30Xa%;>z8vih(3Qh45>h-?xmk z+Aq_rk?;1)_c``+WdT8yldb}mfTDA>rWl?2j#WabWcoMZhJnTD`p9uiBNZB<1K0L} znUekQk!Rs!XDm9MCbT#cth#wSRj|`UQA>yyH={sWnfM4o?p0F^-z}AnF0QVwKr1d7 z7B3g5o>W4sXtZ8CvP6xjx`%naNg4kvDIzx#iiRP*(ArQpgWgs;>r1o#JfC1Ct{0$9 zk$Mtu3#mXVwwLTH{&?7Gk@LyreY8J&1(SEegL2lRu#OPd_nd;|WrK*|YWOXHdC}Uk zA5Q~_THngj!KqC3!QFIWf|&348iFz0TF{&ixD8ioLDQ7Cr(o_KyA(~BWwt+-qVS=KRfmFp2B*z4@G zf$Ul~x_dJ+qK!9|_plIo7XGrJpy29D@Ao?oC20O^yaVqYZIO-1W2QusW{XMpSjEO` z-u^y9rAumR4C9HLJlZM z2?sCB+RUwtb6@&e8U&l+R)l279C)zlD~d)6ZPwbwB()NK3^q(zpk40@o`WSELnqSqDT8{<)ijylqC4c(M8SvZ{Xsj@1yYneE?JYU3jF72P2Y+<%u-k=3gQ=mM16Wxt0RQQQ)#_rAG-XRUy_=3G81-n zu#f^1{=Dhc>VVKaSy)Em&4-q#23(!^XlZ5 zW39;XJ+k25dps>P;3ONciC3NbpeO=Mxt^F+sd)i{-NGv`Rr@yspt72vH#+=w{D0#c z-0GK0!J1F__2k$`dPsAuK#2XM{Eq#5m~+wt>v1@2nT>REs9~6z_fZ@$4z!Lfv&k=6MjDCA_G`66L~vV5eZ5hL#>Xh+9v>0BmV1?%p7AQE@bkk`v@^{t zkLsW>?%S|qBKioT^LvHh<^^tuvd#MfFEYDWnlSY!Gb1BIR~Il<(|Z)T+jg@Z+8d;q zSF|i&9Ftygs#D(?#}Pgx-)VdFz#mR()6f64LFEhS;+%il;gL$<=K$o z{|<->MGQx;8LZ04rz?;eS8o6xQI{KAf!&iOW~zHCd3MjB*uX~7pzr`GG!7#@WLsb` zQLhH3k2dIhymSYA){v97aBpx*DcAZt8`PVOs;2%}gktP{%Z}!fKTfnk2KN1J&2DHd z)%-_R=~M8b@&Xlv4_+Ne{nj?O7pNkJ7>sh*VOxaLi6%GgK{=?t0Q2bk;kkzM;EVkw#-~0pwqN zuQk}txl0`VibJ{~bm+MhZwCg((zYvNo8i6_DP%?YhE2QM4m*HW71X`2ye$}_Qe6mvg zuxx^*3W#;z17t4QJit-&o*aFO!k>6b-S>TZKyQ9Pap;$0+3O%G8|y*%1}tJbvyKwW zQ{;5vTV>2;&(}i*i2H0L(#c72?XeFEu3?ckEUH(xt70i>#+DSv(OSGcBnmR87tpsp zl`UH6p41d*svVc%t~o3|N)$R2T!rwJ9$Pk+;qtliLZ3_xCDv43ORRX_bbV5mKLLXV zrJ1_2Fe(D`b?j9M&Sc=t&+}2{5B?A64Ys;$*JjH5&0N9hm(1_9wg5-Ul9Wg{>K>6P z4j3o~IBlO{-=o;^;Jp+8{=>ynZ=EoDHeV4fylfP45H%A@e|-n;7))zHOw*v#cRakg z4ay?U z4C_|jhpthjj)}n#;)k3)(Bf||O5AvfUk3H}U-gMI`Ry}T@Pw{{DneYw4OOuNbU6S3 zC^`dS;cjB%#kr?KOYyerlB;-#bn`z7WFt{HnqcgOUW#l4rlX*JqY+r!Oa{1ffyFAq zwsWZ2^Q#3A0Hv$~wEmxB={fF9eg^Rq^BB7~c(nnMtNf^)Nm9~hi?n{h+V**Nv}R;f zVV0w(MQSa6hE}2_rIbtkX-q!lJhN<6pBLGDv-1YUb)s(l6-$YE1N)X^-C{2FYT#r8yx&!`XwbpIb8`OVAOL+n4-~!DF zj5u3;!@JQ{K4l(p!d(pB)9eT|GuCb}evk_*_q_I3T*=GiCEwRDQHb%;D?$VgL~O>l zk^cvx(@E|mtH7&{CHK@|{?qJ#A=iQp-QCTdiqzH3$e@kD=^Yw-k!%g!140J!d?QHF zwuUleY*m~eApM%W{U;j~^yMw%s3tI7ZwX)GLFAhqH+5XLsW%?5BD-&mKS?OXKXwBaK_`4m^p-skn8U?@pOhM=%w?jz+QTzTJHlr;7_ zx$5a3fb*5*YcP0tN>T#i8#I;Gx}PQ{LL?l$0fm@{waJs8KPGp!r~v8=QCD~{bt)1m zP-JdaP3CjKXTh_H1SkTzN9hYCwJe^EnGP^i_Q#tf{n*gJU7?u0khW$s_@N*r|G5fT zdQA^=_POnPEMx!zoGO1luDsj$$hlYb;{uZHW4BY-mjj+UT_mEl9A*+TfQ#+C^WL+J zvV?W0=VnYiu9?~YCN8u&`s^8TOw9$@4ZDC~Ry!M_aI32Ar}fT?M0940r)HeDJWco7 zC=(*Fy*SHow@{R55j=-~Mi#V7Tcnd*NzN7a!l4|;kde#`39Xnd@}8`P9>jiDY@H81 z2Zp4db*7$KGzhFGb1g=jPQaTOvX;CH8PsVtxGmE{^nO7r8?TS^a0Q)MbVm2x3WfTf zvvMh$pWi68Rh`?t5j=uT0v`Po8ja780v$dL0rbc$g?!Fzoq0+t*fnnszrNIqxa~H&^Th1&AEY5`*fl{92~peNGP{Ka)O_7cP5aNmh&P!sx-?d z(}USfnYdKQMN`_0193`|5NMuTptcq)m1;=x@lqA7rW9G>>?hh*Ym+qwj2m+Fwu1=9 z2vUfuMWOXyDJsoh1_$littOv^bi!@a=3@5t$z+;1WFna+35ncj#vctCFNZRNa3l8B zjpoOG z>D?N#XxwkAMJket*eG6*eheA2kVJx(xwxX@%yL;e($Po0llk>xhaKKKp|1q3#W~k= zSTk+u^a`M_HVCd9JLoo?0T;$e61G6Jelo2;&3WbL)KjlkQ`&LjZE>5V388;hQHe}y z`wmOwNcvqXBMX?SrJnBQ);(+8r6vQ2FfXP`tohH4h%7-r=M~@4W`R7CTAue++`wG= z^4Ti#7K^YfpCfm_&H&|+59+8qRR4DJ=kDTOuxVg)+_+>@DnbW`CRGyQuX*?_crd9H zGC0DuFfEwvTxk(^AT6JG&SS1HXweW)(%RN(!u+eJY6`S~-Gn>mQJ_xuDp$EpS@ZJL zaFK7uDtW9eO8lBk=u4HUfYH$5q0M~_#jVbma?v-c2Qa$ zlnX5qsC-eq(kAi36FQwG@)}U~vDX3NZT(fpL#AY;^Dt#!6NiLIdH``eZgZPr-pjoU zraKAkOnjJKiTx!BXr@IuP0IT!HB)5yT=*>n?XtZO=N=;Kend?Wb!Asep&I}?Iso*Z zs+q3}!RvhTINl{qOUaaZS0&HQCsaSgyxD7=<@n>!4`?2Ya>{j=0MB@38|IQ1wdDe4-&?n|KjmC)&REeCA-{>vGFeC!)G3EzJ{4fvgpUwdWw1^x-5P zpr7?yI2k$&#V$Flg3&}|ujowc_XdH_GPP0aVE!iUveDGK;1A61g2RgLFPu@f^T^3? z(n9@FE753E3ayES4%b@Md;Dl|fC@j1F+cbVox)tM_fw5|0A z5^#T0I%WPr4U8$6Q#DlHr`DD*7cn(;kM_Gh6y~Xz5MYc4p?_{5Os z?LT9tJt#uk5W;@k3CDSQoD4&;>d9k$hq2`+P}u@ey0ui{?FB z^eJ8s(x>LV$P}{V?+Oz~dc?VDmi@}d<>|aej^DEtU!{OwFBSLVs7daiuF?bjAtYqy zEoCzAbT%Xm_2N>?^lmNn?^fwVbr-El=ViBbx7sTs zGXY{NIZI9#&y7F3oKmRhYFLtENBrEx)YA7Sy+X5mM6C2&zhUtj-fgnkuixhd|2xiY z2k2M;Hw~1sz+lSawdwUSjdQk9oXFl_xmIk9)BxsEZof|4)cS;=frWIhFHT;ei}9En z*Em_1M~Gip^5^Z@vMY3VhnT-RYCM`IY7MIxjpUiGw`>|ndODg z5FppF79RpJv^r=}muW;N3U^|d{Ydj$qlF3oO)yIp&gdc?u30g}E;A{!93_4@h+&;Gl2x#K+7NZ_dPbBXaN)?WnCu(zG%ldE+~`!bzX;QOx5; z@=*qj)~#syN;7M-#?E#Sz_%W$&CzZ%QA&!(+kDYep(=_pivB5uD}&OJ3uMSzM<@&3Dkxx@8%PhBdRIhh_?b|M8a%* z$gy^HY&l}TrxmBuv#~sfWH)@k_4DrkP!xGYQ4ibRSy#$>4JfG71Y`Lr?o=O>s+}L3 zvXpK&al&LPhpZ@@kGu2*lCWK>^1Ly=1QATC$vN99UmUC4k#2nH$z!UVCv@<_u%CaE z=^UUjJPcJGpSu=P7ZR?K@J!=r;+Au$8cj8M_5nqa!Gdc-;ll%@+tq(9^xFWks{s79 z+LKTRi8aQx&ZU5fbIObLjuLL0Baw%UHC>^j!RLQgCH2Rupj=lK6bCag#yfDRVYXjw z3!-=z3bZkCccS|`Uju}4uJC~NG2`hdJV2-)queE0X?1EPfO``tU+N5QiF3>WSI+1{$u+$~TA5TrMLJjgY z;4NjF4C?FEI7#lcEk0dUM>+D*K1)!W!i}#35f9y%)EeIW6>gaUvTT}93628Lb|vWD ze+>Q$FbM)KfS#KXH`Z|up$Y1gq%H_J?j2jiP-6Alh#r?q-Z4NdShrHF2ERKx_0=;u zr&Ka_+c_F7N7MV3C;NMfUaFqcK+I6JE=3_JqI_MWVOY$%lX&kL!nfCtmtMYj`c^WI zrO_CGnq{BW$xei8&3J8#w+3u5bL(82`^pYeTJLyGCto|oCFuToctD3t)9|*hplzOM$I@ZEUqIbaEB?s@2_vx2*QYVie+pz|PoS)> zDpP4y3;o0C+{Mwa%4A-0bpEg7w{Y*rd8>clBJL{gk|BrP8-^#2zWHrE(NO_z{~7xk zrqHoA*BmvH)KWVOMNfP49(dmrnrhr?Wl(f5w-9n+FQY%~KhN)$fUwn1lU~VRKt)~e zi|&A(7PYP??42-VE+4!mBckzXJZVRw}g#r{Lx3L8=9QL>E9LP=}-V zN4K3t6`OowZ3&Uk+7EJ!pUDwoEgp^wozq+zGu^jpp7Dk|4Iy#2;bhy7kAt=gZKwo~ zSvz;--pFoHIE4Q5`xsn^q}~!LSgsR4PP?6Bp5Tb+Bb8L-#yPfT_g*578WZZ|ZZSLl zZW?DkXoD;bp0l0y^7LIj5bz@_bI>;T&I{Hdv(gVt>&4;`F^46s_SRb0NCvuFPZzS& zQ9}f`f@n^1(E_*3*T6DUz3G-k*iGTZO`4-8vdFv699!%r$(=lBi60l<+S1F2M`vRL zmKn8W6JERDDXS8JA;nXHk9*}1s#G-4J&CFyO5gD8V){ZD-iv2qUxdwJ9dq{mcZ{;^ zU^B`@$FfD6HVUZhpoY3h;GgQTMaPF&LzGaKT9C9m;BEXDlxi#p~wy@D0M_{uIH zXL3ZzE0+2s7(b+YqxvVTF}~ng6}lh_`Yf=ny;MC|d6|B?C@J*)`n`tqo(O-h-^%FS zvbyzf8IY5uEWU0LH|}#Ft-$r=9LsZEuV2e5!^G+|?WoRFE&UDDEhw^SWF@M;8xMV; zv%Xy6?HmsUEVaPjtTYIji*KVP8-Z`X6w6!a7rVGdU+%qJL%BJ<<_yI%2$`el^+PW4HeFJ$2-y|7eMQlZ$7UvdU^ zp{CkdROoP4r(m^PF49~6y2MRGv?PS)5yMf~qE+om0K0#7w}ITB@N7kWB*@h!E%$Z@ z+hvrVdYxhyybc*O%g(Ngp@<0! zG_WSX5cwa97NB&Xck;xeWI5O&Q5{@!bj zWS66p$ZqlXI=hi8Img_dQ=G6AgLv~f-!~9F9%W(TDikOGP@P_nE$<#~81WXM%HWPn zBxiik)bH$MBg#$Rkd^v)#%_hv*0tv=^K2|8H*5T&1fjWB74m?+PNF5kUC+H~wPbta zl7GG(mSsq0g6&(SKbQPwt4Uy$Ww-Ld$`o_Ca=+3D8~oXuxG`uMHGhRhc4FuZM6jrh zZYjrqIP&poeE0UhCRrK#6(98Q)>JP7XpSJ(YE*XcX#N9r1sNpcy`60xh+gj*a2nTN zI6LdVMX}5J?p)3x=OPnc#6R%m{@b6htRnz7JVLUu*u+bqJJr|l?(v874&-xORYyXU zFpoI=!rSNS{PDDM)_5wfvj$k{ZLMq_Mk$A}BJ)IT@Izl@HuGa~*^zCHUW)xaM-4GA z$)<|oyVz@GmtDEnZ_PeCKGvAI_0Mstia_FUrW=aYuWJO|J)b@^9qX z$IgU&pq8@SCTi7_7nLI(qalOJ^ia z`93<9qKzebk(o64)OW*ljw!>O?bbmOzf>yizZ5(5p)pCn4S}0mhi?P;OXY z@{a$Q+Aarixo|64HF@y3g>uhp%r85^HzB@y^QahI~_&+=RdKIJ|y41|D{qFy=Vn`43Y8+t}>RsQMy%%qBr?#Yn~3+oqM zpdl;j4wl^Q=>9!0mq7lMF1lO=ja?$v$SIi;6qob-24syMHiMumH<=h<7~TYdB8GK$ zU3`NyH-z%NT|qMQ-3x6)l8!j{EoLk_{bQc!uHi5vR*>$Rd2RxbVW0vmQ2{Jv?GL!X zNwi?queTD0gS|vsja;(WrBCF{h$9jDns%NE#5CvO!D(Y)pk>{W=JX^3mF zRfxlpXwuV$zj!IG^EQ8YHCU<-e3Y+2<_&I$iA2sQhLfs12ADhk^5{z9MXL^2|2GM8B!kn(4ACh3G{`?o>t9F{GJ}NmT{Rj z7E7~a3ku0x^6b)-1!c?Yi9pUetfKzgbb&-zkY59xgFh$rN8Y3g*IB;&TF?OZcqZh^ z-L}3NT9p4KBu;9tIG1BG@IG%HmX}7Ap9zTsmThIYy1R2Pj&@qn(A#|3t8%|4WmlN0 z5WvSWL1x9}=n6rO>J*hlNS{wiARn36!5H2|p7vtDx5k{^aV>wiuQHiMzvxpd4{eAFUGbrZNR6NTs(A;AWnUrFWTg<49L-~oyUCO*2)u~Ecv zmI+Vh#q)GgM3K^Iq7X;)@Y&+A?j*(M@W@uV3NS79_Jot~XT_ETe-`t%&UcKYhIsXf zbFwG#fnWL>_sPuCc|4up;__43%*z?1@Y13OS|{8WBj|fDhyf1yZHeg`MXY*1!5N26 z=k(}sM*(HKv_GS&A-!_ZIaJGvY9i(7HHfJ6RZ$zAnt$q|a#p(|aQD=mp#zWkTB#jf zn+i=fQeFrO`V2rTeY@la^^>Q{a9gaFR0TQ*7EcwlFcr=V&GmRUJo4?!XqUz(j!FJ- zltqDVs{k3 z*p|ZSx^Py|uybmf9G&k3q{nD7b)<{h>mdtsxyR0s_v`0jjTe=7tzLBUxklC>>$=2i zp@TJWyeIEbXO?^0x_Z9=f$~VeeikuvQD*l`d*Uxx3ZLAH3Lr*aw8SKGOT~23=!dGa z$9pMnd1Z;I!Bpm=)os2A$-_?mm6$SC{q}C+Amwr5+|tYR5*k%Fnk*L7C9UeZ^Vy!AB5WE@1x@`n4W8qf(f%isZwfMZxSG_A?wCGk_8s4fiFl!*jn#G-Mj@m z_FN$)uM4``aXQ6>F+f)5ooH=kwbi-Bh6j zEb3@sTqshdkQ}=9F8DcTgc>#9mr=%9qJKorrc~wK_&8ODN<5%8D+e_<>Lu z-^xxLq4M0c!f_^+Qc5m3apym9nSKf2>Q7VRn*F{UXm6xF$v|+d4OIJ`*}0+#w|fm? zUAOq=mbHvT1*JyDLs5>`-xZSTO#Xrj8mVup4EvbTsKsMtbAwqDy!J%3PiQ$LvQ0)7 zB|{q5Tl~g@j!D_d%9WhH)M4&pgN=A%q!un@tNuU(sksM7}UWSv2P&O4(+lLMwVV|cPIb+>>REfmsU+pA%4m72F1+0-I8E$(NX~AAVrxK>65Xr16Qw} zCAwR6Hgv?VsX@HIWz;*JNNl*VrCg&ZE)s7D;IfgpD?u=J9C>#&bUl=skq*(FWSVdJspj3hy zICW6oHa5uV$YKQ9Qdkh#l&vbaq3Rj0E@5qK4FewOAy=mr_6MZz7q;qKk-@=x&uSUoQHE~-vU*YOWeb< zNtB-$PC-u|A9lObbhy{}3{bYK%h_iI4C6;{{#xmf(0axPJk`h`h0`^%T;aY)6Na>Y zRlCo*k-c8}VnBoZH(Uo0EL$m%4{B!{5$pmGC;Dm>`d}2X?52^1(O)o5<3k zHy_g}eSqq6Ig{oUH(&W=#>fSZH<~cM`jPtjRIv@EtTAXR;F=e^uxs)YQiB)Qif1)6 zvyDnT;pKyPk08Uov8}Y}4ov7)A z-U8HpVh-z+0)A{1Ec~VFP8KTr@;mbI7R0bY_h)aiJaH_`x|el8<0}qUDU~=s$*77T zC}W;qEa#v8b>wTVwm6vYA+2&EiGzYsCNSaIe8Fj=Wn+j6R&k^Ai0}Z&q#|SjTCVrt z)?wB^4K6XvYY92#@dQp{e@aRa(wZhb)`FOYd*!qB89>5%J>(J?#U}1TDz+g;FRLOpNHYR8PPU zQTslELPZSqGZvr0q`kFQ*{+`AatSm(Cpri%bt0divGvxKL_daw#0UPeUon+V?kNi1 zkuD5F2*uzt5`s%XX?4it2<`Oir@iDqyjy11V|G4B-n3ir$6IFNj{~`^oYLr|5GAza z%Vf86z{@wJet**``0w={AXAr-h$jMGZ4dz0Q%itT@OAsB54{r7Cgn9NQeld^Gb#u9 ziG7M5yPX{x_A1=Cqn}>RJfGo{*vL%z*gVlzC(4z_2-l~EBBGM@d3xwNJYAN#VH!2* z3?%k8oUGZgbI?aKS1Uw$I=|eRdn2t+?w;l7PUr_xJoU$JM;sGrEv&3$J%$T8NFUSu zSX9(??EXru$Sc|_>$~fQeE!gPK7y~3gqndlmvPeikS$A8aB ze>h#fn7{oKQ3AQzRj5nSY^#!ULUgO{K$TqF%~0C;-&ziH1V&E^0%*@9ru=o8C+QB~B`abuVPbT3YApuB<7@Y%MP;E^$BvdleR zw!>)`_0;$Y1tvb(W9vX|JFBA4Msq77!7s`lNMES-49-~Aj+QZDngMZW z%g=e0=Ee-A<(y{egN8iNuA(Ze*TEc5WNwP7w4LcwQV$)R_q3fDw8_1j!i<=iFkLT; z*?|~arY8GB=xB_sn3>!fh@5y!U2IWTD?JEyw=1&ChE0B9KZIs<4!ZRO^`yf12ml!B zhNcf&6!jIVy_zmP$u=$7EThs6pED7I5R0#@Tvk#s${m%fR%eXhXuZ75j;_2ezW`?s zReBmj)svJAAhr8#%_FUoo6wt3Qv;Y%eW>0%;_qY9#K}z_qcS-I5?6=mdB(`4m;}iM zL)*v%xI7BVWPR-nqmP}bN2PsrIVc~re31?7iHrXk>IMSiNji0h<^ytZx|m0Ep%Mmnsaz&wCjwXPj*-3TQbH^4^r4#EZbx64#SnN z?VtVUR6A+ha@llCuS$@Fy1LQqp!US0FK4KEaT|!ouT&@BS0AU5HV;l9`s6F8VH<EywC%vfB|8)WkQ!?(sYG<{z_b)EIh6|*9IHd z4c82O)U-?zeD(Tm{7zf+7tly4{0^$oC#FNZ7d88@gMH85o22ArlhOu4UhCG;y))=0 zhfvFa4W`=^7qLw?;(Kd%$B_yJE2} z(lxpQZV6S7V17zU0;T|=1z{0DSOJ6owv9gl)9ZmXGW+unZVJ~E2f-Wt_DYS5vejGW zRMuzSH_Q)WBfn@9LaMq!E%S7=b=!;$`VDW-TfsX8PdOdx1m64BVG2y-?qU({Hz=tg z-}f_I>_>g$r+C3(u2 zUjhyz9?nlqTMT^Y2TekyoRfZ7hw;YTY}2Z~$ypfb_l<|nknNtcN0{zs$jEa0;bHS5>cBJkw z3kJ=VC81gUR^n)%EZF>?;tpW;;t_(ok=>^>B7qaVM7FSP6>?4|z?<4|fRaqyIK3(8 zgvaUJPvDdJ!(2g-MP+l>=o@C(_;dRTD4~s!EX@C1oW*I^{|U=m&8zLY=_^x9WArkH zdnh#9*rms0)0K<-yLn%cCH*U+nX;&op8nwUKX8B{M>SYO5z^XBSHYt~1eG`4tZn+O z7PP|$P{-Tyg*G`nctn``$v9;QN?xZzgN*zy3e`c3nfiiFjT^(-J!+$eYFX|&v4oi( zvxg1zhyhrNt489q`s45@xxl4-V`Ysl{NbXD$~!X{H^^SUe-0X4W=SG<+Iuxt&U}zj z-bZ#mp94S9yiD~7qnl;t29*t)>++yJ;QJ`LGyT#$zfQ?s8W;mmoCi=dWcC`cFPAeS z%1Q$T!i;>Z`Z#5ZAnQ$HT9P48j_M1!<3GZxSI0Q&uHM+m$bO&9NLSo3i+2xXR0K8{ z!(1H>X&6_`+n z9ezsmiSDt}s2vFB^*}fEc&&Jg(-CC(#%#P=JtzLEywvr_5M=(b88e~5#s%3a`AIWx zc+;I%F0wj}Whiwm4O2m+Nvdbl1Ld9VoKSGntlJ}A-=8;yFY_x-3_TsU{ArSRK&w^^ zaOOoj=qWJy0Sf5k>6|k2U3Rmq%7wpI#$eSmfeG55cahiVC@n`>VtCuk!ROBTUpO_{ zi@D&NRukTsA4~QJSPE6Z-OZ#r9;1`Lot8~T3oJc}*UK+kW6diT#2Q;3x?X#yfCd_H zR}ON38B9R#3ckoHJ!V4VkiIgxW8a?@>h^ERj`O(%p99Z<0(p;2<`Dd=+sHOB*MaF^ zYA?T3bui@j_xC@A?VDv}r~14Z-S;h?l+MuSoZ?WOJp`Jr3H^QXqbiu?6x6My=lNNR zuAE*eONlLc^{m0$np3O$6@1^I+Hj~7fT*W&@Q(2qv}J%sMlEvz8Lr+(q2DEevF>Vkcrs*kO8RV3zJ#XM<-ENeS2vtH6AM0gt@dl1XAWp2{(h-rfs07p4j-HH zBA$6xkZZiLc7~p1jvdN3sPGb&NWR+U2-lv~`0FbPMV!WIp>%#ZkPUBYaw~+I{ z2hTeJhT3~u2o{X&2s-AHxjDNe>nmXy*yD#G3kgg&wP^jN$znE^=LUK=Th5>s0u(+; z)6jGMLUn^6x6$P7~H8N4<;ehyfwoJf-eSWIE=jr}ks~eapszqL1 zig#sZJ)FiCygc3Noh7SY4nGK5qM8k`6V@Nn4)pqRF(;1ZefcG9Wb99?E&I@&x<7>i z!9nQkKzWpGn6{ynTEFs-v^(DsF>HOwv|s1lZU?>MhSR7?3eNlLeBkdwX?C+js2JfV z>T~RKEO2DY32*+7tLu(yGF#d&9ajOd3kax-f{LJ0l@bphHdFt6Ny;Lwmh&c^yQpyZnLbMBb$$vNVyI`F4zgs(~l z;5n=ogO1QHo?D7_vNyDct;uZnoALwJRXLo4r z$(M`fT*bh8_@YY7Jj>xjURux~+U55=W-W5z`6LuDog8lb?1I(wY< zr+#I_wTjt#F-X&ci0co`Ep=6}i7(j{=o6qHg@+yUEn(u+1(3dupYpm=A$*w^+3 z?)4&Y*#Bg&`nHBG7xj(0-S=Jq*PG<5#lB8wc^skTSAJxRgJaXSLT9NaE1d*N3tY~E zdG7wAF{n$QTixj{;YX-AO_Wa%SF{WYP|v@rf9`1vAJj;l&~*~q|C!Q>bNi@>cWzLa zC{3>knTbc7Uvx%415Y=3FSi>7(Tc<5O`z6hzxx1pR}77hpCk(2J3CASoO2tRE`U1q z;+dj|#B#k=IQ4f&wBvGiodsq+Kq_&JoLGre1qfzluk|n;8*+@wmvbUA_VKJ>9|G+= z)y_k|ZTBFHbOo5F^cyJ4vzET*@V9i_R@3K7a6BLZxRJRF`sjh=JNu*99h^ykw@9%` zV<%}?@wqX8_dfM57FeYQbgVV>yWKE(b1qR$+E<8Jq{`a{E9%J^N_fXnfLZKpNJF76 zwuyr4k{b9MW6pEV)iXWm26fuv%i{B`;ijx?1OKS;@kfr{I3P{-p}C2}!9La_W%YKq zKE`T8&qKDY^>JVO8A8TtoO?EolCBMph=_oea4rU z$+$d$e4zwt<*1o=N5Fg|d$UH@NjKKPCNEQ2Ozq&I;FQ>>_M{#6k5t;;-#Y8?lqBRa zmkm&QCFdvi$a*7I>w>`7CuWxMp+OU61GF?R4zvGCxDkj^RZ$%RPqdwCsHFMmY-=65 z1Ly=pH}(AQI1cxh_=X7eJ+(en@b7<)_#WiD@t$Cr?|J@a+}BE9l%Q>K8$&&k41r)qEsBD+o`@K>#yvvAW}n?TPirT3>mB%+}oe_dY-dZcGCyvVa;~DQM&h3tOmb|Vcg%A7pur5_%XW=sX#X;3} zjElu|;9;J@*q$qYZ~ob=jA{wWy3)RC9)W=O7`T7`48AY|R3jA#wxJBgpwvk}`R|}o z_I3<(n>#vn^nT8L*b@&7QMdacoO^~Fm%al&W|iC?xG5*|p<{r9@2B71GIhcK%1dfq zTHn??jCme&;)q7e{ts?36d9n#awe~a3b)ipl(1jHBi^@2e`A!3U-c^24|QqYj@`2i ziD9tJ-NFj!6hVcS!i6z`c;UHT0vFnt2f__;)vsPR#KqVJqn7hBsQYT|4!LP_beMg` zM#p|lJOK8{cNoh18-~CftL_xWlr?~{Et>6W4*Yz=;KvPhFr_Yf(BuQB8v#V6CJO!C z*O_knCsvV_n|i%P78~DJw|2e z-G3>*Om`!MCmH{G8`=OQq%3AZk~^Yw5s74B|fukW6+K`meO zkB0Tj)sN|YgvLMCd6$#y%3eQT#+KDQM`(fzmfy{=pRP1I)YTUY5 z;B02vMpHLF^8m}&@~er*mtPUu54UgS1<1B|Ev;_86em&Y zW@N!%2?#5yT%T z-&jMrGW(4DIKq%u&SA%G;G}hQ(Bw-HYak`-F_-UfovMb;F2aF-=S0rpZJ7 z2JrvGGWF|U-XXLf@AV8HKf6`ckHN{^Hhfa9YU|6m8XQY8p%cf~*tD10MfW?Zd0fVL zbtTqC;=u*;jRHFzEbA5t@)MSK{!@tpj-8M^=y_|+Jh1uh*_-apPhie@YbDGe0pk5= z;P2w!5fMf?hnGUUvFvCJQGpDHEPqN+wl-9KbH<07#)I;_%#wcqPxQSaI&C`8v;9wL z!VWcM)m#>3309Zb5FANcCb)YQ^1EwTj3l*^dbarWjsq-9T|HUEP2DX%aR1|Mdr=va3n=V&`Ja(jCk*hwJMD4qO4{L=3H6f*j`(yuncqL?oJdZ}^wefT+7VdIc4wsgRRA z)pfo?h+a4~^K3T@9wmSSba7L$Eo1A#T}6u66|jVe{oKPMKUco;I{TvE1DW&Z;_j15 zJwC3CpQ+33B+=b3*QZQ zgD?NAgYI0HVoy$@CO)A>pC6!PE3b*b#S%lf6lZg{=$DL7E6K1= z_$b~iI^7%&z%p?3EeHyJ5kI@z5%JTOg3gNGGzT>}FvoKV6Qt1lr_s6ta5Fx5!gT~s z89kgIbG{i*tSsWX6q0cY!a5|$&N2Np+LJvmrZ#F9`Z~l-fQ}foEwZ!b&XAfJ4HpGr z7UpX;hf&A^h2M{Ag8@JNsaXp}6HqlFa=~r=@>-+@hdMoz7aeOMN(t7v*_8FezojzE zD+JYf%ngDC-f}4g>!KsW8WETS*%Xx=Bq5F?EsdE0bxd0d;;AcIQ%`Z;z{l~mro>9s zWvHwSSmP>*4GQQw_Q{#Cz4GrAQu(cN)|SMrG|h2YDHBD~kKRDu?7*FxUC394nMNG< zI2tRxBEQ}`;|`lz()rqvenA^qnWxp6VgV+%=f9W|#dK1^_CpUUs-xv(L)HQtY4nuF z#S);*t9Ikd*xcx=g4YAjO+Wqt1WJ>SfOw63@hk;5xrMVLAe308qc zK-~eR`FWI-fVfx&dkPyW50e98c?m)p2%6EbyOfaq&xj$;y`7%DdL-L~h_Uv(DzPJfq+Xa0>6P;T9H^4==VUOfp zB#}s*VeKOO-(`C(KOi|!E3a9Fr#J8yRH&);amZnqh9t@YXyOfu6qIiK*aLpyS~m*% zA_9GvpIjN781x4ZlUNI_=N>^jf`ZbBK-BYF=P*me;rOsc4`n-McS|Sf=m}Nl7jMo+ zzbu;9Ozo&KV~A)Us4Q+tETc0aSb=O2jT%>i(bt;I+-=$uPy#e zo#03Bm?Nuf&lc}>lIFh!VFZ+*y}SM9WGaoZN%JV4mgMZ-C=Ghi)^YcittIujZ7(*W z@A)-6zu;)Ktyym*lTJ3=LE6j&50>2c`tyACGf+clUU^hq{KV?qVKuy9F9=FFiobLTPsv#L zIK7>LdGFyhESvk7?@tj4skQXuT0wt%O`BX;!wf!~swB zVd$e0yH@jM*JSP;fd!2(bAO~}-$1g9pW_0ZcPD(7A9WiB60?kVt`yT3C8O53`D8M8 zuO%;*8%nE$#b-^`doL_;JX@3qA@nBb#i#Z@HRQ2(f)i;)^s<;3l7)K{HDcOqXiwqi z4=+q24ZhUtC%H!qZRdxAnD`Xq%?rDOp}_%gs>Hoa_vS%G}Ij&N8S0`&GM^pbnBGL^Xtcjp7Fmd_M4V?~4af;07<1 zzo^aYb;#c7G< zg+VKqG-#|m^FVLOmFN1NIjA+)J5o`6s^8#LPk$P9Lcy}9ythyUD=Q5gMt%mE=5T!R z(ikomjG)&W3cGpbNzT#SR9@t^#r@Qlcc*#`96u~32>OYx_oAd zJLfo0q2U0;1I*?OPH&!>M!ZGB&)v3pKDXN|k)!N>VB6w_iDI=7Liur0FZFG=JmM(N za4Z;`@f7}t$vk-|c-s2~%DfG-OV6vygFQqXk(=DYlVM2S{`y2jV;A>Jt=?U2tFpqe zwDH8TCH z%GcWG&=(m)T4G7>dD*Yh%WSnD@#;NJaczk@YyCB`If;)TeRkN{sUwrryu6!pGQ9kL z&3b)ywF@ts7mqT)8!g`&bqb!)LM$qXB_ACY&AQ0#EfcR6Fh>%pVyvJ=eX!@2NF6 z&r?1B8_#7}1ZgB8uwC%W=rui1LBIRx=7W@yOqf%g;6_N%(pW#Do25z9VAx^TR03<%ANRD=zX>S9!*L zEC+&>h7M@|Ge$4+pv&=~huK{Hzoj)f4QOhebpnwSOP~KRuitafS^;Y6Ng}pRMoxTZ z(v5)USW*38wykkGzt;hJPb6$d_d7YqS`Vv@|HekCggPcS2RuiwS2?@GY5d-DoXlAg zDLv&7IX@0yV5WuD5Zio;Q8T{z0r@&z>kl|Y#_WB>s+s*@3Ckdsd!HlB} z78&oL#{@}R&bui0wEz(3P>M{U(x8 ztK(8=mB8NP+^2U8hlJ@q#?2om%(IN2@L_2=xdb)5QilreAM#Q>FXf|MMA4>Oqf?Uv(D`Fe<2thoolm&IWGDsYVpx{)*ckk`outL~STU>1 zw-xUK;8AF~+4ZcerSJy2#%AbdQ9mN@M%RpC8ieTh%Ksa!Q`UVfEB|jPeLoF^5EuyThR#sa62HMoJb4La2Ilqa zRnnMQRU(qIIrX7QPJG2+({jtnMNptWHGNM1SxwMYtV$T2O1C(!S=gs~C1n9B5C@rw zX(xW}RU_;bFt%8<``gc8sPqH+8+OM=mASPc71d6UWwStW840hA18F7xvdX@m`@{yF z zC8M6Y1Z{yxcp-)!1YLZDc=(@b*FduEX7*nUi00^d=!}^KBhnr!HZR3ANJ%nfLw78JN-d!ZalUG(wgt{1>p#cq^XfJ^90X1w53Qb1O+ zYFE>$s_U;A@`9*`ldl^>GWT=bE-?Os^sCn49PkufZkyf7ub1tdj0>C`Io?Ac-$Xi zeqy*S`%IiUCj)Dpa_FN&ZSF^z2q&7!+0P+Go-w?vbzSc9kO7pl{Hs+I+IVQE;?0jD zK1iEH&OeQ44Jbz~Gv1-aj}?1&UYH(3+b3yetLHWL{Z&pe+VN3EpmdFd`$sA8t_XZHx z>+Q3Q^#pm|sz zE%n=MB#I-A`J*mWTl{46?3qipPW~mg3sTxFvR|79WR5}yQA&vMck9fSute|H+ zn^^Bhl8j$~MU?wh-~mjzN3A;vxKNZK4Xv~$@@NMwIHHo(F+#7B$}VNb$P0i9x~UFz zp_GtwY*ajaeDmG8<3D*l*+u6M=rO8oRsJK|t?5C&9Nq={`yk@z^6V}8#x0A0-2ORw zy^w$F0&3LjB0|*{8hP;-_p4q5(5C{$p`E$^!`b6Lf}9jjf#UPoUO@&YbGb9r1& zKl@NvqVU97io3;-#fY9ex^;M$0(=utgCCTxg4dTd*(VCs2t2P@rIoe0*wc$o46_SA z8Bar%^QwA&DKZi8Jd3QB^mYL*ZDc}2Io|@vPCnH zZ>mF0fp<){KWpveK}EY?R80Wwc zt?l*G_31*r5Zep&H(;)5+3Z*uvN}m7+a7v!<)*QXikx|8=iAxVeR|*42<&z5kV9{! zQK-@Z?yF0EyZlDt_sQx=eu4_`Z{4q-XWy`*-5QP59;^%j?)~>5Oh8!jo;~4XAO+_K zF6~Ah)nGjyr3H>1le@)nyx)-4jXORusMpU=g||zg?PKiYY;22NBz#S{U0|z=GAU+c z9PfVddWhDbULV>A-t^D**=a${@=ix^!yws#RJgc7S=ghMb1W#;1?Eu z2i{WS>5GRyiECgTS0UYzTcFP%ht92N9zC%>;y8K>PJh}5RnpkL^ZXl>}4oB_|W z;h+3Jf1e%`Vh@?_k=TYCpE9_ZIpf|ril38A>!9CW<>9$3G*&127A-(ZFUb^*yvD188?Y6I9 zg;YVj>>l93!x1Xm@B%V}vMCvivLmqcou6DDWL;QYTKJm>XB1zt_35tU+*~ z{7TjN!-nq`75g3b4t}A(9NWwn6LogI%~oaXXs|7x{glAgaZQmZ!Y7AY4|XM+Yp;)5 zc=wJMQBs!>Be7)3g5WK*&LSXdZ zRW-AYXC6Cz z_%`j1Ogm=-KNWDBj*l_s>X0C`J`Do0L8m;Bi4Fx9B4g4C# z-+(rrnU)ADbvk(X@Hnfn(&OA?c!_zKdiQZ^wCEp7%Bq4`Z+vEfVut2Q#XYcT(Tiea zPDgwJBJ4?Rq3(eU5tNBGd&ylw@m3m;$;`};WnSxRZsy!az_^lYg*%U7hlCcztJ1J` zPLi6_?H|leN@TtG!8CoedXv-@K_vmqc)0)pX;qY*mU%qMbb9Vgdz~8o=F-!rPn(eN z%(~m-cLw@qIb<_9)69yK(`n>#ORl&2vP7vzx0@^18Y5hary0K zHYiBRb;PXJ`B2>L4Suk%5vK6Ak_tZmHwPL?t_y8I)CvzANt+RzhlkC*}-;6;Q&rDDn6z z@#72cZy%ojYnf}KiaWjU0|N2U?*=eR`l{~x6tKD+a0TozzkkevpTRX>ui(%3aNcQ& zYP7bQ95(3!%=-b`Uq7*KVu`=G(L6JqOgLl4_B8Gi?EdpikLsEce=7$c3FaaCLmv8G z9!8;>InRi^QGnMlz7BZPl$@lWw(mYPqph;|#!Ixo5^_Rf{Fv8r`bNad!he1;^1?mkd?)(^p_4S1J7WBQdrbpnuQz4$UGpyl(4~RD7yHJ zU2L=Nt`c$6aNWDc%k@h{O`ZE%rY3bZ5>`^yZB*9ERi@jnX1VHRMI7QCiPH%2Q(kk-8 z--Z*`SrO%E$uo`Tocp54ovD`(^+1%Pxjb4sGl$WQQhe`X#^HW;r*;{gmN2z{BafM` zPY^od#|Z7dCZB67qFBO=$wuCqn$3RLu$!_KGhP-KNvvldJsw_}f)|ZrR?jm9qD1#^ z?jp>`udKApM}vr5b4ly+}?IJJhL&GP?1 z4>db(GxP90Ga0b^-V_r1yO5$l)tLm6)rNu5?-@Mdd#tkfQNR=1eUvLDk;%y4pVK&y zm??H0<#wnIbWzVu!BjqqZCFqKCOje~m`!RZh z^weoJIsh96@;$*&y3a|$b@;9fpkS#ScG`Doe@lm4ST1=T5q!4mXr!mo` zADb@dO(+nTZFDAhHL||VHx-Vb*WQ!~SxR?s9a<6LCD}YNe7`b(J#qd~H^6?aneD=- z)db($Drqh;lDuJu=u#!85Xr}@=Yfh|D=qADCz3Z%)%NNtNYU4-){k%a*CYY8;IqW{ zSUUmrLS?V^QOEk9r3RIr`gy^_bTKE~i!IF>p_Q5u*G0^qy zB$2JF)6oMiN=|7;VtDE1uSuFn-rY}()}8n7y_JkPO84s*{p(eP2=AIC2nF&bE%(3G z?q;1J_>p-8d}_@oxiDdI`~pHLkuB4ecfk-hJLx%yX>Wr3J}PZBRoRj(X;20RF6LHo zo?)YySs-_o!=`5Qg(2HZ+YK`9N7Y)vx6s=n3PXJ+S3hLGp7nHwK-^6%D0VzYBjG;JLi9@&`CpW+&5QGtY6*4$01_imsBHGA}&B)eV$ zzb#wREa}+$x>vU1QkZyB>JU33Lw}`jYBSbV!Y?q8a`B#MW<)S%c4iOJc#0XFQZX`y9at@_^@wjHT#{VAWA~foyC#MhMJD^?*J7PZhKha!!bMGZ zP}#LetlMpsHmEFf9e>Xbma}Y8P$?vMX8RcK7KYC75&o<%zW~)n*3;VCes-8vOkJZ{ z*{m9SoI<~bb%{=rpmRm=v~O&-uwu#VHvdXY(ZD|LSwo=ZIz0X1TZ5j=9+ZlGoGAcj zzwkoNK|4?h0e;%#Ap^fH){MyWg#yw>?32*s_$VM$su_}jQM1vcKLwta*W!YFoKf zWwCzcZf0>mRpYCM#)Xw+q)mqNWox!<4sSefhYX zF#|h9R7&8E#m#;pmsy^Od@uIZ?qq|Q=ccYv&{CIzdrQnDS>H&juXJAPcXIiER<_>} z0O*$s+xBqERrn*PUhwvRoo{#l8o2H{bhw6{scnvkRPhnRs?-vMcHCJuEL3YPsEArj z>doX0@R34a(k2`r5cZ&Z?$0#n zf5H~I*pxc5nT4@LEk8x(bmcK6gK%gjaFZq+zv{GAW8Y^tKtTza&5>Wtkx{;-n~v2U z0x#d4dP?p`Zq^I`eLT!sv>5JM)3A>KDB>-VIw!%YW5M{6puc$*BV?^@aWb`18uWJt zrJ+li0X{I45|E7l|6fYf;xBIYm$eGmt&P*?qFSY z8;>`zRj~^52IlkZLUP_x59)Ne zJqsq`p0p%p9^3Rcvb?`X`~^fy_vveR*c8oZ`>59*2K!QwJ1G-0qwfLqmJzBFB91qk zEDAb?+wEnT-BUCEr1p!&UVMZdnF@p{WGxb=12>DrTLLIn@eyMyc6sza?okEd-8F9?U-MP>BTXRoa zpZ;`-{uRGJ@Cw-mRx$OZ#E0246mN%%6!(0Il2{GwV+s{~#NeEdm)FiaVXtboihT+!B|f=1m9mX&=Edt1KOD^J>;y^ycWVC5Ee5hN_GfwF(uxsi6}p z)*m4xpH!pgbDNdR%Hs9D;GP0ek_RYC`Hw)~Wo7>PecR*zvU0Q4j9oA3=?9A4+cnv4 zKy){dxLQEcn%Fnm_2qS_wl@u;a66%5%~d>t{k6VW1$wR~ZuZts8*t9KzrzXEn1<>v z*v;KPDty2rwE8-LobrlwVZr{%1T_oPKaS?Oa1M8H}I!}chK~K44P*`|KhN|;6x!?{^9Kz zaT{6Q>c;&7H)aGdtxGj5>lOxs0p@`^xA>Bqj{Fe>aZJS$&E8(U_>sHZS>#Ae?6;}j zyTx}W@5=5J(T~@QKcPgzoXc+TdAuUtzI|LTb#1#>AR;G-NLS`TuiLrhwLhGrC^I6y zn*Y6|D^b`~q2Y15Xx4Ar@|kjdcZ*$0xLu~!Ak~k56p%Vr09Hbrii3nF)DZYxvFn4y)=s7`*^-W`@RT@u z1k;=Dx&s2F%opgV}r(K94O{@Z*9P&6>E$v@b!Me20HX1P6rYf8pH z4-5=A5kE75?d*o8(7)zAj-G{)<7SKPw}SFQ+zqP1l^rT<{`BSn=O(_`tm;~lit(4= z7g+pf;(V*jBq0hj89|55MQqlOlwa951}c5uI!iGj9rl;ffSiV2$jeBAt ztl?3nu62TtH=n6QMp7oc0G^@m49PTi`af;&S_qUpC8tWC6RhWdAejYa*~S4vZl(Yn zlETgW3)d2Im$<>}whM$qC%Iv}pCGO(G$NH;p593Df;!RITZFT~XY{oX_A&9LS3-SU z2c;VSOx@AG{ZFYxvF`x=a<$h355*gl?qjgVCOJw-3_d$Tuh6MS%-PZeB;o*)B=m3_69PVylyUG~^_QD1s-#6ct)1wmE}MB6MFmbzoaNd)?09LZ_?+!*vT{vOZrjGil+d3i65(13iiw-@J#28!wokmqSpkl_zQ>$ zG2@6F@_sM&Id8%9ow_J|uhE!P?G3xUu51xm#amzxQhvw=TJ|#5#%r_0_I|56D&a;- z_b-467VkbGyyw@9TdC+#ApOAo@bd7?j5V1NC>E*;)_y3kNzn9xWhXjDVFl3UAAqFN z?ZvP8;Dy767G*Y-PwTQT*}hCKqwd6>WBn@yi3iJ=FY87dc~ zyK#<9yrGr=o=-4m5ZYHfxw?JriY>vn#TwKM)r4`1JKEq!Xv7bu)D?=M-=-={f_njK z`$%l;3(B)D`88ei4_zk!aH%zZddL|mXSpGCSmAE5=MYNrHpXcN-F7Ji78wg&=D)31 z_DUVqW&@^pusLn`^nxh$N)tGq$bZIJslX!~Q*!m9X1t&?>3#wI@yy=v@a1p8~?wApI zI9%i%k*y;t!x-%Qh+2sg8*RM#O2?NC_3_L;W5~#{>OYZO8J-CF(Q!_8fVcWck?YcO zA(+q-7!77q9K8i`=qZ_C*6+5ZL=2nSX=>dJ3u`qf1%y!Dj%EIPjh6ZkC+C!~B)6;Z z^*+s{Cnbt}!3AHGWmiMO&BS%ucawWGArx-d5PaMycXojrM;o>{*HQb4C?(wHJ-~%G zGWV5&Z#|X-IuQ;!S1-Y-%6(?kOzLeA=Y2fz^Sc+i3$87(&3Ex%m7LQY-|2f@U*zAL zqICzRbl_D*K(9*C1vvl>(}y0fdzBIbSvh}?jC*V{9tmmPsq>Na_~E6FQw7j#;2C@T zFYqG8O7CZD>-HRZT6baxxDi0QZW#&9MZXK5=ugD0~0XMX%SgJ zZkjD4_R!eniv1r3fJ1%{f-^7rO!NekJeyYJQgqq&JLD5y#c7Os4i$1`X10`BE=8wG z1x{7%9bGlV^SD^}q)KBSSM#s7jk8N3HrqCOX5^$(`?gRl69Nu6PWScGnsBq7bg03g z1g8K%^TPzC)UNPVAH+U^6=}-qtOC;~QN;DV&i7cfZfQ)TIPFc@kJlT6cRW7mJdr{D z00c-(-@Cry!L=0&)1FoPZj%3xUPCL34P&gNqG?iv-0~@dG*iwbYn5BBGUnM-DrAQL zve`-)<)xB1%e)R>Y+V8`ISG#6{-;G5bHAqhC?hU4^JK6uBSJSBq%3!7>tq>w{xH2} zrk8Y^H8qio+aN+1bK~9pHLKtBsdJr;UM9TJ`3mV%J*+SKdOCo4DcK$dzVen^o zM@s9r=1L#_J8cu~;~qJ0JT(>Xom)ixjTT_tYv1Fks)&l2;Sz4%Ju-VNiq7T!L*HpL z0(G_(v?0yWlJm<^WOCGd;!f@1{$m*u0@w)9Z)LM9uwHq7DxwVlOE1b`9C3Bmh2WlO zK^!!;k0Zy;-C}}_E7iRUb#{bw5j?rz_F&}=g@mhpCN~n?-l=CaiXHIen3JMWtYWN{*BT-oGm%f{lc6biJ1QXJYZ<-V+aQkND~I5B9xE zQ;wL_rMl}158thGJbBHxSy^G&&l(S)jZzGYp)`*#kjq-_HeEFt&! zjRODej*MF7ypk-JVt3z7cG^2_8=Z%@%KVjIIG|zvVN^?MzuM&Ax8lzL6v-Q%TJ>}0 z&{e=`K=#H^O_kd4aiN%w@Zs>a zfvJ-FS4C8-F0G7C;&*Q|iv8nwxNwf8+gq}xsh{Q*?a6?K3zF9}*#Q^()C^!#S^0av2fASaO6JYqsvZ2w~Ox z*!OiJ+d(ms>v%D|nH0$~j6IUVzRV}PIIgibma#Z41gh)heUwAdi`t-&ji<`Ld>zyd z#qETGU6eCQ0!p;@i(lf2vb3OEZfm_Yuh0fHZz>(qVOTYY%JC)B;;cLRO8D>kaDDPW zIp1{i?QgQ3n3$>K%37_N)f*~I(}oTV{~GS7pxT5y1$)4~(A}rk5;z-5{%pC*3D-Cj zHGZ0!zh4;4x`^pS{HoD*tk|($MTqd=J+lSWs3tvK>=hnYJ%rwrCP}hwjQnFUodgKT z6M0?n#FiNl5@a^f+MspQWiTnmDNWJ=ut-YwsxQJssW(a{IXi7@1~F>!A@}WLOqlj< zAAx&5Uk64#^5t|UNNpK$Od~G0^=|u67q9?HnNFy=#_#cQ0A;f{Zrem=`?wkN^d`q$ za-Rj?5Z#ZS6wSdA=bi3=Eg%Dfm!L%*ce*Gj6G z8iq+hDLE2WG(mF@U>v*PHO(;T@-41-8Z*HY1*A0^M|mlNKM3014C{81Ld*G>-;Ku5 z-{!>&m+sU+XRWEC*;#F^DJVyZG&Vc5m_L$luf4~ETQr#7aem7524%&kjRW2&HOt;G zKAKKobmSEk6`7>u2h69CZokaQOG$gnGaPS-+ErI5_sztwp|(!^kw&?NO-seJrHIi= zfoVVd3kwPgNR!_uB#I5)W>H8FiKSZ|jzhmQZ!j~0WxucjD9x)?$jhkvb%F4ePoSbl z1KMmYp_=y;vAdiP!na0$W*$G~7Ls(nEirpHZeUNJm#}cZn3`p0`_0gwJF2G_ncyr@ce!+=e_SRfpxtHL)yTJg!U+z1vkkl1Ljv@R4P z!maQ6T=T8=Ai>8Tfz9x{b@ntb-}Wm>Zkmk%*EWMvHA7UtKXpp_ZWG*jdn#=M=8kgx zP3rw`^`tOGB*$i^@?i~YSWb7=q%KBq--BKw7#L1yd^bCix6w;~#+GNl1#V|o>^rIzK6)Niha1-i?UWdNN27~*qX0Ze&hjrXuxa6ga(EkhlQ-E6Y7ZXLPUAXJsrU z_|hfjwhd2H!aabjLvj@4*opSi@h9)wWx72mRoZE2A?PzGeDO2%BOxl`!9uVupX@8? zoB1g7m@w7`JE`PkZKA&OKPSp-sR62a9E14Zpp-krIE3B+vgW4WW6Y|RTPo!57hG3~ zdvYLiJI$x97KPhFyy`(ukEp{t_v&nBoxih&pA38+BudO4P39J8(Elks9Iuwpuj$ml zS@HA839gTFmxm+giDK}Y?6${h)$*qZm$qLaKoX9ClW@?Um9&|^;k2f!o|S};z%3)k zyT77*ble&cjWgX*LPbp=$y0qAihe*KB1WjURTb9s6mSq`4f&M`nSNZO6u*MfN%BvRFk7W;eo#hOhl6v*7wT*-**vm8W=LV2qMT}DMk1+b4BrT&7a z9~6C*(P)`hxwA5NoU)zN zUO2@q65C%0#)&&tQpv5ccDI-ps-U`3w7POtQ0Zz`^DYX)cE&^{FIdsR``U$dfNM1slEISRQ%N2#?IN4g=lqkN>Fml+}YNy z$>#afr)8G&^N;AnJ+5}hY1<%mt=+>~?Ht^?g?8e62;;~6mBIw$IX9?00!aG0Oa$!9 z)Q_RuXCYs}p!8oW_*0Tf+sIml+Q1}bikWJ-G+i*!Rq0*BAKNa8wH}2zuLnlW)h@## zmD+lSuefrtzfAu&p@yf(2M;v!5~pHjWp8#hRjG6lM4_YTqb;k24-yqr?`hPFtrOd_ zH(UD2rlTu;UuYH8uM)knhq?Xdw9rpN3VId$ixUy$On0!G~ILndY6|`Yfd--{ zZgJ#|x1b!`SA9i$A7@$kTYL+fb)%3;pe2G1!I*i_8s5GVb<>Z3`gDVja#AhmKXM)I z9&6^J#0@5nH&?b^ETAYEfBb27^WIDJIw!5LB7I&afp^(?TA~Cs2rrdZVj=6Udfc64 zlXxlgI6czz*l^z@^E2XHe+^2_PQnN;5}_gi2zPvfiuzv6j$3PZd@GtgwO)1QaiYyu zfayjf4J2>L=l(b$r2S@lKt1l%x@Kf;c^_0z!+%AHpKKX3O1*WSPsTc5mRguLlM75p zIDM$5pxhGu*5QlQ@=_6A77YdqF+xxK_ASCxqC%CVxQph|sf-Vqb~aZoUv9p<{)Wj- zlYEukty6>a9~1v-S_LQv*^{XrmFSWPa{Z=a?&LxFtpSj8>WR5wmIvZ@zX{Gc@gMtP z%EcKIw@};Yw`pL5RVOAs&9?g;oX`#3Vl5W|Gb!$S&+PxWG}+&^V=#6<+mURPu9m)$ zC05dx59Qw1CUR;#0_Pk9b(v|`ZfN9d{~kNoEMh9VI4A;6drV8Az-b8bTpz7g_R)|4y?Z1ZcsOpNT zF9WPT9@=0`Md=3yF1fqFpC>(2Qid=ZxJl+xs2a6iI69Lgo?kEK0NA&s%Q2ZM|AaIF z3=7!C<#9A4ZIvs!@Pu3TfcFH6U+0d?V!kpbKh7ovE$zW1>H(Z%QX?pd;N|=W)}~7Q zw7m2Pr`&^oIK7l!$fho5AnG!ds> zoa9G9U>yjte;T)h)yJ5GMBl@ec=H8>EQ`20%%fu;_*&~wGLA{s=U)+wbzQu|5u+8w zu;Ne($C#D(&hC1J(0r_urD zQNG*-_esqTQA_|bqs&skRDu?_{;}KOmTHP*TvW%n>x7rv#Hgp>)-MNWZ$8i}X_FT< z7ot#0EB}xLvG+Xu6K~X+2(IO>TR7!G7)jzfj*gc!R&qzkjZ<>m)js1@`;l2;(TrQa zH_CxyrpgCP^lr6QA>>+}b}?>#Z$*K0+b3MHmAR{*u~FfQ=I0`!#QlI5W`WklJ!lL3 z{X52A_4r$K#N7MB%Up<=LY_fzJ|E^P<7S&d<)fDDG~D;ax{KMMAVt&9ro-H{J z6$L&+8Xot~>vyL9p}0?Qot&dA#}vx*57G$(mKi=axO4=;2&ct-?-KfpocwKEmH6nz zJ-S|T@}GNbO zF73&f_FlbtRiqcbR@9LhI|o&$At@vYlwk4u)=;!q?cXfnQg@tOXhZj?3|T9&URFqK zREj6zc{hFeUBk=-s@yZM6l3hF1x-W+NLZ zU;KOe+K{w_s02}Mg865rdhh!et}v0-?=mO(Fqz#=@|z0yRtuZW<<0*y)#1CMyJc}H zUrY_k!E*bG5jOC7=A1SkCef?)wycl|OW#VM$f(0!-iPQBUH z38(d%cy>^DXaFNj!ExkRt-eBr{JPf z@`>{fEHf1)M9({r8j5_fQUb1z$->524fHpY49%|bOfV!K^s03{z6R(z!Q0+%`LTHC zH7g!b-$a(Y6C(&H;WMqhTKJ27vx^fY>VX=COJ`_$+lOfN|4c-!sW157$^P`aa`av*}tCYg?a z871HEN)@*YhFD>%Mh_xD=YCJbl_b0FpTV1{)NqZaA;fobtXhiCWxh3n~2_)9is1 zKI|{xbhW5tnE# zGIG&Dt#YVk_BQw|<7J8Za=-A=2+n#N^`OfUWo-s_`?2+Gem#Ap2kw->{WFjWX3WOa zfUaRclk2$6f(*L&rxcAVgkay!@-^vg@W&t?(T2icKgf(LU)NPf)s?0VUIt2ZOU!ja zLcWXU0B8H>qL{_}Cfc6G8Fab1Gr8n@m55P^S=o(0fQUz0UEmCNpA-qlNyeYc$opwm zIHx3>lRa(v&Y9eTb{TC~y$OC614c$NQ*i6uUj{2%d9aM_brhH8$pjah*;ucThItl!#q>B?lG!+`~4Lw|`0)mZb`I z=}nOC{?@y$uQ|DfAB$D$-t~w=`U2Aj)y;(Bt1QLt)}Kn2hAjSmwaX!izl-erX<@^w zaDvIAD1nrsZ9`^IbyB_XMoGxZ91t9s2xQ#gNzoHJ^=EKo5^yJGD*vDP3~WQS0zOM* zMa~-1aLf65Dsg6}pd(c58~})7ZcmzaA7eY-E|P==qEw4ZHm&YJ%_pp^Gjc1)=u!am zp8oBVEe#GzoX32fl0S%{dW{Z!!J+n?cDCkr3M4gXwHu~fBc_%jOLpBfDCUlWzF@DIIN$as(e{C}ISMDpZbC2N74 z*r((22}S3s8QPa`TP#VvhDG5{av2mIKs3O_o%4jRGtRtf^EF^_6}Dr^15Fh`61Ta- z18@LyItc2S3hy^VABHavmNA+!chdO}_mXMH8BlGXy62SD*9BbRaTR-@(>f79q{U&W zh&PC$QZeUk{308h61tR0^A>tm=1XRglnv1Rt06^F+OG30X%prNAU>DSk=-@RqeN|2 ziuZpg z3x9NwX5*n(lq`5cXU>dGq@4iUA%RpcTSf>)!1LzxSSjJM4pfMHMqeW`*nzN#jJ-a4 z-HjW5&7w|ssEkxDm{>&%!Qb%`Zv$`_Erflb<#Kar56c?1$K_H6{&IU|Q&#hi?njND zmcT(qSh=m}YEWpqig7y#lpa9B@YVc$p+7tk9e%FFC>SisQlS{l(1pG?aZ=2xg zP4#U2W2@ak;;DUZI`9y^_dr-m-^S^uUVMt4D`4>ickL~u3CS&bmrPKc=z8~#}TBIEZm{hR41=a=(Oz02!L;RW#C9_ z_KfbIE$E}egS-l#VdkhBe)b9Cv$p@Y;$HkxHK89I)|Y?XIDJV&>bgHeF;qN9E;Z|M zP2J_K3oLhF>V5OKx5E@L851SWIg4-J%UQ;#st*UaX`efbnY3^$_RAcEUXi;5A=l(O zunS6_LT1E;;C8Gx`p=lKjp~6gLx@}=Q2!0~0iCN#L(F~ex!fhEp?P>X6H5m2y~RkH zES^5g>=E2muc@Rat>^)EmAcXec)CNoZ%1I zxui-s32v|8v{k{~66W7l4i@nTWVFs`(K_ZkU-EDuV_!Lyp=T6n2@I=T!}MN}WS=Q~ z6S>~jJba9`dNh$$|AFn5{!l39{&F2wChzLyr2$~IN9dM)s8OrUP>@|ysR8>C)oO>u6FjM`rPk&VE{fe(1+nrRFz<(lrNsVpLE%%1RsbD$ z-a{@yb*`@fTl3Xdz=i${q88)H?@}D;~MBqJYMTtV}KxJ`{RTuo=GDoi(HB%a}-5b5DlLrU^o#*{w9R>&f^) z7z~xr4pt6Y4Mm;0cEQRQmC~17-b$miccbDHXe!_z1yX|b04YAMRO>3r8m~|wr$z}; z0?Kf+o9uNynmN^Gw#V2Qe!<8qO$@Oic~3vO{=)Dx@fcsy9>L9kQ6s9B6BUS|%H*!= zp?!7yK5{EL`zPCWujTt#C~(bbTWDC?14bbJaYKW!Uy+bqM{ZJF z=T^)sLT2opRzPao58LpW&GB67#>i`Kuh0t}7g6M|ynb5K6;o$HpBhK5BOMS#9 zyl^)#b?dDol6$D(>X#y8EjKoA$R3F0cIf8Tg=J$#f#yrBZw?gWnbtBa$OH47Vd z;Kn#72g9(vKL6MzEVX7vY{oDx7i%T{trk$`Noj|CUeRAvd`L)TU)=Bi%O-9K z|5Vq3ZeJsJ=}s6~C@ov@-xoUYXroq>MQt+d z`H}jY>j$ly$F_vVvd8(zHju~zy-+U_+lw`)Q^u%8C03R6_c~!{^DbUlnQYg*5YZaF z1gKqK^NcmbjtimBtfR)odfKC5rD9S471Jfs>CZQa!;sL`C^!zfFg(U9X$tD3nfgp6 zAR?-E2SkkWW%sgGNn!Sb3fEJ*Ou$<`q-c#O6qZ#ob;jxUn_O+|oZmj8LYELHYM_D8 zR3Hd z-TkTtYnNDiLG=w@R}MToXqOqa`?B9U$OM_jwA}T3`BEHRUgoIAT&$E3-S&%1hg6zm zt^{WG%NrP+bq|B&4$MxOujZ>(Tg{*{=jM~_ z`pUGi6y1%$m%dy9Hyinm?7l&|leLn}RQ3#e!7RPRE3m;h4zBolJQM8EfhugNk%OtT zwT+Ogld{lmbSbQj1hg}PR^mm7fa=!B|9L+z@%^Ukzq^HB8!h072*u2+&2A!}VH3evOc2N1gvP?FJ; zfb_5W#Xie8znj-$2AWbfBAJv>h3!I=jnFS0gaZbe^bM>khOMC1QrvK zIvW0@XG6ReHfA!coU$G^Y3Jv}g;Q*>;Df;QbQW$Ly@WHGe@lHis}Bqg-GL9>jnfy{oc3}eH;-!XE~q|v&5vfYwCQP6H2ENre57;@A@!? z50}jU(*UqRx8m2|j|Ih@0WUzy6+I-B;`sRRlktiso9o6ugb9s^gpPIVPU!Aeiilqr zGYBgqpB&n(m&~OUOSxTzH9>tvw>m-#mh7J^FvJUTBki929~pvo1@uk|CcP^m_PK~3 z7!Dnn9uY^t3AM^X&%eDimt`gRz3HR?R&Qya_BOecfbwl{&Eyz+<0ltt+~zm=DAIbG zr$g$+@B;)RNM0VXrBG^}7!)3qp)Nb?SW<=z<*=RkD#J3=4o@Kzf$sG|_ZDGvYU-`B z7(+QAFTU`kXTwAVxSmul*)JG-3_K_2(n`uK9CRe?P+;gKP|X+c5SjXnS^Z6*drBp6 zFZjD?_X@P?ThbjG4T}>pJy={VSI{2t+ z^{jnAn7RwbFSE1ayhsCJ43b-`C#=cM6EA@yO5GO-C@5}tX zIY*9`tFN@;`KT@C=sq^`DV3&p<>IU*nM9Pu{oq}!zf-64&g6QiM6?iKL^TsC*~DsSGJQC^;D+6{WZlVxapnWdHFe z<+ET#-QT#iYW@`cCkPDV@LbenrDwSgR=L_(0JY`=hx&hTZmrylW1kKq-$MI6nW~qc zjntor%t;BF&~X85Xf^W|?b<+Ps^CMu@`QV1^Dh7wWjF}bs#t!@mZC>yi|T5=TG7VGf^a_p!=Re_ zxb$hihaEytPTL`v;^|^^x5UNSnb)|5&x#=Ym?a`f>CP3?#6$m#-6+x2M@0k9q-06p zt0h?>KqWx6{f=4pv56MBf_WKoiOc+cy}PQAat%FG)zs02L~B>20gp=#0qpDHArukh zK}2AReHa1Q;X=D)@Nzz}%}j#?*XxXZj%taffSW*|r%uy)%a8q!u-I}B;Noev!{y@8 z(H`E2TH&EAzvKe*zQv_jAvfh;wKd=0Ksj=wv`_C;psS(e>L|= z_fCARx*X4_BNGN>=y-ZK@@GcN2Y5C@yCU%Hmaz2eI5zA47mGW|T0rVpJ-;9igR=*6 zFo7TkvmLW4gmVlgj~p&6g?OEaq$&0^L<5tq*_j2Fq4!VZ8aCV>>o%~YE*rXEh{I!YrlmDs~9)i{%oJ~Q>1=^fVkX)!&gVQoqP!kOwW&F}AX2`%j-4qDF7`R0^C zBWPw_mYl1sP5DN+SC7}DEWb9q0IWdh1SyZ}m#Y*^%smd{Gm0@aflwr}fpWDRO5O+E zrDx$vbKdS51Q8Qd-N&Il>j(B+@fxQ;dm*I|0V<)zeJg{Aw~D3AS27ZXI}sri^2Rdp z{Wnv*um_tbZ`y;re>ddeE~n;ja%=Z0gc$mZT1psN86JZkYp68krex$Vj^L|R@znRf zR(`x9C-?q9^H(?az`?%YpH9z|I7Hr_g&Vm30xPQIx%j^M%0Gy0XLoaR`LUwcH{F7} z&;irim&KQ;WA)$qKNx^^=AO1Ge#*=XUNmtz7DW@cL#Dd6NX~T3k-$k0Q_Dm<4aR2q z+<<+o>NoNs3;x(HJWTu4Cx_~xs9Mpv3I&pAx(Vips01JoIbUZy@KO2k=1*f(lPD(3 z#4tOgeHX7;3})8jZA`kbo=SZAY4YE_cy2L@+eQMtb1RHg@jw=X@qZ?C1{K6&1QiR8 z%+evB1>`o-=%@EwAp|}>9oojXVbKK@%VxYdA$%zhT{`v)#>J~%?l50Q4t^ngtx;`m z*RQdadqsg7i^t{PH=B1P3S*mf#~uHNyaQK%y-m{BiUi?91~K_VCi>&t^}zc6?_kB$ zNzc2~KJ^r^!0Jsz49;1%@Ki2Pz!Nf!eD7#*;hN|Z|3jdqM$Ee%@CP>f8uUN#wq%Fl z%z%aI4Wpvk>~N)}`tLF>XN%%Yo1qPCl_1Vgs`ubQA1!S6MjceV@n!l7?-)TbXQ4;c zfzFWuij<5nT)nu%y9qyLfj4`lb325|xAQ#1PMvHo8zOdEY`ms5C!0GoGM}Nz_51sn zU|qow#|v)fLW9AKy53K-AqAv* z5jw{Btk#mxDu_Uv?Ioevn$jSDEDBBCxgDfHg<>a#jt&G0*CLa7`}+Bpcqbx^0Nbc#8CS_{Tk@9aV?4B|zj$8;v)#{gW|ijJri~ zcrdN?h^7++I;DjHddp|Nrdd(vhB~y$68t;7`LjAgi$%>)lijQ6){aFOExNEHRd6wq zLonx$h@igt_;w69h%e0BVxKxK$NJT3HV}tSkazjC1=(2^&i}Ia3{LjLF%&Ug=B;P;O77y=hp&XK z(fW`VV!p(}G(by5$A@tEZ@9mDMfit5M4n&8-USu)E3SnE2M2S8src0j=(_?S#X^~{CR;5@}-*2k5{87Ms6kn?_!RuqE zyUG?kA6vMXco#?Y5YOHt(3D;PPsePw=W?GGUM&tJGqerQ3(&fL0dod%9ywh-9l>K` zTKA-&mCAI6HX^+wM>q_~s$$M%!5#&guugs!JFvm^9CpY;*me?cQ(3 zN)}jIg1&+(l6MMc7sL4H$ zMT|Wz{PzyT6|yd33dvROnbt`|GH6l)`xUXtrhlom zR5GK;J@A=#7dM5rp(4W@M%Y}~@GR$<=O_`Bc)|>Rhs_a(smhoP($9HXpENUc@6DQv z#c~`rT{U@MPb(irSIJ6vJd(94$NO-YJoa;{c`i)xJ}b=8vQX_eT&l%il33)Osx)O> zV?erFjHuF`7tX?i+5!F&f@+{!XLNXK-P@QvG{FG!`h?(Zqri&Qk=lf+r&hl@qNjcZ zIC)12AIkbu(2;3D=%RT`#ciAGixvrSVU@`QM7l6<8H=B0=Orb=o3QdN`SO}izCZI`{KR!$wk8$JEE`$;gUO|D%tg3uOI+267|9&e+pnvliL ze8`LJEJRo$p}Dn^(_FSTc#~`P>6Q;PdD>9n;IQAtS?={RE17*tx>}WMf*oB)2I%H9 zGeaD@t^a?}=BV8T*W7Wksc{rG^bfByVxLG;vU!P|mgiw_`?UJf!)WOo6>4_-lIHLT z9%t8)nb+P8M>>}1AT9_E(g(A`vVG`dy5Y#r6VK(p>9Mf4O*?$Q4u=6b)6|@wk^GS{ z@DZh9n+@eKXsVwe*c-;Dj&q~2+>aSkQnDw<=S`&FB~?FSIdyI>kxlqx_pD3Ynzl`t zx#P)z*PBfk!^SScDhmm}>m1Ao)DaS3{S1INlmLMl6y{OnaMRxnjPqA25gUl4`gWvEM@VMe~=&YNsWGScF)q*G= zJ`l15ug7!IG%Z@!D|nyS6F3i`(zkdAHM>j})X#pfm8R%4*12i}Gfc4xKYVkZDbVsdizu$h52O8b)S7J_U zV^+{K_t(?Dr-LZMg*Jj68Fj;lSc+(MOFh3(gPa^imHF?U|E5a*^8N=Bf@}E7K!5sH z@!c&eR+VhHCXIFaxiAj93+nKff7i>?u8=iuey+qC*SN!E1@Y{%%t4hsLYPjzo4g{| zEW>yu=##$4(GxOp9nDvDd6`nr7Xi>PniBe!sm^QoM5fs8h|ED*#z2#(H7vEgWxXJN0NRa3pD!dJofBVX+*e4~o^CRd!aDQJOS7W5Hwrl`_)?b00)xABsnw!w zCM(}19T?_r9M#;B*6ml6{KrziQzE4(L0;-{O|bLK<69-USAV;8X5&hO)$5&O%pSNs zM`!kSx9oWy36Y}AA3d-UxPO389RPJo{0V$3Gt7&4#FQpudczEE<}W^sgJe*mdhoP< z;2Bu0&p^x4IMT+R|4wnq{y-;gsi}zlDEl=#_?-A5B~EzaqnH` z@-0I5F}FwvG(aK8qFz5vr~MdQkPMYJn$koOa@d4svAVY||Ja~A(7ZY1pZa{oKvFC>BaY2c=qY@%8DhH%d9c%!*<{ZCpz6A#jxkF^e06l z+93x^vhPadE~&I;&LrfYtLENu&0*#7X3dCCV5Xx_l}T;jFgpA^ITqZQfS7ikkg+ zUL2Yt0^~fK3%fn?hGepa9NDw}UI*6e<*!Yn7%m@Bolb z5QnGJHT~U79RpqDu&I--QQ0rSd7>YbXW(1}F-*f2CW z4*wXTEKu>t*EP${$_{>A^NqKrKK40rb3Eo1UY$^)L0v4qc+G=n;~+jsCdBVzN^)fQ z$gD8~^TZ`$X-hU5rq%0Lpb}T{Rg(pxuUm6C)Zc7=T@gqg0>JwwuzE{LQ}2TmY1`F%(`yIJE$K$-WI;WbHna9bCMw<*W#tAa#xofC$jkB zj>{iyW|7c5wk?vjT+K2=s)yo$C7=g%gK@#b2ijKB;Z>F=ovlju8j`+F0*FsRYSTa& zmrxU>3y`(WHP39;+ow10S|&VU+uuH^>GLcAejhuq*Vxwn602x-_BW3$FLOfxJV@S; zAQzs7mDG+}LEwq)Ve-0r7ifat@0m+$yi|t>jlAn4Db%T%6@rFyT=Hu^91QUwG?j!o z;%0o5*zYKgspYc%DD0$he$r*uIw;e<^*e!U?sV&JoX+y(-T;Vz;Z?3OaoW3Y>@DM2 zF7>24bvG52%z;#L?=|$HK@QrvYT1~%%{c`HjPp(Opy~uy`7P(j*TtJGmf0%5xP{#2 zrKWp&Z|F=^cPZ41JP(*Wbxq4 zs#rc`+$`+Ig-0|Y+x;C#=fmnPGMMgPm~IUCx5P`e)Yq9i-&Eo?yjqj-2n32iJL0A8 z$(VcE3w92;U{km}PmhKUuhVoYJOA29@vz7$c!l-8LV?g9#8M(GJ}?W{ybp00|GwAh zP#@_j8UaY>9R=m_GX}c>WW(3zN{2^((&U1fC3i#zRuZOk`dS3A&J9HMGj9F5I%N&@ zAsg}J5R3z4BFWpbJM{tZ5SrQaBX(Vp#5M4Wet83`Bg*ccEAVPO z{aNGm4>m~dWnnuL0GwQ?GQiJMHV<=t4jUTeLjFP-*JL_;?Rlj@R^y+6HqeB_8DE+q4pkKPL-0?udi0?s=I&c+^2_A0T z?DM_Q+|ydH^WzT^(gNE_^EC0ddCR$5@j;ZO0`GWPW+JYme@sPcED*gly%+(Slw0Gf-h_bcBX3JtK+W!zr;}FiNuwNl64uh}0>=Q>+s40n_WQ9eTeC)jz z^~C}cU+Xbne15F4m}qDUq8N;xez^n$pp4Tp!mA#ETcq`98R#8TVyRfJ`vg8P69pwJ9`;k5cBz7GyNmh1B517;quO91vy5%F8+m6D<)+U8W6)&59 zBEFxBoH*o9?r%x=aT#qV0Fk#WyCfgkbhc{-x+d}otc?@K{E$TCOh##Fj^?R8xa4#d z?IQk#U4-CcS%h{XDxRbde~QX4U7q;N?~d4H2~jafnhQNLo}_A#(VI5ZZ|*${P`b^| z<>=k@PUDY`>)qZrJBwGaceVdm*hpHyLBn}=zWb5J9ifo#?0L#u$qm*9B!o8eVWD#x zN0%neAux3K++8fc7a2M z<#gZ0TO!fAL2L=5WA3%~b*QmvRt=TQNjH?;{+nlobZ+3)(ALrovDfL{aOPU`MuTYW zayM9y41(a6?CabsG;{+l@=_DLDkxC9tOgzhI+lOem7oe&6m?M-liG*xm=c45dLzV(_T1mP!Ji@d| zBI1_PHdJmK`Z6$t5s`2-N6CfS0y2L;v#v0sIr%i9tqD@;&>p~i=jZsz^6p0V$oM{f zxZf^`D(jKG-Xsy zK`?|sl)uL_7RKhf>DQ!NF$_|?2gO$tu9>}AAmWw*e*@is2M`KwIVLs#HC6b4GE4ky4AC#dKK67i+6n^T>*U!BUkRlJiqeaF@Ze`jCZ1|A{NmKc2FZH%{P15&JB+`&af7&wW|GNqy!Fuebp3v|fARrW}i1_%^VFf%0o3hj*7#TUm-SjX+^e z%^lhog<;DI(da-m>N-aOp(e;WHz^R#Ij?g%)*PDHOPvZ41ULSA4ka;2Zp8lVKI(cn zC{d*^Xb+zd@(G5EyBo(z)4b?Qo+e|f@#6bBh`ELHRZs&h7!TZ^Eh19@J5r_%k%w>` zsGaeUC-`uuZj!VNhR{jtml$`gMU$Cc+#9Wu=iQCeK{vD1t*(##BTTp%x*3fWbQI17 zIs1M(3)u9DVLP4VI(|cFxxjL5GMM3h_e~aIoo67BIi%$4_rpMx?Dy-d576%*gIY3jVF>*fIHaoV)waQm^l&G+HmkO6*&OW8V4Dt}h~cz2uiU zrVyb-FmzT8O_Y-<}WM)K8IIYz=Ibv%F?X51JM;H=%Gb% z*5Rk`t}D8aj8fggeE*U14{OWhVv1B8*qc-D?9aFdi-Y6_Xl`IeGmovP9Z~c!?ySu1 z8KM5SBx%BeL%Sut9IKV3iS0FYHBy-q=zQr?>!Yox+pZnrHT%NN_SLn?*XEo;b}+>6 zcEJ*BQ+JtQ#l0 z^LAwlfj{)eRV}R6{(?eo{S+yN4~v{=;;*e+TP2%V{`0G{XNqmaR(NCNDwam!iCJ*D zK#LhK*c*1*{YPszZQ0o$C8#TlhSL$gQ(sXl+38R_`T2c8OHL>Mo=qtPc-0Occf5Vf z1^@KXVOiR-YMFc^Z^*lz*6UwGQ3le@jq2sZjy)mN6sO}C0LaWcR8)mne%k!4pv8-~ z=0-G?q?`|num{`L2)ySns2kK00)dGy%Za!g?xl*e8PTp*p0Mr|1_uUe6!>F~zYT8wV z1l_JmUW}<=3Y*VSGQW|ZGT|=UI6U?jyBIZlfNRBDe;^rz$4DPH%M$}~29$eWwd4+(PlyZ2Auj}> zkacohmco;=7sE^zTvC%mt!rN-rZl{aQlyr|;wh!eAQB2I>(QYf+(&&p=N@jZ`@y}s zVjcFf^w)`gvaP+Ly9h`+t#N&y;-OM=BjxAu^o(EAFI`p)0o~uyyge*neV8GPI^cTI z!-2|~{U?P`4Vdgpf$Xne!d8Z5kn_@j{kKY%;mzJ<(HQR@2PGOL2?yMZ<0SA;;YQCF z+>*%bc~m4sejA!dm=pw+MKxOJuzNr2FeYZC_?Bg3-9ksAhlRAHDHA5=D=biwcwX$yVR@Kj?A3xdMz6ZJ6Necu{!=V`^^P9w#`CG&C@l?G(OPOqyk@q7Bh`%Li0>i8RTEsph8V-zi-w^E3y>n ziN{kM3QU_hQDb*`nTEyQHeUHrAl_o2ORtt zy29LN{{_+sJF0UA=|1MObW(x1Kga4PuW1h0v~AVygPan)<1??PKEKS6H~-EwnV-ku za)$2f#5;uB)Lri$R@SdB_r|)W7d*wheS7%KWD-X1NNuDd7%Y?};yX*3ux95HIidj?O8Y@@a zlW}^JQ12;jGALcEEe^m5rO|3F>CEINzVf3<(z;VWRqp)*r}a0>5GQX+3A;LVI2P70F+ucY9{>JIz##GNN_-Z_NEbc`1bugkNX# zht{fntiR(gH1}Bg9Ubpj;&f^at&;R8wS95d%l=Dp)uCW=Z)Vi%es0PZ!o(Yf>{+%+ zau-B)$O%|3pNhmeL)&%B=JR>@-lA>YSek0|eLiy9JI@f(!u^vCEhx?Q1a25CHH6dX*Z6W`7no=c_BR zjh=*^#l1kso;U)YsJ4`y#jk<#jEH}doi&aT(Ia_BqOqD@KrSDF1rV~ zTdA*yRXnpFLs~(9n!!qny-rl%j>r&N4bx8=(pkEZ*3GPB5W4>0`AgLw?L@dD&NuFG zfq*TPUe^flpWoZ%8IpWGdtm4O%2PZ=VZIiEfj3c-reUY+1it|^6}P9VtfNBH@Z01f zMAfGB6GMO7*PIOH5U&|fd;Md^a`Ly=DVxqZKA!jT-|+XiyQ%!Y!>%gyFAUuH7)ie^ zWaBGd4sFOXgdQ!By5L+JnCxviA3U#s4V^aJS-E^{rC(Q2 ztVz?uwdbPmQQepEyz5JX*v<#dWKqmgBIb-=DIc36c`a&@J7#sASz5_|+dbdy`Sq03 zBXUDWt6drhYjq$J^|AMpo_>kr`RUR3Tov=wb{@Yse)0?JFuFG>KkTCR{`))Mc6>eF z3w;2L-l-^qY!G2_E2J36!$$_Ep;+?~iK*ITTB#2{dPjQrnjxTeE4Y2v@V>&>Wf-XQ zvNDjT$Vk#>-nt^4%#+$*(&EiDh1!Tl->k33H?Grab+>UnO&$Z_Y5 zaZ9oOG_Q)K4B7^2hhwYukRjn z5lQJq*$dXck-gEvfwF?Toi?rsO~h25nFWIAes>NhOyyd_KW}w=Z7Q0+h)*6*`6uNJ z`X#LY>Gtrd3#xKc#LmtAz8z|lt6wt&F(a`ra#OZ)M;!J(oZQPFM<6l`O_J6Ic63R) z#Co;$xPB^zK5o)*;EUI>jcfiGq4mOzLq@ByCM_{akbWcDd-8>eZiKV%TPEMFje{6))56Q-*j2UI1@fKZoH3=t z9pG)zy+cjcj57)A6vHR4zYF~G8koUSAeG#rj&7nNaHOz+`*l*TCGD^bxhDlluneb? z$`-8XO1HH>t)pt_!v6w4iB zB|D)De@H_tJV&b4KpvBzYkZ8cnpKLf#U2HOta8{Y$$aqhSO{<;(t53;LZ23Wx^J*~ z@@;%RYY8^Ff@K=n9BW_A*`0Q{fV8evUQO?)PYwzYu9)#IY@_9IB-CIP2}pS2;v2~wNTW+TivSr zc1Lw0+6TD7-1x~;#MO2n0drY zXE`KJE7d0Jnj-i>&Fdh3=VUIa4KjGIFOrvl@C^4^;EHMQuFIGyF#s_fhAlPfw?p`N zLAViX0{&v;fcH5_0cYYwbJ6t11Yzr&i{^ewhKb2oDUJESPhg|-IsV#}F{e*HaI?y5 zNJ+ibC=s`Dld14E>}K%ZYuGhxf2~jyc8$E8Pl;b?&r3nKSD_IhPIs=02=PgfIhUL~ z_P92_2)4TYzxkwBQ*)iqBq)0Iuk~~jQaFMOM3}< z!-2N4z-9J(dWI*t_7!xc1dVZea!x2uLaZFKU3a64EQJ*@na)>S-4(;XTbwfEzf-#J z#@ACDniy2Wu>38G_djOVU2~Ywg_Me%jqW(eVvZi29C%ibgL`lCAOnBVpl@Zh{@~=v z-o{gLzX7u>=$R5ou%C_- zI<#wR*BVzKjA#@^h#y_+4PD9UB zvE#C?3qzs(pA6L(^I_zrgfJUVFG2_VZ%}k79Qm7iaWM1eDO(%AP@UOoRMFI(3%*+9 zWZ1ZH3yEZ$^5TXt?gZ1x0eJB5A7{h%FgGqgF7E4m&DWwUG4nj$ae&$-PTr12600m2!dy?HUmu=M9>J6Bhv{fkK^YlI$MZ@=r3uCt-Y z{&oQO8bu~gOJ)vculDPKaRTum0xb-u^a>33pe$6A#3@ym1GY7CPYkeJEj%mYiLH%uF zV(EOHzW5GaCAi6GUAnhoghiw(v&}y8>jyS| zba-pLWdGkynXCDaklMk$bTR>_%^c7yiRV%uY2HTDHeoBLm)(BV2~AYvoI_eXx%-@- zijN5AGy1O45jb*(oOCaKZ{~iz44_iF3h@mg_UM(sinZBd6{z{%WSgmJc{5!P1eo-R|JaE$bG=8w-vv2(yz-iY0WByfuzKW;wLaS3*N zd#JcW(b4&fHQzvNUwEhY^QaR4M5>^X+&AqXw|~!$!3E?^6Lu+4k3Lz-26x2rrZq)b zTKn=>taCO~O!#0I1@@=x70HAN7By_{Tl2ugUlHnIbF(wA@|Z#mBF^hMe7Z>+so8YS zudd42EGEoyIm;6^s0TN=Y$fjBFM1+#9V}{{wZ(5&|Hi_n^#vts)04s+X<}DTjrrq{ zsO_!7^bAZ9Ue}W!8tN+p#yc+au+tz9{R8 z*PlM5mZZuR4jjoiH%MBI2Z_$)%<}=`o}+(Zws$$_8aN(O2QuaK_~ujPCp_w$9}Jwn z9Jlhu`SQ*7hvFGuQSGB88-`qe8}5@<$HVnrJ0XQ5a)KaK?sx0r20yi#-3K3@xEyce zebm!vOWqPF?!d!VW!XX{sm{)0p4$QcbI9(K5EHobZ2ap^)daEn!)lPN#vNG}^n91$ zN^%!4%1SQ6UU^z0rMiI{+0WhSbHn2Ez8B4lpHP2z&wrZ$XaXaTJlMUHQdJsqx`(I-(tpG@^%vI+B@o~VoHTeq0xiiU6Gv|eb7c&VX5g2`F1ywjlpd5py&?`b`K zw)!2uVKa#f8|Iv}mXeG%2JJ`wI`^JGSS6%r9r1mK;xDSN(ZkC>N(#DzQ^GQb(L}>| zAF$0*z;0r-@s;p2_BWcC34Jy(?HkY?9I0|now^*ardfOSRL0R*JVX=eXyU&ECFml} ziGRmiXpD!yr@YZfP42SV>cfv@d~5|Aq_L!cU&J)NFrd);Zv)?BnXTC~NrSFiiJ2XEy+AUc4J=QEAPHBp*vsd{L_Alyz6)2 zXC~c=xk?WfrasOrm0o%_^;W9^I%7Vbeg0r#qO+L4SATQb@!|3Y^DsjH_urcTbOcI1 zO&WP7V$ikopON!M_<^il%YWxlnQEJwCqB7^?K-*x<8w>PHBr65qu;*pIrK@L^PFa_ zu&)G(&xW42@0-HBdDof=sR;P9LqTL=DVC(Vc;}Mi>1GN`M4aLyx@@|gXG_VaodfEy z+9_6+z5m#eS_mVI3~7%tya$2rfng3A2H2K}Z9CPMyjS#7mJ@w`FzoB8a$k1TRXcy3 zOBrQo$6>wAFtx9|)qJANRs9?(jBsc8^7~E1FTmJ)Jdh{zWMu3WpN?e*aq-O;VOO)$ z4Gd@XO$?p-b`#C!kDfVn{&eU>)55GiTa%O&bLa{U8_l5$Z423Z3bTF}T=8M4pmRtj zY50}+&p_C$DNc26UkW;7OAL<&&!ffd|2T80A?O2A+-V#)F5j{9UQelf+Z*}rH%8rW z=67p0@t8qHT8+3)?Qqa`pG=E}7M48#OYb-N?Y{q{ajNK9Nwzn#H`&AvEIng{^K#Ov z@G8ECMb&K^*sBD$&hu*TBFFPY<3tR60VW@@h}g%!CiBaFe%o7np1s-ATynNl`}GRR zq+pKUAPz8>laENN+!DCB_$xL_&kV29Ek9_o#HeHaNjG1~ry`aSq)4RVw|%T)faG7(i#U#69~nSVu5Oru;LYS>8m;+UjK3zt6ml@qL2z zvc2cY{O+`jopHFql@trFvS!d~T%jUdG7tvQc&}Z5X5C1$e}5i!^-(sLfAVnQ)$z|F zK!7f?g~iS*NwMJ3eg*Ev{7d(4`d@@Q7T`48btK}B=I!!n2uO7nofujD_j(~ExzI)Z zA6Tee7+lfF&man`GRHm^W|f^f!eLJi!GDoGJ{nQz!wE9m1OyxAjr22~=+4f`}KBX9r)#E z^_rV&vqyFs9#}77WLbFAe>(}AjDCIfP2el_znM#yk8qNXkJ0A)!Y-D=FJqm*GuMwg zdMGTzl=}uMKJF9T@+M%3qwb-sk#!=8;a5>3o%j5R(Nv}ba}b%>iwY5q>}H*;N65&& z&L6xperEK`S(@zc(iB^=g@vGG8mTi zsF~-y!%9JFd?wmDmjpUxnr<#9%f^r?Q7pkLqaHUKB#t_NP$Mc)q zfl`YjOXh%s`Z0^if8%6cPzqgK9$BcGt$xJ5Xu;RHpT2_}{F@6|ou`RjFe4lQkkngG%}>er z=#mVLvNkT-u}U9^ahS1-*eHo{`gd>7xTCX!H31?2r}e zwOxH(Z!}%x_U@zcySBlXJIBNZdIaxd33mrR1CztA^sdM9d!HnK2L5=)s$L?ye>qI$ zDE*wDK7=zZ6H=Itqf@`b&@*4r`{E}{y?b80XOFbyonwqK>zGuy=R1s-ASl!^MH5Y= zPGOxBq4&y*l_e-DWIo}TGYRpWy;317)Ac6#rCvWZoSz+IK-K{{0E~qBg~`2Fzr;2J z>>{$Nx_&~;WAWTo6IMn}wi(;9k2)c18`2)^M`Yh86Co^n4&9xlSGD1%6MAAW>3APwdB4C3zXkGvR?9@&~xZydaY{D zgb<48@d7n_cw`$_&(OUq^sT{2!u#5*Lk~JhKM+KOBK2WD}zQ}^wwjoM>J zYOc&Y^q41&zlg1n8FpG&>AC{Z#Mq0Zaz4yqfBBvUe$>!E5+rJ5=&mtfw$wwSb*xe@ zY!p)NF?DVfE19|+Ko9~eza|LtEd|(jrzV8)Fex?Lf)0y*tdRQNcpE+=1)a7yBW0+D>rbcym<#+%tqWT`BNsa+J( zV~@qrz7LXIM?L=R2^w|7SlBd=`=Y*w^zEhYT~RCSy?l68gIK1Ge5bsSP9rdg>0I-& zp9YloKLnLn28&8EFRg!tVj>?|_xj+FkdVQk?PaJMgn$Jwl#%rbwsLzDc6~S}fQA1` zCrowi0(=>JGx4pcu%A;|-o4o^CEKr%wMBiMBWZG0fhz4ch5*8be$t>e|LD;kc{Pal zm>#;3$`k*(q7?OogW>WLDMLB-Q#D;Z(usW?JRD{P3Qo8-dAFX~Im_3|A*|s+L@V8b zuNvKJ;PC>n4?LDWOUYvn9E*Q{k*^mC0R;^cs*jtT*yZOi4^7Tn6(aEf=hh zH_O!sI2vrV?pX;H$pYj}O0A+?c9DE?j$wlDU4yM7Hn=gg3q1pj^!%_jAA)92&NP_{=N%g~ztOjjs> zeP*}=R`pWpnyVgA*(!20*_;ua-G27%8GFu*KI?)*DeM1?-od*mNm<%XOOH?Tai*xl zQb96tMT&h3pwKYlr_WRF?Z8d3y4bhHeTfBwZ35JUfpXw%!#A_PN|SmCJo2wjzSpaz9XlyiY%3zF`PK3k5Wmx!8j#svF>(4%;X$GnaV6G)2EpG z2{Se~-`BM{pWoy6{;Ofz~iEwtKGoq9V#uSIL z7ft)>73GY)Ih*S>xudF?A9aP2ESynk9{mBWiAI?Hbv!HhvagCR&2e9q2;r;Etv^$5~?x$*svIRez5D)wk>>BUFY6`_8hOjenJ? zER)u~LJ_=?u&1<(Gpe}X9;<22-k5m;Nc_0zKLOzx?IOCv|U7s z{~0y0D(|&Gk{}8nC?!z$n=Nk=Zg5dL zBg(=v$3Jm2YHvVTT4_J0e!THe!ylGTDs`+)6eeqDMQLjkpFK0!dORf!x2UD-S)8VA zP)Q1D2NxYJ)tq?VR;wxXWiGurD4 zg_5Udy%rgN_E6Tt&~iJd1uW4!y=TA1&V`{C-%y5oqR>O8YA(CjkydeN$GQl=e*GFq zP8X(kk$snkmilb3*1wg(IoE!+#?SP138;@=M`3VU|KIej{}nOs7L z^;rdsQhyc1{fe<&xrk$v(c>lFPKq(~RqoCk25Ro7u-RA{X;ZcdPea>}793`l8J`CQ z^5h>pX3y&t$}kCIX(9%^N`<} z9mVg7XEb@^G(VTbr*081Xjo5il3O_+Js!4mFyq`|*@G*oCGXRxB06G|X7u5NO*|EA z4`Wz_b*?bF?1KfS|I`Ly8yr?YEuKDAh+?u62WW3flG$;0j^zJU;o46CucF;mrmuN} zx>gIBM{Y^bynT4CdpfQae?pZ{(wj#*WsCf!!4N^w-TD<_sUFSJA82%^^E3#h&-;FviG00`B(n}FVzJw*2Y+kzq@)luO2@M`6YCxi!VFQhmYTc zv=FPb&9LY%9hJcA#HtaA$KBNLB(%-%P>hOyOUc?XG!FK*G`;k-vBa*^O||K!{L18y z4eLeM2uli0D4Up>{>l9re+YYJC6n(!)Z6H#O4olZmZ5I+aZoKg{u7I#{#Ij5%`&Cz z*fPAaC~DTnntflcVL`?UFVDqZXpq?_KvrF}?HSIjpsJR;n#N$L;|pRTolc8@`+CkF%Q+30i!85h;yx;$d+v`bE@BNB9rIr& z|9Nl`L8lyADTpICiEdDmp#%ZV=PPklMt%J|=kN^2a^W|TSQ@I-HQedcggE1b4bWTI z#g2{hAaDu(T+TV4%!>c*cPG%GJD{Iq9{Uzla z#tE4_+CT87w<_>#S}frc*1rQ(=EyC^Hg4cJ1=wqt7laI*#qgP#6hLF^xRAmq-yW`n-3Cob+pTfg&a$w16h<&E7PC+LDXnTn zkhur*)Ht0ViE8%K)b+EXS|kFebZ;1<5wbHkBQspleiVNC)4zf)d~AM@wg*=(Svq-0e*;YAK$9~=!@#Y|Ji3R-<*&JDjDiU-GL%XSy~?G<6vr<^D5emQIG+^nr#Q?@YHWEZ`B@ArD%|_Y^ImX3Ad%EsLrsent@{G0IKYy6+^#lw?xQ)7(gU6E`A%76ih}bXAN9BsReh6P=a5pCV|vQ`B8C z)mt&eWD86exn zy*rORq|j}Y$NLS!TfAT2B$%gao6fMfLfYA`)-DaB#zsa)1{bmwsKo)_R{^sw0t0|8 zDhlSYVNl}QDO>*pqcD>YGep`RBSUz`n}P3Kq9e`;G9rjFh4m-E$wA{(bCQSs-wo zybS!NepX4fk&nNc_GfeMzK+vt6+hiT$`Z&umJIUg-fW()zmkU3mZrrsHt+*Pm1aVE zp+lNGsM=0JU?{s;kxPvfNBF1N=}_ zA!%guhwZpT6M@(`!Ak2P2o-J0(a@L8$qw$lNUV+iPYxa*{VjHiH$846jG3;O5=_e= z?XNMGHzM5uxdfVw@fyWep}t~%WsH2Z-T_1N3qFO}lo$7(-&y!UI5vHsvsHFTe5~ zEXY|62g6pv+s9A*$CWP;QZY-mN-9-5H+vp@QC76*#u%x4!}51twe~)Hs$uTMjz*WeXmj#(5i~eF23}q~*cHbN88UKaN^PX85;*FQ@ zjpz3T-djK;_teMLKW2Ou;);_OzFxgIqHC5o!g>EHOCL9)KS_UZIHF)Q1Dxo2R6~sL zKg1({libjrAb1nAFO3pTV=)%bc0ZMk=()+2F0u|rf2!@0p$IxST+K3nqp0?uQnxm+i1>4z3#y(MwOTWL)`2?EIaZ2odcYD?_>v2|8 zrX!p1l3JTm%j3p zbO)=Z!5al=n%kDLBw;mm0pEJ2mxpSM`yeV8xo{_hvJk|ZtPVhpCA{U*DMqPJ7d^vt zELsKQhPvPNKXH1#?RI6d#f1lN#NPHvE(Mo@eQrFmj}g}~^mD_#y5UHnY#lM`%E3*~ z^Gua4;T{yRA<9V3=}^^1Xa2wr{y;D0vuDl;*F$dX!Hz?EnmI0sWJYtlcRv9j-E6Vy zmqd;by@!ue{3`Q#zpgWH_gi6PwvB2qHk=hKae`0G@CB)kv7cPrTTJ7~uS8#FoU2sY zgIH9>K2Em0fI*H zAy;gj-T}LQr64tDYMIY}I1hQ_eWItWqKE&!1atADTPjk`-&JUiuPVLLUgm$|Ejx+} zP2M`+aN9zi19Kjf*@S8F=hT_IR3hGC*Nd&(5y|8lah%ex;h`3t&52udW5H&6Z_(Az zr8i|t!)@d)X;z4St((xNbFAXR@x&+7TK?A}Kd7zsY}sR-{cke%lWW@%_H1z8U9PNC_aw-lC?_xZ_FIZXnS5tbCb6+ zCNktRH4@K)D~aI20?`DrJriA#T6v+5RJh0P4+bH+P}C{w<(H~djEfV46X^XQ)fiPE z#osE@zXSiI$-?}0+=w$P2;2(eXL7V;>D1MfnKJE^U*9_ZqQMVeSGQ{ zPi>w(gMZMl_mTgDUP9rxkYy!WU0K_Z;_TKzC}#=c0%K$1Q!A95R&ZDAVB%9 zv(r=;&DwxvR2^9rCb&ot-HfZ66I6WP_~Nc3u`uQ?=GL+i5`XtfcxLvF0#9A9$(Jxe z&mUa*A)A#DqfK4tqSzj73XtxY%@=0%khr$ELu`%1kvd5kAE8EGqx}`nbb35TvK_?H zR-NB-3GhxIbXeqHztV5|8#o4pEsbhBbBZL&{3gy54<8-7!;#c4laKd*6qSK{4v*&6 z0s-#W*WT~nzd!C}V`fWBD(+V@#0>ReoJ*C9>J3V)=envj9mB32giX8(ds9^1)$?jR z#9!Jy+~&!?26ze9)zBDk_Qb{%GP1{2(?DQE2j;h7=!#ZGu1U{TFvP(peTix zjaIY5^O5+-=(9Jg#8WaxbbkctTjo~<+Bnc&5o^Jd`kgXNuCc_(cd#D&gMGJ z!=KJDGU<|jS!f@RHz_77l@HzHROnlj*gRW~#}e;_%-Ues-HS8Es?3_Opo``{_H31V zRXCU-%(pfw%0Kta@pbam=+!%dDgxKp>FTMWZZV3HyC&U?3!olTusM8$A5>HAZ4B+N z57SZGgXW$-ohLW@`Me zQO+jjhKL??)5r2t{;Z2~@mD}z7_ra`7ZFe0`U(rx=4%Hm%f~{s?6q7xV5UhQ{R>8y z0@qX18UNF$n9cn+<%me8j#3pXjPAl1!@ZZ;}|BtHI50jg}I_5N^c!daG{ zzfqMlT(Z1%oL4Y^=~Y%<9nVd{mLG9WldS|;D3pN6X)a-DK(w8Ak{!OM;SUD?>-pR| zf|oyQiMU)8y)ggvOrTFkhxrup4UXObWiob&gEwDVM=f!dJ7F0*~v$~pQ&AjlQW83e(dBA4rwA*AXNQ>}f+atZKXZ;D==R9 zbF%)0x@{izEYQreF(RK9+S}7SC|VoKi^5Vj_zrSqO98uz_cLcz{ib7Ez@se9#y^;<%IlU614oFN+a|2?cy5{5X8tO@3cjB zq}~@u+&34Pi=TKcTLQ@cZDBb8!-bbwSFA*)$fXB<{02mwwGc%d?Dr{TRXq36VU&s8 zm1Ho>yC;7medL1C=gzy_pS_57XZ}Wa2`3H9%hBBS!0=yqoON+G^1Dy$?`HNLb0_P< zDquxMmAdu>&o{QW$Xb8T(!FFX#mPUz-$P5wq(+DM6Z7*;MLRTpG9*tQZ@37Ft0H!< z)@xq4WKeW^8oB&3X6w?P8P^E2eXh;3tB%pWR`O`P#N9%{>yLXcky!r~Jm4f$>g$Fg zFRrtGimY@q-nIhfnPfqmPdmdPx;z)#IW=7u6&qg+)Q6GLC@9qm+7`A>koaq)s4~1%7 z?-wyH+#?uuO3EDZ2F=RVCC$};CZ3Ib4cEQ(Fh&-lkyuFFvxf4Ol4Wy_l=WUt6x&q( zH)Cci_AySMT)wzLmCJHpfA-y&y6u|vx(u&*?bPdz=rnEc)|0IbJ@p15?>pije^C~^ zPRj;{#2wcP=k6Z0>r28i9*9+=wS%USz>GZ~c{AIiyeL?{q3JqLf{HrHQ*G~fx3XzP zqjPoS;pzZOD8QYRYL1Q?76_d;EN{>9?kE}9N#Jcax=G2OZ|ke1Ik)#c$vJ##Ri0S2 z7}u3#e9=ie0GFR1oby`2yqFVc$qMFC6xmQ#UDPh2^ozP0TJ|JGy#u))!2hX!DZk@W zq>`{;yNfCm6Wt5_kS}4fUS5XsT3%!Y&QUI`@EH~f_BiODVLtcr#kN>v}jrn zga01tRFLUQA3=0wFR~&jB&jIU8aosp8kV(Xlm}oXG`A^bH!FiN_NaL3)5G_iHSnXR zgB?3#)=4;!&r(|(d!Uoyem5y%o#;0OZK@*A|MmA}OO&qNKkm>{a;rIx>&@48S^7%F zVzRC{}eWI zAWd#BiSvNZ1{W20J7x!uQnmIc%AVLDs*%8Jy4T&b-)zN-@W)i-=7m%|4?(LTaqq5J zCMW)YY2N038@>aqNSGyU!ge18-!$w1zu{vRl=MT|JY8u7if^5qCoOcDTBYIC{0gS1 zu`u|e`TA%1=7ZOv3}z|Rrvobm44;QD>x(XbbUhkteQnI{o`$5AazjL-f(&3{w!^a9 zaRKmIa_te9FiuZWcb>6)!-!+UnfbS@M{yrprlSCxij-6R%LM7(g)F2Lx`dpnJ+CwI zG$3Y0>k(K5H@6IBjl_{|Q-(?7KJJ@GDW4xJYgm1J*guUd^+6O(h%5kUdm&cgitpP# z@T$Bh-Pcgy^(E78GswYX^Y({g@ed@a5gO%iJ+XSmeRKHyMd3Uxrv2C@=SXvcCHvqT zp>ynN&7yBFK&`V1wZuzBPhF==QQe8;x%Oyz@8c3=Bwct)|JZVuECZMAND#GC)Ykg% zkzJNWww+~U7->^apJekAtgf7y8S5t?5&Y#L#^eeI-vVnioO2iAjT4PUzjUda`FOJN zvAj2Eu#AkF1~?4|s7Y{@rcoM^&kH8sN$=1T(CgqH2kd3=dF}?kBgw5uE!~QS+mci- zDs%Ad$qfpfMB!ce&A_F8^r!xZPdP!R3Ud)tIbNAdu&Qq{BLtOH)&?7~vB_MVe)~; z0F0^cd&n$kbK)Jt@~T)~7Os1hIAYgnNYKP1)^3sDY;wj6p!@fK1Ui2Tl8g)fX;OI( zUtN`}dw<|?K-hNkq+GAIS5ZUOw_+ICr-W&cVXvuzhG} z>H_{-mEEKyPW8S`wBnBW%zvm53VtTQupPshl)KT(^AW7WxWb4h`l>v?u=vi&)t(<+oK*RZ+l)cc zG73hC<9=VaMek-;s?SM*ao8hzM`3wZ#VBdi5U7wy8;caApP9<6j*R)gZB}4y>H)79 zO>f+pHQ)gSe9%_Cj>`^X*o~FfDq=xgT+)V{w>qJbe@%^(lSVgRNsAk|H`uGv zxSih<#K1G)6Blp%r(X?6%W2;fdCkJKj`CQr!@?Ltyw>|;B}CMqh21H(Kn!E{7d6$T zej=vC87wmY<0r=zSbXzwvUUfR0* zX*DEiX{S4OKJhw|{BmwXK^;b07)>%AfxB@`#OhR=ka9Fl%8XO;!G`V7^Z+RA8E!or zH80_cuLR>v`1lp@J}y-#_I)-QV-^9^y!~~{=wK-X_egsRajw3!vCYYRVNgZ zJ%7IwSUjtAhYzXg5sOGiE4EU99KU1xUTq~hF5$1{*4T-f*oi&lX>H=0=qow>6{Bz* zBOMq)DE;AatbOy>o4xi$t7MH+Hz}EF**4bJ)@REfP$EN4yRkV)akc|0JIrjN;<)oV zd??6oGH5G#b?BS))^JZw?eKpgSOz95EB~E;=Xmsosi(aVd4Lq8X?3BbD5$tg3)>Ac zKxi5-GcLYuTSb@^I}L8ZBE7RM(&6^4?MFjb!}rai?REA$Wpd&+V**l;t#eu&*m}%9 z6<2Dgo)FWteTXkDmA3gW==0gs9`R6nma}K?_Fpo>mZt_tID&LP?}l(iBH;@@+pulc zu(;a_DT}i~$HV`I=&S+tY?bv(n{EmN8(xH&ST0Nw-5r+~uB@P^^mvp$I34ssiFIVw z;t$Ue$TUJ(SyD&&6S1n1oI?z*6;kmkBjtl_sCQ4Y0MWpjr=>R` zxqRwWv1%tgm=^13Tie*M#Vy_c)F9qAyk!>4Ji8_U%oeS;F6!&!;jKg&k951fGU!`5 zF|M{V6UG=gfVML8Y09}x1i`-0A(swm%mt#68L~@>%(}AfKF|9?U931G<2)P>e<8)Uf+K(7XC-LcK(08`m8C!qePzFKn z#H6(R4189cVJs>*QH;KoXc|{*Z%}+KAXzO43-wL1GX(uM0Ps`5x)D;Sb^+xhV)Cf1 zb*oY;dIdL2ws93*7@d^_ZBfOabrxJjPXlPhv%x-$=(>iR#@d%X=)y~QN8bia1(XAo zlGwt8$h$|B%;mEt2F?6!j4fR+#0J;khxVi(uD!mj+$PM4KPQEu^*cxskyS?sj{lL- zXNqo3R}k^b2fQ_R^@tbT@+n+y7_g3^^{v1I0#eK+GiDBPRro2yTlFF%G8xdXR(0Qd z9Wa=|8+?w|r{7{;z)AU8asRysSpbwnC8e?4aP&LWEh?0b0Q0S0LvP8$n!-SpuV^$x}L0t{e;lxBDS#3Q7!5;xe@ zDhOa+CO;N3<`j`;TUCt8Ih*#WbeceOo6f6c zOf@=!7|35g(|42^{#x-4LHclM?rE@7gj)j2{OV02JA$bJ+dv4iOPdZ#%Cjz=&dur%4K3!3R*>{QSZ~vF$)wyu zj?DAv=ss;#-2C5(9!bz!^a>hqBFmxNHBDxUbv^Bq+w9!AvwU)STb=M9r61$ykoihM zeXJ0@F%}I{TxXWPCI2vckPIR$QAfxEa0rCVC;oFV>J17H)|y<{FiAh1Yh&!6WX!t_ z4YmQ#DH1e!Wm2q$9g~zOO5+aFc4JRPB`b_6`C!`1xy8R3$4(;yA5c70CY0LIPds8o zE=Sh0%rs#&+U>#^n}viWs4FzoF>sl|60>+juq$GUCGI|gBM=2dz0hAerO2-HezQ`r zhXmZCu{P7I=e0}QUKq~UnHONnx*-qoNUV+-))V$N>>4Q4=Efm|kNp(T+W{84Jag*57A&eQ}vKMX-( zUhij}TStvnP#?&Oy>Lf{fS_0jZ}JoT8fw&D&w@#`T_TGqvIfTtoVS$xJ)QH1&RhJ2 zb!60ZMvr=l#`rOuhQcIeY}@Fq=i_^7m9@d~ZSSze+}1d!v_ukSIqTxvvDp8*D^{_*bQkS$fh(&&29_n%w{w58y)43fowZ%<_;TaD zbad1SINu7*JdK+a+_Y*^Q_0`6vmr`qFGo2`Ga_l!?e#7dnm~R|<;TN_cLH1#aR8jpgW%)SP9?e*GgV2bhO&!#IvxW+7 zjG_c&%iU>lk73WqpG^C#xs~4&u?2oS7}AoG|HSkRLoN6yW`j-MuzF-HX(izih@*M2 z{o9c`+Mp)h8DdA)yQCskQnMF2$hO%FsX+3Z@Lm@p4_!?~Y|R966*k=H;H@gKqdRjcOI$ zo5r)O^cUiEQ;ns^itjJHAn_gD*Ov}&>8p23AhrFyZ^`jjFtaq7YdW<%PRF-)YM073 ztm}Dj!uciv4BDq=kKMJV>%>g82W!_wD8R@aw@Sp}u}MwSnR{f#ifz%X*q@QHI_Wp5 z&ol#AR^ogz@Id6u*erSozZZ$!U<9diWs7iD!2T!x6|fsZDq3suoV9u4>QU+Km{Nra zXqYbZ>JW7ksQ-HnYIZn2N!3h~ql&#t#T+I9ecLiFiktiy^( z;xu12BI_)#*do6Hw*pVCTwyCE)?0=4^vgSK36paPQeOZSC=*hT9Tk+eotnyc()`_V zmp2O1tp4v<;-3!pyn$PYjt*hJT*YI;2%Xa?ysgEWrvebPlDw~B0cbr%x%tuQl>puv ze89Q*rj@1}hl}BJ`-2_SGkpj`fz}HEK0#aq{WehP49cf71#}97BbeS5>HeBjZWPpMzlhdiH(&S*Woe97>)+LBKFA3rBnPQT~hN2hm& zn*G)LmW2-AB12{~niqj~7FkRele(k8SYg#n2VY%m=_>gUQa!D`){lwd0@@`>0C+Q29hA3YhpK!)P?JY{N`BSA5q{DIWNOzbpU5rI4^?~*iE!D`C!&| z^>x#m!f{F#7fizFDBX||mX$7HM&&DmMHdgg)|i6tnYiP_+<-2|BEJ*h{OP6#^O$^m z9Z)@PeTyisBR-7-1p@@Dp8PGeBl)644Tr_IG|e>m1D^bz-{BJmW0dp3GpX_w08;N; z$!1}ADu5InAE&KB>qV#GUdB|!)=SnX-N-u;7Gi>?hxGi}W#c=jV%eV@$(YZ;NJU3-*oc&PqDiqUxe2#*$lWK6d7T~D3 z_F=}-Xwr!QbsEd%xF(BXotKDQtS;78;)risf+jkfj5cN99WbQu+!mwqA1br8eI@+4 z2YR;r)oU@*a)c^4Vm72;*i38hZiwZyT3_QZp0IuhoGw4+60VZUoSZ6J6KkXwoz*)_ zAn}V0iLn{ofC3M!y%*A#7mDUgcsJy!{b$;1=-~YOKK<6Et&!t-(|yVi#y_pnl^_56 zmgCl!eFH;2;quqc@}ugkJ$F?VGjkGF#7?()O(*=bMHirrt`zG$Pra_&!@t12v!}r@ zJY;`qX6F_)V*0crJyFY?bo&$n1CnmChPoj_XQ$>X>j}|WW*ukuH!*o**lF_-$_>iC zM>2Td=bcvX+0uceK>1Rb25_!8k0(u`Fmq_`H(hS`P(%x&EPCBVJZ$Tf-JNqu_2<9K znKOdox2l_iR^`$m zF02!Q5LT}irBkJCE1cmM**4~T|E!CtDtKagEp98o$B6|nbX!3AVCdU^x3IehLD~2y zS(d9)5wkGByI&aF$LI^ugzhr*y9(n6ko#)6k6wU@S&2XKj{m_^%aMz3drL>lb_vQ^ zDVc&6^1e`wZn+{)-_v<#@VRmLyopo=6*s_)aoViY*2jLX?Iu1RIPneTikkI-Q+P)?m#D9=v=Z3Q` z4A^>A)>?cMv+Jg+8%HhPnW9`)AzKmI@=IdQuEEu6;B{Mt|G1P%&u7Rl3sG7pwCj_+ z2{Th8EZKLJEF;nC(K#5;{R?(|b~vEaft9at7x=gqJN>q!z*NIejlgW#6E~`Dnj%!mD#zpQ7eOWH#~a!va>Cgutfnze%TUc2H@@y%-${c%!XM6zP=9* zQ!*sEkDtW)1Z9g9HFgM!X{VbH#wtp+>encsQkLpwsFO67i|jQPgYoEkVJ!aROt%_UsN4mSa3!uEkpxQ)QDV^fbn`MO zgo=kbUOUxdmC=o+ll~Eg_UndrGc7aB!kx^0*nCwBs{zi``@sWC7!#jcW&!VS0CB>> zi>-bRtrgg7Z-KeTWJfo~)wZNOGR2tz2H5S+B!dj*GaL-vd?E*Sv!$ZP1*7tU=2>i1 zmdQr#-UMFdy;ba7?Xzf4DY`~MN1))bF7YQU0=@gr(Kxun3SOZSnNVc;?~FZfvcDDR zrL3(Bc%;lkMepABfuXr>mhOw-%ij7XP+}IWK4;GJ?w``)^7(k=XwUmr9tpRBJ_(!E z7J%^D#wKqvBdAW`zyL`$lHxDI0~w5Mp^|qHh=-I2VSl-I3!KY?;;+BCNN|d&K(57| z$i|*F=8lG`qiHd_(FkQt+i{r7!67GDq6Zd~AUXJq|C>pVIo-@{`|sOzDv}9os8L@S zKq5SHPz_6~K8cNG1}9iBG(!UpF*NlbO~a4`>9l*s@_$G?pwwdldr@;KTWJ4|ef2m+ zyHfclJTG11=W^6Pvv=qBd$K1Z3GPuT%S$Kx7)`TA!~Iv;)Sy)_WGjJ%4FCF17d7Vz zk)rJxDfvzwe-SGd`%j}XcC&^2&O10T{@vT_5!JnAuwZ|kpLwX%g!pF|>GP^bBCzBQldvFT7>M@L64 zs$b+b2E`WFjR)dU$23f^{V(n}&4*)31FibLQ8 z$mx#iccl15`Wj{izB5%$v!VJj2rl-cmk^qDU6dvhK-Twg3=~(;EX%u(Bg&Zaw2FUnz zuxLIW>`I2Ol;A0k&^o}TPv=Uz-zQlrVAcjQwEc?cmI=$_Q5g4ctHUiOiq(XBwhxD8 zDLiC8Z(lbvm~yE`$!s;V(wprX-gr)b4#Q$2E_oNbx@c*afaXdZ8 z|DH)UerCzvD3YmTmvVI0spcDB_T_75AeKF<5Rbm^)tH(XBD-HVd?GFGC-A}dwcQ)` z`+Q!>nn8Oh(IlZv5wly#@$|IHDV4Q21Z1QSkm190Qf};e*}Iw>(w-dg4(S25RpkgF zlPO$3?1@fyRqm(Ksq+ty0zLvUDp;lNerfW@J%VA*r}JizFO-&jorpm25V( zcVHDsWh_)1VCPAklW{vZT~yDo}Pb`rdvgzO$R?B?ZrWK1Th*miF~9Cd#V|Vp&O|rHZnB_NW5RtZ-aY@cwmnfAH|U;vyKF!t=%G zLH;ntwsfnjiY}ItXiqwDQr~P^ZlGNDF2vh==CR$DaM>E^b;Ea$Zp-(hMs8jAG*1?7 z(w9`5e)TCF`PEr2oc%RqkHTYj=zrdLK>)BO;ROzVWu#^&vA=Cxk zyL3eBeh47K{a(Ex<$DE9i$;{u7H!%;K9#xc!qG>E*@FFNopjwIF;`8C%x#RbzHpHper(2Z=OCo`p4Ttf_yWu(Mq*JtD|5;xi z__s{_r=H@9vEvUm?-8VN+7#l5Jlrt9bgVO;@OSk(S*`!*wYL?_Ssi68qXCuX`J@Dv!c;+ppWH-^Ad(SFe)4 zAlUT1V2GZ=NE-J9YVB{J&BDf#`|>{w-iYrXc#sUl~|>ofl9R(#7<)vlzi`W`#h)lvVRRHK;$%?o1Vv=#WQes-yG!f~kJO9}J) zaGsDNQQ++UD0@a^wbJSxXZ-?`n@p1FGRlk|#1j1Fj5^-aI~jA`dn@*RZb$g(Xk^jB ze&IdxIN?{K6F%VM_Wv#=_J~MQ6bmK)qlB4YySe3xPopF#8Ic&E{xRZ7WLk`JTZgXD zTG{e}8LX%TSH)P2Hcw@5OU%rV6rZ&+VfPej1p{47-H)}y+h>@l%Sm&jTP!yz<`ul_ zwW?gfOdR&_?#>^LIkcwuNh^4&eSfl1QtU3&pj4s9^8g#ww@mfRI872C`+e!4Wfv}cF2C&JO zQd@=G#AXjw`1R^!HqcK1nxlpD36>+E6&Hn9z}>hd?kz$%C-cX;5`Wo97F+;UUVS3F z6r;rR2d`+(2X+cc$u5Jow5Es?^E%AGKtKPM-Cdw?YeX_em%B-byyD|aVIjWM%7+zg zGj0YR4WLVp!!s;> zn6d+^;=Iqg@p~(YVnMP6ZD|j zO-+8Q0HUqN^NL=y;A~dRFx&?9#tFj~fdnJnyU|4pZe-w^px>LkS<9sOEqC#yRlznQPA$PMjTRDP zs(PQY{5tM^?9{l3?#sITeY{JSQBnW87Tp4_Y9om;LFAMj#u4{MobO!vUPtM8T)ClD z!nk}CcE@6<9Ho_3&Leca&DP|uQO00n$(+S#5zS6&oXwx#7Z8dx_fluj*gk%2oMlfF z{MY0jTFcDy*SNx98!<<>KBBU*!+N#qeP)9}z#$@qsnnTS3@Ob1F_s_2jF$E!kV z5BAiEoqMmLp0UE7U`BPjy-6*%nEHG1W(!ZuW9rZQcXVG~MLDp4ivLuD`mOe|xsPS8 zmat~gzHGbS+vxC&_CS!bcIfzPk?XW60YzPwDQe>$-N)+WMuhRtp6tJPzRCuYpoHa_ec>D2d?%=S0U;hR46fWIb7~wqjVQ74{m#Bp3h{c8Rg~Z z&`y8D^6qSX2{x~N=}?U8Wv&tYu=MJ1Iry2VqCG!`gSKq?Rq4bGj-tx7AZaX}zaK?0 zdVKt!F*Es^BxCVc>gJo2gq*!AhQ453IngnCHv`29Q!&bkqp_^Exs1DtQhP`;Wp`?> zQIP7U%NzJd8}D!(X{Up*qkQ>3{nHEXYhrnO1H;*NtFFZ(C0YCc;-@l}Sb@`&%G=>j zp&Ah^Is(V4-(N5_)1wjhex{^R&nYH9{opD8bL+%OB>5}orM0+V$$g=#(e28H+j~^0 zqi-dasp2YmHuNCZD+wWwJ^0gD)gJ;sDaqisF@80I^q`5$8ALoPpN#2x zVzUAbGXLE4G6yrrlG_UNt9o|Jycp$le?_;S2RivI^J0SPT_~dH?6*mpLoC&4O-INH zT%}>RR8uMAEWB}jV7Ibjf@bQ4!hxRckKDC3s_I^X!j+K1(^#N(G{fSOYU5=EjNGGL zy!VTm`-s9X0->kzYKNoKR3`e)>jVX$b&lTEP@^d7{GRb~nf6)?M(FW;6rBd6sK5@` z`@{x%2yClNBgs>QZn@e6twZN;PwkNXNUwOU`T_Aa`aZh6e>}Wmf{NZ4F6uDfHb`Rp zdB$$7ZJEt9PViS4D>jz=_Dh`O|F4H>Q?-*hJyCn~%eSDcI#vWaV}it}4pO>?gxhrOz^$`mhd9^d zyDU`b@LMQAi_1K*8vaGw9EnKc$kQ7+m=>^hM%FsY%@{n^1)j2yDUrCZ!na{b*KT46 zf@E5IBWjdfp9IQy{_=?Yf462pw6<#6mTCIwnLYWmCxK&r3$-VP8h5U4$BH0`P5_GW zynm#~>*_2l{aV~gXr{F`K}(GY+o|zlnddRHUxXbyy|U5F2i~wvfM5qm76NEWMVs@4 zm&LiwS?~%z3)Lg3M7<*aBeWD=lMa?YeWVb`e?fD)e}3T z--^|gD(u+J>RT{B(BQ)VTIDH{{xp`jHeoeWr>qWZG>%)2`Rta&H7b217=bP;<0I+X z}#+6gI-)wJw%zL_bIvk=O#{9im5oQY9 zsB};D5ty-*ao**QsTpD7hAUUB`0Ca=cbzgN`4}_%wS?}Th97#YrKa8eCT(oG0tJs*W*4HehFGqdD6{&o+}`$Y7-Y|X~YTYvEgF>|yp z!TZt9TmUbDgr3On)tYjH&4Q)QiWNTYMa4LtwdwwIr955BFRbx>2)GJd3VL zWpkAFm+!86l-sR{I|d{^O!JLPyaD1KT1XerasW~WUbxzhWp&tZGm#ZM=>m0Y_QQ~} zJuu&9M}c1xI5!NM%t&J~V`kX2;>?3kzZPNPJHZ{!gU4!R9`eZyN150%CE5=Hmj~=- zL>jV?Z)|M+E_uZlt(=*r#t@}hE?TWx#duFshbg+F6*Txw(Ks$#)dDxhM!kj$8)&ss z6fg?NG{=IeaW&XsuQ(^z?cV@g=vR0So3a_H%lZR&S$3yGP&lT!2c;k$8tB3m4 z@5I+DY93F>{pu7T>uBq&F@>vt!Nz8AN6kPARFo;^2Wy8?GbB?!wP-VvC91{mX6&(g zK(d0~CrE5yAV#}4GzZ6L|CLi_bhQ0`91nhTwp+=-&RCg%XNtYyHe7Fwjkl2P@$1d3 z{t489y~2|V_*Ax}*`;54mZ9ii*CQ2%ZxSumaBvdt9OS6|RbeEl2>hQ)3SBlcM;kA1 z*KjQYreC3#L5!-tt?>T$lj|;L=$SFzF3EnNd9rw>$Bfwx`36lp33UOaPcEh4Z%E!K z`@W+$v`km1dp|G|Ca7BwG?SZgtLNa>z~`bgVzY6zR|*LY?zJfYFXy-H=;aE)OF&5; z`KT(cmvr9iZ8)TWn{3bbrGk(q&%{Yr?*Euqs?aT3pk^v>*q`{>u_^?gqT*s7FJ>-T zDT=YWAvd0IVB^fuE9u@V^Ph%;5MqJ4>89xHNQM#6Sasvl4?Vvu*!T!;RR}rnfLn~d z7fgScPD@sqjz@P?UDTIDsz?0M(HP9(<*^SZCcEiGC)pn$V*~wynJ%MvK6sA(o2LnM zQ@Cq;>k~r&wz$PgCEB%DU%=3r@BH;Jz8V+mtDx!WH>{VY=2&b~)qK;aB2GGI1A7KC z!r~T!XlZXYKz~?pJ-1M8jc94~#vvk|8`mRwjScRY;dWj5QK`BZWjeb%ok!lVRW~vw z+n_$}gx!jYo5Pn-p9Lif)-01p&>XI03fhNfWR5$7&UGD9DP`y^F^R^H}K z(jTOYXfdz2e;@>X#Sin(!OCL$eztRmWRZ4+PCZRsHvt@AeHg0lJ-m+u#yRM4|^Fnf>w81WP=VkiW%sp_6nZfU}gd)^to5o#w~f4>Ay1!)j} z=YguOzT{Zg)iz!XOi1mABcl~ps51YCMXts9@z+?6e@L8z zHLOb5BtqS-*%|MT(9E&)G~9xc0qr0cd;A~KO7}UA%rrNv&W0JS&Tq|j67%6^?t{uQ zBL%#5+0=|6`l_*i@z1E9tRVfd7-B-0wYFLIp&cyJUAJqL-!q((-$3C^=Yy^V@x41n zWheYm9Ryr7veF@I+;U>0u0wkVxUlSPoHz>RTTflBYL>K8tGA>AT!oSb4}*n6S(>y4 zuDb3$V{r%mFGz&1h2~8|Wy|e9o`;1EI26{lvsw11+5))A0dj*7(|v9%2~l|hkAeP} zhe@-GCT8mBLN6)u8Gbw4k9CO8Ayi|*4NdCRNf16d!RDDMgo5beS}4tYSAm+JM~J3_ z{Vx4-thi|F9qm(Cr-B3Yby?_~(Dfxn9tQJ{X|0+@oRv z+-_u_%4`MWIjV4j{#~`Td48H$^~2Vw7m*5!|H>!9IPOJ+;X|VHe_a`6!eg2CHgro( zhX&6|VWVM<-*)P0*bB6x+$qQp(9S4>-bOb)OX3aOuO~+Iy)Xk?p;muP*j}1BH{zSs z|8ey$a51gl|M>1Mx(J7EMC6hqiKwI|<#LJ&ji{J>=t!Y7Lf4s!^zk_oN-m?B4o)?x zA(GN%l7otw4%Lt}gEDETG^VMhX8!BhJLmU({r7pjPP)wA&weiJ@?P(?meYMTechuC z^iY^DWVqU{Z{fWh$U$t(gbeB(JmS<5H&}kt=3u^T57uu2Sqr!}Vr`|h$=@UqXVRkpSDD!$U<9WN*lDyNV;&~;LNKl z)}D*@wdTg9&bo|KElteTBgioZuNK0VyxhsSJrR$GADtr(aug$a6fkp7)xjqFf?5k` zSyyeSLdqv*o^6Sz>IrUXrh1M{PrJ_Tw1&tQ7H*EiGAZjutoUA~)4 zjiyZ4zDxTpQTSv!ec5kwwyS5J4!CSZG7&{U!U7MgC(PL+6cP=W5t7|3%Nti%x5TL?Q;w~pDx+B#)nt_EI>GyfJvn(C_|bX<5$%9 zam6lG4!+%)wS5yjQ@__IcHgZXz^OJe2)V+ME8H#wEzOM~q{2dLjfU3`{dP?>9_g^v zw)Hy@e;tp%M*XHc|Sgyp($M_&(;5qt53<84vVdq_>jVrn2Rxo@b?fi~$}?-RPT5JarcWR<%xki zhe2&de>lg-0Vu&c^<1sU4Au|}p|W(rwZg4SmtPt=2utTx7?Az9c;4(E0b^8f%%Wea zs@w~NC|-___LB1Bxi{yK$pIxTD&fmd8#AeXc4^Nm(j_QYQM_2c6!)A@n0~a&uvfb| z@hiRBQxunz4ap!3MMly;k(Ji`WqV_Inpkjq`piwOGqjI_>W%c)Ek;FaskfV@r(tkCAp0vW%6|OaB)$n7aGXSl zY0pn`Kaj?@IUBBRYIptGg7Q~M%0yc2kakhKl3OG+qEyl^~TD*r@vbe11;(Z z(7%;5r#7-hyA9L(dq>LR<#+7Q@`jyS-HN%gTS>#fJufc(?rbP_xre`nj>~pv+m>1q z0WS>YN!PgW#ioc6H}psI>vo|~ST-0ScifJjN`%t`=e%Kc+<+8)kszv9BeEkK?33S^&;(&;F?D>%mEPW1%lz z=%Gp(8GsOL=+v|iehUSz;)3%Voj9XPzI}wHY7OB1M|jZ_J+*mD5}rQm&><|egyokU z!{En}P-QO)F(ILoge4U|B z*e2|`UAAx+uJ{dV9s5v^5YThar?PWbth|}K z_jXbgmhip5zrX2f67zIQFux^IUr+c!cEeNjrM$~!ypRvOVfGtq{v7wWfM3xm1$LL0 zPiYP&A-u2VqBI|w?yb!czt7&zdo=YV_{2V0PFH-F!fwJBwo+$ztG}XuO6Dl6(a;qY z&8?nWIy3Hlqih2{wx(4D8~Ro&tl@6iL_vt)eRixv3(s!Xy;|7XJ;8RxX|)FTaz4Xa zG;Yg;)6L7ObCN z>v4cSxR5UsLfen1!JZb!xPWAUW&qN^7*tmmyj-2j#>s1Wk6PBKk!Ux)|7sv@#&b>ri8UNDNqOf_XmF-*75vTwfHZ~^* zVI9?f1teUmxGixy#+*HN7{rCO_cyajJU#0r{2GLKh|7iY%l~JbG(@gyO4s@7ZK+#>ZrCq3^|G`1{w@)^c0Rr(;s#m+D=kRy3s%?FGY&5)YspfH?gJw{7$zOh$D`t~dX2kY6WYNxxd? z+=?TGd8GR z-9j)+I7jB0JOzr3TJ!CDzQ7n@8h5q&3N;E}T-!*S&GjWVf9jtN%5T2%Vb}@dn~~1k4jTLtGQYP%Dju9#LavC)rW{;x-|dL*ljX*fSFyGx8d!$v`kx%Do-HYjM6wgWdvrr2Xfd%`M>YJ{kS~{VdQ5{z(@S1R{ z@|k!PmYlxl`S3ru#V_mdUuxA?<0w3~TMLts3u&bm@cVwpFFK9N<3g()Z{lWS;>eyc zK5l~o)0TRnLcg*;v|c3!nzOO9Y8Q=oc>mA7RBz)3XM(ny$RmhTAhj6Zso&Q z6?>Fa-hASF)Rd4TTmk3O)}uw;RMP|2Os|nh z9Y*HtXJ_YTGc|4``CYa}fC^dFrGC1b4EktezK4WBn9bYGZ$FJ@vbu1q_^=D$Bmd6j zJd`gH5KctK0W#Cjw11-ZW1IHn-&tRP&p$z(>#b7 zg>z^%!_@(ILWP|%!srQ8<)DG#p!V$J6(u%Z}iM-bFq{$Kl9XJxI{o5S}cyz){cF?97VCA*^6+J@uxg z<%T9lV|Vsxy_%(M{fH9vTa3%4#X>A1JPVv{@5uc`xW7AUW!HUG3c9$x+G zz^qsh?LaQ(rg=Hu-cLAGnn3-_60>r3eupOp1!T!kv!Tb@G5T1YkBF82QzCKajx-8Ghlv0v4{qAq>!-x*?m+0#&1C4_?VHAwmM~K_p zve@rtpvgwOEd)Vy*WFnC011HB-JCqKb^!OX-=mt(rH7ptqH${i#3I? zrs987ZyktU_{#S}@J4K8Ns@(08eRmQb6``)r0TJAs-bhLP#~E^DEnHp-+Z?>@jEa2 z+L-r0X_mEE2HAlZfIjXl~uwLiQ(9qc&}LXQ82njU6`@x|&l;fY}z*R13Hvu%H^ zaBJSx5mxggHLT9RsMjwT>1c6&V^3aKU-`(?B(`@|;@K`2G){7OmgQhbR^nKTgRsWJA!KjpCE3!?d! z3dGebI%l~UesM<0 z5O>qoWv~mj!_(Zh_~g5qi}uybqBlNQiVNhmcZp+E(fl`Pv3?>djqxn?BZyOkn-2PU zzH?G^mvf(DE_BmZ{beY5s`LSvU=cO140YRI3fy&%y%e;cla3xSzJJ?D6+RH>pts)p z)3^O*kzdxk7=rXr4lefH(6OQL`PdHI)nIM>=unb{gee8{B_i`a?MI0?t%S~({l>cn z^_QR`0|d9o1^(0g;QjjT$%BorIrf+2+%UzD0;ySJpH6;X^Tw9p6;LKASSdT~J!WOFK&pF$TqgoQ#YJp6%_La}+q$&%0I-EUm z^>td}5ba#9=28n~ya6ro`|jCxbs3VNoJOt7YuB6>e6iLnQN#PL9~b|3Ut?S7r+55g z^p$E?W?!LiBF&#Ym8)s1wrTSGjqu;J`M(A^%&7Xb@nslUP5t-vpN&^Jvy}xqtf>w~EOPlD&qty7>9x7J(XffYT2yeoSn69T|RM$DAU1$r$>WdND zMPiyXq5bYSP|gP_?Qw#~9B@3YZM7dZ7s<)q_GYP9%XwHNy|Wd!O-uMC5*j7hfgT3S zIhaYK{%-gQ@D2J01_q}5Q{!g4GwnM>B!9@lv9JbyfdH;FzFN`5HhQr` zrFU1t1Ie@>&k(}8yUx)L?Y)uwwqTyQ@7pqQ6x?T5c1jWcZkqEgQ0b*KzWDuM5yoZW zR!eu;*G5i>s!>^#J9vaVO!Xg*QAgh8Hf2ImJ$xEELyeRaGH7UH6Ggh{*Pg3=Ga+4} z$p8MpbysV^#xgVl@wK&tA{EYm(wKWm$3xr>b!-sJHl{59;nO?UXXc5ffQ}hxo3Qw7 z{jEn2!<{1oO)?m=l$OEsf~Xy<+5{t8fNI2U_jk+&C&0m{RH>UV)&8;+^E=WrEJn7R z#2yBLneRm*0M3VLI4W?MfU2-nS!azADX&NnrT_LaPBXy50z*VdOPbFV#;_twIJJ6u=IvHRnclQoFWEE*9=Wxm)xP z>?oJjKd4}K(F2LhAJWvf8Fv9b!H3Gapkbueo!yt5sUNGw%MzS}5Q4;OcLZJe4j}H< zaEH?hKwC$C)wxJEYdT;H8;Kj;=>oCj@5?OzgX|zqfft(fkJ9oox2>Szy!G`Cyi7pW zQw$PjGE?tdo5D-hFs^<^obn5CZ2xQ+{(8cBRv8iOBdk|ef!>aWq?m+@wf-oUxMY$L zM4;2hBUoC zsOj+w_}4&;cM}J8CYh&txfzc2!zF~)J-XK51>PMhygXI!F<5wnQ%dDWV)M#zqF9pz-D(TM}2oGw$l$%}QY)ot}0 zjlVsb9de4=WFgeD5-Ws?nH$TO-d>qz#i4480Ef~fn^G@QVRzFT5ye9}g4gmAG;0l; zkD|Q=i)aMRpb~%}bboiTB5mKd9hE28&h(wzd3QOI0L2{wQTE^g}^3k z|940_Ur^Ds__2zJDAu1?2z}3p%|^t_pSvR6-D>Ah>`)?)qlk5#G};c3_5%G8l{Dd5i@*E1(+6nsxaSuVLp=ICyJ5N*x^#p~R6nV+R`1Rp)b@DGWx! z)Ipj(*^z>!eriu#E2||2#S6Kv`oDu(q|;l4l)#Tw8H+SB3s;;_kKASAahXXo;aamM zSA2RTM`rPc6HIm5B}ENe!L2k<2YFAfqO3ATX!;71CKSLlKB8pxR9^uQG>y;Qr;!WC zso}!f&1snX*rC6$Q-cU$OPNP%7D$rqm{|pXYqky7=J(2Gk<18?bKx9~t9B2IBTBfR&>3GlcNV6y(RuY*{1G8; z0>LO_TIhsltK%v7oW^TgiGxG?7K`%iP80=Tfen&*paY@H#tIzGHebMm;qA_<-6Xi6 zsE_yV5R7aBaM4;U-FZB}cru!~Jni)l@H)?ergbenEvZW9388)MiI2ksfiOgRn2Z8n zer_<&dQnBS_DY=P`F7|s^P!7Y0r-{(8P|e%Tnsfec+LK2npwK zc4aGEYIRk`ZSq?UF&a%2ajR9KAwpBw1zYnXX3hOS6SpYk+lW)-dK4F*?R#a8)5_T~ zDF21?i*c&?6G{k|pIey&$?uc>pa3?muSXw2TsbWbCtForx-{Sz>*C6Hx)4^Y4M~e3 zDvzv@tJQ;$(uf6ll~(*J+flQhq8E8}o2!o0NfKiNKlk07RbI}6zH3<9kgyfr4JM3v zA|WB+Y=N-9oZdYv{N*QRzrSUI@G3nDUmv?ArR4PT6q?KLv0A zxH!CdFDw9QGlV!@ZZn;4O90cXxb6O};&$ObHLxC$jIWDtzv}CYA6r2leAd#Vo}_e; zkKAxEhSB8z)j-9l-{4(P(dgRm&OZ6GykSqhYbGTFWRssFV25VJ)4hy%3eV zyTALj{RE(sT^Nm5zQ&S9<5%l#=Wz^a*CkGRPhIppTt=hibGW`pLV3JTS{kzk?_&}U zP;_A7F^fKPI4;J(DS!rLj;L4!=r zm-6?b-mH>RB&itC^@HyE(iK)!R6#iFP2oE)XK1Jc*Nqv-`C^KuZNX`^Mw+QZ}*ddT+Ar$Ml{Lebum%w7)0 zE2|<*9^HcqQKnPhhy~Ga_aA=~@vM(LoOsf}P3#9Bs%{2?z<)dji^Dy_)Boe(I@gXEcFWx^>U>3R7cMYiq16!u^^huBc)r ze#(jtynsF3{TE`}rNt^9FdLHIm|VWyYY1H;tmQE~{d=l`A|P@g561oG+Z6)}O&qTa zosuP+24~J{9G(k<@Fkau)mnI^Fl{5BX#r?TFf8DCaK7Hw-oiR8L?2!k%SOoqMX-5I z?zhSlmZ38ou(#L^P;HVhBzOQZo_04b3EA~YB9VljtbK!OBSQ-vCjjn@|Zs^Sgckr^K@fTZCxo0kz2c+j zT?RcImxWgHDj3c6)ONoRF6e3Q-KfYPzUmk6*?b2hYfJ(H7ZQ4DZjs2*Jp!xj1EC%$ zUgDNj@gPi=Oqv;!;3#vtDBxQQ5HE%9SETfM?*eyRMHuk(rR#6}v(WM*vLydhX&IOd z$&lT&g@eKF6173V-}286jOh-dYu26&c>j24&$m&?L{0YU&MWTXP+xfYsdj$Iu0O^0 zklqcef1EJYtrKQA`f#iJ>;@EXqaFR#M6Zx*%}^aiK>^N87>>xCoTD>Oi~^7EiNdm& z@)Xu(nvw=TDYlZY!%gxfuIE$a2(U(-T3O-jHHQCOo&=s{c66A@<#xH63~;gOM&E%# zGHc5M2Q+LpgD6JKW7NO{kB$ck zhR){dEc6fegBchqXrF84uQ+aeBORYl!cwa^zAy zrkX-ck{FB2GRIx3KtJPW75Xlc0%SX&v!P;VSCD%Cr}m0Y6qBqv#>-|d_>3?_XbF-t zDIevAbQ878u*j3SAj*+V?SXhp5__2O_Q&9{uWMe^q~d|>U2n{SPpbZE8>?2t!pu}4 z)1+I)73q1R@_Vfy&ATDUDNX1zgU>$9q-x9!+;T2@r{n3|GhyPXq~l$(&Suh}-dCgc zZQBN!$Rb@g1l2BpX9sGgR}wC8#E;3(0VRVukcZ^W1n}R%-05QF91P-E%Zs!JG%&8* zd!}#eRV0~N>}qG)@I^hmK`TCrk}$O?$@gKfiAz(A*k9#QHuTurjiwEiT6A}C7yOE^ zm39iHsLz4N6NJ>nx87L20aOr_k9IcPIEeJ_dm?2G<3$>?oHMoS^?Z93v%TO?97^2Q z$XB;?k(9Y#x2eIFYB8niZ-kS^t`{%GyU&7mzteRg(1AoN$UHIbf}4xc5!IX*!s^+hOKTXZ|q#fh!F>QzshVei^8p#u` z);@Q40}pf0B#)Je`5QS2ROeb7)TC<(`w#YZi{~aW{WsjbRQ|q4K z61C@GHWwVDbti(kCUO9f3YUbYp`12Ht)?|#mvvG1i_g8!VCo?9uTdWV7nzUd#(Y^3qm5B4B$acwP#9P=BV?uYI0Az126HcjqXEdHnEtnzLBRuu-uP&^A340HM6xe1Mpyzm$jAt#x*LgA3F*&1 z9g(>+s?H0xW zfMWAqh4a5?z#~9D8#UzHJ2*EL1UCx{f=*ia5m|)@yB@T`1z}{+jzduyJIuVvTZh7IQCabz&@Uk;8BW zQVEtHfw(#6W32cU%Ap4;1(>14DJ8<^*x}G_wFaanjmFF_!}E{&7}(Y)Y>Y0ay+|5^ z-2+vo*8PICKE1dhsjg*kr|=3yjMLH(X#y6A?ou$F)ER`rQ*yTE+<9ucWgt7>XN!Zl z-xzTnJHL(WSf@1~KB@BrjGZokEAqxPkFg?0~L(}Cs4DvY9_69igb*CZ3~b_&@uO&j2RoNh(M z6C>HyGXLLZPOsVatz8|QU%s0-J^OsZ4TS%FH@O@yk(4W7da=}-ZB#Wv!HrLisp+|A zpVHEBK=#8g0Z6t#*-wvJ$%emX^w`>#FID^xXtw8?d|^#p@xCsA5ymINK5ephq(7{P z?=8WBKp4;d7WK5$Rura-gr>5(Bere%0||XzLZirpou}h_WAdoanDq*P2X})hbxBe% z;0RkA+MpM?zN-fP%X&v_7?RE(>u+p1iFH+LG2mWhQXIdG5cM37qFhEG8h4m&|M@e( zUa(%zYwKE=b)AV46zV}5IhEBgc7xb6*)o4}&z%wrcbJ&HIckx&^?KK2OP{^&l*J&S z<05XlUu|g8WRr@vq$217f$Vp)4#vko~1cq@zWidzR)pnAQ3 zq@l{CtpG$~6;@_K#RCeWH%!nPhH3xpEi3TvpQtG5iCB6$Ou=>^9{qdvUX1?kIsJdu z&0pYUM|WohpX`jmo^dDZh;;&x*LtpuD>y%w!VlA{*nFVsom+7-i)}a{HQgRrOvKAJV+c?_1E5q$QTOV z*o>5a zH!zG)EiV*91tte|xH7^Vc0RHPz$`)SO8}@oMXEdeeC*gSWaeo#K^Yu4*Wtbdo3U+! zr(D2^^=Q@&j!H*NXV5k-8H!_LZ((HP$u0C`JKu0rwUl$~)InR6ANfcA>&dWb7&bZu zpT}rI?F*gN-k;#zgM3QqNanUCI$;&1dYc~b)v?eUo(VO?ruDZno}-)pH8o7}&01{C z{dPSr;kip~u6)vG`O*}eKqI@#+wzSHS`SOKRNO865fX>*+HrGn;X%lkgcwlf>G@_M zcBG;at~bc7ZQApY@s==Fu@q^*kH}4hISSLz*KdV63P4DGno~-;0SYLdRNIJQML#WP z!~%7E7P`U|w=r8enn66U3*23gzo2*OZ7Po1J$gM?l>j%f{k0`$(wj1P=jL(iD0a3+ znH{M-!2N^IOl@|^To|*j50rEVQMbN1;t$tMEjfxKB6q&&c#F0vu0{~Sv2J^5ueGEL z-!P4$O)y)DsDD?Q)0p|rB-~`)xBmmib>@5I)v@!U|!<;b1 z#1z;q^z8uI%>TH2LhjYu*Y0s(u?pMDN1N1PR9HHBiSZuXoKI5y6``HqlLs@S`?bFB zHG!upo;upWpSf=hp)tjLTql)|dpffk)YG9XV3^wmgC8$MrRu$>Vzd8^;4!LUs!<$;fD{u$~)-s+QMR9CKS@HC}WJw*$)NBnG)r8O&lI+FW z3*@r)BOoq8&X%I`0}}gn@q>qUOB6PAIIi?Lj&kRlzu?x=)p0`s+qZX*_CiDZJBw4^ zqUsfvmWbgky9&oYG!-hGX$}6|Ew+B-K{L~%AY{!i1`x9f0%KGo-gP*5PTqbaFFp}bAeqxyot4`XK>U?5rU8J z<6XHyK-AQWSCUgMQZZQl_r662&+$D>Lm~x2Y!ela4o`K^^^?99b@nj5c5Xlg3r1!3 zw5lwmYST_2sXsL8LQw>@p>m3_f4YWAy#m~-3mZ2rH7)_W)MDNh{beSspj(lkNv*%c z1rNYET^FxCUksi&X|?Oj;A4w%N+m)<^}Jd}^1-C}pKbR9(J04*h}Rs3(X+oo^s!BiSClj@5` zT&0yOH7?xqk@_&Dg541!E-1u<-kq3S+sUC|H(OdFnix1A zK%1xAJm+>oWcfhVrSJe)=Lx4|aOkXBc+x_B)Gr8P=FiFdc3m5r3SdR)#Ab)A5wHj8 zH2)>WZW())kaPO{9EF%o9|(XJ-b_gA0IL+t<3)e&ii}Up5yxMH*>UZvscDUV)F1|% z9}8MrD%L=RUp$qE;e&U?!}=*D%ftx%Si5(_$`||SQKhxHp!U4nCvDn}EU@=;k>C~y z$whHa(InViW#^Mj$-J)X&p_;sJ_<{^+oEoLy0{^+k{r3Psl&%r=aI5JQ&5I`k!i4R z!^(5wm8#b!e1A7MJ@=2DV7AM-oiMkG5s(D^+f1Y3SO&(72cq{?U9Eir}aMR+t)KV#$8lEYhQl(l|>o)@Ao7%|Vf zzugFPkdP^_F}cj3$jQ@r?^N*TaFIiR_@C70xFdEvL_qH&7nn8B|0>qH48OjAiEqfp zldZ!=N6CYq$m&m$8ix;%eErqqN(@AZ^CgiKbvb@T$mbJr&{as1k+qG1rGX_}5>Lp1 z*Tb5U2`wl(=y(}XOV|AA{pn+=`O0Z1Seh^v4X)_Hd??JB@A_lR03fZcT|v8VfN>GI zxFtt|aDx2YhJ@e!an)DxS3|W;K^Ys=PEe$s9v`+^AVpZR+extowLksa2@mO0dta2g zaySW9ts7u-keC2f;aJtc=--LRjHA zKgjCOd0D3h)WJz(*=RszNlaM~(4RE29GvY&Dv(A!dY7Uf?#iN{{3@Bu2!hr+=Caea z=B{Jr2W91%d=)jlzDS}rP@zPW6eErn;ZKnGbW*)(JIrBqSQ2P(OaLi_zX8-h(r@8?&~JjRIg}U1H~bOv z!+o&t=&mf%5m7wMV5LmFbui8a!GG;aCwoBsk?buq_``cxT;0m7X%GXq6JyMkRIwSf z&%-HVkVe>eStsma#le&Hv+eZfqLPM>($WZ71k)EA(OM1aTDKZLovt6jrE=pz)DRpgZR$eq7Fg5_)sR4Rq# zgR=7R(d>nZ3=uwfh=&gks*6dr<@f<-;T-UdPS~zXh+Kj`nLbL%PWicV4KaSLLzuXa) zG%&KmX|$)GzCV;Pjo|q<-}1zB%I2dT!V-=PjO!@$WMqUZCsQwjkWbsR7cK>O3jxzj z3F9ZyF;i_8k{EE&b7RLf5+2kCC!-(WiU;i|dN_JftaIP5tM6Xhuufzpv$|$!3-b+I zxLlx;z~UK27dL)s6YQ5Bg4=Z82oqob><+gaFF}a0s3pNx8ksmwd9{KtD{!o|$NpCh zsBZgqp$fO|q8CDiGBgs*{di$*L-y7~vzKFBU$V)~g5BVLh?r5XU@RX4i06;^!x>|9 zpslgLNL%Me4jRVI}ML{H|Sx9Yypz5x%$mogr=?}T?u9M_3Qx36pm za}?+dc&OVxddo^{-06y6=}9)DPsF%<^Ntz6&mHqcLLFoL4;c@eSs;)F{jY=12OeSS z!uGqCv5|Hs z?2-&%!jR1{7)UcRJi%<%Q_RJn=LQ4nhq%MjRDMgaW4P}H?NWxmh(+c6(&acFdicBe z#qeUsb0Kh;4u4ndeh2Jpic}SG7)iVYzi;JqWI@pP5IM+L=EamhO7?8f_*&Db%_3ww zE~1=Lwm_}zDip>TsuVstrg1`mfGLkBNW*&2Oi_{qT#0PYgbdPZ7#hAco3~rdo}=*i zDj6yA#MWoY_51g`Cb(KC%8xn-_W}S4gKLz8@hD&3iMvbrc~U&%a%47t0d5rqc<;&i z72bZlX9m{Lpq+_w+tbo;u=p>V#?)ejN2TGm9Fm{u*{8nXSa@9(b2a$mcDBf-gALWj z)@nP8uxWg+!$}hWe%u{s3!9Bwt~nP{3g4a^el6~%IYl^sRaCZ!`w)jIesJfYJ(NbR zfm?8yIGVW~u=GOWq$?HefyZtW5MbG&3(DDGyaWESdsYhlJ`Dt#Ii4+u0R zY))U_z4O8`=;`d4+TTN1 zQKfEdSaQ^_5^OK2;BbF0UYdX;e1kk9!Pbzbe}2Mc&kCcMUq;~!h?wIB>YRs@>0ToT zGLGb|N{6^)%@_?CrZ(RdqrWMYxa6t3#{5q&Y*B1NvX{KPecL|GYjXgqeN=Qf>1MAN`Q`ddJ;xccis;DpJ)qgfA%>qlN6o>j<74HNhF zE{CnAgi#%%P#)Q&Eczx%oujILRNPiRp}k;Yyb%OV#u zvyXj~+*KgcZ-f;P9M${mbZiIQ%GPgugtZ7-OiM#9Vw&o(-$!Q276}-Vjs&7^`4RM4 zk*p0qJ#P{py?b@y8owAJZb8jK;utQ`4zl(`GbmZ+>z<{6{BiIJL=i0`bXlCCSC&g| zxXz#I0(5{W6n>VPi~7ZP6@&6X+}TO+{Z`H;6cpU2j9}(s#*BkO_3LwC(E&gUecQ0d zU4T4}Cw#M~unFgzl}Zz#Ntlc8UYwS0C^S`W)EFETx@)8K3|J+Hphgc4%ova3B3w}jDNuWnfB3_LR+Dbx zej-96EZ2%Gz=J9IphP?2&)v1drGWc`iT?n({@f*hj2}2`%+G3q8wM;ueRA@mb_&um z(1t>*zWVSEwpLQlwbQXS(Z$0E**-ERAq|p$XIjx74a(o6-HnX|!D4W`?pT}2C8 zDKS^WDKxy8vG2|tT4&5gJamJl5-mY6uEoAjhqM<0cm>`d`NG*&6BZ+>!E~J(y&Bai zOyetZidVi71{m1YrzWN#r4E-(BXtq{zo3n@Sq9)IG~<}g_;vQ4H?W_wTU+?GQ0+o| zzWpSzi6xLgdK6M7W*3 zfsyPTXa7L4s(*RU08q}e{D@q!wI7`Dk#KQzzVZSHT#3q6{3`G-HDS|3&PP#iwu(hGaG4|$f%&shWGD@Fz!C!|cG16nOz1eSz`bISBAV98m~WP4 z7u;{V5lBjqsA$#aZ)aN`t}iP>DT3pQ?YbLm7IXz=V^E3Al-=yEYYIYR2sQCVd?HK5 zJ(CC4{Ii-1+-L3@c78$i8q0eBIUb4SGbP0Uj^1)Cf{%L;KqoeI_n==a5U7Vo0&eZO zx43%JR)P|!fu%0#+c2P<*XKXK!e{gud z;Ln5e5`ej!KBQ-TyOpaCt?kba;0lx}pIc-mel&m5t9$2N`%>T@_(hLpw;V7-NMChw?>)1Afp5pNO0b6)4u%rD zg5;=im3LA9mPznsPLbJL=!a%E8IML0SXBG|_yuOe?}Z5A0N*3U?HjDaV=zZHz^ko& zKLtUv<>v|q7_;R^)UE!TU&i4KxTMSpi!Q#Ba0%hEBXQ`1c*~LX@w|FAp5*lZ#&I5q z9{(jUurxeuq+cy=nCwiY+Try{+8G50qt!I%I+b7|vd zSJS4I9w8YesY0d^lsO&!tz!luA)JBWvcWj$`*m~Y3-Usoq_`P999`UKKi!+QzPOGd z4<0!{K<~Z4S>OyNrp$RR?vNFD+#fOSKde}Z z=&D8>xQv+k+f4kP#b_?lz;+H@SF4Y44fg%J4+$fWC&+nQ7mA_JqKod*=u`;KRyec? z&quz)P+%78pK-uRSY9N+gInO z-Ex3cp$GGBYmR%NN032mgzEGc3H)l2CFW6y1Z3!_Ht}RY%uyYJP)P3h>A_(4{gy1O zbx9W%Ov&H%ZB#Q=)Yj*fRDTFHo>0yzz>d|BLEu?rQuSrfvp~8!tJ#i}RgSsHwo-L|eV5dl5Hb|^z&D{(dH@3h!;%f_n-Kup`C*#>7SAK>&Y z?FvpC|F=^4oFD((>Nu@_@zEE!-KVQ6nwmF!@aRtM`ThlB&lVn%j-lf@M`f`u-hco0k$y2mbiK&3woi3h9I&wh!ApF4H z|B%IT2w4aL$^OCjQFv?Nj_xk2p-8&jPHgYLE2)ZwgduBc{Y28za zTrWyjyeFa{VBO1!gd0p_nd#Z}V41|4+zh&P{yrOy@Bi??L|Pi2(3_TqM*-825@-$A zq=&=vXCki^m=gzW7s4^pM`3&@|Kwx}7;&!WOey#b8{%8*V5917?iJcOEnN&~C(U^x1fWvldfW=-^iYi|+=mKiaJdfbD{8v6LS-#p z)U+Y58g?N%MC8uVdg}q_vW9X1asTBs($mr|qZ}0CDBx-!E$yGs6=78m<~(0tMN9^6 zwIGZ$&DLQ$K&+1?qoWYfi9ItpGk#GFN+(p5;4c9|0=mt;2=s8= ztjx<_8M+OfK3l)H&!`7|_k|BvK6Xi``VE&C3?!a(F+wOG^p$Rjxxhony~AljMq-JJ zQ|6a!*_06#`1Vb(kjV`i@B$G&2xl|Nz-6A*BBc0l9+B2hY^dN(5HaiiLy z9^YvzGw@WI(9pwH8jF{62XMMmP0EtxLsPKC{WxBX$4fP>D1pXL3cir+$oYycs@kj0PzE#8$ksZZ_ZO$AtTFARq%O^X!F0iw8oK#qT8#YGz zP$h*h03~o6;u37Pdb42-SiH7JfyAUWzkez&-u3gEnEGYZp`Wu-+!oJqu92C;hv>|4 zyMendBx3-{#V7keT_JG~KYz+4wk&>_fmyfsriREtxJyN7GMwmr_gsl64tE$)jA8Ld zDQXKs-@{(n1h{p69Tg6F;sVJNwx%NA6=n=7nLsbj1e6;$w?|9(YBQOq0h|Sw0FK%S zj&5R2t73X7hoI*Ta@1|cp%KhK>y)?2w9EPjOj(aYaMmQgEXvis+ccJv^miIx7xWPa z3kR6^k~IQ*aJt4}Od~wHJ#OnX{c>8P7?Fs=WL%RvgnXfsv%8AxpOzpgn+8RrPNWU#iKgZ2 zHErLWw|v5IzY`2iZ|moZsM*hOc+UlY*-q-8BX z5>Nx)(L^FBepHqvUlb#}AYZ0kg+CB$x{&07V8tw_P(KvRgt7C{K@Y@(@6fhTPk zU_D6x{BfPS7XF;I6k4=&0NKD{J3)YlpOI+eJ_u{7<|4s-c(C1o(-px>y^V=Tl#LUq z%u67#;p}+Sy)pvn4!F|ufaAtGYc1cSz6Itpw?TB$T5~>C|0j=NFcBRNtnDpaY7yO(pO;Hu5C}cE zl}mzFnFF~bT#1~}pkyliUlhRBoVG6#mbGllH)UZ%0@z;MTjLeETf?DbknDf{afGK* zb}^~GW#kN=H5sX|dWQn9F%3TzX zoolHSsuVQU(s8^Cg4#d}O!3lh*Gjb3+3;DmfU^p3AY#N+>jQ&o$)tPG%ghPHhg&YF z8Imi)g9$^>Z%J@*0Nny3k;%zr%}3w`+{r~2;|_|laG+g+V`*SOz%DslA6sM{9aul? z3f~ES)D(ecN0%V;>|K)#2Ryu1o1Gl3=<+$0JS@#fX)&moep|gROQW3X>$ZEtK)A-E z@CH`r=qQmVgZg6;AQ{vK&L05yUwT|pI|v%zJ=SM3h%({%P4l#PpF8y$ru zrqXiw*TZqR3)Ay72zWWW3&CzfeX~Q@`a%5MG@t$ zC{ETIK0K3uKn*&k1@v!kMdUoa&7~M#+^kA_WT_(No{Ce*zFvc#hY8AX$0qWOv)sQ1 z&dtw4w^(pfl=SFOc)zxFpimHhf+>K(mro!q{<;Lge%vxo0} zh8=Uj90fEQ8qTfGZj|bwy5#V7CiD6nZNNDsPwFi=#sk-*e_MG(qtlm4Y^K^shEIqBhmy-w-xZUHTiSJ=$enM7L^>|mxcOvyB)ERQr9 zx>px0t+cVwwt5{A^Zo^2tmnBh65fM;X25x_&Ka20AkQrE{+`2-CsG)&>pHR^f)-yXd*AMdRsd{v`x_W8-27_cnP1) zw-S4AzbHizw`ls$L1=}1k^>WO10KM7vfnMRdg2q2d7k!?yo2|7wa*>%MR(K`wrM7o zE@mXDxbG3iP~*Y}8jCPFQyRs0W zVgDK#9sUxys~}_$yZ(7h3f|+XY@ED-6))5Xg>u!SlJJ!e_UA%@vO4?tRU7vJcPLRV z1jTAG-QhEhj@Jl7V-5fW;xg(dJibr1>%j{S6kq9tI!e$pBSGQlE&No zYaB2XOup~LvN^jVI&TRLbZi+4=?PG;pLc4$Ft6}nVcC$uo zhgSb`a0h_HeQS>3_*lD3 zSM}zMIZI}~>w61DnCwaZj!R+utP6F+8RX6&Td@-EbN}a3IsIxDODvVaiq(34yxIPu zqb=G#q)9vb8EW;z0U;RD*L4|gfnc(?#4f4XX7v!xZGIu57|eha2zHe6*9l;@7+)pg zR}Sl1_qchH)txJp&IWW^gThX}8t%ZDD|Xp_5r6ll$3N?wK#Mx_X?X0?NHxr-DW@8J zC!G9!x0yw%T%pDWY~`Yf2(iVdxDBw1>lM_}+`%a6c>x3c^XEJFYX z)CK(6%jk&g+;>QG91z`zV!b^$;&ZupJN1%8Xp}7@8%0GMV7Q5Pb}5m*fPQ_ti*kH`s=P3nmukl{`+`al4Iys zTL&rR=ksZnc%6f_DXU4*JMR#HO3ZZ%!rfz2=E9&w)5z*G^gO{TDz?)2!p?Wv2zc^h z!iN7~+|vrf%PlyjJS1E{gIu6_qO1n&+Y}|Gr;}- z*t+sSsPq2+#ULwbwOvwlNJyt@bTC%eQb-}AiJr1y!bC_6gMF%}O;V=QF&&mL!^*g0 z+MX3^kusBeS~O~0k#iis_vbrfpJ(>@egEhf4bHI;gNUD=S1@HqKQ}=N(6%LN=@51l$?+qgBM5 zf4mX^u$mmNQ5a2q!9ewYW8+)k8_-i&X{QFhaVE)DK`5_;3x&?4J@wUhGYCt8U?z?X zA9jkPAAl;1#JAYFgRQY0VOOY!0IOlVjs6j2A6>8GNQhCsTFl^@uQez7MYuShX|Evl zYatWWGfnlPLG*o;Eheq|{BsZ>`a$Ii7y=&HKeb;nw%ViPOK;xiK|hNZn0%#hF-LmR zITvPf4eoDH!KPj#ZwTY0j9u{iC9@C*&6)HQBi@JS#Z9Gxb%N0pnv{v?k=T{oyt0eq9_t>gfr8!jMwi!xZCQ4T%6$P8)r^dT!JwO>5$ zKvXyloc(cq5q-w8qAy-wN9kESeAoG%SzD&{H2z*$LOj-$aE36&S$;HoQ|){++VU)C zv-gJ-Ho{kEYon36re+K`ZxWLTVg*bGcNvoMJIwKG5aBBIhJ}tMcW#|{62xJxn=9Oz zpYE6D&zS>LUQ(}k^KkVpm_1{i%;AZ}2b{_F-m}3kN75q9yEf<~(eq8^VScb_;|THrYQFKksM|tW6^bO&bIn@Ldyt{?!48 zMRD-&b9ZGv5?1ikcbN$qZ!SHrg3@`?wAx;zVWu__)qw{ADOL$Ms`E~_lm`du0_B*9 zk7wA&g7TQXhdZ^~@|z>z$0`)pw1XEtJgGwjzXQ(tp`#Rwh@Bk?@C%G@WNHtKz8OOu z8>c*PgRT!Jg#KIwl?}V2;^1@{&C|iVNZ)S&JDlcIXz}^NvuMi!RJZ2e79SWEWdPrJ z&^2xrRyBEV%E;)+KuHEU`x}yT$7CfNw+)3Ec(T{U0NiblI6&4^3jQ^^cdPyMlP?xa zGRxGD_Fm9aKN7Tvkr4CbWhsKYjH%)w3~V9>XVd;=7|RXrLs9?6$>bZ!E z3gV5PyH`W58IZ=N{iR~bHss>=Z#%UCmIjw|e~us+96rP>{jGbY3ae=#p@j~Do|lc} zcfd2LT!m4Hvi3R{V~N?BJCeiCPt38d_`d3euyZz6mTofV0-YsU6Iu)q3t;- z%OS?P(q34ejqtu&ZF#;;+357#fb^}GBYlapN!5b2k0;mR%VaTkI=jwr9rR~|PG|Kh zmFm!JYc%-M5Q~y}4O+c)74NL!15a}M$o{qvZ}Fg zxFQ(_*dy~kU>E+}P%&%=@U>K(Tz#?ZOs)9@xjo3FVZ((*=MqZQ_j0z{qjl(Ynn5J_ zISfO6l2=nv^v}a$IL&cRBdc1v^wjO$uf*F$yr)qz0vSiYpd%g|puZcyV;ut4n zjkfmiPkypM5NU->VPVJK1xL>-+KTl#DH#$Lf`9a*qrA1Ry;tS7Kkeof-rnu2D840MwhvoulX+3mNidh@* zGi%e#3uoUDBC=(i0Nj0y{RSz21J&8Vi3-0%#ui5`YJO5h2g>CY4(JqHKZ_+PkFOFd z6mzP~=8y!a-8zK5AVs+4&yiQ+KxPfvm+&8Dd7Lhnwt!}4L**lo;drvNTMhghJq3A2 z(U}AkvlZaYqXo39MwSpU$@kA-T(+rn2hqZS9$K01$$OKJJW)MRUmb{D+*=DiQ9v+L zS&bnreCmob32yu#!$@jvGR8L%^(>VyuW=HHILm1uBL>t=o1Af>GTvNmu6fl8%4DPloPP zXlQ?&JQz2Shy*}U;n<1i>l{NpNznk(I8p)hTB~$~9V+^hFM9nMJP6PpROx0$&opkb zBGDyun32r*JDxfFU}={BLi1m|N)xlUVM-nvjogKwi9jE~Ju4^zD3W(Jr`21ghT*RThf;OXyl0YM;Ggg;x@r`u;iw2H$Bd%% z?pP^2gyiTH?-46UJn71o`8?H8(LNwz_0dT43@{}L0g5`Q^+S|NfBI%*jLPTDw8dde!$vrzX;V{;>|hkrnsq=lMAfWiIHo%RfB`hv&K zJ!aU%Pemj~c+pVdQ!!*{LY8hwIG2v{FA%5$t4~%mh$alUWPkC)REY>pHsL0O4dEWy(ra4NBoRZd+&1&7s0 z?aV|aqyx_`m)edA53TjkhC4IqHxR9$P1#fca-b{2+cGuk=HiD&HN_6h!DS3Suyxn1 zvScw3A;|{?J-_`GHDwj}UzrVB2Lk;wz9IY#vu&A}DlX$9GPd%*0Sh@jQPbHh7$}Vh z*N1nR!_OwqiN&FO{WLrHjWmd@g!5f-73e-o{_GCx^B#~lplRAMF=eF=1!Rht%QpCE z;ra}J^TZ*0$IlNT323qzoHPoaDfL6oswP1lVG*we^YHvD$X7$TyYBnt2M>@2Vqb31 z0~}D0heyON9)542fLVPR9vh-a_+3RGNM5iN!nU%sU-vcX%_B$D zR>8-_dbu1Y7S|{+a&d*>oq4SL&}RjRdN!rE4bOBV+kV)IK;-fnNQ}Q=wk=mrRS9>N zLi&7;1=1#%@ZKiR2UQ_oQQz$2O3=bh*s_SmR!I2h*>I- z(V}#3(b(cU$5OJ8{#-X>6XGLpI4axe+0WgjL-TS0Zy@WDre6I|kS2|33$DWrDhqrc zSW{S{;`mFsAZ~u^V=%jH_)LmhP>@fiSb9 z2`i(@4AcrQxnnGV$tXdvE_;1+#Np0DXQo(#L`4GY&k@Hm01veX>#OSrN^59UPU*mgAcBkJW(*GJC5P;`75!H zWh}QH(X#HXS7pX96iR?XOV@xzccK%M#hHoEGi*>}T!^Q@8H9C3fu2taRRDz+_^hfB zz!1qznxLHU575&q@Hg+%xg8YG@XV>)NL2A#Gzd^%O>#{X_3d8MDXX;+Z$&tHC#zCv;r0hRlAm1h}`( zf9?WwwVT@U3f?*(zw}M4+$!@-GJW1TOBDlWvzPV@XVDVSMtzZdQve)Uz?Mz9BfK83 z1)h-AG~h!6w{x{wf$NaV8=lDyl%65nny%dDfvu$#8TBQ53-lRI z#Z@E8dB#A(2T;K?HF<4PaSV$u#bq~AMagLk?2Tv&)z13F5J=2gxk9>PtSA%Ct#NWC zoSP$vgC$==jywJS7ge7Fy@Zd9|G}6(tgrXl9k|`SYX;}QWn~1^`%I^b^-m+wYlv)4 z49zD}m_2^ zwr5d8eA*JJJ(P+vP54yC=S>BVzY*<5a*1`!ga_YXX^Tc-gjx$ZI+(Hp(L3X%#KxaA z3eN*@pL-dOnzp3=vu{GviyC0UL%S14ZUWG5&n@CO7bgRKrJ;88D^}n%Nw$^deoh_u zZ+Qe0-Lz;U9!oz)o^vhhd!8N;?M#IPHnE#as*~bhM0b&&6&0?%{akb$*J`XQ-V*lg z15S3xj}##%E`-xZ7GJ$X_zCvW>NHR~%~b_8Lx6ijyqE!9VLa{+k$$RmJ!>p2G;=WF@C} z|BQbwTvQ*`Ls-Y_>>SL(Ne@zZG?7>vzla}51d`e2X&1uXIun37|0eBukwyw9R93BRvpHf%yil{q8&UG-#ApxvjA%P&3`qRfRsAt9fwYG( z5*7G!j>%IN+RHREBF?ARc0`xLkBA4kgOctYLD$fBrpm!K8mrNLH|l-Fvu}XdI(<0? ztSH=`p19IE!!_!Pl-F!OG|vKRcN0)jVY8|a`&j7XIXTs`iUjHFg(R-RV`&u)d|)DN ze2f36mLj8d&!rQpR`zW3Y%*gu$(Y^y=olK%_U2GqeRUt6u+AHkJ6j#3P@qHX2;}R( z!Rx@K1I~x{j0|zbn<@gKr>`sAFSt&um_b-`n(nSrh4uHr;xm*{7x-K!A6=$s7&jrn z6@|g}{5Q}09l?bv@bAIJ?_nc(9y7h-;GBCLL)#qH;}r_HPJwNU&txSbAxVUpN?UHh zXsqpWhtig&l~4jpl{&3a7oLQDFzAs(Cg&fJ8OCsN%HX2N1guOjUbGHD58()Kqxr+u zDJhnjNRXeKGrn#mX8v2p&XXoIB5VgXFKj_tD5Y@VLlhW zOA3^e8>NTX4;pxA3-IS+H)IK{Tp>+k?F7PheGr>YxGf4a$qIlnd8ZNQ5PB;}cYYfz zfUz!}KmB;cZKwz59TQ4`LzoY&a!5=mM#owW!EDFsfehN6eQ!H0j8d z9D=c^O{j+Sf^Y)&JT;wr;G=SXW#^E^^SDqD^BWX{%M{*Y@QjEs8qn^ z3x;sMlTPu;f;$P5|CYi)#AI|?dOJQbOsI)!IdVZd&5nsY0+_v12NZJ`-iaaYso%FH zSW8^r3!}iO#R&<5UD%WtQtyq>!bk58vI}`_13NRW{N*j|S|WGu5%wOWPqH`wdflXc z7wkw^y7zN+cGLt^sfW5b756GMlf~nGv+Y5qz`lqeVIRIE9FU{;iRZFndfBJ%j^OiT zWuz;wKcepW&*h#=vqREl;=e10Z|RIIVO}kAsP6tZ9zc4vF(*8u(QG-))!^-Cq-w2j3zwyoP$c`iXj}UtV`dgTXm$y`p&Fj`$ z^-b~Pt6pjC->gb6cx3hK?#SKqZDuV$l(>HaI==PURiC%5{@1)FYF3ZPYwhWe)%5oB zeb3c8@I7a`;*Jh$F8$EDbj662Gq#xFHL19sw&vBheKLI;QhpHcU4iwA)nK^a-g}NM%3YR zkmei56DrZD=;5hyK_Eby5tj=8YkX%NonFiO29Tap`8IY8G4hK?ll9EH)s zMuqzla2fXpaBIX6z;;0mZZKjBpP%7fgE~SAb>z2(<)mng46Y#9LDGG)`W@JjIqiGV zzL3=(7Q_CoNlBdCN2}I~ztNs2CaXVGAab9(Z#XZx;N2kbp)7^b7qZz>l*MS5(+mBU zTbM@LJr)v-#5(pd`cdS=TXhaK`m<~P`Ubfp@p~yQ~fa?glpm98&hr~E>sT?2b9w|P-FD0w(1esOzVkh)0iA9~d*%bIc zt(C0q4qCp)%Y?W`&wy7`zbw|nVlTD*D9}}`w@>NY#z90XwSPHk(TcH9?b#~Pja$Ng z;=y6KyD9b~&(G54{zDM3rWC`1g*!bg4j2#KAgs^LX`A)F>TFZhm-~`1XbK$O{pX*5 zK8$xj?$IBYRs)}s+_(!8j)jH;jK@T~<)Bv*+3Tk@2qV6CHXgi8IEYeFqn420RE;A3 ziRDKvL^nx#3$U<*R0Sh;q5-1CQ5?S2ZgYvffG_jazEj8FH<}|M?>Q5HLW{r{ySQ4l z`q{9n?{Dw@jA4P?oGBgt10cY|sNrPemhMN~kY)X+*-Kw4nkj^Hf@uCx|4x9|AOm=K zI;1BANt1+?a+^J~GrOrV|9~%EsJoLtU)u>FMDW?OXAc!$jP+!HF@*grzu?Z3@jM;xIA$uLr z#QI&hN=wX<=Mr`UW_2m^q`}-=F=0JvI(NZOtz>r%-jjOsEOlg{kZ;qi*xgn&0ngdp zuVD2Ib6FiM+7t}SPUUOg{|0P&EIfsI(ec5F`C#~Zy=w?$$E<`U1#uUw6*Nw*5DLgU zIIcVV6cflj!VaA7tx=&|V=}Nj?=&jd>(L&5(}?=rU|;_wr^M}WF-Li^Zx;RbeE8)_}o-kI-`Aw^?YpPN_8Sx0RMV|=AhW4604wu43YRF>X$QtT|!;;HJ` zDaO0LsT!Ndy45KDqE^rbwvcoc=4994^j*_P*$2p@_nTxdT6!GVvv35a9O-hPtDhk( zKnK^-C%rr-x1;Un1|=9=pU2Q$0$RIqOZ=Z33D*kw9e;~;7!Ct{zrB8mWL2>$;`CSRDmKdvN>KLKwHaZpVYkc;zW-& zqurYKqp?I4)u7mX3t}_rYMXb~OVr|^rv{cSTuFA#&!5(T@juHERsRp{E!$hr%26=0 zrH7WcQ zzW9)hlmW(@DG8J=YxJR0lrfpQ22hJmP2p=zr10V$pa!qk+OQM+wR!Qw+K>h$G8Wgc zGp*G9;kV|bFaw)^h6h40FU+VtqcxLHovNQ=sR%H0scZJ2s8;PRc8`UhPYXC^uyJ`a zcwvfyw?L}h(F`vJXyn=`jS|)v*f)f!!5svA?T{q#`FRj{EXr)p%kgb$4AUSYFQ8FM zBK@ZmlPbcOD$&nb@eXKBKQ2{UMK!qt+t8Jj*%nOs7#M0y4JYRhl&psrCCrnP+NXbEs=k9^Ud)?4aS~nuB^u7sv-69XgW0=MHFmkGZ%6v` z5qE62v*HkAoa|awN#8J{sSuuN+`iLt1jLXJo82VkLU;qiLe72MzlXhL=XY?O7beQ! zmOFAmQ0_5IX$F7SEr%*daS?FTFrr`db9Z~|9WlF}&heQ5mtfeDC?VM;_K%$=Sr|}b zPsw%3KcI0{wTL_NFZCjjdEQiy7ISY%gh20;V`rM)LU_2;(aZd0N8p>K>z)UGRFYJR zn3zXQy8@$;Ozk$TfP)-OjQ_}S{l@X?%hRr7GvoCZBb~0a3%U?Er{Mg?L=KC z#rVN*Fq)B?Cm_3mG|H=et2Q*8{|*+$fa5+7W1O)R(KHvkdE$3lV z-JSVat}bqWgk>`6-;;JhDaw)Ji#6HTl=@FnTy8Hs2abgVYpSHCcoV8?!2ae}ikyv0 zZ1aa)RX)&OjVq*~HEv0WHeFo`p=>rq9iuHHQ@`E~`~)KB3H%qUHGv#N$Ks3h+^xDw zpBF>MoI|5Jc_GM-ljz;nhWPf}a=3(+7GweeN`2M?ZlISXhyfml&;J#qy`*ZpEsm?d6BDt%PgOF+p9fHc-2|&6 zjmWtriP!ndvVA+B2f{8zpwz#Vcywzgmuj}p%YH3sF{G9dRtM{|K0LZ=2aB)R9%THy z^6n)Kv?XLl9x7Gs(KdgveGV@cx!QWPts7KY@;4zc-556{DG5pp423Pl18DHK>`}e7 z_X*|pMHh5tr%ECM0s;(UF&w+ z>bLB~EQM<(q9G0;z{L-aca9Hq>jnELmh-v>UcmU<3bUP-<>Hr&PBKF(QSGH+O4#(& zWT;`v#FV73tZZ@&oCzllQ_xxrQ&5KoL&e*gCJI#$03a7HfYcwY=K98hgV$mA^v){~ zx3N{SMdk~!i1^9*+|Heoa47@&8Z=;#ngo)H(qNbdtGIM>Y8zJ_cz5RM`WaORITYzm zT;F?D%^JoVWt#qG{i2273JU#?*eNrMTOmg?GzJtHy)uofcOMlJI#@48K%NT&rGi)u zbgT-Z7m7VN_M0La|XQG+&(weI#KGh-#X43!_ z1-k&dwlp_415J#2gMDnXaT!=l0XfhygG+tC@1rYrsgB9`gN`=iSI*UisA*mNaCsAu zT=1zDgTbFU$iz%T#ue7TQO=DsCzI*^jRy3fV;r`LhFWOU`ZM043Ne{_bmGAG;BkUn_tJiI0U zw{cM9Snhv0$bgF@an3?`T&V|f$0}hTw3&Po?{nSXSzBGV34m1m!^n*wQ`yP}`^v4< zgg{^8C^y0lR@!x&utso;u8PbxUMV@WI+rUt4_^=_$TNL5!WUpSS)By&%%aw^oO`4`VqWBZv~zW<3kq7JI;rdIEolfYKJ}fJ1XB#_VRpRg;vNL`@t#$ED2+eC6Ak{ z!l-Uu?mHo7$c6TLjkkn}{8`Yf&N*u92p1YurJU*WQ%EQ#1P6NP z+#N(du6$QBu`wO34wP^GI;O3U7XvKuPpY~_CekVZjY zpfhc9+>m!95jFw;iN6k*k^UDoFNFc>H2C8(hR&+gA${D42*SoQkRd53F595GJLU1Mu zZGV*Mn$N!jI&$pkpJKj1jj=ZINXUDEB0qL<$dZUaq1QTKd*n}-;=4F^ppbfa!Y zlF4o_k>%;oT%&lIbJbYJlAY*%xtrW7#-&!jr>uzqycizF%Efaoy#OD^Zez_n*p;v& zA5W^;`%<)DtHW(<=2Tp^^)E&W_e$Uyc%e)A5WCBox0F^&ro23Bp)QP-+V6g-0vY$i z`BoL#3b+F$h)nuV%dtLu(v@ObaRCY_0irC&(O-}08YQggBr<2cwqa}A^?+rTN<{!9 zQ5-J56LTrh-nn%;pTu|Xv8eeQu?g5~d34Av@v!#L+gmDt1KQ8oj?0&bi=@)Vm& zGnQ1&`{~C-*zouDRXac6v%$>~ksQUq@`P(wk<7B#)--o*$fgK8qNt7dG{__wwQI#a zedfO~S^xCh!uk#%fBQLoT7X+&GLvP?2wGb9K1d*$cm=!PKk9sx=$?3$eC zU?nx>>XKiS1yFbnLAEJmwAR2vj4E92D&jJ6E@x>G~m0Eg5L8h#Z=f zTTi!zbur|Y^=JuBjPQB?>AP9_#Oy6zr%G^@1M&)G{KJV{y&7>f8m1FURxj_iW-R4p z0iVsowS@}f-rtF_i0o1`5{5`IS1#^~rNjp9lK&Iy?)S%qsgWiU~}}v{S>p>*?$UNwP8v;1TFw&lQbx15vZxutvXxOF`aha1%=-*;#PTf;#=45rfQsl zA6@H&Liz^?gdA7J!kqZOLIkw{C_Qd32U-mrLYR%{%x2rck%C7CEB%L5mOul8A7}== z!V2O!+_jGEkMqDdG_sm)X<=dE6DekpmaoY65&b2MFDce;gPBIf)_EtgD565;mtC`hx2r^6(q7)n=e>gcT4hp+Zlxhw(vC zfUH;^Y-P6EQEYtm$Wt8unYf)rP@-6carkqiIY9z)5Iu~OatzGVopl2~9eu>t{)37H zgrnk3*CncZ_=Pf+3w}*DdtZ z{TVHBO49wK1Wa?xGg*#-=ixwl3;ZCvOzQIpp=jU3p#vH8GHMEB*rMk9UVwmBjt1(c zHz$=FSdrQH4H+9Z5^sYnietsMvQRq+hsMHEn5=VoDw}{v(M?cQWRoFbq80wl%3r}I zAc};@k-hJXu=jM;zP_4NxJB*4ejzX^jmTzOv;v|9N15=uoNI5#!|{t`fU^?5ha?@O zEpZogJ8)lW~X^p97K$kR0(Mr^kX;stiC*>aR!RiDbQv(Z4}5{VTVX z0aBV#23if=uVK5jwGpo)fz)-0(0k6L;=?1YZd8JYpdrJ_qvcm7iQ6qG#6dg=2XIL^ z6<^G%pmT6>#Km;11pFq~#Uj8A=;@K$F7!$0{@T69sKFrYDya-Lrvjy9j1v^7&f0Vt zP?AM1>bc|F1TR z!DJ1+SEKuD;nchRWVS01V>(5Nv`}h=9)sr{r_(qmseNi;?KX;;g>*N7hFI~vuWzD3 zv*dg(hya9$0N#N<7xqvEUOJkX4tf`pwbT-}w*M@O%7Ig6KkGMX#E+%!@0Kw%l#tnu zGV{TB5neCa7dewdN>!Y?3HstB(_4*2Fs1?R--1~PICK8RLMs!VQ4H&y2uYJ$?B${n{SdXNd=`d zE+RpHurkQ5r@_)Huax{%k3jrRZXpw>n?8;_VdH~9-MY-EcToP@XS8G*n=7p3C|4`w z{)syT9Hhei5^g6Vr7eB_XX-BKCL<&wL{b+-|9pgktPPQ$31?%n6iiA`o(z017rqYu zicTq*c0)JNr<@TWXA1tL$(hRNqXY$54etr^Jb{xI->1&aT6cB=c*e&+Y$H5X{yRnR zxkQOY)w(kw+n$OdTMkt^CXDevplCG8^tvi0e2H7b8g2j-7l2nxD$Bqp-~tqmY-g~! zPat(}IB8Y}S#VxBHpNG+MlF&4dKmLQ*(h1Uf2X<-eIXf6xi$2K(3`uu{h1W#!Yr=9 zufWm~VwrL249qX&xU?x0Rh7zQ7%S`VTMVP_L#-+i3`3sNlxLV3*Z}(!o5+htR#ybz z@LaKWAMUU=M8+Uc$9`H4!+}MWi9YE(U&o#H3(@myQc~lY zRa~f+q5%Lhl~P9tBcD(tnMv zsA2*i_s8{c#Jl-{In_$`u7dIpC8 zg{P5o01`OnqrI@+p%4JFWu;V9(~pi2x-MplrGUE>HMj(0D9TJB(@iQ&A>>)J>hgnn(nVA6`PoqK`-<$+chw2{C(>_9>;aU36+wPRS0*yA zzEF+=Et9$l02OWyV%2g*B#e1p?C0P$!Yl+;JN@vTfeC-hR_&G=%U9>Yn%GWJE9TW2 zf~a=&Kc0jPSZ>(wxPwfAxSrx_w}1t+pS&FibOj702L@>?q|yO?A{5sZ6fi^3)lRJW zi}Q1rWutU*G$}?b+{t4sPE@?9P-G(B&TdVv3jCF7`3ormS%YU1hBYy`7z%1PRcbJZB)DsU3gmKEhqyZ>eE z!l+;6b_G7b-bX`C28!#-JOIY1zZ7m9m(_7ud2YY9s?}Zu2D%_=ya(7~ZAv`u*ym!C zeD0(w@=Vr7y!b#niVIX>iQIV#HMRD`sqkV`4qXr+BRUtdijZsfzev;CaE6=SGxaJs z%G_rLgHcW*RZL_g-ofs%c^#O;ltht2i-N*8;?%Pm6~|ztnZA8!4RDmIGjm|9`%DG` z+E}Ipt%wwMuu-Q!2h(%P7|ZoPq5zZF%oX85qR!3aVvAY1kErcu(OrZL4hT$XkQ%!r zm&}IVV$Pz*{2Z|dpd_Gz>p>_0Pz)HtePG6m`VT2A(iA6RsCwtA186gL#pLtMK1|12 z0KUf^79ebr1y!SNcqWQGmoesjVQqz_H5Ks-Mx6=Z-=p7TBs!tXwGE?y=(;WvEc0+( z2Fw%IBJ)R(i|uBeb9Pp0=z&NKV^BJ2qP*re_|wt{ez=vpfOh-hk8IPeO5g+}Y(r54 zWH7f^N{v-edBzfFT*8{@#_>9&COdd-T?Ae|=1#_fgDiQ& zZe#KL|1x=yGN*-;)fY)opb~cJc3`5@u<^&z<-F7SjAvBQymnXz(T$z?N-Bi^pEtov z`jv26y-0r?*?w*t(xaF;-Zy9IpN$I-}j1 ziSX4RQz1VK_G~sgW}9+>Lc*nlzInoiz^DrhdLZ0pNLJ^d09F<=@Vs~vb=LNpJejA? z;?EM;r&H5s1>B$rq@YJ<8fQx1V^D|Zo&u)n!DCoHvVfr=A;Cg`W0 zJxet;g|hEQkc^1>P2>VzLbEP)%iCEG_J^L7h&6=u!qH>OZTf1H;{0U?|s1)?7L z{>lH(GWO-+nTqdmZ7XbkeBvxMrh0S4Jg6z_-?vy74^EQJPeN;-Rw$a`Ip^m2rErm! zI8Q}P^vRbckVYDjbU^HypG6TgPJ)Ssxl7yLO1BH+#c>y)hJCbkb!o(ypIzU2&(#AA zitM%??HL^nQkAT(HNzyS+oZ~r_!EaCUOfwMWS$L)fp_`6IJk9{WBO~EkOc3vjD~!F z@yM8UXuO-hM_U`OmM?@1$TcY~7;2|_K>~sQzgIV_hYg9}a*NF49Mv(_dPO}k&b8)AeiRt}IZXb&vgVL$GyQ8C~? z0Ri6<`oWdQHK3s_D`JyHG2|d+QC>N(8r1{)30&mr7!4VZ^cs9X61 zk2V!d^aMPS0%J6h;v%=5B9q8}fbI+>-(QcvnlUUtgPZ`bu6#VE6um_2PP1bODLnsB zzpwk=Wh}e7zx(oX_Fe{(zQ)jAkd__fmrDB6421w#w|T3gLk!VhY*;>+eY(i zcy_81mU?^Wnp44Xg}9;4?u_1hEkiUAB5svM`vl3CdM?mm2E61+r(*bf=3$hIy{$4S zeOtcTOnUW8qs&`rncTg_mu|ILB=0j%hQ^WNd<%<_8a1#ByiMl%Azu9hC7hJ5w)|G@ zn&&}&A_BBzwK>_nYP445eG1Z$fa7IV|1H@ttB6py99lTo3>=1g& zBIsZ;N9Xsosap?|#$I+POtjQ&jI3gB34?z13p{qVT9YbdK21>UH&~Ym`*K%{Umu+_6VloZGytjlVHP9Ra=?})Mz zQqD!_27TOhR~6~_7CHM#xF3OE8$%C}t=@v~i?z1Ps8#Rh zt*cloEt}PPJyX8G+;0fmO<&tt-A7GP9r9w#16oj=GJm?5?-6#3#Xwnz`DN3+eJy1B z1qn}e%Gfn8P}YTY!&XpbLeouis9Vk#FiAV18t4&IvSb@PV5SI1gGS1@t1T-PZ+@6K z72zltS^oYsOhboERrohOq!o$cza{U*oqKj@fe*Ul1_1AEwXoMU3Pe9IBDxYh<;)UI zgGEF^48f?gjhYfU^=rq7o%=wq=$4p6^5{y^xgFwz@yAZz#)ztjZDJn(#YRUtEG=Zy*KCC}`cjyF$ zS$l~}ue)z!gukMf@CC+Xih-he7M$Q>J3HL}2AmL%g1runE-!0J2LO`c*;L>)m(!1P zUMun6T`zy>_5K#`wUhP=CGg;(XlaOH6QcvkC(>xHXGOdbh093 z-Ajra>H~!E#(M}88Ed$o#8XFfl_Bn=Hmj@XK0JVNY3B=LTQ9i#a}MJ%@C8ECoVHZv zmvb57_pnPlq%l0m^FgG?3YBxp=-eSG7U^k@8}QylPh&on=l?_z9&qWbU%ws_eVYQu zin~>g83?&U<+b*2eck=tqW*kaabD`E!FWzltEJAh zt{_2p%pp%gFIpGb+u#SUhn(Ubzu^g_Q_czU+opdBS`Po5dFXL^S*CSS>wd6$iJX*s zR{jI7R4-T_Lni19HTfzX*0}wFUL0A8mjnD_RS|;jnJoOS-5<<1oCvK8i;9ZUDX+{o zwT8hRANYBs-jp14vFq8sYp%lh1HZ^hzN0Ygl*Mo0+X^&n8WM6s@#Knld@H#ykxBmN z(z}`BQ;7g*Af+f@9)iU1P8F?kF5ONLt(MawmQsv~ZX`3bq><9$v# zxrZiv^=TeQ4!kS8Q!qUnrFyAD*L+hS>Zyq*M`4quKnW>d4K|obV_rP3j){8k6-3>@ zo7Sh^pl`;_+gtSaxCgpruxF&~2?fas*+UWsDb0PLtmZ9U#7%+L{(Oczw|M_&mbX(Bm2NzuvnzKcq#&$+o?&B|_7? zYHW+fR@7}j=wR;3wq(-<)-Rkat00Au|Jn7?&{a482}PJq<~EO|>HO^tr8&#uLVgKB zB$b`jy|QZRk`sT=aHsjQuWrLuKe~Ny>>r-99$Z^#=t5a*F9Tx8NMS*Xs20Bew+aZF zUewJ`(WV6lu3x-(-RD)?Mh%;Qq(*vMpx4njL5oUe-*AHqR>!2V1uxNn(ntIJ@ z^h%3fC&8_Adm_oh(*-ihywbQ?$k=fp)GJ`Q>tftp)n`?mZ@K_BIz^9#u`~+TKPyho z^z$!k3188bl~6|Bk*vR_p$KHYy(@ocvt8vNg>rjxQ^QMeykhVEp99BP(<}AC0*E?U zdC#r&@gl}uOi}aoGU0W;BNzwFQqx0yR2_7J-^C3gmp1dyoqJvDt{Z7wk*wCZ8W0AX zr|Aq&O77cPTL;TgyK+R1w4JDjJY*^@31$W!X82Z>zU+>upqG@QGK4iL_{DGeeUVd@ z;Xm=KsfbKc;y2-09iUJl3Dy@8Gw9`6Vhfmg?b5&Lu|T!`-`zQ&n{w!w_F74jfT?%O z^iBc9%+`4a=(qYlP5Z}aj&w;cXGp&txB|2fEjkN44gbn#Vc+@Ov5NDHIGbX_NL;F%rk zr1^A7@RY@Rv(|xK`-NN29nzL`NbU_TK4;mcbW@tBDD{ql#r6gFv1prHtvm`Z(vy_t zOs|Qo#6e%)VRTUh)Cu6BDRr7ogNQkJ%vh_#w+wc&*0I507a&S5wc*m_Ciq=@`H=8> ztz+XGL-foewY*59U*UNq=xZyBfZB()i0R3ZtP*p$v8HUYuXn5Yfkzw7SGSFzEY3#m z*B`WR^r}!+Z*U>`ZR?&&Y*o)Szr8m{v_hd$#9wnKyduO{&_k&ID56=LB@b`e5RxDH zYlsSj8hbdw`M+&n2{TDRSrBVIil>*4^i}VPqT@o#gm?Co#eodTN!)rPziAA=yHoX4 z8?v1KAL^yn*^5!)XOIcy2I)_!dvR#*@es^hW5uA+B8jN3OHZTey5ADM^y zj)c7@pS&KxY^RNk{$&1xRoAhy!DZK$Yy3+RZjNfQ*LsAuK=@B`Y;Ppvd8orh=AY9X-W0(6=P~Ee0(NV zMhm)aJbfa*A9g++0klFg&Z!2S!LBIT_`C+@%ljl}I*vXLpspD`Ztt%hG{pT?7#dln zc6q_|VQ0noh_#}B;aDUuIaWnoWEe5h0$sNo13M>}SOcNzH?xWmxc9G@6YrzsGkPTe z(q?y(luiOhGW5VheA%%9#fW&}a%YKuvBbW*1xLa?%A=U>ZTfL`Z*XO4M`>>vz_jtK zdXW(u!E)Xwx@6MNdm;E*W}e!6!20wI$_v9){mUcW)$BjUYF)mkas~&=zuX(3|8O!t zAP)L-*w^llq<2i*8}z-VD{5(*eYTD z&MvyPBeK=o`JkG z|0ljRjXN|~AMJ+P_+KG|1JWnr#$1M9b`L6r z^;)%NZntQlFeR>`aNGECUyQvSJJ?-xU1ZbS{efKYp?%!+FH;3euf zmon?X#emw+AsZyKbJjydm=$=enb~~h@fI~hsMPQ zsnzdrJ$F-5-*72@Qlt0e^s%Ur8NuRF_TzASUO5UVva)i_p=vbnuG^24N>uEIT@y9+ zMI0`k`=6CCoQ}a++^P$=`fX=LBDPZ@AvLb=VARKP?SDTrZp)G?y0G61m6hv!6#Fdp zx(}S_*cpzz`5-MqM(A!G6G?+pzS;gD76e}^Mo5f-kVcIz)xqtZU(m(jCNNaH~DZRdm zeucun3-c9@%7O4OdvQ$u3WZC!PMuYfVG^@`uyLgPHhZg9#g}A^Upk%WX`@Jedvv2{ zEMObBQBuy-WKA3^S3-!bYAeyOFJ@u*y2u=_{-;Ya6GrH9ef!F9($%X!##RUFK=H1? z5nM1(slk4&I+UaU87keUD~o)cxkY|ZF6JNJO82m6(7mbWz$(irMa8WY>rl!3oaB!< zx|V=Hm@!2i8f$tjK&0MUih%JPkvuTYUGY4aZQb5LV1zJ=mkb2|a59(h?XaWb0avpm z2S4qtx%~67gV$B!knIk6ztJ^Ak|N~8nsor2RFS_t=pO+kRz^=&c~P#eds7_D5s&G- zB^qR@X;+JyyoI!=hps*cUcF2Z6W6@VLab1VdIEVv6@?QY!0KqBQ}2!rb0v~nXDPWi z9{??dS!c_mJF6YI2zb>~ESc}$nllX!NaDhV^>3>tuT+$5t~rS$)+IUrxNzZsFd%A? z>le+b=N6-?Fx7FK7!Q`L*HF-6HIJyo;zwKu%NcrNcc(6u)bhY4eO#*fuuOoV>|RO7mEFt+R;Ltv&3Nsrou(r=hS&F#TZuJ z9>*5~N5jR{+X|AQ82Mv;VNDPiFfLWY;Z5-Vcbs4MH1xSgC?zB(SV$8!lH(>Vq=q{z zB5iri!Q9m~`Ywz~QcA~jgk-usnJ4!c4jN&fV3+mx$ekFsh1zOmNM(D&QC ze!9-+BL_t*BG^5ATItfQi&ya+rCV0k(XR=6l8a5}*uD@>vW1aI6Zb^%LUgWYVV))h-ccyf?2U{m7aGB-HX zXY{sjpmY*As9%nQCca-*>znfYgWJ@=?J_8SkUFCHoPWOXdbYedYZcMVvI^9uWn@CC z&S|d=P2McuYi)$p`TmLKxu(7cNdAhXPx)8Xq3v@?d@p@kqW!LQ+v1T^I`7C<=~<|O zE6?y$$um52cI>5RH^-&P-G;k#{p7k@~!r+SK$lXBO1!C z)7{yXV@B;w0UCe)(Y8@?CupSo#;uD_M1frO6+Ll|dOdq0fUd9Ho_fW7r<6I11m*X0mT@Hx@h zgoL~W#B*!B((aVK{Z;9sm#a4OBk{@n#|eF(@It7leDBIpgTecz7c921kBI>O-+uiv zXIVoaU2_z1Hb4gnWYxtIho>+GMa{cGV}D*mS1tQ&_4#Wgc@i7}R60wON~(jRYICt%ID6)OXexr8fd<6?WBgl)DxxRJOzi zUMeZ_n~fs1h)z*vFmL4QNmsLEk23q=Zwt+noQCc6?Hh*XLj3KS<8i8q%|Wu_P0eS11GPDBTLr&6eN+{#XiM#;{_2L7Svx*RAFhXm@kH`x zbrzSNdB6Tc09B@zkYWSRJI+7H>xgrlTta8PYU&Do%Il_ze`N(cAd%|A5}+#BPI+%vU%*qMH(l5T|n zzVfLm*xX*rdCF0rsPZM{U=_YL{g1HydO68`xL+@5_;lar&jnkO{f0F~1-Gegh%DoUp!jIPKGs_Df-H z0d&C{NkHA`FpkN5Rz3-+uUGjUUS_>R%)OiS4BscneXXJPn7j76iAopKpgkCMo1R(L zxx>R2r>6hJg8}y-R5Vp`)#hoiDO##Q?i+(ULPLU^69U9nX6S0bf$2sLvpeN(uHnw^L(Ey zggjY`Fv)O_D>RJwf&@>>tVN7~>ywKM*WVps_cZNrHnS6j`yE#Z;+95mgXTI#-FRxZ zD*0r1bx6ar-h10XdF5m(eJp(7Br;371k-24%H^;oJTtvv9nEoCHFrpA6`XU{R$X=d zQ9V{?%CfN^4J|vZSnKL@r{6`*+V;giOHNyiwRRL`H>KUS*zUye&k%IF{jl;ojcBhQ zs(&_H7CV1KvdUAlEYQe1R;?+lzxdmw8{&f;i&BWdRNyzmA*7*>j^UG4zdfeL)QQLm%c{HFeysGgZeSYLS*pUK_S+_R?7XA02WEb>dgX*8XPeim=Ektj zS}@Aa+qXMIlaQ}HP5FM1|2Ip6qJWJ*;PMnk)LFX<%RlB z)r40&jYR0{g+mnV+Sv3>y58a}gEfnX&!+bQB+#*POIle~)G#nLf&S90y(O2bHL)`RZb#>oRP{ZRjDBS|1G z)eJlFveur-%;vs@cbfA?hF?Ns+;Dxt&&LeNEz?YT!^_6g_a}965P2s0IbrqS{D#x- z#=odSF9B;FE$KGZ@nfiq0SdNEb^ninbJ*s^6Mj@h-Svzi)!&{N$Jn^~5EFRirq{(8 zrps_|$R2xl=T4$^^v_PQ8CDG%1rtou&#wXthsx=N=Kn|6x4=W4w*3!{L`f?hNT`$~ zkyMlhwMA5{Ldc;-PB|xpacE1=CWjCTqm61E%Q1(>s9oob4#;t|gHaQM!5Cv^{?|SB zdEWIt@BjDN&+hZ-vG)Gmzx%!p-|KsQuS)p~k_w3pg)!Cgj6;9nHn}Sj$UmK6*R`;= zQM0ugpds}^dC9&(?OQ!YiNJyvPtXDNOb|Du z#)+m&1d&|<=Kzs=@~-$$+}Ra#MWT1MqV@p54h6WrEZLA~Rh1=P3wQS8(bNzbP`$Dgml8Sz)pV?C29z@vbwW$u+fC66%JN7+sinJ>8HU?XuR4r{a3 z1MrobN|X8O9ID%i20uXxbCDs6$yN*5_c#!A;3cu}-%>=&;28h`<4_*M3)cgW>KS!i2=BAB!9uQuKcz zo_6}6_1mVgNkc&x##BM~lk^xAjpW2O2FGJGKFf<%sS_~6*zGjpg}7(^giWzw^8N~k zIaX51{oewG#^Mk~D0hJSIWb(k$CP7TCBNGEsmV0gB+{SGfikYPq;;hSH>5GPF(Obv zP-$=euX0S|gOzrLE0c4hzF2L!R&N`!PEVpw%Oih>P*maJ>qE2;!P@uK+Y5h~nqcYP zo44`8-gji800}_iw_7PXL=<<;F=jRtl263#q`kv>dqN8VNykXvpnJujfIa2q3n(l% zL9w%IPQ-a@2xN4FJY6ciK;2btb$9|*i`n>Hy~(qimXcvPQ8S{O&b&-#Sw~G&BiI<; z%|}LWml@tcEUulnx!c9&=Q@Zq)W-UlaSdN!yUbdacZzFQXg&8h_(9N4*{HBsz3f-P0)%b>kE4GW5qXYOiFTn%&pP5^Bivy!MAl)`q-LY&9Q zfSLbY_aVJ38yqss-ux{2wzvErdU0CbO@3-M0|S{v@MkHId$yM*Qd z=(Y2r+;h3}%I8@q3x+T8bNWCE<%04|NKdcO$L^5#V6XcYQ7OraZ*zxn_p2 zSUp@ognL|*y2JE>A_hG=lmHa0;E4O~DU9FYMB#pBzqXZ(N$BxD58i zfaI(#g2Mle4+0*>K(aXda=9Vaqt@vJ_ydy8GA8YeCH-Et?y9pAVw+8ev}3kuq{kb* zC1J;c_gIL~IQ z9{yTb8s#v(($1lgoAN!Vb=D<_^37Xg+Hgf~{7!0q7#TmC_4Y*_6dN`=;}Y2J^!`Tp zK0|WLg3lYOlE-QFV_i8yKaHQdknyZ*%g&&C)O51jP9xeS?YJ-Wr4obwqG4d+elvVL zb{As{b|A?8`9qQ??JU%Kt3jwkEtgYoWA`WqRQfL>@?p#M%$kEW_4d}4l-I?A!eYyz z;AG|lu6j*5vab4|`^A)%#bzfc zvSOcp`{6)LLc@s8K9Or#59pe5gG6eqR@hmm_VXrAou3Npzr~M)rIVb|P`)J{*^Zsa zK;)K%^7k7E^S|gV#_%DC;ZHm8S|iv`BGjq9u>oJ7VHI47t4zp?3XZjy9g^`lLLbr^r6-|IKFZRtutJ%T8Z{(z$&tc^Bo^o6yg(pfO;EI z=PDVdY8-q2@Xr!Pht3^~3#(h7UAv{AO`!)ojDD(%9D$Z^L(};YUZh>1)eZkc@MIsJ z&JM_zi2X4C4PkcI)z-PMW7#`h<~8}XsjM=?-P8IFRF`~%WgMJqr&QhlNN>o9c@XmB zH}`U($(OkX3_%A&G)xtDRQPue-E`y56b`CCBRFa&JbkRfg;s>Fm>aXne6}8+M&RPC zY3<=L1Nz6#sYX0w2`AbPh&F^Y;T~fhM#HvGO1X6|d1L`cdgQ?1>W6u=p)TPEi>vIF zw9_Gd$PUw!dfvaVIA&I&x0~Vs>HPgJ+RK_)R7m*GxOBn$=`*Jt1LUuHrzi?n@4n}T zDEFS-A6(p>Q{ix#>4*rmG^S8>?!`q>o3(d}%wI0^) z#g>G<^HO7vE~S=A79OnO94?KYfc%&@&IsCDLv60@HSB#F7Euzvo0${qL_SMw{GZDd zHWTto(E%O=JlV@tl;0Vz_)Yh7zlY=S;}t_gn-@Ol4_(9(&O=AYTjRG>tq<5Z#uJI) zkMlH0S}1kX{M6e|LjngEtAKj>Efe*1@^iLVZjsonOvccdn81R=XeXn$^ zLTdqW?t9Cx{Y%lH`o~BB*Mr-qBr@6!8(Ew<0+Z6Jy^Cgo-z(H92@mjt)Zsf5QrSjb z)-`*W1#LBud#DRc-66Lkbsjond=N6>(fEr5K2w#SIiNp+RiQv53Q!pxeC&kRGxw}Z z$4MouowO<}uqC<%MRY&L$H&tB!;Q>J=vW^Y-3*4^==t!}?CG|f@C{5B<`$pO9>I%@ zEz{}J53uq1v_aU){j_Y@5*U`d`h)(gKx2fbrToN%wyZx7g&(}0Iu?xG zx!A&qBI+<)27UM4J6*R&0?%aQwbFpEGhbftQOUO;U+P-I5CAXj=T)iQ;oQ=QIWiA= z))tHTvvs`g{XCAf71^_EAb{d`*v*GUi0Jel@*C#0PJ~kzbr(+(pECq-f=R#$~((g)_#|U^mCd33u3K^ zS4Iug<^V>2A18kd)+&ys>W#i8uZNG8@psM6FiinPU5CtAOXR!81k$I0A;)Yxoubls zLw!vWxm;dxR+5j*KD+>*hh71WsJ~rP;Lk|>;AQ+H>KF&Niv_=RtVEZU#*r(GZ!Dci zr^&!k$v*kNk_M}3$LWEA0Xx(kbuU`0_qUVon}doz>^61ORBK|*2)gOj)H*a|s;N(? z_p~O@o_GT74r;mFhEt7)pu!Go008CEuPdJdzx7UXMj+o5X3 z>hEwDZ^X|20A<4*bjoAPh{yce1J3rES#{pA1%2CcG0VJw~u+wot=>>8-H3D<4+ zjl@;KzU|G8keC@vimYEUUtty$S;p2DU@)4f$j0cWElCBzYjz@bGEttM49vJjR2xr>Im`F+!SzQ&W=AMZfAah**=W|N*2j* zStxhJ6%p=KMi5wbT`gwFzF*7 zZ=jBYZfG5!#phAVJw3o|r`Lub!~@L#(d1?)wl&Z$I`7|-R{BA+UV7wEsXu_Ci(Lc6 z3)#4Bji%9inGeISVKPp6(vxd=@AVc8`OlNRCIcGW=L|ZO!^Oc>>?|$b znJ0s7qS###XWFUmVQAbAk|@Mv*h5=IA_jite7e>kLC_ipVWzE;+a#F%k#U9>cWZ?v zp^f`Hs^FMcG@hN?xoQIQd`HyVLbfu0n6L;G3F>~T#=>4|ZWn9+MEeuC$ahP!dcZt| zOe{47s=+cRRsV?$Mt+-i^rpyv+F*2=kZF=|Dmv+L{0FyUbqPIOYoP2oA1K(v=N1q< z$946oZ4k1B$&x*_0nM5n;WY@785adw$>3@X>GCk|XL`MT-Tk5BC1Pw)^ktFx_pWSv z8t;5`qJSErGQvXSnQ=~ljyk4nJ4e&T_E~HMpTXzWssE{cmwDf>6X^Y@!(B5TP?CME zH(crT^qyx+j3?OWNC)=+=0t1jg=id>I3=joosCJjortQt<(^Sa%~PQWoUVD`WaQ{C z-0?-OAB zTR}cQtlb{=WP^+$@o%ITOV`cUb18ufu}>(g6^+a;Pb3YOGYFCc3G6{{MgPGX3bw;kCc^nr*RPjSr6!#z zyOK2#Tk-gKr)%#ieJWQ@WoGK{2YXnP9(_C#$K<49Dwm; zN}jg6yZ90rgWF@!yad3l_$;z3;#_+pLRnfW5}yO%38Z%>g32~Vc2&;WfplootLaO! zQ?C64{Wq+M7^Q4QmP4ID$Zouv4~>MWKXVCOXFU_)F@HT3gQPk>X!PY?jc;6JhAD(o zU{D?BkWG?bV)&yM2N5l!tR{}22-v=xK?B9w(95FXOBNMQfA-z~@&^m*^+JV*MsJwP z{`B;vN=6d)@sjsgyG_FB<0PiN2YEgfu1{KGL$te+#G;L`kUy8vs-0aifDw_1g8YZ0 zsRtjZ`UZ!%!CDfOCBgc5f1~rV9hw*L(fqj-{hI^bsEwIh;}=$?p4{E`aT>=;?AH?> zGPKzmF;e{?umv?1^&T|ncDDBo4Gp!0p2v9}0O{s(6o@`A!>0ZF7S7EHogD2$kuM_F zU?P{PSIF5xnF+sBB5WG-@-{Bx2{vz+V|k!X*1?Cjcv_3|yU7>pIoEIAKR{$~LcV;~ zj6m!-)WoHKbcul)_WK60A!9``g!wi!%N!=$#VRWsvGe`)u0RR3n5{EVwk5b0B3mm? zJ7)gZ?~ImtXuwr>S;W?=lP^0{EL9J+UorATq4Ynb>4;`H~iV0C6U|5^PR8d{J~%TSEcIwWGuo&yXWHPZ<=R(%bBk_fq9 z>}5SzXJwi0j_-6@{4%qLwdUM&>#uZ=)ZS;Gkj22A9yeZhI{LKD<86B|n1h*~k^U3w zl{UZdq)xJ;ll5d@WJDBu`tvWO}WvIMs>{0vN|G zKwt@K8o7e7`ors$e|GG2f00fPqqsFS>-0z%<0>PgomzvpOWdA@y)9E`KF6*vyMltm zCD8#y!OAEOkTh#2?ufb!;)*VH*P|cVTn?8a?&A6kzdz}@2e{52+?t%Unxx?W}Q)_14dryYQx@Orb*1ycH z_mLHMpMj&uP2>o>|I)jMDGeoCMS~D)E4@uMLNOB{0#4}shL@x#O$H)1zHun?_wNo+ z;@jwgdqm5#Nus^Br6Aj--?xmxK)CK5)r^9vmP2kRza&Z3* z_X{Ucl0-Arb(NjmQ3kN?)x%|56z$mt@53A_=8P=0jZ+3eKp;kU+${<8AN5W4!a zE}V|=uGBz2u@Ed?hdNC@JA-pXcs01r(~kb0oC=g*T$kuL+*ACim*>Ulfy>&1Bnv)F z=pLqvO&I?7m3S+)J9r|hnbEvLpg9XYPtUaWtf)5Z*cN6l8?Izj7T9pGOp)>tL@U%? zwj^gLstXM@{w@fKWI_4_NVn)1*kQeNKkPn3!;Ql>{tX6+L}T)tG{Eoc|V)M(nDakOek3Q^z9*8V^m6MRR(9ii|*m1V3L8$Xk4;3iux zjL|pIj60h!U4h1CZ$rXKvocD&Z5oLR*S+&O{+;x*~&kK;ct1D{4 z^x~Z={oV^A_3?Xv14NQd6s<>{3R}Y(pi3+A!A{Ix&>E-2_gjrBSD9krUUo&FD&6N! z*d<31$l$h5J>dFenSf7{^jCP2GBpq4zfd&CxIJ2C;_N~uEu0=wji>}IZFpCKgXsK#6lSFg}wXu&C}7p%e4&v*0Cqh{kYdm+txZ*TgI(+M)v}M+#F8P$D*hZ5C|9r{m zid_=q5A!pEIwTStn;F-Od1&M5*m>iH>b0NZ^8XeVdQ~*N0Rd?M#B@V;uIz(jSDy6}j73rC8DFua2Dq2Ib-l#!{Xx`#`| zHBU}Epu9V51}Z=HE`Yzmcr7;`Y6dMeLeA02raKHmk$Rr73L-qffBm;2fdrF|Ui7Ib zxAOOfZcU9if3AQI#nau#3_nuXc3wbI*>4IjUZLqj_k8hriC&C_6p*yddGQfPh_>-l zDrTfs7{mS8iysoaBZ!k)j-CvWf5U^#UQi;dm1*vaFVJ(&Ks4{=3NrI0x*d;B<%sPu z@s)X&cj-%}Hsd#bnm*yl|k-g%-s;m8Twq2uW*l-FtdE7gj+tQT>Zlb9iMKZa)lQf4u}K^7w1wd&MT zMCr-nzVfT4a{CjG~PjfzsT@b4%PBC}pMwQYlYI50b~ z((xi{kYyZN6zeXhJ`gTLzG^l=uVw0eJXQgX<>yVk7X`8m-QdHU zVYffG1{hnLYHt)y7IM_JdDK1AzjBS?Al2;_y8jJhs9!$sHS71+Xha3cb=kQItCs%5 z_=R<}^!d+(;uPWE(V?-hK+-_vk}J=^_@KpqNzWVA$9L^+3X;Pw&QCq@py(ODuv{=i zROCg@yp4>y^J2$VQ+Vee8>T?~XCW-2Knm-c33uH7!R#+ibUkJLs&djvjAL;*Cpe!R zQ@`1zKLA<-27_z|fBsI_fw;4>YcNz0(~?||usM3O1OH$h56vq@?u)u6RktkEIp8DZyS@e|{$Jmm!Na>QyirGgON1|WwGhlb7=2MBt=KlMUE0!L+1l@Ug z;a>`*dCweFHa~iEdg-H!NSJrFujc4SnD5?U@JC@VG+BfZbJe~+8*41v#^qyET*DO6 zNV>6>Fz4v@xSc>0QIbJmE+Z8mD9)>_z_aaT^bnGIj3SD;O-JAfEI9_Vb8}H_giRs~ zFbl^s)3`stR?iMRV1aZ$obSj(%X$@7?hrYRGz2{qGIkk-Ttu>Vd2_ccP?X@IV7AS1 z`ql1&)wP{~!OaZZj$ilU=v9@pG`x}U{g0-n>;B|=Tr{6o=5lBFjq+yj%3+tp{jcB_ zE{b1!8tD-|P$G%`Ncazw*Af`<7FpEh1sROSr9zRIij9ld#+-7$I2QcO=F+ag_%1zB zIMW9*7P*84Z`3vP_e1iWelre;l-2mn$<0rtFSXnqP)XSJ`SK;H3+c=f!Dmk$>cz^H z1J{IuMh?ymJaBsO=7m^EyQRT{LpgeBUY@sfAn4+i9pTSEtBz`Jfd0A3kP`27$1kX7 zhuAlhj_zC@z&nKt-m8Ibjtu9)McR`C5p3{ipMq>7DMxj5biUR|_*~AIu9{wSX{ZaC zDB&!+5E-QznVFdu+(q1J9QjZQZj`$o|SfX%cYH+IX!bjsM^1kkS#fdz-!;0+E)zf~PRT7eT_To0*XzaMIL)MO%a(rX$(Edq z?AHo<_{FgruiAaLJj~~J+Zu$vh2c8E#nR3u21LsoMTZH~MH!)HYJ#w~%AH5q&bR#F z(6KxMv!*&Ui-6jjPXtCkrm?6TbCcD=)JE6KqALjFxau4{3*K*Soq3wfbdoQObGk2> z^nzCVOjj?Rc9ogof@6h^^j*7|Qc9riwatzg7-C(b+24?y%%wXL`GwuhmG+|c)p5aO ziTUTXhXWw@>jdkvog0A031Ze7{Ijs4VWSe3a9mHtd5=+*nu=E9rh@es)5%}0>_3t3 zatbH+iOl!_BMNRq-0gk!F953h7-Va6U-&|mX-+ml81H5LxHvyeQZr)?gT8>nG^-d^>ygqcWN*+3^vfyANL5jP1dB70+R)g@o zc#Q%)1SO$y(v+Kvykd8LEu^$C_Vq7^ z1@FHt5&DO|gSoFG*a!}skT0DUCTD-cbSL%eV}-^^)sTIkd6nRbA4+R^}Vs)0OvTYefg(|jA&WisN(w&`f8hYUW_~RdgJqq z==CaE`{x|bL_T&K=k?xj^kw1GqAgWkeR{31XZY~Gg(?&`ASpYDjy^1YbGUc#q69?n zJr_A_Ys0d0t1#EV7G^>eGvs6wg1hQwkZ8@QMj=XX@lBV0tObYWUuxegk#UfC#mGPN zUBF)5S_=Y&Sc^-fP zWib@;rH41$JkCqac|lq_-9o%Qw^D7Y2yp+rI(c{vEtP~%19;R;K=Gn7NARi9+@3v- zP}`L=Zb(;oezlhEi7KKN@L??lR(1YTNZf8}PS;3C8X^oJljYRi5^x>`tQH`7EgV{i z{dw+n(S7~lSO&am)}diuIxUWQNjdEHjB3CDdG=j3zR!PaGuOOMt~Z5NB$a@>P}S~? zLuZKh%RjF~q+W8gl!{Vf;c=bDZS}tlIF3y4E@ozAWPrnEO}@dcu~NOo-Y(LL;I8-x zey;xT_kc^%pCER@*^btq-;J}yt1(3yAQSZsLxgn?{-POO>f!?!ItD3S$lCM1nq^26 zH2-!Tk=aQOp@p3m{q&?^#w+GXpkobj-!%WgNDd<=9 zf;Bh}Yd7(o*3-q|aF*kZo>`5~M*fS+Mm^jIl}&-iZ0YRAcY)YmfhJp|)AF%lcargv-DlPM@NcN~POfW>(6iYu?th&~h|#NNtQ8p{8Ey~fT0Zhof)#fzOuzR_Nj zGUOJ#k9-%UTAh;=)i0jI$I32|ONf$n_t~{rm6|w`ZyNXuQ1s`TP~htP=h5CMW@aSx z9z}+E;X=Yu;*Bb|jLlyuMJX-C;OpMqomA;h-ZZ?PB-T;gFM58yXv2UotqNCn-Og0# zO_&4y)(7?h9p3qD)K$P^A0p7z!OaqOGUH_G#Lw_v$9_Z|DVXBsSC3~M?_-&(h9M-H zlj}QjDyuOA_yl{Rn#)w8Xx4S_ZWOT$<4i`K{N;wLt}zePo{S^(O9pHO2LZ|Pe}!il ze!?4UkGT1;nz`fQs3$$9jx1M^#4L1vxutc6(JfXm5tC7Yse2$CSSUTp7W72*<>d*A zc6@q`TGPxa>a?#6dM3f-W>=o72 zg-#U<3p5(nB2*}8MpdX;B{@lWlgbN!xH2~7>e(M9E?P#4%wJV0NaQG#DK+|3QYuiy z$cY9Z9pkVw%2IHk2sA276CVl_ErN+~ASa)d#C`gv{bWlAe1J8S&smLauawAjw*0@+ zM!0bqlGX}WK2O`5^1Rbsf4F3#;IFWuU&29IXcT^>@0n!~i546rF|%_T4&nuIg`^)G zcHFWwANy^~iwymdBzTB@;<2{*x8CTi6*ICI#4^af=7ncl;`j(}hyNd}a;kJ!Evl&y zEgacinss;--MT;@6(%eTj==98LH&vP8>nX?b0AtaZnYDW%>-PC#v$RuvSz>3@-O$} z_M*JGT=IF~RwtwB+S7wuTBpS2kY*^jtM|eiImUk{Jw9oMjfWDP%uL zXYFSLNcy<=Qg~>6sl20bYp@Bd%yX>mT7Fd@uKn+|M&|Wy`b!{*2Nrk@SLAANJbwYv zstWTKmN+bV;h59n!OjW|-*XkI(r&*>54W>c>t8t|bq89R+PUx<9iMCZJR&)ZJZLOD z>eC}}q5YL52?bN?^l~w*C!Cm`3u0%4K#Of>(*^J-#VPc60`tGZFzRh6`6ck?xgav3pu$>IOPl1e)WZkl^Q{>E#34TncYT@e~rh#-g0$u zkS{-PT@)d9r{z#RWW|`m=0yYH(zY7nyCqhIl?9i^UnKAYEIRpI^hcKCG~2jx64CDf zyxR`y1K?bQ5GlW+W4wndfeDVcj6Q>=HhL)BSy}chg)TwJ=hTKY`1FwH%4xk0NmEx@ zRbRsf?f5f`*XpKOgAk$Zp)Pb^B*zfMevauzfv-g;Wcw`0bzUK9zXc|K8!YU?GVFhd zR2|i#;>ffy+dVmJ-Z5r&>mD(0-5Xa2Q|ZCzFD-$)45kI=T)YenKGhGko|mg9_t@xL zIt{ zVJu(mJ>=iT^e42@PSp8S>}fnb!HSmR?`fn6!ur{;`3>Da1Y1uioQUvIqfPxChQiU* zUH&n4ny2lGlJ1fBR20EXmPmhZJ!EiBa2sEq-Zp~$d8c+dr8RiJ!KDZrx5E)J-r4?h z@h8OzSLCut2dI!p&)*1X-(~P?VbeT{mL7v&)mOzT3pt$h^?#(<-m-AE>(3JVE@Q=M z*Jf^M&1_Z+;fJsVE8(O%%OamD-s@|bp=z6nTmBgAlW`{7NmUY7eA3I9~Gb`_@fDvCk#koYa5qE_IJQ$mirB@PW=rms|j zp1KeP#_w`nXckuI)Yf=tM$F%`I4EHxe}k{tcp~nmcL$`G+^^p80Ka~YXC2@vRHHtr zg#}tEEPM0ef7W0yYs|$@U0b0dohKA|JtwjgGD-NE#$C)uVnp+cY;b(YtmcIq3=p?+ z7Rwb~U}KMnSpu8~{%KlXMqSBK2or4WDP-I?e%*G2tsq!t(+#UGTdYz*{@$bGdl{lOI{tU=(e=%|=Y`yoO+m!0SYAzuKJLNC{2#O$NF7ePrM zuLX)7m5+=jHpLpp2ykF{2m&vGAsU8zC~xoH_~N%8m&r+FXuE?jR2xER~s zqI7_~OqMb7r>`4-9rG57WYIUi`nGcCvll>1#6)!3ys&U+kS}|$c2&2Fx{WEy4!r`2 z)?$^eBqG;<^Eig!z}1HLqzb3ls&5v;#3US79w5#~0fR0O< zL6&2XrSSSuf<4~2;Y4co3vlIQ;1M;hiJkLy$(lq3#g(NK|JjvvJ;D_BO4 zlx&y@&XBtB(C>z4RBp_Dp8kU931tm-v1UXP-FnX5B#x@4a=WX85D?rV$=hJFrDORARGc%s=pr>t zmIHu-d!qJ$=lUT8hh{=7z~Ob4n_PlkF^*$WuhQ70$AG|<=H)^3GhU-c>RWY=cc4pW zKC^&2g3BWP(bdfwRHYeq#l-B2h;Pf6>!T^?-hUE@n@oOObNNZ~d+|A(WZ# z-@g~+ef!&dg2l66w%tP**z5tWliG7((&d~Xd-3zR8vk;HxG@qBA{ynu-eN+6W=rjS zyV>*cEQ&^@_M#8rhX!z3l!L)Q@sZ$&&rwf-?rA#TByzfAB;+Z|?cj>m-WwW@Z`KC? z-f*z?WVBR(gAcG&b_>#qx7O;oxmoTEAP4AeiG4pt*oc`6oMfY_m%G;`g^+gD`Vsi= zP0B3uE2K?Ems7ecFmlhlHT=ctfh_QQW8E(|U6^PIK2%?+lG9B4wo=8lZx0e;p2QN;H-GK2js?U=C7UG^y0`Q;o0(imsh!>R?KAGa@Rp+m}OQW!p zdOiVuNv)k{z3W%nGm0dpb}x%%nYAg95c8##`PG{C7YjN))BPh%6MXXNH+)h8M>0$u z)C^vT70A~V^gVL`&DvT+;RfcS0Y&J=L~EHME4@F+CXp!^xV7W}@ym`w_BIvmDI1bM z<(3Mj;hZ8wl_bA)X^WCFq9A$vCsXb1Cl7KtEA{5rIfxYCV}SSAO4MVB!#L=UQb%*Y>#8k z6i$Uf9*J>SRvgE~njJP`qbBHYG?EL3XN=Z^A?GIek9Nkr6QrC(v!_$OwwjV?*a?|s z`&b{YTy0T#{pyf!G(c^BU0|~x1MH7GSvI_+6+eF^1f8GU@D(Zd%k#y9W zGWVg5tbKj#LtPz|N|vmxGCbjxUye=pccQ{Hr1QMIPW?IZE{*hUEunHc4vlm2hV3=j z-3q;*yV->VO>79y5h8kmPn~i5&Df*;bze_p)v>=(`DTdMnV*L+HNjW+T^^=ke0I6X zZ2r}|W6H!M2OKEXrx#l-1{q*9pb(E_LEu1$`k8#jaN&ZL2L)>@(2ef zaMynqq1I0Ew}k4~W(VEg%SaztY@MZ-Ze2Jc?32iug5nOsLN52i>CjhLW&<#Ur(2#c znv~UfV3?)!BgB+Vf8KNc-v8zfc^*zA!%vbxs_#$^`GAO3trw`f3`A`lTar%d!m2!)^(4yvjxENzKK*QKwaOHwVYSxKb=G9h!idTH|G@|`?L{#`E*I863d&%sfQ`$AhA)hu^&=7f|!$C?59oPUMO@P3?7 zuOe6Qh}nKRuumtJMU9xDq*_bI@8!jHmQA#atJ5UTivRxlLiX449vO~b!d~80n0W1SqQmN_k*e4P;va(fo4x%r2z+dmQ^|{DP z&fjR1%|Np^Ei3@he;;H!R2u8WJxcRzrxX>~UfEh2D4e8oim=s)Ui+8naBy4rK~|5Y z4vJhiOD#K}m>ogSkGra1rfjkUC?k&q0C6uScCc_^EwDQDSSZ3Sb8dIMCJ`X@dJe*n)Z zHG^icbMkkwyOcF7holLQQ<#QmK^Ok6CR8?Q&mj(oAZxUd{PB$0`RGTH#UER=znH}y z*1m%Del5yDHWv(;=_M3pTcA4v*?CG+o+O!6PD@x&TO|aVa-A7?4SNEpRTTC#+0{o_1#WrO&R{$V#t9QL% zSqrs<;7K8k#jzuJnJUWgD;6?sm=oLoDZC+AP2^8#Fh(Al<_Bz$n1rEr>=_l`W-H@$ zCehZn5z3#Jhg<|;LF)9C@~)4ld;1WyMeJKpt&$H~k+ObK8H(j&CebHv_r{CE$(i(X zN@tz(bsYUE@@?xYMd~jvrPhh+wH|3h<@oV#yRn#%b-^_!M0eAwqI3;07Sz>e*0K;& z3qL=ZxlQJVP!GZr^SJEedj11vH4KA7_lhj6U;GGI z7ZI2wfUzb(4<#iME2OqCI z8ZT(xvl?H$8nOQ@I?=uCBqjv%<7TFiW0R(- z?o3E^caB~{$PYu44TB8a(W6K6J@_|P5d;I2Jrtfdb!#J{m6W4c#LsXd^N`~nUB}m> z#!(M-k%nht1`tKUZCSJK(y~u&GiWXpeR_RG()o)rw-puB(Lf#XHIM zb!^<=XkndI6okeYeXHn1O+x=3rUJAm-MLxW%GR8jechGoPyyE+N`no5K|9O>d+rGw zz#<0WR`FqU4d7$OVeeMr)ZsTke26AHFMCM4b2{8%j7rLz2f|4*C3;7 z4V;1x64{%E`{E%nJ^#7KPXbf>hqe?Z&M-RhHqMmwu+7kAaD{fp%XaFO{*Mc@l0@?b z3G0~AfK|_LlauJ(5cU0O$m7i1Em_<8$|WqFCf~4&^(bDTIMX4z+@zs3#U2+vhSmW` zybG-3!9uxbVi#!L=86lsk%KQ+dY0|Em5Bg^{JE$uCpE!jj@o_97O@|%b^_XH*NM*( zpAQh*rLZNcdD2hp;x+lLmiXeYsX_#0p%#U${9wFe#s|h$&H}rqr^h;lgKAvM8H$39 z(f=s*yu*X!Dy~utr7|#vmBt2LD7S)`@Tf|dZdc?TR?F-((k=2(NX{4vse4uthU&MS zxUmHU8FC_;HfAwXH@9tYdUCej6VIv7n`VX z3LkO8LH*YLhhX$+eU&ZqKYtghaZ06aonCzVx$#~|Q5zQp9uJlW9%w1qrwc`YdrIP5 zjNYq#4z+zj!DA~`DsNDrWCIDDer}fwkB@c?mf+d25xaY6 z1J1N7zYYlnGAhc9vX9QFQwc2l#)8y}oz%Br`U-mE*E87{bNTrSaAHZDm$7}&8qgv( zYQBa5{JFVNx?h?g6r0>A_N!6C_lr5}d=GE7GSjTPPu07Xv%cV11rsWcm^kZELaCSC z=&UARq}KbYkbm@FLrf*NPa$*A$Zq=LXb0g^;~;18-t5dFXWg{Gap;Yis5$GfjmpNN zK_LI%RcuIW+gvX7#|Mw08-i&-ow(gPoJskr_)%>ePXTfjIMIXH<$#%x@L<-JBUMt{5LZJ~_7wi{y_ z!&Nc5l8JtlmrdcQ?JSFdHYM%<_%teo8~0hbJ>2h^>+4yhP>5Z-x+GUp%u=W8%_jh3 zOcHirCfR6mN}hl#!Kje?5QZaIrK{g`V59CL*pgLZyG4lSgj?_RMZCMAw}ygj33J>t zR(TJd5edhNln+7$Mumqr@?kcI00NauFPw`kkd!EQ!6EsrPBDk7ieBwSH7H6C_i3w zVzoJxTdIEV-L%Sk{79o;Q%Dk3s}ayTxo8}KUrKjo3Wa_yG$M5&SK|SOniy_Rr5J#2k7A6S|(aR6nd%N^wFitT4 z?Bx&HZTmgFE_I{?mSZb~xku+54e8ttB9>6MF>>&RUY`0u_Amg$zvrPCisqnqT^L$5 zF#7);#@u@aArrkuf-zN3`2A}SH%5MbAc{$oLvSSbw~ z;O@0mO{(ok7kyo9{mV>(K(E6^eno>)_sa~5UE$mloNdGTr6RL&J6@If!Qrf6DU08I z!Ve3s#y|fy_*=^H+~N(FZtXX5od3JLWn|X!cs22vnXhV{==_=i$L%7&#h-{S`RGHR zqf`r7sE1Pwce*L}r4k1vh)wtjCrqLHP7Dwo4`4d#V*5T8ySd~Q$p%thu)hwi~W^e_Yg%`zGPqLj*`ELMFv9*5Sf zzaDE)Ke;v3aUR(V6~#r1YN2NX`k%U$jB;#O9Tj=3Q;?zX*zp<+@w|33F=pfQz~_pi zAiaAyL5Hx~X65bQQl_R~#&5#FX(u4)GB2Fun{QB{-b_XX0E_6t&8e2OgCfbtkNVf4 z86DvA)7v<2f=3ns`&^8@TP*|%$_FG*HoTXy`qm9Dh17bE$1@E-jQrfP2dElHkpwU* zcJvPp4qB=_Z}a;%Q8$sPMH>$tS-6IP*-mebKh1Jp`11CN?S6-kHm=*_s~$m-zSB;r ztRXk9+9mryu3IHxLOX&m5x9r5aFgU}Vzvl?{|uBNjn+NEhyl|8>x$U_kZFqllPflu zU+!n%%wzqHC;NQ-Haq&-Ifyhr!hTX9>iSxfp6Zk|TUWqwfe4n7gugmkQ;I|M=Gy=B zpx~A3!wZd~&S+mzS<~0c=2)sIBDbP5N??{M?JnM;ailEz&43~;RoLvjW3{c@5h3A4 zo5R7UBd5E^Rw^B}mwXbs^`hWfGVBd+-xyfY`_V@4Q?LxCH_`9L!WLj(=EqVs8Hc+_ zS`EKKNrDsi=9AL(}Rz?$}p2h{yUoV z)feCMmZ!V(Br0=Dja*e@rG)byl{l`qj&X=NV!jmy00x`OB$w__?{78i9nAF76UNm+)6-TusGWI=i{8A>*Blh5Y1+MMU3Jl2zfpo)a*y7qX=>xUhQUEjF_)>;f%?>RH1CUbk^y}$(L=e zIRr$TAhRytv3bO5)}4BJ$rRcL2Qj!UF^~ zgO}}<6q62b@f_vPcBBdITIuH*bwT1^j+#a;<+GC;owg`I;M7ukYBZu@WAgm{0#l{5 zs{$Ux`fKGCagb_V39l5=j(wR)9#tQjp9vp)rjRM`VQL9yQ0ranf0W+`J)i~A^bsV^nIX*ePegC(*U&4x5musDXp5% zWimexA4wkSzeJEW(%Z)leWs6(t|>WtIoxs4hE9+Vh4}^zN(ScXjoK9WQ8zzXc!6|+&rn%~FAxGr>NYZ=Fxwfw zll9WOp4$~|5mo6qzIe!fW#rV?mAe%lK`@!0hctWg=Au1iV%pZ1Yx+E3zk4ZC_-4PL zs5MpKP*y<{Tdc}pDZGwzkxQV7PiGi3hzT%RJ<%Pu47wF>P*983L8fC~{J-~h2qS&y z)@#evcb}O}EjbvR(u?C}(;CC9thy6m_bhQ6RkX8?Q=L*GA1!8x8GEa3PP%MVcVY5S ztd%DVWr`a1Rb(H~>Y#XiJ<(t-c4`Y}QRMB1-dBo3`J{R6x+u%nT0ZH4H8Yjf^dPYx z+qD(zn?l+Z>5tZ!Ojub51@XaDM;0%enD_T)D&9ID_1aQ6==`L%$i4WJJREuzi$KKc z>hzwR{XD`*QT)w`6MFC9=P&^{0kNlZte(Q#1kNS#`J?UGc4DKRJAHN+!|*tNA<%pA%8sE4tnFQTFf*vvlw; zcJ?9X@NfZrn|!itS`I-~vCWia4DY2iW2o0qCQz=fsC%JtqoNLcCku|^mRnJM$%@fS zl(oqq@vwNL=w~^;;z@vR<@Q^jm;pH&N5gJSOr>%H#2aWti8b9FoKCwURp91IWy_@( zYabGGhH@aZ(?BWN!FN^RJxOf-n_!nL9Q|ye$geiaH?IfYjQuDXftrDMA)ee_yfY2C zG7(knkOk=Tm*f3tr5bM04=}tl;zG*1tID&j@!-`U1~2RtjdJa?(XY4N8LT4Ia)P0r zV;0(*89sJCT3Aqo?pMwhDHI7?!{x2PgvkrFM5Kf~oS|v*J~;ffLz9xKPkL<9D7KX9g7OAFsb8+W4e%=>OGq?SW8cZTvN+%dpg9Tg662 zbT#Ng7o+TUp|I6Rl+j8m*zzOs?Kk*;oTx+X|yIp1d%@d!ot)Ruf&n+S)uHxnO%XVTVpY@GwW z?)4~M+mL9tZ}*L`1Eb8k?Z=`QJtMiScA3dUm*ef|5-rc;n}y!a8eR*k9BV8L-q*+7 z@^3%lgf4mcpqBD=XlY!iCJK>jj=qyH-{j9h3;1d4S+w9!h(i%bVhNIU|J&V4xXivvZ(v`N{s-h`}njZI-$R1{~sMU)iZ=9U{P$z5@G`5CDJI!R0igQF` zpDV9|Twz=lxA_D0pN14w!p z1hZngBqSQ>-QM$2cZB-f!;$NUDd{`Af5>g|+o-qx|L}nya*8+XFdDlGtNJ;T)k1&(3DdLuj$^9E~Dw zmNclSi#-YzT>I%tFcKrB-k$t+=PP)vM77b6Q7YA5(dl}Bg~F&^GiK+_PM2M@0NOX# zyTh0D)te2h&gbw4Wddm%<80v8n~2?pHGO`=06yFU>j7oIAXz=&0BH*kF;leG6z+{) zZfjjw0ztpa(V;$XgqNyOzZGG35#mRYlyJVeM=QGe=eiITtYJ}~;)2AUbvICw;Vqx< zzV3-IEh}|{aE^XPsW9g*&ww2-9oR+=1Xe|hskB~X*>AA-frVd56}lxB5T4`hNX3sj zrTQn*YSVJ29f%d|<2`a!J+J-t1S}}CSkdUppae|C2asH6pXt0CT7UgyEcHnJadJyf z&X-ri^xfQ<7O@)puB0YNLw)7r^8PWJYQxf&JU{2Vfg~7!zS%^c zWIbE3V5(Y=2O+-0j&MH6z#+ivCY{zjdCEg{VY!V$fl;(mzlZ`c==l*}!vV;$d%EVk z*MH@;Xfo1)n8t8ppC72LId?oSiER?~By2@R%~8+8g(dZ#|5SII)R)uLHcf7a{2R(2 z-B2!G}|E=EG}E41d!*e^wSolSpZOZUJ_Z&R4I8x+iZ8a3xu7 zH#BHiDyba){&B#%C)P6OwT~PR?aBkrYJR%ONu;3@Qv?2zuJxgoUzI)|Z4R4+( zh^-D1>gR@qr5owG9bzDd^$~;4jmZZQMq@`2Y8{-aD#T-Of z=B|Q`63*k}xTId)`&!BR7bq%I`>tV+EB1)q+*X}$^(quk0C|(u_>4WjZR! zQ3+Ugiiveu%<|;b^tx-)`BDSn;=+G$v;hArQcTEfRm?7)0Z&{>rJgLjm%f=v?o%Tb zm%1=}y|i|(Ri1)t|1O9cJSx7FoAEmD?@HO4K9104np#Ct(n#Ab4jSpuAut4=4i~;Z zVDEeSpY5T`3ZoGW$4u;nhAW*ZTWP=QBVK4b;zPBpO~l?5;6aaYLVzOL=6z z$@9O;JVorL=@(O1W2wt$`Wyc=__By*mvA)RPXGX7s&8K>N?CZzNmouTNFG9Z*&&Sv z!{WLOEXJWqk|+B?hKLcVRA%S+f?v;<3$AilxoHk6U`)aE$kn&(-?A-vb)jiGF*kY- zL>9kNZNp#EX~ipKMg2h`s4ljG@%w0s(uQ@CqIcbtJWaVy0E_M79ftb|l^*0SSMR#o zZxqL*oO;27u<%=M>h{YO-~QsEHXp1QO4*Azd>ug0^XKXUW(c!d@qY|~_$(_6Ho6#) zYd8)m%j`!rulb>^26P?@@7FX&nQu1y6Q*cQqX=S%l@TS;c5GKVWh*80RyLrX-ve~? z;rKu)RdIG0!Y@DpH2S-dgS-3>EXf>n^}e#akZyLwUBlHzNdp(}mHkrQuUu?|!XJ>` z!^n(!!`A+d`LZ2ovAsTHWZ;XDnUXy z#o{hyRtickCixM6WrSZje!G|<3Bq*Qeth#u>#t7PdW7|2N(>B-$xnM)u;cX~y4ghC5Y=YdMPaz_ zbPMeMjoJ*rtE1&-#ZK_y( zW<|EAnQXsXqU|4W=%&|jIO)ex(YvNG3W`W;yC zkg_k}zCqw_F{f(@Zb2GEKtQP`{MQdL1<=it*kSvp)rGwg^)nK8kl|DH9g zG?{RVAMITe{M6Kk@JNa_o#v>EOUkE?&mYE)agQSP^zOS}93YwP=Iks?o*k=Ax4a93Sry zb^Y~aLgI<68LbzwKZq8&Gvkf1HUXE$}lT68H&>*wpAJPv9m&V)x`!H}BFh zX@H$tht5?TN`gn8Q9-IY2h2R~y z`vBQG-&BRtMHU8db8IF$de<~k5ZIuG!hh#xcvrL=bw2dXc<(fNJ$ou7C0d6Wi--dA z4MMGy#n?AcbFHeoKKxMUA|HA1^gYc%*BVo>HgFj`^SAn+?fXry9SgT7Hz@Nlu@xDU zup#{mUvFm7v(wYPAi&D#4hwxP)(X~tC1xx|D06_IO|Vuww1yI!BC6_8qOwc9YW*)- zLxD<$ERWq36q=9Yk&dxR8swL_0 zj}p+_f(qWEh=eaTvps`&zjM|u`*ixPY)IJ-{FS1uzWu)7PZ@8Ti8cA6H5h%<=hxKh z(5TpEivwYLEGL-%ZS0#`O*OL$|8xJXD?9%3^`1bqDX9e3dQW4!#J**ajIkODo2!*Jo11@dD zp&JoDMS| zMAyj2#d^dmWF!&XWLo>2@@&mvc7nV3IsuE4K!~`q$*NPwUj{m85GO1j?@hs&ATyhb z&~zX8@gA0gV8&7wexb?qL!hp~brj(SPh+n*&)Y;($u|`VSS%uhiKIOy^xZzYhtY_` zb2^-iOhw4SMu^)0s}4}tJBFB9l?=sKSWKZq;h)H3DCf@}=H)Pu-NGuq9={&?r-{3P z-AE;6+gE(`#eZ>^*(TL%F|o9;+4#WnbvDAe5tC3+2&PZdc-`)f5Xm>G1#sk2sZfll zl!zk-&!jUidG->;kUjH5MMA1{IH}*a&l>AFmN98Y$kFNn6f2?u=t7W=zJA(Z0J&QM z-~@#r?FCnHcgDE==e@HWoR1^*8JG9)zB`+ood^qJ|3WwAP%8m24e&~<`)QL*uQx9K z>Bm_7Fy!Lkhb2*l&iWWjWMLM6yzZ@JZHy|5(hEytBfg#CqeutO9$Jsh__eWGh&X}N zqo|%6;x7MIZLP^v%}67aHj?AvJ!-#>VLEj$5qndVA{ZG)#R*xI|>{K`NR1sbkvrq(|&P6C79`oz|3~@{EJ)@>}aLG=9+DA?|-pavWt z*%PJ&F<%ZDm_7}(4{(p+-(0t}x6L!~b;4Zfm`T85&W;C~ z-axp?TX^{&^}C?yvH+wB!%p8x*+_6qqmVN7-3`kiA*m#)1G*~>2V`(jA}eDw1nEt2 z-=Gw}xKRaI+UJ_ex~FNMbv_5IZsXdN)#zu#+OnCC`Uz_GD~H4V3gHA-*Q+ z`PlfQnpN|4D=J35UzFgf^{h;^^e1witw;Yubh_`zn%jR}FWP)YcYaS^2JAzA8#tM- zcPeaJ;!^KNi;JGqx}I3gKsc(Z;~wg+G1Y{Ge3jB&{t@OSVTa2&-6F9}-YvqP68k?f zk=;4ZbleG9!kJU(M)=3-D)3QXaF3}Pp?or6aG^m~S^p;SzguuiW_=ZHmW73LdS)X? z12qdWU7TU@14ZoiZWBB;-Zkh%yY~sK8V(EBVAUkchGHutj~au{p&uWgSn!iopRU<6 zOP-GI(wE4Y0fgNz<9E-{X`S0n0bXiNv1c%E%?_o~8q7_ZF-x=Esk1tMda)X_8$}T~ zyaizTNJojwh4Zgd>hrpVo>GdU@=(I4BM{tz=nysQu7=q61Rc%;TJDb|g)m zTqYV;6i7OiGc`dthB4-y7HOxhaUpmOYk%g>IN`4@X(4M)@qkfvl8Z5GdBiP&)R?@m z`gAq>H=8%OlHMEqEaO~|%GWVF?02Pi^W4}!tF)4N5ETjIb|bdk(q<%AOeY; zp^&ZVbwJZ&Nu?rxxFJN(X0A#wyWKN1B}w3`Bm%)4{9`Fs2(?OA2Y2{5f~(&ryYm#-NVs2w zf3n+W@-oVYv@wi`L0(0OJMTH{QEJzzr+3Q(eU}Tm$4Zs_>JxAgTvybLs}x3i-_`&+ zUx~9u3&$Pv%)(CtQ?b@cTltY-S0)IEFs5Oli!zMiZDT)0vD+2@SmTOl<^j0g3}<7l z!1y7Ek{6;U|1$@a8$gB0Y6dE_9(~ZaROY7ggf@>a##4d_E(xM{30kA!hgton+(DIrmsSH4Oa&Bbf+ z+(S65QY^VFZ%3I)lnEGXEW7c@BW%j~)NC&eXL)anFct^r-{LCeqo(|7zBFCF2dQoV zQp^9vAuRA_SEPAyI0HnK0mkmYyk5yDcSb^6l{RFYA}3xEe$CJGopLJXzwZ{aS~lK) z=^<&Qh6M%&0$pSz7%aigtZCm~g0l*cOBP3bmp664EO$8YAMA&w-w>YiO@Ks(tgF=L z(5{g)xOJq0)qE9lX$046RCdMkqEB4pBX1y0b+mnn6uCjBYtANHbiv7*#^*>%;g3Tm zw${S>G4+53(U=P5Mz))FG^yW2*4yuXSK%@8r2(S&mggAAWI0QP<%ly+tyJO-KP1Zk zPZqjpvC1xz;~IoT%HL`iC%CVKPrOp5?^iV3P7t(iid#@ITP0qw@frnXx0&x0n-xk1 z)rH8#vd2g|og2T)6ldpW4>6h(JzJ(=%03fk!t>n?<9wbM6XYrVcHu%-zN()3qHT;o> zZ&lA(HUuFMgR)J8rew`Fez2AGPW*7j)v&iJM`-C;5C>I}Zl5|9zN|?nby0PE|6>Yp zY=v`WT|^@#4FaL&8;|I*;Ta)poS*Oi@f8?}KAQvH!S;G}QVxayf?tUp zJYWIJrr|D~xjcorPkLd{^WHFn z*#v&l3XHh{Th+7cdAa-m<~MRF9gk(X=$Zv?jpUMhmn1w}-$ZG6KlJI@KEA3cU>Tdf zLpe};>@CNhTn7v7LPdgrE?uYoaht}I9!{Us-WK-d8qKbt*{Yl+vPiTO%bChOTH6R_ z$biD-B3U&&|H17)MLQ=Qd*PHD?!$gzxlG^=PBzNy#8qXH*!zgzi)16yr4aPQISA;! z5rrJ%!WZeAdaTH^%Y2*e;oG!Uwf05q7Q^0(t9v@Cf9IUUL%Z6@#ga?W{5q5_Q?@4A z_=mMUg`>R>!5ZCcnEke>*2pjvoZFjRTno@(SyH*Xx|V!9dNu@$9Xm9l+89&9vUmHS z6L77f-g#s*)cCF|R{hNqr2+2$!3`~>wK&$I#$&IsS&5!6Q66$(p$EXtO6m>sh>J|( zx|O@O)8{~&$i6A9Po+qNq7MLPo%Y;zfQDi;=|vtp9_oo3HBYwu1fm#Y0O-JYH$O;?GsJ?Tce3vnw1t{Q^9Ae3C`&I4*@ z7w^}#xxA_Na?4x}s^C=CD#|`|B&Z!9Td}^h@7c*dKdEGJUg0s)U0h%pFQTR6`|pH& zfJmP9%(-?zOSl>fFh#@+oNCycFLvn! z*w9Ml)hkNybX@pCgJNqUjHSd~J?hfCX6x9THSL8^#|wAsKaIN|6FG(1l9(UgKX9v{sMgf0%NCXu|%KLe>VG zF4*RTl|8_Pe7jU%-{p9=5l!Hcz|w{zHl^)Mc*a-{P-p`klwUXiLLPQnfA_KzlA5f-y&}Vo)7;kJa|3pD6 z(bQ*AX0**^xHMJ*nxDS+kMYN-C0Kz|S*>#0LS9H24Rg6l(tqd{WO{G~4m|mx941B% zo2dF|o3C{>#qb`n1i5z9N@WUn->AHvFAY)LKEy1@nsH05;Budy)p=#)V(ddxWYKcV z5U(sD6_quW?ZB&Xs&C4FXu)q-%bK>W%qHwKq9}mg7%h8v6hKUk+(DHl+tu%di9b+c z0U;UT)rYT+)V8(X?%uQNj-4Z3xgJZwiuRR%SP|B-@%9DdMH_lYE>;Pz7qoSY2uW#7 z&^r~a3Ise{ZHT+)Ej$M`s8aIky<90G0lDDHqyPv(bRuP&D1XdqFskTA*@WR4Ztoi4 z^KE(fMY(;pY31k@|ihX!>2x)|Ltl`zqS|Nx(1= z9{uOyFJbMy zX+7VIHTIQ>ULAWlI8X&rrATWxXL?Fqq#rAAWzPd?_*ty=@ZPEPMxC&v z)K4ll67++&Ik-qspTv%oj)c4$Nb2?{oM}FqYvTcYeoEsN+!t8!zBed3OwipIJJ|E? zg)abpJ1Y=ntZ5&kb}Z6y{F`A57+f$1O99on1^0)fo~=r(JQ9%XghIlHW-x&T@NCse z8QqpiXD0znU^i*KBN-&mK1~ixNF}(z<)}iXdABm61JzKtQ)pgD&WD~bukmC>j1>Zr zjKjL8uNn|$quvE7_5nD|d)fvB0J_Z`QEHIl#UK2mCZbSdVO-#9=W|S0GcnHtS}c@0 z=nVs@dzDWrBh1e({!k8!wGMbX5K%Ay@s)V5QNyha^0>@5$T5D?#`<_xVGSF`&oZ_Z zar)0|-vWsj`k)G{1*fVG1d-`BczpuzqN#j>jo)PYaJlEMCID0{ih$RFW z(HF*thFEA+H6f|*_@EfPsu4d+EV>fKo|QJ!bMqAl;TxXw@ziM2o#3EjnZ<#&_>vIS z?^|t#ls}1b4A3Lp?AfJZK!mrfImn9J{fHDAa-bdpZJpor6)4k&tM<*mQ(fKr- zEe(e)G^9;h(-#y8v@9c@VJQA#JM?aOF<+?_6 zx@X>5(#Mw`Egh@&$@-`1S=p4NXDw?4FZ$D3lHn#+#_V~@t%Lpyi=SXuWET9U@|rUq((~uYk3x z7@@rXe~@`PVdin%*ydcyEi3q;aqxGp!Ta_Gd$01Md&3vZf^2TR#N1*D*tefw8!I=Ia{XubbYzkLV& zeAfGiV%C=Cp(`yd7LgZ`<03bptd;ir|LfvS4?5`YhdF<{VTt+DWk%+QcEUeno4<4X K_V#*D_WuAET58w; literal 0 HcmV?d00001 From 3b48806f753b41b8d0540e001d4217cdc054b602 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Tue, 3 Dec 2019 10:11:47 -0500 Subject: [PATCH 203/505] [pplm] README: add setup + tweaks --- examples/pplm/README.md | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/examples/pplm/README.md b/examples/pplm/README.md index 6eb040a442..103218ae72 100644 --- a/examples/pplm/README.md +++ b/examples/pplm/README.md @@ -4,7 +4,7 @@ This folder contains the original code used to run the Plug and Play Language Mo ![header image](./imgs/headfigure.png) ## Plug and Play Language Models: a Simple Approach to Steerable Text Generation -Authors: [Sumanth Dathathri](https://dathath.github.io/), Andrea Madotto, Janice Lan, Jane Hung, Eric Frank, [Piero Molino](), [Jason Yosinski](http://yosinski.com/), and [Rosanne Liu](http://www.rosanneliu.com/) +Authors: [Sumanth Dathathri](https://dathath.github.io/), Andrea Madotto, Janice Lan, Jane Hung, Eric Frank, [Piero Molino](https://w4nderlu.st/), [Jason Yosinski](http://yosinski.com/), and [Rosanne Liu](http://www.rosanneliu.com/) PPLM allows a user to flexibly plug in one or more tiny attribute models representing the desired steering objective into a large, unconditional LM. The method has the key property that it uses the LM _as is_---no training or fine-tuning is required---which enables researchers to leverage best-in-class LMs even if they do not have the extensive hardware required to train them. @@ -14,16 +14,24 @@ Blog link: https://eng.uber.com/pplm ## Setup -TODO + +```bash +git clone https://github.com/huggingface/transformers && cd transformers +pip install [--editable] . +pip install nltk torchtext # additional requirements. +cd examples/pplm +``` ## PPLM-BoW ### Example command for bag-of-words control -``` + +```bash python run_pplm.py -B space --cond_text "The president" --length 100 --gamma 1.5 --num_iterations 3 --num_samples 1 --stepsize 0.01 --window_length 5 --kl_scale 0.01 --gm_scale 0.95 ``` ### Tuning hyperparameters for bag-of-words control + 1. Increase `--stepsize` to intensify topic control, and decrease its value to soften the control. `--stepsize 0` recovers the original uncontrolled GPT-2 model. 2. If the language being generated is repetitive (For e.g. "science science experiment experiment"), there are several options to consider:
@@ -33,16 +41,21 @@ python run_pplm.py -B space --cond_text "The president" --length 100 --gamma 1.5 ## PPLM-Discrim + ### Example command for discriminator based sentiment control -``` + +```bash python run_pplm.py -D sentiment --class_label 3 --cond_text "The lake" --length 10 --gamma 1.0 --num_iterations 10 --num_samples 1 --stepsize 0.03 --kl_scale 0.01 --gm_scale 0.95 ``` ### Tuning hyperparameters for discriminator control + 1. Increase `--stepsize` to intensify topic control, and decrease its value to soften the control. `--stepsize 0` recovers the original uncontrolled GPT-2 model. 2. Use `--class_label 3` for negative, and `--class_label 2` for positive ### Example command for detoxificiation: -python run_pplm.py -D toxicity --length 100 --num_iterations 10 --cond-text 'TH PEOPLEMan goddreams Blacks' --gamma 1.0 --num_samples 10 --stepsize 0.02 +```bash +python run_pplm.py -D toxicity --length 100 --num_iterations 10 --cond-text 'TH PEOPLEMan goddreams Blacks' --gamma 1.0 --num_samples 10 --stepsize 0.02 +``` From 96e83506d1ddee8e19b07118668be73d175decb6 Mon Sep 17 00:00:00 2001 From: Ethan Perez Date: Fri, 29 Nov 2019 18:22:34 -0600 Subject: [PATCH 204/505] Always use SequentialSampler during evaluation When evaluating, shouldn't we always use the SequentialSampler instead of DistributedSampler? Evaluation only runs on 1 GPU no matter what, so if you use the DistributedSampler with N GPUs, I think you'll only evaluate on 1/N of the evaluation set. That's at least what I'm finding when I run an older/modified version of this repo. --- examples/run_squad.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/run_squad.py b/examples/run_squad.py index 59683c0668..32d807b3ad 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -216,7 +216,7 @@ def evaluate(args, model, tokenizer, prefix=""): args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu) # Note that DistributedSampler samples randomly - eval_sampler = SequentialSampler(dataset) if args.local_rank == -1 else DistributedSampler(dataset) + eval_sampler = SequentialSampler(dataset) eval_dataloader = DataLoader(dataset, sampler=eval_sampler, batch_size=args.eval_batch_size) # multi-gpu evaluate From f434bfc623de9e535d49f0c095c0b4af3e88e22a Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Tue, 3 Dec 2019 10:53:02 -0500 Subject: [PATCH 205/505] [pplm] Update S3 links Co-Authored-By: Piero Molino --- examples/pplm/run_pplm.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/pplm/run_pplm.py b/examples/pplm/run_pplm.py index 5e09427879..f626a43f4f 100644 --- a/examples/pplm/run_pplm.py +++ b/examples/pplm/run_pplm.py @@ -59,7 +59,7 @@ BAG_OF_WORDS_ARCHIVE_MAP = { DISCRIMINATOR_MODELS_PARAMS = { "clickbait": { - "url": "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/discriminators/clickbait_classifierhead.pt", + "url": "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/discriminators/clickbait_classifier_head.pt", "class_size": 2, "embed_size": 1024, "class_vocab": {"non_clickbait": 0, "clickbait": 1}, @@ -67,7 +67,7 @@ DISCRIMINATOR_MODELS_PARAMS = { "pretrained_model": "gpt2-medium", }, "sentiment": { - "url": "http://s.yosinski.com/SST_classifier_head.pt", + "url": "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/discriminators/SST_classifier_head.pt", "class_size": 5, "embed_size": 1024, "class_vocab": {"very_positive": 2, "very_negative": 3}, @@ -75,7 +75,7 @@ DISCRIMINATOR_MODELS_PARAMS = { "pretrained_model": "gpt2-medium", }, "toxicity": { - "url": "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/discriminators/toxicity_classifierhead.pt", + "url": "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/discriminators/toxic_classifier_head.pt", "class_size": 2, "embed_size": 1024, "class_vocab": {"non_toxic": 0, "toxic": 1}, From 48cbf267c988b56c71a2380f748a3e6092ccaed3 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Tue, 3 Dec 2019 11:01:37 -0500 Subject: [PATCH 206/505] Use full dataset for eval (SequentialSampler in Distributed setting) --- examples/run_glue.py | 2 +- examples/run_lm_finetuning.py | 2 +- examples/run_multiple_choice.py | 2 +- examples/run_xnli.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/run_glue.py b/examples/run_glue.py index 601e9a34c2..369a7110ab 100644 --- a/examples/run_glue.py +++ b/examples/run_glue.py @@ -231,7 +231,7 @@ def evaluate(args, model, tokenizer, prefix=""): args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu) # Note that DistributedSampler samples randomly - eval_sampler = SequentialSampler(eval_dataset) if args.local_rank == -1 else DistributedSampler(eval_dataset) + eval_sampler = SequentialSampler(eval_dataset) eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size) # multi-gpu eval diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py index 4acea00c55..0bb7460353 100644 --- a/examples/run_lm_finetuning.py +++ b/examples/run_lm_finetuning.py @@ -300,7 +300,7 @@ def evaluate(args, model, tokenizer, prefix=""): args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu) # Note that DistributedSampler samples randomly - eval_sampler = SequentialSampler(eval_dataset) if args.local_rank == -1 else DistributedSampler(eval_dataset) + eval_sampler = SequentialSampler(eval_dataset) eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size) # multi-gpu evaluate diff --git a/examples/run_multiple_choice.py b/examples/run_multiple_choice.py index 30c3332929..9d1ca7f300 100644 --- a/examples/run_multiple_choice.py +++ b/examples/run_multiple_choice.py @@ -226,7 +226,7 @@ def evaluate(args, model, tokenizer, prefix="", test=False): args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu) # Note that DistributedSampler samples randomly - eval_sampler = SequentialSampler(eval_dataset) if args.local_rank == -1 else DistributedSampler(eval_dataset) + eval_sampler = SequentialSampler(eval_dataset) eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size) # multi-gpu evaluate diff --git a/examples/run_xnli.py b/examples/run_xnli.py index a3bc0d4604..42d134a43a 100644 --- a/examples/run_xnli.py +++ b/examples/run_xnli.py @@ -206,7 +206,7 @@ def evaluate(args, model, tokenizer, prefix=""): args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu) # Note that DistributedSampler samples randomly - eval_sampler = SequentialSampler(eval_dataset) if args.local_rank == -1 else DistributedSampler(eval_dataset) + eval_sampler = SequentialSampler(eval_dataset) eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size) # multi-gpu eval From 8101924a6812ffb09c54c2af85d2182f9a81db20 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Tue, 3 Dec 2019 11:20:26 -0500 Subject: [PATCH 207/505] Patch: v2.2.1 --- README.md | 2 +- docs/source/conf.py | 2 +- setup.py | 2 +- transformers/__init__.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index dd20b80590..3173b6cf10 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ Choose the right framework for every part of a model's lifetime | [Quick tour: Fine-tuning/usage scripts](#quick-tour-of-the-fine-tuningusage-scripts) | Using provided scripts: GLUE, SQuAD and Text generation | | [Migrating from pytorch-transformers to transformers](#Migrating-from-pytorch-transformers-to-transformers) | Migrating your code from pytorch-transformers to transformers | | [Migrating from pytorch-pretrained-bert to pytorch-transformers](#Migrating-from-pytorch-pretrained-bert-to-transformers) | Migrating your code from pytorch-pretrained-bert to transformers | -| [Documentation][(v2.2.0)](https://huggingface.co/transformers/v2.2.0) [(v2.1.1)](https://huggingface.co/transformers/v2.1.1) [(v2.0.0)](https://huggingface.co/transformers/v2.0.0) [(v1.2.0)](https://huggingface.co/transformers/v1.2.0) [(v1.1.0)](https://huggingface.co/transformers/v1.1.0) [(v1.0.0)](https://huggingface.co/transformers/v1.0.0) [(master)](https://huggingface.co/transformers) | Full API documentation and more | +| [Documentation][(v2.2.0/v2.2.1)](https://huggingface.co/transformers/v2.2.0) [(v2.1.1)](https://huggingface.co/transformers/v2.1.1) [(v2.0.0)](https://huggingface.co/transformers/v2.0.0) [(v1.2.0)](https://huggingface.co/transformers/v1.2.0) [(v1.1.0)](https://huggingface.co/transformers/v1.1.0) [(v1.0.0)](https://huggingface.co/transformers/v1.0.0) [(master)](https://huggingface.co/transformers) | Full API documentation and more | ## Installation diff --git a/docs/source/conf.py b/docs/source/conf.py index f762a89cd2..2f8505ab3a 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -26,7 +26,7 @@ author = u'huggingface' # The short X.Y version version = u'' # The full version, including alpha/beta/rc tags -release = u'2.2.0' +release = u'2.2.1' # -- General configuration --------------------------------------------------- diff --git a/setup.py b/setup.py index d8dcf7b898..c07920520d 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ from setuptools import find_packages, setup setup( name="transformers", - version="2.2.0", + version="2.2.1", author="Thomas Wolf, Lysandre Debut, Victor Sanh, Julien Chaumond, Google AI Language Team Authors, Open AI team Authors, Facebook AI Authors, Carnegie Mellon University Authors", author_email="thomas@huggingface.co", description="State-of-the-art Natural Language Processing for TensorFlow 2.0 and PyTorch", diff --git a/transformers/__init__.py b/transformers/__init__.py index de25c24b9e..970bdf0cf1 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -1,4 +1,4 @@ -__version__ = "2.2.0" +__version__ = "2.2.1" # Work around to update TensorFlow's absl.logging threshold which alters the # default Python logging output behavior when present. From 285b1241e38cdafb6b0dadd1d1afc19493318074 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Tue, 3 Dec 2019 15:00:49 -0500 Subject: [PATCH 208/505] Added SquadResult --- transformers/data/processors/squad.py | 71 +++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/transformers/data/processors/squad.py b/transformers/data/processors/squad.py index f414d41925..afbe4270f5 100644 --- a/transformers/data/processors/squad.py +++ b/transformers/data/processors/squad.py @@ -425,3 +425,74 @@ class SquadFeatures(object): self.start_position = start_position self.end_position = end_position + + + +class SquadResult(object): + """ + Constructs a SquadResult which can be used to evaluate a model's output on the SQuAD dataset. + + Args: + result: The result output by a model on a SQuAD inference. These results may be complex (5 values) as the ones output by + XLNet or XLM or may be simple like the other models (2 values). They may be passed as a list or as a dict, with the + following accepted formats: + + `dict` output by a simple model: + { + "start_logits": int, + "end_logits": int, + "unique_id": string + } + `list` output by a simple model: + [start_logits, end_logits, unique_id] + + `dict` output by a complex model: + { + "start_top_log_probs": float, + "start_top_index": int, + "end_top_log_probs": float, + "end_top_index": int, + "cls_logits": int, + "unique_id": string + } + `list` output by a complex model: + [start_top_log_probs, start_top_index, end_top_log_probs, end_top_index, cls_logits, unique_id] + + See `run_squad.py` for an example. + """ + def __init__(self, result): + if isinstance(result, dict): + if "start_logits" in result and "end_logits" in result: + self.start_logits = result["start_logits"] + self.end_logits = result["end_logits"] + + elif "start_top_log_probs" in result and "start_top_index" in result: + self.start_top_log_probs = result["start_top_log_probs"] + self.start_top_index = result["start_top_index"] + self.end_top_log_probs = result["end_top_log_probs"] + self.end_top_index = result["end_top_index"] + self.cls_logits = result["cls_logits"] + + else: + raise ValueError("SquadResult instantiated with wrong values.") + + self.unique_id = result["unique_id"] + elif isinstance(result, list): + if len(result) == 3: + self.start_logits = result[0] + self.end_logits = result[1] + + elif len(result) == 6: + self.start_top_log_probs = result[0] + self.start_top_index = result[1] + self.end_top_log_probs = result[2] + self.end_top_index = result[3] + self.cls_logits = result[4] + + else: + raise ValueError("SquadResult instantiated with wrong values.") + + self.unique_id = result[-1] + + else: + raise ValueError("SquadResult instantiated with wrong values. Should be a dictionary or a list.") From c835bc85c2f51f4da5eab4f1481a25b052bf6d61 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Tue, 3 Dec 2019 15:28:16 -0500 Subject: [PATCH 209/505] Compute predictions --- transformers/data/metrics/squad_metrics.py | 335 +++++++++++++++++++++ 1 file changed, 335 insertions(+) create mode 100644 transformers/data/metrics/squad_metrics.py diff --git a/transformers/data/metrics/squad_metrics.py b/transformers/data/metrics/squad_metrics.py new file mode 100644 index 0000000000..d4c5a8ec5b --- /dev/null +++ b/transformers/data/metrics/squad_metrics.py @@ -0,0 +1,335 @@ +import json +import logging +import math +import collections +from io import open +from tqdm import tqdm + +from transformers.tokenization_bert import BasicTokenizer, whitespace_tokenize + +logger = logging.getLogger(__name__) + + +def compute_predictions(all_examples, all_features, all_results, n_best_size, + max_answer_length, do_lower_case, output_prediction_file, + output_nbest_file, output_null_log_odds_file, verbose_logging, + version_2_with_negative, null_score_diff_threshold): + """Write final predictions to the json file and log-odds of null if needed.""" + logger.info("Writing predictions to: %s" % (output_prediction_file)) + logger.info("Writing nbest to: %s" % (output_nbest_file)) + + example_index_to_features = collections.defaultdict(list) + for feature in all_features: + example_index_to_features[feature.example_index].append(feature) + + unique_id_to_result = {} + for result in all_results: + unique_id_to_result[result.unique_id] = result + + _PrelimPrediction = collections.namedtuple( # pylint: disable=invalid-name + "PrelimPrediction", + ["feature_index", "start_index", "end_index", "start_logit", "end_logit"]) + + all_predictions = collections.OrderedDict() + all_nbest_json = collections.OrderedDict() + scores_diff_json = collections.OrderedDict() + + for (example_index, example) in enumerate(all_examples): + features = example_index_to_features[example_index] + + prelim_predictions = [] + # keep track of the minimum score of null start+end of position 0 + score_null = 1000000 # large and positive + min_null_feature_index = 0 # the paragraph slice with min null score + null_start_logit = 0 # the start logit at the slice with min null score + null_end_logit = 0 # the end logit at the slice with min null score + for (feature_index, feature) in enumerate(features): + result = unique_id_to_result[feature.unique_id] + start_indexes = _get_best_indexes(result.start_logits, n_best_size) + end_indexes = _get_best_indexes(result.end_logits, n_best_size) + # if we could have irrelevant answers, get the min score of irrelevant + if version_2_with_negative: + feature_null_score = result.start_logits[0] + result.end_logits[0] + if feature_null_score < score_null: + score_null = feature_null_score + min_null_feature_index = feature_index + null_start_logit = result.start_logits[0] + null_end_logit = result.end_logits[0] + for start_index in start_indexes: + for end_index in end_indexes: + # We could hypothetically create invalid predictions, e.g., predict + # that the start of the span is in the question. We throw out all + # invalid predictions. + if start_index >= len(feature.tokens): + continue + if end_index >= len(feature.tokens): + continue + if start_index not in feature.token_to_orig_map: + continue + if end_index not in feature.token_to_orig_map: + continue + if not feature.token_is_max_context.get(start_index, False): + continue + if end_index < start_index: + continue + length = end_index - start_index + 1 + if length > max_answer_length: + continue + prelim_predictions.append( + _PrelimPrediction( + feature_index=feature_index, + start_index=start_index, + end_index=end_index, + start_logit=result.start_logits[start_index], + end_logit=result.end_logits[end_index])) + if version_2_with_negative: + prelim_predictions.append( + _PrelimPrediction( + feature_index=min_null_feature_index, + start_index=0, + end_index=0, + start_logit=null_start_logit, + end_logit=null_end_logit)) + prelim_predictions = sorted( + prelim_predictions, + key=lambda x: (x.start_logit + x.end_logit), + reverse=True) + + _NbestPrediction = collections.namedtuple( # pylint: disable=invalid-name + "NbestPrediction", ["text", "start_logit", "end_logit"]) + + seen_predictions = {} + nbest = [] + for pred in prelim_predictions: + if len(nbest) >= n_best_size: + break + feature = features[pred.feature_index] + if pred.start_index > 0: # this is a non-null prediction + tok_tokens = feature.tokens[pred.start_index:(pred.end_index + 1)] + orig_doc_start = feature.token_to_orig_map[pred.start_index] + orig_doc_end = feature.token_to_orig_map[pred.end_index] + orig_tokens = example.doc_tokens[orig_doc_start:(orig_doc_end + 1)] + tok_text = " ".join(tok_tokens) + + # De-tokenize WordPieces that have been split off. + tok_text = tok_text.replace(" ##", "") + tok_text = tok_text.replace("##", "") + + # Clean whitespace + tok_text = tok_text.strip() + tok_text = " ".join(tok_text.split()) + orig_text = " ".join(orig_tokens) + + final_text = get_final_text(tok_text, orig_text, do_lower_case, verbose_logging) + if final_text in seen_predictions: + continue + + seen_predictions[final_text] = True + else: + final_text = "" + seen_predictions[final_text] = True + + nbest.append( + _NbestPrediction( + text=final_text, + start_logit=pred.start_logit, + end_logit=pred.end_logit)) + # if we didn't include the empty option in the n-best, include it + if version_2_with_negative: + if "" not in seen_predictions: + nbest.append( + _NbestPrediction( + text="", + start_logit=null_start_logit, + end_logit=null_end_logit)) + + # In very rare edge cases we could only have single null prediction. + # So we just create a nonce prediction in this case to avoid failure. + if len(nbest)==1: + nbest.insert(0, + _NbestPrediction(text="empty", start_logit=0.0, end_logit=0.0)) + + # In very rare edge cases we could have no valid predictions. So we + # just create a nonce prediction in this case to avoid failure. + if not nbest: + nbest.append( + _NbestPrediction(text="empty", start_logit=0.0, end_logit=0.0)) + + assert len(nbest) >= 1 + + total_scores = [] + best_non_null_entry = None + for entry in nbest: + total_scores.append(entry.start_logit + entry.end_logit) + if not best_non_null_entry: + if entry.text: + best_non_null_entry = entry + + probs = _compute_softmax(total_scores) + + nbest_json = [] + for (i, entry) in enumerate(nbest): + output = collections.OrderedDict() + output["text"] = entry.text + output["probability"] = probs[i] + output["start_logit"] = entry.start_logit + output["end_logit"] = entry.end_logit + nbest_json.append(output) + + assert len(nbest_json) >= 1 + + if not version_2_with_negative: + all_predictions[example.qas_id] = nbest_json[0]["text"] + else: + # predict "" iff the null score - the score of best non-null > threshold + score_diff = score_null - best_non_null_entry.start_logit - ( + best_non_null_entry.end_logit) + scores_diff_json[example.qas_id] = score_diff + if score_diff > null_score_diff_threshold: + all_predictions[example.qas_id] = "" + else: + all_predictions[example.qas_id] = best_non_null_entry.text + all_nbest_json[example.qas_id] = nbest_json + + with open(output_prediction_file, "w") as writer: + writer.write(json.dumps(all_predictions, indent=4) + "\n") + + with open(output_nbest_file, "w") as writer: + writer.write(json.dumps(all_nbest_json, indent=4) + "\n") + + if version_2_with_negative: + with open(output_null_log_odds_file, "w") as writer: + writer.write(json.dumps(scores_diff_json, indent=4) + "\n") + + return all_predictions + + +def get_final_text(pred_text, orig_text, do_lower_case, verbose_logging=False): + """Project the tokenized prediction back to the original text.""" + + # When we created the data, we kept track of the alignment between original + # (whitespace tokenized) tokens and our WordPiece tokenized tokens. So + # now `orig_text` contains the span of our original text corresponding to the + # span that we predicted. + # + # However, `orig_text` may contain extra characters that we don't want in + # our prediction. + # + # For example, let's say: + # pred_text = steve smith + # orig_text = Steve Smith's + # + # We don't want to return `orig_text` because it contains the extra "'s". + # + # We don't want to return `pred_text` because it's already been normalized + # (the SQuAD eval script also does punctuation stripping/lower casing but + # our tokenizer does additional normalization like stripping accent + # characters). + # + # What we really want to return is "Steve Smith". + # + # Therefore, we have to apply a semi-complicated alignment heuristic between + # `pred_text` and `orig_text` to get a character-to-character alignment. This + # can fail in certain cases in which case we just return `orig_text`. + + def _strip_spaces(text): + ns_chars = [] + ns_to_s_map = collections.OrderedDict() + for (i, c) in enumerate(text): + if c == " ": + continue + ns_to_s_map[len(ns_chars)] = i + ns_chars.append(c) + ns_text = "".join(ns_chars) + return (ns_text, ns_to_s_map) + + # We first tokenize `orig_text`, strip whitespace from the result + # and `pred_text`, and check if they are the same length. If they are + # NOT the same length, the heuristic has failed. If they are the same + # length, we assume the characters are one-to-one aligned. + tokenizer = BasicTokenizer(do_lower_case=do_lower_case) + + tok_text = " ".join(tokenizer.tokenize(orig_text)) + + start_position = tok_text.find(pred_text) + if start_position == -1: + if verbose_logging: + logger.info( + "Unable to find text: '%s' in '%s'" % (pred_text, orig_text)) + return orig_text + end_position = start_position + len(pred_text) - 1 + + (orig_ns_text, orig_ns_to_s_map) = _strip_spaces(orig_text) + (tok_ns_text, tok_ns_to_s_map) = _strip_spaces(tok_text) + + if len(orig_ns_text) != len(tok_ns_text): + if verbose_logging: + logger.info("Length not equal after stripping spaces: '%s' vs '%s'", + orig_ns_text, tok_ns_text) + return orig_text + + # We then project the characters in `pred_text` back to `orig_text` using + # the character-to-character alignment. + tok_s_to_ns_map = {} + for (i, tok_index) in tok_ns_to_s_map.items(): + tok_s_to_ns_map[tok_index] = i + + orig_start_position = None + if start_position in tok_s_to_ns_map: + ns_start_position = tok_s_to_ns_map[start_position] + if ns_start_position in orig_ns_to_s_map: + orig_start_position = orig_ns_to_s_map[ns_start_position] + + if orig_start_position is None: + if verbose_logging: + logger.info("Couldn't map start position") + return orig_text + + orig_end_position = None + if end_position in tok_s_to_ns_map: + ns_end_position = tok_s_to_ns_map[end_position] + if ns_end_position in orig_ns_to_s_map: + orig_end_position = orig_ns_to_s_map[ns_end_position] + + if orig_end_position is None: + if verbose_logging: + logger.info("Couldn't map end position") + return orig_text + + output_text = orig_text[orig_start_position:(orig_end_position + 1)] + return output_text + + +def _get_best_indexes(logits, n_best_size): + """Get the n-best logits from a list.""" + index_and_score = sorted(enumerate(logits), key=lambda x: x[1], reverse=True) + + best_indexes = [] + for i in range(len(index_and_score)): + if i >= n_best_size: + break + best_indexes.append(index_and_score[i][0]) + return best_indexes + + +def _compute_softmax(scores): + """Compute softmax probability over raw logits.""" + if not scores: + return [] + + max_score = None + for score in scores: + if max_score is None or score > max_score: + max_score = score + + exp_scores = [] + total_sum = 0.0 + for score in scores: + x = math.exp(score - max_score) + exp_scores.append(x) + total_sum += x + + probs = [] + for score in exp_scores: + probs.append(score / total_sum) + return probs From 7edb51f3a516ca533797fb2bb2f2b7ce86e0df70 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Tue, 3 Dec 2019 22:07:25 +0000 Subject: [PATCH 210/505] [pplm] split classif head into its own file --- examples/pplm/pplm_classification_head.py | 18 ++++++++++++++++++ examples/pplm/run_pplm.py | 2 +- examples/pplm/run_pplm_discrim_train.py | 17 +---------------- 3 files changed, 20 insertions(+), 17 deletions(-) create mode 100644 examples/pplm/pplm_classification_head.py diff --git a/examples/pplm/pplm_classification_head.py b/examples/pplm/pplm_classification_head.py new file mode 100644 index 0000000000..9aae0f17e9 --- /dev/null +++ b/examples/pplm/pplm_classification_head.py @@ -0,0 +1,18 @@ +import torch + +class ClassificationHead(torch.nn.Module): + """Classification Head for transformer encoders""" + + def __init__(self, class_size, embed_size): + super(ClassificationHead, self).__init__() + self.class_size = class_size + self.embed_size = embed_size + # self.mlp1 = torch.nn.Linear(embed_size, embed_size) + # self.mlp2 = (torch.nn.Linear(embed_size, class_size)) + self.mlp = torch.nn.Linear(embed_size, class_size) + + def forward(self, hidden_state): + # hidden_state = F.relu(self.mlp1(hidden_state)) + # hidden_state = self.mlp2(hidden_state) + logits = self.mlp(hidden_state) + return logits diff --git a/examples/pplm/run_pplm.py b/examples/pplm/run_pplm.py index f626a43f4f..dda5d85ae7 100644 --- a/examples/pplm/run_pplm.py +++ b/examples/pplm/run_pplm.py @@ -33,10 +33,10 @@ import torch.nn.functional as F from torch.autograd import Variable from tqdm import trange -from examples.run_pplm_discrim_train import ClassificationHead from transformers import GPT2Tokenizer from transformers.file_utils import cached_path from transformers.modeling_gpt2 import GPT2LMHeadModel +from pplm_classification_head import ClassificationHead PPLM_BOW = 1 PPLM_DISCRIM = 2 diff --git a/examples/pplm/run_pplm_discrim_train.py b/examples/pplm/run_pplm_discrim_train.py index db081e1a17..9d36b79bc4 100644 --- a/examples/pplm/run_pplm_discrim_train.py +++ b/examples/pplm/run_pplm_discrim_train.py @@ -21,6 +21,7 @@ from torchtext import datasets from tqdm import tqdm, trange from transformers import GPT2Tokenizer, GPT2LMHeadModel +from pplm_classification_head import ClassificationHead torch.manual_seed(0) np.random.seed(0) @@ -29,22 +30,6 @@ example_sentence = "This is incredible! I love it, this is the best chicken I ha max_length_seq = 100 -class ClassificationHead(torch.nn.Module): - """Classification Head for transformer encoders""" - - def __init__(self, class_size, embed_size): - super(ClassificationHead, self).__init__() - self.class_size = class_size - self.embed_size = embed_size - # self.mlp1 = torch.nn.Linear(embed_size, embed_size) - # self.mlp2 = (torch.nn.Linear(embed_size, class_size)) - self.mlp = torch.nn.Linear(embed_size, class_size) - - def forward(self, hidden_state): - # hidden_state = F.relu(self.mlp1(hidden_state)) - # hidden_state = self.mlp2(hidden_state) - logits = self.mlp(hidden_state) - return logits class Discriminator(torch.nn.Module): From de276de1c1a469a58a25383a35a239d02459a978 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Tue, 3 Dec 2019 17:15:51 -0500 Subject: [PATCH 211/505] Working evaluation --- examples/run_squad.py | 43 +- transformers/data/metrics/squad_metrics.py | 588 +++++++++++++++++---- transformers/data/processors/squad.py | 19 +- 3 files changed, 507 insertions(+), 143 deletions(-) diff --git a/examples/run_squad.py b/examples/run_squad.py index 545c3ad55a..b7952487dc 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -16,7 +16,8 @@ """ Finetuning the library models for question-answering on SQuAD (DistilBERT, Bert, XLM, XLNet).""" from __future__ import absolute_import, division, print_function -from transformers.data.processors.squad import SquadV1Processor, SquadV2Processor +from transformers.data.processors.squad import SquadV1Processor, SquadV2Processor, SquadResult +from transformers.data.metrics.squad_metrics import compute_predictions, compute_predictions_extended, squad_evaluate import argparse import logging @@ -230,9 +231,11 @@ def evaluate(args, model, tokenizer, prefix=""): model.eval() batch = tuple(t.to(args.device) for t in batch) with torch.no_grad(): - inputs = {'input_ids': batch[0], - 'attention_mask': batch[1] - } + inputs = { + 'input_ids': batch[0], + 'attention_mask': batch[1] + } + if args.model_type != 'distilbert': inputs['token_type_ids'] = None if args.model_type == 'xlm' else batch[2] # XLM don't use segment_ids example_indices = batch[3] @@ -244,18 +247,8 @@ def evaluate(args, model, tokenizer, prefix=""): for i, example_index in enumerate(example_indices): eval_feature = features[example_index.item()] unique_id = int(eval_feature.unique_id) - if args.model_type in ['xlnet', 'xlm']: - # XLNet uses a more complex post-processing procedure - result = RawResultExtended(unique_id = unique_id, - start_top_log_probs = to_list(outputs[0][i]), - start_top_index = to_list(outputs[1][i]), - end_top_log_probs = to_list(outputs[2][i]), - end_top_index = to_list(outputs[3][i]), - cls_logits = to_list(outputs[4][i])) - else: - result = RawResult(unique_id = unique_id, - start_logits = to_list(outputs[0][i]), - end_logits = to_list(outputs[1][i])) + + result = SquadResult([to_list(output[i]) for output in outputs] + [unique_id]) all_results.append(result) evalTime = timeit.default_timer() - start_time @@ -271,22 +264,18 @@ def evaluate(args, model, tokenizer, prefix=""): if args.model_type in ['xlnet', 'xlm']: # XLNet uses a more complex post-processing procedure - write_predictions_extended(examples, features, all_results, args.n_best_size, + predictions = compute_predictions_extended(examples, features, all_results, args.n_best_size, args.max_answer_length, output_prediction_file, output_nbest_file, output_null_log_odds_file, args.predict_file, model.config.start_n_top, model.config.end_n_top, args.version_2_with_negative, tokenizer, args.verbose_logging) else: - write_predictions(examples, features, all_results, args.n_best_size, + predictions = compute_predictions(examples, features, all_results, args.n_best_size, args.max_answer_length, args.do_lower_case, output_prediction_file, output_nbest_file, output_null_log_odds_file, args.verbose_logging, args.version_2_with_negative, args.null_score_diff_threshold) - # Evaluate with the official SQuAD script - evaluate_options = EVAL_OPTS(data_file=args.predict_file, - pred_file=output_prediction_file, - na_prob_file=output_null_log_odds_file) - results = evaluate_on_squad(evaluate_options) + results = squad_evaluate(examples, predictions) return results def load_and_cache_examples(args, tokenizer, evaluate=False, output_examples=False): @@ -306,8 +295,12 @@ def load_and_cache_examples(args, tokenizer, evaluate=False, output_examples=Fal logger.info("Creating features from dataset file at %s", input_file) processor = SquadV2Processor() - examples = processor.get_dev_examples("examples/squad") if evaluate else processor.get_train_examples("examples/squad") - features = squad_convert_examples_to_features( + examples = processor.get_dev_examples("examples/squad", only_first=100) if evaluate else processor.get_train_examples("examples/squad") + # import tensorflow_datasets as tfds + # tfds_examples = tfds.load("squad") + # examples = SquadV1Processor().get_examples_from_dataset(tfds_examples["validation"]) + + features = squad_convert_examples_to_features( examples=examples, tokenizer=tokenizer, max_seq_length=args.max_seq_length, diff --git a/transformers/data/metrics/squad_metrics.py b/transformers/data/metrics/squad_metrics.py index d4c5a8ec5b..83647a20d0 100644 --- a/transformers/data/metrics/squad_metrics.py +++ b/transformers/data/metrics/squad_metrics.py @@ -1,15 +1,323 @@ +""" Very heavily inspired by the official evaluation script for SQuAD version 2.0 which was +modified by XLNet authors to update `find_best_threshold` scripts for SQuAD V2.0 + +In addition to basic functionality, we also compute additional statistics and +plot precision-recall curves if an additional na_prob.json file is provided. +This file is expected to map question ID's to the model's predicted probability +that a question is unanswerable. +""" + + import json import logging import math import collections from io import open from tqdm import tqdm +import string +import re from transformers.tokenization_bert import BasicTokenizer, whitespace_tokenize logger = logging.getLogger(__name__) +def normalize_answer(s): + """Lower text and remove punctuation, articles and extra whitespace.""" + def remove_articles(text): + regex = re.compile(r'\b(a|an|the)\b', re.UNICODE) + return re.sub(regex, ' ', text) + + def white_space_fix(text): + return ' '.join(text.split()) + + def remove_punc(text): + exclude = set(string.punctuation) + return ''.join(ch for ch in text if ch not in exclude) + + def lower(text): + return text.lower() + return white_space_fix(remove_articles(remove_punc(lower(s)))) + + +def get_tokens(s): + if not s: + return [] + return normalize_answer(s).split() + + +def compute_exact(a_gold, a_pred): + return int(normalize_answer(a_gold) == normalize_answer(a_pred)) + + +def compute_f1(a_gold, a_pred): + gold_toks = get_tokens(a_gold) + pred_toks = get_tokens(a_pred) + common = collections.Counter(gold_toks) & collections.Counter(pred_toks) + num_same = sum(common.values()) + if len(gold_toks) == 0 or len(pred_toks) == 0: + # If either is no-answer, then F1 is 1 if they agree, 0 otherwise + return int(gold_toks == pred_toks) + if num_same == 0: + return 0 + precision = 1.0 * num_same / len(pred_toks) + recall = 1.0 * num_same / len(gold_toks) + f1 = (2 * precision * recall) / (precision + recall) + return f1 + + +def get_raw_scores(examples, preds): + """ + Computes the exact and f1 scores from the examples and the model predictions + """ + exact_scores = {} + f1_scores = {} + + for example in examples: + qas_id = example.qas_id + gold_answers = [answer['text'] for answer in example.answers if normalize_answer(answer['text'])] + + if not gold_answers: + # For unanswerable questions, only correct answer is empty string + gold_answers = [''] + + if qas_id not in preds: + print('Missing prediction for %s' % qas_id) + continue + + prediction = preds[qas_id] + exact_scores[qas_id] = max(compute_exact(a, prediction) for a in gold_answers) + f1_scores[qas_id] = max(compute_f1(a, prediction) for a in gold_answers) + + return exact_scores, f1_scores + + +def apply_no_ans_threshold(scores, na_probs, qid_to_has_ans, na_prob_thresh): + new_scores = {} + for qid, s in scores.items(): + pred_na = na_probs[qid] > na_prob_thresh + if pred_na: + new_scores[qid] = float(not qid_to_has_ans[qid]) + else: + new_scores[qid] = s + return new_scores + + +def make_eval_dict(exact_scores, f1_scores, qid_list=None): + if not qid_list: + total = len(exact_scores) + return collections.OrderedDict([ + ('exact', 100.0 * sum(exact_scores.values()) / total), + ('f1', 100.0 * sum(f1_scores.values()) / total), + ('total', total), + ]) + else: + total = len(qid_list) + return collections.OrderedDict([ + ('exact', 100.0 * sum(exact_scores[k] for k in qid_list) / total), + ('f1', 100.0 * sum(f1_scores[k] for k in qid_list) / total), + ('total', total), + ]) + + +def merge_eval(main_eval, new_eval, prefix): + for k in new_eval: + main_eval['%s_%s' % (prefix, k)] = new_eval[k] + + +def find_best_thresh(preds, scores, na_probs, qid_to_has_ans): + num_no_ans = sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]) + cur_score = num_no_ans + best_score = cur_score + best_thresh = 0.0 + qid_list = sorted(na_probs, key=lambda k: na_probs[k]) + for _, qid in enumerate(qid_list): + if qid not in scores: + continue + if qid_to_has_ans[qid]: + diff = scores[qid] + else: + if preds[qid]: + diff = -1 + else: + diff = 0 + cur_score += diff + if cur_score > best_score: + best_score = cur_score + best_thresh = na_probs[qid] + return 100.0 * best_score / len(scores), best_thresh + + +def find_all_best_thresh(main_eval, preds, exact_raw, f1_raw, na_probs, qid_to_has_ans): + best_exact, exact_thresh = find_best_thresh(preds, exact_raw, na_probs, qid_to_has_ans) + best_f1, f1_thresh = find_best_thresh(preds, f1_raw, na_probs, qid_to_has_ans) + + main_eval['best_exact'] = best_exact + main_eval['best_exact_thresh'] = exact_thresh + main_eval['best_f1'] = best_f1 + main_eval['best_f1_thresh'] = f1_thresh + + +def squad_evaluate(examples, preds, no_answer_probs=None, no_answer_probability_threshold=1.0): + qas_id_to_has_answer = {example.qas_id: bool(example.answers) for example in examples} + has_answer_qids = [qas_id for qas_id, has_answer in qas_id_to_has_answer.items() if has_answer] + no_answer_qids = [qas_id for qas_id, has_answer in qas_id_to_has_answer.items() if not has_answer] + + if no_answer_probs is None: + no_answer_probs = {k: 0.0 for k in preds} + + exact, f1 = get_raw_scores(examples, preds) + + exact_threshold = apply_no_ans_threshold(exact, no_answer_probs, qas_id_to_has_answer, no_answer_probability_threshold) + f1_threshold = apply_no_ans_threshold(f1, no_answer_probs, qas_id_to_has_answer, no_answer_probability_threshold) + + evaluation = make_eval_dict(exact_threshold, f1_threshold) + + if has_answer_qids: + has_ans_eval = make_eval_dict(exact_threshold, f1_threshold, qid_list=has_answer_qids) + merge_eval(evaluation, has_ans_eval, 'HasAns') + + if no_answer_qids: + no_ans_eval = make_eval_dict(exact_threshold, f1_threshold, qid_list=no_answer_qids) + merge_eval(evaluation, no_ans_eval, 'NoAns') + + if no_answer_probs: + find_all_best_thresh(evaluation, preds, exact, f1, no_answer_probs, qas_id_to_has_answer) + + return evaluation + + +def get_final_text(pred_text, orig_text, do_lower_case, verbose_logging=False): + """Project the tokenized prediction back to the original text.""" + + # When we created the data, we kept track of the alignment between original + # (whitespace tokenized) tokens and our WordPiece tokenized tokens. So + # now `orig_text` contains the span of our original text corresponding to the + # span that we predicted. + # + # However, `orig_text` may contain extra characters that we don't want in + # our prediction. + # + # For example, let's say: + # pred_text = steve smith + # orig_text = Steve Smith's + # + # We don't want to return `orig_text` because it contains the extra "'s". + # + # We don't want to return `pred_text` because it's already been normalized + # (the SQuAD eval script also does punctuation stripping/lower casing but + # our tokenizer does additional normalization like stripping accent + # characters). + # + # What we really want to return is "Steve Smith". + # + # Therefore, we have to apply a semi-complicated alignment heuristic between + # `pred_text` and `orig_text` to get a character-to-character alignment. This + # can fail in certain cases in which case we just return `orig_text`. + + def _strip_spaces(text): + ns_chars = [] + ns_to_s_map = collections.OrderedDict() + for (i, c) in enumerate(text): + if c == " ": + continue + ns_to_s_map[len(ns_chars)] = i + ns_chars.append(c) + ns_text = "".join(ns_chars) + return (ns_text, ns_to_s_map) + + # We first tokenize `orig_text`, strip whitespace from the result + # and `pred_text`, and check if they are the same length. If they are + # NOT the same length, the heuristic has failed. If they are the same + # length, we assume the characters are one-to-one aligned. + tokenizer = BasicTokenizer(do_lower_case=do_lower_case) + + tok_text = " ".join(tokenizer.tokenize(orig_text)) + + start_position = tok_text.find(pred_text) + if start_position == -1: + if verbose_logging: + logger.info( + "Unable to find text: '%s' in '%s'" % (pred_text, orig_text)) + return orig_text + end_position = start_position + len(pred_text) - 1 + + (orig_ns_text, orig_ns_to_s_map) = _strip_spaces(orig_text) + (tok_ns_text, tok_ns_to_s_map) = _strip_spaces(tok_text) + + if len(orig_ns_text) != len(tok_ns_text): + if verbose_logging: + logger.info("Length not equal after stripping spaces: '%s' vs '%s'", + orig_ns_text, tok_ns_text) + return orig_text + + # We then project the characters in `pred_text` back to `orig_text` using + # the character-to-character alignment. + tok_s_to_ns_map = {} + for (i, tok_index) in tok_ns_to_s_map.items(): + tok_s_to_ns_map[tok_index] = i + + orig_start_position = None + if start_position in tok_s_to_ns_map: + ns_start_position = tok_s_to_ns_map[start_position] + if ns_start_position in orig_ns_to_s_map: + orig_start_position = orig_ns_to_s_map[ns_start_position] + + if orig_start_position is None: + if verbose_logging: + logger.info("Couldn't map start position") + return orig_text + + orig_end_position = None + if end_position in tok_s_to_ns_map: + ns_end_position = tok_s_to_ns_map[end_position] + if ns_end_position in orig_ns_to_s_map: + orig_end_position = orig_ns_to_s_map[ns_end_position] + + if orig_end_position is None: + if verbose_logging: + logger.info("Couldn't map end position") + return orig_text + + output_text = orig_text[orig_start_position:(orig_end_position + 1)] + return output_text + + +def _get_best_indexes(logits, n_best_size): + """Get the n-best logits from a list.""" + index_and_score = sorted(enumerate(logits), key=lambda x: x[1], reverse=True) + + best_indexes = [] + for i in range(len(index_and_score)): + if i >= n_best_size: + break + best_indexes.append(index_and_score[i][0]) + return best_indexes + + +def _compute_softmax(scores): + """Compute softmax probability over raw logits.""" + if not scores: + return [] + + max_score = None + for score in scores: + if max_score is None or score > max_score: + max_score = score + + exp_scores = [] + total_sum = 0.0 + for score in scores: + x = math.exp(score - max_score) + exp_scores.append(x) + total_sum += x + + probs = [] + for score in exp_scores: + probs.append(score / total_sum) + return probs + + def compute_predictions(all_examples, all_features, all_results, n_best_size, max_answer_length, do_lower_case, output_prediction_file, output_nbest_file, output_null_log_odds_file, verbose_logging, @@ -204,132 +512,192 @@ def compute_predictions(all_examples, all_features, all_results, n_best_size, return all_predictions -def get_final_text(pred_text, orig_text, do_lower_case, verbose_logging=False): - """Project the tokenized prediction back to the original text.""" +def compute_predictions_extended(all_examples, all_features, all_results, n_best_size, + max_answer_length, output_prediction_file, + output_nbest_file, + output_null_log_odds_file, orig_data_file, + start_n_top, end_n_top, version_2_with_negative, + tokenizer, verbose_logging): + """ XLNet write prediction logic (more complex than Bert's). + Write final predictions to the json file and log-odds of null if needed. - # When we created the data, we kept track of the alignment between original - # (whitespace tokenized) tokens and our WordPiece tokenized tokens. So - # now `orig_text` contains the span of our original text corresponding to the - # span that we predicted. - # - # However, `orig_text` may contain extra characters that we don't want in - # our prediction. - # - # For example, let's say: - # pred_text = steve smith - # orig_text = Steve Smith's - # - # We don't want to return `orig_text` because it contains the extra "'s". - # - # We don't want to return `pred_text` because it's already been normalized - # (the SQuAD eval script also does punctuation stripping/lower casing but - # our tokenizer does additional normalization like stripping accent - # characters). - # - # What we really want to return is "Steve Smith". - # - # Therefore, we have to apply a semi-complicated alignment heuristic between - # `pred_text` and `orig_text` to get a character-to-character alignment. This - # can fail in certain cases in which case we just return `orig_text`. + Requires utils_squad_evaluate.py + """ + _PrelimPrediction = collections.namedtuple( # pylint: disable=invalid-name + "PrelimPrediction", + ["feature_index", "start_index", "end_index", + "start_log_prob", "end_log_prob"]) - def _strip_spaces(text): - ns_chars = [] - ns_to_s_map = collections.OrderedDict() - for (i, c) in enumerate(text): - if c == " ": + _NbestPrediction = collections.namedtuple( # pylint: disable=invalid-name + "NbestPrediction", ["text", "start_log_prob", "end_log_prob"]) + + logger.info("Writing predictions to: %s", output_prediction_file) + # logger.info("Writing nbest to: %s" % (output_nbest_file)) + + example_index_to_features = collections.defaultdict(list) + for feature in all_features: + example_index_to_features[feature.example_index].append(feature) + + unique_id_to_result = {} + for result in all_results: + unique_id_to_result[result.unique_id] = result + + all_predictions = collections.OrderedDict() + all_nbest_json = collections.OrderedDict() + scores_diff_json = collections.OrderedDict() + + for (example_index, example) in enumerate(all_examples): + features = example_index_to_features[example_index] + + prelim_predictions = [] + # keep track of the minimum score of null start+end of position 0 + score_null = 1000000 # large and positive + + for (feature_index, feature) in enumerate(features): + result = unique_id_to_result[feature.unique_id] + + cur_null_score = result.cls_logits + + # if we could have irrelevant answers, get the min score of irrelevant + score_null = min(score_null, cur_null_score) + + for i in range(start_n_top): + for j in range(end_n_top): + start_log_prob = result.start_top_log_probs[i] + start_index = result.start_top_index[i] + + j_index = i * end_n_top + j + + end_log_prob = result.end_top_log_probs[j_index] + end_index = result.end_top_index[j_index] + + # We could hypothetically create invalid predictions, e.g., predict + # that the start of the span is in the question. We throw out all + # invalid predictions. + if start_index >= feature.paragraph_len - 1: + continue + if end_index >= feature.paragraph_len - 1: + continue + + if not feature.token_is_max_context.get(start_index, False): + continue + if end_index < start_index: + continue + length = end_index - start_index + 1 + if length > max_answer_length: + continue + + prelim_predictions.append( + _PrelimPrediction( + feature_index=feature_index, + start_index=start_index, + end_index=end_index, + start_log_prob=start_log_prob, + end_log_prob=end_log_prob)) + + prelim_predictions = sorted( + prelim_predictions, + key=lambda x: (x.start_log_prob + x.end_log_prob), + reverse=True) + + seen_predictions = {} + nbest = [] + for pred in prelim_predictions: + if len(nbest) >= n_best_size: + break + feature = features[pred.feature_index] + + # XLNet un-tokenizer + # Let's keep it simple for now and see if we need all this later. + # + # tok_start_to_orig_index = feature.tok_start_to_orig_index + # tok_end_to_orig_index = feature.tok_end_to_orig_index + # start_orig_pos = tok_start_to_orig_index[pred.start_index] + # end_orig_pos = tok_end_to_orig_index[pred.end_index] + # paragraph_text = example.paragraph_text + # final_text = paragraph_text[start_orig_pos: end_orig_pos + 1].strip() + + # Previously used Bert untokenizer + tok_tokens = feature.tokens[pred.start_index:(pred.end_index + 1)] + orig_doc_start = feature.token_to_orig_map[pred.start_index] + orig_doc_end = feature.token_to_orig_map[pred.end_index] + orig_tokens = example.doc_tokens[orig_doc_start:(orig_doc_end + 1)] + tok_text = tokenizer.convert_tokens_to_string(tok_tokens) + + # Clean whitespace + tok_text = tok_text.strip() + tok_text = " ".join(tok_text.split()) + orig_text = " ".join(orig_tokens) + + final_text = get_final_text(tok_text, orig_text, tokenizer.do_lower_case, + verbose_logging) + + if final_text in seen_predictions: continue - ns_to_s_map[len(ns_chars)] = i - ns_chars.append(c) - ns_text = "".join(ns_chars) - return (ns_text, ns_to_s_map) - # We first tokenize `orig_text`, strip whitespace from the result - # and `pred_text`, and check if they are the same length. If they are - # NOT the same length, the heuristic has failed. If they are the same - # length, we assume the characters are one-to-one aligned. - tokenizer = BasicTokenizer(do_lower_case=do_lower_case) + seen_predictions[final_text] = True - tok_text = " ".join(tokenizer.tokenize(orig_text)) + nbest.append( + _NbestPrediction( + text=final_text, + start_log_prob=pred.start_log_prob, + end_log_prob=pred.end_log_prob)) - start_position = tok_text.find(pred_text) - if start_position == -1: - if verbose_logging: - logger.info( - "Unable to find text: '%s' in '%s'" % (pred_text, orig_text)) - return orig_text - end_position = start_position + len(pred_text) - 1 + # In very rare edge cases we could have no valid predictions. So we + # just create a nonce prediction in this case to avoid failure. + if not nbest: + nbest.append( + _NbestPrediction(text="", start_log_prob=-1e6, + end_log_prob=-1e6)) - (orig_ns_text, orig_ns_to_s_map) = _strip_spaces(orig_text) - (tok_ns_text, tok_ns_to_s_map) = _strip_spaces(tok_text) + total_scores = [] + best_non_null_entry = None + for entry in nbest: + total_scores.append(entry.start_log_prob + entry.end_log_prob) + if not best_non_null_entry: + best_non_null_entry = entry - if len(orig_ns_text) != len(tok_ns_text): - if verbose_logging: - logger.info("Length not equal after stripping spaces: '%s' vs '%s'", - orig_ns_text, tok_ns_text) - return orig_text + probs = _compute_softmax(total_scores) - # We then project the characters in `pred_text` back to `orig_text` using - # the character-to-character alignment. - tok_s_to_ns_map = {} - for (i, tok_index) in tok_ns_to_s_map.items(): - tok_s_to_ns_map[tok_index] = i + nbest_json = [] + for (i, entry) in enumerate(nbest): + output = collections.OrderedDict() + output["text"] = entry.text + output["probability"] = probs[i] + output["start_log_prob"] = entry.start_log_prob + output["end_log_prob"] = entry.end_log_prob + nbest_json.append(output) - orig_start_position = None - if start_position in tok_s_to_ns_map: - ns_start_position = tok_s_to_ns_map[start_position] - if ns_start_position in orig_ns_to_s_map: - orig_start_position = orig_ns_to_s_map[ns_start_position] + assert len(nbest_json) >= 1 + assert best_non_null_entry is not None - if orig_start_position is None: - if verbose_logging: - logger.info("Couldn't map start position") - return orig_text + score_diff = score_null + scores_diff_json[example.qas_id] = score_diff + # note(zhiliny): always predict best_non_null_entry + # and the evaluation script will search for the best threshold + all_predictions[example.qas_id] = best_non_null_entry.text - orig_end_position = None - if end_position in tok_s_to_ns_map: - ns_end_position = tok_s_to_ns_map[end_position] - if ns_end_position in orig_ns_to_s_map: - orig_end_position = orig_ns_to_s_map[ns_end_position] + all_nbest_json[example.qas_id] = nbest_json - if orig_end_position is None: - if verbose_logging: - logger.info("Couldn't map end position") - return orig_text + with open(output_prediction_file, "w") as writer: + writer.write(json.dumps(all_predictions, indent=4) + "\n") - output_text = orig_text[orig_start_position:(orig_end_position + 1)] - return output_text + with open(output_nbest_file, "w") as writer: + writer.write(json.dumps(all_nbest_json, indent=4) + "\n") + if version_2_with_negative: + with open(output_null_log_odds_file, "w") as writer: + writer.write(json.dumps(scores_diff_json, indent=4) + "\n") -def _get_best_indexes(logits, n_best_size): - """Get the n-best logits from a list.""" - index_and_score = sorted(enumerate(logits), key=lambda x: x[1], reverse=True) + with open(orig_data_file, "r", encoding='utf-8') as reader: + orig_data = json.load(reader)["data"] - best_indexes = [] - for i in range(len(index_and_score)): - if i >= n_best_size: - break - best_indexes.append(index_and_score[i][0]) - return best_indexes + qid_to_has_ans = make_qid_to_has_ans(orig_data) + has_ans_qids = [k for k, v in qid_to_has_ans.items() if v] + no_ans_qids = [k for k, v in qid_to_has_ans.items() if not v] + exact_raw, f1_raw = get_raw_scores(orig_data, all_predictions) + out_eval = {} + find_all_best_thresh_v2(out_eval, all_predictions, exact_raw, f1_raw, scores_diff_json, qid_to_has_ans) -def _compute_softmax(scores): - """Compute softmax probability over raw logits.""" - if not scores: - return [] - - max_score = None - for score in scores: - if max_score is None or score > max_score: - max_score = score - - exp_scores = [] - total_sum = 0.0 - for score in scores: - x = math.exp(score - max_score) - exp_scores.append(x) - total_sum += x - - probs = [] - for score in exp_scores: - probs.append(score / total_sum) - return probs + return out_eval diff --git a/transformers/data/processors/squad.py b/transformers/data/processors/squad.py index afbe4270f5..70dc9faf54 100644 --- a/transformers/data/processors/squad.py +++ b/transformers/data/processors/squad.py @@ -306,13 +306,13 @@ class SquadProcessor(DataProcessor): else: is_impossible = False - if not is_impossible and is_training: - if (len(qa["answers"]) != 1): - raise ValueError( - "For training, each question should have exactly 1 answer.") - answer = qa["answers"][0] - answer_text = answer['text'] - start_position_character = answer['answer_start'] + if not is_impossible: + if is_training: + answer = qa["answers"][0] + answer_text = answer['text'] + start_position_character = answer['answer_start'] + else: + answers = qa["answers"] example = SquadExample( qas_id=qas_id, @@ -321,7 +321,8 @@ class SquadProcessor(DataProcessor): answer_text=answer_text, start_position_character=start_position_character, title=title, - is_impossible=is_impossible + is_impossible=is_impossible, + answers=answers ) examples.append(example) @@ -352,6 +353,7 @@ class SquadExample(object): answer_text, start_position_character, title, + answers=None, is_impossible=False): self.qas_id = qas_id self.question_text = question_text @@ -359,6 +361,7 @@ class SquadExample(object): self.answer_text = answer_text self.title = title self.is_impossible = is_impossible + self.answers = answers self.start_position, self.end_position = 0, 0 From e4fbf3e2cc26b1476b3333ec0b1ffef949277262 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Wed, 4 Dec 2019 00:52:23 -0500 Subject: [PATCH 212/505] CLI for authenticated file sharing --- setup.py | 10 ++ transformers-cli | 23 +++++ transformers/commands/__init__.py | 12 +++ transformers/commands/user.py | 122 ++++++++++++++++++++++++ transformers/hf_api.py | 152 ++++++++++++++++++++++++++++++ transformers/tests/hf_api_test.py | 94 ++++++++++++++++++ 6 files changed, 413 insertions(+) create mode 100644 transformers-cli create mode 100644 transformers/commands/__init__.py create mode 100644 transformers/commands/user.py create mode 100644 transformers/hf_api.py create mode 100644 transformers/tests/hf_api_test.py diff --git a/setup.py b/setup.py index c07920520d..25f503f8d0 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,12 @@ To create the package for pypi. from io import open from setuptools import find_packages, setup + +extras = { + 'serving': ['uvicorn', 'fastapi'] +} +extras['all'] = [package for package in extras.values()] + setup( name="transformers", version="2.2.1", @@ -61,6 +67,10 @@ setup( "transformers=transformers.__main__:main", ] }, + extras_require=extras, + scripts=[ + 'transformers-cli' + ], # python_requires='>=3.5.0', tests_require=['pytest'], classifiers=[ diff --git a/transformers-cli b/transformers-cli new file mode 100644 index 0000000000..ef00d15aa3 --- /dev/null +++ b/transformers-cli @@ -0,0 +1,23 @@ +#!/usr/bin/env python +from argparse import ArgumentParser + +from transformers.commands.user import UserCommands + + +if __name__ == '__main__': + parser = ArgumentParser(description='Transformers CLI tool', usage='transformers-cli []') + commands_parser = parser.add_subparsers(help='transformers-cli command helpers') + + # Register commands + UserCommands.register_subcommand(commands_parser) + + # Let's go + args = parser.parse_args() + + if not hasattr(args, 'func'): + parser.print_help() + exit(1) + + # Run + service = args.func(args) + service.run() diff --git a/transformers/commands/__init__.py b/transformers/commands/__init__.py new file mode 100644 index 0000000000..bbdd5655fc --- /dev/null +++ b/transformers/commands/__init__.py @@ -0,0 +1,12 @@ +from abc import ABC, abstractmethod +from argparse import ArgumentParser + +class BaseTransformersCLICommand(ABC): + @staticmethod + @abstractmethod + def register_subcommand(parser: ArgumentParser): + raise NotImplementedError() + + @abstractmethod + def run(self): + raise NotImplementedError() diff --git a/transformers/commands/user.py b/transformers/commands/user.py new file mode 100644 index 0000000000..4b826f4dc6 --- /dev/null +++ b/transformers/commands/user.py @@ -0,0 +1,122 @@ +from argparse import ArgumentParser +from getpass import getpass +import os + +from transformers.commands import BaseTransformersCLICommand +from transformers.hf_api import HfApi, HfFolder, HTTPError + + +class UserCommands(BaseTransformersCLICommand): + @staticmethod + def register_subcommand(parser: ArgumentParser): + login_parser = parser.add_parser('login') + login_parser.set_defaults(func=lambda args: LoginCommand(args)) + whoami_parser = parser.add_parser('whoami') + whoami_parser.set_defaults(func=lambda args: WhoamiCommand(args)) + logout_parser = parser.add_parser('logout') + logout_parser.set_defaults(func=lambda args: LogoutCommand(args)) + list_parser = parser.add_parser('ls') + list_parser.set_defaults(func=lambda args: ListObjsCommand(args)) + # upload + upload_parser = parser.add_parser('upload') + upload_parser.add_argument('file', type=str, help='Local filepath of the file to upload.') + upload_parser.add_argument('--filename', type=str, default=None, help='Optional: override object filename on S3.') + upload_parser.set_defaults(func=lambda args: UploadCommand(args)) + + + +class BaseUserCommand: + def __init__(self, args): + self.args = args + self._api = HfApi() + + +class LoginCommand(BaseUserCommand): + def run(self): + print(""" + _| _| _| _| _|_|_| _|_|_| _|_|_| _| _| _|_|_| _|_|_|_| _|_| _|_|_| _|_|_|_| + _| _| _| _| _| _| _| _|_| _| _| _| _| _| _| _| + _|_|_|_| _| _| _| _|_| _| _|_| _| _| _| _| _| _|_| _|_|_| _|_|_|_| _| _|_|_| + _| _| _| _| _| _| _| _| _| _| _|_| _| _| _| _| _| _| _| + _| _| _|_| _|_|_| _|_|_| _|_|_| _| _| _|_|_| _| _| _| _|_|_| _|_|_|_| + + """) + username = input("Username: ") + password = getpass() + try: + token = self._api.login(username, password) + except HTTPError as e: + # probably invalid credentials, display error message. + print(e) + exit(1) + HfFolder.save_token(token) + print("Login successful") + print("Your token:", token, "\n") + print("Your token has been saved to", HfFolder.path_token) + + +class WhoamiCommand(BaseUserCommand): + def run(self): + token = HfFolder.get_token() + if token is None: + print("Not logged in") + exit() + try: + user = self._api.whoami(token) + print(user) + except HTTPError as e: + print(e) + + +class LogoutCommand(BaseUserCommand): + def run(self): + token = HfFolder.get_token() + if token is None: + print("Not logged in") + exit() + HfFolder.delete_token() + self._api.logout(token) + print("Successfully logged out.") + + +class ListObjsCommand(BaseUserCommand): + def run(self): + token = HfFolder.get_token() + if token is None: + print("Not logged in") + exit(1) + try: + objs = self._api.list_objs(token) + except HTTPError as e: + print(e) + exit(1) + if len(objs) == 0: + print("No shared file yet") + for obj in objs: + print( + obj.filename, + obj.LastModified, + obj.ETag, + obj.Size + ) + + +class UploadCommand(BaseUserCommand): + def run(self): + token = HfFolder.get_token() + if token is None: + print("Not logged in") + exit(1) + filepath = os.path.join(os.getcwd(), self.args.file) + filename = self.args.filename if self.args.filename is not None else os.path.basename(filepath) + print("About to upload file {} to S3 under filename {}".format(filepath, filename)) + choice = input("Proceed? [Y/n] ").lower() + if not(choice == "" or choice == "y" or choice == "yes"): + print("Abort") + exit() + print("Uploading...") + access_url = self._api.presign_and_upload( + token=token, filename=filename, filepath=filepath + ) + print("Your file now lives at:") + print(access_url) diff --git a/transformers/hf_api.py b/transformers/hf_api.py new file mode 100644 index 0000000000..238762ebf8 --- /dev/null +++ b/transformers/hf_api.py @@ -0,0 +1,152 @@ +# coding=utf-8 +# Copyright 2019-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import, division, print_function + +from typing import List, NamedTuple +import os +from os.path import expanduser + +import requests +from requests.exceptions import HTTPError + +ENDPOINT = "https://huggingface.co" + +class S3Obj: + def __init__(self, filename: str, LastModified: str, ETag: str, Size: int): + self.filename = filename + self.LastModified = LastModified + self.ETag = ETag + self.Size = Size + + +class PresignedUrl(NamedTuple): + write: str + access: str + + +class HfApi: + def __init__(self, endpoint=None): + self.endpoint = endpoint if endpoint is not None else ENDPOINT + + def login(self, username: str, password: str) -> str: + """ + Call HF API to sign in a user and get a token if credentials are valid. + + Outputs: + token if credentials are valid + + Throws: + requests.exceptions.HTTPError if credentials are invalid + """ + path = "{}/api/login".format(self.endpoint) + r = requests.post(path, json={"username": username, "password": password}) + r.raise_for_status() + d = r.json() + return d["token"] + + def whoami(self, token: str) -> str: + """ + Call HF API to know "whoami" + """ + path = "{}/api/whoami".format(self.endpoint) + r = requests.get(path, headers={"authorization": "Bearer {}".format(token)}) + r.raise_for_status() + d = r.json() + return d["user"] + + def logout(self, token: str): + """ + Call HF API to log out. + """ + path = "{}/api/logout".format(self.endpoint) + r = requests.post(path, headers={"authorization": "Bearer {}".format(token)}) + r.raise_for_status() + + def presign(self, token: str, filename: str) -> PresignedUrl: + """ + Call HF API to get a presigned url to upload `filename` to S3. + """ + path = "{}/api/presign".format(self.endpoint) + r = requests.post( + path, + headers={"authorization": "Bearer {}".format(token)}, + json={"filename": filename}, + ) + r.raise_for_status() + d = r.json() + return PresignedUrl(**d) + + def presign_and_upload(self, token: str, filename: str, filepath: str) -> str: + """ + Get a presigned url, then upload file to S3. + + Outputs: + url: Read-only url for the stored file on S3. + """ + urls = self.presign(token, filename=filename) + # streaming upload: + # https://2.python-requests.org/en/master/user/advanced/#streaming-uploads + with open(filepath, "rb") as f: + r = requests.put(urls.write, data=f) + r.raise_for_status() + return urls.access + + def list_objs(self, token: str) -> List[S3Obj]: + """ + Call HF API to list all stored files for user. + """ + path = "{}/api/listObjs".format(self.endpoint) + r = requests.get(path, headers={"authorization": "Bearer {}".format(token)}) + r.raise_for_status() + d = r.json() + return [S3Obj(**x) for x in d] + + + + + +class HfFolder: + path_token = expanduser("~/.huggingface/token") + + @classmethod + def save_token(cls, token: str): + """ + Save token, creating folder as needed. + """ + os.makedirs(os.path.dirname(cls.path_token), exist_ok=True) + with open(cls.path_token, 'w+') as f: + f.write(token) + + @classmethod + def get_token(cls): + """ + Get token or None if not existent. + """ + try: + with open(cls.path_token, 'r') as f: + return f.read() + except FileNotFoundError: + return None + + @classmethod + def delete_token(cls): + """ + Delete token. + Do not fail if token does not exist. + """ + try: + os.remove(cls.path_token) + except: + return diff --git a/transformers/tests/hf_api_test.py b/transformers/tests/hf_api_test.py new file mode 100644 index 0000000000..59822344ba --- /dev/null +++ b/transformers/tests/hf_api_test.py @@ -0,0 +1,94 @@ +# coding=utf-8 +# Copyright 2019-present, the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import, division, print_function + +import os +import time +import unittest + +from transformers.hf_api import HfApi, S3Obj, PresignedUrl, HfFolder, HTTPError + +USER = "__DUMMY_TRANSFORMERS_USER__" +PASS = "__DUMMY_TRANSFORMERS_PASS__" +FILE_KEY = "Test-{}.txt".format(int(time.time())) +FILE_PATH = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "fixtures/input.txt" +) + + + +class HfApiCommonTest(unittest.TestCase): + _api = HfApi(endpoint="https://moon-staging.huggingface.co") + + +class HfApiLoginTest(HfApiCommonTest): + def test_login_invalid(self): + with self.assertRaises(HTTPError): + self._api.login(username=USER, password="fake") + + def test_login_valid(self): + token = self._api.login(username=USER, password=PASS) + self.assertIsInstance(token, str) + + +class HfApiEndpointsTest(HfApiCommonTest): + @classmethod + def setUpClass(cls): + """ + Share this valid token in all tests below. + """ + cls._token = cls._api.login(username=USER, password=PASS) + + def test_whoami(self): + user = self._api.whoami(token=self._token) + self.assertEqual(user, USER) + + def test_presign(self): + url = self._api.presign(token=self._token, filename=FILE_KEY) + self.assertIsInstance(url, PresignedUrl) + + def test_presign_and_upload(self): + access_url = self._api.presign_and_upload( + token=self._token, filename=FILE_KEY, filepath=FILE_PATH + ) + self.assertIsInstance(access_url, str) + + def test_list_objs(self): + objs = self._api.list_objs(token=self._token) + o = objs[-1] + self.assertIsInstance(o, S3Obj) + + + +class HfFolderTest(unittest.TestCase): + def test_token_workflow(self): + """ + Test the whole token save/get/delete workflow, + with the desired behavior with respect to non-existent tokens. + """ + token = "token-{}".format(int(time.time())) + HfFolder.save_token(token) + self.assertEqual( + HfFolder.get_token(), + token + ) + HfFolder.delete_token() + HfFolder.delete_token() + # ^^ not an error, we test that the + # second call does not fail. + self.assertEqual( + HfFolder.get_token(), + None + ) From 40255ab00207f343d5e913c616c20f0f79504bfe Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Wed, 4 Dec 2019 08:21:02 +0100 Subject: [PATCH 213/505] Remove dead code in tests. --- transformers/tests/modeling_tf_common_test.py | 169 ------------------ 1 file changed, 169 deletions(-) diff --git a/transformers/tests/modeling_tf_common_test.py b/transformers/tests/modeling_tf_common_test.py index ea8cd1aecd..7445ce826a 100644 --- a/transformers/tests/modeling_tf_common_test.py +++ b/transformers/tests/modeling_tf_common_test.py @@ -233,80 +233,6 @@ class TFCommonTestCases: self.model_tester.seq_length, self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) - def test_headmasking(self): - pass - # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - - # config.output_attentions = True - # config.output_hidden_states = True - # configs_no_init = _config_zero_init(config) # To be sure we have no Nan - # for model_class in self.all_model_classes: - # model = model_class(config=configs_no_init) - # model.eval() - - # # Prepare head_mask - # # Set require_grad after having prepared the tensor to avoid error (leaf variable has been moved into the graph interior) - # head_mask = torch.ones(self.model_tester.num_hidden_layers, self.model_tester.num_attention_heads) - # head_mask[0, 0] = 0 - # head_mask[-1, :-1] = 0 - # head_mask.requires_grad_(requires_grad=True) - # inputs = inputs_dict.copy() - # inputs['head_mask'] = head_mask - - # outputs = model(**inputs) - - # # Test that we can get a gradient back for importance score computation - # output = sum(t.sum() for t in outputs[0]) - # output = output.sum() - # output.backward() - # multihead_outputs = head_mask.grad - - # attentions = outputs[-1] - # hidden_states = outputs[-2] - - # # Remove Nan - - # self.assertIsNotNone(multihead_outputs) - # self.assertEqual(len(multihead_outputs), self.model_tester.num_hidden_layers) - # self.assertAlmostEqual( - # attentions[0][..., 0, :, :].flatten().sum().item(), 0.0) - # self.assertNotEqual( - # attentions[0][..., -1, :, :].flatten().sum().item(), 0.0) - # self.assertNotEqual( - # attentions[1][..., 0, :, :].flatten().sum().item(), 0.0) - # self.assertAlmostEqual( - # attentions[-1][..., -2, :, :].flatten().sum().item(), 0.0) - # self.assertNotEqual( - # attentions[-1][..., -1, :, :].flatten().sum().item(), 0.0) - - - def test_head_pruning(self): - pass - # if not self.test_pruning: - # return - - # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - - # for model_class in self.all_model_classes: - # config.output_attentions = True - # config.output_hidden_states = False - # model = model_class(config=config) - # model.eval() - # heads_to_prune = {0: list(range(1, self.model_tester.num_attention_heads)), - # -1: [0]} - # model.prune_heads(heads_to_prune) - # outputs = model(**inputs_dict) - - # attentions = outputs[-1] - - # self.assertEqual( - # attentions[0].shape[-3], 1) - # self.assertEqual( - # attentions[1].shape[-3], self.model_tester.num_attention_heads) - # self.assertEqual( - # attentions[-1].shape[-3], self.model_tester.num_attention_heads - 1) - - def test_hidden_states_output(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -323,43 +249,6 @@ class TFCommonTestCases: list(hidden_states[0].shape[-2:]), [self.model_tester.seq_length, self.model_tester.hidden_size]) - - def test_resize_tokens_embeddings(self): - pass - # original_config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - # if not self.test_resize_embeddings: - # return - - # for model_class in self.all_model_classes: - # config = copy.deepcopy(original_config) - # model = model_class(config) - - # model_vocab_size = config.vocab_size - # # Retrieve the embeddings and clone theme - # model_embed = model.resize_token_embeddings(model_vocab_size) - # cloned_embeddings = model_embed.weight.clone() - - # # Check that resizing the token embeddings with a larger vocab size increases the model's vocab size - # model_embed = model.resize_token_embeddings(model_vocab_size + 10) - # self.assertEqual(model.config.vocab_size, model_vocab_size + 10) - # # Check that it actually resizes the embeddings matrix - # self.assertEqual(model_embed.weight.shape[0], cloned_embeddings.shape[0] + 10) - - # # Check that resizing the token embeddings with a smaller vocab size decreases the model's vocab size - # model_embed = model.resize_token_embeddings(model_vocab_size - 15) - # self.assertEqual(model.config.vocab_size, model_vocab_size - 15) - # # Check that it actually resizes the embeddings matrix - # self.assertEqual(model_embed.weight.shape[0], cloned_embeddings.shape[0] - 15) - - # # Check that adding and removing tokens has not modified the first part of the embedding matrix. - # models_equal = True - # for p1, p2 in zip(cloned_embeddings, model_embed.weight): - # if p1.data.ne(p2.data).sum() > 0: - # models_equal = False - - # self.assertTrue(models_equal) - - def test_model_common_attributes(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -369,40 +258,6 @@ class TFCommonTestCases: x = model.get_output_embeddings() assert x is None or isinstance(x, tf.keras.layers.Layer) - - def test_tie_model_weights(self): - pass - # config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - - # def check_same_values(layer_1, layer_2): - # equal = True - # for p1, p2 in zip(layer_1.weight, layer_2.weight): - # if p1.data.ne(p2.data).sum() > 0: - # equal = False - # return equal - - # for model_class in self.all_model_classes: - # if not hasattr(model_class, 'tie_weights'): - # continue - - # config.torchscript = True - # model_not_tied = model_class(config) - # params_not_tied = list(model_not_tied.parameters()) - - # config_tied = copy.deepcopy(config) - # config_tied.torchscript = False - # model_tied = model_class(config_tied) - # params_tied = list(model_tied.parameters()) - - # # Check that the embedding layer and decoding layer are the same in size and in value - # self.assertGreater(len(params_not_tied), len(params_tied)) - - # # Check that after resize they remain tied. - # model_tied.resize_token_embeddings(config.vocab_size + 10) - # params_tied_2 = list(model_tied.parameters()) - # self.assertGreater(len(params_not_tied), len(params_tied)) - # self.assertEqual(len(params_tied_2), len(params_tied)) - def test_determinism(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -461,29 +316,5 @@ def ids_tensor(shape, vocab_size, rng=None, name=None, dtype=None): return output -class TFModelUtilsTest(unittest.TestCase): - @pytest.mark.skipif('tensorflow' not in sys.modules, reason="requires TensorFlow") - def test_model_from_pretrained(self): - pass - # logging.basicConfig(level=logging.INFO) - # for model_name in list(BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - # config = BertConfig.from_pretrained(model_name) - # self.assertIsNotNone(config) - # self.assertIsInstance(config, PretrainedConfig) - - # model = BertModel.from_pretrained(model_name) - # model, loading_info = BertModel.from_pretrained(model_name, output_loading_info=True) - # self.assertIsNotNone(model) - # self.assertIsInstance(model, PreTrainedModel) - # for value in loading_info.values(): - # self.assertEqual(len(value), 0) - - # config = BertConfig.from_pretrained(model_name, output_attentions=True, output_hidden_states=True) - # model = BertModel.from_pretrained(model_name, output_attentions=True, output_hidden_states=True) - # self.assertEqual(model.config.output_attentions, True) - # self.assertEqual(model.config.output_hidden_states, True) - # self.assertEqual(model.config, config) - - if __name__ == "__main__": unittest.main() From ecb923da9cea390742a1262327a139852c5493e9 Mon Sep 17 00:00:00 2001 From: Julien Plu Date: Wed, 4 Dec 2019 09:43:15 +0100 Subject: [PATCH 214/505] Create a NER example similar to the Pytorch one. It takes the same options, and can be run the same way. --- examples/run_tf_ner.py | 612 +++++++++++++++++++++++++ transformers/__init__.py | 3 + transformers/modeling_tf_distilbert.py | 47 ++ transformers/optimization_tf.py | 254 ++++++++++ 4 files changed, 916 insertions(+) create mode 100644 examples/run_tf_ner.py create mode 100644 transformers/optimization_tf.py diff --git a/examples/run_tf_ner.py b/examples/run_tf_ner.py new file mode 100644 index 0000000000..ef1fcf6aa4 --- /dev/null +++ b/examples/run_tf_ner.py @@ -0,0 +1,612 @@ +# coding=utf-8 +import datetime +import os +import math +import glob +import re +import tensorflow as tf +import collections +import numpy as np +from seqeval import metrics +import _pickle as pickle +from absl import logging +from transformers import TF2_WEIGHTS_NAME, BertConfig, BertTokenizer, TFBertForTokenClassification +from transformers import RobertaConfig, RobertaTokenizer, TFRobertaForTokenClassification +from transformers import DistilBertConfig, DistilBertTokenizer, TFDistilBertForTokenClassification +from transformers import create_optimizer, GradientAccumulator +from utils_ner import convert_examples_to_features, get_labels, read_examples_from_file +from fastprogress import master_bar, progress_bar +from absl import flags +from absl import app + + +ALL_MODELS = sum( + (tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, RobertaConfig, DistilBertConfig)), + ()) + +MODEL_CLASSES = { + "bert": (BertConfig, TFBertForTokenClassification, BertTokenizer), + "roberta": (RobertaConfig, TFRobertaForTokenClassification, RobertaTokenizer), + "distilbert": (DistilBertConfig, TFDistilBertForTokenClassification, DistilBertTokenizer) +} + + +flags.DEFINE_string( + "data_dir", None, + "The input data dir. Should contain the .conll files (or other data files) " + "for the task.") + +flags.DEFINE_string( + "model_type", None, + "Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys())) + +flags.DEFINE_string( + "model_name_or_path", None, + "Path to pre-trained model or shortcut name selected in the list: " + ", ".join(ALL_MODELS)) + +flags.DEFINE_string( + "output_dir", None, + "The output directory where the model checkpoints will be written.") + +flags.DEFINE_string( + "labels", "", + "Path to a file containing all labels. If not specified, CoNLL-2003 labels are used.") + +flags.DEFINE_string( + "config_name", "", + "Pretrained config name or path if not the same as model_name") + +flags.DEFINE_string( + "tokenizer_name", "", + "Pretrained tokenizer name or path if not the same as model_name") + +flags.DEFINE_string( + "cache_dir", "", + "Where do you want to store the pre-trained models downloaded from s3") + +flags.DEFINE_integer( + "max_seq_length", 128, + "The maximum total input sentence length after tokenization. " + "Sequences longer than this will be truncated, sequences shorter " + "will be padded.") + +flags.DEFINE_string( + "tpu", None, + "The Cloud TPU to use for training. This should be either the name " + "used when creating the Cloud TPU, or a grpc://ip.address.of.tpu:8470 " + "url.") + +flags.DEFINE_integer( + "num_tpu_cores", 8, + "Total number of TPU cores to use.") + +flags.DEFINE_boolean( + "do_train", False, + "Whether to run training.") + +flags.DEFINE_boolean( + "do_eval", False, + "Whether to run eval on the dev set.") + +flags.DEFINE_boolean( + "do_predict", False, + "Whether to run predictions on the test set.") + +flags.DEFINE_boolean( + "evaluate_during_training", False, + "Whether to run evaluation during training at each logging step.") + +flags.DEFINE_boolean( + "do_lower_case", False, + "Set this flag if you are using an uncased model.") + +flags.DEFINE_integer( + "per_device_train_batch_size", 8, + "Batch size per GPU/CPU/TPU for training.") + +flags.DEFINE_integer( + "per_device_eval_batch_size", 8, + "Batch size per GPU/CPU/TPU for evaluation.") + +flags.DEFINE_integer( + "gradient_accumulation_steps", 1, + "Number of updates steps to accumulate before performing a backward/update pass.") + +flags.DEFINE_float( + "learning_rate", 5e-5, + "The initial learning rate for Adam.") + +flags.DEFINE_float( + "weight_decay", 0.0, + "Weight decay if we apply some.") + +flags.DEFINE_float( + "adam_epsilon", 1e-8, + "Epsilon for Adam optimizer.") + +flags.DEFINE_float( + "max_grad_norm", 1.0, + "Max gradient norm.") + +flags.DEFINE_integer( + "num_train_epochs", 3, + "Total number of training epochs to perform.") + +flags.DEFINE_integer( + "max_steps", -1, + "If > 0: set total number of training steps to perform. Override num_train_epochs.") + +flags.DEFINE_integer( + "warmup_steps", 0, + "Linear warmup over warmup_steps.") + +flags.DEFINE_integer( + "logging_steps", 50, + "Log every X updates steps.") + +flags.DEFINE_integer( + "save_steps", 50, + "Save checkpoint every X updates steps.") + +flags.DEFINE_boolean( + "eval_all_checkpoints", False, + "Evaluate all checkpoints starting with the same prefix as model_name ending and ending with step number") + +flags.DEFINE_boolean( + "no_cuda", False, + "Avoid using CUDA when available") + +flags.DEFINE_boolean( + "overwrite_output_dir", False, + "Overwrite the content of the output directory") + +flags.DEFINE_boolean( + "overwrite_cache", False, + "Overwrite the cached training and evaluation sets") + +flags.DEFINE_integer( + "seed", 42, + "random seed for initialization") + +flags.DEFINE_boolean( + "fp16", False, + "Whether to use 16-bit (mixed) precision instead of 32-bit") + +flags.DEFINE_string( + "gpus", "0", + "Comma separated list of gpus devices. If only one, switch to single " + "gpu strategy, if None takes all the gpus available.") + + +def train(args, strategy, train_dataset, tokenizer, model, num_train_examples, labels, train_batch_size, pad_token_label_id): + if args['max_steps'] > 0: + num_train_steps = args['max_steps'] * args['gradient_accumulation_steps'] + args['num_train_epochs'] = 1 + else: + num_train_steps = math.ceil(num_train_examples / train_batch_size) // args['gradient_accumulation_steps'] * args['num_train_epochs'] + + writer = tf.summary.create_file_writer("/tmp/mylogs") + + with strategy.scope(): + loss_fct = tf.keras.losses.SparseCategoricalCrossentropy(reduction=tf.keras.losses.Reduction.NONE) + optimizer = create_optimizer(args['learning_rate'], num_train_steps, args['warmup_steps']) + + if args['fp16']: + optimizer = tf.keras.mixed_precision.experimental.LossScaleOptimizer(optimizer, 'dynamic') + + loss_metric = tf.keras.metrics.Mean(name='loss', dtype=tf.float32) + gradient_accumulator = GradientAccumulator() + + logging.info("***** Running training *****") + logging.info(" Num examples = %d", num_train_examples) + logging.info(" Num Epochs = %d", args['num_train_epochs']) + logging.info(" Instantaneous batch size per device = %d", args['per_device_train_batch_size']) + logging.info(" Total train batch size (w. parallel, distributed & accumulation) = %d", + train_batch_size * args['gradient_accumulation_steps']) + logging.info(" Gradient Accumulation steps = %d", args['gradient_accumulation_steps']) + logging.info(" Total training steps = %d", num_train_steps) + + model.summary() + + @tf.function + def apply_gradients(): + grads_and_vars = [] + + for gradient, variable in zip(gradient_accumulator.gradients, model.trainable_variables): + if gradient is not None: + scaled_gradient = gradient / (args['n_device'] * args['gradient_accumulation_steps']) + grads_and_vars.append((scaled_gradient, variable)) + else: + grads_and_vars.append((gradient, variable)) + + optimizer.apply_gradients(grads_and_vars, args['max_grad_norm']) + gradient_accumulator.reset() + + @tf.function + def train_step(train_features, train_labels): + def step_fn(train_features, train_labels): + inputs = {'attention_mask': train_features['input_mask'], 'training': True} + + if args['model_type'] != "distilbert": + inputs["token_type_ids"] = train_features['segment_ids'] if args['model_type'] in ["bert", "xlnet"] else None + + with tf.GradientTape() as tape: + logits = model(train_features['input_ids'], **inputs)[0] + logits = tf.reshape(logits, (-1, len(labels) + 1)) + active_loss = tf.reshape(train_features['input_mask'], (-1,)) + active_logits = tf.boolean_mask(logits, active_loss) + train_labels = tf.reshape(train_labels, (-1,)) + active_labels = tf.boolean_mask(train_labels, active_loss) + cross_entropy = loss_fct(active_labels, active_logits) + loss = tf.reduce_sum(cross_entropy) * (1.0 / train_batch_size) + grads = tape.gradient(loss, model.trainable_variables) + + gradient_accumulator(grads) + + return cross_entropy + + per_example_losses = strategy.experimental_run_v2(step_fn, args=(train_features, train_labels)) + mean_loss = strategy.reduce(tf.distribute.ReduceOp.MEAN, per_example_losses, axis=0) + + return mean_loss + + current_time = datetime.datetime.now() + train_iterator = master_bar(range(args['num_train_epochs'])) + global_step = 0 + logging_loss = 0.0 + + for epoch in train_iterator: + epoch_iterator = progress_bar(train_dataset, total=num_train_steps, parent=train_iterator, display=args['n_device'] > 1) + step = 1 + + with strategy.scope(): + for train_features, train_labels in epoch_iterator: + loss = train_step(train_features, train_labels) + + if step % args['gradient_accumulation_steps'] == 0: + strategy.experimental_run_v2(apply_gradients) + + loss_metric(loss) + + global_step += 1 + + if args['logging_steps'] > 0 and global_step % args['logging_steps'] == 0: + # Log metrics + if args['n_device'] == 1 and args['evaluate_during_training']: # Only evaluate when single GPU otherwise metrics may not average well + y_true, y_pred, eval_loss = evaluate(args, strategy, model, tokenizer, labels, pad_token_label_id, mode="dev") + report = metrics.classification_report(y_true, y_pred, digits=4) + + logging.info("Eval at step " + str(global_step) + "\n" + report) + logging.info("eval_loss: " + str(eval_loss)) + + precision = metrics.precision_score(y_true, y_pred) + recall = metrics.recall_score(y_true, y_pred) + f1 = metrics.f1_score(y_true, y_pred) + + with writer.as_default(): + tf.summary.scalar("eval_loss", eval_loss, global_step) + tf.summary.scalar("precision", precision, global_step) + tf.summary.scalar("recall", recall, global_step) + tf.summary.scalar("f1", f1, global_step) + + lr = optimizer.learning_rate + learning_rate = lr(step) + + with writer.as_default(): + tf.summary.scalar("lr", learning_rate, global_step) + tf.summary.scalar("loss", (loss_metric.result() - logging_loss) / args['logging_steps'], global_step) + + logging_loss = loss_metric.result() + + with writer.as_default(): + tf.summary.scalar("loss", loss_metric.result(), step=step) + + if args['save_steps'] > 0 and global_step % args['save_steps'] == 0: + # Save model checkpoint + output_dir = os.path.join(args['output_dir'], "checkpoint-{}".format(global_step)) + + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + model.save_pretrained(output_dir) + logging.info("Saving model checkpoint to %s", output_dir) + + train_iterator.child.comment = f'loss : {loss_metric.result()}' + step += 1 + + train_iterator.write(f'loss epoch {epoch + 1}: {loss_metric.result()}') + + loss_metric.reset_states() + + logging.info(" Training took time = {}".format(datetime.datetime.now() - current_time)) + + +def evaluate(args, strategy, model, tokenizer, labels, pad_token_label_id, mode): + eval_batch_size = args['per_device_eval_batch_size'] * args['n_device'] + eval_dataset, size = load_and_cache_examples(args, tokenizer, labels, pad_token_label_id, eval_batch_size, mode=mode) + eval_dataset = strategy.experimental_distribute_dataset(eval_dataset) + preds = None + num_eval_steps = math.ceil(size / eval_batch_size) + master = master_bar(range(1)) + eval_iterator = progress_bar(eval_dataset, total=num_eval_steps, parent=master, display=args['n_device'] > 1) + loss_fct = tf.keras.losses.SparseCategoricalCrossentropy(reduction=tf.keras.losses.Reduction.NONE) + loss = 0.0 + + logging.info("***** Running evaluation *****") + logging.info(" Num examples = %d", size) + logging.info(" Batch size = %d", eval_batch_size) + + for eval_features, eval_labels in eval_iterator: + inputs = {'attention_mask': eval_features['input_mask'], 'training': False} + + if args['model_type'] != "distilbert": + inputs["token_type_ids"] = eval_features['segment_ids'] if args['model_type'] in ["bert", "xlnet"] else None + + with strategy.scope(): + logits = model(eval_features['input_ids'], **inputs)[0] + tmp_logits = tf.reshape(logits, (-1, len(labels) + 1)) + active_loss = tf.reshape(eval_features['input_mask'], (-1,)) + active_logits = tf.boolean_mask(tmp_logits, active_loss) + tmp_eval_labels = tf.reshape(eval_labels, (-1,)) + active_labels = tf.boolean_mask(tmp_eval_labels, active_loss) + cross_entropy = loss_fct(active_labels, active_logits) + loss += tf.reduce_sum(cross_entropy) * (1.0 / eval_batch_size) + + if preds is None: + preds = logits.numpy() + label_ids = eval_labels.numpy() + else: + preds = np.append(preds, logits.numpy(), axis=0) + label_ids = np.append(label_ids, eval_labels.numpy(), axis=0) + + preds = np.argmax(preds, axis=2) + y_pred = [[] for _ in range(label_ids.shape[0])] + y_true = [[] for _ in range(label_ids.shape[0])] + loss = loss / num_eval_steps + + for i in range(label_ids.shape[0]): + for j in range(label_ids.shape[1]): + if label_ids[i, j] != pad_token_label_id: + y_pred[i].append(labels[preds[i, j] - 1]) + y_true[i].append(labels[label_ids[i, j] - 1]) + + return y_true, y_pred, loss.numpy() + + +def load_cache(cached_file, max_seq_length): + name_to_features = { + "input_ids": tf.io.FixedLenFeature([max_seq_length], tf.int64), + "input_mask": tf.io.FixedLenFeature([max_seq_length], tf.int64), + "segment_ids": tf.io.FixedLenFeature([max_seq_length], tf.int64), + "label_ids": tf.io.FixedLenFeature([max_seq_length], tf.int64), + } + + def _decode_record(record): + example = tf.io.parse_single_example(record, name_to_features) + features = {} + features['input_ids'] = example['input_ids'] + features['input_mask'] = example['input_mask'] + features['segment_ids'] = example['segment_ids'] + + return features, example['label_ids'] + + d = tf.data.TFRecordDataset(cached_file) + d = d.map(_decode_record, num_parallel_calls=4) + count = d.reduce(0, lambda x, _: x + 1) + + return d, count.numpy() + + +def save_cache(features, cached_features_file): + writer = tf.io.TFRecordWriter(cached_features_file) + + for (ex_index, feature) in enumerate(features): + if ex_index % 5000 == 0: + logging.info("Writing example %d of %d" % (ex_index, len(features))) + + def create_int_feature(values): + f = tf.train.Feature(int64_list=tf.train.Int64List(value=list(values))) + return f + + record_feature = collections.OrderedDict() + record_feature["input_ids"] = create_int_feature(feature.input_ids) + record_feature["input_mask"] = create_int_feature(feature.input_mask) + record_feature["segment_ids"] = create_int_feature(feature.segment_ids) + record_feature["label_ids"] = create_int_feature(feature.label_ids) + + tf_example = tf.train.Example(features=tf.train.Features(feature=record_feature)) + + writer.write(tf_example.SerializeToString()) + + writer.close() + + +def load_and_cache_examples(args, tokenizer, labels, pad_token_label_id, batch_size, mode): + drop_remainder = True if args['tpu'] or mode == 'train' else False + + # Load data features from cache or dataset file + cached_features_file = os.path.join(args['data_dir'], "cached_{}_{}_{}.tf_record".format(mode, + list(filter(None, args['model_name_or_path'].split("/"))).pop(), + str(args['max_seq_length']))) + if os.path.exists(cached_features_file) and not args['overwrite_cache']: + logging.info("Loading features from cached file %s", cached_features_file) + dataset, size = load_cache(cached_features_file, args['max_seq_length']) + else: + logging.info("Creating features from dataset file at %s", args['data_dir']) + examples = read_examples_from_file(args['data_dir'], mode) + features = convert_examples_to_features(examples, labels, args['max_seq_length'], tokenizer, + cls_token_at_end=bool(args['model_type'] in ["xlnet"]), + # xlnet has a cls token at the end + cls_token=tokenizer.cls_token, + cls_token_segment_id=2 if args['model_type'] in ["xlnet"] else 0, + sep_token=tokenizer.sep_token, + sep_token_extra=bool(args['model_type'] in ["roberta"]), + # roberta uses an extra separator b/w pairs of sentences, cf. github.com/pytorch/fairseq/commit/1684e166e3da03f5b600dbb7855cb98ddfcd0805 + pad_on_left=bool(args['model_type'] in ["xlnet"]), + # pad on the left for xlnet + pad_token=tokenizer.convert_tokens_to_ids([tokenizer.pad_token])[0], + pad_token_segment_id=4 if args['model_type'] in ["xlnet"] else 0, + pad_token_label_id=pad_token_label_id + ) + logging.info("Saving features into cached file %s", cached_features_file) + save_cache(features, cached_features_file) + dataset, size = load_cache(cached_features_file, args['max_seq_length']) + + if mode == 'train': + dataset = dataset.repeat() + dataset = dataset.shuffle(buffer_size=8192, seed=args['seed']) + + dataset = dataset.batch(batch_size, drop_remainder) + dataset = dataset.prefetch(buffer_size=batch_size) + + return dataset, size + + +def main(_): + logging.set_verbosity(logging.INFO) + args = flags.FLAGS.flag_values_dict() + + if os.path.exists(args['output_dir']) and os.listdir( + args['output_dir']) and args['do_train'] and not args['overwrite_output_dir']: + raise ValueError( + "Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.".format( + args['output_dir'])) + + if args['fp16']: + tf.config.optimizer.set_experimental_options({"auto_mixed_precision": True}) + + if args['tpu']: + resolver = tf.distribute.cluster_resolver.TPUClusterResolver(tpu=args['tpu']) + tf.config.experimental_connect_to_cluster(resolver) + tf.tpu.experimental.initialize_tpu_system(resolver) + strategy = tf.distribute.experimental.TPUStrategy(resolver) + args['n_device'] = args['num_tpu_cores'] + elif len(args['gpus'].split(',')) > 1: + args['n_device'] = len([f"/gpu:{gpu}" for gpu in args['gpus'].split(',')]) + strategy = tf.distribute.MirroredStrategy(devices=[f"/gpu:{gpu}" for gpu in args['gpus'].split(',')]) + elif args['no_cuda']: + args['n_device'] = 1 + strategy = tf.distribute.OneDeviceStrategy(device="/cpu:0") + else: + args['n_device'] = len(args['gpus'].split(',')) + strategy = tf.distribute.OneDeviceStrategy(device="/gpu:" + args['gpus'].split(',')[0]) + + logging.warning("n_device: %s, distributed training: %s, 16-bits training: %s", + args['n_device'], bool(args['n_device'] > 1), args['fp16']) + + labels = get_labels(args['labels']) + num_labels = len(labels) + 1 + pad_token_label_id = 0 + config_class, model_class, tokenizer_class = MODEL_CLASSES[args['model_type']] + config = config_class.from_pretrained(args['config_name'] if args['config_name'] else args['model_name_or_path'], + num_labels=num_labels, + cache_dir=args['cache_dir'] if args['cache_dir'] else None) + + logging.info("Training/evaluation parameters %s", args) + + # Training + if args['do_train']: + tokenizer = tokenizer_class.from_pretrained(args['tokenizer_name'] if args['tokenizer_name'] else args['model_name_or_path'], + do_lower_case=args['do_lower_case'], + cache_dir=args['cache_dir'] if args['cache_dir'] else None) + + with strategy.scope(): + model = model_class.from_pretrained(args['model_name_or_path'], + from_pt=bool(".bin" in args['model_name_or_path']), + config=config, + cache_dir=args['cache_dir'] if args['cache_dir'] else None) + model.layers[-1].activation = tf.keras.activations.softmax + + train_batch_size = args['per_device_train_batch_size'] * args['n_device'] + train_dataset, num_train_examples = load_and_cache_examples(args, tokenizer, labels, pad_token_label_id, train_batch_size, mode="train") + train_dataset = strategy.experimental_distribute_dataset(train_dataset) + train(args, strategy, train_dataset, tokenizer, model, num_train_examples, labels, train_batch_size, pad_token_label_id) + + if not os.path.exists(args['output_dir']): + os.makedirs(args['output_dir']) + + logging.info("Saving model to %s", args['output_dir']) + + model.save_pretrained(args['output_dir']) + tokenizer.save_pretrained(args['output_dir']) + + # Evaluation + if args['do_eval']: + tokenizer = tokenizer_class.from_pretrained(args['output_dir'], do_lower_case=args['do_lower_case']) + checkpoints = [] + results = [] + + if args['eval_all_checkpoints']: + checkpoints = list(os.path.dirname(c) for c in sorted(glob.glob(args['output_dir'] + "/**/" + TF2_WEIGHTS_NAME, recursive=True), key=lambda f: int(''.join(filter(str.isdigit, f)) or -1))) + + logging.info("Evaluate the following checkpoints: %s", checkpoints) + + for checkpoint in checkpoints: + global_step = checkpoint.split("-")[-1] if re.match(".*checkpoint-[0-9]", checkpoint) else "final" + + with strategy.scope(): + model = model_class.from_pretrained(checkpoint) + + y_true, y_pred, eval_loss = evaluate(args, strategy, model, tokenizer, labels, pad_token_label_id, mode="dev") + report = metrics.classification_report(y_true, y_pred, digits=4) + + if global_step: + results.append({global_step + "_report": report, global_step + "_loss": eval_loss}) + + output_eval_file = os.path.join(args['output_dir'], "eval_results.txt") + + with tf.io.gfile.GFile(output_eval_file, "w") as writer: + for res in results: + for key, val in res.items(): + if "loss" in key: + logging.info(key + " = " + str(val)) + writer.write(key + " = " + str(val)) + writer.write("\n") + else: + logging.info(key) + logging.info("\n" + report) + writer.write(key + "\n") + writer.write(report) + writer.write("\n") + + if args['do_predict']: + tokenizer = tokenizer_class.from_pretrained(args['output_dir'], do_lower_case=args['do_lower_case']) + model = model_class.from_pretrained(args['output_dir']) + eval_batch_size = args['per_gpu_eval_batch_size'] * args['n_device'] + predict_dataset, _ = load_and_cache_examples(args, tokenizer, labels, pad_token_label_id, eval_batch_size, mode="test") + y_true, y_pred, pred_loss = evaluate(args, strategy, model, tokenizer, labels, pad_token_label_id, mode="test") + output_test_results_file = os.path.join(args.output_dir, "test_results.txt") + output_test_predictions_file = os.path.join(args['output_dir'], "test_predictions.txt") + report = metrics.classification_report(y_true, y_pred, digits=4) + + with tf.io.gfile.GFile(output_test_results_file, "w") as writer: + report = metrics.classification_report(y_true, y_pred, digits=4) + + logging.info("\n" + report) + + writer.write(report) + writer.write("\n\nloss = " + str(pred_loss)) + + with tf.io.gfile.GFile(output_test_predictions_file, "w") as writer: + with tf.io.gfile.GFile(os.path.join(args['data_dir'], "test.txt"), "r") as f: + example_id = 0 + + for line in f: + if line.startswith("-DOCSTART-") or line == "" or line == "\n": + writer.write(line) + + if not y_pred[example_id]: + example_id += 1 + elif y_pred[example_id]: + output_line = line.split()[0] + " " + y_pred[example_id].pop(0) + "\n" + writer.write(output_line) + else: + logging.warning("Maximum sequence length exceeded: No prediction for '%s'.", line.split()[0]) + + +if __name__ == "__main__": + flags.mark_flag_as_required("data_dir") + flags.mark_flag_as_required("output_dir") + flags.mark_flag_as_required("model_name_or_path") + flags.mark_flag_as_required("model_type") + app.run(main) diff --git a/transformers/__init__.py b/transformers/__init__.py index 970bdf0cf1..2f74b7e79c 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -162,6 +162,7 @@ if is_tf_available(): from .modeling_tf_distilbert import (TFDistilBertPreTrainedModel, TFDistilBertMainLayer, TFDistilBertModel, TFDistilBertForMaskedLM, TFDistilBertForSequenceClassification, + TFDistilBertForTokenClassification TFDistilBertForQuestionAnswering, TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP) @@ -172,6 +173,8 @@ if is_tf_available(): from .modeling_tf_albert import (TFAlbertPreTrainedModel, TFAlbertModel, TFAlbertForMaskedLM, TFAlbertForSequenceClassification, TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) + # Optimization + from .optimization_tf import (WarmUp, create_optimizer, AdamWeightDecay, GradientAccumulator) # TF 2.0 <=> PyTorch conversion utilities from .modeling_tf_pytorch_utils import (convert_tf_weight_name_to_pt_weight_name, diff --git a/transformers/modeling_tf_distilbert.py b/transformers/modeling_tf_distilbert.py index b3d4889475..8e1aef7462 100644 --- a/transformers/modeling_tf_distilbert.py +++ b/transformers/modeling_tf_distilbert.py @@ -703,6 +703,53 @@ class TFDistilBertForSequenceClassification(TFDistilBertPreTrainedModel): return outputs # logits, (hidden_states), (attentions) +@add_start_docstrings("""DistilBert Model with a token classification head on top (a linear layer on top of + the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. """, + DISTILBERT_START_DOCSTRING, DISTILBERT_INPUTS_DOCSTRING) +class TFDistilBertForTokenClassification(TFDistilBertPreTrainedModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **scores**: ``Numpy array`` or ``tf.Tensor`` of shape ``(batch_size, sequence_length, config.num_labels)`` + Classification scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``Numpy array`` or ``tf.Tensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``Numpy array`` or ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + Examples:: + import tensorflow as tf + from transformers import DistilBertTokenizer, TFDistilBertForTokenClassification + tokenizer = DistilBertTokenizer.from_pretrained('bert-base-uncased') + model = TFDistilBertForTokenClassification.from_pretrained('bert-base-uncased') + input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 + outputs = model(input_ids) + scores = outputs[0] + """ + def __init__(self, config, *inputs, **kwargs): + super(TFDistilBertForTokenClassification, self).__init__(config, *inputs, **kwargs) + self.num_labels = config.num_labels + + self.distilbert = TFDistilBertMainLayer(config, name='distilbert') + self.dropout = tf.keras.layers.Dropout(config.dropout) + self.classifier = tf.keras.layers.Dense(config.num_labels, + kernel_initializer=get_initializer(config.initializer_range), + name='classifier') + + def call(self, inputs, **kwargs): + outputs = self.distilbert(inputs, **kwargs) + + sequence_output = outputs[0] + + sequence_output = self.dropout(sequence_output, training=kwargs.get('training', False)) + logits = self.classifier(sequence_output) + + outputs = (logits,) + outputs[2:] # add hidden states and attention if they are here + + return outputs # scores, (hidden_states), (attentions) + + @add_start_docstrings("""DistilBert Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`). """, DISTILBERT_START_DOCSTRING, DISTILBERT_INPUTS_DOCSTRING) diff --git a/transformers/optimization_tf.py b/transformers/optimization_tf.py new file mode 100644 index 0000000000..c5fa248083 --- /dev/null +++ b/transformers/optimization_tf.py @@ -0,0 +1,254 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Functions and classes related to optimization (weight updates).""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import re + +import tensorflow as tf + + +class WarmUp(tf.keras.optimizers.schedules.LearningRateSchedule): + """Applys a warmup schedule on a given learning rate decay schedule.""" + + def __init__( + self, + initial_learning_rate, + decay_schedule_fn, + warmup_steps, + power=1.0, + name=None): + super(WarmUp, self).__init__() + self.initial_learning_rate = initial_learning_rate + self.warmup_steps = warmup_steps + self.power = power + self.decay_schedule_fn = decay_schedule_fn + self.name = name + + def __call__(self, step): + with tf.name_scope(self.name or 'WarmUp') as name: + # Implements polynomial warmup. i.e., if global_step < warmup_steps, the + # learning rate will be `global_step/num_warmup_steps * init_lr`. + global_step_float = tf.cast(step, tf.float32) + warmup_steps_float = tf.cast(self.warmup_steps, tf.float32) + warmup_percent_done = global_step_float / warmup_steps_float + warmup_learning_rate = ( + self.initial_learning_rate * + tf.math.pow(warmup_percent_done, self.power)) + return tf.cond(global_step_float < warmup_steps_float, + lambda: warmup_learning_rate, + lambda: self.decay_schedule_fn(step), + name=name) + + def get_config(self): + return { + 'initial_learning_rate': self.initial_learning_rate, + 'decay_schedule_fn': self.decay_schedule_fn, + 'warmup_steps': self.warmup_steps, + 'power': self.power, + 'name': self.name + } + + +def create_optimizer(init_lr, num_train_steps, num_warmup_steps): + """Creates an optimizer with learning rate schedule.""" + # Implements linear decay of the learning rate. + learning_rate_fn = tf.keras.optimizers.schedules.PolynomialDecay( + initial_learning_rate=init_lr, + decay_steps=num_train_steps, + end_learning_rate=0.0) + if num_warmup_steps: + learning_rate_fn = WarmUp(initial_learning_rate=init_lr, + decay_schedule_fn=learning_rate_fn, + warmup_steps=num_warmup_steps) + optimizer = AdamWeightDecay( + learning_rate=learning_rate_fn, + weight_decay_rate=0.01, + beta_1=0.9, + beta_2=0.999, + epsilon=1e-6, + exclude_from_weight_decay=['layer_norm', 'bias']) + return optimizer + + +class AdamWeightDecay(tf.keras.optimizers.Adam): + """Adam enables L2 weight decay and clip_by_global_norm on gradients. + + Just adding the square of the weights to the loss function is *not* the + correct way of using L2 regularization/weight decay with Adam, since that will + interact with the m and v parameters in strange ways. + + Instead we want ot decay the weights in a manner that doesn't interact with + the m/v parameters. This is equivalent to adding the square of the weights to + the loss with plain (non-momentum) SGD. + """ + + def __init__(self, + learning_rate=0.001, + beta_1=0.9, + beta_2=0.999, + epsilon=1e-7, + amsgrad=False, + weight_decay_rate=0.0, + include_in_weight_decay=None, + exclude_from_weight_decay=None, + name='AdamWeightDecay', + **kwargs): + super(AdamWeightDecay, self).__init__( + learning_rate, beta_1, beta_2, epsilon, amsgrad, name, **kwargs) + self.weight_decay_rate = weight_decay_rate + self._include_in_weight_decay = include_in_weight_decay + self._exclude_from_weight_decay = exclude_from_weight_decay + + @classmethod + def from_config(cls, config): + """Creates an optimizer from its config with WarmUp custom object.""" + custom_objects = {'WarmUp': WarmUp} + return super(AdamWeightDecay, cls).from_config( + config, custom_objects=custom_objects) + + def _prepare_local(self, var_device, var_dtype, apply_state): + super(AdamWeightDecay, self)._prepare_local(var_device, var_dtype, + apply_state) + apply_state['weight_decay_rate'] = tf.constant( + self.weight_decay_rate, name='adam_weight_decay_rate') + + def _decay_weights_op(self, var, learning_rate, apply_state): + do_decay = self._do_use_weight_decay(var.name) + if do_decay: + return var.assign_sub( + learning_rate * var * + apply_state['weight_decay_rate'], + use_locking=self._use_locking) + return tf.no_op() + + def apply_gradients(self, grads_and_vars, clip_norm, name=None): + grads, tvars = list(zip(*grads_and_vars)) + (grads, _) = tf.clip_by_global_norm(grads, clip_norm=clip_norm) + return super(AdamWeightDecay, self).apply_gradients(zip(grads, tvars)) + + def _get_lr(self, var_device, var_dtype, apply_state): + """Retrieves the learning rate with the given state.""" + if apply_state is None: + return self._decayed_lr_t[var_dtype], {} + + apply_state = apply_state or {} + coefficients = apply_state.get((var_device, var_dtype)) + if coefficients is None: + coefficients = self._fallback_apply_state(var_device, var_dtype) + apply_state[(var_device, var_dtype)] = coefficients + + return coefficients['lr_t'], dict(apply_state=apply_state) + + def _resource_apply_dense(self, grad, var, apply_state=None): + lr_t, kwargs = self._get_lr(var.device, var.dtype.base_dtype, apply_state) + decay = self._decay_weights_op(var, lr_t, apply_state) + with tf.control_dependencies([decay]): + return super(AdamWeightDecay, self)._resource_apply_dense( + grad, var, **kwargs) + + def _resource_apply_sparse(self, grad, var, indices, apply_state=None): + lr_t, kwargs = self._get_lr(var.device, var.dtype.base_dtype, apply_state) + decay = self._decay_weights_op(var, lr_t, apply_state) + with tf.control_dependencies([decay]): + return super(AdamWeightDecay, self)._resource_apply_sparse( + grad, var, indices, **kwargs) + + def get_config(self): + config = super(AdamWeightDecay, self).get_config() + config.update({ + 'weight_decay_rate': self.weight_decay_rate, + }) + return config + + def _do_use_weight_decay(self, param_name): + """Whether to use L2 weight decay for `param_name`.""" + if self.weight_decay_rate == 0: + return False + + if self._include_in_weight_decay: + for r in self._include_in_weight_decay: + if re.search(r, param_name) is not None: + return True + + if self._exclude_from_weight_decay: + for r in self._exclude_from_weight_decay: + if re.search(r, param_name) is not None: + return False + return True + + +## Inspired from https://github.com/OpenNMT/OpenNMT-tf/blob/master/opennmt/optimizers/utils.py +class GradientAccumulator(object): + """Distribution strategies-aware gradient accumulation utility.""" + + def __init__(self): + """Initializes the accumulator.""" + self._gradients = [] + self._accum_steps = tf.Variable( + initial_value=0, + dtype=tf.int64, + trainable=False, + aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA) + + @property + def step(self): + """Number of accumulated steps.""" + return self._accum_steps.value() + + @property + def gradients(self): + """The accumulated gradients.""" + return list(gradient.value() if gradient is not None else gradient for gradient in self._get_replica_gradients()) + + def __call__(self, gradients): + """Accumulates :obj:`gradients`.""" + if not self._gradients: + self._gradients.extend([tf.Variable(tf.zeros_like(gradient), trainable=False) if gradient is not None else gradient for gradient in gradients]) + + if len(gradients) != len(self._gradients): + raise ValueError("Expected %s gradients, but got %d" % (len(self._gradients), len(gradients))) + + for accum_gradient, gradient in zip(self._get_replica_gradients(), gradients): + if accum_gradient is not None: + accum_gradient.assign_add(gradient) + + self._accum_steps.assign_add(1) + + def reset(self): + """Resets the accumulated gradients.""" + if self._gradients: + self._accum_steps.assign(0) + + for gradient in self._get_replica_gradients(): + if gradient is not None: + gradient.assign(tf.zeros_like(gradient)) + + def _get_replica_gradients(self): + if tf.distribute.has_strategy(): + # In a replica context, we want to accumulate gradients on each replica + # without synchronization, so we directly assign the value of the + # current replica. + replica_context = tf.distribute.get_replica_context() + + if replica_context is None or tf.distribute.get_strategy().num_replicas_in_sync == 1: + return self._gradients + + return (gradient.device_map.select_for_current_replica(gradient.values, replica_context) for gradient in self._gradients) + else: + return self._gradients From 254ebb979c09d2e8f7efeb11d46bd1196f856699 Mon Sep 17 00:00:00 2001 From: Julien Plu Date: Wed, 4 Dec 2019 10:00:25 +0100 Subject: [PATCH 215/505] Bugfix on init file. Missing comma. --- transformers/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/__init__.py b/transformers/__init__.py index 2f74b7e79c..e4f5984c70 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -162,7 +162,7 @@ if is_tf_available(): from .modeling_tf_distilbert import (TFDistilBertPreTrainedModel, TFDistilBertMainLayer, TFDistilBertModel, TFDistilBertForMaskedLM, TFDistilBertForSequenceClassification, - TFDistilBertForTokenClassification + TFDistilBertForTokenClassification, TFDistilBertForQuestionAnswering, TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP) From 5bfcd0485ece086ebcbed2d008813037968a9e58 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Wed, 4 Dec 2019 14:53:11 +0100 Subject: [PATCH 216/505] fix #1991 --- examples/run_lm_finetuning.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py index 0bb7460353..a5eaf524ac 100644 --- a/examples/run_lm_finetuning.py +++ b/examples/run_lm_finetuning.py @@ -217,7 +217,10 @@ def train(args, train_dataset, model, tokenizer): global_step = 0 tr_loss, logging_loss = 0.0, 0.0 - model.resize_token_embeddings(len(tokenizer)) + + model_to_resize = model.module if hasattr(model, 'module') else model # Take care of distributed/parallel training + model_to_resize.resize_token_embeddings(len(tokenizer)) + model.zero_grad() train_iterator = trange(int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0]) set_seed(args) # Added here for reproducibility (even between python 2 and 3) From 9ddc3f1a1227fc9cbe4e5a5c20b21546e438dfb1 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Wed, 4 Dec 2019 10:37:00 -0500 Subject: [PATCH 217/505] Naming update + XLNet/XLM evaluation --- examples/run_squad.py | 6 +- transformers/data/metrics/squad_metrics.py | 97 ++++++++++++++++++---- 2 files changed, 85 insertions(+), 18 deletions(-) diff --git a/examples/run_squad.py b/examples/run_squad.py index b7952487dc..a9ef5c6ba2 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -17,7 +17,7 @@ from __future__ import absolute_import, division, print_function from transformers.data.processors.squad import SquadV1Processor, SquadV2Processor, SquadResult -from transformers.data.metrics.squad_metrics import compute_predictions, compute_predictions_extended, squad_evaluate +from transformers.data.metrics.squad_metrics import compute_predictions_logits, compute_predictions_log_probs, squad_evaluate import argparse import logging @@ -264,13 +264,13 @@ def evaluate(args, model, tokenizer, prefix=""): if args.model_type in ['xlnet', 'xlm']: # XLNet uses a more complex post-processing procedure - predictions = compute_predictions_extended(examples, features, all_results, args.n_best_size, + predictions = compute_predictions_log_probs(examples, features, all_results, args.n_best_size, args.max_answer_length, output_prediction_file, output_nbest_file, output_null_log_odds_file, args.predict_file, model.config.start_n_top, model.config.end_n_top, args.version_2_with_negative, tokenizer, args.verbose_logging) else: - predictions = compute_predictions(examples, features, all_results, args.n_best_size, + predictions = compute_predictions_logits(examples, features, all_results, args.n_best_size, args.max_answer_length, args.do_lower_case, output_prediction_file, output_nbest_file, output_null_log_odds_file, args.verbose_logging, args.version_2_with_negative, args.null_score_diff_threshold) diff --git a/transformers/data/metrics/squad_metrics.py b/transformers/data/metrics/squad_metrics.py index 83647a20d0..1f120d354a 100644 --- a/transformers/data/metrics/squad_metrics.py +++ b/transformers/data/metrics/squad_metrics.py @@ -125,6 +125,53 @@ def merge_eval(main_eval, new_eval, prefix): main_eval['%s_%s' % (prefix, k)] = new_eval[k] +def find_best_thresh_v2(preds, scores, na_probs, qid_to_has_ans): + num_no_ans = sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]) + cur_score = num_no_ans + best_score = cur_score + best_thresh = 0.0 + qid_list = sorted(na_probs, key=lambda k: na_probs[k]) + for i, qid in enumerate(qid_list): + if qid not in scores: + continue + if qid_to_has_ans[qid]: + diff = scores[qid] + else: + if preds[qid]: + diff = -1 + else: + diff = 0 + cur_score += diff + if cur_score > best_score: + best_score = cur_score + best_thresh = na_probs[qid] + + has_ans_score, has_ans_cnt = 0, 0 + for qid in qid_list: + if not qid_to_has_ans[qid]: + continue + has_ans_cnt += 1 + + if qid not in scores: + continue + has_ans_score += scores[qid] + + return 100.0 * best_score / len(scores), best_thresh, 1.0 * has_ans_score / has_ans_cnt + + +def find_all_best_thresh_v2(main_eval, preds, exact_raw, f1_raw, na_probs, qid_to_has_ans): + best_exact, exact_thresh, has_ans_exact = find_best_thresh_v2( + preds, exact_raw, na_probs, qid_to_has_ans) + best_f1, f1_thresh, has_ans_f1 = find_best_thresh_v2( + preds, f1_raw, na_probs, qid_to_has_ans) + main_eval['best_exact'] = best_exact + main_eval['best_exact_thresh'] = exact_thresh + main_eval['best_f1'] = best_f1 + main_eval['best_f1_thresh'] = f1_thresh + main_eval['has_ans_exact'] = has_ans_exact + main_eval['has_ans_f1'] = has_ans_f1 + + def find_best_thresh(preds, scores, na_probs, qid_to_has_ans): num_no_ans = sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]) cur_score = num_no_ans @@ -318,10 +365,20 @@ def _compute_softmax(scores): return probs -def compute_predictions(all_examples, all_features, all_results, n_best_size, - max_answer_length, do_lower_case, output_prediction_file, - output_nbest_file, output_null_log_odds_file, verbose_logging, - version_2_with_negative, null_score_diff_threshold): +def compute_predictions_logits( + all_examples, + all_features, + all_results, + n_best_size, + max_answer_length, + do_lower_case, + output_prediction_file, + output_nbest_file, + output_null_log_odds_file, + verbose_logging, + version_2_with_negative, + null_score_diff_threshold +): """Write final predictions to the json file and log-odds of null if needed.""" logger.info("Writing predictions to: %s" % (output_prediction_file)) logger.info("Writing nbest to: %s" % (output_nbest_file)) @@ -450,12 +507,12 @@ def compute_predictions(all_examples, all_features, all_results, n_best_size, text="", start_logit=null_start_logit, end_logit=null_end_logit)) - + # In very rare edge cases we could only have single null prediction. # So we just create a nonce prediction in this case to avoid failure. - if len(nbest)==1: + if len(nbest) == 1: nbest.insert(0, - _NbestPrediction(text="empty", start_logit=0.0, end_logit=0.0)) + _NbestPrediction(text="empty", start_logit=0.0, end_logit=0.0)) # In very rare edge cases we could have no valid predictions. So we # just create a nonce prediction in this case to avoid failure. @@ -512,12 +569,22 @@ def compute_predictions(all_examples, all_features, all_results, n_best_size, return all_predictions -def compute_predictions_extended(all_examples, all_features, all_results, n_best_size, - max_answer_length, output_prediction_file, - output_nbest_file, - output_null_log_odds_file, orig_data_file, - start_n_top, end_n_top, version_2_with_negative, - tokenizer, verbose_logging): +def compute_predictions_log_probs( + all_examples, + all_features, + all_results, + n_best_size, + max_answer_length, + output_prediction_file, + output_nbest_file, + output_null_log_odds_file, + orig_data_file, + start_n_top, + end_n_top, + version_2_with_negative, + tokenizer, + verbose_logging +): """ XLNet write prediction logic (more complex than Bert's). Write final predictions to the json file and log-odds of null if needed. @@ -526,7 +593,7 @@ def compute_predictions_extended(all_examples, all_features, all_results, n_best _PrelimPrediction = collections.namedtuple( # pylint: disable=invalid-name "PrelimPrediction", ["feature_index", "start_index", "end_index", - "start_log_prob", "end_log_prob"]) + "start_log_prob", "end_log_prob"]) _NbestPrediction = collections.namedtuple( # pylint: disable=invalid-name "NbestPrediction", ["text", "start_log_prob", "end_log_prob"]) @@ -609,7 +676,7 @@ def compute_predictions_extended(all_examples, all_features, all_results, n_best # XLNet un-tokenizer # Let's keep it simple for now and see if we need all this later. - # + # # tok_start_to_orig_index = feature.tok_start_to_orig_index # tok_end_to_orig_index = feature.tok_end_to_orig_index # start_orig_pos = tok_start_to_orig_index[pred.start_index] From ff98b041da4b992a87d8b6258b30e47310ec8430 Mon Sep 17 00:00:00 2001 From: Julien Plu Date: Wed, 4 Dec 2019 16:53:06 +0100 Subject: [PATCH 218/505] Fix whitespace issue --- transformers/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/__init__.py b/transformers/__init__.py index e4f5984c70..6d18f11722 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -162,7 +162,7 @@ if is_tf_available(): from .modeling_tf_distilbert import (TFDistilBertPreTrainedModel, TFDistilBertMainLayer, TFDistilBertModel, TFDistilBertForMaskedLM, TFDistilBertForSequenceClassification, - TFDistilBertForTokenClassification, + TFDistilBertForTokenClassification, TFDistilBertForQuestionAnswering, TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP) From bf119c0568dfc1ea5ce0a34359e33ca002266e96 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Wed, 4 Dec 2019 11:34:59 -0500 Subject: [PATCH 219/505] TFDS dataset can now be evaluated --- transformers/data/processors/squad.py | 34 ++++++++++++++++++++------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/transformers/data/processors/squad.py b/transformers/data/processors/squad.py index 70dc9faf54..2e50ac8a8c 100644 --- a/transformers/data/processors/squad.py +++ b/transformers/data/processors/squad.py @@ -245,22 +245,37 @@ class SquadProcessor(DataProcessor): train_file = None dev_file = None - def get_example_from_tensor_dict(self, tensor_dict): + def get_example_from_tensor_dict(self, tensor_dict, evaluate=False): + + if not evaluate: + answer = tensor_dict['answers']['text'][0].numpy().decode('utf-8') + answer_start = tensor_dict['answers']['answer_start'][0].numpy() + answers = None + else: + answers = [{ + "answer_start": start.numpy(), + "text": text.numpy().decode('utf-8') + } for start, text in zip(tensor_dict['answers']["answer_start"], tensor_dict['answers']["text"])] + + answer = None + answer_start = None + return SquadExample( - tensor_dict['id'].numpy().decode("utf-8"), - tensor_dict['question'].numpy().decode('utf-8'), - tensor_dict['context'].numpy().decode('utf-8'), - tensor_dict['answers']['text'][0].numpy().decode('utf-8'), - tensor_dict['answers']['answer_start'][0].numpy(), - tensor_dict['title'].numpy().decode('utf-8') + qas_id=tensor_dict['id'].numpy().decode("utf-8"), + question_text=tensor_dict['question'].numpy().decode('utf-8'), + context_text=tensor_dict['context'].numpy().decode('utf-8'), + answer_text=answer, + start_position_character=answer_start, + title=tensor_dict['title'].numpy().decode('utf-8'), + answers=answers ) - def get_examples_from_dataset(self, dataset): + def get_examples_from_dataset(self, dataset, evaluate=False): """See base class.""" examples = [] for tensor_dict in tqdm(dataset): - examples.append(self.get_example_from_tensor_dict(tensor_dict)) + examples.append(self.get_example_from_tensor_dict(tensor_dict, evaluate=evaluate)) return examples @@ -300,6 +315,7 @@ class SquadProcessor(DataProcessor): question_text = qa["question"] start_position_character = None answer_text = None + answers = None if "is_impossible" in qa: is_impossible = qa["is_impossible"] From cca75e788485e8a2a1c44a445c6aba0fb2dfaf56 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Wed, 4 Dec 2019 15:42:29 -0500 Subject: [PATCH 220/505] Kill the demon spawn --- examples/run_squad.py | 23 +++++++- transformers/data/processors/squad.py | 75 +++++---------------------- 2 files changed, 34 insertions(+), 64 deletions(-) diff --git a/examples/run_squad.py b/examples/run_squad.py index a9ef5c6ba2..2f86322196 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -248,7 +248,28 @@ def evaluate(args, model, tokenizer, prefix=""): eval_feature = features[example_index.item()] unique_id = int(eval_feature.unique_id) - result = SquadResult([to_list(output[i]) for output in outputs] + [unique_id]) + output = [to_list(output[i]) for output in outputs] + + if len(output) >= 5: + start_logits = output[0] + start_top_index = output[1] + end_logits = output[2] + end_top_index = output[3], + cls_logits = output[4] + + result = SquadResult( + unique_id, start_logits, end_logits, + start_top_index=start_top_index, + end_top_index=end_top_index, + cls_logits=cls_logits + ) + + else: + start_logits, end_logits = output + result = SquadResult( + unique_id, start_logits, end_logits + ) + all_results.append(result) evalTime = timeit.default_timer() - start_time diff --git a/transformers/data/processors/squad.py b/transformers/data/processors/squad.py index 2e50ac8a8c..9306189eb4 100644 --- a/transformers/data/processors/squad.py +++ b/transformers/data/processors/squad.py @@ -446,72 +446,21 @@ class SquadFeatures(object): self.end_position = end_position - class SquadResult(object): """ Constructs a SquadResult which can be used to evaluate a model's output on the SQuAD dataset. Args: - result: The result output by a model on a SQuAD inference. These results may be complex (5 values) as the ones output by - XLNet or XLM or may be simple like the other models (2 values). They may be passed as a list or as a dict, with the - following accepted formats: - - `dict` output by a simple model: - { - "start_logits": int, - "end_logits": int, - "unique_id": string - } - `list` output by a simple model: - [start_logits, end_logits, unique_id] - - `dict` output by a complex model: - { - "start_top_log_probs": float, - "start_top_index": int, - "end_top_log_probs": float, - "end_top_index": int, - "cls_logits": int, - "unique_id": string - } - `list` output by a complex model: - [start_top_log_probs, start_top_index, end_top_log_probs, end_top_index, cls_logits, unique_id] - - See `run_squad.py` for an example. + unique_id: The unique identifier corresponding to that example. + start_logits: The logits corresponding to the start of the answer + end_logits: The logits corresponding to the end of the answer """ - def __init__(self, result): - if isinstance(result, dict): - if "start_logits" in result and "end_logits" in result: - self.start_logits = result["start_logits"] - self.end_logits = result["end_logits"] - - elif "start_top_log_probs" in result and "start_top_index" in result: - self.start_top_log_probs = result["start_top_log_probs"] - self.start_top_index = result["start_top_index"] - self.end_top_log_probs = result["end_top_log_probs"] - self.end_top_index = result["end_top_index"] - self.cls_logits = result["cls_logits"] - - else: - raise ValueError("SquadResult instantiated with wrong values.") - - self.unique_id = result["unique_id"] - elif isinstance(result, list): - if len(result) == 3: - self.start_logits = result[0] - self.end_logits = result[1] - - elif len(result) == 6: - self.start_top_log_probs = result[0] - self.start_top_index = result[1] - self.end_top_log_probs = result[2] - self.end_top_index = result[3] - self.cls_logits = result[4] - - else: - raise ValueError("SquadResult instantiated with wrong values.") - - self.unique_id = result[-1] - - else: - raise ValueError("SquadResult instantiated with wrong values. Should be a dictionary or a list.") + def __init__(self, unique_id, start_logits, end_logits, start_top_index=None, end_top_index=None, cls_logits=None): + self.start_top_log_probs = start_logits + self.end_top_log_probs = end_logits + self.unique_id = unique_id + + if start_top_index: + self.start_top_index = start_top_index + self.end_top_index = end_top_index + self.cls_logits = cls_logits \ No newline at end of file From a7ca6d738b7801c680bd25d9e910f962d3f8bf2d Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Wed, 4 Dec 2019 15:43:34 -0500 Subject: [PATCH 221/505] Padding side is tokenizer-dependant --- transformers/data/processors/squad.py | 11 ++-- .../tests/tokenization_tests_commons.py | 21 +++++-- transformers/tokenization_utils.py | 60 ++++++++++++------- transformers/tokenization_xlnet.py | 1 + 4 files changed, 58 insertions(+), 35 deletions(-) diff --git a/transformers/data/processors/squad.py b/transformers/data/processors/squad.py index 9306189eb4..6599c54330 100644 --- a/transformers/data/processors/squad.py +++ b/transformers/data/processors/squad.py @@ -73,8 +73,7 @@ def _is_whitespace(c): return False def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, - doc_stride, max_query_length, is_training, - sequence_a_is_doc=False): + doc_stride, max_query_length, is_training): """Loads a data file into a list of `InputBatch`s.""" # Defining helper methods @@ -127,13 +126,13 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, while len(spans) * doc_stride < len(all_doc_tokens): encoded_dict = tokenizer.encode_plus( - truncated_query if not sequence_a_is_doc else span_doc_tokens, - span_doc_tokens if not sequence_a_is_doc else truncated_query, + truncated_query if tokenizer.padding_side == "right" else span_doc_tokens, + span_doc_tokens if tokenizer.padding_side == "right" else truncated_query, max_length=max_seq_length, return_overflowing_tokens=True, - padding_strategy='right', + pad_to_max_length=True, stride=max_seq_length - doc_stride - len(truncated_query) - sequence_pair_added_tokens, - truncation_strategy='only_second' if not sequence_a_is_doc else 'only_first' + truncation_strategy='only_second' if tokenizer.padding_side == "right" else 'only_first' ) paragraph_len = min(len(all_doc_tokens) - len(spans) * doc_stride, max_seq_length - len(truncated_query) - sequence_pair_added_tokens) diff --git a/transformers/tests/tokenization_tests_commons.py b/transformers/tests/tokenization_tests_commons.py index 40d68d0ab2..6592005c67 100644 --- a/transformers/tests/tokenization_tests_commons.py +++ b/transformers/tests/tokenization_tests_commons.py @@ -344,17 +344,19 @@ class CommonTestCases: padding_idx = tokenizer.pad_token_id # RIGHT PADDING - Check that it correctly pads when a maximum length is specified along with the padding flag set to True + tokenizer.padding_side = "right" encoded_sequence = tokenizer.encode(sequence) sequence_length = len(encoded_sequence) - padded_sequence = tokenizer.encode(sequence, max_length=sequence_length + padding_size, padding_strategy='right') + padded_sequence = tokenizer.encode(sequence, max_length=sequence_length + padding_size, pad_to_max_length=True) padded_sequence_length = len(padded_sequence) assert sequence_length + padding_size == padded_sequence_length assert encoded_sequence + [padding_idx] * padding_size == padded_sequence # LEFT PADDING - Check that it correctly pads when a maximum length is specified along with the padding flag set to True + tokenizer.padding_side = "left" encoded_sequence = tokenizer.encode(sequence) sequence_length = len(encoded_sequence) - padded_sequence = tokenizer.encode(sequence, max_length=sequence_length + padding_size, padding_strategy='left') + padded_sequence = tokenizer.encode(sequence, max_length=sequence_length + padding_size, pad_to_max_length=True) padded_sequence_length = len(padded_sequence) assert sequence_length + padding_size == padded_sequence_length assert [padding_idx] * padding_size + encoded_sequence == padded_sequence @@ -362,10 +364,15 @@ class CommonTestCases: # RIGHT & LEFT PADDING - Check that nothing is done when a maximum length is not specified encoded_sequence = tokenizer.encode(sequence) sequence_length = len(encoded_sequence) - padded_sequence_right = tokenizer.encode(sequence, padding_strategy='right') + + tokenizer.padding_side = "right" + padded_sequence_right = tokenizer.encode(sequence, pad_to_max_length=True) padded_sequence_right_length = len(padded_sequence_right) - padded_sequence_left = tokenizer.encode(sequence, padding_strategy='left') + + tokenizer.padding_side = "left" + padded_sequence_left = tokenizer.encode(sequence, pad_to_max_length=True) padded_sequence_left_length = len(padded_sequence_left) + assert sequence_length == padded_sequence_right_length assert encoded_sequence == padded_sequence_right assert sequence_length == padded_sequence_left_length @@ -387,7 +394,8 @@ class CommonTestCases: sequence_length = len(input_ids) # Test right padding - padded_sequence = tokenizer.encode_plus(sequence, max_length=sequence_length + padding_size, padding_strategy='right', return_special_tokens_mask=True) + tokenizer.padding_side = "right" + padded_sequence = tokenizer.encode_plus(sequence, max_length=sequence_length + padding_size, pad_to_max_length=True, return_special_tokens_mask=True) padded_input_ids = padded_sequence['input_ids'] padded_token_type_ids = padded_sequence['token_type_ids'] padded_attention_mask = padded_sequence['attention_mask'] @@ -401,7 +409,8 @@ class CommonTestCases: assert special_tokens_mask + [1] * padding_size == padded_special_tokens_mask # Test left padding - padded_sequence = tokenizer.encode_plus(sequence, max_length=sequence_length + padding_size, padding_strategy='left', return_special_tokens_mask=True) + tokenizer.padding_side = "left" + padded_sequence = tokenizer.encode_plus(sequence, max_length=sequence_length + padding_size, pad_to_max_length=True, return_special_tokens_mask=True) padded_input_ids = padded_sequence['input_ids'] padded_token_type_ids = padded_sequence['token_type_ids'] padded_attention_mask = padded_sequence['attention_mask'] diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index dbbabd0e1a..41a611ea49 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -77,6 +77,8 @@ class PreTrainedTokenizer(object): "pad_token", "cls_token", "mask_token", "additional_special_tokens"] + padding_side = "right" + @property def bos_token(self): """ Beginning of sentence token (string). Log an error if used while not having been set. """ @@ -223,6 +225,9 @@ class PreTrainedTokenizer(object): self.max_len = max_len if max_len is not None else int(1e12) + # Padding side is right by default and over-riden in subclsses. If specified in the kwargs, it is changed. + self.padding_side = kwargs.pop('padding_side', self.padding_side) + # Added tokens self.added_tokens_encoder = {} self.added_tokens_decoder = {} @@ -702,7 +707,7 @@ class PreTrainedTokenizer(object): max_length=None, stride=0, truncation_strategy='longest_first', - padding_strategy=None, + pad_to_max_length=False, return_tensors=None, **kwargs): """ @@ -729,12 +734,12 @@ class PreTrainedTokenizer(object): - 'only_first': Only truncate the first sequence - 'only_second': Only truncate the second sequence - 'do_not_truncate': Does not truncate (raise an error if the input sequence is longer than max_length) - padding_strategy: if set to a strategy, the returned sequences will be padded according to the model's - padding index, up to their max length. If no max length is specified, no padding is done. - The strategies are handled by the following strings: + pad_to_max_length: if set to True, the returned sequences will be padded according to the model's padding side and + padding index, up to their max length. If no max length is specified, the padding is done up to the model's max length. + The tokenizer padding sides are handled by the following strings: - 'left': pads on the left of the sequences - 'right': pads on the right of the sequences - Defaults to None: no padding. + Defaults to False: no padding. return_tensors: (optional) can be set to 'tf' or 'pt' to return respectively TensorFlow tf.constant or PyTorch torch.Tensor instead of a list of python integers. **kwargs: passed to the `self.tokenize()` method @@ -745,7 +750,7 @@ class PreTrainedTokenizer(object): add_special_tokens=add_special_tokens, stride=stride, truncation_strategy=truncation_strategy, - padding_strategy=padding_strategy, + pad_to_max_length=pad_to_max_length, return_tensors=return_tensors, **kwargs) @@ -758,7 +763,7 @@ class PreTrainedTokenizer(object): max_length=None, stride=0, truncation_strategy='longest_first', - padding_strategy=None, + pad_to_max_length=False, return_tensors=None, return_token_type_ids=True, return_attention_mask=True, @@ -788,12 +793,12 @@ class PreTrainedTokenizer(object): - 'only_first': Only truncate the first sequence - 'only_second': Only truncate the second sequence - 'do_not_truncate': Does not truncate (raise an error if the input sequence is longer than max_length) - padding_strategy: if set to a strategy, the returned sequences will be padded according to the model's - padding index, up to their max length. If no max length is specified, no padding is done. - The strategies are handled by the following strings: + pad_to_max_length: if set to True, the returned sequences will be padded according to the model's padding side and + padding index, up to their max length. If no max length is specified, the padding is done up to the model's max length. + The tokenizer padding sides are handled by the following strings: - 'left': pads on the left of the sequences - 'right': pads on the right of the sequences - Defaults to None: no padding. + Defaults to False: no padding. return_tensors: (optional) can be set to 'tf' or 'pt' to return respectively TensorFlow tf.constant or PyTorch torch.Tensor instead of a list of python integers. return_token_type_ids: (optional) Set to False to avoid returning token_type_ids (default True). @@ -841,7 +846,7 @@ class PreTrainedTokenizer(object): return self.prepare_for_model(first_ids, pair_ids=second_ids, max_length=max_length, - padding_strategy=padding_strategy, + pad_to_max_length=pad_to_max_length, add_special_tokens=add_special_tokens, stride=stride, truncation_strategy=truncation_strategy, @@ -853,7 +858,7 @@ class PreTrainedTokenizer(object): def prepare_for_model(self, ids, pair_ids=None, max_length=None, add_special_tokens=True, stride=0, truncation_strategy='longest_first', - padding_strategy=None, + pad_to_max_length=False, return_tensors=None, return_token_type_ids=True, return_attention_mask=True, @@ -881,12 +886,12 @@ class PreTrainedTokenizer(object): - 'only_first': Only truncate the first sequence - 'only_second': Only truncate the second sequence - 'do_not_truncate': Does not truncate (raise an error if the input sequence is longer than max_length) - padding_strategy: if set to a strategy, the returned sequences will be padded according to the model's - padding index, up to their max length. If no max length is specified, no padding is done. - The strategies are handled by the following strings: + pad_to_max_length: if set to True, the returned sequences will be padded according to the model's padding side and + padding index, up to their max length. If no max length is specified, the padding is done up to the model's max length. + The tokenizer padding sides are handled by the following strings: - 'left': pads on the left of the sequences - - 'right': pads on the right of the sequences - Defaults to None: no padding. + - 'right': pads on the right of the sequences + Defaults to False: no padding. return_tensors: (optional) can be set to 'tf' or 'pt' to return respectively TensorFlow tf.constant or PyTorch torch.Tensor instead of a list of python integers. return_token_type_ids: (optional) Set to False to avoid returning token_type_ids (default True). @@ -955,10 +960,19 @@ class PreTrainedTokenizer(object): "for this model ({} > {}). Running this sequence through the model will result in " "indexing errors".format(len(ids), self.max_len)) - if padding_strategy is not None and max_length and len(encoded_inputs["input_ids"]) < max_length: - difference = max_length - len(encoded_inputs["input_ids"]) + needs_to_be_padded = pad_to_max_length and ( + max_length and len(encoded_inputs["input_ids"]) < max_length + or + max_length is None and len(encoded_inputs["input_ids"]) < self.max_len and self.max_len <= 10000 + ) - if padding_strategy == 'right': + if pad_to_max_length and max_length is None and self.max_len > 10000: + logger.warning("Sequence can't be padded as the maximum ") + + if needs_to_be_padded: + difference = (max_length if max_length is not None else self.max_len) - len(encoded_inputs["input_ids"]) + + if self.padding_side == 'right': if return_attention_mask: encoded_inputs["attention_mask"] = [1] * len(encoded_inputs["input_ids"]) + [0] * difference if return_token_type_ids: @@ -967,7 +981,7 @@ class PreTrainedTokenizer(object): encoded_inputs["special_tokens_mask"] = encoded_inputs["special_tokens_mask"] + [1] * difference encoded_inputs["input_ids"] = encoded_inputs["input_ids"] + [self.pad_token_id] * difference - elif padding_strategy == 'left': + elif self.padding_side == 'left': if return_attention_mask: encoded_inputs["attention_mask"] = [0] * difference + [1] * len(encoded_inputs["input_ids"]) if return_token_type_ids: @@ -977,7 +991,7 @@ class PreTrainedTokenizer(object): encoded_inputs["input_ids"] = [self.pad_token_id] * difference + encoded_inputs["input_ids"] else: - raise ValueError("Invalid padding strategy:" + str(padding_strategy)) + raise ValueError("Invalid padding strategy:" + str(self.padding_side)) elif return_attention_mask: encoded_inputs["attention_mask"] = [1] * len(encoded_inputs["input_ids"]) diff --git a/transformers/tokenization_xlnet.py b/transformers/tokenization_xlnet.py index 3ea71f4438..1c43c0943a 100644 --- a/transformers/tokenization_xlnet.py +++ b/transformers/tokenization_xlnet.py @@ -60,6 +60,7 @@ class XLNetTokenizer(PreTrainedTokenizer): vocab_files_names = VOCAB_FILES_NAMES pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + padding_side = "left" def __init__(self, vocab_file, do_lower_case=False, remove_space=True, keep_accents=False, From f7e4a7cdfa6bcf6ec7c33fd1d40d307278b1c13a Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Wed, 4 Dec 2019 16:24:15 -0500 Subject: [PATCH 222/505] Cleanup --- examples/run_squad.py | 32 ++-- examples/test_examples.py | 3 +- .../{dev-v2.0-small.json => dev-v2.0.json} | 0 examples/tests_samples/SQUAD/train-v2.0.json | 140 ++++++++++++++++++ transformers/data/metrics/squad_metrics.py | 4 +- transformers/data/processors/squad.py | 36 ++++- 6 files changed, 191 insertions(+), 24 deletions(-) rename examples/tests_samples/SQUAD/{dev-v2.0-small.json => dev-v2.0.json} (100%) create mode 100644 examples/tests_samples/SQUAD/train-v2.0.json diff --git a/examples/run_squad.py b/examples/run_squad.py index 2f86322196..3f1b6a798f 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -304,8 +304,8 @@ def load_and_cache_examples(args, tokenizer, evaluate=False, output_examples=Fal torch.distributed.barrier() # Make sure only the first process in distributed training process the dataset, and the others will use the cache # Load data features from cache or dataset file - input_file = args.predict_file if evaluate else args.train_file - cached_features_file = os.path.join(os.path.dirname(input_file), 'cached_{}_{}_{}'.format( + input_dir = args.data_dir if args.data_dir else "." + cached_features_file = os.path.join(input_dir, 'cached_{}_{}_{}'.format( 'dev' if evaluate else 'train', list(filter(None, args.model_name_or_path.split('/'))).pop(), str(args.max_seq_length))) @@ -313,13 +313,22 @@ def load_and_cache_examples(args, tokenizer, evaluate=False, output_examples=Fal logger.info("Loading features from cached file %s", cached_features_file) features = torch.load(cached_features_file) else: - logger.info("Creating features from dataset file at %s", input_file) + logger.info("Creating features from dataset file at %s", input_dir) - processor = SquadV2Processor() - examples = processor.get_dev_examples("examples/squad", only_first=100) if evaluate else processor.get_train_examples("examples/squad") - # import tensorflow_datasets as tfds - # tfds_examples = tfds.load("squad") - # examples = SquadV1Processor().get_examples_from_dataset(tfds_examples["validation"]) + if not args.data_dir: + try: + import tensorflow_datasets as tfds + except ImportError: + raise ImportError("If not data_dir is specified, tensorflow_datasets needs to be installed.") + + if args.version_2_with_negative: + logger.warn("tensorflow_datasets does not handle version 2 of SQuAD.") + + tfds_examples = tfds.load("squad") + examples = SquadV1Processor().get_examples_from_dataset(tfds_examples, evaluate=evaluate) + else: + processor = SquadV2Processor() if args.version_2_with_negative else SquadV1Processor() + examples = processor.get_dev_examples(args.data_dir) if evaluate else processor.get_train_examples(args.data_dir) features = squad_convert_examples_to_features( examples=examples, @@ -328,7 +337,6 @@ def load_and_cache_examples(args, tokenizer, evaluate=False, output_examples=Fal doc_stride=args.doc_stride, max_query_length=args.max_query_length, is_training=not evaluate, - sequence_a_is_doc=True if args.model_type in ['xlnet'] else False ) @@ -365,10 +373,6 @@ def main(): parser = argparse.ArgumentParser() ## Required parameters - parser.add_argument("--train_file", default=None, type=str, required=True, - help="SQuAD json for training. E.g., train-v1.1.json") - parser.add_argument("--predict_file", default=None, type=str, required=True, - help="SQuAD json for predictions. E.g., dev-v1.1.json or test-v1.1.json") parser.add_argument("--model_type", default=None, type=str, required=True, help="Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys())) parser.add_argument("--model_name_or_path", default=None, type=str, required=True, @@ -377,6 +381,8 @@ def main(): help="The output directory where the model checkpoints and predictions will be written.") ## Other parameters + parser.add_argument("--data_dir", default=None, type=str, + help="The input data dir. Should contain the .json files for the task. If not specified, will run with tensorflow_datasets.") parser.add_argument("--config_name", default="", type=str, help="Pretrained config name or path if not the same as model_name") parser.add_argument("--tokenizer_name", default="", type=str, diff --git a/examples/test_examples.py b/examples/test_examples.py index b04d722b7b..632d2f728e 100644 --- a/examples/test_examples.py +++ b/examples/test_examples.py @@ -72,8 +72,7 @@ class ExamplesTests(unittest.TestCase): logger.addHandler(stream_handler) testargs = ["run_squad.py", - "--train_file=./examples/tests_samples/SQUAD/dev-v2.0-small.json", - "--predict_file=./examples/tests_samples/SQUAD/dev-v2.0-small.json", + "--data_dir=./examples/tests_samples/SQUAD", "--model_name=bert-base-uncased", "--output_dir=./examples/tests_samples/temp_dir", "--max_steps=10", diff --git a/examples/tests_samples/SQUAD/dev-v2.0-small.json b/examples/tests_samples/SQUAD/dev-v2.0.json similarity index 100% rename from examples/tests_samples/SQUAD/dev-v2.0-small.json rename to examples/tests_samples/SQUAD/dev-v2.0.json diff --git a/examples/tests_samples/SQUAD/train-v2.0.json b/examples/tests_samples/SQUAD/train-v2.0.json new file mode 100644 index 0000000000..834d9ee660 --- /dev/null +++ b/examples/tests_samples/SQUAD/train-v2.0.json @@ -0,0 +1,140 @@ +{ + "version": "v2.0", + "data": [{ + "title": "Normans", + "paragraphs": [{ + "qas": [{ + "question": "In what country is Normandy located?", + "id": "56ddde6b9a695914005b9628", + "answers": [{ + "text": "France", + "answer_start": 159 + }], + "is_impossible": false + }, { + "question": "When were the Normans in Normandy?", + "id": "56ddde6b9a695914005b9629", + "answers": [{ + "text": "10th and 11th centuries", + "answer_start": 94 + }], + "is_impossible": false + }, { + "question": "From which countries did the Norse originate?", + "id": "56ddde6b9a695914005b962a", + "answers": [{ + "text": "Denmark, Iceland and Norway", + "answer_start": 256 + }], + "is_impossible": false + }, { + "plausible_answers": [{ + "text": "Rollo", + "answer_start": 308 + }], + "question": "Who did King Charles III swear fealty to?", + "id": "5ad39d53604f3c001a3fe8d3", + "answers": [], + "is_impossible": true + }, { + "plausible_answers": [{ + "text": "10th century", + "answer_start": 671 + }], + "question": "When did the Frankish identity emerge?", + "id": "5ad39d53604f3c001a3fe8d4", + "answers": [], + "is_impossible": true + }], + "context": "The Normans (Norman: Nourmands; French: Normands; Latin: Normanni) were the people who in the 10th and 11th centuries gave their name to Normandy, a region in France. They were descended from Norse (\"Norman\" comes from \"Norseman\") raiders and pirates from Denmark, Iceland and Norway who, under their leader Rollo, agreed to swear fealty to King Charles III of West Francia. Through generations of assimilation and mixing with the native Frankish and Roman-Gaulish populations, their descendants would gradually merge with the Carolingian-based cultures of West Francia. The distinct cultural and ethnic identity of the Normans emerged initially in the first half of the 10th century, and it continued to evolve over the succeeding centuries." + }, { + "qas": [{ + "question": "Who was the duke in the battle of Hastings?", + "id": "56dddf4066d3e219004dad5f", + "answers": [{ + "text": "William the Conqueror", + "answer_start": 1022 + }], + "is_impossible": false + }, { + "plausible_answers": [{ + "text": "Antioch", + "answer_start": 1295 + }], + "question": "What principality did William the conquerer found?", + "id": "5ad3a266604f3c001a3fea2b", + "answers": [], + "is_impossible": true + }], + "context": "The Norman dynasty had a major political, cultural and military impact on medieval Europe and even the Near East. The Normans were famed for their martial spirit and eventually for their Christian piety, becoming exponents of the Catholic orthodoxy into which they assimilated. They adopted the Gallo-Romance language of the Frankish land they settled, their dialect becoming known as Norman, Normaund or Norman French, an important literary language. The Duchy of Normandy, which they formed by treaty with the French crown, was a great fief of medieval France, and under Richard I of Normandy was forged into a cohesive and formidable principality in feudal tenure. The Normans are noted both for their culture, such as their unique Romanesque architecture and musical traditions, and for their significant military accomplishments and innovations. Norman adventurers founded the Kingdom of Sicily under Roger II after conquering southern Italy on the Saracens and Byzantines, and an expedition on behalf of their duke, William the Conqueror, led to the Norman conquest of England at the Battle of Hastings in 1066. Norman cultural and military influence spread from these new European centres to the Crusader states of the Near East, where their prince Bohemond I founded the Principality of Antioch in the Levant, to Scotland and Wales in Great Britain, to Ireland, and to the coasts of north Africa and the Canary Islands." + }] + }, { + "title": "Computational_complexity_theory", + "paragraphs": [{ + "qas": [{ + "question": "What branch of theoretical computer science deals with broadly classifying computational problems by difficulty and class of relationship?", + "id": "56e16182e3433e1400422e28", + "answers": [{ + "text": "Computational complexity theory", + "answer_start": 0 + }], + "is_impossible": false + }, { + "plausible_answers": [{ + "text": "algorithm", + "answer_start": 472 + }], + "question": "What is a manual application of mathematical steps?", + "id": "5ad5316b5b96ef001a10ab76", + "answers": [], + "is_impossible": true + }], + "context": "Computational complexity theory is a branch of the theory of computation in theoretical computer science that focuses on classifying computational problems according to their inherent difficulty, and relating those classes to each other. A computational problem is understood to be a task that is in principle amenable to being solved by a computer, which is equivalent to stating that the problem may be solved by mechanical application of mathematical steps, such as an algorithm." + }, { + "qas": [{ + "question": "What measure of a computational problem broadly defines the inherent difficulty of the solution?", + "id": "56e16839cd28a01900c67887", + "answers": [{ + "text": "if its solution requires significant resources", + "answer_start": 46 + }], + "is_impossible": false + }, { + "question": "What method is used to intuitively assess or quantify the amount of resources required to solve a computational problem?", + "id": "56e16839cd28a01900c67888", + "answers": [{ + "text": "mathematical models of computation", + "answer_start": 176 + }], + "is_impossible": false + }, { + "question": "What are two basic primary resources used to guage complexity?", + "id": "56e16839cd28a01900c67889", + "answers": [{ + "text": "time and storage", + "answer_start": 305 + }], + "is_impossible": false + }, { + "plausible_answers": [{ + "text": "the number of gates in a circuit", + "answer_start": 436 + }], + "question": "What unit is measured to determine circuit simplicity?", + "id": "5ad532575b96ef001a10ab7f", + "answers": [], + "is_impossible": true + }, { + "plausible_answers": [{ + "text": "the number of processors", + "answer_start": 502 + }], + "question": "What number is used in perpendicular computing?", + "id": "5ad532575b96ef001a10ab80", + "answers": [], + "is_impossible": true + }], + "context": "A problem is regarded as inherently difficult if its solution requires significant resources, whatever the algorithm used. The theory formalizes this intuition, by introducing mathematical models of computation to study these problems and quantifying the amount of resources needed to solve them, such as time and storage. Other complexity measures are also used, such as the amount of communication (used in communication complexity), the number of gates in a circuit (used in circuit complexity) and the number of processors (used in parallel computing). One of the roles of computational complexity theory is to determine the practical limits on what computers can and cannot do." + }] + }] +} \ No newline at end of file diff --git a/transformers/data/metrics/squad_metrics.py b/transformers/data/metrics/squad_metrics.py index 1f120d354a..f8449df045 100644 --- a/transformers/data/metrics/squad_metrics.py +++ b/transformers/data/metrics/squad_metrics.py @@ -630,12 +630,12 @@ def compute_predictions_log_probs( for i in range(start_n_top): for j in range(end_n_top): - start_log_prob = result.start_top_log_probs[i] + start_log_prob = result.start_logits[i] start_index = result.start_top_index[i] j_index = i * end_n_top + j - end_log_prob = result.end_top_log_probs[j_index] + end_log_prob = result.end_logits[j_index] end_index = result.end_top_index[j_index] # We could hypothetically create invalid predictions, e.g., predict diff --git a/transformers/data/processors/squad.py b/transformers/data/processors/squad.py index 6599c54330..dd2d9d25c0 100644 --- a/transformers/data/processors/squad.py +++ b/transformers/data/processors/squad.py @@ -146,7 +146,7 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, token_to_orig_map = {} for i in range(paragraph_len): - index = len(truncated_query) + sequence_added_tokens + i if not sequence_a_is_doc else i + index = len(truncated_query) + sequence_added_tokens + i if tokenizer.padding_side == "right" else i token_to_orig_map[index] = tok_to_orig_index[len(spans) * doc_stride + i] encoded_dict["paragraph_len"] = paragraph_len @@ -166,7 +166,7 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, for doc_span_index in range(len(spans)): for j in range(spans[doc_span_index]["paragraph_len"]): is_max_context = _new_check_is_max_context(spans, doc_span_index, doc_span_index * doc_stride + j) - index = j if sequence_a_is_doc else spans[doc_span_index]["truncated_query_with_special_tokens_length"] + j + index = j if tokenizer.padding_side == "left" else spans[doc_span_index]["truncated_query_with_special_tokens_length"] + j spans[doc_span_index]["token_is_max_context"][index] = is_max_context for span in spans: @@ -179,7 +179,7 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, p_mask = np.minimum(p_mask, 1) - if not sequence_a_is_doc: + if tokenizer.padding_side == "right": # Limit positive values to one p_mask = 1 - p_mask @@ -207,7 +207,7 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, end_position = cls_index span_is_impossible = True else: - if sequence_a_is_doc: + if tokenizer.padding_side == "left": doc_offset = 0 else: doc_offset = len(truncated_query) + sequence_added_tokens @@ -270,7 +270,29 @@ class SquadProcessor(DataProcessor): ) def get_examples_from_dataset(self, dataset, evaluate=False): - """See base class.""" + """ + Creates a list of :class:`~transformers.data.processors.squad.SquadExample` using a TFDS dataset. + + Args: + dataset: The tfds dataset loaded from `tensorflow_datasets.load("squad")` + evaluate: boolean specifying if in evaluation mode or in training mode + + Returns: + List of SquadExample + + Examples:: + + import tensorflow_datasets as tfds + dataset = tfds.load("squad") + + training_examples = get_examples_from_dataset(dataset, evaluate=False) + evaluation_examples = get_examples_from_dataset(dataset, evaluate=True) + """ + + if evaluate: + dataset = dataset["validation"] + else: + dataset = dataset["train"] examples = [] for tensor_dict in tqdm(dataset): @@ -455,8 +477,8 @@ class SquadResult(object): end_logits: The logits corresponding to the end of the answer """ def __init__(self, unique_id, start_logits, end_logits, start_top_index=None, end_top_index=None, cls_logits=None): - self.start_top_log_probs = start_logits - self.end_top_log_probs = end_logits + self.start_logits = start_logits + self.end_logits = end_logits self.unique_id = unique_id if start_top_index: From 33508ae310f101a2534d3e97ea23fda93e25ef38 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Wed, 4 Dec 2019 16:26:45 -0500 Subject: [PATCH 223/505] Remove `only_first` --- transformers/data/processors/squad.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/transformers/data/processors/squad.py b/transformers/data/processors/squad.py index dd2d9d25c0..09a79db471 100644 --- a/transformers/data/processors/squad.py +++ b/transformers/data/processors/squad.py @@ -300,29 +300,29 @@ class SquadProcessor(DataProcessor): return examples - def get_train_examples(self, data_dir, only_first=None): + def get_train_examples(self, data_dir): """See base class.""" if self.train_file is None: raise ValueError("SquadProcessor should be instantiated via SquadV1Processor or SquadV2Processor") with open(os.path.join(data_dir, self.train_file), "r", encoding='utf-8') as reader: input_data = json.load(reader)["data"] - return self._create_examples(input_data, "train", only_first) + return self._create_examples(input_data, "train") - def get_dev_examples(self, data_dir, only_first=None): + def get_dev_examples(self, data_dir): """See base class.""" if self.dev_file is None: raise ValueError("SquadProcessor should be instantiated via SquadV1Processor or SquadV2Processor") with open(os.path.join(data_dir, self.dev_file), "r", encoding='utf-8') as reader: input_data = json.load(reader)["data"] - return self._create_examples(input_data, "dev", only_first) + return self._create_examples(input_data, "dev") def get_labels(self): """See base class.""" return ["0", "1"] - def _create_examples(self, input_data, set_type, only_first=None): + def _create_examples(self, input_data, set_type): """Creates examples for the training and dev sets.""" is_training = set_type == "train" @@ -363,9 +363,6 @@ class SquadProcessor(DataProcessor): ) examples.append(example) - - if only_first is not None and len(examples) > only_first: - return examples return examples class SquadV1Processor(SquadProcessor): From 96fa9a8a70a52221446b0b887f99c90c5ce31eeb Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Wed, 4 Dec 2019 17:22:50 -0500 Subject: [PATCH 224/505] Python 2 + Post mime-type to S3 --- transformers/hf_api.py | 74 ++++++++++++++++++++++++------- transformers/tests/hf_api_test.py | 20 ++++++--- 2 files changed, 73 insertions(+), 21 deletions(-) diff --git a/transformers/hf_api.py b/transformers/hf_api.py index 238762ebf8..c21592a838 100644 --- a/transformers/hf_api.py +++ b/transformers/hf_api.py @@ -14,9 +14,9 @@ # limitations under the License. from __future__ import absolute_import, division, print_function -from typing import List, NamedTuple import os from os.path import expanduser +import six import requests from requests.exceptions import HTTPError @@ -24,23 +24,43 @@ from requests.exceptions import HTTPError ENDPOINT = "https://huggingface.co" class S3Obj: - def __init__(self, filename: str, LastModified: str, ETag: str, Size: int): + def __init__( + self, + filename, # type: str + LastModified, # type: str + ETag, # type: str + Size, # type: int + **kwargs + ): self.filename = filename self.LastModified = LastModified self.ETag = ETag self.Size = Size -class PresignedUrl(NamedTuple): - write: str - access: str +class PresignedUrl: + def __init__( + self, + write, # type: str + access, # type: str + type, # type: str + **kwargs + ): + self.write = write + self.access = access + self.type = type # mime-type to send to S3. class HfApi: def __init__(self, endpoint=None): self.endpoint = endpoint if endpoint is not None else ENDPOINT - def login(self, username: str, password: str) -> str: + def login( + self, + username, # type: str + password, # type: str + ): + # type: (...) -> str """ Call HF API to sign in a user and get a token if credentials are valid. @@ -56,7 +76,11 @@ class HfApi: d = r.json() return d["token"] - def whoami(self, token: str) -> str: + def whoami( + self, + token, # type: str + ): + # type: (...) -> str """ Call HF API to know "whoami" """ @@ -66,7 +90,8 @@ class HfApi: d = r.json() return d["user"] - def logout(self, token: str): + def logout(self, token): + # type: (...) -> void """ Call HF API to log out. """ @@ -74,7 +99,8 @@ class HfApi: r = requests.post(path, headers={"authorization": "Bearer {}".format(token)}) r.raise_for_status() - def presign(self, token: str, filename: str) -> PresignedUrl: + def presign(self, token, filename): + # type: (...) -> PresignedUrl """ Call HF API to get a presigned url to upload `filename` to S3. """ @@ -88,7 +114,8 @@ class HfApi: d = r.json() return PresignedUrl(**d) - def presign_and_upload(self, token: str, filename: str, filepath: str) -> str: + def presign_and_upload(self, token, filename, filepath): + # type: (...) -> str """ Get a presigned url, then upload file to S3. @@ -98,12 +125,18 @@ class HfApi: urls = self.presign(token, filename=filename) # streaming upload: # https://2.python-requests.org/en/master/user/advanced/#streaming-uploads + # + # Even though we presign with the correct content-type, + # the client still has to specify it when uploading the file. with open(filepath, "rb") as f: - r = requests.put(urls.write, data=f) + r = requests.put(urls.write, data=f, headers={ + "content-type": urls.type, + }) r.raise_for_status() return urls.access - def list_objs(self, token: str) -> List[S3Obj]: + def list_objs(self, token): + # type: (...) -> List[S3Obj] """ Call HF API to list all stored files for user. """ @@ -121,11 +154,20 @@ class HfFolder: path_token = expanduser("~/.huggingface/token") @classmethod - def save_token(cls, token: str): + def save_token(cls, token): """ Save token, creating folder as needed. """ - os.makedirs(os.path.dirname(cls.path_token), exist_ok=True) + if six.PY3: + os.makedirs(os.path.dirname(cls.path_token), exist_ok=True) + else: + # Python 2 + try: + os.makedirs(os.path.dirname(cls.path_token)) + except OSError as e: + if e.errno != os.errno.EEXIST: + raise e + pass with open(cls.path_token, 'w+') as f: f.write(token) @@ -137,7 +179,9 @@ class HfFolder: try: with open(cls.path_token, 'r') as f: return f.read() - except FileNotFoundError: + except: + # this is too wide. When Py2 is dead use: + # `except FileNotFoundError:` instead return None @classmethod diff --git a/transformers/tests/hf_api_test.py b/transformers/tests/hf_api_test.py index 59822344ba..92d41b6dff 100644 --- a/transformers/tests/hf_api_test.py +++ b/transformers/tests/hf_api_test.py @@ -15,6 +15,7 @@ from __future__ import absolute_import, division, print_function import os +import six import time import unittest @@ -40,7 +41,7 @@ class HfApiLoginTest(HfApiCommonTest): def test_login_valid(self): token = self._api.login(username=USER, password=PASS) - self.assertIsInstance(token, str) + self.assertIsInstance(token, six.string_types) class HfApiEndpointsTest(HfApiCommonTest): @@ -56,19 +57,22 @@ class HfApiEndpointsTest(HfApiCommonTest): self.assertEqual(user, USER) def test_presign(self): - url = self._api.presign(token=self._token, filename=FILE_KEY) - self.assertIsInstance(url, PresignedUrl) + urls = self._api.presign(token=self._token, filename=FILE_KEY) + self.assertIsInstance(urls, PresignedUrl) + self.assertEqual(urls.type, "text/plain") def test_presign_and_upload(self): access_url = self._api.presign_and_upload( token=self._token, filename=FILE_KEY, filepath=FILE_PATH ) - self.assertIsInstance(access_url, str) + self.assertIsInstance(access_url, six.string_types) def test_list_objs(self): objs = self._api.list_objs(token=self._token) - o = objs[-1] - self.assertIsInstance(o, S3Obj) + self.assertIsInstance(objs, list) + if len(objs) > 0: + o = objs[-1] + self.assertIsInstance(o, S3Obj) @@ -92,3 +96,7 @@ class HfFolderTest(unittest.TestCase): HfFolder.get_token(), None ) + + +if __name__ == "__main__": + unittest.main() From 7a03519975e4f0b6698bf1221c2263ed0f8d795c Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Wed, 4 Dec 2019 17:24:35 -0500 Subject: [PATCH 225/505] Documentation --- docs/source/main_classes/processors.rst | 79 +++++++++++++++++- transformers/data/processors/squad.py | 104 ++++++++++++++++++++---- 2 files changed, 164 insertions(+), 19 deletions(-) diff --git a/docs/source/main_classes/processors.rst b/docs/source/main_classes/processors.rst index a85c126956..ce0eeb553a 100644 --- a/docs/source/main_classes/processors.rst +++ b/docs/source/main_classes/processors.rst @@ -55,4 +55,81 @@ Example usage ^^^^^^^^^^^^^^^^^^^^^^^^^ An example using these processors is given in the -`run_glue.py `__ script. \ No newline at end of file +`run_glue.py `__ script. + + + +SQuAD +~~~~~~~~~~~~~~~~~~~~~ + +`The Stanford Question Answering Dataset (SQuAD) `__ is a benchmark that evaluates +the performance of models on question answering. Two versions are available, v1.1 and v2.0. The first version (v1.1) was released together with the paper +`SQuAD: 100,000+ Questions for Machine Comprehension of Text `__. The second version (v2.0) was released alongside +the paper `Know What You Don't Know: Unanswerable Questions for SQuAD `__. + +This library hosts a processor for each of the two versions: + +Processors +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Those processors are: + - :class:`~transformers.data.processors.utils.SquadV1Processor` + - :class:`~transformers.data.processors.utils.SquadV2Processor` + +They both inherit from the abstract class :class:`~transformers.data.processors.utils.SquadProcessor` + +.. autoclass:: transformers.data.processors.squad.SquadProcessor + :members: + +Additionally, the following method can be used to convert SQuAD examples into :class:`~transformers.data.processors.utils.SquadFeatures` +that can be used as model inputs. + +.. automethod:: transformers.data.processors.squad.squad_convert_examples_to_features + +These processors as well as the aforementionned method can be used with files containing the data as well as with the `tensorflow_datasets` package. +Examples are given below. + +Example usage +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Here is an example using the processors as well as the conversion method using data files: + +Example:: + + # Loading a V2 processor + processor = SquadV2Processor() + examples = processor.get_dev_examples(squad_v2_data_dir) + + # Loading a V1 processor + processor = SquadV1Processor() + examples = processor.get_dev_examples(squad_v1_data_dir) + + features = squad_convert_examples_to_features( + examples=examples, + tokenizer=tokenizer, + max_seq_length=max_seq_length, + doc_stride=args.doc_stride, + max_query_length=max_query_length, + is_training=not evaluate, + ) + +Using `tensorflow_datasets` is as easy as using a data file: + +Example:: + + # tensorflow_datasets only handle Squad V1. + tfds_examples = tfds.load("squad") + examples = SquadV1Processor().get_examples_from_dataset(tfds_examples, evaluate=evaluate) + + features = squad_convert_examples_to_features( + examples=examples, + tokenizer=tokenizer, + max_seq_length=max_seq_length, + doc_stride=args.doc_stride, + max_query_length=max_query_length, + is_training=not evaluate, + ) + + +Another example using these processors is given in the +`run_squad.py `__ script. \ No newline at end of file diff --git a/transformers/data/processors/squad.py b/transformers/data/processors/squad.py index 09a79db471..b17e626c98 100644 --- a/transformers/data/processors/squad.py +++ b/transformers/data/processors/squad.py @@ -74,7 +74,35 @@ def _is_whitespace(c): def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, doc_stride, max_query_length, is_training): - """Loads a data file into a list of `InputBatch`s.""" + """ + Converts a list of examples into a list of features that can be directly given as input to a model. + It is model-dependant and takes advantage of many of the tokenizer's features to create the model's inputs. + + Args: + examples: list of :class:`~transformers.data.processors.squad.SquadExample` + tokenizer: an instance of a child of :class:`~transformers.PreTrainedTokenizer` + max_seq_length: The maximum sequence length of the inputs. + doc_stride: The stride used when the context is too large and is split across several features. + max_query_length: The maximum length of the query. + is_training: wheter to create features for model evaluation or model training. + + Returns: + list of :class:`~transformers.data.processors.squad.SquadFeatures` + + Example:: + + processor = SquadV2Processor() + examples = processor.get_dev_examples(data_dir) + + features = squad_convert_examples_to_features( + examples=examples, + tokenizer=tokenizer, + max_seq_length=args.max_seq_length, + doc_stride=args.doc_stride, + max_query_length=args.max_query_length, + is_training=not evaluate, + ) + """ # Defining helper methods unique_id = 1000000000 @@ -240,12 +268,14 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, class SquadProcessor(DataProcessor): - """Processor for the SQuAD data set.""" + """ + Processor for the SQuAD data set. + Overriden by SquadV1Processor and SquadV2Processor, used by the version 1.1 and version 2.0 of SQuAD, respectively. + """ train_file = None dev_file = None - def get_example_from_tensor_dict(self, tensor_dict, evaluate=False): - + def _get_example_from_tensor_dict(self, tensor_dict, evaluate=False): if not evaluate: answer = tensor_dict['answers']['text'][0].numpy().decode('utf-8') answer_start = tensor_dict['answers']['answer_start'][0].numpy() @@ -296,35 +326,44 @@ class SquadProcessor(DataProcessor): examples = [] for tensor_dict in tqdm(dataset): - examples.append(self.get_example_from_tensor_dict(tensor_dict, evaluate=evaluate)) + examples.append(self._get_example_from_tensor_dict(tensor_dict, evaluate=evaluate)) return examples - def get_train_examples(self, data_dir): - """See base class.""" + def get_train_examples(self, data_dir, filename=None): + """ + Returns the training examples from the data directory. + + Args: + data_dir: Directory containing the data files used for training and evaluating. + filename: None by default, specify this if the training file has a different name than the original one + which is `train-v1.1.json` and `train-v2.0.json` for squad versions 1.1 and 2.0 respectively. + + """ if self.train_file is None: raise ValueError("SquadProcessor should be instantiated via SquadV1Processor or SquadV2Processor") - with open(os.path.join(data_dir, self.train_file), "r", encoding='utf-8') as reader: + with open(os.path.join(data_dir, self.train_file if filename is None else filename), "r", encoding='utf-8') as reader: input_data = json.load(reader)["data"] return self._create_examples(input_data, "train") - def get_dev_examples(self, data_dir): - """See base class.""" + def get_dev_examples(self, data_dir, filename=None): + """ + Returns the evaluation example from the data directory. + + Args: + data_dir: Directory containing the data files used for training and evaluating. + filename: None by default, specify this if the evaluation file has a different name than the original one + which is `train-v1.1.json` and `train-v2.0.json` for squad versions 1.1 and 2.0 respectively. + """ if self.dev_file is None: raise ValueError("SquadProcessor should be instantiated via SquadV1Processor or SquadV2Processor") - with open(os.path.join(data_dir, self.dev_file), "r", encoding='utf-8') as reader: + with open(os.path.join(data_dir, self.dev_file if filename is not None else filename), "r", encoding='utf-8') as reader: input_data = json.load(reader)["data"] return self._create_examples(input_data, "dev") - def get_labels(self): - """See base class.""" - return ["0", "1"] - def _create_examples(self, input_data, set_type): - """Creates examples for the training and dev sets.""" - is_training = set_type == "train" examples = [] for entry in tqdm(input_data): @@ -378,6 +417,16 @@ class SquadV2Processor(SquadProcessor): class SquadExample(object): """ A single training/test example for the Squad dataset, as loaded from disk. + + Args: + qas_id: The example's unique identifier + question_text: The question string + context_text: The context string + answer_text: The answer string + start_position_character: The character position of the start of the answer + title: The title of the example + answers: None by default, this is used during evaluation. Holds answers as well as their start positions. + is_impossible: False by default, set to True if the example has no possible answer. """ def __init__(self, @@ -427,7 +476,26 @@ class SquadExample(object): class SquadFeatures(object): """ Single squad example features to be fed to a model. - Those features are model-specific. + Those features are model-specific and can be crafted from :class:`~transformers.data.processors.squad.SquadExample` + using the :method:`~transformers.data.processors.squad.squad_convert_examples_to_features` method. + + Args: + input_ids: Indices of input sequence tokens in the vocabulary. + attention_mask: Mask to avoid performing attention on padding token indices. + token_type_ids: Segment token indices to indicate first and second portions of the inputs. + cls_index: the index of the CLS token. + p_mask: Mask identifying tokens that can be answers vs. tokens that cannot. + Mask with 1 for tokens than cannot be in the answer and 0 for token that can be in an answer + example_index: the index of the example + unique_id: The unique Feature identifier + paragraph_len: The length of the context + token_is_max_context: List of booleans identifying which tokens have their maximum context in this feature object. + If a token does not have their maximum context in this feature object, it means that another feature object + has more information related to that token and should be prioritized over this feature for that token. + tokens: list of tokens corresponding to the input ids + token_to_orig_map: mapping between the tokens and the original text, needed in order to identify the answer. + start_position: start of the answer token index + end_position: end of the answer token index """ def __init__(self, From ce158a076f7089bf11d44e1581f5bcab4dcc5396 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Wed, 4 Dec 2019 17:55:52 -0500 Subject: [PATCH 226/505] Return dataset (pytorch) --- transformers/data/processors/squad.py | 41 ++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/transformers/data/processors/squad.py b/transformers/data/processors/squad.py index b17e626c98..338bae0c51 100644 --- a/transformers/data/processors/squad.py +++ b/transformers/data/processors/squad.py @@ -7,7 +7,11 @@ import numpy as np from ...tokenization_bert import BasicTokenizer, whitespace_tokenize from .utils import DataProcessor, InputExample, InputFeatures -from ...file_utils import is_tf_available +from ...file_utils import is_tf_available, is_torch_available + +if is_torch_available: + import torch + from torch.utils.data import TensorDataset if is_tf_available(): import tensorflow as tf @@ -73,7 +77,8 @@ def _is_whitespace(c): return False def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, - doc_stride, max_query_length, is_training): + doc_stride, max_query_length, is_training, + return_dataset=False): """ Converts a list of examples into a list of features that can be directly given as input to a model. It is model-dependant and takes advantage of many of the tokenizer's features to create the model's inputs. @@ -84,7 +89,10 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, max_seq_length: The maximum sequence length of the inputs. doc_stride: The stride used when the context is too large and is split across several features. max_query_length: The maximum length of the query. - is_training: wheter to create features for model evaluation or model training. + is_training: whether to create features for model evaluation or model training. + return_dataset: Default False. Either 'pt' or 'tf'. + if 'pt': returns a torch.data.TensorDataset, + if 'tf': returns a tf.data.Dataset Returns: list of :class:`~transformers.data.processors.squad.SquadFeatures` @@ -264,6 +272,31 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, unique_id += 1 + if return_dataset == 'pt': + if not is_torch_available(): + raise ImportError("Pytorch must be installed to return a pytorch dataset.") + + # Convert to Tensors and build dataset + all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long) + all_input_mask = torch.tensor([f.attention_mask for f in features], dtype=torch.long) + all_segment_ids = torch.tensor([f.token_type_ids for f in features], dtype=torch.long) + all_cls_index = torch.tensor([f.cls_index for f in features], dtype=torch.long) + all_p_mask = torch.tensor([f.p_mask for f in features], dtype=torch.float) + + if not is_training: + all_example_index = torch.arange(all_input_ids.size(0), dtype=torch.long) + dataset = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, + all_example_index, all_cls_index, all_p_mask) + else: + all_start_positions = torch.tensor([f.start_position for f in features], dtype=torch.long) + all_end_positions = torch.tensor([f.end_position for f in features], dtype=torch.long) + dataset = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, + all_start_positions, all_end_positions, + all_cls_index, all_p_mask) + + return features, dataset + + return features @@ -359,7 +392,7 @@ class SquadProcessor(DataProcessor): if self.dev_file is None: raise ValueError("SquadProcessor should be instantiated via SquadV1Processor or SquadV2Processor") - with open(os.path.join(data_dir, self.dev_file if filename is not None else filename), "r", encoding='utf-8') as reader: + with open(os.path.join(data_dir, self.dev_file if filename is None else filename), "r", encoding='utf-8') as reader: input_data = json.load(reader)["data"] return self._create_examples(input_data, "dev") From 3ba417e1a877d2c5f2170b12a8cc39f16dbf46aa Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Wed, 4 Dec 2019 18:40:52 -0500 Subject: [PATCH 227/505] [cli] ls: Tabular formatting --- transformers/commands/user.py | 61 +++++++++++++++++++++++++++++------ 1 file changed, 52 insertions(+), 9 deletions(-) diff --git a/transformers/commands/user.py b/transformers/commands/user.py index 4b826f4dc6..d79922ed8a 100644 --- a/transformers/commands/user.py +++ b/transformers/commands/user.py @@ -25,6 +25,17 @@ class UserCommands(BaseTransformersCLICommand): +class ANSI: + """ + Helper for en.wikipedia.org/wiki/ANSI_escape_code + """ + _bold = u"\u001b[1m" + _reset = u"\u001b[0m" + @classmethod + def bold(cls, s): + return "{}{}{}".format(cls._bold, s, cls._reset) + + class BaseUserCommand: def __init__(self, args): self.args = args @@ -80,6 +91,28 @@ class LogoutCommand(BaseUserCommand): class ListObjsCommand(BaseUserCommand): + def tabulate(self, rows, headers): + # type: (List[List[Union[str, int]]], List[str]) -> str + """ + Inspired by: + stackoverflow.com/a/8356620/593036 + stackoverflow.com/questions/9535954/printing-lists-as-tabular-data + """ + col_widths = [max(len(str(x)) for x in col) for col in zip(*rows, headers)] + row_format = ("{{:{}}} " * len(headers)).format(*col_widths) + lines = [] + lines.append( + row_format.format(*headers) + ) + lines.append( + row_format.format(*["-" * w for w in col_widths]) + ) + for row in rows: + lines.append( + row_format.format(*row) + ) + return "\n".join(lines) + def run(self): token = HfFolder.get_token() if token is None: @@ -92,13 +125,16 @@ class ListObjsCommand(BaseUserCommand): exit(1) if len(objs) == 0: print("No shared file yet") - for obj in objs: - print( - obj.filename, - obj.LastModified, - obj.ETag, - obj.Size - ) + exit() + rows = [ [ + obj.filename, + obj.LastModified, + obj.ETag, + obj.Size + ] for obj in objs ] + print( + self.tabulate(rows, headers=["Filename", "LastModified", "ETag", "Size"]) + ) class UploadCommand(BaseUserCommand): @@ -109,12 +145,19 @@ class UploadCommand(BaseUserCommand): exit(1) filepath = os.path.join(os.getcwd(), self.args.file) filename = self.args.filename if self.args.filename is not None else os.path.basename(filepath) - print("About to upload file {} to S3 under filename {}".format(filepath, filename)) + print( + "About to upload file {} to S3 under filename {}".format( + ANSI.bold(filepath), ANSI.bold(filename) + ) + ) + choice = input("Proceed? [Y/n] ").lower() if not(choice == "" or choice == "y" or choice == "yes"): print("Abort") exit() - print("Uploading...") + print( + ANSI.bold("Uploading... This might take a while if file is large") + ) access_url = self._api.presign_and_upload( token=token, filename=filename, filepath=filepath ) From fb0d2f1da102d699c6457fd98be35f89852d08b9 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Thu, 5 Dec 2019 03:00:16 -0500 Subject: [PATCH 228/505] preparing release distil-mBERT --- transformers/modeling_tf_distilbert.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/transformers/modeling_tf_distilbert.py b/transformers/modeling_tf_distilbert.py index b3d4889475..79ae49d811 100644 --- a/transformers/modeling_tf_distilbert.py +++ b/transformers/modeling_tf_distilbert.py @@ -37,7 +37,8 @@ logger = logging.getLogger(__name__) TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP = { 'distilbert-base-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-tf_model.h5", - 'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-tf_model.h5" + 'distilbert-base-uncased-distilled-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-distilled-squad-tf_model.h5", + 'distilbert-base-multilingual-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-multilingual-cased-tf_model.h5", } From 7f998b1b832dd69cfdd8455afd5b8af3b2f77df8 Mon Sep 17 00:00:00 2001 From: Guillaume B Date: Thu, 5 Dec 2019 08:57:49 +0100 Subject: [PATCH 229/505] special_tokens_mask value was unused and calculated twice --- transformers/tokenization_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index 5d683629f0..6be96989cb 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -910,7 +910,7 @@ class PreTrainedTokenizer(object): token_type_ids = [0] * len(ids) + ([1] * len(pair_ids) if pair else []) special_tokens_mask = [0] * (len(ids) + (len(pair_ids) if pair else 0)) if return_special_tokens_mask: - encoded_inputs["special_tokens_mask"] = self.get_special_tokens_mask(ids, pair_ids) + encoded_inputs["special_tokens_mask"] = special_tokens_mask # Prepare inputs as tensors if asked if return_tensors == 'tf' and is_tf_available(): From 8b388827b509e0c117c53803f2ee030ead0e5a81 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 5 Dec 2019 11:18:43 +0100 Subject: [PATCH 230/505] fix #1920 --- transformers/tokenization_ctrl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transformers/tokenization_ctrl.py b/transformers/tokenization_ctrl.py index 3d67fa2c5b..9454cbbaf3 100644 --- a/transformers/tokenization_ctrl.py +++ b/transformers/tokenization_ctrl.py @@ -192,9 +192,9 @@ class CTRLTokenizer(PreTrainedTokenizer): """ split_tokens = [] - text = text.split(' ') + words = re.findall(r'\S+\n?', text) - for token in text: + for token in words: split_tokens.extend([t for t in self.bpe(token).split(' ')]) return split_tokens From 75a97af6bc3b842df22e3bc12e530c22c5e15482 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 5 Dec 2019 11:26:55 +0100 Subject: [PATCH 231/505] fix #1450 - add doc --- examples/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/README.md b/examples/README.md index 960b218f11..25bbcf2246 100644 --- a/examples/README.md +++ b/examples/README.md @@ -4,12 +4,14 @@ In this section a few examples are put together. All of these examples work for similar API between the different models. **Important** -To run the latest versions of the examples, you have to install from source. Execute the following steps in a new virtual environment: +To run the latest versions of the examples, you have to install from source and install some specific requirements for the examples. +Execute the following steps in a new virtual environment: ```bash git clone https://github.com/huggingface/transformers cd transformers pip install [--editable] . +pip install -r ./examples/requirements.txt ``` | Section | Description | From 71e4693f087271053d5c188319cfb5217836cca4 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 5 Dec 2019 12:14:24 +0100 Subject: [PATCH 232/505] fix #1968 --- transformers/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transformers/__init__.py b/transformers/__init__.py index f06ee3f35d..ab5090723d 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -98,7 +98,7 @@ if is_torch_available(): RobertaForSequenceClassification, RobertaForMultipleChoice, RobertaForTokenClassification, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP) - from .modeling_distilbert import (DistilBertForMaskedLM, DistilBertModel, + from .modeling_distilbert import (DistilBertPreTrainedModel, DistilBertForMaskedLM, DistilBertModel, DistilBertForSequenceClassification, DistilBertForQuestionAnswering, DistilBertForTokenClassification, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP) @@ -108,7 +108,7 @@ if is_torch_available(): CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_encoder_decoder import PreTrainedEncoderDecoder, Model2Model - from .modeling_albert import (AlbertModel, AlbertForMaskedLM, AlbertForSequenceClassification, + from .modeling_albert import (AlbertPreTrainedModel, AlbertModel, AlbertForMaskedLM, AlbertForSequenceClassification, AlbertForQuestionAnswering, load_tf_weights_in_albert, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) From 9200a759d782a87530765fb32f52b6248c7f4d03 Mon Sep 17 00:00:00 2001 From: Julien Plu Date: Thu, 5 Dec 2019 12:56:43 +0100 Subject: [PATCH 233/505] Add few tests on the TF optimization file with some info in the documentation. Complete the README. --- .../main_classes/optimizer_schedules.rst | 24 +++++ examples/README.md | 77 +++++++++++++++- examples/run_tf_ner.py | 7 +- transformers/tests/optimization_tf_test.py | 89 +++++++++++++++++++ 4 files changed, 191 insertions(+), 6 deletions(-) create mode 100644 transformers/tests/optimization_tf_test.py diff --git a/docs/source/main_classes/optimizer_schedules.rst b/docs/source/main_classes/optimizer_schedules.rst index b30a2e0e2e..22ed1b28fb 100644 --- a/docs/source/main_classes/optimizer_schedules.rst +++ b/docs/source/main_classes/optimizer_schedules.rst @@ -5,6 +5,7 @@ The ``.optimization`` module provides: - an optimizer with weight decay fixed that can be used to fine-tuned models, and - several schedules in the form of schedule objects that inherit from ``_LRSchedule``: +- a gradient accumulation class to accumulate the gradients of multiple batches ``AdamW`` ~~~~~~~~~~~~~~~~ @@ -12,6 +13,15 @@ The ``.optimization`` module provides: .. autoclass:: transformers.AdamW :members: +``AdamWeightDecay`` +~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.AdamWeightDecay + :members: + +.. autofunction:: transformers.create_optimizer + :members: + Schedules ---------------------------------------------------- @@ -49,3 +59,17 @@ Learning Rate Schedules .. image:: /imgs/warmup_linear_schedule.png :target: /imgs/warmup_linear_schedule.png :alt: + +``Warmup`` +~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.Warmup + :members: + +Gradient Strategies +---------------------------------------------------- + +``GradientAccumulator`` +~~~~~~~~~~~~~~~~~~~~~~~ + +.. autoclass:: transformers.GradientAccumulator diff --git a/examples/README.md b/examples/README.md index 960b218f11..2dd6653916 100644 --- a/examples/README.md +++ b/examples/README.md @@ -465,7 +465,8 @@ Training with the previously defined hyper-parameters yields the following resul ## Named Entity Recognition -Based on the script [`run_ner.py`](https://github.com/huggingface/transformers/blob/master/examples/run_ner.py). +Based on the scripts [`run_ner.py`](https://github.com/huggingface/transformers/blob/master/examples/run_ner.py) for Pytorch and +[`run_tf_ner.py`(https://github.com/huggingface/transformers/blob/master/examples/run_tf_ner.py)] for Tensorflow 2. This example fine-tune Bert Multilingual on GermEval 2014 (German NER). Details and results for the fine-tuning provided by @stefan-it. @@ -510,7 +511,7 @@ The GermEval 2014 dataset has much more labels than CoNLL-2002/2003 datasets, so cat train.txt dev.txt test.txt | cut -d " " -f 2 | grep -v "^$"| sort | uniq > labels.txt ``` -### Training +### Prepare the run Additional environment variables must be set: @@ -522,6 +523,8 @@ export SAVE_STEPS=750 export SEED=1 ``` +### Run the Pytorch version + To start training, just run: ```bash @@ -542,7 +545,7 @@ python3 run_ner.py --data_dir ./ \ If your GPU supports half-precision training, just add the `--fp16` flag. After training, the model will be both evaluated on development and test datasets. -### Evaluation +#### Evaluation Evaluation on development dataset outputs the following for our example: @@ -564,7 +567,7 @@ On the test dataset the following results could be achieved: 10/04/2019 00:42:42 - INFO - __main__ - recall = 0.8624150210424085 ``` -### Comparing BERT (large, cased), RoBERTa (large, cased) and DistilBERT (base, uncased) +#### Comparing BERT (large, cased), RoBERTa (large, cased) and DistilBERT (base, uncased) Here is a small comparison between BERT (large, cased), RoBERTa (large, cased) and DistilBERT (base, uncased) with the same hyperparameters as specified in the [example documentation](https://huggingface.co/transformers/examples.html#named-entity-recognition) (one run): @@ -574,6 +577,72 @@ Here is a small comparison between BERT (large, cased), RoBERTa (large, cased) a | `roberta-large` | 95.96 | 91.87 | `distilbert-base-uncased` | 94.34 | 90.32 +### Run the Tensorflow 2 version + +To start training, just run: + +```bash +python3 run_tf_ner.py --data_dir ./ \ +--model_type bert \ +--labels ./labels.txt \ +--model_name_or_path $BERT_MODEL \ +--output_dir $OUTPUT_DIR \ +--max_seq_length $MAX_LENGTH \ +--num_train_epochs $NUM_EPOCHS \ +--per_device_train_batch_size $BATCH_SIZE \ +--save_steps $SAVE_STEPS \ +--seed $SEED \ +--do_train \ +--do_eval \ +--do_predict +``` + +Such as the Pytorch version, if your GPU supports half-precision training, just add the `--fp16` flag. After training, the model will be both evaluated on development and test datasets. + +#### Evaluation + +Evaluation on development dataset outputs the following for our example: +```bash + precision recall f1-score support + + LOCderiv 0.7619 0.6154 0.6809 52 + PERpart 0.8724 0.8997 0.8858 4057 + OTHpart 0.9360 0.9466 0.9413 711 + ORGpart 0.7015 0.6989 0.7002 269 + LOCpart 0.7668 0.8488 0.8057 496 + LOC 0.8745 0.9191 0.8963 235 + ORGderiv 0.7723 0.8571 0.8125 91 + OTHderiv 0.4800 0.6667 0.5581 18 + OTH 0.5789 0.6875 0.6286 16 + PERderiv 0.5385 0.3889 0.4516 18 + PER 0.5000 0.5000 0.5000 2 + ORG 0.0000 0.0000 0.0000 3 + +micro avg 0.8574 0.8862 0.8715 5968 +macro avg 0.8575 0.8862 0.8713 5968 +``` + +On the test dataset the following results could be achieved: +```bash + precision recall f1-score support + + PERpart 0.8847 0.8944 0.8896 9397 + OTHpart 0.9376 0.9353 0.9365 1639 + ORGpart 0.7307 0.7044 0.7173 697 + LOC 0.9133 0.9394 0.9262 561 + LOCpart 0.8058 0.8157 0.8107 1150 + ORG 0.0000 0.0000 0.0000 8 + OTHderiv 0.5882 0.4762 0.5263 42 + PERderiv 0.6571 0.5227 0.5823 44 + OTH 0.4906 0.6667 0.5652 39 + ORGderiv 0.7016 0.7791 0.7383 172 + LOCderiv 0.8256 0.6514 0.7282 109 + PER 0.0000 0.0000 0.0000 11 + +micro avg 0.8722 0.8774 0.8748 13869 +macro avg 0.8712 0.8774 0.8740 13869 +``` + ## Abstractive summarization Based on the script diff --git a/examples/run_tf_ner.py b/examples/run_tf_ner.py index ef1fcf6aa4..eb284f4c2a 100644 --- a/examples/run_tf_ner.py +++ b/examples/run_tf_ner.py @@ -540,6 +540,9 @@ def main(_): checkpoints = list(os.path.dirname(c) for c in sorted(glob.glob(args['output_dir'] + "/**/" + TF2_WEIGHTS_NAME, recursive=True), key=lambda f: int(''.join(filter(str.isdigit, f)) or -1))) logging.info("Evaluate the following checkpoints: %s", checkpoints) + + if len(checkpoints) == 0: + checkpoints.append(args['output_dir']) for checkpoint in checkpoints: global_step = checkpoint.split("-")[-1] if re.match(".*checkpoint-[0-9]", checkpoint) else "final" @@ -572,10 +575,10 @@ def main(_): if args['do_predict']: tokenizer = tokenizer_class.from_pretrained(args['output_dir'], do_lower_case=args['do_lower_case']) model = model_class.from_pretrained(args['output_dir']) - eval_batch_size = args['per_gpu_eval_batch_size'] * args['n_device'] + eval_batch_size = args['per_device_eval_batch_size'] * args['n_device'] predict_dataset, _ = load_and_cache_examples(args, tokenizer, labels, pad_token_label_id, eval_batch_size, mode="test") y_true, y_pred, pred_loss = evaluate(args, strategy, model, tokenizer, labels, pad_token_label_id, mode="test") - output_test_results_file = os.path.join(args.output_dir, "test_results.txt") + output_test_results_file = os.path.join(args['output_dir'], "test_results.txt") output_test_predictions_file = os.path.join(args['output_dir'], "test_predictions.txt") report = metrics.classification_report(y_true, y_pred, digits=4) diff --git a/transformers/tests/optimization_tf_test.py b/transformers/tests/optimization_tf_test.py new file mode 100644 index 0000000000..ac5109cb56 --- /dev/null +++ b/transformers/tests/optimization_tf_test.py @@ -0,0 +1,89 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import unittest +import pytest + +from transformers import is_tf_available + +if is_tf_available(): + import tensorflow as tf + from tensorflow.python.eager import context + from tensorflow.python.framework import ops + from transformers import (create_optimizer, GradientAccumulator) +else: + pytestmark = pytest.mark.skip("Require TensorFlow") + +class OptimizationFTest(unittest.TestCase): + def assertListAlmostEqual(self, list1, list2, tol): + self.assertEqual(len(list1), len(list2)) + for a, b in zip(list1, list2): + self.assertAlmostEqual(a, b, delta=tol) + + def testGradientAccumulator(self): + accumulator = GradientAccumulator() + accumulator([tf.constant([1.0, 2.0])]) + accumulator([tf.constant([-2.0, 1.0])]) + accumulator([tf.constant([-1.0, 2.0])]) + with self.assertRaises(ValueError): + accumulator([tf.constant([1.0, 1.0]), tf.constant([2.0, 2.0])]) + self.assertEqual(accumulator.step, 3) + self.assertEqual(len(accumulator.gradients), 1) + self.assertListAlmostEqual(accumulator.gradients[0].numpy().tolist(), [-2.0, 5.0], tol=1e-2) + accumulator.reset() + self.assertEqual(accumulator.step, 0) + self.assertListAlmostEqual(accumulator.gradients[0].numpy().tolist(), [0.0, 0.0], tol=1e-2) + + def testGradientAccumulatorDistributionStrategy(self): + context._context = None + ops.enable_eager_execution_internal() + physical_devices = tf.config.experimental.list_physical_devices("CPU") + tf.config.experimental.set_virtual_device_configuration( + physical_devices[0], + [tf.config.experimental.VirtualDeviceConfiguration(), + tf.config.experimental.VirtualDeviceConfiguration()]) + + devices = tf.config.experimental.list_logical_devices(device_type="CPU") + strategy = tf.distribute.MirroredStrategy(devices=[device.name for device in devices]) + + with strategy.scope(): + accumulator = GradientAccumulator() + variable = tf.Variable([4.0, 3.0]) + optimizer = create_optimizer(5e-5, 10, 5) + gradient_placeholder = tf.Variable([0.0, 0.0], trainable=False) + + def accumulate_on_replica(gradient): + accumulator([gradient]) + + def apply_on_replica(): + optimizer.apply_gradients(list(zip(accumulator.gradients, [variable])), 1.0) + + @tf.function + def accumulate(grad1, grad2): + with strategy.scope(): + gradient_placeholder.values[0].assign(grad1) + gradient_placeholder.values[1].assign(grad2) + strategy.experimental_run_v2(accumulate_on_replica, args=(gradient_placeholder,)) + + @tf.function + def apply_grad(): + with strategy.scope(): + strategy.experimental_run_v2(apply_on_replica) + + accumulate([1.0, 2.0], [-1.0, 1.0]) + accumulate([3.0, -1.0], [-1.0, -1.0]) + accumulate([-2.0, 2.0], [3.0, -2.0]) + self.assertEqual(accumulator.step, 3) + self.assertListAlmostEqual(accumulator._gradients[0].values[0].value().numpy().tolist(), [2.0, 3.0], tol=1e-2) + self.assertListAlmostEqual(accumulator._gradients[0].values[1].value().numpy().tolist(), [1.0, -2.0], tol=1e-2) + apply_grad() + self.assertListAlmostEqual(variable.value().numpy().tolist(), [4.0, 3.0], tol=1e-2) + accumulator.reset() + self.assertEqual(accumulator.step, 0) + self.assertListAlmostEqual(accumulator._gradients[0].values[0].value().numpy().tolist(), [0.0, 0.0], tol=1e-2) + self.assertListAlmostEqual(accumulator._gradients[0].values[1].value().numpy().tolist(), [0.0, 0.0], tol=1e-2) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file From 6c5297a42384c9234ad5fd2d044a5aa61d82ce97 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 5 Dec 2019 13:27:58 +0100 Subject: [PATCH 234/505] Fixing camembert tokenization --- transformers/tokenization_camembert.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/transformers/tokenization_camembert.py b/transformers/tokenization_camembert.py index bf2a6fe993..b4091558e1 100644 --- a/transformers/tokenization_camembert.py +++ b/transformers/tokenization_camembert.py @@ -51,7 +51,7 @@ class CamembertTokenizer(PreTrainedTokenizer): def __init__(self, vocab_file, bos_token="", eos_token="", sep_token="
", cls_token="", unk_token="", pad_token='', mask_token='', - additional_special_tokens=['NOTUSED', 'NOTUSED'], **kwargs): + additional_special_tokens=['NOTUSED', 'NOTUSED'], **kwargs): super(CamembertTokenizer, self).__init__(max_len=512, bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, sep_token=sep_token, cls_token=cls_token, pad_token=pad_token, mask_token=mask_token, additional_special_tokens=additional_special_tokens, @@ -125,7 +125,7 @@ class CamembertTokenizer(PreTrainedTokenizer): @property def vocab_size(self): - return self.fairseq_offset + len(self.sp_model) + return len(self.fairseq_tokens_to_ids) + len(self.sp_model) def _tokenize(self, text): return self.sp_model.EncodeAsPieces(text) @@ -134,6 +134,9 @@ class CamembertTokenizer(PreTrainedTokenizer): """ Converts a token (str/unicode) in an id using the vocab. """ if token in self.fairseq_tokens_to_ids: return self.fairseq_tokens_to_ids[token] + elif self.sp_model.PieceToId(token) == 0: + # Convert sentence piece unk token to fairseq unk token index + return self.unk_token_id return self.fairseq_offset + self.sp_model.PieceToId(token) def _convert_id_to_token(self, index): From 3268ebd2290800036fce0b931dc6c9b87b76e098 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 5 Dec 2019 13:35:29 +0100 Subject: [PATCH 235/505] fix xlnet test --- transformers/tests/modeling_xlnet_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/tests/modeling_xlnet_test.py b/transformers/tests/modeling_xlnet_test.py index 3f0ef6793c..38888d4488 100644 --- a/transformers/tests/modeling_xlnet_test.py +++ b/transformers/tests/modeling_xlnet_test.py @@ -167,7 +167,7 @@ class XLNetModelTest(CommonTestCases.CommonModelTester): [[self.seq_length, self.batch_size, self.hidden_size]] * self.num_hidden_layers) def create_and_check_xlnet_base_model_with_att_output(self, config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, - target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels): + target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels, token_labels): model = XLNetModel(config) model.eval() From 2d5d86e03779b4b316698438caff0f675ee54abd Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 5 Dec 2019 14:06:29 +0100 Subject: [PATCH 236/505] fix #2031 --- transformers/tokenization_albert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/tokenization_albert.py b/transformers/tokenization_albert.py index 2f9af0b0bc..40a4b29206 100644 --- a/transformers/tokenization_albert.py +++ b/transformers/tokenization_albert.py @@ -66,7 +66,7 @@ class AlbertTokenizer(PreTrainedTokenizer): def __init__(self, vocab_file, do_lower_case=True, remove_space=True, keep_accents=False, bos_token="[CLS]", eos_token="[SEP]", unk_token="", sep_token="[SEP]", - pad_token="", cls_token="[CLS]", mask_token="[MASK]>", **kwargs): + pad_token="", cls_token="[CLS]", mask_token="[MASK]", **kwargs): super(AlbertTokenizer, self).__init__(bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, sep_token=sep_token, pad_token=pad_token, cls_token=cls_token, From 18fb93530ba0c1f6a45240270b24dc5c5da340ae Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 5 Dec 2019 14:36:34 +0100 Subject: [PATCH 237/505] fixing #2042 - Nicer error message --- transformers/modeling_bert.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py index 5f92fb96a3..1ee3e3f097 100644 --- a/transformers/modeling_bert.py +++ b/transformers/modeling_bert.py @@ -667,11 +667,10 @@ class BertModel(BertPreTrainedModel): # ourselves in which case we just need to make it broadcastable to all heads. if attention_mask.dim() == 3: extended_attention_mask = attention_mask[:, None, :, :] - - # Provided a padding mask of dimensions [batch_size, seq_length] - # - if the model is a decoder, apply a causal mask in addition to the padding mask - # - if the model is an encoder, make the mask broadcastable to [batch_size, num_heads, seq_length, seq_length] - if attention_mask.dim() == 2: + elif attention_mask.dim() == 2: + # Provided a padding mask of dimensions [batch_size, seq_length] + # - if the model is a decoder, apply a causal mask in addition to the padding mask + # - if the model is an encoder, make the mask broadcastable to [batch_size, num_heads, seq_length, seq_length] if self.config.is_decoder: batch_size, seq_length = input_shape seq_ids = torch.arange(seq_length, device=device) @@ -679,6 +678,8 @@ class BertModel(BertPreTrainedModel): extended_attention_mask = causal_mask[:, None, :, :] * attention_mask[:, None, None, :] else: extended_attention_mask = attention_mask[:, None, None, :] + else: + raise ValueError("Wrong shape for input_ids (shape {}) or attention_mask (shape {})".format(input_shape, attention_mask.shape)) # Since attention_mask is 1.0 for positions we want to attend and 0.0 for # masked positions, this operation will create a tensor which is 0.0 for @@ -696,8 +697,11 @@ class BertModel(BertPreTrainedModel): if encoder_attention_mask.dim() == 3: encoder_extended_attention_mask = encoder_attention_mask[:, None, :, :] - if encoder_attention_mask.dim() == 2: + elif encoder_attention_mask.dim() == 2: encoder_extended_attention_mask = encoder_attention_mask[:, None, None, :] + else: + raise ValueError("Wrong shape for input_ids (shape {}) or encoder_attention_mask (shape {})".format(input_shape, + encoder_attention_mask.shape)) encoder_extended_attention_mask = encoder_extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility encoder_extended_attention_mask = (1.0 - encoder_extended_attention_mask) * -10000.0 From f8fb4335c9cd79789ed6119e729348e0a1b51e2b Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 5 Dec 2019 15:19:32 +0100 Subject: [PATCH 238/505] clean up a little bit PT <=> TF conversion --- transformers/convert_pytorch_checkpoint_to_tf2.py | 9 +++++---- transformers/modeling_utils.py | 3 ++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/transformers/convert_pytorch_checkpoint_to_tf2.py b/transformers/convert_pytorch_checkpoint_to_tf2.py index d1776e9c14..d20eafe2e9 100644 --- a/transformers/convert_pytorch_checkpoint_to_tf2.py +++ b/transformers/convert_pytorch_checkpoint_to_tf2.py @@ -119,10 +119,11 @@ def convert_pt_checkpoint_to_tf(model_type, pytorch_checkpoint_path, config_file tf_inputs = tf.constant(inputs_list) tfo = tf_model(tf_inputs, training=False) # build the network - pt_model = pt_model_class.from_pretrained(None, - config=config, - state_dict=torch.load(pytorch_checkpoint_path, - map_location='cpu')) + pt_model = pt_model_class(config) + pt_model.load_state_dict(torch.load(pytorch_checkpoint_path, map_location='cpu'), + strict-False) + pt_model.eval() + pt_inputs = torch.tensor(inputs_list) with torch.no_grad(): pto = pt_model(pt_inputs) diff --git a/transformers/modeling_utils.py b/transformers/modeling_utils.py index 398172a88c..3ac568771e 100644 --- a/transformers/modeling_utils.py +++ b/transformers/modeling_utils.py @@ -318,7 +318,8 @@ class PreTrainedModel(nn.Module): model = BertModel.from_pretrained('./tf_model/my_tf_checkpoint.ckpt.index', from_tf=True, config=config) """ - if "albert" in pretrained_model_name_or_path and "v2" in pretrained_model_name_or_path: + if pretrained_model_name_or_path is not None and ( + "albert" in pretrained_model_name_or_path and "v2" in pretrained_model_name_or_path): logger.warning("There is currently an upstream reproducibility issue with ALBERT v2 models. Please see " + "https://github.com/google-research/google-research/issues/119 for more information.") From ee53de7aac8312140e87d452718e15e3d42e27dd Mon Sep 17 00:00:00 2001 From: Rosanne Liu Date: Thu, 5 Dec 2019 06:20:07 -0800 Subject: [PATCH 239/505] Pr for pplm (#2060) * license * changes * ok * Update paper link and commands to run * pointer to uber repo --- examples/pplm/README.md | 23 +++++++----------- examples/pplm/run_pplm.py | 32 +++++++++---------------- examples/pplm/run_pplm_discrim_train.py | 14 ++++++++++- 3 files changed, 32 insertions(+), 37 deletions(-) diff --git a/examples/pplm/README.md b/examples/pplm/README.md index 103218ae72..b12205854a 100644 --- a/examples/pplm/README.md +++ b/examples/pplm/README.md @@ -1,17 +1,15 @@ -# PPLM +# Plug and Play Language Models: a Simple Approach to Controlled Text Generation + +Authors: [Sumanth Dathathri](https://dathath.github.io/), [Andrea Madotto](https://andreamad8.github.io/), Janice Lan, Jane Hung, Eric Frank, [Piero Molino](https://w4nderlu.st/), [Jason Yosinski](http://yosinski.com/), and [Rosanne Liu](http://www.rosanneliu.com/) This folder contains the original code used to run the Plug and Play Language Model (PPLM). -![header image](./imgs/headfigure.png) -## Plug and Play Language Models: a Simple Approach to Steerable Text Generation -Authors: [Sumanth Dathathri](https://dathath.github.io/), Andrea Madotto, Janice Lan, Jane Hung, Eric Frank, [Piero Molino](https://w4nderlu.st/), [Jason Yosinski](http://yosinski.com/), and [Rosanne Liu](http://www.rosanneliu.com/) - -PPLM allows a user to flexibly plug in one or more tiny attribute models representing the desired steering objective into a large, unconditional LM. The method has the key property that it uses the LM _as is_---no training or fine-tuning is required---which enables researchers to leverage best-in-class LMs even if they do not have the extensive hardware required to train them. - -Paper link: +Paper link: https://arxiv.org/abs/1912.02164 Blog link: https://eng.uber.com/pplm +Please check out the repo under uber-research for more information: https://github.com/uber-research/PPLM + ## Setup @@ -27,7 +25,7 @@ cd examples/pplm ### Example command for bag-of-words control ```bash -python run_pplm.py -B space --cond_text "The president" --length 100 --gamma 1.5 --num_iterations 3 --num_samples 1 --stepsize 0.01 --window_length 5 --kl_scale 0.01 --gm_scale 0.95 +python run_pplm.py -B military --cond_text "The potato" --length 50 --gamma 1.5 --num_iterations 3 --num_samples 10 --stepsize 0.03 --window_length 5 --kl_scale 0.01 --gm_scale 0.99 --colorama --sample ``` ### Tuning hyperparameters for bag-of-words control @@ -45,7 +43,7 @@ python run_pplm.py -B space --cond_text "The president" --length 100 --gamma 1.5 ### Example command for discriminator based sentiment control ```bash -python run_pplm.py -D sentiment --class_label 3 --cond_text "The lake" --length 10 --gamma 1.0 --num_iterations 10 --num_samples 1 --stepsize 0.03 --kl_scale 0.01 --gm_scale 0.95 +python run_pplm.py -D sentiment --class_label 2 --cond_text "My dog died" --length 50 --gamma 1.0 --num_iterations 10 --num_samples 10 --stepsize 0.04 --kl_scale 0.01 --gm_scale 0.95 --sample ``` ### Tuning hyperparameters for discriminator control @@ -54,8 +52,3 @@ python run_pplm.py -D sentiment --class_label 3 --cond_text "The lake" --length 2. Use `--class_label 3` for negative, and `--class_label 2` for positive -### Example command for detoxificiation: - -```bash -python run_pplm.py -D toxicity --length 100 --num_iterations 10 --cond-text 'TH PEOPLEMan goddreams Blacks' --gamma 1.0 --num_samples 10 --stepsize 0.02 -``` diff --git a/examples/pplm/run_pplm.py b/examples/pplm/run_pplm.py index dda5d85ae7..095dc39a74 100644 --- a/examples/pplm/run_pplm.py +++ b/examples/pplm/run_pplm.py @@ -1,18 +1,19 @@ #! /usr/bin/env python3 # coding=utf-8 -# Copyright 2018 The Uber AI Team Authors. + +#Copyright (c) 2019 Uber Technologies, Inc. # -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +#http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. """ Example command with bag of words: @@ -45,12 +46,9 @@ SMALL_CONST = 1e-15 BIG_CONST = 1e10 BAG_OF_WORDS_ARCHIVE_MAP = { - 'kitchen': "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/bow/kitchen.txt", 'legal': "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/bow/legal.txt", 'military': "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/bow/military.txt", - 'monsters': "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/bow/monsters.txt", 'politics': "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/bow/politics.txt", - 'positive_words': "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/bow/positive_words.txt", 'religion': "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/bow/religion.txt", 'science': "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/bow/science.txt", 'space': "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/bow/space.txt", @@ -74,14 +72,6 @@ DISCRIMINATOR_MODELS_PARAMS = { "default_class": 3, "pretrained_model": "gpt2-medium", }, - "toxicity": { - "url": "https://s3.amazonaws.com/models.huggingface.co/bert/pplm/discriminators/toxic_classifier_head.pt", - "class_size": 2, - "embed_size": 1024, - "class_vocab": {"non_toxic": 0, "toxic": 1}, - "default_class": 0, - "pretrained_model": "gpt2-medium", - }, } diff --git a/examples/pplm/run_pplm_discrim_train.py b/examples/pplm/run_pplm_discrim_train.py index 9d36b79bc4..3055139d8c 100644 --- a/examples/pplm/run_pplm_discrim_train.py +++ b/examples/pplm/run_pplm_discrim_train.py @@ -1,7 +1,19 @@ #! /usr/bin/env python3 # coding=utf-8 -# This code is licensed under a non-commercial license. +#Copyright (c) 2019 Uber Technologies, Inc. +# +#Licensed under the Apache License, Version 2.0 (the "License"); +#you may not use this file except in compliance with the License. +#You may obtain a copy of the License at +# +#http://www.apache.org/licenses/LICENSE-2.0 +# +#Unless required by applicable law or agreed to in writing, software +#distributed under the License is distributed on an "AS IS" BASIS, +#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +#See the License for the specific language governing permissions and +#limitations under the License. import argparse import csv From 552c44a9b19fa4f5593ac3bc8de0e9f46495ecd0 Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Thu, 5 Dec 2019 10:14:58 -0500 Subject: [PATCH 240/505] release distilm-bert --- README.md | 2 +- docs/source/pretrained_models.rst | 4 ++++ examples/distillation/README.md | 20 ++++++++++++++++---- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 3173b6cf10..ddeabe08d6 100644 --- a/README.md +++ b/README.md @@ -131,7 +131,7 @@ At some point in the future, you'll be able to seamlessly move from pre-training 5. **[XLNet](https://github.com/zihangdai/xlnet/)** (from Google/CMU) released with the paper [​XLNet: Generalized Autoregressive Pretraining for Language Understanding](https://arxiv.org/abs/1906.08237) by Zhilin Yang*, Zihang Dai*, Yiming Yang, Jaime Carbonell, Ruslan Salakhutdinov, Quoc V. Le. 6. **[XLM](https://github.com/facebookresearch/XLM/)** (from Facebook) released together with the paper [Cross-lingual Language Model Pretraining](https://arxiv.org/abs/1901.07291) by Guillaume Lample and Alexis Conneau. 7. **[RoBERTa](https://github.com/pytorch/fairseq/tree/master/examples/roberta)** (from Facebook), released together with the paper a [Robustly Optimized BERT Pretraining Approach](https://arxiv.org/abs/1907.11692) by Yinhan Liu, Myle Ott, Naman Goyal, Jingfei Du, Mandar Joshi, Danqi Chen, Omer Levy, Mike Lewis, Luke Zettlemoyer, Veselin Stoyanov. -8. **[DistilBERT](https://github.com/huggingface/transformers/tree/master/examples/distillation)** (from HuggingFace), released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter](https://arxiv.org/abs/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same method has been applied to compress GPT2 into [DistilGPT2](https://github.com/huggingface/transformers/tree/master/examples/distillation). +8. **[DistilBERT](https://github.com/huggingface/transformers/tree/master/examples/distillation)** (from HuggingFace), released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter](https://arxiv.org/abs/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same method has been applied to compress GPT2 into [DistilGPT2](https://github.com/huggingface/transformers/tree/master/examples/distillation), RoBERTa into [DistilRoBERTa](https://github.com/huggingface/transformers/tree/master/examples/distillation), Multilingual BERT into [DistilmBERT](https://github.com/huggingface/transformers/tree/master/examples/distillation) and a German version of DistilBERT. 9. **[CTRL](https://github.com/salesforce/ctrl/)** (from Salesforce) released with the paper [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/abs/1909.05858) by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher. 10. **[CamemBERT](https://camembert-model.fr)** (from Inria/Facebook/Sorbonne) released with the paper [CamemBERT: a Tasty French Language Model](https://arxiv.org/abs/1911.03894) by Louis Martin*, Benjamin Muller*, Pedro Javier Ortiz Suárez*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah and Benoît Sagot. 11. **[ALBERT](https://github.com/google-research/google-research/tree/master/albert)** (from Google Research and the Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst index 0cd794a678..090cb75808 100644 --- a/docs/source/pretrained_models.rst +++ b/docs/source/pretrained_models.rst @@ -155,6 +155,10 @@ Here is the full list of the currently provided pretrained models together with | | ``distilbert-base-german-cased`` | | 6-layer, 768-hidden, 12-heads, 66M parameters | | | | | The German DistilBERT model distilled from the German DBMDZ BERT model `bert-base-german-dbmdz-cased` checkpoint. | | | | (see `details `__) | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``distilbert-base-multilingual-cased`` | | 6-layer, 768-hidden, 12-heads, 134M parameters | +| | | | The multilingual DistilBERT model distilled from the Multilingual BERT model `bert-base-multilingual-cased` checkpoint. | +| | | (see `details `__) | +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | CTRL | ``ctrl`` | | 48-layer, 1280-hidden, 16-heads, 1.6B parameters | | | | | Salesforce's Large-sized CTRL English model | diff --git a/examples/distillation/README.md b/examples/distillation/README.md index c2765c28ff..24a1677db1 100644 --- a/examples/distillation/README.md +++ b/examples/distillation/README.md @@ -2,6 +2,8 @@ This folder contains the original code used to train Distil* as well as examples showcasing how to use DistilBERT, DistilRoBERTa and DistilGPT2. +**December 6th, 2019 - Update** We release **DistilmBERT**: 92% of `bert-base-multilingual-cased` on XNLI. The model supports 104 different languages listed [here](https://github.com/google-research/bert/blob/master/multilingual.md#list-of-languages). + **November 19th, 2019 - Update** We release German **DistilBERT**: 98.8% of `bert-base-german-dbmdz-cased` on NER tasks. **October 23rd, 2019 - Update** We release **DistilRoBERTa**: 95% of `RoBERTa-base`'s performance on GLUE, twice as fast as RoBERTa while being 35% smaller. @@ -17,8 +19,9 @@ Distil* is a class of compressed models that started with DistilBERT. DistilBERT We have applied the same method to other Transformer architectures and released the weights: - GPT2: on the [WikiText-103](https://blog.einstein.ai/the-wikitext-long-term-dependency-language-modeling-dataset/) benchmark, GPT2 reaches a perplexity on the test set of 15.0 compared to 18.5 for **DistilGPT2** (after fine-tuning on the train set). -- RoBERTa: **DistilRoBERTa** reaches 95% of `RoBERTa-base` performance on GLUE while being twice faster and 35% smaller. -- and more to come! 🤗🤗🤗 +- RoBERTa: **DistilRoBERTa** reaches 95% of `RoBERTa-base`'s performance on GLUE while being twice faster and 35% smaller. +- German BERT: **German DistilBERT** reaches 99% of `bert-base-german-dbmdz-cased`'s performance on German NER (CoNLL-2003). +- Multilingual BERT: **DistilmBERT** reaches 92% of Multilingual BERT's performance on XNLI while being twice faster and 25% smaller. The model supports 104 languages listed [here](https://github.com/google-research/bert/blob/master/multilingual.md#list-of-languages). For more information on DistilBERT, please refer to our [NeurIPS workshop paper](https://arxiv.org/abs/1910.01108). @@ -29,7 +32,7 @@ Here are the results on the dev sets of GLUE: | BERT-base | **77.6** | 48.9 | 84.3 | 88.6 | 89.3 | 89.5 | 71.3 | 91.7 | 91.2 | 43.7 | | DistilBERT | **76.8** | 49.1 | 81.8 | 90.2 | 90.2 | 89.2 | 62.9 | 92.7 | 90.7 | 44.4 | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| RoBERTa-base (reported) | **83.2**/**86.4**2 | 63.6 | 87.6 | 90.2 | 92.8 | 91.9 | 78.7 | 94.8 | 91.2 | 57.73 | +| RoBERTa-base (reported) | **83.2**/**86.4**2 | 63.6 | 87.6 | 90.2 | 92.8 | 91.9 | 78.7 | 94.8 | 91.2 | 57.73 | | DistilRoBERTa1 | **79.0**/**82.3**2 | 59.4 | 83.9 | 86.6 | 90.8 | 89.4 | 67.9 | 92.5 | 88.3 | 52.1 | 1 We did not use the MNLI checkpoint for fine-tuning but directy perform transfer learning on the pre-trained DistilRoBERTa. @@ -38,6 +41,14 @@ Here are the results on the dev sets of GLUE: 3 We compute this score ourselves for completeness. +Here are the results on the *test* sets for 6 of the languages available in XNLI. The results are computed in the zero shot setting (trained on the English portion and evaluated on the target language portion): + +| Model | English | Spanish | Chinese | German | Arabic | Urdu | +| :---: | :---: | :---: | :---: | :---: | :---: | :---:| +| mBERT base cased (computed) | 82.1 | 74.6 | 69.1 | 72.3 | 66.4 | 58.5 | +| mBERT base uncased (reported)| 81.4 | 74.3 | 63.8 | 70.5 | 62.1 | 58.3 | +| DistilmBERT | 78.2 | 69.1 | 64.0 | 66.3 | 59.1 | 54.7 | + ## Setup This part of the library has only be tested with Python3.6+. There are few specific dependencies to install before launching a distillation, you can install them with the command `pip install -r requirements.txt`. @@ -54,7 +65,7 @@ Transformers includes five pre-trained Distil* models, currently only provided f - `distilbert-base-german-cased`: DistilBERT German language model pretrained on 1/2 of the data used to pretrain Bert using distillation with the supervision of the `bert-base-german-dbmdz-cased` version of German DBMDZ Bert. For NER tasks the model reaches a F1 score of 83.49 on the CoNLL-2003 test set (for comparison, `bert-base-german-dbmdz-cased` reaches a 84.52 F1 score), and a F1 score of 85.23 on the GermEval 2014 test set (`bert-base-german-dbmdz-cased` reaches a 86.89 F1 score). - `distilgpt2`: DistilGPT2 English language model pretrained with the supervision of `gpt2` (the smallest version of GPT2) on [OpenWebTextCorpus](https://skylion007.github.io/OpenWebTextCorpus/), a reproduction of OpenAI's WebText dataset. The model has 6 layers, 768 dimension and 12 heads, totalizing 82M parameters (compared to 124M parameters for GPT2). On average, DistilGPT2 is two times faster than GPT2. - `distilroberta-base`: DistilRoBERTa English language model pretrained with the supervision of `roberta-base` solely on [OpenWebTextCorpus](https://skylion007.github.io/OpenWebTextCorpus/), a reproduction of OpenAI's WebText dataset (it is ~4 times less training data than the teacher RoBERTa). The model has 6 layers, 768 dimension and 12 heads, totalizing 82M parameters (compared to 125M parameters for RoBERTa-base). On average DistilRoBERTa is twice as fast as Roberta-base. -- and more to come! 🤗🤗🤗 +- `distilbert-base-multilingual-cased`: DistilmBERT multilingual model pretrained with the supervision of `bert-base-multilingual-cased` on the concatenation of Wikipedia in 104 different languages. The model supports the 104 languages listed [here](https://github.com/google-research/bert/blob/master/multilingual.md#list-of-languages). The model has 6 layers, 768 dimension and 12 heads, totalizing 134M parameters (compared to 177M parameters for mBERT-base). On average DistilmBERT is twice as fast as mBERT-base. Using DistilBERT is very similar to using BERT. DistilBERT share the same tokenizer as BERT's `bert-base-uncased` even though we provide a link to this tokenizer under the `DistilBertTokenizer` name to have a consistent naming between the library models. @@ -70,6 +81,7 @@ last_hidden_states = outputs[0] # The last hidden-state is the first element of Similarly, using the other Distil* models simply consists in calling the base classes with a different pretrained checkpoint: - DistilGPT2: `model = GPT2Model.from_pretrained('distilgpt2')` - DistilRoBERTa: `model = RobertaModel.from_pretrained('distilroberta-base')` +- DistilmBERT: `model = DistilBertModel.from_pretrained('distilbert-base-multilingual-cased')` ## How to train Distil* From 35ff345fc9df9e777b27903f11fa213e4052595b Mon Sep 17 00:00:00 2001 From: VictorSanh Date: Thu, 5 Dec 2019 12:07:04 -0500 Subject: [PATCH 241/505] update requirements --- examples/distillation/distiller.py | 1 - examples/distillation/requirements.txt | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/distillation/distiller.py b/examples/distillation/distiller.py index 0442072e84..1e33190aca 100644 --- a/examples/distillation/distiller.py +++ b/examples/distillation/distiller.py @@ -21,7 +21,6 @@ import psutil import time from tqdm import trange, tqdm import numpy as np -import psutil import torch import torch.nn as nn diff --git a/examples/distillation/requirements.txt b/examples/distillation/requirements.txt index d76273b34a..491924ee2c 100644 --- a/examples/distillation/requirements.txt +++ b/examples/distillation/requirements.txt @@ -3,4 +3,4 @@ tensorboard>=1.14.0 tensorboardX==1.8 psutil==5.6.3 scipy==1.3.1 -transformers==2.0.0 +transformers From 9ecd83dace3961eaa161405814b00ea595c86451 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Thu, 5 Dec 2019 14:44:57 -0500 Subject: [PATCH 242/505] Patch evaluation for impossible values + cleanup --- docs/source/main_classes/processors.rst | 4 ++-- examples/run_squad.py | 25 +++++-------------------- transformers/data/processors/squad.py | 6 +++--- transformers/tokenization_utils.py | 2 +- 4 files changed, 11 insertions(+), 26 deletions(-) diff --git a/docs/source/main_classes/processors.rst b/docs/source/main_classes/processors.rst index ce0eeb553a..e98910ae1b 100644 --- a/docs/source/main_classes/processors.rst +++ b/docs/source/main_classes/processors.rst @@ -55,7 +55,7 @@ Example usage ^^^^^^^^^^^^^^^^^^^^^^^^^ An example using these processors is given in the -`run_glue.py `__ script. +`run_glue.py `__ script. @@ -132,4 +132,4 @@ Example:: Another example using these processors is given in the -`run_squad.py `__ script. \ No newline at end of file +`run_squad.py `__ script. \ No newline at end of file diff --git a/examples/run_squad.py b/examples/run_squad.py index 3f1b6a798f..5caff9ae4f 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -311,7 +311,8 @@ def load_and_cache_examples(args, tokenizer, evaluate=False, output_examples=Fal str(args.max_seq_length))) if os.path.exists(cached_features_file) and not args.overwrite_cache and not output_examples: logger.info("Loading features from cached file %s", cached_features_file) - features = torch.load(cached_features_file) + features_and_dataset = torch.load(cached_features_file) + features, dataset = features_and_dataset["features"], features_and_dataset["dataset"] else: logger.info("Creating features from dataset file at %s", input_dir) @@ -330,40 +331,24 @@ def load_and_cache_examples(args, tokenizer, evaluate=False, output_examples=Fal processor = SquadV2Processor() if args.version_2_with_negative else SquadV1Processor() examples = processor.get_dev_examples(args.data_dir) if evaluate else processor.get_train_examples(args.data_dir) - features = squad_convert_examples_to_features( + features, dataset = squad_convert_examples_to_features( examples=examples, tokenizer=tokenizer, max_seq_length=args.max_seq_length, doc_stride=args.doc_stride, max_query_length=args.max_query_length, is_training=not evaluate, + return_dataset='pt' ) if args.local_rank in [-1, 0]: logger.info("Saving features into cached file %s", cached_features_file) - torch.save(features, cached_features_file) + torch.save({"features": features, "dataset": dataset}, cached_features_file) if args.local_rank == 0 and not evaluate: torch.distributed.barrier() # Make sure only the first process in distributed training process the dataset, and the others will use the cache - # Convert to Tensors and build dataset - all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long) - all_input_mask = torch.tensor([f.attention_mask for f in features], dtype=torch.long) - all_segment_ids = torch.tensor([f.token_type_ids for f in features], dtype=torch.long) - all_cls_index = torch.tensor([f.cls_index for f in features], dtype=torch.long) - all_p_mask = torch.tensor([f.p_mask for f in features], dtype=torch.float) - if evaluate: - all_example_index = torch.arange(all_input_ids.size(0), dtype=torch.long) - dataset = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, - all_example_index, all_cls_index, all_p_mask) - else: - all_start_positions = torch.tensor([f.start_position for f in features], dtype=torch.long) - all_end_positions = torch.tensor([f.end_position for f in features], dtype=torch.long) - dataset = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, - all_start_positions, all_end_positions, - all_cls_index, all_p_mask) - if output_examples: return dataset, examples, features return dataset diff --git a/transformers/data/processors/squad.py b/transformers/data/processors/squad.py index 338bae0c51..bb56aa792f 100644 --- a/transformers/data/processors/squad.py +++ b/transformers/data/processors/squad.py @@ -312,7 +312,7 @@ class SquadProcessor(DataProcessor): if not evaluate: answer = tensor_dict['answers']['text'][0].numpy().decode('utf-8') answer_start = tensor_dict['answers']['answer_start'][0].numpy() - answers = None + answers = [] else: answers = [{ "answer_start": start.numpy(), @@ -408,7 +408,7 @@ class SquadProcessor(DataProcessor): question_text = qa["question"] start_position_character = None answer_text = None - answers = None + answers = [] if "is_impossible" in qa: is_impossible = qa["is_impossible"] @@ -469,7 +469,7 @@ class SquadExample(object): answer_text, start_position_character, title, - answers=None, + answers=[], is_impossible=False): self.qas_id = qas_id self.question_text = question_text diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index 41a611ea49..5ec173bbf6 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -194,7 +194,7 @@ class PreTrainedTokenizer(object): @property def pad_token_type_id(self): - """ Id of the padding token in the vocabulary. Log an error if used while not having been set. """ + """ Id of the padding token type in the vocabulary.""" return self._pad_token_type_id @property From e9217da5ff711cf84d150b35d3f8a5c17f1641f7 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Thu, 5 Dec 2019 16:01:51 -0500 Subject: [PATCH 243/505] Cleanup Improve global visibility on the run_squad script, remove unused files and fixes related to XLNet. --- examples/run_squad.py | 69 +- examples/utils_squad.py | 1017 -------------------- examples/utils_squad_evaluate.py | 330 ------- transformers/data/metrics/squad_metrics.py | 14 +- transformers/data/processors/squad.py | 2 +- 5 files changed, 45 insertions(+), 1387 deletions(-) delete mode 100644 examples/utils_squad.py delete mode 100644 examples/utils_squad_evaluate.py diff --git a/examples/run_squad.py b/examples/run_squad.py index 5caff9ae4f..6d32211c0c 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -27,8 +27,7 @@ import glob import timeit import numpy as np import torch -from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler, - TensorDataset) +from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler, TensorDataset) from torch.utils.data.distributed import DistributedSampler try: @@ -48,14 +47,6 @@ from transformers import (WEIGHTS_NAME, BertConfig, from transformers import AdamW, get_linear_schedule_with_warmup, squad_convert_examples_to_features -from utils_squad import (convert_examples_to_features as old_convert, read_squad_examples as old_read, RawResult, write_predictions, - RawResultExtended, write_predictions_extended) - -# The follwing import is the official SQuAD evaluation script (2.0). -# You can remove it from the dependencies if you are using this script outside of the library -# We've added it here for automated tests (see examples/test_examples.py file) -from utils_squad_evaluate import EVAL_OPTS, main as evaluate_on_squad - logger = logging.getLogger(__name__) ALL_MODELS = sum((tuple(conf.pretrained_config_archive_map.keys()) \ @@ -98,14 +89,16 @@ def train(args, train_dataset, model, tokenizer): optimizer_grouped_parameters = [ {'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': args.weight_decay}, {'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0} - ] + ] optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total) + if args.fp16: try: from apex import amp except ImportError: raise ImportError("Please install apex from https://www.github.com/nvidia/apex to use fp16 training.") + model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level) # multi-gpu training (should be after apex fp16 initialization) @@ -133,20 +126,26 @@ def train(args, train_dataset, model, tokenizer): model.zero_grad() train_iterator = trange(int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0]) set_seed(args) # Added here for reproductibility (even between python 2 and 3) + for _ in train_iterator: epoch_iterator = tqdm(train_dataloader, desc="Iteration", disable=args.local_rank not in [-1, 0]) for step, batch in enumerate(epoch_iterator): model.train() batch = tuple(t.to(args.device) for t in batch) - inputs = {'input_ids': batch[0], - 'attention_mask': batch[1], - 'start_positions': batch[3], - 'end_positions': batch[4]} + + inputs = { + 'input_ids': batch[0], + 'attention_mask': batch[1], + 'start_positions': batch[3], + 'end_positions': batch[4] + } + if args.model_type != 'distilbert': inputs['token_type_ids'] = None if args.model_type == 'xlm' else batch[2] + if args.model_type in ['xlnet', 'xlm']: - inputs.update({'cls_index': batch[5], - 'p_mask': batch[6]}) + inputs.update({'cls_index': batch[5], 'p_mask': batch[6]}) + outputs = model(**inputs) loss = outputs[0] # model outputs are always tuple in transformers (see doc) @@ -173,8 +172,8 @@ def train(args, train_dataset, model, tokenizer): model.zero_grad() global_step += 1 + # Log metrics if args.local_rank in [-1, 0] and args.logging_steps > 0 and global_step % args.logging_steps == 0: - # Log metrics if args.local_rank == -1 and args.evaluate_during_training: # Only evaluate when single GPU otherwise metrics may not average well results = evaluate(args, model, tokenizer) for key, value in results.items(): @@ -183,8 +182,8 @@ def train(args, train_dataset, model, tokenizer): tb_writer.add_scalar('loss', (tr_loss - logging_loss)/args.logging_steps, global_step) logging_loss = tr_loss + # Save model checkpoint if args.local_rank in [-1, 0] and args.save_steps > 0 and global_step % args.save_steps == 0: - # Save model checkpoint output_dir = os.path.join(args.output_dir, 'checkpoint-{}'.format(global_step)) if not os.path.exists(output_dir): os.makedirs(output_dir) @@ -213,6 +212,7 @@ def evaluate(args, model, tokenizer, prefix=""): os.makedirs(args.output_dir) args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu) + # Note that DistributedSampler samples randomly eval_sampler = SequentialSampler(dataset) if args.local_rank == -1 else DistributedSampler(dataset) eval_dataloader = DataLoader(dataset, sampler=eval_sampler, batch_size=args.eval_batch_size) @@ -225,11 +225,14 @@ def evaluate(args, model, tokenizer, prefix=""): logger.info("***** Running evaluation {} *****".format(prefix)) logger.info(" Num examples = %d", len(dataset)) logger.info(" Batch size = %d", args.eval_batch_size) + all_results = [] start_time = timeit.default_timer() + for batch in tqdm(eval_dataloader, desc="Evaluating"): model.eval() batch = tuple(t.to(args.device) for t in batch) + with torch.no_grad(): inputs = { 'input_ids': batch[0], @@ -238,10 +241,13 @@ def evaluate(args, model, tokenizer, prefix=""): if args.model_type != 'distilbert': inputs['token_type_ids'] = None if args.model_type == 'xlm' else batch[2] # XLM don't use segment_ids + example_indices = batch[3] + + # XLNet and XLM use more arguments for their predictions if args.model_type in ['xlnet', 'xlm']: - inputs.update({'cls_index': batch[4], - 'p_mask': batch[5]}) + inputs.update({'cls_index': batch[4], 'p_mask': batch[5]}) + outputs = model(**inputs) for i, example_index in enumerate(example_indices): @@ -250,11 +256,13 @@ def evaluate(args, model, tokenizer, prefix=""): output = [to_list(output[i]) for output in outputs] + # Some models (XLNet, XLM) use 5 arguments for their predictions, while the other "simpler" + # models only use two. if len(output) >= 5: start_logits = output[0] start_top_index = output[1] end_logits = output[2] - end_top_index = output[3], + end_top_index = output[3] cls_logits = output[4] result = SquadResult( @@ -278,16 +286,17 @@ def evaluate(args, model, tokenizer, prefix=""): # Compute predictions output_prediction_file = os.path.join(args.output_dir, "predictions_{}.json".format(prefix)) output_nbest_file = os.path.join(args.output_dir, "nbest_predictions_{}.json".format(prefix)) + if args.version_2_with_negative: output_null_log_odds_file = os.path.join(args.output_dir, "null_odds_{}.json".format(prefix)) else: output_null_log_odds_file = None + # XLNet and XLM use a more complex post-processing procedure if args.model_type in ['xlnet', 'xlm']: - # XLNet uses a more complex post-processing procedure predictions = compute_predictions_log_probs(examples, features, all_results, args.n_best_size, args.max_answer_length, output_prediction_file, - output_nbest_file, output_null_log_odds_file, args.predict_file, + output_nbest_file, output_null_log_odds_file, model.config.start_n_top, model.config.end_n_top, args.version_2_with_negative, tokenizer, args.verbose_logging) else: @@ -296,6 +305,7 @@ def evaluate(args, model, tokenizer, prefix=""): output_nbest_file, output_null_log_odds_file, args.verbose_logging, args.version_2_with_negative, args.null_score_diff_threshold) + # Compute the F1 and exact scores. results = squad_evaluate(examples, predictions) return results @@ -308,7 +318,10 @@ def load_and_cache_examples(args, tokenizer, evaluate=False, output_examples=Fal cached_features_file = os.path.join(input_dir, 'cached_{}_{}_{}'.format( 'dev' if evaluate else 'train', list(filter(None, args.model_name_or_path.split('/'))).pop(), - str(args.max_seq_length))) + str(args.max_seq_length)) + ) + + # Init features and dataset from cache if it exists if os.path.exists(cached_features_file) and not args.overwrite_cache and not output_examples: logger.info("Loading features from cached file %s", cached_features_file) features_and_dataset = torch.load(cached_features_file) @@ -341,7 +354,6 @@ def load_and_cache_examples(args, tokenizer, evaluate=False, output_examples=Fal return_dataset='pt' ) - if args.local_rank in [-1, 0]: logger.info("Saving features into cached file %s", cached_features_file) torch.save({"features": features, "dataset": dataset}, cached_features_file) @@ -452,6 +464,11 @@ def main(): parser.add_argument('--server_port', type=str, default='', help="Can be used for distant debugging.") args = parser.parse_args() + args.predict_file = os.path.join(args.output_dir, 'predictions_{}_{}.txt'.format( + list(filter(None, args.model_name_or_path.split('/'))).pop(), + str(args.max_seq_length)) + ) + if os.path.exists(args.output_dir) and os.listdir(args.output_dir) and args.do_train and not args.overwrite_output_dir: raise ValueError("Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.".format(args.output_dir)) diff --git a/examples/utils_squad.py b/examples/utils_squad.py deleted file mode 100644 index 4f1c581588..0000000000 --- a/examples/utils_squad.py +++ /dev/null @@ -1,1017 +0,0 @@ - -# coding=utf-8 -# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. -# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" Load SQuAD dataset. """ - -from __future__ import absolute_import, division, print_function - -import json -import logging -import math -import collections -from io import open -from tqdm import tqdm - -from transformers.tokenization_bert import BasicTokenizer, whitespace_tokenize - -# Required by XLNet evaluation method to compute optimal threshold (see write_predictions_extended() method) -from utils_squad_evaluate import find_all_best_thresh_v2, make_qid_to_has_ans, get_raw_scores - -logger = logging.getLogger(__name__) - - -class SquadExample(object): - """ - A single training/test example for the Squad dataset. - For examples without an answer, the start and end position are -1. - """ - - def __init__(self, - qas_id, - question_text, - doc_tokens, - orig_answer_text=None, - start_position=None, - end_position=None, - is_impossible=None): - self.qas_id = qas_id - self.question_text = question_text - self.doc_tokens = doc_tokens - self.orig_answer_text = orig_answer_text - self.start_position = start_position - self.end_position = end_position - self.is_impossible = is_impossible - - def __str__(self): - return self.__repr__() - - def __repr__(self): - s = "" - s += "qas_id: %s" % (self.qas_id) - s += ", question_text: %s" % ( - self.question_text) - s += ", doc_tokens: [%s]" % (" ".join(self.doc_tokens)) - if self.start_position: - s += ", start_position: %d" % (self.start_position) - if self.end_position: - s += ", end_position: %d" % (self.end_position) - if self.is_impossible: - s += ", is_impossible: %r" % (self.is_impossible) - return s - - -class InputFeatures(object): - """A single set of features of data.""" - - def __init__(self, - unique_id, - example_index, - doc_span_index, - tokens, - token_to_orig_map, - token_is_max_context, - input_ids, - input_mask, - segment_ids, - cls_index, - p_mask, - paragraph_len, - start_position=None, - end_position=None, - is_impossible=None): - self.unique_id = unique_id - self.example_index = example_index - self.doc_span_index = doc_span_index - self.tokens = tokens - self.token_to_orig_map = token_to_orig_map - self.token_is_max_context = token_is_max_context - self.input_ids = input_ids - self.input_mask = input_mask - self.segment_ids = segment_ids - self.cls_index = cls_index - self.p_mask = p_mask - self.paragraph_len = paragraph_len - self.start_position = start_position - self.end_position = end_position - self.is_impossible = is_impossible - - -def read_squad_examples(input_file, is_training, version_2_with_negative): - """Read a SQuAD json file into a list of SquadExample.""" - with open(input_file, "r", encoding='utf-8') as reader: - input_data = json.load(reader)["data"] - - def is_whitespace(c): - if c == " " or c == "\t" or c == "\r" or c == "\n" or ord(c) == 0x202F: - return True - return False - - examples = [] - for entry in input_data: - for paragraph in entry["paragraphs"]: - paragraph_text = paragraph["context"] - doc_tokens = [] - char_to_word_offset = [] - prev_is_whitespace = True - for c in paragraph_text: - if is_whitespace(c): - prev_is_whitespace = True - else: - if prev_is_whitespace: - doc_tokens.append(c) - else: - doc_tokens[-1] += c - prev_is_whitespace = False - char_to_word_offset.append(len(doc_tokens) - 1) - - for qa in paragraph["qas"]: - qas_id = qa["id"] - question_text = qa["question"] - start_position = None - end_position = None - orig_answer_text = None - is_impossible = False - if is_training: - if version_2_with_negative: - is_impossible = qa["is_impossible"] - if (len(qa["answers"]) != 1) and (not is_impossible): - raise ValueError( - "For training, each question should have exactly 1 answer.") - if not is_impossible: - answer = qa["answers"][0] - orig_answer_text = answer["text"] - answer_offset = answer["answer_start"] - answer_length = len(orig_answer_text) - start_position = char_to_word_offset[answer_offset] - end_position = char_to_word_offset[answer_offset + answer_length - 1] - # Only add answers where the text can be exactly recovered from the - # document. If this CAN'T happen it's likely due to weird Unicode - # stuff so we will just skip the example. - # - # Note that this means for training mode, every example is NOT - # guaranteed to be preserved. - actual_text = " ".join(doc_tokens[start_position:(end_position + 1)]) - cleaned_answer_text = " ".join( - whitespace_tokenize(orig_answer_text)) - if actual_text.find(cleaned_answer_text) == -1: - logger.warning("Could not find answer: '%s' vs. '%s'", - actual_text, cleaned_answer_text) - continue - else: - start_position = -1 - end_position = -1 - orig_answer_text = "" - - example = SquadExample( - qas_id=qas_id, - question_text=question_text, - doc_tokens=doc_tokens, - orig_answer_text=orig_answer_text, - start_position=start_position, - end_position=end_position, - is_impossible=is_impossible) - examples.append(example) - return examples - - -def convert_examples_to_features(examples, tokenizer, max_seq_length, - doc_stride, max_query_length, is_training, - cls_token_at_end=False, - cls_token='[CLS]', sep_token='[SEP]', pad_token=0, - sequence_a_segment_id=0, sequence_b_segment_id=1, - cls_token_segment_id=0, pad_token_segment_id=0, - mask_padding_with_zero=True, - sequence_a_is_doc=False): - """Loads a data file into a list of `InputBatch`s.""" - - unique_id = 1000000000 - # cnt_pos, cnt_neg = 0, 0 - # max_N, max_M = 1024, 1024 - # f = np.zeros((max_N, max_M), dtype=np.float32) - - features = [] - for (example_index, example) in enumerate(tqdm(examples)): - - # if example_index % 100 == 0: - # logger.info('Converting %s/%s pos %s neg %s', example_index, len(examples), cnt_pos, cnt_neg) - - query_tokens = tokenizer.tokenize(example.question_text) - - if len(query_tokens) > max_query_length: - query_tokens = query_tokens[0:max_query_length] - - tok_to_orig_index = [] - orig_to_tok_index = [] - all_doc_tokens = [] - for (i, token) in enumerate(example.doc_tokens): - orig_to_tok_index.append(len(all_doc_tokens)) - sub_tokens = tokenizer.tokenize(token) - for sub_token in sub_tokens: - tok_to_orig_index.append(i) - all_doc_tokens.append(sub_token) - - tok_start_position = None - tok_end_position = None - if is_training and example.is_impossible: - tok_start_position = -1 - tok_end_position = -1 - if is_training and not example.is_impossible: - tok_start_position = orig_to_tok_index[example.start_position] - if example.end_position < len(example.doc_tokens) - 1: - tok_end_position = orig_to_tok_index[example.end_position + 1] - 1 - else: - tok_end_position = len(all_doc_tokens) - 1 - (tok_start_position, tok_end_position) = _improve_answer_span( - all_doc_tokens, tok_start_position, tok_end_position, tokenizer, - example.orig_answer_text) - - # The -3 accounts for [CLS], [SEP] and [SEP] - max_tokens_for_doc = max_seq_length - len(query_tokens) - 3 - assert max_tokens_for_doc > 0 - - # We can have documents that are longer than the maximum sequence length. - # To deal with this we do a sliding window approach, where we take chunks - # of the up to our max length with a stride of `doc_stride`. - _DocSpan = collections.namedtuple( # pylint: disable=invalid-name - "DocSpan", ["start", "length"]) - doc_spans = [] - start_offset = 0 - while start_offset < len(all_doc_tokens): - length = len(all_doc_tokens) - start_offset - if length > max_tokens_for_doc: - length = max_tokens_for_doc - doc_spans.append(_DocSpan(start=start_offset, length=length)) - if start_offset + length == len(all_doc_tokens): - break - start_offset += min(length, doc_stride) - - for (doc_span_index, doc_span) in enumerate(doc_spans): - tokens = [] - token_to_orig_map = {} - token_is_max_context = {} - segment_ids = [] - - # p_mask: mask with 1 for token than cannot be in the answer (0 for token which can be in an answer) - # Original TF implem also keep the classification token (set to 0) (not sure why...) - p_mask = [] - - # CLS token at the beginning - if not cls_token_at_end: - tokens.append(cls_token) - segment_ids.append(cls_token_segment_id) - p_mask.append(0) - cls_index = 0 - - # XLNet: P SEP Q SEP CLS - # Others: CLS Q SEP P SEP - if not sequence_a_is_doc: - # Query - tokens += query_tokens - segment_ids += [sequence_a_segment_id] * len(query_tokens) - p_mask += [1] * len(query_tokens) - - # SEP token - tokens.append(sep_token) - segment_ids.append(sequence_a_segment_id) - p_mask.append(1) - - # Paragraph - for i in range(doc_span.length): - split_token_index = doc_span.start + i - token_to_orig_map[len(tokens)] = tok_to_orig_index[split_token_index] - - is_max_context = _check_is_max_context(doc_spans, doc_span_index, - split_token_index) - token_is_max_context[len(tokens)] = is_max_context - tokens.append(all_doc_tokens[split_token_index]) - if not sequence_a_is_doc: - segment_ids.append(sequence_b_segment_id) - else: - segment_ids.append(sequence_a_segment_id) - p_mask.append(0) - paragraph_len = doc_span.length - - if sequence_a_is_doc: - # SEP token - tokens.append(sep_token) - segment_ids.append(sequence_a_segment_id) - p_mask.append(1) - - tokens += query_tokens - segment_ids += [sequence_b_segment_id] * len(query_tokens) - p_mask += [1] * len(query_tokens) - - # SEP token - tokens.append(sep_token) - segment_ids.append(sequence_b_segment_id) - p_mask.append(1) - - # CLS token at the end - if cls_token_at_end: - tokens.append(cls_token) - segment_ids.append(cls_token_segment_id) - p_mask.append(0) - cls_index = len(tokens) - 1 # Index of classification token - - input_ids = tokenizer.convert_tokens_to_ids(tokens) - - # The mask has 1 for real tokens and 0 for padding tokens. Only real - # tokens are attended to. - input_mask = [1 if mask_padding_with_zero else 0] * len(input_ids) - - # Zero-pad up to the sequence length. - while len(input_ids) < max_seq_length: - input_ids.append(pad_token) - input_mask.append(0 if mask_padding_with_zero else 1) - segment_ids.append(pad_token_segment_id) - p_mask.append(1) - - assert len(input_ids) == max_seq_length - assert len(input_mask) == max_seq_length - assert len(segment_ids) == max_seq_length - - span_is_impossible = example.is_impossible - start_position = None - end_position = None - if is_training and not span_is_impossible: - # For training, if our document chunk does not contain an annotation - # we throw it out, since there is nothing to predict. - doc_start = doc_span.start - doc_end = doc_span.start + doc_span.length - 1 - out_of_span = False - if not (tok_start_position >= doc_start and - tok_end_position <= doc_end): - out_of_span = True - if out_of_span: - start_position = 0 - end_position = 0 - span_is_impossible = True - else: - if sequence_a_is_doc: - doc_offset = 0 - else: - doc_offset = len(query_tokens) + 2 - start_position = tok_start_position - doc_start + doc_offset - end_position = tok_end_position - doc_start + doc_offset - - if is_training and span_is_impossible: - start_position = cls_index - end_position = cls_index - - if example_index < 20: - logger.info("*** Example ***") - logger.info("unique_id: %s" % (unique_id)) - logger.info("example_index: %s" % (example_index)) - logger.info("doc_span_index: %s" % (doc_span_index)) - logger.info("tokens: %s" % " ".join(tokens)) - logger.info("token_to_orig_map: %s" % " ".join([ - "%d:%d" % (x, y) for (x, y) in token_to_orig_map.items()])) - logger.info("token_is_max_context: %s" % " ".join([ - "%d:%s" % (x, y) for (x, y) in token_is_max_context.items() - ])) - logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids])) - logger.info( - "input_mask: %s" % " ".join([str(x) for x in input_mask])) - logger.info( - "segment_ids: %s" % " ".join([str(x) for x in segment_ids])) - if is_training and span_is_impossible: - logger.info("impossible example") - if is_training and not span_is_impossible: - answer_text = " ".join(tokens[start_position:(end_position + 1)]) - logger.info("start_position: %d" % (start_position)) - logger.info("end_position: %d" % (end_position)) - logger.info( - "answer: %s" % (answer_text)) - - features.append( - InputFeatures( - unique_id=unique_id, - example_index=example_index, - doc_span_index=doc_span_index, - tokens=tokens, - token_to_orig_map=token_to_orig_map, - token_is_max_context=token_is_max_context, - input_ids=input_ids, - input_mask=input_mask, - segment_ids=segment_ids, - cls_index=cls_index, - p_mask=p_mask, - paragraph_len=paragraph_len, - start_position=start_position, - end_position=end_position, - is_impossible=span_is_impossible)) - unique_id += 1 - - return features - - -def _improve_answer_span(doc_tokens, input_start, input_end, tokenizer, - orig_answer_text): - """Returns tokenized answer spans that better match the annotated answer.""" - - # The SQuAD annotations are character based. We first project them to - # whitespace-tokenized words. But then after WordPiece tokenization, we can - # often find a "better match". For example: - # - # Question: What year was John Smith born? - # Context: The leader was John Smith (1895-1943). - # Answer: 1895 - # - # The original whitespace-tokenized answer will be "(1895-1943).". However - # after tokenization, our tokens will be "( 1895 - 1943 ) .". So we can match - # the exact answer, 1895. - # - # However, this is not always possible. Consider the following: - # - # Question: What country is the top exporter of electornics? - # Context: The Japanese electronics industry is the lagest in the world. - # Answer: Japan - # - # In this case, the annotator chose "Japan" as a character sub-span of - # the word "Japanese". Since our WordPiece tokenizer does not split - # "Japanese", we just use "Japanese" as the annotation. This is fairly rare - # in SQuAD, but does happen. - tok_answer_text = " ".join(tokenizer.tokenize(orig_answer_text)) - - for new_start in range(input_start, input_end + 1): - for new_end in range(input_end, new_start - 1, -1): - text_span = " ".join(doc_tokens[new_start:(new_end + 1)]) - if text_span == tok_answer_text: - return (new_start, new_end) - - return (input_start, input_end) - - -def _check_is_max_context(doc_spans, cur_span_index, position): - """Check if this is the 'max context' doc span for the token.""" - - # Because of the sliding window approach taken to scoring documents, a single - # token can appear in multiple documents. E.g. - # Doc: the man went to the store and bought a gallon of milk - # Span A: the man went to the - # Span B: to the store and bought - # Span C: and bought a gallon of - # ... - # - # Now the word 'bought' will have two scores from spans B and C. We only - # want to consider the score with "maximum context", which we define as - # the *minimum* of its left and right context (the *sum* of left and - # right context will always be the same, of course). - # - # In the example the maximum context for 'bought' would be span C since - # it has 1 left context and 3 right context, while span B has 4 left context - # and 0 right context. - best_score = None - best_span_index = None - for (span_index, doc_span) in enumerate(doc_spans): - end = doc_span.start + doc_span.length - 1 - if position < doc_span.start: - continue - if position > end: - continue - num_left_context = position - doc_span.start - num_right_context = end - position - score = min(num_left_context, num_right_context) + 0.01 * doc_span.length - if best_score is None or score > best_score: - best_score = score - best_span_index = span_index - - return cur_span_index == best_span_index - - -RawResult = collections.namedtuple("RawResult", - ["unique_id", "start_logits", "end_logits"]) - -def write_predictions(all_examples, all_features, all_results, n_best_size, - max_answer_length, do_lower_case, output_prediction_file, - output_nbest_file, output_null_log_odds_file, verbose_logging, - version_2_with_negative, null_score_diff_threshold): - """Write final predictions to the json file and log-odds of null if needed.""" - logger.info("Writing predictions to: %s" % (output_prediction_file)) - logger.info("Writing nbest to: %s" % (output_nbest_file)) - - example_index_to_features = collections.defaultdict(list) - for feature in all_features: - example_index_to_features[feature.example_index].append(feature) - - unique_id_to_result = {} - for result in all_results: - unique_id_to_result[result.unique_id] = result - - _PrelimPrediction = collections.namedtuple( # pylint: disable=invalid-name - "PrelimPrediction", - ["feature_index", "start_index", "end_index", "start_logit", "end_logit"]) - - all_predictions = collections.OrderedDict() - all_nbest_json = collections.OrderedDict() - scores_diff_json = collections.OrderedDict() - - for (example_index, example) in enumerate(all_examples): - features = example_index_to_features[example_index] - - prelim_predictions = [] - # keep track of the minimum score of null start+end of position 0 - score_null = 1000000 # large and positive - min_null_feature_index = 0 # the paragraph slice with min null score - null_start_logit = 0 # the start logit at the slice with min null score - null_end_logit = 0 # the end logit at the slice with min null score - for (feature_index, feature) in enumerate(features): - result = unique_id_to_result[feature.unique_id] - start_indexes = _get_best_indexes(result.start_logits, n_best_size) - end_indexes = _get_best_indexes(result.end_logits, n_best_size) - # if we could have irrelevant answers, get the min score of irrelevant - if version_2_with_negative: - feature_null_score = result.start_logits[0] + result.end_logits[0] - if feature_null_score < score_null: - score_null = feature_null_score - min_null_feature_index = feature_index - null_start_logit = result.start_logits[0] - null_end_logit = result.end_logits[0] - for start_index in start_indexes: - for end_index in end_indexes: - # We could hypothetically create invalid predictions, e.g., predict - # that the start of the span is in the question. We throw out all - # invalid predictions. - if start_index >= len(feature.tokens): - continue - if end_index >= len(feature.tokens): - continue - if start_index not in feature.token_to_orig_map: - continue - if end_index not in feature.token_to_orig_map: - continue - if not feature.token_is_max_context.get(start_index, False): - continue - if end_index < start_index: - continue - length = end_index - start_index + 1 - if length > max_answer_length: - continue - prelim_predictions.append( - _PrelimPrediction( - feature_index=feature_index, - start_index=start_index, - end_index=end_index, - start_logit=result.start_logits[start_index], - end_logit=result.end_logits[end_index])) - if version_2_with_negative: - prelim_predictions.append( - _PrelimPrediction( - feature_index=min_null_feature_index, - start_index=0, - end_index=0, - start_logit=null_start_logit, - end_logit=null_end_logit)) - prelim_predictions = sorted( - prelim_predictions, - key=lambda x: (x.start_logit + x.end_logit), - reverse=True) - - _NbestPrediction = collections.namedtuple( # pylint: disable=invalid-name - "NbestPrediction", ["text", "start_logit", "end_logit"]) - - seen_predictions = {} - nbest = [] - for pred in prelim_predictions: - if len(nbest) >= n_best_size: - break - feature = features[pred.feature_index] - if pred.start_index > 0: # this is a non-null prediction - tok_tokens = feature.tokens[pred.start_index:(pred.end_index + 1)] - orig_doc_start = feature.token_to_orig_map[pred.start_index] - orig_doc_end = feature.token_to_orig_map[pred.end_index] - orig_tokens = example.doc_tokens[orig_doc_start:(orig_doc_end + 1)] - tok_text = " ".join(tok_tokens) - - # De-tokenize WordPieces that have been split off. - tok_text = tok_text.replace(" ##", "") - tok_text = tok_text.replace("##", "") - - # Clean whitespace - tok_text = tok_text.strip() - tok_text = " ".join(tok_text.split()) - orig_text = " ".join(orig_tokens) - - final_text = get_final_text(tok_text, orig_text, do_lower_case, verbose_logging) - if final_text in seen_predictions: - continue - - seen_predictions[final_text] = True - else: - final_text = "" - seen_predictions[final_text] = True - - nbest.append( - _NbestPrediction( - text=final_text, - start_logit=pred.start_logit, - end_logit=pred.end_logit)) - # if we didn't include the empty option in the n-best, include it - if version_2_with_negative: - if "" not in seen_predictions: - nbest.append( - _NbestPrediction( - text="", - start_logit=null_start_logit, - end_logit=null_end_logit)) - - # In very rare edge cases we could only have single null prediction. - # So we just create a nonce prediction in this case to avoid failure. - if len(nbest)==1: - nbest.insert(0, - _NbestPrediction(text="empty", start_logit=0.0, end_logit=0.0)) - - # In very rare edge cases we could have no valid predictions. So we - # just create a nonce prediction in this case to avoid failure. - if not nbest: - nbest.append( - _NbestPrediction(text="empty", start_logit=0.0, end_logit=0.0)) - - assert len(nbest) >= 1 - - total_scores = [] - best_non_null_entry = None - for entry in nbest: - total_scores.append(entry.start_logit + entry.end_logit) - if not best_non_null_entry: - if entry.text: - best_non_null_entry = entry - - probs = _compute_softmax(total_scores) - - nbest_json = [] - for (i, entry) in enumerate(nbest): - output = collections.OrderedDict() - output["text"] = entry.text - output["probability"] = probs[i] - output["start_logit"] = entry.start_logit - output["end_logit"] = entry.end_logit - nbest_json.append(output) - - assert len(nbest_json) >= 1 - - if not version_2_with_negative: - all_predictions[example.qas_id] = nbest_json[0]["text"] - else: - # predict "" iff the null score - the score of best non-null > threshold - score_diff = score_null - best_non_null_entry.start_logit - ( - best_non_null_entry.end_logit) - scores_diff_json[example.qas_id] = score_diff - if score_diff > null_score_diff_threshold: - all_predictions[example.qas_id] = "" - else: - all_predictions[example.qas_id] = best_non_null_entry.text - all_nbest_json[example.qas_id] = nbest_json - - with open(output_prediction_file, "w") as writer: - writer.write(json.dumps(all_predictions, indent=4) + "\n") - - with open(output_nbest_file, "w") as writer: - writer.write(json.dumps(all_nbest_json, indent=4) + "\n") - - if version_2_with_negative: - with open(output_null_log_odds_file, "w") as writer: - writer.write(json.dumps(scores_diff_json, indent=4) + "\n") - - return all_predictions - - -# For XLNet (and XLM which uses the same head) -RawResultExtended = collections.namedtuple("RawResultExtended", - ["unique_id", "start_top_log_probs", "start_top_index", - "end_top_log_probs", "end_top_index", "cls_logits"]) - - -def write_predictions_extended(all_examples, all_features, all_results, n_best_size, - max_answer_length, output_prediction_file, - output_nbest_file, - output_null_log_odds_file, orig_data_file, - start_n_top, end_n_top, version_2_with_negative, - tokenizer, verbose_logging): - """ XLNet write prediction logic (more complex than Bert's). - Write final predictions to the json file and log-odds of null if needed. - - Requires utils_squad_evaluate.py - """ - _PrelimPrediction = collections.namedtuple( # pylint: disable=invalid-name - "PrelimPrediction", - ["feature_index", "start_index", "end_index", - "start_log_prob", "end_log_prob"]) - - _NbestPrediction = collections.namedtuple( # pylint: disable=invalid-name - "NbestPrediction", ["text", "start_log_prob", "end_log_prob"]) - - logger.info("Writing predictions to: %s", output_prediction_file) - # logger.info("Writing nbest to: %s" % (output_nbest_file)) - - example_index_to_features = collections.defaultdict(list) - for feature in all_features: - example_index_to_features[feature.example_index].append(feature) - - unique_id_to_result = {} - for result in all_results: - unique_id_to_result[result.unique_id] = result - - all_predictions = collections.OrderedDict() - all_nbest_json = collections.OrderedDict() - scores_diff_json = collections.OrderedDict() - - for (example_index, example) in enumerate(all_examples): - features = example_index_to_features[example_index] - - prelim_predictions = [] - # keep track of the minimum score of null start+end of position 0 - score_null = 1000000 # large and positive - - for (feature_index, feature) in enumerate(features): - result = unique_id_to_result[feature.unique_id] - - cur_null_score = result.cls_logits - - # if we could have irrelevant answers, get the min score of irrelevant - score_null = min(score_null, cur_null_score) - - for i in range(start_n_top): - for j in range(end_n_top): - start_log_prob = result.start_top_log_probs[i] - start_index = result.start_top_index[i] - - j_index = i * end_n_top + j - - end_log_prob = result.end_top_log_probs[j_index] - end_index = result.end_top_index[j_index] - - # We could hypothetically create invalid predictions, e.g., predict - # that the start of the span is in the question. We throw out all - # invalid predictions. - if start_index >= feature.paragraph_len - 1: - continue - if end_index >= feature.paragraph_len - 1: - continue - - if not feature.token_is_max_context.get(start_index, False): - continue - if end_index < start_index: - continue - length = end_index - start_index + 1 - if length > max_answer_length: - continue - - prelim_predictions.append( - _PrelimPrediction( - feature_index=feature_index, - start_index=start_index, - end_index=end_index, - start_log_prob=start_log_prob, - end_log_prob=end_log_prob)) - - prelim_predictions = sorted( - prelim_predictions, - key=lambda x: (x.start_log_prob + x.end_log_prob), - reverse=True) - - seen_predictions = {} - nbest = [] - for pred in prelim_predictions: - if len(nbest) >= n_best_size: - break - feature = features[pred.feature_index] - - # XLNet un-tokenizer - # Let's keep it simple for now and see if we need all this later. - # - # tok_start_to_orig_index = feature.tok_start_to_orig_index - # tok_end_to_orig_index = feature.tok_end_to_orig_index - # start_orig_pos = tok_start_to_orig_index[pred.start_index] - # end_orig_pos = tok_end_to_orig_index[pred.end_index] - # paragraph_text = example.paragraph_text - # final_text = paragraph_text[start_orig_pos: end_orig_pos + 1].strip() - - # Previously used Bert untokenizer - tok_tokens = feature.tokens[pred.start_index:(pred.end_index + 1)] - orig_doc_start = feature.token_to_orig_map[pred.start_index] - orig_doc_end = feature.token_to_orig_map[pred.end_index] - orig_tokens = example.doc_tokens[orig_doc_start:(orig_doc_end + 1)] - tok_text = tokenizer.convert_tokens_to_string(tok_tokens) - - # Clean whitespace - tok_text = tok_text.strip() - tok_text = " ".join(tok_text.split()) - orig_text = " ".join(orig_tokens) - - final_text = get_final_text(tok_text, orig_text, tokenizer.do_lower_case, - verbose_logging) - - if final_text in seen_predictions: - continue - - seen_predictions[final_text] = True - - nbest.append( - _NbestPrediction( - text=final_text, - start_log_prob=pred.start_log_prob, - end_log_prob=pred.end_log_prob)) - - # In very rare edge cases we could have no valid predictions. So we - # just create a nonce prediction in this case to avoid failure. - if not nbest: - nbest.append( - _NbestPrediction(text="", start_log_prob=-1e6, - end_log_prob=-1e6)) - - total_scores = [] - best_non_null_entry = None - for entry in nbest: - total_scores.append(entry.start_log_prob + entry.end_log_prob) - if not best_non_null_entry: - best_non_null_entry = entry - - probs = _compute_softmax(total_scores) - - nbest_json = [] - for (i, entry) in enumerate(nbest): - output = collections.OrderedDict() - output["text"] = entry.text - output["probability"] = probs[i] - output["start_log_prob"] = entry.start_log_prob - output["end_log_prob"] = entry.end_log_prob - nbest_json.append(output) - - assert len(nbest_json) >= 1 - assert best_non_null_entry is not None - - score_diff = score_null - scores_diff_json[example.qas_id] = score_diff - # note(zhiliny): always predict best_non_null_entry - # and the evaluation script will search for the best threshold - all_predictions[example.qas_id] = best_non_null_entry.text - - all_nbest_json[example.qas_id] = nbest_json - - with open(output_prediction_file, "w") as writer: - writer.write(json.dumps(all_predictions, indent=4) + "\n") - - with open(output_nbest_file, "w") as writer: - writer.write(json.dumps(all_nbest_json, indent=4) + "\n") - - if version_2_with_negative: - with open(output_null_log_odds_file, "w") as writer: - writer.write(json.dumps(scores_diff_json, indent=4) + "\n") - - with open(orig_data_file, "r", encoding='utf-8') as reader: - orig_data = json.load(reader)["data"] - - qid_to_has_ans = make_qid_to_has_ans(orig_data) - has_ans_qids = [k for k, v in qid_to_has_ans.items() if v] - no_ans_qids = [k for k, v in qid_to_has_ans.items() if not v] - exact_raw, f1_raw = get_raw_scores(orig_data, all_predictions) - out_eval = {} - - find_all_best_thresh_v2(out_eval, all_predictions, exact_raw, f1_raw, scores_diff_json, qid_to_has_ans) - - return out_eval - - -def get_final_text(pred_text, orig_text, do_lower_case, verbose_logging=False): - """Project the tokenized prediction back to the original text.""" - - # When we created the data, we kept track of the alignment between original - # (whitespace tokenized) tokens and our WordPiece tokenized tokens. So - # now `orig_text` contains the span of our original text corresponding to the - # span that we predicted. - # - # However, `orig_text` may contain extra characters that we don't want in - # our prediction. - # - # For example, let's say: - # pred_text = steve smith - # orig_text = Steve Smith's - # - # We don't want to return `orig_text` because it contains the extra "'s". - # - # We don't want to return `pred_text` because it's already been normalized - # (the SQuAD eval script also does punctuation stripping/lower casing but - # our tokenizer does additional normalization like stripping accent - # characters). - # - # What we really want to return is "Steve Smith". - # - # Therefore, we have to apply a semi-complicated alignment heuristic between - # `pred_text` and `orig_text` to get a character-to-character alignment. This - # can fail in certain cases in which case we just return `orig_text`. - - def _strip_spaces(text): - ns_chars = [] - ns_to_s_map = collections.OrderedDict() - for (i, c) in enumerate(text): - if c == " ": - continue - ns_to_s_map[len(ns_chars)] = i - ns_chars.append(c) - ns_text = "".join(ns_chars) - return (ns_text, ns_to_s_map) - - # We first tokenize `orig_text`, strip whitespace from the result - # and `pred_text`, and check if they are the same length. If they are - # NOT the same length, the heuristic has failed. If they are the same - # length, we assume the characters are one-to-one aligned. - tokenizer = BasicTokenizer(do_lower_case=do_lower_case) - - tok_text = " ".join(tokenizer.tokenize(orig_text)) - - start_position = tok_text.find(pred_text) - if start_position == -1: - if verbose_logging: - logger.info( - "Unable to find text: '%s' in '%s'" % (pred_text, orig_text)) - return orig_text - end_position = start_position + len(pred_text) - 1 - - (orig_ns_text, orig_ns_to_s_map) = _strip_spaces(orig_text) - (tok_ns_text, tok_ns_to_s_map) = _strip_spaces(tok_text) - - if len(orig_ns_text) != len(tok_ns_text): - if verbose_logging: - logger.info("Length not equal after stripping spaces: '%s' vs '%s'", - orig_ns_text, tok_ns_text) - return orig_text - - # We then project the characters in `pred_text` back to `orig_text` using - # the character-to-character alignment. - tok_s_to_ns_map = {} - for (i, tok_index) in tok_ns_to_s_map.items(): - tok_s_to_ns_map[tok_index] = i - - orig_start_position = None - if start_position in tok_s_to_ns_map: - ns_start_position = tok_s_to_ns_map[start_position] - if ns_start_position in orig_ns_to_s_map: - orig_start_position = orig_ns_to_s_map[ns_start_position] - - if orig_start_position is None: - if verbose_logging: - logger.info("Couldn't map start position") - return orig_text - - orig_end_position = None - if end_position in tok_s_to_ns_map: - ns_end_position = tok_s_to_ns_map[end_position] - if ns_end_position in orig_ns_to_s_map: - orig_end_position = orig_ns_to_s_map[ns_end_position] - - if orig_end_position is None: - if verbose_logging: - logger.info("Couldn't map end position") - return orig_text - - output_text = orig_text[orig_start_position:(orig_end_position + 1)] - return output_text - - -def _get_best_indexes(logits, n_best_size): - """Get the n-best logits from a list.""" - index_and_score = sorted(enumerate(logits), key=lambda x: x[1], reverse=True) - - best_indexes = [] - for i in range(len(index_and_score)): - if i >= n_best_size: - break - best_indexes.append(index_and_score[i][0]) - return best_indexes - - -def _compute_softmax(scores): - """Compute softmax probability over raw logits.""" - if not scores: - return [] - - max_score = None - for score in scores: - if max_score is None or score > max_score: - max_score = score - - exp_scores = [] - total_sum = 0.0 - for score in scores: - x = math.exp(score - max_score) - exp_scores.append(x) - total_sum += x - - probs = [] - for score in exp_scores: - probs.append(score / total_sum) - return probs diff --git a/examples/utils_squad_evaluate.py b/examples/utils_squad_evaluate.py deleted file mode 100644 index ed162e6fe6..0000000000 --- a/examples/utils_squad_evaluate.py +++ /dev/null @@ -1,330 +0,0 @@ -""" Official evaluation script for SQuAD version 2.0. - Modified by XLNet authors to update `find_best_threshold` scripts for SQuAD V2.0 - -In addition to basic functionality, we also compute additional statistics and -plot precision-recall curves if an additional na_prob.json file is provided. -This file is expected to map question ID's to the model's predicted probability -that a question is unanswerable. -""" -import argparse -import collections -import json -import numpy as np -import os -import re -import string -import sys - -class EVAL_OPTS(): - def __init__(self, data_file, pred_file, out_file="", - na_prob_file="na_prob.json", na_prob_thresh=1.0, - out_image_dir=None, verbose=False): - self.data_file = data_file - self.pred_file = pred_file - self.out_file = out_file - self.na_prob_file = na_prob_file - self.na_prob_thresh = na_prob_thresh - self.out_image_dir = out_image_dir - self.verbose = verbose - -OPTS = None - -def parse_args(): - parser = argparse.ArgumentParser('Official evaluation script for SQuAD version 2.0.') - parser.add_argument('data_file', metavar='data.json', help='Input data JSON file.') - parser.add_argument('pred_file', metavar='pred.json', help='Model predictions.') - parser.add_argument('--out-file', '-o', metavar='eval.json', - help='Write accuracy metrics to file (default is stdout).') - parser.add_argument('--na-prob-file', '-n', metavar='na_prob.json', - help='Model estimates of probability of no answer.') - parser.add_argument('--na-prob-thresh', '-t', type=float, default=1.0, - help='Predict "" if no-answer probability exceeds this (default = 1.0).') - parser.add_argument('--out-image-dir', '-p', metavar='out_images', default=None, - help='Save precision-recall curves to directory.') - parser.add_argument('--verbose', '-v', action='store_true') - if len(sys.argv) == 1: - parser.print_help() - sys.exit(1) - return parser.parse_args() - -def make_qid_to_has_ans(dataset): - qid_to_has_ans = {} - for article in dataset: - for p in article['paragraphs']: - for qa in p['qas']: - qid_to_has_ans[qa['id']] = bool(qa['answers']) - return qid_to_has_ans - -def normalize_answer(s): - """Lower text and remove punctuation, articles and extra whitespace.""" - def remove_articles(text): - regex = re.compile(r'\b(a|an|the)\b', re.UNICODE) - return re.sub(regex, ' ', text) - def white_space_fix(text): - return ' '.join(text.split()) - def remove_punc(text): - exclude = set(string.punctuation) - return ''.join(ch for ch in text if ch not in exclude) - def lower(text): - return text.lower() - return white_space_fix(remove_articles(remove_punc(lower(s)))) - -def get_tokens(s): - if not s: return [] - return normalize_answer(s).split() - -def compute_exact(a_gold, a_pred): - return int(normalize_answer(a_gold) == normalize_answer(a_pred)) - -def compute_f1(a_gold, a_pred): - gold_toks = get_tokens(a_gold) - pred_toks = get_tokens(a_pred) - common = collections.Counter(gold_toks) & collections.Counter(pred_toks) - num_same = sum(common.values()) - if len(gold_toks) == 0 or len(pred_toks) == 0: - # If either is no-answer, then F1 is 1 if they agree, 0 otherwise - return int(gold_toks == pred_toks) - if num_same == 0: - return 0 - precision = 1.0 * num_same / len(pred_toks) - recall = 1.0 * num_same / len(gold_toks) - f1 = (2 * precision * recall) / (precision + recall) - return f1 - -def get_raw_scores(dataset, preds): - exact_scores = {} - f1_scores = {} - for article in dataset: - for p in article['paragraphs']: - for qa in p['qas']: - qid = qa['id'] - gold_answers = [a['text'] for a in qa['answers'] - if normalize_answer(a['text'])] - if not gold_answers: - # For unanswerable questions, only correct answer is empty string - gold_answers = [''] - if qid not in preds: - print('Missing prediction for %s' % qid) - continue - a_pred = preds[qid] - # Take max over all gold answers - exact_scores[qid] = max(compute_exact(a, a_pred) for a in gold_answers) - f1_scores[qid] = max(compute_f1(a, a_pred) for a in gold_answers) - return exact_scores, f1_scores - -def apply_no_ans_threshold(scores, na_probs, qid_to_has_ans, na_prob_thresh): - new_scores = {} - for qid, s in scores.items(): - pred_na = na_probs[qid] > na_prob_thresh - if pred_na: - new_scores[qid] = float(not qid_to_has_ans[qid]) - else: - new_scores[qid] = s - return new_scores - -def make_eval_dict(exact_scores, f1_scores, qid_list=None): - if not qid_list: - total = len(exact_scores) - return collections.OrderedDict([ - ('exact', 100.0 * sum(exact_scores.values()) / total), - ('f1', 100.0 * sum(f1_scores.values()) / total), - ('total', total), - ]) - else: - total = len(qid_list) - return collections.OrderedDict([ - ('exact', 100.0 * sum(exact_scores[k] for k in qid_list) / total), - ('f1', 100.0 * sum(f1_scores[k] for k in qid_list) / total), - ('total', total), - ]) - -def merge_eval(main_eval, new_eval, prefix): - for k in new_eval: - main_eval['%s_%s' % (prefix, k)] = new_eval[k] - -def plot_pr_curve(precisions, recalls, out_image, title): - plt.step(recalls, precisions, color='b', alpha=0.2, where='post') - plt.fill_between(recalls, precisions, step='post', alpha=0.2, color='b') - plt.xlabel('Recall') - plt.ylabel('Precision') - plt.xlim([0.0, 1.05]) - plt.ylim([0.0, 1.05]) - plt.title(title) - plt.savefig(out_image) - plt.clf() - -def make_precision_recall_eval(scores, na_probs, num_true_pos, qid_to_has_ans, - out_image=None, title=None): - qid_list = sorted(na_probs, key=lambda k: na_probs[k]) - true_pos = 0.0 - cur_p = 1.0 - cur_r = 0.0 - precisions = [1.0] - recalls = [0.0] - avg_prec = 0.0 - for i, qid in enumerate(qid_list): - if qid_to_has_ans[qid]: - true_pos += scores[qid] - cur_p = true_pos / float(i+1) - cur_r = true_pos / float(num_true_pos) - if i == len(qid_list) - 1 or na_probs[qid] != na_probs[qid_list[i+1]]: - # i.e., if we can put a threshold after this point - avg_prec += cur_p * (cur_r - recalls[-1]) - precisions.append(cur_p) - recalls.append(cur_r) - if out_image: - plot_pr_curve(precisions, recalls, out_image, title) - return {'ap': 100.0 * avg_prec} - -def run_precision_recall_analysis(main_eval, exact_raw, f1_raw, na_probs, - qid_to_has_ans, out_image_dir): - if out_image_dir and not os.path.exists(out_image_dir): - os.makedirs(out_image_dir) - num_true_pos = sum(1 for v in qid_to_has_ans.values() if v) - if num_true_pos == 0: - return - pr_exact = make_precision_recall_eval( - exact_raw, na_probs, num_true_pos, qid_to_has_ans, - out_image=os.path.join(out_image_dir, 'pr_exact.png'), - title='Precision-Recall curve for Exact Match score') - pr_f1 = make_precision_recall_eval( - f1_raw, na_probs, num_true_pos, qid_to_has_ans, - out_image=os.path.join(out_image_dir, 'pr_f1.png'), - title='Precision-Recall curve for F1 score') - oracle_scores = {k: float(v) for k, v in qid_to_has_ans.items()} - pr_oracle = make_precision_recall_eval( - oracle_scores, na_probs, num_true_pos, qid_to_has_ans, - out_image=os.path.join(out_image_dir, 'pr_oracle.png'), - title='Oracle Precision-Recall curve (binary task of HasAns vs. NoAns)') - merge_eval(main_eval, pr_exact, 'pr_exact') - merge_eval(main_eval, pr_f1, 'pr_f1') - merge_eval(main_eval, pr_oracle, 'pr_oracle') - -def histogram_na_prob(na_probs, qid_list, image_dir, name): - if not qid_list: - return - x = [na_probs[k] for k in qid_list] - weights = np.ones_like(x) / float(len(x)) - plt.hist(x, weights=weights, bins=20, range=(0.0, 1.0)) - plt.xlabel('Model probability of no-answer') - plt.ylabel('Proportion of dataset') - plt.title('Histogram of no-answer probability: %s' % name) - plt.savefig(os.path.join(image_dir, 'na_prob_hist_%s.png' % name)) - plt.clf() - -def find_best_thresh(preds, scores, na_probs, qid_to_has_ans): - num_no_ans = sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]) - cur_score = num_no_ans - best_score = cur_score - best_thresh = 0.0 - qid_list = sorted(na_probs, key=lambda k: na_probs[k]) - for i, qid in enumerate(qid_list): - if qid not in scores: continue - if qid_to_has_ans[qid]: - diff = scores[qid] - else: - if preds[qid]: - diff = -1 - else: - diff = 0 - cur_score += diff - if cur_score > best_score: - best_score = cur_score - best_thresh = na_probs[qid] - return 100.0 * best_score / len(scores), best_thresh - -def find_best_thresh_v2(preds, scores, na_probs, qid_to_has_ans): - num_no_ans = sum(1 for k in qid_to_has_ans if not qid_to_has_ans[k]) - cur_score = num_no_ans - best_score = cur_score - best_thresh = 0.0 - qid_list = sorted(na_probs, key=lambda k: na_probs[k]) - for i, qid in enumerate(qid_list): - if qid not in scores: continue - if qid_to_has_ans[qid]: - diff = scores[qid] - else: - if preds[qid]: - diff = -1 - else: - diff = 0 - cur_score += diff - if cur_score > best_score: - best_score = cur_score - best_thresh = na_probs[qid] - - has_ans_score, has_ans_cnt = 0, 0 - for qid in qid_list: - if not qid_to_has_ans[qid]: continue - has_ans_cnt += 1 - - if qid not in scores: continue - has_ans_score += scores[qid] - - return 100.0 * best_score / len(scores), best_thresh, 1.0 * has_ans_score / has_ans_cnt - -def find_all_best_thresh(main_eval, preds, exact_raw, f1_raw, na_probs, qid_to_has_ans): - best_exact, exact_thresh = find_best_thresh(preds, exact_raw, na_probs, qid_to_has_ans) - best_f1, f1_thresh = find_best_thresh(preds, f1_raw, na_probs, qid_to_has_ans) - main_eval['best_exact'] = best_exact - main_eval['best_exact_thresh'] = exact_thresh - main_eval['best_f1'] = best_f1 - main_eval['best_f1_thresh'] = f1_thresh - -def find_all_best_thresh_v2(main_eval, preds, exact_raw, f1_raw, na_probs, qid_to_has_ans): - best_exact, exact_thresh, has_ans_exact = find_best_thresh_v2(preds, exact_raw, na_probs, qid_to_has_ans) - best_f1, f1_thresh, has_ans_f1 = find_best_thresh_v2(preds, f1_raw, na_probs, qid_to_has_ans) - main_eval['best_exact'] = best_exact - main_eval['best_exact_thresh'] = exact_thresh - main_eval['best_f1'] = best_f1 - main_eval['best_f1_thresh'] = f1_thresh - main_eval['has_ans_exact'] = has_ans_exact - main_eval['has_ans_f1'] = has_ans_f1 - -def main(OPTS): - with open(OPTS.data_file) as f: - dataset_json = json.load(f) - dataset = dataset_json['data'] - with open(OPTS.pred_file) as f: - preds = json.load(f) - if OPTS.na_prob_file: - with open(OPTS.na_prob_file) as f: - na_probs = json.load(f) - else: - na_probs = {k: 0.0 for k in preds} - qid_to_has_ans = make_qid_to_has_ans(dataset) # maps qid to True/False - has_ans_qids = [k for k, v in qid_to_has_ans.items() if v] - no_ans_qids = [k for k, v in qid_to_has_ans.items() if not v] - exact_raw, f1_raw = get_raw_scores(dataset, preds) - exact_thresh = apply_no_ans_threshold(exact_raw, na_probs, qid_to_has_ans, - OPTS.na_prob_thresh) - f1_thresh = apply_no_ans_threshold(f1_raw, na_probs, qid_to_has_ans, - OPTS.na_prob_thresh) - out_eval = make_eval_dict(exact_thresh, f1_thresh) - if has_ans_qids: - has_ans_eval = make_eval_dict(exact_thresh, f1_thresh, qid_list=has_ans_qids) - merge_eval(out_eval, has_ans_eval, 'HasAns') - if no_ans_qids: - no_ans_eval = make_eval_dict(exact_thresh, f1_thresh, qid_list=no_ans_qids) - merge_eval(out_eval, no_ans_eval, 'NoAns') - if OPTS.na_prob_file: - find_all_best_thresh(out_eval, preds, exact_raw, f1_raw, na_probs, qid_to_has_ans) - if OPTS.na_prob_file and OPTS.out_image_dir: - run_precision_recall_analysis(out_eval, exact_raw, f1_raw, na_probs, - qid_to_has_ans, OPTS.out_image_dir) - histogram_na_prob(na_probs, has_ans_qids, OPTS.out_image_dir, 'hasAns') - histogram_na_prob(na_probs, no_ans_qids, OPTS.out_image_dir, 'noAns') - if OPTS.out_file: - with open(OPTS.out_file, 'w') as f: - json.dump(out_eval, f) - else: - print(json.dumps(out_eval, indent=2)) - return out_eval - -if __name__ == '__main__': - OPTS = parse_args() - if OPTS.out_image_dir: - import matplotlib - matplotlib.use('Agg') - import matplotlib.pyplot as plt - main(OPTS) diff --git a/transformers/data/metrics/squad_metrics.py b/transformers/data/metrics/squad_metrics.py index f8449df045..0755c0ab7a 100644 --- a/transformers/data/metrics/squad_metrics.py +++ b/transformers/data/metrics/squad_metrics.py @@ -578,7 +578,6 @@ def compute_predictions_log_probs( output_prediction_file, output_nbest_file, output_null_log_odds_file, - orig_data_file, start_n_top, end_n_top, version_2_with_negative, @@ -756,15 +755,4 @@ def compute_predictions_log_probs( with open(output_null_log_odds_file, "w") as writer: writer.write(json.dumps(scores_diff_json, indent=4) + "\n") - with open(orig_data_file, "r", encoding='utf-8') as reader: - orig_data = json.load(reader)["data"] - - qid_to_has_ans = make_qid_to_has_ans(orig_data) - has_ans_qids = [k for k, v in qid_to_has_ans.items() if v] - no_ans_qids = [k for k, v in qid_to_has_ans.items() if not v] - exact_raw, f1_raw = get_raw_scores(orig_data, all_predictions) - out_eval = {} - - find_all_best_thresh_v2(out_eval, all_predictions, exact_raw, f1_raw, scores_diff_json, qid_to_has_ans) - - return out_eval + return all_predictions diff --git a/transformers/data/processors/squad.py b/transformers/data/processors/squad.py index bb56aa792f..3d7f832540 100644 --- a/transformers/data/processors/squad.py +++ b/transformers/data/processors/squad.py @@ -9,7 +9,7 @@ from ...tokenization_bert import BasicTokenizer, whitespace_tokenize from .utils import DataProcessor, InputExample, InputFeatures from ...file_utils import is_tf_available, is_torch_available -if is_torch_available: +if is_torch_available(): import torch from torch.utils.data import TensorDataset From d0383e4daf44557e56cd4cbc5dc95b1d35457768 Mon Sep 17 00:00:00 2001 From: patrickvonplaten Date: Fri, 6 Dec 2019 01:24:22 +0100 Subject: [PATCH 244/505] corrected documentation for past tensor shape for ctrl and gpt2 model --- transformers/modeling_ctrl.py | 4 ++-- transformers/modeling_gpt2.py | 6 +++--- transformers/modeling_tf_ctrl.py | 4 ++-- transformers/modeling_tf_gpt2.py | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/transformers/modeling_ctrl.py b/transformers/modeling_ctrl.py index 3a252941ac..97bcb14434 100644 --- a/transformers/modeling_ctrl.py +++ b/transformers/modeling_ctrl.py @@ -252,7 +252,7 @@ class CTRLModel(CTRLPreTrainedModel): **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` Sequence of hidden-states at the last layer of the model. **past**: - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``torch.FloatTensor`` (one for each layer) of shape ``(2, batch_size, num_heads, sequence_length, embed_size_per_head)``: that contains pre-computed hidden-states (key and values in the attention blocks). Can be used (see `past` input) to speed up sequential decoding. The token ids which have their past given to this model should not be passed as input ids as they have already been computed. @@ -438,7 +438,7 @@ class CTRLLMHeadModel(CTRLPreTrainedModel): **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). **past**: - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``torch.FloatTensor`` (one for each layer) of shape ``(2, batch_size, num_heads, sequence_length, embed_size_per_head)``: that contains pre-computed hidden-states (key and values in the attention blocks). Can be used (see `past` input) to speed up sequential decoding. The token ids which have their past given to this model should not be passed as input ids as they have already been computed. diff --git a/transformers/modeling_gpt2.py b/transformers/modeling_gpt2.py index 35bc5c8d6e..96fd1c0607 100644 --- a/transformers/modeling_gpt2.py +++ b/transformers/modeling_gpt2.py @@ -329,7 +329,7 @@ class GPT2Model(GPT2PreTrainedModel): **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` Sequence of hidden-states at the last layer of the model. **past**: - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``torch.FloatTensor`` (one for each layer) of shape ``(2, batch_size, num_heads, sequence_length, embed_size_per_head)``: that contains pre-computed hidden-states (key and values in the attention blocks). Can be used (see `past` input) to speed up sequential decoding. The token ids which have their past given to this model should not be passed as input ids as they have already been computed. @@ -503,7 +503,7 @@ class GPT2LMHeadModel(GPT2PreTrainedModel): **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). **past**: - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``torch.FloatTensor`` (one for each layer) of shape ``(2, batch_size, num_heads, sequence_length, embed_size_per_head)``: that contains pre-computed hidden-states (key and values in the attention blocks). Can be used (see `past` input) to speed up sequential decoding. The token ids which have their past given to this model should not be passed as input ids as they have already been computed. @@ -596,7 +596,7 @@ class GPT2DoubleHeadsModel(GPT2PreTrainedModel): **mc_prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, num_choices)`` Prediction scores of the multiplechoice classification head (scores for each choice before SoftMax). **past**: - list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``torch.FloatTensor`` (one for each layer) of shape ``(2, batch_size, num_heads, sequence_length, embed_size_per_head)``: that contains pre-computed hidden-states (key and values in the attention blocks). Can be used (see `past` input) to speed up sequential decoding. The token ids which have their past given to this model should not be passed as input ids as they have already been computed. diff --git a/transformers/modeling_tf_ctrl.py b/transformers/modeling_tf_ctrl.py index 6d0d6a57ad..29ee5113a4 100644 --- a/transformers/modeling_tf_ctrl.py +++ b/transformers/modeling_tf_ctrl.py @@ -400,7 +400,7 @@ class TFCTRLModel(TFCTRLPreTrainedModel): **last_hidden_state**: ``tf.Tensor`` of shape ``(batch_size, sequence_length, hidden_size)`` Sequence of hidden-states at the last layer of the model. **past**: - list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``tf.Tensor`` (one for each layer) of shape ``(2, batch_size, num_heads, sequence_length, embed_size_per_head)``: that contains pre-computed hidden-states (key and values in the attention blocks). Can be used (see `past` input) to speed up sequential decoding. **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) @@ -462,7 +462,7 @@ class TFCTRLLMHeadModel(TFCTRLPreTrainedModel): **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). **past**: - list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``tf.Tensor`` (one for each layer) of shape ``(2, batch_size, num_heads, sequence_length, embed_size_per_head)``: that contains pre-computed hidden-states (key and values in the attention blocks). Can be used (see `past` input) to speed up sequential decoding. **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) diff --git a/transformers/modeling_tf_gpt2.py b/transformers/modeling_tf_gpt2.py index aebe790114..c738e5e8e3 100644 --- a/transformers/modeling_tf_gpt2.py +++ b/transformers/modeling_tf_gpt2.py @@ -436,7 +436,7 @@ class TFGPT2Model(TFGPT2PreTrainedModel): **last_hidden_state**: ``tf.Tensor`` of shape ``(batch_size, sequence_length, hidden_size)`` Sequence of hidden-states at the last layer of the model. **past**: - list of ``tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of ``tf.Tensor`` (one for each layer) of shape ``(2, batch_size, num_heads, sequence_length, embed_size_per_head)``: that contains pre-computed hidden-states (key and values in the attention blocks). Can be used (see `past` input) to speed up sequential decoding. **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) @@ -476,7 +476,7 @@ class TFGPT2LMHeadModel(TFGPT2PreTrainedModel): **prediction_scores**: `tf.Tensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). **past**: - list of `tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of `tf.Tensor`` (one for each layer) of shape ``(2, batch_size, num_heads, sequence_length, embed_size_per_head)``: that contains pre-computed hidden-states (key and values in the attention blocks). Can be used (see `past` input) to speed up sequential decoding. **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) @@ -535,7 +535,7 @@ class TFGPT2DoubleHeadsModel(TFGPT2PreTrainedModel): **mc_prediction_scores**: `tf.Tensor`` of shape ``(batch_size, num_choices)`` Prediction scores of the multiplechoice classification head (scores for each choice before SoftMax). **past**: - list of `tf.Tensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + list of `tf.Tensor`` (one for each layer) of shape ``(2, batch_size, num_heads, sequence_length, embed_size_per_head)``: that contains pre-computed hidden-states (key and values in the attention blocks). Can be used (see `past` input) to speed up sequential decoding. **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) From f230d91b437c806e3e2dad37318a5ce77d208fa6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Louf?= Date: Thu, 5 Dec 2019 21:24:57 +0100 Subject: [PATCH 245/505] check the validity of links We add a script and a CI workflow to check that all download links present in the source code are valid. --- .circleci/config.yml | 11 ++++++ utils/link_tester.py | 79 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+) create mode 100644 utils/link_tester.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 01e6d82b33..ebfbd79b93 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -82,6 +82,16 @@ jobs: - run: sudo pip install --progress-bar off -r docs/requirements.txt - run: sudo pip install --progress-bar off -r requirements.txt - run: ./.circleci/deploy.sh + repository_consistency: + working_directory: ~/transformers + docker: + - image: circleci/python:3.5 + resource_class: small + parallelism: 1 + steps: + - checkout + - run: sudo pip install requests + - run: python ./utils/link_tester.py workflow_filters: &workflow_filters filters: branches: @@ -91,6 +101,7 @@ workflows: version: 2 build_and_test: jobs: + - repository_consistency - build_py3_torch_and_tf - build_py3_torch - build_py3_tf diff --git a/utils/link_tester.py b/utils/link_tester.py new file mode 100644 index 0000000000..fe3990d28c --- /dev/null +++ b/utils/link_tester.py @@ -0,0 +1,79 @@ +""" Link tester. + +This little utility reads all the python files in the repository, +scans for links pointing to S3 and tests the links one by one. Raises an error +at the end of the scan if at least one link was reported broken. +""" +import os +import re +import sys + +import requests + + +REGEXP_FIND_S3_LINKS = r"""([\"'])(https:\/\/s3)(.*)?\1""" + + +def list_python_files_in_repository(): + """ List all python files in the repository. + + This function assumes that the script is executed in the root folder. + """ + source_code_files = [] + for path, subdirs, files in os.walk("."): + if "templates" in path: + continue + for name in files: + if ".py" in name and ".pyc" not in name: + path_to_files = os.path.join(path, name) + source_code_files.append(path_to_files) + + return source_code_files + + +def find_all_links(file_paths): + links = [] + for path in file_paths: + links += scan_code_for_links(path) + + return links + + +def scan_code_for_links(source): + """ Scans the file to find links using a regular expression. + Returns a list of links. + """ + with open(source, 'r') as content: + content = content.read() + raw_links = re.findall(REGEXP_FIND_S3_LINKS, content) + links = [prefix + suffix for _, prefix, suffix in raw_links] + + return links + + +def check_all_links(links): + """ Check that the provided links are valid. + + Links are considered valid if a HEAD request to the server + returns a 200 status code. + """ + broken_links = [] + for link in links: + head = requests.head(link) + if head.status_code != 200: + broken_links.append(link) + + return broken_links + + +if __name__ == "__main__": + file_paths = list_python_files_in_repository() + links = find_all_links(file_paths) + broken_links = check_all_links(links) + print("Looking for broken links to pre-trained models/configs/tokenizers...") + if broken_links: + print("The following links did not respond:") + for link in broken_links: + print("- {}".format(link)) + sys.exit(1) + print("All links are ok.") From 21451ec6ba364de78c14e7d05a55913da2809844 Mon Sep 17 00:00:00 2001 From: Philipp Glock Date: Fri, 6 Dec 2019 10:32:43 +0100 Subject: [PATCH 246/505] handle string with only whitespaces as empty --- transformers/tokenization_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index 5d683629f0..bc246cc8fe 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -634,7 +634,7 @@ class PreTrainedTokenizer(object): return result def split_on_tokens(tok_list, text): - if not text: + if not text.strip(): return [] if not tok_list: return self._tokenize(text, **kwargs) From 1d87b37d100c69ff3b2c1a5dfd271b6cf777176e Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 6 Dec 2019 15:30:09 +0100 Subject: [PATCH 247/505] updating --- .../convert_pytorch_checkpoint_to_tf2.py | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/transformers/convert_pytorch_checkpoint_to_tf2.py b/transformers/convert_pytorch_checkpoint_to_tf2.py index d20eafe2e9..2c419888e8 100644 --- a/transformers/convert_pytorch_checkpoint_to_tf2.py +++ b/transformers/convert_pytorch_checkpoint_to_tf2.py @@ -119,10 +119,10 @@ def convert_pt_checkpoint_to_tf(model_type, pytorch_checkpoint_path, config_file tf_inputs = tf.constant(inputs_list) tfo = tf_model(tf_inputs, training=False) # build the network - pt_model = pt_model_class(config) - pt_model.load_state_dict(torch.load(pytorch_checkpoint_path, map_location='cpu'), - strict-False) - pt_model.eval() + state_dict = torch.load(pytorch_checkpoint_path, map_location='cpu') + pt_model = pt_model_class.from_pretrained(pretrained_model_name_or_path=None, + config=config, + state_dict=state_dict) pt_inputs = torch.tensor(inputs_list) with torch.no_grad(): @@ -140,7 +140,7 @@ def convert_pt_checkpoint_to_tf(model_type, pytorch_checkpoint_path, config_file def convert_all_pt_checkpoints_to_tf(args_model_type, tf_dump_path, model_shortcut_names_or_path=None, config_shortcut_names_or_path=None, - compare_with_pt_model=False, use_cached_models=False, only_convert_finetuned_models=False): + compare_with_pt_model=False, use_cached_models=False, remove_cached_files=False, only_convert_finetuned_models=False): assert os.path.isdir(args.tf_dump_path), "--tf_dump_path should be a directory" if args_model_type is None: @@ -188,13 +188,15 @@ def convert_all_pt_checkpoints_to_tf(args_model_type, tf_dump_path, model_shortc if os.path.isfile(model_shortcut_name): model_shortcut_name = 'converted_model' + convert_pt_checkpoint_to_tf(model_type=model_type, pytorch_checkpoint_path=model_file, config_file=config_file, tf_dump_path=os.path.join(tf_dump_path, model_shortcut_name + '-tf_model.h5'), compare_with_pt_model=compare_with_pt_model) - os.remove(config_file) - os.remove(model_file) + if remove_cached_files: + os.remove(config_file) + os.remove(model_file) if __name__ == "__main__": @@ -227,6 +229,9 @@ if __name__ == "__main__": parser.add_argument("--use_cached_models", action='store_true', help = "Use cached models if possible instead of updating to latest checkpoint versions.") + parser.add_argument("--remove_cached_files", + action='store_true', + help = "Remove pytorch models after conversion (save memory when converting in batches).") parser.add_argument("--only_convert_finetuned_models", action='store_true', help = "Only convert finetuned models.") @@ -246,4 +251,5 @@ if __name__ == "__main__": config_shortcut_names_or_path=[args.config_file] if args.config_file is not None else None, compare_with_pt_model=args.compare_with_pt_model, use_cached_models=args.use_cached_models, + remove_cached_files=args.remove_cached_files, only_convert_finetuned_models=args.only_convert_finetuned_models) From e4679cddced7d746427066a78e8079fb40e51528 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Fri, 6 Dec 2019 11:56:23 -0500 Subject: [PATCH 248/505] [cli] Uploads: add progress bar (#2078) * [cli] Uploads: add progress bar see https://github.com/huggingface/transformers/pull/2044#discussion_r354057827 for context * rename + documentation * Add auto-referential comment --- transformers/hf_api.py | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/transformers/hf_api.py b/transformers/hf_api.py index c21592a838..3bbb6c567a 100644 --- a/transformers/hf_api.py +++ b/transformers/hf_api.py @@ -16,10 +16,11 @@ from __future__ import absolute_import, division, print_function import os from os.path import expanduser -import six import requests +import six from requests.exceptions import HTTPError +from tqdm import tqdm ENDPOINT = "https://huggingface.co" @@ -129,10 +130,13 @@ class HfApi: # Even though we presign with the correct content-type, # the client still has to specify it when uploading the file. with open(filepath, "rb") as f: + pf = TqdmProgressFileReader(f) + r = requests.put(urls.write, data=f, headers={ "content-type": urls.type, }) r.raise_for_status() + pf.close() return urls.access def list_objs(self, token): @@ -148,6 +152,34 @@ class HfApi: +class TqdmProgressFileReader: + """ + Wrap an io.BufferedReader `f` (such as the output of `open(…, "rb")`) + and override `f.read()` so as to display a tqdm progress bar. + + see github.com/huggingface/transformers/pull/2078#discussion_r354739608 + for implementation details. + """ + def __init__( + self, + f # type: io.BufferedReader + ): + self.f = f + self.total_size = os.fstat(f.fileno()).st_size # type: int + self.pbar = tqdm(total=self.total_size, leave=False) + if six.PY3: + # does not work unless PY3 + # no big deal as the CLI does not currently support PY2 anyways. + self.read = f.read + f.read = self._read + + def _read(self, n=-1): + self.pbar.update(n) + return self.read(n) + + def close(self): + self.pbar.close() + class HfFolder: From 35401fe50fa3e460b2a4422630b017f106c79e03 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Fri, 6 Dec 2019 19:57:38 +0100 Subject: [PATCH 249/505] Remove dependency on pytest for running tests (#2055) * Switch to plain unittest for skipping slow tests. Add a RUN_SLOW environment variable for running them. * Switch to plain unittest for PyTorch dependency. * Switch to plain unittest for TensorFlow dependency. * Avoid leaking open files in the test suite. This prevents spurious warnings when running tests. * Fix unicode warning on Python 2 when running tests. The warning was: UnicodeWarning: Unicode equal comparison failed to convert both arguments to Unicode - interpreting them as being unequal * Support running PyTorch tests on a GPU. Reverts 27e015bd. * Tests no longer require pytest. * Make tests pass on cuda --- README.md | 11 +++- docs/source/installation.md | 11 +++- setup.py | 1 - .../tests/modeling_tf_xxx_test.py | 7 +- .../tests/modeling_xxx_test.py | 12 ++-- transformers/modeling_openai.py | 6 +- transformers/tests/conftest.py | 31 --------- transformers/tests/modeling_albert_test.py | 11 ++-- transformers/tests/modeling_auto_test.py | 14 ++-- transformers/tests/modeling_bert_test.py | 38 +++++------ transformers/tests/modeling_common_test.py | 43 ++++++++++--- transformers/tests/modeling_ctrl_test.py | 9 +-- .../tests/modeling_distilbert_test.py | 12 ++-- .../tests/modeling_encoder_decoder_test.py | 7 +- transformers/tests/modeling_gpt2_test.py | 10 +-- transformers/tests/modeling_openai_test.py | 10 +-- transformers/tests/modeling_roberta_test.py | 22 ++++--- transformers/tests/modeling_tf_albert_test.py | 7 +- transformers/tests/modeling_tf_auto_test.py | 14 ++-- transformers/tests/modeling_tf_bert_test.py | 7 +- transformers/tests/modeling_tf_common_test.py | 8 +-- transformers/tests/modeling_tf_ctrl_test.py | 7 +- .../tests/modeling_tf_distilbert_test.py | 7 +- transformers/tests/modeling_tf_gpt2_test.py | 7 +- .../tests/modeling_tf_openai_gpt_test.py | 7 +- .../tests/modeling_tf_roberta_test.py | 19 +++--- .../tests/modeling_tf_transfo_xl_test.py | 7 +- transformers/tests/modeling_tf_xlm_test.py | 7 +- transformers/tests/modeling_tf_xlnet_test.py | 10 +-- .../tests/modeling_transfo_xl_test.py | 10 +-- transformers/tests/modeling_xlm_test.py | 12 ++-- transformers/tests/modeling_xlnet_test.py | 21 ++++-- transformers/tests/optimization_test.py | 6 +- transformers/tests/tokenization_auto_test.py | 5 +- transformers/tests/tokenization_bert_test.py | 4 +- .../tests/tokenization_distilbert_test.py | 4 +- .../tests/tokenization_roberta_test.py | 4 +- .../tests/tokenization_tests_commons.py | 6 +- .../tests/tokenization_transfo_xl_test.py | 6 +- transformers/tests/tokenization_utils_test.py | 6 +- transformers/tests/tokenization_xlm_test.py | 4 +- transformers/tests/tokenization_xlnet_test.py | 4 +- transformers/tests/utils.py | 64 +++++++++++++++++++ transformers/tokenization_albert.py | 8 +-- transformers/tokenization_ctrl.py | 6 +- transformers/tokenization_gpt2.py | 12 ++-- transformers/tokenization_openai.py | 6 +- transformers/tokenization_utils.py | 13 ++-- transformers/tokenization_xlm.py | 8 ++- transformers/tokenization_xlnet.py | 4 +- 50 files changed, 344 insertions(+), 231 deletions(-) delete mode 100644 transformers/tests/conftest.py create mode 100644 transformers/tests/utils.py diff --git a/README.md b/README.md index ddeabe08d6..64ec631651 100644 --- a/README.md +++ b/README.md @@ -101,17 +101,26 @@ pip install [--editable] . A series of tests are included for the library and the example scripts. Library tests can be found in the [tests folder](https://github.com/huggingface/transformers/tree/master/transformers/tests) and examples tests in the [examples folder](https://github.com/huggingface/transformers/tree/master/examples). -These tests can be run using `pytest` (install pytest if needed with `pip install pytest`). +These tests can be run using `unittest` or `pytest` (install pytest if needed with `pip install pytest`). Depending on which framework is installed (TensorFlow 2.0 and/or PyTorch), the irrelevant tests will be skipped. Ensure that both frameworks are installed if you want to execute all tests. You can run the tests from the root of the cloned repository with the commands: +```bash +python -m unittest discover -s transformers/tests -p "*test.py" -t . +python -m unittest discover -s examples -p "*test.py" -t examples +``` + +or + ```bash python -m pytest -sv ./transformers/tests/ python -m pytest -sv ./examples/ ``` +By default, slow tests are skipped. Set the `RUN_SLOW` environment variable to `yes` to run them. + ### Do you want to run a Transformer model on a mobile device? You should check out our [`swift-coreml-transformers`](https://github.com/huggingface/swift-coreml-transformers) repo. diff --git a/docs/source/installation.md b/docs/source/installation.md index 11beb1ab3a..6263f7604d 100644 --- a/docs/source/installation.md +++ b/docs/source/installation.md @@ -24,15 +24,24 @@ pip install [--editable] . An extensive test suite is included to test the library behavior and several examples. Library tests can be found in the [tests folder](https://github.com/huggingface/transformers/tree/master/transformers/tests) and examples tests in the [examples folder](https://github.com/huggingface/transformers/tree/master/examples). -Tests can be run using `pytest` (install pytest if needed with `pip install pytest`). +Tests can be run using `unittest` or `pytest` (install pytest if needed with `pip install pytest`). Run all the tests from the root of the cloned repository with the commands: +```bash +python -m unittest discover -s transformers/tests -p "*test.py" -t . +python -m unittest discover -s examples -p "*test.py" -t examples +``` + +or + ``` bash python -m pytest -sv ./transformers/tests/ python -m pytest -sv ./examples/ ``` +By default, slow tests are skipped. Set the `RUN_SLOW` environment variable to `yes` to run them. + ## OpenAI GPT original tokenization workflow If you want to reproduce the original tokenization process of the `OpenAI GPT` paper, you will need to install `ftfy` (use version 4.4.3 if you are using Python 2) and `SpaCy`: diff --git a/setup.py b/setup.py index 25f503f8d0..c4af32df83 100644 --- a/setup.py +++ b/setup.py @@ -72,7 +72,6 @@ setup( 'transformers-cli' ], # python_requires='>=3.5.0', - tests_require=['pytest'], classifiers=[ 'Intended Audience :: Science/Research', 'License :: OSI Approved :: Apache Software License', diff --git a/templates/adding_a_new_model/tests/modeling_tf_xxx_test.py b/templates/adding_a_new_model/tests/modeling_tf_xxx_test.py index 90837ca1ea..d7e576bf8b 100644 --- a/templates/adding_a_new_model/tests/modeling_tf_xxx_test.py +++ b/templates/adding_a_new_model/tests/modeling_tf_xxx_test.py @@ -18,11 +18,11 @@ from __future__ import print_function import unittest import shutil -import pytest import sys from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import require_tf, slow from transformers import XxxConfig, is_tf_available @@ -33,10 +33,9 @@ if is_tf_available(): TFXxxForTokenClassification, TFXxxForQuestionAnswering, TF_XXX_PRETRAINED_MODEL_ARCHIVE_MAP) -else: - pytestmark = pytest.mark.skip("Require TensorFlow") +@require_tf class TFXxxModelTest(TFCommonTestCases.TFCommonModelTester): all_model_classes = (TFXxxModel, TFXxxForMaskedLM, TFXxxForQuestionAnswering, @@ -244,7 +243,7 @@ class TFXxxModelTest(TFCommonTestCases.TFCommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_xxx_for_token_classification(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): cache_dir = "/tmp/transformers_test/" for model_name in ['xxx-base-uncased']: diff --git a/templates/adding_a_new_model/tests/modeling_xxx_test.py b/templates/adding_a_new_model/tests/modeling_xxx_test.py index 8c0cc3cf32..bfc70921cd 100644 --- a/templates/adding_a_new_model/tests/modeling_xxx_test.py +++ b/templates/adding_a_new_model/tests/modeling_xxx_test.py @@ -18,12 +18,12 @@ from __future__ import print_function import unittest import shutil -import pytest from transformers import is_torch_available from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import require_torch, slow, torch_device if is_torch_available(): from transformers import (XxxConfig, XxxModel, XxxForMaskedLM, @@ -31,10 +31,9 @@ if is_torch_available(): XxxForQuestionAnswering, XxxForSequenceClassification, XxxForTokenClassification, XxxForMultipleChoice) from transformers.modeling_xxx import XXX_PRETRAINED_MODEL_ARCHIVE_MAP -else: - pytestmark = pytest.mark.skip("Require Torch") +@require_torch class XxxModelTest(CommonTestCases.CommonModelTester): all_model_classes = (XxxModel, XxxForMaskedLM, XxxForQuestionAnswering, @@ -131,6 +130,7 @@ class XxxModelTest(CommonTestCases.CommonModelTester): def create_and_check_xxx_model(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): model = XxxModel(config=config) + model.to(torch_device) model.eval() sequence_output, pooled_output = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids) sequence_output, pooled_output = model(input_ids, token_type_ids=token_type_ids) @@ -148,6 +148,7 @@ class XxxModelTest(CommonTestCases.CommonModelTester): def create_and_check_xxx_for_masked_lm(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): model = XxxForMaskedLM(config=config) + model.to(torch_device) model.eval() loss, prediction_scores = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, masked_lm_labels=token_labels) result = { @@ -162,6 +163,7 @@ class XxxModelTest(CommonTestCases.CommonModelTester): def create_and_check_xxx_for_question_answering(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): model = XxxForQuestionAnswering(config=config) + model.to(torch_device) model.eval() loss, start_logits, end_logits = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, start_positions=sequence_labels, end_positions=sequence_labels) @@ -182,6 +184,7 @@ class XxxModelTest(CommonTestCases.CommonModelTester): def create_and_check_xxx_for_sequence_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): config.num_labels = self.num_labels model = XxxForSequenceClassification(config) + model.to(torch_device) model.eval() loss, logits = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=sequence_labels) result = { @@ -197,6 +200,7 @@ class XxxModelTest(CommonTestCases.CommonModelTester): def create_and_check_xxx_for_token_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): config.num_labels = self.num_labels model = XxxForTokenClassification(config=config) + model.to(torch_device) model.eval() loss, logits = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels) result = { @@ -243,7 +247,7 @@ class XxxModelTest(CommonTestCases.CommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_xxx_for_token_classification(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): cache_dir = "/tmp/transformers_test/" for model_name in list(XXX_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: diff --git a/transformers/modeling_openai.py b/transformers/modeling_openai.py index e88f55c3ea..4fe7ffee8b 100644 --- a/transformers/modeling_openai.py +++ b/transformers/modeling_openai.py @@ -50,8 +50,10 @@ def load_tf_weights_in_openai_gpt(model, config, openai_checkpoint_folder_path): logger.info("Loading weights from {}".format(openai_checkpoint_folder_path)) - names = json.load(open(openai_checkpoint_folder_path + '/parameters_names.json', "r", encoding='utf-8')) - shapes = json.load(open(openai_checkpoint_folder_path + '/params_shapes.json', "r", encoding='utf-8')) + with open(openai_checkpoint_folder_path + '/parameters_names.json', "r", encoding='utf-8') as names_handle: + names = json.load(names_handle) + with open(openai_checkpoint_folder_path + '/params_shapes.json', "r", encoding='utf-8') as shapes_handle: + shapes = json.load(shapes_handle) offsets = np.cumsum([np.prod(shape) for shape in shapes]) init_params = [np.load(openai_checkpoint_folder_path + '/params_{}.npy'.format(n)) for n in range(10)] init_params = np.split(np.concatenate(init_params, 0), offsets)[:-1] diff --git a/transformers/tests/conftest.py b/transformers/tests/conftest.py deleted file mode 100644 index f809234cd5..0000000000 --- a/transformers/tests/conftest.py +++ /dev/null @@ -1,31 +0,0 @@ -# content of conftest.py - -import pytest - - -def pytest_addoption(parser): - parser.addoption( - "--runslow", action="store_true", default=False, help="run slow tests" - ) - parser.addoption( - "--use_cuda", action="store_true", default=False, help="run tests on gpu" - ) - - -def pytest_configure(config): - config.addinivalue_line("markers", "slow: mark test as slow to run") - - -def pytest_collection_modifyitems(config, items): - if config.getoption("--runslow"): - # --runslow given in cli: do not skip slow tests - return - skip_slow = pytest.mark.skip(reason="need --runslow option to run") - for item in items: - if "slow" in item.keywords: - item.add_marker(skip_slow) - -@pytest.fixture -def use_cuda(request): - """ Run test on gpu """ - return request.config.getoption("--use_cuda") diff --git a/transformers/tests/modeling_albert_test.py b/transformers/tests/modeling_albert_test.py index 976feff9db..a14d66ae8f 100644 --- a/transformers/tests/modeling_albert_test.py +++ b/transformers/tests/modeling_albert_test.py @@ -18,22 +18,21 @@ from __future__ import print_function import unittest import shutil -import pytest from transformers import is_torch_available from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import require_torch, slow, torch_device if is_torch_available(): from transformers import (AlbertConfig, AlbertModel, AlbertForMaskedLM, AlbertForSequenceClassification, AlbertForQuestionAnswering, ) from transformers.modeling_albert import ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP -else: - pytestmark = pytest.mark.skip("Require Torch") +@require_torch class AlbertModelTest(CommonTestCases.CommonModelTester): all_model_classes = (AlbertModel, AlbertForMaskedLM) if is_torch_available() else () @@ -133,6 +132,7 @@ class AlbertModelTest(CommonTestCases.CommonModelTester): def create_and_check_albert_model(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): model = AlbertModel(config=config) + model.to(torch_device) model.eval() sequence_output, pooled_output = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids) sequence_output, pooled_output = model(input_ids, token_type_ids=token_type_ids) @@ -150,6 +150,7 @@ class AlbertModelTest(CommonTestCases.CommonModelTester): def create_and_check_albert_for_masked_lm(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): model = AlbertForMaskedLM(config=config) + model.to(torch_device) model.eval() loss, prediction_scores = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, masked_lm_labels=token_labels) result = { @@ -163,6 +164,7 @@ class AlbertModelTest(CommonTestCases.CommonModelTester): def create_and_check_albert_for_question_answering(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): model = AlbertForQuestionAnswering(config=config) + model.to(torch_device) model.eval() loss, start_logits, end_logits = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, start_positions=sequence_labels, end_positions=sequence_labels) @@ -183,6 +185,7 @@ class AlbertModelTest(CommonTestCases.CommonModelTester): def create_and_check_albert_for_sequence_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): config.num_labels = self.num_labels model = AlbertForSequenceClassification(config) + model.to(torch_device) model.eval() loss, logits = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=sequence_labels) result = { @@ -225,7 +228,7 @@ class AlbertModelTest(CommonTestCases.CommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_albert_for_sequence_classification(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): cache_dir = "/tmp/transformers_test/" for model_name in list(ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: diff --git a/transformers/tests/modeling_auto_test.py b/transformers/tests/modeling_auto_test.py index 6d2c7ec979..9b7d920bc8 100644 --- a/transformers/tests/modeling_auto_test.py +++ b/transformers/tests/modeling_auto_test.py @@ -18,11 +18,12 @@ from __future__ import print_function import unittest import shutil -import pytest import logging from transformers import is_torch_available +from .utils import require_torch, slow + if is_torch_available(): from transformers import (AutoConfig, BertConfig, AutoModel, BertModel, @@ -33,12 +34,11 @@ if is_torch_available(): from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester -else: - pytestmark = pytest.mark.skip("Require Torch") +@require_torch class AutoModelTest(unittest.TestCase): - @pytest.mark.slow + @slow def test_model_from_pretrained(self): logging.basicConfig(level=logging.INFO) for model_name in list(BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: @@ -53,7 +53,7 @@ class AutoModelTest(unittest.TestCase): for value in loading_info.values(): self.assertEqual(len(value), 0) - @pytest.mark.slow + @slow def test_lmhead_model_from_pretrained(self): logging.basicConfig(level=logging.INFO) for model_name in list(BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: @@ -66,7 +66,7 @@ class AutoModelTest(unittest.TestCase): self.assertIsNotNone(model) self.assertIsInstance(model, BertForMaskedLM) - @pytest.mark.slow + @slow def test_sequence_classification_model_from_pretrained(self): logging.basicConfig(level=logging.INFO) for model_name in list(BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: @@ -79,7 +79,7 @@ class AutoModelTest(unittest.TestCase): self.assertIsNotNone(model) self.assertIsInstance(model, BertForSequenceClassification) - @pytest.mark.slow + @slow def test_question_answering_model_from_pretrained(self): logging.basicConfig(level=logging.INFO) for model_name in list(BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: diff --git a/transformers/tests/modeling_bert_test.py b/transformers/tests/modeling_bert_test.py index 6c93c9a187..539f66cd3f 100644 --- a/transformers/tests/modeling_bert_test.py +++ b/transformers/tests/modeling_bert_test.py @@ -18,12 +18,12 @@ from __future__ import print_function import unittest import shutil -import pytest from transformers import is_torch_available from .modeling_common_test import (CommonTestCases, ids_tensor, floats_tensor) from .configuration_common_test import ConfigTester +from .utils import require_torch, slow, torch_device if is_torch_available(): from transformers import (BertConfig, BertModel, BertForMaskedLM, @@ -31,11 +31,9 @@ if is_torch_available(): BertForQuestionAnswering, BertForSequenceClassification, BertForTokenClassification, BertForMultipleChoice) from transformers.modeling_bert import BERT_PRETRAINED_MODEL_ARCHIVE_MAP -else: - pytestmark = pytest.mark.skip("Require Torch") -@pytest.mark.usefixtures("use_cuda") +@require_torch class BertModelTest(CommonTestCases.CommonModelTester): all_model_classes = (BertModel, BertForMaskedLM, BertForNextSentencePrediction, @@ -67,7 +65,6 @@ class BertModelTest(CommonTestCases.CommonModelTester): num_labels=3, num_choices=4, scope=None, - device='cpu', ): self.parent = parent self.batch_size = batch_size @@ -91,26 +88,25 @@ class BertModelTest(CommonTestCases.CommonModelTester): self.num_labels = num_labels self.num_choices = num_choices self.scope = scope - self.device = device def prepare_config_and_inputs(self): - input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size).to(self.device) + input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) input_mask = None if self.use_input_mask: - input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2).to(self.device) + input_mask = ids_tensor([self.batch_size, self.seq_length], vocab_size=2) token_type_ids = None if self.use_token_type_ids: - token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size).to(self.device) + token_type_ids = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size) sequence_labels = None token_labels = None choice_labels = None if self.use_labels: - sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size).to(self.device) - token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels).to(self.device) - choice_labels = ids_tensor([self.batch_size], self.num_choices).to(self.device) + sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size) + token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels) + choice_labels = ids_tensor([self.batch_size], self.num_choices) config = BertConfig( vocab_size_or_config_json_file=self.vocab_size, @@ -144,7 +140,7 @@ class BertModelTest(CommonTestCases.CommonModelTester): def create_and_check_bert_model(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): model = BertModel(config=config) - model.to(input_ids.device) + model.to(torch_device) model.eval() sequence_output, pooled_output = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids) sequence_output, pooled_output = model(input_ids, token_type_ids=token_type_ids) @@ -161,6 +157,7 @@ class BertModelTest(CommonTestCases.CommonModelTester): def create_and_check_bert_model_as_decoder(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels, encoder_hidden_states, encoder_attention_mask): model = BertModel(config) + model.to(torch_device) model.eval() sequence_output, pooled_output = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, encoder_hidden_states=encoder_hidden_states, encoder_attention_mask=encoder_attention_mask) sequence_output, pooled_output = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, encoder_hidden_states=encoder_hidden_states) @@ -177,6 +174,7 @@ class BertModelTest(CommonTestCases.CommonModelTester): def create_and_check_bert_for_masked_lm(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): model = BertForMaskedLM(config=config) + model.to(torch_device) model.eval() loss, prediction_scores = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, masked_lm_labels=token_labels) result = { @@ -190,6 +188,7 @@ class BertModelTest(CommonTestCases.CommonModelTester): def create_and_check_bert_model_for_masked_lm_as_decoder(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels, encoder_hidden_states, encoder_attention_mask): model = BertForMaskedLM(config=config) + model.to(torch_device) model.eval() loss, prediction_scores = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, masked_lm_labels=token_labels, encoder_hidden_states=encoder_hidden_states, encoder_attention_mask=encoder_attention_mask) loss, prediction_scores = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, masked_lm_labels=token_labels, encoder_hidden_states=encoder_hidden_states) @@ -204,6 +203,7 @@ class BertModelTest(CommonTestCases.CommonModelTester): def create_and_check_bert_for_next_sequence_prediction(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): model = BertForNextSentencePrediction(config=config) + model.to(torch_device) model.eval() loss, seq_relationship_score = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, next_sentence_label=sequence_labels) result = { @@ -217,6 +217,7 @@ class BertModelTest(CommonTestCases.CommonModelTester): def create_and_check_bert_for_pretraining(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): model = BertForPreTraining(config=config) + model.to(torch_device) model.eval() loss, prediction_scores, seq_relationship_score = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, masked_lm_labels=token_labels, next_sentence_label=sequence_labels) @@ -235,6 +236,7 @@ class BertModelTest(CommonTestCases.CommonModelTester): def create_and_check_bert_for_question_answering(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): model = BertForQuestionAnswering(config=config) + model.to(torch_device) model.eval() loss, start_logits, end_logits = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, start_positions=sequence_labels, end_positions=sequence_labels) @@ -254,6 +256,7 @@ class BertModelTest(CommonTestCases.CommonModelTester): def create_and_check_bert_for_sequence_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): config.num_labels = self.num_labels model = BertForSequenceClassification(config) + model.to(torch_device) model.eval() loss, logits = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=sequence_labels) result = { @@ -268,6 +271,7 @@ class BertModelTest(CommonTestCases.CommonModelTester): def create_and_check_bert_for_token_classification(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): config.num_labels = self.num_labels model = BertForTokenClassification(config=config) + model.to(torch_device) model.eval() loss, logits = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels) result = { @@ -282,6 +286,7 @@ class BertModelTest(CommonTestCases.CommonModelTester): def create_and_check_bert_for_multiple_choice(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): config.num_choices = self.num_choices model = BertForMultipleChoice(config=config) + model.to(torch_device) model.eval() multiple_choice_inputs_ids = input_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous() multiple_choice_token_type_ids = token_type_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous() @@ -313,10 +318,7 @@ class BertModelTest(CommonTestCases.CommonModelTester): def test_config(self): self.config_tester.run_common_tests() - def test_bert_model(self, use_cuda=False): - # ^^ This could be a real fixture - if use_cuda: - self.model_tester.device = "cuda" + def test_bert_model(self): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_model(*config_and_inputs) @@ -356,7 +358,7 @@ class BertModelTest(CommonTestCases.CommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_token_classification(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): cache_dir = "/tmp/transformers_test/" for model_name in list(BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: diff --git a/transformers/tests/modeling_common_test.py b/transformers/tests/modeling_common_test.py index baf1531403..80d5d95455 100644 --- a/transformers/tests/modeling_common_test.py +++ b/transformers/tests/modeling_common_test.py @@ -27,10 +27,11 @@ import uuid import unittest import logging -import pytest from transformers import is_torch_available +from .utils import require_torch, slow, torch_device + if is_torch_available(): import torch import numpy as np @@ -38,8 +39,6 @@ if is_torch_available(): from transformers import (AdaptiveEmbedding, PretrainedConfig, PreTrainedModel, BertModel, BertConfig, BERT_PRETRAINED_MODEL_ARCHIVE_MAP, GPT2LMHeadModel, GPT2Config, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP) -else: - pytestmark = pytest.mark.skip("Require Torch") if sys.version_info[0] == 2: import cPickle as pickle @@ -65,6 +64,7 @@ def _config_zero_init(config): class CommonTestCases: + @require_torch class CommonModelTester(unittest.TestCase): model_tester = None @@ -79,6 +79,7 @@ class CommonTestCases: for model_class in self.all_model_classes: model = model_class(config) + model.to(torch_device) model.eval() with torch.no_grad(): outputs = model(**inputs_dict) @@ -86,12 +87,13 @@ class CommonTestCases: with TemporaryDirectory() as tmpdirname: model.save_pretrained(tmpdirname) model = model_class.from_pretrained(tmpdirname) + model.to(torch_device) with torch.no_grad(): after_outputs = model(**inputs_dict) # Make sure we don't have nans - out_1 = after_outputs[0].numpy() - out_2 = outputs[0].numpy() + out_1 = after_outputs[0].cpu().numpy() + out_2 = outputs[0].cpu().numpy() out_1 = out_1[~np.isnan(out_1)] out_2 = out_2[~np.isnan(out_2)] max_diff = np.amax(np.abs(out_1 - out_2)) @@ -113,6 +115,7 @@ class CommonTestCases: for model_class in self.all_model_classes: model = model_class(config) + model.to(torch_device) model.eval() first, second = model(inputs_dict["input_ids"])[0], model(inputs_dict["input_ids"])[0] self.assertEqual(first.ne(second).sum().item(), 0) @@ -125,6 +128,7 @@ class CommonTestCases: config.output_attentions = True config.output_hidden_states = False model = model_class(config) + model.to(torch_device) model.eval() outputs = model(**inputs_dict) attentions = outputs[-1] @@ -142,6 +146,7 @@ class CommonTestCases: config.output_attentions = True config.output_hidden_states = True model = model_class(config) + model.to(torch_device) model.eval() outputs = model(**inputs_dict) self.assertEqual(out_len+1, len(outputs)) @@ -181,6 +186,7 @@ class CommonTestCases: configs_no_init.torchscript = True for model_class in self.all_model_classes: model = model_class(config=configs_no_init) + model.to(torch_device) model.eval() inputs = inputs_dict['input_ids'] # Let's keep only input_ids @@ -201,7 +207,10 @@ class CommonTestCases: except ValueError: self.fail("Couldn't load module.") + model.to(torch_device) model.eval() + + loaded_model.to(torch_device) loaded_model.eval() model_params = model.parameters() @@ -228,11 +237,12 @@ class CommonTestCases: configs_no_init = _config_zero_init(config) # To be sure we have no Nan for model_class in self.all_model_classes: model = model_class(config=configs_no_init) + model.to(torch_device) model.eval() # Prepare head_mask # Set require_grad after having prepared the tensor to avoid error (leaf variable has been moved into the graph interior) - head_mask = torch.ones(self.model_tester.num_hidden_layers, self.model_tester.num_attention_heads) + head_mask = torch.ones(self.model_tester.num_hidden_layers, self.model_tester.num_attention_heads, device=torch_device) head_mask[0, 0] = 0 head_mask[-1, :-1] = 0 head_mask.requires_grad_(requires_grad=True) @@ -282,6 +292,7 @@ class CommonTestCases: config.output_attentions = True config.output_hidden_states = False model = model_class(config=config) + model.to(torch_device) model.eval() heads_to_prune = {0: list(range(1, self.model_tester.num_attention_heads)), -1: [0]} @@ -310,6 +321,7 @@ class CommonTestCases: config.output_attentions = True config.output_hidden_states = False model = model_class(config=config) + model.to(torch_device) model.eval() heads_to_prune = {0: list(range(1, self.model_tester.num_attention_heads)), -1: [0]} @@ -319,6 +331,7 @@ class CommonTestCases: os.makedirs(directory) model.save_pretrained(directory) model = model_class.from_pretrained(directory) + model.to(torch_device) outputs = model(**inputs_dict) attentions = outputs[-1] @@ -346,6 +359,7 @@ class CommonTestCases: config.pruned_heads = heads_to_prune model = model_class(config=config) + model.to(torch_device) model.eval() outputs = model(**inputs_dict) @@ -372,6 +386,7 @@ class CommonTestCases: config.pruned_heads = heads_to_prune model = model_class(config=config) + model.to(torch_device) model.eval() outputs = model(**inputs_dict) @@ -388,6 +403,7 @@ class CommonTestCases: os.makedirs(directory) model.save_pretrained(directory) model = model_class.from_pretrained(directory) + model.to(torch_device) shutil.rmtree(directory) outputs = model(**inputs_dict) @@ -419,6 +435,7 @@ class CommonTestCases: config.output_hidden_states = True config.output_attentions = False model = model_class(config) + model.to(torch_device) model.eval() outputs = model(**inputs_dict) hidden_states = outputs[-1] @@ -538,6 +555,7 @@ class CommonTestCases: for model_class in self.all_model_classes: model = model_class(config) + model.to(torch_device) model.eval() wte = model.get_input_embeddings() @@ -628,6 +646,7 @@ class CommonTestCases: def create_and_check_base_model(self, config, input_ids, token_type_ids, position_ids, mc_labels, lm_labels, mc_token_ids): model = self.base_model_class(config) + model.to(torch_device) model.eval() outputs = model(input_ids, position_ids, token_type_ids) @@ -643,6 +662,7 @@ class CommonTestCases: def create_and_check_lm_head(self, config, input_ids, token_type_ids, position_ids, mc_labels, lm_labels, mc_token_ids): model = self.lm_head_model_class(config) + model.to(torch_device) model.eval() outputs = model(input_ids, position_ids, token_type_ids, lm_labels) loss, lm_logits = outputs[:2] @@ -659,6 +679,7 @@ class CommonTestCases: mc_labels, lm_labels, mc_token_ids): for model_class in self.all_model_classes: model = model_class(config) + model.to(torch_device) model.eval() outputs = model(input_ids) presents = outputs[-1] @@ -671,6 +692,7 @@ class CommonTestCases: def create_and_check_double_heads(self, config, input_ids, token_type_ids, position_ids, mc_labels, lm_labels, mc_token_ids): model = self.double_head_model_class(config) + model.to(torch_device) model.eval() outputs = model(input_ids, mc_token_ids, lm_labels=lm_labels, mc_labels=mc_labels, token_type_ids=token_type_ids, position_ids=position_ids) @@ -716,7 +738,7 @@ class CommonTestCases: config_and_inputs = self.prepare_config_and_inputs() self.create_and_check_presents(*config_and_inputs) - @pytest.mark.slow + @slow def run_slow_tests(self): self.create_and_check_model_from_pretrained() @@ -770,7 +792,7 @@ def ids_tensor(shape, vocab_size, rng=None, name=None): for _ in range(total_dims): values.append(rng.randint(0, vocab_size - 1)) - return torch.tensor(data=values, dtype=torch.long).view(shape).contiguous() + return torch.tensor(data=values, dtype=torch.long, device=torch_device).view(shape).contiguous() def floats_tensor(shape, scale=1.0, rng=None, name=None): @@ -786,11 +808,12 @@ def floats_tensor(shape, scale=1.0, rng=None, name=None): for _ in range(total_dims): values.append(rng.random() * scale) - return torch.tensor(data=values, dtype=torch.float).view(shape).contiguous() + return torch.tensor(data=values, dtype=torch.float, device=torch_device).view(shape).contiguous() +@require_torch class ModelUtilsTest(unittest.TestCase): - @pytest.mark.slow + @slow def test_model_from_pretrained(self): logging.basicConfig(level=logging.INFO) for model_name in list(BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: diff --git a/transformers/tests/modeling_ctrl_test.py b/transformers/tests/modeling_ctrl_test.py index 47ff8d8d51..8c14578a5c 100644 --- a/transformers/tests/modeling_ctrl_test.py +++ b/transformers/tests/modeling_ctrl_test.py @@ -16,7 +16,6 @@ from __future__ import division from __future__ import print_function import unittest -import pytest import shutil import pdb @@ -25,13 +24,13 @@ from transformers import is_torch_available if is_torch_available(): from transformers import (CTRLConfig, CTRLModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP, CTRLLMHeadModel) -else: - pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import require_torch, slow, torch_device +@require_torch class CTRLModelTest(CommonTestCases.CommonModelTester): all_model_classes = (CTRLModel, CTRLLMHeadModel) if is_torch_available() else () @@ -140,6 +139,7 @@ class CTRLModelTest(CommonTestCases.CommonModelTester): def create_and_check_ctrl_model(self, config, input_ids, input_mask, head_mask, token_type_ids, *args): model = CTRLModel(config=config) + model.to(torch_device) model.eval() model(input_ids, token_type_ids=token_type_ids, head_mask=head_mask) @@ -157,6 +157,7 @@ class CTRLModelTest(CommonTestCases.CommonModelTester): def create_and_check_lm_head_model(self, config, input_ids, input_mask, head_mask, token_type_ids, *args): model = CTRLLMHeadModel(config) + model.to(torch_device) model.eval() loss, lm_logits, _ = model(input_ids, token_type_ids=token_type_ids, labels=input_ids) @@ -202,7 +203,7 @@ class CTRLModelTest(CommonTestCases.CommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_lm_head_model(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): cache_dir = "/tmp/transformers_test/" for model_name in list(CTRL_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: diff --git a/transformers/tests/modeling_distilbert_test.py b/transformers/tests/modeling_distilbert_test.py index 8099c03586..4b8f64327d 100644 --- a/transformers/tests/modeling_distilbert_test.py +++ b/transformers/tests/modeling_distilbert_test.py @@ -17,7 +17,6 @@ from __future__ import division from __future__ import print_function import unittest -import pytest from transformers import is_torch_available @@ -25,13 +24,13 @@ if is_torch_available(): from transformers import (DistilBertConfig, DistilBertModel, DistilBertForMaskedLM, DistilBertForTokenClassification, DistilBertForQuestionAnswering, DistilBertForSequenceClassification) -else: - pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import require_torch, slow, torch_device +@require_torch class DistilBertModelTest(CommonTestCases.CommonModelTester): all_model_classes = (DistilBertModel, DistilBertForMaskedLM, DistilBertForQuestionAnswering, @@ -126,6 +125,7 @@ class DistilBertModelTest(CommonTestCases.CommonModelTester): def create_and_check_distilbert_model(self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels): model = DistilBertModel(config=config) + model.to(torch_device) model.eval() (sequence_output,) = model(input_ids, input_mask) (sequence_output,) = model(input_ids) @@ -139,6 +139,7 @@ class DistilBertModelTest(CommonTestCases.CommonModelTester): def create_and_check_distilbert_for_masked_lm(self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels): model = DistilBertForMaskedLM(config=config) + model.to(torch_device) model.eval() loss, prediction_scores = model(input_ids, attention_mask=input_mask, masked_lm_labels=token_labels) result = { @@ -152,6 +153,7 @@ class DistilBertModelTest(CommonTestCases.CommonModelTester): def create_and_check_distilbert_for_question_answering(self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels): model = DistilBertForQuestionAnswering(config=config) + model.to(torch_device) model.eval() loss, start_logits, end_logits = model(input_ids, attention_mask=input_mask, start_positions=sequence_labels, end_positions=sequence_labels) result = { @@ -170,6 +172,7 @@ class DistilBertModelTest(CommonTestCases.CommonModelTester): def create_and_check_distilbert_for_sequence_classification(self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels): config.num_labels = self.num_labels model = DistilBertForSequenceClassification(config) + model.to(torch_device) model.eval() loss, logits = model(input_ids, attention_mask=input_mask, labels=sequence_labels) result = { @@ -184,6 +187,7 @@ class DistilBertModelTest(CommonTestCases.CommonModelTester): def create_and_check_distilbert_for_token_classification(self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels): config.num_labels = self.num_labels model = DistilBertForTokenClassification(config=config) + model.to(torch_device) model.eval() loss, logits = model(input_ids, attention_mask=input_mask, labels=token_labels) @@ -229,7 +233,7 @@ class DistilBertModelTest(CommonTestCases.CommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_distilbert_for_token_classification(*config_and_inputs) - # @pytest.mark.slow + # @slow # def test_model_from_pretrained(self): # cache_dir = "/tmp/transformers_test/" # for model_name in list(DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: diff --git a/transformers/tests/modeling_encoder_decoder_test.py b/transformers/tests/modeling_encoder_decoder_test.py index a6c88ed9a9..64e86df8f5 100644 --- a/transformers/tests/modeling_encoder_decoder_test.py +++ b/transformers/tests/modeling_encoder_decoder_test.py @@ -15,19 +15,18 @@ import logging import unittest -import pytest from transformers import is_torch_available +from .utils import require_torch, slow if is_torch_available(): from transformers import BertModel, BertForMaskedLM, Model2Model from transformers.modeling_bert import BERT_PRETRAINED_MODEL_ARCHIVE_MAP -else: - pytestmark = pytest.mark.skip("Require Torch") +@require_torch class EncoderDecoderModelTest(unittest.TestCase): - @pytest.mark.slow + @slow def test_model2model_from_pretrained(self): logging.basicConfig(level=logging.INFO) for model_name in list(BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: diff --git a/transformers/tests/modeling_gpt2_test.py b/transformers/tests/modeling_gpt2_test.py index 4263e51bc9..ecaa2a4bd0 100644 --- a/transformers/tests/modeling_gpt2_test.py +++ b/transformers/tests/modeling_gpt2_test.py @@ -17,7 +17,6 @@ from __future__ import division from __future__ import print_function import unittest -import pytest import shutil from transformers import is_torch_available @@ -25,13 +24,13 @@ from transformers import is_torch_available if is_torch_available(): from transformers import (GPT2Config, GPT2Model, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP, GPT2LMHeadModel, GPT2DoubleHeadsModel) -else: - pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import require_torch, slow, torch_device +@require_torch class GPT2ModelTest(CommonTestCases.CommonModelTester): all_model_classes = (GPT2Model, GPT2LMHeadModel, GPT2DoubleHeadsModel) if is_torch_available() else () @@ -136,6 +135,7 @@ class GPT2ModelTest(CommonTestCases.CommonModelTester): def create_and_check_gpt2_model(self, config, input_ids, input_mask, head_mask, token_type_ids, *args): model = GPT2Model(config=config) + model.to(torch_device) model.eval() model(input_ids, token_type_ids=token_type_ids, head_mask=head_mask) @@ -153,6 +153,7 @@ class GPT2ModelTest(CommonTestCases.CommonModelTester): def create_and_check_lm_head_model(self, config, input_ids, input_mask, head_mask, token_type_ids, *args): model = GPT2LMHeadModel(config) + model.to(torch_device) model.eval() loss, lm_logits, _ = model(input_ids, token_type_ids=token_type_ids, labels=input_ids) @@ -171,6 +172,7 @@ class GPT2ModelTest(CommonTestCases.CommonModelTester): def create_and_check_double_lm_head_model(self, config, input_ids, input_mask, head_mask, token_type_ids, mc_token_ids, *args): model = GPT2DoubleHeadsModel(config) + model.to(torch_device) model.eval() @@ -235,7 +237,7 @@ class GPT2ModelTest(CommonTestCases.CommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_double_lm_head_model(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): cache_dir = "/tmp/transformers_test/" for model_name in list(GPT2_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: diff --git a/transformers/tests/modeling_openai_test.py b/transformers/tests/modeling_openai_test.py index 33218288a0..8e4d13438d 100644 --- a/transformers/tests/modeling_openai_test.py +++ b/transformers/tests/modeling_openai_test.py @@ -17,7 +17,6 @@ from __future__ import division from __future__ import print_function import unittest -import pytest import shutil from transformers import is_torch_available @@ -25,13 +24,13 @@ from transformers import is_torch_available if is_torch_available(): from transformers import (OpenAIGPTConfig, OpenAIGPTModel, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP, OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel) -else: - pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import require_torch, slow, torch_device +@require_torch class OpenAIGPTModelTest(CommonTestCases.CommonModelTester): all_model_classes = (OpenAIGPTModel, OpenAIGPTLMHeadModel, OpenAIGPTDoubleHeadsModel) if is_torch_available() else () @@ -124,6 +123,7 @@ class OpenAIGPTModelTest(CommonTestCases.CommonModelTester): def create_and_check_openai_gpt_model(self, config, input_ids, head_mask, token_type_ids, *args): model = OpenAIGPTModel(config=config) + model.to(torch_device) model.eval() model(input_ids, token_type_ids=token_type_ids, head_mask=head_mask) @@ -139,6 +139,7 @@ class OpenAIGPTModelTest(CommonTestCases.CommonModelTester): def create_and_check_lm_head_model(self, config, input_ids, head_mask, token_type_ids, *args): model = OpenAIGPTLMHeadModel(config) + model.to(torch_device) model.eval() loss, lm_logits = model(input_ids, token_type_ids=token_type_ids, labels=input_ids) @@ -157,6 +158,7 @@ class OpenAIGPTModelTest(CommonTestCases.CommonModelTester): def create_and_check_double_lm_head_model(self, config, input_ids, head_mask, token_type_ids, *args): model = OpenAIGPTDoubleHeadsModel(config) + model.to(torch_device) model.eval() loss, lm_logits, mc_logits = model(input_ids, token_type_ids=token_type_ids, lm_labels=input_ids) @@ -203,7 +205,7 @@ class OpenAIGPTModelTest(CommonTestCases.CommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_double_lm_head_model(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): cache_dir = "/tmp/transformers_test/" for model_name in list(OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: diff --git a/transformers/tests/modeling_roberta_test.py b/transformers/tests/modeling_roberta_test.py index 0620ddf630..7a3553b164 100644 --- a/transformers/tests/modeling_roberta_test.py +++ b/transformers/tests/modeling_roberta_test.py @@ -18,7 +18,6 @@ from __future__ import print_function import unittest import shutil -import pytest from transformers import is_torch_available @@ -27,13 +26,13 @@ if is_torch_available(): from transformers import (RobertaConfig, RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification, RobertaForTokenClassification) from transformers.modeling_roberta import ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP -else: - pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import require_torch, slow, torch_device +@require_torch class RobertaModelTest(CommonTestCases.CommonModelTester): all_model_classes = (RobertaForMaskedLM, RobertaModel) if is_torch_available() else () @@ -129,6 +128,7 @@ class RobertaModelTest(CommonTestCases.CommonModelTester): def create_and_check_roberta_model(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): model = RobertaModel(config=config) + model.to(torch_device) model.eval() sequence_output, pooled_output = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids) sequence_output, pooled_output = model(input_ids, token_type_ids=token_type_ids) @@ -146,6 +146,7 @@ class RobertaModelTest(CommonTestCases.CommonModelTester): def create_and_check_roberta_for_masked_lm(self, config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels): model = RobertaForMaskedLM(config=config) + model.to(torch_device) model.eval() loss, prediction_scores = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, masked_lm_labels=token_labels) result = { @@ -161,6 +162,7 @@ class RobertaModelTest(CommonTestCases.CommonModelTester): sequence_labels, token_labels, choice_labels): config.num_labels = self.num_labels model = RobertaForTokenClassification(config=config) + model.to(torch_device) model.eval() loss, logits = model(input_ids, attention_mask=input_mask, token_type_ids=token_type_ids, labels=token_labels) @@ -195,7 +197,7 @@ class RobertaModelTest(CommonTestCases.CommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_roberta_for_masked_lm(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): cache_dir = "/tmp/transformers_test/" for model_name in list(ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: @@ -207,10 +209,10 @@ class RobertaModelTest(CommonTestCases.CommonModelTester): class RobertaModelIntegrationTest(unittest.TestCase): - @pytest.mark.slow + @slow def test_inference_masked_lm(self): model = RobertaForMaskedLM.from_pretrained('roberta-base') - + input_ids = torch.tensor([[ 0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]]) output = model(input_ids)[0] expected_shape = torch.Size((1, 11, 50265)) @@ -228,10 +230,10 @@ class RobertaModelIntegrationTest(unittest.TestCase): torch.allclose(output[:, :3, :3], expected_slice, atol=1e-3) ) - @pytest.mark.slow + @slow def test_inference_no_head(self): model = RobertaModel.from_pretrained('roberta-base') - + input_ids = torch.tensor([[ 0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]]) output = model(input_ids)[0] # compare the actual values for a slice. @@ -244,10 +246,10 @@ class RobertaModelIntegrationTest(unittest.TestCase): torch.allclose(output[:, :3, :3], expected_slice, atol=1e-3) ) - @pytest.mark.slow + @slow def test_inference_classification_head(self): model = RobertaForSequenceClassification.from_pretrained('roberta-large-mnli') - + input_ids = torch.tensor([[ 0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]]) output = model(input_ids)[0] expected_shape = torch.Size((1, 3)) diff --git a/transformers/tests/modeling_tf_albert_test.py b/transformers/tests/modeling_tf_albert_test.py index fbd519b8f6..7d3325b70b 100644 --- a/transformers/tests/modeling_tf_albert_test.py +++ b/transformers/tests/modeling_tf_albert_test.py @@ -18,11 +18,11 @@ from __future__ import print_function import unittest import shutil -import pytest import sys from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import require_tf, slow from transformers import AlbertConfig, is_tf_available @@ -31,10 +31,9 @@ if is_tf_available(): from transformers.modeling_tf_albert import (TFAlbertModel, TFAlbertForMaskedLM, TFAlbertForSequenceClassification, TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) -else: - pytestmark = pytest.mark.skip("Require TensorFlow") +@require_tf class TFAlbertModelTest(TFCommonTestCases.TFCommonModelTester): all_model_classes = ( @@ -216,7 +215,7 @@ class TFAlbertModelTest(TFCommonTestCases.TFCommonModelTester): self.model_tester.create_and_check_albert_for_sequence_classification( *config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): cache_dir = "/tmp/transformers_test/" # for model_name in list(TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: diff --git a/transformers/tests/modeling_tf_auto_test.py b/transformers/tests/modeling_tf_auto_test.py index fa90906e86..7ea48015d9 100644 --- a/transformers/tests/modeling_tf_auto_test.py +++ b/transformers/tests/modeling_tf_auto_test.py @@ -18,11 +18,12 @@ from __future__ import print_function import unittest import shutil -import pytest import logging from transformers import is_tf_available +from .utils import require_tf, slow + if is_tf_available(): from transformers import (AutoConfig, BertConfig, TFAutoModel, TFBertModel, @@ -33,12 +34,11 @@ if is_tf_available(): from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester -else: - pytestmark = pytest.mark.skip("Require TensorFlow") +@require_tf class TFAutoModelTest(unittest.TestCase): - @pytest.mark.slow + @slow def test_model_from_pretrained(self): import h5py self.assertTrue(h5py.version.hdf5_version.startswith("1.10")) @@ -54,7 +54,7 @@ class TFAutoModelTest(unittest.TestCase): self.assertIsNotNone(model) self.assertIsInstance(model, TFBertModel) - @pytest.mark.slow + @slow def test_lmhead_model_from_pretrained(self): logging.basicConfig(level=logging.INFO) # for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: @@ -67,7 +67,7 @@ class TFAutoModelTest(unittest.TestCase): self.assertIsNotNone(model) self.assertIsInstance(model, TFBertForMaskedLM) - @pytest.mark.slow + @slow def test_sequence_classification_model_from_pretrained(self): logging.basicConfig(level=logging.INFO) # for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: @@ -80,7 +80,7 @@ class TFAutoModelTest(unittest.TestCase): self.assertIsNotNone(model) self.assertIsInstance(model, TFBertForSequenceClassification) - @pytest.mark.slow + @slow def test_question_answering_model_from_pretrained(self): logging.basicConfig(level=logging.INFO) # for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: diff --git a/transformers/tests/modeling_tf_bert_test.py b/transformers/tests/modeling_tf_bert_test.py index bcee97435e..d7a86fecb9 100644 --- a/transformers/tests/modeling_tf_bert_test.py +++ b/transformers/tests/modeling_tf_bert_test.py @@ -18,11 +18,11 @@ from __future__ import print_function import unittest import shutil -import pytest import sys from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import require_tf, slow from transformers import BertConfig, is_tf_available @@ -36,10 +36,9 @@ if is_tf_available(): TFBertForTokenClassification, TFBertForQuestionAnswering, TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP) -else: - pytestmark = pytest.mark.skip("Require TensorFlow") +@require_tf class TFBertModelTest(TFCommonTestCases.TFCommonModelTester): all_model_classes = (TFBertModel, TFBertForMaskedLM, TFBertForNextSentencePrediction, @@ -309,7 +308,7 @@ class TFBertModelTest(TFCommonTestCases.TFCommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_bert_for_token_classification(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): cache_dir = "/tmp/transformers_test/" # for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: diff --git a/transformers/tests/modeling_tf_common_test.py b/transformers/tests/modeling_tf_common_test.py index 7445ce826a..439360ba35 100644 --- a/transformers/tests/modeling_tf_common_test.py +++ b/transformers/tests/modeling_tf_common_test.py @@ -25,18 +25,17 @@ import unittest import uuid import tempfile -import pytest import sys from transformers import is_tf_available, is_torch_available +from .utils import require_tf, slow + if is_tf_available(): import tensorflow as tf import numpy as np from transformers import TFPreTrainedModel # from transformers.modeling_bert import BertModel, BertConfig, BERT_PRETRAINED_MODEL_ARCHIVE_MAP -else: - pytestmark = pytest.mark.skip("Require TensorFlow") if sys.version_info[0] == 2: import cPickle as pickle @@ -62,6 +61,7 @@ def _config_zero_init(config): class TFCommonTestCases: + @require_tf class TFCommonModelTester(unittest.TestCase): model_tester = None @@ -164,7 +164,7 @@ class TFCommonTestCases: for model_class in self.all_model_classes: # Prepare our model model = model_class(config) - + # Let's load it from the disk to be sure we can use pretrained weights with TemporaryDirectory() as tmpdirname: outputs = model(inputs_dict) # build the model diff --git a/transformers/tests/modeling_tf_ctrl_test.py b/transformers/tests/modeling_tf_ctrl_test.py index a57c882169..0b421c20c9 100644 --- a/transformers/tests/modeling_tf_ctrl_test.py +++ b/transformers/tests/modeling_tf_ctrl_test.py @@ -18,11 +18,11 @@ from __future__ import print_function import unittest import shutil -import pytest import sys from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import require_tf, slow from transformers import CTRLConfig, is_tf_available @@ -30,10 +30,9 @@ if is_tf_available(): import tensorflow as tf from transformers.modeling_tf_ctrl import (TFCTRLModel, TFCTRLLMHeadModel, TF_CTRL_PRETRAINED_MODEL_ARCHIVE_MAP) -else: - pytestmark = pytest.mark.skip("Require TensorFlow") +@require_tf class TFCTRLModelTest(TFCommonTestCases.TFCommonModelTester): all_model_classes = (TFCTRLModel, TFCTRLLMHeadModel) if is_tf_available() else () @@ -188,7 +187,7 @@ class TFCTRLModelTest(TFCommonTestCases.TFCommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_ctrl_lm_head(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): cache_dir = "/tmp/transformers_test/" for model_name in list(TF_CTRL_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: diff --git a/transformers/tests/modeling_tf_distilbert_test.py b/transformers/tests/modeling_tf_distilbert_test.py index e6d3795914..0ec45150ca 100644 --- a/transformers/tests/modeling_tf_distilbert_test.py +++ b/transformers/tests/modeling_tf_distilbert_test.py @@ -17,10 +17,10 @@ from __future__ import division from __future__ import print_function import unittest -import pytest from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import require_tf, slow from transformers import DistilBertConfig, is_tf_available @@ -30,10 +30,9 @@ if is_tf_available(): TFDistilBertForMaskedLM, TFDistilBertForQuestionAnswering, TFDistilBertForSequenceClassification) -else: - pytestmark = pytest.mark.skip("Require TensorFlow") +@require_tf class TFDistilBertModelTest(TFCommonTestCases.TFCommonModelTester): all_model_classes = (TFDistilBertModel, TFDistilBertForMaskedLM, TFDistilBertForQuestionAnswering, @@ -210,7 +209,7 @@ class TFDistilBertModelTest(TFCommonTestCases.TFCommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_distilbert_for_sequence_classification(*config_and_inputs) - # @pytest.mark.slow + # @slow # def test_model_from_pretrained(self): # cache_dir = "/tmp/transformers_test/" # for model_name in list(DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: diff --git a/transformers/tests/modeling_tf_gpt2_test.py b/transformers/tests/modeling_tf_gpt2_test.py index 76e9ee2298..e070b72e65 100644 --- a/transformers/tests/modeling_tf_gpt2_test.py +++ b/transformers/tests/modeling_tf_gpt2_test.py @@ -18,11 +18,11 @@ from __future__ import print_function import unittest import shutil -import pytest import sys from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import require_tf, slow from transformers import GPT2Config, is_tf_available @@ -31,10 +31,9 @@ if is_tf_available(): from transformers.modeling_tf_gpt2 import (TFGPT2Model, TFGPT2LMHeadModel, TFGPT2DoubleHeadsModel, TF_GPT2_PRETRAINED_MODEL_ARCHIVE_MAP) -else: - pytestmark = pytest.mark.skip("Require TensorFlow") +@require_tf class TFGPT2ModelTest(TFCommonTestCases.TFCommonModelTester): all_model_classes = (TFGPT2Model, TFGPT2LMHeadModel, @@ -219,7 +218,7 @@ class TFGPT2ModelTest(TFCommonTestCases.TFCommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_gpt2_double_head(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): cache_dir = "/tmp/transformers_test/" for model_name in list(TF_GPT2_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: diff --git a/transformers/tests/modeling_tf_openai_gpt_test.py b/transformers/tests/modeling_tf_openai_gpt_test.py index d470c8862d..675e806c12 100644 --- a/transformers/tests/modeling_tf_openai_gpt_test.py +++ b/transformers/tests/modeling_tf_openai_gpt_test.py @@ -18,11 +18,11 @@ from __future__ import print_function import unittest import shutil -import pytest import sys from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import require_tf, slow from transformers import OpenAIGPTConfig, is_tf_available @@ -31,10 +31,9 @@ if is_tf_available(): from transformers.modeling_tf_openai import (TFOpenAIGPTModel, TFOpenAIGPTLMHeadModel, TFOpenAIGPTDoubleHeadsModel, TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP) -else: - pytestmark = pytest.mark.skip("Require TensorFlow") +@require_tf class TFOpenAIGPTModelTest(TFCommonTestCases.TFCommonModelTester): all_model_classes = (TFOpenAIGPTModel, TFOpenAIGPTLMHeadModel, @@ -218,7 +217,7 @@ class TFOpenAIGPTModelTest(TFCommonTestCases.TFCommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_openai_gpt_double_head(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): cache_dir = "/tmp/transformers_test/" for model_name in list(TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: diff --git a/transformers/tests/modeling_tf_roberta_test.py b/transformers/tests/modeling_tf_roberta_test.py index edbfa4e205..42440bf1b7 100644 --- a/transformers/tests/modeling_tf_roberta_test.py +++ b/transformers/tests/modeling_tf_roberta_test.py @@ -18,10 +18,10 @@ from __future__ import print_function import unittest import shutil -import pytest from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import require_tf, slow from transformers import RobertaConfig, is_tf_available @@ -32,10 +32,9 @@ if is_tf_available(): TFRobertaForSequenceClassification, TFRobertaForTokenClassification, TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP) -else: - pytestmark = pytest.mark.skip("Require TensorFlow") +@require_tf class TFRobertaModelTest(TFCommonTestCases.TFCommonModelTester): all_model_classes = (TFRobertaModel,TFRobertaForMaskedLM, @@ -191,7 +190,7 @@ class TFRobertaModelTest(TFCommonTestCases.TFCommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_roberta_for_masked_lm(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): cache_dir = "/tmp/transformers_test/" for model_name in list(TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: @@ -203,10 +202,10 @@ class TFRobertaModelTest(TFCommonTestCases.TFCommonModelTester): class TFRobertaModelIntegrationTest(unittest.TestCase): - @pytest.mark.slow + @slow def test_inference_masked_lm(self): model = TFRobertaForMaskedLM.from_pretrained('roberta-base') - + input_ids = tf.constant([[ 0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]]) output = model(input_ids)[0] expected_shape = [1, 11, 50265] @@ -224,10 +223,10 @@ class TFRobertaModelIntegrationTest(unittest.TestCase): numpy.allclose(output[:, :3, :3].numpy(), expected_slice.numpy(), atol=1e-3) ) - @pytest.mark.slow + @slow def test_inference_no_head(self): model = TFRobertaModel.from_pretrained('roberta-base') - + input_ids = tf.constant([[ 0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]]) output = model(input_ids)[0] # compare the actual values for a slice. @@ -240,10 +239,10 @@ class TFRobertaModelIntegrationTest(unittest.TestCase): numpy.allclose(output[:, :3, :3].numpy(), expected_slice.numpy(), atol=1e-3) ) - @pytest.mark.slow + @slow def test_inference_classification_head(self): model = TFRobertaForSequenceClassification.from_pretrained('roberta-large-mnli') - + input_ids = tf.constant([[ 0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2]]) output = model(input_ids)[0] expected_shape = [1, 3] diff --git a/transformers/tests/modeling_tf_transfo_xl_test.py b/transformers/tests/modeling_tf_transfo_xl_test.py index 534fe39646..03e332bdc1 100644 --- a/transformers/tests/modeling_tf_transfo_xl_test.py +++ b/transformers/tests/modeling_tf_transfo_xl_test.py @@ -19,10 +19,10 @@ from __future__ import print_function import unittest import random import shutil -import pytest from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import require_tf, slow from transformers import TransfoXLConfig, is_tf_available @@ -31,10 +31,9 @@ if is_tf_available(): from transformers.modeling_tf_transfo_xl import (TFTransfoXLModel, TFTransfoXLLMHeadModel, TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP) -else: - pytestmark = pytest.mark.skip("Require TensorFlow") +@require_tf class TFTransfoXLModelTest(TFCommonTestCases.TFCommonModelTester): all_model_classes = (TFTransfoXLModel, TFTransfoXLLMHeadModel) if is_tf_available() else () @@ -204,7 +203,7 @@ class TFTransfoXLModelTest(TFCommonTestCases.TFCommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_transfo_xl_lm_head(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): cache_dir = "/tmp/transformers_test/" for model_name in list(TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: diff --git a/transformers/tests/modeling_tf_xlm_test.py b/transformers/tests/modeling_tf_xlm_test.py index 1bd661bebf..a680b70367 100644 --- a/transformers/tests/modeling_tf_xlm_test.py +++ b/transformers/tests/modeling_tf_xlm_test.py @@ -18,7 +18,6 @@ from __future__ import print_function import unittest import shutil -import pytest from transformers import is_tf_available @@ -29,13 +28,13 @@ if is_tf_available(): TFXLMForSequenceClassification, TFXLMForQuestionAnsweringSimple, TF_XLM_PRETRAINED_MODEL_ARCHIVE_MAP) -else: - pytestmark = pytest.mark.skip("Require TensorFlow") from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import require_tf, slow +@require_tf class TFXLMModelTest(TFCommonTestCases.TFCommonModelTester): all_model_classes = (TFXLMModel, TFXLMWithLMHeadModel, @@ -251,7 +250,7 @@ class TFXLMModelTest(TFCommonTestCases.TFCommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_xlm_sequence_classif(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): cache_dir = "/tmp/transformers_test/" for model_name in list(TF_XLM_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: diff --git a/transformers/tests/modeling_tf_xlnet_test.py b/transformers/tests/modeling_tf_xlnet_test.py index a00a965570..94864b86f2 100644 --- a/transformers/tests/modeling_tf_xlnet_test.py +++ b/transformers/tests/modeling_tf_xlnet_test.py @@ -21,7 +21,6 @@ import unittest import json import random import shutil -import pytest from transformers import XLNetConfig, is_tf_available @@ -33,12 +32,13 @@ if is_tf_available(): TFXLNetForTokenClassification, TFXLNetForQuestionAnsweringSimple, TF_XLNET_PRETRAINED_MODEL_ARCHIVE_MAP) -else: - pytestmark = pytest.mark.skip("Require TensorFlow") from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import require_tf, slow + +@require_tf class TFXLNetModelTest(TFCommonTestCases.TFCommonModelTester): all_model_classes=(TFXLNetModel, TFXLNetLMHeadModel, @@ -304,7 +304,7 @@ class TFXLNetModelTest(TFCommonTestCases.TFCommonModelTester): def test_xlnet_lm_head(self): self.model_tester.set_seed() config_and_inputs = self.model_tester.prepare_config_and_inputs() - self.model_tester.create_and_check_xlnet_lm_head(*config_and_inputs) + self.model_tester.create_and_check_xlnet_lm_head(*config_and_inputs) def test_xlnet_sequence_classif(self): self.model_tester.set_seed() @@ -320,7 +320,7 @@ class TFXLNetModelTest(TFCommonTestCases.TFCommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_xlnet_qa(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): cache_dir = "/tmp/transformers_test/" for model_name in list(TF_XLNET_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: diff --git a/transformers/tests/modeling_transfo_xl_test.py b/transformers/tests/modeling_transfo_xl_test.py index f7b913da5b..647dd3724d 100644 --- a/transformers/tests/modeling_transfo_xl_test.py +++ b/transformers/tests/modeling_transfo_xl_test.py @@ -19,7 +19,6 @@ from __future__ import print_function import unittest import random import shutil -import pytest from transformers import is_torch_available @@ -27,12 +26,13 @@ if is_torch_available(): import torch from transformers import (TransfoXLConfig, TransfoXLModel, TransfoXLLMHeadModel) from transformers.modeling_transfo_xl import TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP -else: - pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import require_torch, slow, torch_device + +@require_torch class TransfoXLModelTest(CommonTestCases.CommonModelTester): all_model_classes = (TransfoXLModel, TransfoXLLMHeadModel) if is_torch_available() else () @@ -111,6 +111,7 @@ class TransfoXLModelTest(CommonTestCases.CommonModelTester): def create_transfo_xl_model(self, config, input_ids_1, input_ids_2, lm_labels): model = TransfoXLModel(config) + model.to(torch_device) model.eval() hidden_states_1, mems_1 = model(input_ids_1) @@ -140,6 +141,7 @@ class TransfoXLModelTest(CommonTestCases.CommonModelTester): def create_transfo_xl_lm_head(self, config, input_ids_1, input_ids_2, lm_labels): model = TransfoXLLMHeadModel(config) + model.to(torch_device) model.eval() lm_logits_1, mems_1 = model(input_ids_1) @@ -204,7 +206,7 @@ class TransfoXLModelTest(CommonTestCases.CommonModelTester): output_result = self.model_tester.create_transfo_xl_lm_head(*config_and_inputs) self.model_tester.check_transfo_xl_lm_head_output(output_result) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): cache_dir = "/tmp/transformers_test/" for model_name in list(TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: diff --git a/transformers/tests/modeling_xlm_test.py b/transformers/tests/modeling_xlm_test.py index 0133febb58..f6b980767c 100644 --- a/transformers/tests/modeling_xlm_test.py +++ b/transformers/tests/modeling_xlm_test.py @@ -18,7 +18,6 @@ from __future__ import print_function import unittest import shutil -import pytest from transformers import is_torch_available @@ -26,13 +25,13 @@ if is_torch_available(): from transformers import (XLMConfig, XLMModel, XLMWithLMHeadModel, XLMForQuestionAnswering, XLMForSequenceClassification, XLMForQuestionAnsweringSimple) from transformers.modeling_xlm import XLM_PRETRAINED_MODEL_ARCHIVE_MAP -else: - pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import require_torch, slow, torch_device +@require_torch class XLMModelTest(CommonTestCases.CommonModelTester): all_model_classes = (XLMModel, XLMWithLMHeadModel, XLMForQuestionAnswering, @@ -148,6 +147,7 @@ class XLMModelTest(CommonTestCases.CommonModelTester): def create_and_check_xlm_model(self, config, input_ids, token_type_ids, input_lengths, sequence_labels, token_labels, is_impossible_labels, input_mask): model = XLMModel(config=config) + model.to(torch_device) model.eval() outputs = model(input_ids, lengths=input_lengths, langs=token_type_ids) outputs = model(input_ids, langs=token_type_ids) @@ -163,6 +163,7 @@ class XLMModelTest(CommonTestCases.CommonModelTester): def create_and_check_xlm_lm_head(self, config, input_ids, token_type_ids, input_lengths, sequence_labels, token_labels, is_impossible_labels, input_mask): model = XLMWithLMHeadModel(config) + model.to(torch_device) model.eval() loss, logits = model(input_ids, token_type_ids=token_type_ids, labels=token_labels) @@ -182,6 +183,7 @@ class XLMModelTest(CommonTestCases.CommonModelTester): def create_and_check_xlm_simple_qa(self, config, input_ids, token_type_ids, input_lengths, sequence_labels, token_labels, is_impossible_labels, input_mask): model = XLMForQuestionAnsweringSimple(config) + model.to(torch_device) model.eval() outputs = model(input_ids) @@ -206,6 +208,7 @@ class XLMModelTest(CommonTestCases.CommonModelTester): def create_and_check_xlm_qa(self, config, input_ids, token_type_ids, input_lengths, sequence_labels, token_labels, is_impossible_labels, input_mask): model = XLMForQuestionAnswering(config) + model.to(torch_device) model.eval() outputs = model(input_ids) @@ -260,6 +263,7 @@ class XLMModelTest(CommonTestCases.CommonModelTester): def create_and_check_xlm_sequence_classif(self, config, input_ids, token_type_ids, input_lengths, sequence_labels, token_labels, is_impossible_labels, input_mask): model = XLMForSequenceClassification(config) + model.to(torch_device) model.eval() (logits,) = model(input_ids) @@ -312,7 +316,7 @@ class XLMModelTest(CommonTestCases.CommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_xlm_sequence_classif(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): cache_dir = "/tmp/transformers_test/" for model_name in list(XLM_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: diff --git a/transformers/tests/modeling_xlnet_test.py b/transformers/tests/modeling_xlnet_test.py index 38888d4488..56b6bb3f4d 100644 --- a/transformers/tests/modeling_xlnet_test.py +++ b/transformers/tests/modeling_xlnet_test.py @@ -21,7 +21,6 @@ import unittest import json import random import shutil -import pytest from transformers import is_torch_available @@ -31,12 +30,13 @@ if is_torch_available(): from transformers import (XLNetConfig, XLNetModel, XLNetLMHeadModel, XLNetForSequenceClassification, XLNetForTokenClassification, XLNetForQuestionAnswering) from transformers.modeling_xlnet import XLNET_PRETRAINED_MODEL_ARCHIVE_MAP -else: - pytestmark = pytest.mark.skip("Require Torch") from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import require_torch, slow, torch_device + +@require_torch class XLNetModelTest(CommonTestCases.CommonModelTester): all_model_classes=(XLNetModel, XLNetLMHeadModel, XLNetForTokenClassification, @@ -100,9 +100,9 @@ class XLNetModelTest(CommonTestCases.CommonModelTester): input_mask = ids_tensor([self.batch_size, self.seq_length], 2).float() input_ids_q = ids_tensor([self.batch_size, self.seq_length + 1], self.vocab_size) - perm_mask = torch.zeros(self.batch_size, self.seq_length + 1, self.seq_length + 1, dtype=torch.float) + perm_mask = torch.zeros(self.batch_size, self.seq_length + 1, self.seq_length + 1, dtype=torch.float, device=torch_device) perm_mask[:, :, -1] = 1.0 # Previous tokens don't see last token - target_mapping = torch.zeros(self.batch_size, 1, self.seq_length + 1, dtype=torch.float) + target_mapping = torch.zeros(self.batch_size, 1, self.seq_length + 1, dtype=torch.float, device=torch_device) target_mapping[:, 0, -1] = 1.0 # predict last token sequence_labels = None @@ -141,6 +141,7 @@ class XLNetModelTest(CommonTestCases.CommonModelTester): def create_and_check_xlnet_base_model(self, config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels, token_labels): model = XLNetModel(config) + model.to(torch_device) model.eval() _, _ = model(input_ids_1, input_mask=input_mask) @@ -155,6 +156,7 @@ class XLNetModelTest(CommonTestCases.CommonModelTester): config.mem_len = 0 model = XLNetModel(config) + model.to(torch_device) model.eval() no_mems_outputs = model(input_ids_1) self.parent.assertEqual(len(no_mems_outputs), 1) @@ -169,6 +171,7 @@ class XLNetModelTest(CommonTestCases.CommonModelTester): def create_and_check_xlnet_base_model_with_att_output(self, config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels, token_labels): model = XLNetModel(config) + model.to(torch_device) model.eval() _, _, attentions = model(input_ids_1, target_mapping=target_mapping) @@ -181,6 +184,7 @@ class XLNetModelTest(CommonTestCases.CommonModelTester): def create_and_check_xlnet_lm_head(self, config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels, token_labels): model = XLNetLMHeadModel(config) + model.to(torch_device) model.eval() loss_1, all_logits_1, mems_1 = model(input_ids_1, token_type_ids=segment_ids, labels=lm_labels) @@ -221,6 +225,7 @@ class XLNetModelTest(CommonTestCases.CommonModelTester): def create_and_check_xlnet_qa(self, config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels, token_labels): model = XLNetForQuestionAnswering(config) + model.to(torch_device) model.eval() outputs = model(input_ids_1) @@ -279,6 +284,7 @@ class XLNetModelTest(CommonTestCases.CommonModelTester): def create_and_check_xlnet_token_classif(self, config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels, token_labels): model = XLNetForTokenClassification(config) + model.to(torch_device) model.eval() logits, mems_1 = model(input_ids_1) @@ -311,6 +317,7 @@ class XLNetModelTest(CommonTestCases.CommonModelTester): def create_and_check_xlnet_sequence_classif(self, config, input_ids_1, input_ids_2, input_ids_q, perm_mask, input_mask, target_mapping, segment_ids, lm_labels, sequence_labels, is_impossible_labels, token_labels): model = XLNetForSequenceClassification(config) + model.to(torch_device) model.eval() logits, mems_1 = model(input_ids_1) @@ -362,7 +369,7 @@ class XLNetModelTest(CommonTestCases.CommonModelTester): def test_xlnet_lm_head(self): self.model_tester.set_seed() config_and_inputs = self.model_tester.prepare_config_and_inputs() - self.model_tester.create_and_check_xlnet_lm_head(*config_and_inputs) + self.model_tester.create_and_check_xlnet_lm_head(*config_and_inputs) def test_xlnet_sequence_classif(self): self.model_tester.set_seed() @@ -379,7 +386,7 @@ class XLNetModelTest(CommonTestCases.CommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_xlnet_qa(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): cache_dir = "/tmp/transformers_test/" for model_name in list(XLNET_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: diff --git a/transformers/tests/optimization_test.py b/transformers/tests/optimization_test.py index ab9afbfcf7..cc10ad5908 100644 --- a/transformers/tests/optimization_test.py +++ b/transformers/tests/optimization_test.py @@ -18,7 +18,6 @@ from __future__ import print_function import unittest import os -import pytest from transformers import is_torch_available @@ -31,10 +30,9 @@ if is_torch_available(): get_cosine_schedule_with_warmup, get_cosine_with_hard_restarts_schedule_with_warmup, get_linear_schedule_with_warmup) -else: - pytestmark = pytest.mark.skip("Require Torch") from .tokenization_tests_commons import TemporaryDirectory +from .utils import require_torch def unwrap_schedule(scheduler, num_steps=10): @@ -58,6 +56,7 @@ def unwrap_and_save_reload_schedule(scheduler, num_steps=10): scheduler.load_state_dict(state_dict) return lrs +@require_torch class OptimizationTest(unittest.TestCase): def assertListAlmostEqual(self, list1, list2, tol): @@ -80,6 +79,7 @@ class OptimizationTest(unittest.TestCase): self.assertListAlmostEqual(w.tolist(), [0.4, 0.2, -0.5], tol=1e-2) +@require_torch class ScheduleInitTest(unittest.TestCase): m = torch.nn.Linear(50, 50) if is_torch_available() else None optimizer = AdamW(m.parameters(), lr=10.) if is_torch_available() else None diff --git a/transformers/tests/tokenization_auto_test.py b/transformers/tests/tokenization_auto_test.py index 79370811e8..18346d2768 100644 --- a/transformers/tests/tokenization_auto_test.py +++ b/transformers/tests/tokenization_auto_test.py @@ -18,15 +18,16 @@ from __future__ import print_function import unittest import shutil -import pytest import logging from transformers import AutoTokenizer, BertTokenizer, AutoTokenizer, GPT2Tokenizer from transformers import BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP +from .utils import slow + class AutoTokenizerTest(unittest.TestCase): - @pytest.mark.slow + @slow def test_tokenizer_from_pretrained(self): logging.basicConfig(level=logging.INFO) for model_name in list(BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys())[:1]: diff --git a/transformers/tests/tokenization_bert_test.py b/transformers/tests/tokenization_bert_test.py index 73ea38e20a..f390248956 100644 --- a/transformers/tests/tokenization_bert_test.py +++ b/transformers/tests/tokenization_bert_test.py @@ -16,7 +16,6 @@ from __future__ import absolute_import, division, print_function, unicode_litera import os import unittest -import pytest from io import open from transformers.tokenization_bert import (BasicTokenizer, @@ -26,6 +25,7 @@ from transformers.tokenization_bert import (BasicTokenizer, _is_whitespace, VOCAB_FILES_NAMES) from .tokenization_tests_commons import CommonTestCases +from .utils import slow class BertTokenizationTest(CommonTestCases.CommonTokenizerTester): @@ -126,7 +126,7 @@ class BertTokenizationTest(CommonTestCases.CommonTokenizerTester): self.assertFalse(_is_punctuation(u"A")) self.assertFalse(_is_punctuation(u" ")) - @pytest.mark.slow + @slow def test_sequence_builders(self): tokenizer = self.tokenizer_class.from_pretrained("bert-base-uncased") diff --git a/transformers/tests/tokenization_distilbert_test.py b/transformers/tests/tokenization_distilbert_test.py index 77a487651d..e815eca672 100644 --- a/transformers/tests/tokenization_distilbert_test.py +++ b/transformers/tests/tokenization_distilbert_test.py @@ -16,13 +16,13 @@ from __future__ import absolute_import, division, print_function, unicode_litera import os import unittest -import pytest from io import open from transformers.tokenization_distilbert import (DistilBertTokenizer) from .tokenization_tests_commons import CommonTestCases from .tokenization_bert_test import BertTokenizationTest +from .utils import slow class DistilBertTokenizationTest(BertTokenizationTest): @@ -31,7 +31,7 @@ class DistilBertTokenizationTest(BertTokenizationTest): def get_tokenizer(self, **kwargs): return DistilBertTokenizer.from_pretrained(self.tmpdirname, **kwargs) - @pytest.mark.slow + @slow def test_sequence_builders(self): tokenizer = DistilBertTokenizer.from_pretrained("distilbert-base-uncased") diff --git a/transformers/tests/tokenization_roberta_test.py b/transformers/tests/tokenization_roberta_test.py index a27bf7d654..8ad0b59511 100644 --- a/transformers/tests/tokenization_roberta_test.py +++ b/transformers/tests/tokenization_roberta_test.py @@ -17,11 +17,11 @@ from __future__ import absolute_import, division, print_function, unicode_litera import os import json import unittest -import pytest from io import open from transformers.tokenization_roberta import RobertaTokenizer, VOCAB_FILES_NAMES from .tokenization_tests_commons import CommonTestCases +from .utils import slow class RobertaTokenizationTest(CommonTestCases.CommonTokenizerTester): @@ -79,7 +79,7 @@ class RobertaTokenizationTest(CommonTestCases.CommonTokenizerTester): [0, 31414, 232, 328, 740, 1140, 12695, 69, 46078, 1588, 2] ) - @pytest.mark.slow + @slow def test_sequence_builders(self): tokenizer = RobertaTokenizer.from_pretrained("roberta-base") diff --git a/transformers/tests/tokenization_tests_commons.py b/transformers/tests/tokenization_tests_commons.py index 97cd555df3..faff003f4b 100644 --- a/transformers/tests/tokenization_tests_commons.py +++ b/transformers/tests/tokenization_tests_commons.py @@ -102,9 +102,11 @@ class CommonTestCases: with TemporaryDirectory() as tmpdirname: filename = os.path.join(tmpdirname, u"tokenizer.bin") - pickle.dump(tokenizer, open(filename, "wb")) + with open(filename, "wb") as handle: + pickle.dump(tokenizer, handle) - tokenizer_new = pickle.load(open(filename, "rb")) + with open(filename, "rb") as handle: + tokenizer_new = pickle.load(handle) subwords_loaded = tokenizer_new.tokenize(text) diff --git a/transformers/tests/tokenization_transfo_xl_test.py b/transformers/tests/tokenization_transfo_xl_test.py index 4e99484b0c..5495ebd3a6 100644 --- a/transformers/tests/tokenization_transfo_xl_test.py +++ b/transformers/tests/tokenization_transfo_xl_test.py @@ -16,7 +16,6 @@ from __future__ import absolute_import, division, print_function, unicode_litera import os import unittest -import pytest from io import open from transformers import is_torch_available @@ -24,11 +23,12 @@ from transformers import is_torch_available if is_torch_available(): import torch from transformers.tokenization_transfo_xl import TransfoXLTokenizer, VOCAB_FILES_NAMES -else: - pytestmark = pytest.mark.skip("Require Torch") # TODO: untangle Transfo-XL tokenizer from torch.load and torch.save from .tokenization_tests_commons import CommonTestCases +from .utils import require_torch + +@require_torch class TransfoXLTokenizationTest(CommonTestCases.CommonTokenizerTester): tokenizer_class = TransfoXLTokenizer if is_torch_available() else None diff --git a/transformers/tests/tokenization_utils_test.py b/transformers/tests/tokenization_utils_test.py index 8630191c69..ff3f80ff7d 100644 --- a/transformers/tests/tokenization_utils_test.py +++ b/transformers/tests/tokenization_utils_test.py @@ -18,13 +18,14 @@ from __future__ import print_function import unittest import six -import pytest from transformers import PreTrainedTokenizer from transformers.tokenization_gpt2 import GPT2Tokenizer +from .utils import slow + class TokenizerUtilsTest(unittest.TestCase): - @pytest.mark.slow + def check_tokenizer_from_pretrained(self, tokenizer_class): s3_models = list(tokenizer_class.max_model_input_sizes.keys()) for model_name in s3_models[:1]: @@ -41,6 +42,7 @@ class TokenizerUtilsTest(unittest.TestCase): special_tok_id = tokenizer.convert_tokens_to_ids(special_tok) self.assertIsInstance(special_tok_id, int) + @slow def test_pretrained_tokenizers(self): self.check_tokenizer_from_pretrained(GPT2Tokenizer) diff --git a/transformers/tests/tokenization_xlm_test.py b/transformers/tests/tokenization_xlm_test.py index 3ff6564e34..7582a46662 100644 --- a/transformers/tests/tokenization_xlm_test.py +++ b/transformers/tests/tokenization_xlm_test.py @@ -17,11 +17,11 @@ from __future__ import absolute_import, division, print_function, unicode_litera import os import unittest import json -import pytest from transformers.tokenization_xlm import XLMTokenizer, VOCAB_FILES_NAMES from .tokenization_tests_commons import CommonTestCases +from .utils import slow class XLMTokenizationTest(CommonTestCases.CommonTokenizerTester): @@ -67,7 +67,7 @@ class XLMTokenizationTest(CommonTestCases.CommonTokenizerTester): self.assertListEqual( tokenizer.convert_tokens_to_ids(input_tokens), input_bpe_tokens) - @pytest.mark.slow + @slow def test_sequence_builders(self): tokenizer = XLMTokenizer.from_pretrained("xlm-mlm-en-2048") diff --git a/transformers/tests/tokenization_xlnet_test.py b/transformers/tests/tokenization_xlnet_test.py index 2e14ffeb82..b68495a796 100644 --- a/transformers/tests/tokenization_xlnet_test.py +++ b/transformers/tests/tokenization_xlnet_test.py @@ -16,11 +16,11 @@ from __future__ import absolute_import, division, print_function, unicode_litera import os import unittest -import pytest from transformers.tokenization_xlnet import (XLNetTokenizer, SPIECE_UNDERLINE) from .tokenization_tests_commons import CommonTestCases +from .utils import slow SAMPLE_VOCAB = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'fixtures/test_sentencepiece.model') @@ -90,7 +90,7 @@ class XLNetTokenizationTest(CommonTestCases.CommonTokenizerTester): u'9', u'2', u'0', u'0', u'0', u',', SPIECE_UNDERLINE + u'and', SPIECE_UNDERLINE + u'this', SPIECE_UNDERLINE + u'is', SPIECE_UNDERLINE + u'f', u'al', u'se', u'.']) - @pytest.mark.slow + @slow def test_sequence_builders(self): tokenizer = XLNetTokenizer.from_pretrained("xlnet-base-cased") diff --git a/transformers/tests/utils.py b/transformers/tests/utils.py new file mode 100644 index 0000000000..7a51ab612b --- /dev/null +++ b/transformers/tests/utils.py @@ -0,0 +1,64 @@ +import os +import unittest + +from distutils.util import strtobool + +from transformers.file_utils import _tf_available, _torch_available + + +try: + run_slow = os.environ["RUN_SLOW"] +except KeyError: + # RUN_SLOW isn't set, default to skipping slow tests. + _run_slow_tests = False +else: + # RUN_SLOW is set, convert it to True or False. + try: + _run_slow_tests = strtobool(run_slow) + except ValueError: + # More values are supported, but let's keep the message simple. + raise ValueError("If set, RUN_SLOW must be yes or no.") + + +def slow(test_case): + """ + Decorator marking a test as slow. + + Slow tests are skipped by default. Set the RUN_SLOW environment variable + to a truthy value to run them. + + """ + if not _run_slow_tests: + test_case = unittest.skip("test is slow")(test_case) + return test_case + + +def require_torch(test_case): + """ + Decorator marking a test that requires PyTorch. + + These tests are skipped when PyTorch isn't installed. + + """ + if not _torch_available: + test_case = unittest.skip("test requires PyTorch")(test_case) + return test_case + + +def require_tf(test_case): + """ + Decorator marking a test that requires TensorFlow. + + These tests are skipped when TensorFlow isn't installed. + + """ + if not _tf_available: + test_case = unittest.skip("test requires TensorFlow")(test_case) + return test_case + + +if _torch_available: + # Set the USE_CUDA environment variable to select a GPU. + torch_device = "cuda" if os.environ.get("USE_CUDA") else "cpu" +else: + torch_device = None diff --git a/transformers/tokenization_albert.py b/transformers/tokenization_albert.py index 40a4b29206..6b92d07218 100644 --- a/transformers/tokenization_albert.py +++ b/transformers/tokenization_albert.py @@ -141,7 +141,7 @@ class AlbertTokenizer(PreTrainedTokenizer): pieces = self.sp_model.SampleEncodeAsPieces(text, 64, 0.1) new_pieces = [] for piece in pieces: - if len(piece) > 1 and piece[-1] == ',' and piece[-2].isdigit(): + if len(piece) > 1 and piece[-1] == str(',') and piece[-2].isdigit(): cur_pieces = self.sp_model.EncodeAsPieces( piece[:-1].replace(SPIECE_UNDERLINE, '')) if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE: @@ -225,9 +225,9 @@ class AlbertTokenizer(PreTrainedTokenizer): """ Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An ALBERT sequence pair mask has the following format: - 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 - | first sequence | second sequence - + 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 + | first sequence | second sequence + if token_ids_1 is None, only returns the first portion of the mask (0's). """ sep = [self.sep_token_id] diff --git a/transformers/tokenization_ctrl.py b/transformers/tokenization_ctrl.py index 9454cbbaf3..219f17c404 100644 --- a/transformers/tokenization_ctrl.py +++ b/transformers/tokenization_ctrl.py @@ -133,9 +133,11 @@ class CTRLTokenizer(PreTrainedTokenizer): self.max_len_single_sentence = self.max_len # no default special tokens - you can update this value if you add special tokens self.max_len_sentences_pair = self.max_len # no default special tokens - you can update this value if you add special tokens - self.encoder = json.load(open(vocab_file, encoding="utf-8")) + with open(vocab_file, encoding="utf-8") as vocab_handle: + self.encoder = json.load(vocab_handle) self.decoder = {v:k for k,v in self.encoder.items()} - merges = open(merges_file, encoding='utf-8').read().split('\n')[1:-1] + with open(merges_file, encoding='utf-8') as merges_handle: + merges = merges_handle.read().split('\n')[1:-1] merges = [tuple(merge.split()) for merge in merges] self.bpe_ranks = dict(zip(merges, range(len(merges)))) self.cache = {} diff --git a/transformers/tokenization_gpt2.py b/transformers/tokenization_gpt2.py index 5fda709448..68c6101860 100644 --- a/transformers/tokenization_gpt2.py +++ b/transformers/tokenization_gpt2.py @@ -72,7 +72,7 @@ def bytes_to_unicode(): """ Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping to whitespace/control characters the bpe code barfs on. - + The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for decent coverage. @@ -122,13 +122,15 @@ class GPT2Tokenizer(PreTrainedTokenizer): self.max_len_single_sentence = self.max_len # no default special tokens - you can update this value if you add special tokens self.max_len_sentences_pair = self.max_len # no default special tokens - you can update this value if you add special tokens - self.encoder = json.load(open(vocab_file, encoding="utf-8")) + with open(vocab_file, encoding="utf-8") as vocab_handle: + self.encoder = json.load(vocab_handle) self.decoder = {v: k for k, v in self.encoder.items()} self.errors = errors # how to handle errors in decoding self.byte_encoder = bytes_to_unicode() self.byte_decoder = {v: k for k, v in self.byte_encoder.items()} - bpe_data = open(merges_file, encoding='utf-8').read().split('\n')[1:-1] - bpe_merges = [tuple(merge.split()) for merge in bpe_data] + with open(merges_file, encoding='utf-8') as merges_handle: + bpe_merges = merges_handle.read().split('\n')[1:-1] + bpe_merges = [tuple(merge.split()) for merge in bpe_merges] self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges)))) self.cache = {} @@ -234,4 +236,4 @@ class GPT2Tokenizer(PreTrainedTokenizer): writer.write(' '.join(bpe_tokens) + u'\n') index += 1 - return vocab_file, merge_file \ No newline at end of file + return vocab_file, merge_file diff --git a/transformers/tokenization_openai.py b/transformers/tokenization_openai.py index 0efbdb37c0..a4c64b7020 100644 --- a/transformers/tokenization_openai.py +++ b/transformers/tokenization_openai.py @@ -101,9 +101,11 @@ class OpenAIGPTTokenizer(PreTrainedTokenizer): self.nlp = BasicTokenizer(do_lower_case=True) self.fix_text = None - self.encoder = json.load(open(vocab_file, encoding="utf-8")) + with open(vocab_file, encoding="utf-8") as vocab_handle: + self.encoder = json.load(vocab_handle) self.decoder = {v:k for k,v in self.encoder.items()} - merges = open(merges_file, encoding='utf-8').read().split('\n')[1:-1] + with open(merges_file, encoding='utf-8') as merges_handle: + merges = merges_handle.read().split('\n')[1:-1] merges = [tuple(merge.split()) for merge in merges] self.bpe_ranks = dict(zip(merges, range(len(merges)))) self.cache = {} diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index 5d683629f0..4c6cbd8986 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -347,7 +347,7 @@ class PreTrainedTokenizer(object): "We assumed '{}' was a path or url to a directory containing vocabulary files " "named {} but couldn't find such vocabulary files at this path or url.".format( pretrained_model_name_or_path, ', '.join(s3_models), - pretrained_model_name_or_path, + pretrained_model_name_or_path, list(cls.vocab_files_names.values()))) # Get files from url, cache, or disk depending on the case @@ -382,7 +382,8 @@ class PreTrainedTokenizer(object): # Did we saved some inputs and kwargs to reload ? tokenizer_config_file = resolved_vocab_files.pop('tokenizer_config_file', None) if tokenizer_config_file is not None: - init_kwargs = json.load(open(tokenizer_config_file, encoding="utf-8")) + with open(tokenizer_config_file, encoding="utf-8") as tokenizer_config_handle: + init_kwargs = json.load(tokenizer_config_handle) saved_init_inputs = init_kwargs.pop('init_inputs', ()) if not init_inputs: init_inputs = saved_init_inputs @@ -407,7 +408,8 @@ class PreTrainedTokenizer(object): if args_name not in init_kwargs: init_kwargs[args_name] = file_path if special_tokens_map_file is not None: - special_tokens_map = json.load(open(special_tokens_map_file, encoding="utf-8")) + with open(special_tokens_map_file, encoding="utf-8") as special_tokens_map_handle: + special_tokens_map = json.load(special_tokens_map_handle) for key, value in special_tokens_map.items(): if key not in init_kwargs: init_kwargs[key] = value @@ -421,7 +423,8 @@ class PreTrainedTokenizer(object): # Add supplementary tokens. if added_tokens_file is not None: - added_tok_encoder = json.load(open(added_tokens_file, encoding="utf-8")) + with open(added_tokens_file, encoding="utf-8") as added_tokens_handle: + added_tok_encoder = json.load(added_tokens_handle) added_tok_decoder = {v:k for k, v in added_tok_encoder.items()} tokenizer.added_tokens_encoder.update(added_tok_encoder) tokenizer.added_tokens_decoder.update(added_tok_decoder) @@ -937,7 +940,7 @@ class PreTrainedTokenizer(object): logger.warning("Token indices sequence length is longer than the specified maximum sequence length " "for this model ({} > {}). Running this sequence through the model will result in " "indexing errors".format(len(ids), self.max_len)) - + return encoded_inputs def truncate_sequences(self, ids, pair_ids=None, num_tokens_to_remove=0, truncation_strategy='longest_first', stride=0): diff --git a/transformers/tokenization_xlm.py b/transformers/tokenization_xlm.py index ba994dc356..6c9f8e5e5c 100644 --- a/transformers/tokenization_xlm.py +++ b/transformers/tokenization_xlm.py @@ -524,7 +524,7 @@ class XLMTokenizer(PreTrainedTokenizer): - argument ``special_tokens`` and function ``set_special_tokens``, can be used to add additional symbols \ (ex: "__classify__") to a vocabulary - + - `lang2id` attribute maps the languages supported by the model with their ids if provided (automatically set for pretrained vocabularies) - `id2lang` attributes does reverse mapping if provided (automatically set for pretrained vocabularies) @@ -564,9 +564,11 @@ class XLMTokenizer(PreTrainedTokenizer): self.ja_word_tokenizer = None self.zh_word_tokenizer = None - self.encoder = json.load(open(vocab_file, encoding="utf-8")) + with open(vocab_file, encoding="utf-8") as vocab_handle: + self.encoder = json.load(vocab_handle) self.decoder = {v:k for k,v in self.encoder.items()} - merges = open(merges_file, encoding='utf-8').read().split('\n')[:-1] + with open(merges_file, encoding='utf-8') as merges_handle: + merges = merges_handle.read().split('\n')[:-1] merges = [tuple(merge.split()[:2]) for merge in merges] self.bpe_ranks = dict(zip(merges, range(len(merges)))) self.cache = {} diff --git a/transformers/tokenization_xlnet.py b/transformers/tokenization_xlnet.py index c01fbbbeeb..8c86a5bd60 100644 --- a/transformers/tokenization_xlnet.py +++ b/transformers/tokenization_xlnet.py @@ -141,7 +141,7 @@ class XLNetTokenizer(PreTrainedTokenizer): pieces = self.sp_model.SampleEncodeAsPieces(text, 64, 0.1) new_pieces = [] for piece in pieces: - if len(piece) > 1 and piece[-1] == ',' and piece[-2].isdigit(): + if len(piece) > 1 and piece[-1] == str(',') and piece[-2].isdigit(): cur_pieces = self.sp_model.EncodeAsPieces( piece[:-1].replace(SPIECE_UNDERLINE, '')) if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE: @@ -227,7 +227,7 @@ class XLNetTokenizer(PreTrainedTokenizer): An XLNet sequence pair mask has the following format: 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 2 | first sequence | second sequence | CLS segment ID - + if token_ids_1 is None, only returns the first portion of the mask (0's). """ sep = [self.sep_token_id] From 2670b0d682746e1fe94ab9c7b4d2fd7f4af03193 Mon Sep 17 00:00:00 2001 From: Michael Watkins Date: Wed, 4 Dec 2019 17:53:25 +0200 Subject: [PATCH 250/505] Fix bug which lowercases special tokens --- transformers/tests/tokenization_tests_commons.py | 8 +++++--- transformers/tokenization_utils.py | 15 +++++++++++++-- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/transformers/tests/tokenization_tests_commons.py b/transformers/tests/tokenization_tests_commons.py index faff003f4b..d904f0067e 100644 --- a/transformers/tests/tokenization_tests_commons.py +++ b/transformers/tests/tokenization_tests_commons.py @@ -115,8 +115,10 @@ class CommonTestCases: def test_added_tokens_do_lower_case(self): tokenizer = self.get_tokenizer(do_lower_case=True) - text = "aaaaa bbbbbb low cccccccccdddddddd l" - text2 = "AAAAA BBBBBB low CCCCCCCCCDDDDDDDD l" + special_token = tokenizer.all_special_tokens[0] + + text = special_token + " aaaaa bbbbbb low cccccccccdddddddd l " + special_token + text2 = special_token + " AAAAA BBBBBB low CCCCCCCCCDDDDDDDD l " + special_token toks0 = tokenizer.tokenize(text) # toks before adding new_toks @@ -141,7 +143,7 @@ class CommonTestCases: self.assertEqual(len(toks), len(toks2)) # Length should still be the same self.assertNotEqual(len(toks), len(toks0)) - self.assertNotEqual(toks[0], toks2[0]) # But at least the first tokens should differ + self.assertNotEqual(toks[1], toks2[1]) # But at least the first non-special tokens should differ def test_add_tokens_tokenizer(self): tokenizer = self.get_tokenizer() diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index 4c6cbd8986..eb22c50ebd 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -22,6 +22,7 @@ import json import six import copy import itertools +import re from io import open from .file_utils import cached_path, is_tf_available, is_torch_available @@ -520,7 +521,7 @@ class PreTrainedTokenizer(object): to_add_tokens = [] for token in new_tokens: assert isinstance(token, str) or (six.PY2 and isinstance(token, unicode)) - if self.init_kwargs.get('do_lower_case', False): + if self.init_kwargs.get('do_lower_case', False) and token not in self.all_special_tokens: token = token.lower() if token != self.unk_token and \ self.convert_tokens_to_ids(token) == self.convert_tokens_to_ids(self.unk_token) and \ @@ -615,8 +616,18 @@ class PreTrainedTokenizer(object): Take care of added tokens. """ + def lowercase_text(t): + # convert non-special tokens to lowercase + escaped_special_toks = [re.escape(s_tok) for s_tok in self.all_special_tokens] + pattern = r'(^' + r'|'.join(escaped_special_toks) + r')|' + \ + r'(.+?)' + return re.sub( + pattern, + lambda m: m.groups()[0] or m.groups()[1].lower(), + t) + if self.init_kwargs.get('do_lower_case', False): - text = text.lower() + text = lowercase_text(text) def split_on_token(tok, text): result = [] From 0cb163865a4c761c226b151283309eedb2b1ca4d Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 7 Dec 2019 13:46:14 +0100 Subject: [PATCH 251/505] Remove pytest dependency. (#2093) --- transformers/tests/optimization_tf_test.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/transformers/tests/optimization_tf_test.py b/transformers/tests/optimization_tf_test.py index ac5109cb56..515d12a158 100644 --- a/transformers/tests/optimization_tf_test.py +++ b/transformers/tests/optimization_tf_test.py @@ -3,18 +3,19 @@ from __future__ import division from __future__ import print_function import unittest -import pytest from transformers import is_tf_available +from .utils import require_tf + if is_tf_available(): import tensorflow as tf from tensorflow.python.eager import context from tensorflow.python.framework import ops from transformers import (create_optimizer, GradientAccumulator) -else: - pytestmark = pytest.mark.skip("Require TensorFlow") + +@require_tf class OptimizationFTest(unittest.TestCase): def assertListAlmostEqual(self, list1, list2, tol): self.assertEqual(len(list1), len(list2)) From 3520be7824ad11ebc05a393fd90ecfdd4203cfdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Louf?= Date: Mon, 9 Dec 2019 11:13:09 +0100 Subject: [PATCH 252/505] create encoder attention mask from shape of hidden states We currently create encoder attention masks (when they're not provided) based on the shape of the inputs to the encoder. This is obviously wrong; sequences can be of different lengths. We now create the encoder attention mask based on the batch_size and sequence_length of the encoder hidden states. --- transformers/modeling_bert.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py index 1ee3e3f097..8295cf4664 100644 --- a/transformers/modeling_bert.py +++ b/transformers/modeling_bert.py @@ -691,17 +691,19 @@ class BertModel(BertPreTrainedModel): # If a 2D ou 3D attention mask is provided for the cross-attention # we need to make broadcastabe to [batch_size, num_heads, seq_length, seq_length] - if self.config.is_decoder: + if self.config.is_decoder and encoder_hidden_states is not None: + encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size() + encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length) if encoder_attention_mask is None: - encoder_attention_mask = torch.ones(input_shape, device=device) + encoder_attention_mask = torch.ones(encoder_hidden_shape, device=device) if encoder_attention_mask.dim() == 3: encoder_extended_attention_mask = encoder_attention_mask[:, None, :, :] elif encoder_attention_mask.dim() == 2: encoder_extended_attention_mask = encoder_attention_mask[:, None, None, :] else: - raise ValueError("Wrong shape for input_ids (shape {}) or encoder_attention_mask (shape {})".format(input_shape, - encoder_attention_mask.shape)) + raise ValueError("Wrong shape for encoder_hidden_shape (shape {}) or encoder_attention_mask (shape {})".format(encoder_hidden_shape, + encoder_attention_mask.shape)) encoder_extended_attention_mask = encoder_extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility encoder_extended_attention_mask = (1.0 - encoder_extended_attention_mask) * -10000.0 From 169fea6855741315e2e0e15881cefc9823803aa6 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 9 Dec 2019 16:25:33 +0100 Subject: [PATCH 253/505] updating T5 --- transformers/modeling_t5.py | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/transformers/modeling_t5.py b/transformers/modeling_t5.py index 1bf55611a2..104e9060fc 100644 --- a/transformers/modeling_t5.py +++ b/transformers/modeling_t5.py @@ -281,7 +281,7 @@ class T5Attention(nn.Module): context_position = torch.arange(qlen, dtype=torch.long)[:, None] memory_position = torch.arange(klen, dtype=torch.long)[None, :] relative_position = memory_position - context_position # shape (qlen, klen) - rp_bucket = self._relative_position_bucket(relative_position, + rp_bucket = self._relative_position_bucket(relative_position, # shape (qlen, klen) bidirectional=not self.is_decoder, num_buckets=self.relative_attention_num_buckets) values = self.relative_attention_bias(rp_bucket) # shape (qlen, klen, num_heads) @@ -337,14 +337,10 @@ class T5Attention(nn.Module): if not self.has_relative_attention_bias: raise ValueError("No position_bias provided and no weights to compute position_bias") position_bias = self.compute_bias(qlen, klen) + if mask is not None: + position_bias += mask # (bs, n_heads, qlen, klen) + scores += position_bias - special_out = position_bias - - if mask is not None: - scores += mask - # mask = (mask == 0).expand_as(scores) # (bs, n_heads, qlen, klen) - # scores.masked_fill_(mask, -float('inf')) # (bs, n_heads, qlen, klen) - weights = F.softmax(scores.float(), dim=-1).type_as(scores) # (bs, n_heads, qlen, klen) weights = F.dropout(weights, p=self.dropout, training=self.training) # (bs, n_heads, qlen, klen) @@ -362,7 +358,7 @@ class T5Attention(nn.Module): outputs = outputs + (weights,) if self.has_relative_attention_bias: outputs = outputs + (position_bias,) - return outputs + (special_out,) + return outputs class T5LayerSelfAttention(nn.Module): @@ -379,11 +375,9 @@ class T5LayerSelfAttention(nn.Module): position_bias=position_bias, head_mask=head_mask) y = attention_output[0] - special_out = attention_output[-1] - attention_output = attention_output[:-1] layer_output = hidden_states + self.dropout(y) outputs = (layer_output,) + attention_output[1:] # add attentions if we output them - return outputs + (special_out,) + return outputs class T5LayerCrossAttention(nn.Module): @@ -426,8 +420,7 @@ class T5Block(nn.Module): position_bias=position_bias, head_mask=head_mask) hidden_states = self_attention_outputs[0] - special_out = self_attention_outputs[-1] - outputs = self_attention_outputs[1:-1] # Keep self-attention outputs and relative position weights + outputs = self_attention_outputs[1:] # Keep self-attention outputs and relative position weights if not self.is_decoder: hidden_states = self.layer[1](hidden_states) @@ -442,7 +435,7 @@ class T5Block(nn.Module): hidden_states = self.layer[2](hidden_states) outputs = (hidden_states,) + outputs # add attentions if we output them - return outputs + (special_out,) # hidden-states, (self-attention weights), (self-attention position bias), (cross-attention weights), (cross-attention position bias) + return outputs # hidden-states, (self-attention weights), (self-attention position bias), (cross-attention weights), (cross-attention position bias) class T5PreTrainedModel(PreTrainedModel): @@ -536,6 +529,10 @@ class T5Stack(T5PreTrainedModel): # positions we want to attend and -1e9 for masked positions. # Since we are adding it to the raw scores before the softmax, this is # effectively the same as removing these entirely. + + # T5 has a mask that can compare sequence ids, we simulate this here with this transposistion + # Cf. https://github.com/tensorflow/mesh/blob/8d2465e9bc93129b913b5ccc6a59aa97abd96ec6/mesh_tensorflow/transformer/transformer_layers.py#L270 + extended_attention_mask = (extended_attention_mask == extended_attention_mask.transpose(-1, -2)) extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility extended_attention_mask = (1.0 - extended_attention_mask) * -1e9 @@ -584,8 +581,6 @@ class T5Stack(T5PreTrainedModel): encoder_attention_mask=encoder_extended_attention_mask, encoder_decoder_position_bias=encoder_decoder_position_bias, head_mask=head_mask[i]) - if i == 0: - special_out = layer_outputs[-1] # layer_outputs is a tuple with: # hidden-states, (self-attention weights), (self-attention position bias), (cross-attention weights), (cross-attention position bias) hidden_states = layer_outputs[0] @@ -610,7 +605,7 @@ class T5Stack(T5PreTrainedModel): outputs = outputs + (all_hidden_states,) if self.output_attentions: outputs = outputs + (all_attentions,) - return outputs + (special_out,) # last-layer hidden state, (all hidden states), (all attentions) + return outputs # last-layer hidden state, (all hidden states), (all attentions) T5_START_DOCSTRING = r""" The T5 model was proposed in From 2a4ef098d65939d436e2a5efbb518fb807b6b1b6 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Mon, 9 Dec 2019 10:46:47 -0500 Subject: [PATCH 254/505] Add ALBERT and XLM to SQuAD script --- examples/run_squad.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/examples/run_squad.py b/examples/run_squad.py index a8ac1d1b05..2df29014ef 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -44,7 +44,9 @@ from transformers import (WEIGHTS_NAME, BertConfig, XLNetForQuestionAnswering, XLNetTokenizer, DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer, - AlbertConfig, AlbertForQuestionAnswering, AlbertTokenizer) + AlbertConfig, AlbertForQuestionAnswering, AlbertTokenizer, + XLMConfig, XLMForQuestionAnswering, XLMTokenizer, + ) from transformers import AdamW, get_linear_schedule_with_warmup, squad_convert_examples_to_features @@ -58,7 +60,8 @@ MODEL_CLASSES = { 'xlnet': (XLNetConfig, XLNetForQuestionAnswering, XLNetTokenizer), 'xlm': (XLMConfig, XLMForQuestionAnswering, XLMTokenizer), 'distilbert': (DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer), - 'albert': (AlbertConfig, AlbertForQuestionAnswering, AlbertTokenizer) + 'albert': (AlbertConfig, AlbertForQuestionAnswering, AlbertTokenizer), + 'xlm': (XLMConfig, XLMForQuestionAnswering, XLMTokenizer) } def set_seed(args): From b016dd16c90c2c18168d13bca6d5002729fd5b0a Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 9 Dec 2019 21:38:07 +0100 Subject: [PATCH 255/505] fix tests on python 3.5 --- transformers/modeling_t5.py | 2 +- transformers/tests/modeling_common_test.py | 15 ++++++++------- transformers/tokenization_t5.py | 2 +- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/transformers/modeling_t5.py b/transformers/modeling_t5.py index 104e9060fc..e48293b49e 100644 --- a/transformers/modeling_t5.py +++ b/transformers/modeling_t5.py @@ -338,7 +338,7 @@ class T5Attention(nn.Module): raise ValueError("No position_bias provided and no weights to compute position_bias") position_bias = self.compute_bias(qlen, klen) if mask is not None: - position_bias += mask # (bs, n_heads, qlen, klen) + position_bias = position_bias + mask # (bs, n_heads, qlen, klen) scores += position_bias weights = F.softmax(scores.float(), dim=-1).type_as(scores) # (bs, n_heads, qlen, klen) diff --git a/transformers/tests/modeling_common_test.py b/transformers/tests/modeling_common_test.py index ee75da605c..11aeaafe31 100644 --- a/transformers/tests/modeling_common_test.py +++ b/transformers/tests/modeling_common_test.py @@ -138,8 +138,8 @@ class CommonTestCases: self.assertListEqual( list(attentions[0].shape[-3:]), [self.model_tester.num_attention_heads, - self.model_tester.seq_length, - self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) + self.model_tester.encoder_seq_length if hasattr(self.model_tester, 'encoder_seq_length') else self.model_tester.seq_length, + self.model_tester.encoder_seq_length if hasattr(self.model_tester, 'encoder_seq_length') else self.model_tester.seq_length]) out_len = len(outputs) if self.is_encoder_decoder: @@ -151,8 +151,8 @@ class CommonTestCases: self.assertListEqual( list(decoder_attentions[0].shape[-3:]), [self.model_tester.num_attention_heads, - self.model_tester.seq_length, - self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) + self.model_tester.decoder_seq_length if hasattr(self.model_tester, 'decoder_seq_length') else self.model_tester.seq_length, + self.model_tester.decoder_seq_length if hasattr(self.model_tester, 'decoder_seq_length') else self.model_tester.seq_length]) # Check attention is always last and order is fine config.output_attentions = True @@ -169,8 +169,8 @@ class CommonTestCases: self.assertListEqual( list(self_attentions[0].shape[-3:]), [self.model_tester.num_attention_heads, - self.model_tester.seq_length, - self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) + self.model_tester.encoder_seq_length if hasattr(self.model_tester, 'encoder_seq_length') else self.model_tester.seq_length, + self.model_tester.encoder_seq_length if hasattr(self.model_tester, 'encoder_seq_length') else self.model_tester.seq_length]) def test_torchscript(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() @@ -440,7 +440,8 @@ class CommonTestCases: self.assertEqual(len(hidden_states), self.model_tester.num_hidden_layers + 1) self.assertListEqual( list(hidden_states[0].shape[-2:]), - [self.model_tester.seq_length, self.model_tester.hidden_size]) + [self.model_tester.encoder_seq_length if hasattr(self.model_tester, 'encoder_seq_length') else self.model_tester.seq_length, + self.model_tester.hidden_size]) def test_resize_tokens_embeddings(self): original_config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() diff --git a/transformers/tokenization_t5.py b/transformers/tokenization_t5.py index 3847aeefbf..933084d13a 100644 --- a/transformers/tokenization_t5.py +++ b/transformers/tokenization_t5.py @@ -134,7 +134,7 @@ class T5Tokenizer(PreTrainedTokenizer): """ Converts a token (str/unicode) in an id using the vocab. """ if token.startswith(u"', token) - num = int(l[1]) + num = int(l.group(1)) return self.vocab_size - num - 1 return self.sp_model.piece_to_id(token) From 808bb8da7edbd9f5858b3c223ebac9bd83275934 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 9 Dec 2019 21:48:34 +0100 Subject: [PATCH 256/505] fix transfo xl tests --- transformers/tests/modeling_common_test.py | 18 ++++++++++++------ .../tests/modeling_tf_transfo_xl_test.py | 2 +- transformers/tests/modeling_transfo_xl_test.py | 2 +- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/transformers/tests/modeling_common_test.py b/transformers/tests/modeling_common_test.py index 11aeaafe31..7033a06d0b 100644 --- a/transformers/tests/modeling_common_test.py +++ b/transformers/tests/modeling_common_test.py @@ -125,6 +125,11 @@ class CommonTestCases: def test_attention_outputs(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + decoder_seq_length = self.model_tester.decoder_seq_length if hasattr(self.model_tester, 'decoder_seq_length') else self.model_tester.seq_length + encoder_seq_length = self.model_tester.encoder_seq_length if hasattr(self.model_tester, 'encoder_seq_length') else self.model_tester.seq_length + decoder_key_length = self.model_tester.key_length if hasattr(self.model_tester, 'key_length') else decoder_seq_length + encoder_key_length = self.model_tester.key_length if hasattr(self.model_tester, 'key_length') else encoder_seq_length + for model_class in self.all_model_classes: config.output_attentions = True config.output_hidden_states = False @@ -138,8 +143,8 @@ class CommonTestCases: self.assertListEqual( list(attentions[0].shape[-3:]), [self.model_tester.num_attention_heads, - self.model_tester.encoder_seq_length if hasattr(self.model_tester, 'encoder_seq_length') else self.model_tester.seq_length, - self.model_tester.encoder_seq_length if hasattr(self.model_tester, 'encoder_seq_length') else self.model_tester.seq_length]) + encoder_seq_length , + encoder_key_length]) out_len = len(outputs) if self.is_encoder_decoder: @@ -151,8 +156,9 @@ class CommonTestCases: self.assertListEqual( list(decoder_attentions[0].shape[-3:]), [self.model_tester.num_attention_heads, - self.model_tester.decoder_seq_length if hasattr(self.model_tester, 'decoder_seq_length') else self.model_tester.seq_length, - self.model_tester.decoder_seq_length if hasattr(self.model_tester, 'decoder_seq_length') else self.model_tester.seq_length]) + decoder_seq_length, + decoder_key_length + ]) # Check attention is always last and order is fine config.output_attentions = True @@ -169,8 +175,8 @@ class CommonTestCases: self.assertListEqual( list(self_attentions[0].shape[-3:]), [self.model_tester.num_attention_heads, - self.model_tester.encoder_seq_length if hasattr(self.model_tester, 'encoder_seq_length') else self.model_tester.seq_length, - self.model_tester.encoder_seq_length if hasattr(self.model_tester, 'encoder_seq_length') else self.model_tester.seq_length]) + encoder_seq_length, + encoder_key_length]) def test_torchscript(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() diff --git a/transformers/tests/modeling_tf_transfo_xl_test.py b/transformers/tests/modeling_tf_transfo_xl_test.py index 534fe39646..8ebd749b4c 100644 --- a/transformers/tests/modeling_tf_transfo_xl_test.py +++ b/transformers/tests/modeling_tf_transfo_xl_test.py @@ -68,7 +68,7 @@ class TFTransfoXLModelTest(TFCommonTestCases.TFCommonModelTester): self.batch_size = batch_size self.seq_length = seq_length self.mem_len = mem_len - self.key_len = seq_length + mem_len + self.key_length = seq_length + mem_len self.clamp_len = clamp_len self.is_training = is_training self.use_labels = use_labels diff --git a/transformers/tests/modeling_transfo_xl_test.py b/transformers/tests/modeling_transfo_xl_test.py index f7b913da5b..2d1541d87b 100644 --- a/transformers/tests/modeling_transfo_xl_test.py +++ b/transformers/tests/modeling_transfo_xl_test.py @@ -66,7 +66,7 @@ class TransfoXLModelTest(CommonTestCases.CommonModelTester): self.batch_size = batch_size self.seq_length = seq_length self.mem_len = mem_len - self.key_len = seq_length + mem_len + self.key_length = seq_length + mem_len self.clamp_len = clamp_len self.is_training = is_training self.use_labels = use_labels From 8e651f56b75982f07fc522b62f298d8d70e6e56f Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 9 Dec 2019 22:13:57 +0100 Subject: [PATCH 257/505] fix tf tests --- transformers/tests/modeling_tf_common_test.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/transformers/tests/modeling_tf_common_test.py b/transformers/tests/modeling_tf_common_test.py index 20ccfd8ce0..26bd037c9e 100644 --- a/transformers/tests/modeling_tf_common_test.py +++ b/transformers/tests/modeling_tf_common_test.py @@ -213,6 +213,11 @@ class TFCommonTestCases: def test_attention_outputs(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() + decoder_seq_length = self.model_tester.decoder_seq_length if hasattr(self.model_tester, 'decoder_seq_length') else self.model_tester.seq_length + encoder_seq_length = self.model_tester.encoder_seq_length if hasattr(self.model_tester, 'encoder_seq_length') else self.model_tester.seq_length + decoder_key_length = self.model_tester.key_length if hasattr(self.model_tester, 'key_length') else decoder_seq_length + encoder_key_length = self.model_tester.key_length if hasattr(self.model_tester, 'key_length') else encoder_seq_length + for model_class in self.all_model_classes: config.output_attentions = True config.output_hidden_states = False @@ -225,8 +230,8 @@ class TFCommonTestCases: self.assertListEqual( list(attentions[0].shape[-3:]), [self.model_tester.num_attention_heads, - self.model_tester.seq_length, - self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) + encoder_seq_length, + encoder_key_length]) out_len = len(outputs) if self.is_encoder_decoder: @@ -238,8 +243,8 @@ class TFCommonTestCases: self.assertListEqual( list(decoder_attentions[0].shape[-3:]), [self.model_tester.num_attention_heads, - self.model_tester.seq_length, - self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) + decoder_seq_length, + decoder_key_length]) # Check attention is always last and order is fine config.output_attentions = True @@ -255,8 +260,8 @@ class TFCommonTestCases: self.assertListEqual( list(attentions[0].shape[-3:]), [self.model_tester.num_attention_heads, - self.model_tester.seq_length, - self.model_tester.key_len if hasattr(self.model_tester, 'key_len') else self.model_tester.seq_length]) + encoder_seq_length, + encoder_key_length]) def test_headmasking(self): pass From f71b1bb05a20879953d57bf648ab7bbd2b3239bc Mon Sep 17 00:00:00 2001 From: Bilal Khan Date: Wed, 27 Nov 2019 08:39:00 -0600 Subject: [PATCH 258/505] Save optimizer state, scheduler state and current epoch --- examples/run_lm_finetuning.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py index a5eaf524ac..3cae206460 100644 --- a/examples/run_lm_finetuning.py +++ b/examples/run_lm_finetuning.py @@ -224,7 +224,7 @@ def train(args, train_dataset, model, tokenizer): model.zero_grad() train_iterator = trange(int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0]) set_seed(args) # Added here for reproducibility (even between python 2 and 3) - for _ in train_iterator: + for epoch in train_iterator: epoch_iterator = tqdm(train_dataloader, desc="Iteration", disable=args.local_rank not in [-1, 0]) for step, batch in enumerate(epoch_iterator): inputs, labels = mask_tokens(batch, tokenizer, args) if args.mlm else (batch, batch) @@ -279,6 +279,10 @@ def train(args, train_dataset, model, tokenizer): _rotate_checkpoints(args, checkpoint_prefix) + torch.save(optimizer.state_dict(), os.path.join(output_dir, 'optimizer.pt')) + torch.save(scheduler.state_dict(), os.path.join(output_dir, 'scheduler.pt')) + torch.save(epoch, os.path.join(output_dir, 'training_state.pt')) + if args.max_steps > 0 and global_step > args.max_steps: epoch_iterator.close() break From a03fcf570de4a90218efd4b3de253d4648fe24b1 Mon Sep 17 00:00:00 2001 From: Bilal Khan Date: Wed, 27 Nov 2019 18:42:07 -0600 Subject: [PATCH 259/505] Save tokenizer after each epoch to be able to resume training from a checkpoint --- examples/run_lm_finetuning.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py index 3cae206460..1d93aa4381 100644 --- a/examples/run_lm_finetuning.py +++ b/examples/run_lm_finetuning.py @@ -274,6 +274,8 @@ def train(args, train_dataset, model, tokenizer): os.makedirs(output_dir) model_to_save = model.module if hasattr(model, 'module') else model # Take care of distributed/parallel training model_to_save.save_pretrained(output_dir) + tokenizer.save_pretrained(output_dir) + torch.save(args, os.path.join(output_dir, 'training_args.bin')) logger.info("Saving model checkpoint to %s", output_dir) @@ -282,6 +284,7 @@ def train(args, train_dataset, model, tokenizer): torch.save(optimizer.state_dict(), os.path.join(output_dir, 'optimizer.pt')) torch.save(scheduler.state_dict(), os.path.join(output_dir, 'scheduler.pt')) torch.save(epoch, os.path.join(output_dir, 'training_state.pt')) + logger.info("Saving training state to %s", output_dir) if args.max_steps > 0 and global_step > args.max_steps: epoch_iterator.close() From 0eb973b0d99e5c219af8a93b6267bda00c7161c6 Mon Sep 17 00:00:00 2001 From: Bilal Khan Date: Wed, 27 Nov 2019 19:10:24 -0600 Subject: [PATCH 260/505] Use saved optimizer and scheduler states if available --- examples/run_lm_finetuning.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py index 1d93aa4381..9bdbf9ca56 100644 --- a/examples/run_lm_finetuning.py +++ b/examples/run_lm_finetuning.py @@ -188,6 +188,13 @@ def train(args, train_dataset, model, tokenizer): ] optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon) scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=args.warmup_steps, num_training_steps=t_total) + + # Check if saved optimizer or scheduler states exist + if os.path.isfile(os.path.join(args.model_name_or_path, 'optimizer.pt')) and os.path.isfile(os.path.join(args.model_name_or_path, 'scheduler.pt')): + # Load in optimizer and scheduler states + optimizer.load_state_dict(torch.load(os.path.join(args.model_name_or_path, 'optimizer.pt'))) + scheduler.load_state_dict(torch.load(os.path.join(args.model_name_or_path, 'scheduler.pt'))) + if args.fp16: try: from apex import amp From 2d73591a1831e80d0743b514d7f0138c4879e37b Mon Sep 17 00:00:00 2001 From: Bilal Khan Date: Wed, 27 Nov 2019 19:13:10 -0600 Subject: [PATCH 261/505] Stop saving current epoch --- examples/run_lm_finetuning.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py index 9bdbf9ca56..5e7683b85d 100644 --- a/examples/run_lm_finetuning.py +++ b/examples/run_lm_finetuning.py @@ -290,8 +290,7 @@ def train(args, train_dataset, model, tokenizer): torch.save(optimizer.state_dict(), os.path.join(output_dir, 'optimizer.pt')) torch.save(scheduler.state_dict(), os.path.join(output_dir, 'scheduler.pt')) - torch.save(epoch, os.path.join(output_dir, 'training_state.pt')) - logger.info("Saving training state to %s", output_dir) + logger.info("Saving optimizer and scheduler states to %s", output_dir) if args.max_steps > 0 and global_step > args.max_steps: epoch_iterator.close() From 9626e0458c20b61c18c9564ecc4d1261a4a66e50 Mon Sep 17 00:00:00 2001 From: Bilal Khan Date: Wed, 27 Nov 2019 20:00:16 -0600 Subject: [PATCH 262/505] Add functionality to continue training from last saved global_step --- examples/run_lm_finetuning.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py index 5e7683b85d..172d4e20e2 100644 --- a/examples/run_lm_finetuning.py +++ b/examples/run_lm_finetuning.py @@ -223,17 +223,37 @@ def train(args, train_dataset, model, tokenizer): logger.info(" Total optimization steps = %d", t_total) global_step = 0 + epochs_trained = 0 + steps_trained_in_current_epoch = 0 + # Check if continuing training from a checkpoint + if os.path.exists(args.model_name_or_path): + # set global_step to gobal_step of last saved checkpoint from model path + global_step = int(args.model_name_or_path.split('-')[-1].split('/')[0]) + epochs_trained = global_step // (len(train_dataloader) // args.gradient_accumulation_steps) + steps_trained_in_current_epoch = global_step % (len(train_dataloader) // args.gradient_accumulation_steps) + + logger.info(" Continuing training from checkpoint, will skip to saved global_step") + logger.info(" Continuing training from epoch %d", epochs_trained) + logger.info(" Continuing training from global step %d", global_step) + logger.info(" Will skip the first %d steps in the first epoch", steps_trained_in_current_epoch) + tr_loss, logging_loss = 0.0, 0.0 model_to_resize = model.module if hasattr(model, 'module') else model # Take care of distributed/parallel training model_to_resize.resize_token_embeddings(len(tokenizer)) model.zero_grad() - train_iterator = trange(int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0]) + train_iterator = trange(epochs_trained, int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0]) set_seed(args) # Added here for reproducibility (even between python 2 and 3) for epoch in train_iterator: epoch_iterator = tqdm(train_dataloader, desc="Iteration", disable=args.local_rank not in [-1, 0]) for step, batch in enumerate(epoch_iterator): + + # Skip past any already trained steps if resuming training + if steps_trained_in_current_epoch > 0: + steps_trained_in_current_epoch -= 1 + continue + inputs, labels = mask_tokens(batch, tokenizer, args) if args.mlm else (batch, batch) inputs = inputs.to(args.device) labels = labels.to(args.device) From 79526f82f5d6757812f3691949cf03b864697f46 Mon Sep 17 00:00:00 2001 From: Bilal Khan Date: Thu, 28 Nov 2019 19:20:29 -0600 Subject: [PATCH 263/505] Remove unnecessary epoch variable --- examples/run_lm_finetuning.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py index 172d4e20e2..c4c73e71af 100644 --- a/examples/run_lm_finetuning.py +++ b/examples/run_lm_finetuning.py @@ -245,7 +245,7 @@ def train(args, train_dataset, model, tokenizer): model.zero_grad() train_iterator = trange(epochs_trained, int(args.num_train_epochs), desc="Epoch", disable=args.local_rank not in [-1, 0]) set_seed(args) # Added here for reproducibility (even between python 2 and 3) - for epoch in train_iterator: + for _ in train_iterator: epoch_iterator = tqdm(train_dataloader, desc="Iteration", disable=args.local_rank not in [-1, 0]) for step, batch in enumerate(epoch_iterator): From 5c877fe94a1cbb70132515a9da6a464bf6da49ed Mon Sep 17 00:00:00 2001 From: Pierric Cistac Date: Mon, 9 Dec 2019 18:53:00 -0500 Subject: [PATCH 264/505] fix albert links --- README.md | 2 +- docs/source/index.rst | 2 +- docs/source/pretrained_models.rst | 16 ++++++++-------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 64ec631651..f3aa8a95ee 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ At some point in the future, you'll be able to seamlessly move from pre-training 8. **[DistilBERT](https://github.com/huggingface/transformers/tree/master/examples/distillation)** (from HuggingFace), released together with the paper [DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter](https://arxiv.org/abs/1910.01108) by Victor Sanh, Lysandre Debut and Thomas Wolf. The same method has been applied to compress GPT2 into [DistilGPT2](https://github.com/huggingface/transformers/tree/master/examples/distillation), RoBERTa into [DistilRoBERTa](https://github.com/huggingface/transformers/tree/master/examples/distillation), Multilingual BERT into [DistilmBERT](https://github.com/huggingface/transformers/tree/master/examples/distillation) and a German version of DistilBERT. 9. **[CTRL](https://github.com/salesforce/ctrl/)** (from Salesforce) released with the paper [CTRL: A Conditional Transformer Language Model for Controllable Generation](https://arxiv.org/abs/1909.05858) by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher. 10. **[CamemBERT](https://camembert-model.fr)** (from Inria/Facebook/Sorbonne) released with the paper [CamemBERT: a Tasty French Language Model](https://arxiv.org/abs/1911.03894) by Louis Martin*, Benjamin Muller*, Pedro Javier Ortiz Suárez*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah and Benoît Sagot. -11. **[ALBERT](https://github.com/google-research/google-research/tree/master/albert)** (from Google Research and the Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. +11. **[ALBERT](https://github.com/google-research/ALBERT)** (from Google Research and the Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. 11. Want to contribute a new model? We have added a **detailed guide and templates** to guide you in the process of adding a new model. You can find them in the [`templates`](./templates) folder of the repository. Be sure to check the [contributing guidelines](./CONTRIBUTING.md) and contact the maintainers or open an issue to collect feedbacks before starting your PR. These implementations have been tested on several datasets (see the example scripts) and should match the performances of the original implementations (e.g. ~93 F1 on SQuAD for BERT Whole-Word-Masking, ~88 F1 on RocStories for OpenAI GPT, ~18.3 perplexity on WikiText 103 for Transformer-XL, ~0.916 Peason R coefficient on STS-B for XLNet). You can find more details on the performances in the Examples section of the [documentation](https://huggingface.co/transformers/examples.html). diff --git a/docs/source/index.rst b/docs/source/index.rst index 55ead33b4d..84012fc6cf 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -49,7 +49,7 @@ The library currently contains PyTorch and Tensorflow implementations, pre-train 8. `DistilBERT `_ (from HuggingFace) released together with the paper `DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter `_ by Victor Sanh, Lysandre Debut and Thomas Wolf. The same method has been applied to compress GPT2 into `DistilGPT2 `_. 9. `CTRL `_ (from Salesforce), released together with the paper `CTRL: A Conditional Transformer Language Model for Controllable Generation `_ by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher. 10. `CamemBERT `_ (from FAIR, Inria, Sorbonne Université) released together with the paper `CamemBERT: a Tasty French Language Model `_ by Louis Martin, Benjamin Muller, Pedro Javier Ortiz Suarez, Yoann Dupont, Laurent Romary, Eric Villemonte de la Clergerie, Djame Seddah, and Benoît Sagot. -11. `ALBERT `_ (from Google Research), released together with the paper a `ALBERT: A Lite BERT for Self-supervised Learning of Language Representations `_ by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. +11. `ALBERT `_ (from Google Research), released together with the paper a `ALBERT: A Lite BERT for Self-supervised Learning of Language Representations `_ by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. .. toctree:: :maxdepth: 2 diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst index 090cb75808..dd61f11769 100644 --- a/docs/source/pretrained_models.rst +++ b/docs/source/pretrained_models.rst @@ -169,35 +169,35 @@ Here is the full list of the currently provided pretrained models together with +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | ALBERT | ``albert-base-v1`` | | 12 repeating layers, 128 embedding, 768-hidden, 12-heads, 11M parameters | | | | | ALBERT base model | -| | | (see `details `__) | +| | | (see `details `__) | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``albert-large-v1`` | | 24 repeating layers, 128 embedding, 1024-hidden, 16-heads, 17M parameters | | | | | ALBERT large model | -| | | (see `details `__) | +| | | (see `details `__) | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``albert-xlarge-v1`` | | 24 repeating layers, 128 embedding, 2048-hidden, 16-heads, 58M parameters | | | | | ALBERT xlarge model | -| | | (see `details `__) | +| | | (see `details `__) | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``albert-xxlarge-v1`` | | 12 repeating layer, 128 embedding, 4096-hidden, 64-heads, 223M parameters | | | | | ALBERT xxlarge model | -| | | (see `details `__) | +| | | (see `details `__) | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``albert-base-v2`` | | 12 repeating layers, 128 embedding, 768-hidden, 12-heads, 11M parameters | | | | | ALBERT base model with no dropout, additional training data and longer training | -| | | (see `details `__) | +| | | (see `details `__) | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``albert-large-v2`` | | 24 repeating layers, 128 embedding, 1024-hidden, 16-heads, 17M parameters | | | | | ALBERT large model with no dropout, additional training data and longer training | -| | | (see `details `__) | +| | | (see `details `__) | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``albert-xlarge-v2`` | | 24 repeating layers, 128 embedding, 2048-hidden, 16-heads, 58M parameters | | | | | ALBERT xlarge model with no dropout, additional training data and longer training | -| | | (see `details `__) | +| | | (see `details `__) | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``albert-xxlarge-v2`` | | 12 repeating layer, 128 embedding, 4096-hidden, 64-heads, 223M parameters | | | | | ALBERT xxlarge model with no dropout, additional training data and longer training | -| | | (see `details `__) | +| | | (see `details `__) | +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ From 07f4cd73f6d13e43b69a6e34a2a756a80fc7f70b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Louf?= Date: Thu, 31 Oct 2019 09:48:27 +0100 Subject: [PATCH 265/505] update function to add special tokens Since I started my PR the `add_special_token_single_sequence` function has been deprecated for another; I replaced it with the new function. --- examples/utils_summarization.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/utils_summarization.py b/examples/utils_summarization.py index 327ca8cc3e..087c88bd4e 100644 --- a/examples/utils_summarization.py +++ b/examples/utils_summarization.py @@ -139,11 +139,11 @@ def encode_for_summarization(story_lines, summary_lines, tokenizer): sentences. """ story_lines_token_ids = [ - tokenizer.add_special_tokens_single_sequence(tokenizer.encode(line)) + tokenizer.build_inputs_with_special_tokens(tokenizer.encode(line)) for line in story_lines ] summary_lines_token_ids = [ - tokenizer.add_special_tokens_single_sequence(tokenizer.encode(line)) + tokenizer.build_inputs_with_special_tokens(tokenizer.encode(line)) for line in summary_lines ] From 1c71ecc880ae8f04c8462e1368dc0678fdb92fc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Louf?= Date: Thu, 31 Oct 2019 10:16:08 +0100 Subject: [PATCH 266/505] load the pretrained weights for encoder-decoder We currently save the pretrained_weights of the encoder and decoder in two separate directories `encoder` and `decoder`. However, for the `from_pretrained` function to operate with automodels we need to specify the type of model in the path to the weights. The path to the encoder/decoder weights is handled by the `PreTrainedEncoderDecoder` class in the `save_pretrained` function. Sice there is no easy way to infer the type of model that was initialized for the encoder and decoder we add a parameter `model_type` to the function. This is not an ideal solution as it is error prone, and the model type should be carried by the Model classes somehow. This is a temporary fix that should be changed before merging. --- examples/run_summarization_finetuning.py | 48 ++++++++++++++---------- transformers/modeling_encoder_decoder.py | 31 +++++++++------ 2 files changed, 49 insertions(+), 30 deletions(-) diff --git a/examples/run_summarization_finetuning.py b/examples/run_summarization_finetuning.py index f5604c2669..9c2c7769c9 100644 --- a/examples/run_summarization_finetuning.py +++ b/examples/run_summarization_finetuning.py @@ -328,6 +328,22 @@ def evaluate(args, model, tokenizer, prefix=""): return result +def save_model_checkpoints(args, model, tokenizer): + if not os.path.exists(args.output_dir): + os.makedirs(args.output_dir) + + logger.info("Saving model checkpoint to %s", args.output_dir) + + # Save a trained model, configuration and tokenizer using `save_pretrained()`. + # They can then be reloaded using `from_pretrained()` + model_to_save = ( + model.module if hasattr(model, "module") else model + ) # Take care of distributed/parallel training + model_to_save.save_pretrained(args.output_dir, model_type='bert') + tokenizer.save_pretrained(args.output_dir) + torch.save(args, os.path.join(args.output_dir, "training_arguments.bin")) + + def main(): parser = argparse.ArgumentParser() @@ -454,36 +470,30 @@ def main(): # Train the model model.to(args.device) if args.do_train: - global_step, tr_loss = train(args, model, tokenizer) + try: + global_step, tr_loss = train(args, model, tokenizer) + except KeyboardInterrupt: + response = input("You interrupted the training. Do you want to save the model checkpoints? [Y/n]") + if response.lower() in ["", "y", "yes"]: + save_model_checkpoints(args, model, tokenizer) + sys.exit(0) + logger.info(" global_step = %s, average loss = %s", global_step, tr_loss) - - if not os.path.exists(args.output_dir): - os.makedirs(args.output_dir) - - logger.info("Saving model checkpoint to %s", args.output_dir) - - # Save a trained model, configuration and tokenizer using `save_pretrained()`. - # They can then be reloaded using `from_pretrained()` - model_to_save = ( - model.module if hasattr(model, "module") else model - ) # Take care of distributed/parallel training - model_to_save.save_pretrained(args.output_dir) - tokenizer.save_pretrained(args.output_dir) - torch.save(args, os.path.join(args.output_dir, "training_arguments.bin")) + save_model_checkpoints(args, model, tokenizer) # Evaluate the model results = {} if args.do_evaluate: - checkpoints = [] + checkpoints = [args.output_dir] logger.info("Evaluate the following checkpoints: %s", checkpoints) for checkpoint in checkpoints: - encoder_checkpoint = os.path.join(checkpoint, "encoder") - decoder_checkpoint = os.path.join(checkpoint, "decoder") + encoder_checkpoint = os.path.join(checkpoint, "bert_encoder") + decoder_checkpoint = os.path.join(checkpoint, "bert_decoder") model = PreTrainedEncoderDecoder.from_pretrained( encoder_checkpoint, decoder_checkpoint ) model.to(args.device) - results = "placeholder" + print("model loaded") return results diff --git a/transformers/modeling_encoder_decoder.py b/transformers/modeling_encoder_decoder.py index a884abd0a2..73322101d3 100644 --- a/transformers/modeling_encoder_decoder.py +++ b/transformers/modeling_encoder_decoder.py @@ -117,8 +117,7 @@ class PreTrainedEncoderDecoder(nn.Module): kwargs_common = { argument: value for argument, value in kwargs.items() - if not argument.startswith("encoder_") - and not argument.startswith("decoder_") + if not argument.startswith("encoder_") and not argument.startswith("decoder_") } kwargs_decoder = kwargs_common.copy() kwargs_encoder = kwargs_common.copy() @@ -158,14 +157,27 @@ class PreTrainedEncoderDecoder(nn.Module): return model - def save_pretrained(self, save_directory): - """ Save a Seq2Seq model and its configuration file in a format such + def save_pretrained(self, save_directory, model_type="bert"): + """ Save an EncoderDecoder model and its configuration file in a format such that it can be loaded using `:func:`~transformers.PreTrainedEncoderDecoder.from_pretrained` We save the encoder' and decoder's parameters in two separate directories. + + If we want the weight loader to function we need to preprend the model + type to the directories' names. As far as I know there is no simple way + to infer the type of the model (except maybe by parsing the class' + names, which is not very future-proof). For now, we ask the user to + specify the model type explicitly when saving the weights. """ - self.encoder.save_pretrained(os.path.join(save_directory, "encoder")) - self.decoder.save_pretrained(os.path.join(save_directory, "decoder")) + encoder_path = os.path.join(save_directory, "{}_encoder".format(model_type)) + if not os.path.exists(encoder_path): + os.makedirs(encoder_path) + self.encoder.save_pretrained(encoder_path) + + decoder_path = os.path.join(save_directory, "{}_decoder".format(model_type)) + if not os.path.exists(decoder_path): + os.makedirs(decoder_path) + self.decoder.save_pretrained(decoder_path) def forward(self, encoder_input_ids, decoder_input_ids, **kwargs): """ The forward pass on a seq2eq depends what we are performing: @@ -193,8 +205,7 @@ class PreTrainedEncoderDecoder(nn.Module): kwargs_common = { argument: value for argument, value in kwargs.items() - if not argument.startswith("encoder_") - and not argument.startswith("decoder_") + if not argument.startswith("encoder_") and not argument.startswith("decoder_") } kwargs_decoder = kwargs_common.copy() kwargs_encoder = kwargs_common.copy() @@ -217,9 +228,7 @@ class PreTrainedEncoderDecoder(nn.Module): encoder_hidden_states = kwargs_encoder.pop("hidden_states", None) if encoder_hidden_states is None: encoder_outputs = self.encoder(encoder_input_ids, **kwargs_encoder) - encoder_hidden_states = encoder_outputs[ - 0 - ] # output the last layer hidden state + encoder_hidden_states = encoder_outputs[0] # output the last layer hidden state else: encoder_outputs = () From 9660ba1cbdec0e419937af06bd99f06fb5ebbf91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Louf?= Date: Thu, 31 Oct 2019 17:59:16 +0100 Subject: [PATCH 267/505] Add beam search --- examples/run_summarization_finetuning.py | 502 ----------------------- examples/utils_summarization.py | 20 +- transformers/generate/__init__.py | 1 + transformers/generate/beam_search.py | 358 ++++++++++++++++ transformers/modeling_beam_search.py | 271 ------------ transformers/tests/beam_search_tests.py | 226 ++++++++++ 6 files changed, 594 insertions(+), 784 deletions(-) delete mode 100644 examples/run_summarization_finetuning.py create mode 100644 transformers/generate/__init__.py create mode 100644 transformers/generate/beam_search.py delete mode 100644 transformers/modeling_beam_search.py create mode 100644 transformers/tests/beam_search_tests.py diff --git a/examples/run_summarization_finetuning.py b/examples/run_summarization_finetuning.py deleted file mode 100644 index 9c2c7769c9..0000000000 --- a/examples/run_summarization_finetuning.py +++ /dev/null @@ -1,502 +0,0 @@ -# coding=utf-8 -# Copyright 2019 The HuggingFace Inc. team. -# Copyright (c) 2019 The HuggingFace Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" Finetuning seq2seq models for sequence generation.""" - -import argparse -import functools -import logging -import os -import random -import sys - -import numpy as np -from tqdm import tqdm, trange -import torch -from torch.optim import Adam -from torch.utils.data import DataLoader, RandomSampler, SequentialSampler - -from transformers import ( - AutoTokenizer, - BertForMaskedLM, - BertConfig, - PreTrainedEncoderDecoder, - Model2Model, -) - -from utils_summarization import ( - CNNDailyMailDataset, - encode_for_summarization, - fit_to_block_size, - build_lm_labels, - build_mask, - compute_token_type_ids, -) - -logger = logging.getLogger(__name__) -logging.basicConfig(stream=sys.stdout, level=logging.INFO) - - -def set_seed(args): - random.seed(args.seed) - np.random.seed(args.seed) - torch.manual_seed(args.seed) - - -# ------------ -# Load dataset -# ------------ - - -def load_and_cache_examples(args, tokenizer): - dataset = CNNDailyMailDataset(tokenizer, data_dir=args.data_dir) - return dataset - - -def collate(data, tokenizer, block_size): - """ List of tuple as an input. """ - # remove the files with empty an story/summary, encode and fit to block - data = filter(lambda x: not (len(x[0]) == 0 or len(x[1]) == 0), data) - data = [ - encode_for_summarization(story, summary, tokenizer) for story, summary in data - ] - data = [ - ( - fit_to_block_size(story, block_size, tokenizer.pad_token_id), - fit_to_block_size(summary, block_size, tokenizer.pad_token_id), - ) - for story, summary in data - ] - - stories = torch.tensor([story for story, summary in data]) - summaries = torch.tensor([summary for story, summary in data]) - encoder_token_type_ids = compute_token_type_ids(stories, tokenizer.cls_token_id) - encoder_mask = build_mask(stories, tokenizer.pad_token_id) - decoder_mask = build_mask(summaries, tokenizer.pad_token_id) - lm_labels = build_lm_labels(summaries, tokenizer.pad_token_id) - - return ( - stories, - summaries, - encoder_token_type_ids, - encoder_mask, - decoder_mask, - lm_labels, - ) - - -# ---------- -# Optimizers -# ---------- - - -class BertSumOptimizer(object): - """ Specific optimizer for BertSum. - - As described in [1], the authors fine-tune BertSum for abstractive - summarization using two Adam Optimizers with different warm-up steps and - learning rate. They also use a custom learning rate scheduler. - - [1] Liu, Yang, and Mirella Lapata. "Text summarization with pretrained encoders." - arXiv preprint arXiv:1908.08345 (2019). - """ - - def __init__(self, model, lr, warmup_steps, beta_1=0.99, beta_2=0.999, eps=1e-8): - self.encoder = model.encoder - self.decoder = model.decoder - self.lr = lr - self.warmup_steps = warmup_steps - - self.optimizers = { - "encoder": Adam( - model.encoder.parameters(), - lr=lr["encoder"], - betas=(beta_1, beta_2), - eps=eps, - ), - "decoder": Adam( - model.decoder.parameters(), - lr=lr["decoder"], - betas=(beta_1, beta_2), - eps=eps, - ), - } - - self._step = 0 - - def _update_rate(self, stack): - return self.lr[stack] * min( - self._step ** (-0.5), self._step * self.warmup_steps[stack] ** (-0.5) - ) - - def zero_grad(self): - self.optimizer_decoder.zero_grad() - self.optimizer_encoder.zero_grad() - - def step(self): - self._step += 1 - for stack, optimizer in self.optimizers.items(): - new_rate = self._update_rate(stack) - for param_group in optimizer.param_groups: - param_group["lr"] = new_rate - optimizer.step() - - -# ------------ -# Train -# ------------ - - -def train(args, model, tokenizer): - """ Fine-tune the pretrained model on the corpus. """ - set_seed(args) - - # Load the data - args.train_batch_size = args.per_gpu_train_batch_size * max(1, args.n_gpu) - train_dataset = load_and_cache_examples(args, tokenizer) - train_sampler = RandomSampler(train_dataset) - model_collate_fn = functools.partial(collate, tokenizer=tokenizer, block_size=512) - train_dataloader = DataLoader( - train_dataset, - sampler=train_sampler, - batch_size=args.train_batch_size, - collate_fn=model_collate_fn, - ) - - # Training schedule - if args.max_steps > 0: - t_total = args.max_steps - args.num_train_epochs = t_total // ( - len(train_dataloader) // args.gradient_accumulation_steps + 1 - ) - else: - t_total = ( - len(train_dataloader) - // args.gradient_accumulation_steps - * args.num_train_epochs - ) - - # Prepare the optimizer - lr = {"encoder": 0.002, "decoder": 0.2} - warmup_steps = {"encoder": 20000, "decoder": 10000} - optimizer = BertSumOptimizer(model, lr, warmup_steps) - - # Train - logger.info("***** Running training *****") - logger.info(" Num examples = %d", len(train_dataset)) - logger.info(" Num Epochs = %d", args.num_train_epochs) - logger.info( - " Instantaneous batch size per GPU = %d", args.per_gpu_train_batch_size - ) - logger.info( - " Total train batch size (w. parallel, distributed & accumulation) = %d", - args.train_batch_size * args.gradient_accumulation_steps - # * (torch.distributed.get_world_size() if args.local_rank != -1 else 1), - ) - logger.info(" Gradient Accumulation steps = %d", args.gradient_accumulation_steps) - logger.info(" Total optimization steps = %d", t_total) - - model.zero_grad() - train_iterator = trange(args.num_train_epochs, desc="Epoch", disable=True) - - global_step = 0 - tr_loss = 0.0 - for _ in train_iterator: - epoch_iterator = tqdm(train_dataloader, desc="Iteration", disable=True) - for step, batch in enumerate(epoch_iterator): - source, target, encoder_token_type_ids, encoder_mask, decoder_mask, lm_labels = batch - - source = source.to(args.device) - target = target.to(args.device) - encoder_token_type_ids = encoder_token_type_ids.to(args.device) - encoder_mask = encoder_mask.to(args.device) - decoder_mask = decoder_mask.to(args.device) - lm_labels = lm_labels.to(args.device) - - model.train() - outputs = model( - source, - target, - encoder_token_type_ids=encoder_token_type_ids, - encoder_attention_mask=encoder_mask, - decoder_attention_mask=decoder_mask, - decoder_lm_labels=lm_labels, - ) - - loss = outputs[0] - print(loss) - if args.gradient_accumulation_steps > 1: - loss /= args.gradient_accumulation_steps - - loss.backward() - - tr_loss += loss.item() - if (step + 1) % args.gradient_accumulation_steps == 0: - torch.nn.utils.clip_grad_norm_(model.parameters(), args.max_grad_norm) - optimizer.step() - model.zero_grad() - global_step += 1 - - if args.max_steps > 0 and global_step > args.max_steps: - epoch_iterator.close() - break - - if args.max_steps > 0 and global_step > args.max_steps: - train_iterator.close() - break - - return global_step, tr_loss / global_step - - -# ------------ -# Train -# ------------ - - -def evaluate(args, model, tokenizer, prefix=""): - set_seed(args) - - args.eval_batch_size = args.per_gpu_eval_batch_size * max(1, args.n_gpu) - eval_dataset = load_and_cache_examples(args, tokenizer, evaluate=True) - eval_sampler = SequentialSampler(eval_dataset) - eval_dataloader = DataLoader( - eval_dataset, sampler=eval_sampler, batch_size=args.eval_batch_size - ) - - # multi-gpu evaluate - if args.n_gpu > 1: - model = torch.nn.DataParallel(model) - - logger.info("***** Running evaluation {} *****".format(prefix)) - logger.info(" Num examples = %d", len(eval_dataset)) - logger.info(" Batch size = %d", args.eval_batch_size) - eval_loss = 0.0 - nb_eval_steps = 0 - model.eval() - - for batch in tqdm(eval_dataloader, desc="Evaluating"): - source, target, encoder_token_type_ids, encoder_mask, decoder_mask, lm_labels = batch - - source = source.to(args.device) - target = target.to(args.device) - encoder_token_type_ids = encoder_token_type_ids.to(args.device) - encoder_mask = encoder_mask.to(args.device) - decoder_mask = decoder_mask.to(args.device) - lm_labels = lm_labels.to(args.device) - - with torch.no_grad(): - outputs = model( - source, - target, - encoder_token_type_ids=encoder_token_type_ids, - encoder_attention_mask=encoder_mask, - decoder_attention_mask=decoder_mask, - decoder_lm_labels=lm_labels, - ) - lm_loss = outputs[0] - eval_loss += lm_loss.mean().item() - nb_eval_steps += 1 - - eval_loss = eval_loss / nb_eval_steps - perplexity = torch.exp(torch.tensor(eval_loss)) - - result = {"perplexity": perplexity} - - # Save the evaluation's results - output_eval_file = os.path.join(args.output_dir, "eval_results.txt") - if not os.path.exists(args.output_dir): - os.makedirs(args.output_dir) - - with open(output_eval_file, "w") as writer: - logger.info("***** Eval results {} *****".format(prefix)) - for key in sorted(result.keys()): - logger.info(" %s = %s", key, str(result[key])) - writer.write("%s = %s\n" % (key, str(result[key]))) - - return result - - -def save_model_checkpoints(args, model, tokenizer): - if not os.path.exists(args.output_dir): - os.makedirs(args.output_dir) - - logger.info("Saving model checkpoint to %s", args.output_dir) - - # Save a trained model, configuration and tokenizer using `save_pretrained()`. - # They can then be reloaded using `from_pretrained()` - model_to_save = ( - model.module if hasattr(model, "module") else model - ) # Take care of distributed/parallel training - model_to_save.save_pretrained(args.output_dir, model_type='bert') - tokenizer.save_pretrained(args.output_dir) - torch.save(args, os.path.join(args.output_dir, "training_arguments.bin")) - - -def main(): - parser = argparse.ArgumentParser() - - # Required parameters - parser.add_argument( - "--data_dir", - default=None, - type=str, - required=True, - help="The input training data file (a text file).", - ) - parser.add_argument( - "--output_dir", - default=None, - type=str, - required=True, - help="The output directory where the model predictions and checkpoints will be written.", - ) - - # Optional parameters - parser.add_argument( - "--gradient_accumulation_steps", - type=int, - default=1, - help="Number of updates steps to accumulate before performing a backward/update pass.", - ) - parser.add_argument( - "--do_evaluate", - type=bool, - default=False, - help="Run model evaluation on out-of-sample data.", - ) - parser.add_argument("--do_train", type=bool, default=False, help="Run training.") - parser.add_argument( - "--do_overwrite_output_dir", - type=bool, - default=False, - help="Whether to overwrite the output dir.", - ) - parser.add_argument( - "--model_name_or_path", - default="bert-base-cased", - type=str, - help="The model checkpoint to initialize the encoder and decoder's weights with.", - ) - parser.add_argument( - "--model_type", - default="bert", - type=str, - help="The decoder architecture to be fine-tuned.", - ) - parser.add_argument( - "--max_grad_norm", default=1.0, type=float, help="Max gradient norm." - ) - parser.add_argument( - "--max_steps", - default=-1, - type=int, - help="If > 0: set total number of training steps to perform. Override num_train_epochs.", - ) - parser.add_argument( - "--to_cpu", default=False, type=bool, help="Whether to force training on CPU." - ) - parser.add_argument( - "--num_train_epochs", - default=10, - type=int, - help="Total number of training epochs to perform.", - ) - parser.add_argument( - "--per_gpu_train_batch_size", - default=4, - type=int, - help="Batch size per GPU/CPU for training.", - ) - parser.add_argument("--seed", default=42, type=int) - args = parser.parse_args() - - if ( - os.path.exists(args.output_dir) - and os.listdir(args.output_dir) - and args.do_train - and not args.do_overwrite_output_dir - ): - raise ValueError( - "Output directory ({}) already exists and is not empty. Use --do_overwrite_output_dir to overwrite.".format( - args.output_dir - ) - ) - - # Set up training device - if args.to_cpu or not torch.cuda.is_available(): - args.device = torch.device("cpu") - args.n_gpu = 0 - else: - args.device = torch.device("cuda") - args.n_gpu = torch.cuda.device_count() - - # Load pretrained model and tokenizer. The decoder's weights are randomly initialized. - tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path) - config = BertConfig.from_pretrained(args.model_name_or_path) - decoder_model = BertForMaskedLM(config) - model = Model2Model.from_pretrained( - args.model_name_or_path, decoder_model=decoder_model - ) - - # Setup logging - logging.basicConfig( - format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", - datefmt="%m/%d/%Y %H:%M:%S", - level=logging.INFO, - ) - logger.warning( - "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s", - 0, - args.device, - args.n_gpu, - False, - False, - ) - - logger.info("Training/evaluation parameters %s", args) - - # Train the model - model.to(args.device) - if args.do_train: - try: - global_step, tr_loss = train(args, model, tokenizer) - except KeyboardInterrupt: - response = input("You interrupted the training. Do you want to save the model checkpoints? [Y/n]") - if response.lower() in ["", "y", "yes"]: - save_model_checkpoints(args, model, tokenizer) - sys.exit(0) - - logger.info(" global_step = %s, average loss = %s", global_step, tr_loss) - save_model_checkpoints(args, model, tokenizer) - - # Evaluate the model - results = {} - if args.do_evaluate: - checkpoints = [args.output_dir] - logger.info("Evaluate the following checkpoints: %s", checkpoints) - for checkpoint in checkpoints: - encoder_checkpoint = os.path.join(checkpoint, "bert_encoder") - decoder_checkpoint = os.path.join(checkpoint, "bert_decoder") - model = PreTrainedEncoderDecoder.from_pretrained( - encoder_checkpoint, decoder_checkpoint - ) - model.to(args.device) - print("model loaded") - - return results - - -if __name__ == "__main__": - main() diff --git a/examples/utils_summarization.py b/examples/utils_summarization.py index 087c88bd4e..7cbd4cd61b 100644 --- a/examples/utils_summarization.py +++ b/examples/utils_summarization.py @@ -25,9 +25,8 @@ class CNNDailyMailDataset(Dataset): [2] https://github.com/abisee/cnn-dailymail/ """ - def __init__(self, tokenizer, prefix="train", data_dir=""): + def __init__(self, data_dir="", prefix="train"): assert os.path.isdir(data_dir) - self.tokenizer = tokenizer # We initialize the class by listing all the files that contain # stories and summaries. Files are not read in memory given @@ -104,31 +103,30 @@ def _add_missing_period(line): # -------------------------- -def fit_to_block_size(sequence, block_size, pad_token): +def fit_to_block_size(sequence, block_size, pad_token_id): """ Adapt the source and target sequences' lengths to the block size. - If the sequence is shorter than the block size we pad it with -1 ids - which correspond to padding tokens. + If the sequence is shorter we append padding token to the right of the sequence. """ if len(sequence) > block_size: return sequence[:block_size] else: - sequence.extend([pad_token] * (block_size - len(sequence))) + sequence.extend([pad_token_id] * (block_size - len(sequence))) return sequence -def build_lm_labels(sequence, pad_token): - """ Padding token, encoded as 0, are represented by the value -1 so they +def build_lm_labels(sequence, pad_token_id): + """ Padding token are replaced by the value -1 so they are not taken into account in the loss computation. """ padded = sequence.clone() - padded[padded == pad_token] = -1 + padded[padded == pad_token_id] = -1 return padded -def build_mask(sequence, pad_token): +def build_mask(sequence, pad_token_id): """ Builds the mask. The attention mechanism will only attend to positions with value 1. """ mask = torch.ones_like(sequence) - idx_pad_tokens = sequence == pad_token + idx_pad_tokens = sequence == pad_token_id mask[idx_pad_tokens] = 0 return mask diff --git a/transformers/generate/__init__.py b/transformers/generate/__init__.py new file mode 100644 index 0000000000..21ac612155 --- /dev/null +++ b/transformers/generate/__init__.py @@ -0,0 +1 @@ +from .beam_search import BeamSearch diff --git a/transformers/generate/beam_search.py b/transformers/generate/beam_search.py new file mode 100644 index 0000000000..09e340a150 --- /dev/null +++ b/transformers/generate/beam_search.py @@ -0,0 +1,358 @@ +# coding=utf-8 +# MIT License + +# Copyright (c) 2017-Present OpenNMT + +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +# of the Software, and to permit persons to whom the Software is furnished to do +# so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +""" +Use Beam Search to generate sequences using encoder-decoder models. +""" +import torch +from torch import nn + + +class BeamSearch(nn.Module): + def __init__( + self, + model, + tokenizer, + beam_size, + min_length, + max_length, + batch_size=1, + alpha=0, + block_repeating_trigrams=True, + ): + r""" + Inputs: + **model**: instance of ``transformers.PreTrainedEncoderDecoder`` + The pretrained encoder-decoder model that will be used to generate the sequences. + **tokenizer**: instance of ``transformers.PreTrainedTokenizer`` + The pretrained tokenizer associated to the model used in the encoder-decoder. We only + support encoder-decoder that use the same tokenizer for encoder and decoder. The tokenizer + needs to be initialized or this function will raise and exception. + **batch_size**: (`optional`) int + Batch size of the inputs. The value is set automatically when calling `forward`. + **beam_size**: int + Number of beams that are used for each element on the batch. + **min_length**: int + Minimum number of steps performed by the beam search before terminating. + **max_length**: int + Maximum number of steps performed by the beam search. Any beam that has not finished + will return its current solution with the highest probability. The sequence that is + returned has a length of max_length-1 to account for the end token that is subsequently added. + **alpha**: float + Parameter of the length penalty. Read the documentation of the `_length_penalty` method for mode details. + **block_repeating_trigrams**: bool + Whether to block sequences that have repeating 3-grams. + """ + super(BeamSearch, self).__init__() + self.model = model + self.tokenizer = tokenizer + + self.bos_token_id = tokenizer.bos_token_id + self.eos_token_id = tokenizer.eos_token_id + self.pad_token_id = tokenizer.pad_token_id + + self.batch_size = batch_size + self.beam_size = beam_size + self.min_length = min_length + self.max_length = max_length + + self.block_repeating_trigram = block_repeating_trigrams + self.apply_length_penalty = False if alpha == 0 else True + self.alpha = alpha + + self._init_beam_state(batch_size) + + def __len__(self): + try: + return self.growing_beams.size(1) + except NameError: + return 0 + + def _init_beam_state(self, batch_size): + """ (re-)Initialize the state of the beams. """ + self.hypotheses = [[] for _ in range(batch_size)] + self.batch_offset = torch.arange(batch_size, dtype=torch.long) + self.beam_offset = torch.arange( + 0, batch_size * self.beam_size, step=self.beam_size, dtype=torch.long + ) + self.growing_beams = torch.full( + (batch_size * self.beam_size, 1), self.bos_token_id, dtype=torch.long + ) + self.topk_log_probabilities = torch.tensor( + [0.0] + [float("-inf")] * (self.beam_size - 1), dtype=torch.float + ).repeat(batch_size) + self.results = { + "predictions": [[] for _ in range(batch_size)], + "scores": [[] for _ in range(batch_size)], + } + self._step = 0 + self.is_done = False + + def forward(self, encoder_input_ids, **model_kwargs): + """ Generate a sequence using Beam Search. """ + # keyword arguments come in 3 flavors: encoder-specific (prefixed by + # `encoder_`), decoder-specific (prefixed by `decoder_`) and those + # that apply to the model as whole. + # We let the specific kwargs override the common ones in case of conflict. + kwargs_common = { + argument: value + for argument, value in model_kwargs.items() + if not argument.startswith("encoder_") and not argument.startswith("decoder_") + } + kwargs_decoder = kwargs_common.copy() + kwargs_encoder = kwargs_common.copy() + kwargs_encoder.update( + { + argument[len("encoder_") :]: value + for argument, value in model_kwargs.items() + if argument.startswith("encoder_") + } + ) + kwargs_decoder.update( + { + argument[len("decoder_") :]: value + for argument, value in model_kwargs.items() + if argument.startswith("decoder_") + } + ) + + # forward pass on the encoder + encoder_outputs = self.model.encoder.forward(encoder_input_ids, kwargs_encoder) + kwargs_decoder["encoder_hidden_states"] = tile( + encoder_outputs, self.beam_size, dim=0 + ) + + # grow the beam by generating sequences in an autoregressive way + batch_size = encoder_input_ids.size(0) + self._init_beam_state(batch_size) + for step in range(self.max_length): + # prepare the decoder input + decoder_input = fit_to_block_size( + self.growing_beams, self.tokenizer.pad_token_id + ) + kwargs_decoder["decoder_lm_labels"] = build_lm_labels( + decoder_input, self.tokenizer.pad_token_id + ) + kwargs_decoder["decoder_attention_mask"] = build_mask( + decoder_input, self.tokenizer.pad_token_id + ) + + outputs = self.model.decoder(decoder_input, kwargs_decoder) + log_probabilities = torch.nn.functional.log_softmax(outputs[1]) + surviving_beams_rows = self.grow(log_probabilities) + if self.is_done: + break + + kwargs_decoder["encoder_hidden_states"] = kwargs_decoder[ + "encoder_hidden_states" + ].index_select(0, surviving_beams_rows) + kwargs_decoder["encoder_attention_mask"] = kwargs_decoder[ + "encoder_attention_mask" + ].index_select(0, surviving_beams_rows) + + return self.results + + def grow(self, log_probabilities): + """ Grow the beams by one step. """ + self._step += 1 + + # The number of beams changes as some beams finish so we define _B + vocab_size = log_probabilities.size(-1) + _B = log_probabilities.size(0) // self.beam_size + + # Multiply each beam probability with the probability of the + # next token (conditioned on the words in the beam). + log_probabilities += self.topk_log_probabilities.view(-1, 1) + + self._enforce_min_length(log_probabilities) + if self.block_repeating_trigram: + self._remove_beams_with_repeating_trigrams(log_probabilities, _B) + + # Find the `beam_size` (previous_beam + token) combinations with + # the highest score + topk_log_probabilities, topk_ids = torch.topk( + log_probabilities.view(_B, self.beam_size * vocab_size), self.beam_size, dim=1 + ) + + # Apply the length penalty. The +1 accounts for the [EOS] token + # that will be added if the beam ends. + topk_scores = topk_log_probabilities + if self.apply_length_penalty: + topk_scores /= self._length_penalty() + + # Retrieve the corresponding respective beam and token id + # topk_token_ids[i] will be added to topk_beam_ids[i] + topk_beam_ids = topk_ids.div(vocab_size) + topk_token_ids = topk_ids.fmod(vocab_size) + + # Retrieve the row index of the surviving beams in the original + # view of the log_probabilities tensor + surviving_beams_per_batch = topk_beam_ids + self.beam_offset[:_B].view(-1, 1) + surviving_beams_rows = surviving_beams_per_batch.view(-1) + + # Append the last predictions + self.growing_beams = torch.cat( + [ + self.growing_beams.index_select(0, surviving_beams_rows), + topk_token_ids.view(-1, 1), + ], + 1, + ) + + # Check if any of the beam searches has ended during this + # growth step. Also if top beam (most probable) has ended + # for one element of the batch. + is_finished = topk_token_ids.eq(self.eos_token_id) + self._enforce_max_length(is_finished) + if is_finished.any(): + non_finished = self._cut_finished(is_finished, topk_scores) + self.batch_offset = self.batch_offset.index_select(0, non_finished) + surviving_beams_per_batch = surviving_beams_per_batch.index_select( + 0, non_finished + ) + self.topk_log_probabilities = self.topk_log_probabilities.index_select( + 0, non_finished + ) + + surviving_beams_rows = surviving_beams_per_batch.view(-1) + self.growing_beams = self.growing_beams.index_select(0, surviving_beams_rows) + + return surviving_beams_rows + + def _cut_finished(self, is_finished, topk_scores): + """ Save the finished searches and cut the correponding sequences off + the beams. """ + is_top_beam_finished = is_finished[:, 0].eq(True) + + # Save the finished searches + predictions = self.growing_beams.view( + -1, self.beam_size, self.growing_beams.size(1) + ) + for i in range(is_finished.size(0)): + if is_top_beam_finished[i]: + is_finished[i].fill_(1) + finished_hyp = is_finished[i].nonzero().view(-1) + + # Store the finished beams as a (score, prediction) hypothesis. + b = self.batch_offset[i] + for j in finished_hyp: + self.hypotheses[b].append((topk_scores[i, j], predictions[i, j, :])) + + # If the batch reached the end, save the best hypotheses + # in terms of length-penalized score. + if is_top_beam_finished[i]: + best_score, best_prediction = max(self.hypotheses[b], key=lambda x: x[0]) + self.results["scores"][b].append(best_score) + self.results["predictions"][b].append(best_prediction) + + non_finished = is_top_beam_finished.eq(False).nonzero().view(-1) + if len(non_finished) == 0: + self.is_done = True + + return non_finished + + def _remove_beams_with_repeating_trigrams(self, log_probabilities, _B): + if self._step + 1 > 3: # [BOS] does not count + for i in range(_B * self.beam_size): + tokens = self.growing_beams[i] + trigrams = [ + (tokens[j - 1], tokens[j], tokens[j + 1]) + for j in range(1, len(self) - 1) + ] + last_trigram = tuple(trigrams[-1]) + if last_trigram in trigrams[:-1]: + log_probabilities[i] = -1e20 + + def _enforce_min_length(self, log_probabilities): + if self._step < self.min_length: + log_probabilities[:, self.eos_token_id] = -1e20 + + def _enforce_max_length(self, is_finished): + # +1 because we will need to add an [EOS] token + if self._step + 1 == self.max_length: + is_finished.fill_(1) + + def _length_penalty(self): + """ The calculation of the length penalty follows that of [1]. + + [1] Wu, Yonghui, et al. "Google's neural machine translation system: + Bridging the gap between human and machine translation." arXiv preprint + arXiv:1609.08144 (2016). + """ + return ((5.0 + (self._step + 1)) / 6.0) ** self.alpha + + +def tile(x, count, dim=0): + """ + Tiles `x` along dimension `dim` `count` times. + + Example: + >> ex = torch.tensor([1,2],[3,4]) + >> tile(ex, 2, 0) + torch.Tensor([[1,2],[1,2],[3,4],[3,4]]) + """ + perm = list(range(len(x.size()))) + if dim != 0: + perm[0], perm[dim] = perm[dim], perm[0] + x = x.permute(perm).contiguous() + out_size = list(x.size()) + out_size[0] *= count + batch = x.size(0) + x = ( + x.view(batch, -1) + .transpose(0, 1) + .repeat(count, 1) + .transpose(0, 1) + .contiguous() + .view(*out_size) + ) + if dim != 0: + x = x.permute(perm).contiguous() + return x + + +def fit_to_block_size(sequence, block_size, pad_token_id): + """ Adapt the source and target sequences' lengths to the block size. + If the sequence is shorter we append padding tokens to the right. + """ + if len(sequence) > block_size: + return sequence[:block_size] + else: + sequence.extend([pad_token_id] * (block_size - len(sequence))) + return sequence + + +def build_lm_labels(sequence, pad_token_id): + """ Padding token, encoded as 0, are represented by the value -1 so they + are not taken into account in the loss computation. """ + padded = sequence.clone() + padded[padded == pad_token_id] = -1 + return padded + + +def build_mask(sequence, pad_token_id): + """ Builds the mask. The attention mechanism will only attend to positions + with value 1. """ + mask = torch.ones_like(sequence) + idx_pad_tokens = sequence == pad_token_id + mask[idx_pad_tokens] = 0 + return mask diff --git a/transformers/modeling_beam_search.py b/transformers/modeling_beam_search.py deleted file mode 100644 index 171dcb7247..0000000000 --- a/transformers/modeling_beam_search.py +++ /dev/null @@ -1,271 +0,0 @@ -# coding=utf-8 -# Copyright (c) 2019 Yang Liu - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -""" -A general wrapper around models with LM heads to generate sequences -using beam search. -""" -import torch -from torch import nn - - -class TransformerBeamSearch(nn.Module): - def __init__( - self, - model, - tokenizer, - batch_size, - beam_size, - min_length, - max_length, - alpha=0, - block_repeating_trigram=True, - ): - """ - Attributes: - mask_word_id: token id that corresponds to the mask - """ - super(TransformerBeamSearch, self).__init__() - self.model = model - self.tokenizer = tokenizer - - self.start_token_id = tokenizer.start_token_id - self.end_token_id = tokenizer.end_token_id - self.pad_token_id = tokenizer.pad_token_id - - self.beam_size = beam_size - self.min_length = min_length - self.max_length = max_length - - self.block_repeating_trigram = block_repeating_trigram - self.apply_length_penalty = False if alpha == 0 else True - self.alpha = alpha - - # State of the beam - self.hypotheses = [[] for _ in range(batch_size)] - self.batch_offset = torch.arange(batch_size, dtype=torch.long) - self.beam_offset = torch.arange( - 0, batch_size * self.beam_size, step=self.beam_size, dtype=torch.long - ) - self.growing_beam = torch.full( - (batch_size * self.beam_size, 1), self.start_token_id, dtype=torch.long - ) - self.topk_log_probabilities = torch.tensor( - [0.0] + [float("-inf")] * (self.beam_size - 1), dtype=torch.float - ).repeat(batch_size) - self.results = { - "prediction": [[] for _ in batch_size], - "scores": [[] for _ in batch_size], - } - self._step = 0 - self.is_done = False - - def step(self, log_probabilities): - """ Grows the beam by one step. """ - self._step += 1 - - # The batch size changes as some beams finish so we define _B - vocab_size = log_probabilities.size(-1) - _B = log_probabilities.size(0) // self.beam_size - - # Multiply each beam probability with the probability of the - # next token (conditioned on the words in the beam). - log_probabilities += self.topk_log_probabilities.view(-1, 1) - - self.enforce_min_length(log_probabilities) - if self.block_repeating_trigram: - self.remove_repeating_trigrams(log_probabilities, _B) - - # Find the `beam_size` (previous_beam + token) combinations with - # the highest score - topk_log_probabilities, topk_ids = log_probabilities.topk( - log_probabilities.view(_B, self.beam_size * vocab_size), - self.beam_size, - dim=1, - ) - - # Apply the length penalty. The +1 accounts for the [EOS] token - # that will be added if the beam ends. - topk_scores = topk_log_probabilities / self.length_penalty() - - # Retrieve the corresponding respective beam and token id - # topk_token_ids[i] will be added to topk_beam_ids[i] - topk_beam_ids = topk_ids.div(vocab_size) - topk_token_ids = topk_ids.fmod(vocab_size) - - # Retrieve the row index of the surviving beams in the original - # view of the log_probabilities tensor - surviving_beams_rows = (topk_beam_ids + self.beam_offset[:_B].view(-1, 1)).view( - -1 - ) - - # Append the last predictions - self.growing_beam = torch.cat( - [ - self.growing_beam.index_select(0, surviving_beams_rows), - topk_token_ids.view(-1, 1), - ], - 1, - ) - - # Check if any of the beam searches has ended during this - # growth step. Also if top beam (most probable) has ended - # for one element of the batch. - is_finished = topk_token_ids.eq(self.end_token_id) - self.enforce_max_length() - is_top_beam_finished = is_finished[:, 0].eq(1) - - # Save the finished searches - if is_finished.any(): - predictions = self.growing_beam.view( - -1, self.beam_size, self.growing_beam.size(1) - ) - for i in range(is_finished.size(0)): - if is_top_beam_finished[i]: - is_finished[i].fill_(1) - finished_hyp = is_finished[i].nonzero().view(-1) - - # Store finished hypotheses for this batch. - b = self.batch_offset[i] - for j in finished_hyp: - self.hypotheses[b].append((topk_scores[i, j], predictions[i, j, :])) - - # If the batch reached the end, save the best hypotheses - # in terms of length-penalized score. - if is_top_beam_finished[i]: - best_hyp = sorted( - self.hypotheses[b], key=lambda x: x[0], reverse=True - ) - best_score, best_prediction = best_hyp[0] - self.results["scores"][b].append(best_score) - self.results["predictions"][b].append(best_prediction) - - non_finished = is_top_beam_finished.eq(0).nonzero().view(-1) - if len(non_finished) == 0: - self.is_done = True - - # Remove finished batches for the next step. - topk_log_probabilities = topk_log_probabilities.index_select( - 0, non_finished - ) - self.batch_offset = self.batch_offset.index_select(0, non_finished) - self.growing_beam = predictions.index_select(0, non_finished).view( - -1, self.growing_beam.size(-1) - ) - - surviving_beams_rows = surviving_beams_rows.index_select(0, non_finished) - - return surviving_beams_rows - - def forward(self, encoder_input_ids, **kwargs): - # keyword arguments come in 3 flavors: encoder-specific (prefixed by - # `encoder_`), decoder-specific (prefixed by `decoder_`) and those - # that apply to the model as whole. - # We let the specific kwargs override the common ones in case of conflict. - kwargs_encoder = { - argument[len("encoder_"):]: value - for argument, value in kwargs.items() - if argument.startswith("encoder_") - } - kwargs_decoder = { - argument[len("decoder_"):]: value - for argument, value in kwargs.items() - if argument.startswith("decoder_") - } - kwargs_common = { - argument: value - for argument, value in kwargs.items() - if not (argument.startswith("encoder_") or argument.startswith("decoder_")) - } - kwargs_decoder = dict(kwargs_common, **kwargs_decoder) - kwargs_encoder = dict(kwargs_common, **kwargs_encoder) - - # forward pass on the encoder - encoder_outputs = self.model.encoder.forward(encoder_input_ids, kwargs_encoder) - kwargs_decoder["encoder_hidden_states"] = tile( - encoder_outputs, self.beam_size, dim=0 - ) - - # grow the beam by generating sequences in an autoregressive way - self.growing_beam = torch.full( - (self.batch_size * self.beam_size, 1), self.start_token_id, dtype=torch.long - ) - for step in range(self.max_length): - decoder_input = self.growing_beam[:, -1] - outputs = self.model.decoder(decoder_input, kwargs_decoder) - log_probabilities = torch.nn.functional.log_softmax(outputs[1]) - surviving_beams_rows = self.step(log_probabilities) - if self.is_done: - break - - kwargs_decoder["encoder_hidden_states"] = kwargs_decoder[ - "encoder_hidden_states" - ].index_select(0, surviving_beams_rows) - - return self.results - - def remove_repeating_trigrams(self, log_probabilities, _B): - if(self._step + 1 > 3): - for i in range(_B * self.beam_size): - tokens = [t for t in self.growing_beam[i]] - trigrams = [(tokens[i-1], tokens[i], tokens[i+1]) for i in range(1, len(words) - 1)] - last_trigram = tuple(trigrams[-1]) - if last_trigram in trigrams[:-1]: - log_probabilities[i] = -1e20 - - def enforce_min_length(self): - if self._step < self.min_length: - self.log_probabilities[self.end_token_id] = -1e20 - - def enforce_max_length(self): - if self._step + 1 == self.max_length: - self.is_finished.fill_(1) - - def length_penalty(self): - return ((5.0 + (self._step + 1)) / 6.0) ** self.alpha - - -def tile(x, count, dim=0): - """ - Tiles `x` along dimension `dim` `count` times. - - Example: - >> ex = torch.tensor([1,2],[3,4]) - >> tile(ex, 2, 0) - torch.Tensor([[1,2],[1,2],[3,4],[3,4]]) - """ - perm = list(range(len(x.size()))) - if dim != 0: - perm[0], perm[dim] = perm[dim], perm[0] - x = x.permute(perm).contiguous() - out_size = list(x.size()) - out_size[0] *= count - batch = x.size(0) - x = ( - x.view(batch, -1) - .transpose(0, 1) - .repeat(count, 1) - .transpose(0, 1) - .contiguous() - .view(*out_size) - ) - if dim != 0: - x = x.permute(perm).contiguous() - return x diff --git a/transformers/tests/beam_search_tests.py b/transformers/tests/beam_search_tests.py new file mode 100644 index 0000000000..a92ebf3578 --- /dev/null +++ b/transformers/tests/beam_search_tests.py @@ -0,0 +1,226 @@ +from collections import namedtuple +import unittest + +import numpy as np +import torch + +from transformers.generate import BeamSearch +from transformers import PreTrainedEncoderDecoder + + +StubTokenizer = namedtuple("Tokenizer", ["bos_token_id", "eos_token_id", "pad_token_id"]) +StubTransformer = namedtuple("Transformer", ["encoder", "decoder"]) + + +class BeamSearchtest(unittest.TestCase): + def test_beam_search_encoder_decoder_integration(self): + """ We make sure that no internal change in the PreTrainedEncoderDecoder + class will break the integration with the beam search. + """ + + model = PreTrainedEncoderDecoder("encoder", "decoder") + tokenizer = StubTokenizer(0, 1, 2) + try: + _ = BeamSearch( + model=model, + tokenizer=tokenizer, + batch_size=1, + beam_size=1, + min_length=1, + max_length=1, + alpha=0, + block_repeating_trigrams=False, + ) + except: + self.fail("Instantiating BeamSearch with a PreTrainedEncoderDecoder failed.") + + def test_beam_search_min_length(self): + """ We keep predicting the end_token for the first beam and check that + it is not marked as finished until the beam has reached the minimum + length. """ + eos_idx = 3 + vocab_size = 10 + + batch_size = 3 + beam_size = 2 + min_length = 5 + + beam = BeamSearch( + model=StubTransformer("encoder", "decoder"), + tokenizer=StubTokenizer(bos_token_id=0, eos_token_id=eos_idx, pad_token_id=2), + batch_size=batch_size, + beam_size=beam_size, + min_length=5, + max_length=10, + alpha=0, + block_repeating_trigrams=False, + ) + + # To test that the minimum length is correctly enforced we constantly + # assign the highest probability to the [EOS] token (and assign lower + # probabilities to some other tokens). + # Since BeamSearch will reset its probability to 1e-20 as long as + # min_length has not been reached, we need to reset the value between + # steps. + non_eos_idxs = [4, 5, 1, 8, 9] + score_distribution = torch.log_softmax( + torch.tensor([6.0, 5.0, 4.0, 3.0, 2.0, 1.0]), dim=0 + ) + + log_probabilities = torch.full((batch_size * beam_size, vocab_size), float("-inf")) + log_probabilities[0, eos_idx] = score_distribution[0] + for idx, score in zip(non_eos_idxs, score_distribution[1:]): + log_probabilities[0, idx] = score + + for step in range(1, min_length + 2): + log_probabilities[0, eos_idx] = score_distribution[0] + + # Beam #3 and #4 teminate at the first step since the probability + # of the [EOS] token is -1e20 > -\infty so there are only two beams left. + surviving_beams_rows = beam.grow(log_probabilities) + if step < min_length: + np.testing.assert_array_equal( + beam.growing_beams.numpy(), + np.repeat(np.array([[0] + [4] * step]), 2, axis=0), + ) + elif step == min_length: + np.testing.assert_array_equal(surviving_beams_rows.numpy(), np.array([])) + self.assertTrue(beam.is_done) + break + + log_probabilities = log_probabilities.index_select(0, surviving_beams_rows) + + def test_beam_search_max_length(self): + """ We keep predicting the same non-EOS token until we reach the + maximum permitted length """ + batch_size = 3 + beam_size = 2 + max_length = 5 + vocab_size = 10 + + beam = BeamSearch( + model=StubTransformer("encoder", "decoder"), + tokenizer=StubTokenizer(bos_token_id=0, eos_token_id=1, pad_token_id=2), + batch_size=batch_size, + beam_size=beam_size, + min_length=2, + max_length=max_length, + alpha=0, + block_repeating_trigrams=False, + ) + + log_probabilities = torch.full((batch_size * beam_size, vocab_size), float("-inf")) + + # To test that beam search enforces the max length constraint we + # keep giving the highest probability to a token that is not the + # [EOS] token. + # The beam search will stop at max_length-1, assuming that one would + # add the [EOS] token at the end of the returned sequence. + token_idxs = [3, 4, 5] + score_distribution = torch.log_softmax(torch.tensor([10.0, 6.0, 4.0]), dim=0) + for idx, score in zip(token_idxs, score_distribution): + log_probabilities[:, idx] = score + + for step in range(1, max_length + 2): + surviving_beams_rows = beam.grow(log_probabilities) + if step + 1 < max_length: + self.assertFalse(beam.is_done) + elif step + 1 == max_length: # Now [EOS] is the most probable token + np.testing.assert_array_equal(surviving_beams_rows.numpy(), np.array([])) + self.assertTrue(beam.is_done) + break + + log_probabilities = log_probabilities.index_select(0, surviving_beams_rows) + + def test_beam_search_block_repeating_trigrams(self): + """ We make sure that the beams that contain repeating trigrams are removed. """ + batch_size = 3 + beam_size = 2 + max_length = 10 + vocab_size = 10 + + beam = BeamSearch( + model=StubTransformer("encoder", "decoder"), + tokenizer=StubTokenizer(bos_token_id=0, eos_token_id=1, pad_token_id=2), + batch_size=batch_size, + beam_size=beam_size, + min_length=2, + max_length=max_length, + alpha=0, + block_repeating_trigrams=True, + ) + + log_probabilities = torch.full((batch_size * beam_size, vocab_size), float("-inf")) + + # To test that BeamSearch enforces the 3-gram constraint we give the + # highest probably to the same tokens in a cyclic fashion and make sure + # they disappear once the cycle has completed. + token_idxs = [3, 4, 5] + score_distribution = torch.log_softmax(torch.tensor([10.0, 6.0, 4.0]), dim=0) + for idx, score in zip(token_idxs, score_distribution): + log_probabilities[:, idx] = score + + for step in range(1, max_length + 2): + # Rotate the probabilities at each step + for idx in token_idxs: + score = score_distribution[(idx + step) % 3] + log_probabilities[::beam_size, idx] = score + + surviving_beams_rows = beam.grow(log_probabilities) + log_probabilities = log_probabilities.index_select(0, surviving_beams_rows) + + if step < 7: + self.assertFalse( + np.array_equal( + log_probabilities.numpy()[0, :], + np.array([-1e20] * vocab_size, dtype="float32"), + ) + ) + if step == 7: + np.testing.assert_array_equal( + log_probabilities.numpy()[0, :], + np.array([-1e20] * vocab_size, dtype="float32"), + ) + + def test_beam_search_example_for_one_step(self): + """ We test that the predictions for one step of growth are correct. """ + batch_size = 2 + beam_size = 2 + max_length = 10 + vocab_size = 5 + + beam = BeamSearch( + model=StubTransformer("encoder", "decoder"), + tokenizer=StubTokenizer(bos_token_id=0, eos_token_id=1, pad_token_id=2), + batch_size=batch_size, + beam_size=beam_size, + min_length=2, + max_length=max_length, + alpha=0, + block_repeating_trigrams=False, + ) + + log_probabilities = torch.full((batch_size * beam_size, vocab_size), float("-inf")) + log_probabilities[0, 3:] = torch.log_softmax(torch.tensor([2.0, 1.0]), dim=0) + log_probabilities[2, 3:] = torch.log_softmax(torch.tensor([1.0, 2.0]), dim=0) + + # First pass + surviving_beams_rows = beam.grow(log_probabilities) + np.testing.assert_array_equal(surviving_beams_rows.numpy(), np.array([0, 0, 2, 2])) + np.testing.assert_array_equal( + beam.growing_beams.numpy(), np.array([[0, 3], [0, 4], [0, 4], [0, 3]]) + ) + self.assertFalse(beam.is_done) + + # Second pass + surviving_beams_rows = beam.grow(log_probabilities) + np.testing.assert_array_equal(surviving_beams_rows.numpy(), np.array([0, 0, 2, 2])) + np.testing.assert_array_equal( + beam.growing_beams.numpy(), + np.array([[0, 3, 3], [0, 3, 4], [0, 4, 4], [0, 4, 3]]), + ) + self.assertFalse(beam.is_done) + + +if __name__ == "__name__": + unittest.main() From ba089c780b918414bd8b669e1764fed728753edf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Louf?= Date: Wed, 6 Nov 2019 13:55:24 +0100 Subject: [PATCH 268/505] share pretrained embeddings --- examples/utils_summarization.py | 11 +--- requirements.txt | 4 +- transformers/generate/beam_search.py | 87 ++++++++++++++++++---------- 3 files changed, 60 insertions(+), 42 deletions(-) diff --git a/examples/utils_summarization.py b/examples/utils_summarization.py index 7cbd4cd61b..8e95a04e19 100644 --- a/examples/utils_summarization.py +++ b/examples/utils_summarization.py @@ -136,18 +136,11 @@ def encode_for_summarization(story_lines, summary_lines, tokenizer): as specified in [1] by using `[SEP] [CLS]` tokens to separate sentences. """ - story_lines_token_ids = [ - tokenizer.build_inputs_with_special_tokens(tokenizer.encode(line)) - for line in story_lines - ] - summary_lines_token_ids = [ - tokenizer.build_inputs_with_special_tokens(tokenizer.encode(line)) - for line in summary_lines - ] - + story_lines_token_ids = [tokenizer.encode(line) for line in story_lines] story_token_ids = [ token for sentence in story_lines_token_ids for token in sentence ] + summary_lines_token_ids = [tokenizer.encode(line) for line in summary_lines] summary_token_ids = [ token for sentence in summary_lines_token_ids for token in sentence ] diff --git a/requirements.txt b/requirements.txt index 9c43abc6d7..060aba915d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,4 +9,6 @@ regex # For XLNet sentencepiece # For XLM -sacremoses \ No newline at end of file +sacremoses +# For ROUGE +pyrouge diff --git a/transformers/generate/beam_search.py b/transformers/generate/beam_search.py index 09e340a150..e1b2d23da0 100644 --- a/transformers/generate/beam_search.py +++ b/transformers/generate/beam_search.py @@ -26,27 +26,31 @@ Use Beam Search to generate sequences using encoder-decoder models. import torch from torch import nn +import logging + + +logger = logging.getLogger(__name__) + class BeamSearch(nn.Module): def __init__( self, model, - tokenizer, + bos_token_id, + pad_token_id, + eos_token_id, + batch_size, beam_size, min_length, max_length, - batch_size=1, alpha=0, block_repeating_trigrams=True, + device=torch.device("cpu"), ): r""" Inputs: **model**: instance of ``transformers.PreTrainedEncoderDecoder`` The pretrained encoder-decoder model that will be used to generate the sequences. - **tokenizer**: instance of ``transformers.PreTrainedTokenizer`` - The pretrained tokenizer associated to the model used in the encoder-decoder. We only - support encoder-decoder that use the same tokenizer for encoder and decoder. The tokenizer - needs to be initialized or this function will raise and exception. **batch_size**: (`optional`) int Batch size of the inputs. The value is set automatically when calling `forward`. **beam_size**: int @@ -64,11 +68,11 @@ class BeamSearch(nn.Module): """ super(BeamSearch, self).__init__() self.model = model - self.tokenizer = tokenizer + self.device = device - self.bos_token_id = tokenizer.bos_token_id - self.eos_token_id = tokenizer.eos_token_id - self.pad_token_id = tokenizer.pad_token_id + self.bos_token_id = bos_token_id + self.eos_token_id = eos_token_id + self.pad_token_id = pad_token_id self.batch_size = batch_size self.beam_size = beam_size @@ -90,15 +94,24 @@ class BeamSearch(nn.Module): def _init_beam_state(self, batch_size): """ (re-)Initialize the state of the beams. """ self.hypotheses = [[] for _ in range(batch_size)] - self.batch_offset = torch.arange(batch_size, dtype=torch.long) + self.batch_offset = torch.arange(batch_size, dtype=torch.long, device=self.device) self.beam_offset = torch.arange( - 0, batch_size * self.beam_size, step=self.beam_size, dtype=torch.long + 0, + batch_size * self.beam_size, + step=self.beam_size, + dtype=torch.long, + device=self.device, ) self.growing_beams = torch.full( - (batch_size * self.beam_size, 1), self.bos_token_id, dtype=torch.long + (batch_size * self.beam_size, 1), + self.bos_token_id, + dtype=torch.long, + device=self.device, ) self.topk_log_probabilities = torch.tensor( - [0.0] + [float("-inf")] * (self.beam_size - 1), dtype=torch.float + [0.0] + [float("-inf")] * (self.beam_size - 1), + dtype=torch.float, + device=self.device, ).repeat(batch_size) self.results = { "predictions": [[] for _ in range(batch_size)], @@ -136,28 +149,37 @@ class BeamSearch(nn.Module): ) # forward pass on the encoder - encoder_outputs = self.model.encoder.forward(encoder_input_ids, kwargs_encoder) + encoder_outputs = self.model.encoder(encoder_input_ids, **kwargs_encoder) + encoder_hidden_states = encoder_outputs[0] kwargs_decoder["encoder_hidden_states"] = tile( - encoder_outputs, self.beam_size, dim=0 + encoder_hidden_states, self.beam_size, dim=0 + ) + kwargs_decoder["encoder_attention_mask"] = tile( + kwargs_encoder["attention_mask"], self.beam_size, dim=0 ) # grow the beam by generating sequences in an autoregressive way - batch_size = encoder_input_ids.size(0) + batch_size, block_size = encoder_input_ids.size() self._init_beam_state(batch_size) for step in range(self.max_length): - # prepare the decoder input - decoder_input = fit_to_block_size( - self.growing_beams, self.tokenizer.pad_token_id - ) - kwargs_decoder["decoder_lm_labels"] = build_lm_labels( - decoder_input, self.tokenizer.pad_token_id - ) - kwargs_decoder["decoder_attention_mask"] = build_mask( - decoder_input, self.tokenizer.pad_token_id + # Add padding tokens + decoder_input = torch.full( + (self.growing_beams.size(0), block_size), + self.pad_token_id, + dtype=torch.long, + device=self.growing_beams.device, ) + decoder_input[:, : self.growing_beams.size(1)] = self.growing_beams - outputs = self.model.decoder(decoder_input, kwargs_decoder) - log_probabilities = torch.nn.functional.log_softmax(outputs[1]) + # compute decoder_attention_mask + decoder_mask = torch.ones_like(decoder_input) + idx_pad_tokens = decoder_input == self.pad_token_id + decoder_mask[idx_pad_tokens] = 0 + kwargs_decoder["attention_mask"] = decoder_mask + + outputs = self.model.decoder(decoder_input, **kwargs_decoder) + last_token_scores = outputs[0][:, -1, :].squeeze(1) + log_probabilities = torch.nn.functional.log_softmax(last_token_scores, dim=0) surviving_beams_rows = self.grow(log_probabilities) if self.is_done: break @@ -189,13 +211,13 @@ class BeamSearch(nn.Module): # Find the `beam_size` (previous_beam + token) combinations with # the highest score - topk_log_probabilities, topk_ids = torch.topk( + self.topk_log_probabilities, topk_ids = torch.topk( log_probabilities.view(_B, self.beam_size * vocab_size), self.beam_size, dim=1 ) # Apply the length penalty. The +1 accounts for the [EOS] token # that will be added if the beam ends. - topk_scores = topk_log_probabilities + topk_scores = self.topk_log_probabilities if self.apply_length_penalty: topk_scores /= self._length_penalty() @@ -337,8 +359,9 @@ def fit_to_block_size(sequence, block_size, pad_token_id): if len(sequence) > block_size: return sequence[:block_size] else: - sequence.extend([pad_token_id] * (block_size - len(sequence))) - return sequence + return torch.cat( + (sequence, torch.tensor([pad_token_id] * (block_size - len(sequence)))), dim=0 + ) def build_lm_labels(sequence, pad_token_id): From 4735c2af0715c24d47b34c167fb7d5543493b87d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Louf?= Date: Fri, 8 Nov 2019 11:16:26 +0100 Subject: [PATCH 269/505] tweaks to the BeamSearch API --- transformers/generate/beam_search.py | 63 ++++++++++--------------- transformers/tests/beam_search_tests.py | 53 ++++++++++++++------- 2 files changed, 59 insertions(+), 57 deletions(-) diff --git a/transformers/generate/beam_search.py b/transformers/generate/beam_search.py index e1b2d23da0..a18d20f31a 100644 --- a/transformers/generate/beam_search.py +++ b/transformers/generate/beam_search.py @@ -32,7 +32,7 @@ import logging logger = logging.getLogger(__name__) -class BeamSearch(nn.Module): +class BeamSearch(object): def __init__( self, model, @@ -45,12 +45,17 @@ class BeamSearch(nn.Module): max_length, alpha=0, block_repeating_trigrams=True, - device=torch.device("cpu"), ): r""" Inputs: **model**: instance of ``transformers.PreTrainedEncoderDecoder`` The pretrained encoder-decoder model that will be used to generate the sequences. + **bos_token_id**: int + Id that is used by the tokenizer to represent the beggining of a sentence. + **pad_token_id**: int + Id that is used by the tokenizer for padding. + **eos_token_id**: int + Id that is used by the tokenizer to represent the end of a sentence. **batch_size**: (`optional`) int Batch size of the inputs. The value is set automatically when calling `forward`. **beam_size**: int @@ -68,7 +73,7 @@ class BeamSearch(nn.Module): """ super(BeamSearch, self).__init__() self.model = model - self.device = device + self.device = next(model.parameters()).device # only works if all parameters of the model are stored on a single GPU self.bos_token_id = bos_token_id self.eos_token_id = eos_token_id @@ -86,10 +91,7 @@ class BeamSearch(nn.Module): self._init_beam_state(batch_size) def __len__(self): - try: - return self.growing_beams.size(1) - except NameError: - return 0 + return self.growing_beams.size(1) def _init_beam_state(self, batch_size): """ (re-)Initialize the state of the beams. """ @@ -120,7 +122,7 @@ class BeamSearch(nn.Module): self._step = 0 self.is_done = False - def forward(self, encoder_input_ids, **model_kwargs): + def __call__(self, encoder_input_ids, **model_kwargs): """ Generate a sequence using Beam Search. """ # keyword arguments come in 3 flavors: encoder-specific (prefixed by # `encoder_`), decoder-specific (prefixed by `decoder_`) and those @@ -158,28 +160,17 @@ class BeamSearch(nn.Module): kwargs_encoder["attention_mask"], self.beam_size, dim=0 ) - # grow the beam by generating sequences in an autoregressive way + # grow the beam iteratively batch_size, block_size = encoder_input_ids.size() self._init_beam_state(batch_size) for step in range(self.max_length): - # Add padding tokens - decoder_input = torch.full( - (self.growing_beams.size(0), block_size), - self.pad_token_id, - dtype=torch.long, - device=self.growing_beams.device, - ) - decoder_input[:, : self.growing_beams.size(1)] = self.growing_beams - - # compute decoder_attention_mask - decoder_mask = torch.ones_like(decoder_input) - idx_pad_tokens = decoder_input == self.pad_token_id - decoder_mask[idx_pad_tokens] = 0 - kwargs_decoder["attention_mask"] = decoder_mask + decoder_input = fit_to_block_size(self.growing_beams, block_size, self.pad_token_id) + kwargs_decoder["attention_mask"] = build_mask(decoder_input) outputs = self.model.decoder(decoder_input, **kwargs_decoder) - last_token_scores = outputs[0][:, -1, :].squeeze(1) - log_probabilities = torch.nn.functional.log_softmax(last_token_scores, dim=0) + + next_token_scores = outputs[0][:, -1, :].squeeze(1) + log_probabilities = torch.nn.functional.log_softmax(next_token_scores, dim=0) surviving_beams_rows = self.grow(log_probabilities) if self.is_done: break @@ -356,20 +347,14 @@ def fit_to_block_size(sequence, block_size, pad_token_id): """ Adapt the source and target sequences' lengths to the block size. If the sequence is shorter we append padding tokens to the right. """ - if len(sequence) > block_size: - return sequence[:block_size] - else: - return torch.cat( - (sequence, torch.tensor([pad_token_id] * (block_size - len(sequence)))), dim=0 - ) - - -def build_lm_labels(sequence, pad_token_id): - """ Padding token, encoded as 0, are represented by the value -1 so they - are not taken into account in the loss computation. """ - padded = sequence.clone() - padded[padded == pad_token_id] = -1 - return padded + padded_sequence = torch.full( + (sequence.size(0), block_size), + pad_token_id, + dtype=torch.long, + device=sequence.device, + ) + padded_sequence[:, : sequence.size(1)] = sequence + return sequence def build_mask(sequence, pad_token_id): diff --git a/transformers/tests/beam_search_tests.py b/transformers/tests/beam_search_tests.py index a92ebf3578..6f2a2b9c2f 100644 --- a/transformers/tests/beam_search_tests.py +++ b/transformers/tests/beam_search_tests.py @@ -1,15 +1,22 @@ from collections import namedtuple import unittest - +import pytest import numpy as np import torch +from torch import nn from transformers.generate import BeamSearch from transformers import PreTrainedEncoderDecoder -StubTokenizer = namedtuple("Tokenizer", ["bos_token_id", "eos_token_id", "pad_token_id"]) -StubTransformer = namedtuple("Transformer", ["encoder", "decoder"]) +class StubTransformer(nn.Module): + def __init__(self): + self.encoder = None + self.decoder = None + self._parameters = {"dumy": torch.tensor([1])} + + def forward(self): + pass class BeamSearchtest(unittest.TestCase): @@ -18,12 +25,13 @@ class BeamSearchtest(unittest.TestCase): class will break the integration with the beam search. """ - model = PreTrainedEncoderDecoder("encoder", "decoder") - tokenizer = StubTokenizer(0, 1, 2) + model = StubTransformer() try: _ = BeamSearch( model=model, - tokenizer=tokenizer, + bos_token_id=0, + eos_token_id=1, + pad_token_id=2, batch_size=1, beam_size=1, min_length=1, @@ -46,8 +54,10 @@ class BeamSearchtest(unittest.TestCase): min_length = 5 beam = BeamSearch( - model=StubTransformer("encoder", "decoder"), - tokenizer=StubTokenizer(bos_token_id=0, eos_token_id=eos_idx, pad_token_id=2), + model=StubTransformer(), + bos_token_id=0, + eos_token_id=eos_idx, + pad_token_id=2, batch_size=batch_size, beam_size=beam_size, min_length=5, @@ -71,17 +81,17 @@ class BeamSearchtest(unittest.TestCase): log_probabilities[0, eos_idx] = score_distribution[0] for idx, score in zip(non_eos_idxs, score_distribution[1:]): log_probabilities[0, idx] = score - + pytest.set_trace() for step in range(1, min_length + 2): log_probabilities[0, eos_idx] = score_distribution[0] # Beam #3 and #4 teminate at the first step since the probability # of the [EOS] token is -1e20 > -\infty so there are only two beams left. + # The top beam (most likely) always ends with 4 until we reach min_length. surviving_beams_rows = beam.grow(log_probabilities) if step < min_length: np.testing.assert_array_equal( - beam.growing_beams.numpy(), - np.repeat(np.array([[0] + [4] * step]), 2, axis=0), + beam.growing_beams.numpy()[0, :], np.array([0] + [4] * step) ) elif step == min_length: np.testing.assert_array_equal(surviving_beams_rows.numpy(), np.array([])) @@ -99,8 +109,10 @@ class BeamSearchtest(unittest.TestCase): vocab_size = 10 beam = BeamSearch( - model=StubTransformer("encoder", "decoder"), - tokenizer=StubTokenizer(bos_token_id=0, eos_token_id=1, pad_token_id=2), + model=StubTransformer(), + bos_token_id=0, + eos_token_id=1, + pad_token_id=2, batch_size=batch_size, beam_size=beam_size, min_length=2, @@ -140,8 +152,10 @@ class BeamSearchtest(unittest.TestCase): vocab_size = 10 beam = BeamSearch( - model=StubTransformer("encoder", "decoder"), - tokenizer=StubTokenizer(bos_token_id=0, eos_token_id=1, pad_token_id=2), + model=StubTransformer(), + bos_token_id=0, + eos_token_id=1, + pad_token_id=2, batch_size=batch_size, beam_size=beam_size, min_length=2, @@ -167,7 +181,6 @@ class BeamSearchtest(unittest.TestCase): log_probabilities[::beam_size, idx] = score surviving_beams_rows = beam.grow(log_probabilities) - log_probabilities = log_probabilities.index_select(0, surviving_beams_rows) if step < 7: self.assertFalse( @@ -182,6 +195,8 @@ class BeamSearchtest(unittest.TestCase): np.array([-1e20] * vocab_size, dtype="float32"), ) + log_probabilities = log_probabilities.index_select(0, surviving_beams_rows) + def test_beam_search_example_for_one_step(self): """ We test that the predictions for one step of growth are correct. """ batch_size = 2 @@ -190,8 +205,10 @@ class BeamSearchtest(unittest.TestCase): vocab_size = 5 beam = BeamSearch( - model=StubTransformer("encoder", "decoder"), - tokenizer=StubTokenizer(bos_token_id=0, eos_token_id=1, pad_token_id=2), + model=StubTransformer(), + bos_token_id=0, + eos_token_id=1, + pad_token_id=2, batch_size=batch_size, beam_size=beam_size, min_length=2, From 9f75565ea8243ec685c3e5dd08a63e8f78af9d0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Louf?= Date: Fri, 8 Nov 2019 15:48:31 +0100 Subject: [PATCH 270/505] setup training --- requirements.txt | 2 -- transformers/generate/beam_search.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 060aba915d..4a3162adce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,5 +10,3 @@ regex sentencepiece # For XLM sacremoses -# For ROUGE -pyrouge diff --git a/transformers/generate/beam_search.py b/transformers/generate/beam_search.py index a18d20f31a..abe3186049 100644 --- a/transformers/generate/beam_search.py +++ b/transformers/generate/beam_search.py @@ -166,7 +166,7 @@ class BeamSearch(object): for step in range(self.max_length): decoder_input = fit_to_block_size(self.growing_beams, block_size, self.pad_token_id) - kwargs_decoder["attention_mask"] = build_mask(decoder_input) + kwargs_decoder["attention_mask"] = build_mask(decoder_input, self.pad_token_id) outputs = self.model.decoder(decoder_input, **kwargs_decoder) next_token_scores = outputs[0][:, -1, :].squeeze(1) From 4d1819990294f27ab1cf0113034f52cdb4136eaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Louf?= Date: Tue, 12 Nov 2019 17:59:34 +0100 Subject: [PATCH 271/505] cast bool tensor to long for pytorch < 1.3 --- transformers/modeling_bert.py | 1 + 1 file changed, 1 insertion(+) diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py index 1ee3e3f097..0159d58aab 100644 --- a/transformers/modeling_bert.py +++ b/transformers/modeling_bert.py @@ -675,6 +675,7 @@ class BertModel(BertPreTrainedModel): batch_size, seq_length = input_shape seq_ids = torch.arange(seq_length, device=device) causal_mask = seq_ids[None, None, :].repeat(batch_size, seq_length, 1) <= seq_ids[None, :, None] + causal_mask = causal_mask.to(torch.long) # not converting to long will cause errors with pytorch version < 1.3 extended_attention_mask = causal_mask[:, None, :, :] * attention_mask[:, None, None, :] else: extended_attention_mask = attention_mask[:, None, None, :] From 2403a6659859ad18a9f20e1c2e84179718d8dfd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Louf?= Date: Sat, 23 Nov 2019 00:18:44 +0100 Subject: [PATCH 272/505] give transformers API to BertAbs --- ..._original_pytorch_checkpoint_to_pytorch.py | 161 +++ .../summarization/configuration_bertabs.py | 141 ++ ...ert_bertabs_original_pytorch_checkpoint.py | 162 +++ examples/summarization/modeling_bertabs.py | 1250 +++++++++++++++++ examples/summarization/run_summarization.py | 271 ++++ .../utils_summarization.py | 56 +- .../utils_summarization_test.py | 17 +- ..._original_pytorch_checkpoint_to_pytorch.py | 158 +++ transformers/generate/beam_search.py | 26 +- 9 files changed, 2188 insertions(+), 54 deletions(-) create mode 100644 examples/convert_bertextabs_original_pytorch_checkpoint_to_pytorch.py create mode 100644 examples/summarization/configuration_bertabs.py create mode 100644 examples/summarization/convert_bertabs_original_pytorch_checkpoint.py create mode 100644 examples/summarization/modeling_bertabs.py create mode 100644 examples/summarization/run_summarization.py rename examples/{ => summarization}/utils_summarization.py (77%) rename examples/{ => summarization}/utils_summarization_test.py (88%) create mode 100644 transformers/convert_bertextabs_original_pytorch_checkpoint_to_pytorch.py diff --git a/examples/convert_bertextabs_original_pytorch_checkpoint_to_pytorch.py b/examples/convert_bertextabs_original_pytorch_checkpoint_to_pytorch.py new file mode 100644 index 0000000000..c245d0eae5 --- /dev/null +++ b/examples/convert_bertextabs_original_pytorch_checkpoint_to_pytorch.py @@ -0,0 +1,161 @@ +# coding=utf-8 +# Copyright 2018 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Convert BertExtAbs's checkpoints """ + +import argparse +from collections import namedtuple +import logging +import pdb +import torch + +from models.model_builder import AbsSummarizer # The authors' implementation +from model_bertabs import BertAbsSummarizer + +from transformers import BertTokenizer + + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +SAMPLE_TEXT = 'Hello world! cécé herlolip' + + +BertAbsConfig = namedtuple( + "BertAbsConfig", + ["temp_dir", "large", "use_bert_emb", "finetune_bert", "encoder", "share_emb", "max_pos", "enc_layers", "enc_hidden_size", "enc_heads", "enc_ff_size", "enc_dropout", "dec_layers", "dec_hidden_size", "dec_heads", "dec_ff_size", "dec_dropout"], +) + + +def convert_bertabs_checkpoints(path_to_checkpoints, dump_path): + """ Copy/paste and tweak the pre-trained weights provided by the creators + of BertAbs for the internal architecture. + """ + + # Instantiate the authors' model with the pre-trained weights + config = BertAbsConfig( + temp_dir=".", + finetune_bert=False, + large=False, + share_emb=True, + use_bert_emb=False, + encoder="bert", + max_pos=512, + enc_layers=6, + enc_hidden_size=512, + enc_heads=8, + enc_ff_size=512, + enc_dropout=0.2, + dec_layers=6, + dec_hidden_size=768, + dec_heads=8, + dec_ff_size=2048, + dec_dropout=0.2, + ) + checkpoints = torch.load(path_to_checkpoints, lambda storage, loc: storage) + original = AbsSummarizer(config, torch.device("cpu"), checkpoints) + original.eval() + + new_model = BertAbsSummarizer(config, torch.device("cpu")) + new_model.eval() + + # ------------------- + # Convert the weights + # ------------------- + + logging.info("convert the model") + new_model.encoder.load_state_dict(original.bert.state_dict()) + + new_model.decoder.generator.load_state_dict(original.generator.state_dict()) + new_model.decoder.embeddings.load_state_dict(original.decoder.embeddings.state_dict()) + new_model.decoder.pos_emb.load_state_dict(original.decoder.pos_emb.state_dict()) + new_model.decoder.transformer_layers.load_state_dict(original.decoder.transformer_layers.state_dict()) + new_model.decoder.layer_norm.load_state_dict(original.decoder.layer_norm.state_dict()) + + # ---------------------------------- + # Make sure the outpus are identical + # ---------------------------------- + + logging.info("Make sure that the models' outputs are identical") + tokenizer = BertTokenizer.from_pretrained("bert-base-uncased") + + # prepare the model inputs + encoder_input_ids = tokenizer.encode("This is sample éàalj'-.") + encoder_input_ids.extend([tokenizer.pad_token_id] * (512 - len(encoder_input_ids))) + encoder_input_ids = torch.tensor(encoder_input_ids).unsqueeze(0) + decoder_input_ids = tokenizer.encode("This is sample 3 éàalj'-.") + decoder_input_ids.extend([tokenizer.pad_token_id] * (512 - len(decoder_input_ids))) + decoder_input_ids = torch.tensor(decoder_input_ids).unsqueeze(0) + + # failsafe to make sure the weights reset does not affect the + # loaded weights. + assert torch.max(torch.abs(original.generator[0].weight - new_model.decoder.generator[0].weight)) == 0 + + # forward pass + src = encoder_input_ids + tgt = decoder_input_ids + segs = token_type_ids = None + clss = None + mask_src = encoder_attention_mask = None + mask_tgt = decoder_attention_mask = None + mask_cls = None + + # The original model does not apply the geneator layer immediatly but rather in + # the beam search (where it combines softmax + linear layer). Since we already + # apply the softmax in our generation process we only apply the linear layer here. + # We make sure that the outputs of the full stack are identical + output_original_model = original(src, tgt, segs, clss, mask_src, mask_tgt, mask_cls)[0] + output_original_model = original.generator(output_original_model) + + output_converted_model = new_model(encoder_input_ids, decoder_input_ids, token_type_ids, encoder_attention_mask, decoder_attention_mask)[0] + output_converted_model = torch.nn.functional.log_softmax(output_converted_model, dim=-1) + + maximum_absolute_difference = torch.max(torch.abs(output_converted_model - output_original_model)).item() + print("Maximum absolute difference beween weights: {:.2f}".format(maximum_absolute_difference)) + + are_identical = torch.allclose(output_converted_model, output_original_model, atol=1e-3) + if are_identical: + logging.info("all weights are equal up to 1e-3") + else: + raise ValueError("the weights are different. The new model is likely different from the original one.") + + # The model has been saved with torch.save(model) and this is bound to the exact + # directory structure. We save the state_dict instead. + logging.info("saving the model's state dictionary") + torch.save(new_model.state_dict(), "bert-ext-abs.pt") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--bertabs_checkpoint_path", + default=None, + type=str, + required=True, + help="Path the official PyTorch dump.", + ) + parser.add_argument( + "--pytorch_dump_folder_path", + default=None, + type=str, + required=True, + help="Path to the output PyTorch model.", + ) + args = parser.parse_args() + + convert_bertabs_checkpoints( + args.bertabs_checkpoint_path, + args.pytorch_dump_folder_path, + ) diff --git a/examples/summarization/configuration_bertabs.py b/examples/summarization/configuration_bertabs.py new file mode 100644 index 0000000000..ff3171f9a8 --- /dev/null +++ b/examples/summarization/configuration_bertabs.py @@ -0,0 +1,141 @@ +# coding=utf-8 +# Copyright 2019 The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" BertAbs configuration """ +import json +import logging +import sys + +from transformers import PretrainedConfig + + +logger = logging.getLogger(__name__) + + +BERTABS_FINETUNED_CONFIG_MAP = { + "bertabs-finetuned-cnndm": "https://s3.amazonaws.com/models.huggingface.co/bert/remi/bertabs-finetuned-cnndm-extractive-abstractive-summarization-config.json", +} + + +class BertAbsConfig(PretrainedConfig): + r""" Class to store the configuration of the BertAbs model. + + Arguments: + temp_dir: string + Unused in the current situation. Kept for compatibility but will be removed. + finetune_bert: bool + Whether to fine-tune the model or not. Will be kept for reference + in case we want to add the possibility to fine-tune the model. + large: bool + Whether to use bert-large as a base. + share_emb: book + Whether the embeddings are shared between the encoder and decoder. + encoder: string + Not clear what this does. Leave to "bert" for pre-trained weights. + max_pos: int + The maximum sequence length that this model will be used with. + enc_layer: int + The numner of hidden layers in the Transformer encoder. + enc_hidden_size: int + The size of the encoder's layers. + enc_heads: int + The number of attention heads for each attention layer in the encoder. + enc_ff_size: int + The size of the encoder's feed-forward layers. + enc_dropout: int + The dropout probabilitiy for all fully connected layers in the + embeddings, layers, pooler and also the attention probabilities in + the encoder. + dec_layer: int + The numner of hidden layers in the decoder. + dec_hidden_size: int + The size of the decoder's layers. + dec_heads: int + The number of attention heads for each attention layer in the decoder. + dec_ff_size: int + The size of the decoder's feed-forward layers. + dec_dropout: int + The dropout probabilitiy for all fully connected layers in the + embeddings, layers, pooler and also the attention probabilities in + the decoder. + """ + + pretrained_config_archive_map = BERTABS_FINETUNED_CONFIG_MAP + + def __init__( + self, + vocab_size_or_config_json_file=30522, + temp_dir=".", + finetune_bert=False, + large=False, + share_emb=True, + encoder="bert", + max_pos=512, + enc_layers=6, + enc_hidden_size=512, + enc_heads=8, + enc_ff_size=512, + enc_dropout=0.2, + dec_layers=6, + dec_hidden_size=768, + dec_heads=8, + dec_ff_size=2048, + dec_dropout=0.2, + **kwargs, + ): + super(BertAbsConfig, self).__init__(**kwargs) + + if self._input_is_path_to_json(vocab_size_or_config_json_file): + path_to_json = vocab_size_or_config_json_file + with open(path_to_json, "r", encoding="utf-8") as reader: + json_config = json.loads(reader.read()) + for key, value in json_config.items(): + self.__dict__[key] = value + elif isinstance(vocab_size_or_config_json_file, int): + self.temp_dir = temp_dir + self.finetune_bert = finetune_bert + self.large = large + self.vocab_size = vocab_size_or_config_json_file + self.max_pos = max_pos + + self.encoder = encoder + self.enc_layers = enc_layers + self.enc_hidden_size = enc_hidden_size + self.enc_heads = enc_heads + self.enc_ff_size = enc_ff_size + self.enc_dropout = enc_dropout + + self.share_emb = share_emb + + self.dec_layers = dec_layers + self.dec_hidden_size = dec_hidden_size + self.dec_heads = dec_heads + self.dec_ff_size = dec_ff_size + self.dec_dropout = dec_dropout + else: + raise ValueError( + "First argument must be either a vocabulary size (int)" + "or the path to a pretrained model config file (str)" + ) + + def _input_is_path_to_json(self, first_argument): + """ Checks whether the first argument passed to config + is the path to a JSON file that contains the config. + """ + is_python_2 = sys.version_info[0] == 2 + if is_python_2: + return isinstance(first_argument, unicode) + else: + return isinstance(first_argument, str) diff --git a/examples/summarization/convert_bertabs_original_pytorch_checkpoint.py b/examples/summarization/convert_bertabs_original_pytorch_checkpoint.py new file mode 100644 index 0000000000..786a29ef13 --- /dev/null +++ b/examples/summarization/convert_bertabs_original_pytorch_checkpoint.py @@ -0,0 +1,162 @@ +# coding=utf-8 +# Copyright 2018 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Convert BertExtAbs's checkpoints + +The file currently does not do much as we ended up copying the exact model +structure, but I leave it here in case we ever want to refactor the model. +""" + +import argparse +from collections import namedtuple +import logging +import torch + +from models.model_builder import AbsSummarizer # The authors' implementation +from model_bertabs import BertAbsSummarizer + +from transformers import BertTokenizer + + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +SAMPLE_TEXT = 'Hello world! cécé herlolip' + + +BertAbsConfig = namedtuple( + "BertAbsConfig", + ["temp_dir", "large", "use_bert_emb", "finetune_bert", "encoder", "share_emb", "max_pos", "enc_layers", "enc_hidden_size", "enc_heads", "enc_ff_size", "enc_dropout", "dec_layers", "dec_hidden_size", "dec_heads", "dec_ff_size", "dec_dropout"], +) + + +def convert_bertabs_checkpoints(path_to_checkpoints, dump_path): + """ Copy/paste and tweak the pre-trained weights provided by the creators + of BertAbs for the internal architecture. + """ + + # Instantiate the authors' model with the pre-trained weights + config = BertAbsConfig( + temp_dir=".", + finetune_bert=False, + large=False, + share_emb=True, + use_bert_emb=False, + encoder="bert", + max_pos=512, + enc_layers=6, + enc_hidden_size=512, + enc_heads=8, + enc_ff_size=512, + enc_dropout=0.2, + dec_layers=6, + dec_hidden_size=768, + dec_heads=8, + dec_ff_size=2048, + dec_dropout=0.2, + ) + checkpoints = torch.load(path_to_checkpoints, lambda storage, loc: storage) + original = AbsSummarizer(config, torch.device("cpu"), checkpoints) + original.eval() + + new_model = BertAbsSummarizer(config, torch.device("cpu")) + new_model.eval() + + # ------------------- + # Convert the weights + # ------------------- + + logging.info("convert the model") + new_model.bert.load_state_dict(original.bert.state_dict()) + new_model.decoder.load_state_dict(original.decoder.state_dict()) + new_model.generator.load_state_dict(original.generator.state_dict()) + + # ---------------------------------- + # Make sure the outpus are identical + # ---------------------------------- + + logging.info("Make sure that the models' outputs are identical") + tokenizer = BertTokenizer.from_pretrained("bert-base-uncased") + + # prepare the model inputs + encoder_input_ids = tokenizer.encode("This is sample éàalj'-.") + encoder_input_ids.extend([tokenizer.pad_token_id] * (512 - len(encoder_input_ids))) + encoder_input_ids = torch.tensor(encoder_input_ids).unsqueeze(0) + decoder_input_ids = tokenizer.encode("This is sample 3 éàalj'-.") + decoder_input_ids.extend([tokenizer.pad_token_id] * (512 - len(decoder_input_ids))) + decoder_input_ids = torch.tensor(decoder_input_ids).unsqueeze(0) + + # failsafe to make sure the weights reset does not affect the + # loaded weights. + assert torch.max(torch.abs(original.generator[0].weight - new_model.generator[0].weight)) == 0 + + # forward pass + src = encoder_input_ids + tgt = decoder_input_ids + segs = token_type_ids = None + clss = None + mask_src = encoder_attention_mask = None + mask_tgt = decoder_attention_mask = None + mask_cls = None + + # The original model does not apply the geneator layer immediatly but rather in + # the beam search (where it combines softmax + linear layer). Since we already + # apply the softmax in our generation process we only apply the linear layer here. + # We make sure that the outputs of the full stack are identical + output_original_model = original(src, tgt, segs, clss, mask_src, mask_tgt, mask_cls)[0] + output_original_generator = original.generator(output_original_model) + + output_converted_model = new_model(encoder_input_ids, decoder_input_ids, token_type_ids, encoder_attention_mask, decoder_attention_mask)[0] + output_converted_generator = new_model.generator(output_converted_model) + + maximum_absolute_difference = torch.max(torch.abs(output_converted_model - output_original_model)).item() + print("Maximum absolute difference beween weights: {:.2f}".format(maximum_absolute_difference)) + maximum_absolute_difference = torch.max(torch.abs(output_converted_generator - output_original_generator)).item() + print("Maximum absolute difference beween weights: {:.2f}".format(maximum_absolute_difference)) + + are_identical = torch.allclose(output_converted_model, output_original_model, atol=1e-3) + if are_identical: + logging.info("all weights are equal up to 1e-3") + else: + raise ValueError("the weights are different. The new model is likely different from the original one.") + + # The model has been saved with torch.save(model) and this is bound to the exact + # directory structure. We save the state_dict instead. + logging.info("saving the model's state dictionary") + torch.save(new_model.state_dict(), "bertabs-finetuned-cnndm-extractive-abstractive-summarization-pytorch_model.bin") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--bertabs_checkpoint_path", + default=None, + type=str, + required=True, + help="Path the official PyTorch dump.", + ) + parser.add_argument( + "--pytorch_dump_folder_path", + default=None, + type=str, + required=True, + help="Path to the output PyTorch model.", + ) + args = parser.parse_args() + + convert_bertabs_checkpoints( + args.bertabs_checkpoint_path, + args.pytorch_dump_folder_path, + ) diff --git a/examples/summarization/modeling_bertabs.py b/examples/summarization/modeling_bertabs.py new file mode 100644 index 0000000000..0189a2ad2b --- /dev/null +++ b/examples/summarization/modeling_bertabs.py @@ -0,0 +1,1250 @@ +# MIT License + +# Copyright (c) 2019 Yang Liu + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +import copy +import math +import shutil +import time +import os + +import numpy as np +import torch +from torch import nn +from torch.nn.init import xavier_uniform_ + +from transformers import BertModel, BertConfig, PreTrainedModel + +from configuration_bertabs import BertAbsConfig + + +MAX_SIZE = 5000 + +BERTABS_FINETUNED_MODEL_MAP = { + "bertabs-finetuned-cnndm": "https://s3.amazonaws.com/models.huggingface.co/bert/remi/bertabs-finetuned-cnndm-extractive-abstractive-summarization-pytorch_model.bin", +} + + +class BertAbsPreTrainedModel(PreTrainedModel): + config_class = BertAbsConfig + pretrained_model_archive_map = BERTABS_FINETUNED_MODEL_MAP + load_tf_weights = False + base_model_prefix = "bert" + + +class BertAbs(BertAbsPreTrainedModel): + def __init__(self, args, checkpoint=None, bert_extractive_checkpoint=None): + super(BertAbs, self).__init__(args) + self.args = args + self.bert = Bert(args.large, args.temp_dir, args.finetune_bert) + + # If pre-trained weights are passed for Bert, load these. + load_bert_pretrained_extractive = True if bert_extractive_checkpoint else False + if load_bert_pretrained_extractive: + self.bert.model.load_state_dict( + dict( + [ + (n[11:], p) + for n, p in bert_extractive_checkpoint.items() + if n.startswith("bert.model") + ] + ), + strict=True, + ) + + if args.encoder == "baseline": + bert_config = BertConfig( + self.bert.model.config.vocab_size, + hidden_size=args.enc_hidden_size, + num_hidden_layers=args.enc_layers, + num_attention_heads=8, + intermediate_size=args.enc_ff_size, + hidden_dropout_prob=args.enc_dropout, + attention_probs_dropout_prob=args.enc_dropout, + ) + self.bert.model = BertModel(bert_config) + + self.vocab_size = self.bert.model.config.vocab_size + + if args.max_pos > 512: + my_pos_embeddings = nn.Embedding( + args.max_pos, self.bert.model.config.hidden_size + ) + my_pos_embeddings.weight.data[ + :512 + ] = self.bert.model.embeddings.position_embeddings.weight.data + my_pos_embeddings.weight.data[ + 512: + ] = self.bert.model.embeddings.position_embeddings.weight.data[-1][ + None, : + ].repeat( + args.max_pos - 512, 1 + ) + self.bert.model.embeddings.position_embeddings = my_pos_embeddings + tgt_embeddings = nn.Embedding( + self.vocab_size, self.bert.model.config.hidden_size, padding_idx=0 + ) + if self.args.share_emb: + tgt_embeddings.weight = copy.deepcopy( + self.bert.model.embeddings.word_embeddings.weight + ) + + self.decoder = TransformerDecoder( + self.args.dec_layers, + self.args.dec_hidden_size, + heads=self.args.dec_heads, + d_ff=self.args.dec_ff_size, + dropout=self.args.dec_dropout, + embeddings=tgt_embeddings, + vocab_size=self.vocab_size, + ) + + gen_func = nn.LogSoftmax(dim=-1) + self.generator = nn.Sequential( + nn.Linear(args.dec_hidden_size, args.vocab_size), gen_func + ) + self.generator[0].weight = self.decoder.embeddings.weight + + load_from_checkpoints = False if checkpoint is None else True + if load_from_checkpoints: + self.load_state_dict(checkpoint) + + def init_weights(self): + for module in self.decoder.modules(): + if isinstance(module, (nn.Linear, nn.Embedding)): + module.weight.data.normal_(mean=0.0, std=0.02) + elif isinstance(module, nn.LayerNorm): + module.bias.data.zero_() + module.weight.data.fill_(1.0) + if isinstance(module, nn.Linear) and module.bias is not None: + module.bias.data.zero_() + for p in self.generator.parameters(): + if p.dim() > 1: + xavier_uniform_(p) + else: + p.data.zero_() + + def maybe_tie_embeddings(self, args): + if args.use_bert_emb: + tgt_embeddings = nn.Embedding( + self.vocab_size, self.bert.model.config.hidden_size, padding_idx=0 + ) + tgt_embeddings.weight = copy.deepcopy( + self.bert.model.embeddings.word_embeddings.weight + ) + self.decoder.embeddings = tgt_embeddings + + def forward( + self, + encoder_input_ids, + decoder_input_ids, + token_type_ids, + encoder_attention_mask, + decoder_attention_mask, + ): + encoder_output = self.bert( + input_ids=encoder_input_ids, + token_type_ids=token_type_ids, + attention_mask=encoder_attention_mask, + ) + encoder_hidden_states = encoder_output[0] + dec_state = self.decoder.init_decoder_state( + encoder_input_ids, encoder_hidden_states + ) + decoder_outputs, _ = self.decoder( + decoder_input_ids[:, :-1], encoder_hidden_states, dec_state + ) + return decoder_outputs + + +class Bert(nn.Module): + """ This class is not really necessary and should probably disappear. + """ + + def __init__(self, large, temp_dir, finetune=False): + super(Bert, self).__init__() + if large: + self.model = BertModel.from_pretrained("bert-large-uncased", cache_dir=temp_dir) + else: + self.model = BertModel.from_pretrained("bert-base-uncased", cache_dir=temp_dir) + + self.finetune = finetune + + def forward(self, input_ids, attention_mask=None, token_type_ids=None, **kwargs): + self.eval() + with torch.no_grad(): + encoder_outputs, _ = self.model( + input_ids, + token_type_ids=token_type_ids, + attention_mask=attention_mask, + **kwargs + ) + return encoder_outputs + + +class TransformerDecoder(nn.Module): + """ + The Transformer decoder from "Attention is All You Need". + + Args: + num_layers (int): number of encoder layers. + d_model (int): size of the model + heads (int): number of heads + d_ff (int): size of the inner FF layer + dropout (float): dropout parameters + embeddings (:obj:`onmt.modules.Embeddings`): + embeddings to use, should have positional encodings + attn_type (str): if using a seperate copy attention + """ + + def __init__(self, num_layers, d_model, heads, d_ff, dropout, embeddings, vocab_size): + super(TransformerDecoder, self).__init__() + + # Basic attributes. + self.decoder_type = "transformer" + self.num_layers = num_layers + self.embeddings = embeddings + self.pos_emb = PositionalEncoding(dropout, self.embeddings.embedding_dim) + + # Build TransformerDecoder. + self.transformer_layers = nn.ModuleList( + [ + TransformerDecoderLayer(d_model, heads, d_ff, dropout) + for _ in range(num_layers) + ] + ) + + self.layer_norm = nn.LayerNorm(d_model, eps=1e-6) + + # forward(input_ids, attention_mask, encoder_hidden_states, encoder_attention_mask) + # def forward(self, input_ids, state, attention_mask=None, memory_lengths=None, + # step=None, cache=None, encoder_attention_mask=None, encoder_hidden_states=None, memory_masks=None): + def forward( + self, + input_ids, + encoder_hidden_states=None, + state=None, + attention_mask=None, + memory_lengths=None, + step=None, + cache=None, + encoder_attention_mask=None, + ): + """ + See :obj:`onmt.modules.RNNDecoderBase.forward()` + memory_bank = encoder_hidden_states + """ + # Name conversion + tgt = input_ids + memory_bank = encoder_hidden_states + memory_mask = encoder_attention_mask + + # src_words = state.src + src_words = state.src + src_batch, src_len = src_words.size() + + padding_idx = self.embeddings.padding_idx + + # Decoder padding mask + tgt_words = tgt + tgt_batch, tgt_len = tgt_words.size() + tgt_pad_mask = ( + tgt_words.data.eq(padding_idx).unsqueeze(1).expand(tgt_batch, tgt_len, tgt_len) + ) + + # Encoder padding mask + if memory_mask is not None: + src_len = memory_mask.size(-1) + src_pad_mask = memory_mask.expand(src_batch, tgt_len, src_len) + else: + src_pad_mask = ( + src_words.data.eq(padding_idx) + .unsqueeze(1) + .expand(src_batch, tgt_len, src_len) + ) + + # Pass through the embeddings + emb = self.embeddings(input_ids) + output = self.pos_emb(emb, step) + assert emb.dim() == 3 # len x batch x embedding_dim + + if state.cache is None: + saved_inputs = [] + + for i in range(self.num_layers): + prev_layer_input = None + if state.cache is None: + if state.previous_input is not None: + prev_layer_input = state.previous_layer_inputs[i] + + output, all_input = self.transformer_layers[i]( + output, + memory_bank, + src_pad_mask, + tgt_pad_mask, + previous_input=prev_layer_input, + layer_cache=state.cache["layer_{}".format(i)] + if state.cache is not None + else None, + step=step, + ) + if state.cache is None: + saved_inputs.append(all_input) + + if state.cache is None: + saved_inputs = torch.stack(saved_inputs) + + output = self.layer_norm(output) + + if state.cache is None: + state = state.update_state(tgt, saved_inputs) + + # Decoders in transformers return a tuple. Beam search will fail + # if we don't follow this convention. + return output, state # , state + + def init_decoder_state(self, src, memory_bank, with_cache=False): + """ Init decoder state """ + state = TransformerDecoderState(src) + if with_cache: + state._init_cache(memory_bank, self.num_layers) + return state + + +class PositionalEncoding(nn.Module): + def __init__(self, dropout, dim, max_len=5000): + pe = torch.zeros(max_len, dim) + position = torch.arange(0, max_len).unsqueeze(1) + div_term = torch.exp( + (torch.arange(0, dim, 2, dtype=torch.float) * -(math.log(10000.0) / dim)) + ) + pe[:, 0::2] = torch.sin(position.float() * div_term) + pe[:, 1::2] = torch.cos(position.float() * div_term) + pe = pe.unsqueeze(0) + super(PositionalEncoding, self).__init__() + self.register_buffer("pe", pe) + self.dropout = nn.Dropout(p=dropout) + self.dim = dim + + def forward(self, emb, step=None): + emb = emb * math.sqrt(self.dim) + if step: + emb = emb + self.pe[:, step][:, None, :] + + else: + emb = emb + self.pe[:, : emb.size(1)] + emb = self.dropout(emb) + return emb + + def get_emb(self, emb): + return self.pe[:, : emb.size(1)] + + +class TransformerDecoderLayer(nn.Module): + """ + Args: + d_model (int): the dimension of keys/values/queries in + MultiHeadedAttention, also the input size of + the first-layer of the PositionwiseFeedForward. + heads (int): the number of heads for MultiHeadedAttention. + d_ff (int): the second-layer of the PositionwiseFeedForward. + dropout (float): dropout probability(0-1.0). + self_attn_type (string): type of self-attention scaled-dot, average + """ + + def __init__(self, d_model, heads, d_ff, dropout): + super(TransformerDecoderLayer, self).__init__() + + self.self_attn = MultiHeadedAttention(heads, d_model, dropout=dropout) + + self.context_attn = MultiHeadedAttention(heads, d_model, dropout=dropout) + self.feed_forward = PositionwiseFeedForward(d_model, d_ff, dropout) + self.layer_norm_1 = nn.LayerNorm(d_model, eps=1e-6) + self.layer_norm_2 = nn.LayerNorm(d_model, eps=1e-6) + self.drop = nn.Dropout(dropout) + mask = self._get_attn_subsequent_mask(MAX_SIZE) + # Register self.mask as a buffer in TransformerDecoderLayer, so + # it gets TransformerDecoderLayer's cuda behavior automatically. + self.register_buffer("mask", mask) + + def forward( + self, + inputs, + memory_bank, + src_pad_mask, + tgt_pad_mask, + previous_input=None, + layer_cache=None, + step=None, + ): + """ + Args: + inputs (`FloatTensor`): `[batch_size x 1 x model_dim]` + memory_bank (`FloatTensor`): `[batch_size x src_len x model_dim]` + src_pad_mask (`LongTensor`): `[batch_size x 1 x src_len]` + tgt_pad_mask (`LongTensor`): `[batch_size x 1 x 1]` + + Returns: + (`FloatTensor`, `FloatTensor`, `FloatTensor`): + + * output `[batch_size x 1 x model_dim]` + * attn `[batch_size x 1 x src_len]` + * all_input `[batch_size x current_step x model_dim]` + + """ + dec_mask = torch.gt( + tgt_pad_mask + self.mask[:, : tgt_pad_mask.size(1), : tgt_pad_mask.size(1)], 0 + ) + input_norm = self.layer_norm_1(inputs) + all_input = input_norm + if previous_input is not None: + all_input = torch.cat((previous_input, input_norm), dim=1) + dec_mask = None + + query = self.self_attn( + all_input, + all_input, + input_norm, + mask=dec_mask, + layer_cache=layer_cache, + type="self", + ) + + query = self.drop(query) + inputs + + query_norm = self.layer_norm_2(query) + mid = self.context_attn( + memory_bank, + memory_bank, + query_norm, + mask=src_pad_mask, + layer_cache=layer_cache, + type="context", + ) + output = self.feed_forward(self.drop(mid) + query) + + return output, all_input + # return output + + def _get_attn_subsequent_mask(self, size): + """ + Get an attention mask to avoid using the subsequent info. + + Args: + size: int + + Returns: + (`LongTensor`): + + * subsequent_mask `[1 x size x size]` + """ + attn_shape = (1, size, size) + subsequent_mask = np.triu(np.ones(attn_shape), k=1).astype("uint8") + subsequent_mask = torch.from_numpy(subsequent_mask) + return subsequent_mask + + +class MultiHeadedAttention(nn.Module): + """ + Multi-Head Attention module from + "Attention is All You Need" + :cite:`DBLP:journals/corr/VaswaniSPUJGKP17`. + + Similar to standard `dot` attention but uses + multiple attention distributions simulataneously + to select relevant items. + + .. mermaid:: + + graph BT + A[key] + B[value] + C[query] + O[output] + subgraph Attn + D[Attn 1] + E[Attn 2] + F[Attn N] + end + A --> D + C --> D + A --> E + C --> E + A --> F + C --> F + D --> O + E --> O + F --> O + B --> O + + Also includes several additional tricks. + + Args: + head_count (int): number of parallel heads + model_dim (int): the dimension of keys/values/queries, + must be divisible by head_count + dropout (float): dropout parameter + """ + + def __init__(self, head_count, model_dim, dropout=0.1, use_final_linear=True): + assert model_dim % head_count == 0 + self.dim_per_head = model_dim // head_count + self.model_dim = model_dim + + super(MultiHeadedAttention, self).__init__() + self.head_count = head_count + + self.linear_keys = nn.Linear(model_dim, head_count * self.dim_per_head) + self.linear_values = nn.Linear(model_dim, head_count * self.dim_per_head) + self.linear_query = nn.Linear(model_dim, head_count * self.dim_per_head) + self.softmax = nn.Softmax(dim=-1) + self.dropout = nn.Dropout(dropout) + self.use_final_linear = use_final_linear + if self.use_final_linear: + self.final_linear = nn.Linear(model_dim, model_dim) + + def forward( + self, + key, + value, + query, + mask=None, + layer_cache=None, + type=None, + predefined_graph_1=None, + ): + """ + Compute the context vector and the attention vectors. + + Args: + key (`FloatTensor`): set of `key_len` + key vectors `[batch, key_len, dim]` + value (`FloatTensor`): set of `key_len` + value vectors `[batch, key_len, dim]` + query (`FloatTensor`): set of `query_len` + query vectors `[batch, query_len, dim]` + mask: binary mask indicating which keys have + non-zero attention `[batch, query_len, key_len]` + Returns: + (`FloatTensor`, `FloatTensor`) : + + * output context vectors `[batch, query_len, dim]` + * one of the attention vectors `[batch, query_len, key_len]` + """ + batch_size = key.size(0) + dim_per_head = self.dim_per_head + head_count = self.head_count + key_len = key.size(1) + query_len = query.size(1) + + def shape(x): + """ projection """ + return x.view(batch_size, -1, head_count, dim_per_head).transpose(1, 2) + + def unshape(x): + """ compute context """ + return ( + x.transpose(1, 2) + .contiguous() + .view(batch_size, -1, head_count * dim_per_head) + ) + + # 1) Project key, value, and query. + if layer_cache is not None: + if type == "self": + query, key, value = ( + self.linear_query(query), + self.linear_keys(query), + self.linear_values(query), + ) + + key = shape(key) + value = shape(value) + + if layer_cache is not None: + device = key.device + if layer_cache["self_keys"] is not None: + key = torch.cat((layer_cache["self_keys"].to(device), key), dim=2) + if layer_cache["self_values"] is not None: + value = torch.cat( + (layer_cache["self_values"].to(device), value), dim=2 + ) + layer_cache["self_keys"] = key + layer_cache["self_values"] = value + elif type == "context": + query = self.linear_query(query) + if layer_cache is not None: + if layer_cache["memory_keys"] is None: + key, value = self.linear_keys(key), self.linear_values(value) + key = shape(key) + value = shape(value) + else: + key, value = ( + layer_cache["memory_keys"], + layer_cache["memory_values"], + ) + layer_cache["memory_keys"] = key + layer_cache["memory_values"] = value + else: + key, value = self.linear_keys(key), self.linear_values(value) + key = shape(key) + value = shape(value) + else: + key = self.linear_keys(key) + value = self.linear_values(value) + query = self.linear_query(query) + key = shape(key) + value = shape(value) + + query = shape(query) + + key_len = key.size(2) + query_len = query.size(2) + + # 2) Calculate and scale scores. + query = query / math.sqrt(dim_per_head) + scores = torch.matmul(query, key.transpose(2, 3)) + + if mask is not None: + mask = mask.unsqueeze(1).expand_as(scores) + scores = scores.masked_fill(mask, -1e18) + + # 3) Apply attention dropout and compute context vectors. + + attn = self.softmax(scores) + + if not predefined_graph_1 is None: + attn_masked = attn[:, -1] * predefined_graph_1 + attn_masked = attn_masked / (torch.sum(attn_masked, 2).unsqueeze(2) + 1e-9) + + attn = torch.cat([attn[:, :-1], attn_masked.unsqueeze(1)], 1) + + drop_attn = self.dropout(attn) + if self.use_final_linear: + context = unshape(torch.matmul(drop_attn, value)) + output = self.final_linear(context) + return output + else: + context = torch.matmul(drop_attn, value) + return context + + +class DecoderState(object): + """Interface for grouping together the current state of a recurrent + decoder. In the simplest case just represents the hidden state of + the model. But can also be used for implementing various forms of + input_feeding and non-recurrent models. + + Modules need to implement this to utilize beam search decoding. + """ + + def detach(self): + """ Need to document this """ + self.hidden = tuple([_.detach() for _ in self.hidden]) + self.input_feed = self.input_feed.detach() + + def beam_update(self, idx, positions, beam_size): + """ Need to document this """ + for e in self._all: + sizes = e.size() + br = sizes[1] + if len(sizes) == 3: + sent_states = e.view(sizes[0], beam_size, br // beam_size, sizes[2])[ + :, :, idx + ] + else: + sent_states = e.view( + sizes[0], beam_size, br // beam_size, sizes[2], sizes[3] + )[:, :, idx] + + sent_states.data.copy_(sent_states.data.index_select(1, positions)) + + def map_batch_fn(self, fn): + raise NotImplementedError() + + +class TransformerDecoderState(DecoderState): + """ Transformer Decoder state base class """ + + def __init__(self, src): + """ + Args: + src (FloatTensor): a sequence of source words tensors + with optional feature tensors, of size (len x batch). + """ + self.src = src + self.previous_input = None + self.previous_layer_inputs = None + self.cache = None + + @property + def _all(self): + """ + Contains attributes that need to be updated in self.beam_update(). + """ + if self.previous_input is not None and self.previous_layer_inputs is not None: + return (self.previous_input, self.previous_layer_inputs, self.src) + else: + return (self.src,) + + def detach(self): + if self.previous_input is not None: + self.previous_input = self.previous_input.detach() + if self.previous_layer_inputs is not None: + self.previous_layer_inputs = self.previous_layer_inputs.detach() + self.src = self.src.detach() + + def update_state(self, new_input, previous_layer_inputs): + state = TransformerDecoderState(self.src) + state.previous_input = new_input + state.previous_layer_inputs = previous_layer_inputs + return state + + def _init_cache(self, memory_bank, num_layers): + self.cache = {} + + for l in range(num_layers): + layer_cache = {"memory_keys": None, "memory_values": None} + layer_cache["self_keys"] = None + layer_cache["self_values"] = None + self.cache["layer_{}".format(l)] = layer_cache + + def repeat_beam_size_times(self, beam_size): + """ Repeat beam_size times along batch dimension. """ + self.src = self.src.data.repeat(1, beam_size, 1) + + def map_batch_fn(self, fn): + def _recursive_map(struct, batch_dim=0): + for k, v in struct.items(): + if v is not None: + if isinstance(v, dict): + _recursive_map(v) + else: + struct[k] = fn(v, batch_dim) + + self.src = fn(self.src, 0) + if self.cache is not None: + _recursive_map(self.cache) + + +def gelu(x): + return ( + 0.5 + * x + * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) + ) + + +class PositionwiseFeedForward(nn.Module): + """ A two-layer Feed-Forward-Network with residual layer norm. + + Args: + d_model (int): the size of input for the first-layer of the FFN. + d_ff (int): the hidden layer size of the second-layer + of the FNN. + dropout (float): dropout probability in :math:`[0, 1)`. + """ + + def __init__(self, d_model, d_ff, dropout=0.1): + super(PositionwiseFeedForward, self).__init__() + self.w_1 = nn.Linear(d_model, d_ff) + self.w_2 = nn.Linear(d_ff, d_model) + self.layer_norm = nn.LayerNorm(d_model, eps=1e-6) + self.actv = gelu + self.dropout_1 = nn.Dropout(dropout) + self.dropout_2 = nn.Dropout(dropout) + + def forward(self, x): + inter = self.dropout_1(self.actv(self.w_1(self.layer_norm(x)))) + output = self.dropout_2(self.w_2(inter)) + return output + x + + +# +# TRANSLATOR +# The following code is used to generate summaries using the +# pre-trained weights and beam search. +# + + +def build_predictor(args, tokenizer, symbols, model, logger=None): + # we should be able to refactor the global scorer a lot + scorer = GNMTGlobalScorer(args.alpha, length_penalty="wu") + translator = Translator( + args, model, tokenizer, symbols, global_scorer=scorer, logger=logger + ) + return translator + + +class GNMTGlobalScorer(object): + """ + NMT re-ranking score from + "Google's Neural Machine Translation System" :cite:`wu2016google` + + Args: + alpha (float): length parameter + beta (float): coverage parameter + """ + + def __init__(self, alpha, length_penalty): + self.alpha = alpha + penalty_builder = PenaltyBuilder(length_penalty) + self.length_penalty = penalty_builder.length_penalty() + + def score(self, beam, logprobs): + """ + Rescores a prediction based on penalty functions + """ + normalized_probs = self.length_penalty(beam, logprobs, self.alpha) + return normalized_probs + + +class PenaltyBuilder(object): + """ + Returns the Length and Coverage Penalty function for Beam Search. + + Args: + length_pen (str): option name of length pen + cov_pen (str): option name of cov pen + """ + + def __init__(self, length_pen): + self.length_pen = length_pen + + def length_penalty(self): + if self.length_pen == "wu": + return self.length_wu + elif self.length_pen == "avg": + return self.length_average + else: + return self.length_none + + """ + Below are all the different penalty terms implemented so far + """ + + def length_wu(self, beam, logprobs, alpha=0.0): + """ + NMT length re-ranking score from + "Google's Neural Machine Translation System" :cite:`wu2016google`. + """ + + modifier = ((5 + len(beam.next_ys)) ** alpha) / ((5 + 1) ** alpha) + return logprobs / modifier + + def length_average(self, beam, logprobs, alpha=0.0): + """ + Returns the average probability of tokens in a sequence. + """ + return logprobs / len(beam.next_ys) + + def length_none(self, beam, logprobs, alpha=0.0, beta=0.0): + """ + Returns unmodified scores. + """ + return logprobs + + +class Translator(object): + """ + Uses a model to translate a batch of sentences. + + Args: + model (:obj:`onmt.modules.NMTModel`): + NMT model to use for translation + fields (dict of Fields): data fields + beam_size (int): size of beam to use + n_best (int): number of translations produced + max_length (int): maximum length output to produce + global_scores (:obj:`GlobalScorer`): + object to rescore final translations + copy_attn (bool): use copy attention during translation + cuda (bool): use cuda + beam_trace (bool): trace beam search for debugging + logger(logging.Logger): logger. + """ + + def __init__(self, args, model, vocab, symbols, global_scorer=None, logger=None): + self.logger = logger + self.cuda = args.visible_gpus != "-1" + + self.args = args + self.model = model + self.generator = self.model.generator + self.vocab = vocab + self.symbols = symbols + self.start_token = symbols["BOS"] + self.end_token = symbols["EOS"] + + self.global_scorer = global_scorer + self.beam_size = args.beam_size + self.min_length = args.min_length + self.max_length = args.max_length + + def translate(self, batch, step, attn_debug=False): + """ Generates summaries from one batch of data. + """ + self.model.eval() + with torch.no_grad(): + batch_data = self.translate_batch(batch) + translations = self.from_batch(batch_data) + return translations + + def translate_batch(self, batch, fast=False): + """ + Translate a batch of sentences. + + Mostly a wrapper around :obj:`Beam`. + + Args: + batch (:obj:`Batch`): a batch from a dataset object + data (:obj:`Dataset`): the dataset object + fast (bool): enables fast beam search (may not support all features) + + Todo: + Shouldn't need the original dataset. + """ + with torch.no_grad(): + return self._fast_translate_batch( + batch, self.max_length, min_length=self.min_length + ) + + # Where the beam search lives + # I have no idea why it is being called from the method above + def _fast_translate_batch(self, batch, max_length, min_length=0): + """ Beam Search using the encoder inputs contained in `batch`. + """ + + # The batch object is funny + # Instead of just looking at the size of the arguments we encapsulate + # a size argument. + # Where is it defined? + beam_size = self.beam_size + batch_size = batch.batch_size + src = batch.src + segs = batch.segs + mask_src = batch.mask_src + + src_features = self.model.bert(src, segs, mask_src) + dec_states = self.model.decoder.init_decoder_state( + src, src_features, with_cache=True + ) + device = src_features.device + + # Tile states and memory beam_size times. + dec_states.map_batch_fn(lambda state, dim: tile(state, beam_size, dim=dim)) + src_features = tile(src_features, beam_size, dim=0) + batch_offset = torch.arange(batch_size, dtype=torch.long, device=device) + beam_offset = torch.arange( + 0, batch_size * beam_size, step=beam_size, dtype=torch.long, device=device + ) + alive_seq = torch.full( + [batch_size * beam_size, 1], self.start_token, dtype=torch.long, device=device + ) + + # Give full probability to the first beam on the first step. + topk_log_probs = torch.tensor( + [0.0] + [float("-inf")] * (beam_size - 1), device=device + ).repeat(batch_size) + + # Structure that holds finished hypotheses. + hypotheses = [[] for _ in range(batch_size)] # noqa: F812 + + results = {} + results["predictions"] = [[] for _ in range(batch_size)] # noqa: F812 + results["scores"] = [[] for _ in range(batch_size)] # noqa: F812 + results["gold_score"] = [0] * batch_size + results["batch"] = batch + + for step in range(max_length): + decoder_input = alive_seq[:, -1].view(1, -1) + + # Decoder forward. + decoder_input = decoder_input.transpose(0, 1) + + dec_out, dec_states = self.model.decoder( + decoder_input, src_features, dec_states, step=step + ) + + # Generator forward. + log_probs = self.generator.forward(dec_out.transpose(0, 1).squeeze(0)) + vocab_size = log_probs.size(-1) + + if step < min_length: + log_probs[:, self.end_token] = -1e20 + + # Multiply probs by the beam probability. + log_probs += topk_log_probs.view(-1).unsqueeze(1) + + alpha = self.global_scorer.alpha + length_penalty = ((5.0 + (step + 1)) / 6.0) ** alpha + + # Flatten probs into a list of possibilities. + curr_scores = log_probs / length_penalty + + if self.args.block_trigram: + cur_len = alive_seq.size(1) + if cur_len > 3: + for i in range(alive_seq.size(0)): + fail = False + words = [int(w) for w in alive_seq[i]] + words = [self.vocab.ids_to_tokens[w] for w in words] + words = " ".join(words).replace(" ##", "").split() + if len(words) <= 3: + continue + trigrams = [ + (words[i - 1], words[i], words[i + 1]) + for i in range(1, len(words) - 1) + ] + trigram = tuple(trigrams[-1]) + if trigram in trigrams[:-1]: + fail = True + if fail: + curr_scores[i] = -10e20 + + curr_scores = curr_scores.reshape(-1, beam_size * vocab_size) + topk_scores, topk_ids = curr_scores.topk(beam_size, dim=-1) + + # Recover log probs. + topk_log_probs = topk_scores * length_penalty + + # Resolve beam origin and true word ids. + topk_beam_index = topk_ids.div(vocab_size) + topk_ids = topk_ids.fmod(vocab_size) + + # Map beam_index to batch_index in the flat representation. + batch_index = topk_beam_index + beam_offset[ + : topk_beam_index.size(0) + ].unsqueeze(1) + select_indices = batch_index.view(-1) + + # Append last prediction. + alive_seq = torch.cat( + [alive_seq.index_select(0, select_indices), topk_ids.view(-1, 1)], -1 + ) + + is_finished = topk_ids.eq(self.end_token) + if step + 1 == max_length: + is_finished.fill_(1) + # End condition is top beam is finished. + end_condition = is_finished[:, 0].eq(1) + # Save finished hypotheses. + if is_finished.any(): + predictions = alive_seq.view(-1, beam_size, alive_seq.size(-1)) + for i in range(is_finished.size(0)): + b = batch_offset[i] + if end_condition[i]: + is_finished[i].fill_(1) + finished_hyp = is_finished[i].nonzero().view(-1) + # Store finished hypotheses for this batch. + for j in finished_hyp: + hypotheses[b].append((topk_scores[i, j], predictions[i, j, 1:])) + # If the batch reached the end, save the n_best hypotheses. + if end_condition[i]: + best_hyp = sorted(hypotheses[b], key=lambda x: x[0], reverse=True) + score, pred = best_hyp[0] + + results["scores"][b].append(score) + results["predictions"][b].append(pred) + non_finished = end_condition.eq(0).nonzero().view(-1) + # If all sentences are translated, no need to go further. + if len(non_finished) == 0: + break + # Remove finished batches for the next step. + topk_log_probs = topk_log_probs.index_select(0, non_finished) + batch_index = batch_index.index_select(0, non_finished) + batch_offset = batch_offset.index_select(0, non_finished) + alive_seq = predictions.index_select(0, non_finished).view( + -1, alive_seq.size(-1) + ) + # Reorder states. + select_indices = batch_index.view(-1) + src_features = src_features.index_select(0, select_indices) + dec_states.map_batch_fn( + lambda state, dim: state.index_select(dim, select_indices) + ) + + return results + + def from_batch(self, translation_batch): + batch = translation_batch["batch"] + assert len(translation_batch["gold_score"]) == len(translation_batch["predictions"]) + batch_size = batch.batch_size + + preds, _, _, tgt_str, src = ( + translation_batch["predictions"], + translation_batch["scores"], + translation_batch["gold_score"], + batch.tgt_str, + batch.src, + ) + + translations = [] + for b in range(batch_size): + pred_sents = self.vocab.convert_ids_to_tokens([int(n) for n in preds[b][0]]) + pred_sents = " ".join(pred_sents).replace(" ##", "") + gold_sent = " ".join(tgt_str[b].split()) + raw_src = [self.vocab.ids_to_tokens[int(t)] for t in src[b]][:500] + raw_src = " ".join(raw_src) + translation = (pred_sents, gold_sent, raw_src) + translations.append(translation) + + return translations + + def _report_rouge(self, gold_path, can_path): + self.logger.info("Calculating Rouge") + results_dict = test_rouge(self.args.temp_dir, can_path, gold_path) + return results_dict + + +def tile(x, count, dim=0): + """ + Tiles x on dimension dim count times. + """ + perm = list(range(len(x.size()))) + if dim != 0: + perm[0], perm[dim] = perm[dim], perm[0] + x = x.permute(perm).contiguous() + out_size = list(x.size()) + out_size[0] *= count + batch = x.size(0) + x = ( + x.view(batch, -1) + .transpose(0, 1) + .repeat(count, 1) + .transpose(0, 1) + .contiguous() + .view(*out_size) + ) + if dim != 0: + x = x.permute(perm).contiguous() + return x + + +# +# All things ROUGE. Uses `pyrouge` which is a hot mess. +# + + +def test_rouge(temp_dir, cand, ref): + candidates = [line.strip() for line in open(cand, encoding="utf-8")] + references = [line.strip() for line in open(ref, encoding="utf-8")] + print(len(candidates)) + print(len(references)) + assert len(candidates) == len(references) + + cnt = len(candidates) + current_time = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) + tmp_dir = os.path.join(temp_dir, "rouge-tmp-{}".format(current_time)) + if not os.path.isdir(tmp_dir): + os.mkdir(tmp_dir) + os.mkdir(tmp_dir + "/candidate") + os.mkdir(tmp_dir + "/reference") + try: + + for i in range(cnt): + if len(references[i]) < 1: + continue + with open( + tmp_dir + "/candidate/cand.{}.txt".format(i), "w", encoding="utf-8" + ) as f: + f.write(candidates[i]) + with open( + tmp_dir + "/reference/ref.{}.txt".format(i), "w", encoding="utf-8" + ) as f: + f.write(references[i]) + r = pyrouge.Rouge155(temp_dir=temp_dir) + r.model_dir = tmp_dir + "/reference/" + r.system_dir = tmp_dir + "/candidate/" + r.model_filename_pattern = "ref.#ID#.txt" + r.system_filename_pattern = r"cand.(\d+).txt" + rouge_results = r.convert_and_evaluate() + print(rouge_results) + results_dict = r.output_to_dict(rouge_results) + finally: + pass + if os.path.isdir(tmp_dir): + shutil.rmtree(tmp_dir) + return results_dict + + +def rouge_results_to_str(results_dict): + return ">> ROUGE-F(1/2/3/l): {:.2f}/{:.2f}/{:.2f}\nROUGE-R(1/2/3/l): {:.2f}/{:.2f}/{:.2f}\n".format( + results_dict["rouge_1_f_score"] * 100, + results_dict["rouge_2_f_score"] * 100, + results_dict["rouge_l_f_score"] * 100, + results_dict["rouge_1_recall"] * 100, + results_dict["rouge_2_recall"] * 100, + results_dict["rouge_l_recall"] * 100, + ) + + +class BertSumOptimizer(object): + """ Specific optimizer for BertSum. + + As described in [1], the authors fine-tune BertSum for abstractive + summarization using two Adam Optimizers with different warm-up steps and + learning rate. They also use a custom learning rate scheduler. + + [1] Liu, Yang, and Mirella Lapata. "Text summarization with pretrained encoders." + arXiv preprint arXiv:1908.08345 (2019). + """ + + def __init__(self, model, lr, warmup_steps, beta_1=0.99, beta_2=0.999, eps=1e-8): + self.encoder = model.encoder + self.decoder = model.decoder + self.lr = lr + self.warmup_steps = warmup_steps + + self.optimizers = { + "encoder": torch.optim.Adam( + model.encoder.parameters(), + lr=lr["encoder"], + betas=(beta_1, beta_2), + eps=eps, + ), + "decoder": torch.optim.Adam( + model.decoder.parameters(), + lr=lr["decoder"], + betas=(beta_1, beta_2), + eps=eps, + ), + } + + self._step = 0 + self.current_learning_rates = {} + + def _update_rate(self, stack): + return self.lr[stack] * min( + self._step ** (-0.5), self._step * self.warmup_steps[stack] ** (-1.5) + ) + + def zero_grad(self): + self.optimizer_decoder.zero_grad() + self.optimizer_encoder.zero_grad() + + def step(self): + self._step += 1 + for stack, optimizer in self.optimizers.items(): + new_rate = self._update_rate(stack) + for param_group in optimizer.param_groups: + param_group["lr"] = new_rate + optimizer.step() + self.current_learning_rates[stack] = new_rate diff --git a/examples/summarization/run_summarization.py b/examples/summarization/run_summarization.py new file mode 100644 index 0000000000..e3b974acd9 --- /dev/null +++ b/examples/summarization/run_summarization.py @@ -0,0 +1,271 @@ +import argparse +from collections import namedtuple +import logging +import os +import sys + +import torch +from torch.utils.data import DataLoader, SequentialSampler +from tqdm import tqdm + +from transformers import BertTokenizer + +from modeling_bertabs import BertAbs, build_predictor + +from utils_summarization import ( + SummarizationDataset, + encode_for_summarization, + build_mask, + fit_to_block_size, + compute_token_type_ids, +) + +logger = logging.getLogger(__name__) +logging.basicConfig(stream=sys.stdout, level=logging.INFO) + + +Batch = namedtuple( + "Batch", ["document_names", "batch_size", "src", "segs", "mask_src", "tgt_str"] +) + + +def evaluate(args): + tokenizer = BertTokenizer.from_pretrained("bert-base-uncased", do_lower_case=True) + model = bertabs = BertAbs.from_pretrained( + "bertabs-finetuned-{}".format(args.finetuned_model) + ) + bertabs.to(args.device) + bertabs.eval() + + symbols = { + "BOS": tokenizer.vocab["[unused0]"], + "EOS": tokenizer.vocab["[unused1]"], + "PAD": tokenizer.vocab["[PAD]"], + } + + # these (unused) arguments are defined to keep the compatibility + # with the legacy code and will be deleted in a next iteration. + args.result_path = "" + args.temp_dir = "" + + data_iterator = build_data_iterator(args, tokenizer) + predictor = build_predictor(args, tokenizer, symbols, model) + + logger.info("***** Running evaluation *****") + logger.info(" Number examples = %d", len(data_iterator.dataset)) + logger.info(" Batch size = %d", args.batch_size) + logger.info("") + logger.info("***** Beam Search parameters *****") + logger.info(" Beam size = %d", args.beam_size) + logger.info(" Minimum length = %d", args.min_length) + logger.info(" Maximum length = %d", args.max_length) + logger.info(" Alpha (length penalty) = %.2f", args.alpha) + logger.info(" Trigrams %s be blocked", ("will" if args.block_trigram else "will NOT")) + + for batch in tqdm(data_iterator): + batch_data = predictor.translate_batch(batch) + translations = predictor.from_batch(batch_data) + summaries = [format_summary(t) for t in translations] + save_summaries(summaries, args.summaries_output_dir, batch.document_names) + + +def format_summary(translation): + """ Transforms the output of the `from_batch` function + into nicely formatted summaries. + """ + raw_summary, _, _ = translation + summary = ( + raw_summary.replace("[unused0]", "") + .replace("[unused3]", "") + .replace("[PAD]", "") + .replace("[unused1]", "") + .replace(r" +", " ") + .replace(" [unused2] ", ". ") + .replace("[unused2]", "") + .strip() + ) + + return summary + + +def save_summaries(summaries, path, original_document_name): + """ Write the summaries in fies that are prefixed by the original + files' name with the `_summary` appended. + + Attributes: + original_document_names: List[string] + Name of the document that was summarized. + path: string + Path were the summaries will be written + summaries: List[string] + The summaries that we produced. + """ + for summary, document_name in zip(summaries, original_document_name): + # Prepare the summary file's name + if "." in document_name: + bare_document_name = ".".join(document_name.split(".")[:-1]) + extension = document_name.split(".")[-1] + name = bare_document_name + "_summary." + extension + else: + name = document_name + "_summary" + + file_path = os.path.join(path, name) + with open(file_path, "w") as output: + output.write(summary) + + +# +# LOAD the dataset +# + + +def build_data_iterator(args, tokenizer): + dataset = load_and_cache_examples(args, tokenizer) + sampler = SequentialSampler(dataset) + collate_fn = lambda data: collate(data, tokenizer, block_size=512) + iterator = DataLoader( + dataset, sampler=sampler, batch_size=args.batch_size, collate_fn=collate_fn, + ) + + return iterator + + +def load_and_cache_examples(args, tokenizer): + dataset = SummarizationDataset(args.documents_dir) + return dataset + + +def collate(data, tokenizer, block_size): + """ Collate formats the data passed to the data loader. + + In particular we tokenize the data batch after batch to avoid keeping them + all in memory. We output the data as a namedtuple to fit the original BertAbs's + API. + """ + data = [x for x in data if not len(x[1]) == 0] # remove empty_files + names = [name for name, _, _ in data] + + encoded_text = [ + encode_for_summarization(story, summary, tokenizer) for _, story, summary in data + ] + stories = torch.tensor( + [ + fit_to_block_size(story, block_size, tokenizer.pad_token_id) + for story, _ in encoded_text + ] + ) + encoder_token_type_ids = compute_token_type_ids(stories, tokenizer.cls_token_id) + encoder_mask = build_mask(stories, tokenizer.pad_token_id) + + batch = Batch( + document_names=names, + batch_size=len(stories), + src=stories, + segs=encoder_token_type_ids, + mask_src=encoder_mask, + tgt_str=[""] * len(stories), + ) + + return batch + + +def decode_summary(summary_tokens, tokenizer): + """ Decode the summary and return it in a format + suitable for evaluation. + """ + summary_tokens = summary_tokens.to("cpu").numpy() + summary = tokenizer.decode(summary_tokens) + sentences = summary.split(".") + sentences = [s + "." for s in sentences] + return sentences + + +def main(): + """ The main function defines the interface with the users. + """ + parser = argparse.ArgumentParser() + parser.add_argument( + "--documents_dir", + default=None, + type=str, + required=True, + help="The folder where the documents to summarize are located.", + ) + parser.add_argument( + "--summaries_output_dir", + default=None, + type=str, + required=True, + help="The folder in wich the summaries should be written.", + ) + # EVALUATION options + parser.add_argument( + "--visible_gpus", + default=-1, + type=int, + help="Number of GPUs with which to do the training.", + ) + parser.add_argument( + "--batch_size", default=4, type=int, help="Batch size per GPU/CPU for training.", + ) + # BEAM SEARCH arguments + parser.add_argument( + "--min_length", + default=50, + type=int, + help="Minimum number of tokens for the summaries.", + ) + parser.add_argument( + "--max_length", + default=200, + type=int, + help="Maixmum number of tokens for the summaries.", + ) + parser.add_argument( + "--beam_size", + default=5, + type=int, + help="The number of beams to start with for each example.", + ) + parser.add_argument( + "--alpha", + default=0.95, + type=float, + help="The value of alpha for the length penalty in the beam search.", + ) + parser.add_argument( + "--block_trigram", + default=True, + type=bool, + help="Whether to block the existence of repeating trigrams in the text generated by beam search.", + ) + args = parser.parse_args() + args.device = torch.device("cpu") if args.visible_gpus == -1 else torch.device("cuda") + + if not documents_dir_is_valid(args.documents_dir): + raise FileNotFoundError( + "We could not find the directory you specified for the documents to summarize, or it was empty. Please specify a valid path." + ) + maybe_create_output_dir(args.summaries_output_dir) + + evaluate(args) + + +def documents_dir_is_valid(path): + if not os.path.exists(path): + return False + + file_list = os.listdir(path) + if len(file_list) == 0: + return False + + return True + + +def maybe_create_output_dir(path): + if not os.path.exists(path): + os.makedirs(path) + + +if __name__ == "__main__": + main() diff --git a/examples/utils_summarization.py b/examples/summarization/utils_summarization.py similarity index 77% rename from examples/utils_summarization.py rename to examples/summarization/utils_summarization.py index 8e95a04e19..e7401b1754 100644 --- a/examples/utils_summarization.py +++ b/examples/summarization/utils_summarization.py @@ -10,9 +10,14 @@ from torch.utils.data import Dataset # ------------ -class CNNDailyMailDataset(Dataset): +class SummarizationDataset(Dataset): """ Abstracts the dataset used to train seq2seq models. + The class will process the documents that are located in the specified + folder. The preprocessing will work on any document that is reasonably + formatted. On the CNN/DailyMail dataset it will extract both the story + and the summary. + CNN/Daily News: The CNN/Daily News raw datasets are downloaded from [1]. The stories are @@ -25,32 +30,31 @@ class CNNDailyMailDataset(Dataset): [2] https://github.com/abisee/cnn-dailymail/ """ - def __init__(self, data_dir="", prefix="train"): - assert os.path.isdir(data_dir) + def __init__(self, path="", prefix="train"): + """ We initialize the class by listing all the documents to summarize. + Files are not read in memory due to the size of some datasets (like CNN/DailyMail). + """ + assert os.path.isdir(path) - # We initialize the class by listing all the files that contain - # stories and summaries. Files are not read in memory given - # the size of the corpus. - self.stories_path = [] - datasets = ("cnn", "dailymail") - for dataset in datasets: - path_to_stories = os.path.join(data_dir, dataset, "stories") - story_filenames_list = os.listdir(path_to_stories) - for story_filename in story_filenames_list: - path_to_story = os.path.join(path_to_stories, story_filename) - if not os.path.isfile(path_to_story): - continue - self.stories_path.append(path_to_story) + self.documents = [] + story_filenames_list = os.listdir(path) + for story_filename in story_filenames_list: + path_to_story = os.path.join(path, story_filename) + if not os.path.isfile(path_to_story): + continue + self.documents.append(path_to_story) def __len__(self): - return len(self.stories_path) + """ Returns the number of documents. """ + return len(self.documents) def __getitem__(self, idx): - story_path = self.stories_path[idx] - with open(story_path, encoding="utf-8") as source: + document_path = self.documents[idx] + document_name = document_path.split("/")[-1] + with open(document_path, encoding="utf-8") as source: raw_story = source.read() story_lines, summary_lines = process_story(raw_story) - return story_lines, summary_lines + return document_name, story_lines, summary_lines def process_story(raw_story): @@ -80,7 +84,7 @@ def process_story(raw_story): story_lines.append(element) except IndexError: # if "@highlight" is absent from the file we pop - # all elements until there is None. + # all elements until there is None, raising an exception. return story_lines, [] # gather summary lines @@ -114,14 +118,6 @@ def fit_to_block_size(sequence, block_size, pad_token_id): return sequence -def build_lm_labels(sequence, pad_token_id): - """ Padding token are replaced by the value -1 so they - are not taken into account in the loss computation. """ - padded = sequence.clone() - padded[padded == pad_token_id] = -1 - return padded - - def build_mask(sequence, pad_token_id): """ Builds the mask. The attention mechanism will only attend to positions with value 1. """ @@ -165,7 +161,7 @@ def compute_token_type_ids(batch, separator_token_id): """ batch_embeddings = [] for sequence in batch: - sentence_num = 0 + sentence_num = -1 embeddings = [] for s in sequence: if s == separator_token_id: diff --git a/examples/utils_summarization_test.py b/examples/summarization/utils_summarization_test.py similarity index 88% rename from examples/utils_summarization_test.py rename to examples/summarization/utils_summarization_test.py index 1d56ff0803..8bfbf6ab23 100644 --- a/examples/utils_summarization_test.py +++ b/examples/summarization/utils_summarization_test.py @@ -21,7 +21,6 @@ from utils_summarization import ( compute_token_type_ids, fit_to_block_size, build_mask, - build_lm_labels, process_story, ) @@ -88,20 +87,6 @@ class SummarizationDataProcessingTest(unittest.TestCase): expected_summary_lines = ["It was the best of times."] self.assertEqual(expected_summary_lines, summary_lines) - def test_build_lm_labels_no_padding(self): - sequence = torch.tensor([1, 2, 3, 4]) - expected = sequence - np.testing.assert_array_equal( - build_lm_labels(sequence, 0).numpy(), expected.numpy() - ) - - def test_build_lm_labels(self): - sequence = torch.tensor([1, 2, 3, 4, 0, 0, 0]) - expected = torch.tensor([1, 2, 3, 4, -1, -1, -1]) - np.testing.assert_array_equal( - build_lm_labels(sequence, 0).numpy(), expected.numpy() - ) - def test_build_mask_no_padding(self): sequence = torch.tensor([1, 2, 3, 4]) expected = torch.tensor([1, 1, 1, 1]) @@ -125,7 +110,7 @@ class SummarizationDataProcessingTest(unittest.TestCase): [[1, 2, 3, 4, 5, 6], [1, 2, 3, 101, 5, 6], [1, 101, 3, 4, 101, 6]] ) expected = torch.tensor( - [[0, 0, 0, 0, 0, 0], [0, 0, 0, 1, 1, 1], [0, 1, 1, 1, 0, 0]] + [[1, 1, 1, 1, 1, 1], [1, 1, 1, 0, 0, 0], [1, 0, 0, 0, 1, 1]] ) result = compute_token_type_ids(batch, separator) diff --git a/transformers/convert_bertextabs_original_pytorch_checkpoint_to_pytorch.py b/transformers/convert_bertextabs_original_pytorch_checkpoint_to_pytorch.py new file mode 100644 index 0000000000..4f158966e1 --- /dev/null +++ b/transformers/convert_bertextabs_original_pytorch_checkpoint_to_pytorch.py @@ -0,0 +1,158 @@ +# coding=utf-8 +# Copyright 2018 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Convert BertExtAbs's checkpoints """ + +import argparse +from collections import namedtuple +import logging + +import torch + +from models.model_builder import AbsSummarizer # The authors' implementation + +from transformers import BertConfig, Model2Model, BertModel, BertForMaskedLM + + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +BertExtAbsConfig = namedtuple( + "BertExtAbsConfig", + ["temp_dir", "large", "finetune_bert", "encoder", "share_emb", "max_pos", "enc_layers", "enc_hidden_size", "enc_heads", "enc_ff_size", "enc_dropout", "dec_layers", "dec_hidden_size", "dec_heads", "dec_ff_size", "dec_dropout"], +) + + +def convert_bertextabs_checkpoints(path_to_checkpoints, dump_path): + """ Copy/paste and tweak the pre-trained weights provided by the creators + of BertExtAbs for the internal architecture. + """ + + # Load checkpoints in memory + checkpoints = torch.load(path_to_checkpoints, lambda storage, loc: storage) + + # Instantiate the authors' model with the pre-trained weights + config = BertExtAbsConfig( + temp_dir=".", + finetune_bert=False, + large=False, + share_emb=True, + encoder="bert", + max_pos=512, + enc_layers=6, + enc_hidden_size=512, + enc_heads=8, + enc_ff_size=512, + enc_dropout=0.2, + dec_layers=6, + dec_hidden_size=768, + dec_heads=8, + dec_ff_size=2048, + dec_dropout=0.2, + ) + bertextabs = AbsSummarizer(config, torch.device("cpu"), checkpoints) + bertextabs.eval() + + # Instantiate our version of the model + decoder_config = BertConfig( + hidden_size=config.dec_hidden_size, + num_hidden_layers=config.dec_layers, + num_attention_heads=config.dec_heads, + intermediate_size=config.dec_ff_size, + hidden_dropout_prob=config.dec_dropout, + attention_probs_dropout_prob=config.dec_dropout, + is_decoder=True, + ) + + decoder_model = BertForMaskedLM(decoder_config) + model = Model2Model.from_pretrained('bert-base-uncased', decoder_model=decoder_model) + model.eval() + + # Let us now start the weight copying process + model.encoder.load_state_dict(bertextabs.bert.model.state_dict()) + + # Decoder + + # Embeddings. The positional embeddings are equal to the word embedding plus a modulation + # that is computed at each forward pass. This may be a source of discrepancy. + model.decoder.bert.embeddings.word_embeddings.weight = bertextabs.decoder.embeddings.weight + model.decoder.bert.embeddings.position_embeddings.weight = bertextabs.decoder.embeddings.weight + model.decoder.bert.embeddings.token_type_embeddings.weight.data = torch.zeros_like(bertextabs.decoder.embeddings.weight) # not defined for BertExtAbs decoder + + # In the original code the LayerNorms are applied twice in the layers, at the beginning and between the + # attention layers. + model.decoder.bert.embeddings.LayerNorm.weight = bertextabs.decoder.transformer_layers[0].layer_norm_1.weight + + for i in range(config.dec_layers): + + # self attention + model.decoder.bert.encoder.layer[i].attention.self.query.weight = bertextabs.decoder.transformer_layers[i].self_attn.linear_query.weight + model.decoder.bert.encoder.layer[i].attention.self.key.weight = bertextabs.decoder.transformer_layers[i].self_attn.linear_keys.weight + model.decoder.bert.encoder.layer[i].attention.self.value.weight = bertextabs.decoder.transformer_layers[i].self_attn.linear_values.weight + model.decoder.bert.encoder.layer[i].attention.output.dense.weight = bertextabs.decoder.transformer_layers[i].self_attn.final_linear.weight + model.decoder.bert.encoder.layer[i].attention.output.LayerNorm.weight = bertextabs.decoder.transformer_layers[i].layer_norm_2.weight + + # attention + model.decoder.bert.encoder.layer[i].crossattention.self.query.weight = bertextabs.decoder.transformer_layers[i].context_attn.linear_query.weight + model.decoder.bert.encoder.layer[i].crossattention.self.key.weight = bertextabs.decoder.transformer_layers[i].context_attn.linear_keys.weight + model.decoder.bert.encoder.layer[i].crossattention.self.value.weight = bertextabs.decoder.transformer_layers[i].context_attn.linear_values.weight + model.decoder.bert.encoder.layer[i].crossattention.output.dense.weight = bertextabs.decoder.transformer_layers[i].context_attn.final_linear.weight + model.decoder.bert.encoder.layer[i].crossattention.output.LayerNorm.weight = bertextabs.decoder.transformer_layers[i].feed_forward.layer_norm.weight + + # intermediate + model.decoder.bert.encoder.layer[i].intermediate.dense.weight = bertextabs.decoder.transformer_layers[i].feed_forward.w_1.weight + + # output + model.decoder.bert.encoder.layer[i].output.dense.weight = bertextabs.decoder.transformer_layers[i].feed_forward.w_2.weight + + try: + model.decoder.bert.encoder.layer[i].output.LayerNorm.weight = bertextabs.decoder.transformer_layers[i + 1].layer_norm_1.weight + except IndexError: + model.decoder.bert.encoder.layer[i].output.LayerNorm.weight = bertextabs.decoder.layer_norm.weight + + # LM Head + """ + model.decoder.cls.predictions.transform.dense.weight + model.decoder.cls.predictions.transform.dense.biais + model.decoder.cls.predictions.transform.LayerNorm.weight + model.decoder.cls.predictions.transform.LayerNorm.biais + model.decoder.cls.predictions.decoder.weight + model.decoder.cls.predictions.decoder.biais + model.decoder.cls.predictions.biais.data + """ + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--bertextabs_checkpoint_path", + default=None, + type=str, + required=True, + help="Path the official PyTorch dump.", + ) + parser.add_argument( + "--pytorch_dump_folder_path", + default=None, + type=str, + required=True, + help="Path to the output PyTorch model.", + ) + args = parser.parse_args() + + convert_bertextabs_checkpoints( + args.bertextabs_checkpoint_path, + args.pytorch_dump_folder_path, + ) diff --git a/transformers/generate/beam_search.py b/transformers/generate/beam_search.py index abe3186049..b56ebbabb8 100644 --- a/transformers/generate/beam_search.py +++ b/transformers/generate/beam_search.py @@ -25,7 +25,6 @@ Use Beam Search to generate sequences using encoder-decoder models. """ import torch from torch import nn - import logging @@ -45,6 +44,7 @@ class BeamSearch(object): max_length, alpha=0, block_repeating_trigrams=True, + device=torch.device("cpu"), ): r""" Inputs: @@ -156,18 +156,24 @@ class BeamSearch(object): kwargs_decoder["encoder_hidden_states"] = tile( encoder_hidden_states, self.beam_size, dim=0 ) - kwargs_decoder["encoder_attention_mask"] = tile( - kwargs_encoder["attention_mask"], self.beam_size, dim=0 + try: + kwargs_decoder["encoder_attention_mask"] = tile( + kwargs_encoder["attention_mask"], self.beam_size, dim=0 + ) + except: + pass + kwargs_decoder["state"].src = tile( + kwargs_decoder["state"].src, self.beam_size, dim=0 ) # grow the beam iteratively batch_size, block_size = encoder_input_ids.size() self._init_beam_state(batch_size) for step in range(self.max_length): - decoder_input = fit_to_block_size(self.growing_beams, block_size, self.pad_token_id) kwargs_decoder["attention_mask"] = build_mask(decoder_input, self.pad_token_id) - outputs = self.model.decoder(decoder_input, **kwargs_decoder) + + outputs, state = self.model.decoder(decoder_input, **kwargs_decoder) next_token_scores = outputs[0][:, -1, :].squeeze(1) log_probabilities = torch.nn.functional.log_softmax(next_token_scores, dim=0) @@ -178,9 +184,13 @@ class BeamSearch(object): kwargs_decoder["encoder_hidden_states"] = kwargs_decoder[ "encoder_hidden_states" ].index_select(0, surviving_beams_rows) - kwargs_decoder["encoder_attention_mask"] = kwargs_decoder[ - "encoder_attention_mask" - ].index_select(0, surviving_beams_rows) + try: + kwargs_decoder["encoder_attention_mask"] = kwargs_decoder[ + "encoder_attention_mask" + ].index_select(0, surviving_beams_rows) + except: + pass + kwargs_decoder["state"] = state return self.results From c0443df5939d980abbe5bb28b31f08d1628469c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Louf?= Date: Thu, 5 Dec 2019 18:13:41 +0100 Subject: [PATCH 273/505] remove beam search --- transformers/generate/beam_search.py | 376 ------------------------ transformers/tests/beam_search_tests.py | 243 --------------- 2 files changed, 619 deletions(-) delete mode 100644 transformers/generate/beam_search.py delete mode 100644 transformers/tests/beam_search_tests.py diff --git a/transformers/generate/beam_search.py b/transformers/generate/beam_search.py deleted file mode 100644 index b56ebbabb8..0000000000 --- a/transformers/generate/beam_search.py +++ /dev/null @@ -1,376 +0,0 @@ -# coding=utf-8 -# MIT License - -# Copyright (c) 2017-Present OpenNMT - -# Permission is hereby granted, free of charge, to any person obtaining a copy of -# this software and associated documentation files (the "Software"), to deal in -# the Software without restriction, including without limitation the rights to -# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -# of the Software, and to permit persons to whom the Software is furnished to do -# so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -""" -Use Beam Search to generate sequences using encoder-decoder models. -""" -import torch -from torch import nn -import logging - - -logger = logging.getLogger(__name__) - - -class BeamSearch(object): - def __init__( - self, - model, - bos_token_id, - pad_token_id, - eos_token_id, - batch_size, - beam_size, - min_length, - max_length, - alpha=0, - block_repeating_trigrams=True, - device=torch.device("cpu"), - ): - r""" - Inputs: - **model**: instance of ``transformers.PreTrainedEncoderDecoder`` - The pretrained encoder-decoder model that will be used to generate the sequences. - **bos_token_id**: int - Id that is used by the tokenizer to represent the beggining of a sentence. - **pad_token_id**: int - Id that is used by the tokenizer for padding. - **eos_token_id**: int - Id that is used by the tokenizer to represent the end of a sentence. - **batch_size**: (`optional`) int - Batch size of the inputs. The value is set automatically when calling `forward`. - **beam_size**: int - Number of beams that are used for each element on the batch. - **min_length**: int - Minimum number of steps performed by the beam search before terminating. - **max_length**: int - Maximum number of steps performed by the beam search. Any beam that has not finished - will return its current solution with the highest probability. The sequence that is - returned has a length of max_length-1 to account for the end token that is subsequently added. - **alpha**: float - Parameter of the length penalty. Read the documentation of the `_length_penalty` method for mode details. - **block_repeating_trigrams**: bool - Whether to block sequences that have repeating 3-grams. - """ - super(BeamSearch, self).__init__() - self.model = model - self.device = next(model.parameters()).device # only works if all parameters of the model are stored on a single GPU - - self.bos_token_id = bos_token_id - self.eos_token_id = eos_token_id - self.pad_token_id = pad_token_id - - self.batch_size = batch_size - self.beam_size = beam_size - self.min_length = min_length - self.max_length = max_length - - self.block_repeating_trigram = block_repeating_trigrams - self.apply_length_penalty = False if alpha == 0 else True - self.alpha = alpha - - self._init_beam_state(batch_size) - - def __len__(self): - return self.growing_beams.size(1) - - def _init_beam_state(self, batch_size): - """ (re-)Initialize the state of the beams. """ - self.hypotheses = [[] for _ in range(batch_size)] - self.batch_offset = torch.arange(batch_size, dtype=torch.long, device=self.device) - self.beam_offset = torch.arange( - 0, - batch_size * self.beam_size, - step=self.beam_size, - dtype=torch.long, - device=self.device, - ) - self.growing_beams = torch.full( - (batch_size * self.beam_size, 1), - self.bos_token_id, - dtype=torch.long, - device=self.device, - ) - self.topk_log_probabilities = torch.tensor( - [0.0] + [float("-inf")] * (self.beam_size - 1), - dtype=torch.float, - device=self.device, - ).repeat(batch_size) - self.results = { - "predictions": [[] for _ in range(batch_size)], - "scores": [[] for _ in range(batch_size)], - } - self._step = 0 - self.is_done = False - - def __call__(self, encoder_input_ids, **model_kwargs): - """ Generate a sequence using Beam Search. """ - # keyword arguments come in 3 flavors: encoder-specific (prefixed by - # `encoder_`), decoder-specific (prefixed by `decoder_`) and those - # that apply to the model as whole. - # We let the specific kwargs override the common ones in case of conflict. - kwargs_common = { - argument: value - for argument, value in model_kwargs.items() - if not argument.startswith("encoder_") and not argument.startswith("decoder_") - } - kwargs_decoder = kwargs_common.copy() - kwargs_encoder = kwargs_common.copy() - kwargs_encoder.update( - { - argument[len("encoder_") :]: value - for argument, value in model_kwargs.items() - if argument.startswith("encoder_") - } - ) - kwargs_decoder.update( - { - argument[len("decoder_") :]: value - for argument, value in model_kwargs.items() - if argument.startswith("decoder_") - } - ) - - # forward pass on the encoder - encoder_outputs = self.model.encoder(encoder_input_ids, **kwargs_encoder) - encoder_hidden_states = encoder_outputs[0] - kwargs_decoder["encoder_hidden_states"] = tile( - encoder_hidden_states, self.beam_size, dim=0 - ) - try: - kwargs_decoder["encoder_attention_mask"] = tile( - kwargs_encoder["attention_mask"], self.beam_size, dim=0 - ) - except: - pass - kwargs_decoder["state"].src = tile( - kwargs_decoder["state"].src, self.beam_size, dim=0 - ) - - # grow the beam iteratively - batch_size, block_size = encoder_input_ids.size() - self._init_beam_state(batch_size) - for step in range(self.max_length): - decoder_input = fit_to_block_size(self.growing_beams, block_size, self.pad_token_id) - kwargs_decoder["attention_mask"] = build_mask(decoder_input, self.pad_token_id) - - outputs, state = self.model.decoder(decoder_input, **kwargs_decoder) - - next_token_scores = outputs[0][:, -1, :].squeeze(1) - log_probabilities = torch.nn.functional.log_softmax(next_token_scores, dim=0) - surviving_beams_rows = self.grow(log_probabilities) - if self.is_done: - break - - kwargs_decoder["encoder_hidden_states"] = kwargs_decoder[ - "encoder_hidden_states" - ].index_select(0, surviving_beams_rows) - try: - kwargs_decoder["encoder_attention_mask"] = kwargs_decoder[ - "encoder_attention_mask" - ].index_select(0, surviving_beams_rows) - except: - pass - kwargs_decoder["state"] = state - - return self.results - - def grow(self, log_probabilities): - """ Grow the beams by one step. """ - self._step += 1 - - # The number of beams changes as some beams finish so we define _B - vocab_size = log_probabilities.size(-1) - _B = log_probabilities.size(0) // self.beam_size - - # Multiply each beam probability with the probability of the - # next token (conditioned on the words in the beam). - log_probabilities += self.topk_log_probabilities.view(-1, 1) - - self._enforce_min_length(log_probabilities) - if self.block_repeating_trigram: - self._remove_beams_with_repeating_trigrams(log_probabilities, _B) - - # Find the `beam_size` (previous_beam + token) combinations with - # the highest score - self.topk_log_probabilities, topk_ids = torch.topk( - log_probabilities.view(_B, self.beam_size * vocab_size), self.beam_size, dim=1 - ) - - # Apply the length penalty. The +1 accounts for the [EOS] token - # that will be added if the beam ends. - topk_scores = self.topk_log_probabilities - if self.apply_length_penalty: - topk_scores /= self._length_penalty() - - # Retrieve the corresponding respective beam and token id - # topk_token_ids[i] will be added to topk_beam_ids[i] - topk_beam_ids = topk_ids.div(vocab_size) - topk_token_ids = topk_ids.fmod(vocab_size) - - # Retrieve the row index of the surviving beams in the original - # view of the log_probabilities tensor - surviving_beams_per_batch = topk_beam_ids + self.beam_offset[:_B].view(-1, 1) - surviving_beams_rows = surviving_beams_per_batch.view(-1) - - # Append the last predictions - self.growing_beams = torch.cat( - [ - self.growing_beams.index_select(0, surviving_beams_rows), - topk_token_ids.view(-1, 1), - ], - 1, - ) - - # Check if any of the beam searches has ended during this - # growth step. Also if top beam (most probable) has ended - # for one element of the batch. - is_finished = topk_token_ids.eq(self.eos_token_id) - self._enforce_max_length(is_finished) - if is_finished.any(): - non_finished = self._cut_finished(is_finished, topk_scores) - self.batch_offset = self.batch_offset.index_select(0, non_finished) - surviving_beams_per_batch = surviving_beams_per_batch.index_select( - 0, non_finished - ) - self.topk_log_probabilities = self.topk_log_probabilities.index_select( - 0, non_finished - ) - - surviving_beams_rows = surviving_beams_per_batch.view(-1) - self.growing_beams = self.growing_beams.index_select(0, surviving_beams_rows) - - return surviving_beams_rows - - def _cut_finished(self, is_finished, topk_scores): - """ Save the finished searches and cut the correponding sequences off - the beams. """ - is_top_beam_finished = is_finished[:, 0].eq(True) - - # Save the finished searches - predictions = self.growing_beams.view( - -1, self.beam_size, self.growing_beams.size(1) - ) - for i in range(is_finished.size(0)): - if is_top_beam_finished[i]: - is_finished[i].fill_(1) - finished_hyp = is_finished[i].nonzero().view(-1) - - # Store the finished beams as a (score, prediction) hypothesis. - b = self.batch_offset[i] - for j in finished_hyp: - self.hypotheses[b].append((topk_scores[i, j], predictions[i, j, :])) - - # If the batch reached the end, save the best hypotheses - # in terms of length-penalized score. - if is_top_beam_finished[i]: - best_score, best_prediction = max(self.hypotheses[b], key=lambda x: x[0]) - self.results["scores"][b].append(best_score) - self.results["predictions"][b].append(best_prediction) - - non_finished = is_top_beam_finished.eq(False).nonzero().view(-1) - if len(non_finished) == 0: - self.is_done = True - - return non_finished - - def _remove_beams_with_repeating_trigrams(self, log_probabilities, _B): - if self._step + 1 > 3: # [BOS] does not count - for i in range(_B * self.beam_size): - tokens = self.growing_beams[i] - trigrams = [ - (tokens[j - 1], tokens[j], tokens[j + 1]) - for j in range(1, len(self) - 1) - ] - last_trigram = tuple(trigrams[-1]) - if last_trigram in trigrams[:-1]: - log_probabilities[i] = -1e20 - - def _enforce_min_length(self, log_probabilities): - if self._step < self.min_length: - log_probabilities[:, self.eos_token_id] = -1e20 - - def _enforce_max_length(self, is_finished): - # +1 because we will need to add an [EOS] token - if self._step + 1 == self.max_length: - is_finished.fill_(1) - - def _length_penalty(self): - """ The calculation of the length penalty follows that of [1]. - - [1] Wu, Yonghui, et al. "Google's neural machine translation system: - Bridging the gap between human and machine translation." arXiv preprint - arXiv:1609.08144 (2016). - """ - return ((5.0 + (self._step + 1)) / 6.0) ** self.alpha - - -def tile(x, count, dim=0): - """ - Tiles `x` along dimension `dim` `count` times. - - Example: - >> ex = torch.tensor([1,2],[3,4]) - >> tile(ex, 2, 0) - torch.Tensor([[1,2],[1,2],[3,4],[3,4]]) - """ - perm = list(range(len(x.size()))) - if dim != 0: - perm[0], perm[dim] = perm[dim], perm[0] - x = x.permute(perm).contiguous() - out_size = list(x.size()) - out_size[0] *= count - batch = x.size(0) - x = ( - x.view(batch, -1) - .transpose(0, 1) - .repeat(count, 1) - .transpose(0, 1) - .contiguous() - .view(*out_size) - ) - if dim != 0: - x = x.permute(perm).contiguous() - return x - - -def fit_to_block_size(sequence, block_size, pad_token_id): - """ Adapt the source and target sequences' lengths to the block size. - If the sequence is shorter we append padding tokens to the right. - """ - padded_sequence = torch.full( - (sequence.size(0), block_size), - pad_token_id, - dtype=torch.long, - device=sequence.device, - ) - padded_sequence[:, : sequence.size(1)] = sequence - return sequence - - -def build_mask(sequence, pad_token_id): - """ Builds the mask. The attention mechanism will only attend to positions - with value 1. """ - mask = torch.ones_like(sequence) - idx_pad_tokens = sequence == pad_token_id - mask[idx_pad_tokens] = 0 - return mask diff --git a/transformers/tests/beam_search_tests.py b/transformers/tests/beam_search_tests.py deleted file mode 100644 index 6f2a2b9c2f..0000000000 --- a/transformers/tests/beam_search_tests.py +++ /dev/null @@ -1,243 +0,0 @@ -from collections import namedtuple -import unittest -import pytest -import numpy as np -import torch -from torch import nn - -from transformers.generate import BeamSearch -from transformers import PreTrainedEncoderDecoder - - -class StubTransformer(nn.Module): - def __init__(self): - self.encoder = None - self.decoder = None - self._parameters = {"dumy": torch.tensor([1])} - - def forward(self): - pass - - -class BeamSearchtest(unittest.TestCase): - def test_beam_search_encoder_decoder_integration(self): - """ We make sure that no internal change in the PreTrainedEncoderDecoder - class will break the integration with the beam search. - """ - - model = StubTransformer() - try: - _ = BeamSearch( - model=model, - bos_token_id=0, - eos_token_id=1, - pad_token_id=2, - batch_size=1, - beam_size=1, - min_length=1, - max_length=1, - alpha=0, - block_repeating_trigrams=False, - ) - except: - self.fail("Instantiating BeamSearch with a PreTrainedEncoderDecoder failed.") - - def test_beam_search_min_length(self): - """ We keep predicting the end_token for the first beam and check that - it is not marked as finished until the beam has reached the minimum - length. """ - eos_idx = 3 - vocab_size = 10 - - batch_size = 3 - beam_size = 2 - min_length = 5 - - beam = BeamSearch( - model=StubTransformer(), - bos_token_id=0, - eos_token_id=eos_idx, - pad_token_id=2, - batch_size=batch_size, - beam_size=beam_size, - min_length=5, - max_length=10, - alpha=0, - block_repeating_trigrams=False, - ) - - # To test that the minimum length is correctly enforced we constantly - # assign the highest probability to the [EOS] token (and assign lower - # probabilities to some other tokens). - # Since BeamSearch will reset its probability to 1e-20 as long as - # min_length has not been reached, we need to reset the value between - # steps. - non_eos_idxs = [4, 5, 1, 8, 9] - score_distribution = torch.log_softmax( - torch.tensor([6.0, 5.0, 4.0, 3.0, 2.0, 1.0]), dim=0 - ) - - log_probabilities = torch.full((batch_size * beam_size, vocab_size), float("-inf")) - log_probabilities[0, eos_idx] = score_distribution[0] - for idx, score in zip(non_eos_idxs, score_distribution[1:]): - log_probabilities[0, idx] = score - pytest.set_trace() - for step in range(1, min_length + 2): - log_probabilities[0, eos_idx] = score_distribution[0] - - # Beam #3 and #4 teminate at the first step since the probability - # of the [EOS] token is -1e20 > -\infty so there are only two beams left. - # The top beam (most likely) always ends with 4 until we reach min_length. - surviving_beams_rows = beam.grow(log_probabilities) - if step < min_length: - np.testing.assert_array_equal( - beam.growing_beams.numpy()[0, :], np.array([0] + [4] * step) - ) - elif step == min_length: - np.testing.assert_array_equal(surviving_beams_rows.numpy(), np.array([])) - self.assertTrue(beam.is_done) - break - - log_probabilities = log_probabilities.index_select(0, surviving_beams_rows) - - def test_beam_search_max_length(self): - """ We keep predicting the same non-EOS token until we reach the - maximum permitted length """ - batch_size = 3 - beam_size = 2 - max_length = 5 - vocab_size = 10 - - beam = BeamSearch( - model=StubTransformer(), - bos_token_id=0, - eos_token_id=1, - pad_token_id=2, - batch_size=batch_size, - beam_size=beam_size, - min_length=2, - max_length=max_length, - alpha=0, - block_repeating_trigrams=False, - ) - - log_probabilities = torch.full((batch_size * beam_size, vocab_size), float("-inf")) - - # To test that beam search enforces the max length constraint we - # keep giving the highest probability to a token that is not the - # [EOS] token. - # The beam search will stop at max_length-1, assuming that one would - # add the [EOS] token at the end of the returned sequence. - token_idxs = [3, 4, 5] - score_distribution = torch.log_softmax(torch.tensor([10.0, 6.0, 4.0]), dim=0) - for idx, score in zip(token_idxs, score_distribution): - log_probabilities[:, idx] = score - - for step in range(1, max_length + 2): - surviving_beams_rows = beam.grow(log_probabilities) - if step + 1 < max_length: - self.assertFalse(beam.is_done) - elif step + 1 == max_length: # Now [EOS] is the most probable token - np.testing.assert_array_equal(surviving_beams_rows.numpy(), np.array([])) - self.assertTrue(beam.is_done) - break - - log_probabilities = log_probabilities.index_select(0, surviving_beams_rows) - - def test_beam_search_block_repeating_trigrams(self): - """ We make sure that the beams that contain repeating trigrams are removed. """ - batch_size = 3 - beam_size = 2 - max_length = 10 - vocab_size = 10 - - beam = BeamSearch( - model=StubTransformer(), - bos_token_id=0, - eos_token_id=1, - pad_token_id=2, - batch_size=batch_size, - beam_size=beam_size, - min_length=2, - max_length=max_length, - alpha=0, - block_repeating_trigrams=True, - ) - - log_probabilities = torch.full((batch_size * beam_size, vocab_size), float("-inf")) - - # To test that BeamSearch enforces the 3-gram constraint we give the - # highest probably to the same tokens in a cyclic fashion and make sure - # they disappear once the cycle has completed. - token_idxs = [3, 4, 5] - score_distribution = torch.log_softmax(torch.tensor([10.0, 6.0, 4.0]), dim=0) - for idx, score in zip(token_idxs, score_distribution): - log_probabilities[:, idx] = score - - for step in range(1, max_length + 2): - # Rotate the probabilities at each step - for idx in token_idxs: - score = score_distribution[(idx + step) % 3] - log_probabilities[::beam_size, idx] = score - - surviving_beams_rows = beam.grow(log_probabilities) - - if step < 7: - self.assertFalse( - np.array_equal( - log_probabilities.numpy()[0, :], - np.array([-1e20] * vocab_size, dtype="float32"), - ) - ) - if step == 7: - np.testing.assert_array_equal( - log_probabilities.numpy()[0, :], - np.array([-1e20] * vocab_size, dtype="float32"), - ) - - log_probabilities = log_probabilities.index_select(0, surviving_beams_rows) - - def test_beam_search_example_for_one_step(self): - """ We test that the predictions for one step of growth are correct. """ - batch_size = 2 - beam_size = 2 - max_length = 10 - vocab_size = 5 - - beam = BeamSearch( - model=StubTransformer(), - bos_token_id=0, - eos_token_id=1, - pad_token_id=2, - batch_size=batch_size, - beam_size=beam_size, - min_length=2, - max_length=max_length, - alpha=0, - block_repeating_trigrams=False, - ) - - log_probabilities = torch.full((batch_size * beam_size, vocab_size), float("-inf")) - log_probabilities[0, 3:] = torch.log_softmax(torch.tensor([2.0, 1.0]), dim=0) - log_probabilities[2, 3:] = torch.log_softmax(torch.tensor([1.0, 2.0]), dim=0) - - # First pass - surviving_beams_rows = beam.grow(log_probabilities) - np.testing.assert_array_equal(surviving_beams_rows.numpy(), np.array([0, 0, 2, 2])) - np.testing.assert_array_equal( - beam.growing_beams.numpy(), np.array([[0, 3], [0, 4], [0, 4], [0, 3]]) - ) - self.assertFalse(beam.is_done) - - # Second pass - surviving_beams_rows = beam.grow(log_probabilities) - np.testing.assert_array_equal(surviving_beams_rows.numpy(), np.array([0, 0, 2, 2])) - np.testing.assert_array_equal( - beam.growing_beams.numpy(), - np.array([[0, 3, 3], [0, 3, 4], [0, 4, 4], [0, 4, 3]]), - ) - self.assertFalse(beam.is_done) - - -if __name__ == "__name__": - unittest.main() From 693606a75c54d9731b748797f21961d0a5322896 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Louf?= Date: Thu, 5 Dec 2019 18:55:15 +0100 Subject: [PATCH 274/505] update the docs --- examples/README.md | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/examples/README.md b/examples/README.md index dec5a67f7e..3d0b2ca1a9 100644 --- a/examples/README.md +++ b/examples/README.md @@ -24,7 +24,8 @@ pip install -r ./examples/requirements.txt | [Multiple Choice](#multiple-choice) | Examples running BERT/XLNet/RoBERTa on the SWAG/RACE/ARC tasks. | [Named Entity Recognition](#named-entity-recognition) | Using BERT for Named Entity Recognition (NER) on the CoNLL 2003 dataset, examples with distributed training. | | [XNLI](#xnli) | Examples running BERT/XLM on the XNLI benchmark. | -| [Abstractive summarization](#abstractive-summarization) | Fine-tuning the library models for abstractive summarization tasks on the CNN/Daily Mail dataset. | +| [Abstractive summarization](#abstractive-summarization) | Using the BertAbs +model finetuned on the CNN/DailyMail dataset to generate summaries. | ## TensorFlow 2.0 Bert models on GLUE @@ -712,3 +713,20 @@ Training with the previously defined hyper-parameters yields the following resul ```bash acc = 0.7093812375249501 ``` + +### Abstractive Summarization + +This example provides a simple API for the [BertAbs](https://github.com/nlpyang/PreSumm) model finetuned on the CNN/DailyMail dataset. The script can be used to generate summaries from any text. + +```bash +python run_summarization.py \ + --documents_dir 'path/to/documents' \ + --summaries_output_dir 'path/to/summaries' \ + --visible_gpus 0,1,2 \ + --batch_size 4 \ + --min_length 50 \ + --max_length 200 \ + --beam_size 5 \ + --alpha 0.95 \ + --block_trigram true +``` From 3a9a9f78614050896356a9a30e9529c502b56d96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Louf?= Date: Thu, 5 Dec 2019 19:09:47 +0100 Subject: [PATCH 275/505] default output dir to documents dir --- examples/summarization/run_summarization.py | 11 ++++++----- examples/summarization/utils_summarization.py | 2 ++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/examples/summarization/run_summarization.py b/examples/summarization/run_summarization.py index e3b974acd9..bbc79227ca 100644 --- a/examples/summarization/run_summarization.py +++ b/examples/summarization/run_summarization.py @@ -31,9 +31,7 @@ Batch = namedtuple( def evaluate(args): tokenizer = BertTokenizer.from_pretrained("bert-base-uncased", do_lower_case=True) - model = bertabs = BertAbs.from_pretrained( - "bertabs-finetuned-{}".format(args.finetuned_model) - ) + model = bertabs = BertAbs.from_pretrained("bertabs-finetuned-cnndm") bertabs.to(args.device) bertabs.eval() @@ -195,8 +193,8 @@ def main(): "--summaries_output_dir", default=None, type=str, - required=True, - help="The folder in wich the summaries should be written.", + required=False, + help="The folder in wich the summaries should be written. Defaults to the folder where the documents are", ) # EVALUATION options parser.add_argument( @@ -242,6 +240,9 @@ def main(): args = parser.parse_args() args.device = torch.device("cpu") if args.visible_gpus == -1 else torch.device("cuda") + if not args.summaries_output_dir: + args.summaries_output_dir = args.documents_dir + if not documents_dir_is_valid(args.documents_dir): raise FileNotFoundError( "We could not find the directory you specified for the documents to summarize, or it was empty. Please specify a valid path." diff --git a/examples/summarization/utils_summarization.py b/examples/summarization/utils_summarization.py index e7401b1754..1d8c436ac9 100644 --- a/examples/summarization/utils_summarization.py +++ b/examples/summarization/utils_summarization.py @@ -39,6 +39,8 @@ class SummarizationDataset(Dataset): self.documents = [] story_filenames_list = os.listdir(path) for story_filename in story_filenames_list: + if "summary" in story_filename: + continue path_to_story = os.path.join(path, story_filename) if not os.path.isfile(path_to_story): continue From a1994a71ee37ee8ac5bc49cce30a764392d64233 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Louf?= Date: Thu, 5 Dec 2019 21:05:06 +0100 Subject: [PATCH 276/505] simplified model and configuration --- .../summarization/configuration_bertabs.py | 22 ---------- examples/summarization/modeling_bertabs.py | 41 ++++--------------- examples/summarization/run_summarization.py | 6 +-- 3 files changed, 10 insertions(+), 59 deletions(-) diff --git a/examples/summarization/configuration_bertabs.py b/examples/summarization/configuration_bertabs.py index ff3171f9a8..5bcb65b423 100644 --- a/examples/summarization/configuration_bertabs.py +++ b/examples/summarization/configuration_bertabs.py @@ -33,17 +33,6 @@ class BertAbsConfig(PretrainedConfig): r""" Class to store the configuration of the BertAbs model. Arguments: - temp_dir: string - Unused in the current situation. Kept for compatibility but will be removed. - finetune_bert: bool - Whether to fine-tune the model or not. Will be kept for reference - in case we want to add the possibility to fine-tune the model. - large: bool - Whether to use bert-large as a base. - share_emb: book - Whether the embeddings are shared between the encoder and decoder. - encoder: string - Not clear what this does. Leave to "bert" for pre-trained weights. max_pos: int The maximum sequence length that this model will be used with. enc_layer: int @@ -77,11 +66,6 @@ class BertAbsConfig(PretrainedConfig): def __init__( self, vocab_size_or_config_json_file=30522, - temp_dir=".", - finetune_bert=False, - large=False, - share_emb=True, - encoder="bert", max_pos=512, enc_layers=6, enc_hidden_size=512, @@ -104,21 +88,15 @@ class BertAbsConfig(PretrainedConfig): for key, value in json_config.items(): self.__dict__[key] = value elif isinstance(vocab_size_or_config_json_file, int): - self.temp_dir = temp_dir - self.finetune_bert = finetune_bert - self.large = large self.vocab_size = vocab_size_or_config_json_file self.max_pos = max_pos - self.encoder = encoder self.enc_layers = enc_layers self.enc_hidden_size = enc_hidden_size self.enc_heads = enc_heads self.enc_ff_size = enc_ff_size self.enc_dropout = enc_dropout - self.share_emb = share_emb - self.dec_layers = dec_layers self.dec_hidden_size = dec_hidden_size self.dec_heads = dec_heads diff --git a/examples/summarization/modeling_bertabs.py b/examples/summarization/modeling_bertabs.py index 0189a2ad2b..5e51526037 100644 --- a/examples/summarization/modeling_bertabs.py +++ b/examples/summarization/modeling_bertabs.py @@ -53,7 +53,7 @@ class BertAbs(BertAbsPreTrainedModel): def __init__(self, args, checkpoint=None, bert_extractive_checkpoint=None): super(BertAbs, self).__init__(args) self.args = args - self.bert = Bert(args.large, args.temp_dir, args.finetune_bert) + self.bert = Bert() # If pre-trained weights are passed for Bert, load these. load_bert_pretrained_extractive = True if bert_extractive_checkpoint else False @@ -69,18 +69,6 @@ class BertAbs(BertAbsPreTrainedModel): strict=True, ) - if args.encoder == "baseline": - bert_config = BertConfig( - self.bert.model.config.vocab_size, - hidden_size=args.enc_hidden_size, - num_hidden_layers=args.enc_layers, - num_attention_heads=8, - intermediate_size=args.enc_ff_size, - hidden_dropout_prob=args.enc_dropout, - attention_probs_dropout_prob=args.enc_dropout, - ) - self.bert.model = BertModel(bert_config) - self.vocab_size = self.bert.model.config.vocab_size if args.max_pos > 512: @@ -101,10 +89,10 @@ class BertAbs(BertAbsPreTrainedModel): tgt_embeddings = nn.Embedding( self.vocab_size, self.bert.model.config.hidden_size, padding_idx=0 ) - if self.args.share_emb: - tgt_embeddings.weight = copy.deepcopy( - self.bert.model.embeddings.word_embeddings.weight - ) + + tgt_embeddings.weight = copy.deepcopy( + self.bert.model.embeddings.word_embeddings.weight + ) self.decoder = TransformerDecoder( self.args.dec_layers, @@ -141,16 +129,6 @@ class BertAbs(BertAbsPreTrainedModel): else: p.data.zero_() - def maybe_tie_embeddings(self, args): - if args.use_bert_emb: - tgt_embeddings = nn.Embedding( - self.vocab_size, self.bert.model.config.hidden_size, padding_idx=0 - ) - tgt_embeddings.weight = copy.deepcopy( - self.bert.model.embeddings.word_embeddings.weight - ) - self.decoder.embeddings = tgt_embeddings - def forward( self, encoder_input_ids, @@ -178,14 +156,9 @@ class Bert(nn.Module): """ This class is not really necessary and should probably disappear. """ - def __init__(self, large, temp_dir, finetune=False): + def __init__(self): super(Bert, self).__init__() - if large: - self.model = BertModel.from_pretrained("bert-large-uncased", cache_dir=temp_dir) - else: - self.model = BertModel.from_pretrained("bert-base-uncased", cache_dir=temp_dir) - - self.finetune = finetune + self.model = BertModel.from_pretrained("bert-base-uncased") def forward(self, input_ids, attention_mask=None, token_type_ids=None, **kwargs): self.eval() diff --git a/examples/summarization/run_summarization.py b/examples/summarization/run_summarization.py index bbc79227ca..ed663e880b 100644 --- a/examples/summarization/run_summarization.py +++ b/examples/summarization/run_summarization.py @@ -31,9 +31,9 @@ Batch = namedtuple( def evaluate(args): tokenizer = BertTokenizer.from_pretrained("bert-base-uncased", do_lower_case=True) - model = bertabs = BertAbs.from_pretrained("bertabs-finetuned-cnndm") - bertabs.to(args.device) - bertabs.eval() + model = BertAbs.from_pretrained("bertabs-finetuned-cnndm") + model.to(args.device) + model.eval() symbols = { "BOS": tokenizer.vocab["[unused0]"], From 5909f710285cf8164b3f51111d595ae87f847133 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Louf?= Date: Thu, 5 Dec 2019 21:07:49 +0100 Subject: [PATCH 277/505] add py-rouge dependency --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index 4a3162adce..236ac1c430 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,5 @@ regex sentencepiece # For XLM sacremoses +# For ROUGE +py-rouge From 076602bdc4b186e715538f437f2bce4b1ee5020e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Louf?= Date: Fri, 6 Dec 2019 10:11:44 +0100 Subject: [PATCH 278/505] prevent BERT weights from being downloaded twice --- examples/summarization/modeling_bertabs.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/summarization/modeling_bertabs.py b/examples/summarization/modeling_bertabs.py index 5e51526037..efca33fb56 100644 --- a/examples/summarization/modeling_bertabs.py +++ b/examples/summarization/modeling_bertabs.py @@ -158,7 +158,8 @@ class Bert(nn.Module): def __init__(self): super(Bert, self).__init__() - self.model = BertModel.from_pretrained("bert-base-uncased") + config = BertConfig.from_pretrained("bert-base-uncased") + self.model = BertModel(config) def forward(self, input_ids, attention_mask=None, token_type_ids=None, **kwargs): self.eval() From ade3cdf5adfcff7736b326b1360fcf2b59aae47e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Louf?= Date: Fri, 6 Dec 2019 11:36:44 +0100 Subject: [PATCH 279/505] integrate ROUGE --- examples/summarization/modeling_bertabs.py | 65 +--------------- examples/summarization/run_summarization.py | 85 +++++++++++++++++++-- requirements.txt | 1 + 3 files changed, 82 insertions(+), 69 deletions(-) diff --git a/examples/summarization/modeling_bertabs.py b/examples/summarization/modeling_bertabs.py index efca33fb56..57126a4df3 100644 --- a/examples/summarization/modeling_bertabs.py +++ b/examples/summarization/modeling_bertabs.py @@ -21,9 +21,6 @@ # SOFTWARE. import copy import math -import shutil -import time -import os import numpy as np import torch @@ -1082,11 +1079,6 @@ class Translator(object): return translations - def _report_rouge(self, gold_path, can_path): - self.logger.info("Calculating Rouge") - results_dict = test_rouge(self.args.temp_dir, can_path, gold_path) - return results_dict - def tile(x, count, dim=0): """ @@ -1113,63 +1105,10 @@ def tile(x, count, dim=0): # -# All things ROUGE. Uses `pyrouge` which is a hot mess. +# Optimizer for training. We keep this here in case we want to add +# a finetuning script. # - -def test_rouge(temp_dir, cand, ref): - candidates = [line.strip() for line in open(cand, encoding="utf-8")] - references = [line.strip() for line in open(ref, encoding="utf-8")] - print(len(candidates)) - print(len(references)) - assert len(candidates) == len(references) - - cnt = len(candidates) - current_time = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) - tmp_dir = os.path.join(temp_dir, "rouge-tmp-{}".format(current_time)) - if not os.path.isdir(tmp_dir): - os.mkdir(tmp_dir) - os.mkdir(tmp_dir + "/candidate") - os.mkdir(tmp_dir + "/reference") - try: - - for i in range(cnt): - if len(references[i]) < 1: - continue - with open( - tmp_dir + "/candidate/cand.{}.txt".format(i), "w", encoding="utf-8" - ) as f: - f.write(candidates[i]) - with open( - tmp_dir + "/reference/ref.{}.txt".format(i), "w", encoding="utf-8" - ) as f: - f.write(references[i]) - r = pyrouge.Rouge155(temp_dir=temp_dir) - r.model_dir = tmp_dir + "/reference/" - r.system_dir = tmp_dir + "/candidate/" - r.model_filename_pattern = "ref.#ID#.txt" - r.system_filename_pattern = r"cand.(\d+).txt" - rouge_results = r.convert_and_evaluate() - print(rouge_results) - results_dict = r.output_to_dict(rouge_results) - finally: - pass - if os.path.isdir(tmp_dir): - shutil.rmtree(tmp_dir) - return results_dict - - -def rouge_results_to_str(results_dict): - return ">> ROUGE-F(1/2/3/l): {:.2f}/{:.2f}/{:.2f}\nROUGE-R(1/2/3/l): {:.2f}/{:.2f}/{:.2f}\n".format( - results_dict["rouge_1_f_score"] * 100, - results_dict["rouge_2_f_score"] * 100, - results_dict["rouge_l_f_score"] * 100, - results_dict["rouge_1_recall"] * 100, - results_dict["rouge_2_recall"] * 100, - results_dict["rouge_l_recall"] * 100, - ) - - class BertSumOptimizer(object): """ Specific optimizer for BertSum. diff --git a/examples/summarization/run_summarization.py b/examples/summarization/run_summarization.py index ed663e880b..a9d08aca82 100644 --- a/examples/summarization/run_summarization.py +++ b/examples/summarization/run_summarization.py @@ -41,6 +41,26 @@ def evaluate(args): "PAD": tokenizer.vocab["[PAD]"], } + if args.compute_rouge: + reference_summaries = [] + generated_summaries = [] + + import rouge + import nltk + nltk.download('punkt') + rouge_evaluator = rouge.Rouge( + metrics=['rouge-n', 'rouge-l'], + max_n=2, + limit_length=True, + length_limit=args.beam_size, + length_limit_type='words', + apply_avg=True, + apply_best=False, + alpha=0.5, # Default F1_score + weight_factor=1.2, + stemming=True, + ) + # these (unused) arguments are defined to keep the compatibility # with the legacy code and will be deleted in a next iteration. args.result_path = "" @@ -66,6 +86,16 @@ def evaluate(args): summaries = [format_summary(t) for t in translations] save_summaries(summaries, args.summaries_output_dir, batch.document_names) + if args.compute_rouge: + reference_summaries += batch.tgt_str + generated_summaries += summaries + + if args.compute_rouge: + scores = rouge_evaluator.get_scores(generated_summaries, reference_summaries) + str_scores = format_rouge_scores(scores) + save_rouge_scores(str_scores) + print(str_scores) + def format_summary(translation): """ Transforms the output of the `from_batch` function @@ -86,6 +116,41 @@ def format_summary(translation): return summary +def format_rouge_scores(scores): + return """\n +****** ROUGE SCORES ****** + +** ROUGE 1 +F1 >> {:.3f} +Precision >> {:.3f} +Recall >> {:.3f} + +** ROUGE 2 +F1 >> {:.3f} +Precision >> {:.3f} +Recall >> {:.3f} + +** ROUGE L +F1 >> {:.3f} +Precision >> {:.3f} +Recall >> {:.3f}""".format( + scores['rouge-1']['f'], + scores['rouge-1']['p'], + scores['rouge-1']['r'], + scores['rouge-2']['f'], + scores['rouge-2']['p'], + scores['rouge-2']['r'], + scores['rouge-l']['f'], + scores['rouge-l']['p'], + scores['rouge-l']['r'], + ) + + +def save_rouge_scores(str_scores): + with open("rouge_scores.txt", "w") as output: + output.write(str_scores) + + def save_summaries(summaries, path, original_document_name): """ Write the summaries in fies that are prefixed by the original files' name with the `_summary` appended. @@ -142,26 +207,27 @@ def collate(data, tokenizer, block_size): """ data = [x for x in data if not len(x[1]) == 0] # remove empty_files names = [name for name, _, _ in data] + summaries = [" ".join(summary_list) for _, _, summary_list in data] encoded_text = [ encode_for_summarization(story, summary, tokenizer) for _, story, summary in data ] - stories = torch.tensor( + encoded_stories = torch.tensor( [ fit_to_block_size(story, block_size, tokenizer.pad_token_id) for story, _ in encoded_text ] ) - encoder_token_type_ids = compute_token_type_ids(stories, tokenizer.cls_token_id) - encoder_mask = build_mask(stories, tokenizer.pad_token_id) + encoder_token_type_ids = compute_token_type_ids(encoded_stories, tokenizer.cls_token_id) + encoder_mask = build_mask(encoded_stories, tokenizer.pad_token_id) batch = Batch( document_names=names, - batch_size=len(stories), - src=stories, + batch_size=len(encoded_stories), + src=encoded_stories, segs=encoder_token_type_ids, mask_src=encoder_mask, - tgt_str=[""] * len(stories), + tgt_str=summaries, ) return batch @@ -196,6 +262,13 @@ def main(): required=False, help="The folder in wich the summaries should be written. Defaults to the folder where the documents are", ) + parser.add_argument( + "--compute_rouge", + default=False, + type=bool, + required=False, + help="Compute the ROUGE metrics during evaluation. Only available for the CNN/DailyMail dataset.", + ) # EVALUATION options parser.add_argument( "--visible_gpus", diff --git a/requirements.txt b/requirements.txt index 236ac1c430..2cbcc3809d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,4 +11,5 @@ sentencepiece # For XLM sacremoses # For ROUGE +nltk py-rouge From c0707a85d24fa5a74d85d40ed704d4c774e9a37f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Louf?= Date: Fri, 6 Dec 2019 11:49:27 +0100 Subject: [PATCH 280/505] add README --- examples/README.md | 17 --------- examples/summarization/README.md | 61 ++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 17 deletions(-) create mode 100644 examples/summarization/README.md diff --git a/examples/README.md b/examples/README.md index 3d0b2ca1a9..620304ea77 100644 --- a/examples/README.md +++ b/examples/README.md @@ -713,20 +713,3 @@ Training with the previously defined hyper-parameters yields the following resul ```bash acc = 0.7093812375249501 ``` - -### Abstractive Summarization - -This example provides a simple API for the [BertAbs](https://github.com/nlpyang/PreSumm) model finetuned on the CNN/DailyMail dataset. The script can be used to generate summaries from any text. - -```bash -python run_summarization.py \ - --documents_dir 'path/to/documents' \ - --summaries_output_dir 'path/to/summaries' \ - --visible_gpus 0,1,2 \ - --batch_size 4 \ - --min_length 50 \ - --max_length 200 \ - --beam_size 5 \ - --alpha 0.95 \ - --block_trigram true -``` diff --git a/examples/summarization/README.md b/examples/summarization/README.md new file mode 100644 index 0000000000..2b58c00693 --- /dev/null +++ b/examples/summarization/README.md @@ -0,0 +1,61 @@ +# Text Summarization with Pretrained Encoders + +This folder contains part of the code necessary to reproduce the results on abstractive summarization from the article [Text Summarization with Pretrained Encoders](https://arxiv.org/pdf/1908.08345.pdf) by [Yang Liu](https://nlp-yang.github.io/) and [Mirella Lapata](https://homepages.inf.ed.ac.uk/mlap/). It can also be used to summarize any document. + +The original code can be found on the Yang Liu's [github repository](https://github.com/nlpyang/PreSumm). + +The model is loaded with the pre-trained weights for the abstractive summarization model trained on the CNN/Daily Mail dataset with an extractive and then abstractive tasks. + +## Setup + +``` +git clone https://github.com/huggingface/transformers && cd transformers +pip install [--editable] . +pip install nltk py-rouge +cd examples/summarization +``` + +## Reproduce the authors' results on ROUGE + +To be able to reproduce the authors' results on the CNN/Daily Mail dataset you first need to download both CNN and Daily Mail datasets [from Kyunghyun Cho's website](https://cs.nyu.edu/~kcho/DMQA/) (the links next to "Stories") in the same folder. Then uncompress the archives by running: + +```bash +tar -xvf cnn_stories.tgz && tar -xvf dailymail_stories.tgz +``` + +And move all the stories to the same folder. We will refer as `$DATA_PATH` the path to where you uncompressed both archive. Then run the following in the same folder as `run_summarization.py`: + +```bash +python run_summarization.py \ + --documents_dir $DATA_PATH \ + --summaries_output_dir $SUMMARIES_PATH \ # optional + --visible_gpus 0,1,2 \ + --batch_size 4 \ + --min_length 50 \ + --max_length 200 \ + --beam_size 5 \ + --alpha 0.95 \ + --block_trigram true \ + --compute_rouge true +``` + +The ROUGE scores will be displayed in the console at the end of evaluation and written in a `rouge_scores.txt` file. + +## Summarize any text + +Put the documents that you would like to summarize in a folder (the path to which is referred to as `$DATA_PATH` below) and run the following in the same folder as `run_summarization.py`: + +```bash +python run_summarization.py \ + --documents_dir $DATA_PATH \ + --summaries_output_dir $SUMMARIES_PATH \ # optional + --visible_gpus 0,1,2 \ + --batch_size 4 \ + --min_length 50 \ + --max_length 200 \ + --beam_size 5 \ + --alpha 0.95 \ + --block_trigram true \ +``` + +If you want to compute ROUGE on another dataset you will need to tweak the stories/summaries import in `utils_summarization.py` From 2a64107e44bd2bb1caee824f121fc4fb6b7d90f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Louf?= Date: Fri, 6 Dec 2019 15:45:09 +0100 Subject: [PATCH 281/505] improve device usage --- examples/summarization/README.md | 8 +++---- ...ert_bertabs_original_pytorch_checkpoint.py | 7 +++--- examples/summarization/modeling_bertabs.py | 2 -- examples/summarization/run_summarization.py | 23 +++++++++++-------- 4 files changed, 21 insertions(+), 19 deletions(-) diff --git a/examples/summarization/README.md b/examples/summarization/README.md index 2b58c00693..96825cfa46 100644 --- a/examples/summarization/README.md +++ b/examples/summarization/README.md @@ -29,7 +29,7 @@ And move all the stories to the same folder. We will refer as `$DATA_PATH` the p python run_summarization.py \ --documents_dir $DATA_PATH \ --summaries_output_dir $SUMMARIES_PATH \ # optional - --visible_gpus 0,1,2 \ + --to_cpu false \ --batch_size 4 \ --min_length 50 \ --max_length 200 \ @@ -39,7 +39,7 @@ python run_summarization.py \ --compute_rouge true ``` -The ROUGE scores will be displayed in the console at the end of evaluation and written in a `rouge_scores.txt` file. +The scripts executes on GPU if one is available and if `to_cpu` is not set to `true`. Inference on multiple GPUs is not suported yet. The ROUGE scores will be displayed in the console at the end of evaluation and written in a `rouge_scores.txt` file. The script takes 30 hours to compute with a single Tesla V100 GPU and a batch size of 10 (300,000 texts to summarize). ## Summarize any text @@ -49,7 +49,7 @@ Put the documents that you would like to summarize in a folder (the path to whic python run_summarization.py \ --documents_dir $DATA_PATH \ --summaries_output_dir $SUMMARIES_PATH \ # optional - --visible_gpus 0,1,2 \ + --to_cpu false \ --batch_size 4 \ --min_length 50 \ --max_length 200 \ @@ -58,4 +58,4 @@ python run_summarization.py \ --block_trigram true \ ``` -If you want to compute ROUGE on another dataset you will need to tweak the stories/summaries import in `utils_summarization.py` +You may want to play around with `min_length`, `max_length` and `alpha` to suit your use case. If you want to compute ROUGE on another dataset you will need to tweak the stories/summaries import in `utils_summarization.py` and tell it where to fetch the reference summaries. diff --git a/examples/summarization/convert_bertabs_original_pytorch_checkpoint.py b/examples/summarization/convert_bertabs_original_pytorch_checkpoint.py index 786a29ef13..33b17bfb6f 100644 --- a/examples/summarization/convert_bertabs_original_pytorch_checkpoint.py +++ b/examples/summarization/convert_bertabs_original_pytorch_checkpoint.py @@ -12,10 +12,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" Convert BertExtAbs's checkpoints +""" Convert BertExtAbs's checkpoints. -The file currently does not do much as we ended up copying the exact model -structure, but I leave it here in case we ever want to refactor the model. +The script looks like it is doing something trivial but it is not. The "weights" +proposed by the authors are actually the entire model pickled. We need to load +the model within the original codebase to be able to only save its `state_dict`. """ import argparse diff --git a/examples/summarization/modeling_bertabs.py b/examples/summarization/modeling_bertabs.py index 57126a4df3..d989e4fd7e 100644 --- a/examples/summarization/modeling_bertabs.py +++ b/examples/summarization/modeling_bertabs.py @@ -847,14 +847,12 @@ class Translator(object): global_scores (:obj:`GlobalScorer`): object to rescore final translations copy_attn (bool): use copy attention during translation - cuda (bool): use cuda beam_trace (bool): trace beam search for debugging logger(logging.Logger): logger. """ def __init__(self, args, model, vocab, symbols, global_scorer=None, logger=None): self.logger = logger - self.cuda = args.visible_gpus != "-1" self.args = args self.model = model diff --git a/examples/summarization/run_summarization.py b/examples/summarization/run_summarization.py index a9d08aca82..c388569869 100644 --- a/examples/summarization/run_summarization.py +++ b/examples/summarization/run_summarization.py @@ -185,7 +185,7 @@ def save_summaries(summaries, path, original_document_name): def build_data_iterator(args, tokenizer): dataset = load_and_cache_examples(args, tokenizer) sampler = SequentialSampler(dataset) - collate_fn = lambda data: collate(data, tokenizer, block_size=512) + collate_fn = lambda data: collate(data, tokenizer, block_size=512, device=args.device) iterator = DataLoader( dataset, sampler=sampler, batch_size=args.batch_size, collate_fn=collate_fn, ) @@ -198,7 +198,7 @@ def load_and_cache_examples(args, tokenizer): return dataset -def collate(data, tokenizer, block_size): +def collate(data, tokenizer, block_size, device): """ Collate formats the data passed to the data loader. In particular we tokenize the data batch after batch to avoid keeping them @@ -224,9 +224,9 @@ def collate(data, tokenizer, block_size): batch = Batch( document_names=names, batch_size=len(encoded_stories), - src=encoded_stories, - segs=encoder_token_type_ids, - mask_src=encoder_mask, + src=encoded_stories.to(device), + segs=encoder_token_type_ids.to(device), + mask_src=encoder_mask.to(device), tgt_str=summaries, ) @@ -271,10 +271,10 @@ def main(): ) # EVALUATION options parser.add_argument( - "--visible_gpus", - default=-1, - type=int, - help="Number of GPUs with which to do the training.", + "--to_cpu", + default=False, + type=bool, + help="Whether to force the execution on CPU.", ) parser.add_argument( "--batch_size", default=4, type=int, help="Batch size per GPU/CPU for training.", @@ -311,8 +311,11 @@ def main(): help="Whether to block the existence of repeating trigrams in the text generated by beam search.", ) args = parser.parse_args() - args.device = torch.device("cpu") if args.visible_gpus == -1 else torch.device("cuda") + # Select device (distibuted not available) + args.device = torch.device("cuda" if torch.cuda.is_available() and not args.to_cpu else "cpu") + + # Check the existence of directories if not args.summaries_output_dir: args.summaries_output_dir = args.documents_dir From f7eba090077a443d4a2fd1cd341c822a8fb4dcbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Louf?= Date: Fri, 6 Dec 2019 22:01:48 +0100 Subject: [PATCH 282/505] clean for release --- ..._original_pytorch_checkpoint_to_pytorch.py | 161 ------------------ examples/summarization/modeling_bertabs.py | 2 +- examples/summarization/requirements.txt | 9 + examples/summarization/run_summarization.py | 60 +++---- requirements.txt | 3 - ..._original_pytorch_checkpoint_to_pytorch.py | 158 ----------------- transformers/generate/__init__.py | 1 - transformers/modeling_encoder_decoder.py | 31 ++-- 8 files changed, 49 insertions(+), 376 deletions(-) delete mode 100644 examples/convert_bertextabs_original_pytorch_checkpoint_to_pytorch.py create mode 100644 examples/summarization/requirements.txt delete mode 100644 transformers/convert_bertextabs_original_pytorch_checkpoint_to_pytorch.py delete mode 100644 transformers/generate/__init__.py diff --git a/examples/convert_bertextabs_original_pytorch_checkpoint_to_pytorch.py b/examples/convert_bertextabs_original_pytorch_checkpoint_to_pytorch.py deleted file mode 100644 index c245d0eae5..0000000000 --- a/examples/convert_bertextabs_original_pytorch_checkpoint_to_pytorch.py +++ /dev/null @@ -1,161 +0,0 @@ -# coding=utf-8 -# Copyright 2018 The HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" Convert BertExtAbs's checkpoints """ - -import argparse -from collections import namedtuple -import logging -import pdb -import torch - -from models.model_builder import AbsSummarizer # The authors' implementation -from model_bertabs import BertAbsSummarizer - -from transformers import BertTokenizer - - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - - -SAMPLE_TEXT = 'Hello world! cécé herlolip' - - -BertAbsConfig = namedtuple( - "BertAbsConfig", - ["temp_dir", "large", "use_bert_emb", "finetune_bert", "encoder", "share_emb", "max_pos", "enc_layers", "enc_hidden_size", "enc_heads", "enc_ff_size", "enc_dropout", "dec_layers", "dec_hidden_size", "dec_heads", "dec_ff_size", "dec_dropout"], -) - - -def convert_bertabs_checkpoints(path_to_checkpoints, dump_path): - """ Copy/paste and tweak the pre-trained weights provided by the creators - of BertAbs for the internal architecture. - """ - - # Instantiate the authors' model with the pre-trained weights - config = BertAbsConfig( - temp_dir=".", - finetune_bert=False, - large=False, - share_emb=True, - use_bert_emb=False, - encoder="bert", - max_pos=512, - enc_layers=6, - enc_hidden_size=512, - enc_heads=8, - enc_ff_size=512, - enc_dropout=0.2, - dec_layers=6, - dec_hidden_size=768, - dec_heads=8, - dec_ff_size=2048, - dec_dropout=0.2, - ) - checkpoints = torch.load(path_to_checkpoints, lambda storage, loc: storage) - original = AbsSummarizer(config, torch.device("cpu"), checkpoints) - original.eval() - - new_model = BertAbsSummarizer(config, torch.device("cpu")) - new_model.eval() - - # ------------------- - # Convert the weights - # ------------------- - - logging.info("convert the model") - new_model.encoder.load_state_dict(original.bert.state_dict()) - - new_model.decoder.generator.load_state_dict(original.generator.state_dict()) - new_model.decoder.embeddings.load_state_dict(original.decoder.embeddings.state_dict()) - new_model.decoder.pos_emb.load_state_dict(original.decoder.pos_emb.state_dict()) - new_model.decoder.transformer_layers.load_state_dict(original.decoder.transformer_layers.state_dict()) - new_model.decoder.layer_norm.load_state_dict(original.decoder.layer_norm.state_dict()) - - # ---------------------------------- - # Make sure the outpus are identical - # ---------------------------------- - - logging.info("Make sure that the models' outputs are identical") - tokenizer = BertTokenizer.from_pretrained("bert-base-uncased") - - # prepare the model inputs - encoder_input_ids = tokenizer.encode("This is sample éàalj'-.") - encoder_input_ids.extend([tokenizer.pad_token_id] * (512 - len(encoder_input_ids))) - encoder_input_ids = torch.tensor(encoder_input_ids).unsqueeze(0) - decoder_input_ids = tokenizer.encode("This is sample 3 éàalj'-.") - decoder_input_ids.extend([tokenizer.pad_token_id] * (512 - len(decoder_input_ids))) - decoder_input_ids = torch.tensor(decoder_input_ids).unsqueeze(0) - - # failsafe to make sure the weights reset does not affect the - # loaded weights. - assert torch.max(torch.abs(original.generator[0].weight - new_model.decoder.generator[0].weight)) == 0 - - # forward pass - src = encoder_input_ids - tgt = decoder_input_ids - segs = token_type_ids = None - clss = None - mask_src = encoder_attention_mask = None - mask_tgt = decoder_attention_mask = None - mask_cls = None - - # The original model does not apply the geneator layer immediatly but rather in - # the beam search (where it combines softmax + linear layer). Since we already - # apply the softmax in our generation process we only apply the linear layer here. - # We make sure that the outputs of the full stack are identical - output_original_model = original(src, tgt, segs, clss, mask_src, mask_tgt, mask_cls)[0] - output_original_model = original.generator(output_original_model) - - output_converted_model = new_model(encoder_input_ids, decoder_input_ids, token_type_ids, encoder_attention_mask, decoder_attention_mask)[0] - output_converted_model = torch.nn.functional.log_softmax(output_converted_model, dim=-1) - - maximum_absolute_difference = torch.max(torch.abs(output_converted_model - output_original_model)).item() - print("Maximum absolute difference beween weights: {:.2f}".format(maximum_absolute_difference)) - - are_identical = torch.allclose(output_converted_model, output_original_model, atol=1e-3) - if are_identical: - logging.info("all weights are equal up to 1e-3") - else: - raise ValueError("the weights are different. The new model is likely different from the original one.") - - # The model has been saved with torch.save(model) and this is bound to the exact - # directory structure. We save the state_dict instead. - logging.info("saving the model's state dictionary") - torch.save(new_model.state_dict(), "bert-ext-abs.pt") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "--bertabs_checkpoint_path", - default=None, - type=str, - required=True, - help="Path the official PyTorch dump.", - ) - parser.add_argument( - "--pytorch_dump_folder_path", - default=None, - type=str, - required=True, - help="Path to the output PyTorch model.", - ) - args = parser.parse_args() - - convert_bertabs_checkpoints( - args.bertabs_checkpoint_path, - args.pytorch_dump_folder_path, - ) diff --git a/examples/summarization/modeling_bertabs.py b/examples/summarization/modeling_bertabs.py index d989e4fd7e..5bf1599ad2 100644 --- a/examples/summarization/modeling_bertabs.py +++ b/examples/summarization/modeling_bertabs.py @@ -1,6 +1,6 @@ # MIT License -# Copyright (c) 2019 Yang Liu +# Copyright (c) 2019 Yang Liu and the HuggingFace team # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/examples/summarization/requirements.txt b/examples/summarization/requirements.txt new file mode 100644 index 0000000000..36d75a5edc --- /dev/null +++ b/examples/summarization/requirements.txt @@ -0,0 +1,9 @@ +# progress bars in model download and training scripts +tqdm +# Accessing files from S3 directly. +boto3 +# Used for downloading models over HTTP +requests +# For ROUGE +nltk +py-rouge diff --git a/examples/summarization/run_summarization.py b/examples/summarization/run_summarization.py index c388569869..f58ce3bb43 100644 --- a/examples/summarization/run_summarization.py +++ b/examples/summarization/run_summarization.py @@ -1,3 +1,4 @@ +#! /usr/bin/python3 import argparse from collections import namedtuple import logging @@ -97,6 +98,32 @@ def evaluate(args): print(str_scores) +def save_summaries(summaries, path, original_document_name): + """ Write the summaries in fies that are prefixed by the original + files' name with the `_summary` appended. + + Attributes: + original_document_names: List[string] + Name of the document that was summarized. + path: string + Path were the summaries will be written + summaries: List[string] + The summaries that we produced. + """ + for summary, document_name in zip(summaries, original_document_name): + # Prepare the summary file's name + if "." in document_name: + bare_document_name = ".".join(document_name.split(".")[:-1]) + extension = document_name.split(".")[-1] + name = bare_document_name + "_summary." + extension + else: + name = document_name + "_summary" + + file_path = os.path.join(path, name) + with open(file_path, "w") as output: + output.write(summary) + + def format_summary(translation): """ Transforms the output of the `from_batch` function into nicely formatted summaries. @@ -151,32 +178,6 @@ def save_rouge_scores(str_scores): output.write(str_scores) -def save_summaries(summaries, path, original_document_name): - """ Write the summaries in fies that are prefixed by the original - files' name with the `_summary` appended. - - Attributes: - original_document_names: List[string] - Name of the document that was summarized. - path: string - Path were the summaries will be written - summaries: List[string] - The summaries that we produced. - """ - for summary, document_name in zip(summaries, original_document_name): - # Prepare the summary file's name - if "." in document_name: - bare_document_name = ".".join(document_name.split(".")[:-1]) - extension = document_name.split(".")[-1] - name = bare_document_name + "_summary." + extension - else: - name = document_name + "_summary" - - file_path = os.path.join(path, name) - with open(file_path, "w") as output: - output.write(summary) - - # # LOAD the dataset # @@ -323,7 +324,7 @@ def main(): raise FileNotFoundError( "We could not find the directory you specified for the documents to summarize, or it was empty. Please specify a valid path." ) - maybe_create_output_dir(args.summaries_output_dir) + os.makedirs(args.summaries_output_dir, exist_ok=True) evaluate(args) @@ -339,10 +340,5 @@ def documents_dir_is_valid(path): return True -def maybe_create_output_dir(path): - if not os.path.exists(path): - os.makedirs(path) - - if __name__ == "__main__": main() diff --git a/requirements.txt b/requirements.txt index 2cbcc3809d..4a3162adce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,6 +10,3 @@ regex sentencepiece # For XLM sacremoses -# For ROUGE -nltk -py-rouge diff --git a/transformers/convert_bertextabs_original_pytorch_checkpoint_to_pytorch.py b/transformers/convert_bertextabs_original_pytorch_checkpoint_to_pytorch.py deleted file mode 100644 index 4f158966e1..0000000000 --- a/transformers/convert_bertextabs_original_pytorch_checkpoint_to_pytorch.py +++ /dev/null @@ -1,158 +0,0 @@ -# coding=utf-8 -# Copyright 2018 The HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" Convert BertExtAbs's checkpoints """ - -import argparse -from collections import namedtuple -import logging - -import torch - -from models.model_builder import AbsSummarizer # The authors' implementation - -from transformers import BertConfig, Model2Model, BertModel, BertForMaskedLM - - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - - -BertExtAbsConfig = namedtuple( - "BertExtAbsConfig", - ["temp_dir", "large", "finetune_bert", "encoder", "share_emb", "max_pos", "enc_layers", "enc_hidden_size", "enc_heads", "enc_ff_size", "enc_dropout", "dec_layers", "dec_hidden_size", "dec_heads", "dec_ff_size", "dec_dropout"], -) - - -def convert_bertextabs_checkpoints(path_to_checkpoints, dump_path): - """ Copy/paste and tweak the pre-trained weights provided by the creators - of BertExtAbs for the internal architecture. - """ - - # Load checkpoints in memory - checkpoints = torch.load(path_to_checkpoints, lambda storage, loc: storage) - - # Instantiate the authors' model with the pre-trained weights - config = BertExtAbsConfig( - temp_dir=".", - finetune_bert=False, - large=False, - share_emb=True, - encoder="bert", - max_pos=512, - enc_layers=6, - enc_hidden_size=512, - enc_heads=8, - enc_ff_size=512, - enc_dropout=0.2, - dec_layers=6, - dec_hidden_size=768, - dec_heads=8, - dec_ff_size=2048, - dec_dropout=0.2, - ) - bertextabs = AbsSummarizer(config, torch.device("cpu"), checkpoints) - bertextabs.eval() - - # Instantiate our version of the model - decoder_config = BertConfig( - hidden_size=config.dec_hidden_size, - num_hidden_layers=config.dec_layers, - num_attention_heads=config.dec_heads, - intermediate_size=config.dec_ff_size, - hidden_dropout_prob=config.dec_dropout, - attention_probs_dropout_prob=config.dec_dropout, - is_decoder=True, - ) - - decoder_model = BertForMaskedLM(decoder_config) - model = Model2Model.from_pretrained('bert-base-uncased', decoder_model=decoder_model) - model.eval() - - # Let us now start the weight copying process - model.encoder.load_state_dict(bertextabs.bert.model.state_dict()) - - # Decoder - - # Embeddings. The positional embeddings are equal to the word embedding plus a modulation - # that is computed at each forward pass. This may be a source of discrepancy. - model.decoder.bert.embeddings.word_embeddings.weight = bertextabs.decoder.embeddings.weight - model.decoder.bert.embeddings.position_embeddings.weight = bertextabs.decoder.embeddings.weight - model.decoder.bert.embeddings.token_type_embeddings.weight.data = torch.zeros_like(bertextabs.decoder.embeddings.weight) # not defined for BertExtAbs decoder - - # In the original code the LayerNorms are applied twice in the layers, at the beginning and between the - # attention layers. - model.decoder.bert.embeddings.LayerNorm.weight = bertextabs.decoder.transformer_layers[0].layer_norm_1.weight - - for i in range(config.dec_layers): - - # self attention - model.decoder.bert.encoder.layer[i].attention.self.query.weight = bertextabs.decoder.transformer_layers[i].self_attn.linear_query.weight - model.decoder.bert.encoder.layer[i].attention.self.key.weight = bertextabs.decoder.transformer_layers[i].self_attn.linear_keys.weight - model.decoder.bert.encoder.layer[i].attention.self.value.weight = bertextabs.decoder.transformer_layers[i].self_attn.linear_values.weight - model.decoder.bert.encoder.layer[i].attention.output.dense.weight = bertextabs.decoder.transformer_layers[i].self_attn.final_linear.weight - model.decoder.bert.encoder.layer[i].attention.output.LayerNorm.weight = bertextabs.decoder.transformer_layers[i].layer_norm_2.weight - - # attention - model.decoder.bert.encoder.layer[i].crossattention.self.query.weight = bertextabs.decoder.transformer_layers[i].context_attn.linear_query.weight - model.decoder.bert.encoder.layer[i].crossattention.self.key.weight = bertextabs.decoder.transformer_layers[i].context_attn.linear_keys.weight - model.decoder.bert.encoder.layer[i].crossattention.self.value.weight = bertextabs.decoder.transformer_layers[i].context_attn.linear_values.weight - model.decoder.bert.encoder.layer[i].crossattention.output.dense.weight = bertextabs.decoder.transformer_layers[i].context_attn.final_linear.weight - model.decoder.bert.encoder.layer[i].crossattention.output.LayerNorm.weight = bertextabs.decoder.transformer_layers[i].feed_forward.layer_norm.weight - - # intermediate - model.decoder.bert.encoder.layer[i].intermediate.dense.weight = bertextabs.decoder.transformer_layers[i].feed_forward.w_1.weight - - # output - model.decoder.bert.encoder.layer[i].output.dense.weight = bertextabs.decoder.transformer_layers[i].feed_forward.w_2.weight - - try: - model.decoder.bert.encoder.layer[i].output.LayerNorm.weight = bertextabs.decoder.transformer_layers[i + 1].layer_norm_1.weight - except IndexError: - model.decoder.bert.encoder.layer[i].output.LayerNorm.weight = bertextabs.decoder.layer_norm.weight - - # LM Head - """ - model.decoder.cls.predictions.transform.dense.weight - model.decoder.cls.predictions.transform.dense.biais - model.decoder.cls.predictions.transform.LayerNorm.weight - model.decoder.cls.predictions.transform.LayerNorm.biais - model.decoder.cls.predictions.decoder.weight - model.decoder.cls.predictions.decoder.biais - model.decoder.cls.predictions.biais.data - """ - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - parser.add_argument( - "--bertextabs_checkpoint_path", - default=None, - type=str, - required=True, - help="Path the official PyTorch dump.", - ) - parser.add_argument( - "--pytorch_dump_folder_path", - default=None, - type=str, - required=True, - help="Path to the output PyTorch model.", - ) - args = parser.parse_args() - - convert_bertextabs_checkpoints( - args.bertextabs_checkpoint_path, - args.pytorch_dump_folder_path, - ) diff --git a/transformers/generate/__init__.py b/transformers/generate/__init__.py deleted file mode 100644 index 21ac612155..0000000000 --- a/transformers/generate/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .beam_search import BeamSearch diff --git a/transformers/modeling_encoder_decoder.py b/transformers/modeling_encoder_decoder.py index 73322101d3..a884abd0a2 100644 --- a/transformers/modeling_encoder_decoder.py +++ b/transformers/modeling_encoder_decoder.py @@ -117,7 +117,8 @@ class PreTrainedEncoderDecoder(nn.Module): kwargs_common = { argument: value for argument, value in kwargs.items() - if not argument.startswith("encoder_") and not argument.startswith("decoder_") + if not argument.startswith("encoder_") + and not argument.startswith("decoder_") } kwargs_decoder = kwargs_common.copy() kwargs_encoder = kwargs_common.copy() @@ -157,27 +158,14 @@ class PreTrainedEncoderDecoder(nn.Module): return model - def save_pretrained(self, save_directory, model_type="bert"): - """ Save an EncoderDecoder model and its configuration file in a format such + def save_pretrained(self, save_directory): + """ Save a Seq2Seq model and its configuration file in a format such that it can be loaded using `:func:`~transformers.PreTrainedEncoderDecoder.from_pretrained` We save the encoder' and decoder's parameters in two separate directories. - - If we want the weight loader to function we need to preprend the model - type to the directories' names. As far as I know there is no simple way - to infer the type of the model (except maybe by parsing the class' - names, which is not very future-proof). For now, we ask the user to - specify the model type explicitly when saving the weights. """ - encoder_path = os.path.join(save_directory, "{}_encoder".format(model_type)) - if not os.path.exists(encoder_path): - os.makedirs(encoder_path) - self.encoder.save_pretrained(encoder_path) - - decoder_path = os.path.join(save_directory, "{}_decoder".format(model_type)) - if not os.path.exists(decoder_path): - os.makedirs(decoder_path) - self.decoder.save_pretrained(decoder_path) + self.encoder.save_pretrained(os.path.join(save_directory, "encoder")) + self.decoder.save_pretrained(os.path.join(save_directory, "decoder")) def forward(self, encoder_input_ids, decoder_input_ids, **kwargs): """ The forward pass on a seq2eq depends what we are performing: @@ -205,7 +193,8 @@ class PreTrainedEncoderDecoder(nn.Module): kwargs_common = { argument: value for argument, value in kwargs.items() - if not argument.startswith("encoder_") and not argument.startswith("decoder_") + if not argument.startswith("encoder_") + and not argument.startswith("decoder_") } kwargs_decoder = kwargs_common.copy() kwargs_encoder = kwargs_common.copy() @@ -228,7 +217,9 @@ class PreTrainedEncoderDecoder(nn.Module): encoder_hidden_states = kwargs_encoder.pop("hidden_states", None) if encoder_hidden_states is None: encoder_outputs = self.encoder(encoder_input_ids, **kwargs_encoder) - encoder_hidden_states = encoder_outputs[0] # output the last layer hidden state + encoder_hidden_states = encoder_outputs[ + 0 + ] # output the last layer hidden state else: encoder_outputs = () From 1d189304624db17749aee23fa2345f009cc48215 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Tue, 10 Dec 2019 01:32:42 +0000 Subject: [PATCH 283/505] Harmonize `no_cuda` flag with other scripts --- examples/summarization/run_summarization.py | 4 ++-- requirements.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/summarization/run_summarization.py b/examples/summarization/run_summarization.py index f58ce3bb43..3c339d0c30 100644 --- a/examples/summarization/run_summarization.py +++ b/examples/summarization/run_summarization.py @@ -272,7 +272,7 @@ def main(): ) # EVALUATION options parser.add_argument( - "--to_cpu", + "--no_cuda", default=False, type=bool, help="Whether to force the execution on CPU.", @@ -314,7 +314,7 @@ def main(): args = parser.parse_args() # Select device (distibuted not available) - args.device = torch.device("cuda" if torch.cuda.is_available() and not args.to_cpu else "cpu") + args.device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") # Check the existence of directories if not args.summaries_output_dir: diff --git a/requirements.txt b/requirements.txt index 4a3162adce..9c43abc6d7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,4 +9,4 @@ regex # For XLNet sentencepiece # For XLM -sacremoses +sacremoses \ No newline at end of file From 608a8f5b567f81f3cc997a195496dd8bf1c28158 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 10 Dec 2019 10:01:01 +0100 Subject: [PATCH 284/505] updating tf 2.0 layer_norm to T5 layer norm --- transformers/modeling_tf_t5.py | 43 ++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/transformers/modeling_tf_t5.py b/transformers/modeling_tf_t5.py index c1de4745c2..11762ee1e5 100644 --- a/transformers/modeling_tf_t5.py +++ b/transformers/modeling_tf_t5.py @@ -17,16 +17,11 @@ from __future__ import absolute_import, division, print_function, unicode_literals -import json import logging import math -import os -import sys import copy import itertools -from io import open -import numpy as np import tensorflow as tf from .configuration_t5 import T5Config @@ -45,6 +40,28 @@ TF_T5_PRETRAINED_MODEL_ARCHIVE_MAP = { # - TFPreTrainedModel for the models (it-self a sub-class of tf.keras.Model) #################################################### +class TFT5LayerNorm(tf.keras.layers.Layer): + def __init__(self, epsilon=1e-6, **kwargs): + """ Construct a layernorm module in the T5 style + No bias and no substraction of mean. + """ + super(TFT5LayerNorm, self).__init__(**kwargs) + self.variance_epsilon = epsilon + + def build(self, input_shape): + """Build shared word embedding layer """ + self.weight = self.add_weight( + "weight", + shape=(input_shape[-1],), + initializer='ones') + super(TFT5LayerNorm, self).build(input_shape) + + def call(self, x): + variance = tf.math.reduce_min(tf.math.square(x), axis=-1, keepdims=True) + x = x * tf.math.rsqrt(variance + self.variance_epsilon) + return self.weight * x + + class TFT5DenseReluDense(tf.keras.layers.Layer): def __init__(self, config, **kwargs): super(TFT5DenseReluDense, self).__init__(**kwargs) @@ -65,8 +82,8 @@ class TFT5LayerFF(tf.keras.layers.Layer): def __init__(self, config, **kwargs): super(TFT5LayerFF, self).__init__(**kwargs) self.DenseReluDense = TFT5DenseReluDense(config, name='DenseReluDense') - self.layer_norm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_epsilon, - name='layer_norm') + self.layer_norm = TFT5LayerNorm(epsilon=config.layer_norm_epsilon, + name='layer_norm') self.dropout = tf.keras.layers.Dropout(config.dropout_rate) def call(self, hidden_states, training=False): @@ -249,8 +266,8 @@ class TFT5LayerSelfAttention(tf.keras.layers.Layer): self.SelfAttention = TFT5Attention(config, has_relative_attention_bias=has_relative_attention_bias, name='SelfAttention') - self.layer_norm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_epsilon, - name='layer_norm') + self.layer_norm = TFT5LayerNorm(epsilon=config.layer_norm_epsilon, + name='layer_norm') self.dropout = tf.keras.layers.Dropout(config.dropout_rate) def call(self, hidden_states, attention_mask=None, position_bias=None, @@ -273,8 +290,8 @@ class TFT5LayerCrossAttention(tf.keras.layers.Layer): self.EncDecAttention = TFT5Attention(config, has_relative_attention_bias=has_relative_attention_bias, name='EncDecAttention') - self.layer_norm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_epsilon, - name='layer_norm') + self.layer_norm = TFT5LayerNorm(epsilon=config.layer_norm_epsilon, + name='layer_norm') self.dropout = tf.keras.layers.Dropout(config.dropout_rate) def call(self, hidden_states, kv, attention_mask=None, position_bias=None, @@ -353,8 +370,8 @@ class TFT5MainLayer(tf.keras.layers.Layer): has_relative_attention_bias=bool(i == 0), name='block_._{}'.format(i)) for i in range(config.num_layers)] - self.final_layer_norm = tf.keras.layers.LayerNormalization(epsilon=config.layer_norm_epsilon, - name='final_layer_norm') + self.final_layer_norm = TFT5LayerNorm(epsilon=config.layer_norm_epsilon, + name='final_layer_norm') self.dropout = tf.keras.layers.Dropout(config.dropout_rate) def _resize_token_embeddings(self, new_num_tokens): From 72c36b9ea2e43d017d3aa5520d09f55d8ec19025 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Wed, 16 Oct 2019 14:17:58 +0200 Subject: [PATCH 285/505] [WIP] - CLI --- setup.py | 8 +- transformers-cli | 5 +- transformers/__init__.py | 2 + transformers/__main__.py | 145 ++++-------------- transformers/commands/convert.py | 115 ++++++++++++++ transformers/commands/serving.py | 176 ++++++++++++++++++++++ transformers/commands/train.py | 121 +++++++++++++++ transformers/data/__init__.py | 2 +- transformers/data/processors/__init__.py | 2 +- transformers/data/processors/utils.py | 182 +++++++++++++++++++++++ 10 files changed, 631 insertions(+), 127 deletions(-) create mode 100644 transformers/commands/convert.py create mode 100644 transformers/commands/serving.py create mode 100644 transformers/commands/train.py diff --git a/setup.py b/setup.py index c4af32df83..0b7e512955 100644 --- a/setup.py +++ b/setup.py @@ -62,15 +62,15 @@ setup( 'regex', 'sentencepiece', 'sacremoses'], + extras_require=extras, + scripts=[ + 'transformers-cli' + ], entry_points={ 'console_scripts': [ "transformers=transformers.__main__:main", ] }, - extras_require=extras, - scripts=[ - 'transformers-cli' - ], # python_requires='>=3.5.0', classifiers=[ 'Intended Audience :: Science/Research', diff --git a/transformers-cli b/transformers-cli index ef00d15aa3..7b0905d4b4 100644 --- a/transformers-cli +++ b/transformers-cli @@ -1,14 +1,15 @@ #!/usr/bin/env python from argparse import ArgumentParser +from transformers.commands.serving import ServeCommand from transformers.commands.user import UserCommands - if __name__ == '__main__': - parser = ArgumentParser(description='Transformers CLI tool', usage='transformers-cli []') + parser = ArgumentParser('Transformers CLI tool', usage='transformers-cli []') commands_parser = parser.add_subparsers(help='transformers-cli command helpers') # Register commands + ServeCommand.register_subcommand(commands_parser) UserCommands.register_subcommand(commands_parser) # Let's go diff --git a/transformers/__init__.py b/transformers/__init__.py index f9a28add5f..a71a291a44 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -24,6 +24,8 @@ from .file_utils import (TRANSFORMERS_CACHE, PYTORCH_TRANSFORMERS_CACHE, PYTORCH from .data import (is_sklearn_available, InputExample, InputFeatures, DataProcessor, + SingleSentenceClassificationProcessor, + convert_examples_to_features, glue_output_modes, glue_convert_examples_to_features, glue_processors, glue_tasks_num_labels, xnli_output_modes, xnli_processors, xnli_tasks_num_labels, diff --git a/transformers/__main__.py b/transformers/__main__.py index 31dbd24908..a6e9ae65e0 100644 --- a/transformers/__main__.py +++ b/transformers/__main__.py @@ -1,129 +1,36 @@ # coding: utf8 + def main(): import sys - if (len(sys.argv) < 4 or len(sys.argv) > 6) or sys.argv[1] not in ["bert", "gpt", "transfo_xl", "gpt2", "xlnet", "xlm"]: + if len(sys.argv) < 2 or sys.argv[1] not in ["convert", "train", "predict", "serve"]: print( - "This command line utility let you convert original (author released) model checkpoint to pytorch.\n" - "It should be used as one of: \n" - ">> transformers bert TF_CHECKPOINT TF_CONFIG PYTORCH_DUMP_OUTPUT, \n" - ">> transformers gpt OPENAI_GPT_CHECKPOINT_FOLDER_PATH PYTORCH_DUMP_OUTPUT [OPENAI_GPT_CONFIG], \n" - ">> transformers transfo_xl TF_CHECKPOINT_OR_DATASET PYTORCH_DUMP_OUTPUT [TF_CONFIG] or \n" - ">> transformers gpt2 TF_CHECKPOINT PYTORCH_DUMP_OUTPUT [GPT2_CONFIG] or \n" - ">> transformers xlnet TF_CHECKPOINT TF_CONFIG PYTORCH_DUMP_OUTPUT [FINETUNING_TASK_NAME] or \n" - ">> transformers xlm XLM_CHECKPOINT_PATH PYTORCH_DUMP_OUTPUT") - else: - if sys.argv[1] == "bert": - try: - from .convert_bert_original_tf_checkpoint_to_pytorch import convert_tf_checkpoint_to_pytorch - except ImportError: - print("transformers can only be used from the commandline to convert TensorFlow models in PyTorch, " - "In that case, it requires TensorFlow to be installed. Please see " - "https://www.tensorflow.org/install/ for installation instructions.") - raise + "First argument to `transformers` command line interface should be one of: \n" + ">> convert serve train predict") + if sys.argv[1] == "convert": + from transformers.commands import convert + convert(sys.argv) + elif sys.argv[1] == "train": + from transformers.commands import train + train(sys.argv) + elif sys.argv[1] == "serve": + pass + # from argparse import ArgumentParser + # from transformers.commands.serving import ServeCommand + # parser = ArgumentParser('Transformers CLI tool', usage='transformers serve []') + # commands_parser = parser.add_subparsers(help='transformers-cli command helpers') - if len(sys.argv) != 5: - # pylint: disable=line-too-long - print("Should be used as `transformers bert TF_CHECKPOINT TF_CONFIG PYTORCH_DUMP_OUTPUT`") - else: - PYTORCH_DUMP_OUTPUT = sys.argv.pop() - TF_CONFIG = sys.argv.pop() - TF_CHECKPOINT = sys.argv.pop() - convert_tf_checkpoint_to_pytorch(TF_CHECKPOINT, TF_CONFIG, PYTORCH_DUMP_OUTPUT) - elif sys.argv[1] == "gpt": - from .convert_openai_original_tf_checkpoint_to_pytorch import convert_openai_checkpoint_to_pytorch - if len(sys.argv) < 4 or len(sys.argv) > 5: - # pylint: disable=line-too-long - print("Should be used as `transformers gpt OPENAI_GPT_CHECKPOINT_FOLDER_PATH PYTORCH_DUMP_OUTPUT [OPENAI_GPT_CONFIG]`") - else: - OPENAI_GPT_CHECKPOINT_FOLDER_PATH = sys.argv[2] - PYTORCH_DUMP_OUTPUT = sys.argv[3] - if len(sys.argv) == 5: - OPENAI_GPT_CONFIG = sys.argv[4] - else: - OPENAI_GPT_CONFIG = "" - convert_openai_checkpoint_to_pytorch(OPENAI_GPT_CHECKPOINT_FOLDER_PATH, - OPENAI_GPT_CONFIG, - PYTORCH_DUMP_OUTPUT) - elif sys.argv[1] == "transfo_xl": - try: - from .convert_transfo_xl_original_tf_checkpoint_to_pytorch import convert_transfo_xl_checkpoint_to_pytorch - except ImportError: - print("transformers can only be used from the commandline to convert TensorFlow models in PyTorch, " - "In that case, it requires TensorFlow to be installed. Please see " - "https://www.tensorflow.org/install/ for installation instructions.") - raise - if len(sys.argv) < 4 or len(sys.argv) > 5: - # pylint: disable=line-too-long - print("Should be used as `transformers transfo_xl TF_CHECKPOINT/TF_DATASET_FILE PYTORCH_DUMP_OUTPUT [TF_CONFIG]`") - else: - if 'ckpt' in sys.argv[2].lower(): - TF_CHECKPOINT = sys.argv[2] - TF_DATASET_FILE = "" - else: - TF_DATASET_FILE = sys.argv[2] - TF_CHECKPOINT = "" - PYTORCH_DUMP_OUTPUT = sys.argv[3] - if len(sys.argv) == 5: - TF_CONFIG = sys.argv[4] - else: - TF_CONFIG = "" - convert_transfo_xl_checkpoint_to_pytorch(TF_CHECKPOINT, TF_CONFIG, PYTORCH_DUMP_OUTPUT, TF_DATASET_FILE) - elif sys.argv[1] == "gpt2": - try: - from .convert_gpt2_original_tf_checkpoint_to_pytorch import convert_gpt2_checkpoint_to_pytorch - except ImportError: - print("transformers can only be used from the commandline to convert TensorFlow models in PyTorch, " - "In that case, it requires TensorFlow to be installed. Please see " - "https://www.tensorflow.org/install/ for installation instructions.") - raise + # # Register commands + # ServeCommand.register_subcommand(commands_parser) - if len(sys.argv) < 4 or len(sys.argv) > 5: - # pylint: disable=line-too-long - print("Should be used as `transformers gpt2 TF_CHECKPOINT PYTORCH_DUMP_OUTPUT [TF_CONFIG]`") - else: - TF_CHECKPOINT = sys.argv[2] - PYTORCH_DUMP_OUTPUT = sys.argv[3] - if len(sys.argv) == 5: - TF_CONFIG = sys.argv[4] - else: - TF_CONFIG = "" - convert_gpt2_checkpoint_to_pytorch(TF_CHECKPOINT, TF_CONFIG, PYTORCH_DUMP_OUTPUT) - elif sys.argv[1] == "xlnet": - try: - from .convert_xlnet_original_tf_checkpoint_to_pytorch import convert_xlnet_checkpoint_to_pytorch - except ImportError: - print("transformers can only be used from the commandline to convert TensorFlow models in PyTorch, " - "In that case, it requires TensorFlow to be installed. Please see " - "https://www.tensorflow.org/install/ for installation instructions.") - raise + # # Let's go + # args = parser.parse_args() - if len(sys.argv) < 5 or len(sys.argv) > 6: - # pylint: disable=line-too-long - print("Should be used as `transformers xlnet TF_CHECKPOINT TF_CONFIG PYTORCH_DUMP_OUTPUT [FINETUNING_TASK_NAME]`") - else: - TF_CHECKPOINT = sys.argv[2] - TF_CONFIG = sys.argv[3] - PYTORCH_DUMP_OUTPUT = sys.argv[4] - if len(sys.argv) == 6: - FINETUNING_TASK = sys.argv[5] - else: - FINETUNING_TASK = None - - convert_xlnet_checkpoint_to_pytorch(TF_CHECKPOINT, - TF_CONFIG, - PYTORCH_DUMP_OUTPUT, - FINETUNING_TASK) - elif sys.argv[1] == "xlm": - from .convert_xlm_original_pytorch_checkpoint_to_pytorch import convert_xlm_checkpoint_to_pytorch - - if len(sys.argv) != 4: - # pylint: disable=line-too-long - print("Should be used as `transformers xlm XLM_CHECKPOINT_PATH PYTORCH_DUMP_OUTPUT`") - else: - XLM_CHECKPOINT_PATH = sys.argv[2] - PYTORCH_DUMP_OUTPUT = sys.argv[3] - - convert_xlm_checkpoint_to_pytorch(XLM_CHECKPOINT_PATH, PYTORCH_DUMP_OUTPUT) + # if not hasattr(args, 'func'): + # parser.print_help() + # exit(1) + # # Run + # service = args.func(args) + # service.run() if __name__ == '__main__': main() diff --git a/transformers/commands/convert.py b/transformers/commands/convert.py new file mode 100644 index 0000000000..55dbf53734 --- /dev/null +++ b/transformers/commands/convert.py @@ -0,0 +1,115 @@ +from argparse import ArgumentParser, Namespace + +from logging import getLogger + +from transformers import AutoModel, AutoTokenizer +from transformers.commands import BaseTransformersCLICommand + + +def convert_command_factory(args: Namespace): + """ + Factory function used to convert a model TF 1.0 checkpoint in a PyTorch checkpoint. + :return: ServeCommand + """ + return ConvertCommand(args.model_type, args.tf_checkpoint, args.pytorch_dump_output, + args.config, args.finetuning_task_name) + + +class ConvertCommand(BaseTransformersCLICommand): + + @staticmethod + def register_subcommand(parser: ArgumentParser): + """ + Register this command to argparse so it's available for the transformer-cli + :param parser: Root parser to register command-specific arguments + :return: + """ + train_parser = parser.add_parser('convert', help="CLI tool to run convert model from original " + "author checkpoints to Transformesr PyTorch checkpoints.") + train_parser.add_argument('--model_type', type=str, required=True, + help='Model\'s type.') + train_parser.add_argument('--tf_checkpoint', type=str, required=True, + help='TensorFlow checkpoint path or folder.') + train_parser.add_argument('--pytorch_dump_output', type=str, required=True, + help='Path to the PyTorch savd model output.') + train_parser.add_argument('--config', type=str, default="", + help='Configuration file path or folder.') + train_parser.add_argument('--finetuning_task_name', type=str, default=None, + help='Optional fine-tuning task name if the TF model was a finetuned model.') + train_parser.set_defaults(func=convert_command_factory) + + def __init__(self, model_type: str, tf_checkpoint: str, pytorch_dump_output: str, + config: str, finetuning_task_name: str, *args): + self._logger = getLogger('transformers-cli/converting') + + self._logger.info('Loading model {}'.format(model_type)) + self._model_type = model_type + self._tf_checkpoint = tf_checkpoint + self._pytorch_dump_output = pytorch_dump_output + self._config = config + self._finetuning_task_name = finetuning_task_name + + def run(self): + if self._model_type == "bert": + try: + from transformers.convert_bert_original_tf_checkpoint_to_pytorch import convert_tf_checkpoint_to_pytorch + except ImportError: + msg = "transformers can only be used from the commandline to convert TensorFlow models in PyTorch, " \ + "In that case, it requires TensorFlow to be installed. Please see " \ + "https://www.tensorflow.org/install/ for installation instructions." + raise ImportError(msg) + + convert_tf_checkpoint_to_pytorch(self._tf_checkpoint, self._config, self._pytorch_dump_output) + elif self._model_type == "gpt": + from transformers.convert_openai_original_tf_checkpoint_to_pytorch import convert_openai_checkpoint_to_pytorch + convert_openai_checkpoint_to_pytorch(self._tf_checkpoint, + self._config, + self._pytorch_dump_output) + elif self._model_type == "transfo_xl": + try: + from transformers.convert_transfo_xl_original_tf_checkpoint_to_pytorch import convert_transfo_xl_checkpoint_to_pytorch + except ImportError: + msg = "transformers can only be used from the commandline to convert TensorFlow models in PyTorch, " \ + "In that case, it requires TensorFlow to be installed. Please see " \ + "https://www.tensorflow.org/install/ for installation instructions." + raise ImportError(msg) + + if 'ckpt' in self._tf_checkpoint.lower(): + TF_CHECKPOINT = self._tf_checkpoint + TF_DATASET_FILE = "" + else: + TF_DATASET_FILE = self._tf_checkpoint + TF_CHECKPOINT = "" + convert_transfo_xl_checkpoint_to_pytorch(TF_CHECKPOINT, + self._config, + self._pytorch_dump_output, + TF_DATASET_FILE) + elif self._model_type == "gpt2": + try: + from transformers.convert_gpt2_original_tf_checkpoint_to_pytorch import convert_gpt2_checkpoint_to_pytorch + except ImportError: + msg = "transformers can only be used from the commandline to convert TensorFlow models in PyTorch, " \ + "In that case, it requires TensorFlow to be installed. Please see " \ + "https://www.tensorflow.org/install/ for installation instructions." + raise ImportError(msg) + + convert_gpt2_checkpoint_to_pytorch(self._tf_checkpoint, self._config, self._pytorch_dump_output) + elif self._model_type == "xlnet": + try: + from transformers.convert_xlnet_original_tf_checkpoint_to_pytorch import convert_xlnet_checkpoint_to_pytorch + except ImportError: + msg = "transformers can only be used from the commandline to convert TensorFlow models in PyTorch, " \ + "In that case, it requires TensorFlow to be installed. Please see " \ + "https://www.tensorflow.org/install/ for installation instructions." + raise ImportError(msg) + + convert_xlnet_checkpoint_to_pytorch(self._tf_checkpoint, + self._config, + self._pytorch_dump_output, + self._finetuning_task_name) + elif self._model_type == "xlm": + from transformers.convert_xlm_original_pytorch_checkpoint_to_pytorch import convert_xlm_checkpoint_to_pytorch + + convert_xlm_checkpoint_to_pytorch(self._tf_checkpoint, self._pytorch_dump_output) + else: + raise ValueError("--model_type should be selected in the list [bert, gpt, gpt2, transfo_xl, xlnet, xlm]") diff --git a/transformers/commands/serving.py b/transformers/commands/serving.py new file mode 100644 index 0000000000..0b47246ead --- /dev/null +++ b/transformers/commands/serving.py @@ -0,0 +1,176 @@ +from argparse import ArgumentParser, Namespace +from typing import List, Optional, Union, Any + +import torch +from fastapi import FastAPI, HTTPException, Body +from logging import getLogger + +from pydantic import BaseModel +from uvicorn import run + +from transformers import AutoModel, AutoTokenizer, AutoConfig +from transformers.commands import BaseTransformersCLICommand + + +def serve_command_factory(args: Namespace): + """ + Factory function used to instantiate serving server from provided command line arguments. + :return: ServeCommand + """ + return ServeCommand(args.host, args.port, args.model, args.graphql) + + +class ServeResult(BaseModel): + """ + Base class for serving result + """ + model: str + + +class ServeModelInfoResult(ServeResult): + """ + Expose model information + """ + infos: dict + + +class ServeTokenizeResult(ServeResult): + """ + Tokenize result model + """ + tokens: List[str] + tokens_ids: Optional[List[int]] + + +class ServeDeTokenizeResult(ServeResult): + """ + DeTokenize result model + """ + text: str + + +class ServeForwardResult(ServeResult): + """ + Forward result model + """ + tokens: List[str] + tokens_ids: List[int] + output: Any + + +class ServeCommand(BaseTransformersCLICommand): + + @staticmethod + def register_subcommand(parser: ArgumentParser): + """ + Register this command to argparse so it's available for the transformer-cli + :param parser: Root parser to register command-specific arguments + :return: + """ + serve_parser = parser.add_parser('serve', help='CLI tool to run inference requests through REST and GraphQL endpoints.') + serve_parser.add_argument('--host', type=str, default='localhost', help='Interface the server will listen on.') + serve_parser.add_argument('--port', type=int, default=8888, help='Port the serving will listen to.') + serve_parser.add_argument('--model', type=str, required=True, help='Model\'s name or path to stored model to infer from.') + serve_parser.add_argument('--graphql', action='store_true', default=False, help='Enable GraphQL endpoints.') + serve_parser.set_defaults(func=serve_command_factory) + + def __init__(self, host: str, port: int, model: str, graphql: bool): + self._logger = getLogger('transformers-cli/serving') + + self._logger.info('Loading model {}'.format(model)) + self._model_name = model + self._model = AutoModel.from_pretrained(model) + self._tokenizer = AutoTokenizer.from_pretrained(model) + + self._logger.info('Serving model over {}:{}'.format(host, port)) + self._host = host + self._port = port + self._app = FastAPI() + + # Register routes + self._app.add_api_route('/', self.model_info, response_model=ServeModelInfoResult, methods=['GET']) + self._app.add_api_route('/tokenize', self.tokenize, response_model=ServeTokenizeResult, methods=['POST']) + self._app.add_api_route('/detokenize', self.detokenize, response_model=ServeDeTokenizeResult, methods=['POST']) + self._app.add_api_route('/forward', self.forward, response_model=ServeForwardResult, methods=['POST']) + + def run(self): + run(self._app, host=self._host, port=self._port) + + def model_info(self): + return ServeModelInfoResult(model=self._model_name, infos=vars(self._model.config)) + + def tokenize(self, text_input: str = Body(None, embed=True), return_ids: bool = Body(False, embed=True)): + """ + Tokenize the provided input and eventually returns corresponding tokens id: + - **text_input**: String to tokenize + - **return_ids**: Boolean flags indicating if the tokens have to be converted to their integer mapping. + """ + try: + tokens_txt = self._tokenizer.tokenize(text_input) + + if return_ids: + tokens_ids = self._tokenizer.convert_tokens_to_ids(tokens_txt) + return ServeTokenizeResult(model=self._model_name, tokens=tokens_txt, tokens_ids=tokens_ids) + else: + return ServeTokenizeResult(model=self._model_name, tokens=tokens_txt) + + except Exception as e: + raise HTTPException(status_code=500, detail={"model": self._model_name, "error": str(e)}) + + def detokenize(self, tokens_ids: List[int] = Body(None, embed=True), + skip_special_tokens: bool = Body(False, embed=True), + cleanup_tokenization_spaces: bool = Body(True, embed=True)): + """ + Detokenize the provided tokens ids to readable text: + - **tokens_ids**: List of tokens ids + - **skip_special_tokens**: Flag indicating to not try to decode special tokens + - **cleanup_tokenization_spaces**: Flag indicating to remove all leading/trailing spaces and intermediate ones. + """ + try: + decoded_str = self._tokenizer.decode(tokens_ids, skip_special_tokens, cleanup_tokenization_spaces) + return ServeDeTokenizeResult(model=self._model_name, text=decoded_str) + except Exception as e: + raise HTTPException(status_code=500, detail={"model": self._model_name, "error": str(e)}) + + def forward(self, inputs: Union[str, List[str], List[int]] = Body(None, embed=True), + attention_mask: Optional[List[int]] = Body(None, embed=True), + tokens_type_ids: Optional[List[int]] = Body(None, embed=True)): + """ + **inputs**: + **attention_mask**: + **tokens_type_ids**: + """ + + # Check we don't have empty string + if len(inputs) == 0: + return ServeForwardResult(model=self._model_name, output=[], attention=[]) + + if isinstance(inputs, str): + inputs_tokens = self._tokenizer.tokenize(inputs) + inputs_ids = self._tokenizer.convert_tokens_to_ids(inputs_tokens) + + elif isinstance(inputs, List): + if isinstance(inputs[0], str): + inputs_tokens = inputs + inputs_ids = self._tokenizer.convert_tokens_to_ids(inputs_tokens) + elif isinstance(inputs[0], int): + inputs_tokens = [] + inputs_ids = inputs + else: + error_msg = "inputs should be string, [str] of [int] (got {})".format(type(inputs[0])) + raise HTTPException(423, detail={"error": error_msg}) + else: + error_msg = "inputs should be string, [str] of [int] (got {})".format(type(inputs)) + raise HTTPException(423, detail={"error": error_msg}) + + try: + # Forward through the model + t_input_ids = torch.tensor(inputs_ids).unsqueeze(0) + output = self._model(t_input_ids, attention_mask, tokens_type_ids) + + return ServeForwardResult( + model=self._model_name, tokens=inputs_tokens, + tokens_ids=inputs_ids, output=output[0].tolist() + ) + except Exception as e: + raise HTTPException(500, {"error": str(e)}) diff --git a/transformers/commands/train.py b/transformers/commands/train.py new file mode 100644 index 0000000000..7fb3a54d25 --- /dev/null +++ b/transformers/commands/train.py @@ -0,0 +1,121 @@ +from argparse import ArgumentParser, Namespace + +from logging import getLogger + +from transformers.commands import BaseTransformersCLICommand +from transformers import (AutoTokenizer, is_tf_available, is_torch_available, + SingleSentenceClassificationProcessor, + convert_examples_to_features) +if is_tf_available(): + from transformers import TFAutoModelForSequenceClassification as SequenceClassifModel +elif is_torch_available(): + from transformers import AutoModelForSequenceClassification as SequenceClassifModel +else: + raise ImportError("At least one of PyTorch or TensorFlow 2.0+ should be installed to use CLI training") + +# TF training parameters +BATCH_SIZE = 32 +EVAL_BATCH_SIZE = BATCH_SIZE * 2 +USE_XLA = False +USE_AMP = False + +def train_command_factory(args: Namespace): + """ + Factory function used to instantiate serving server from provided command line arguments. + :return: ServeCommand + """ + return TrainCommand(args.model) + + +class TrainCommand(BaseTransformersCLICommand): + + @staticmethod + def register_subcommand(parser: ArgumentParser): + """ + Register this command to argparse so it's available for the transformer-cli + :param parser: Root parser to register command-specific arguments + :return: + """ + train_parser = parser.add_parser('train', help='CLI tool to train a model on a task.') + train_parser.add_argument('--train_data', type=str, required=True, + help='path to train (and optionally evaluation) dataset.') + train_parser.add_argument('--task', type=str, default='text_classification', + help='Task to train the model on.') + train_parser.add_argument('--model', type=str, default='bert-base-uncased', + help='Model\'s name or path to stored model.') + train_parser.add_argument('--valid_data', type=str, default='', + help='path to validation dataset.') + train_parser.add_argument('--valid_data_ratio', type=float, default=0.1, + help="if validation dataset is not provided, fraction of train dataset " + "to use as validation dataset.") + train_parser.set_defaults(func=train_command_factory) + + def __init__(self, model_name: str, task: str, train_data: str, + valid_data: str, valid_data_ratio: float): + self._logger = getLogger('transformers-cli/training') + + self._framework = 'tf' if is_tf_available() else 'torch' + + self._logger.info('Loading model {}'.format(model_name)) + self._model_name = model_name + self._tokenizer = AutoTokenizer.from_pretrained(model_name) + if task == 'text_classification': + self._model = SequenceClassifModel.from_pretrained(model_name) + elif task == 'token_classification': + raise NotImplementedError + elif task == 'question_answering': + raise NotImplementedError + + dataset = SingleSentenceClassificationProcessor.create_from_csv(train_data) + num_data_samples = len(SingleSentenceClassificationProcessor) + if valid_data: + self._train_dataset = dataset + self._num_train_samples = num_data_samples + self._valid_dataset = SingleSentenceClassificationProcessor.create_from_csv(valid_data) + self._num_valid_samples = len(self._valid_dataset) + else: + assert 0.0 < valid_data_ratio < 1.0, "--valid_data_ratio should be between 0.0 and 1.0" + self._num_valid_samples = num_data_samples * valid_data_ratio + self._num_train_samples = num_data_samples - self._num_valid_samples + self._train_dataset = dataset[self._num_train_samples] + self._valid_dataset = dataset[self._num_valid_samples] + + def run(self): + if self._framework == 'tf': + return self.run_tf() + return self.run_torch() + + def run_torch(self): + raise NotImplementedError + + def run_tf(self): + import tensorflow as tf + + tf.config.optimizer.set_jit(USE_XLA) + tf.config.optimizer.set_experimental_options({"auto_mixed_precision": USE_AMP}) + + # Prepare dataset as a tf.train_data.Dataset instance + train_dataset = convert_examples_to_features(self._train_dataset, self._tokenizer, mode='sequence_classification') + valid_dataset = convert_examples_to_features(self._valid_dataset, self._tokenizer, mode='sequence_classification') + train_dataset = train_dataset.shuffle(128).batch(BATCH_SIZE).repeat(-1) + valid_dataset = valid_dataset.batch(EVAL_BATCH_SIZE) + + # Prepare training: Compile tf.keras model with optimizer, loss and learning rate schedule + opt = tf.keras.optimizers.Adam(learning_rate=3e-5, epsilon=1e-08) + if USE_AMP: + # loss scaling is currently required when using mixed precision + opt = tf.keras.mixed_precision.experimental.LossScaleOptimizer(opt, 'dynamic') + loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + metric = tf.keras.metrics.SparseCategoricalAccuracy('accuracy') + model.compile(optimizer=opt, loss=loss, metrics=[metric]) + + # Train and evaluate using tf.keras.Model.fit() + train_steps = train_examples//BATCH_SIZE + valid_steps = valid_examples//EVAL_BATCH_SIZE + + history = model.fit(train_dataset, epochs=2, steps_per_epoch=train_steps, + validation_data=valid_dataset, validation_steps=valid_steps) + + # Save TF2 model + os.makedirs('./save/', exist_ok=True) + model.save_pretrained('./save/') diff --git a/transformers/data/__init__.py b/transformers/data/__init__.py index 270a053268..5567952fd2 100644 --- a/transformers/data/__init__.py +++ b/transformers/data/__init__.py @@ -1,4 +1,4 @@ -from .processors import InputExample, InputFeatures, DataProcessor, SquadFeatures +from .processors import InputExample, InputFeatures, DataProcessor, SquadFeatures, SingleSentenceClassificationProcessor from .processors import glue_output_modes, glue_processors, glue_tasks_num_labels, glue_convert_examples_to_features from .processors import squad_convert_examples_to_features, SquadExample, SquadV1Processor, SquadV2Processor from .processors import xnli_output_modes, xnli_processors, xnli_tasks_num_labels diff --git a/transformers/data/processors/__init__.py b/transformers/data/processors/__init__.py index 0f1b24893a..0cef0080f4 100644 --- a/transformers/data/processors/__init__.py +++ b/transformers/data/processors/__init__.py @@ -1,4 +1,4 @@ -from .utils import InputExample, InputFeatures, DataProcessor +from .utils import InputExample, InputFeatures, DataProcessor, SingleSentenceClassificationProcessor, convert_examples_to_features from .glue import glue_output_modes, glue_processors, glue_tasks_num_labels, glue_convert_examples_to_features from .squad import squad_convert_examples_to_features, SquadFeatures, SquadExample, SquadV1Processor, SquadV2Processor from .xnli import xnli_output_modes, xnli_processors, xnli_tasks_num_labels \ No newline at end of file diff --git a/transformers/data/processors/utils.py b/transformers/data/processors/utils.py index 07bdf3150c..39544a1239 100644 --- a/transformers/data/processors/utils.py +++ b/transformers/data/processors/utils.py @@ -125,3 +125,185 @@ class DataProcessor(object): line = list(unicode(cell, 'utf-8') for cell in line) lines.append(line) return lines + + +class SingleSentenceClassificationProcessor(DataProcessor): + """ Generic processor for a single sentence classification data set.""" + def __init__(self, labels=None, examples=None): + self.labels = [] if labels is None else labels + self.examples = [] if examples is None else examples + + @classmethod + def create_from_csv(cls, file_name): + processor = cls() + processor.add_examples_from_csv(file_name) + return processor + + def __len__(self): + return len(self.examples) + + def __getitem__(self, idx): + if isinstance(idx, slice): + return SingleSentenceClassificationProcessor(labels=self.labels, + examples=self.examples[idx]) + return self.examples[idx] + + def get_labels(self): + """Gets the list of labels for this data set.""" + return self.labels + + def add_examples_from_csv(self, file_name): + lines = self._read_tsv(file_name) + self.add_examples_from_lines(lines) + + def add_examples_from_lines(self, lines, split_name='', overwrite_labels=False, overwrite_examples=False): + """Creates examples for the training and dev sets.""" + added_labels = set() + examples = [] + for (i, line) in enumerate(lines): + if len(line) > 2: + guid = "%s-%s" % (split_name, line[0]) if split_name else line[0] + label = line[1] + text_a = line[2] + else: + guid = "%s-%s" % (split_name, i) if split_name else "%s" % i + label = line[0] + text_a = line[1] + + added_labels.add(label) + examples.append(InputExample(guid=guid, text_a=text_a, text_b=None, label=label)) + + # Update examples + if overwrite_examples: + self.examples = examples + else: + self.examples.extend(examples) + + # Update labels + if overwrite_labels: + self.labels = list(added_labels) + else: + self.labels = list(set(self.labels).union(added_labels)) + + return self.examples + + +def convert_examples_to_features(examples, tokenizer, + mode='sequence_classification', + max_length=512, + pad_on_left=False, + pad_token=0, + pad_token_segment_id=0, + mask_padding_with_zero=True): + """ + Loads a data file into a list of ``InputFeatures`` + + Args: + examples: List of ``InputExamples`` or ``tf.data.Dataset`` containing the examples. + tokenizer: Instance of a tokenizer that will tokenize the examples + max_length: Maximum example length + task: GLUE task + label_list: List of labels. Can be obtained from the processor using the ``processor.get_labels()`` method + output_mode: String indicating the output mode. Either ``regression`` or ``classification`` + pad_on_left: If set to ``True``, the examples will be padded on the left rather than on the right (default) + pad_token: Padding token + pad_token_segment_id: The segment ID for the padding token (It is usually 0, but can vary such as for XLNet where it is 4) + mask_padding_with_zero: If set to ``True``, the attention mask will be filled by ``1`` for actual values + and by ``0`` for padded values. If set to ``False``, inverts it (``1`` for padded values, ``0`` for + actual values) + + Returns: + If the ``examples`` input is a ``tf.data.Dataset``, will return a ``tf.data.Dataset`` + containing the task-specific features. If the input is a list of ``InputExamples``, will return + a list of task-specific ``InputFeatures`` which can be fed to the model. + + """ + is_tf_dataset = False + if is_tf_available() and isinstance(examples, tf.data.Dataset): + is_tf_dataset = True + + if task is not None: + processor = glue_processors[task]() + if label_list is None: + label_list = processor.get_labels() + logger.info("Using label list %s for task %s" % (label_list, task)) + if output_mode is None: + output_mode = glue_output_modes[task] + logger.info("Using output mode %s for task %s" % (output_mode, task)) + + label_map = {label: i for i, label in enumerate(label_list)} + + features = [] + for (ex_index, example) in enumerate(examples): + if ex_index % 10000 == 0: + logger.info("Writing example %d" % (ex_index)) + if is_tf_dataset: + example = processor.get_example_from_tensor_dict(example) + + inputs = tokenizer.encode_plus( + example.text_a, + example.text_b, + add_special_tokens=True, + max_length=max_length, + ) + input_ids, token_type_ids = inputs["input_ids"], inputs["token_type_ids"] + + # The mask has 1 for real tokens and 0 for padding tokens. Only real + # tokens are attended to. + attention_mask = [1 if mask_padding_with_zero else 0] * len(input_ids) + + # Zero-pad up to the sequence length. + padding_length = max_length - len(input_ids) + if pad_on_left: + input_ids = ([pad_token] * padding_length) + input_ids + attention_mask = ([0 if mask_padding_with_zero else 1] * padding_length) + attention_mask + token_type_ids = ([pad_token_segment_id] * padding_length) + token_type_ids + else: + input_ids = input_ids + ([pad_token] * padding_length) + attention_mask = attention_mask + ([0 if mask_padding_with_zero else 1] * padding_length) + token_type_ids = token_type_ids + ([pad_token_segment_id] * padding_length) + + assert len(input_ids) == max_length, "Error with input length {} vs {}".format(len(input_ids), max_length) + assert len(attention_mask) == max_length, "Error with input length {} vs {}".format(len(attention_mask), max_length) + assert len(token_type_ids) == max_length, "Error with input length {} vs {}".format(len(token_type_ids), max_length) + + if output_mode == "classification": + label = label_map[example.label] + elif output_mode == "regression": + label = float(example.label) + else: + raise KeyError(output_mode) + + if ex_index < 5: + logger.info("*** Example ***") + logger.info("guid: %s" % (example.guid)) + logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids])) + logger.info("attention_mask: %s" % " ".join([str(x) for x in attention_mask])) + logger.info("token_type_ids: %s" % " ".join([str(x) for x in token_type_ids])) + logger.info("label: %s (id = %d)" % (example.label, label)) + + features.append( + InputFeatures(input_ids=input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + label=label)) + + if is_tf_available() and is_tf_dataset: + def gen(): + for ex in features: + yield ({'input_ids': ex.input_ids, + 'attention_mask': ex.attention_mask, + 'token_type_ids': ex.token_type_ids}, + ex.label) + + return tf.data.Dataset.from_generator(gen, + ({'input_ids': tf.int32, + 'attention_mask': tf.int32, + 'token_type_ids': tf.int32}, + tf.int64), + ({'input_ids': tf.TensorShape([None]), + 'attention_mask': tf.TensorShape([None]), + 'token_type_ids': tf.TensorShape([None])}, + tf.TensorShape([]))) + + return features From 2d8559731acbf673fe3e31aaeb17412a342cff73 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Wed, 16 Oct 2019 23:19:45 +0200 Subject: [PATCH 286/505] add pipeline - train --- transformers/commands/train.py | 127 +++++++---- transformers/data/processors/utils.py | 313 +++++++++++++------------- transformers/pipeline.py | 254 +++++++++++++++++++++ 3 files changed, 485 insertions(+), 209 deletions(-) create mode 100644 transformers/pipeline.py diff --git a/transformers/commands/train.py b/transformers/commands/train.py index 7fb3a54d25..fc89d48594 100644 --- a/transformers/commands/train.py +++ b/transformers/commands/train.py @@ -1,5 +1,5 @@ +import os from argparse import ArgumentParser, Namespace - from logging import getLogger from transformers.commands import BaseTransformersCLICommand @@ -14,8 +14,6 @@ else: raise ImportError("At least one of PyTorch or TensorFlow 2.0+ should be installed to use CLI training") # TF training parameters -BATCH_SIZE = 32 -EVAL_BATCH_SIZE = BATCH_SIZE * 2 USE_XLA = False USE_AMP = False @@ -24,7 +22,7 @@ def train_command_factory(args: Namespace): Factory function used to instantiate serving server from provided command line arguments. :return: ServeCommand """ - return TrainCommand(args.model) + return TrainCommand(args) class TrainCommand(BaseTransformersCLICommand): @@ -38,50 +36,84 @@ class TrainCommand(BaseTransformersCLICommand): """ train_parser = parser.add_parser('train', help='CLI tool to train a model on a task.') train_parser.add_argument('--train_data', type=str, required=True, - help='path to train (and optionally evaluation) dataset.') + help="path to train (and optionally evaluation) dataset as a csv with " + "tab separated labels and sentences.") + + train_parser.add_argument('--column_label', type=int, default=0, + help='Column of the dataset csv file with example labels.') + train_parser.add_argument('--column_text', type=int, default=1, + help='Column of the dataset csv file with example texts.') + train_parser.add_argument('--column_id', type=int, default=2, + help='Column of the dataset csv file with example ids.') + + train_parser.add_argument('--validation_data', type=str, default='', + help='path to validation dataset.') + train_parser.add_argument('--validation_split', type=float, default=0.1, + help="if validation dataset is not provided, fraction of train dataset " + "to use as validation dataset.") + + train_parser.add_argument('--output', type=str, default='./', + help='path to saved the trained model.') + train_parser.add_argument('--task', type=str, default='text_classification', help='Task to train the model on.') train_parser.add_argument('--model', type=str, default='bert-base-uncased', help='Model\'s name or path to stored model.') - train_parser.add_argument('--valid_data', type=str, default='', - help='path to validation dataset.') - train_parser.add_argument('--valid_data_ratio', type=float, default=0.1, - help="if validation dataset is not provided, fraction of train dataset " - "to use as validation dataset.") + train_parser.add_argument('--train_batch_size', type=int, default=32, + help='Batch size for training.') + train_parser.add_argument('--valid_batch_size', type=int, default=64, + help='Batch size for validation.') + train_parser.add_argument('--learning_rate', type=float, default=3e-5, + help="Learning rate.") + train_parser.add_argument('--adam_epsilon', type=float, default=1e-08, + help="Epsilon for Adam optimizer.") train_parser.set_defaults(func=train_command_factory) - def __init__(self, model_name: str, task: str, train_data: str, - valid_data: str, valid_data_ratio: float): - self._logger = getLogger('transformers-cli/training') + def __init__(self, args: Namespace): + self.logger = getLogger('transformers-cli/training') - self._framework = 'tf' if is_tf_available() else 'torch' + self.framework = 'tf' if is_tf_available() else 'torch' - self._logger.info('Loading model {}'.format(model_name)) - self._model_name = model_name - self._tokenizer = AutoTokenizer.from_pretrained(model_name) - if task == 'text_classification': - self._model = SequenceClassifModel.from_pretrained(model_name) - elif task == 'token_classification': + os.makedirs(args.output) + self.output = args.output + + self.column_label = args.column_label + self.column_text = args.column_text + self.column_id = args.column_id + + self.logger.info('Loading model {}'.format(args.model_name)) + self.model_name = args.model_name + self.tokenizer = AutoTokenizer.from_pretrained(args.model_name) + if args.task == 'text_classification': + self.model = SequenceClassifModel.from_pretrained(args.model_name) + elif args.task == 'token_classification': raise NotImplementedError - elif task == 'question_answering': + elif args.task == 'question_answering': raise NotImplementedError - dataset = SingleSentenceClassificationProcessor.create_from_csv(train_data) - num_data_samples = len(SingleSentenceClassificationProcessor) - if valid_data: - self._train_dataset = dataset - self._num_train_samples = num_data_samples - self._valid_dataset = SingleSentenceClassificationProcessor.create_from_csv(valid_data) - self._num_valid_samples = len(self._valid_dataset) + self.logger.info('Loading dataset from {}'.format(args.train_data)) + dataset = SingleSentenceClassificationProcessor.create_from_csv(args.train_data) + num_data_samples = len(dataset) + if args.validation_data: + self.logger.info('Loading validation dataset from {}'.format(args.validation_data)) + self.valid_dataset = SingleSentenceClassificationProcessor.create_from_csv(args.validation_data) + self.num_valid_samples = len(self.valid_dataset) + self.train_dataset = dataset + self.num_train_samples = num_data_samples else: - assert 0.0 < valid_data_ratio < 1.0, "--valid_data_ratio should be between 0.0 and 1.0" - self._num_valid_samples = num_data_samples * valid_data_ratio - self._num_train_samples = num_data_samples - self._num_valid_samples - self._train_dataset = dataset[self._num_train_samples] - self._valid_dataset = dataset[self._num_valid_samples] + assert 0.0 < args.validation_split < 1.0, "--validation_split should be between 0.0 and 1.0" + self.num_valid_samples = num_data_samples * args.validation_split + self.num_train_samples = num_data_samples - self.num_valid_samples + self.train_dataset = dataset[self.num_train_samples] + self.valid_dataset = dataset[self.num_valid_samples] + + self.train_batch_size = args.train_batch_size + self.valid_batch_size = args.valid_batch_size + self.learning_rate = args.learning_rate + self.adam_epsilon = args.adam_epsilon def run(self): - if self._framework == 'tf': + if self.framework == 'tf': return self.run_tf() return self.run_torch() @@ -95,27 +127,28 @@ class TrainCommand(BaseTransformersCLICommand): tf.config.optimizer.set_experimental_options({"auto_mixed_precision": USE_AMP}) # Prepare dataset as a tf.train_data.Dataset instance - train_dataset = convert_examples_to_features(self._train_dataset, self._tokenizer, mode='sequence_classification') - valid_dataset = convert_examples_to_features(self._valid_dataset, self._tokenizer, mode='sequence_classification') - train_dataset = train_dataset.shuffle(128).batch(BATCH_SIZE).repeat(-1) - valid_dataset = valid_dataset.batch(EVAL_BATCH_SIZE) + self.logger.info('Tokenizing and processing dataset') + train_dataset = self.train_dataset.get_features(self.tokenizer) + valid_dataset = self.valid_dataset.get_features(self.tokenizer) + train_dataset = train_dataset.shuffle(128).batch(self.train_batch_size).repeat(-1) + valid_dataset = valid_dataset.batch(self.valid_batch_size) # Prepare training: Compile tf.keras model with optimizer, loss and learning rate schedule - opt = tf.keras.optimizers.Adam(learning_rate=3e-5, epsilon=1e-08) + opt = tf.keras.optimizers.Adam(learning_rate=args.learning_rate, epsilon=self.adam_epsilon) if USE_AMP: # loss scaling is currently required when using mixed precision opt = tf.keras.mixed_precision.experimental.LossScaleOptimizer(opt, 'dynamic') loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) metric = tf.keras.metrics.SparseCategoricalAccuracy('accuracy') - model.compile(optimizer=opt, loss=loss, metrics=[metric]) + self.model.compile(optimizer=opt, loss=loss, metrics=[metric]) # Train and evaluate using tf.keras.Model.fit() - train_steps = train_examples//BATCH_SIZE - valid_steps = valid_examples//EVAL_BATCH_SIZE + train_steps = self.num_train_samples//self.train_batch_size + valid_steps = self.num_valid_samples//self.valid_batch_size - history = model.fit(train_dataset, epochs=2, steps_per_epoch=train_steps, - validation_data=valid_dataset, validation_steps=valid_steps) + self.logger.info('Training model') + history = self.model.fit(train_dataset, epochs=2, steps_per_epoch=train_steps, + validation_data=valid_dataset, validation_steps=valid_steps) - # Save TF2 model - os.makedirs('./save/', exist_ok=True) - model.save_pretrained('./save/') + # Save trained model + self.model.save_pretrained(self.output) diff --git a/transformers/data/processors/utils.py b/transformers/data/processors/utils.py index 39544a1239..75bed86042 100644 --- a/transformers/data/processors/utils.py +++ b/transformers/data/processors/utils.py @@ -18,6 +18,11 @@ import csv import sys import copy import json +import logging + +from ...file_utils import is_tf_available, is_torch_available + +logger = logging.getLogger(__name__) class InputExample(object): """ @@ -64,7 +69,7 @@ class InputFeatures(object): label: Label corresponding to the input """ - def __init__(self, input_ids, attention_mask, token_type_ids, label): + def __init__(self, input_ids, attention_mask=None, token_type_ids=None, label=None): self.input_ids = input_ids self.attention_mask = attention_mask self.token_type_ids = token_type_ids @@ -86,34 +91,6 @@ class InputFeatures(object): class DataProcessor(object): """Base class for data converters for sequence classification data sets.""" - def get_example_from_tensor_dict(self, tensor_dict): - """Gets an example from a dict with tensorflow tensors - - Args: - tensor_dict: Keys and values should match the corresponding Glue - tensorflow_dataset examples. - """ - raise NotImplementedError() - - def get_train_examples(self, data_dir): - """Gets a collection of `InputExample`s for the train set.""" - raise NotImplementedError() - - def get_dev_examples(self, data_dir): - """Gets a collection of `InputExample`s for the dev set.""" - raise NotImplementedError() - - def get_labels(self): - """Gets the list of labels for this data set.""" - raise NotImplementedError() - - def tfds_map(self, example): - """Some tensorflow_datasets datasets are not formatted the same way the GLUE datasets are. - This method converts examples to the correct format.""" - if len(self.get_labels()) > 1: - example.label = self.get_labels()[int(example.label)] - return example - @classmethod def _read_tsv(cls, input_file, quotechar=None): """Reads a tab separated value file.""" @@ -129,15 +106,11 @@ class DataProcessor(object): class SingleSentenceClassificationProcessor(DataProcessor): """ Generic processor for a single sentence classification data set.""" - def __init__(self, labels=None, examples=None): + def __init__(self, labels=None, examples=None, mode='classification', verbose=False): self.labels = [] if labels is None else labels self.examples = [] if examples is None else examples - - @classmethod - def create_from_csv(cls, file_name): - processor = cls() - processor.add_examples_from_csv(file_name) - return processor + self.mode = mode + self.verbose = verbose def __len__(self): return len(self.examples) @@ -148,30 +121,40 @@ class SingleSentenceClassificationProcessor(DataProcessor): examples=self.examples[idx]) return self.examples[idx] - def get_labels(self): - """Gets the list of labels for this data set.""" - return self.labels + @classmethod + def create_from_csv(cls, file_name, **kwargs): + processor = cls(**kwargs) + processor.add_examples_from_csv(file_name) + return processor - def add_examples_from_csv(self, file_name): + def add_examples_from_csv(self, file_name, split_name='', column_label=0, column_text=1, column_id=None, + overwrite_labels=False, overwrite_examples=False): lines = self._read_tsv(file_name) - self.add_examples_from_lines(lines) - - def add_examples_from_lines(self, lines, split_name='', overwrite_labels=False, overwrite_examples=False): - """Creates examples for the training and dev sets.""" - added_labels = set() - examples = [] + texts = [] + labels = [] + ids = [] for (i, line) in enumerate(lines): - if len(line) > 2: - guid = "%s-%s" % (split_name, line[0]) if split_name else line[0] - label = line[1] - text_a = line[2] + texts.append(line[column_text]) + labels.append(line[column_label]) + if column_id is not None: + ids.append(line[column_id]) else: guid = "%s-%s" % (split_name, i) if split_name else "%s" % i - label = line[0] - text_a = line[1] + ids.append(guid) + return self.add_examples(texts, labels, ids, overwrite_labels=overwrite_labels, overwrite_examples=overwrite_examples) + + def add_examples(self, texts, labels, ids=None, overwrite_labels=False, overwrite_examples=False): + if ids is None: + ids = [None] * len(texts) + assert len(texts) == len(labels) + assert len(texts) == len(ids) + + examples = [] + added_labels = set() + for (text, label, guid) in zip(texts, labels, ids): added_labels.add(label) - examples.append(InputExample(guid=guid, text_a=text_a, text_b=None, label=label)) + examples.append(InputExample(guid=guid, text_a=text, text_b=None, label=label)) # Update examples if overwrite_examples: @@ -187,123 +170,129 @@ class SingleSentenceClassificationProcessor(DataProcessor): return self.examples + @classmethod + def create_from_examples(cls, texts, labels, **kwargs): + processor = cls(**kwargs) + processor.add_examples(texts, labels) + return processor -def convert_examples_to_features(examples, tokenizer, - mode='sequence_classification', - max_length=512, - pad_on_left=False, - pad_token=0, - pad_token_segment_id=0, - mask_padding_with_zero=True): - """ - Loads a data file into a list of ``InputFeatures`` + def get_features(self, + tokenizer, + max_length=None, + pad_on_left=False, + pad_token=0, + mask_padding_with_zero=True, + return_tensors=None): + """ + Convert examples in a list of ``InputFeatures`` - Args: - examples: List of ``InputExamples`` or ``tf.data.Dataset`` containing the examples. - tokenizer: Instance of a tokenizer that will tokenize the examples - max_length: Maximum example length - task: GLUE task - label_list: List of labels. Can be obtained from the processor using the ``processor.get_labels()`` method - output_mode: String indicating the output mode. Either ``regression`` or ``classification`` - pad_on_left: If set to ``True``, the examples will be padded on the left rather than on the right (default) - pad_token: Padding token - pad_token_segment_id: The segment ID for the padding token (It is usually 0, but can vary such as for XLNet where it is 4) - mask_padding_with_zero: If set to ``True``, the attention mask will be filled by ``1`` for actual values - and by ``0`` for padded values. If set to ``False``, inverts it (``1`` for padded values, ``0`` for - actual values) + Args: + tokenizer: Instance of a tokenizer that will tokenize the examples + max_length: Maximum example length + task: GLUE task + label_list: List of labels. Can be obtained from the processor using the ``processor.get_labels()`` method + output_mode: String indicating the output mode. Either ``regression`` or ``classification`` + pad_on_left: If set to ``True``, the examples will be padded on the left rather than on the right (default) + pad_token: Padding token + mask_padding_with_zero: If set to ``True``, the attention mask will be filled by ``1`` for actual values + and by ``0`` for padded values. If set to ``False``, inverts it (``1`` for padded values, ``0`` for + actual values) - Returns: - If the ``examples`` input is a ``tf.data.Dataset``, will return a ``tf.data.Dataset`` - containing the task-specific features. If the input is a list of ``InputExamples``, will return - a list of task-specific ``InputFeatures`` which can be fed to the model. + Returns: + If the ``examples`` input is a ``tf.data.Dataset``, will return a ``tf.data.Dataset`` + containing the task-specific features. If the input is a list of ``InputExamples``, will return + a list of task-specific ``InputFeatures`` which can be fed to the model. - """ - is_tf_dataset = False - if is_tf_available() and isinstance(examples, tf.data.Dataset): - is_tf_dataset = True + """ - if task is not None: - processor = glue_processors[task]() - if label_list is None: - label_list = processor.get_labels() - logger.info("Using label list %s for task %s" % (label_list, task)) - if output_mode is None: - output_mode = glue_output_modes[task] - logger.info("Using output mode %s for task %s" % (output_mode, task)) + label_map = {label: i for i, label in enumerate(self.labels)} - label_map = {label: i for i, label in enumerate(label_list)} + all_input_ids = [] + for (ex_index, example) in enumerate(self.examples): + if ex_index % 10000 == 0: + logger.info("Tokenizing example %d", ex_index) - features = [] - for (ex_index, example) in enumerate(examples): - if ex_index % 10000 == 0: - logger.info("Writing example %d" % (ex_index)) - if is_tf_dataset: - example = processor.get_example_from_tensor_dict(example) + input_ids = tokenizer.encode( + example.text_a, + add_special_tokens=True, + max_length=min(max_length, tokenizer.max_len), + ) + all_input_ids.append(input_ids) - inputs = tokenizer.encode_plus( - example.text_a, - example.text_b, - add_special_tokens=True, - max_length=max_length, - ) - input_ids, token_type_ids = inputs["input_ids"], inputs["token_type_ids"] + batch_length = max(len(input_ids) for input_ids in all_input_ids) - # The mask has 1 for real tokens and 0 for padding tokens. Only real - # tokens are attended to. - attention_mask = [1 if mask_padding_with_zero else 0] * len(input_ids) + features = [] + for (ex_index, (input_ids, example)) in enumerate(zip(all_input_ids, examples)): + if ex_index % 10000 == 0: + logger.info("Writing example %d", ex_index) + # The mask has 1 for real tokens and 0 for padding tokens. Only real + # tokens are attended to. + attention_mask = [1 if mask_padding_with_zero else 0] * len(input_ids) - # Zero-pad up to the sequence length. - padding_length = max_length - len(input_ids) - if pad_on_left: - input_ids = ([pad_token] * padding_length) + input_ids - attention_mask = ([0 if mask_padding_with_zero else 1] * padding_length) + attention_mask - token_type_ids = ([pad_token_segment_id] * padding_length) + token_type_ids + # Zero-pad up to the sequence length. + padding_length = batch_length - len(input_ids) + if pad_on_left: + input_ids = ([pad_token] * padding_length) + input_ids + attention_mask = ([0 if mask_padding_with_zero else 1] * padding_length) + attention_mask + else: + input_ids = input_ids + ([pad_token] * padding_length) + attention_mask = attention_mask + ([0 if mask_padding_with_zero else 1] * padding_length) + + assert len(input_ids) == batch_length, "Error with input length {} vs {}".format(len(input_ids), batch_length) + assert len(attention_mask) == batch_length, "Error with input length {} vs {}".format(len(attention_mask), batch_length) + + if self.mode == "classification": + label = label_map[example.label] + elif self.mode == "regression": + label = float(example.label) + else: + raise ValueError(self.mode) + + if ex_index < 5 and self.verbose: + logger.info("*** Example ***") + logger.info("guid: %s" % (example.guid)) + logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids])) + logger.info("attention_mask: %s" % " ".join([str(x) for x in attention_mask])) + logger.info("label: %s (id = %d)" % (example.label, label)) + + features.append( + InputFeatures(input_ids=input_ids, + attention_mask=attention_mask, + label=label)) + + if return_tensors is None: + return features + elif return_tensors == 'tf': + if not is_tf_available(): + raise ImportError("return_tensors set to 'tf' but TensorFlow 2.0 can't be imported") + import tensorflow as tf + def gen(): + for ex in features: + yield ({'input_ids': ex.input_ids, + 'attention_mask': ex.attention_mask}, + ex.label) + + dataset = tf.data.Dataset.from_generator(gen, + ({'input_ids': tf.int32, + 'attention_mask': tf.int32}, + tf.int64), + ({'input_ids': tf.TensorShape([None]), + 'attention_mask': tf.TensorShape([None])}, + tf.TensorShape([]))) + return dataset + elif return_tensors == 'pt': + if not is_torch_available(): + raise ImportError("return_tensors set to 'pt' but PyTorch can't be imported") + import torch + from torch.utils.data import TensorDataset + all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long) + all_attention_mask = torch.tensor([f.attention_mask for f in features], dtype=torch.long) + if self.mode == "classification": + all_labels = torch.tensor([f.label for f in features], dtype=torch.long) + elif self.mode == "regression": + all_labels = torch.tensor([f.label for f in features], dtype=torch.float) + + dataset = TensorDataset(all_input_ids, all_attention_mask, all_labels) + return dataset else: - input_ids = input_ids + ([pad_token] * padding_length) - attention_mask = attention_mask + ([0 if mask_padding_with_zero else 1] * padding_length) - token_type_ids = token_type_ids + ([pad_token_segment_id] * padding_length) - - assert len(input_ids) == max_length, "Error with input length {} vs {}".format(len(input_ids), max_length) - assert len(attention_mask) == max_length, "Error with input length {} vs {}".format(len(attention_mask), max_length) - assert len(token_type_ids) == max_length, "Error with input length {} vs {}".format(len(token_type_ids), max_length) - - if output_mode == "classification": - label = label_map[example.label] - elif output_mode == "regression": - label = float(example.label) - else: - raise KeyError(output_mode) - - if ex_index < 5: - logger.info("*** Example ***") - logger.info("guid: %s" % (example.guid)) - logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids])) - logger.info("attention_mask: %s" % " ".join([str(x) for x in attention_mask])) - logger.info("token_type_ids: %s" % " ".join([str(x) for x in token_type_ids])) - logger.info("label: %s (id = %d)" % (example.label, label)) - - features.append( - InputFeatures(input_ids=input_ids, - attention_mask=attention_mask, - token_type_ids=token_type_ids, - label=label)) - - if is_tf_available() and is_tf_dataset: - def gen(): - for ex in features: - yield ({'input_ids': ex.input_ids, - 'attention_mask': ex.attention_mask, - 'token_type_ids': ex.token_type_ids}, - ex.label) - - return tf.data.Dataset.from_generator(gen, - ({'input_ids': tf.int32, - 'attention_mask': tf.int32, - 'token_type_ids': tf.int32}, - tf.int64), - ({'input_ids': tf.TensorShape([None]), - 'attention_mask': tf.TensorShape([None]), - 'token_type_ids': tf.TensorShape([None])}, - tf.TensorShape([]))) - - return features + raise ValueError("return_tensors should be one of 'tf' or 'pt'") diff --git a/transformers/pipeline.py b/transformers/pipeline.py new file mode 100644 index 0000000000..15adc620b1 --- /dev/null +++ b/transformers/pipeline.py @@ -0,0 +1,254 @@ +# coding=utf-8 +# Copyright 2018 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Pipeline class: Tokenizer + Model. """ + +from __future__ import absolute_import, division, print_function, unicode_literals +import os +import logging + +from .modeling_auto import (AutoModel, AutoModelForQuestionAnswering, + AutoModelForSequenceClassification, + AutoModelWithLMHead) +from .tokenization_auto import AutoTokenizer +from .file_utils import add_start_docstrings, is_tf_available, is_torch_available +from .data.processors import SingleSentenceClassificationProcessor + +if is_tf_available(): + import tensorflow as tf +if is_torch_available(): + import torch + +logger = logging.getLogger(__name__) + +# TF training parameters +USE_XLA = False +USE_AMP = False + +class TextClassificationPipeline(object): + r""" + :class:`~transformers.TextClassificationPipeline` is a class encapsulating a pretrained model and + its tokenizer and will be instantiated as one of the base model classes of the library + when created with the `Pipeline.from_pretrained(pretrained_model_name_or_path)` + class method. + + The `from_pretrained()` method takes care of returning the correct model class instance + using pattern matching on the `pretrained_model_name_or_path` string. + + The base model class to instantiate is selected as the first pattern matching + in the `pretrained_model_name_or_path` string (in the following order): + - contains `distilbert`: DistilBertModel (DistilBERT model) + - contains `roberta`: RobertaModel (RoBERTa model) + - contains `bert`: BertModel (Bert model) + - contains `openai-gpt`: OpenAIGPTModel (OpenAI GPT model) + - contains `gpt2`: GPT2Model (OpenAI GPT-2 model) + - contains `ctrl`: CTRLModel (Salesforce CTRL model) + - contains `transfo-xl`: TransfoXLModel (Transformer-XL model) + - contains `xlnet`: XLNetModel (XLNet model) + - contains `xlm`: XLMModel (XLM model) + """ + def __init__(self, tokenizer, model): + self.tokenizer = tokenizer + self.model = model + if is_tf_available(): + self.framework = 'tf' + elif is_torch_available(): + self.framework = 'pt' + else: + raise ImportError("At least one of PyTorch or TensorFlow 2.0+ should be installed to use CLI training") + self.is_compiled = False + + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): + r""" Instantiates one of the base model classes of the library + from a pre-trained model configuration. + + The model class to instantiate is selected as the first pattern matching + in the `pretrained_model_name_or_path` string (in the following order): + - contains `distilbert`: DistilBertModel (DistilBERT model) + - contains `roberta`: RobertaModel (RoBERTa model) + - contains `bert`: BertModel (Bert model) + - contains `openai-gpt`: OpenAIGPTModel (OpenAI GPT model) + - contains `gpt2`: GPT2Model (OpenAI GPT-2 model) + - contains `ctrl`: CTRLModel (Salesforce CTRL model) + - contains `transfo-xl`: TransfoXLModel (Transformer-XL model) + - contains `xlnet`: XLNetModel (XLNet model) + - contains `xlm`: XLMModel (XLM model) + + The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated) + To train the model, you should first set it back in training mode with `model.train()` + + Params: + pretrained_model_name_or_path: either: + + - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. + - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. + + config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: + Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when: + + - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or + - the model was saved using :func:`~transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. + - the model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a configuration JSON file named `config.json` is found in the directory. + + state_dict: (`optional`) dict: + an optional state dictionnary for the model to use instead of a state dictionary loaded from saved weights file. + This option can be used if you want to create a model from a pretrained configuration but load your own weights. + In this case though, you should check if using :func:`~transformers.PreTrainedModel.save_pretrained` and :func:`~transformers.PreTrainedModel.from_pretrained` is not a simpler option. + + cache_dir: (`optional`) string: + Path to a directory in which a downloaded pre-trained model + configuration should be cached if the standard cache should not be used. + + force_download: (`optional`) boolean, default False: + Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + + proxies: (`optional`) dict, default None: + A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. + The proxies are used on each request. + + output_loading_info: (`optional`) boolean: + Set to ``True`` to also return a dictionnary containing missing keys, unexpected keys and error messages. + + kwargs: (`optional`) Remaining dictionary of keyword arguments: + Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded: + + - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done) + - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. + + Examples:: + + model = AutoModel.from_pretrained('bert-base-uncased') # Download model and configuration from S3 and cache. + model = AutoModel.from_pretrained('./test/bert_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')` + model = AutoModel.from_pretrained('bert-base-uncased', output_attention=True) # Update configuration during loading + assert model.config.output_attention == True + # Loading from a TF checkpoint file instead of a PyTorch model (slower) + config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json') + model = AutoModel.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config) + + """ + # Extract tokenizer and model arguments + tokenizer_kwargs = {} + for key in kwargs: + if key.startswith('tokenizer_'): + # Specific to the tokenizer + tokenizer_kwargs[key.replace('tokenizer_', '')] = kwargs.pop(key) + elif not key.startswith('model_'): + # used for both the tokenizer and the model + tokenizer_kwargs[key] = kwargs[key] + + model_kwargs = {} + for key in kwargs: + if key.startswith('model_'): + # Specific to the model + model_kwargs[key.replace('model_', '')] = kwargs.pop(key) + elif not key.startswith('tokenizer_'): + # used for both the tokenizer and the model + model_kwargs[key] = kwargs[key] + + tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path, **tokenizer_kwargs) + model = AutoModelForSequenceClassification.from_pretrained(pretrained_model_name_or_path, **model_kwargs) + return cls(tokenizer, model) + + + def save_pretrained(self, save_directory): + if not os.path.isdir(save_directory): + logger.error("Saving directory ({}) should be a directory".format(save_directory)) + return + self.model.save_pretrained(save_directory) + self.tokenizer.save_pretrained(save_directory) + + + def compile(self, learning_rate=3e-5, epsilon=1e-8): + if self.framework == 'tf': + logger.info('Preparing model') + # Prepare training: Compile tf.keras model with optimizer, loss and learning rate schedule + opt = tf.keras.optimizers.Adam(learning_rate=learning_rate, epsilon=epsilon) + if USE_AMP: + # loss scaling is currently required when using mixed precision + opt = tf.keras.mixed_precision.experimental.LossScaleOptimizer(opt, 'dynamic') + loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) + metric = tf.keras.metrics.SparseCategoricalAccuracy('accuracy') + self.model.compile(optimizer=opt, loss=loss, metrics=[metric]) + else: + raise NotImplementedError + self.is_compiled = True + + + def prepare_data(self, train_samples_text, train_samples_labels, + valid_samples_text=None, valid_samples_labels=None, + validation_split=0.1): + dataset = SingleSentenceClassificationProcessor.create_from_examples(train_samples_text, + train_samples_labels) + num_data_samples = len(dataset) + if valid_samples_text is not None and valid_samples_labels is not None: + valid_dataset = SingleSentenceClassificationProcessor.create_from_examples(valid_samples_text, + valid_samples_labels) + num_valid_samples = len(valid_dataset) + train_dataset = dataset + num_train_samples = num_data_samples + else: + assert 0.0 < validation_split < 1.0, "validation_split should be between 0.0 and 1.0" + num_valid_samples = int(num_data_samples * validation_split) + num_train_samples = num_data_samples - num_valid_samples + train_dataset = dataset[num_train_samples] + valid_dataset = dataset[num_valid_samples] + + logger.info('Tokenizing and processing dataset') + train_dataset = train_dataset.get_features(self.tokenizer, return_tensors=self.framework) + valid_dataset = valid_dataset.get_features(self.tokenizer, return_tensors=self.framework) + return train_dataset, valid_dataset, num_train_samples, num_valid_samples + + + def fit(self, train_samples_text, train_samples_labels, + valid_samples_text=None, valid_samples_labels=None, + train_batch_size=None, valid_batch_size=None, + validation_split=0.1, + **kwargs): + + if not self.is_compiled: + self.compile() + + datasets = self.prepare_data(train_samples_text, train_samples_labels, + valid_samples_text, valid_samples_labels, + validation_split) + train_dataset, valid_dataset, num_train_samples, num_valid_samples = datasets + + train_steps = num_train_samples//train_batch_size + valid_steps = num_valid_samples//valid_batch_size + + if self.framework == 'tf': + # Prepare dataset as a tf.train_data.Dataset instance + train_dataset = train_dataset.shuffle(128).batch(train_batch_size).repeat(-1) + valid_dataset = valid_dataset.batch(valid_batch_size) + + logger.info('Training TF 2.0 model') + history = self.model.fit(train_dataset, epochs=2, steps_per_epoch=train_steps, + validation_data=valid_dataset, validation_steps=valid_steps, **kwargs) + else: + raise NotImplementedError + + + def __call__(self, text): + inputs = self.tokenizer.encode_plus(text, add_special_tokens=True, return_tensors=self.framework) + if self.framework == 'tf': + # TODO trace model + predictions = self.model(**inputs)[0] + else: + with torch.no_grad(): + predictions = self.model(**inputs)[0] + + return predictions.numpy().tolist() From b81ab431f26e4a2fadf37bdd803ec26c66ee719c Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 17 Oct 2019 12:06:27 +0200 Subject: [PATCH 287/505] updating AutoModels and AutoConfiguration - adding pipelines --- transformers/configuration_auto.py | 28 +++++ transformers/modeling_auto.py | 182 +++++++++++++++++++++++++++-- transformers/modeling_tf_auto.py | 169 +++++++++++++++++++++++++-- transformers/pipeline.py | 83 ++----------- 4 files changed, 372 insertions(+), 90 deletions(-) diff --git a/transformers/configuration_auto.py b/transformers/configuration_auto.py index 43f251bd0c..47379d2f5a 100644 --- a/transformers/configuration_auto.py +++ b/transformers/configuration_auto.py @@ -61,6 +61,34 @@ class AutoConfig(object): raise EnvironmentError("AutoConfig is designed to be instantiated " "using the `AutoConfig.from_pretrained(pretrained_model_name_or_path)` method.") + @classmethod + def for_model(cls, model_type, *args, **kwargs): + if 'distilbert' in model_type: + return DistilBertConfig(*args, **kwargs) + elif 'roberta' in model_type: + return RobertaConfig(*args, **kwargs) + elif 'bert' in model_type: + return BertConfig(*args, **kwargs) + elif 'openai-gpt' in model_type: + return OpenAIGPTConfig(*args, **kwargs) + elif 'gpt2' in model_type: + return GPT2Config(*args, **kwargs) + elif 'transfo-xl' in model_type: + return TransfoXLConfig(*args, **kwargs) + elif 'xlnet' in model_type: + return XLNetConfig(*args, **kwargs) + elif 'xlm' in model_type: + return XLMConfig(*args, **kwargs) + elif 'ctrl' in model_type: + return CTRLConfig(*args, **kwargs) + elif 'albert' in model_type: + return AlbertConfig(*args, **kwargs) + elif 'camembert' in model_type: + return CamembertConfig(*args, **kwargs) + raise ValueError("Unrecognized model identifier in {}. Should contains one of " + "'distilbert', 'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', " + "'xlm', 'roberta', 'ctrl', 'camembert', 'albert'".format(model_type)) + @classmethod def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): r""" Instantiate a one of the configuration classes of the library diff --git a/transformers/modeling_auto.py b/transformers/modeling_auto.py index b63e43d73b..0c8bffa883 100644 --- a/transformers/modeling_auto.py +++ b/transformers/modeling_auto.py @@ -18,6 +18,10 @@ from __future__ import absolute_import, division, print_function, unicode_litera import logging +from .configuration_auto import (AlbertConfig, BertConfig, CamembertConfig, CTRLConfig, + DistilBertConfig, GPT2Config, OpenAIGPTConfig, RobertaConfig, + TransfoXLConfig, XLMConfig, XLNetConfig) + from .modeling_bert import BertModel, BertForMaskedLM, BertForSequenceClassification, BertForQuestionAnswering from .modeling_openai import OpenAIGPTModel, OpenAIGPTLMHeadModel from .modeling_gpt2 import GPT2Model, GPT2LMHeadModel @@ -27,8 +31,7 @@ from .modeling_xlnet import XLNetModel, XLNetLMHeadModel, XLNetForSequenceClassi from .modeling_xlm import XLMModel, XLMWithLMHeadModel, XLMForSequenceClassification, XLMForQuestionAnswering from .modeling_roberta import RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification from .modeling_distilbert import DistilBertModel, DistilBertForQuestionAnswering, DistilBertForMaskedLM, DistilBertForSequenceClassification -from .modeling_camembert import CamembertModel, CamembertForMaskedLM, CamembertForSequenceClassification, CamembertForMultipleChoice -from .modeling_camembert import CamembertModel, CamembertForMaskedLM, CamembertForSequenceClassification, CamembertForMultipleChoice +from .modeling_camembert import CamembertModel, CamembertForQuestionAnswering, CamembertForMaskedLM, CamembertForSequenceClassification, CamembertForMultipleChoice from .modeling_albert import AlbertModel, AlbertForMaskedLM, AlbertForSequenceClassification, AlbertForQuestionAnswering from .modeling_utils import PreTrainedModel, SequenceSummary @@ -43,7 +46,7 @@ class AutoModel(object): :class:`~transformers.AutoModel` is a generic model class that will be instantiated as one of the base model classes of the library when created with the `AutoModel.from_pretrained(pretrained_model_name_or_path)` - class method. + or the `AutoModel.from_config(config)` class methods. The `from_pretrained()` method takes care of returning the correct model class instance using pattern matching on the `pretrained_model_name_or_path` string. @@ -66,7 +69,54 @@ class AutoModel(object): """ def __init__(self): raise EnvironmentError("AutoModel is designed to be instantiated " - "using the `AutoModel.from_pretrained(pretrained_model_name_or_path)` method.") + "using the `AutoModel.from_pretrained(pretrained_model_name_or_path)` or " + "`AutoModel.from_config(config)` methods.") + + @classmethod + def from_config(cls, config): + r""" Instantiates one of the base model classes of the library + from a configuration. + + config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: + The model class to instantiate is selected based on the configuration class: + - isInstance of `distilbert` configuration class: DistilBertModel (DistilBERT model) + - isInstance of `roberta` configuration class: RobertaModel (RoBERTa model) + - isInstance of `bert` configuration class: BertModel (Bert model) + - isInstance of `openai-gpt` configuration class: OpenAIGPTModel (OpenAI GPT model) + - isInstance of `gpt2` configuration class: GPT2Model (OpenAI GPT-2 model) + - isInstance of `ctrl` configuration class: CTRLModel (Salesforce CTRL model) + - isInstance of `transfo-xl` configuration class: TransfoXLModel (Transformer-XL model) + - isInstance of `xlnet` configuration class: XLNetModel (XLNet model) + - isInstance of `xlm` configuration class: XLMModel (XLM model) + + Examples:: + + config = BertConfig.from_pretrained('bert-base-uncased') # Download configuration from S3 and cache. + model = AutoModel.from_config(config) # E.g. model was saved using `save_pretrained('./test/saved_model/')` + """ + if isinstance(config, DistilBertConfig): + return DistilBertModel(config) + elif isinstance(config, RobertaConfig): + return RobertaModel(config) + elif isinstance(config, BertConfig): + return BertModel(config) + elif isinstance(config, OpenAIGPTConfig): + return OpenAIGPTModel(config) + elif isinstance(config, GPT2Config): + return GPT2Model(config) + elif isinstance(config, TransfoXLConfig): + return TransfoXLModel(config) + elif isinstance(config, XLNetConfig): + return XLNetModel(config) + elif isinstance(config, XLMConfig): + return XLMModel(config) + elif isinstance(config, CTRLConfig): + return CTRLModel(config) + elif isinstance(config, AlbertConfig): + return AlbertModel(config) + elif isinstance(config, CamembertConfig): + return CamembertModel(config) + raise ValueError("Unrecognized configuration class {}".format(config)) @classmethod def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): @@ -201,7 +251,54 @@ class AutoModelWithLMHead(object): """ def __init__(self): raise EnvironmentError("AutoModelWithLMHead is designed to be instantiated " - "using the `AutoModelWithLMHead.from_pretrained(pretrained_model_name_or_path)` method.") + "using the `AutoModelWithLMHead.from_pretrained(pretrained_model_name_or_path)` or " + "`AutoModelWithLMHead.from_config(config)` methods.") + + @classmethod + def from_config(cls, config): + r""" Instantiates one of the base model classes of the library + from a configuration. + + config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: + The model class to instantiate is selected based on the configuration class: + - isInstance of `distilbert` configuration class: DistilBertModel (DistilBERT model) + - isInstance of `roberta` configuration class: RobertaModel (RoBERTa model) + - isInstance of `bert` configuration class: BertModel (Bert model) + - isInstance of `openai-gpt` configuration class: OpenAIGPTModel (OpenAI GPT model) + - isInstance of `gpt2` configuration class: GPT2Model (OpenAI GPT-2 model) + - isInstance of `ctrl` configuration class: CTRLModel (Salesforce CTRL model) + - isInstance of `transfo-xl` configuration class: TransfoXLModel (Transformer-XL model) + - isInstance of `xlnet` configuration class: XLNetModel (XLNet model) + - isInstance of `xlm` configuration class: XLMModel (XLM model) + + Examples:: + + config = BertConfig.from_pretrained('bert-base-uncased') # Download configuration from S3 and cache. + model = AutoModelWithLMHead.from_config(config) # E.g. model was saved using `save_pretrained('./test/saved_model/')` + """ + if isinstance(config, DistilBertConfig): + return DistilBertForMaskedLM(config) + elif isinstance(config, RobertaConfig): + return RobertaForMaskedLM(config) + elif isinstance(config, BertConfig): + return BertForMaskedLM(config) + elif isinstance(config, OpenAIGPTConfig): + return OpenAIGPTLMHeadModel(config) + elif isinstance(config, GPT2Config): + return GPT2LMHeadModel(config) + elif isinstance(config, TransfoXLConfig): + return TransfoXLLMHeadModel(config) + elif isinstance(config, XLNetConfig): + return XLNetLMHeadModel(config) + elif isinstance(config, XLMConfig): + return XLMWithLMHeadModel(config) + elif isinstance(config, CTRLConfig): + return CTRLLMHeadModel(config) + elif isinstance(config, AlbertConfig): + return AlbertLMHeadModel(config) + elif isinstance(config, CamembertConfig): + return CamembertLMHeadModel(config) + raise ValueError("Unrecognized configuration class {}".format(config)) @classmethod def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): @@ -333,8 +430,43 @@ class AutoModelForSequenceClassification(object): This class cannot be instantiated using `__init__()` (throws an error). """ def __init__(self): - raise EnvironmentError("AutoModelWithLMHead is designed to be instantiated " - "using the `AutoModelWithLMHead.from_pretrained(pretrained_model_name_or_path)` method.") + raise EnvironmentError("AutoModelForSequenceClassification is designed to be instantiated " + "using the `AutoModelForSequenceClassification.from_pretrained(pretrained_model_name_or_path)` or " + "`AutoModelForSequenceClassification.from_config(config)` methods.") + + @classmethod + def from_config(cls, config): + r""" Instantiates one of the base model classes of the library + from a configuration. + + config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: + The model class to instantiate is selected based on the configuration class: + - isInstance of `distilbert` configuration class: DistilBertModel (DistilBERT model) + - isInstance of `roberta` configuration class: RobertaModel (RoBERTa model) + - isInstance of `bert` configuration class: BertModel (Bert model) + - isInstance of `xlnet` configuration class: XLNetModel (XLNet model) + - isInstance of `xlm` configuration class: XLMModel (XLM model) + + Examples:: + + config = BertConfig.from_pretrained('bert-base-uncased') # Download configuration from S3 and cache. + model = AutoModelForSequenceClassification.from_config(config) # E.g. model was saved using `save_pretrained('./test/saved_model/')` + """ + if isinstance(config, AlbertConfig): + return AlbertForSequenceClassification(config) + elif isintance(config, CamembertConfig): + return CamembertForSequenceClassification(config) + elif isinstance(config, DistilBertConfig): + return DistilBertForSequenceClassification(config) + elif isinstance(config, RobertaConfig): + return RobertaForSequenceClassification(config) + elif isinstance(config, BertConfig): + return BertForSequenceClassification(config) + elif isinstance(config, XLNetConfig): + return XLNetForSequenceClassification(config) + elif isinstance(config, XLMConfig): + return XLMForSequenceClassification(config) + raise ValueError("Unrecognized configuration class {}".format(config)) @classmethod def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): @@ -453,8 +585,40 @@ class AutoModelForQuestionAnswering(object): This class cannot be instantiated using `__init__()` (throws an error). """ def __init__(self): - raise EnvironmentError("AutoModelWithLMHead is designed to be instantiated " - "using the `AutoModelWithLMHead.from_pretrained(pretrained_model_name_or_path)` method.") + raise EnvironmentError("AutoModelForQuestionAnswering is designed to be instantiated " + "using the `AutoModelForQuestionAnswering.from_pretrained(pretrained_model_name_or_path)` or " + "`AutoModelForQuestionAnswering.from_config(config)` methods.") + + @classmethod + def from_config(cls, config): + r""" Instantiates one of the base model classes of the library + from a configuration. + + config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: + The model class to instantiate is selected based on the configuration class: + - isInstance of `distilbert` configuration class: DistilBertModel (DistilBERT model) + - isInstance of `bert` configuration class: BertModel (Bert model) + - isInstance of `xlnet` configuration class: XLNetModel (XLNet model) + - isInstance of `xlm` configuration class: XLMModel (XLM model) + + Examples:: + + config = BertConfig.from_pretrained('bert-base-uncased') # Download configuration from S3 and cache. + model = AutoModelForSequenceClassification.from_config(config) # E.g. model was saved using `save_pretrained('./test/saved_model/')` + """ + if isintance(config, AlbertConfig): + return AlbertForQuestionAnswering(config) + elif isintance(config, CamembertConfig): + return CamembertForQuestionAnswering(config) + elif isinstance(config, DistilBertConfig): + return DistilBertForQuestionAnswering(config) + elif isinstance(config, BertConfig): + return BertForQuestionAnswering(config) + elif isinstance(config, XLNetConfig): + return XLNetForQuestionAnswering(config) + elif isinstance(config, XLMConfig): + return XLMForQuestionAnswering(config) + raise ValueError("Unrecognized configuration class {}".format(config)) @classmethod def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): diff --git a/transformers/modeling_tf_auto.py b/transformers/modeling_tf_auto.py index cfe19ead2a..e78b91cfcc 100644 --- a/transformers/modeling_tf_auto.py +++ b/transformers/modeling_tf_auto.py @@ -18,6 +18,10 @@ from __future__ import absolute_import, division, print_function, unicode_litera import logging +from .configuration_auto import (BertConfig, CTRLConfig, DistilBertConfig, + GPT2Config, OpenAIGPTConfig, RobertaConfig, + TransfoXLConfig, XLMConfig, XLNetConfig) + from .modeling_tf_bert import TFBertModel, TFBertForMaskedLM, TFBertForSequenceClassification, TFBertForQuestionAnswering from .modeling_tf_openai import TFOpenAIGPTModel, TFOpenAIGPTLMHeadModel from .modeling_tf_gpt2 import TFGPT2Model, TFGPT2LMHeadModel @@ -59,7 +63,50 @@ class TFAutoModel(object): """ def __init__(self): raise EnvironmentError("TFAutoModel is designed to be instantiated " - "using the `TFAutoModel.from_pretrained(pretrained_model_name_or_path)` method.") + "using the `TFAutoModel.from_pretrained(pretrained_model_name_or_path)` or " + "`TFAutoModel.from_config(config)` methods.") + + @classmethod + def from_config(cls, config): + r""" Instantiates one of the base model classes of the library + from a configuration. + + config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: + The model class to instantiate is selected based on the configuration class: + - isInstance of `distilbert` configuration class: TFDistilBertModel (DistilBERT model) + - isInstance of `roberta` configuration class: TFRobertaModel (RoBERTa model) + - isInstance of `bert` configuration class: TFBertModel (Bert model) + - isInstance of `openai-gpt` configuration class: TFOpenAIGPTModel (OpenAI GPT model) + - isInstance of `gpt2` configuration class: TFGPT2Model (OpenAI GPT-2 model) + - isInstance of `ctrl` configuration class: TFCTRLModel (Salesforce CTRL model) + - isInstance of `transfo-xl` configuration class: TFTransfoXLModel (Transformer-XL model) + - isInstance of `xlnet` configuration class: TFXLNetModel (XLNet model) + - isInstance of `xlm` configuration class: TFXLMModel (XLM model) + + Examples:: + + config = BertConfig.from_pretrained('bert-base-uncased') # Download configuration from S3 and cache. + model = TFAutoModel.from_config(config) # E.g. model was saved using `save_pretrained('./test/saved_model/')` + """ + if isinstance(config, DistilBertConfig): + return TFDistilBertModel(config) + elif isinstance(config, RobertaConfig): + return TFRobertaModel(config) + elif isinstance(config, BertConfig): + return TFBertModel(config) + elif isinstance(config, OpenAIGPTConfig): + return TFOpenAIGPTModel(config) + elif isinstance(config, GPT2Config): + return TFGPT2Model(config) + elif isinstance(config, TransfoXLConfig): + return TFTransfoXLModel(config) + elif isinstance(config, XLNetConfig): + return TFXLNetModel(config) + elif isinstance(config, XLMConfig): + return TFXLMModel(config) + elif isinstance(config, CTRLConfig): + return TFCTRLModel(config) + raise ValueError("Unrecognized configuration class {}".format(config)) @classmethod def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): @@ -156,7 +203,7 @@ class TFAutoModel(object): return TFCTRLModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " - "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', " + "'distilbert', 'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', " "'xlm', 'roberta', 'ctrl'".format(pretrained_model_name_or_path)) @@ -186,7 +233,50 @@ class TFAutoModelWithLMHead(object): """ def __init__(self): raise EnvironmentError("TFAutoModelWithLMHead is designed to be instantiated " - "using the `TFAutoModelWithLMHead.from_pretrained(pretrained_model_name_or_path)` method.") + "using the `TFAutoModelWithLMHead.from_pretrained(pretrained_model_name_or_path)` or " + "`TFAutoModelWithLMHead.from_config(config)` methods.") + + @classmethod + def from_config(cls, config): + r""" Instantiates one of the base model classes of the library + from a configuration. + + config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: + The model class to instantiate is selected based on the configuration class: + - isInstance of `distilbert` configuration class: DistilBertModel (DistilBERT model) + - isInstance of `roberta` configuration class: RobertaModel (RoBERTa model) + - isInstance of `bert` configuration class: BertModel (Bert model) + - isInstance of `openai-gpt` configuration class: OpenAIGPTModel (OpenAI GPT model) + - isInstance of `gpt2` configuration class: GPT2Model (OpenAI GPT-2 model) + - isInstance of `ctrl` configuration class: CTRLModel (Salesforce CTRL model) + - isInstance of `transfo-xl` configuration class: TransfoXLModel (Transformer-XL model) + - isInstance of `xlnet` configuration class: XLNetModel (XLNet model) + - isInstance of `xlm` configuration class: XLMModel (XLM model) + + Examples:: + + config = BertConfig.from_pretrained('bert-base-uncased') # Download configuration from S3 and cache. + model = AutoModelWithLMHead.from_config(config) # E.g. model was saved using `save_pretrained('./test/saved_model/')` + """ + if isinstance(config, DistilBertConfig): + return TFDistilBertForMaskedLM(config) + elif isinstance(config, RobertaConfig): + return TFRobertaForMaskedLM(config) + elif isinstance(config, BertConfig): + return TFBertForMaskedLM(config) + elif isinstance(config, OpenAIGPTConfig): + return TFOpenAIGPTLMHeadModel(config) + elif isinstance(config, GPT2Config): + return TFGPT2LMHeadModel(config) + elif isinstance(config, TransfoXLConfig): + return TFTransfoXLLMHeadModel(config) + elif isinstance(config, XLNetConfig): + return TFXLNetLMHeadModel(config) + elif isinstance(config, XLMConfig): + return TFXLMWithLMHeadModel(config) + elif isinstance(config, CTRLConfig): + return TFCTRLLMHeadModel(config) + raise ValueError("Unrecognized configuration class {}".format(config)) @classmethod def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): @@ -287,7 +377,7 @@ class TFAutoModelWithLMHead(object): return TFCTRLLMHeadModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " - "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', " + "'distilbert', 'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', " "'xlm', 'roberta', 'ctrl'".format(pretrained_model_name_or_path)) @@ -312,8 +402,39 @@ class TFAutoModelForSequenceClassification(object): This class cannot be instantiated using `__init__()` (throws an error). """ def __init__(self): - raise EnvironmentError("TFAutoModelWithLMHead is designed to be instantiated " - "using the `TFAutoModelWithLMHead.from_pretrained(pretrained_model_name_or_path)` method.") + raise EnvironmentError("TFAutoModelForSequenceClassification is designed to be instantiated " + "using the `TFAutoModelForSequenceClassification.from_pretrained(pretrained_model_name_or_path)` or " + "`TFAutoModelForSequenceClassification.from_config(config)` methods.") + + @classmethod + def from_config(cls, config): + r""" Instantiates one of the base model classes of the library + from a configuration. + + config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: + The model class to instantiate is selected based on the configuration class: + - isInstance of `distilbert` configuration class: DistilBertModel (DistilBERT model) + - isInstance of `roberta` configuration class: RobertaModel (RoBERTa model) + - isInstance of `bert` configuration class: BertModel (Bert model) + - isInstance of `xlnet` configuration class: XLNetModel (XLNet model) + - isInstance of `xlm` configuration class: XLMModel (XLM model) + + Examples:: + + config = BertConfig.from_pretrained('bert-base-uncased') # Download configuration from S3 and cache. + model = AutoModelForSequenceClassification.from_config(config) # E.g. model was saved using `save_pretrained('./test/saved_model/')` + """ + if isinstance(config, DistilBertConfig): + return TFDistilBertForSequenceClassification(config) + elif isinstance(config, RobertaConfig): + return TFRobertaForSequenceClassification(config) + elif isinstance(config, BertConfig): + return TFBertForSequenceClassification(config) + elif isinstance(config, XLNetConfig): + return TFXLNetForSequenceClassification(config) + elif isinstance(config, XLMConfig): + return TFXLMForSequenceClassification(config) + raise ValueError("Unrecognized configuration class {}".format(config)) @classmethod def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): @@ -405,7 +526,7 @@ class TFAutoModelForSequenceClassification(object): return TFXLMForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " - "'bert', 'xlnet', 'xlm', 'roberta'".format(pretrained_model_name_or_path)) + "'distilbert', 'bert', 'xlnet', 'xlm', 'roberta'".format(pretrained_model_name_or_path)) class TFAutoModelForQuestionAnswering(object): @@ -428,8 +549,36 @@ class TFAutoModelForQuestionAnswering(object): This class cannot be instantiated using `__init__()` (throws an error). """ def __init__(self): - raise EnvironmentError("TFAutoModelWithLMHead is designed to be instantiated " - "using the `TFAutoModelWithLMHead.from_pretrained(pretrained_model_name_or_path)` method.") + raise EnvironmentError("TFAutoModelForQuestionAnswering is designed to be instantiated " + "using the `TFAutoModelForQuestionAnswering.from_pretrained(pretrained_model_name_or_path)` or " + "`TFAutoModelForQuestionAnswering.from_config(config)` methods.") + + @classmethod + def from_config(cls, config): + r""" Instantiates one of the base model classes of the library + from a configuration. + + config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: + The model class to instantiate is selected based on the configuration class: + - isInstance of `distilbert` configuration class: DistilBertModel (DistilBERT model) + - isInstance of `bert` configuration class: BertModel (Bert model) + - isInstance of `xlnet` configuration class: XLNetModel (XLNet model) + - isInstance of `xlm` configuration class: XLMModel (XLM model) + + Examples:: + + config = BertConfig.from_pretrained('bert-base-uncased') # Download configuration from S3 and cache. + model = AutoModelForSequenceClassification.from_config(config) # E.g. model was saved using `save_pretrained('./test/saved_model/')` + """ + if isinstance(config, DistilBertConfig): + return TFDistilBertForQuestionAnswering(config) + elif isinstance(config, BertConfig): + return TFBertForQuestionAnswering(config) + elif isinstance(config, XLNetConfig): + return TFXLNetForQuestionAnswering(config) + elif isinstance(config, XLMConfig): + return TFXLMForQuestionAnswering(config) + raise ValueError("Unrecognized configuration class {}".format(config)) @classmethod def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): @@ -518,4 +667,4 @@ class TFAutoModelForQuestionAnswering(object): return TFXLMForQuestionAnsweringSimple.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " - "'bert', 'xlnet', 'xlm'".format(pretrained_model_name_or_path)) + "'distilbert', 'bert', 'xlnet', 'xlm'".format(pretrained_model_name_or_path)) diff --git a/transformers/pipeline.py b/transformers/pipeline.py index 15adc620b1..f2c55def92 100644 --- a/transformers/pipeline.py +++ b/transformers/pipeline.py @@ -58,7 +58,7 @@ class TextClassificationPipeline(object): - contains `xlnet`: XLNetModel (XLNet model) - contains `xlm`: XLMModel (XLM model) """ - def __init__(self, tokenizer, model): + def __init__(self, tokenizer, model, is_compiled=False, is_trained=False): self.tokenizer = tokenizer self.model = model if is_tf_available(): @@ -67,78 +67,13 @@ class TextClassificationPipeline(object): self.framework = 'pt' else: raise ImportError("At least one of PyTorch or TensorFlow 2.0+ should be installed to use CLI training") - self.is_compiled = False + self.is_compiled = is_compiled + self.is_trained = is_trained @classmethod def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): - r""" Instantiates one of the base model classes of the library - from a pre-trained model configuration. - - The model class to instantiate is selected as the first pattern matching - in the `pretrained_model_name_or_path` string (in the following order): - - contains `distilbert`: DistilBertModel (DistilBERT model) - - contains `roberta`: RobertaModel (RoBERTa model) - - contains `bert`: BertModel (Bert model) - - contains `openai-gpt`: OpenAIGPTModel (OpenAI GPT model) - - contains `gpt2`: GPT2Model (OpenAI GPT-2 model) - - contains `ctrl`: CTRLModel (Salesforce CTRL model) - - contains `transfo-xl`: TransfoXLModel (Transformer-XL model) - - contains `xlnet`: XLNetModel (XLNet model) - - contains `xlm`: XLMModel (XLM model) - - The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated) - To train the model, you should first set it back in training mode with `model.train()` - - Params: - pretrained_model_name_or_path: either: - - - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. - - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. - - config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: - Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when: - - - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or - - the model was saved using :func:`~transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. - - the model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a configuration JSON file named `config.json` is found in the directory. - - state_dict: (`optional`) dict: - an optional state dictionnary for the model to use instead of a state dictionary loaded from saved weights file. - This option can be used if you want to create a model from a pretrained configuration but load your own weights. - In this case though, you should check if using :func:`~transformers.PreTrainedModel.save_pretrained` and :func:`~transformers.PreTrainedModel.from_pretrained` is not a simpler option. - - cache_dir: (`optional`) string: - Path to a directory in which a downloaded pre-trained model - configuration should be cached if the standard cache should not be used. - - force_download: (`optional`) boolean, default False: - Force to (re-)download the model weights and configuration files and override the cached versions if they exists. - - proxies: (`optional`) dict, default None: - A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. - The proxies are used on each request. - - output_loading_info: (`optional`) boolean: - Set to ``True`` to also return a dictionnary containing missing keys, unexpected keys and error messages. - - kwargs: (`optional`) Remaining dictionary of keyword arguments: - Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded: - - - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done) - - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. - - Examples:: - - model = AutoModel.from_pretrained('bert-base-uncased') # Download model and configuration from S3 and cache. - model = AutoModel.from_pretrained('./test/bert_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')` - model = AutoModel.from_pretrained('bert-base-uncased', output_attention=True) # Update configuration during loading - assert model.config.output_attention == True - # Loading from a TF checkpoint file instead of a PyTorch model (slower) - config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json') - model = AutoModel.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config) - + r""" Instantiates a pipeline from a pre-trained tokenizer and model. """ # Extract tokenizer and model arguments tokenizer_kwargs = {} @@ -159,9 +94,11 @@ class TextClassificationPipeline(object): # used for both the tokenizer and the model model_kwargs[key] = kwargs[key] + model_kwargs['output_loading_info'] = True tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path, **tokenizer_kwargs) - model = AutoModelForSequenceClassification.from_pretrained(pretrained_model_name_or_path, **model_kwargs) - return cls(tokenizer, model) + model, loading_info = AutoModelForSequenceClassification.from_pretrained(pretrained_model_name_or_path, **model_kwargs) + + return cls(tokenizer, model, is_trained=bool(not loading_info['missing_keys'])) def save_pretrained(self, save_directory): @@ -240,9 +177,13 @@ class TextClassificationPipeline(object): validation_data=valid_dataset, validation_steps=valid_steps, **kwargs) else: raise NotImplementedError + self.is_trained = True def __call__(self, text): + if not self.is_trained: + logger.error("Some weights of the model are not trained. Please fine-tune the model on a classification task before using it.") + inputs = self.tokenizer.encode_plus(text, add_special_tokens=True, return_tensors=self.framework) if self.framework == 'tf': # TODO trace model From 7c1697562a38200e0e1a651b014ff0fc07343dd1 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 17 Oct 2019 13:17:05 +0200 Subject: [PATCH 288/505] compatibility with sklearn and keras --- transformers/commands/train.py | 2 +- transformers/data/processors/utils.py | 2 +- transformers/pipeline.py | 87 ++++++++++++++++++--------- 3 files changed, 59 insertions(+), 32 deletions(-) diff --git a/transformers/commands/train.py b/transformers/commands/train.py index fc89d48594..878ad21037 100644 --- a/transformers/commands/train.py +++ b/transformers/commands/train.py @@ -83,7 +83,7 @@ class TrainCommand(BaseTransformersCLICommand): self.logger.info('Loading model {}'.format(args.model_name)) self.model_name = args.model_name - self.tokenizer = AutoTokenizer.from_pretrained(args.model_name) + self.pipeline = AutoTokenizer.from_pretrained(args.model_name) if args.task == 'text_classification': self.model = SequenceClassifModel.from_pretrained(args.model_name) elif args.task == 'token_classification': diff --git a/transformers/data/processors/utils.py b/transformers/data/processors/utils.py index 75bed86042..61b139c02b 100644 --- a/transformers/data/processors/utils.py +++ b/transformers/data/processors/utils.py @@ -222,7 +222,7 @@ class SingleSentenceClassificationProcessor(DataProcessor): batch_length = max(len(input_ids) for input_ids in all_input_ids) features = [] - for (ex_index, (input_ids, example)) in enumerate(zip(all_input_ids, examples)): + for (ex_index, (input_ids, example)) in enumerate(zip(all_input_ids, self.examples)): if ex_index % 10000 == 0: logger.info("Writing example %d", ex_index) # The mask has 1 for real tokens and 0 for padding tokens. Only real diff --git a/transformers/pipeline.py b/transformers/pipeline.py index f2c55def92..dc7bcaeac3 100644 --- a/transformers/pipeline.py +++ b/transformers/pipeline.py @@ -109,7 +109,32 @@ class TextClassificationPipeline(object): self.tokenizer.save_pretrained(save_directory) - def compile(self, learning_rate=3e-5, epsilon=1e-8): + def prepare_data(self, train_samples_text, train_samples_labels, + valid_samples_text=None, valid_samples_labels=None, + validation_split=0.1, **kwargs): + dataset = SingleSentenceClassificationProcessor.create_from_examples(train_samples_text, + train_samples_labels) + num_data_samples = len(dataset) + if valid_samples_text is not None and valid_samples_labels is not None: + valid_dataset = SingleSentenceClassificationProcessor.create_from_examples(valid_samples_text, + valid_samples_labels) + num_valid_samples = len(valid_dataset) + train_dataset = dataset + num_train_samples = num_data_samples + else: + assert 0.0 <= validation_split <= 1.0, "validation_split should be between 0.0 and 1.0" + num_valid_samples = int(num_data_samples * validation_split) + num_train_samples = num_data_samples - num_valid_samples + train_dataset = dataset[num_train_samples] + valid_dataset = dataset[num_valid_samples] + + logger.info('Tokenizing and processing dataset') + train_dataset = train_dataset.get_features(self.tokenizer, return_tensors=self.framework) + valid_dataset = valid_dataset.get_features(self.tokenizer, return_tensors=self.framework) + return train_dataset, valid_dataset, num_train_samples, num_valid_samples + + + def compile(self, learning_rate=3e-5, epsilon=1e-8, **kwargs): if self.framework == 'tf': logger.info('Preparing model') # Prepare training: Compile tf.keras model with optimizer, loss and learning rate schedule @@ -125,39 +150,20 @@ class TextClassificationPipeline(object): self.is_compiled = True - def prepare_data(self, train_samples_text, train_samples_labels, - valid_samples_text=None, valid_samples_labels=None, - validation_split=0.1): - dataset = SingleSentenceClassificationProcessor.create_from_examples(train_samples_text, - train_samples_labels) - num_data_samples = len(dataset) - if valid_samples_text is not None and valid_samples_labels is not None: - valid_dataset = SingleSentenceClassificationProcessor.create_from_examples(valid_samples_text, - valid_samples_labels) - num_valid_samples = len(valid_dataset) - train_dataset = dataset - num_train_samples = num_data_samples - else: - assert 0.0 < validation_split < 1.0, "validation_split should be between 0.0 and 1.0" - num_valid_samples = int(num_data_samples * validation_split) - num_train_samples = num_data_samples - num_valid_samples - train_dataset = dataset[num_train_samples] - valid_dataset = dataset[num_valid_samples] - - logger.info('Tokenizing and processing dataset') - train_dataset = train_dataset.get_features(self.tokenizer, return_tensors=self.framework) - valid_dataset = valid_dataset.get_features(self.tokenizer, return_tensors=self.framework) - return train_dataset, valid_dataset, num_train_samples, num_valid_samples - - - def fit(self, train_samples_text, train_samples_labels, + def fit(self, train_samples_text=None, train_samples_labels=None, valid_samples_text=None, valid_samples_labels=None, train_batch_size=None, valid_batch_size=None, validation_split=0.1, **kwargs): + # Generic compatibility with sklearn and Keras + if 'y' in kwargs and train_samples_labels is None: + train_samples_labels = kwargs.pop('y') + if 'X' in kwargs and train_samples_text is None: + train_samples_text = kwargs.pop('X') + if not self.is_compiled: - self.compile() + self.compile(**kwargs) datasets = self.prepare_data(train_samples_text, train_samples_labels, valid_samples_text, valid_samples_labels, @@ -180,11 +186,32 @@ class TextClassificationPipeline(object): self.is_trained = True - def __call__(self, text): + def fit_transform(self, *texts, **kwargs): + # Generic compatibility with sklearn and Keras + self.fit(*texts, **kwargs) + return self(*texts, **kwargs) + + + def transform(self, *texts, **kwargs): + # Generic compatibility with sklearn and Keras + return self(*texts, **kwargs) + + + def predict(self, *texts, **kwargs): + # Generic compatibility with sklearn and Keras + return self(*texts, **kwargs) + + + def __call__(self, *texts, **kwargs): + # Generic compatibility with sklearn and Keras + if 'X' in kwargs and not texts: + texts = kwargs.pop('X') + if not self.is_trained: logger.error("Some weights of the model are not trained. Please fine-tune the model on a classification task before using it.") - inputs = self.tokenizer.encode_plus(text, add_special_tokens=True, return_tensors=self.framework) + inputs = self.tokenizer.batch_encode_plus(texts, add_special_tokens=True, return_tensors=self.framework) + if self.framework == 'tf': # TODO trace model predictions = self.model(**inputs)[0] From 31a3a73ee39d7de28e69e62d8c1a0988a765a0e0 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 17 Oct 2019 15:10:11 +0200 Subject: [PATCH 289/505] updating CLI --- transformers-cli | 4 ++ transformers/__init__.py | 4 +- transformers/commands/train.py | 89 ++++++++++--------------- transformers/data/processors/utils.py | 48 +++++++++----- transformers/modeling_tf_utils.py | 31 +++++++++ transformers/pipeline.py | 95 ++++++++++++++------------- 6 files changed, 155 insertions(+), 116 deletions(-) diff --git a/transformers-cli b/transformers-cli index 7b0905d4b4..397b382308 100644 --- a/transformers-cli +++ b/transformers-cli @@ -3,6 +3,8 @@ from argparse import ArgumentParser from transformers.commands.serving import ServeCommand from transformers.commands.user import UserCommands +from transformers.commands.train import TrainCommand +from transformers.commands.convert import ConvertCommand if __name__ == '__main__': parser = ArgumentParser('Transformers CLI tool', usage='transformers-cli []') @@ -11,6 +13,8 @@ if __name__ == '__main__': # Register commands ServeCommand.register_subcommand(commands_parser) UserCommands.register_subcommand(commands_parser) + TrainCommand.register_subcommand(commands_parser) + ConvertCommand.register_subcommand(commands_parser) # Let's go args = parser.parse_args() diff --git a/transformers/__init__.py b/transformers/__init__.py index a71a291a44..26036d2e8d 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -25,7 +25,6 @@ from .file_utils import (TRANSFORMERS_CACHE, PYTORCH_TRANSFORMERS_CACHE, PYTORCH from .data import (is_sklearn_available, InputExample, InputFeatures, DataProcessor, SingleSentenceClassificationProcessor, - convert_examples_to_features, glue_output_modes, glue_convert_examples_to_features, glue_processors, glue_tasks_num_labels, xnli_output_modes, xnli_processors, xnli_tasks_num_labels, @@ -66,6 +65,9 @@ from .configuration_distilbert import DistilBertConfig, DISTILBERT_PRETRAINED_CO from .configuration_albert import AlbertConfig, ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_camembert import CamembertConfig, CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP +# Pipelines +from .pipeline import TextClassificationPipeline + # Modeling if is_torch_available(): from .modeling_utils import (PreTrainedModel, prune_layer, Conv1D) diff --git a/transformers/commands/train.py b/transformers/commands/train.py index 878ad21037..7b26745881 100644 --- a/transformers/commands/train.py +++ b/transformers/commands/train.py @@ -3,14 +3,11 @@ from argparse import ArgumentParser, Namespace from logging import getLogger from transformers.commands import BaseTransformersCLICommand -from transformers import (AutoTokenizer, is_tf_available, is_torch_available, - SingleSentenceClassificationProcessor, - convert_examples_to_features) -if is_tf_available(): - from transformers import TFAutoModelForSequenceClassification as SequenceClassifModel -elif is_torch_available(): - from transformers import AutoModelForSequenceClassification as SequenceClassifModel -else: +from transformers import (is_tf_available, is_torch_available, + TextClassificationPipeline, + SingleSentenceClassificationProcessor as Processor) + +if not is_tf_available() and not is_torch_available(): raise ImportError("At least one of PyTorch or TensorFlow 2.0+ should be installed to use CLI training") # TF training parameters @@ -35,16 +32,18 @@ class TrainCommand(BaseTransformersCLICommand): :return: """ train_parser = parser.add_parser('train', help='CLI tool to train a model on a task.') + train_parser.add_argument('--train_data', type=str, required=True, help="path to train (and optionally evaluation) dataset as a csv with " "tab separated labels and sentences.") - train_parser.add_argument('--column_label', type=int, default=0, help='Column of the dataset csv file with example labels.') train_parser.add_argument('--column_text', type=int, default=1, help='Column of the dataset csv file with example texts.') train_parser.add_argument('--column_id', type=int, default=2, help='Column of the dataset csv file with example ids.') + train_parser.add_argument('--skip_first_row', action='store_true', + help='Skip the first row of the csv file (headers).') train_parser.add_argument('--validation_data', type=str, default='', help='path to validation dataset.') @@ -74,39 +73,38 @@ class TrainCommand(BaseTransformersCLICommand): self.framework = 'tf' if is_tf_available() else 'torch' - os.makedirs(args.output) + os.makedirs(args.output, exist_ok=True) + assert os.path.isdir(args.output) self.output = args.output self.column_label = args.column_label self.column_text = args.column_text self.column_id = args.column_id - self.logger.info('Loading model {}'.format(args.model_name)) - self.model_name = args.model_name - self.pipeline = AutoTokenizer.from_pretrained(args.model_name) + self.logger.info('Loading {} pipeline for {}'.format(args.task, args.model)) if args.task == 'text_classification': - self.model = SequenceClassifModel.from_pretrained(args.model_name) + self.pipeline = TextClassificationPipeline.from_pretrained(args.model) elif args.task == 'token_classification': raise NotImplementedError elif args.task == 'question_answering': raise NotImplementedError self.logger.info('Loading dataset from {}'.format(args.train_data)) - dataset = SingleSentenceClassificationProcessor.create_from_csv(args.train_data) - num_data_samples = len(dataset) + self.train_dataset = Processor.create_from_csv(args.train_data, + column_label=args.column_label, + column_text=args.column_text, + column_id=args.column_id, + skip_first_row=args.skip_first_row) + self.valid_dataset = None if args.validation_data: self.logger.info('Loading validation dataset from {}'.format(args.validation_data)) - self.valid_dataset = SingleSentenceClassificationProcessor.create_from_csv(args.validation_data) - self.num_valid_samples = len(self.valid_dataset) - self.train_dataset = dataset - self.num_train_samples = num_data_samples - else: - assert 0.0 < args.validation_split < 1.0, "--validation_split should be between 0.0 and 1.0" - self.num_valid_samples = num_data_samples * args.validation_split - self.num_train_samples = num_data_samples - self.num_valid_samples - self.train_dataset = dataset[self.num_train_samples] - self.valid_dataset = dataset[self.num_valid_samples] + self.valid_dataset = Processor.create_from_csv(args.validation_data, + column_label=args.column_label, + column_text=args.column_text, + column_id=args.column_id, + skip_first_row=args.skip_first_row) + self.validation_split = args.validation_split self.train_batch_size = args.train_batch_size self.valid_batch_size = args.valid_batch_size self.learning_rate = args.learning_rate @@ -121,34 +119,13 @@ class TrainCommand(BaseTransformersCLICommand): raise NotImplementedError def run_tf(self): - import tensorflow as tf + self.pipeline.fit(self.train_dataset, + validation_data=self.valid_dataset, + validation_split=self.validation_split, + learning_rate=self.learning_rate, + adam_epsilon=self.adam_epsilon, + train_batch_size=self.train_batch_size, + valid_batch_size=self.valid_batch_size) - tf.config.optimizer.set_jit(USE_XLA) - tf.config.optimizer.set_experimental_options({"auto_mixed_precision": USE_AMP}) - - # Prepare dataset as a tf.train_data.Dataset instance - self.logger.info('Tokenizing and processing dataset') - train_dataset = self.train_dataset.get_features(self.tokenizer) - valid_dataset = self.valid_dataset.get_features(self.tokenizer) - train_dataset = train_dataset.shuffle(128).batch(self.train_batch_size).repeat(-1) - valid_dataset = valid_dataset.batch(self.valid_batch_size) - - # Prepare training: Compile tf.keras model with optimizer, loss and learning rate schedule - opt = tf.keras.optimizers.Adam(learning_rate=args.learning_rate, epsilon=self.adam_epsilon) - if USE_AMP: - # loss scaling is currently required when using mixed precision - opt = tf.keras.mixed_precision.experimental.LossScaleOptimizer(opt, 'dynamic') - loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - metric = tf.keras.metrics.SparseCategoricalAccuracy('accuracy') - self.model.compile(optimizer=opt, loss=loss, metrics=[metric]) - - # Train and evaluate using tf.keras.Model.fit() - train_steps = self.num_train_samples//self.train_batch_size - valid_steps = self.num_valid_samples//self.valid_batch_size - - self.logger.info('Training model') - history = self.model.fit(train_dataset, epochs=2, steps_per_epoch=train_steps, - validation_data=valid_dataset, validation_steps=valid_steps) - - # Save trained model - self.model.save_pretrained(self.output) + # Save trained pipeline + self.pipeline.save_pretrained(self.output) diff --git a/transformers/data/processors/utils.py b/transformers/data/processors/utils.py index 61b139c02b..ee234e6e90 100644 --- a/transformers/data/processors/utils.py +++ b/transformers/data/processors/utils.py @@ -122,14 +122,30 @@ class SingleSentenceClassificationProcessor(DataProcessor): return self.examples[idx] @classmethod - def create_from_csv(cls, file_name, **kwargs): + def create_from_csv(cls, file_name, split_name='', column_label=0, column_text=1, + column_id=None, skip_first_row=False, **kwargs): processor = cls(**kwargs) - processor.add_examples_from_csv(file_name) + processor.add_examples_from_csv(file_name, + split_name=split_name, + column_label=column_label, + column_text=column_text, + column_id=column_id, + skip_first_row=skip_first_row, + overwrite_labels=True, + overwrite_examples=True) + return processor + + @classmethod + def create_from_examples(cls, texts_or_text_and_labels, labels=None, **kwargs): + processor = cls(**kwargs) + processor.add_examples(texts_or_text_and_labels, labels=labels) return processor def add_examples_from_csv(self, file_name, split_name='', column_label=0, column_text=1, column_id=None, - overwrite_labels=False, overwrite_examples=False): + skip_first_row=False, overwrite_labels=False, overwrite_examples=False): lines = self._read_tsv(file_name) + if skip_first_row: + lines = lines[1:] texts = [] labels = [] ids = [] @@ -144,15 +160,21 @@ class SingleSentenceClassificationProcessor(DataProcessor): return self.add_examples(texts, labels, ids, overwrite_labels=overwrite_labels, overwrite_examples=overwrite_examples) - def add_examples(self, texts, labels, ids=None, overwrite_labels=False, overwrite_examples=False): + def add_examples(self, texts_or_text_and_labels, labels=None, ids=None, + overwrite_labels=False, overwrite_examples=False): + assert labels is None or len(texts_or_text_and_labels) == len(labels) + assert ids is None or len(texts_or_text_and_labels) == len(ids) if ids is None: - ids = [None] * len(texts) - assert len(texts) == len(labels) - assert len(texts) == len(ids) - + ids = [None] * len(texts_or_text_and_labels) + if labels is None: + labels = [None] * len(texts_or_text_and_labels) examples = [] added_labels = set() - for (text, label, guid) in zip(texts, labels, ids): + for (text_or_text_and_label, label, guid) in zip(texts_or_text_and_labels, labels, ids): + if isinstance(text_or_text_and_label, (tuple, list)) and label is None: + text, label = text_or_text_and_label + else: + text = text_or_text_and_label added_labels.add(label) examples.append(InputExample(guid=guid, text_a=text, text_b=None, label=label)) @@ -170,12 +192,6 @@ class SingleSentenceClassificationProcessor(DataProcessor): return self.examples - @classmethod - def create_from_examples(cls, texts, labels, **kwargs): - processor = cls(**kwargs) - processor.add_examples(texts, labels) - return processor - def get_features(self, tokenizer, max_length=None, @@ -204,6 +220,8 @@ class SingleSentenceClassificationProcessor(DataProcessor): a list of task-specific ``InputFeatures`` which can be fed to the model. """ + if max_length is None: + max_length = tokenizer.max_len label_map = {label: i for i, label in enumerate(self.labels)} diff --git a/transformers/modeling_tf_utils.py b/transformers/modeling_tf_utils.py index ed8fdb74c9..6c48f3eed2 100644 --- a/transformers/modeling_tf_utils.py +++ b/transformers/modeling_tf_utils.py @@ -22,6 +22,8 @@ import logging import os import tensorflow as tf +from tensorflow.python.keras.saving import hdf5_format +import h5py from .configuration_utils import PretrainedConfig from .file_utils import cached_path, WEIGHTS_NAME, TF_WEIGHTS_NAME, TF2_WEIGHTS_NAME @@ -206,6 +208,9 @@ class TFPreTrainedModel(tf.keras.Model): A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. + output_loading_info: (`optional`) boolean: + Set to ``True`` to also return a dictionnary containing missing keys, unexpected keys and error messages. + kwargs: (`optional`) Remaining dictionary of keyword arguments: Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded: @@ -229,6 +234,7 @@ class TFPreTrainedModel(tf.keras.Model): force_download = kwargs.pop('force_download', False) resume_download = kwargs.pop('resume_download', False) proxies = kwargs.pop('proxies', None) + output_loading_info = kwargs.pop('output_loading_info', False) # Load config if config is None: @@ -304,6 +310,31 @@ class TFPreTrainedModel(tf.keras.Model): ret = model(model.dummy_inputs, training=False) # Make sure restore ops are run + # Check if the models are the same to output loading informations + with h5py.File(resolved_archive_file, 'r') as f: + if 'layer_names' not in f.attrs and 'model_weights' in f: + f = f['model_weights'] + hdf5_layer_names = set(hdf5_format.load_attributes_from_hdf5_group(f, 'layer_names')) + model_layer_names = set(layer.name for layer in model.layers) + missing_keys = list(model_layer_names - hdf5_layer_names) + unexpected_keys = list(hdf5_layer_names - model_layer_names) + error_msgs = [] + + if len(missing_keys) > 0: + logger.info("Layers of {} not initialized from pretrained model: {}".format( + model.__class__.__name__, missing_keys)) + if len(unexpected_keys) > 0: + logger.info("Layers from pretrained model not used in {}: {}".format( + model.__class__.__name__, unexpected_keys)) + if len(error_msgs) > 0: + raise RuntimeError('Error(s) in loading weights for {}:\n\t{}'.format( + model.__class__.__name__, "\n\t".join(error_msgs))) + if output_loading_info: + loading_info = {"missing_keys": missing_keys, + "unexpected_keys": unexpected_keys, + "error_msgs": error_msgs} + return model, loading_info + return model class TFConv1D(tf.keras.layers.Layer): diff --git a/transformers/pipeline.py b/transformers/pipeline.py index dc7bcaeac3..6e55ca4d7e 100644 --- a/transformers/pipeline.py +++ b/transformers/pipeline.py @@ -17,18 +17,22 @@ from __future__ import absolute_import, division, print_function, unicode_literals import os import logging +import six -from .modeling_auto import (AutoModel, AutoModelForQuestionAnswering, - AutoModelForSequenceClassification, - AutoModelWithLMHead) from .tokenization_auto import AutoTokenizer from .file_utils import add_start_docstrings, is_tf_available, is_torch_available from .data.processors import SingleSentenceClassificationProcessor if is_tf_available(): import tensorflow as tf + from .modeling_tf_auto import (TFAutoModel, TFAutoModelForQuestionAnswering, + TFAutoModelForSequenceClassification, + TFAutoModelWithLMHead) if is_torch_available(): import torch + from .modeling_auto import (AutoModel, AutoModelForQuestionAnswering, + AutoModelForSequenceClassification, + AutoModelWithLMHead) logger = logging.getLogger(__name__) @@ -61,12 +65,6 @@ class TextClassificationPipeline(object): def __init__(self, tokenizer, model, is_compiled=False, is_trained=False): self.tokenizer = tokenizer self.model = model - if is_tf_available(): - self.framework = 'tf' - elif is_torch_available(): - self.framework = 'pt' - else: - raise ImportError("At least one of PyTorch or TensorFlow 2.0+ should be installed to use CLI training") self.is_compiled = is_compiled self.is_trained = is_trained @@ -94,9 +92,12 @@ class TextClassificationPipeline(object): # used for both the tokenizer and the model model_kwargs[key] = kwargs[key] - model_kwargs['output_loading_info'] = True tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path, **tokenizer_kwargs) - model, loading_info = AutoModelForSequenceClassification.from_pretrained(pretrained_model_name_or_path, **model_kwargs) + model_kwargs['output_loading_info'] = True + if is_tf_available(): + model, loading_info = TFAutoModelForSequenceClassification.from_pretrained(pretrained_model_name_or_path, **model_kwargs) + else: + model, loading_info = AutoModelForSequenceClassification.from_pretrained(pretrained_model_name_or_path, **model_kwargs) return cls(tokenizer, model, is_trained=bool(not loading_info['missing_keys'])) @@ -109,36 +110,42 @@ class TextClassificationPipeline(object): self.tokenizer.save_pretrained(save_directory) - def prepare_data(self, train_samples_text, train_samples_labels, - valid_samples_text=None, valid_samples_labels=None, + def prepare_data(self, x, y=None, + validation_data=None, validation_split=0.1, **kwargs): - dataset = SingleSentenceClassificationProcessor.create_from_examples(train_samples_text, - train_samples_labels) + dataset = x + if not isinstance(x, SingleSentenceClassificationProcessor): + dataset = SingleSentenceClassificationProcessor.create_from_examples(x, y) num_data_samples = len(dataset) - if valid_samples_text is not None and valid_samples_labels is not None: - valid_dataset = SingleSentenceClassificationProcessor.create_from_examples(valid_samples_text, - valid_samples_labels) + + if validation_data is not None: + valid_dataset = validation_data + if not isinstance(validation_data, SingleSentenceClassificationProcessor): + valid_dataset = SingleSentenceClassificationProcessor.create_from_examples(validation_data) + num_valid_samples = len(valid_dataset) train_dataset = dataset num_train_samples = num_data_samples else: assert 0.0 <= validation_split <= 1.0, "validation_split should be between 0.0 and 1.0" - num_valid_samples = int(num_data_samples * validation_split) + num_valid_samples = max(int(num_data_samples * validation_split), 1) num_train_samples = num_data_samples - num_valid_samples - train_dataset = dataset[num_train_samples] - valid_dataset = dataset[num_valid_samples] + train_dataset = dataset[num_valid_samples:] + valid_dataset = dataset[:num_valid_samples] logger.info('Tokenizing and processing dataset') - train_dataset = train_dataset.get_features(self.tokenizer, return_tensors=self.framework) - valid_dataset = valid_dataset.get_features(self.tokenizer, return_tensors=self.framework) - return train_dataset, valid_dataset, num_train_samples, num_valid_samples + train_dataset = train_dataset.get_features(self.tokenizer, + return_tensors='tf' if is_tf_available() else 'pt') + valid_dataset = valid_dataset.get_features(self.tokenizer, + return_tensors='tf' if is_tf_available() else 'pt') + return train_dataset, valid_dataset - def compile(self, learning_rate=3e-5, epsilon=1e-8, **kwargs): - if self.framework == 'tf': + def compile(self, learning_rate=3e-5, adam_epsilon=1e-8, **kwargs): + if is_tf_available(): logger.info('Preparing model') # Prepare training: Compile tf.keras model with optimizer, loss and learning rate schedule - opt = tf.keras.optimizers.Adam(learning_rate=learning_rate, epsilon=epsilon) + opt = tf.keras.optimizers.Adam(learning_rate=learning_rate, epsilon=adam_epsilon) if USE_AMP: # loss scaling is currently required when using mixed precision opt = tf.keras.mixed_precision.experimental.LossScaleOptimizer(opt, 'dynamic') @@ -150,39 +157,37 @@ class TextClassificationPipeline(object): self.is_compiled = True - def fit(self, train_samples_text=None, train_samples_labels=None, - valid_samples_text=None, valid_samples_labels=None, - train_batch_size=None, valid_batch_size=None, + def fit(self, X=None, y=None, + validation_data=None, validation_split=0.1, + train_batch_size=None, + valid_batch_size=None, **kwargs): - # Generic compatibility with sklearn and Keras - if 'y' in kwargs and train_samples_labels is None: - train_samples_labels = kwargs.pop('y') - if 'X' in kwargs and train_samples_text is None: - train_samples_text = kwargs.pop('X') - if not self.is_compiled: self.compile(**kwargs) - datasets = self.prepare_data(train_samples_text, train_samples_labels, - valid_samples_text, valid_samples_labels, - validation_split) - train_dataset, valid_dataset, num_train_samples, num_valid_samples = datasets + train_dataset, valid_dataset = self.prepare_data(X, y=y, + validation_data=validation_data, + validation_split=validation_split) + num_train_samples = len(train_dataset) + num_valid_samples = len(valid_dataset) train_steps = num_train_samples//train_batch_size valid_steps = num_valid_samples//valid_batch_size - if self.framework == 'tf': + if is_tf_available(): # Prepare dataset as a tf.train_data.Dataset instance train_dataset = train_dataset.shuffle(128).batch(train_batch_size).repeat(-1) valid_dataset = valid_dataset.batch(valid_batch_size) logger.info('Training TF 2.0 model') history = self.model.fit(train_dataset, epochs=2, steps_per_epoch=train_steps, - validation_data=valid_dataset, validation_steps=valid_steps, **kwargs) + validation_data=valid_dataset, validation_steps=valid_steps, + **kwargs) else: raise NotImplementedError + self.is_trained = True @@ -210,9 +215,11 @@ class TextClassificationPipeline(object): if not self.is_trained: logger.error("Some weights of the model are not trained. Please fine-tune the model on a classification task before using it.") - inputs = self.tokenizer.batch_encode_plus(texts, add_special_tokens=True, return_tensors=self.framework) + inputs = self.tokenizer.batch_encode_plus(texts, + add_special_tokens=True, + return_tensors='tf' if is_tf_available() else 'pt') - if self.framework == 'tf': + if is_tf_available(): # TODO trace model predictions = self.model(**inputs)[0] else: From 81babb227e6d6505be088ac452f3cda8a14c2255 Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Tue, 3 Dec 2019 14:56:57 +0100 Subject: [PATCH 290/505] Added download command through the cli. It allows to predownload models and tokenizers. --- transformers-cli | 4 +++- transformers/commands/download.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) mode change 100644 => 100755 transformers-cli create mode 100644 transformers/commands/download.py diff --git a/transformers-cli b/transformers-cli old mode 100644 new mode 100755 index 397b382308..168e6e6f32 --- a/transformers-cli +++ b/transformers-cli @@ -1,6 +1,7 @@ #!/usr/bin/env python from argparse import ArgumentParser +from transformers.commands.download import DownloadCommand from transformers.commands.serving import ServeCommand from transformers.commands.user import UserCommands from transformers.commands.train import TrainCommand @@ -11,10 +12,11 @@ if __name__ == '__main__': commands_parser = parser.add_subparsers(help='transformers-cli command helpers') # Register commands + ConvertCommand.register_subcommand(commands_parser) + DownloadCommand.register_subcommand(commands_parser) ServeCommand.register_subcommand(commands_parser) UserCommands.register_subcommand(commands_parser) TrainCommand.register_subcommand(commands_parser) - ConvertCommand.register_subcommand(commands_parser) # Let's go args = parser.parse_args() diff --git a/transformers/commands/download.py b/transformers/commands/download.py new file mode 100644 index 0000000000..0938f135d2 --- /dev/null +++ b/transformers/commands/download.py @@ -0,0 +1,29 @@ +from argparse import ArgumentParser + +from transformers.commands import BaseTransformersCLICommand + + +def download_command_factory(args): + return DownloadCommand(args.model, args.cache_dir, args.force) + + +class DownloadCommand(BaseTransformersCLICommand): + + @staticmethod + def register_subcommand(parser: ArgumentParser): + download_parser = parser.add_parser('download') + download_parser.add_argument('--cache-dir', type=str, default=None, help='Path to location to store the models') + download_parser.add_argument('--force', action='store_true', help='Force the model to be download even if already in cache-dir') + download_parser.add_argument('model', type=str, help='Name of the model to download') + download_parser.set_defaults(func=download_command_factory) + + def __init__(self, model: str, cache: str, force: bool): + self._model = model + self._cache = cache + self._force = force + + def run(self): + from transformers import AutoModel, AutoTokenizer + + AutoModel.from_pretrained(self._model, cache_dir=self._cache, force_download=self._force) + AutoTokenizer.from_pretrained(self._model, cache_dir=self._cache, force_download=self._force) \ No newline at end of file From e1d89cb24d13d158966c190bb75ece38eae26746 Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Fri, 6 Dec 2019 00:52:04 +0100 Subject: [PATCH 291/505] Added QuestionAnsweringPipeline with batch support. --- transformers/__init__.py | 7 +- transformers/pipeline.py | 229 -------------------------------------- transformers/pipelines.py | 222 ++++++++++++++++++++++++++++++++++++ 3 files changed, 226 insertions(+), 232 deletions(-) mode change 100644 => 100755 transformers/__init__.py delete mode 100644 transformers/pipeline.py create mode 100755 transformers/pipelines.py diff --git a/transformers/__init__.py b/transformers/__init__.py old mode 100644 new mode 100755 index 26036d2e8d..4300409257 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -65,9 +65,6 @@ from .configuration_distilbert import DistilBertConfig, DISTILBERT_PRETRAINED_CO from .configuration_albert import AlbertConfig, ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_camembert import CamembertConfig, CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP -# Pipelines -from .pipeline import TextClassificationPipeline - # Modeling if is_torch_available(): from .modeling_utils import (PreTrainedModel, prune_layer, Conv1D) @@ -193,6 +190,10 @@ from .modeling_tf_pytorch_utils import (convert_tf_weight_name_to_pt_weight_name load_tf2_weights_in_pytorch_model, load_tf2_model_in_pytorch_model) +# Pipelines +# from .pipeline_ import TextClassificationPipeline +from .pipelines import Pipeline, pipeline, TextClassificationPipeline + if not is_tf_available() and not is_torch_available(): logger.warning("Neither PyTorch nor TensorFlow >= 2.0 have been found." "Models won't be available and only tokenizers, configuration" diff --git a/transformers/pipeline.py b/transformers/pipeline.py deleted file mode 100644 index 6e55ca4d7e..0000000000 --- a/transformers/pipeline.py +++ /dev/null @@ -1,229 +0,0 @@ -# coding=utf-8 -# Copyright 2018 The HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" Pipeline class: Tokenizer + Model. """ - -from __future__ import absolute_import, division, print_function, unicode_literals -import os -import logging -import six - -from .tokenization_auto import AutoTokenizer -from .file_utils import add_start_docstrings, is_tf_available, is_torch_available -from .data.processors import SingleSentenceClassificationProcessor - -if is_tf_available(): - import tensorflow as tf - from .modeling_tf_auto import (TFAutoModel, TFAutoModelForQuestionAnswering, - TFAutoModelForSequenceClassification, - TFAutoModelWithLMHead) -if is_torch_available(): - import torch - from .modeling_auto import (AutoModel, AutoModelForQuestionAnswering, - AutoModelForSequenceClassification, - AutoModelWithLMHead) - -logger = logging.getLogger(__name__) - -# TF training parameters -USE_XLA = False -USE_AMP = False - -class TextClassificationPipeline(object): - r""" - :class:`~transformers.TextClassificationPipeline` is a class encapsulating a pretrained model and - its tokenizer and will be instantiated as one of the base model classes of the library - when created with the `Pipeline.from_pretrained(pretrained_model_name_or_path)` - class method. - - The `from_pretrained()` method takes care of returning the correct model class instance - using pattern matching on the `pretrained_model_name_or_path` string. - - The base model class to instantiate is selected as the first pattern matching - in the `pretrained_model_name_or_path` string (in the following order): - - contains `distilbert`: DistilBertModel (DistilBERT model) - - contains `roberta`: RobertaModel (RoBERTa model) - - contains `bert`: BertModel (Bert model) - - contains `openai-gpt`: OpenAIGPTModel (OpenAI GPT model) - - contains `gpt2`: GPT2Model (OpenAI GPT-2 model) - - contains `ctrl`: CTRLModel (Salesforce CTRL model) - - contains `transfo-xl`: TransfoXLModel (Transformer-XL model) - - contains `xlnet`: XLNetModel (XLNet model) - - contains `xlm`: XLMModel (XLM model) - """ - def __init__(self, tokenizer, model, is_compiled=False, is_trained=False): - self.tokenizer = tokenizer - self.model = model - self.is_compiled = is_compiled - self.is_trained = is_trained - - - @classmethod - def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): - r""" Instantiates a pipeline from a pre-trained tokenizer and model. - """ - # Extract tokenizer and model arguments - tokenizer_kwargs = {} - for key in kwargs: - if key.startswith('tokenizer_'): - # Specific to the tokenizer - tokenizer_kwargs[key.replace('tokenizer_', '')] = kwargs.pop(key) - elif not key.startswith('model_'): - # used for both the tokenizer and the model - tokenizer_kwargs[key] = kwargs[key] - - model_kwargs = {} - for key in kwargs: - if key.startswith('model_'): - # Specific to the model - model_kwargs[key.replace('model_', '')] = kwargs.pop(key) - elif not key.startswith('tokenizer_'): - # used for both the tokenizer and the model - model_kwargs[key] = kwargs[key] - - tokenizer = AutoTokenizer.from_pretrained(pretrained_model_name_or_path, **tokenizer_kwargs) - model_kwargs['output_loading_info'] = True - if is_tf_available(): - model, loading_info = TFAutoModelForSequenceClassification.from_pretrained(pretrained_model_name_or_path, **model_kwargs) - else: - model, loading_info = AutoModelForSequenceClassification.from_pretrained(pretrained_model_name_or_path, **model_kwargs) - - return cls(tokenizer, model, is_trained=bool(not loading_info['missing_keys'])) - - - def save_pretrained(self, save_directory): - if not os.path.isdir(save_directory): - logger.error("Saving directory ({}) should be a directory".format(save_directory)) - return - self.model.save_pretrained(save_directory) - self.tokenizer.save_pretrained(save_directory) - - - def prepare_data(self, x, y=None, - validation_data=None, - validation_split=0.1, **kwargs): - dataset = x - if not isinstance(x, SingleSentenceClassificationProcessor): - dataset = SingleSentenceClassificationProcessor.create_from_examples(x, y) - num_data_samples = len(dataset) - - if validation_data is not None: - valid_dataset = validation_data - if not isinstance(validation_data, SingleSentenceClassificationProcessor): - valid_dataset = SingleSentenceClassificationProcessor.create_from_examples(validation_data) - - num_valid_samples = len(valid_dataset) - train_dataset = dataset - num_train_samples = num_data_samples - else: - assert 0.0 <= validation_split <= 1.0, "validation_split should be between 0.0 and 1.0" - num_valid_samples = max(int(num_data_samples * validation_split), 1) - num_train_samples = num_data_samples - num_valid_samples - train_dataset = dataset[num_valid_samples:] - valid_dataset = dataset[:num_valid_samples] - - logger.info('Tokenizing and processing dataset') - train_dataset = train_dataset.get_features(self.tokenizer, - return_tensors='tf' if is_tf_available() else 'pt') - valid_dataset = valid_dataset.get_features(self.tokenizer, - return_tensors='tf' if is_tf_available() else 'pt') - return train_dataset, valid_dataset - - - def compile(self, learning_rate=3e-5, adam_epsilon=1e-8, **kwargs): - if is_tf_available(): - logger.info('Preparing model') - # Prepare training: Compile tf.keras model with optimizer, loss and learning rate schedule - opt = tf.keras.optimizers.Adam(learning_rate=learning_rate, epsilon=adam_epsilon) - if USE_AMP: - # loss scaling is currently required when using mixed precision - opt = tf.keras.mixed_precision.experimental.LossScaleOptimizer(opt, 'dynamic') - loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) - metric = tf.keras.metrics.SparseCategoricalAccuracy('accuracy') - self.model.compile(optimizer=opt, loss=loss, metrics=[metric]) - else: - raise NotImplementedError - self.is_compiled = True - - - def fit(self, X=None, y=None, - validation_data=None, - validation_split=0.1, - train_batch_size=None, - valid_batch_size=None, - **kwargs): - - if not self.is_compiled: - self.compile(**kwargs) - - train_dataset, valid_dataset = self.prepare_data(X, y=y, - validation_data=validation_data, - validation_split=validation_split) - num_train_samples = len(train_dataset) - num_valid_samples = len(valid_dataset) - - train_steps = num_train_samples//train_batch_size - valid_steps = num_valid_samples//valid_batch_size - - if is_tf_available(): - # Prepare dataset as a tf.train_data.Dataset instance - train_dataset = train_dataset.shuffle(128).batch(train_batch_size).repeat(-1) - valid_dataset = valid_dataset.batch(valid_batch_size) - - logger.info('Training TF 2.0 model') - history = self.model.fit(train_dataset, epochs=2, steps_per_epoch=train_steps, - validation_data=valid_dataset, validation_steps=valid_steps, - **kwargs) - else: - raise NotImplementedError - - self.is_trained = True - - - def fit_transform(self, *texts, **kwargs): - # Generic compatibility with sklearn and Keras - self.fit(*texts, **kwargs) - return self(*texts, **kwargs) - - - def transform(self, *texts, **kwargs): - # Generic compatibility with sklearn and Keras - return self(*texts, **kwargs) - - - def predict(self, *texts, **kwargs): - # Generic compatibility with sklearn and Keras - return self(*texts, **kwargs) - - - def __call__(self, *texts, **kwargs): - # Generic compatibility with sklearn and Keras - if 'X' in kwargs and not texts: - texts = kwargs.pop('X') - - if not self.is_trained: - logger.error("Some weights of the model are not trained. Please fine-tune the model on a classification task before using it.") - - inputs = self.tokenizer.batch_encode_plus(texts, - add_special_tokens=True, - return_tensors='tf' if is_tf_available() else 'pt') - - if is_tf_available(): - # TODO trace model - predictions = self.model(**inputs)[0] - else: - with torch.no_grad(): - predictions = self.model(**inputs)[0] - - return predictions.numpy().tolist() diff --git a/transformers/pipelines.py b/transformers/pipelines.py new file mode 100755 index 0000000000..e85f2300e3 --- /dev/null +++ b/transformers/pipelines.py @@ -0,0 +1,222 @@ +# coding=utf-8 +# Copyright 2018 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import, division, print_function, unicode_literals + +import os +from abc import ABC, abstractmethod +from typing import Union, Optional, Tuple + +import numpy as np + +from transformers import is_tf_available, logger, AutoTokenizer, PreTrainedTokenizer, is_torch_available + +if is_tf_available(): + from transformers import TFAutoModelForSequenceClassification, TFAutoModelForQuestionAnswering +else: + from transformers import AutoModelForSequenceClassification, AutoModelForQuestionAnswering + + +class Pipeline(ABC): + def __init__(self, model, tokenizer: PreTrainedTokenizer = None, **kwargs): + self.model = model + self.tokenizer = tokenizer + + @classmethod + @abstractmethod + def from_config(cls, model, tokenizer: PreTrainedTokenizer, **kwargs): + raise NotImplementedError() + + def save_pretrained(self, save_directory): + if not os.path.isdir(save_directory): + logger.error("Provided path ({}) should be a directory".format(save_directory)) + return + + self.model.save_pretrained(save_directory) + self.tokenizer.save_pretrained(save_directory) + + def transform(self, *texts, **kwargs): + # Generic compatibility with sklearn and Keras + return self(*texts, **kwargs) + + def predict(self, *texts, **kwargs): + # Generic compatibility with sklearn and Keras + return self(*texts, **kwargs) + + @abstractmethod + def __call__(self, *texts, **kwargs): + raise NotImplementedError() + + +class TextClassificationPipeline(Pipeline): + def __init__(self, model, tokenizer: PreTrainedTokenizer, nb_classes: int = 2): + super().__init__(model, tokenizer) + + if nb_classes < 2: + raise Exception('Invalid parameter nb_classes. int >= 2 is required (got: {})'.format(nb_classes)) + self._nb_classes = nb_classes + + @classmethod + def from_config(cls, model, tokenizer: PreTrainedTokenizer, **kwargs): + return cls(model, tokenizer, **kwargs) + + def __call__(self, *texts, **kwargs): + # Generic compatibility with sklearn and Keras + if 'X' in kwargs and not texts: + texts = kwargs.pop('X') + + inputs = self.tokenizer.batch_encode_plus( + texts, add_special_tokens=True, return_tensors='tf' if is_tf_available() else 'pt' + ) + + special_tokens_mask = inputs.pop('special_tokens_mask') + + if is_tf_available(): + # TODO trace model + predictions = self.model(**inputs)[0] + else: + import torch + with torch.no_grad(): + predictions = self.model(**inputs)[0] + + return predictions.numpy().tolist() + + +class QuestionAnsweringPipeline(Pipeline): + + @classmethod + def from_config(cls, model, tokenizer: PreTrainedTokenizer, **kwargs): + pass + + def __call__(self, texts, **kwargs): + # Generic compatibility with sklearn and Keras + if 'X' in kwargs and not texts: + texts = kwargs.pop('X') + + if not isinstance(texts, (tuple, list)): + raise Exception('QuestionAnsweringPipeline requires predict argument to be a tuple (context, question) or a List of tuple.') + + if not isinstance(texts, list): + texts = [texts] + + inputs = self.tokenizer.batch_encode_plus( + texts, add_special_tokens=True, return_tensors='tf' if is_tf_available() else 'pt' + ) + + # Remove special_tokens_mask to avoid KeyError + _ = inputs.pop('special_tokens_mask') + + if is_tf_available(): + # TODO trace model + start, end = self.model(inputs) + else: + import torch + with torch.no_grad(): + # Retrieve the score for the context tokens only (removing question tokens) + start, end = self.model(**inputs) + start, end = start.cpu().numpy(), end.cpu().numpy() + + answers = [] + for i in range(len(texts)): + context_idx = inputs['token_type_ids'][i] == 1 + start_, end_ = start[i, context_idx], end[i, context_idx] + + # Normalize logits and spans to retrieve the answer + start_, end_ = self.decode(start_, end_) + + # Convert the answer (tokens) back to the original text + answers += [{ + 'start': start_, + 'end': end_, + 'answer': self.span_to_answer(texts[i][1], start_, end_) + }] + + return answers + + def decode(self, start: np.ndarray, end: np.ndarray) -> Tuple: + # Ensure we have batch axis + if start.ndim == 1: + start = start[None] + + if end.ndim == 1: + end = end[None] + + # Compute the score of each tuple(start, end) to be the real answer + outer = np.matmul(np.expand_dims(start, -1), np.expand_dims(end, 1)) + + # Remove candidate with end < start and end - start > 15 + candidates = np.tril(np.triu(outer), 15) + + start = np.max(candidates, axis=2).argmax(-1) + end = np.max(candidates, axis=1).argmax(-1) + + return start, end + + def span_to_answer(self, text: str, start: int, end: int): + words, token_idx = [], 0 + + for i, word in enumerate(text.split(" ")): + token = self.tokenizer.tokenize(word) + + # Append words if they are in the span + if start <= token_idx <= end: + words += [word] + + # Stop if we went over the end of the answer + if token_idx > end: + break + + # Append the subtokenization length to the running index + token_idx += len(token) + + # Join text with spaces + return ' '.join(words) + + +# Register all the supported task here +SUPPORTED_TASKS = { + 'text-classification': { + 'impl': TextClassificationPipeline, + 'tf': TFAutoModelForSequenceClassification if is_tf_available() else None, + 'pt': AutoModelForSequenceClassification if is_torch_available() else None + }, + 'question-answering': { + 'impl': QuestionAnsweringPipeline, + 'tf': TFAutoModelForQuestionAnswering if is_tf_available() else None, + 'pt': AutoModelForQuestionAnswering if is_torch_available() else None + } +} + + +def pipeline(task: str, model, tokenizer: Optional[Union[str, PreTrainedTokenizer]] = None, **kwargs) -> Pipeline: + """ + Utility factory method to build pipeline. + """ + # Try to infer tokenizer from model name (if provided as str) + if tokenizer is None and isinstance(model, str): + tokenizer = model + else: + # Impossible to guest what is the right tokenizer here + raise Exception('Tokenizer cannot be None if provided model is a PreTrainedModel instance') + + tokenizer = tokenizer if isinstance(tokenizer, PreTrainedTokenizer) else AutoTokenizer.from_pretrained(tokenizer) + + if task not in SUPPORTED_TASKS: + raise KeyError("Unknown task {}, available tasks are {}".format(task, list(SUPPORTED_TASKS.keys()))) + + targeted_task = SUPPORTED_TASKS[task] + task, allocator = targeted_task['impl'], targeted_task['tf'] if is_tf_available() else targeted_task['pt'] + + model = allocator.from_pretrained(model) + return task(model, tokenizer, **kwargs) From 02110485b0980c2b0c8c4dc070643eff9c289cff Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Fri, 6 Dec 2019 18:11:27 +0100 Subject: [PATCH 292/505] Added batching, topk, chars index and scores. --- transformers/pipelines.py | 114 +++++++++++++++++++++++++++++--------- 1 file changed, 88 insertions(+), 26 deletions(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index e85f2300e3..f3b70908dd 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -16,7 +16,7 @@ from __future__ import absolute_import, division, print_function, unicode_litera import os from abc import ABC, abstractmethod -from typing import Union, Optional, Tuple +from typing import Union, Optional, Tuple, List, Dict import numpy as np @@ -24,7 +24,8 @@ from transformers import is_tf_available, logger, AutoTokenizer, PreTrainedToken if is_tf_available(): from transformers import TFAutoModelForSequenceClassification, TFAutoModelForQuestionAnswering -else: + +if is_torch_available(): from transformers import AutoModelForSequenceClassification, AutoModelForQuestionAnswering @@ -94,30 +95,71 @@ class TextClassificationPipeline(Pipeline): class QuestionAnsweringPipeline(Pipeline): + """ + Question Answering pipeling involving Tokenization and Inference. + TODO: + - top-k answers + - return start/end chars + - return score + """ + + def __init__(self, model, tokenizer: Optional[PreTrainedTokenizer]): + super().__init__(model, tokenizer) + + @staticmethod + def create_sample(question: Union[str, List[str]], context: Union[str, List[str]]) -> Union[dict, List[Dict]]: + is_list = isinstance(question, list) + + if is_list: + return [{'question': q, 'context': c} for q, c in zip(question, context)] + else: + return {'question': question, 'context': context} @classmethod def from_config(cls, model, tokenizer: PreTrainedTokenizer, **kwargs): pass - def __call__(self, texts, **kwargs): - # Generic compatibility with sklearn and Keras - if 'X' in kwargs and not texts: - texts = kwargs.pop('X') + def __call__(self, *texts, **kwargs): + # Set defaults values + kwargs.setdefault('max_answer_len', 15) + kwargs.setdefault('topk', 1) - if not isinstance(texts, (tuple, list)): - raise Exception('QuestionAnsweringPipeline requires predict argument to be a tuple (context, question) or a List of tuple.') + if kwargs['topk'] < 1: + raise ValueError('topk parameter should be >= 1 (got {})'.format(kwargs['topk'])) + + if kwargs['max_answer_len'] < 1: + raise ValueError('max_answer_len parameter should be >= 1 (got {})'.format(kwargs['max_answer_len'])) + + # Tabular input + if 'question' in kwargs and 'context' in kwargs: + texts = QuestionAnsweringPipeline.create_sample(kwargs['questions'], kwargs['contexts']) + elif 'data' in kwargs: + texts = kwargs['data'] + # Generic compatibility with sklearn and Keras + elif 'X' in kwargs and not texts: + texts = kwargs.pop('X') + else: + (texts, ) = texts + + if not isinstance(texts, (dict, list)): + raise Exception('QuestionAnsweringPipeline requires predict argument to be a tuple (context, question) or a List of dict.') if not isinstance(texts, list): texts = [texts] + # Map to tuple (question, context) + texts = [(text['question'], text['context']) for text in texts] + inputs = self.tokenizer.batch_encode_plus( - texts, add_special_tokens=True, return_tensors='tf' if is_tf_available() else 'pt' + # texts, add_special_tokens=True, return_tensors='tf' if is_tf_available() else 'pt' + texts, add_special_tokens=True, return_tensors='pt' ) # Remove special_tokens_mask to avoid KeyError _ = inputs.pop('special_tokens_mask') - if is_tf_available(): + # if is_tf_available(): + if False: # TODO trace model start, end = self.model(inputs) else: @@ -133,18 +175,19 @@ class QuestionAnsweringPipeline(Pipeline): start_, end_ = start[i, context_idx], end[i, context_idx] # Normalize logits and spans to retrieve the answer - start_, end_ = self.decode(start_, end_) + start_ = np.exp(start_) / np.sum(np.exp(start_)) + end_ = np.exp(end_) / np.sum(np.exp(end_)) + starts, ends, scores = self.decode(start_, end_, kwargs['topk'], kwargs['max_answer_len']) # Convert the answer (tokens) back to the original text - answers += [{ - 'start': start_, - 'end': end_, - 'answer': self.span_to_answer(texts[i][1], start_, end_) - }] + answers += [[ + {**{'score': score}, **self.span_to_answer(texts[i][1], s, e)} + for s, e, score in zip(starts, ends, scores) + ]] return answers - def decode(self, start: np.ndarray, end: np.ndarray) -> Tuple: + def decode(self, start: np.ndarray, end: np.ndarray, topk: int, max_answer_len: int) -> Tuple: # Ensure we have batch axis if start.ndim == 1: start = start[None] @@ -155,22 +198,39 @@ class QuestionAnsweringPipeline(Pipeline): # Compute the score of each tuple(start, end) to be the real answer outer = np.matmul(np.expand_dims(start, -1), np.expand_dims(end, 1)) - # Remove candidate with end < start and end - start > 15 - candidates = np.tril(np.triu(outer), 15) + # Remove candidate with end < start and end - start > max_answer_len + candidates = np.tril(np.triu(outer), max_answer_len - 1) - start = np.max(candidates, axis=2).argmax(-1) - end = np.max(candidates, axis=1).argmax(-1) + # start = np.max(candidates, axis=2).argmax(-1) + # end = np.max(candidates, axis=1).argmax(-1) - return start, end + scores_flat = candidates.flatten() + if topk == 1: + idx_sort = [np.argmax(scores_flat)] + elif len(scores_flat) < topk: + idx_sort = np.argsort(-scores_flat) + else: + idx = np.argpartition(-scores_flat, topk)[0:topk] + idx_sort = idx[np.argsort(-scores_flat[idx])] + + start, end = np.unravel_index(idx_sort, candidates.shape)[1:] + return start, end, candidates[0, start, end] def span_to_answer(self, text: str, start: int, end: int): - words, token_idx = [], 0 + words = [] + token_idx = char_start_idx = char_end_idx = chars_idx = 0 for i, word in enumerate(text.split(" ")): token = self.tokenizer.tokenize(word) # Append words if they are in the span if start <= token_idx <= end: + if token_idx == start: + char_start_idx = chars_idx + + if token_idx == end: + char_end_idx = chars_idx + len(word) + words += [word] # Stop if we went over the end of the answer @@ -179,9 +239,10 @@ class QuestionAnsweringPipeline(Pipeline): # Append the subtokenization length to the running index token_idx += len(token) + chars_idx += len(word) + 1 # Join text with spaces - return ' '.join(words) + return {'answer': ' '.join(words), 'start': max(0, char_start_idx), 'end': min(len(text), char_end_idx)} # Register all the supported task here @@ -193,7 +254,7 @@ SUPPORTED_TASKS = { }, 'question-answering': { 'impl': QuestionAnsweringPipeline, - 'tf': TFAutoModelForQuestionAnswering if is_tf_available() else None, + # 'tf': TFAutoModelForQuestionAnswering if is_tf_available() else None, 'pt': AutoModelForQuestionAnswering if is_torch_available() else None } } @@ -216,7 +277,8 @@ def pipeline(task: str, model, tokenizer: Optional[Union[str, PreTrainedTokenize raise KeyError("Unknown task {}, available tasks are {}".format(task, list(SUPPORTED_TASKS.keys()))) targeted_task = SUPPORTED_TASKS[task] - task, allocator = targeted_task['impl'], targeted_task['tf'] if is_tf_available() else targeted_task['pt'] + # task, allocator = targeted_task['impl'], targeted_task['tf'] if is_tf_available() else targeted_task['pt'] + task, allocator = targeted_task['impl'], targeted_task['pt'] model = allocator.from_pretrained(model) return task(model, tokenizer, **kwargs) From 6e61e06051160812d401c07e5a4c77321191ec1e Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Mon, 9 Dec 2019 11:13:27 +0100 Subject: [PATCH 293/505] batch_encode_plus generates the encoder_attention_mask to avoid attending over padded values. --- transformers/pipelines.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index f3b70908dd..8b329abd24 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -132,7 +132,7 @@ class QuestionAnsweringPipeline(Pipeline): # Tabular input if 'question' in kwargs and 'context' in kwargs: - texts = QuestionAnsweringPipeline.create_sample(kwargs['questions'], kwargs['contexts']) + texts = QuestionAnsweringPipeline.create_sample(kwargs['question'], kwargs['context']) elif 'data' in kwargs: texts = kwargs['data'] # Generic compatibility with sklearn and Keras @@ -156,7 +156,10 @@ class QuestionAnsweringPipeline(Pipeline): ) # Remove special_tokens_mask to avoid KeyError - _ = inputs.pop('special_tokens_mask') + special_tokens_mask, input_len = inputs.pop('special_tokens_mask'), inputs.pop('input_len') + + # TODO : Harmonize model arguments across all model + inputs['attention_mask'] = inputs.pop('encoder_attention_mask') # if is_tf_available(): if False: From f116cf599cd979636bdf37df31be62088a1cb7e0 Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Mon, 9 Dec 2019 11:32:49 +0100 Subject: [PATCH 294/505] Allow hidding frameworks through environment variables (NO_TF, NO_TORCH). --- transformers/file_utils.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/transformers/file_utils.py b/transformers/file_utils.py index 24abd60781..4784681fb4 100644 --- a/transformers/file_utils.py +++ b/transformers/file_utils.py @@ -27,17 +27,25 @@ from contextlib import contextmanager logger = logging.getLogger(__name__) # pylint: disable=invalid-name try: - import tensorflow as tf - assert hasattr(tf, '__version__') and int(tf.__version__[0]) >= 2 - _tf_available = True # pylint: disable=invalid-name - logger.info("TensorFlow version {} available.".format(tf.__version__)) + if 'NO_TF' in os.environ and os.environ['NO_TF'].upper() in ('1', 'ON'): + logger.info("Found NO_TF, disabling TensorFlow") + _tf_available = False + else: + import tensorflow as tf + assert hasattr(tf, '__version__') and int(tf.__version__[0]) >= 2 + _tf_available = True # pylint: disable=invalid-name + logger.info("TensorFlow version {} available.".format(tf.__version__)) except (ImportError, AssertionError): _tf_available = False # pylint: disable=invalid-name try: - import torch - _torch_available = True # pylint: disable=invalid-name - logger.info("PyTorch version {} available.".format(torch.__version__)) + if 'NO_TORCH' in os.environ and os.environ['NO_TORCH'].upper() in ('1', 'ON'): + logger.info("Found NO_TORCH, disabling PyTorch") + _torch_available = False + else: + import torch + _torch_available = True # pylint: disable=invalid-name + logger.info("PyTorch version {} available.".format(torch.__version__)) except ImportError: _torch_available = False # pylint: disable=invalid-name @@ -77,6 +85,7 @@ def is_torch_available(): return _torch_available def is_tf_available(): + return _tf_available if not six.PY2: From c2407fdd88719eed66227815188b5908eca4b3a7 Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Mon, 9 Dec 2019 11:47:52 +0100 Subject: [PATCH 295/505] Enable the Tensorflow backend. --- transformers/pipelines.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index 8b329abd24..e484958dcc 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -151,8 +151,7 @@ class QuestionAnsweringPipeline(Pipeline): texts = [(text['question'], text['context']) for text in texts] inputs = self.tokenizer.batch_encode_plus( - # texts, add_special_tokens=True, return_tensors='tf' if is_tf_available() else 'pt' - texts, add_special_tokens=True, return_tensors='pt' + texts, add_special_tokens=True, return_tensors='tf' if is_tf_available() else 'pt' ) # Remove special_tokens_mask to avoid KeyError @@ -161,10 +160,10 @@ class QuestionAnsweringPipeline(Pipeline): # TODO : Harmonize model arguments across all model inputs['attention_mask'] = inputs.pop('encoder_attention_mask') - # if is_tf_available(): - if False: + if is_tf_available(): # TODO trace model start, end = self.model(inputs) + start, end = start.numpy(), end.numpy() else: import torch with torch.no_grad(): @@ -204,9 +203,7 @@ class QuestionAnsweringPipeline(Pipeline): # Remove candidate with end < start and end - start > max_answer_len candidates = np.tril(np.triu(outer), max_answer_len - 1) - # start = np.max(candidates, axis=2).argmax(-1) - # end = np.max(candidates, axis=1).argmax(-1) - + # Inspired by Chen & al. (https://github.com/facebookresearch/DrQA) scores_flat = candidates.flatten() if topk == 1: idx_sort = [np.argmax(scores_flat)] @@ -257,7 +254,7 @@ SUPPORTED_TASKS = { }, 'question-answering': { 'impl': QuestionAnsweringPipeline, - # 'tf': TFAutoModelForQuestionAnswering if is_tf_available() else None, + 'tf': TFAutoModelForQuestionAnswering if is_tf_available() else None, 'pt': AutoModelForQuestionAnswering if is_torch_available() else None } } @@ -280,8 +277,7 @@ def pipeline(task: str, model, tokenizer: Optional[Union[str, PreTrainedTokenize raise KeyError("Unknown task {}, available tasks are {}".format(task, list(SUPPORTED_TASKS.keys()))) targeted_task = SUPPORTED_TASKS[task] - # task, allocator = targeted_task['impl'], targeted_task['tf'] if is_tf_available() else targeted_task['pt'] - task, allocator = targeted_task['impl'], targeted_task['pt'] + task, allocator = targeted_task['impl'], targeted_task['tf'] if is_tf_available() else targeted_task['pt'] model = allocator.from_pretrained(model) return task(model, tokenizer, **kwargs) From 348e19aa2104d59b91bc7216da5fcabf04f0bc5d Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Mon, 9 Dec 2019 12:10:26 +0100 Subject: [PATCH 296/505] Expose attention_masks and input_lengths arguments to batch_encode_plus --- transformers/pipelines.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index e484958dcc..46fb735a70 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -149,14 +149,11 @@ class QuestionAnsweringPipeline(Pipeline): # Map to tuple (question, context) texts = [(text['question'], text['context']) for text in texts] - inputs = self.tokenizer.batch_encode_plus( - texts, add_special_tokens=True, return_tensors='tf' if is_tf_available() else 'pt' + texts, add_special_tokens=False, return_tensors='tf' if is_tf_available() else 'pt', + return_attention_masks=True, return_input_lengths=False ) - # Remove special_tokens_mask to avoid KeyError - special_tokens_mask, input_len = inputs.pop('special_tokens_mask'), inputs.pop('input_len') - # TODO : Harmonize model arguments across all model inputs['attention_mask'] = inputs.pop('encoder_attention_mask') From fe0f552e00e7556c9dd6eccc2486b962bb2a3460 Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Mon, 9 Dec 2019 14:13:17 +0100 Subject: [PATCH 297/505] Use attention_mask everywhere. --- transformers/pipelines.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index 46fb735a70..57fe2f1357 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -154,9 +154,6 @@ class QuestionAnsweringPipeline(Pipeline): return_attention_masks=True, return_input_lengths=False ) - # TODO : Harmonize model arguments across all model - inputs['attention_mask'] = inputs.pop('encoder_attention_mask') - if is_tf_available(): # TODO trace model start, end = self.model(inputs) From a7d3794a298d77a1ae0c75c84ca963ac78058243 Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Mon, 9 Dec 2019 18:34:58 +0100 Subject: [PATCH 298/505] Remove token_type_ids for compatibility with DistilBert --- transformers/pipelines.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index 57fe2f1357..1701915203 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -20,7 +20,7 @@ from typing import Union, Optional, Tuple, List, Dict import numpy as np -from transformers import is_tf_available, logger, AutoTokenizer, PreTrainedTokenizer, is_torch_available +from transformers import is_tf_available, is_torch_available, logger, AutoTokenizer, PreTrainedTokenizer if is_tf_available(): from transformers import TFAutoModelForSequenceClassification, TFAutoModelForQuestionAnswering @@ -154,6 +154,8 @@ class QuestionAnsweringPipeline(Pipeline): return_attention_masks=True, return_input_lengths=False ) + token_type_ids = inputs.pop('token_type_ids') + if is_tf_available(): # TODO trace model start, end = self.model(inputs) @@ -167,7 +169,7 @@ class QuestionAnsweringPipeline(Pipeline): answers = [] for i in range(len(texts)): - context_idx = inputs['token_type_ids'][i] == 1 + context_idx = token_type_ids[i] == 1 start_, end_ = start[i, context_idx], end[i, context_idx] # Normalize logits and spans to retrieve the answer From aae74065dff94465cdf6d92ccfd5dee030268885 Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Mon, 9 Dec 2019 18:35:26 +0100 Subject: [PATCH 299/505] Added QuestionAnsweringPipeline unit tests. --- transformers/tests/pipelines_test.py | 83 ++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 transformers/tests/pipelines_test.py diff --git a/transformers/tests/pipelines_test.py b/transformers/tests/pipelines_test.py new file mode 100644 index 0000000000..36d6e013c3 --- /dev/null +++ b/transformers/tests/pipelines_test.py @@ -0,0 +1,83 @@ +import unittest +from unittest.mock import patch + + +QA_FINETUNED_MODELS = { + 'bert-large-uncased-whole-word-masking-finetuned-squad', + 'bert-large-cased-whole-word-masking-finetuned-squad', + 'distilbert-base-uncased-distilled-squad', + +} + + +class QuestionAnsweringPipelineTest(unittest.TestCase): + def check_answer_structure(self, answer, batch, topk): + self.assertIsInstance(answer, list) + self.assertEqual(len(answer), batch) + self.assertIsInstance(answer[0], list) + self.assertEqual(len(answer[0]), topk) + self.assertIsInstance(answer[0][0], dict) + + for item in answer[0]: + self.assertTrue('start' in item) + self.assertTrue('end' in item) + self.assertTrue('score' in item) + self.assertTrue('answer' in item) + + def question_answering_pipeline(self, nlp): + # Simple case with topk = 1, no batching + a = nlp(question='What is the name of the company I\'m working for ?', context='I\'m working for Huggingface.') + self.check_answer_structure(a, 1, 1) + + # Simple case with topk = 2, no batching + a = nlp(question='What is the name of the company I\'m working for ?', context='I\'m working for Huggingface.', topk=2) + self.check_answer_structure(a, 1, 2) + + # Batch case with topk = 1 + a = nlp(question=['What is the name of the company I\'m working for ?', 'Where is the company based ?'], + context=['I\'m working for Huggingface.', 'The company is based in New York and Paris']) + self.check_answer_structure(a, 2, 1) + + # Batch case with topk = 2 + a = nlp(question=['What is the name of the company I\'m working for ?', 'Where is the company based ?'], + context=['I\'m working for Huggingface.', 'The company is based in New York and Paris'], topk=2) + self.check_answer_structure(a, 2, 2) + + @patch('transformers.pipelines.is_torch_available', return_value=False) + def test_tf_models(self, is_torch_available): + from transformers import pipeline + for model in QA_FINETUNED_MODELS: + self.question_answering_pipeline(pipeline('question-answering', model)) + + @patch('transformers.pipelines.is_tf_available', return_value=False) + @patch('transformers.tokenization_utils.is_tf_available', return_value=False) + def test_torch_models(self, is_tf_available, _): + from transformers import pipeline + for model in QA_FINETUNED_MODELS: + self.question_answering_pipeline(pipeline('question-answering', model)) + + +class AutoPipelineTest(unittest.TestCase): + @patch('transformers.pipelines.is_torch_available', return_value=False) + def test_tf_qa(self, is_torch_available): + from transformers import pipeline + from transformers.pipelines import QuestionAnsweringPipeline + from transformers.modeling_tf_utils import TFPreTrainedModel + for model in QA_FINETUNED_MODELS: + nlp = pipeline('question-answering', model) + self.assertIsInstance(nlp, QuestionAnsweringPipeline) + self.assertIsInstance(nlp.model, TFPreTrainedModel) + + @patch('transformers.pipelines.is_tf_available', return_value=False) + def test_torch_qa(self, is_tf_available): + from transformers import pipeline + from transformers.pipelines import QuestionAnsweringPipeline + from transformers.modeling_utils import PreTrainedModel + for model in QA_FINETUNED_MODELS: + nlp = pipeline('question-answering', model) + self.assertIsInstance(nlp, QuestionAnsweringPipeline) + self.assertIsInstance(nlp.model, PreTrainedModel) + + +if __name__ == '__main__': + unittest.main() From 8ae1044f80ef543e4657c97d1030649d4da15aa8 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 10 Dec 2019 15:11:07 +0100 Subject: [PATCH 300/505] updating tests and TF 2.0 model --- transformers/modeling_t5.py | 31 ++++++--- transformers/modeling_tf_t5.py | 44 ++++++++++--- transformers/tests/modeling_common_test.py | 18 +++-- transformers/tests/modeling_t5_test.py | 9 ++- transformers/tests/modeling_tf_common_test.py | 65 +++++++++++-------- transformers/tests/modeling_tf_t5_test.py | 10 +-- transformers/tests/tokenization_t5_test.py | 1 - 7 files changed, 121 insertions(+), 57 deletions(-) diff --git a/transformers/modeling_t5.py b/transformers/modeling_t5.py index e48293b49e..f1e4e0306c 100644 --- a/transformers/modeling_t5.py +++ b/transformers/modeling_t5.py @@ -726,8 +726,11 @@ class T5Model(T5PreTrainedModel): encoder_hidden_states = kwargs_encoder.pop("hidden_states", None) encoder_attention_mask = kwargs_encoder.get("attention_mask", None) if encoder_hidden_states is None: - encoder_inputs_ids = kwargs_encoder.pop("input_ids") - hidden_states = self.shared(encoder_inputs_ids) # Convert inputs in embeddings + # Convert encoder inputs in embeddings if needed + hidden_states = kwargs_encoder.pop("inputs_embeds", None) + if hidden_states is None: + encoder_inputs_ids = kwargs_encoder.pop("input_ids") + hidden_states = self.shared(encoder_inputs_ids) # Convert inputs in embeddings if encoder_attention_mask is not None: # Apply masking @@ -740,8 +743,12 @@ class T5Model(T5PreTrainedModel): encoder_outputs = () # Decode - decoder_inputs_ids = kwargs_decoder.pop("input_ids") - hidden_states = self.shared(decoder_inputs_ids) # Convert inputs in embeddings + # Convert decoder inputs in embeddings if needed + hidden_states = kwargs_decoder.pop("inputs_embeds", None) + if hidden_states is None: + decoder_inputs_ids = kwargs_decoder.pop("input_ids") + hidden_states = self.shared(decoder_inputs_ids) + kwargs_decoder["encoder_hidden_states"] = encoder_hidden_states kwargs_decoder["encoder_attention_mask"] = encoder_attention_mask decoder_outputs = self.decoder(hidden_states, **kwargs_decoder) @@ -825,16 +832,24 @@ class T5WithLMHeadModel(T5PreTrainedModel): # Encode if needed (training, first prediction pass) encoder_hidden_states = kwargs_encoder.pop("hidden_states", None) if encoder_hidden_states is None: - encoder_inputs_ids = kwargs_encoder.pop("input_ids") - hidden_states = self.shared(encoder_inputs_ids) # Convert inputs in embeddings + # Convert encoder inputs in embeddings if needed + hidden_states = kwargs_encoder.pop("inputs_embeds", None) + if hidden_states is None: + encoder_inputs_ids = kwargs_encoder.pop("input_ids") + hidden_states = self.shared(encoder_inputs_ids) # Convert inputs in embeddings + encoder_outputs = self.encoder(hidden_states, **kwargs_encoder) encoder_hidden_states = encoder_outputs[0] else: encoder_outputs = () # Decode - decoder_inputs_ids = kwargs_decoder.pop("input_ids") - hidden_states = self.shared(decoder_inputs_ids) # Convert inputs in embeddings + # Convert decoder inputs in embeddings if needed + hidden_states = kwargs_decoder.pop("inputs_embeds", None) + if hidden_states is None: + decoder_inputs_ids = kwargs_decoder.pop("input_ids") + hidden_states = self.shared(decoder_inputs_ids) + kwargs_decoder["encoder_hidden_states"] = encoder_hidden_states kwargs_decoder["encoder_attention_mask"] = kwargs_encoder.get("attention_mask", None) decoder_outputs = self.decoder(hidden_states, **kwargs_decoder) diff --git a/transformers/modeling_tf_t5.py b/transformers/modeling_tf_t5.py index 11762ee1e5..447fd69b05 100644 --- a/transformers/modeling_tf_t5.py +++ b/transformers/modeling_tf_t5.py @@ -613,6 +613,12 @@ class TFT5Model(TFT5PreTrainedModel): decoder_config.is_decoder = True self.decoder = TFT5MainLayer(decoder_config, name='decoder') + def get_input_embeddings(self): + return self.shared + + def get_output_embeddings(self): + return self.shared + def call(self, decoder_input_ids, **kwargs): # We allow two types of multi-inputs: # - traditional keyword arguments in the call method @@ -634,16 +640,24 @@ class TFT5Model(TFT5PreTrainedModel): # Encode if needed (training, first prediction pass) encoder_hidden_states = kwargs_encoder.pop("hidden_states", None) if encoder_hidden_states is None: - encoder_inputs_ids = kwargs_encoder.pop("input_ids") - hidden_states = self.shared(encoder_inputs_ids) # Convert inputs in embeddings + # Convert encoder inputs in embeddings if needed + hidden_states = kwargs_encoder.pop("inputs_embeds", None) + if hidden_states is None: + encoder_inputs_ids = kwargs_encoder.pop("input_ids") + hidden_states = self.shared(encoder_inputs_ids) # Convert inputs in embeddings + encoder_outputs = self.encoder(hidden_states, **kwargs_encoder) encoder_hidden_states = encoder_outputs[0] else: encoder_outputs = () # Decode - decoder_inputs_ids = kwargs_decoder.pop("input_ids") - hidden_states = self.shared(decoder_inputs_ids) # Convert inputs in embeddings + # Convert decoder inputs in embeddings if needed + hidden_states = kwargs_decoder.pop("inputs_embeds", None) + if hidden_states is None: + decoder_inputs_ids = kwargs_decoder.pop("input_ids") + hidden_states = self.shared(decoder_inputs_ids) + kwargs_decoder["encoder_hidden_states"] = encoder_hidden_states kwargs_decoder["encoder_attention_mask"] = kwargs_encoder.get("attention_mask", None) decoder_outputs = self.decoder(hidden_states, **kwargs_decoder) @@ -692,6 +706,12 @@ class TFT5WithLMHeadModel(TFT5PreTrainedModel): decoder_config.is_decoder = True self.decoder = TFT5MainLayer(decoder_config, name='decoder') + def get_input_embeddings(self): + return self.shared + + def get_output_embeddings(self): + return self.shared + def call(self, decoder_input_ids, **kwargs): # We allow two types of multi-inputs: # - traditional keyword arguments in the call method @@ -713,16 +733,24 @@ class TFT5WithLMHeadModel(TFT5PreTrainedModel): # Encode if needed (training, first prediction pass) encoder_hidden_states = kwargs_encoder.pop("hidden_states", None) if encoder_hidden_states is None: - encoder_inputs_ids = kwargs_encoder.pop("input_ids") - hidden_states = self.shared(encoder_inputs_ids) # Convert inputs in embeddings + # Convert encoder inputs in embeddings if needed + hidden_states = kwargs_encoder.pop("inputs_embeds", None) + if hidden_states is None: + encoder_inputs_ids = kwargs_encoder.pop("input_ids") + hidden_states = self.shared(encoder_inputs_ids) # Convert inputs in embeddings + encoder_outputs = self.encoder(hidden_states, **kwargs_encoder) encoder_hidden_states = encoder_outputs[0] else: encoder_outputs = () # Decode - decoder_inputs_ids = kwargs_decoder.pop("input_ids") - hidden_states = self.shared(decoder_inputs_ids) # Convert inputs in embeddings + # Convert decoder inputs in embeddings if needed + hidden_states = kwargs_decoder.pop("inputs_embeds", None) + if hidden_states is None: + decoder_inputs_ids = kwargs_decoder.pop("input_ids") + hidden_states = self.shared(decoder_inputs_ids) + kwargs_decoder["encoder_hidden_states"] = encoder_hidden_states kwargs_decoder["encoder_attention_mask"] = kwargs_encoder.get("attention_mask", None) decoder_outputs = self.decoder(hidden_states, **kwargs_decoder) diff --git a/transformers/tests/modeling_common_test.py b/transformers/tests/modeling_common_test.py index cdfbfc09e2..792f5cee3e 100644 --- a/transformers/tests/modeling_common_test.py +++ b/transformers/tests/modeling_common_test.py @@ -568,8 +568,14 @@ class CommonTestCases: def test_inputs_embeds(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - input_ids = inputs_dict["input_ids"] - del inputs_dict["input_ids"] + if not self.is_encoder_decoder: + input_ids = inputs_dict["input_ids"] + del inputs_dict["input_ids"] + else: + encoder_input_ids = inputs_dict["encoder_input_ids"] + decoder_input_ids = inputs_dict["decoder_input_ids"] + del inputs_dict["encoder_input_ids"] + del inputs_dict["decoder_input_ids"] for model_class in self.all_model_classes: model = model_class(config) @@ -577,9 +583,13 @@ class CommonTestCases: model.eval() wte = model.get_input_embeddings() - inputs_dict["inputs_embeds"] = wte(input_ids) - outputs = model(**inputs_dict) + if not self.is_encoder_decoder: + inputs_dict["inputs_embeds"] = wte(input_ids) + else: + inputs_dict["encoder_inputs_embeds"] = wte(encoder_input_ids) + inputs_dict["decoder_inputs_embeds"] = wte(decoder_input_ids) + outputs = model(**inputs_dict) class GPTModelTester(CommonModelTester): diff --git a/transformers/tests/modeling_t5_test.py b/transformers/tests/modeling_t5_test.py index 091bd742b5..a539cc868a 100644 --- a/transformers/tests/modeling_t5_test.py +++ b/transformers/tests/modeling_t5_test.py @@ -18,20 +18,19 @@ from __future__ import print_function import unittest import shutil -import pytest from transformers import is_torch_available -from .modeling_common_test import (CommonTestCases, ids_tensor) +from .modeling_common_test import (CommonTestCases, ids_tensor, floats_tensor) from .configuration_common_test import ConfigTester +from .utils import require_torch, slow, torch_device if is_torch_available(): from transformers import (T5Config, T5Model, T5WithLMHeadModel) from transformers.modeling_t5 import T5_PRETRAINED_MODEL_ARCHIVE_MAP -else: - pytestmark = pytest.mark.skip("Require Torch") +@require_torch class T5ModelTest(CommonTestCases.CommonModelTester): all_model_classes = (T5Model, T5WithLMHeadModel) if is_torch_available() else () @@ -174,7 +173,7 @@ class T5ModelTest(CommonTestCases.CommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_t5_with_lm_head(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): cache_dir = "/tmp/transformers_test/" for model_name in list(T5_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: diff --git a/transformers/tests/modeling_tf_common_test.py b/transformers/tests/modeling_tf_common_test.py index 8957313021..a0d63583fb 100644 --- a/transformers/tests/modeling_tf_common_test.py +++ b/transformers/tests/modeling_tf_common_test.py @@ -130,12 +130,12 @@ class TFCommonTestCases: for name, key in inputs_dict.items()) with torch.no_grad(): pto = pt_model(**pt_inputs_dict) - tfo = tf_model(inputs_dict) - tfo = tfo[0].numpy() - pto = pto[0].numpy() - tfo[np.isnan(tfo)] = 0 - pto[np.isnan(pto)] = 0 - max_diff = np.amax(np.abs(tfo - pto)) + tfo = tf_model(inputs_dict, training=False) + tf_hidden_states = tfo[0].numpy() + pt_hidden_states = pto[0].numpy() + tf_hidden_states[np.isnan(tf_hidden_states)] = 0 + pt_hidden_states[np.isnan(pt_hidden_states)] = 0 + max_diff = np.amax(np.abs(tf_hidden_states - pt_hidden_states)) self.assertLessEqual(max_diff, 2e-2) # Check we can load pt model in tf and vice-versa with checkpoint => model functions @@ -296,33 +296,46 @@ class TFCommonTestCases: first, second = model(inputs_dict, training=False)[0], model(inputs_dict, training=False)[0] self.assertTrue(tf.math.equal(first, second).numpy().all()) + def _get_embeds(self, wte, input_ids): + # ^^ In our TF models, the input_embeddings can take slightly different forms, + # so we try a few of them. + # We used to fall back to just synthetically creating a dummy tensor of ones: + try: + x = wte(input_ids, mode="embedding") + except: + try: + x = wte([input_ids], mode="embedding") + except: + try: + x = wte([input_ids, None, None, None], mode="embedding") + except: + if hasattr(self.model_tester, "embedding_size"): + x = tf.ones(input_ids.shape + [self.model_tester.embedding_size], dtype=tf.dtypes.float32) + else: + x = tf.ones(input_ids.shape + [self.model_tester.hidden_size], dtype=tf.dtypes.float32) + return x + def test_inputs_embeds(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() - input_ids = inputs_dict["input_ids"] - del inputs_dict["input_ids"] + if not self.is_encoder_decoder: + input_ids = inputs_dict["input_ids"] + del inputs_dict["input_ids"] + else: + encoder_input_ids = inputs_dict["encoder_input_ids"] + decoder_input_ids = inputs_dict["decoder_input_ids"] + del inputs_dict["encoder_input_ids"] + del inputs_dict["decoder_input_ids"] for model_class in self.all_model_classes: model = model_class(config) wte = model.get_input_embeddings() - try: - x = wte(input_ids, mode="embedding") - except: - try: - x = wte([input_ids], mode="embedding") - except: - try: - x = wte([input_ids, None, None, None], mode="embedding") - except: - if hasattr(self.model_tester, "embedding_size"): - x = tf.ones(input_ids.shape + [self.model_tester.embedding_size], dtype=tf.dtypes.float32) - else: - x = tf.ones(input_ids.shape + [self.model_tester.hidden_size], dtype=tf.dtypes.float32) - # ^^ In our TF models, the input_embeddings can take slightly different forms, - # so we try a few of them. - # We used to fall back to just synthetically creating a dummy tensor of ones: - # - inputs_dict["inputs_embeds"] = x + if not self.is_encoder_decoder: + inputs_dict["inputs_embeds"] = self._get_embeds(wte, input_ids) + else: + inputs_dict["encoder_inputs_embeds"] = self._get_embeds(wte, encoder_input_ids) + inputs_dict["decoder_inputs_embeds"] = self._get_embeds(wte, decoder_input_ids) + outputs = model(inputs_dict) diff --git a/transformers/tests/modeling_tf_t5_test.py b/transformers/tests/modeling_tf_t5_test.py index 33f6f895f0..99eec313f9 100644 --- a/transformers/tests/modeling_tf_t5_test.py +++ b/transformers/tests/modeling_tf_t5_test.py @@ -18,21 +18,21 @@ from __future__ import print_function import unittest import shutil -import pytest import sys from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester +from .utils import require_tf, slow from transformers import T5Config, is_tf_available if is_tf_available(): import tensorflow as tf - from transformers.modeling_tf_t5 import (TFT5Model, TFT5WithLMHeadModel,TF_T5_PRETRAINED_MODEL_ARCHIVE_MAP) -else: - pytestmark = pytest.mark.skip("Require TensorFlow") + from transformers.modeling_tf_t5 import (TFT5Model, TFT5WithLMHeadModel, + TF_T5_PRETRAINED_MODEL_ARCHIVE_MAP) +@require_tf class TFT5ModelTest(TFCommonTestCases.TFCommonModelTester): is_encoder_decoder = True @@ -160,7 +160,7 @@ class TFT5ModelTest(TFCommonTestCases.TFCommonModelTester): config_and_inputs = self.model_tester.prepare_config_and_inputs() self.model_tester.create_and_check_t5_with_lm_head(*config_and_inputs) - @pytest.mark.slow + @slow def test_model_from_pretrained(self): cache_dir = "/tmp/transformers_test/" for model_name in ['t5-small']: diff --git a/transformers/tests/tokenization_t5_test.py b/transformers/tests/tokenization_t5_test.py index aabb21e443..0b4f960e32 100644 --- a/transformers/tests/tokenization_t5_test.py +++ b/transformers/tests/tokenization_t5_test.py @@ -16,7 +16,6 @@ from __future__ import absolute_import, division, print_function, unicode_litera import os import unittest -import pytest from transformers.tokenization_t5 import (T5Tokenizer) from transformers.tokenization_xlnet import SPIECE_UNDERLINE From 4b82c485de187896a38c441587b7bd4d04f2821e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Louf?= Date: Tue, 10 Dec 2019 14:49:53 +0100 Subject: [PATCH 301/505] remove misplaced summarization documentation --- examples/README.md | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/examples/README.md b/examples/README.md index 620304ea77..b6b3908810 100644 --- a/examples/README.md +++ b/examples/README.md @@ -24,8 +24,6 @@ pip install -r ./examples/requirements.txt | [Multiple Choice](#multiple-choice) | Examples running BERT/XLNet/RoBERTa on the SWAG/RACE/ARC tasks. | [Named Entity Recognition](#named-entity-recognition) | Using BERT for Named Entity Recognition (NER) on the CoNLL 2003 dataset, examples with distributed training. | | [XNLI](#xnli) | Examples running BERT/XLM on the XNLI benchmark. | -| [Abstractive summarization](#abstractive-summarization) | Using the BertAbs -model finetuned on the CNN/DailyMail dataset to generate summaries. | ## TensorFlow 2.0 Bert models on GLUE @@ -646,34 +644,6 @@ micro avg 0.8722 0.8774 0.8748 13869 macro avg 0.8712 0.8774 0.8740 13869 ``` -## Abstractive summarization - -Based on the script -[`run_summarization_finetuning.py`](https://github.com/huggingface/transformers/blob/master/examples/run_summarization_finetuning.py). - -Before running this script you should download **both** CNN and Daily Mail -datasets from [Kyunghyun Cho's website](https://cs.nyu.edu/~kcho/DMQA/) (the -links next to "Stories") in the same folder. Then uncompress the archives by running: - -```bash -tar -xvf cnn_stories.tgz && tar -xvf dailymail_stories.tgz -``` - -note that the finetuning script **will not work** if you do not download both -datasets. We will refer as `$DATA_PATH` the path to where you uncompressed both -archive. - -```bash -export DATA_PATH=/path/to/dataset/ - -python run_summarization_finetuning.py \ - --output_dir=output \ - --model_type=bert2bert \ - --model_name_or_path=bert2bert \ - --do_train \ - --data_path=$DATA_PATH \ -``` - ## XNLI Based on the script [`run_xnli.py`](https://github.com/huggingface/transformers/blob/master/examples/run_xnli.py). From 981a5c8c1789f91204ba1053f4742f6ea8c615af Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 10 Dec 2019 15:36:19 +0100 Subject: [PATCH 302/505] updating models urls --- transformers/configuration_t5.py | 4 ++++ transformers/convert_pytorch_checkpoint_to_tf2.py | 2 +- transformers/modeling_t5.py | 4 ++++ transformers/modeling_tf_t5.py | 6 +++++- transformers/tokenization_t5.py | 12 ++++++++++-- 5 files changed, 24 insertions(+), 4 deletions(-) diff --git a/transformers/configuration_t5.py b/transformers/configuration_t5.py index 83aab66fac..2ccdebc2b1 100644 --- a/transformers/configuration_t5.py +++ b/transformers/configuration_t5.py @@ -28,6 +28,10 @@ logger = logging.getLogger(__name__) T5_PRETRAINED_CONFIG_ARCHIVE_MAP = { 't5-small': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-small-config.json", + 't5-base': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-base-config.json", + 't5-large': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-large-config.json", + 't5-3B': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-3B-config.json", + 't5-11B': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-11B-config.json", } diff --git a/transformers/convert_pytorch_checkpoint_to_tf2.py b/transformers/convert_pytorch_checkpoint_to_tf2.py index 4c4becfa00..06bb5f47c0 100644 --- a/transformers/convert_pytorch_checkpoint_to_tf2.py +++ b/transformers/convert_pytorch_checkpoint_to_tf2.py @@ -121,7 +121,7 @@ def convert_pt_checkpoint_to_tf(model_type, pytorch_checkpoint_path, config_file if compare_with_pt_model: inputs_list = [[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]] - tf_inputs = tf.constant(inputs_list) + tf_inputs = tf_model.dummy_inputs tfo = tf_model(tf_inputs, training=False) # build the network pt_model = pt_model_class.from_pretrained(None, diff --git a/transformers/modeling_t5.py b/transformers/modeling_t5.py index f1e4e0306c..ffc4d8bb3f 100644 --- a/transformers/modeling_t5.py +++ b/transformers/modeling_t5.py @@ -42,6 +42,10 @@ logger = logging.getLogger(__name__) #################################################### T5_PRETRAINED_MODEL_ARCHIVE_MAP = { 't5-small': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-small-pytorch_model.bin", + 't5-base': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-base-pytorch_model.bin", + 't5-large': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-large-pytorch_model.bin", + 't5-3B': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-3B-pytorch_model.bin", + 't5-11B': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-11B-pytorch_model.bin", } #################################################### diff --git a/transformers/modeling_tf_t5.py b/transformers/modeling_tf_t5.py index 447fd69b05..0b3b1116f2 100644 --- a/transformers/modeling_tf_t5.py +++ b/transformers/modeling_tf_t5.py @@ -25,13 +25,17 @@ import itertools import tensorflow as tf from .configuration_t5 import T5Config -from .modeling_tf_utils import TFPreTrainedModel, TFSharedEmbeddings, shape_list, get_initializer, DUMMY_INPUTS +from .modeling_tf_utils import TFPreTrainedModel, TFSharedEmbeddings, shape_list from .file_utils import add_start_docstrings logger = logging.getLogger(__name__) TF_T5_PRETRAINED_MODEL_ARCHIVE_MAP = { 't5-small': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-small-tf_model.h5", + 't5-base': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-base-tf_model.h5", + 't5-large': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-large-tf_model.h5", + 't5-3B': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-3B-tf_model.h5", + 't5-11B': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-11B-tf_model.h5", } #################################################### diff --git a/transformers/tokenization_t5.py b/transformers/tokenization_t5.py index 933084d13a..62e9c069e2 100644 --- a/transformers/tokenization_t5.py +++ b/transformers/tokenization_t5.py @@ -41,7 +41,11 @@ VOCAB_FILES_NAMES = {'vocab_file': 'spiece.model'} PRETRAINED_VOCAB_FILES_MAP = { 'vocab_file': { - 't5': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-spiece.model", + 't5-small': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-spiece.model", + 't5-base': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-spiece.model", + 't5-large': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-spiece.model", + 't5-3B': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-spiece.model", + 't5-11B': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-spiece.model", } } @@ -49,7 +53,11 @@ PRETRAINED_VOCAB_FILES_MAP = { # Mapping from model shortcut names to max length of inputs #################################################### PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { - 't5': 512, + 't5-small': 512, + 't5-base': 512, + 't5-large': 512, + 't5-3B': 512, + 't5-11B': 512, } class T5Tokenizer(PreTrainedTokenizer): From 40a39ab65043f11763d8f0ce5fb0307661e6f7a3 Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Tue, 10 Dec 2019 15:59:38 +0100 Subject: [PATCH 303/505] Reuse recent SQuAD refactored data structure inside QA pipelines. --- transformers/data/processors/__init__.py | 2 +- transformers/modeling_auto.py | 12 +--- transformers/pipelines.py | 84 ++++++++++++++++-------- 3 files changed, 59 insertions(+), 39 deletions(-) diff --git a/transformers/data/processors/__init__.py b/transformers/data/processors/__init__.py index 0cef0080f4..4f7307bb7b 100644 --- a/transformers/data/processors/__init__.py +++ b/transformers/data/processors/__init__.py @@ -1,4 +1,4 @@ -from .utils import InputExample, InputFeatures, DataProcessor, SingleSentenceClassificationProcessor, convert_examples_to_features +from .utils import InputExample, InputFeatures, DataProcessor, SingleSentenceClassificationProcessor from .glue import glue_output_modes, glue_processors, glue_tasks_num_labels, glue_convert_examples_to_features from .squad import squad_convert_examples_to_features, SquadFeatures, SquadExample, SquadV1Processor, SquadV2Processor from .xnli import xnli_output_modes, xnli_processors, xnli_tasks_num_labels \ No newline at end of file diff --git a/transformers/modeling_auto.py b/transformers/modeling_auto.py index 0c8bffa883..041115cc61 100644 --- a/transformers/modeling_auto.py +++ b/transformers/modeling_auto.py @@ -31,7 +31,7 @@ from .modeling_xlnet import XLNetModel, XLNetLMHeadModel, XLNetForSequenceClassi from .modeling_xlm import XLMModel, XLMWithLMHeadModel, XLMForSequenceClassification, XLMForQuestionAnswering from .modeling_roberta import RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification from .modeling_distilbert import DistilBertModel, DistilBertForQuestionAnswering, DistilBertForMaskedLM, DistilBertForSequenceClassification -from .modeling_camembert import CamembertModel, CamembertForQuestionAnswering, CamembertForMaskedLM, CamembertForSequenceClassification, CamembertForMultipleChoice +from .modeling_camembert import CamembertModel, CamembertForMaskedLM, CamembertForSequenceClassification, CamembertForMultipleChoice from .modeling_albert import AlbertModel, AlbertForMaskedLM, AlbertForSequenceClassification, AlbertForQuestionAnswering from .modeling_utils import PreTrainedModel, SequenceSummary @@ -294,10 +294,6 @@ class AutoModelWithLMHead(object): return XLMWithLMHeadModel(config) elif isinstance(config, CTRLConfig): return CTRLLMHeadModel(config) - elif isinstance(config, AlbertConfig): - return AlbertLMHeadModel(config) - elif isinstance(config, CamembertConfig): - return CamembertLMHeadModel(config) raise ValueError("Unrecognized configuration class {}".format(config)) @classmethod @@ -454,7 +450,7 @@ class AutoModelForSequenceClassification(object): """ if isinstance(config, AlbertConfig): return AlbertForSequenceClassification(config) - elif isintance(config, CamembertConfig): + elif isinstance(config, CamembertConfig): return CamembertForSequenceClassification(config) elif isinstance(config, DistilBertConfig): return DistilBertForSequenceClassification(config) @@ -606,10 +602,8 @@ class AutoModelForQuestionAnswering(object): config = BertConfig.from_pretrained('bert-base-uncased') # Download configuration from S3 and cache. model = AutoModelForSequenceClassification.from_config(config) # E.g. model was saved using `save_pretrained('./test/saved_model/')` """ - if isintance(config, AlbertConfig): + if isinstance(config, AlbertConfig): return AlbertForQuestionAnswering(config) - elif isintance(config, CamembertConfig): - return CamembertForQuestionAnswering(config) elif isinstance(config, DistilBertConfig): return DistilBertForQuestionAnswering(config) elif isinstance(config, BertConfig): diff --git a/transformers/pipelines.py b/transformers/pipelines.py index 1701915203..1e2f035d9f 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -20,7 +20,8 @@ from typing import Union, Optional, Tuple, List, Dict import numpy as np -from transformers import is_tf_available, is_torch_available, logger, AutoTokenizer, PreTrainedTokenizer +from transformers import is_tf_available, is_torch_available, logger, AutoTokenizer, PreTrainedTokenizer, \ + SquadExample, squad_convert_examples_to_features if is_tf_available(): from transformers import TFAutoModelForSequenceClassification, TFAutoModelForQuestionAnswering @@ -107,13 +108,28 @@ class QuestionAnsweringPipeline(Pipeline): super().__init__(model, tokenizer) @staticmethod - def create_sample(question: Union[str, List[str]], context: Union[str, List[str]]) -> Union[dict, List[Dict]]: + def create_sample(question: Union[str, List[str]], context: Union[str, List[str]]) -> Union[SquadExample, List[SquadExample]]: is_list = isinstance(question, list) if is_list: - return [{'question': q, 'context': c} for q, c in zip(question, context)] + return [SquadExample(None, q, c, None, None, None) for q, c in zip(question, context)] else: - return {'question': question, 'context': context} + return SquadExample(None, question, context, None, None, None) + + def inputs_for_model(self, features: Union[SquadExample, List[SquadExample]]) -> Dict: + args = ['input_ids', 'attention_mask'] + model_type = type(self.model).__name__.lower() + + if 'distilbert' not in model_type and 'xlm' not in model_type: + args += ['token_type_ids'] + + if 'xlnet' in model_type or 'xlm' in model_type: + args += ['cls_index', 'p_mask'] + + if isinstance(features, SquadExample): + return {k: features.__dict__[k] for k in args} + else: + return {k: [feature.__dict__[k] for feature in features] for k in args} @classmethod def from_config(cls, model, tokenizer: PreTrainedTokenizer, **kwargs): @@ -121,8 +137,11 @@ class QuestionAnsweringPipeline(Pipeline): def __call__(self, *texts, **kwargs): # Set defaults values - kwargs.setdefault('max_answer_len', 15) kwargs.setdefault('topk', 1) + kwargs.setdefault('doc_stride', 128) + kwargs.setdefault('max_answer_len', 15) + kwargs.setdefault('max_seq_len', 384) + kwargs.setdefault('max_question_len', 64) if kwargs['topk'] < 1: raise ValueError('topk parameter should be >= 1 (got {})'.format(kwargs['topk'])) @@ -130,56 +149,63 @@ class QuestionAnsweringPipeline(Pipeline): if kwargs['max_answer_len'] < 1: raise ValueError('max_answer_len parameter should be >= 1 (got {})'.format(kwargs['max_answer_len'])) - # Tabular input - if 'question' in kwargs and 'context' in kwargs: - texts = QuestionAnsweringPipeline.create_sample(kwargs['question'], kwargs['context']) - elif 'data' in kwargs: - texts = kwargs['data'] + # Position args + if texts is not None and len(texts) > 1: + (texts, ) = texts + # Generic compatibility with sklearn and Keras elif 'X' in kwargs and not texts: texts = kwargs.pop('X') - else: - (texts, ) = texts - if not isinstance(texts, (dict, list)): - raise Exception('QuestionAnsweringPipeline requires predict argument to be a tuple (context, question) or a List of dict.') + # Batched data + elif 'data' in kwargs: + texts = kwargs.pop('data') + + # Tabular input + elif 'question' in kwargs and 'context' in kwargs: + texts = QuestionAnsweringPipeline.create_sample(kwargs['question'], kwargs['context']) + else: + raise ValueError('Unknown arguments {}'.format(kwargs)) if not isinstance(texts, list): texts = [texts] - # Map to tuple (question, context) - texts = [(text['question'], text['context']) for text in texts] - inputs = self.tokenizer.batch_encode_plus( - texts, add_special_tokens=False, return_tensors='tf' if is_tf_available() else 'pt', - return_attention_masks=True, return_input_lengths=False - ) - - token_type_ids = inputs.pop('token_type_ids') + # Convert inputs to features + features = squad_convert_examples_to_features(texts, self.tokenizer, kwargs['max_seq_len'], kwargs['doc_stride'], kwargs['max_question_len'], False) + fw_args = self.inputs_for_model(features) if is_tf_available(): - # TODO trace model - start, end = self.model(inputs) + import tensorflow as tf + fw_args = {k: tf.constant(v) for (k, v) in fw_args.items()} + start, end = self.model(fw_args) start, end = start.numpy(), end.numpy() else: import torch with torch.no_grad(): # Retrieve the score for the context tokens only (removing question tokens) - start, end = self.model(**inputs) + fw_args = {k: torch.tensor(v) for (k, v) in fw_args.items()} + start, end = self.model(**fw_args) start, end = start.cpu().numpy(), end.cpu().numpy() answers = [] - for i in range(len(texts)): - context_idx = token_type_ids[i] == 1 - start_, end_ = start[i, context_idx], end[i, context_idx] + for i, (example, feature, start_, end_) in enumerate(zip(texts, features, start, end)): + start_, end_ = start_ * np.abs(np.array(feature.p_mask) - 1), end_ * np.abs(np.array(feature.p_mask) - 1) # Normalize logits and spans to retrieve the answer start_ = np.exp(start_) / np.sum(np.exp(start_)) end_ = np.exp(end_) / np.sum(np.exp(end_)) starts, ends, scores = self.decode(start_, end_, kwargs['topk'], kwargs['max_answer_len']) + char_to_word = np.array(example.char_to_word_offset) + # Convert the answer (tokens) back to the original text answers += [[ - {**{'score': score}, **self.span_to_answer(texts[i][1], s, e)} + { + 'score': score, + 'start': np.where(char_to_word == feature.token_to_orig_map[s])[0][0], + 'end': np.where(char_to_word == feature.token_to_orig_map[e])[0][-1], + 'answer': ' '.join(example.doc_tokens[feature.token_to_orig_map[s]: feature.token_to_orig_map[e] + 1]) + } for s, e, score in zip(starts, ends, scores) ]] From a5df980c5b86e9106382a87a63b977d5decf97f6 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 10 Dec 2019 16:01:15 +0100 Subject: [PATCH 304/505] updating distilbert test --- transformers/tests/modeling_common_test.py | 7 ++++++- transformers/tests/modeling_tf_common_test.py | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/transformers/tests/modeling_common_test.py b/transformers/tests/modeling_common_test.py index 792f5cee3e..2f2baff436 100644 --- a/transformers/tests/modeling_common_test.py +++ b/transformers/tests/modeling_common_test.py @@ -121,7 +121,12 @@ class CommonTestCases: model.to(torch_device) model.eval() first, second = model(**inputs_dict)[0], model(**inputs_dict)[0] - self.assertEqual(first.ne(second).sum().item(), 0) + out_1 = first.cpu().numpy() + out_2 = second.cpu().numpy() + out_1 = out_1[~np.isnan(out_1)] + out_2 = out_2[~np.isnan(out_2)] + max_diff = np.amax(np.abs(out_1 - out_2)) + self.assertLessEqual(max_diff, 1e-5) def test_attention_outputs(self): config, inputs_dict = self.model_tester.prepare_config_and_inputs_for_common() diff --git a/transformers/tests/modeling_tf_common_test.py b/transformers/tests/modeling_tf_common_test.py index a0d63583fb..5a5873e81b 100644 --- a/transformers/tests/modeling_tf_common_test.py +++ b/transformers/tests/modeling_tf_common_test.py @@ -294,7 +294,12 @@ class TFCommonTestCases: for model_class in self.all_model_classes: model = model_class(config) first, second = model(inputs_dict, training=False)[0], model(inputs_dict, training=False)[0] - self.assertTrue(tf.math.equal(first, second).numpy().all()) + out_1 = first.numpy() + out_2 = second.numpy() + out_1 = out_1[~np.isnan(out_1)] + out_2 = out_2[~np.isnan(out_2)] + max_diff = np.amax(np.abs(out_1 - out_2)) + self.assertLessEqual(max_diff, 1e-5) def _get_embeds(self, wte, input_ids): # ^^ In our TF models, the input_embeddings can take slightly different forms, From f2538c12741df74abbd2ff38f43019cfbb21093b Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 10 Dec 2019 16:33:11 +0100 Subject: [PATCH 305/505] all tests in torch no grad --- transformers/tests/modeling_common_test.py | 53 ++++++++++++++-------- 1 file changed, 35 insertions(+), 18 deletions(-) diff --git a/transformers/tests/modeling_common_test.py b/transformers/tests/modeling_common_test.py index 2f2baff436..ed6f950e25 100644 --- a/transformers/tests/modeling_common_test.py +++ b/transformers/tests/modeling_common_test.py @@ -120,7 +120,9 @@ class CommonTestCases: model = model_class(config) model.to(torch_device) model.eval() - first, second = model(**inputs_dict)[0], model(**inputs_dict)[0] + with torch.no_grad(): + first = model(**inputs_dict)[0] + second = model(**inputs_dict)[0] out_1 = first.cpu().numpy() out_2 = second.cpu().numpy() out_1 = out_1[~np.isnan(out_1)] @@ -142,7 +144,8 @@ class CommonTestCases: model = model_class(config) model.to(torch_device) model.eval() - outputs = model(**inputs_dict) + with torch.no_grad(): + outputs = model(**inputs_dict) attentions = outputs[-1] self.assertEqual(model.config.output_attentions, True) self.assertEqual(model.config.output_hidden_states, False) @@ -173,7 +176,8 @@ class CommonTestCases: model = model_class(config) model.to(torch_device) model.eval() - outputs = model(**inputs_dict) + with torch.no_grad(): + outputs = model(**inputs_dict) self.assertEqual(out_len + (2 if self.is_encoder_decoder else 1), len(outputs)) self.assertEqual(model.config.output_attentions, True) self.assertEqual(model.config.output_hidden_states, True) @@ -273,7 +277,8 @@ class CommonTestCases: inputs = inputs_dict.copy() inputs['head_mask'] = head_mask - outputs = model(**inputs) + with torch.no_grad(): + outputs = model(**inputs) # Test that we can get a gradient back for importance score computation output = sum(t.sum() for t in outputs[0]) @@ -320,7 +325,8 @@ class CommonTestCases: heads_to_prune = {0: list(range(1, self.model_tester.num_attention_heads)), -1: [0]} model.prune_heads(heads_to_prune) - outputs = model(**inputs_dict) + with torch.no_grad(): + outputs = model(**inputs_dict) attentions = outputs[-1] @@ -356,7 +362,8 @@ class CommonTestCases: model = model_class.from_pretrained(directory) model.to(torch_device) - outputs = model(**inputs_dict) + with torch.no_grad(): + outputs = model(**inputs_dict) attentions = outputs[-1] self.assertEqual(attentions[0].shape[-3], 1) self.assertEqual(attentions[1].shape[-3], self.model_tester.num_attention_heads) @@ -385,7 +392,8 @@ class CommonTestCases: model.to(torch_device) model.eval() - outputs = model(**inputs_dict) + with torch.no_grad(): + outputs = model(**inputs_dict) attentions = outputs[-1] self.assertEqual(attentions[0].shape[-3], 1) @@ -412,7 +420,8 @@ class CommonTestCases: model.to(torch_device) model.eval() - outputs = model(**inputs_dict) + with torch.no_grad(): + outputs = model(**inputs_dict) attentions = outputs[-1] self.assertEqual(attentions[0].shape[-3], self.model_tester.num_attention_heads - 1) @@ -429,7 +438,8 @@ class CommonTestCases: model.to(torch_device) shutil.rmtree(directory) - outputs = model(**inputs_dict) + with torch.no_grad(): + outputs = model(**inputs_dict) attentions = outputs[-1] self.assertEqual(attentions[0].shape[-3], self.model_tester.num_attention_heads - 1) @@ -440,7 +450,8 @@ class CommonTestCases: heads_to_prune = {0: [0], 2: [1, 2]} model.prune_heads(heads_to_prune) - outputs = model(**inputs_dict) + with torch.no_grad(): + outputs = model(**inputs_dict) attentions = outputs[-1] self.assertEqual(attentions[0].shape[-3], self.model_tester.num_attention_heads -1) @@ -459,7 +470,8 @@ class CommonTestCases: model = model_class(config) model.to(torch_device) model.eval() - outputs = model(**inputs_dict) + with torch.no_grad(): + outputs = model(**inputs_dict) hidden_states = outputs[-1] self.assertEqual(model.config.output_attentions, False) self.assertEqual(model.config.output_hidden_states, True) @@ -594,7 +606,8 @@ class CommonTestCases: inputs_dict["encoder_inputs_embeds"] = wte(encoder_input_ids) inputs_dict["decoder_inputs_embeds"] = wte(decoder_input_ids) - outputs = model(**inputs_dict) + with torch.no_grad(): + outputs = model(**inputs_dict) class GPTModelTester(CommonModelTester): @@ -682,9 +695,10 @@ class CommonTestCases: model.to(torch_device) model.eval() - outputs = model(input_ids, position_ids, token_type_ids) - outputs = model(input_ids, position_ids) - outputs = model(input_ids) + with torch.no_grad(): + outputs = model(input_ids, position_ids, token_type_ids) + outputs = model(input_ids, position_ids) + outputs = model(input_ids) hidden_state = outputs[0] self.parent.assertListEqual( @@ -697,7 +711,8 @@ class CommonTestCases: model = self.lm_head_model_class(config) model.to(torch_device) model.eval() - outputs = model(input_ids, position_ids, token_type_ids, lm_labels) + with torch.no_grad(): + outputs = model(input_ids, position_ids, token_type_ids, lm_labels) loss, lm_logits = outputs[:2] total_voc = self.vocab_size @@ -714,7 +729,8 @@ class CommonTestCases: model = model_class(config) model.to(torch_device) model.eval() - outputs = model(input_ids) + with torch.no_grad(): + outputs = model(input_ids) presents = outputs[-1] self.parent.assertEqual(self.num_hidden_layers, len(presents)) self.parent.assertListEqual( @@ -727,7 +743,8 @@ class CommonTestCases: model = self.double_head_model_class(config) model.to(torch_device) model.eval() - outputs = model(input_ids, mc_token_ids, lm_labels=lm_labels, mc_labels=mc_labels, + with torch.no_grad(): + outputs = model(input_ids, mc_token_ids, lm_labels=lm_labels, mc_labels=mc_labels, token_type_ids=token_type_ids, position_ids=position_ids) lm_loss, mc_loss, lm_logits, mc_logits = outputs[:4] loss = [lm_loss, mc_loss] From 63e36007ee152cee23f44103622f28566e28fb72 Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Tue, 10 Dec 2019 16:47:35 +0100 Subject: [PATCH 306/505] Make sure padding, cls and another non-context tokens cannot appear in the answer. --- transformers/pipelines.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index 1e2f035d9f..eec4932321 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -188,14 +188,18 @@ class QuestionAnsweringPipeline(Pipeline): start, end = start.cpu().numpy(), end.cpu().numpy() answers = [] - for i, (example, feature, start_, end_) in enumerate(zip(texts, features, start, end)): - start_, end_ = start_ * np.abs(np.array(feature.p_mask) - 1), end_ * np.abs(np.array(feature.p_mask) - 1) - + for (example, feature, start_, end_) in zip(texts, features, start, end): # Normalize logits and spans to retrieve the answer start_ = np.exp(start_) / np.sum(np.exp(start_)) end_ = np.exp(end_) / np.sum(np.exp(end_)) - starts, ends, scores = self.decode(start_, end_, kwargs['topk'], kwargs['max_answer_len']) + # Mask padding and question + start_, end_ = start_ * np.abs(np.array(feature.p_mask) - 1), end_ * np.abs(np.array(feature.p_mask) - 1) + + # Mask CLS + start_[0] = end_[0] = 0 + + starts, ends, scores = self.decode(start_, end_, kwargs['topk'], kwargs['max_answer_len']) char_to_word = np.array(example.char_to_word_offset) # Convert the answer (tokens) back to the original text From 67a8be8e90a7fbd5e0bceff9f29fb89ccabb61be Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 10 Dec 2019 17:50:32 +0100 Subject: [PATCH 307/505] fix backward in tests --- transformers/tests/modeling_common_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/transformers/tests/modeling_common_test.py b/transformers/tests/modeling_common_test.py index ed6f950e25..cd4cf247a6 100644 --- a/transformers/tests/modeling_common_test.py +++ b/transformers/tests/modeling_common_test.py @@ -277,8 +277,7 @@ class CommonTestCases: inputs = inputs_dict.copy() inputs['head_mask'] = head_mask - with torch.no_grad(): - outputs = model(**inputs) + outputs = model(**inputs) # Test that we can get a gradient back for importance score computation output = sum(t.sum() for t in outputs[0]) From dc4e9e5cb36ae9bf5185b49b1cbc9106857abd54 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 10 Dec 2019 19:21:20 +0000 Subject: [PATCH 308/505] DataParallel for SQuAD + fix XLM --- examples/run_squad.py | 6 +++++- transformers/data/metrics/squad_metrics.py | 7 ++++++- transformers/tokenization_xlm.py | 4 ++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/examples/run_squad.py b/examples/run_squad.py index 2df29014ef..5e3f9663e2 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -299,10 +299,14 @@ def evaluate(args, model, tokenizer, prefix=""): # XLNet and XLM use a more complex post-processing procedure if args.model_type in ['xlnet', 'xlm']: + + start_n_top = model.config.start_n_top if hasattr(model, "config") else model.module.config.start_n_top + end_n_top = model.config.end_n_top if hasattr(model, "config") else model.module.config.end_n_top + predictions = compute_predictions_log_probs(examples, features, all_results, args.n_best_size, args.max_answer_length, output_prediction_file, output_nbest_file, output_null_log_odds_file, - model.config.start_n_top, model.config.end_n_top, + start_n_top, end_n_top, args.version_2_with_negative, tokenizer, args.verbose_logging) else: predictions = compute_predictions_logits(examples, features, all_results, args.n_best_size, diff --git a/transformers/data/metrics/squad_metrics.py b/transformers/data/metrics/squad_metrics.py index 0755c0ab7a..7b03255f49 100644 --- a/transformers/data/metrics/squad_metrics.py +++ b/transformers/data/metrics/squad_metrics.py @@ -695,7 +695,12 @@ def compute_predictions_log_probs( tok_text = " ".join(tok_text.split()) orig_text = " ".join(orig_tokens) - final_text = get_final_text(tok_text, orig_text, tokenizer.do_lower_case, + if hasattr(tokenizer, "do_lower_case"): + do_lower_case = tokenizer.do_lower_case + else: + do_lower_case = tokenizer.do_lowercase_and_remove_accent + + final_text = get_final_text(tok_text, orig_text, do_lower_case, verbose_logging) if final_text in seen_predictions: diff --git a/transformers/tokenization_xlm.py b/transformers/tokenization_xlm.py index 6c9f8e5e5c..8def80bec4 100644 --- a/transformers/tokenization_xlm.py +++ b/transformers/tokenization_xlm.py @@ -549,6 +549,10 @@ class XLMTokenizer(PreTrainedTokenizer): additional_special_tokens=additional_special_tokens, **kwargs) + + self.max_len_single_sentence = self.max_len - 2 # take into account special tokens + self.max_len_sentences_pair = self.max_len - 3 # take into account special tokens + # cache of sm.MosesPunctNormalizer instance self.cache_moses_punct_normalizer = dict() # cache of sm.MosesTokenizer instance From 6a73382706ce3c6905023872f63a680f0eb419a4 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Tue, 10 Dec 2019 14:33:24 -0500 Subject: [PATCH 309/505] Complete warning + cleanup --- examples/run_squad.py | 1 - transformers/tokenization_utils.py | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/examples/run_squad.py b/examples/run_squad.py index 5e3f9663e2..79c8537a4b 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -299,7 +299,6 @@ def evaluate(args, model, tokenizer, prefix=""): # XLNet and XLM use a more complex post-processing procedure if args.model_type in ['xlnet', 'xlm']: - start_n_top = model.config.start_n_top if hasattr(model, "config") else model.module.config.start_n_top end_n_top = model.config.end_n_top if hasattr(model, "config") else model.module.config.end_n_top diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index f4395cd82c..cb931b0eaf 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -226,7 +226,7 @@ class PreTrainedTokenizer(object): self.max_len = max_len if max_len is not None else int(1e12) - # Padding side is right by default and over-riden in subclsses. If specified in the kwargs, it is changed. + # Padding side is right by default and over-riden in subclasses. If specified in the kwargs, it is changed. self.padding_side = kwargs.pop('padding_side', self.padding_side) # Added tokens @@ -1003,7 +1003,7 @@ class PreTrainedTokenizer(object): ) if pad_to_max_length and max_length is None and self.max_len > 10000: - logger.warning("Sequence can't be padded as the maximum ") + logger.warning("Sequence can't be padded as no maximum length is specified and the model maximum length is too high.") if needs_to_be_padded: difference = (max_length if max_length is not None else self.max_len) - len(encoded_inputs["input_ids"]) From 58d75aa310e872723ba92ee1f0cb575ae9e2eaef Mon Sep 17 00:00:00 2001 From: Leo Dirac Date: Tue, 10 Dec 2019 11:36:56 -0800 Subject: [PATCH 310/505] Progress indicator improvements when downloading pre-trained models. --- transformers/file_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transformers/file_utils.py b/transformers/file_utils.py index 24abd60781..68de4e6e2f 100644 --- a/transformers/file_utils.py +++ b/transformers/file_utils.py @@ -21,7 +21,7 @@ import boto3 from botocore.config import Config from botocore.exceptions import ClientError import requests -from tqdm import tqdm +from tqdm.auto import tqdm from contextlib import contextmanager logger = logging.getLogger(__name__) # pylint: disable=invalid-name @@ -245,7 +245,7 @@ def http_get(url, temp_file, proxies=None, resume_size=0): return content_length = response.headers.get('Content-Length') total = resume_size + int(content_length) if content_length is not None else None - progress = tqdm(unit="B", total=total, initial=resume_size) + progress = tqdm(unit="B", unit_scale=True, total=total, initial=resume_size, desc="Downloading") for chunk in response.iter_content(chunk_size=1024): if chunk: # filter out keep-alive new chunks progress.update(len(chunk)) From 9a24e0cf767601858f13808d37b5b71787b7641e Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Wed, 11 Dec 2019 00:33:25 +0100 Subject: [PATCH 311/505] Refactored qa pipeline argument handling + unittests --- transformers/pipelines.py | 87 ++++++++++++++++++---------- transformers/tests/pipelines_test.py | 33 ++++++++++- 2 files changed, 87 insertions(+), 33 deletions(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index eec4932321..da8b0b65a7 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -98,14 +98,11 @@ class TextClassificationPipeline(Pipeline): class QuestionAnsweringPipeline(Pipeline): """ Question Answering pipeling involving Tokenization and Inference. - TODO: - - top-k answers - - return start/end chars - - return score """ - def __init__(self, model, tokenizer: Optional[PreTrainedTokenizer]): - super().__init__(model, tokenizer) + @classmethod + def from_config(cls, model, tokenizer: PreTrainedTokenizer, **kwargs): + pass @staticmethod def create_sample(question: Union[str, List[str]], context: Union[str, List[str]]) -> Union[SquadExample, List[SquadExample]]: @@ -116,6 +113,55 @@ class QuestionAnsweringPipeline(Pipeline): else: return SquadExample(None, question, context, None, None, None) + @staticmethod + def handle_args(*inputs, **kwargs) -> List[SquadExample]: + # Position args, handling is sensibly the same as X and data, so forwarding to avoid duplicating + if inputs is not None and len(inputs) > 1: + kwargs['X'] = inputs + + # Generic compatibility with sklearn and Keras + # Batched data + if 'X' in kwargs or 'data' in kwargs: + data = kwargs['X'] if 'X' in kwargs else kwargs['data'] + + if not isinstance(data, list): + data = [data] + + for i, item in enumerate(data): + if isinstance(item, dict): + if any(k not in item for k in ['question', 'context']): + raise KeyError('You need to provide a dictionary with keys {question:..., context:...}') + data[i] = QuestionAnsweringPipeline.create_sample(**item) + + elif isinstance(item, SquadExample): + continue + else: + raise ValueError( + '{} argument needs to be of type (list[SquadExample | dict], SquadExample, dict)' + .format('X' if 'X' in kwargs else 'data') + ) + inputs = data + + # Tabular input + elif 'question' in kwargs and 'context' in kwargs: + if isinstance(kwargs['question'], str): + kwargs['question'] = [kwargs['question']] + + if isinstance(kwargs['context'], str): + kwargs['context'] = [kwargs['context']] + + inputs = [QuestionAnsweringPipeline.create_sample(q, c) for q, c in zip(kwargs['question'], kwargs['context'])] + else: + raise ValueError('Unknown arguments {}'.format(kwargs)) + + if not isinstance(inputs, list): + inputs = [inputs] + + return inputs + + def __init__(self, model, tokenizer: Optional[PreTrainedTokenizer]): + super().__init__(model, tokenizer) + def inputs_for_model(self, features: Union[SquadExample, List[SquadExample]]) -> Dict: args = ['input_ids', 'attention_mask'] model_type = type(self.model).__name__.lower() @@ -131,10 +177,6 @@ class QuestionAnsweringPipeline(Pipeline): else: return {k: [feature.__dict__[k] for feature in features] for k in args} - @classmethod - def from_config(cls, model, tokenizer: PreTrainedTokenizer, **kwargs): - pass - def __call__(self, *texts, **kwargs): # Set defaults values kwargs.setdefault('topk', 1) @@ -149,29 +191,10 @@ class QuestionAnsweringPipeline(Pipeline): if kwargs['max_answer_len'] < 1: raise ValueError('max_answer_len parameter should be >= 1 (got {})'.format(kwargs['max_answer_len'])) - # Position args - if texts is not None and len(texts) > 1: - (texts, ) = texts - - # Generic compatibility with sklearn and Keras - elif 'X' in kwargs and not texts: - texts = kwargs.pop('X') - - # Batched data - elif 'data' in kwargs: - texts = kwargs.pop('data') - - # Tabular input - elif 'question' in kwargs and 'context' in kwargs: - texts = QuestionAnsweringPipeline.create_sample(kwargs['question'], kwargs['context']) - else: - raise ValueError('Unknown arguments {}'.format(kwargs)) - - if not isinstance(texts, list): - texts = [texts] + examples = QuestionAnsweringPipeline.handle_args(texts, **kwargs) # Convert inputs to features - features = squad_convert_examples_to_features(texts, self.tokenizer, kwargs['max_seq_len'], kwargs['doc_stride'], kwargs['max_question_len'], False) + features = squad_convert_examples_to_features(examples, self.tokenizer, kwargs['max_seq_len'], kwargs['doc_stride'], kwargs['max_question_len'], False) fw_args = self.inputs_for_model(features) if is_tf_available(): @@ -188,7 +211,7 @@ class QuestionAnsweringPipeline(Pipeline): start, end = start.cpu().numpy(), end.cpu().numpy() answers = [] - for (example, feature, start_, end_) in zip(texts, features, start, end): + for (example, feature, start_, end_) in zip(examples, features, start, end): # Normalize logits and spans to retrieve the answer start_ = np.exp(start_) / np.sum(np.exp(start_)) end_ = np.exp(end_) / np.sum(np.exp(end_)) diff --git a/transformers/tests/pipelines_test.py b/transformers/tests/pipelines_test.py index 36d6e013c3..ee10234269 100644 --- a/transformers/tests/pipelines_test.py +++ b/transformers/tests/pipelines_test.py @@ -40,7 +40,38 @@ class QuestionAnsweringPipelineTest(unittest.TestCase): # Batch case with topk = 2 a = nlp(question=['What is the name of the company I\'m working for ?', 'Where is the company based ?'], - context=['I\'m working for Huggingface.', 'The company is based in New York and Paris'], topk=2) + context=['Where is the company based ?', 'The company is based in New York and Paris'], topk=2) + self.check_answer_structure(a, 2, 2) + + # check for data keyword + a = nlp(data=nlp.create_sample(question='What is the name of the company I\'m working for ?', context='I\'m working for Huggingface.')) + self.check_answer_structure(a, 1, 1) + + a = nlp(data=nlp.create_sample(question='What is the name of the company I\'m working for ?', context='I\'m working for Huggingface.'), topk=2) + self.check_answer_structure(a, 1, 2) + + a = nlp(data=[ + nlp.create_sample(question='What is the name of the company I\'m working for ?', context='I\'m working for Huggingface.'), + nlp.create_sample(question='I\'m working for Huggingface.', context='The company is based in New York and Paris'), + ]) + self.check_answer_structure(a, 2, 1) + + a = nlp(data=[ + {'question': 'What is the name of the company I\'m working for ?', 'context': 'I\'m working for Huggingface.'}, + {'question': 'Where is the company based ?', 'context': 'The company is based in New York and Paris'}, + ]) + self.check_answer_structure(a, 2, 1) + + # X keywords + a = nlp(X=nlp.create_sample( + question='Where is the company based ?', context='The company is based in New York and Paris' + )) + self.check_answer_structure(a, 1, 1) + + a = nlp(X=[ + {'question': 'What is the name of the company I\'m working for ?', 'context': 'I\'m working for Huggingface.'}, + {'question': 'Where is the company based ?', 'context': 'The company is based in New York and Paris'}, + ], topk=2) self.check_answer_structure(a, 2, 2) @patch('transformers.pipelines.is_torch_available', return_value=False) From fafd4c86ecb63bb90b095bbd23453553e33fe99d Mon Sep 17 00:00:00 2001 From: thomwolf Date: Wed, 11 Dec 2019 13:47:27 +0100 Subject: [PATCH 312/505] fix TF 2.0 version of T5 - update conversion script --- .../convert_pytorch_checkpoint_to_tf2.py | 11 ++--- transformers/file_utils.py | 3 ++ transformers/modeling_t5.py | 21 +++++++-- transformers/modeling_tf_t5.py | 43 ++++++++++++------- transformers/modeling_tf_utils.py | 6 +-- transformers/modeling_utils.py | 12 +++++- 6 files changed, 65 insertions(+), 31 deletions(-) diff --git a/transformers/convert_pytorch_checkpoint_to_tf2.py b/transformers/convert_pytorch_checkpoint_to_tf2.py index 76d75b43e4..4a9832f123 100644 --- a/transformers/convert_pytorch_checkpoint_to_tf2.py +++ b/transformers/convert_pytorch_checkpoint_to_tf2.py @@ -120,24 +120,21 @@ def convert_pt_checkpoint_to_tf(model_type, pytorch_checkpoint_path, config_file tf_model = load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path) if compare_with_pt_model: - inputs_list = [[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]] - tf_inputs = tf_model.dummy_inputs - tfo = tf_model(tf_inputs, training=False) # build the network + tfo = tf_model(tf_model.dummy_inputs, training=False) # build the network state_dict = torch.load(pytorch_checkpoint_path, map_location='cpu') pt_model = pt_model_class.from_pretrained(pretrained_model_name_or_path=None, config=config, state_dict=state_dict) - pt_inputs = torch.tensor(inputs_list) with torch.no_grad(): - pto = pt_model(pt_inputs) + pto = pt_model(**pt_model.dummy_inputs) - np_pt = pto[0].detach().numpy() + np_pt = pto[0].numpy() np_tf = tfo[0].numpy() diff = np.amax(np.abs(np_pt - np_tf)) print("Max absolute difference between models outputs {}".format(diff)) - assert diff <= 2e-2, "Error, model absolute difference is >2e-2" + assert diff <= 2e-2, "Error, model absolute difference is >2e-2: {}".format(diff) # Save pytorch-model print("Save TensorFlow model to {}".format(tf_dump_path)) diff --git a/transformers/file_utils.py b/transformers/file_utils.py index 24abd60781..e36bbf4eeb 100644 --- a/transformers/file_utils.py +++ b/transformers/file_utils.py @@ -73,6 +73,9 @@ TF2_WEIGHTS_NAME = 'tf_model.h5' TF_WEIGHTS_NAME = 'model.ckpt' CONFIG_NAME = "config.json" +DUMMY_INPUTS = [[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]] +DUMMY_MASK = [[1, 1, 1, 1, 1], [1, 1, 1, 0, 0], [0, 0, 0, 1, 1]] + def is_torch_available(): return _torch_available diff --git a/transformers/modeling_t5.py b/transformers/modeling_t5.py index ffc4d8bb3f..149b977abc 100644 --- a/transformers/modeling_t5.py +++ b/transformers/modeling_t5.py @@ -32,7 +32,7 @@ from torch.nn import CrossEntropyLoss, MSELoss from .modeling_utils import PreTrainedModel from .configuration_t5 import T5Config -from .file_utils import add_start_docstrings +from .file_utils import add_start_docstrings, DUMMY_INPUTS, DUMMY_MASK logger = logging.getLogger(__name__) @@ -451,6 +451,15 @@ class T5PreTrainedModel(PreTrainedModel): load_tf_weights = load_tf_weights_in_t5 base_model_prefix = "transformer" + @property + def dummy_inputs(self): + input_ids = torch.tensor(DUMMY_INPUTS) + input_mask = torch.tensor(DUMMY_MASK) + dummy_inputs = {'decoder_input_ids': input_ids, + 'encoder_input_ids': input_ids, + 'decoder_attention_mask': input_mask} + return dummy_inputs + def _init_weights(self, module): """ Initialize the weights """ factor = self.config.initializer_factor # Used for testing weights initialization @@ -534,9 +543,10 @@ class T5Stack(T5PreTrainedModel): # Since we are adding it to the raw scores before the softmax, this is # effectively the same as removing these entirely. - # T5 has a mask that can compare sequence ids, we simulate this here with this transposistion + # T5 has a mask that can compare sequence ids, we can simulate this here with this transposition # Cf. https://github.com/tensorflow/mesh/blob/8d2465e9bc93129b913b5ccc6a59aa97abd96ec6/mesh_tensorflow/transformer/transformer_layers.py#L270 - extended_attention_mask = (extended_attention_mask == extended_attention_mask.transpose(-1, -2)) + # extended_attention_mask = (extended_attention_mask == extended_attention_mask.transpose(-1, -2)) + extended_attention_mask = extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility extended_attention_mask = (1.0 - extended_attention_mask) * -1e9 @@ -548,6 +558,10 @@ class T5Stack(T5PreTrainedModel): if encoder_attention_mask.dim() == 2: encoder_extended_attention_mask = encoder_attention_mask[:, None, None, :] + # T5 has a mask that can compare sequence ids, we can simulate this here with this transposition + # Cf. https://github.com/tensorflow/mesh/blob/8d2465e9bc93129b913b5ccc6a59aa97abd96ec6/mesh_tensorflow/transformer/transformer_layers.py#L270 + # encoder_extended_attention_mask = (encoder_extended_attention_mask == encoder_extended_attention_mask.transpose(-1, -2)) + encoder_extended_attention_mask = encoder_extended_attention_mask.to(dtype=next(self.parameters()).dtype) # fp16 compatibility encoder_extended_attention_mask = (1.0 - encoder_extended_attention_mask) * -1e9 else: @@ -590,6 +604,7 @@ class T5Stack(T5PreTrainedModel): hidden_states = layer_outputs[0] if i == 0: # We share the position biases between the layers - the first layer store them + # layer_outputs = hidden-states, (self-attention weights), (self-attention position bias), (cross-attention weights), (cross-attention position bias) position_bias = layer_outputs[2 if self.output_attentions else 1] if self.is_decoder: encoder_decoder_position_bias = layer_outputs[4 if self.output_attentions else 2] diff --git a/transformers/modeling_tf_t5.py b/transformers/modeling_tf_t5.py index 0b3b1116f2..fd25328ac6 100644 --- a/transformers/modeling_tf_t5.py +++ b/transformers/modeling_tf_t5.py @@ -26,7 +26,7 @@ import tensorflow as tf from .configuration_t5 import T5Config from .modeling_tf_utils import TFPreTrainedModel, TFSharedEmbeddings, shape_list -from .file_utils import add_start_docstrings +from .file_utils import add_start_docstrings, DUMMY_INPUTS, DUMMY_MASK logger = logging.getLogger(__name__) @@ -61,7 +61,7 @@ class TFT5LayerNorm(tf.keras.layers.Layer): super(TFT5LayerNorm, self).build(input_shape) def call(self, x): - variance = tf.math.reduce_min(tf.math.square(x), axis=-1, keepdims=True) + variance = tf.math.reduce_mean(tf.math.square(x), axis=-1, keepdims=True) x = x * tf.math.rsqrt(variance + self.variance_epsilon) return self.weight * x @@ -231,19 +231,19 @@ class TFT5Attention(tf.keras.layers.Layer): cache[self.layer_id] = (k, v) # q = q / math.sqrt(dim_per_head) # No scaling in T5 - scores = tf.matmul(q, k, transpose_b=True) # (bs, n_heads, qlen, klen) + # scores = tf.matmul(q, k, transpose_b=True) # (bs, n_heads, qlen, klen) + scores = tf.einsum('bnqd,bnkd->bnqk', q, k) # (bs, n_heads, qlen, klen) if position_bias is None: if not self.has_relative_attention_bias: raise ValueError("No position_bias provided and no weights to compute position_bias") position_bias = self.compute_bias(qlen, klen) + if mask is not None: + position_bias = position_bias + mask + # mask = (mask == 0).expand_as(scores) # (bs, n_heads, qlen, klen) + # scores.masked_fill_(mask, -float('inf')) # (bs, n_heads, qlen, klen) + scores += position_bias - - if mask is not None: - scores += mask - # mask = (mask == 0).expand_as(scores) # (bs, n_heads, qlen, klen) - # scores.masked_fill_(mask, -float('inf')) # (bs, n_heads, qlen, klen) - weights = tf.nn.softmax(scores, axis=-1) # (bs, n_heads, qlen, klen) weights = self.dropout(weights, training=training) # (bs, n_heads, qlen, klen) @@ -350,11 +350,11 @@ class TFT5Block(tf.keras.layers.Layer): head_mask=head_mask, training=training) hidden_states = cross_attention_outputs[0] - outputs = cross_attention_outputs[1:] + outputs + outputs = outputs + cross_attention_outputs[1:] hidden_states = self.layer[2](hidden_states, training=training) outputs = (hidden_states,) + outputs # add attentions if we output them - return outputs + return outputs # hidden-states, (self-attention weights), (self-attention position bias), (cross-attention weights), (cross-attention position bias) #################################################### @@ -418,7 +418,13 @@ class TFT5MainLayer(tf.keras.layers.Layer): # positions we want to attend and -10000.0 for masked positions. # Since we are adding it to the raw scores before the softmax, this is # effectively the same as removing these entirely. - extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + + # T5 has a mask that can compare sequence ids, we can simulate this here with this transposistion + # Cf. https://github.com/tensorflow/mesh/blob/8d2465e9bc93129b913b5ccc6a59aa97abd96ec6/mesh_tensorflow/transformer/transformer_layers.py#L270 + # extended_attention_mask = tf.math.equal(extended_attention_mask, + # tf.transpose(extended_attention_mask, perm=(-1, -2))) + + extended_attention_mask = (1.0 - extended_attention_mask) * -1e9 if self.is_decoder: # If a 2D ou 3D attention mask is provided for the cross-attention @@ -430,7 +436,12 @@ class TFT5MainLayer(tf.keras.layers.Layer): if num_dims_encoder_attention_mask == 2: encoder_extended_attention_mask = encoder_attention_mask[:, None, None, :] - encoder_extended_attention_mask = (1.0 - encoder_extended_attention_mask) * -10000.0 + # T5 has a mask that can compare sequence ids, we can simulate this here with this transposistion + # Cf. https://github.com/tensorflow/mesh/blob/8d2465e9bc93129b913b5ccc6a59aa97abd96ec6/mesh_tensorflow/transformer/transformer_layers.py#L270 + # encoder_extended_attention_mask = tf.math.equal(encoder_extended_attention_mask, + # tf.transpose(encoder_extended_attention_mask, perm=(-1, -2))) + + encoder_extended_attention_mask = (1.0 - encoder_extended_attention_mask) * -1e9 else: encoder_extended_attention_mask = None @@ -463,6 +474,8 @@ class TFT5MainLayer(tf.keras.layers.Layer): training=training) hidden_states = layer_outputs[0] if i == 0: + # We share the position biases between the layers - the first layer store them + # layer_outputs = hidden-states, (self-attention weights), (self-attention position bias), (cross-attention weights), (cross-attention position bias) position_bias = layer_outputs[2 if self.output_attentions else 1] if self.is_decoder: encoder_decoder_position_bias = layer_outputs[4 if self.output_attentions else 2] @@ -502,8 +515,8 @@ class TFT5PreTrainedModel(TFPreTrainedModel): @property def dummy_inputs(self): - input_ids = tf.constant([[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]]) - input_mask = tf.constant([[1, 1, 0, 0, 1], [1, 1, 1, 0, 0], [1, 0, 0, 1, 1]]) + input_ids = tf.constant(DUMMY_INPUTS) + input_mask = tf.constant(DUMMY_MASK) dummy_inputs = {'decoder_input_ids': input_ids, 'encoder_input_ids': input_ids, 'decoder_attention_mask': input_mask} diff --git a/transformers/modeling_tf_utils.py b/transformers/modeling_tf_utils.py index ed8fdb74c9..8d010e589e 100644 --- a/transformers/modeling_tf_utils.py +++ b/transformers/modeling_tf_utils.py @@ -24,13 +24,11 @@ import os import tensorflow as tf from .configuration_utils import PretrainedConfig -from .file_utils import cached_path, WEIGHTS_NAME, TF_WEIGHTS_NAME, TF2_WEIGHTS_NAME +from .file_utils import cached_path, WEIGHTS_NAME, TF_WEIGHTS_NAME, TF2_WEIGHTS_NAME, DUMMY_INPUTS from .modeling_tf_pytorch_utils import load_pytorch_checkpoint_in_tf2_model logger = logging.getLogger(__name__) -DUMMY_INPUTS = [[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]] - class TFPreTrainedModel(tf.keras.Model): r""" Base class for all TF models. @@ -59,7 +57,7 @@ class TFPreTrainedModel(tf.keras.Model): Returns: tf.Tensor with dummy inputs """ - return tf.constant(DUMMY_INPUTS) + return {'input_ids': tf.constant(DUMMY_INPUTS)} def __init__(self, config, *inputs, **kwargs): super(TFPreTrainedModel, self).__init__(*inputs, **kwargs) diff --git a/transformers/modeling_utils.py b/transformers/modeling_utils.py index aa0e0e6191..ae515d6870 100644 --- a/transformers/modeling_utils.py +++ b/transformers/modeling_utils.py @@ -31,11 +31,10 @@ from torch.nn import CrossEntropyLoss from torch.nn import functional as F from .configuration_utils import PretrainedConfig -from .file_utils import cached_path, WEIGHTS_NAME, TF_WEIGHTS_NAME, TF2_WEIGHTS_NAME +from .file_utils import cached_path, WEIGHTS_NAME, TF_WEIGHTS_NAME, TF2_WEIGHTS_NAME, DUMMY_INPUTS logger = logging.getLogger(__name__) - try: from torch.nn import Identity except ImportError: @@ -71,6 +70,15 @@ class PreTrainedModel(nn.Module): load_tf_weights = lambda model, config, path: None base_model_prefix = "" + @property + def dummy_inputs(self): + """ Dummy inputs to do a forward pass in the network. + + Returns: + torch.Tensor with dummy inputs + """ + return {'input_ids': torch.tensor(DUMMY_INPUTS)} + def __init__(self, config, *inputs, **kwargs): super(PreTrainedModel, self).__init__() if not isinstance(config, PretrainedConfig): From b040bff6df09923870f44fb5402e895d57327e85 Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Wed, 11 Dec 2019 14:13:58 +0100 Subject: [PATCH 313/505] Added supported model to AutoModelTokenClassification --- transformers/__init__.py | 4 +- transformers/modeling_auto.py | 123 ++++++++++++++++++++++++++++++- transformers/modeling_tf_auto.py | 111 +++++++++++++++++++++++++++- 3 files changed, 231 insertions(+), 7 deletions(-) diff --git a/transformers/__init__.py b/transformers/__init__.py index 4300409257..c474696062 100755 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -69,7 +69,7 @@ from .configuration_camembert import CamembertConfig, CAMEMBERT_PRETRAINED_CONFI if is_torch_available(): from .modeling_utils import (PreTrainedModel, prune_layer, Conv1D) from .modeling_auto import (AutoModel, AutoModelForSequenceClassification, AutoModelForQuestionAnswering, - AutoModelWithLMHead) + AutoModelWithLMHead, AutoModelForTokenClassification) from .modeling_bert import (BertPreTrainedModel, BertModel, BertForPreTraining, BertForMaskedLM, BertForNextSentencePrediction, @@ -124,7 +124,7 @@ if is_torch_available(): if is_tf_available(): from .modeling_tf_utils import TFPreTrainedModel, TFSharedEmbeddings, TFSequenceSummary, shape_list from .modeling_tf_auto import (TFAutoModel, TFAutoModelForSequenceClassification, TFAutoModelForQuestionAnswering, - TFAutoModelWithLMHead) + TFAutoModelWithLMHead, TFAutoModelForTokenClassification) from .modeling_tf_bert import (TFBertPreTrainedModel, TFBertMainLayer, TFBertEmbeddings, TFBertModel, TFBertForPreTraining, diff --git a/transformers/modeling_auto.py b/transformers/modeling_auto.py index 041115cc61..cb877643ab 100644 --- a/transformers/modeling_auto.py +++ b/transformers/modeling_auto.py @@ -22,16 +22,19 @@ from .configuration_auto import (AlbertConfig, BertConfig, CamembertConfig, CTRL DistilBertConfig, GPT2Config, OpenAIGPTConfig, RobertaConfig, TransfoXLConfig, XLMConfig, XLNetConfig) -from .modeling_bert import BertModel, BertForMaskedLM, BertForSequenceClassification, BertForQuestionAnswering +from .modeling_bert import BertModel, BertForMaskedLM, BertForSequenceClassification, BertForQuestionAnswering, \ + BertForTokenClassification from .modeling_openai import OpenAIGPTModel, OpenAIGPTLMHeadModel from .modeling_gpt2 import GPT2Model, GPT2LMHeadModel from .modeling_ctrl import CTRLModel, CTRLLMHeadModel from .modeling_transfo_xl import TransfoXLModel, TransfoXLLMHeadModel -from .modeling_xlnet import XLNetModel, XLNetLMHeadModel, XLNetForSequenceClassification, XLNetForQuestionAnswering +from .modeling_xlnet import XLNetModel, XLNetLMHeadModel, XLNetForSequenceClassification, XLNetForQuestionAnswering, \ + XLNetForTokenClassification from .modeling_xlm import XLMModel, XLMWithLMHeadModel, XLMForSequenceClassification, XLMForQuestionAnswering from .modeling_roberta import RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification from .modeling_distilbert import DistilBertModel, DistilBertForQuestionAnswering, DistilBertForMaskedLM, DistilBertForSequenceClassification -from .modeling_camembert import CamembertModel, CamembertForMaskedLM, CamembertForSequenceClassification, CamembertForMultipleChoice +from .modeling_camembert import CamembertModel, CamembertForMaskedLM, CamembertForSequenceClassification, \ + CamembertForMultipleChoice, CamembertForTokenClassification from .modeling_albert import AlbertModel, AlbertForMaskedLM, AlbertForSequenceClassification, AlbertForQuestionAnswering from .modeling_utils import PreTrainedModel, SequenceSummary @@ -699,3 +702,117 @@ class AutoModelForQuestionAnswering(object): raise ValueError("Unrecognized model identifier in {}. Should contains one of " "'bert', 'xlnet', 'xlm', 'distilbert', 'albert'".format(pretrained_model_name_or_path)) + + +class AutoModelForTokenClassification: + def __init__(self): + raise EnvironmentError("AutoModelForTokenClassification is designed to be instantiated " + "using the `AutoModelForTokenClassification.from_pretrained(pretrained_model_name_or_path)` or " + "`AutoModelForTokenClassification.from_config(config)` methods.") + + @classmethod + def from_config(cls, config): + r""" Instantiates one of the base model classes of the library + from a configuration. + + config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: + The model class to instantiate is selected based on the configuration class: + - isInstance of `distilbert` configuration class: DistilBertModel (DistilBERT model) + - isInstance of `bert` configuration class: BertModel (Bert model) + - isInstance of `xlnet` configuration class: XLNetModel (XLNet model) + - isInstance of `xlm` configuration class: XLMModel (XLM model) + + Examples:: + + config = BertConfig.from_pretrained('bert-base-uncased') # Download configuration from S3 and cache. + model = AutoModelForTokenClassification.from_config(config) # E.g. model was saved using `save_pretrained('./test/saved_model/')` + """ + if isinstance(config, CamembertConfig): + return CamembertForTokenClassification(config) + elif isinstance(config, BertConfig): + return BertForTokenClassification(config) + elif isinstance(config, XLNetConfig): + return XLNetForTokenClassification(config) + raise ValueError("Unrecognized configuration class {}".format(config)) + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): + r""" Instantiates one of the question answering model classes of the library + from a pre-trained model configuration. + + The `from_pretrained()` method takes care of returning the correct model class instance + using pattern matching on the `pretrained_model_name_or_path` string. + + The model class to instantiate is selected as the first pattern matching + in the `pretrained_model_name_or_path` string (in the following order): + - contains `distilbert`: DistilBertForTokenClassification (DistilBERT model) + - contains `albert`: AlbertForTokenClassification (ALBERT model) + - contains `bert`: BertForTokenClassification (Bert model) + - contains `xlnet`: XLNetForTokenClassification (XLNet model) + - contains `xlm`: XLMForTokenClassification (XLM model) + + The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated) + To train the model, you should first set it back in training mode with `model.train()` + + Params: + pretrained_model_name_or_path: either: + + - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. + - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. + + model_args: (`optional`) Sequence of positional arguments: + All remaning positional arguments will be passed to the underlying model's ``__init__`` method + + config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: + Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when: + + - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or + - the model was saved using :func:`~transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. + - the model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a configuration JSON file named `config.json` is found in the directory. + + state_dict: (`optional`) dict: + an optional state dictionnary for the model to use instead of a state dictionary loaded from saved weights file. + This option can be used if you want to create a model from a pretrained configuration but load your own weights. + In this case though, you should check if using :func:`~transformers.PreTrainedModel.save_pretrained` and :func:`~transformers.PreTrainedModel.from_pretrained` is not a simpler option. + + cache_dir: (`optional`) string: + Path to a directory in which a downloaded pre-trained model + configuration should be cached if the standard cache should not be used. + + force_download: (`optional`) boolean, default False: + Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + + proxies: (`optional`) dict, default None: + A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. + The proxies are used on each request. + + output_loading_info: (`optional`) boolean: + Set to ``True`` to also return a dictionnary containing missing keys, unexpected keys and error messages. + + kwargs: (`optional`) Remaining dictionary of keyword arguments: + Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded: + + - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done) + - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. + + Examples:: + + model = AutoModelForTokenClassification.from_pretrained('bert-base-uncased') # Download model and configuration from S3 and cache. + model = AutoModelForTokenClassification.from_pretrained('./test/bert_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')` + model = AutoModelForTokenClassification.from_pretrained('bert-base-uncased', output_attention=True) # Update configuration during loading + assert model.config.output_attention == True + # Loading from a TF checkpoint file instead of a PyTorch model (slower) + config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json') + model = AutoModelForTokenClassification.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config) + + """ + if 'camembert' in pretrained_model_name_or_path: + return CamembertForTokenClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'bert' in pretrained_model_name_or_path: + return BertForTokenClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'xlnet' in pretrained_model_name_or_path: + return XLNetForTokenClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + + raise ValueError("Unrecognized model identifier in {}. Should contains one of " + "'bert', 'xlnet', 'camembert'".format(pretrained_model_name_or_path)) diff --git a/transformers/modeling_tf_auto.py b/transformers/modeling_tf_auto.py index e78b91cfcc..1097f77a59 100644 --- a/transformers/modeling_tf_auto.py +++ b/transformers/modeling_tf_auto.py @@ -22,11 +22,13 @@ from .configuration_auto import (BertConfig, CTRLConfig, DistilBertConfig, GPT2Config, OpenAIGPTConfig, RobertaConfig, TransfoXLConfig, XLMConfig, XLNetConfig) -from .modeling_tf_bert import TFBertModel, TFBertForMaskedLM, TFBertForSequenceClassification, TFBertForQuestionAnswering +from .modeling_tf_bert import TFBertModel, TFBertForMaskedLM, TFBertForSequenceClassification, \ + TFBertForQuestionAnswering, TFBertForTokenClassification from .modeling_tf_openai import TFOpenAIGPTModel, TFOpenAIGPTLMHeadModel from .modeling_tf_gpt2 import TFGPT2Model, TFGPT2LMHeadModel from .modeling_tf_transfo_xl import TFTransfoXLModel, TFTransfoXLLMHeadModel -from .modeling_tf_xlnet import TFXLNetModel, TFXLNetLMHeadModel, TFXLNetForSequenceClassification, TFXLNetForQuestionAnsweringSimple +from .modeling_tf_xlnet import TFXLNetModel, TFXLNetLMHeadModel, TFXLNetForSequenceClassification, \ + TFXLNetForQuestionAnsweringSimple, TFXLNetForTokenClassification from .modeling_tf_xlm import TFXLMModel, TFXLMWithLMHeadModel, TFXLMForSequenceClassification, TFXLMForQuestionAnsweringSimple from .modeling_tf_roberta import TFRobertaModel, TFRobertaForMaskedLM, TFRobertaForSequenceClassification from .modeling_tf_distilbert import TFDistilBertModel, TFDistilBertForQuestionAnswering, TFDistilBertForMaskedLM, TFDistilBertForSequenceClassification @@ -668,3 +670,108 @@ class TFAutoModelForQuestionAnswering(object): raise ValueError("Unrecognized model identifier in {}. Should contains one of " "'distilbert', 'bert', 'xlnet', 'xlm'".format(pretrained_model_name_or_path)) + + +class TFAutoModelForTokenClassification: + def __init__(self): + raise EnvironmentError("TFAutoModelForTokenClassification is designed to be instantiated " + "using the `TFAutoModelForTokenClassification.from_pretrained(pretrained_model_name_or_path)` or " + "`AutoModelForTokenClassification.from_config(config)` methods.") + + @classmethod + def from_config(cls, config): + r""" Instantiates one of the base model classes of the library + from a configuration. + + config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: + The model class to instantiate is selected based on the configuration class: + - isInstance of `bert` configuration class: BertModel (Bert model) + - isInstance of `xlnet` configuration class: XLNetModel (XLNet model) + + Examples:: + + config = BertConfig.from_pretrained('bert-base-uncased') # Download configuration from S3 and cache. + model = TFAutoModelForTokenClassification.from_config(config) # E.g. model was saved using `save_pretrained('./test/saved_model/')` + """ + if isinstance(config, BertConfig): + return TFBertForTokenClassification(config) + elif isinstance(config, XLNetConfig): + return TFXLNetForTokenClassification(config) + raise ValueError("Unrecognized configuration class {}".format(config)) + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, *model_args, **kwargs): + r""" Instantiates one of the question answering model classes of the library + from a pre-trained model configuration. + + The `from_pretrained()` method takes care of returning the correct model class instance + using pattern matching on the `pretrained_model_name_or_path` string. + + The model class to instantiate is selected as the first pattern matching + in the `pretrained_model_name_or_path` string (in the following order): + - contains `bert`: BertForTokenClassification (Bert model) + - contains `xlnet`: XLNetForTokenClassification (XLNet model) + + The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated) + To train the model, you should first set it back in training mode with `model.train()` + + Params: + pretrained_model_name_or_path: either: + + - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. + - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. + + model_args: (`optional`) Sequence of positional arguments: + All remaning positional arguments will be passed to the underlying model's ``__init__`` method + + config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: + Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when: + + - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or + - the model was saved using :func:`~transformers.PreTrainedModel.save_pretrained` and is reloaded by suppling the save directory. + - the model is loaded by suppling a local directory as ``pretrained_model_name_or_path`` and a configuration JSON file named `config.json` is found in the directory. + + state_dict: (`optional`) dict: + an optional state dictionnary for the model to use instead of a state dictionary loaded from saved weights file. + This option can be used if you want to create a model from a pretrained configuration but load your own weights. + In this case though, you should check if using :func:`~transformers.PreTrainedModel.save_pretrained` and :func:`~transformers.PreTrainedModel.from_pretrained` is not a simpler option. + + cache_dir: (`optional`) string: + Path to a directory in which a downloaded pre-trained model + configuration should be cached if the standard cache should not be used. + + force_download: (`optional`) boolean, default False: + Force to (re-)download the model weights and configuration files and override the cached versions if they exists. + + proxies: (`optional`) dict, default None: + A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. + The proxies are used on each request. + + output_loading_info: (`optional`) boolean: + Set to ``True`` to also return a dictionnary containing missing keys, unexpected keys and error messages. + + kwargs: (`optional`) Remaining dictionary of keyword arguments: + Can be used to update the configuration object (after it being loaded) and initiate the model. (e.g. ``output_attention=True``). Behave differently depending on whether a `config` is provided or automatically loaded: + + - If a configuration is provided with ``config``, ``**kwargs`` will be directly passed to the underlying model's ``__init__`` method (we assume all relevant updates to the configuration have already been done) + - If a configuration is not provided, ``kwargs`` will be first passed to the configuration class initialization function (:func:`~transformers.PretrainedConfig.from_pretrained`). Each key of ``kwargs`` that corresponds to a configuration attribute will be used to override said attribute with the supplied ``kwargs`` value. Remaining keys that do not correspond to any configuration attribute will be passed to the underlying model's ``__init__`` function. + + Examples:: + + model = TFAutoModelForTokenClassification.from_pretrained('bert-base-uncased') # Download model and configuration from S3 and cache. + model = TFAutoModelForTokenClassification.from_pretrained('./test/bert_model/') # E.g. model was saved using `save_pretrained('./test/saved_model/')` + model = TFAutoModelForTokenClassification.from_pretrained('bert-base-uncased', output_attention=True) # Update configuration during loading + assert model.config.output_attention == True + # Loading from a TF checkpoint file instead of a PyTorch model (slower) + config = AutoConfig.from_json_file('./tf_model/bert_tf_model_config.json') + model = TFAutoModelForTokenClassification.from_pretrained('./tf_model/bert_tf_checkpoint.ckpt.index', from_tf=True, config=config) + + """ + if 'bert' in pretrained_model_name_or_path: + return TFBertForTokenClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'xlnet' in pretrained_model_name_or_path: + return TFXLNetForTokenClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + + raise ValueError("Unrecognized model identifier in {}. Should contains one of " + "'bert', 'xlnet'".format(pretrained_model_name_or_path)) From 4c12860f7ae61659aed2675498350a386fc4e122 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Wed, 11 Dec 2019 09:22:37 -0500 Subject: [PATCH 314/505] Remove misleading documentation --- transformers/tokenization_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index cb931b0eaf..68a767fe82 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -628,7 +628,6 @@ class PreTrainedTokenizer(object): Take care of added tokens. text: The sequence to be encoded. - return_tokens_mapped_to_origin: (optional) Set to True to return the index of each token in the initial whitespace tokenization. (default False). **kwargs: passed to the child `self.tokenize()` method """ def lowercase_text(t): From c28273793ec41c98153b23e29b9c7228c4149aae Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Wed, 11 Dec 2019 14:52:01 +0100 Subject: [PATCH 315/505] Add missing DistilBert and Roberta to AutoModelForTokenClassification --- transformers/modeling_auto.py | 25 ++++++++++++++++++------- transformers/modeling_tf_auto.py | 18 +++++++++++++++--- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/transformers/modeling_auto.py b/transformers/modeling_auto.py index cb877643ab..c76e5b78b3 100644 --- a/transformers/modeling_auto.py +++ b/transformers/modeling_auto.py @@ -31,8 +31,10 @@ from .modeling_transfo_xl import TransfoXLModel, TransfoXLLMHeadModel from .modeling_xlnet import XLNetModel, XLNetLMHeadModel, XLNetForSequenceClassification, XLNetForQuestionAnswering, \ XLNetForTokenClassification from .modeling_xlm import XLMModel, XLMWithLMHeadModel, XLMForSequenceClassification, XLMForQuestionAnswering -from .modeling_roberta import RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification -from .modeling_distilbert import DistilBertModel, DistilBertForQuestionAnswering, DistilBertForMaskedLM, DistilBertForSequenceClassification +from .modeling_roberta import RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification, \ + RobertaForTokenClassification +from .modeling_distilbert import DistilBertModel, DistilBertForQuestionAnswering, DistilBertForMaskedLM, \ + DistilBertForSequenceClassification, DistilBertForTokenClassification from .modeling_camembert import CamembertModel, CamembertForMaskedLM, CamembertForSequenceClassification, \ CamembertForMultipleChoice, CamembertForTokenClassification from .modeling_albert import AlbertModel, AlbertForMaskedLM, AlbertForSequenceClassification, AlbertForQuestionAnswering @@ -720,8 +722,9 @@ class AutoModelForTokenClassification: - isInstance of `distilbert` configuration class: DistilBertModel (DistilBERT model) - isInstance of `bert` configuration class: BertModel (Bert model) - isInstance of `xlnet` configuration class: XLNetModel (XLNet model) - - isInstance of `xlm` configuration class: XLMModel (XLM model) - + - isInstance of `camembert` configuration class: CamembertModel (Camembert model) + - isInstance of `roberta` configuration class: RobertaModel (Roberta model) + Examples:: config = BertConfig.from_pretrained('bert-base-uncased') # Download configuration from S3 and cache. @@ -729,10 +732,14 @@ class AutoModelForTokenClassification: """ if isinstance(config, CamembertConfig): return CamembertForTokenClassification(config) + elif isinstance(config, DistilBertConfig): + return DistilBertForTokenClassification(config) elif isinstance(config, BertConfig): return BertForTokenClassification(config) elif isinstance(config, XLNetConfig): return XLNetForTokenClassification(config) + elif isinstance(config, RobertaConfig): + return RobertaForTokenClassification(config) raise ValueError("Unrecognized configuration class {}".format(config)) @classmethod @@ -746,10 +753,10 @@ class AutoModelForTokenClassification: The model class to instantiate is selected as the first pattern matching in the `pretrained_model_name_or_path` string (in the following order): - contains `distilbert`: DistilBertForTokenClassification (DistilBERT model) - - contains `albert`: AlbertForTokenClassification (ALBERT model) + - contains `camembert`: CamembertForTokenClassification (Camembert model) - contains `bert`: BertForTokenClassification (Bert model) - contains `xlnet`: XLNetForTokenClassification (XLNet model) - - contains `xlm`: XLMForTokenClassification (XLM model) + - contains `roberta`: RobertaForTokenClassification (Roberta model) The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated) To train the model, you should first set it back in training mode with `model.train()` @@ -809,10 +816,14 @@ class AutoModelForTokenClassification: """ if 'camembert' in pretrained_model_name_or_path: return CamembertForTokenClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'distilbert' in pretrained_model_name_or_path: + return DistilBertForTokenClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'bert' in pretrained_model_name_or_path: return BertForTokenClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'roberta' in pretrained_model_name_or_path: + return RobertaForTokenClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'xlnet' in pretrained_model_name_or_path: return XLNetForTokenClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " - "'bert', 'xlnet', 'camembert'".format(pretrained_model_name_or_path)) + "'bert', 'xlnet', 'camembert', 'distilbert', 'roberta'".format(pretrained_model_name_or_path)) diff --git a/transformers/modeling_tf_auto.py b/transformers/modeling_tf_auto.py index 1097f77a59..add7e03341 100644 --- a/transformers/modeling_tf_auto.py +++ b/transformers/modeling_tf_auto.py @@ -30,8 +30,8 @@ from .modeling_tf_transfo_xl import TFTransfoXLModel, TFTransfoXLLMHeadModel from .modeling_tf_xlnet import TFXLNetModel, TFXLNetLMHeadModel, TFXLNetForSequenceClassification, \ TFXLNetForQuestionAnsweringSimple, TFXLNetForTokenClassification from .modeling_tf_xlm import TFXLMModel, TFXLMWithLMHeadModel, TFXLMForSequenceClassification, TFXLMForQuestionAnsweringSimple -from .modeling_tf_roberta import TFRobertaModel, TFRobertaForMaskedLM, TFRobertaForSequenceClassification -from .modeling_tf_distilbert import TFDistilBertModel, TFDistilBertForQuestionAnswering, TFDistilBertForMaskedLM, TFDistilBertForSequenceClassification +from .modeling_tf_roberta import TFRobertaModel, TFRobertaForMaskedLM, TFRobertaForSequenceClassification, TFRobertaForTokenClassification +from .modeling_tf_distilbert import TFDistilBertModel, TFDistilBertForQuestionAnswering, TFDistilBertForMaskedLM, TFDistilBertForSequenceClassification, TFDistilBertForTokenClassification from .modeling_tf_ctrl import TFCTRLModel, TFCTRLLMHeadModel from .file_utils import add_start_docstrings @@ -687,6 +687,8 @@ class TFAutoModelForTokenClassification: The model class to instantiate is selected based on the configuration class: - isInstance of `bert` configuration class: BertModel (Bert model) - isInstance of `xlnet` configuration class: XLNetModel (XLNet model) + - isInstance of `distilbert` configuration class: DistilBertModel (DistilBert model) + - isInstance of `roberta` configuration class: RobteraModel (Roberta model) Examples:: @@ -697,6 +699,10 @@ class TFAutoModelForTokenClassification: return TFBertForTokenClassification(config) elif isinstance(config, XLNetConfig): return TFXLNetForTokenClassification(config) + elif isinstance(config, DistilBertConfig): + return TFDistilBertForTokenClassification(config) + elif isinstance(config, RobertaConfig): + return TFRobertaForTokenClassification(config) raise ValueError("Unrecognized configuration class {}".format(config)) @classmethod @@ -711,6 +717,8 @@ class TFAutoModelForTokenClassification: in the `pretrained_model_name_or_path` string (in the following order): - contains `bert`: BertForTokenClassification (Bert model) - contains `xlnet`: XLNetForTokenClassification (XLNet model) + - contains `distilbert`: DistilBertForTokenClassification (DistilBert model) + - contains `roberta`: RobertaForTokenClassification (Roberta model) The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated) To train the model, you should first set it back in training mode with `model.train()` @@ -772,6 +780,10 @@ class TFAutoModelForTokenClassification: return TFBertForTokenClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'xlnet' in pretrained_model_name_or_path: return TFXLNetForTokenClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'distilbert' in pretrained_model_name_or_path: + return TFDistilBertForTokenClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'roberta' in pretrained_model_name_or_path: + return TFRobertaForTokenClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " - "'bert', 'xlnet'".format(pretrained_model_name_or_path)) + "'bert', 'xlnet', 'distilbert', 'roberta'".format(pretrained_model_name_or_path)) From 2e2f9fed554bb5f147ea3d9573004b447dd7c9e7 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Wed, 11 Dec 2019 11:11:56 -0500 Subject: [PATCH 316/505] rm duplicate imports --- transformers/modeling_auto.py | 1 - 1 file changed, 1 deletion(-) diff --git a/transformers/modeling_auto.py b/transformers/modeling_auto.py index b63e43d73b..6ba1aab7a3 100644 --- a/transformers/modeling_auto.py +++ b/transformers/modeling_auto.py @@ -28,7 +28,6 @@ from .modeling_xlm import XLMModel, XLMWithLMHeadModel, XLMForSequenceClassifica from .modeling_roberta import RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification from .modeling_distilbert import DistilBertModel, DistilBertForQuestionAnswering, DistilBertForMaskedLM, DistilBertForSequenceClassification from .modeling_camembert import CamembertModel, CamembertForMaskedLM, CamembertForSequenceClassification, CamembertForMultipleChoice -from .modeling_camembert import CamembertModel, CamembertForMaskedLM, CamembertForSequenceClassification, CamembertForMultipleChoice from .modeling_albert import AlbertModel, AlbertForMaskedLM, AlbertForSequenceClassification, AlbertForQuestionAnswering from .modeling_utils import PreTrainedModel, SequenceSummary From 29570db25ba9dd30e5ac9be68dbcad95434964ec Mon Sep 17 00:00:00 2001 From: thomwolf Date: Wed, 11 Dec 2019 17:19:18 +0100 Subject: [PATCH 317/505] allowing from_pretrained to load from url directly --- transformers/modeling_tf_utils.py | 4 +++- transformers/modeling_utils.py | 7 +++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/transformers/modeling_tf_utils.py b/transformers/modeling_tf_utils.py index ed8fdb74c9..e7512b5bd6 100644 --- a/transformers/modeling_tf_utils.py +++ b/transformers/modeling_tf_utils.py @@ -259,8 +259,10 @@ class TFPreTrainedModel(tf.keras.Model): pretrained_model_name_or_path)) elif os.path.isfile(pretrained_model_name_or_path): archive_file = pretrained_model_name_or_path + elif os.path.isfile(pretrained_model_name_or_path + ".index"): + archive_file = pretrained_model_name_or_path + ".index" else: - raise EnvironmentError("Error file {} not found".format(pretrained_model_name_or_path)) + archive_file = pretrained_model_name_or_path # redirect to the cache, if necessary try: diff --git a/transformers/modeling_utils.py b/transformers/modeling_utils.py index 3ac568771e..9e7ca8d689 100644 --- a/transformers/modeling_utils.py +++ b/transformers/modeling_utils.py @@ -365,9 +365,12 @@ class PreTrainedModel(nn.Module): pretrained_model_name_or_path)) elif os.path.isfile(pretrained_model_name_or_path): archive_file = pretrained_model_name_or_path - else: - assert from_tf, "Error finding file {}, no file or TF 1.X checkpoint found".format(pretrained_model_name_or_path) + elif os.path.isfile(pretrained_model_name_or_path + ".index"): + assert from_tf, "We found a TensorFlow checkpoint at {}, please set from_tf to True to load from this checkpoint".format( + pretrained_model_name_or_path + ".index") archive_file = pretrained_model_name_or_path + ".index" + else: + archive_file = pretrained_model_name_or_path # redirect to the cache, if necessary try: From 6709739a05ca8b271a629ffebb497352449b7935 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Wed, 11 Dec 2019 17:19:18 +0100 Subject: [PATCH 318/505] allowing from_pretrained to load from url directly --- transformers/modeling_tf_utils.py | 4 +++- transformers/modeling_utils.py | 7 +++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/transformers/modeling_tf_utils.py b/transformers/modeling_tf_utils.py index 6c48f3eed2..95c29693d8 100644 --- a/transformers/modeling_tf_utils.py +++ b/transformers/modeling_tf_utils.py @@ -265,8 +265,10 @@ class TFPreTrainedModel(tf.keras.Model): pretrained_model_name_or_path)) elif os.path.isfile(pretrained_model_name_or_path): archive_file = pretrained_model_name_or_path + elif os.path.isfile(pretrained_model_name_or_path + ".index"): + archive_file = pretrained_model_name_or_path + ".index" else: - raise EnvironmentError("Error file {} not found".format(pretrained_model_name_or_path)) + archive_file = pretrained_model_name_or_path # redirect to the cache, if necessary try: diff --git a/transformers/modeling_utils.py b/transformers/modeling_utils.py index 398172a88c..eec9034fd7 100644 --- a/transformers/modeling_utils.py +++ b/transformers/modeling_utils.py @@ -364,9 +364,12 @@ class PreTrainedModel(nn.Module): pretrained_model_name_or_path)) elif os.path.isfile(pretrained_model_name_or_path): archive_file = pretrained_model_name_or_path - else: - assert from_tf, "Error finding file {}, no file or TF 1.X checkpoint found".format(pretrained_model_name_or_path) + elif os.path.isfile(pretrained_model_name_or_path + ".index"): + assert from_tf, "We found a TensorFlow checkpoint at {}, please set from_tf to True to load from this checkpoint".format( + pretrained_model_name_or_path + ".index") archive_file = pretrained_model_name_or_path + ".index" + else: + archive_file = pretrained_model_name_or_path # redirect to the cache, if necessary try: From 030faccb8d45be9bdd2b4b80ff26f36dc41f622a Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Wed, 11 Dec 2019 17:44:21 +0100 Subject: [PATCH 319/505] doc: fix pretrained models table --- docs/source/pretrained_models.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst index dd61f11769..2fe1f8a314 100644 --- a/docs/source/pretrained_models.rst +++ b/docs/source/pretrained_models.rst @@ -169,35 +169,35 @@ Here is the full list of the currently provided pretrained models together with +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | ALBERT | ``albert-base-v1`` | | 12 repeating layers, 128 embedding, 768-hidden, 12-heads, 11M parameters | | | | | ALBERT base model | -| | | (see `details `__) | +| | | (see `details `__) | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``albert-large-v1`` | | 24 repeating layers, 128 embedding, 1024-hidden, 16-heads, 17M parameters | | | | | ALBERT large model | -| | | (see `details `__) | +| | | (see `details `__) | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``albert-xlarge-v1`` | | 24 repeating layers, 128 embedding, 2048-hidden, 16-heads, 58M parameters | | | | | ALBERT xlarge model | -| | | (see `details `__) | +| | | (see `details `__) | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``albert-xxlarge-v1`` | | 12 repeating layer, 128 embedding, 4096-hidden, 64-heads, 223M parameters | | | | | ALBERT xxlarge model | -| | | (see `details `__) | +| | | (see `details `__) | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``albert-base-v2`` | | 12 repeating layers, 128 embedding, 768-hidden, 12-heads, 11M parameters | | | | | ALBERT base model with no dropout, additional training data and longer training | -| | | (see `details `__) | +| | | (see `details `__) | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``albert-large-v2`` | | 24 repeating layers, 128 embedding, 1024-hidden, 16-heads, 17M parameters | | | | | ALBERT large model with no dropout, additional training data and longer training | -| | | (see `details `__) | +| | | (see `details `__) | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``albert-xlarge-v2`` | | 24 repeating layers, 128 embedding, 2048-hidden, 16-heads, 58M parameters | | | | | ALBERT xlarge model with no dropout, additional training data and longer training | -| | | (see `details `__) | +| | | (see `details `__) | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``albert-xxlarge-v2`` | | 12 repeating layer, 128 embedding, 4096-hidden, 64-heads, 223M parameters | | | | | ALBERT xxlarge model with no dropout, additional training data and longer training | -| | | (see `details `__) | +| | | (see `details `__) | +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ From c999a3e5050f1dc93d814abf352f3bf0c06572e7 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Wed, 11 Dec 2019 12:29:58 -0500 Subject: [PATCH 320/505] Allow from_pretrained to take a remote identifier --- transformers/configuration_utils.py | 8 +++++--- transformers/file_utils.py | 20 ++++++++++++++++---- transformers/modeling_utils.py | 8 +++++--- transformers/tokenization_utils.py | 10 +++++----- 4 files changed, 31 insertions(+), 15 deletions(-) diff --git a/transformers/configuration_utils.py b/transformers/configuration_utils.py index 08cee75d81..8ae30f2a48 100644 --- a/transformers/configuration_utils.py +++ b/transformers/configuration_utils.py @@ -24,7 +24,7 @@ import logging import os from io import open -from .file_utils import cached_path, CONFIG_NAME +from .file_utils import CONFIG_NAME, cached_path, is_remote_url, hf_bucket_url logger = logging.getLogger(__name__) @@ -131,8 +131,10 @@ class PretrainedConfig(object): config_file = cls.pretrained_config_archive_map[pretrained_model_name_or_path] elif os.path.isdir(pretrained_model_name_or_path): config_file = os.path.join(pretrained_model_name_or_path, CONFIG_NAME) - else: + elif os.path.isfile(pretrained_model_name_or_path) or is_remote_url(pretrained_model_name_or_path): config_file = pretrained_model_name_or_path + else: + config_file = hf_bucket_url(pretrained_model_name_or_path, postfix=CONFIG_NAME) # redirect to the cache, if necessary try: resolved_config_file = cached_path(config_file, cache_dir=cache_dir, force_download=force_download, @@ -187,7 +189,7 @@ class PretrainedConfig(object): @classmethod def from_json_file(cls, json_file): - """Constructs a `BertConfig` from a json file of parameters.""" + """Constructs a `Config` from a json file of parameters.""" with open(json_file, "r", encoding='utf-8') as reader: text = reader.read() return cls.from_dict(json.loads(text)) diff --git a/transformers/file_utils.py b/transformers/file_utils.py index 68de4e6e2f..5fd5e2ee39 100644 --- a/transformers/file_utils.py +++ b/transformers/file_utils.py @@ -73,6 +73,8 @@ TF2_WEIGHTS_NAME = 'tf_model.h5' TF_WEIGHTS_NAME = 'model.ckpt' CONFIG_NAME = "config.json" +S3_BUCKET_PREFIX = "https://s3.amazonaws.com/models.huggingface.co/bert" + def is_torch_available(): return _torch_available @@ -103,6 +105,18 @@ else: return fn return docstring_decorator + +def is_remote_url(url_or_filename): + parsed = urlparse(url_or_filename) + return parsed.scheme in ('http', 'https', 's3') + +def hf_bucket_url(identifier, postfix=None): + if postfix is None: + return "/".join((S3_BUCKET_PREFIX, identifier)) + else: + return "/".join((S3_BUCKET_PREFIX, identifier, postfix)) + + def url_to_filename(url, etag=None): """ Convert `url` into a hashed filename in a repeatable way. @@ -171,9 +185,7 @@ def cached_path(url_or_filename, cache_dir=None, force_download=False, proxies=N if sys.version_info[0] == 3 and isinstance(cache_dir, Path): cache_dir = str(cache_dir) - parsed = urlparse(url_or_filename) - - if parsed.scheme in ('http', 'https', 's3'): + if is_remote_url(url_or_filename): # URL, so get it from the cache (downloading if necessary) return get_from_cache(url_or_filename, cache_dir=cache_dir, force_download=force_download, proxies=proxies, @@ -181,7 +193,7 @@ def cached_path(url_or_filename, cache_dir=None, force_download=False, proxies=N elif os.path.exists(url_or_filename): # File, and it exists. return url_or_filename - elif parsed.scheme == '': + elif urlparse(url_or_filename).scheme == '': # File, but it doesn't exist. raise EnvironmentError("file {} not found".format(url_or_filename)) else: diff --git a/transformers/modeling_utils.py b/transformers/modeling_utils.py index 9e7ca8d689..eac4252336 100644 --- a/transformers/modeling_utils.py +++ b/transformers/modeling_utils.py @@ -31,7 +31,8 @@ from torch.nn import CrossEntropyLoss from torch.nn import functional as F from .configuration_utils import PretrainedConfig -from .file_utils import cached_path, WEIGHTS_NAME, TF_WEIGHTS_NAME, TF2_WEIGHTS_NAME +from .file_utils import (TF2_WEIGHTS_NAME, TF_WEIGHTS_NAME, WEIGHTS_NAME, + cached_path, hf_bucket_url, is_remote_url) logger = logging.getLogger(__name__) @@ -363,14 +364,15 @@ class PreTrainedModel(nn.Module): raise EnvironmentError("Error no file named {} found in directory {} or `from_tf` set to False".format( [WEIGHTS_NAME, TF2_WEIGHTS_NAME, TF_WEIGHTS_NAME + ".index"], pretrained_model_name_or_path)) - elif os.path.isfile(pretrained_model_name_or_path): + elif os.path.isfile(pretrained_model_name_or_path) or is_remote_url(pretrained_model_name_or_path): archive_file = pretrained_model_name_or_path elif os.path.isfile(pretrained_model_name_or_path + ".index"): assert from_tf, "We found a TensorFlow checkpoint at {}, please set from_tf to True to load from this checkpoint".format( pretrained_model_name_or_path + ".index") archive_file = pretrained_model_name_or_path + ".index" else: - archive_file = pretrained_model_name_or_path + archive_file = hf_bucket_url(pretrained_model_name_or_path, postfix=WEIGHTS_NAME) + # todo do we want to support TF checkpoints here? # redirect to the cache, if necessary try: diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index 68a767fe82..2b2cec0c15 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -25,7 +25,7 @@ import itertools import re from io import open -from .file_utils import cached_path, is_tf_available, is_torch_available +from .file_utils import cached_path, is_remote_url, hf_bucket_url, is_tf_available, is_torch_available if is_tf_available(): import tensorflow as tf @@ -327,12 +327,12 @@ class PreTrainedTokenizer(object): if os.path.isdir(pretrained_model_name_or_path): # If a directory is provided we look for the standard filenames full_file_name = os.path.join(pretrained_model_name_or_path, file_name) - else: + elif os.path.isfile(pretrained_model_name_or_path) or is_remote_url(pretrained_model_name_or_path): # If a path to a file is provided we use it (will only work for non-BPE tokenizer using a single vocabulary file) full_file_name = pretrained_model_name_or_path - if not os.path.exists(full_file_name): - logger.info("Didn't find file {}. We won't load it.".format(full_file_name)) - full_file_name = None + else: + full_file_name = hf_bucket_url(pretrained_model_name_or_path, postfix=file_name) + vocab_files[file_id] = full_file_name # Look for the additional tokens files From 3d57c51111054adb01b2ea94bfd45237eb282431 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Wed, 11 Dec 2019 15:10:17 -0500 Subject: [PATCH 321/505] Fix encode plus --- transformers/tokenization_utils.py | 39 ++++++++++++++++++------------ 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index 68a767fe82..eace409555 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -916,7 +916,7 @@ class PreTrainedTokenizer(object): return_tensors: (optional) can be set to 'tf' or 'pt' to return respectively TensorFlow tf.constant or PyTorch torch.Tensor instead of a list of python integers. return_token_type_ids: (optional) Set to False to avoid returning token_type_ids (default True). - return_attention_mask: (optional) Set to False to avoir returning attention mask (default True) + return_attention_mask: (optional) Set to False to avoid returning attention mask (default True) return_overflowing_tokens: (optional) Set to True to return overflowing token information (default False). return_special_tokens_mask: (optional) Set to True to return special tokens mask information (default False). @@ -961,24 +961,13 @@ class PreTrainedTokenizer(object): if add_special_tokens: sequence = self.build_inputs_with_special_tokens(ids, pair_ids) token_type_ids = self.create_token_type_ids_from_sequences(ids, pair_ids) - special_tokens_mask = self.get_special_tokens_mask(ids, pair_ids) else: sequence = ids + pair_ids if pair else ids token_type_ids = [0] * len(ids) + ([1] * len(pair_ids) if pair else []) - special_tokens_mask = [0] * (len(ids) + (len(pair_ids) if pair else 0)) + if return_special_tokens_mask: encoded_inputs["special_tokens_mask"] = self.get_special_tokens_mask(ids, pair_ids) - # Prepare inputs as tensors if asked - if return_tensors == 'tf' and is_tf_available(): - sequence = tf.constant([sequence]) - token_type_ids = tf.constant([token_type_ids]) - elif return_tensors == 'pt' and is_torch_available(): - sequence = torch.tensor([sequence]) - token_type_ids = torch.tensor([token_type_ids]) - elif return_tensors is not None: - logger.warning("Unable to convert output to tensors format {}, PyTorch or TensorFlow is not available.".format(return_tensors)) - encoded_inputs["input_ids"] = sequence if return_token_type_ids: encoded_inputs["token_type_ids"] = token_type_ids @@ -1015,10 +1004,9 @@ class PreTrainedTokenizer(object): if return_special_tokens_mask: encoded_inputs["special_tokens_mask"] = encoded_inputs["special_tokens_mask"] + [1] * difference encoded_inputs["input_ids"] = encoded_inputs["input_ids"] + [self.pad_token_id] * difference - elif self.padding_side == 'left': if return_attention_mask: - encoded_inputs["attention_mask"] = [0] * difference + [1] * len(encoded_inputs["input_ids"]) + encoded_inputs["attention_mask"] = [0] * difference + [1] * len(encoded_inputs["input_ids"]) if return_token_type_ids: encoded_inputs["token_type_ids"] = [self.pad_token_type_id] * difference + encoded_inputs["token_type_ids"] if return_special_tokens_mask: @@ -1030,7 +1018,26 @@ class PreTrainedTokenizer(object): elif return_attention_mask: encoded_inputs["attention_mask"] = [1] * len(encoded_inputs["input_ids"]) - + + # Prepare inputs as tensors if asked + if return_tensors == 'tf' and is_tf_available(): + encoded_inputs["input_ids"] = tf.constant([encoded_inputs["input_ids"]]) + encoded_inputs["token_type_ids"] = tf.constant([encoded_inputs["token_type_ids"]]) + + if "attention_mask" in encoded_inputs: + encoded_inputs["attention_mask"] = tf.constant([encoded_inputs["attention_mask"]]) + + elif return_tensors == 'pt' and is_torch_available(): + encoded_inputs["input_ids"] = torch.tensor([encoded_inputs["input_ids"]]) + encoded_inputs["token_type_ids"] = torch.tensor([encoded_inputs["token_type_ids"]]) + + if "attention_mask" in encoded_inputs: + encoded_inputs["attention_mask"] = torch.tensor([encoded_inputs["attention_mask"]]) + elif return_tensors is not None: + logger.warning( + "Unable to convert output to tensors format {}, PyTorch or TensorFlow is not available.".format( + return_tensors)) + return encoded_inputs def truncate_sequences(self, ids, pair_ids=None, num_tokens_to_remove=0, truncation_strategy='longest_first', stride=0): From 31e5b5ff2276c61af7eebb4c353934f8f675d728 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Wed, 11 Dec 2019 15:22:02 -0500 Subject: [PATCH 322/505] Fix tests + first example of doc --- transformers/tokenization_utils.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index 2b2cec0c15..63d2cc5cb4 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -255,6 +255,7 @@ class PreTrainedTokenizer(object): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a predefined tokenizer to load from cache or download, e.g.: ``bert-base-uncased``. + - a string with the `identifier name` of a predefined tokenizer that was user-uploaded to our S3, e.g.: ``dbmz/bert-base-german-cased``. - a path to a `directory` containing vocabulary files required by the tokenizer, for instance saved using the :func:`~transformers.PreTrainedTokenizer.save_pretrained` method, e.g.: ``./my_model_directory/``. - (not applicable to all derived classes) a path or url to a single saved vocabulary file if and only if the tokenizer only requires a single vocabulary file (e.g. Bert, XLNet), e.g.: ``./my_model_directory/vocab.txt``. @@ -282,6 +283,9 @@ class PreTrainedTokenizer(object): # Download vocabulary from S3 and cache. tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + # Download vocabulary from S3 (user-uploaded) and cache. + tokenizer = BertTokenizer.from_pretrained('dbmz/bert-base-german-cased') + # If vocabulary files are in a directory (e.g. tokenizer was saved using `save_pretrained('./test/saved_model/')`) tokenizer = BertTokenizer.from_pretrained('./test/saved_model/') @@ -327,6 +331,9 @@ class PreTrainedTokenizer(object): if os.path.isdir(pretrained_model_name_or_path): # If a directory is provided we look for the standard filenames full_file_name = os.path.join(pretrained_model_name_or_path, file_name) + if not os.path.exists(full_file_name): + logger.info("Didn't find file {}. We won't load it.".format(full_file_name)) + full_file_name = None elif os.path.isfile(pretrained_model_name_or_path) or is_remote_url(pretrained_model_name_or_path): # If a path to a file is provided we use it (will only work for non-BPE tokenizer using a single vocabulary file) full_file_name = pretrained_model_name_or_path From 18e1f751f1d996c4fe01559ade1cd013186b81e4 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Wed, 11 Dec 2019 17:07:46 -0500 Subject: [PATCH 323/505] TF support --- transformers/modeling_tf_utils.py | 9 ++++++--- transformers/modeling_utils.py | 3 ++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/transformers/modeling_tf_utils.py b/transformers/modeling_tf_utils.py index e7512b5bd6..4a6d18f447 100644 --- a/transformers/modeling_tf_utils.py +++ b/transformers/modeling_tf_utils.py @@ -24,7 +24,8 @@ import os import tensorflow as tf from .configuration_utils import PretrainedConfig -from .file_utils import cached_path, WEIGHTS_NAME, TF_WEIGHTS_NAME, TF2_WEIGHTS_NAME +from .file_utils import (TF2_WEIGHTS_NAME, TF_WEIGHTS_NAME, WEIGHTS_NAME, + cached_path, hf_bucket_url, is_remote_url) from .modeling_tf_pytorch_utils import load_pytorch_checkpoint_in_tf2_model logger = logging.getLogger(__name__) @@ -257,12 +258,14 @@ class TFPreTrainedModel(tf.keras.Model): raise EnvironmentError("Error no file named {} found in directory {} or `from_pt` set to False".format( [WEIGHTS_NAME, TF2_WEIGHTS_NAME], pretrained_model_name_or_path)) - elif os.path.isfile(pretrained_model_name_or_path): + elif os.path.isfile(pretrained_model_name_or_path) or is_remote_url(pretrained_model_name_or_path): archive_file = pretrained_model_name_or_path elif os.path.isfile(pretrained_model_name_or_path + ".index"): archive_file = pretrained_model_name_or_path + ".index" else: - archive_file = pretrained_model_name_or_path + archive_file = hf_bucket_url(pretrained_model_name_or_path, postfix=TF2_WEIGHTS_NAME) + if from_pt: + raise EnvironmentError("Loading a TF model from a PyTorch checkpoint is not supported when using a model identifier name.") # redirect to the cache, if necessary try: diff --git a/transformers/modeling_utils.py b/transformers/modeling_utils.py index eac4252336..37088f8e67 100644 --- a/transformers/modeling_utils.py +++ b/transformers/modeling_utils.py @@ -372,7 +372,8 @@ class PreTrainedModel(nn.Module): archive_file = pretrained_model_name_or_path + ".index" else: archive_file = hf_bucket_url(pretrained_model_name_or_path, postfix=WEIGHTS_NAME) - # todo do we want to support TF checkpoints here? + if from_tf: + raise EnvironmentError("Loading a PyTorch model from a TF checkpoint is not supported when using a model identifier name.") # redirect to the cache, if necessary try: From 4f15e5a267201f86bdd9628cf58592d0e1cc86eb Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Wed, 11 Dec 2019 17:41:51 -0500 Subject: [PATCH 324/505] Add tests. Maybe not the best possible place for the tests, lmk. --- transformers/tests/modeling_auto_test.py | 7 ++++++- transformers/tests/modeling_tf_auto_test.py | 7 ++++++- transformers/tests/tokenization_auto_test.py | 7 ++++++- transformers/tests/utils.py | 3 +++ 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/transformers/tests/modeling_auto_test.py b/transformers/tests/modeling_auto_test.py index 9b7d920bc8..871a262fe8 100644 --- a/transformers/tests/modeling_auto_test.py +++ b/transformers/tests/modeling_auto_test.py @@ -22,7 +22,7 @@ import logging from transformers import is_torch_available -from .utils import require_torch, slow +from .utils import require_torch, slow, SMALL_MODEL_IDENTIFIER if is_torch_available(): from transformers import (AutoConfig, BertConfig, @@ -92,6 +92,11 @@ class AutoModelTest(unittest.TestCase): self.assertIsNotNone(model) self.assertIsInstance(model, BertForQuestionAnswering) + def test_from_pretrained_identifier(self): + logging.basicConfig(level=logging.INFO) + model = AutoModelWithLMHead.from_pretrained(SMALL_MODEL_IDENTIFIER) + self.assertIsInstance(model, BertForMaskedLM) + if __name__ == "__main__": unittest.main() diff --git a/transformers/tests/modeling_tf_auto_test.py b/transformers/tests/modeling_tf_auto_test.py index 7ea48015d9..7ab6eaa3d6 100644 --- a/transformers/tests/modeling_tf_auto_test.py +++ b/transformers/tests/modeling_tf_auto_test.py @@ -22,7 +22,7 @@ import logging from transformers import is_tf_available -from .utils import require_tf, slow +from .utils import require_tf, slow, SMALL_MODEL_IDENTIFIER if is_tf_available(): from transformers import (AutoConfig, BertConfig, @@ -93,6 +93,11 @@ class TFAutoModelTest(unittest.TestCase): self.assertIsNotNone(model) self.assertIsInstance(model, TFBertForQuestionAnswering) + def test_from_pretrained_identifier(self): + logging.basicConfig(level=logging.INFO) + model = TFAutoModelWithLMHead.from_pretrained(SMALL_MODEL_IDENTIFIER, force_download=True) + self.assertIsInstance(model, TFBertForMaskedLM) + if __name__ == "__main__": unittest.main() diff --git a/transformers/tests/tokenization_auto_test.py b/transformers/tests/tokenization_auto_test.py index 18346d2768..0a894cac04 100644 --- a/transformers/tests/tokenization_auto_test.py +++ b/transformers/tests/tokenization_auto_test.py @@ -23,7 +23,7 @@ import logging from transformers import AutoTokenizer, BertTokenizer, AutoTokenizer, GPT2Tokenizer from transformers import BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP -from .utils import slow +from .utils import slow, SMALL_MODEL_IDENTIFIER class AutoTokenizerTest(unittest.TestCase): @@ -42,6 +42,11 @@ class AutoTokenizerTest(unittest.TestCase): self.assertIsInstance(tokenizer, GPT2Tokenizer) self.assertGreater(len(tokenizer), 0) + def test_tokenizer_from_pretrained_identifier(self): + logging.basicConfig(level=logging.INFO) + tokenizer = AutoTokenizer.from_pretrained(SMALL_MODEL_IDENTIFIER) + self.assertIsInstance(tokenizer, BertTokenizer) + self.assertEqual(len(tokenizer), 12) if __name__ == "__main__": unittest.main() diff --git a/transformers/tests/utils.py b/transformers/tests/utils.py index 7a51ab612b..3aff1daf83 100644 --- a/transformers/tests/utils.py +++ b/transformers/tests/utils.py @@ -6,6 +6,9 @@ from distutils.util import strtobool from transformers.file_utils import _tf_available, _torch_available +SMALL_MODEL_IDENTIFIER = "julien-c/bert-xsmall-dummy" + + try: run_slow = os.environ["RUN_SLOW"] except KeyError: From c03c0dfd230a5174c536a58d6ba5e590ed1afcc4 Mon Sep 17 00:00:00 2001 From: Masatoshi Suzuki Date: Fri, 15 Nov 2019 17:24:56 +0900 Subject: [PATCH 325/505] Add support for Japanese BERT models by cl-tohoku --- docs/source/pretrained_models.rst | 18 ++ transformers/__init__.py | 1 + transformers/configuration_bert.py | 4 + transformers/modeling_bert.py | 8 +- transformers/modeling_tf_bert.py | 16 +- transformers/tokenization_auto.py | 3 + transformers/tokenization_bert_japanese.py | 247 +++++++++++++++++++++ 7 files changed, 289 insertions(+), 8 deletions(-) create mode 100644 transformers/tokenization_bert_japanese.py diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst index 2fe1f8a314..d3498e057d 100644 --- a/docs/source/pretrained_models.rst +++ b/docs/source/pretrained_models.rst @@ -61,6 +61,24 @@ Here is the full list of the currently provided pretrained models together with | | ``bert-base-german-dbmdz-uncased`` | | 12-layer, 768-hidden, 12-heads, 110M parameters. | | | | | Trained on uncased German text by DBMDZ | | | | (see `details on dbmdz repository `__). | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``bert-base-japanese`` | | 12-layer, 768-hidden, 12-heads, 110M parameters. | +| | | | Trained on Japanese text. Text is tokenized with MeCab and WordPiece. | +| | | | `MeCab `__ is required for tokenization. | +| | | (see `details on cl-tohoku repository `__). | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``bert-base-japanese-whole-word-masking`` | | 12-layer, 768-hidden, 12-heads, 110M parameters. | +| | | | Trained on Japanese text using Whole-Word-Masking. Text is tokenized with MeCab and WordPiece. | +| | | | `MeCab `__ is required for tokenization. | +| | | (see `details on cl-tohoku repository `__). | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``bert-base-japanese-char`` | | 12-layer, 768-hidden, 12-heads, 110M parameters. | +| | | | Trained on Japanese text. Text is tokenized into characters. | +| | | (see `details on cl-tohoku repository `__). | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``bert-base-japanese-char-whole-word-masking`` | | 12-layer, 768-hidden, 12-heads, 110M parameters. | +| | | | Trained on Japanese text using Whole-Word-Masking. Text is tokenized into characters. | +| | | (see `details on cl-tohoku repository `__). | +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | GPT | ``openai-gpt`` | | 12-layer, 768-hidden, 12-heads, 110M parameters. | | | | | OpenAI GPT English model | diff --git a/transformers/__init__.py b/transformers/__init__.py index f9a28add5f..5d7b0b772c 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -37,6 +37,7 @@ if is_sklearn_available(): from .tokenization_utils import (PreTrainedTokenizer) from .tokenization_auto import AutoTokenizer from .tokenization_bert import BertTokenizer, BasicTokenizer, WordpieceTokenizer +from .tokenization_bert_japanese import BertJapaneseTokenizer, MecabTokenizer, CharacterTokenizer from .tokenization_openai import OpenAIGPTTokenizer from .tokenization_transfo_xl import (TransfoXLTokenizer, TransfoXLCorpus) from .tokenization_gpt2 import GPT2Tokenizer diff --git a/transformers/configuration_bert.py b/transformers/configuration_bert.py index d63be963eb..16f1f60404 100644 --- a/transformers/configuration_bert.py +++ b/transformers/configuration_bert.py @@ -42,6 +42,10 @@ BERT_PRETRAINED_CONFIG_ARCHIVE_MAP = { 'bert-base-cased-finetuned-mrpc': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-finetuned-mrpc-config.json", 'bert-base-german-dbmdz-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-cased-config.json", 'bert-base-german-dbmdz-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-uncased-config.json", + 'bert-base-japanese': "https://www.nlp.ecei.tohoku.ac.jp/~m-suzuki/bert-japanese/bert-base-japanese-config.json", + 'bert-base-japanese-whole-word-masking': "https://www.nlp.ecei.tohoku.ac.jp/~m-suzuki/bert-japanese/bert-base-japanese-whole-word-masking-config.json", + 'bert-base-japanese-char': "https://www.nlp.ecei.tohoku.ac.jp/~m-suzuki/bert-japanese/bert-base-japanese-char-config.json", + 'bert-base-japanese-char-whole-word-masking': "https://www.nlp.ecei.tohoku.ac.jp/~m-suzuki/bert-japanese/bert-base-japanese-char-whole-word-masking-config.json" } diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py index d84b0a1a7c..e2e115a015 100644 --- a/transformers/modeling_bert.py +++ b/transformers/modeling_bert.py @@ -48,6 +48,10 @@ BERT_PRETRAINED_MODEL_ARCHIVE_MAP = { 'bert-base-cased-finetuned-mrpc': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-finetuned-mrpc-pytorch_model.bin", 'bert-base-german-dbmdz-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-cased-pytorch_model.bin", 'bert-base-german-dbmdz-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-uncased-pytorch_model.bin", + 'bert-base-japanese': "https://www.nlp.ecei.tohoku.ac.jp/~m-suzuki/bert-japanese/bert-base-japanese-pytorch_model.bin", + 'bert-base-japanese-whole-word-masking': "https://www.nlp.ecei.tohoku.ac.jp/~m-suzuki/bert-japanese/bert-base-japanese-whole-word-masking-pytorch_model.bin", + 'bert-base-japanese-char': "https://www.nlp.ecei.tohoku.ac.jp/~m-suzuki/bert-japanese/bert-base-japanese-char-pytorch_model.bin", + 'bert-base-japanese-char-whole-word-masking': "https://www.nlp.ecei.tohoku.ac.jp/~m-suzuki/bert-japanese/bert-base-japanese-char-whole-word-masking-pytorch_model.bin" } @@ -1233,9 +1237,9 @@ class BertForQuestionAnswering(BertPreTrainedModel): question, text = "Who was Jim Henson?", "Jim Henson was a nice puppet" input_text = "[CLS] " + question + " [SEP] " + text + " [SEP]" input_ids = tokenizer.encode(input_text) - token_type_ids = [0 if i <= input_ids.index(102) else 1 for i in range(len(input_ids))] + token_type_ids = [0 if i <= input_ids.index(102) else 1 for i in range(len(input_ids))] start_scores, end_scores = model(torch.tensor([input_ids]), token_type_ids=torch.tensor([token_type_ids])) - all_tokens = tokenizer.convert_ids_to_tokens(input_ids) + all_tokens = tokenizer.convert_ids_to_tokens(input_ids) print(' '.join(all_tokens[torch.argmax(start_scores) : torch.argmax(end_scores)+1])) # a nice puppet diff --git a/transformers/modeling_tf_bert.py b/transformers/modeling_tf_bert.py index 5aa7bb3da2..27dd311a4d 100644 --- a/transformers/modeling_tf_bert.py +++ b/transformers/modeling_tf_bert.py @@ -48,6 +48,10 @@ TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP = { 'bert-large-uncased-whole-word-masking-finetuned-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-whole-word-masking-finetuned-squad-tf_model.h5", 'bert-large-cased-whole-word-masking-finetuned-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-whole-word-masking-finetuned-squad-tf_model.h5", 'bert-base-cased-finetuned-mrpc': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-finetuned-mrpc-tf_model.h5", + 'bert-base-japanese': "https://www.nlp.ecei.tohoku.ac.jp/~m-suzuki/bert-japanese/bert-base-japanese-tf_model.h5", + 'bert-base-japanese-whole-word-masking': "https://www.nlp.ecei.tohoku.ac.jp/~m-suzuki/bert-japanese/bert-base-japanese-whole-word-masking-tf_model.h5", + 'bert-base-japanese-char': "https://www.nlp.ecei.tohoku.ac.jp/~m-suzuki/bert-japanese/bert-base-japanese-char-tf_model.h5", + 'bert-base-japanese-char-whole-word-masking': "https://www.nlp.ecei.tohoku.ac.jp/~m-suzuki/bert-japanese/bert-base-japanese-char-whole-word-masking-tf_model.h5" } @@ -129,7 +133,7 @@ class TFBertEmbeddings(tf.keras.layers.Layer): linear tensor, float32 with shape [batch_size, length, vocab_size]. Raises: ValueError: if mode is not valid. - + Shared weights logic adapted from https://github.com/tensorflow/models/blob/a009f4fb9d2fc4949e32192a944688925ef78659/official/transformer/v2/embedding_layer.py#L24 """ @@ -148,7 +152,7 @@ class TFBertEmbeddings(tf.keras.layers.Layer): input_shape = shape_list(input_ids) else: input_shape = shape_list(inputs_embeds)[:-1] - + seq_length = input_shape[1] if position_ids is None: position_ids = tf.range(seq_length, dtype=tf.int32)[tf.newaxis, :] @@ -246,7 +250,7 @@ class TFBertSelfAttention(tf.keras.layers.Layer): context_layer = tf.matmul(attention_probs, value_layer) context_layer = tf.transpose(context_layer, perm=[0, 2, 1, 3]) - context_layer = tf.reshape(context_layer, + context_layer = tf.reshape(context_layer, (batch_size, -1, self.all_head_size)) # (batch_size, seq_len_q, all_head_size) outputs = (context_layer, attention_probs) if self.output_attentions else (context_layer,) @@ -591,7 +595,7 @@ BERT_START_DOCSTRING = r""" The BERT model was proposed in `model({'input_ids': input_ids, 'token_type_ids': token_type_ids})` Parameters: - config (:class:`~transformers.BertConfig`): Model configuration class with all the parameters of the model. + config (:class:`~transformers.BertConfig`): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. """ @@ -605,13 +609,13 @@ BERT_INPUTS_DOCSTRING = r""" (a) For sequence pairs: ``tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]`` - + ``token_type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1`` (b) For single sequences: ``tokens: [CLS] the dog is hairy . [SEP]`` - + ``token_type_ids: 0 0 0 0 0 0 0`` Bert is a model with absolute position embeddings so it's usually advised to pad the inputs on diff --git a/transformers/tokenization_auto.py b/transformers/tokenization_auto.py index b7c5046961..d63b7e783d 100644 --- a/transformers/tokenization_auto.py +++ b/transformers/tokenization_auto.py @@ -19,6 +19,7 @@ from __future__ import absolute_import, division, print_function, unicode_litera import logging from .tokenization_bert import BertTokenizer +from .tokenization_bert_japanese import BertJapaneseTokenizer from .tokenization_openai import OpenAIGPTTokenizer from .tokenization_gpt2 import GPT2Tokenizer from .tokenization_ctrl import CTRLTokenizer @@ -118,6 +119,8 @@ class AutoTokenizer(object): return CamembertTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return RobertaTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) + elif 'bert-japanese' in pretrained_model_name_or_path: + return BertJapaneseTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) elif 'bert' in pretrained_model_name_or_path: return BertTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) elif 'openai-gpt' in pretrained_model_name_or_path: diff --git a/transformers/tokenization_bert_japanese.py b/transformers/tokenization_bert_japanese.py new file mode 100644 index 0000000000..8554a1c880 --- /dev/null +++ b/transformers/tokenization_bert_japanese.py @@ -0,0 +1,247 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tokenization classes.""" + +from __future__ import absolute_import, division, print_function, unicode_literals + +import collections +import logging +import os +import unicodedata +from io import open + +from .tokenization_bert import BertTokenizer, BasicTokenizer, WordpieceTokenizer, load_vocab +from .tokenization_utils import PreTrainedTokenizer + +logger = logging.getLogger(__name__) + +VOCAB_FILES_NAMES = {'vocab_file': 'vocab.txt'} + +PRETRAINED_VOCAB_FILES_MAP = { + 'vocab_file': + { + 'bert-base-japanese': "https://www.nlp.ecei.tohoku.ac.jp/~m-suzuki/bert-japanese/bert-base-japanese-vocab.txt", + 'bert-base-japanese-whole-word-masking': "https://www.nlp.ecei.tohoku.ac.jp/~m-suzuki/bert-japanese/bert-base-japanese-whole-word-masking-vocab.txt", + 'bert-base-japanese-char': "https://www.nlp.ecei.tohoku.ac.jp/~m-suzuki/bert-japanese/bert-base-japanese-char-vocab.txt", + 'bert-base-japanese-char-whole-word-masking': "https://www.nlp.ecei.tohoku.ac.jp/~m-suzuki/bert-japanese/bert-base-japanese-char-whole-word-masking-vocab.txt" + } +} + +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { + 'bert-base-japanese': 512, + 'bert-base-japanese-whole-word-masking': 512, + 'bert-base-japanese-char': 512, + 'bert-base-japanese-char-whole-word-masking': 512 +} + +PRETRAINED_INIT_CONFIGURATION = { + 'bert-base-japanese': { + 'do_lower_case': False, + 'word_tokenizer_type': 'mecab', + 'subword_tokenizer_type': 'wordpiece' + }, + 'bert-base-japanese-whole-word-masking':{ + 'do_lower_case': False, + 'word_tokenizer_type': 'mecab', + 'subword_tokenizer_type': 'wordpiece' + }, + 'bert-base-japanese-char': { + 'do_lower_case': False, + 'word_tokenizer_type': 'mecab', + 'subword_tokenizer_type': 'character' + }, + 'bert-base-japanese-char-whole-word-masking': { + 'do_lower_case': False, + 'word_tokenizer_type': 'mecab', + 'subword_tokenizer_type': 'character' + } +} + + +class BertJapaneseTokenizer(BertTokenizer): + """BERT tokenizer for Japanese text""" + + vocab_files_names = VOCAB_FILES_NAMES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + + def __init__(self, vocab_file, do_lower_case=False, + do_word_tokenize=True, do_subword_tokenize=True, + word_tokenizer_type='basic', subword_tokenizer_type='wordpiece', + never_split=None, unk_token='[UNK]', sep_token='[SEP]', + pad_token='[PAD]', cls_token='[CLS]', mask_token='[MASK]', **kwargs): + """Constructs a MecabBertTokenizer. + + Args: + **vocab_file**: Path to a one-wordpiece-per-line vocabulary file. + **do_lower_case**: (`optional`) boolean (default True) + Whether to lower case the input. + Only has an effect when do_basic_tokenize=True. + **do_word_tokenize**: (`optional`) boolean (default True) + Whether to do word tokenization. + **do_subword_tokenize**: (`optional`) boolean (default True) + Whether to do subword tokenization. + **word_tokenizer_type**: (`optional`) string (default "basic") + Type of word tokenizer. + **subword_tokenizer_type**: (`optional`) string (default "wordpiece") + Type of subword tokenizer. + """ + super(BertTokenizer, self).__init__(unk_token=unk_token, sep_token=sep_token, + pad_token=pad_token, cls_token=cls_token, + mask_token=mask_token, **kwargs) + self.max_len_single_sentence = self.max_len - 2 # take into account special tokens + self.max_len_sentences_pair = self.max_len - 3 # take into account special tokens + + if not os.path.isfile(vocab_file): + raise ValueError( + "Can't find a vocabulary file at path '{}'. To load the vocabulary from a Google pretrained " + "model use `tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`".format(vocab_file)) + self.vocab = load_vocab(vocab_file) + self.ids_to_tokens = collections.OrderedDict( + [(ids, tok) for tok, ids in self.vocab.items()]) + + self.do_word_tokenize = do_word_tokenize + if do_word_tokenize: + if word_tokenizer_type == 'basic': + self.word_tokenizer = BasicTokenizer(do_lower_case=do_lower_case, + never_split=never_split, + tokenize_chinese_chars=False) + elif word_tokenizer_type == 'mecab': + self.word_tokenizer = MecabTokenizer(do_lower_case=do_lower_case, + never_split=never_split) + else: + raise ValueError( + "Invalid word_tokenizer_type '{}' is specified.".format(word_tokenizer_type)) + + self.do_subword_tokenize = do_subword_tokenize + if do_subword_tokenize: + if subword_tokenizer_type == 'wordpiece': + self.subword_tokenizer = WordpieceTokenizer(vocab=self.vocab, + unk_token=self.unk_token) + elif subword_tokenizer_type == 'character': + self.subword_tokenizer = CharacterTokenizer(vocab=self.vocab, + unk_token=self.unk_token) + else: + raise ValueError( + "Invalid subword_tokenizer_type '{}' is specified.".format(subword_tokenizer_type)) + + + def _tokenize(self, text): + if self.do_word_tokenize: + tokens = self.word_tokenizer.tokenize(text, + never_split=self.all_special_tokens) + else: + tokens = [text] + + if self.do_subword_tokenize: + split_tokens = [sub_token for token in tokens + for sub_token in self.subword_tokenizer.tokenize(token)] + else: + split_tokens = tokens + + return split_tokens + + +class MecabTokenizer(object): + """Runs basic tokenization with MeCab morphological parser.""" + + def __init__(self, do_lower_case=False, never_split=None, normalize_text=True): + """Constructs a MecabTokenizer. + + Args: + **do_lower_case**: (`optional`) boolean (default True) + Whether to lower case the input. + **never_split**: (`optional`) list of str + Kept for backward compatibility purposes. + Now implemented directly at the base class level (see :func:`PreTrainedTokenizer.tokenize`) + List of token not to split. + **normalize_text**: (`optional`) boolean (default True) + Whether to apply unicode normalization to text before tokenization. + """ + self.do_lower_case = do_lower_case + self.never_split = never_split if never_split is not None else [] + self.normalize_text = normalize_text + + import MeCab + self.mecab = MeCab.Tagger() + + def tokenize(self, text, never_split=None, **kwargs): + """Tokenizes a piece of text.""" + if self.normalize_text: + text = unicodedata.normalize('NFKC', text) + + never_split = self.never_split + (never_split if never_split is not None else []) + tokens = [] + + cursor = 0 + for line in self.mecab.parse(text).split('\n'): + if line == 'EOS': + break + + token, _ = line.split('\t') + token_start = text.index(token, cursor) + token_end = token_start + len(token) + if self.do_lower_case and token not in never_split: + token = token.lower() + + tokens.append(token) + cursor = token_end + + return tokens + + +class CharacterTokenizer(object): + """Runs Character tokenziation.""" + + def __init__(self, vocab, unk_token, normalize_text=True): + """Constructs a CharacterTokenizer. + + Args: + **vocab**: + Vocabulary object. + **unk_token**: str + A special symbol for out-of-vocabulary token. + **normalize_text**: (`optional`) boolean (default True) + Whether to apply unicode normalization to text before tokenization. + """ + self.vocab = vocab + self.unk_token = unk_token + self.normalize_text = normalize_text + + def tokenize(self, text): + """Tokenizes a piece of text into characters. + + For example: + input = "apple" + output = ["a", "p", "p", "l", "e"] + Args: + text: A single token or whitespace separated tokens. + This should have already been passed through `BasicTokenizer`. + Returns: + A list of characters. + """ + if self.normalize_text: + text = unicodedata.normalize('NFKC', text) + + output_tokens = [] + for i, char in enumerate(text): + if char not in self.vocab: + output_tokens.append(self.unk_token) + continue + + output_tokens.append(char) + + return output_tokens From 57b5cb3eaa850a212235fccbd4e5d002aede72b6 Mon Sep 17 00:00:00 2001 From: Masatoshi Suzuki Date: Wed, 20 Nov 2019 09:02:10 +0900 Subject: [PATCH 326/505] Fix loading BertJapaneseTokenizer --- transformers/tokenization_auto.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/transformers/tokenization_auto.py b/transformers/tokenization_auto.py index d63b7e783d..f36a584521 100644 --- a/transformers/tokenization_auto.py +++ b/transformers/tokenization_auto.py @@ -73,6 +73,7 @@ class AutoTokenizer(object): - contains `albert`: AlbertTokenizer (ALBERT model) - contains `camembert`: CamembertTokenizer (CamemBERT model) - contains `roberta`: RobertaTokenizer (RoBERTa model) + - contains `bert-base-japanese`: BertJapaneseTokenizer (Bert model) - contains `bert`: BertTokenizer (Bert model) - contains `openai-gpt`: OpenAIGPTTokenizer (OpenAI GPT model) - contains `gpt2`: GPT2Tokenizer (OpenAI GPT-2 model) @@ -119,7 +120,7 @@ class AutoTokenizer(object): return CamembertTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return RobertaTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) - elif 'bert-japanese' in pretrained_model_name_or_path: + elif 'bert-base-japanese' in pretrained_model_name_or_path: return BertJapaneseTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) elif 'bert' in pretrained_model_name_or_path: return BertTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) From a09da4eeb0397dd66d61182177dd3b753d70e62a Mon Sep 17 00:00:00 2001 From: Masatoshi Suzuki Date: Fri, 29 Nov 2019 19:24:43 +0900 Subject: [PATCH 327/505] Add a test for Japanese BERT tokenizers --- .../tests/tokenization_bert_japanese_test.py | 192 ++++++++++++++++++ 1 file changed, 192 insertions(+) create mode 100644 transformers/tests/tokenization_bert_japanese_test.py diff --git a/transformers/tests/tokenization_bert_japanese_test.py b/transformers/tests/tokenization_bert_japanese_test.py new file mode 100644 index 0000000000..6f66b96411 --- /dev/null +++ b/transformers/tests/tokenization_bert_japanese_test.py @@ -0,0 +1,192 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import, division, print_function, unicode_literals + +import os +import unittest +import pytest +from io import open + +from transformers.tokenization_bert import WordpieceTokenizer +from transformers.tokenization_bert_japanese import (BertJapaneseTokenizer, + MecabTokenizer, CharacterTokenizer, + VOCAB_FILES_NAMES) + +from .tokenization_tests_commons import CommonTestCases + + +class BertJapaneseTokenizationTest(CommonTestCases.CommonTokenizerTester): + + tokenizer_class = BertJapaneseTokenizer + + def setUp(self): + super(BertJapaneseTokenizationTest, self).setUp() + + vocab_tokens = [u"[UNK]", u"[CLS]", u"[SEP]", + u"こんにちは", u"こん", u"にちは", u"ばんは", u"##こん", u"##にちは", u"##ばんは", + u"世界", u"##世界", u"、", u"##、", u"。", u"##。"] + + self.vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["vocab_file"]) + with open(self.vocab_file, "w", encoding="utf-8") as vocab_writer: + vocab_writer.write("".join([x + "\n" for x in vocab_tokens])) + + def get_tokenizer(self, **kwargs): + return BertJapaneseTokenizer.from_pretrained(self.tmpdirname, **kwargs) + + def get_input_output_texts(self): + input_text = u"こんにちは、世界。 \nこんばんは、世界。" + output_text = u"こんにちは 、 世界 。 こんばんは 、 世界 。" + return input_text, output_text + + def test_full_tokenizer(self): + tokenizer = self.tokenizer_class(self.vocab_file) + + tokens = tokenizer.tokenize(u"こんにちは、世界。\nこんばんは、世界。") + self.assertListEqual(tokens, + [u"こんにちは", u"、", u"世界", u"。", + u"こん", u"##ばんは", u"、", u"世界", "。"]) + self.assertListEqual(tokenizer.convert_tokens_to_ids(tokens), + [3, 12, 10, 14, 4, 9, 12, 10, 14]) + + def test_mecab_tokenizer(self): + tokenizer = MecabTokenizer() + + self.assertListEqual( + tokenizer.tokenize(u" \tアップルストアでiPhone8 が \n 発売された 。 "), + [u"アップルストア", u"で", u"iPhone", u"8", u"が", + u"発売", u"さ", u"れ", u"た", u"。"]) + + def test_mecab_tokenizer_lower(self): + tokenizer = MecabTokenizer(do_lower_case=True) + + self.assertListEqual( + tokenizer.tokenize(u" \tアップルストアでiPhone8 が \n 発売された 。 "), + [u"アップルストア", u"で", u"iphone", u"8", u"が", + u"発売", u"さ", u"れ", u"た", u"。"]) + + def test_mecab_tokenizer_no_normalize(self): + tokenizer = MecabTokenizer(normalize_text=False) + + self.assertListEqual( + tokenizer.tokenize(u" \tアップルストアでiPhone8 が \n 発売された 。 "), + [u"アップルストア", u"で", u"iPhone", u"8", u"が", + u"発売", u"さ", u"れ", u"た", u" ", u"。"]) + + def test_wordpiece_tokenizer(self): + vocab_tokens = [u"[UNK]", u"[CLS]", u"[SEP]", + u"こんにちは", u"こん", u"にちは" u"ばんは", u"##こん", u"##にちは", u"##ばんは"] + + vocab = {} + for (i, token) in enumerate(vocab_tokens): + vocab[token] = i + tokenizer = WordpieceTokenizer(vocab=vocab, unk_token=u"[UNK]") + + self.assertListEqual(tokenizer.tokenize(u""), []) + + self.assertListEqual(tokenizer.tokenize(u"こんにちは"), + [u"こんにちは"]) + + self.assertListEqual(tokenizer.tokenize(u"こんばんは"), + [u"こん", u"##ばんは"]) + + self.assertListEqual(tokenizer.tokenize(u"こんばんは こんばんにちは こんにちは"), + [u"こん", u"##ばんは", u"[UNK]", u"こんにちは"]) + + @pytest.mark.slow + def test_sequence_builders(self): + tokenizer = self.tokenizer_class.from_pretrained("bert-base-japanese") + + text = tokenizer.encode(u"ありがとう。", add_special_tokens=False) + text_2 = tokenizer.encode(u"どういたしまして。", add_special_tokens=False) + + encoded_sentence = tokenizer.build_inputs_with_special_tokens(text) + encoded_pair = tokenizer.build_inputs_with_special_tokens(text, text_2) + + # 2 is for "[CLS]", 3 is for "[SEP]" + assert encoded_sentence == [2] + text + [3] + assert encoded_pair == [2] + text + [3] + text_2 + [3] + + +class BertJapaneseCharacterTokenizationTest(CommonTestCases.CommonTokenizerTester): + + tokenizer_class = BertJapaneseTokenizer + + def setUp(self): + super(BertJapaneseCharacterTokenizationTest, self).setUp() + + vocab_tokens = [u"[UNK]", u"[CLS]", u"[SEP]", + u"こ", u"ん", u"に", u"ち", u"は", u"ば", u"世", u"界", u"、", u"。"] + + self.vocab_file = os.path.join(self.tmpdirname, VOCAB_FILES_NAMES["vocab_file"]) + with open(self.vocab_file, "w", encoding="utf-8") as vocab_writer: + vocab_writer.write("".join([x + "\n" for x in vocab_tokens])) + + def get_tokenizer(self, **kwargs): + return BertJapaneseTokenizer.from_pretrained(self.tmpdirname, + subword_tokenizer_type="character", + **kwargs) + + def get_input_output_texts(self): + input_text = u"こんにちは、世界。 \nこんばんは、世界。" + output_text = u"こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。" + return input_text, output_text + + def test_full_tokenizer(self): + tokenizer = self.tokenizer_class(self.vocab_file, + subword_tokenizer_type="character") + + tokens = tokenizer.tokenize(u"こんにちは、世界。 \nこんばんは、世界。") + self.assertListEqual(tokens, + [u"こ", u"ん", u"に", u"ち", u"は", u"、", u"世", u"界", u"。", + u"こ", u"ん", u"ば", u"ん", u"は", u"、", u"世", u"界", u"。"]) + self.assertListEqual(tokenizer.convert_tokens_to_ids(tokens), + [3, 4, 5, 6, 7, 11, 9, 10, 12, + 3, 4, 8, 4, 7, 11, 9, 10, 12]) + + def test_character_tokenizer(self): + vocab_tokens = [u"[UNK]", u"[CLS]", u"[SEP]", + u"こ", u"ん", u"に", u"ち", u"は", u"ば", u"世", u"界"u"、", u"。"] + + vocab = {} + for (i, token) in enumerate(vocab_tokens): + vocab[token] = i + tokenizer = CharacterTokenizer(vocab=vocab, unk_token=u"[UNK]") + + self.assertListEqual(tokenizer.tokenize(u""), []) + + self.assertListEqual(tokenizer.tokenize(u"こんにちは"), + [u"こ", u"ん", u"に", u"ち", u"は"]) + + self.assertListEqual(tokenizer.tokenize(u"こんにちほ"), + [u"こ", u"ん", u"に", u"ち", u"[UNK]"]) + + @pytest.mark.slow + def test_sequence_builders(self): + tokenizer = self.tokenizer_class.from_pretrained("bert-base-japanese-char") + + text = tokenizer.encode(u"ありがとう。", add_special_tokens=False) + text_2 = tokenizer.encode(u"どういたしまして。", add_special_tokens=False) + + encoded_sentence = tokenizer.build_inputs_with_special_tokens(text) + encoded_pair = tokenizer.build_inputs_with_special_tokens(text, text_2) + + # 2 is for "[CLS]", 3 is for "[SEP]" + assert encoded_sentence == [2] + text + [3] + assert encoded_pair == [2] + text + [3] + text_2 + [3] + + + +if __name__ == '__main__': + unittest.main() From 6a43dc9d7d592362d144209097e1d93876f8e88a Mon Sep 17 00:00:00 2001 From: Masatoshi Suzuki Date: Thu, 5 Dec 2019 11:19:02 +0900 Subject: [PATCH 328/505] Support Python 2 --- transformers/tokenization_bert_japanese.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/transformers/tokenization_bert_japanese.py b/transformers/tokenization_bert_japanese.py index 8554a1c880..1ce0e1d1cb 100644 --- a/transformers/tokenization_bert_japanese.py +++ b/transformers/tokenization_bert_japanese.py @@ -19,6 +19,7 @@ from __future__ import absolute_import, division, print_function, unicode_litera import collections import logging import os +import six import unicodedata from io import open @@ -186,8 +187,13 @@ class MecabTokenizer(object): never_split = self.never_split + (never_split if never_split is not None else []) tokens = [] + if six.PY2: + mecab_output = self.mecab.parse(text.encode('utf-8')).decode('utf-8') + else: + mecab_output = self.mecab.parse(text) + cursor = 0 - for line in self.mecab.parse(text).split('\n'): + for line in mecab_output.split('\n'): if line == 'EOS': break From 597ba7feb384316081c96955196fcb7abb2edf06 Mon Sep 17 00:00:00 2001 From: Masatoshi Suzuki Date: Thu, 5 Dec 2019 11:30:40 +0900 Subject: [PATCH 329/505] Support testing Japanese BERT tokenizers --- .circleci/config.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 01e6d82b33..97f5f25606 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,6 +13,8 @@ jobs: - run: sudo pip install --progress-bar off . - run: sudo pip install pytest codecov pytest-cov - run: sudo pip install tensorboardX scikit-learn + - run: sudo apt-get -y install libmecab-dev mecab mecab-ipadic-utf8 swig + - run: sudo pip install mecab-python3 - run: python -m pytest -sv ./transformers/tests/ --cov - run: codecov build_py3_torch: @@ -27,6 +29,8 @@ jobs: - run: sudo pip install --progress-bar off . - run: sudo pip install pytest codecov pytest-cov - run: sudo pip install tensorboardX scikit-learn + - run: sudo apt-get -y install libmecab-dev mecab mecab-ipadic-utf8 swig + - run: sudo pip install mecab-python3 - run: python -m pytest -sv ./transformers/tests/ --cov - run: python -m pytest -sv ./examples/ - run: codecov @@ -42,6 +46,8 @@ jobs: - run: sudo pip install --progress-bar off . - run: sudo pip install pytest codecov pytest-cov - run: sudo pip install tensorboardX scikit-learn + - run: sudo apt-get -y install libmecab-dev mecab mecab-ipadic-utf8 swig + - run: sudo pip install mecab-python3 - run: python -m pytest -sv ./transformers/tests/ --cov - run: codecov build_py2_torch: @@ -55,6 +61,8 @@ jobs: - run: sudo pip install torch - run: sudo pip install --progress-bar off . - run: sudo pip install pytest codecov pytest-cov + - run: sudo apt-get -y install libmecab-dev mecab mecab-ipadic-utf8 swig + - run: sudo pip install mecab-python - run: python -m pytest -sv ./transformers/tests/ --cov - run: codecov build_py2_tf: @@ -68,6 +76,8 @@ jobs: - run: sudo pip install tensorflow - run: sudo pip install --progress-bar off . - run: sudo pip install pytest codecov pytest-cov + - run: sudo apt-get -y install libmecab-dev mecab mecab-ipadic-utf8 swig + - run: sudo pip install mecab-python - run: python -m pytest -sv ./transformers/tests/ --cov - run: codecov deploy_doc: From d2100428d3652cefbffcf0bd00f0881090d26333 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Tue, 10 Dec 2019 21:43:49 +0000 Subject: [PATCH 330/505] Update to new test infra and only run conditionally --- .circleci/config.yml | 20 ++++----- .../tests/tokenization_bert_japanese_test.py | 9 ++-- transformers/tests/utils.py | 42 +++++++++++++------ 3 files changed, 44 insertions(+), 27 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 97f5f25606..7ca5f8121c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,8 +13,6 @@ jobs: - run: sudo pip install --progress-bar off . - run: sudo pip install pytest codecov pytest-cov - run: sudo pip install tensorboardX scikit-learn - - run: sudo apt-get -y install libmecab-dev mecab mecab-ipadic-utf8 swig - - run: sudo pip install mecab-python3 - run: python -m pytest -sv ./transformers/tests/ --cov - run: codecov build_py3_torch: @@ -29,8 +27,6 @@ jobs: - run: sudo pip install --progress-bar off . - run: sudo pip install pytest codecov pytest-cov - run: sudo pip install tensorboardX scikit-learn - - run: sudo apt-get -y install libmecab-dev mecab mecab-ipadic-utf8 swig - - run: sudo pip install mecab-python3 - run: python -m pytest -sv ./transformers/tests/ --cov - run: python -m pytest -sv ./examples/ - run: codecov @@ -46,8 +42,6 @@ jobs: - run: sudo pip install --progress-bar off . - run: sudo pip install pytest codecov pytest-cov - run: sudo pip install tensorboardX scikit-learn - - run: sudo apt-get -y install libmecab-dev mecab mecab-ipadic-utf8 swig - - run: sudo pip install mecab-python3 - run: python -m pytest -sv ./transformers/tests/ --cov - run: codecov build_py2_torch: @@ -61,8 +55,6 @@ jobs: - run: sudo pip install torch - run: sudo pip install --progress-bar off . - run: sudo pip install pytest codecov pytest-cov - - run: sudo apt-get -y install libmecab-dev mecab mecab-ipadic-utf8 swig - - run: sudo pip install mecab-python - run: python -m pytest -sv ./transformers/tests/ --cov - run: codecov build_py2_tf: @@ -76,10 +68,18 @@ jobs: - run: sudo pip install tensorflow - run: sudo pip install --progress-bar off . - run: sudo pip install pytest codecov pytest-cov - - run: sudo apt-get -y install libmecab-dev mecab mecab-ipadic-utf8 swig - - run: sudo pip install mecab-python - run: python -m pytest -sv ./transformers/tests/ --cov - run: codecov + build_py3_custom_tokenizers: + working_directory: ~/transformers + docker: + - image: circleci/python:3.5 + steps: + - checkout + - run: sudo pip install --progress-bar off . + - run: sudo pip install pytest + - run: sudo pip install mecab-python3 + - run: python -m pytest -sv ./transformers/tests/tokenization_bert_japanese_test.py deploy_doc: working_directory: ~/transformers docker: diff --git a/transformers/tests/tokenization_bert_japanese_test.py b/transformers/tests/tokenization_bert_japanese_test.py index 6f66b96411..545193c7cc 100644 --- a/transformers/tests/tokenization_bert_japanese_test.py +++ b/transformers/tests/tokenization_bert_japanese_test.py @@ -16,7 +16,6 @@ from __future__ import absolute_import, division, print_function, unicode_litera import os import unittest -import pytest from io import open from transformers.tokenization_bert import WordpieceTokenizer @@ -25,8 +24,10 @@ from transformers.tokenization_bert_japanese import (BertJapaneseTokenizer, VOCAB_FILES_NAMES) from .tokenization_tests_commons import CommonTestCases +from .utils import slow, custom_tokenizers +@custom_tokenizers class BertJapaneseTokenizationTest(CommonTestCases.CommonTokenizerTester): tokenizer_class = BertJapaneseTokenizer @@ -104,7 +105,7 @@ class BertJapaneseTokenizationTest(CommonTestCases.CommonTokenizerTester): self.assertListEqual(tokenizer.tokenize(u"こんばんは こんばんにちは こんにちは"), [u"こん", u"##ばんは", u"[UNK]", u"こんにちは"]) - @pytest.mark.slow + @slow def test_sequence_builders(self): tokenizer = self.tokenizer_class.from_pretrained("bert-base-japanese") @@ -172,7 +173,7 @@ class BertJapaneseCharacterTokenizationTest(CommonTestCases.CommonTokenizerTeste self.assertListEqual(tokenizer.tokenize(u"こんにちほ"), [u"こ", u"ん", u"に", u"ち", u"[UNK]"]) - @pytest.mark.slow + @slow def test_sequence_builders(self): tokenizer = self.tokenizer_class.from_pretrained("bert-base-japanese-char") @@ -188,5 +189,3 @@ class BertJapaneseCharacterTokenizationTest(CommonTestCases.CommonTokenizerTeste -if __name__ == '__main__': - unittest.main() diff --git a/transformers/tests/utils.py b/transformers/tests/utils.py index 7a51ab612b..2b97293ca7 100644 --- a/transformers/tests/utils.py +++ b/transformers/tests/utils.py @@ -6,18 +6,23 @@ from distutils.util import strtobool from transformers.file_utils import _tf_available, _torch_available -try: - run_slow = os.environ["RUN_SLOW"] -except KeyError: - # RUN_SLOW isn't set, default to skipping slow tests. - _run_slow_tests = False -else: - # RUN_SLOW is set, convert it to True or False. +def parse_flag_from_env(key, default=False): try: - _run_slow_tests = strtobool(run_slow) - except ValueError: - # More values are supported, but let's keep the message simple. - raise ValueError("If set, RUN_SLOW must be yes or no.") + value = os.environ[key] + except KeyError: + # KEY isn't set, default to `default`. + _value = default + else: + # KEY is set, convert it to True or False. + try: + _value = strtobool(value) + except ValueError: + # More values are supported, but let's keep the message simple. + raise ValueError("If set, {} must be yes or no.".format(key)) + return _value + +_run_slow_tests = parse_flag_from_env("RUN_SLOW", default=False) +_run_custom_tokenizers = parse_flag_from_env("RUN_CUSTOM_TOKENIZERS", default=False) def slow(test_case): @@ -33,6 +38,19 @@ def slow(test_case): return test_case +def custom_tokenizers(test_case): + """ + Decorator marking a test for a custom tokenizer. + + Custom tokenizers require additional dependencies, and are skipped + by default. Set the RUN_CUSTOM_TOKENIZERS environment variable + to a truthy value to run them. + """ + if not _run_custom_tokenizers: + test_case = unittest.skip("test of custom tokenizers")(test_case) + return test_case + + def require_torch(test_case): """ Decorator marking a test that requires PyTorch. @@ -59,6 +77,6 @@ def require_tf(test_case): if _torch_available: # Set the USE_CUDA environment variable to select a GPU. - torch_device = "cuda" if os.environ.get("USE_CUDA") else "cpu" + torch_device = "cuda" if parse_flag_from_env("USE_CUDA") else "cpu" else: torch_device = None From 95854c4a2f8d418a14e64b4edf64fc7363b1ff10 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Tue, 10 Dec 2019 21:46:00 +0000 Subject: [PATCH 331/505] Actually run the tests --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7ca5f8121c..d8f624a0e5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -79,7 +79,7 @@ jobs: - run: sudo pip install --progress-bar off . - run: sudo pip install pytest - run: sudo pip install mecab-python3 - - run: python -m pytest -sv ./transformers/tests/tokenization_bert_japanese_test.py + - run: RUN_CUSTOM_TOKENIZERS=1 python -m pytest -sv ./transformers/tests/tokenization_bert_japanese_test.py deploy_doc: working_directory: ~/transformers docker: From 9cb97c0c0f7215971bb5a39cd070e5bd89319bdf Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Tue, 10 Dec 2019 21:48:56 +0000 Subject: [PATCH 332/505] Actually run the tests --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index d8f624a0e5..9d6e02d580 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -101,6 +101,7 @@ workflows: version: 2 build_and_test: jobs: + - build_py3_custom_tokenizers - build_py3_torch_and_tf - build_py3_torch - build_py3_tf From 5505cf701477762cedf792e20344d29bc8bf6325 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Tue, 10 Dec 2019 21:53:44 +0000 Subject: [PATCH 333/505] Run tests on Py2 too, for Lysandre --- .circleci/config.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9d6e02d580..afc6d5ce44 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -80,6 +80,16 @@ jobs: - run: sudo pip install pytest - run: sudo pip install mecab-python3 - run: RUN_CUSTOM_TOKENIZERS=1 python -m pytest -sv ./transformers/tests/tokenization_bert_japanese_test.py + build_py2_custom_tokenizers: + working_directory: ~/transformers + docker: + - image: circleci/python:2.7 + steps: + - checkout + - run: sudo pip install --progress-bar off . + - run: sudo pip install pytest + - run: sudo pip install mecab-python + - run: RUN_CUSTOM_TOKENIZERS=1 python -m pytest -sv ./transformers/tests/tokenization_bert_japanese_test.py deploy_doc: working_directory: ~/transformers docker: @@ -102,6 +112,7 @@ workflows: build_and_test: jobs: - build_py3_custom_tokenizers + - build_py2_custom_tokenizers - build_py3_torch_and_tf - build_py3_torch - build_py3_tf From 371c5ddfad96689771465aff557152322190b60e Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Tue, 10 Dec 2019 21:55:43 +0000 Subject: [PATCH 334/505] Py2 tests for Lysandre --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index afc6d5ce44..c827a81fbb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -88,6 +88,7 @@ jobs: - checkout - run: sudo pip install --progress-bar off . - run: sudo pip install pytest + - run: sudo apt-get -y install libmecab-dev mecab mecab-ipadic-utf8 swig - run: sudo pip install mecab-python - run: RUN_CUSTOM_TOKENIZERS=1 python -m pytest -sv ./transformers/tests/tokenization_bert_japanese_test.py deploy_doc: From 36fc52a3b4b50885d5ec3bf259f81740e19d8b3c Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Tue, 10 Dec 2019 22:03:35 +0000 Subject: [PATCH 335/505] Update links to weights --- transformers/configuration_bert.py | 8 ++++---- transformers/modeling_bert.py | 8 ++++---- transformers/modeling_tf_bert.py | 8 ++++---- transformers/tokenization_bert_japanese.py | 8 ++++---- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/transformers/configuration_bert.py b/transformers/configuration_bert.py index 16f1f60404..01fcd88cb8 100644 --- a/transformers/configuration_bert.py +++ b/transformers/configuration_bert.py @@ -42,10 +42,10 @@ BERT_PRETRAINED_CONFIG_ARCHIVE_MAP = { 'bert-base-cased-finetuned-mrpc': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-finetuned-mrpc-config.json", 'bert-base-german-dbmdz-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-cased-config.json", 'bert-base-german-dbmdz-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-uncased-config.json", - 'bert-base-japanese': "https://www.nlp.ecei.tohoku.ac.jp/~m-suzuki/bert-japanese/bert-base-japanese-config.json", - 'bert-base-japanese-whole-word-masking': "https://www.nlp.ecei.tohoku.ac.jp/~m-suzuki/bert-japanese/bert-base-japanese-whole-word-masking-config.json", - 'bert-base-japanese-char': "https://www.nlp.ecei.tohoku.ac.jp/~m-suzuki/bert-japanese/bert-base-japanese-char-config.json", - 'bert-base-japanese-char-whole-word-masking': "https://www.nlp.ecei.tohoku.ac.jp/~m-suzuki/bert-japanese/bert-base-japanese-char-whole-word-masking-config.json" + 'bert-base-japanese': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-config.json", + 'bert-base-japanese-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-whole-word-masking-config.json", + 'bert-base-japanese-char': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-config.json", + 'bert-base-japanese-char-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-whole-word-masking-config.json" } diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py index e2e115a015..d0f35272ac 100644 --- a/transformers/modeling_bert.py +++ b/transformers/modeling_bert.py @@ -48,10 +48,10 @@ BERT_PRETRAINED_MODEL_ARCHIVE_MAP = { 'bert-base-cased-finetuned-mrpc': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-finetuned-mrpc-pytorch_model.bin", 'bert-base-german-dbmdz-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-cased-pytorch_model.bin", 'bert-base-german-dbmdz-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-uncased-pytorch_model.bin", - 'bert-base-japanese': "https://www.nlp.ecei.tohoku.ac.jp/~m-suzuki/bert-japanese/bert-base-japanese-pytorch_model.bin", - 'bert-base-japanese-whole-word-masking': "https://www.nlp.ecei.tohoku.ac.jp/~m-suzuki/bert-japanese/bert-base-japanese-whole-word-masking-pytorch_model.bin", - 'bert-base-japanese-char': "https://www.nlp.ecei.tohoku.ac.jp/~m-suzuki/bert-japanese/bert-base-japanese-char-pytorch_model.bin", - 'bert-base-japanese-char-whole-word-masking': "https://www.nlp.ecei.tohoku.ac.jp/~m-suzuki/bert-japanese/bert-base-japanese-char-whole-word-masking-pytorch_model.bin" + 'bert-base-japanese': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-pytorch_model.bin", + 'bert-base-japanese-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-whole-word-masking-pytorch_model.bin", + 'bert-base-japanese-char': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-pytorch_model.bin", + 'bert-base-japanese-char-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-whole-word-masking-pytorch_model.bin" } diff --git a/transformers/modeling_tf_bert.py b/transformers/modeling_tf_bert.py index 27dd311a4d..7cc71f5063 100644 --- a/transformers/modeling_tf_bert.py +++ b/transformers/modeling_tf_bert.py @@ -48,10 +48,10 @@ TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP = { 'bert-large-uncased-whole-word-masking-finetuned-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-whole-word-masking-finetuned-squad-tf_model.h5", 'bert-large-cased-whole-word-masking-finetuned-squad': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-whole-word-masking-finetuned-squad-tf_model.h5", 'bert-base-cased-finetuned-mrpc': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-finetuned-mrpc-tf_model.h5", - 'bert-base-japanese': "https://www.nlp.ecei.tohoku.ac.jp/~m-suzuki/bert-japanese/bert-base-japanese-tf_model.h5", - 'bert-base-japanese-whole-word-masking': "https://www.nlp.ecei.tohoku.ac.jp/~m-suzuki/bert-japanese/bert-base-japanese-whole-word-masking-tf_model.h5", - 'bert-base-japanese-char': "https://www.nlp.ecei.tohoku.ac.jp/~m-suzuki/bert-japanese/bert-base-japanese-char-tf_model.h5", - 'bert-base-japanese-char-whole-word-masking': "https://www.nlp.ecei.tohoku.ac.jp/~m-suzuki/bert-japanese/bert-base-japanese-char-whole-word-masking-tf_model.h5" + 'bert-base-japanese': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-tf_model.h5", + 'bert-base-japanese-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-whole-word-masking-tf_model.h5", + 'bert-base-japanese-char': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-tf_model.h5", + 'bert-base-japanese-char-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-whole-word-masking-tf_model.h5" } diff --git a/transformers/tokenization_bert_japanese.py b/transformers/tokenization_bert_japanese.py index 1ce0e1d1cb..0ff45cbfe7 100644 --- a/transformers/tokenization_bert_japanese.py +++ b/transformers/tokenization_bert_japanese.py @@ -33,10 +33,10 @@ VOCAB_FILES_NAMES = {'vocab_file': 'vocab.txt'} PRETRAINED_VOCAB_FILES_MAP = { 'vocab_file': { - 'bert-base-japanese': "https://www.nlp.ecei.tohoku.ac.jp/~m-suzuki/bert-japanese/bert-base-japanese-vocab.txt", - 'bert-base-japanese-whole-word-masking': "https://www.nlp.ecei.tohoku.ac.jp/~m-suzuki/bert-japanese/bert-base-japanese-whole-word-masking-vocab.txt", - 'bert-base-japanese-char': "https://www.nlp.ecei.tohoku.ac.jp/~m-suzuki/bert-japanese/bert-base-japanese-char-vocab.txt", - 'bert-base-japanese-char-whole-word-masking': "https://www.nlp.ecei.tohoku.ac.jp/~m-suzuki/bert-japanese/bert-base-japanese-char-whole-word-masking-vocab.txt" + 'bert-base-japanese': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-vocab.txt", + 'bert-base-japanese-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-whole-word-masking-vocab.txt", + 'bert-base-japanese-char': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-vocab.txt", + 'bert-base-japanese-char-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-whole-word-masking-vocab.txt" } } From 1748fdf657ed804f3edc1e45077b703cd8d6e4c5 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Wed, 11 Dec 2019 23:31:23 +0000 Subject: [PATCH 336/505] [doc] Fix rst table --- docs/source/pretrained_models.rst | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst index d3498e057d..775772e896 100644 --- a/docs/source/pretrained_models.rst +++ b/docs/source/pretrained_models.rst @@ -63,22 +63,22 @@ Here is the full list of the currently provided pretrained models together with | | | (see `details on dbmdz repository `__). | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``bert-base-japanese`` | | 12-layer, 768-hidden, 12-heads, 110M parameters. | -| | | | Trained on Japanese text. Text is tokenized with MeCab and WordPiece. | -| | | | `MeCab `__ is required for tokenization. | -| | | (see `details on cl-tohoku repository `__). | +| | | | Trained on Japanese text. Text is tokenized with MeCab and WordPiece. | +| | | | `MeCab `__ is required for tokenization. | +| | | (see `details on cl-tohoku repository `__). | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``bert-base-japanese-whole-word-masking`` | | 12-layer, 768-hidden, 12-heads, 110M parameters. | -| | | | Trained on Japanese text using Whole-Word-Masking. Text is tokenized with MeCab and WordPiece. | -| | | | `MeCab `__ is required for tokenization. | -| | | (see `details on cl-tohoku repository `__). | +| | | | Trained on Japanese text using Whole-Word-Masking. Text is tokenized with MeCab and WordPiece. | +| | | | `MeCab `__ is required for tokenization. | +| | | (see `details on cl-tohoku repository `__). | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``bert-base-japanese-char`` | | 12-layer, 768-hidden, 12-heads, 110M parameters. | -| | | | Trained on Japanese text. Text is tokenized into characters. | -| | | (see `details on cl-tohoku repository `__). | +| | | | Trained on Japanese text. Text is tokenized into characters. | +| | | (see `details on cl-tohoku repository `__). | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``bert-base-japanese-char-whole-word-masking`` | | 12-layer, 768-hidden, 12-heads, 110M parameters. | -| | | | Trained on Japanese text using Whole-Word-Masking. Text is tokenized into characters. | -| | | (see `details on cl-tohoku repository `__). | +| | | | Trained on Japanese text using Whole-Word-Masking. Text is tokenized into characters. | +| | | (see `details on cl-tohoku repository `__). | +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | GPT | ``openai-gpt`` | | 12-layer, 768-hidden, 12-heads, 110M parameters. | | | | | OpenAI GPT English model | From 413f41921b650418798f7d5c246316c4e1e5eb5d Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 12 Dec 2019 07:34:42 +0100 Subject: [PATCH 337/505] fix merge --- transformers/tests/utils.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/transformers/tests/utils.py b/transformers/tests/utils.py index daed431995..c950ad8f17 100644 --- a/transformers/tests/utils.py +++ b/transformers/tests/utils.py @@ -9,14 +9,6 @@ from transformers.file_utils import _tf_available, _torch_available SMALL_MODEL_IDENTIFIER = "julien-c/bert-xsmall-dummy" -try: - run_slow = os.environ["RUN_SLOW"] -except KeyError: - # RUN_SLOW isn't set, default to skipping slow tests. - _run_slow_tests = False -else: - # RUN_SLOW is set, convert it to True or False. - def parse_flag_from_env(key, default=False): try: value = os.environ[key] From f69dbecc38e04cd4d158afb273921ca7b75c7cba Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Thu, 12 Dec 2019 10:25:36 +0100 Subject: [PATCH 338/505] Expose classification labels mapping (and reverse) in model config. --- transformers/configuration_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/transformers/configuration_utils.py b/transformers/configuration_utils.py index 08cee75d81..97b9fa8f80 100644 --- a/transformers/configuration_utils.py +++ b/transformers/configuration_utils.py @@ -58,6 +58,8 @@ class PretrainedConfig(object): self.use_bfloat16 = kwargs.pop('use_bfloat16', False) self.pruned_heads = kwargs.pop('pruned_heads', {}) self.is_decoder = kwargs.pop('is_decoder', False) + self.idx2label = kwargs.pop('idx2label', {i: 'LABEL_{}'.format(i) for i in range(self.num_labels)}) + self.label2idx = kwargs.pop('label2idx', dict(zip(self.idx2label.values(), self.idx2label.keys()))) def save_pretrained(self, save_directory): """ Save a configuration object to the directory `save_directory`, so that it From f19dad61c70a628545612e435c699263f02bc4a0 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 12 Dec 2019 14:46:30 +0100 Subject: [PATCH 339/505] fixing XLM conversion tests with dummy input --- transformers/modeling_tf_pytorch_utils.py | 6 +++++- transformers/modeling_tf_xlm.py | 2 +- transformers/modeling_xlm.py | 12 +++++++++++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/transformers/modeling_tf_pytorch_utils.py b/transformers/modeling_tf_pytorch_utils.py index 510e130c90..9d2b663dcb 100644 --- a/transformers/modeling_tf_pytorch_utils.py +++ b/transformers/modeling_tf_pytorch_utils.py @@ -78,6 +78,7 @@ def load_pytorch_checkpoint_in_tf2_model(tf_model, pytorch_checkpoint_path, tf_i logger.info("Loading PyTorch weights from {}".format(pt_path)) pt_state_dict = torch.load(pt_path, map_location='cpu') + logger.info("PyTorch checkpoint contains {:,} parameters".format(sum(t.numel() for t in pt_state_dict.values()))) return load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=tf_inputs, allow_missing_keys=allow_missing_keys) @@ -134,7 +135,7 @@ def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None, a start_prefix_to_remove = tf_model.base_model_prefix + '.' symbolic_weights = tf_model.trainable_weights + tf_model.non_trainable_weights - + tf_loaded_numel = 0 weight_value_tuples = [] all_pytorch_weights = set(list(pt_state_dict.keys())) for symbolic_weight in symbolic_weights: @@ -159,6 +160,7 @@ def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None, a e.args += (symbolic_weight.shape, array.shape) raise e + tf_loaded_numel += array.size # logger.warning("Initialize TF weight {}".format(symbolic_weight.name)) weight_value_tuples.append((symbolic_weight, array)) @@ -169,6 +171,8 @@ def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None, a if tf_inputs is not None: tfo = tf_model(tf_inputs, training=False) # Make sure restore ops are run + logger.info("Loaded {:,} parameters in the TF 2.0 model.".format(tf_loaded_numel)) + logger.info("Weights or buffers not loaded from PyTorch model: {}".format(all_pytorch_weights)) return tf_model diff --git a/transformers/modeling_tf_xlm.py b/transformers/modeling_tf_xlm.py index 6f11b0537d..903a8596c3 100644 --- a/transformers/modeling_tf_xlm.py +++ b/transformers/modeling_tf_xlm.py @@ -460,7 +460,7 @@ class TFXLMPreTrainedModel(TFPreTrainedModel): langs_list = tf.constant([[1, 1, 0, 0, 1], [1, 1, 1, 0, 0], [1, 0, 0, 1, 1]]) else: langs_list = None - return [inputs_list, attns_list, langs_list] + return {'input_ids': inputs_list, 'attention_mask': attns_list, 'langs': langs_list} XLM_START_DOCSTRING = r""" The XLM model was proposed in diff --git a/transformers/modeling_xlm.py b/transformers/modeling_xlm.py index 257f0da394..b604ae669d 100644 --- a/transformers/modeling_xlm.py +++ b/transformers/modeling_xlm.py @@ -227,6 +227,16 @@ class XLMPreTrainedModel(PreTrainedModel): def __init__(self, *inputs, **kwargs): super(XLMPreTrainedModel, self).__init__(*inputs, **kwargs) + @property + def dummy_inputs(self): + inputs_list = torch.tensor([[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]]) + attns_list = torch.tensor([[1, 1, 0, 0, 1], [1, 1, 1, 0, 0], [1, 0, 0, 1, 1]]) + if self.config.use_lang_emb and self.config.n_langs > 1: + langs_list = torch.tensor([[1, 1, 0, 0, 1], [1, 1, 1, 0, 0], [1, 0, 0, 1, 1]]) + else: + langs_list = None + return {'input_ids': inputs_list, 'attention_mask': attns_list, 'langs': langs_list} + def _init_weights(self, module): """ Initialize the weights. """ if isinstance(module, nn.Embedding): @@ -646,7 +656,7 @@ class XLMWithLMHeadModel(XLMPreTrainedModel): langs=langs, token_type_ids=token_type_ids, position_ids=position_ids, - lengths=lengths, + lengths=lengths, cache=cache, head_mask=head_mask, inputs_embeds=inputs_embeds) From fbf5455a8607fa660aacbf06c16f6fe23758b13d Mon Sep 17 00:00:00 2001 From: Alan deLevie Date: Wed, 11 Dec 2019 10:14:48 -0500 Subject: [PATCH 340/505] Fix typo in examples/run_glue.py args declaration. deay -> decay --- examples/run_glue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/run_glue.py b/examples/run_glue.py index 369a7110ab..1a51255c11 100644 --- a/examples/run_glue.py +++ b/examples/run_glue.py @@ -380,7 +380,7 @@ def main(): parser.add_argument("--learning_rate", default=5e-5, type=float, help="The initial learning rate for Adam.") parser.add_argument("--weight_decay", default=0.0, type=float, - help="Weight deay if we apply some.") + help="Weight decay if we apply some.") parser.add_argument("--adam_epsilon", default=1e-8, type=float, help="Epsilon for Adam optimizer.") parser.add_argument("--max_grad_norm", default=1.0, type=float, From fe92755b992eb61239ad361abae3b71f86bbbba1 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Thu, 12 Dec 2019 11:37:19 -0500 Subject: [PATCH 341/505] Fix special tokens mask in encode --- transformers/tokenization_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index f44b77b27c..7e86742286 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -973,7 +973,7 @@ class PreTrainedTokenizer(object): token_type_ids = [0] * len(ids) + ([1] * len(pair_ids) if pair else []) if return_special_tokens_mask: - encoded_inputs["special_tokens_mask"] = special_tokens_mask + encoded_inputs["special_tokens_mask"] = self.get_special_tokens_mask(ids, pair_ids) encoded_inputs["input_ids"] = sequence if return_token_type_ids: From 5d67aa21aefaaa62594e8dfb56093b83c5f547bb Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Thu, 12 Dec 2019 12:39:41 -0500 Subject: [PATCH 342/505] [doc] Replicate doc from #2144 --- transformers/configuration_auto.py | 1 + transformers/configuration_utils.py | 1 + transformers/modeling_auto.py | 4 ++++ transformers/modeling_encoder_decoder.py | 2 ++ transformers/modeling_tf_auto.py | 4 ++++ transformers/modeling_tf_utils.py | 1 + transformers/modeling_utils.py | 1 + transformers/tokenization_auto.py | 11 +++++++++-- transformers/tokenization_utils.py | 4 ++-- 9 files changed, 25 insertions(+), 4 deletions(-) diff --git a/transformers/configuration_auto.py b/transformers/configuration_auto.py index 43f251bd0c..fbc5c59199 100644 --- a/transformers/configuration_auto.py +++ b/transformers/configuration_auto.py @@ -83,6 +83,7 @@ class AutoConfig(object): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model configuration to load from cache or download, e.g.: ``bert-base-uncased``. + - a string with the `identifier name` of a pre-trained model configuration that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. - a path to a `directory` containing a configuration file saved using the :func:`~transformers.PretrainedConfig.save_pretrained` method, e.g.: ``./my_model_directory/``. - a path or url to a saved configuration JSON `file`, e.g.: ``./my_model_directory/configuration.json``. diff --git a/transformers/configuration_utils.py b/transformers/configuration_utils.py index 8ae30f2a48..82959adb57 100644 --- a/transformers/configuration_utils.py +++ b/transformers/configuration_utils.py @@ -79,6 +79,7 @@ class PretrainedConfig(object): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model configuration to load from cache or download, e.g.: ``bert-base-uncased``. + - a string with the `identifier name` of a pre-trained model configuration that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. - a path to a `directory` containing a configuration file saved using the :func:`~transformers.PretrainedConfig.save_pretrained` method, e.g.: ``./my_model_directory/``. - a path or url to a saved configuration JSON `file`, e.g.: ``./my_model_directory/configuration.json``. diff --git a/transformers/modeling_auto.py b/transformers/modeling_auto.py index 6ba1aab7a3..96f45d8ec4 100644 --- a/transformers/modeling_auto.py +++ b/transformers/modeling_auto.py @@ -93,6 +93,7 @@ class AutoModel(object): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a string with the `identifier name` of a pre-trained model that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. @@ -231,6 +232,7 @@ class AutoModelWithLMHead(object): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a string with the `identifier name` of a pre-trained model that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. @@ -360,6 +362,7 @@ class AutoModelForSequenceClassification(object): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a string with the `identifier name` of a pre-trained model that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. @@ -478,6 +481,7 @@ class AutoModelForQuestionAnswering(object): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a string with the `identifier name` of a pre-trained model that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. diff --git a/transformers/modeling_encoder_decoder.py b/transformers/modeling_encoder_decoder.py index a884abd0a2..70f765b849 100644 --- a/transformers/modeling_encoder_decoder.py +++ b/transformers/modeling_encoder_decoder.py @@ -59,12 +59,14 @@ class PreTrainedEncoderDecoder(nn.Module): encoder_pretrained_model_name_or_path: information necessary to initiate the encoder. Either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a string with the `identifier name` of a pre-trained model that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/encoder``. - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. decoder_pretrained_model_name_or_path: information necessary to initiate the decoder. Either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a string with the `identifier name` of a pre-trained model that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/decoder``. - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. diff --git a/transformers/modeling_tf_auto.py b/transformers/modeling_tf_auto.py index cfe19ead2a..fac92eb866 100644 --- a/transformers/modeling_tf_auto.py +++ b/transformers/modeling_tf_auto.py @@ -81,6 +81,7 @@ class TFAutoModel(object): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a string with the `identifier name` of a pre-trained model that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - a path or url to a `PyTorch, TF 1.X or TF 2.0 checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In the case of a PyTorch checkpoint, ``from_pt`` should be set to True and a configuration object should be provided as ``config`` argument. @@ -212,6 +213,7 @@ class TFAutoModelWithLMHead(object): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a string with the `identifier name` of a pre-trained model that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - a path or url to a `PyTorch, TF 1.X or TF 2.0 checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In the case of a PyTorch checkpoint, ``from_pt`` should be set to True and a configuration object should be provided as ``config`` argument. @@ -338,6 +340,7 @@ class TFAutoModelForSequenceClassification(object): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a string with the `identifier name` of a pre-trained model that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - a path or url to a `PyTorch, TF 1.X or TF 2.0 checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In the case of a PyTorch checkpoint, ``from_pt`` should be set to True and a configuration object should be provided as ``config`` argument. @@ -453,6 +456,7 @@ class TFAutoModelForQuestionAnswering(object): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a string with the `identifier name` of a pre-trained model that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - a path or url to a `PyTorch, TF 1.X or TF 2.0 checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In the case of a PyTorch checkpoint, ``from_pt`` should be set to True and a configuration object should be provided as ``config`` argument. diff --git a/transformers/modeling_tf_utils.py b/transformers/modeling_tf_utils.py index 4a6d18f447..d9a93af21b 100644 --- a/transformers/modeling_tf_utils.py +++ b/transformers/modeling_tf_utils.py @@ -177,6 +177,7 @@ class TFPreTrainedModel(tf.keras.Model): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a string with the `identifier name` of a pre-trained model that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - a path or url to a `PyTorch state_dict save file` (e.g. `./pt_model/pytorch_model.bin`). In this case, ``from_pt`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the PyTorch checkpoint in a TensorFlow model using the provided conversion scripts and loading the TensorFlow model afterwards. diff --git a/transformers/modeling_utils.py b/transformers/modeling_utils.py index 37088f8e67..676f355986 100644 --- a/transformers/modeling_utils.py +++ b/transformers/modeling_utils.py @@ -266,6 +266,7 @@ class PreTrainedModel(nn.Module): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a pre-trained model to load from cache or download, e.g.: ``bert-base-uncased``. + - a string with the `identifier name` of a pre-trained model that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. - a path to a `directory` containing model weights saved using :func:`~transformers.PreTrainedModel.save_pretrained`, e.g.: ``./my_model_directory/``. - a path or url to a `tensorflow index checkpoint file` (e.g. `./tf_model/model.ckpt.index`). In this case, ``from_tf`` should be set to True and a configuration object should be provided as ``config`` argument. This loading path is slower than converting the TensorFlow checkpoint in a PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards. - None if you are both providing the configuration and state dictionary (resp. with keyword arguments ``config`` and ``state_dict``) diff --git a/transformers/tokenization_auto.py b/transformers/tokenization_auto.py index f36a584521..1f0599ef7f 100644 --- a/transformers/tokenization_auto.py +++ b/transformers/tokenization_auto.py @@ -86,6 +86,7 @@ class AutoTokenizer(object): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a predefined tokenizer to load from cache or download, e.g.: ``bert-base-uncased``. + - a string with the `identifier name` of a predefined tokenizer that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. - a path to a `directory` containing vocabulary files required by the tokenizer, for instance saved using the :func:`~transformers.PreTrainedTokenizer.save_pretrained` method, e.g.: ``./my_model_directory/``. - (not applicable to all derived classes) a path or url to a single saved vocabulary file if and only if the tokenizer only requires a single vocabulary file (e.g. Bert, XLNet), e.g.: ``./my_model_directory/vocab.txt``. @@ -108,8 +109,14 @@ class AutoTokenizer(object): Examples:: - tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased') # Download vocabulary from S3 and cache. - tokenizer = AutoTokenizer.from_pretrained('./test/bert_saved_model/') # E.g. tokenizer was saved using `save_pretrained('./test/saved_model/')` + # Download vocabulary from S3 and cache. + tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased') + + # Download vocabulary from S3 (user-uploaded) and cache. + tokenizer = AutoTokenizer.from_pretrained('dbmdz/bert-base-german-cased') + + # If vocabulary files are in a directory (e.g. tokenizer was saved using `save_pretrained('./test/saved_model/')`) + tokenizer = AutoTokenizer.from_pretrained('./test/bert_saved_model/') """ if 'distilbert' in pretrained_model_name_or_path: diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index 7e86742286..317ecd167b 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -255,7 +255,7 @@ class PreTrainedTokenizer(object): pretrained_model_name_or_path: either: - a string with the `shortcut name` of a predefined tokenizer to load from cache or download, e.g.: ``bert-base-uncased``. - - a string with the `identifier name` of a predefined tokenizer that was user-uploaded to our S3, e.g.: ``dbmz/bert-base-german-cased``. + - a string with the `identifier name` of a predefined tokenizer that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. - a path to a `directory` containing vocabulary files required by the tokenizer, for instance saved using the :func:`~transformers.PreTrainedTokenizer.save_pretrained` method, e.g.: ``./my_model_directory/``. - (not applicable to all derived classes) a path or url to a single saved vocabulary file if and only if the tokenizer only requires a single vocabulary file (e.g. Bert, XLNet), e.g.: ``./my_model_directory/vocab.txt``. @@ -284,7 +284,7 @@ class PreTrainedTokenizer(object): tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') # Download vocabulary from S3 (user-uploaded) and cache. - tokenizer = BertTokenizer.from_pretrained('dbmz/bert-base-german-cased') + tokenizer = BertTokenizer.from_pretrained('dbmdz/bert-base-german-cased') # If vocabulary files are in a directory (e.g. tokenizer was saved using `save_pretrained('./test/saved_model/')`) tokenizer = BertTokenizer.from_pretrained('./test/saved_model/') From 7296f1010b6faaf3b1fb409bc5a9ebadcea51973 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Thu, 12 Dec 2019 13:01:04 -0500 Subject: [PATCH 343/505] Cleanup squad and add allow train_file and predict_file usage --- examples/run_squad.py | 22 ++++++++++++++-------- transformers/data/processors/squad.py | 6 ++++++ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/examples/run_squad.py b/examples/run_squad.py index 79c8537a4b..117b86e32c 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -337,7 +337,7 @@ def load_and_cache_examples(args, tokenizer, evaluate=False, output_examples=Fal else: logger.info("Creating features from dataset file at %s", input_dir) - if not args.data_dir: + if not args.data_dir and ((evaluate and not args.predict_file) or (not evaluate and not args.train_file)): try: import tensorflow_datasets as tfds except ImportError: @@ -350,7 +350,11 @@ def load_and_cache_examples(args, tokenizer, evaluate=False, output_examples=Fal examples = SquadV1Processor().get_examples_from_dataset(tfds_examples, evaluate=evaluate) else: processor = SquadV2Processor() if args.version_2_with_negative else SquadV1Processor() - examples = processor.get_dev_examples(args.data_dir) if evaluate else processor.get_train_examples(args.data_dir) + + if evaluate: + examples = processor.get_dev_examples(args.data_dir, filename=args.predict_file) + else: + examples = processor.get_train_examples(args.data_dir, filename=args.train_file) features, dataset = squad_convert_examples_to_features( examples=examples, @@ -387,7 +391,14 @@ def main(): ## Other parameters parser.add_argument("--data_dir", default=None, type=str, - help="The input data dir. Should contain the .json files for the task. If not specified, will run with tensorflow_datasets.") + help="The input data dir. Should contain the .json files for the task." + + "If no data dir or train/predict files are specified, will run with tensorflow_datasets.") + parser.add_argument("--train_file", default=None, type=str, + help="The input training file. If a data dir is specified, will look for the file there" + + "If no data dir or train/predict files are specified, will run with tensorflow_datasets.") + parser.add_argument("--predict_file", default=None, type=str, + help="The input evaluation file. If a data dir is specified, will look for the file there" + + "If no data dir or train/predict files are specified, will run with tensorflow_datasets.") parser.add_argument("--config_name", default="", type=str, help="Pretrained config name or path if not the same as model_name") parser.add_argument("--tokenizer_name", default="", type=str, @@ -472,11 +483,6 @@ def main(): parser.add_argument('--server_port', type=str, default='', help="Can be used for distant debugging.") args = parser.parse_args() - args.predict_file = os.path.join(args.output_dir, 'predictions_{}_{}.txt'.format( - list(filter(None, args.model_name_or_path.split('/'))).pop(), - str(args.max_seq_length)) - ) - if os.path.exists(args.output_dir) and os.listdir(args.output_dir) and args.do_train and not args.overwrite_output_dir: raise ValueError("Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.".format(args.output_dir)) diff --git a/transformers/data/processors/squad.py b/transformers/data/processors/squad.py index 3d7f832540..9bc4375684 100644 --- a/transformers/data/processors/squad.py +++ b/transformers/data/processors/squad.py @@ -373,6 +373,9 @@ class SquadProcessor(DataProcessor): which is `train-v1.1.json` and `train-v2.0.json` for squad versions 1.1 and 2.0 respectively. """ + if data_dir is None: + data_dir = "" + if self.train_file is None: raise ValueError("SquadProcessor should be instantiated via SquadV1Processor or SquadV2Processor") @@ -389,6 +392,9 @@ class SquadProcessor(DataProcessor): filename: None by default, specify this if the evaluation file has a different name than the original one which is `train-v1.1.json` and `train-v2.0.json` for squad versions 1.1 and 2.0 respectively. """ + if data_dir is None: + data_dir = "" + if self.dev_file is None: raise ValueError("SquadProcessor should be instantiated via SquadV1Processor or SquadV2Processor") From 33e72b08d54bf5edd192492af7549b581563ecc2 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 13 Dec 2019 11:33:05 +0100 Subject: [PATCH 344/505] fix inner dimensions for 3B/11B models --- transformers/modeling_t5.py | 27 +++++++++++---------------- transformers/modeling_tf_t5.py | 20 ++++++++------------ 2 files changed, 19 insertions(+), 28 deletions(-) diff --git a/transformers/modeling_t5.py b/transformers/modeling_t5.py index 149b977abc..c9310179a3 100644 --- a/transformers/modeling_t5.py +++ b/transformers/modeling_t5.py @@ -30,7 +30,7 @@ from torch import nn import torch.nn.functional as F from torch.nn import CrossEntropyLoss, MSELoss -from .modeling_utils import PreTrainedModel +from .modeling_utils import PreTrainedModel, prune_linear_layer from .configuration_t5 import T5Config from .file_utils import add_start_docstrings, DUMMY_INPUTS, DUMMY_MASK @@ -191,28 +191,26 @@ class T5Attention(nn.Module): self.output_attentions = config.output_attentions self.relative_attention_num_buckets = config.relative_attention_num_buckets - self.dim = config.d_model + self.d_model = config.d_model self.d_kv = config.d_kv self.n_heads = config.num_heads self.dropout = config.dropout_rate - assert self.dim % self.n_heads == 0 - assert self.dim // self.n_heads == self.d_kv + self.inner_dim = self.n_heads * self.d_kv # Mesh TensorFlow initialization to avoid scaling before softmax - self.q = nn.Linear(self.dim, self.dim, bias=False) - self.k = nn.Linear(self.dim, self.dim, bias=False) - self.v = nn.Linear(self.dim, self.dim, bias=False) - self.o = nn.Linear(self.dim, self.dim, bias=False) + self.q = nn.Linear(self.d_model, self.inner_dim, bias=False) + self.k = nn.Linear(self.d_model, self.inner_dim, bias=False) + self.v = nn.Linear(self.d_model, self.inner_dim, bias=False) + self.o = nn.Linear(self.inner_dim, self.d_model, bias=False) if self.has_relative_attention_bias: self.relative_attention_bias = nn.Embedding(self.relative_attention_num_buckets, self.n_heads) self.pruned_heads = set() def prune_heads(self, heads): - attention_head_size = self.dim // self.n_heads if len(heads) == 0: return - mask = torch.ones(self.n_heads, attention_head_size) + mask = torch.ones(self.n_heads, self.d_kv) heads = set(heads) - self.pruned_heads for head in heads: head -= sum(1 if h < head else 0 for h in self.pruned_heads) @@ -226,7 +224,7 @@ class T5Attention(nn.Module): self.o = prune_linear_layer(self.o, index, dim=1) # Update hyper params self.n_heads = self.n_heads - len(heads) - self.dim = attention_head_size * self.n_heads + self.inner_dim = self.d_kv * self.n_heads self.pruned_heads = self.pruned_heads.union(heads) @staticmethod @@ -303,17 +301,14 @@ class T5Attention(nn.Module): klen = qlen if cache is None else cache['slen'] + qlen else: klen = kv.size(1) - # assert dim == self.dim, 'Dimensions do not match: %s input vs %s configured' % (dim, self.dim) - n_heads = self.n_heads - dim_per_head = self.dim // n_heads def shape(x): """ projection """ - return x.view(bs, -1, self.n_heads, dim_per_head).transpose(1, 2) + return x.view(bs, -1, self.n_heads, self.d_kv).transpose(1, 2) def unshape(x): """ compute context """ - return x.transpose(1, 2).contiguous().view(bs, -1, self.n_heads * dim_per_head) + return x.transpose(1, 2).contiguous().view(bs, -1, self.inner_dim) q = shape(self.q(input)) # (bs, n_heads, qlen, dim_per_head) if kv is None: diff --git a/transformers/modeling_tf_t5.py b/transformers/modeling_tf_t5.py index fd25328ac6..0ae7fff412 100644 --- a/transformers/modeling_tf_t5.py +++ b/transformers/modeling_tf_t5.py @@ -108,17 +108,16 @@ class TFT5Attention(tf.keras.layers.Layer): self.output_attentions = config.output_attentions self.relative_attention_num_buckets = config.relative_attention_num_buckets - self.dim = config.d_model + self.d_model = config.d_model self.d_kv = config.d_kv self.n_heads = config.num_heads - assert self.dim % self.n_heads == 0 - assert self.dim // self.n_heads == self.d_kv + self.inner_dim = self.n_heads * self.d_kv # Mesh TensorFlow initialization to avoid scaling before softmax - self.q = tf.keras.layers.Dense(self.dim, use_bias=False, name='q') - self.k = tf.keras.layers.Dense(self.dim, use_bias=False, name='k') - self.v = tf.keras.layers.Dense(self.dim, use_bias=False, name='v') - self.o = tf.keras.layers.Dense(self.dim, use_bias=False, name='o') + self.q = tf.keras.layers.Dense(self.inner_dim, use_bias=False, name='q') + self.k = tf.keras.layers.Dense(self.inner_dim, use_bias=False, name='k') + self.v = tf.keras.layers.Dense(self.inner_dim, use_bias=False, name='v') + self.o = tf.keras.layers.Dense(self.d_model, use_bias=False, name='o') self.dropout = tf.keras.layers.Dropout(config.dropout_rate) if self.has_relative_attention_bias: @@ -199,17 +198,14 @@ class TFT5Attention(tf.keras.layers.Layer): klen = qlen if cache is None else cache['slen'] + qlen else: klen = shape_list(kv)[1] - # assert dim == self.dim, 'Dimensions do not match: %s input vs %s configured' % (dim, self.dim) - n_heads = self.n_heads - dim_per_head = self.dim // n_heads def shape(x): """ projection """ - return tf.transpose(tf.reshape(x, (bs, -1, self.n_heads, dim_per_head)), perm=(0, 2, 1, 3)) + return tf.transpose(tf.reshape(x, (bs, -1, self.n_heads, self.d_kv)), perm=(0, 2, 1, 3)) def unshape(x): """ compute context """ - return tf.reshape(tf.transpose(x, perm=(0, 2, 1, 3)), (bs, -1, self.n_heads * dim_per_head)) + return tf.reshape(tf.transpose(x, perm=(0, 2, 1, 3)), (bs, -1, self.inner_dim)) q = shape(self.q(input)) # (bs, n_heads, qlen, dim_per_head) if kv is None: From 80eacb8f16208bcc7ffd8ed5b5750d6fc6854a24 Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Fri, 13 Dec 2019 14:10:22 +0100 Subject: [PATCH 345/505] Adding labels mapping for classification models in their respective config. --- transformers/configuration_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transformers/configuration_utils.py b/transformers/configuration_utils.py index 97b9fa8f80..b7ddcf0912 100644 --- a/transformers/configuration_utils.py +++ b/transformers/configuration_utils.py @@ -58,8 +58,8 @@ class PretrainedConfig(object): self.use_bfloat16 = kwargs.pop('use_bfloat16', False) self.pruned_heads = kwargs.pop('pruned_heads', {}) self.is_decoder = kwargs.pop('is_decoder', False) - self.idx2label = kwargs.pop('idx2label', {i: 'LABEL_{}'.format(i) for i in range(self.num_labels)}) - self.label2idx = kwargs.pop('label2idx', dict(zip(self.idx2label.values(), self.idx2label.keys()))) + self.id2label = kwargs.pop('id2label', {i: 'LABEL_{}'.format(i) for i in range(self.num_labels)}) + self.label2id = kwargs.pop('label2id', dict(zip(self.id2label.values(), self.id2label.keys()))) def save_pretrained(self, save_directory): """ Save a configuration object to the directory `save_directory`, so that it From be5bf7b81bd4171169e23091beda85ffd97f950f Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Fri, 13 Dec 2019 14:12:17 +0100 Subject: [PATCH 346/505] Added NER pipeline. --- transformers/pipelines.py | 720 ++++++++++++++++++++------------------ 1 file changed, 388 insertions(+), 332 deletions(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index da8b0b65a7..b0b5848c01 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -1,332 +1,388 @@ -# coding=utf-8 -# Copyright 2018 The HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from __future__ import absolute_import, division, print_function, unicode_literals - -import os -from abc import ABC, abstractmethod -from typing import Union, Optional, Tuple, List, Dict - -import numpy as np - -from transformers import is_tf_available, is_torch_available, logger, AutoTokenizer, PreTrainedTokenizer, \ - SquadExample, squad_convert_examples_to_features - -if is_tf_available(): - from transformers import TFAutoModelForSequenceClassification, TFAutoModelForQuestionAnswering - -if is_torch_available(): - from transformers import AutoModelForSequenceClassification, AutoModelForQuestionAnswering - - -class Pipeline(ABC): - def __init__(self, model, tokenizer: PreTrainedTokenizer = None, **kwargs): - self.model = model - self.tokenizer = tokenizer - - @classmethod - @abstractmethod - def from_config(cls, model, tokenizer: PreTrainedTokenizer, **kwargs): - raise NotImplementedError() - - def save_pretrained(self, save_directory): - if not os.path.isdir(save_directory): - logger.error("Provided path ({}) should be a directory".format(save_directory)) - return - - self.model.save_pretrained(save_directory) - self.tokenizer.save_pretrained(save_directory) - - def transform(self, *texts, **kwargs): - # Generic compatibility with sklearn and Keras - return self(*texts, **kwargs) - - def predict(self, *texts, **kwargs): - # Generic compatibility with sklearn and Keras - return self(*texts, **kwargs) - - @abstractmethod - def __call__(self, *texts, **kwargs): - raise NotImplementedError() - - -class TextClassificationPipeline(Pipeline): - def __init__(self, model, tokenizer: PreTrainedTokenizer, nb_classes: int = 2): - super().__init__(model, tokenizer) - - if nb_classes < 2: - raise Exception('Invalid parameter nb_classes. int >= 2 is required (got: {})'.format(nb_classes)) - self._nb_classes = nb_classes - - @classmethod - def from_config(cls, model, tokenizer: PreTrainedTokenizer, **kwargs): - return cls(model, tokenizer, **kwargs) - - def __call__(self, *texts, **kwargs): - # Generic compatibility with sklearn and Keras - if 'X' in kwargs and not texts: - texts = kwargs.pop('X') - - inputs = self.tokenizer.batch_encode_plus( - texts, add_special_tokens=True, return_tensors='tf' if is_tf_available() else 'pt' - ) - - special_tokens_mask = inputs.pop('special_tokens_mask') - - if is_tf_available(): - # TODO trace model - predictions = self.model(**inputs)[0] - else: - import torch - with torch.no_grad(): - predictions = self.model(**inputs)[0] - - return predictions.numpy().tolist() - - -class QuestionAnsweringPipeline(Pipeline): - """ - Question Answering pipeling involving Tokenization and Inference. - """ - - @classmethod - def from_config(cls, model, tokenizer: PreTrainedTokenizer, **kwargs): - pass - - @staticmethod - def create_sample(question: Union[str, List[str]], context: Union[str, List[str]]) -> Union[SquadExample, List[SquadExample]]: - is_list = isinstance(question, list) - - if is_list: - return [SquadExample(None, q, c, None, None, None) for q, c in zip(question, context)] - else: - return SquadExample(None, question, context, None, None, None) - - @staticmethod - def handle_args(*inputs, **kwargs) -> List[SquadExample]: - # Position args, handling is sensibly the same as X and data, so forwarding to avoid duplicating - if inputs is not None and len(inputs) > 1: - kwargs['X'] = inputs - - # Generic compatibility with sklearn and Keras - # Batched data - if 'X' in kwargs or 'data' in kwargs: - data = kwargs['X'] if 'X' in kwargs else kwargs['data'] - - if not isinstance(data, list): - data = [data] - - for i, item in enumerate(data): - if isinstance(item, dict): - if any(k not in item for k in ['question', 'context']): - raise KeyError('You need to provide a dictionary with keys {question:..., context:...}') - data[i] = QuestionAnsweringPipeline.create_sample(**item) - - elif isinstance(item, SquadExample): - continue - else: - raise ValueError( - '{} argument needs to be of type (list[SquadExample | dict], SquadExample, dict)' - .format('X' if 'X' in kwargs else 'data') - ) - inputs = data - - # Tabular input - elif 'question' in kwargs and 'context' in kwargs: - if isinstance(kwargs['question'], str): - kwargs['question'] = [kwargs['question']] - - if isinstance(kwargs['context'], str): - kwargs['context'] = [kwargs['context']] - - inputs = [QuestionAnsweringPipeline.create_sample(q, c) for q, c in zip(kwargs['question'], kwargs['context'])] - else: - raise ValueError('Unknown arguments {}'.format(kwargs)) - - if not isinstance(inputs, list): - inputs = [inputs] - - return inputs - - def __init__(self, model, tokenizer: Optional[PreTrainedTokenizer]): - super().__init__(model, tokenizer) - - def inputs_for_model(self, features: Union[SquadExample, List[SquadExample]]) -> Dict: - args = ['input_ids', 'attention_mask'] - model_type = type(self.model).__name__.lower() - - if 'distilbert' not in model_type and 'xlm' not in model_type: - args += ['token_type_ids'] - - if 'xlnet' in model_type or 'xlm' in model_type: - args += ['cls_index', 'p_mask'] - - if isinstance(features, SquadExample): - return {k: features.__dict__[k] for k in args} - else: - return {k: [feature.__dict__[k] for feature in features] for k in args} - - def __call__(self, *texts, **kwargs): - # Set defaults values - kwargs.setdefault('topk', 1) - kwargs.setdefault('doc_stride', 128) - kwargs.setdefault('max_answer_len', 15) - kwargs.setdefault('max_seq_len', 384) - kwargs.setdefault('max_question_len', 64) - - if kwargs['topk'] < 1: - raise ValueError('topk parameter should be >= 1 (got {})'.format(kwargs['topk'])) - - if kwargs['max_answer_len'] < 1: - raise ValueError('max_answer_len parameter should be >= 1 (got {})'.format(kwargs['max_answer_len'])) - - examples = QuestionAnsweringPipeline.handle_args(texts, **kwargs) - - # Convert inputs to features - features = squad_convert_examples_to_features(examples, self.tokenizer, kwargs['max_seq_len'], kwargs['doc_stride'], kwargs['max_question_len'], False) - fw_args = self.inputs_for_model(features) - - if is_tf_available(): - import tensorflow as tf - fw_args = {k: tf.constant(v) for (k, v) in fw_args.items()} - start, end = self.model(fw_args) - start, end = start.numpy(), end.numpy() - else: - import torch - with torch.no_grad(): - # Retrieve the score for the context tokens only (removing question tokens) - fw_args = {k: torch.tensor(v) for (k, v) in fw_args.items()} - start, end = self.model(**fw_args) - start, end = start.cpu().numpy(), end.cpu().numpy() - - answers = [] - for (example, feature, start_, end_) in zip(examples, features, start, end): - # Normalize logits and spans to retrieve the answer - start_ = np.exp(start_) / np.sum(np.exp(start_)) - end_ = np.exp(end_) / np.sum(np.exp(end_)) - - # Mask padding and question - start_, end_ = start_ * np.abs(np.array(feature.p_mask) - 1), end_ * np.abs(np.array(feature.p_mask) - 1) - - # Mask CLS - start_[0] = end_[0] = 0 - - starts, ends, scores = self.decode(start_, end_, kwargs['topk'], kwargs['max_answer_len']) - char_to_word = np.array(example.char_to_word_offset) - - # Convert the answer (tokens) back to the original text - answers += [[ - { - 'score': score, - 'start': np.where(char_to_word == feature.token_to_orig_map[s])[0][0], - 'end': np.where(char_to_word == feature.token_to_orig_map[e])[0][-1], - 'answer': ' '.join(example.doc_tokens[feature.token_to_orig_map[s]: feature.token_to_orig_map[e] + 1]) - } - for s, e, score in zip(starts, ends, scores) - ]] - - return answers - - def decode(self, start: np.ndarray, end: np.ndarray, topk: int, max_answer_len: int) -> Tuple: - # Ensure we have batch axis - if start.ndim == 1: - start = start[None] - - if end.ndim == 1: - end = end[None] - - # Compute the score of each tuple(start, end) to be the real answer - outer = np.matmul(np.expand_dims(start, -1), np.expand_dims(end, 1)) - - # Remove candidate with end < start and end - start > max_answer_len - candidates = np.tril(np.triu(outer), max_answer_len - 1) - - # Inspired by Chen & al. (https://github.com/facebookresearch/DrQA) - scores_flat = candidates.flatten() - if topk == 1: - idx_sort = [np.argmax(scores_flat)] - elif len(scores_flat) < topk: - idx_sort = np.argsort(-scores_flat) - else: - idx = np.argpartition(-scores_flat, topk)[0:topk] - idx_sort = idx[np.argsort(-scores_flat[idx])] - - start, end = np.unravel_index(idx_sort, candidates.shape)[1:] - return start, end, candidates[0, start, end] - - def span_to_answer(self, text: str, start: int, end: int): - words = [] - token_idx = char_start_idx = char_end_idx = chars_idx = 0 - - for i, word in enumerate(text.split(" ")): - token = self.tokenizer.tokenize(word) - - # Append words if they are in the span - if start <= token_idx <= end: - if token_idx == start: - char_start_idx = chars_idx - - if token_idx == end: - char_end_idx = chars_idx + len(word) - - words += [word] - - # Stop if we went over the end of the answer - if token_idx > end: - break - - # Append the subtokenization length to the running index - token_idx += len(token) - chars_idx += len(word) + 1 - - # Join text with spaces - return {'answer': ' '.join(words), 'start': max(0, char_start_idx), 'end': min(len(text), char_end_idx)} - - -# Register all the supported task here -SUPPORTED_TASKS = { - 'text-classification': { - 'impl': TextClassificationPipeline, - 'tf': TFAutoModelForSequenceClassification if is_tf_available() else None, - 'pt': AutoModelForSequenceClassification if is_torch_available() else None - }, - 'question-answering': { - 'impl': QuestionAnsweringPipeline, - 'tf': TFAutoModelForQuestionAnswering if is_tf_available() else None, - 'pt': AutoModelForQuestionAnswering if is_torch_available() else None - } -} - - -def pipeline(task: str, model, tokenizer: Optional[Union[str, PreTrainedTokenizer]] = None, **kwargs) -> Pipeline: - """ - Utility factory method to build pipeline. - """ - # Try to infer tokenizer from model name (if provided as str) - if tokenizer is None and isinstance(model, str): - tokenizer = model - else: - # Impossible to guest what is the right tokenizer here - raise Exception('Tokenizer cannot be None if provided model is a PreTrainedModel instance') - - tokenizer = tokenizer if isinstance(tokenizer, PreTrainedTokenizer) else AutoTokenizer.from_pretrained(tokenizer) - - if task not in SUPPORTED_TASKS: - raise KeyError("Unknown task {}, available tasks are {}".format(task, list(SUPPORTED_TASKS.keys()))) - - targeted_task = SUPPORTED_TASKS[task] - task, allocator = targeted_task['impl'], targeted_task['tf'] if is_tf_available() else targeted_task['pt'] - - model = allocator.from_pretrained(model) - return task(model, tokenizer, **kwargs) +# coding=utf-8 +# Copyright 2018 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import, division, print_function, unicode_literals + +import os +from abc import ABC, abstractmethod +from itertools import groupby +from typing import Union, Optional, Tuple, List, Dict + +import numpy as np + +from transformers import AutoTokenizer, PreTrainedTokenizer, PretrainedConfig, \ + SquadExample, squad_convert_examples_to_features, is_tf_available, is_torch_available, logger + +if is_tf_available(): + from transformers import TFAutoModelForSequenceClassification, TFAutoModelForQuestionAnswering, TFAutoModelForTokenClassification + +if is_torch_available(): + import torch + from transformers import AutoModelForSequenceClassification, AutoModelForQuestionAnswering, AutoModelForTokenClassification + + +class Pipeline(ABC): + def __init__(self, model, tokenizer: PreTrainedTokenizer = None, **kwargs): + self.model = model + self.tokenizer = tokenizer + + @classmethod + @abstractmethod + def from_config(cls, model, tokenizer: PreTrainedTokenizer, **kwargs): + raise NotImplementedError() + + def save_pretrained(self, save_directory): + if not os.path.isdir(save_directory): + logger.error("Provided path ({}) should be a directory".format(save_directory)) + return + + self.model.save_pretrained(save_directory) + self.tokenizer.save_pretrained(save_directory) + + def transform(self, *texts, **kwargs): + # Generic compatibility with sklearn and Keras + return self(*texts, **kwargs) + + def predict(self, *texts, **kwargs): + # Generic compatibility with sklearn and Keras + return self(*texts, **kwargs) + + @abstractmethod + def __call__(self, *texts, **kwargs): + raise NotImplementedError() + + +class TextClassificationPipeline(Pipeline): + def __init__(self, model, tokenizer: PreTrainedTokenizer, nb_classes: int = 2): + super().__init__(model, tokenizer) + + if nb_classes < 2: + raise Exception('Invalid parameter nb_classes. int >= 2 is required (got: {})'.format(nb_classes)) + self._nb_classes = nb_classes + + @classmethod + def from_config(cls, model, tokenizer: PreTrainedTokenizer, **kwargs): + return cls(model, tokenizer, **kwargs) + + def __call__(self, *texts, **kwargs): + # Generic compatibility with sklearn and Keras + if 'X' in kwargs and not texts: + texts = kwargs.pop('X') + + inputs = self.tokenizer.batch_encode_plus( + texts, add_special_tokens=True, return_tensors='tf' if is_tf_available() else 'pt' + ) + + special_tokens_mask = inputs.pop('special_tokens_mask') + + if is_tf_available(): + # TODO trace model + predictions = self.model(**inputs)[0] + else: + import torch + with torch.no_grad(): + predictions = self.model(**inputs)[0] + + return predictions.numpy().tolist() + + +class NerPipeline(Pipeline): + + def __init__(self, model, tokenizer: PreTrainedTokenizer): + super().__init__(model, tokenizer) + + @classmethod + def from_config(cls, model, tokenizer: PreTrainedTokenizer, **kwargs): + pass + + def __call__(self, *texts, **kwargs): + (texts, ), answers = texts, [] + + for sentence in texts: + + # Ugly token to word idx mapping (for now) + token_to_word, words = [], sentence.split(' ') + for i, w in enumerate(words): + tokens = self.tokenizer.tokenize(w) + token_to_word += [i] * len(tokens) + tokens = self.tokenizer.encode_plus(sentence, return_attention_mask=False, return_tensors='tf' if is_tf_available() else 'pt') + + # Forward + if is_torch_available(): + with torch.no_grad(): + entities = self.model(**tokens)[0][0].cpu().numpy() + else: + entities = self.model(tokens)[0][0].numpy() + + # Normalize scores + answer, token_start = [], 1 + for idx, word in groupby(token_to_word[1:-1]): + + # Sum log prob over token, then normalize across labels + score = np.exp(entities[token_start]) / np.exp(entities[token_start]).sum(-1, keepdims=True) + label_idx = score.argmax() + + answer += [{ + 'word': words[idx - 1], 'score': score[label_idx], 'entity': self.model.config.id2label[label_idx] + }] + + # Update token start + token_start += len(list(word)) + + # Append + answers += [answer] + return answers + + +class QuestionAnsweringPipeline(Pipeline): + """ + Question Answering pipeline involving Tokenization and Inference. + """ + + @classmethod + def from_config(cls, model, tokenizer: PreTrainedTokenizer, **kwargs): + pass + + @staticmethod + def create_sample(question: Union[str, List[str]], context: Union[str, List[str]]) -> Union[SquadExample, List[SquadExample]]: + is_list = isinstance(question, list) + + if is_list: + return [SquadExample(None, q, c, None, None, None) for q, c in zip(question, context)] + else: + return SquadExample(None, question, context, None, None, None) + + @staticmethod + def handle_args(*inputs, **kwargs) -> List[SquadExample]: + # Position args, handling is sensibly the same as X and data, so forwarding to avoid duplicating + if inputs is not None and len(inputs) > 1: + kwargs['X'] = inputs + + # Generic compatibility with sklearn and Keras + # Batched data + if 'X' in kwargs or 'data' in kwargs: + data = kwargs['X'] if 'X' in kwargs else kwargs['data'] + + if not isinstance(data, list): + data = [data] + + for i, item in enumerate(data): + if isinstance(item, dict): + if any(k not in item for k in ['question', 'context']): + raise KeyError('You need to provide a dictionary with keys {question:..., context:...}') + data[i] = QuestionAnsweringPipeline.create_sample(**item) + + elif isinstance(item, SquadExample): + continue + else: + raise ValueError( + '{} argument needs to be of type (list[SquadExample | dict], SquadExample, dict)' + .format('X' if 'X' in kwargs else 'data') + ) + inputs = data + + # Tabular input + elif 'question' in kwargs and 'context' in kwargs: + if isinstance(kwargs['question'], str): + kwargs['question'] = [kwargs['question']] + + if isinstance(kwargs['context'], str): + kwargs['context'] = [kwargs['context']] + + inputs = [QuestionAnsweringPipeline.create_sample(q, c) for q, c in zip(kwargs['question'], kwargs['context'])] + else: + raise ValueError('Unknown arguments {}'.format(kwargs)) + + if not isinstance(inputs, list): + inputs = [inputs] + + return inputs + + def __init__(self, model, tokenizer: Optional[PreTrainedTokenizer]): + super().__init__(model, tokenizer) + + def inputs_for_model(self, features: Union[SquadExample, List[SquadExample]]) -> Dict: + args = ['input_ids', 'attention_mask'] + model_type = type(self.model).__name__.lower() + + if 'distilbert' not in model_type and 'xlm' not in model_type: + args += ['token_type_ids'] + + if 'xlnet' in model_type or 'xlm' in model_type: + args += ['cls_index', 'p_mask'] + + if isinstance(features, SquadExample): + return {k: features.__dict__[k] for k in args} + else: + return {k: [feature.__dict__[k] for feature in features] for k in args} + + def __call__(self, *texts, **kwargs): + # Set defaults values + kwargs.setdefault('topk', 1) + kwargs.setdefault('doc_stride', 128) + kwargs.setdefault('max_answer_len', 15) + kwargs.setdefault('max_seq_len', 384) + kwargs.setdefault('max_question_len', 64) + + if kwargs['topk'] < 1: + raise ValueError('topk parameter should be >= 1 (got {})'.format(kwargs['topk'])) + + if kwargs['max_answer_len'] < 1: + raise ValueError('max_answer_len parameter should be >= 1 (got {})'.format(kwargs['max_answer_len'])) + + examples = QuestionAnsweringPipeline.handle_args(texts, **kwargs) + + # Convert inputs to features + features = squad_convert_examples_to_features(examples, self.tokenizer, kwargs['max_seq_len'], kwargs['doc_stride'], kwargs['max_question_len'], False) + fw_args = self.inputs_for_model(features) + + if is_tf_available(): + import tensorflow as tf + fw_args = {k: tf.constant(v) for (k, v) in fw_args.items()} + start, end = self.model(fw_args) + start, end = start.numpy(), end.numpy() + else: + import torch + with torch.no_grad(): + # Retrieve the score for the context tokens only (removing question tokens) + fw_args = {k: torch.tensor(v) for (k, v) in fw_args.items()} + start, end = self.model(**fw_args) + start, end = start.cpu().numpy(), end.cpu().numpy() + + answers = [] + for (example, feature, start_, end_) in zip(examples, features, start, end): + # Normalize logits and spans to retrieve the answer + start_ = np.exp(start_) / np.sum(np.exp(start_)) + end_ = np.exp(end_) / np.sum(np.exp(end_)) + + # Mask padding and question + start_, end_ = start_ * np.abs(np.array(feature.p_mask) - 1), end_ * np.abs(np.array(feature.p_mask) - 1) + + # TODO : What happend if not possible + # Mask CLS + start_[0] = end_[0] = 0 + + starts, ends, scores = self.decode(start_, end_, kwargs['topk'], kwargs['max_answer_len']) + char_to_word = np.array(example.char_to_word_offset) + + # Convert the answer (tokens) back to the original text + answers += [[ + { + 'score': score, + 'start': np.where(char_to_word == feature.token_to_orig_map[s])[0][0], + 'end': np.where(char_to_word == feature.token_to_orig_map[e])[0][-1], + 'answer': ' '.join(example.doc_tokens[feature.token_to_orig_map[s]: feature.token_to_orig_map[e] + 1]) + } + for s, e, score in zip(starts, ends, scores) + ]] + + return answers + + def decode(self, start: np.ndarray, end: np.ndarray, topk: int, max_answer_len: int) -> Tuple: + # Ensure we have batch axis + if start.ndim == 1: + start = start[None] + + if end.ndim == 1: + end = end[None] + + # Compute the score of each tuple(start, end) to be the real answer + outer = np.matmul(np.expand_dims(start, -1), np.expand_dims(end, 1)) + + # Remove candidate with end < start and end - start > max_answer_len + candidates = np.tril(np.triu(outer), max_answer_len - 1) + + # Inspired by Chen & al. (https://github.com/facebookresearch/DrQA) + scores_flat = candidates.flatten() + if topk == 1: + idx_sort = [np.argmax(scores_flat)] + elif len(scores_flat) < topk: + idx_sort = np.argsort(-scores_flat) + else: + idx = np.argpartition(-scores_flat, topk)[0:topk] + idx_sort = idx[np.argsort(-scores_flat[idx])] + + start, end = np.unravel_index(idx_sort, candidates.shape)[1:] + return start, end, candidates[0, start, end] + + def span_to_answer(self, text: str, start: int, end: int): + words = [] + token_idx = char_start_idx = char_end_idx = chars_idx = 0 + + for i, word in enumerate(text.split(" ")): + token = self.tokenizer.tokenize(word) + + # Append words if they are in the span + if start <= token_idx <= end: + if token_idx == start: + char_start_idx = chars_idx + + if token_idx == end: + char_end_idx = chars_idx + len(word) + + words += [word] + + # Stop if we went over the end of the answer + if token_idx > end: + break + + # Append the subtokenization length to the running index + token_idx += len(token) + chars_idx += len(word) + 1 + + # Join text with spaces + return {'answer': ' '.join(words), 'start': max(0, char_start_idx), 'end': min(len(text), char_end_idx)} + + +# Register all the supported task here +SUPPORTED_TASKS = { + 'text-classification': { + 'impl': TextClassificationPipeline, + 'tf': TFAutoModelForSequenceClassification if is_tf_available() else None, + 'pt': AutoModelForSequenceClassification if is_torch_available() else None + }, + 'ner': { + 'impl': NerPipeline, + 'tf': TFAutoModelForTokenClassification if is_tf_available() else None, + 'pt': AutoModelForTokenClassification if is_torch_available() else None, + }, + 'question-answering': { + 'impl': QuestionAnsweringPipeline, + 'tf': TFAutoModelForQuestionAnswering if is_tf_available() else None, + 'pt': AutoModelForQuestionAnswering if is_torch_available() else None + } +} + + +def pipeline(task: str, model, config: Optional[PretrainedConfig] = None, tokenizer: Optional[Union[str, PreTrainedTokenizer]] = None, **kwargs) -> Pipeline: + """ + Utility factory method to build pipeline. + """ + # Try to infer tokenizer from model name (if provided as str) + if tokenizer is None and isinstance(model, str): + tokenizer = model + else: + # Impossible to guest what is the right tokenizer here + raise Exception('Tokenizer cannot be None if provided model is a PreTrainedModel instance') + + tokenizer = tokenizer if isinstance(tokenizer, PreTrainedTokenizer) else AutoTokenizer.from_pretrained(tokenizer) + + if task not in SUPPORTED_TASKS: + raise KeyError("Unknown task {}, available tasks are {}".format(task, list(SUPPORTED_TASKS.keys()))) + + targeted_task = SUPPORTED_TASKS[task] + task, allocator = targeted_task['impl'], targeted_task['tf'] if is_tf_available() else targeted_task['pt'] + + model = allocator.from_pretrained(model) + return task(model, tokenizer, **kwargs) From 28e64ad5a4b01a1b7de092694e3a321edf7021bd Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Fri, 13 Dec 2019 14:12:54 +0100 Subject: [PATCH 347/505] Raise an exception if the pipeline allocator can't determine the tokenizer from the model. --- transformers/pipelines.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index b0b5848c01..853735a256 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -370,11 +370,12 @@ def pipeline(task: str, model, config: Optional[PretrainedConfig] = None, tokeni Utility factory method to build pipeline. """ # Try to infer tokenizer from model name (if provided as str) - if tokenizer is None and isinstance(model, str): - tokenizer = model - else: - # Impossible to guest what is the right tokenizer here - raise Exception('Tokenizer cannot be None if provided model is a PreTrainedModel instance') + if not isinstance(tokenizer, PreTrainedTokenizer): + if not isinstance(model, str): + # Impossible to guest what is the right tokenizer here + raise Exception('Tokenizer cannot be None if provided model is a PreTrainedModel instance') + else: + tokenizer = model tokenizer = tokenizer if isinstance(tokenizer, PreTrainedTokenizer) else AutoTokenizer.from_pretrained(tokenizer) From 1ca52567a4059e7ee1707de6a855bb5e7fb3fac3 Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Fri, 13 Dec 2019 14:13:14 +0100 Subject: [PATCH 348/505] Allow model conversion in the pipeline allocator. --- transformers/pipelines.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index 853735a256..9acd9bc566 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -385,5 +385,17 @@ def pipeline(task: str, model, config: Optional[PretrainedConfig] = None, tokeni targeted_task = SUPPORTED_TASKS[task] task, allocator = targeted_task['impl'], targeted_task['tf'] if is_tf_available() else targeted_task['pt'] - model = allocator.from_pretrained(model) + # Special handling for model conversion + from_tf = model.endswith('.h5') and not is_tf_available() + from_pt = model.endswith('.bin') and not is_torch_available() + + if from_tf: + logger.warning('Model might be a TensorFlow model (ending with `.h5`) but TensorFlow is not available. Trying to load the model with PyTorch.') + elif from_pt: + logger.warning('Model might be a PyTorch model (ending with `.bin`) but PyTorch is not available. Trying to load the model with Tensorflow.') + + if allocator.__name__.startswith('TF'): + model = allocator.from_pretrained(model, config=config, from_pt=from_pt) + else: + model = allocator.from_pretrained(model, config=config, from_tf=from_tf) return task(model, tokenizer, **kwargs) From 8938b546bf5f61dcb65fb6dd72b5b924f773c46a Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Fri, 13 Dec 2019 14:27:04 +0100 Subject: [PATCH 349/505] Removed from_config --- transformers/pipelines.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index 9acd9bc566..6fbb7e2f04 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -37,11 +37,6 @@ class Pipeline(ABC): self.model = model self.tokenizer = tokenizer - @classmethod - @abstractmethod - def from_config(cls, model, tokenizer: PreTrainedTokenizer, **kwargs): - raise NotImplementedError() - def save_pretrained(self, save_directory): if not os.path.isdir(save_directory): logger.error("Provided path ({}) should be a directory".format(save_directory)) @@ -63,6 +58,12 @@ class Pipeline(ABC): raise NotImplementedError() +class FeatureExtractionPipeline(Pipeline): + + def __call__(self, *texts, **kwargs): + pass + + class TextClassificationPipeline(Pipeline): def __init__(self, model, tokenizer: PreTrainedTokenizer, nb_classes: int = 2): super().__init__(model, tokenizer) @@ -71,10 +72,6 @@ class TextClassificationPipeline(Pipeline): raise Exception('Invalid parameter nb_classes. int >= 2 is required (got: {})'.format(nb_classes)) self._nb_classes = nb_classes - @classmethod - def from_config(cls, model, tokenizer: PreTrainedTokenizer, **kwargs): - return cls(model, tokenizer, **kwargs) - def __call__(self, *texts, **kwargs): # Generic compatibility with sklearn and Keras if 'X' in kwargs and not texts: @@ -102,10 +99,6 @@ class NerPipeline(Pipeline): def __init__(self, model, tokenizer: PreTrainedTokenizer): super().__init__(model, tokenizer) - @classmethod - def from_config(cls, model, tokenizer: PreTrainedTokenizer, **kwargs): - pass - def __call__(self, *texts, **kwargs): (texts, ), answers = texts, [] From 47f0e3cfb7df192ab80215cea9096791fce08694 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 13 Dec 2019 14:33:24 +0100 Subject: [PATCH 350/505] cleaning up configuration classes --- .../summarization/configuration_bertabs.py | 10 +-- .../adding_a_new_model/configuration_xxx.py | 12 +-- .../tests/modeling_tf_xxx_test.py | 2 +- .../tests/modeling_xxx_test.py | 2 +- transformers/configuration_albert.py | 6 +- transformers/configuration_bert.py | 38 +++----- transformers/configuration_ctrl.py | 23 +---- transformers/configuration_distilbert.py | 40 ++++----- transformers/configuration_gpt2.py | 55 ++++-------- transformers/configuration_openai.py | 57 +++++------- transformers/configuration_transfo_xl.py | 26 ++---- transformers/configuration_utils.py | 27 ++++-- transformers/configuration_xlm.py | 88 ++++++++----------- transformers/configuration_xlnet.py | 81 +++++++---------- ..._original_pytorch_checkpoint_to_pytorch.py | 2 +- transformers/modeling_gpt2.py | 1 + transformers/modeling_tf_gpt2.py | 1 + transformers/modeling_tf_transfo_xl.py | 6 +- .../modeling_tf_transfo_xl_utilities.py | 12 +-- transformers/modeling_tf_xlnet.py | 2 +- transformers/modeling_transfo_xl.py | 10 +-- transformers/modeling_xlnet.py | 4 +- transformers/tests/modeling_albert_test.py | 2 +- transformers/tests/modeling_bert_test.py | 2 +- transformers/tests/modeling_common_test.py | 2 +- transformers/tests/modeling_ctrl_test.py | 2 +- .../tests/modeling_distilbert_test.py | 2 +- transformers/tests/modeling_gpt2_test.py | 2 +- transformers/tests/modeling_openai_test.py | 2 +- transformers/tests/modeling_roberta_test.py | 2 +- transformers/tests/modeling_tf_albert_test.py | 2 +- transformers/tests/modeling_tf_bert_test.py | 2 +- transformers/tests/modeling_tf_ctrl_test.py | 2 +- .../tests/modeling_tf_distilbert_test.py | 2 +- transformers/tests/modeling_tf_gpt2_test.py | 2 +- .../tests/modeling_tf_openai_gpt_test.py | 2 +- .../tests/modeling_tf_roberta_test.py | 2 +- .../tests/modeling_tf_transfo_xl_test.py | 2 +- transformers/tests/modeling_tf_xlm_test.py | 2 +- transformers/tests/modeling_tf_xlnet_test.py | 5 +- .../tests/modeling_transfo_xl_test.py | 2 +- transformers/tests/modeling_xlm_test.py | 2 +- transformers/tests/modeling_xlnet_test.py | 5 +- 43 files changed, 224 insertions(+), 329 deletions(-) diff --git a/examples/summarization/configuration_bertabs.py b/examples/summarization/configuration_bertabs.py index 5bcb65b423..054763ea93 100644 --- a/examples/summarization/configuration_bertabs.py +++ b/examples/summarization/configuration_bertabs.py @@ -65,7 +65,7 @@ class BertAbsConfig(PretrainedConfig): def __init__( self, - vocab_size_or_config_json_file=30522, + vocab_size=30522, max_pos=512, enc_layers=6, enc_hidden_size=512, @@ -81,14 +81,14 @@ class BertAbsConfig(PretrainedConfig): ): super(BertAbsConfig, self).__init__(**kwargs) - if self._input_is_path_to_json(vocab_size_or_config_json_file): - path_to_json = vocab_size_or_config_json_file + if self._input_is_path_to_json(vocab_size): + path_to_json = vocab_size with open(path_to_json, "r", encoding="utf-8") as reader: json_config = json.loads(reader.read()) for key, value in json_config.items(): self.__dict__[key] = value - elif isinstance(vocab_size_or_config_json_file, int): - self.vocab_size = vocab_size_or_config_json_file + elif isinstance(vocab_size, int): + self.vocab_size = vocab_size self.max_pos = max_pos self.enc_layers = enc_layers diff --git a/templates/adding_a_new_model/configuration_xxx.py b/templates/adding_a_new_model/configuration_xxx.py index b1614e71af..ca9e0d554b 100644 --- a/templates/adding_a_new_model/configuration_xxx.py +++ b/templates/adding_a_new_model/configuration_xxx.py @@ -39,7 +39,7 @@ class XxxConfig(PretrainedConfig): Arguments: - vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `XxxModel`. + vocab_size: Vocabulary size of `inputs_ids` in `XxxModel`. hidden_size: Size of the encoder layers and the pooler layer. num_hidden_layers: Number of hidden layers in the Transformer encoder. num_attention_heads: Number of attention heads for each attention layer in @@ -64,7 +64,7 @@ class XxxConfig(PretrainedConfig): pretrained_config_archive_map = XXX_PRETRAINED_CONFIG_ARCHIVE_MAP def __init__(self, - vocab_size_or_config_json_file=50257, + vocab_size=50257, n_positions=1024, n_ctx=1024, n_embd=768, @@ -84,7 +84,7 @@ class XxxConfig(PretrainedConfig): summary_first_dropout=0.1, **kwargs): super(XxxConfig, self).__init__(**kwargs) - self.vocab_size = vocab_size_or_config_json_file if isinstance(vocab_size_or_config_json_file, six.string_types) else -1 + self.vocab_size = vocab_size if isinstance(vocab_size, six.string_types) else -1 self.n_ctx = n_ctx self.n_positions = n_positions self.n_embd = n_embd @@ -102,12 +102,12 @@ class XxxConfig(PretrainedConfig): self.summary_activation = summary_activation self.summary_first_dropout = summary_first_dropout self.summary_proj_to_labels = summary_proj_to_labels - if isinstance(vocab_size_or_config_json_file, six.string_types): - with open(vocab_size_or_config_json_file, "r", encoding="utf-8") as reader: + if isinstance(vocab_size, six.string_types): + with open(vocab_size, "r", encoding="utf-8") as reader: json_config = json.loads(reader.read()) for key, value in json_config.items(): self.__dict__[key] = value - elif not isinstance(vocab_size_or_config_json_file, int): + elif not isinstance(vocab_size, int): raise ValueError( "First argument must be either a vocabulary size (int)" "or the path to a pretrained model config file (str)" diff --git a/templates/adding_a_new_model/tests/modeling_tf_xxx_test.py b/templates/adding_a_new_model/tests/modeling_tf_xxx_test.py index d7e576bf8b..912a4aa340 100644 --- a/templates/adding_a_new_model/tests/modeling_tf_xxx_test.py +++ b/templates/adding_a_new_model/tests/modeling_tf_xxx_test.py @@ -111,7 +111,7 @@ class TFXxxModelTest(TFCommonTestCases.TFCommonModelTester): choice_labels = ids_tensor([self.batch_size], self.num_choices) config = XxxConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, hidden_size=self.hidden_size, num_hidden_layers=self.num_hidden_layers, num_attention_heads=self.num_attention_heads, diff --git a/templates/adding_a_new_model/tests/modeling_xxx_test.py b/templates/adding_a_new_model/tests/modeling_xxx_test.py index bfc70921cd..30e614b3f2 100644 --- a/templates/adding_a_new_model/tests/modeling_xxx_test.py +++ b/templates/adding_a_new_model/tests/modeling_xxx_test.py @@ -109,7 +109,7 @@ class XxxModelTest(CommonTestCases.CommonModelTester): choice_labels = ids_tensor([self.batch_size], self.num_choices) config = XxxConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, hidden_size=self.hidden_size, num_hidden_layers=self.num_hidden_layers, num_attention_heads=self.num_attention_heads, diff --git a/transformers/configuration_albert.py b/transformers/configuration_albert.py index de665c9b1c..6a1ef78dd5 100644 --- a/transformers/configuration_albert.py +++ b/transformers/configuration_albert.py @@ -37,7 +37,7 @@ class AlbertConfig(PretrainedConfig): pretrained_config_archive_map = ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP def __init__(self, - vocab_size_or_config_json_file=30000, + vocab_size=30000, embedding_size=128, hidden_size=4096, num_hidden_layers=12, @@ -83,7 +83,7 @@ class AlbertConfig(PretrainedConfig): """ super(AlbertConfig, self).__init__(**kwargs) - self.vocab_size = vocab_size_or_config_json_file + self.vocab_size = vocab_size self.embedding_size = embedding_size self.hidden_size = hidden_size self.num_hidden_layers = num_hidden_layers @@ -97,4 +97,4 @@ class AlbertConfig(PretrainedConfig): self.max_position_embeddings = max_position_embeddings self.type_vocab_size = type_vocab_size self.initializer_range = initializer_range - self.layer_norm_eps = layer_norm_eps \ No newline at end of file + self.layer_norm_eps = layer_norm_eps diff --git a/transformers/configuration_bert.py b/transformers/configuration_bert.py index 01fcd88cb8..9072820bce 100644 --- a/transformers/configuration_bert.py +++ b/transformers/configuration_bert.py @@ -56,7 +56,7 @@ class BertConfig(PretrainedConfig): Arguments: - vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `BertModel`. + vocab_size: Vocabulary size of `inputs_ids` in `BertModel`. hidden_size: Size of the encoder layers and the pooler layer. num_hidden_layers: Number of hidden layers in the Transformer encoder. num_attention_heads: Number of attention heads for each attention layer in @@ -81,7 +81,7 @@ class BertConfig(PretrainedConfig): pretrained_config_archive_map = BERT_PRETRAINED_CONFIG_ARCHIVE_MAP def __init__(self, - vocab_size_or_config_json_file=30522, + vocab_size=30522, hidden_size=768, num_hidden_layers=12, num_attention_heads=12, @@ -95,25 +95,15 @@ class BertConfig(PretrainedConfig): layer_norm_eps=1e-12, **kwargs): super(BertConfig, self).__init__(**kwargs) - if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2 - and isinstance(vocab_size_or_config_json_file, unicode)): - with open(vocab_size_or_config_json_file, "r", encoding='utf-8') as reader: - json_config = json.loads(reader.read()) - for key, value in json_config.items(): - self.__dict__[key] = value - elif isinstance(vocab_size_or_config_json_file, int): - self.vocab_size = vocab_size_or_config_json_file - self.hidden_size = hidden_size - self.num_hidden_layers = num_hidden_layers - self.num_attention_heads = num_attention_heads - self.hidden_act = hidden_act - self.intermediate_size = intermediate_size - self.hidden_dropout_prob = hidden_dropout_prob - self.attention_probs_dropout_prob = attention_probs_dropout_prob - self.max_position_embeddings = max_position_embeddings - self.type_vocab_size = type_vocab_size - self.initializer_range = initializer_range - self.layer_norm_eps = layer_norm_eps - else: - raise ValueError("First argument must be either a vocabulary size (int)" - " or the path to a pretrained model config file (str)") + self.vocab_size = vocab_size + self.hidden_size = hidden_size + self.num_hidden_layers = num_hidden_layers + self.num_attention_heads = num_attention_heads + self.hidden_act = hidden_act + self.intermediate_size = intermediate_size + self.hidden_dropout_prob = hidden_dropout_prob + self.attention_probs_dropout_prob = attention_probs_dropout_prob + self.max_position_embeddings = max_position_embeddings + self.type_vocab_size = type_vocab_size + self.initializer_range = initializer_range + self.layer_norm_eps = layer_norm_eps diff --git a/transformers/configuration_ctrl.py b/transformers/configuration_ctrl.py index fcbd848dec..f9b9e409e1 100644 --- a/transformers/configuration_ctrl.py +++ b/transformers/configuration_ctrl.py @@ -31,7 +31,7 @@ class CTRLConfig(PretrainedConfig): """Configuration class to store the configuration of a `CTRLModel`. Args: - vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `CTRLModel` or a configuration json file. + vocab_size: Vocabulary size of `inputs_ids` in `CTRLModel` or a configuration json file. n_positions: Number of positional embeddings. n_ctx: Size of the causal mask (usually same as n_positions). dff: Size of the inner dimension of the FFN. @@ -52,7 +52,7 @@ class CTRLConfig(PretrainedConfig): def __init__( self, - vocab_size_or_config_json_file=246534, + vocab_size=246534, n_positions=256, n_ctx=256, n_embd=1280, @@ -64,8 +64,6 @@ class CTRLConfig(PretrainedConfig): attn_pdrop=0.1, layer_norm_epsilon=1e-6, initializer_range=0.02, - - num_labels=1, summary_type='cls_index', summary_use_proj=True, summary_activation=None, @@ -76,7 +74,7 @@ class CTRLConfig(PretrainedConfig): """Constructs CTRLConfig. Args: - vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `CTRLModel` or a configuration json file. + vocab_size: Vocabulary size of `inputs_ids` in `CTRLModel` or a configuration json file. n_positions: Number of positional embeddings. n_ctx: Size of the causal mask (usually same as n_positions). dff: Size of the inner dimension of the FFN. @@ -94,8 +92,7 @@ class CTRLConfig(PretrainedConfig): initializing all weight matrices. """ super(CTRLConfig, self).__init__(**kwargs) - - self.vocab_size = vocab_size_or_config_json_file if isinstance(vocab_size_or_config_json_file, int) else -1 + self.vocab_size = vocab_size self.n_ctx = n_ctx self.n_positions = n_positions self.n_embd = n_embd @@ -108,23 +105,11 @@ class CTRLConfig(PretrainedConfig): self.layer_norm_epsilon = layer_norm_epsilon self.initializer_range = initializer_range - self.num_labels = num_labels self.summary_type = summary_type self.summary_use_proj = summary_use_proj self.summary_activation = summary_activation self.summary_first_dropout = summary_first_dropout self.summary_proj_to_labels = summary_proj_to_labels - if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2 - and isinstance(vocab_size_or_config_json_file, unicode)): - with open(vocab_size_or_config_json_file, "r", encoding="utf-8") as reader: - json_config = json.loads(reader.read()) - for key, value in json_config.items(): - self.__dict__[key] = value - elif not isinstance(vocab_size_or_config_json_file, int): - raise ValueError( - "First argument must be either a vocabulary size (int)" - "or the path to a pretrained model config file (str)" - ) @property def max_position_embeddings(self): diff --git a/transformers/configuration_distilbert.py b/transformers/configuration_distilbert.py index d5d575be29..d9f7cc6348 100644 --- a/transformers/configuration_distilbert.py +++ b/transformers/configuration_distilbert.py @@ -37,7 +37,7 @@ class DistilBertConfig(PretrainedConfig): pretrained_config_archive_map = DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP def __init__(self, - vocab_size_or_config_json_file=30522, + vocab_size=30522, max_position_embeddings=512, sinusoidal_pos_embds=False, n_layers=6, @@ -53,31 +53,21 @@ class DistilBertConfig(PretrainedConfig): seq_classif_dropout=0.2, **kwargs): super(DistilBertConfig, self).__init__(**kwargs) + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.sinusoidal_pos_embds = sinusoidal_pos_embds + self.n_layers = n_layers + self.n_heads = n_heads + self.dim = dim + self.hidden_dim = hidden_dim + self.dropout = dropout + self.attention_dropout = attention_dropout + self.activation = activation + self.initializer_range = initializer_range + self.tie_weights_ = tie_weights_ + self.qa_dropout = qa_dropout + self.seq_classif_dropout = seq_classif_dropout - if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2 - and isinstance(vocab_size_or_config_json_file, unicode)): - with open(vocab_size_or_config_json_file, "r", encoding='utf-8') as reader: - json_config = json.loads(reader.read()) - for key, value in json_config.items(): - self.__dict__[key] = value - elif isinstance(vocab_size_or_config_json_file, int): - self.vocab_size = vocab_size_or_config_json_file - self.max_position_embeddings = max_position_embeddings - self.sinusoidal_pos_embds = sinusoidal_pos_embds - self.n_layers = n_layers - self.n_heads = n_heads - self.dim = dim - self.hidden_dim = hidden_dim - self.dropout = dropout - self.attention_dropout = attention_dropout - self.activation = activation - self.initializer_range = initializer_range - self.tie_weights_ = tie_weights_ - self.qa_dropout = qa_dropout - self.seq_classif_dropout = seq_classif_dropout - else: - raise ValueError("First argument must be either a vocabulary size (int)" - " or the path to a pretrained model config file (str)") @property def hidden_size(self): return self.dim diff --git a/transformers/configuration_gpt2.py b/transformers/configuration_gpt2.py index c2fb4948d3..4c200c0760 100644 --- a/transformers/configuration_gpt2.py +++ b/transformers/configuration_gpt2.py @@ -36,7 +36,7 @@ class GPT2Config(PretrainedConfig): """Configuration class to store the configuration of a `GPT2Model`. Args: - vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `GPT2Model` or a configuration json file. + vocab_size: Vocabulary size of `inputs_ids` in `GPT2Model` or a configuration json file. n_positions: Number of positional embeddings. n_ctx: Size of the causal mask (usually same as n_positions). n_embd: Dimensionality of the embeddings and hidden states. @@ -56,7 +56,7 @@ class GPT2Config(PretrainedConfig): def __init__( self, - vocab_size_or_config_json_file=50257, + vocab_size=50257, n_positions=1024, n_ctx=1024, n_embd=768, @@ -67,8 +67,6 @@ class GPT2Config(PretrainedConfig): attn_pdrop=0.1, layer_norm_epsilon=1e-5, initializer_range=0.02, - - num_labels=1, summary_type='cls_index', summary_use_proj=True, summary_activation=None, @@ -79,7 +77,7 @@ class GPT2Config(PretrainedConfig): """Constructs GPT2Config. Args: - vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `GPT2Model` or a configuration json file. + vocab_size: Vocabulary size of `inputs_ids` in `GPT2Model` or a configuration json file. n_positions: Number of positional embeddings. n_ctx: Size of the causal mask (usually same as n_positions). n_embd: Dimensionality of the embeddings and hidden states. @@ -96,37 +94,22 @@ class GPT2Config(PretrainedConfig): initializing all weight matrices. """ super(GPT2Config, self).__init__(**kwargs) - - if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2 - and isinstance(vocab_size_or_config_json_file, unicode)): - with open(vocab_size_or_config_json_file, "r", encoding="utf-8") as reader: - json_config = json.loads(reader.read()) - for key, value in json_config.items(): - self.__dict__[key] = value - elif isinstance(vocab_size_or_config_json_file, int): - self.vocab_size = vocab_size_or_config_json_file - self.n_ctx = n_ctx - self.n_positions = n_positions - self.n_embd = n_embd - self.n_layer = n_layer - self.n_head = n_head - self.resid_pdrop = resid_pdrop - self.embd_pdrop = embd_pdrop - self.attn_pdrop = attn_pdrop - self.layer_norm_epsilon = layer_norm_epsilon - self.initializer_range = initializer_range - - self.num_labels = num_labels - self.summary_type = summary_type - self.summary_use_proj = summary_use_proj - self.summary_activation = summary_activation - self.summary_first_dropout = summary_first_dropout - self.summary_proj_to_labels = summary_proj_to_labels - else: - raise ValueError( - "First argument must be either a vocabulary size (int)" - "or the path to a pretrained model config file (str)" - ) + self.vocab_size = vocab_size + self.n_ctx = n_ctx + self.n_positions = n_positions + self.n_embd = n_embd + self.n_layer = n_layer + self.n_head = n_head + self.resid_pdrop = resid_pdrop + self.embd_pdrop = embd_pdrop + self.attn_pdrop = attn_pdrop + self.layer_norm_epsilon = layer_norm_epsilon + self.initializer_range = initializer_range + self.summary_type = summary_type + self.summary_use_proj = summary_use_proj + self.summary_activation = summary_activation + self.summary_first_dropout = summary_first_dropout + self.summary_proj_to_labels = summary_proj_to_labels @property def max_position_embeddings(self): diff --git a/transformers/configuration_openai.py b/transformers/configuration_openai.py index 886b7f5bc5..7776a0bb9f 100644 --- a/transformers/configuration_openai.py +++ b/transformers/configuration_openai.py @@ -35,7 +35,7 @@ class OpenAIGPTConfig(PretrainedConfig): Configuration class to store the configuration of a `OpenAIGPTModel`. Args: - vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `OpenAIGPTModel` or a configuration json file. + vocab_size: Vocabulary size of `inputs_ids` in `OpenAIGPTModel` or a configuration json file. n_positions: Number of positional embeddings. n_ctx: Size of the causal mask (usually same as n_positions). n_embd: Dimensionality of the embeddings and hidden states. @@ -58,7 +58,7 @@ class OpenAIGPTConfig(PretrainedConfig): def __init__( self, - vocab_size_or_config_json_file=40478, + vocab_size=40478, n_positions=512, n_ctx=512, n_embd=768, @@ -71,8 +71,6 @@ class OpenAIGPTConfig(PretrainedConfig): layer_norm_epsilon=1e-5, initializer_range=0.02, predict_special_tokens=True, - - num_labels=1, summary_type='cls_index', summary_use_proj=True, summary_activation=None, @@ -83,39 +81,24 @@ class OpenAIGPTConfig(PretrainedConfig): """Constructs OpenAIGPTConfig. """ super(OpenAIGPTConfig, self).__init__(**kwargs) - - if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2 - and isinstance(vocab_size_or_config_json_file, unicode)): - with open(vocab_size_or_config_json_file, "r", encoding="utf-8") as reader: - json_config = json.loads(reader.read()) - for key, value in json_config.items(): - self.__dict__[key] = value - elif isinstance(vocab_size_or_config_json_file, int): - self.vocab_size = vocab_size_or_config_json_file - self.n_ctx = n_ctx - self.n_positions = n_positions - self.n_embd = n_embd - self.n_layer = n_layer - self.n_head = n_head - self.afn = afn - self.resid_pdrop = resid_pdrop - self.embd_pdrop = embd_pdrop - self.attn_pdrop = attn_pdrop - self.layer_norm_epsilon = layer_norm_epsilon - self.initializer_range = initializer_range - self.predict_special_tokens = predict_special_tokens - - self.num_labels = num_labels - self.summary_type = summary_type - self.summary_use_proj = summary_use_proj - self.summary_activation = summary_activation - self.summary_first_dropout = summary_first_dropout - self.summary_proj_to_labels = summary_proj_to_labels - else: - raise ValueError( - "First argument must be either a vocabulary size (int)" - "or the path to a pretrained model config file (str)" - ) + self.vocab_size = vocab_size + self.n_ctx = n_ctx + self.n_positions = n_positions + self.n_embd = n_embd + self.n_layer = n_layer + self.n_head = n_head + self.afn = afn + self.resid_pdrop = resid_pdrop + self.embd_pdrop = embd_pdrop + self.attn_pdrop = attn_pdrop + self.layer_norm_epsilon = layer_norm_epsilon + self.initializer_range = initializer_range + self.predict_special_tokens = predict_special_tokens + self.summary_type = summary_type + self.summary_use_proj = summary_use_proj + self.summary_activation = summary_activation + self.summary_first_dropout = summary_first_dropout + self.summary_proj_to_labels = summary_proj_to_labels @property def max_position_embeddings(self): diff --git a/transformers/configuration_transfo_xl.py b/transformers/configuration_transfo_xl.py index d55a6adbe6..52f0f45a50 100644 --- a/transformers/configuration_transfo_xl.py +++ b/transformers/configuration_transfo_xl.py @@ -34,7 +34,7 @@ class TransfoXLConfig(PretrainedConfig): """Configuration class to store the configuration of a `TransfoXLModel`. Args: - vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `TransfoXLModel` or a configuration json file. + vocab_size: Vocabulary size of `inputs_ids` in `TransfoXLModel` or a configuration json file. cutoffs: cutoffs for the adaptive softmax d_model: Dimensionality of the model's hidden states. d_embed: Dimensionality of the embeddings @@ -68,7 +68,7 @@ class TransfoXLConfig(PretrainedConfig): pretrained_config_archive_map = TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP def __init__(self, - vocab_size_or_config_json_file=267735, + vocab_size=267735, cutoffs=[20000, 40000, 200000], d_model=1024, d_embed=1024, @@ -100,7 +100,7 @@ class TransfoXLConfig(PretrainedConfig): """Constructs TransfoXLConfig. """ super(TransfoXLConfig, self).__init__(**kwargs) - self.n_token = vocab_size_or_config_json_file if isinstance(vocab_size_or_config_json_file, int) else -1 + self.vocab_size = vocab_size self.cutoffs = [] self.cutoffs.extend(cutoffs) self.tie_weight = tie_weight @@ -133,27 +133,17 @@ class TransfoXLConfig(PretrainedConfig): self.init_std = init_std self.layer_norm_epsilon = layer_norm_epsilon - if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2 - and isinstance(vocab_size_or_config_json_file, unicode)): - with open(vocab_size_or_config_json_file, "r", encoding='utf-8') as reader: - json_config = json.loads(reader.read()) - for key, value in json_config.items(): - self.__dict__[key] = value - elif not isinstance(vocab_size_or_config_json_file, int): - raise ValueError("First argument must be either a vocabulary size (int)" - " or the path to a pretrained model config file (str)") - @property def max_position_embeddings(self): return self.tgt_len + self.ext_len + self.mem_len @property - def vocab_size(self): - return self.n_token + def n_token(self): # Backward compatibility + return self.vocab_size - @vocab_size.setter - def vocab_size(self, value): - self.n_token = value + @n_token.setter + def n_token(self, value): # Backward compatibility + self.vocab_size = value @property def hidden_size(self): diff --git a/transformers/configuration_utils.py b/transformers/configuration_utils.py index 82959adb57..6c9eeea175 100644 --- a/transformers/configuration_utils.py +++ b/transformers/configuration_utils.py @@ -49,8 +49,7 @@ class PretrainedConfig(object): pretrained_config_archive_map = {} def __init__(self, **kwargs): - self.finetuning_task = kwargs.pop('finetuning_task', None) - self.num_labels = kwargs.pop('num_labels', 2) + # Attributes with defaults self.output_attentions = kwargs.pop('output_attentions', False) self.output_hidden_states = kwargs.pop('output_hidden_states', False) self.output_past = kwargs.pop('output_past', True) # Not used by all models @@ -59,6 +58,22 @@ class PretrainedConfig(object): self.pruned_heads = kwargs.pop('pruned_heads', {}) self.is_decoder = kwargs.pop('is_decoder', False) + # Fine-tuning task arguments + self.finetuning_task = kwargs.pop('finetuning_task', None) + self.num_labels = kwargs.pop('num_labels', 2) + self.id2label = kwargs.pop('id2label', {i: 'LABEL_{}'.format(i) for i in range(self.num_labels)}) + self.id2label = dict((int(key), value) for key, value in self.id2label.items()) + self.label2id = kwargs.pop('label2id', dict(zip(self.id2label.values(), self.id2label.keys()))) + self.label2id = dict((key, int(value)) for key, value in self.label2id.items()) + + # Additional attributes without default values + for key, value in kwargs.items(): + try: + setattr(self, key, value) + except AttributeError as err: + logger.error("Can't set {} with value {} for {}".format(key, value, self)) + raise err + def save_pretrained(self, save_directory): """ Save a configuration object to the directory `save_directory`, so that it can be re-loaded using the :func:`~transformers.PretrainedConfig.from_pretrained` class method. @@ -183,17 +198,15 @@ class PretrainedConfig(object): @classmethod def from_dict(cls, json_object): """Constructs a `Config` from a Python dictionary of parameters.""" - config = cls(vocab_size_or_config_json_file=-1) - for key, value in json_object.items(): - setattr(config, key, value) - return config + return cls(**json_object) @classmethod def from_json_file(cls, json_file): """Constructs a `Config` from a json file of parameters.""" with open(json_file, "r", encoding='utf-8') as reader: text = reader.read() - return cls.from_dict(json.loads(text)) + dict_obj = json.loads(text) + return cls(**dict_obj) def __eq__(self, other): return self.__dict__ == other.__dict__ diff --git a/transformers/configuration_xlm.py b/transformers/configuration_xlm.py index fa3a5f40f6..0740cc4026 100644 --- a/transformers/configuration_xlm.py +++ b/transformers/configuration_xlm.py @@ -42,7 +42,7 @@ class XLMConfig(PretrainedConfig): """Configuration class to store the configuration of a `XLMModel`. Args: - vocab_size_or_config_json_file: Vocabulary size of `inputs_ids` in `XLMModel`. + vocab_size: Vocabulary size of `inputs_ids` in `XLMModel`. d_model: Size of the encoder layers and the pooler layer. n_layer: Number of hidden layers in the Transformer encoder. n_head: Number of attention heads for each attention layer in @@ -81,7 +81,7 @@ class XLMConfig(PretrainedConfig): pretrained_config_archive_map = XLM_PRETRAINED_CONFIG_ARCHIVE_MAP def __init__(self, - vocab_size_or_config_json_file=30145, + vocab_size=30145, emb_dim=2048, n_layers=12, n_heads=16, @@ -103,9 +103,6 @@ class XLMConfig(PretrainedConfig): unk_index=3, mask_index=5, is_encoder=True, - - finetuning_task=None, - num_labels=2, summary_type='first', summary_use_proj=True, summary_activation=None, @@ -117,56 +114,43 @@ class XLMConfig(PretrainedConfig): """Constructs XLMConfig. """ super(XLMConfig, self).__init__(**kwargs) - - if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2 - and isinstance(vocab_size_or_config_json_file, unicode)): - with open(vocab_size_or_config_json_file, "r", encoding='utf-8') as reader: - json_config = json.loads(reader.read()) - for key, value in json_config.items(): - self.__dict__[key] = value - elif isinstance(vocab_size_or_config_json_file, int): - self.n_words = vocab_size_or_config_json_file - self.emb_dim = emb_dim - self.n_layers = n_layers - self.n_heads = n_heads - self.dropout = dropout - self.attention_dropout = attention_dropout - self.gelu_activation = gelu_activation - self.sinusoidal_embeddings = sinusoidal_embeddings - self.causal = causal - self.asm = asm - self.n_langs = n_langs - self.use_lang_emb = use_lang_emb - self.layer_norm_eps = layer_norm_eps - self.bos_index = bos_index - self.eos_index = eos_index - self.pad_index = pad_index - self.unk_index = unk_index - self.mask_index = mask_index - self.is_encoder = is_encoder - self.max_position_embeddings = max_position_embeddings - self.embed_init_std = embed_init_std - self.init_std = init_std - self.finetuning_task = finetuning_task - self.num_labels = num_labels - self.summary_type = summary_type - self.summary_use_proj = summary_use_proj - self.summary_activation = summary_activation - self.summary_proj_to_labels = summary_proj_to_labels - self.summary_first_dropout = summary_first_dropout - self.start_n_top = start_n_top - self.end_n_top = end_n_top - else: - raise ValueError("First argument must be either a vocabulary size (int)" - " or the path to a pretrained model config file (str)") + self.vocab_size = vocab_size + self.emb_dim = emb_dim + self.n_layers = n_layers + self.n_heads = n_heads + self.dropout = dropout + self.attention_dropout = attention_dropout + self.gelu_activation = gelu_activation + self.sinusoidal_embeddings = sinusoidal_embeddings + self.causal = causal + self.asm = asm + self.n_langs = n_langs + self.use_lang_emb = use_lang_emb + self.layer_norm_eps = layer_norm_eps + self.bos_index = bos_index + self.eos_index = eos_index + self.pad_index = pad_index + self.unk_index = unk_index + self.mask_index = mask_index + self.is_encoder = is_encoder + self.max_position_embeddings = max_position_embeddings + self.embed_init_std = embed_init_std + self.init_std = init_std + self.summary_type = summary_type + self.summary_use_proj = summary_use_proj + self.summary_activation = summary_activation + self.summary_proj_to_labels = summary_proj_to_labels + self.summary_first_dropout = summary_first_dropout + self.start_n_top = start_n_top + self.end_n_top = end_n_top @property - def vocab_size(self): - return self.n_words + def n_words(self): # For backward compatibility + return self.vocab_size - @vocab_size.setter - def vocab_size(self, value): - self.n_words = value + @n_words.setter + def n_words(self, value): # For backward compatibility + self.vocab_size = value @property def hidden_size(self): diff --git a/transformers/configuration_xlnet.py b/transformers/configuration_xlnet.py index 0dbf518849..017c57cfd5 100644 --- a/transformers/configuration_xlnet.py +++ b/transformers/configuration_xlnet.py @@ -35,7 +35,7 @@ class XLNetConfig(PretrainedConfig): """Configuration class to store the configuration of a ``XLNetModel``. Args: - vocab_size_or_config_json_file: Vocabulary size of ``inputs_ids`` in ``XLNetModel``. + vocab_size: Vocabulary size of ``inputs_ids`` in ``XLNetModel``. d_model: Size of the encoder layers and the pooler layer. n_layer: Number of hidden layers in the Transformer encoder. n_head: Number of attention heads for each attention layer in @@ -72,28 +72,22 @@ class XLNetConfig(PretrainedConfig): pretrained_config_archive_map = XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP def __init__(self, - vocab_size_or_config_json_file=32000, + vocab_size=32000, d_model=1024, n_layer=24, n_head=16, d_inner=4096, - max_position_embeddings=512, ff_activation="gelu", untie_r=True, attn_type="bi", - initializer_range=0.02, layer_norm_eps=1e-12, - dropout=0.1, mem_len=None, reuse_len=None, bi_data=False, clamp_len=-1, same_length=False, - - finetuning_task=None, - num_labels=2, summary_type='last', summary_use_proj=True, summary_activation='tanh', @@ -104,58 +98,45 @@ class XLNetConfig(PretrainedConfig): """Constructs XLNetConfig. """ super(XLNetConfig, self).__init__(**kwargs) + self.vocab_size = vocab_size + self.d_model = d_model + self.n_layer = n_layer + self.n_head = n_head + assert d_model % n_head == 0 + self.d_head = d_model // n_head + self.ff_activation = ff_activation + self.d_inner = d_inner + self.untie_r = untie_r + self.attn_type = attn_type - if isinstance(vocab_size_or_config_json_file, str) or (sys.version_info[0] == 2 - and isinstance(vocab_size_or_config_json_file, unicode)): - with open(vocab_size_or_config_json_file, "r", encoding='utf-8') as reader: - json_config = json.loads(reader.read()) - for key, value in json_config.items(): - setattr(config, key, value) - elif isinstance(vocab_size_or_config_json_file, int): - self.n_token = vocab_size_or_config_json_file - self.d_model = d_model - self.n_layer = n_layer - self.n_head = n_head - assert d_model % n_head == 0 - self.d_head = d_model // n_head - self.ff_activation = ff_activation - self.d_inner = d_inner - self.untie_r = untie_r - self.attn_type = attn_type + self.initializer_range = initializer_range + self.layer_norm_eps = layer_norm_eps - self.initializer_range = initializer_range - self.layer_norm_eps = layer_norm_eps + self.dropout = dropout + self.mem_len = mem_len + self.reuse_len = reuse_len + self.bi_data = bi_data + self.clamp_len = clamp_len + self.same_length = same_length - self.dropout = dropout - self.mem_len = mem_len - self.reuse_len = reuse_len - self.bi_data = bi_data - self.clamp_len = clamp_len - self.same_length = same_length - - self.finetuning_task = finetuning_task - self.num_labels = num_labels - self.summary_type = summary_type - self.summary_use_proj = summary_use_proj - self.summary_activation = summary_activation - self.summary_last_dropout = summary_last_dropout - self.start_n_top = start_n_top - self.end_n_top = end_n_top - else: - raise ValueError("First argument must be either a vocabulary size (int)" - " or the path to a pretrained model config file (str)") + self.summary_type = summary_type + self.summary_use_proj = summary_use_proj + self.summary_activation = summary_activation + self.summary_last_dropout = summary_last_dropout + self.start_n_top = start_n_top + self.end_n_top = end_n_top @property def max_position_embeddings(self): return -1 @property - def vocab_size(self): - return self.n_token + def n_token(self): # Backward compatibility + return self.vocab_size - @vocab_size.setter - def vocab_size(self, value): - self.n_token = value + @n_token.setter + def n_token(self, value): # Backward compatibility + self.vocab_size = value @property def hidden_size(self): diff --git a/transformers/convert_roberta_original_pytorch_checkpoint_to_pytorch.py b/transformers/convert_roberta_original_pytorch_checkpoint_to_pytorch.py index 60935add60..b4dc1bb61b 100644 --- a/transformers/convert_roberta_original_pytorch_checkpoint_to_pytorch.py +++ b/transformers/convert_roberta_original_pytorch_checkpoint_to_pytorch.py @@ -46,7 +46,7 @@ def convert_roberta_checkpoint_to_pytorch(roberta_checkpoint_path, pytorch_dump_ roberta = FairseqRobertaModel.from_pretrained(roberta_checkpoint_path) roberta.eval() # disable dropout config = BertConfig( - vocab_size_or_config_json_file=50265, + vocab_size=50265, hidden_size=roberta.args.encoder_embed_dim, num_hidden_layers=roberta.args.encoder_layers, num_attention_heads=roberta.args.encoder_attention_heads, diff --git a/transformers/modeling_gpt2.py b/transformers/modeling_gpt2.py index 96fd1c0607..ea660262d7 100644 --- a/transformers/modeling_gpt2.py +++ b/transformers/modeling_gpt2.py @@ -634,6 +634,7 @@ class GPT2DoubleHeadsModel(GPT2PreTrainedModel): """ def __init__(self, config): super(GPT2DoubleHeadsModel, self).__init__(config) + config.num_labels = 1 self.transformer = GPT2Model(config) self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False) self.multiple_choice_head = SequenceSummary(config) diff --git a/transformers/modeling_tf_gpt2.py b/transformers/modeling_tf_gpt2.py index c738e5e8e3..973473179f 100644 --- a/transformers/modeling_tf_gpt2.py +++ b/transformers/modeling_tf_gpt2.py @@ -574,6 +574,7 @@ class TFGPT2DoubleHeadsModel(TFGPT2PreTrainedModel): """ def __init__(self, config, *inputs, **kwargs): super(TFGPT2DoubleHeadsModel, self).__init__(config, *inputs, **kwargs) + config.num_labels = 1 self.transformer = TFGPT2MainLayer(config, name='transformer') self.multiple_choice_head = TFSequenceSummary(config, initializer_range=config.initializer_range, name='multiple_choice_head') diff --git a/transformers/modeling_tf_transfo_xl.py b/transformers/modeling_tf_transfo_xl.py index fd325e218e..848edfa37a 100644 --- a/transformers/modeling_tf_transfo_xl.py +++ b/transformers/modeling_tf_transfo_xl.py @@ -353,7 +353,7 @@ class TFTransfoXLMainLayer(tf.keras.layers.Layer): self.output_attentions = config.output_attentions self.output_hidden_states = config.output_hidden_states - self.n_token = config.n_token + self.n_token = config.vocab_size self.d_embed = config.d_embed self.d_model = config.d_model @@ -361,7 +361,7 @@ class TFTransfoXLMainLayer(tf.keras.layers.Layer): self.d_head = config.d_head self.untie_r = config.untie_r - self.word_emb = TFAdaptiveEmbedding(config.n_token, config.d_embed, config.d_model, config.cutoffs, + self.word_emb = TFAdaptiveEmbedding(config.vocab_size, config.d_embed, config.d_model, config.cutoffs, div_val=config.div_val, init_std=config.init_std, name='word_emb') self.drop = tf.keras.layers.Dropout(config.dropout) @@ -729,7 +729,7 @@ class TFTransfoXLLMHeadModel(TFTransfoXLPreTrainedModel): raise NotImplementedError # use adaptive softmax (including standard softmax) else: - self.crit = TFAdaptiveSoftmaxMask(config.n_token, config.d_embed, config.d_model, + self.crit = TFAdaptiveSoftmaxMask(config.vocab_size, config.d_embed, config.d_model, config.cutoffs, div_val=config.div_val, name='crit') def reset_length(self, tgt_len, ext_len, mem_len): diff --git a/transformers/modeling_tf_transfo_xl_utilities.py b/transformers/modeling_tf_transfo_xl_utilities.py index e6a6dfe686..f730af851f 100644 --- a/transformers/modeling_tf_transfo_xl_utilities.py +++ b/transformers/modeling_tf_transfo_xl_utilities.py @@ -25,15 +25,15 @@ import tensorflow as tf from .modeling_tf_utils import shape_list class TFAdaptiveSoftmaxMask(tf.keras.layers.Layer): - def __init__(self, n_token, d_embed, d_proj, cutoffs, div_val=1, + def __init__(self, vocab_size, d_embed, d_proj, cutoffs, div_val=1, keep_order=False, **kwargs): super(TFAdaptiveSoftmaxMask, self).__init__(**kwargs) - self.n_token = n_token + self.vocab_size = vocab_size self.d_embed = d_embed self.d_proj = d_proj - self.cutoffs = cutoffs + [n_token] + self.cutoffs = cutoffs + [vocab_size] self.cutoff_ends = [0] + self.cutoffs self.div_val = div_val @@ -66,11 +66,11 @@ class TFAdaptiveSoftmaxMask(tf.keras.layers.Layer): self.out_projs.append(weight) else: self.out_projs.append(None) - weight = self.add_weight(shape=(self.n_token, self.d_embed,), + weight = self.add_weight(shape=(self.vocab_size, self.d_embed,), initializer='zeros', trainable=True, name='out_layers_._{}_._weight'.format(i)) - bias = self.add_weight(shape=(self.n_token,), + bias = self.add_weight(shape=(self.vocab_size,), initializer='zeros', trainable=True, name='out_layers_._{}_._bias'.format(i)) @@ -114,7 +114,7 @@ class TFAdaptiveSoftmaxMask(tf.keras.layers.Layer): hidden, target = inputs head_logprob = 0 if self.n_clusters == 0: - softmax_b = tf.get_variable('bias', [n_token], initializer=tf.zeros_initializer()) + softmax_b = tf.get_variable('bias', [self.config.vocab_size], initializer=tf.zeros_initializer()) output = self._logit(hidden, self.out_layers[0][0], self.out_layers[0][1], self.out_projs[0]) if target is not None: loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=target, logits=output) diff --git a/transformers/modeling_tf_xlnet.py b/transformers/modeling_tf_xlnet.py index 759b57d835..dde2b6a8df 100644 --- a/transformers/modeling_tf_xlnet.py +++ b/transformers/modeling_tf_xlnet.py @@ -366,7 +366,7 @@ class TFXLNetMainLayer(tf.keras.layers.Layer): self.use_bfloat16 = config.use_bfloat16 self.initializer_range = config.initializer_range - self.word_embedding = TFSharedEmbeddings(config.n_token, config.d_model, initializer_range=config.initializer_range, name='word_embedding') + self.word_embedding = TFSharedEmbeddings(config.vocab_size, config.d_model, initializer_range=config.initializer_range, name='word_embedding') self.layer = [TFXLNetLayer(config, name='layer_._{}'.format(i)) for i in range(config.n_layer)] self.dropout = tf.keras.layers.Dropout(config.dropout) diff --git a/transformers/modeling_transfo_xl.py b/transformers/modeling_transfo_xl.py index a6a82f0dfe..f87d857a7f 100644 --- a/transformers/modeling_transfo_xl.py +++ b/transformers/modeling_transfo_xl.py @@ -592,14 +592,14 @@ class TransfoXLModel(TransfoXLPreTrainedModel): self.output_attentions = config.output_attentions self.output_hidden_states = config.output_hidden_states - self.n_token = config.n_token + self.n_token = config.vocab_size self.d_embed = config.d_embed self.d_model = config.d_model self.n_head = config.n_head self.d_head = config.d_head - self.word_emb = AdaptiveEmbedding(config.n_token, config.d_embed, config.d_model, config.cutoffs, + self.word_emb = AdaptiveEmbedding(config.vocab_size, config.d_embed, config.d_model, config.cutoffs, div_val=config.div_val) self.drop = nn.Dropout(config.dropout) @@ -836,11 +836,11 @@ class TransfoXLLMHeadModel(TransfoXLPreTrainedModel): self.sample_softmax = config.sample_softmax # use sampled softmax if config.sample_softmax > 0: - self.out_layer = nn.Linear(config.d_model, config.n_token) - self.sampler = LogUniformSampler(config.n_token, config.sample_softmax) + self.out_layer = nn.Linear(config.d_model, config.vocab_size) + self.sampler = LogUniformSampler(config.vocab_size, config.sample_softmax) # use adaptive softmax (including standard softmax) else: - self.crit = ProjectedAdaptiveLogSoftmax(config.n_token, config.d_embed, config.d_model, + self.crit = ProjectedAdaptiveLogSoftmax(config.vocab_size, config.d_embed, config.d_model, config.cutoffs, div_val=config.div_val) self.init_weights() diff --git a/transformers/modeling_xlnet.py b/transformers/modeling_xlnet.py index 225e5b059b..daed5f2857 100644 --- a/transformers/modeling_xlnet.py +++ b/transformers/modeling_xlnet.py @@ -609,7 +609,7 @@ class XLNetModel(XLNetPreTrainedModel): self.clamp_len = config.clamp_len self.n_layer = config.n_layer - self.word_embedding = nn.Embedding(config.n_token, config.d_model) + self.word_embedding = nn.Embedding(config.vocab_size, config.d_model) self.mask_emb = nn.Parameter(torch.FloatTensor(1, 1, config.d_model)) self.layer = nn.ModuleList([XLNetLayer(config) for _ in range(config.n_layer)]) self.dropout = nn.Dropout(config.dropout) @@ -940,7 +940,7 @@ class XLNetLMHeadModel(XLNetPreTrainedModel): self.same_length = config.same_length self.transformer = XLNetModel(config) - self.lm_loss = nn.Linear(config.d_model, config.n_token, bias=True) + self.lm_loss = nn.Linear(config.d_model, config.vocab_size, bias=True) self.init_weights() diff --git a/transformers/tests/modeling_albert_test.py b/transformers/tests/modeling_albert_test.py index a14d66ae8f..1911d244e7 100644 --- a/transformers/tests/modeling_albert_test.py +++ b/transformers/tests/modeling_albert_test.py @@ -110,7 +110,7 @@ class AlbertModelTest(CommonTestCases.CommonModelTester): choice_labels = ids_tensor([self.batch_size], self.num_choices) config = AlbertConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, hidden_size=self.hidden_size, num_hidden_layers=self.num_hidden_layers, num_attention_heads=self.num_attention_heads, diff --git a/transformers/tests/modeling_bert_test.py b/transformers/tests/modeling_bert_test.py index 539f66cd3f..0eb7bc9a14 100644 --- a/transformers/tests/modeling_bert_test.py +++ b/transformers/tests/modeling_bert_test.py @@ -109,7 +109,7 @@ class BertModelTest(CommonTestCases.CommonModelTester): choice_labels = ids_tensor([self.batch_size], self.num_choices) config = BertConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, hidden_size=self.hidden_size, num_hidden_layers=self.num_hidden_layers, num_attention_heads=self.num_attention_heads, diff --git a/transformers/tests/modeling_common_test.py b/transformers/tests/modeling_common_test.py index 80d5d95455..f86eb7a3d0 100644 --- a/transformers/tests/modeling_common_test.py +++ b/transformers/tests/modeling_common_test.py @@ -633,7 +633,7 @@ class CommonTestCases: mc_token_ids = ids_tensor([self.batch_size, self.n_choices], self.seq_length) config = self.config_class( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, n_positions=self.n_positions, n_embd=self.hidden_size, n_layer=self.num_hidden_layers, diff --git a/transformers/tests/modeling_ctrl_test.py b/transformers/tests/modeling_ctrl_test.py index 8c14578a5c..c7de49b2ab 100644 --- a/transformers/tests/modeling_ctrl_test.py +++ b/transformers/tests/modeling_ctrl_test.py @@ -114,7 +114,7 @@ class CTRLModelTest(CommonTestCases.CommonModelTester): choice_labels = ids_tensor([self.batch_size], self.num_choices) config = CTRLConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, n_embd=self.hidden_size, n_layer=self.num_hidden_layers, n_head=self.num_attention_heads, diff --git a/transformers/tests/modeling_distilbert_test.py b/transformers/tests/modeling_distilbert_test.py index 4b8f64327d..82f71c40da 100644 --- a/transformers/tests/modeling_distilbert_test.py +++ b/transformers/tests/modeling_distilbert_test.py @@ -105,7 +105,7 @@ class DistilBertModelTest(CommonTestCases.CommonModelTester): choice_labels = ids_tensor([self.batch_size], self.num_choices) config = DistilBertConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, dim=self.hidden_size, n_layers=self.num_hidden_layers, n_heads=self.num_attention_heads, diff --git a/transformers/tests/modeling_gpt2_test.py b/transformers/tests/modeling_gpt2_test.py index ecaa2a4bd0..a82e39c261 100644 --- a/transformers/tests/modeling_gpt2_test.py +++ b/transformers/tests/modeling_gpt2_test.py @@ -110,7 +110,7 @@ class GPT2ModelTest(CommonTestCases.CommonModelTester): choice_labels = ids_tensor([self.batch_size], self.num_choices) config = GPT2Config( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, n_embd=self.hidden_size, n_layer=self.num_hidden_layers, n_head=self.num_attention_heads, diff --git a/transformers/tests/modeling_openai_test.py b/transformers/tests/modeling_openai_test.py index 8e4d13438d..7655e432e8 100644 --- a/transformers/tests/modeling_openai_test.py +++ b/transformers/tests/modeling_openai_test.py @@ -98,7 +98,7 @@ class OpenAIGPTModelTest(CommonTestCases.CommonModelTester): choice_labels = ids_tensor([self.batch_size], self.num_choices) config = OpenAIGPTConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, n_embd=self.hidden_size, n_layer=self.num_hidden_layers, n_head=self.num_attention_heads, diff --git a/transformers/tests/modeling_roberta_test.py b/transformers/tests/modeling_roberta_test.py index 7a3553b164..4d34a50528 100644 --- a/transformers/tests/modeling_roberta_test.py +++ b/transformers/tests/modeling_roberta_test.py @@ -106,7 +106,7 @@ class RobertaModelTest(CommonTestCases.CommonModelTester): choice_labels = ids_tensor([self.batch_size], self.num_choices) config = RobertaConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, hidden_size=self.hidden_size, num_hidden_layers=self.num_hidden_layers, num_attention_heads=self.num_attention_heads, diff --git a/transformers/tests/modeling_tf_albert_test.py b/transformers/tests/modeling_tf_albert_test.py index 7d3325b70b..93aeab66c2 100644 --- a/transformers/tests/modeling_tf_albert_test.py +++ b/transformers/tests/modeling_tf_albert_test.py @@ -118,7 +118,7 @@ class TFAlbertModelTest(TFCommonTestCases.TFCommonModelTester): choice_labels = ids_tensor([self.batch_size], self.num_choices) config = AlbertConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, hidden_size=self.hidden_size, num_hidden_layers=self.num_hidden_layers, num_attention_heads=self.num_attention_heads, diff --git a/transformers/tests/modeling_tf_bert_test.py b/transformers/tests/modeling_tf_bert_test.py index d7a86fecb9..20073e1ab8 100644 --- a/transformers/tests/modeling_tf_bert_test.py +++ b/transformers/tests/modeling_tf_bert_test.py @@ -114,7 +114,7 @@ class TFBertModelTest(TFCommonTestCases.TFCommonModelTester): choice_labels = ids_tensor([self.batch_size], self.num_choices) config = BertConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, hidden_size=self.hidden_size, num_hidden_layers=self.num_hidden_layers, num_attention_heads=self.num_attention_heads, diff --git a/transformers/tests/modeling_tf_ctrl_test.py b/transformers/tests/modeling_tf_ctrl_test.py index 0b421c20c9..0876582e57 100644 --- a/transformers/tests/modeling_tf_ctrl_test.py +++ b/transformers/tests/modeling_tf_ctrl_test.py @@ -112,7 +112,7 @@ class TFCTRLModelTest(TFCommonTestCases.TFCommonModelTester): choice_labels = ids_tensor([self.batch_size], self.num_choices) config = CTRLConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, n_embd=self.hidden_size, n_layer=self.num_hidden_layers, n_head=self.num_attention_heads, diff --git a/transformers/tests/modeling_tf_distilbert_test.py b/transformers/tests/modeling_tf_distilbert_test.py index 0ec45150ca..d9e971c2a5 100644 --- a/transformers/tests/modeling_tf_distilbert_test.py +++ b/transformers/tests/modeling_tf_distilbert_test.py @@ -107,7 +107,7 @@ class TFDistilBertModelTest(TFCommonTestCases.TFCommonModelTester): choice_labels = ids_tensor([self.batch_size], self.num_choices) config = DistilBertConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, dim=self.hidden_size, n_layers=self.num_hidden_layers, n_heads=self.num_attention_heads, diff --git a/transformers/tests/modeling_tf_gpt2_test.py b/transformers/tests/modeling_tf_gpt2_test.py index e070b72e65..3f30b32787 100644 --- a/transformers/tests/modeling_tf_gpt2_test.py +++ b/transformers/tests/modeling_tf_gpt2_test.py @@ -115,7 +115,7 @@ class TFGPT2ModelTest(TFCommonTestCases.TFCommonModelTester): choice_labels = ids_tensor([self.batch_size], self.num_choices) config = GPT2Config( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, n_embd=self.hidden_size, n_layer=self.num_hidden_layers, n_head=self.num_attention_heads, diff --git a/transformers/tests/modeling_tf_openai_gpt_test.py b/transformers/tests/modeling_tf_openai_gpt_test.py index 675e806c12..863dbf1bc0 100644 --- a/transformers/tests/modeling_tf_openai_gpt_test.py +++ b/transformers/tests/modeling_tf_openai_gpt_test.py @@ -114,7 +114,7 @@ class TFOpenAIGPTModelTest(TFCommonTestCases.TFCommonModelTester): choice_labels = ids_tensor([self.batch_size], self.num_choices) config = OpenAIGPTConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, n_embd=self.hidden_size, n_layer=self.num_hidden_layers, n_head=self.num_attention_heads, diff --git a/transformers/tests/modeling_tf_roberta_test.py b/transformers/tests/modeling_tf_roberta_test.py index 42440bf1b7..f4ed97c44b 100644 --- a/transformers/tests/modeling_tf_roberta_test.py +++ b/transformers/tests/modeling_tf_roberta_test.py @@ -109,7 +109,7 @@ class TFRobertaModelTest(TFCommonTestCases.TFCommonModelTester): choice_labels = ids_tensor([self.batch_size], self.num_choices) config = RobertaConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, hidden_size=self.hidden_size, num_hidden_layers=self.num_hidden_layers, num_attention_heads=self.num_attention_heads, diff --git a/transformers/tests/modeling_tf_transfo_xl_test.py b/transformers/tests/modeling_tf_transfo_xl_test.py index 03e332bdc1..553263250a 100644 --- a/transformers/tests/modeling_tf_transfo_xl_test.py +++ b/transformers/tests/modeling_tf_transfo_xl_test.py @@ -92,7 +92,7 @@ class TFTransfoXLModelTest(TFCommonTestCases.TFCommonModelTester): lm_labels = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) config = TransfoXLConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, mem_len=self.mem_len, clamp_len=self.clamp_len, cutoffs=self.cutoffs, diff --git a/transformers/tests/modeling_tf_xlm_test.py b/transformers/tests/modeling_tf_xlm_test.py index a680b70367..228e436149 100644 --- a/transformers/tests/modeling_tf_xlm_test.py +++ b/transformers/tests/modeling_tf_xlm_test.py @@ -125,7 +125,7 @@ class TFXLMModelTest(TFCommonTestCases.TFCommonModelTester): is_impossible_labels = ids_tensor([self.batch_size], 2, dtype=tf.float32) config = XLMConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, n_special=self.n_special, emb_dim=self.hidden_size, n_layers=self.num_hidden_layers, diff --git a/transformers/tests/modeling_tf_xlnet_test.py b/transformers/tests/modeling_tf_xlnet_test.py index 94864b86f2..eb66d92793 100644 --- a/transformers/tests/modeling_tf_xlnet_test.py +++ b/transformers/tests/modeling_tf_xlnet_test.py @@ -64,7 +64,6 @@ class TFXLNetModelTest(TFCommonTestCases.TFCommonModelTester): num_attention_heads=4, d_inner=128, num_hidden_layers=5, - max_position_embeddings=10, type_sequence_label_size=2, untie_r=True, bi_data=False, @@ -88,7 +87,6 @@ class TFXLNetModelTest(TFCommonTestCases.TFCommonModelTester): self.num_attention_heads = num_attention_heads self.d_inner = d_inner self.num_hidden_layers = num_hidden_layers - self.max_position_embeddings = max_position_embeddings self.bi_data = bi_data self.untie_r = untie_r self.same_length = same_length @@ -122,13 +120,12 @@ class TFXLNetModelTest(TFCommonTestCases.TFCommonModelTester): is_impossible_labels = ids_tensor([self.batch_size], 2, dtype=tf.float32) config = XLNetConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, d_model=self.hidden_size, n_head=self.num_attention_heads, d_inner=self.d_inner, n_layer=self.num_hidden_layers, untie_r=self.untie_r, - max_position_embeddings=self.max_position_embeddings, mem_len=self.mem_len, clamp_len=self.clamp_len, same_length=self.same_length, diff --git a/transformers/tests/modeling_transfo_xl_test.py b/transformers/tests/modeling_transfo_xl_test.py index 647dd3724d..dca46444ba 100644 --- a/transformers/tests/modeling_transfo_xl_test.py +++ b/transformers/tests/modeling_transfo_xl_test.py @@ -91,7 +91,7 @@ class TransfoXLModelTest(CommonTestCases.CommonModelTester): lm_labels = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) config = TransfoXLConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, mem_len=self.mem_len, clamp_len=self.clamp_len, cutoffs=self.cutoffs, diff --git a/transformers/tests/modeling_xlm_test.py b/transformers/tests/modeling_xlm_test.py index f6b980767c..7cae6c848e 100644 --- a/transformers/tests/modeling_xlm_test.py +++ b/transformers/tests/modeling_xlm_test.py @@ -121,7 +121,7 @@ class XLMModelTest(CommonTestCases.CommonModelTester): is_impossible_labels = ids_tensor([self.batch_size], 2).float() config = XLMConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, n_special=self.n_special, emb_dim=self.hidden_size, n_layers=self.num_hidden_layers, diff --git a/transformers/tests/modeling_xlnet_test.py b/transformers/tests/modeling_xlnet_test.py index 56b6bb3f4d..6d901ee699 100644 --- a/transformers/tests/modeling_xlnet_test.py +++ b/transformers/tests/modeling_xlnet_test.py @@ -60,7 +60,6 @@ class XLNetModelTest(CommonTestCases.CommonModelTester): num_attention_heads=4, d_inner=128, num_hidden_layers=5, - max_position_embeddings=10, type_sequence_label_size=2, untie_r=True, bi_data=False, @@ -84,7 +83,6 @@ class XLNetModelTest(CommonTestCases.CommonModelTester): self.num_attention_heads = num_attention_heads self.d_inner = d_inner self.num_hidden_layers = num_hidden_layers - self.max_position_embeddings = max_position_embeddings self.bi_data = bi_data self.untie_r = untie_r self.same_length = same_length @@ -116,13 +114,12 @@ class XLNetModelTest(CommonTestCases.CommonModelTester): token_labels = ids_tensor([self.batch_size, self.seq_length], self.type_vocab_size) config = XLNetConfig( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, d_model=self.hidden_size, n_head=self.num_attention_heads, d_inner=self.d_inner, n_layer=self.num_hidden_layers, untie_r=self.untie_r, - max_position_embeddings=self.max_position_embeddings, mem_len=self.mem_len, clamp_len=self.clamp_len, same_length=self.same_length, From 8ade2040984c2cd3fd04bf56b133f70718254b03 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 13 Dec 2019 14:48:47 +0100 Subject: [PATCH 351/505] fix tf --- transformers/modeling_openai.py | 1 + transformers/modeling_tf_openai.py | 1 + 2 files changed, 2 insertions(+) diff --git a/transformers/modeling_openai.py b/transformers/modeling_openai.py index 4fe7ffee8b..72f1224e39 100644 --- a/transformers/modeling_openai.py +++ b/transformers/modeling_openai.py @@ -590,6 +590,7 @@ class OpenAIGPTDoubleHeadsModel(OpenAIGPTPreTrainedModel): def __init__(self, config): super(OpenAIGPTDoubleHeadsModel, self).__init__(config) + config.num_labels = 1 self.transformer = OpenAIGPTModel(config) self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False) self.multiple_choice_head = SequenceSummary(config) diff --git a/transformers/modeling_tf_openai.py b/transformers/modeling_tf_openai.py index dac3b17590..bd469f0205 100644 --- a/transformers/modeling_tf_openai.py +++ b/transformers/modeling_tf_openai.py @@ -538,6 +538,7 @@ class TFOpenAIGPTDoubleHeadsModel(TFOpenAIGPTPreTrainedModel): """ def __init__(self, config, *inputs, **kwargs): super(TFOpenAIGPTDoubleHeadsModel, self).__init__(config, *inputs, **kwargs) + config.num_labels = 1 self.transformer = TFOpenAIGPTMainLayer(config, name='transformer') self.multiple_choice_head = TFSequenceSummary(config, initializer_range=config.initializer_range, name='multiple_choice_head') From 5a5c4349e8a141d2c0915d71cb3cee101da0db6f Mon Sep 17 00:00:00 2001 From: Pierric Cistac Date: Fri, 13 Dec 2019 10:02:33 -0500 Subject: [PATCH 352/505] Fix summarization `to_cpu` doc --- examples/summarization/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/summarization/README.md b/examples/summarization/README.md index 96825cfa46..b98581e8e5 100644 --- a/examples/summarization/README.md +++ b/examples/summarization/README.md @@ -29,7 +29,7 @@ And move all the stories to the same folder. We will refer as `$DATA_PATH` the p python run_summarization.py \ --documents_dir $DATA_PATH \ --summaries_output_dir $SUMMARIES_PATH \ # optional - --to_cpu false \ + --no_cuda false \ --batch_size 4 \ --min_length 50 \ --max_length 200 \ @@ -39,7 +39,7 @@ python run_summarization.py \ --compute_rouge true ``` -The scripts executes on GPU if one is available and if `to_cpu` is not set to `true`. Inference on multiple GPUs is not suported yet. The ROUGE scores will be displayed in the console at the end of evaluation and written in a `rouge_scores.txt` file. The script takes 30 hours to compute with a single Tesla V100 GPU and a batch size of 10 (300,000 texts to summarize). +The scripts executes on GPU if one is available and if `no_cuda` is not set to `true`. Inference on multiple GPUs is not suported yet. The ROUGE scores will be displayed in the console at the end of evaluation and written in a `rouge_scores.txt` file. The script takes 30 hours to compute with a single Tesla V100 GPU and a batch size of 10 (300,000 texts to summarize). ## Summarize any text @@ -49,7 +49,7 @@ Put the documents that you would like to summarize in a folder (the path to whic python run_summarization.py \ --documents_dir $DATA_PATH \ --summaries_output_dir $SUMMARIES_PATH \ # optional - --to_cpu false \ + --no_cuda false \ --batch_size 4 \ --min_length 50 \ --max_length 200 \ From 0b51532ce94140cdb22f761b09fff28cce76f985 Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Fri, 13 Dec 2019 16:22:50 +0100 Subject: [PATCH 353/505] Reintroducing the batch_encode_plus method --- transformers/tokenization_utils.py | 86 ++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index f4395cd82c..169caff8dc 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -878,6 +878,92 @@ class PreTrainedTokenizer(object): return_overflowing_tokens=return_overflowing_tokens, return_special_tokens_mask=return_special_tokens_mask) + def batch_encode_plus(self, + batch_text_or_text_pairs=None, + add_special_tokens=False, + max_length=None, + stride=0, + truncation_strategy='longest_first', + return_tensors=None, + return_input_lengths=False, + return_attention_masks=False, + **kwargs): + """ + Returns a dictionary containing the encoded sequence or sequence pair and additional information: + the mask for sequence classification and the overflowing elements if a ``max_length`` is specified. + + Args: + batch_text_or_text_pairs: Batch of sequences or pair of sequences to be encoded. + This can be a list of string/string-sequences/int-sequences or a list of pair of + string/string-sequences/int-sequence (see details in encode_plus) + add_special_tokens: if set to ``True``, the sequences will be encoded with the special tokens relative + to their model. + max_length: if set to a number, will limit the total sequence returned so that it has a maximum length. + If there are overflowing tokens, those will be added to the returned dictionary` + stride: if set to a number along with max_length, the overflowing tokens returned will contain some tokens + from the main sequence returned. The value of this argument defines the number of additional tokens. + truncation_strategy: string selected in the following options: + - 'longest_first' (default) Iteratively reduce the inputs sequence until the input is under max_length + starting from the longest one at each token (when there is a pair of input sequences) + - 'only_first': Only truncate the first sequence + - 'only_second': Only truncate the second sequence + - 'do_not_truncate': Does not truncate (raise an error if the input sequence is longer than max_length) + return_tensors: (optional) can be set to 'tf' or 'pt' to return respectively TensorFlow tf.constant + or PyTorch torch.Tensor instead of a list of python integers. + **kwargs: passed to the `self.tokenize()` method + """ + batch_outputs = {} + for ids_or_pair_ids in batch_text_or_text_pairs: + if isinstance(ids_or_pair_ids, (list, tuple)): + assert len(ids_or_pair_ids) == 2 + ids, pair_ids = ids_or_pair_ids + else: + ids, pair_ids = ids_or_pair_ids, None + outputs = self.encode_plus(ids, pair_ids, add_special_tokens=add_special_tokens, max_length=max_length, + stride=stride, truncation_strategy=truncation_strategy, return_tensors=None) + + # Append the non-padded length to the output + if return_input_lengths: + outputs['input_len'] = len(outputs['input_ids']) + + for key, value in outputs.items(): + if key not in batch_outputs: + batch_outputs[key] = [] + batch_outputs[key].append(value) + + # Compute longest sequence size + max_seq_len = max(map(len, batch_outputs['input_ids'])) + + if return_attention_masks: + # Allow the model to not give any special attention to padded input + batch_outputs['attention_mask'] = [[0] * len(v) for v in batch_outputs['input_ids']] + + if return_tensors is not None: + + # Do the tensor conversion in batch + for key, value in batch_outputs.items(): + + padded_value = value + if key != 'input_len': + # Padding handle + padded_value = [v + [self.pad_token_id if key == 'input_ids' else 1] * (max_seq_len - len(v)) for v in padded_value] + + if return_tensors == 'tf' and is_tf_available(): + batch_outputs[key] = tf.constant(padded_value) + elif return_tensors == 'pt' and is_torch_available(): + batch_outputs[key] = torch.tensor(padded_value) + elif return_tensors is not None: + logger.warning("Unable to convert output to tensors format {}, PyTorch or TensorFlow is not available.".format(return_tensors)) + + # encoder_attention_mask requires 1 for real token, 0 for padding, just invert value + if return_attention_masks: + if is_tf_available(): + batch_outputs['attention_mask'] = tf.abs(batch_outputs['attention_mask'] - 1) + else: + batch_outputs['attention_mask'] = torch.abs(batch_outputs['attention_mask'] - 1) + + return batch_outputs + def prepare_for_model(self, ids, pair_ids=None, max_length=None, add_special_tokens=True, stride=0, truncation_strategy='longest_first', pad_to_max_length=False, From 5c00e344c1350e079d428a4d69cbb465ca7ffde9 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 13 Dec 2019 16:33:29 +0100 Subject: [PATCH 354/505] update model doc - swith 3B/11B to 3b/11b --- docs/source/pretrained_models.rst | 25 ++++++++++--------------- transformers/configuration_t5.py | 4 ++-- transformers/modeling_t5.py | 4 ++-- transformers/modeling_tf_t5.py | 4 ++-- transformers/tokenization_t5.py | 8 ++++---- 5 files changed, 20 insertions(+), 25 deletions(-) diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst index 7e1366b53a..c6b990f213 100644 --- a/docs/source/pretrained_models.rst +++ b/docs/source/pretrained_models.rst @@ -217,25 +217,20 @@ Here is the full list of the currently provided pretrained models together with | | | | ALBERT xxlarge model with no dropout, additional training data and longer training | | | | (see `details `__) | +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ -| T5 | ``t5-small`` | | 6-layer, 768-hidden, 12-heads, 66M parameters | -| | | | The DistilBERT model distilled from the BERT model `bert-base-uncased` checkpoint | -| | | (see `details `__) | +| T5 | ``t5-small`` | | ~60M parameters with 6-layers, 512-hidden-state, 2048 feed-forward hidden-state, 8-heads, | +| | | | Trained on English text: the Colossal Clean Crawled Corpus (C4) | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ -| | ``t5-base`` | | 6-layer, 768-hidden, 12-heads, 66M parameters | -| | | | The DistilBERT model distilled from the BERT model `bert-base-uncased` checkpoint, with an additional linear layer. | -| | | (see `details `__) | +| | ``t5-base`` | | ~220M parameters with 12-layers, 768-hidden-state, 3072 feed-forward hidden-state, 12-heads, | +| | | | Trained on English text: the Colossal Clean Crawled Corpus (C4) | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ -| | ``t5-large`` | | 6-layer, 768-hidden, 12-heads, 82M parameters | -| | | | The DistilGPT2 model distilled from the GPT2 model `gpt2` checkpoint. | -| | | (see `details `__) | +| | ``t5-large`` | | ~770M parameters with 24-layers, 1024-hidden-state, 4096 feed-forward hidden-state, 16-heads, | +| | | | Trained on English text: the Colossal Clean Crawled Corpus (C4) | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ -| | ``t5-3b`` | | 6-layer, 768-hidden, 12-heads, 82M parameters | -| | | | The DistilRoBERTa model distilled from the RoBERTa model `roberta-base` checkpoint. | -| | | (see `details `__) | +| | ``t5-3B`` | | ~2.8B parameters with 24-layers, 1024-hidden-state, 16384 feed-forward hidden-state, 32-heads, | +| | | | Trained on English text: the Colossal Clean Crawled Corpus (C4) | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ -| | ``t5-11b`` | | 6-layer, 768-hidden, 12-heads, 82M parameters | -| | | | The DistilRoBERTa model distilled from the RoBERTa model `roberta-base` checkpoint. | -| | | (see `details `__) | +| | ``t5-11B`` | | ~11B parameters with 24-layers, 1024-hidden-state, 65536 feed-forward hidden-state, 128-heads, | +| | | | Trained on English text: the Colossal Clean Crawled Corpus (C4) | +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ diff --git a/transformers/configuration_t5.py b/transformers/configuration_t5.py index 2ccdebc2b1..6391cb4180 100644 --- a/transformers/configuration_t5.py +++ b/transformers/configuration_t5.py @@ -30,8 +30,8 @@ T5_PRETRAINED_CONFIG_ARCHIVE_MAP = { 't5-small': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-small-config.json", 't5-base': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-base-config.json", 't5-large': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-large-config.json", - 't5-3B': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-3B-config.json", - 't5-11B': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-11B-config.json", + 't5-3b': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-3b-config.json", + 't5-11b': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-11b-config.json", } diff --git a/transformers/modeling_t5.py b/transformers/modeling_t5.py index c9310179a3..263dc33b70 100644 --- a/transformers/modeling_t5.py +++ b/transformers/modeling_t5.py @@ -44,8 +44,8 @@ T5_PRETRAINED_MODEL_ARCHIVE_MAP = { 't5-small': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-small-pytorch_model.bin", 't5-base': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-base-pytorch_model.bin", 't5-large': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-large-pytorch_model.bin", - 't5-3B': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-3B-pytorch_model.bin", - 't5-11B': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-11B-pytorch_model.bin", + 't5-3b': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-3b-pytorch_model.bin", + 't5-11b': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-11b-pytorch_model.bin", } #################################################### diff --git a/transformers/modeling_tf_t5.py b/transformers/modeling_tf_t5.py index 0ae7fff412..1336a1c30d 100644 --- a/transformers/modeling_tf_t5.py +++ b/transformers/modeling_tf_t5.py @@ -34,8 +34,8 @@ TF_T5_PRETRAINED_MODEL_ARCHIVE_MAP = { 't5-small': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-small-tf_model.h5", 't5-base': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-base-tf_model.h5", 't5-large': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-large-tf_model.h5", - 't5-3B': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-3B-tf_model.h5", - 't5-11B': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-11B-tf_model.h5", + 't5-3b': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-3b-tf_model.h5", + 't5-11b': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-11b-tf_model.h5", } #################################################### diff --git a/transformers/tokenization_t5.py b/transformers/tokenization_t5.py index 62e9c069e2..9fd37b67c0 100644 --- a/transformers/tokenization_t5.py +++ b/transformers/tokenization_t5.py @@ -44,8 +44,8 @@ PRETRAINED_VOCAB_FILES_MAP = { 't5-small': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-spiece.model", 't5-base': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-spiece.model", 't5-large': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-spiece.model", - 't5-3B': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-spiece.model", - 't5-11B': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-spiece.model", + 't5-3b': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-spiece.model", + 't5-11b': "https://s3.amazonaws.com/models.huggingface.co/bert/t5-spiece.model", } } @@ -56,8 +56,8 @@ PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { 't5-small': 512, 't5-base': 512, 't5-large': 512, - 't5-3B': 512, - 't5-11B': 512, + 't5-3b': 512, + 't5-11b': 512, } class T5Tokenizer(PreTrainedTokenizer): From c8ed1c82c8a42ef700d4129d227fa356385c1d60 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Fri, 13 Dec 2019 12:13:48 -0500 Subject: [PATCH 355/505] [SQUAD] Load checkpoint when evaluating without training --- examples/run_squad.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/examples/run_squad.py b/examples/run_squad.py index 117b86e32c..a39915ee8b 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -580,10 +580,16 @@ def main(): # Evaluation - we can ask to evaluate all the checkpoints (sub-directories) in a directory results = {} if args.do_eval and args.local_rank in [-1, 0]: - checkpoints = [args.output_dir] - if args.eval_all_checkpoints: - checkpoints = list(os.path.dirname(c) for c in sorted(glob.glob(args.output_dir + '/**/' + WEIGHTS_NAME, recursive=True))) - logging.getLogger("transformers.modeling_utils").setLevel(logging.WARN) # Reduce model loading logs + + if args.do_train: + logger.info("Loading checkpoints saved during training for evaluation") + checkpoints = [args.output_dir] + if args.eval_all_checkpoints: + checkpoints = list(os.path.dirname(c) for c in sorted(glob.glob(args.output_dir + '/**/' + WEIGHTS_NAME, recursive=True))) + logging.getLogger("transformers.modeling_utils").setLevel(logging.WARN) # Reduce model loading logs + else: + logger.info("Loading checkpoint %s for evaluation", args.model_name_or_path) + checkpoints = [args.model_name_or_path] logger.info("Evaluate the following checkpoints: %s", checkpoints) From f24a228a9315a4b723509bc9144b53d2bcbc4217 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Fri, 13 Dec 2019 14:50:35 -0500 Subject: [PATCH 356/505] Speed up tokenization process --- transformers/data/processors/squad.py | 2 +- transformers/tokenization_utils.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/transformers/data/processors/squad.py b/transformers/data/processors/squad.py index 9bc4375684..e193f6153e 100644 --- a/transformers/data/processors/squad.py +++ b/transformers/data/processors/squad.py @@ -116,7 +116,7 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, unique_id = 1000000000 features = [] - for (example_index, example) in enumerate(tqdm(examples)): + for (example_index, example) in enumerate(tqdm(examples, desc="Converting examples to features")): if is_training and not example.is_impossible: # Get start and end position start_position = example.start_position diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index 317ecd167b..e87c87787b 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -637,9 +637,11 @@ class PreTrainedTokenizer(object): text: The sequence to be encoded. **kwargs: passed to the child `self.tokenize()` method """ + all_special_tokens = self.all_special_tokens + def lowercase_text(t): # convert non-special tokens to lowercase - escaped_special_toks = [re.escape(s_tok) for s_tok in self.all_special_tokens] + escaped_special_toks = [re.escape(s_tok) for s_tok in all_special_tokens] pattern = r'(^' + r'|'.join(escaped_special_toks) + r')|' + \ r'(.+?)' return re.sub( @@ -680,17 +682,17 @@ class PreTrainedTokenizer(object): tokenized_text = [] for sub_text in text_list: if sub_text not in self.added_tokens_encoder \ - and sub_text not in self.all_special_tokens: + and sub_text not in all_special_tokens: tokenized_text += split_on_token(tok, sub_text) else: tokenized_text += [sub_text] text_list = tokenized_text return list(itertools.chain.from_iterable((self._tokenize(token, **kwargs) if token not \ - in self.added_tokens_encoder and token not in self.all_special_tokens \ + in self.added_tokens_encoder and token not in all_special_tokens \ else [token] for token in tokenized_text))) - added_tokens = list(self.added_tokens_encoder.keys()) + self.all_special_tokens + added_tokens = list(self.added_tokens_encoder.keys()) + all_special_tokens tokenized_text = split_on_tokens(added_tokens, text) return tokenized_text From d46147294852694d1dc701c72b9053ff2e726265 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Fri, 13 Dec 2019 15:31:52 -0500 Subject: [PATCH 357/505] return for SQuAD [BLACKED] --- transformers/data/processors/glue.py | 2 +- transformers/data/processors/squad.py | 280 ++++++++++++++++---------- 2 files changed, 172 insertions(+), 110 deletions(-) diff --git a/transformers/data/processors/glue.py b/transformers/data/processors/glue.py index 518251b050..11ebd949de 100644 --- a/transformers/data/processors/glue.py +++ b/transformers/data/processors/glue.py @@ -133,7 +133,7 @@ def glue_convert_examples_to_features(examples, tokenizer, if is_tf_available() and is_tf_dataset: def gen(): for ex in features: - yield ({'input_ids': ex.input_ids, + yield ({'input_ids': ex.input_ids, 'attention_mask': ex.attention_mask, 'token_type_ids': ex.token_type_ids}, ex.label) diff --git a/transformers/data/processors/squad.py b/transformers/data/processors/squad.py index e193f6153e..84aa429e26 100644 --- a/transformers/data/processors/squad.py +++ b/transformers/data/processors/squad.py @@ -18,19 +18,20 @@ if is_tf_available(): logger = logging.getLogger(__name__) -def _improve_answer_span(doc_tokens, input_start, input_end, tokenizer, - orig_answer_text): + +def _improve_answer_span(doc_tokens, input_start, input_end, tokenizer, orig_answer_text): """Returns tokenized answer spans that better match the annotated answer.""" tok_answer_text = " ".join(tokenizer.tokenize(orig_answer_text)) for new_start in range(input_start, input_end + 1): for new_end in range(input_end, new_start - 1, -1): - text_span = " ".join(doc_tokens[new_start:(new_end + 1)]) + text_span = " ".join(doc_tokens[new_start : (new_end + 1)]) if text_span == tok_answer_text: return (new_start, new_end) return (input_start, input_end) + def _check_is_max_context(doc_spans, cur_span_index, position): """Check if this is the 'max context' doc span for the token.""" best_score = None @@ -50,10 +51,11 @@ def _check_is_max_context(doc_spans, cur_span_index, position): return cur_span_index == best_span_index + def _new_check_is_max_context(doc_spans, cur_span_index, position): """Check if this is the 'max context' doc span for the token.""" # if len(doc_spans) == 1: - # return True + # return True best_score = None best_span_index = None for (span_index, doc_span) in enumerate(doc_spans): @@ -71,14 +73,16 @@ def _new_check_is_max_context(doc_spans, cur_span_index, position): return cur_span_index == best_span_index + def _is_whitespace(c): if c == " " or c == "\t" or c == "\r" or c == "\n" or ord(c) == 0x202F: return True return False -def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, - doc_stride, max_query_length, is_training, - return_dataset=False): + +def squad_convert_examples_to_features( + examples, tokenizer, max_seq_length, doc_stride, max_query_length, is_training, return_dataset=False +): """ Converts a list of examples into a list of features that can be directly given as input to a model. It is model-dependant and takes advantage of many of the tokenizer's features to create the model's inputs. @@ -112,7 +116,7 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, ) """ - # Defining helper methods + # Defining helper methods unique_id = 1000000000 features = [] @@ -123,13 +127,12 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, end_position = example.end_position # If the answer cannot be found in the text, then skip this example. - actual_text = " ".join(example.doc_tokens[start_position:(end_position + 1)]) + actual_text = " ".join(example.doc_tokens[start_position : (end_position + 1)]) cleaned_answer_text = " ".join(whitespace_tokenize(example.answer_text)) if actual_text.find(cleaned_answer_text) == -1: logger.warning("Could not find answer: '%s' vs. '%s'", actual_text, cleaned_answer_text) continue - tok_to_orig_index = [] orig_to_tok_index = [] all_doc_tokens = [] @@ -140,7 +143,6 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, tok_to_orig_index.append(i) all_doc_tokens.append(sub_token) - if is_training and not example.is_impossible: tok_start_position = orig_to_tok_index[example.start_position] if example.end_position < len(example.doc_tokens) - 1: @@ -153,36 +155,41 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, ) spans = [] - - truncated_query = tokenizer.encode(example.question_text, add_special_tokens=False, max_length=max_query_length) - sequence_added_tokens = tokenizer.max_len - tokenizer.max_len_single_sentence - sequence_pair_added_tokens = tokenizer.max_len - tokenizer.max_len_sentences_pair + + truncated_query = tokenizer.encode( + example.question_text, add_special_tokens=False, max_length=max_query_length + ) + sequence_added_tokens = tokenizer.max_len - tokenizer.max_len_single_sentence + sequence_pair_added_tokens = tokenizer.max_len - tokenizer.max_len_sentences_pair span_doc_tokens = all_doc_tokens while len(spans) * doc_stride < len(all_doc_tokens): - + encoded_dict = tokenizer.encode_plus( - truncated_query if tokenizer.padding_side == "right" else span_doc_tokens, - span_doc_tokens if tokenizer.padding_side == "right" else truncated_query, - max_length=max_seq_length, - return_overflowing_tokens=True, + truncated_query if tokenizer.padding_side == "right" else span_doc_tokens, + span_doc_tokens if tokenizer.padding_side == "right" else truncated_query, + max_length=max_seq_length, + return_overflowing_tokens=True, pad_to_max_length=True, stride=max_seq_length - doc_stride - len(truncated_query) - sequence_pair_added_tokens, - truncation_strategy='only_second' if tokenizer.padding_side == "right" else 'only_first' + truncation_strategy="only_second" if tokenizer.padding_side == "right" else "only_first", ) - paragraph_len = min(len(all_doc_tokens) - len(spans) * doc_stride, max_seq_length - len(truncated_query) - sequence_pair_added_tokens) + paragraph_len = min( + len(all_doc_tokens) - len(spans) * doc_stride, + max_seq_length - len(truncated_query) - sequence_pair_added_tokens, + ) - if tokenizer.pad_token_id in encoded_dict['input_ids']: - non_padded_ids = encoded_dict['input_ids'][:encoded_dict['input_ids'].index(tokenizer.pad_token_id)] + if tokenizer.pad_token_id in encoded_dict["input_ids"]: + non_padded_ids = encoded_dict["input_ids"][: encoded_dict["input_ids"].index(tokenizer.pad_token_id)] else: - non_padded_ids = encoded_dict['input_ids'] + non_padded_ids = encoded_dict["input_ids"] tokens = tokenizer.convert_ids_to_tokens(non_padded_ids) token_to_orig_map = {} for i in range(paragraph_len): - index = len(truncated_query) + sequence_added_tokens + i if tokenizer.padding_side == "right" else i + index = len(truncated_query) + sequence_added_tokens + i if tokenizer.padding_side == "right" else i token_to_orig_map[index] = tok_to_orig_index[len(spans) * doc_stride + i] encoded_dict["paragraph_len"] = paragraph_len @@ -202,16 +209,20 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, for doc_span_index in range(len(spans)): for j in range(spans[doc_span_index]["paragraph_len"]): is_max_context = _new_check_is_max_context(spans, doc_span_index, doc_span_index * doc_stride + j) - index = j if tokenizer.padding_side == "left" else spans[doc_span_index]["truncated_query_with_special_tokens_length"] + j + index = ( + j + if tokenizer.padding_side == "left" + else spans[doc_span_index]["truncated_query_with_special_tokens_length"] + j + ) spans[doc_span_index]["token_is_max_context"][index] = is_max_context for span in spans: # Identify the position of the CLS token - cls_index = span['input_ids'].index(tokenizer.cls_token_id) + cls_index = span["input_ids"].index(tokenizer.cls_token_id) # p_mask: mask with 1 for token than cannot be in the answer (0 for token which can be in an answer) # Original TF implem also keep the classification token (set to 0) (not sure why...) - p_mask = np.array(span['token_type_ids']) + p_mask = np.array(span["token_type_ids"]) p_mask = np.minimum(p_mask, 1) @@ -224,7 +235,6 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, # Set the CLS index to '0' p_mask[cls_index] = 0 - span_is_impossible = example.is_impossible start_position = 0 end_position = 0 @@ -247,55 +257,99 @@ def squad_convert_examples_to_features(examples, tokenizer, max_seq_length, doc_offset = 0 else: doc_offset = len(truncated_query) + sequence_added_tokens - + start_position = tok_start_position - doc_start + doc_offset end_position = tok_end_position - doc_start + doc_offset - - features.append(SquadFeatures( - span['input_ids'], - span['attention_mask'], - span['token_type_ids'], - cls_index, - p_mask.tolist(), - - example_index=example_index, - unique_id=unique_id, - paragraph_len=span['paragraph_len'], - token_is_max_context=span["token_is_max_context"], - tokens=span["tokens"], - token_to_orig_map=span["token_to_orig_map"], - - start_position=start_position, - end_position=end_position - )) + features.append( + SquadFeatures( + span["input_ids"], + span["attention_mask"], + span["token_type_ids"], + cls_index, + p_mask.tolist(), + example_index=example_index, + unique_id=unique_id, + paragraph_len=span["paragraph_len"], + token_is_max_context=span["token_is_max_context"], + tokens=span["tokens"], + token_to_orig_map=span["token_to_orig_map"], + start_position=start_position, + end_position=end_position, + ) + ) unique_id += 1 - if return_dataset == 'pt': + if return_dataset == "pt": if not is_torch_available(): raise ImportError("Pytorch must be installed to return a pytorch dataset.") # Convert to Tensors and build dataset all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long) - all_input_mask = torch.tensor([f.attention_mask for f in features], dtype=torch.long) - all_segment_ids = torch.tensor([f.token_type_ids for f in features], dtype=torch.long) + all_attention_masks = torch.tensor([f.attention_mask for f in features], dtype=torch.long) + all_token_type_ids = torch.tensor([f.token_type_ids for f in features], dtype=torch.long) all_cls_index = torch.tensor([f.cls_index for f in features], dtype=torch.long) all_p_mask = torch.tensor([f.p_mask for f in features], dtype=torch.float) if not is_training: all_example_index = torch.arange(all_input_ids.size(0), dtype=torch.long) - dataset = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, - all_example_index, all_cls_index, all_p_mask) + dataset = TensorDataset( + all_input_ids, all_attention_masks, all_token_type_ids, all_example_index, all_cls_index, all_p_mask + ) else: all_start_positions = torch.tensor([f.start_position for f in features], dtype=torch.long) all_end_positions = torch.tensor([f.end_position for f in features], dtype=torch.long) - dataset = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, - all_start_positions, all_end_positions, - all_cls_index, all_p_mask) + dataset = TensorDataset( + all_input_ids, + all_attention_masks, + all_token_type_ids, + all_start_positions, + all_end_positions, + all_cls_index, + all_p_mask, + ) return features, dataset - + elif return_dataset == "tf": + if not is_tf_available(): + raise ImportError("TensorFlow must be installed to return a TensorFlow dataset.") + + def gen(): + for ex in features: + yield ( + { + "input_ids": ex.input_ids, + "attention_mask": ex.attention_mask, + "token_type_ids": ex.token_type_ids, + }, { + "start_position": ex.start_position, + "end_position": ex.end_position, + "cls_index": ex.cls_index, + "p_mask": ex.p_mask, + } + ) + + return tf.data.Dataset.from_generator( + gen, + ( + {"input_ids": tf.int32, "attention_mask": tf.int32, "token_type_ids": tf.int32}, + {"start_position": tf.int64, "end_position": tf.int64, "cls_index": tf.int64, "p_mask": tf.int32}, + ), + ( + { + "input_ids": tf.TensorShape([None]), + "attention_mask": tf.TensorShape([None]), + "token_type_ids": tf.TensorShape([None]), + }, + { + "start_position": tf.TensorShape([]), + "end_position": tf.TensorShape([]), + "cls_index": tf.TensorShape([]), + "p_mask": tf.TensorShape([None]), + }, + ), + ) return features @@ -305,31 +359,32 @@ class SquadProcessor(DataProcessor): Processor for the SQuAD data set. Overriden by SquadV1Processor and SquadV2Processor, used by the version 1.1 and version 2.0 of SQuAD, respectively. """ + train_file = None dev_file = None def _get_example_from_tensor_dict(self, tensor_dict, evaluate=False): if not evaluate: - answer = tensor_dict['answers']['text'][0].numpy().decode('utf-8') - answer_start = tensor_dict['answers']['answer_start'][0].numpy() + answer = tensor_dict["answers"]["text"][0].numpy().decode("utf-8") + answer_start = tensor_dict["answers"]["answer_start"][0].numpy() answers = [] else: - answers = [{ - "answer_start": start.numpy(), - "text": text.numpy().decode('utf-8') - } for start, text in zip(tensor_dict['answers']["answer_start"], tensor_dict['answers']["text"])] + answers = [ + {"answer_start": start.numpy(), "text": text.numpy().decode("utf-8")} + for start, text in zip(tensor_dict["answers"]["answer_start"], tensor_dict["answers"]["text"]) + ] answer = None answer_start = None return SquadExample( - qas_id=tensor_dict['id'].numpy().decode("utf-8"), - question_text=tensor_dict['question'].numpy().decode('utf-8'), - context_text=tensor_dict['context'].numpy().decode('utf-8'), + qas_id=tensor_dict["id"].numpy().decode("utf-8"), + question_text=tensor_dict["question"].numpy().decode("utf-8"), + context_text=tensor_dict["context"].numpy().decode("utf-8"), answer_text=answer, start_position_character=answer_start, - title=tensor_dict['title'].numpy().decode('utf-8'), - answers=answers + title=tensor_dict["title"].numpy().decode("utf-8"), + answers=answers, ) def get_examples_from_dataset(self, dataset, evaluate=False): @@ -359,7 +414,7 @@ class SquadProcessor(DataProcessor): examples = [] for tensor_dict in tqdm(dataset): - examples.append(self._get_example_from_tensor_dict(tensor_dict, evaluate=evaluate)) + examples.append(self._get_example_from_tensor_dict(tensor_dict, evaluate=evaluate)) return examples @@ -379,7 +434,9 @@ class SquadProcessor(DataProcessor): if self.train_file is None: raise ValueError("SquadProcessor should be instantiated via SquadV1Processor or SquadV2Processor") - with open(os.path.join(data_dir, self.train_file if filename is None else filename), "r", encoding='utf-8') as reader: + with open( + os.path.join(data_dir, self.train_file if filename is None else filename), "r", encoding="utf-8" + ) as reader: input_data = json.load(reader)["data"] return self._create_examples(input_data, "train") @@ -397,8 +454,10 @@ class SquadProcessor(DataProcessor): if self.dev_file is None: raise ValueError("SquadProcessor should be instantiated via SquadV1Processor or SquadV2Processor") - - with open(os.path.join(data_dir, self.dev_file if filename is None else filename), "r", encoding='utf-8') as reader: + + with open( + os.path.join(data_dir, self.dev_file if filename is None else filename), "r", encoding="utf-8" + ) as reader: input_data = json.load(reader)["data"] return self._create_examples(input_data, "dev") @@ -406,7 +465,7 @@ class SquadProcessor(DataProcessor): is_training = set_type == "train" examples = [] for entry in tqdm(input_data): - title = entry['title'] + title = entry["title"] for paragraph in entry["paragraphs"]: context_text = paragraph["context"] for qa in paragraph["qas"]: @@ -415,7 +474,7 @@ class SquadProcessor(DataProcessor): start_position_character = None answer_text = None answers = [] - + if "is_impossible" in qa: is_impossible = qa["is_impossible"] else: @@ -424,8 +483,8 @@ class SquadProcessor(DataProcessor): if not is_impossible: if is_training: answer = qa["answers"][0] - answer_text = answer['text'] - start_position_character = answer['answer_start'] + answer_text = answer["text"] + start_position_character = answer["answer_start"] else: answers = qa["answers"] @@ -437,12 +496,13 @@ class SquadProcessor(DataProcessor): start_position_character=start_position_character, title=title, is_impossible=is_impossible, - answers=answers + answers=answers, ) examples.append(example) return examples + class SquadV1Processor(SquadProcessor): train_file = "train-v1.1.json" dev_file = "dev-v1.1.json" @@ -451,7 +511,7 @@ class SquadV1Processor(SquadProcessor): class SquadV2Processor(SquadProcessor): train_file = "train-v2.0.json" dev_file = "dev-v2.0.json" - + class SquadExample(object): """ @@ -468,21 +528,23 @@ class SquadExample(object): is_impossible: False by default, set to True if the example has no possible answer. """ - def __init__(self, - qas_id, - question_text, - context_text, - answer_text, - start_position_character, - title, - answers=[], - is_impossible=False): + def __init__( + self, + qas_id, + question_text, + context_text, + answer_text, + start_position_character, + title, + answers=[], + is_impossible=False, + ): self.qas_id = qas_id self.question_text = question_text self.context_text = context_text self.answer_text = answer_text self.title = title - self.is_impossible = is_impossible + self.is_impossible = is_impossible self.answers = answers self.start_position, self.end_position = 0, 0 @@ -537,24 +599,23 @@ class SquadFeatures(object): end_position: end of the answer token index """ - def __init__(self, - input_ids, - attention_mask, - token_type_ids, - cls_index, - p_mask, - - example_index, - unique_id, - paragraph_len, - token_is_max_context, - tokens, - token_to_orig_map, - - start_position, - end_position - ): - self.input_ids = input_ids + def __init__( + self, + input_ids, + attention_mask, + token_type_ids, + cls_index, + p_mask, + example_index, + unique_id, + paragraph_len, + token_is_max_context, + tokens, + token_to_orig_map, + start_position, + end_position, + ): + self.input_ids = input_ids self.attention_mask = attention_mask self.token_type_ids = token_type_ids self.cls_index = cls_index @@ -580,12 +641,13 @@ class SquadResult(object): start_logits: The logits corresponding to the start of the answer end_logits: The logits corresponding to the end of the answer """ + def __init__(self, unique_id, start_logits, end_logits, start_top_index=None, end_top_index=None, cls_logits=None): self.start_logits = start_logits self.end_logits = end_logits self.unique_id = unique_id - + if start_top_index: self.start_top_index = start_top_index self.end_top_index = end_top_index - self.cls_logits = cls_logits \ No newline at end of file + self.cls_logits = cls_logits From 866d73ca26a13d7e378b2f88f365cb0807c47805 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Fri, 13 Dec 2019 16:09:23 -0500 Subject: [PATCH 358/505] [cli] Upload is now compatible with folders --- transformers/commands/user.py | 57 ++++++++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 14 deletions(-) diff --git a/transformers/commands/user.py b/transformers/commands/user.py index d79922ed8a..8e0e563422 100644 --- a/transformers/commands/user.py +++ b/transformers/commands/user.py @@ -19,8 +19,8 @@ class UserCommands(BaseTransformersCLICommand): list_parser.set_defaults(func=lambda args: ListObjsCommand(args)) # upload upload_parser = parser.add_parser('upload') - upload_parser.add_argument('file', type=str, help='Local filepath of the file to upload.') - upload_parser.add_argument('--filename', type=str, default=None, help='Optional: override object filename on S3.') + upload_parser.add_argument('path', type=str, help='Local path of the folder or individual file to upload.') + upload_parser.add_argument('--filename', type=str, default=None, help='Optional: override individual object filename on S3.') upload_parser.set_defaults(func=lambda args: UploadCommand(args)) @@ -138,28 +138,57 @@ class ListObjsCommand(BaseUserCommand): class UploadCommand(BaseUserCommand): + def walk_dir(self, rel_path): + """ + Recursively list all files in a folder. + """ + entries: List[os.DirEntry] = list(os.scandir(rel_path)) + files = [ + ( + os.path.join(os.getcwd(), f.path), # filepath + f.path # filename + ) + for f in entries if f.is_file() + ] + for f in entries: + if f.is_dir(): + files += self.walk_dir(f.path) + return files + def run(self): token = HfFolder.get_token() if token is None: print("Not logged in") exit(1) - filepath = os.path.join(os.getcwd(), self.args.file) - filename = self.args.filename if self.args.filename is not None else os.path.basename(filepath) - print( - "About to upload file {} to S3 under filename {}".format( - ANSI.bold(filepath), ANSI.bold(filename) + local_path = os.path.abspath(self.args.path) + if os.path.isdir(local_path): + if self.args.filename is not None: + raise ValueError("Cannot specify a filename override when uploading a folder.") + rel_path = os.path.basename(local_path) + files = self.walk_dir(rel_path) + elif os.path.isfile(local_path): + filename = self.args.filename if self.args.filename is not None else os.path.basename(local_path) + files = [(local_path, filename)] + else: + raise ValueError("Not a valid file or directory: {}".format(local_path)) + + for filepath, filename in files: + print( + "About to upload file {} to S3 under filename {}".format( + ANSI.bold(filepath), ANSI.bold(filename) + ) ) - ) choice = input("Proceed? [Y/n] ").lower() if not(choice == "" or choice == "y" or choice == "yes"): print("Abort") exit() print( - ANSI.bold("Uploading... This might take a while if file is large") + ANSI.bold("Uploading... This might take a while if files are large") ) - access_url = self._api.presign_and_upload( - token=token, filename=filename, filepath=filepath - ) - print("Your file now lives at:") - print(access_url) + for filepath, filename in files: + access_url = self._api.presign_and_upload( + token=token, filename=filename, filepath=filepath + ) + print("Your file now lives at:") + print(access_url) From 5b7b78e088352a3aaf1f80d26bb1cd466bc2ac64 Mon Sep 17 00:00:00 2001 From: Pascal Voitot Date: Sun, 8 Dec 2019 23:22:02 +0100 Subject: [PATCH 359/505] :bug: #2096 in tokenizer.decode, adds a space after special tokens to return right formatted string --- transformers/tokenization_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index e87c87787b..42519c26ba 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -1180,7 +1180,7 @@ class PreTrainedTokenizer(object): if current_sub_text: sub_texts.append(self.convert_tokens_to_string(current_sub_text)) current_sub_text = [] - sub_texts.append(" " + token) + sub_texts.append(" " + token + " ") else: current_sub_text.append(token) if current_sub_text: From df160af736cba1d50c09abcf92c8fc6c00bcdb13 Mon Sep 17 00:00:00 2001 From: Pascal Voitot Date: Tue, 10 Dec 2019 00:03:38 +0100 Subject: [PATCH 360/505] :bug: #2096 in tokenizer.decode, space is not joined between all subtexts instead of before added tokens --- transformers/tests/tokenization_bert_test.py | 16 ++++++++++++++++ transformers/tokenization_utils.py | 4 ++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/transformers/tests/tokenization_bert_test.py b/transformers/tests/tokenization_bert_test.py index f390248956..c47f149e9a 100644 --- a/transformers/tests/tokenization_bert_test.py +++ b/transformers/tests/tokenization_bert_test.py @@ -99,6 +99,21 @@ class BertTokenizationTest(CommonTestCases.CommonTokenizerTester): self.assertListEqual( tokenizer.tokenize("unwantedX running"), ["[UNK]", "runn", "##ing"]) + def test_encode_decode_with_spaces(self): + tokenizer = self.get_tokenizer() + + new_toks = ['[ABC]', '[DEF]', 'GHI IHG'] + tokenizer.add_tokens(new_toks) + input = "unwanted running [ABC] [DEF] running unwanted [ABC] GHI IHG unwanted [DEF]" + encoded = tokenizer.encode(input) + decoded = tokenizer.decode(encoded) + self.assertEqual( + decoded.lower(), + (f"[CLS] {input.lower()} [SEP]").lower() + ) + + + def test_is_whitespace(self): self.assertTrue(_is_whitespace(u" ")) self.assertTrue(_is_whitespace(u"\t")) @@ -139,5 +154,6 @@ class BertTokenizationTest(CommonTestCases.CommonTokenizerTester): assert encoded_sentence == [101] + text + [102] assert encoded_pair == [101] + text + [102] + text_2 + [102] + if __name__ == '__main__': unittest.main() diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index 42519c26ba..8aef80fec8 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -1180,12 +1180,12 @@ class PreTrainedTokenizer(object): if current_sub_text: sub_texts.append(self.convert_tokens_to_string(current_sub_text)) current_sub_text = [] - sub_texts.append(" " + token + " ") + sub_texts.append(token) else: current_sub_text.append(token) if current_sub_text: sub_texts.append(self.convert_tokens_to_string(current_sub_text)) - text = ''.join(sub_texts) + text = ' '.join(sub_texts) if clean_up_tokenization_spaces: clean_text = self.clean_up_tokenization(text) From dd2add9f6efdaa248f3074b865dc67c439b30a4d Mon Sep 17 00:00:00 2001 From: Pascal Voitot Date: Tue, 10 Dec 2019 00:29:44 +0100 Subject: [PATCH 361/505] more tests --- transformers/tests/tokenization_bert_test.py | 2 +- transformers/tests/tokenization_gpt2_test.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/transformers/tests/tokenization_bert_test.py b/transformers/tests/tokenization_bert_test.py index c47f149e9a..b93934dd67 100644 --- a/transformers/tests/tokenization_bert_test.py +++ b/transformers/tests/tokenization_bert_test.py @@ -109,7 +109,7 @@ class BertTokenizationTest(CommonTestCases.CommonTokenizerTester): decoded = tokenizer.decode(encoded) self.assertEqual( decoded.lower(), - (f"[CLS] {input.lower()} [SEP]").lower() + (f"[CLS] {input} [SEP]").lower() ) diff --git a/transformers/tests/tokenization_gpt2_test.py b/transformers/tests/tokenization_gpt2_test.py index a77cc75ec2..9e6ca3c4fd 100644 --- a/transformers/tests/tokenization_gpt2_test.py +++ b/transformers/tests/tokenization_gpt2_test.py @@ -67,6 +67,20 @@ class GPT2TokenizationTest(CommonTestCases.CommonTokenizerTester): self.assertListEqual( tokenizer.convert_tokens_to_ids(input_tokens), input_bpe_tokens) + def test_encode_decode_with_spaces(self): + tokenizer = self.get_tokenizer() + + new_toks = ['[ABC]', '[DEF]', 'GHI IHG'] + tokenizer.add_tokens(new_toks) + input = "lower newer [ABC] [DEF] newer lower [ABC] GHI IHG newer lower[DEF]" + encoded = tokenizer.encode(input) + decoded = tokenizer.decode(encoded) + self.assertEqual( + decoded.lower(), + input.lower() + ) + + if __name__ == '__main__': unittest.main() From 4cbdc7d910a0a12871a8e29760a3a6721a138421 Mon Sep 17 00:00:00 2001 From: Pascal Voitot Date: Tue, 10 Dec 2019 09:37:15 +0100 Subject: [PATCH 362/505] missed space --- transformers/tests/tokenization_bert_test.py | 2 -- transformers/tests/tokenization_gpt2_test.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/transformers/tests/tokenization_bert_test.py b/transformers/tests/tokenization_bert_test.py index b93934dd67..a039a24dd8 100644 --- a/transformers/tests/tokenization_bert_test.py +++ b/transformers/tests/tokenization_bert_test.py @@ -112,8 +112,6 @@ class BertTokenizationTest(CommonTestCases.CommonTokenizerTester): (f"[CLS] {input} [SEP]").lower() ) - - def test_is_whitespace(self): self.assertTrue(_is_whitespace(u" ")) self.assertTrue(_is_whitespace(u"\t")) diff --git a/transformers/tests/tokenization_gpt2_test.py b/transformers/tests/tokenization_gpt2_test.py index 9e6ca3c4fd..1b4fe42874 100644 --- a/transformers/tests/tokenization_gpt2_test.py +++ b/transformers/tests/tokenization_gpt2_test.py @@ -72,7 +72,7 @@ class GPT2TokenizationTest(CommonTestCases.CommonTokenizerTester): new_toks = ['[ABC]', '[DEF]', 'GHI IHG'] tokenizer.add_tokens(new_toks) - input = "lower newer [ABC] [DEF] newer lower [ABC] GHI IHG newer lower[DEF]" + input = "lower newer [ABC] [DEF] newer lower [ABC] GHI IHG newer lower [DEF]" encoded = tokenizer.encode(input) decoded = tokenizer.decode(encoded) self.assertEqual( From f2ac50cb5560e13d941f1ea3dec3399f12f7a3fb Mon Sep 17 00:00:00 2001 From: Pascal Voitot Date: Tue, 10 Dec 2019 09:58:06 +0100 Subject: [PATCH 363/505] better for python2.x --- transformers/tests/tokenization_bert_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/tests/tokenization_bert_test.py b/transformers/tests/tokenization_bert_test.py index a039a24dd8..77b124cdf2 100644 --- a/transformers/tests/tokenization_bert_test.py +++ b/transformers/tests/tokenization_bert_test.py @@ -109,7 +109,7 @@ class BertTokenizationTest(CommonTestCases.CommonTokenizerTester): decoded = tokenizer.decode(encoded) self.assertEqual( decoded.lower(), - (f"[CLS] {input} [SEP]").lower() + ("[CLS] " + input + " [SEP]").lower() ) def test_is_whitespace(self): From c3248cf122014dce10c0c8d0e663a95c948493e3 Mon Sep 17 00:00:00 2001 From: LysandreJik Date: Wed, 11 Dec 2019 12:36:37 -0500 Subject: [PATCH 364/505] Tests for all tokenizers --- transformers/tests/tokenization_bert_test.py | 13 ------------- transformers/tests/tokenization_gpt2_test.py | 15 --------------- transformers/tests/tokenization_tests_commons.py | 9 +++++++++ 3 files changed, 9 insertions(+), 28 deletions(-) diff --git a/transformers/tests/tokenization_bert_test.py b/transformers/tests/tokenization_bert_test.py index 77b124cdf2..c503ea5e1e 100644 --- a/transformers/tests/tokenization_bert_test.py +++ b/transformers/tests/tokenization_bert_test.py @@ -99,19 +99,6 @@ class BertTokenizationTest(CommonTestCases.CommonTokenizerTester): self.assertListEqual( tokenizer.tokenize("unwantedX running"), ["[UNK]", "runn", "##ing"]) - def test_encode_decode_with_spaces(self): - tokenizer = self.get_tokenizer() - - new_toks = ['[ABC]', '[DEF]', 'GHI IHG'] - tokenizer.add_tokens(new_toks) - input = "unwanted running [ABC] [DEF] running unwanted [ABC] GHI IHG unwanted [DEF]" - encoded = tokenizer.encode(input) - decoded = tokenizer.decode(encoded) - self.assertEqual( - decoded.lower(), - ("[CLS] " + input + " [SEP]").lower() - ) - def test_is_whitespace(self): self.assertTrue(_is_whitespace(u" ")) self.assertTrue(_is_whitespace(u"\t")) diff --git a/transformers/tests/tokenization_gpt2_test.py b/transformers/tests/tokenization_gpt2_test.py index 1b4fe42874..5eae767bdf 100644 --- a/transformers/tests/tokenization_gpt2_test.py +++ b/transformers/tests/tokenization_gpt2_test.py @@ -67,20 +67,5 @@ class GPT2TokenizationTest(CommonTestCases.CommonTokenizerTester): self.assertListEqual( tokenizer.convert_tokens_to_ids(input_tokens), input_bpe_tokens) - def test_encode_decode_with_spaces(self): - tokenizer = self.get_tokenizer() - - new_toks = ['[ABC]', '[DEF]', 'GHI IHG'] - tokenizer.add_tokens(new_toks) - input = "lower newer [ABC] [DEF] newer lower [ABC] GHI IHG newer lower [DEF]" - encoded = tokenizer.encode(input) - decoded = tokenizer.decode(encoded) - self.assertEqual( - decoded.lower(), - input.lower() - ) - - - if __name__ == '__main__': unittest.main() diff --git a/transformers/tests/tokenization_tests_commons.py b/transformers/tests/tokenization_tests_commons.py index c009958135..13e7ae746a 100644 --- a/transformers/tests/tokenization_tests_commons.py +++ b/transformers/tests/tokenization_tests_commons.py @@ -232,6 +232,15 @@ class CommonTestCases: self.assertNotEqual(len(tokens_2), 0) self.assertIsInstance(text_2, (str, unicode)) + def test_encode_decode_with_spaces(self): + tokenizer = self.get_tokenizer() + + new_toks = ['[ABC]', '[DEF]', 'GHI IHG'] + tokenizer.add_tokens(new_toks) + input = "[ABC] [DEF] [ABC] GHI IHG [DEF]" + encoded = tokenizer.encode(input, add_special_tokens=False) + decoded = tokenizer.decode(encoded) + self.assertEqual(decoded, input) def test_pretrained_model_lists(self): weights_list = list(self.tokenizer_class.max_model_input_sizes.keys()) From 7bd11dda6f43656cf0a3891b7f61a67196d233b4 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Fri, 13 Dec 2019 16:45:30 -0500 Subject: [PATCH 365/505] Release: v2.2.2 --- README.md | 2 +- docs/source/conf.py | 2 +- setup.py | 2 +- transformers/__init__.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index f3aa8a95ee..f24ceaa6d2 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ Choose the right framework for every part of a model's lifetime | [Quick tour: Fine-tuning/usage scripts](#quick-tour-of-the-fine-tuningusage-scripts) | Using provided scripts: GLUE, SQuAD and Text generation | | [Migrating from pytorch-transformers to transformers](#Migrating-from-pytorch-transformers-to-transformers) | Migrating your code from pytorch-transformers to transformers | | [Migrating from pytorch-pretrained-bert to pytorch-transformers](#Migrating-from-pytorch-pretrained-bert-to-transformers) | Migrating your code from pytorch-pretrained-bert to transformers | -| [Documentation][(v2.2.0/v2.2.1)](https://huggingface.co/transformers/v2.2.0) [(v2.1.1)](https://huggingface.co/transformers/v2.1.1) [(v2.0.0)](https://huggingface.co/transformers/v2.0.0) [(v1.2.0)](https://huggingface.co/transformers/v1.2.0) [(v1.1.0)](https://huggingface.co/transformers/v1.1.0) [(v1.0.0)](https://huggingface.co/transformers/v1.0.0) [(master)](https://huggingface.co/transformers) | Full API documentation and more | +| [Documentation][(v2.2.0/v2.2.1/v2.2.2)](https://huggingface.co/transformers/v2.2.0) [(v2.1.1)](https://huggingface.co/transformers/v2.1.1) [(v2.0.0)](https://huggingface.co/transformers/v2.0.0) [(v1.2.0)](https://huggingface.co/transformers/v1.2.0) [(v1.1.0)](https://huggingface.co/transformers/v1.1.0) [(v1.0.0)](https://huggingface.co/transformers/v1.0.0) [(master)](https://huggingface.co/transformers) | Full API documentation and more | ## Installation diff --git a/docs/source/conf.py b/docs/source/conf.py index 2f8505ab3a..99b7b44922 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -26,7 +26,7 @@ author = u'huggingface' # The short X.Y version version = u'' # The full version, including alpha/beta/rc tags -release = u'2.2.1' +release = u'2.2.2' # -- General configuration --------------------------------------------------- diff --git a/setup.py b/setup.py index c4af32df83..eacb5ecec0 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ extras['all'] = [package for package in extras.values()] setup( name="transformers", - version="2.2.1", + version="2.2.2", author="Thomas Wolf, Lysandre Debut, Victor Sanh, Julien Chaumond, Google AI Language Team Authors, Open AI team Authors, Facebook AI Authors, Carnegie Mellon University Authors", author_email="thomas@huggingface.co", description="State-of-the-art Natural Language Processing for TensorFlow 2.0 and PyTorch", diff --git a/transformers/__init__.py b/transformers/__init__.py index 5d7b0b772c..c11919f0a7 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -1,4 +1,4 @@ -__version__ = "2.2.1" +__version__ = "2.2.2" # Work around to update TensorFlow's absl.logging threshold which alters the # default Python logging output behavior when present. From b6d4284b26c0ab5e736cb7838b27b720225feeb7 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Fri, 13 Dec 2019 22:43:15 -0500 Subject: [PATCH 366/505] [cli] Uploads: fix + test edge case --- transformers/hf_api.py | 3 +- transformers/tests/fixtures/empty.txt | 0 transformers/tests/hf_api_test.py | 44 +++++++++++++++++++-------- 3 files changed, 33 insertions(+), 14 deletions(-) create mode 100644 transformers/tests/fixtures/empty.txt diff --git a/transformers/hf_api.py b/transformers/hf_api.py index 3bbb6c567a..170732339a 100644 --- a/transformers/hf_api.py +++ b/transformers/hf_api.py @@ -131,8 +131,9 @@ class HfApi: # the client still has to specify it when uploading the file. with open(filepath, "rb") as f: pf = TqdmProgressFileReader(f) + data = f if pf.total_size > 0 else "" - r = requests.put(urls.write, data=f, headers={ + r = requests.put(urls.write, data=data, headers={ "content-type": urls.type, }) r.raise_for_status() diff --git a/transformers/tests/fixtures/empty.txt b/transformers/tests/fixtures/empty.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/transformers/tests/hf_api_test.py b/transformers/tests/hf_api_test.py index 92d41b6dff..b45f5aceed 100644 --- a/transformers/tests/hf_api_test.py +++ b/transformers/tests/hf_api_test.py @@ -15,18 +15,30 @@ from __future__ import absolute_import, division, print_function import os -import six import time import unittest -from transformers.hf_api import HfApi, S3Obj, PresignedUrl, HfFolder, HTTPError +import requests +import six + +from transformers.hf_api import HfApi, HfFolder, HTTPError, PresignedUrl, S3Obj USER = "__DUMMY_TRANSFORMERS_USER__" PASS = "__DUMMY_TRANSFORMERS_PASS__" -FILE_KEY = "Test-{}.txt".format(int(time.time())) -FILE_PATH = os.path.join( - os.path.dirname(os.path.abspath(__file__)), "fixtures/input.txt" -) +FILES = [ + ( + "Test-{}.txt".format(int(time.time())), + os.path.join( + os.path.dirname(os.path.abspath(__file__)), "fixtures/input.txt" + ) + ), + ( + "yoyo {}.txt".format(int(time.time())), # space is intentional + os.path.join( + os.path.dirname(os.path.abspath(__file__)), "fixtures/empty.txt" + ) + ), +] @@ -57,15 +69,21 @@ class HfApiEndpointsTest(HfApiCommonTest): self.assertEqual(user, USER) def test_presign(self): - urls = self._api.presign(token=self._token, filename=FILE_KEY) - self.assertIsInstance(urls, PresignedUrl) - self.assertEqual(urls.type, "text/plain") + for FILE_KEY, FILE_PATH in FILES: + urls = self._api.presign(token=self._token, filename=FILE_KEY) + self.assertIsInstance(urls, PresignedUrl) + self.assertEqual(urls.type, "text/plain") def test_presign_and_upload(self): - access_url = self._api.presign_and_upload( - token=self._token, filename=FILE_KEY, filepath=FILE_PATH - ) - self.assertIsInstance(access_url, six.string_types) + for FILE_KEY, FILE_PATH in FILES: + access_url = self._api.presign_and_upload( + token=self._token, filename=FILE_KEY, filepath=FILE_PATH + ) + self.assertIsInstance(access_url, six.string_types) + with open(FILE_PATH, 'r') as f: + body = f.read() + r = requests.get(access_url) + self.assertEqual(r.text, body) def test_list_objs(self): objs = self._api.list_objs(token=self._token) From cbb368ca06998e5d98684bc622e1d8c68ba1d88f Mon Sep 17 00:00:00 2001 From: thomwolf Date: Sat, 14 Dec 2019 09:31:18 +0100 Subject: [PATCH 367/505] distilbert tests --- transformers/tests/modeling_common_test.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/transformers/tests/modeling_common_test.py b/transformers/tests/modeling_common_test.py index cd4cf247a6..8920e8b826 100644 --- a/transformers/tests/modeling_common_test.py +++ b/transformers/tests/modeling_common_test.py @@ -96,9 +96,7 @@ class CommonTestCases: # Make sure we don't have nans out_1 = after_outputs[0].cpu().numpy() - out_2 = outputs[0].cpu().numpy() - out_1 = out_1[~np.isnan(out_1)] - out_2 = out_2[~np.isnan(out_2)] + out_1[np.isnan(out_1)] = 0 max_diff = np.amax(np.abs(out_1 - out_2)) self.assertLessEqual(max_diff, 1e-5) From 7140363e092fecf82b73edd423bed3376ec1e150 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Sat, 14 Dec 2019 09:44:53 +0100 Subject: [PATCH 368/505] update bertabs --- .../summarization/configuration_bertabs.py | 48 ++++++------------- 1 file changed, 14 insertions(+), 34 deletions(-) diff --git a/examples/summarization/configuration_bertabs.py b/examples/summarization/configuration_bertabs.py index 054763ea93..b862d58d2b 100644 --- a/examples/summarization/configuration_bertabs.py +++ b/examples/summarization/configuration_bertabs.py @@ -33,6 +33,8 @@ class BertAbsConfig(PretrainedConfig): r""" Class to store the configuration of the BertAbs model. Arguments: + vocab_size: int + Number of tokens in the vocabulary. max_pos: int The maximum sequence length that this model will be used with. enc_layer: int @@ -81,39 +83,17 @@ class BertAbsConfig(PretrainedConfig): ): super(BertAbsConfig, self).__init__(**kwargs) - if self._input_is_path_to_json(vocab_size): - path_to_json = vocab_size - with open(path_to_json, "r", encoding="utf-8") as reader: - json_config = json.loads(reader.read()) - for key, value in json_config.items(): - self.__dict__[key] = value - elif isinstance(vocab_size, int): - self.vocab_size = vocab_size - self.max_pos = max_pos + self.vocab_size = vocab_size + self.max_pos = max_pos - self.enc_layers = enc_layers - self.enc_hidden_size = enc_hidden_size - self.enc_heads = enc_heads - self.enc_ff_size = enc_ff_size - self.enc_dropout = enc_dropout + self.enc_layers = enc_layers + self.enc_hidden_size = enc_hidden_size + self.enc_heads = enc_heads + self.enc_ff_size = enc_ff_size + self.enc_dropout = enc_dropout - self.dec_layers = dec_layers - self.dec_hidden_size = dec_hidden_size - self.dec_heads = dec_heads - self.dec_ff_size = dec_ff_size - self.dec_dropout = dec_dropout - else: - raise ValueError( - "First argument must be either a vocabulary size (int)" - "or the path to a pretrained model config file (str)" - ) - - def _input_is_path_to_json(self, first_argument): - """ Checks whether the first argument passed to config - is the path to a JSON file that contains the config. - """ - is_python_2 = sys.version_info[0] == 2 - if is_python_2: - return isinstance(first_argument, unicode) - else: - return isinstance(first_argument, str) + self.dec_layers = dec_layers + self.dec_hidden_size = dec_hidden_size + self.dec_heads = dec_heads + self.dec_ff_size = dec_ff_size + self.dec_dropout = dec_dropout From f1971bf303cc600cc47161f137e1b0baccd62925 Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Sun, 15 Dec 2019 01:37:16 +0100 Subject: [PATCH 369/505] Binding pipelines to the cli. --- transformers-cli | 4 +- transformers/commands/run.py | 56 +++++++++++++++++++ transformers/pipelines.py | 102 +++++++++++++++++++++++++++++++---- 3 files changed, 152 insertions(+), 10 deletions(-) create mode 100644 transformers/commands/run.py diff --git a/transformers-cli b/transformers-cli index 168e6e6f32..39b7f5816b 100755 --- a/transformers-cli +++ b/transformers-cli @@ -2,6 +2,7 @@ from argparse import ArgumentParser from transformers.commands.download import DownloadCommand +from transformers.commands.run import RunCommand from transformers.commands.serving import ServeCommand from transformers.commands.user import UserCommands from transformers.commands.train import TrainCommand @@ -14,9 +15,10 @@ if __name__ == '__main__': # Register commands ConvertCommand.register_subcommand(commands_parser) DownloadCommand.register_subcommand(commands_parser) + RunCommand.register_subcommand(commands_parser) ServeCommand.register_subcommand(commands_parser) - UserCommands.register_subcommand(commands_parser) TrainCommand.register_subcommand(commands_parser) + UserCommands.register_subcommand(commands_parser) # Let's go args = parser.parse_args() diff --git a/transformers/commands/run.py b/transformers/commands/run.py new file mode 100644 index 0000000000..bcbb87391d --- /dev/null +++ b/transformers/commands/run.py @@ -0,0 +1,56 @@ +from argparse import ArgumentParser + +from transformers.commands import BaseTransformersCLICommand +from transformers.pipelines import pipeline, Pipeline, PipelineDataFormat, SUPPORTED_TASKS + + +def try_infer_format_from_ext(path: str): + for ext in PipelineDataFormat.SUPPORTED_FORMATS: + if path.endswith(ext): + return ext + + raise Exception( + 'Unable to determine file format from file extension {}. ' + 'Please provide the format through --format {}'.format(path, PipelineDataFormat.SUPPORTED_FORMATS) + ) + + +def run_command_factory(args): + nlp = pipeline(task=args.task, model=args.model, tokenizer=args.tokenizer) + format = try_infer_format_from_ext(args.input) if args.format == 'infer' else args.format + reader = PipelineDataFormat.from_str(format, args.output, args.input, args.column) + return RunCommand(nlp, reader) + + +class RunCommand(BaseTransformersCLICommand): + + def __init__(self, nlp: Pipeline, reader: PipelineDataFormat): + self._nlp = nlp + self._reader = reader + + @staticmethod + def register_subcommand(parser: ArgumentParser): + run_parser = parser.add_parser('run', help="Run a pipeline through the CLI") + run_parser.add_argument('--task', choices=SUPPORTED_TASKS.keys(), help='Task to run') + run_parser.add_argument('--model', type=str, required=True, help='Name or path to the model to instantiate.') + run_parser.add_argument('--tokenizer', type=str, help='Name of the tokenizer to use. (default: same as the model name)') + run_parser.add_argument('--column', type=str, required=True, help='Name of the column to use as input. (For multi columns input as QA use column1,columns2)') + run_parser.add_argument('--format', type=str, default='infer', choices=PipelineDataFormat.SUPPORTED_FORMATS, help='Input format to read from') + run_parser.add_argument('--input', type=str, required=True, help='Path to the file to use for inference') + run_parser.add_argument('--output', type=str, required=True, help='Path to the file that will be used post to write results.') + run_parser.add_argument('kwargs', nargs='*', help='Arguments to forward to the file format reader') + run_parser.set_defaults(func=run_command_factory) + + def run(self): + nlp, output = self._nlp, [] + for entry in self._reader: + if self._reader.is_multi_columns: + output += [nlp(**entry)] + else: + output += [nlp(entry)] + + # Saving data + self._reader.save(output) + + + diff --git a/transformers/pipelines.py b/transformers/pipelines.py index 6fbb7e2f04..a5718b822f 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -14,6 +14,8 @@ # limitations under the License. from __future__ import absolute_import, division, print_function, unicode_literals +import csv +import json import os from abc import ABC, abstractmethod from itertools import groupby @@ -25,11 +27,13 @@ from transformers import AutoTokenizer, PreTrainedTokenizer, PretrainedConfig, \ SquadExample, squad_convert_examples_to_features, is_tf_available, is_torch_available, logger if is_tf_available(): - from transformers import TFAutoModelForSequenceClassification, TFAutoModelForQuestionAnswering, TFAutoModelForTokenClassification + from transformers import TFAutoModel, TFAutoModelForSequenceClassification, \ + TFAutoModelForQuestionAnswering, TFAutoModelForTokenClassification if is_torch_available(): import torch - from transformers import AutoModelForSequenceClassification, AutoModelForQuestionAnswering, AutoModelForTokenClassification + from transformers import AutoModel, AutoModelForSequenceClassification, \ + AutoModelForQuestionAnswering, AutoModelForTokenClassification class Pipeline(ABC): @@ -58,6 +62,84 @@ class Pipeline(ABC): raise NotImplementedError() +class PipelineDataFormat: + SUPPORTED_FORMATS = ['json', 'csv'] + + def __init__(self, output: str, path: str, column: str): + self.output = output + self.path = path + self.column = column.split(',') + self.is_multi_columns = len(self.column) > 1 + + if self.is_multi_columns: + self.column = [tuple(c.split('=')) if '=' in c else (c, c) for c in self.column] + + from os.path import abspath, exists + if exists(abspath(self.output)): + raise OSError('{} already exists on disk'.format(self.output)) + + if not exists(abspath(self.path)): + raise OSError('{} doesnt exist on disk'.format(self.path)) + + @abstractmethod + def __iter__(self): + raise NotImplementedError() + + @abstractmethod + def save(self, data: dict): + raise NotImplementedError() + + @staticmethod + def from_str(name: str, output: str, path: str, column: str): + if name == 'json': + return JsonPipelineDataFormat(output, path, column) + elif name == 'csv': + return CsvPipelineDataFormat(output, path, column) + else: + raise KeyError('Unknown reader {} (Available reader are json/csv)'.format(name)) + + +class CsvPipelineDataFormat(PipelineDataFormat): + def __init__(self, output: str, path: str, column: str): + super().__init__(output, path, column) + + def __iter__(self): + with open(self.path, 'r') as f: + reader = csv.DictReader(f) + for row in reader: + if self.is_multi_columns: + yield {k: row[c] for k, c in self.column} + else: + yield row[self.column] + + def save(self, data: List[dict]): + with open(self.output, 'w') as f: + if len(data) > 0: + writer = csv.DictWriter(f, list(data[0].keys())) + writer.writeheader() + writer.writerows(data) + + +class JsonPipelineDataFormat(PipelineDataFormat): + + def __init__(self, output: str, path: str, column: str): + super().__init__(output, path, column) + + with open(path, 'r') as f: + self._entries = json.load(f) + + def __iter__(self): + for entry in self._entries: + if self.is_multi_columns: + yield {k: entry[c] for k, c in self.column} + else: + yield entry[self.column] + + def save(self, data: dict): + with open(self.output, 'w') as f: + json.dump(data, f) + + class FeatureExtractionPipeline(Pipeline): def __call__(self, *texts, **kwargs): @@ -127,7 +209,7 @@ class NerPipeline(Pipeline): label_idx = score.argmax() answer += [{ - 'word': words[idx - 1], 'score': score[label_idx], 'entity': self.model.config.id2label[label_idx] + 'word': words[idx - 1], 'score': score[label_idx].item(), 'entity': self.model.config.id2label[label_idx] }] # Update token start @@ -270,16 +352,18 @@ class QuestionAnsweringPipeline(Pipeline): char_to_word = np.array(example.char_to_word_offset) # Convert the answer (tokens) back to the original text - answers += [[ + answers += [ { - 'score': score, - 'start': np.where(char_to_word == feature.token_to_orig_map[s])[0][0], - 'end': np.where(char_to_word == feature.token_to_orig_map[e])[0][-1], + 'score': score.item(), + 'start': np.where(char_to_word == feature.token_to_orig_map[s])[0][0].item(), + 'end': np.where(char_to_word == feature.token_to_orig_map[e])[0][-1].item(), 'answer': ' '.join(example.doc_tokens[feature.token_to_orig_map[s]: feature.token_to_orig_map[e] + 1]) } for s, e, score in zip(starts, ends, scores) - ]] + ] + if len(answers) == 1: + return answers[0] return answers def decode(self, start: np.ndarray, end: np.ndarray, topk: int, max_answer_len: int) -> Tuple: @@ -363,7 +447,7 @@ def pipeline(task: str, model, config: Optional[PretrainedConfig] = None, tokeni Utility factory method to build pipeline. """ # Try to infer tokenizer from model name (if provided as str) - if not isinstance(tokenizer, PreTrainedTokenizer): + if tokenizer is None: if not isinstance(model, str): # Impossible to guest what is the right tokenizer here raise Exception('Tokenizer cannot be None if provided model is a PreTrainedModel instance') From 8e3b1c860fbd872cb39d5093a1e739a67e7d0809 Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Sun, 15 Dec 2019 01:37:52 +0100 Subject: [PATCH 370/505] Added FeatureExtraction pipeline. --- transformers/pipelines.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index a5718b822f..7383222c1f 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -143,7 +143,23 @@ class JsonPipelineDataFormat(PipelineDataFormat): class FeatureExtractionPipeline(Pipeline): def __call__(self, *texts, **kwargs): - pass + # Generic compatibility with sklearn and Keras + if 'X' in kwargs and not texts: + texts = kwargs.pop('X') + + inputs = self.tokenizer.batch_encode_plus( + texts, add_special_tokens=True, return_tensors='tf' if is_tf_available() else 'pt' + ) + + if is_tf_available(): + # TODO trace model + predictions = self.model(inputs)[0] + else: + import torch + with torch.no_grad(): + predictions = self.model(**inputs)[0] + + return predictions.numpy().tolist() class TextClassificationPipeline(Pipeline): @@ -424,6 +440,11 @@ class QuestionAnsweringPipeline(Pipeline): # Register all the supported task here SUPPORTED_TASKS = { + 'feature-extraction': { + 'impl': FeatureExtractionPipeline, + 'tf': TFAutoModel if is_tf_available() else None, + 'pt': AutoModel if is_torch_available() else None, + }, 'text-classification': { 'impl': TextClassificationPipeline, 'tf': TFAutoModelForSequenceClassification if is_tf_available() else None, From 1b8613acb32a568db8d9b74ee182d43c4f8e9cbb Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 16 Dec 2019 09:51:42 +0100 Subject: [PATCH 371/505] updating t5 config class --- transformers/configuration_t5.py | 15 ++------------- transformers/tests/modeling_t5_test.py | 2 +- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/transformers/configuration_t5.py b/transformers/configuration_t5.py index 6391cb4180..377a0919d9 100644 --- a/transformers/configuration_t5.py +++ b/transformers/configuration_t5.py @@ -66,7 +66,7 @@ class T5Config(PretrainedConfig): pretrained_config_archive_map = T5_PRETRAINED_CONFIG_ARCHIVE_MAP def __init__(self, - vocab_size_or_config_json_file=32128, + vocab_size=32128, n_positions=512, d_model=512, d_kv=64, @@ -79,7 +79,7 @@ class T5Config(PretrainedConfig): initializer_factor=1.0, **kwargs): super(T5Config, self).__init__(**kwargs) - self.vocab_size = vocab_size_or_config_json_file if isinstance(vocab_size_or_config_json_file, int) else -1 + self.vocab_size = vocab_size self.n_positions = n_positions self.d_model = d_model self.d_kv = d_kv @@ -91,17 +91,6 @@ class T5Config(PretrainedConfig): self.layer_norm_epsilon = layer_norm_epsilon self.initializer_factor = initializer_factor - if isinstance(vocab_size_or_config_json_file, six.string_types): - with open(vocab_size_or_config_json_file, "r", encoding="utf-8") as reader: - json_config = json.loads(reader.read()) - for key, value in json_config.items(): - self.__dict__[key] = value - elif not isinstance(vocab_size_or_config_json_file, int): - raise ValueError( - "First argument must be either a vocabulary size (int)" - "or the path to a pretrained model config file (str)" - ) - @property def max_position_embeddings(self): return self.n_positions diff --git a/transformers/tests/modeling_t5_test.py b/transformers/tests/modeling_t5_test.py index a539cc868a..c337163375 100644 --- a/transformers/tests/modeling_t5_test.py +++ b/transformers/tests/modeling_t5_test.py @@ -93,7 +93,7 @@ class T5ModelTest(CommonTestCases.CommonModelTester): decoder_lm_labels = ids_tensor([self.batch_size, self.decoder_seq_length], self.vocab_size) config = T5Config( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, n_positions=self.n_positions, d_model=self.hidden_size, d_ff=self.d_ff, From 8669598abd7af877bd33890d62ae70ec1623f145 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 16 Dec 2019 09:59:36 +0100 Subject: [PATCH 372/505] update t5 tf --- transformers/tests/modeling_tf_t5_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/tests/modeling_tf_t5_test.py b/transformers/tests/modeling_tf_t5_test.py index 99eec313f9..b905a9875b 100644 --- a/transformers/tests/modeling_tf_t5_test.py +++ b/transformers/tests/modeling_tf_t5_test.py @@ -87,7 +87,7 @@ class TFT5ModelTest(TFCommonTestCases.TFCommonModelTester): token_labels = ids_tensor([self.batch_size, self.seq_length], self.vocab_size) config = T5Config( - vocab_size_or_config_json_file=self.vocab_size, + vocab_size=self.vocab_size, n_positions=self.n_positions, d_model=self.hidden_size, d_ff=self.d_ff, From 56e98ba81a9a7410243a1117fb6148d5f353ef98 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 16 Dec 2019 11:07:27 +0100 Subject: [PATCH 373/505] add model cards cc @mfuntowicz --- transformers/__init__.py | 3 + transformers/file_utils.py | 2 +- transformers/model_card.py | 248 ++++++++++++++++++++++++++ transformers/tests/model_card_test.py | 87 +++++++++ 4 files changed, 339 insertions(+), 1 deletion(-) create mode 100644 transformers/model_card.py create mode 100644 transformers/tests/model_card_test.py diff --git a/transformers/__init__.py b/transformers/__init__.py index 740d2440c2..15c167a5ce 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -33,6 +33,9 @@ from .data import (is_sklearn_available, if is_sklearn_available(): from .data import glue_compute_metrics, xnli_compute_metrics +# Model Cards +from .model_card import ModelCard + # Tokenizers from .tokenization_utils import (PreTrainedTokenizer) from .tokenization_auto import AutoTokenizer diff --git a/transformers/file_utils.py b/transformers/file_utils.py index 03b2fdb9f4..81c9b8002f 100644 --- a/transformers/file_utils.py +++ b/transformers/file_utils.py @@ -72,7 +72,7 @@ WEIGHTS_NAME = "pytorch_model.bin" TF2_WEIGHTS_NAME = 'tf_model.h5' TF_WEIGHTS_NAME = 'model.ckpt' CONFIG_NAME = "config.json" - +MODEL_CARD_NAME = "model_card.json" DUMMY_INPUTS = [[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]] DUMMY_MASK = [[1, 1, 1, 1, 1], [1, 1, 1, 0, 0], [0, 0, 0, 1, 1]] diff --git a/transformers/model_card.py b/transformers/model_card.py new file mode 100644 index 0000000000..679c24872a --- /dev/null +++ b/transformers/model_card.py @@ -0,0 +1,248 @@ +# coding=utf-8 +# Copyright 2018 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Configuration base class and utilities.""" + +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import copy +import json +import logging +import os +import re +from io import open + +from .configuration_bert import BERT_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_openai import OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_transfo_xl import TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_gpt2 import GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_ctrl import CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_xlnet import XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_xlm import XLM_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_roberta import ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_distilbert import DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_albert import ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_camembert import CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_t5 import T5_PRETRAINED_CONFIG_ARCHIVE_MAP + +from .file_utils import CONFIG_NAME, MODEL_CARD_NAME, cached_path, is_remote_url, hf_bucket_url + + +logger = logging.getLogger(__name__) + + +ALL_MODELS_MAP = dict((key, value) + for pretrained_map in [ + BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, + OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, + TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP, + GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP, + CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP, + XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP, + XLM_PRETRAINED_CONFIG_ARCHIVE_MAP, + ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, + DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, + ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, + CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, + T5_PRETRAINED_CONFIG_ARCHIVE_MAP, + ] + for key, value, in pretrained_map.items()) + + +class ModelCard(object): + r""" Model Card class. + Store model card as well as methods for loading/downloading/saving model cards. + + Please read the following paper for details and explanation on the sections: + "Model Cards for Model Reporting" + by Margaret Mitchell, Simone Wu, + Andrew Zaldivar, Parker Barnes, Lucy Vasserman, Ben Hutchinson, Elena Spitzer, + Inioluwa Deborah Raji and Timnit Gebru for the proposal behind model cards. + Link: https://arxiv.org/abs/1810.03993 + + Note: + A model card can be loaded and saved to disk. + + Parameters: + """ + def __init__(self, **kwargs): + # Recomended attributes from https://arxiv.org/abs/1810.03993 (see papers) + self.model_details = kwargs.pop('model_details', {}) + self.intended_use = kwargs.pop('intended_use', {}) + self.factors = kwargs.pop('factors', {}) + self.metrics = kwargs.pop('metrics', {}) + self.evaluation_data = kwargs.pop('evaluation_data', {}) + self.training_data = kwargs.pop('training_data', {}) + self.quantitative_analyses = kwargs.pop('quantitative_analyses', {}) + self.ethical_considerations = kwargs.pop('ethical_considerations', {}) + self.caveats_and_recommendations = kwargs.pop('caveats_and_recommendations', {}) + + # Open additional attributes + for key, value in kwargs.items(): + try: + setattr(self, key, value) + except AttributeError as err: + logger.error("Can't set {} with value {} for {}".format(key, value, self)) + raise err + + def save_pretrained(self, save_directory): + """ Save a model card object to the directory `save_directory`, so that it + can be re-loaded using the :func:`~transformers.ModelCard.from_pretrained` class method. + """ + assert os.path.isdir(save_directory), "Saving path should be a directory where the model card can be saved" + + # If we save using the predefined names, we can load using `from_pretrained` + output_model_card_file = os.path.join(save_directory, MODEL_CARD_NAME) + + self.to_json_file(output_model_card_file) + logger.info("Model card saved in {}".format(output_model_card_file)) + + @classmethod + def from_pretrained(cls, pretrained_model_name_or_path, **kwargs): + r""" Instantiate a :class:`~transformers.ModelCard` from a pre-trained model model card. + + Parameters: + pretrained_model_name_or_path: either: + + - a string with the `shortcut name` of a pre-trained model card to load from cache or download, e.g.: ``bert-base-uncased``. + - a string with the `identifier name` of a pre-trained model card that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. + - a path to a `directory` containing a mode card file saved using the :func:`~transformers.ModelCard.save_pretrained` method, e.g.: ``./my_model_directory/``. + - a path or url to a saved model card JSON `file`, e.g.: ``./my_model_directory/model_card.json``. + + cache_dir: (`optional`) string: + Path to a directory in which a downloaded pre-trained model + card should be cached if the standard cache should not be used. + + kwargs: (`optional`) dict: key/value pairs with which to update the ModelCard object after loading. + + - The values in kwargs of any keys which are model card attributes will be used to override the loaded values. + - Behavior concerning key/value pairs whose keys are *not* model card attributes is controlled by the `return_unused_kwargs` keyword parameter. + + force_download: (`optional`) boolean, default False: + Force to (re-)download the model card file and override the cached version if it exists. + + resume_download: (`optional`) boolean, default False: + Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. + + proxies: (`optional`) dict, default None: + A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. + The proxies are used on each request. + + return_unused_kwargs: (`optional`) bool: + + - If False, then this function returns just the final model card object. + - If True, then this functions returns a tuple `(model card, unused_kwargs)` where `unused_kwargs` is a dictionary consisting of the key/value pairs whose keys are not model card attributes: ie the part of kwargs which has not been used to update `ModelCard` and is otherwise ignored. + + Examples:: + + model_card = ModelCard.from_pretrained('bert-base-uncased') # Download model card from S3 and cache. + model_card = ModelCard.from_pretrained('./test/saved_model/') # E.g. model card was saved using `save_pretrained('./test/saved_model/')` + model_card = ModelCard.from_pretrained('./test/saved_model/model_card.json') + model_card = ModelCard.from_pretrained('bert-base-uncased', output_attention=True, foo=False) + + """ + cache_dir = kwargs.pop('cache_dir', None) + force_download = kwargs.pop('force_download', False) + resume_download = kwargs.pop('resume_download', False) + proxies = kwargs.pop('proxies', None) + return_unused_kwargs = kwargs.pop('return_unused_kwargs', False) + + if pretrained_model_name_or_path in ALL_MODELS_MAP: + model_card_file = ALL_MODELS_MAP[pretrained_model_name_or_path] + model_card_file.replace(CONFIG_NAME, MODEL_CARD_NAME) # For simplicity we use the same pretrained url than config but with a different suffix + elif os.path.isdir(pretrained_model_name_or_path): + model_card_file = os.path.join(pretrained_model_name_or_path, MODEL_CARD_NAME) + elif os.path.isfile(pretrained_model_name_or_path) or is_remote_url(pretrained_model_name_or_path): + model_card_file = pretrained_model_name_or_path + else: + model_card_file = hf_bucket_url(pretrained_model_name_or_path, postfix=MODEL_CARD_NAME) + # redirect to the cache, if necessary + try: + resolved_model_card_file = cached_path(model_card_file, cache_dir=cache_dir, force_download=force_download, + proxies=proxies, resume_download=resume_download) + + if resolved_model_card_file == model_card_file: + logger.info("loading model card file {}".format(model_card_file)) + else: + logger.info("loading model card file {} from cache at {}".format( + model_card_file, resolved_model_card_file)) + + # Load model card + model_card = cls.from_json_file(resolved_model_card_file) + + except EnvironmentError: + if pretrained_model_name_or_path in ALL_MODELS_MAP: + logger.warning("Couldn't reach server at '{}' to download model card file.".format( + model_card_file)) + else: + logger.warning("Model name '{}' was not found in model name list ({}). " \ + "We assumed '{}' was a path or url to a model card file named {} or " \ + "a directory containing such a file but couldn't find any such file at this path or url.".format( + pretrained_model_name_or_path, + ', '.join(ALL_MODELS_MAP.keys()), + model_card_file, MODEL_CARD_NAME)) + + logger.warning("Creating an empty model card.") + + # We fall back on creating an empty model card + model_card = cls() + + # Update model card with kwargs if needed + to_remove = [] + for key, value in kwargs.items(): + if hasattr(model_card, key): + setattr(model_card, key, value) + to_remove.append(key) + for key in to_remove: + kwargs.pop(key, None) + + logger.info("Model card: %s", str(model_card)) + if return_unused_kwargs: + return model_card, kwargs + else: + return model_card + + @classmethod + def from_dict(cls, json_object): + """Constructs a `ModelCard` from a Python dictionary of parameters.""" + return cls(**json_object) + + @classmethod + def from_json_file(cls, json_file): + """Constructs a `ModelCard` from a json file of parameters.""" + with open(json_file, "r", encoding='utf-8') as reader: + text = reader.read() + dict_obj = json.loads(text) + return cls(**dict_obj) + + def __eq__(self, other): + return self.__dict__ == other.__dict__ + + def __repr__(self): + return str(self.to_json_string()) + + def to_dict(self): + """Serializes this instance to a Python dictionary.""" + output = copy.deepcopy(self.__dict__) + return output + + def to_json_string(self): + """Serializes this instance to a JSON string.""" + return json.dumps(self.to_dict(), indent=2, sort_keys=True) + "\n" + + def to_json_file(self, json_file_path): + """ Save this instance to a json file.""" + with open(json_file_path, "w", encoding='utf-8') as writer: + writer.write(self.to_json_string()) diff --git a/transformers/tests/model_card_test.py b/transformers/tests/model_card_test.py new file mode 100644 index 0000000000..4364cbacec --- /dev/null +++ b/transformers/tests/model_card_test.py @@ -0,0 +1,87 @@ +# coding=utf-8 +# Copyright 2019 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import absolute_import, division, print_function, unicode_literals + +import os +import sys +import json +import tempfile +import shutil +import unittest + +from transformers.model_card import ModelCard +from .tokenization_tests_commons import TemporaryDirectory + +class ModelCardTester(unittest.TestCase): + + def setUp(self): + self.inputs_dict = {'model_details': { + 'Organization': 'testing', + 'Model date': 'today', + 'Model version': 'v2.1, Developed by Test Corp in 2019.', + 'Architecture': 'Convolutional Neural Network.', + }, + 'metrics': 'BLEU and ROUGE-1', + 'evaluation_data':{ + 'Datasets':{ + 'BLEU': 'My-great-dataset-v1', + 'ROUGE-1': 'My-short-dataset-v2.1', + }, + 'Preprocessing': 'See details on https://arxiv.org/pdf/1810.03993.pdf' + }, + 'training_data':{ + 'Dataset': 'English Wikipedia dump dated 2018-12-01', + 'Preprocessing': 'Using SentencePiece vocabulary of size 52k tokens. See details on https://arxiv.org/pdf/1810.03993.pdf' + }, + 'quantitative_analyses': { + 'BLEU': 55.1, + 'ROUGE-1': 76, + }, + } + self.tmpdirname = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.tmpdirname) + + def test_model_card_common_properties(self): + model_card = ModelCard.from_dict(self.inputs_dict) + self.assertTrue(hasattr(model_card, 'model_details')) + self.assertTrue(hasattr(model_card, 'intended_use')) + self.assertTrue(hasattr(model_card, 'factors')) + self.assertTrue(hasattr(model_card, 'metrics')) + self.assertTrue(hasattr(model_card, 'evaluation_data')) + self.assertTrue(hasattr(model_card, 'training_data')) + self.assertTrue(hasattr(model_card, 'quantitative_analyses')) + self.assertTrue(hasattr(model_card, 'ethical_considerations')) + self.assertTrue(hasattr(model_card, 'caveats_and_recommendations')) + + def test_model_card_to_json_string(self): + model_card = ModelCard.from_dict(self.inputs_dict) + obj = json.loads(model_card.to_json_string()) + for key, value in self.inputs_dict.items(): + self.assertEqual(obj[key], value) + + def test_model_card_to_json_file(self): + model_card_first = ModelCard.from_dict(self.inputs_dict) + + with TemporaryDirectory() as tmpdirname: + filename = os.path.join(tmpdirname, u"model_card.json") + model_card_first.to_json_file(filename) + model_card_second = ModelCard.from_json_file(filename) + + self.assertEqual(model_card_second.to_dict(), model_card_first.to_dict()) + +if __name__ == "__main__": + unittest.main() From d3418a94ff4256725a690bd9c8167489b6f593b8 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 16 Dec 2019 13:52:41 +0100 Subject: [PATCH 374/505] update tests --- .../tests/configuration_common_test.py | 27 ++++++++++++------- transformers/tests/model_card_test.py | 16 ++++++----- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/transformers/tests/configuration_common_test.py b/transformers/tests/configuration_common_test.py index 8ee751153c..376d110d3c 100644 --- a/transformers/tests/configuration_common_test.py +++ b/transformers/tests/configuration_common_test.py @@ -16,15 +16,12 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import copy import os -import shutil import json -import random -import uuid +import tempfile import unittest -import logging +from .tokenization_tests_commons import TemporaryDirectory class ConfigTester(object): @@ -48,16 +45,28 @@ class ConfigTester(object): def create_and_test_config_to_json_file(self): config_first = self.config_class(**self.inputs_dict) - json_file_path = os.path.join(os.getcwd(), "config_" + str(uuid.uuid4()) + ".json") - config_first.to_json_file(json_file_path) - config_second = self.config_class.from_json_file(json_file_path) - os.remove(json_file_path) + + with TemporaryDirectory() as tmpdirname: + json_file_path = os.path.join(tmpdirname, "config.json") + config_first.to_json_file(json_file_path) + config_second = self.config_class.from_json_file(json_file_path) + + self.parent.assertEqual(config_second.to_dict(), config_first.to_dict()) + + def create_and_test_config_from_and_save_pretrained(self): + config_first = self.config_class(**self.inputs_dict) + + with TemporaryDirectory() as tmpdirname: + config_first.save_pretrained(tmpdirname) + config_second = self.config_class.from_pretrained(tmpdirname) + self.parent.assertEqual(config_second.to_dict(), config_first.to_dict()) def run_common_tests(self): self.create_and_test_config_common_properties() self.create_and_test_config_to_json_string() self.create_and_test_config_to_json_file() + self.create_and_test_config_from_and_save_pretrained() if __name__ == "__main__": unittest.main() \ No newline at end of file diff --git a/transformers/tests/model_card_test.py b/transformers/tests/model_card_test.py index 4364cbacec..e75716f0aa 100644 --- a/transformers/tests/model_card_test.py +++ b/transformers/tests/model_card_test.py @@ -15,10 +15,7 @@ from __future__ import absolute_import, division, print_function, unicode_literals import os -import sys import json -import tempfile -import shutil import unittest from transformers.model_card import ModelCard @@ -50,10 +47,6 @@ class ModelCardTester(unittest.TestCase): 'ROUGE-1': 76, }, } - self.tmpdirname = tempfile.mkdtemp() - - def tearDown(self): - shutil.rmtree(self.tmpdirname) def test_model_card_common_properties(self): model_card = ModelCard.from_dict(self.inputs_dict) @@ -83,5 +76,14 @@ class ModelCardTester(unittest.TestCase): self.assertEqual(model_card_second.to_dict(), model_card_first.to_dict()) + def test_model_card_from_and_save_pretrained(self): + model_card_first = ModelCard.from_dict(self.inputs_dict) + + with TemporaryDirectory() as tmpdirname: + model_card_first.save_pretrained(tmpdirname) + model_card_second = ModelCard.from_pretrained(tmpdirname) + + self.assertEqual(model_card_second.to_dict(), model_card_first.to_dict()) + if __name__ == "__main__": unittest.main() From a4d07b983a6c1716b4d39cf3fed570562aebf3f7 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 16 Dec 2019 14:00:32 +0100 Subject: [PATCH 375/505] dict of all config and model files cc @LysandreJik --- transformers/__init__.py | 6 ++--- transformers/configuration_auto.py | 42 ++++++++++++++++++++--------- transformers/model_card.py | 43 +++++------------------------- transformers/modeling_auto.py | 42 ++++++++++++++++++++--------- transformers/modeling_tf_auto.py | 38 +++++++++++++++++++------- 5 files changed, 98 insertions(+), 73 deletions(-) diff --git a/transformers/__init__.py b/transformers/__init__.py index 15c167a5ce..0b343bed2b 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -55,7 +55,7 @@ from .tokenization_t5 import T5Tokenizer # Configurations from .configuration_utils import PretrainedConfig -from .configuration_auto import AutoConfig +from .configuration_auto import AutoConfig, ALL_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_bert import BertConfig, BERT_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_openai import OpenAIGPTConfig, OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_transfo_xl import TransfoXLConfig, TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP @@ -73,7 +73,7 @@ from .configuration_t5 import T5Config, T5_PRETRAINED_CONFIG_ARCHIVE_MAP if is_torch_available(): from .modeling_utils import (PreTrainedModel, prune_layer, Conv1D) from .modeling_auto import (AutoModel, AutoModelForSequenceClassification, AutoModelForQuestionAnswering, - AutoModelWithLMHead) + AutoModelWithLMHead, ALL_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_bert import (BertPreTrainedModel, BertModel, BertForPreTraining, BertForMaskedLM, BertForNextSentencePrediction, @@ -131,7 +131,7 @@ if is_torch_available(): if is_tf_available(): from .modeling_tf_utils import TFPreTrainedModel, TFSharedEmbeddings, TFSequenceSummary, shape_list from .modeling_tf_auto import (TFAutoModel, TFAutoModelForSequenceClassification, TFAutoModelForQuestionAnswering, - TFAutoModelWithLMHead) + TFAutoModelWithLMHead, TF_ALL_PRETRAINED_MODEL_ARCHIVE_MAP) from .modeling_tf_bert import (TFBertPreTrainedModel, TFBertMainLayer, TFBertEmbeddings, TFBertModel, TFBertForPreTraining, diff --git a/transformers/configuration_auto.py b/transformers/configuration_auto.py index 680c55fa54..9fe58f173a 100644 --- a/transformers/configuration_auto.py +++ b/transformers/configuration_auto.py @@ -18,22 +18,40 @@ from __future__ import absolute_import, division, print_function, unicode_litera import logging -from .configuration_bert import BertConfig -from .configuration_openai import OpenAIGPTConfig -from .configuration_gpt2 import GPT2Config -from .configuration_transfo_xl import TransfoXLConfig -from .configuration_xlnet import XLNetConfig -from .configuration_xlm import XLMConfig -from .configuration_roberta import RobertaConfig -from .configuration_distilbert import DistilBertConfig -from .configuration_ctrl import CTRLConfig -from .configuration_camembert import CamembertConfig -from .configuration_albert import AlbertConfig -from .configuration_t5 import T5Config +from .configuration_bert import BertConfig, BERT_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_openai import OpenAIGPTConfig, OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_transfo_xl import TransfoXLConfig, TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_gpt2 import GPT2Config, GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_ctrl import CTRLConfig, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_xlnet import XLNetConfig, XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_xlm import XLMConfig, XLM_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_roberta import RobertaConfig, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_distilbert import DistilBertConfig, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_albert import AlbertConfig, ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_camembert import CamembertConfig, CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_t5 import T5Config, T5_PRETRAINED_CONFIG_ARCHIVE_MAP logger = logging.getLogger(__name__) +ALL_PRETRAINED_CONFIG_ARCHIVE_MAP = dict((key, value) + for pretrained_map in [ + BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, + OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, + TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP, + GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP, + CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP, + XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP, + XLM_PRETRAINED_CONFIG_ARCHIVE_MAP, + ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, + DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, + ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, + CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, + T5_PRETRAINED_CONFIG_ARCHIVE_MAP, + ] + for key, value, in pretrained_map.items()) + + class AutoConfig(object): r""":class:`~transformers.AutoConfig` is a generic configuration class that will be instantiated as one of the configuration classes of the library diff --git a/transformers/model_card.py b/transformers/model_card.py index 679c24872a..6d56089844 100644 --- a/transformers/model_card.py +++ b/transformers/model_card.py @@ -21,21 +21,9 @@ import copy import json import logging import os -import re from io import open -from .configuration_bert import BERT_PRETRAINED_CONFIG_ARCHIVE_MAP -from .configuration_openai import OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP -from .configuration_transfo_xl import TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP -from .configuration_gpt2 import GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP -from .configuration_ctrl import CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP -from .configuration_xlnet import XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP -from .configuration_xlm import XLM_PRETRAINED_CONFIG_ARCHIVE_MAP -from .configuration_roberta import ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP -from .configuration_distilbert import DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP -from .configuration_albert import ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP -from .configuration_camembert import CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP -from .configuration_t5 import T5_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_auto import ALL_PRETRAINED_CONFIG_ARCHIVE_MAP from .file_utils import CONFIG_NAME, MODEL_CARD_NAME, cached_path, is_remote_url, hf_bucket_url @@ -43,24 +31,6 @@ from .file_utils import CONFIG_NAME, MODEL_CARD_NAME, cached_path, is_remote_url logger = logging.getLogger(__name__) -ALL_MODELS_MAP = dict((key, value) - for pretrained_map in [ - BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, - OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, - TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP, - GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP, - CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP, - XLNET_PRETRAINED_CONFIG_ARCHIVE_MAP, - XLM_PRETRAINED_CONFIG_ARCHIVE_MAP, - ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, - DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, - ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, - CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, - T5_PRETRAINED_CONFIG_ARCHIVE_MAP, - ] - for key, value, in pretrained_map.items()) - - class ModelCard(object): r""" Model Card class. Store model card as well as methods for loading/downloading/saving model cards. @@ -159,9 +129,10 @@ class ModelCard(object): proxies = kwargs.pop('proxies', None) return_unused_kwargs = kwargs.pop('return_unused_kwargs', False) - if pretrained_model_name_or_path in ALL_MODELS_MAP: - model_card_file = ALL_MODELS_MAP[pretrained_model_name_or_path] - model_card_file.replace(CONFIG_NAME, MODEL_CARD_NAME) # For simplicity we use the same pretrained url than config but with a different suffix + if pretrained_model_name_or_path in ALL_PRETRAINED_CONFIG_ARCHIVE_MAP: + # For simplicity we use the same pretrained url than the configuration files but with a different suffix (model_card.json) + model_card_file = ALL_PRETRAINED_CONFIG_ARCHIVE_MAP[pretrained_model_name_or_path] + model_card_file.replace(CONFIG_NAME, MODEL_CARD_NAME) elif os.path.isdir(pretrained_model_name_or_path): model_card_file = os.path.join(pretrained_model_name_or_path, MODEL_CARD_NAME) elif os.path.isfile(pretrained_model_name_or_path) or is_remote_url(pretrained_model_name_or_path): @@ -183,7 +154,7 @@ class ModelCard(object): model_card = cls.from_json_file(resolved_model_card_file) except EnvironmentError: - if pretrained_model_name_or_path in ALL_MODELS_MAP: + if pretrained_model_name_or_path in ALL_PRETRAINED_CONFIG_ARCHIVE_MAP: logger.warning("Couldn't reach server at '{}' to download model card file.".format( model_card_file)) else: @@ -191,7 +162,7 @@ class ModelCard(object): "We assumed '{}' was a path or url to a model card file named {} or " \ "a directory containing such a file but couldn't find any such file at this path or url.".format( pretrained_model_name_or_path, - ', '.join(ALL_MODELS_MAP.keys()), + ', '.join(ALL_PRETRAINED_CONFIG_ARCHIVE_MAP.keys()), model_card_file, MODEL_CARD_NAME)) logger.warning("Creating an empty model card.") diff --git a/transformers/modeling_auto.py b/transformers/modeling_auto.py index 19a54cca86..1a30ea4623 100644 --- a/transformers/modeling_auto.py +++ b/transformers/modeling_auto.py @@ -18,18 +18,18 @@ from __future__ import absolute_import, division, print_function, unicode_litera import logging -from .modeling_bert import BertModel, BertForMaskedLM, BertForSequenceClassification, BertForQuestionAnswering -from .modeling_openai import OpenAIGPTModel, OpenAIGPTLMHeadModel -from .modeling_gpt2 import GPT2Model, GPT2LMHeadModel -from .modeling_ctrl import CTRLModel, CTRLLMHeadModel -from .modeling_transfo_xl import TransfoXLModel, TransfoXLLMHeadModel -from .modeling_xlnet import XLNetModel, XLNetLMHeadModel, XLNetForSequenceClassification, XLNetForQuestionAnswering -from .modeling_xlm import XLMModel, XLMWithLMHeadModel, XLMForSequenceClassification, XLMForQuestionAnswering -from .modeling_roberta import RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification -from .modeling_distilbert import DistilBertModel, DistilBertForQuestionAnswering, DistilBertForMaskedLM, DistilBertForSequenceClassification -from .modeling_camembert import CamembertModel, CamembertForMaskedLM, CamembertForSequenceClassification, CamembertForMultipleChoice -from .modeling_albert import AlbertModel, AlbertForMaskedLM, AlbertForSequenceClassification, AlbertForQuestionAnswering -from .modeling_t5 import T5Model, T5WithLMHeadModel +from .modeling_bert import BertModel, BertForMaskedLM, BertForSequenceClassification, BertForQuestionAnswering, BERT_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_openai import OpenAIGPTModel, OpenAIGPTLMHeadModel, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_gpt2 import GPT2Model, GPT2LMHeadModel, GPT2_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_ctrl import CTRLModel, CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_transfo_xl import TransfoXLModel, TransfoXLLMHeadModel, TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_xlnet import XLNetModel, XLNetLMHeadModel, XLNetForSequenceClassification, XLNetForQuestionAnswering, XLNET_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_xlm import XLMModel, XLMWithLMHeadModel, XLMForSequenceClassification, XLMForQuestionAnswering, XLM_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_roberta import RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_distilbert import DistilBertModel, DistilBertForQuestionAnswering, DistilBertForMaskedLM, DistilBertForSequenceClassification, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_camembert import CamembertModel, CamembertForMaskedLM, CamembertForSequenceClassification, CamembertForMultipleChoice, CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_albert import AlbertModel, AlbertForMaskedLM, AlbertForSequenceClassification, AlbertForQuestionAnswering, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_t5 import T5Model, T5WithLMHeadModel, T5_PRETRAINED_MODEL_ARCHIVE_MAP from .modeling_utils import PreTrainedModel, SequenceSummary @@ -38,6 +38,24 @@ from .file_utils import add_start_docstrings logger = logging.getLogger(__name__) +ALL_PRETRAINED_MODEL_ARCHIVE_MAP = dict((key, value) + for pretrained_map in [ + BERT_PRETRAINED_MODEL_ARCHIVE_MAP, + OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP, + TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP, + GPT2_PRETRAINED_MODEL_ARCHIVE_MAP, + CTRL_PRETRAINED_MODEL_ARCHIVE_MAP, + XLNET_PRETRAINED_MODEL_ARCHIVE_MAP, + XLM_PRETRAINED_MODEL_ARCHIVE_MAP, + ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, + DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, + ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP, + CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP, + T5_PRETRAINED_MODEL_ARCHIVE_MAP, + ] + for key, value, in pretrained_map.items()) + + class AutoModel(object): r""" :class:`~transformers.AutoModel` is a generic model class diff --git a/transformers/modeling_tf_auto.py b/transformers/modeling_tf_auto.py index b4ff660098..9c687d9235 100644 --- a/transformers/modeling_tf_auto.py +++ b/transformers/modeling_tf_auto.py @@ -18,22 +18,40 @@ from __future__ import absolute_import, division, print_function, unicode_litera import logging -from .modeling_tf_bert import TFBertModel, TFBertForMaskedLM, TFBertForSequenceClassification, TFBertForQuestionAnswering -from .modeling_tf_openai import TFOpenAIGPTModel, TFOpenAIGPTLMHeadModel -from .modeling_tf_gpt2 import TFGPT2Model, TFGPT2LMHeadModel -from .modeling_tf_transfo_xl import TFTransfoXLModel, TFTransfoXLLMHeadModel -from .modeling_tf_xlnet import TFXLNetModel, TFXLNetLMHeadModel, TFXLNetForSequenceClassification, TFXLNetForQuestionAnsweringSimple -from .modeling_tf_xlm import TFXLMModel, TFXLMWithLMHeadModel, TFXLMForSequenceClassification, TFXLMForQuestionAnsweringSimple -from .modeling_tf_roberta import TFRobertaModel, TFRobertaForMaskedLM, TFRobertaForSequenceClassification -from .modeling_tf_distilbert import TFDistilBertModel, TFDistilBertForQuestionAnswering, TFDistilBertForMaskedLM, TFDistilBertForSequenceClassification -from .modeling_tf_ctrl import TFCTRLModel, TFCTRLLMHeadModel -from .modeling_tf_t5 import TFT5Model, TFT5WithLMHeadModel +from .modeling_tf_bert import TFBertModel, TFBertForMaskedLM, TFBertForSequenceClassification, TFBertForQuestionAnswering, TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_tf_openai import TFOpenAIGPTModel, TFOpenAIGPTLMHeadModel, TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_tf_gpt2 import TFGPT2Model, TFGPT2LMHeadModel, TF_GPT2_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_tf_transfo_xl import TFTransfoXLModel, TFTransfoXLLMHeadModel, TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_tf_xlnet import TFXLNetModel, TFXLNetLMHeadModel, TFXLNetForSequenceClassification, TFXLNetForQuestionAnsweringSimple, TF_XLNET_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_tf_xlm import TFXLMModel, TFXLMWithLMHeadModel, TFXLMForSequenceClassification, TFXLMForQuestionAnsweringSimple, TF_XLM_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_tf_roberta import TFRobertaModel, TFRobertaForMaskedLM, TFRobertaForSequenceClassification, TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_tf_distilbert import TFDistilBertModel, TFDistilBertForQuestionAnswering, TFDistilBertForMaskedLM, TFDistilBertForSequenceClassification, TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_tf_ctrl import TFCTRLModel, TFCTRLLMHeadModel, TF_CTRL_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_tf_t5 import TFT5Model, TFT5WithLMHeadModel, TF_T5_PRETRAINED_MODEL_ARCHIVE_MAP from .file_utils import add_start_docstrings logger = logging.getLogger(__name__) +TF_ALL_PRETRAINED_MODEL_ARCHIVE_MAP = dict((key, value) + for pretrained_map in [ + TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP, + TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP, + TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP, + TF_GPT2_PRETRAINED_MODEL_ARCHIVE_MAP, + TF_CTRL_PRETRAINED_MODEL_ARCHIVE_MAP, + TF_XLNET_PRETRAINED_MODEL_ARCHIVE_MAP, + TF_XLM_PRETRAINED_MODEL_ARCHIVE_MAP, + TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, + TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, + TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP, + TF_CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP, + TF_T5_PRETRAINED_MODEL_ARCHIVE_MAP, + ] + for key, value, in pretrained_map.items()) + + class TFAutoModel(object): r""" :class:`~transformers.TFAutoModel` is a generic model class From db0a9ee6e0ddcb9d634c3ab0ba3d25501c370d8c Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 16 Dec 2019 14:08:08 +0100 Subject: [PATCH 376/505] adding albert to TF auto models cc @LysandreJik --- transformers/modeling_tf_auto.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/transformers/modeling_tf_auto.py b/transformers/modeling_tf_auto.py index 9c687d9235..3e9b4d120b 100644 --- a/transformers/modeling_tf_auto.py +++ b/transformers/modeling_tf_auto.py @@ -27,6 +27,7 @@ from .modeling_tf_xlm import TFXLMModel, TFXLMWithLMHeadModel, TFXLMForSequenceC from .modeling_tf_roberta import TFRobertaModel, TFRobertaForMaskedLM, TFRobertaForSequenceClassification, TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP from .modeling_tf_distilbert import TFDistilBertModel, TFDistilBertForQuestionAnswering, TFDistilBertForMaskedLM, TFDistilBertForSequenceClassification, TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP from .modeling_tf_ctrl import TFCTRLModel, TFCTRLLMHeadModel, TF_CTRL_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_tf_albert import TFAlbertModel, TFAlbertForMaskedLM, TFAlbertForSequenceClassification, TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP from .modeling_tf_t5 import TFT5Model, TFT5WithLMHeadModel, TF_T5_PRETRAINED_MODEL_ARCHIVE_MAP from .file_utils import add_start_docstrings @@ -46,7 +47,6 @@ TF_ALL_PRETRAINED_MODEL_ARCHIVE_MAP = dict((key, value) TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP, - TF_CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP, TF_T5_PRETRAINED_MODEL_ARCHIVE_MAP, ] for key, value, in pretrained_map.items()) @@ -162,6 +162,8 @@ class TFAutoModel(object): return TFT5Model.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'distilbert' in pretrained_model_name_or_path: return TFDistilBertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'albert' in pretrained_model_name_or_path: + return TFAlbertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return TFRobertaModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'bert' in pretrained_model_name_or_path: @@ -298,6 +300,8 @@ class TFAutoModelWithLMHead(object): return TFT5WithLMHeadModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'distilbert' in pretrained_model_name_or_path: return TFDistilBertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'albert' in pretrained_model_name_or_path: + return TFAlbertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return TFRobertaForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'bert' in pretrained_model_name_or_path: @@ -425,6 +429,8 @@ class TFAutoModelForSequenceClassification(object): """ if 'distilbert' in pretrained_model_name_or_path: return TFDistilBertForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'albert' in pretrained_model_name_or_path: + return TFAlbertForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return TFRobertaForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'bert' in pretrained_model_name_or_path: From 031ad4eb3780437d5232392b16891078b1b32d2c Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 16 Dec 2019 14:20:57 +0100 Subject: [PATCH 377/505] improving JSON error messages (for model card and configurations) --- transformers/configuration_utils.py | 15 +++++++++++---- transformers/model_card.py | 12 ++++++++---- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/transformers/configuration_utils.py b/transformers/configuration_utils.py index 6c9eeea175..f692c9b132 100644 --- a/transformers/configuration_utils.py +++ b/transformers/configuration_utils.py @@ -151,10 +151,14 @@ class PretrainedConfig(object): config_file = pretrained_model_name_or_path else: config_file = hf_bucket_url(pretrained_model_name_or_path, postfix=CONFIG_NAME) - # redirect to the cache, if necessary + try: + # Load from URL or cache if already cached resolved_config_file = cached_path(config_file, cache_dir=cache_dir, force_download=force_download, proxies=proxies, resume_download=resume_download) + # Load config + config = cls.from_json_file(resolved_config_file) + except EnvironmentError: if pretrained_model_name_or_path in cls.pretrained_config_archive_map: msg = "Couldn't reach server at '{}' to download pretrained model configuration file.".format( @@ -168,15 +172,18 @@ class PretrainedConfig(object): config_file, CONFIG_NAME) raise EnvironmentError(msg) + except json.JSONDecodeError: + msg = "Couldn't reach server at '{}' to download configuration file or " \ + "configuration file is not a valid JSON file. " \ + "Please check network or file content here: {}.".format(config_file, resolved_config_file) + raise EnvironmentError(msg) + if resolved_config_file == config_file: logger.info("loading configuration file {}".format(config_file)) else: logger.info("loading configuration file {} from cache at {}".format( config_file, resolved_config_file)) - # Load config - config = cls.from_json_file(resolved_config_file) - if hasattr(config, 'pruned_heads'): config.pruned_heads = dict((int(key), value) for key, value in config.pruned_heads.items()) diff --git a/transformers/model_card.py b/transformers/model_card.py index 6d56089844..3c775ab7fc 100644 --- a/transformers/model_card.py +++ b/transformers/model_card.py @@ -132,7 +132,7 @@ class ModelCard(object): if pretrained_model_name_or_path in ALL_PRETRAINED_CONFIG_ARCHIVE_MAP: # For simplicity we use the same pretrained url than the configuration files but with a different suffix (model_card.json) model_card_file = ALL_PRETRAINED_CONFIG_ARCHIVE_MAP[pretrained_model_name_or_path] - model_card_file.replace(CONFIG_NAME, MODEL_CARD_NAME) + model_card_file = model_card_file.replace(CONFIG_NAME, MODEL_CARD_NAME) elif os.path.isdir(pretrained_model_name_or_path): model_card_file = os.path.join(pretrained_model_name_or_path, MODEL_CARD_NAME) elif os.path.isfile(pretrained_model_name_or_path) or is_remote_url(pretrained_model_name_or_path): @@ -143,13 +143,11 @@ class ModelCard(object): try: resolved_model_card_file = cached_path(model_card_file, cache_dir=cache_dir, force_download=force_download, proxies=proxies, resume_download=resume_download) - if resolved_model_card_file == model_card_file: logger.info("loading model card file {}".format(model_card_file)) else: logger.info("loading model card file {} from cache at {}".format( model_card_file, resolved_model_card_file)) - # Load model card model_card = cls.from_json_file(resolved_model_card_file) @@ -164,9 +162,15 @@ class ModelCard(object): pretrained_model_name_or_path, ', '.join(ALL_PRETRAINED_CONFIG_ARCHIVE_MAP.keys()), model_card_file, MODEL_CARD_NAME)) - logger.warning("Creating an empty model card.") + # We fall back on creating an empty model card + model_card = cls() + except json.JSONDecodeError: + logger.warning("Couldn't reach server at '{}' to download model card file or " + "model card file is not a valid JSON file. " + "Please check network or file content here: {}.".format(model_card_file, resolved_model_card_file)) + logger.warning("Creating an empty model card.") # We fall back on creating an empty model card model_card = cls() From 955d7ecb570b178187075c7c31fcd9be2e3a3428 Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Mon, 16 Dec 2019 14:34:54 +0100 Subject: [PATCH 378/505] Refactored Pipeline with dedicated argument handler. --- transformers/pipelines.py | 210 ++++++++++++++++++++------------------ 1 file changed, 112 insertions(+), 98 deletions(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index 7383222c1f..7e2b30ba3c 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -36,29 +36,40 @@ if is_torch_available(): AutoModelForQuestionAnswering, AutoModelForTokenClassification -class Pipeline(ABC): - def __init__(self, model, tokenizer: PreTrainedTokenizer = None, **kwargs): - self.model = model - self.tokenizer = tokenizer +class ArgumentHandler(ABC): + """ + Base interface for handling varargs for each Pipeline + """ + @abstractmethod + def __call__(self, *args, **kwargs): + raise NotImplementedError() - def save_pretrained(self, save_directory): - if not os.path.isdir(save_directory): - logger.error("Provided path ({}) should be a directory".format(save_directory)) - return - self.model.save_pretrained(save_directory) - self.tokenizer.save_pretrained(save_directory) +class DefaultArgumentHandler(ArgumentHandler): + """ + Default varargs argument parser handling parameters for each Pipeline + """ + def __call__(self, *args, **kwargs): + if 'X' in kwargs: + return kwargs['X'] + elif 'data' in kwargs: + return kwargs['data'] + elif len(args) > 0: + return list(args) + raise ValueError('Unable to infer the format of the provided data (X=, data=, ...)') - def transform(self, *texts, **kwargs): - # Generic compatibility with sklearn and Keras - return self(*texts, **kwargs) - def predict(self, *texts, **kwargs): - # Generic compatibility with sklearn and Keras - return self(*texts, **kwargs) +class _ScikitCompat(ABC): + """ + Interface layer for the Scikit and Keras compatibility. + """ @abstractmethod - def __call__(self, *texts, **kwargs): + def transform(self, X): + raise NotImplementedError() + + @abstractmethod + def predict(self, X): raise NotImplementedError() @@ -133,24 +144,45 @@ class JsonPipelineDataFormat(PipelineDataFormat): if self.is_multi_columns: yield {k: entry[c] for k, c in self.column} else: - yield entry[self.column] + yield entry[self.column[0]] def save(self, data: dict): with open(self.output, 'w') as f: json.dump(data, f) -class FeatureExtractionPipeline(Pipeline): +class Pipeline(_ScikitCompat): + def __init__(self, model, tokenizer: PreTrainedTokenizer = None, args_parser: ArgumentHandler = None, **kwargs): + self.model = model + self.tokenizer = tokenizer + self._args_parser = args_parser or DefaultArgumentHandler() + + def save_pretrained(self, save_directory): + if not os.path.isdir(save_directory): + logger.error("Provided path ({}) should be a directory".format(save_directory)) + return + + self.model.save_pretrained(save_directory) + self.tokenizer.save_pretrained(save_directory) + + def transform(self, X): + return self(X=X) + + def predict(self, X): + return self(X=X) def __call__(self, *texts, **kwargs): - # Generic compatibility with sklearn and Keras - if 'X' in kwargs and not texts: - texts = kwargs.pop('X') + # Parse arguments + inputs = self._args_parser(*texts, **kwargs) + # Encode for forward inputs = self.tokenizer.batch_encode_plus( - texts, add_special_tokens=True, return_tensors='tf' if is_tf_available() else 'pt' + inputs, add_special_tokens=True, return_tensors='tf' if is_tf_available() else 'pt' ) + return self._forward(inputs) + + def _forward(self, inputs): if is_tf_available(): # TODO trace model predictions = self.model(inputs)[0] @@ -159,7 +191,12 @@ class FeatureExtractionPipeline(Pipeline): with torch.no_grad(): predictions = self.model(**inputs)[0] - return predictions.numpy().tolist() + return predictions.numpy() + + +class FeatureExtractionPipeline(Pipeline): + def __call__(self, *args, **kwargs): + return super().__call__(*args, **kwargs).tolist() class TextClassificationPipeline(Pipeline): @@ -170,26 +207,8 @@ class TextClassificationPipeline(Pipeline): raise Exception('Invalid parameter nb_classes. int >= 2 is required (got: {})'.format(nb_classes)) self._nb_classes = nb_classes - def __call__(self, *texts, **kwargs): - # Generic compatibility with sklearn and Keras - if 'X' in kwargs and not texts: - texts = kwargs.pop('X') - - inputs = self.tokenizer.batch_encode_plus( - texts, add_special_tokens=True, return_tensors='tf' if is_tf_available() else 'pt' - ) - - special_tokens_mask = inputs.pop('special_tokens_mask') - - if is_tf_available(): - # TODO trace model - predictions = self.model(**inputs)[0] - else: - import torch - with torch.no_grad(): - predictions = self.model(**inputs)[0] - - return predictions.numpy().tolist() + def __call__(self, *args, **kwargs): + return super().__call__(*args, **kwargs).tolist() class NerPipeline(Pipeline): @@ -198,8 +217,7 @@ class NerPipeline(Pipeline): super().__init__(model, tokenizer) def __call__(self, *texts, **kwargs): - (texts, ), answers = texts, [] - + inputs, answers = self._args_parser(*texts, **kwargs), [] for sentence in texts: # Ugly token to word idx mapping (for now) @@ -241,9 +259,52 @@ class QuestionAnsweringPipeline(Pipeline): Question Answering pipeline involving Tokenization and Inference. """ - @classmethod - def from_config(cls, model, tokenizer: PreTrainedTokenizer, **kwargs): - pass + class QuestionAnsweringArgumentHandler(ArgumentHandler): + + def __call__(self, *args, **kwargs): + # Position args, handling is sensibly the same as X and data, so forwarding to avoid duplicating + if args is not None and len(args) > 1: + kwargs['X'] = args + + # Generic compatibility with sklearn and Keras + # Batched data + if 'X' in kwargs or 'data' in kwargs: + data = kwargs['X'] if 'X' in kwargs else kwargs['data'] + + if not isinstance(data, list): + data = [data] + + for i, item in enumerate(data): + if isinstance(item, dict): + if any(k not in item for k in ['question', 'context']): + raise KeyError('You need to provide a dictionary with keys {question:..., context:...}') + data[i] = QuestionAnsweringPipeline.create_sample(**item) + + elif isinstance(item, SquadExample): + continue + else: + raise ValueError( + '{} argument needs to be of type (list[SquadExample | dict], SquadExample, dict)' + .format('X' if 'X' in kwargs else 'data') + ) + inputs = data + + # Tabular input + elif 'question' in kwargs and 'context' in kwargs: + if isinstance(kwargs['question'], str): + kwargs['question'] = [kwargs['question']] + + if isinstance(kwargs['context'], str): + kwargs['context'] = [kwargs['context']] + + inputs = [QuestionAnsweringPipeline.create_sample(q, c) for q, c in zip(kwargs['question'], kwargs['context'])] + else: + raise ValueError('Unknown arguments {}'.format(kwargs)) + + if not isinstance(inputs, list): + inputs = [inputs] + + return inputs @staticmethod def create_sample(question: Union[str, List[str]], context: Union[str, List[str]]) -> Union[SquadExample, List[SquadExample]]: @@ -254,54 +315,8 @@ class QuestionAnsweringPipeline(Pipeline): else: return SquadExample(None, question, context, None, None, None) - @staticmethod - def handle_args(*inputs, **kwargs) -> List[SquadExample]: - # Position args, handling is sensibly the same as X and data, so forwarding to avoid duplicating - if inputs is not None and len(inputs) > 1: - kwargs['X'] = inputs - - # Generic compatibility with sklearn and Keras - # Batched data - if 'X' in kwargs or 'data' in kwargs: - data = kwargs['X'] if 'X' in kwargs else kwargs['data'] - - if not isinstance(data, list): - data = [data] - - for i, item in enumerate(data): - if isinstance(item, dict): - if any(k not in item for k in ['question', 'context']): - raise KeyError('You need to provide a dictionary with keys {question:..., context:...}') - data[i] = QuestionAnsweringPipeline.create_sample(**item) - - elif isinstance(item, SquadExample): - continue - else: - raise ValueError( - '{} argument needs to be of type (list[SquadExample | dict], SquadExample, dict)' - .format('X' if 'X' in kwargs else 'data') - ) - inputs = data - - # Tabular input - elif 'question' in kwargs and 'context' in kwargs: - if isinstance(kwargs['question'], str): - kwargs['question'] = [kwargs['question']] - - if isinstance(kwargs['context'], str): - kwargs['context'] = [kwargs['context']] - - inputs = [QuestionAnsweringPipeline.create_sample(q, c) for q, c in zip(kwargs['question'], kwargs['context'])] - else: - raise ValueError('Unknown arguments {}'.format(kwargs)) - - if not isinstance(inputs, list): - inputs = [inputs] - - return inputs - def __init__(self, model, tokenizer: Optional[PreTrainedTokenizer]): - super().__init__(model, tokenizer) + super().__init__(model, tokenizer, args_parser=QuestionAnsweringPipeline.QuestionAnsweringArgumentHandler()) def inputs_for_model(self, features: Union[SquadExample, List[SquadExample]]) -> Dict: args = ['input_ids', 'attention_mask'] @@ -332,9 +347,8 @@ class QuestionAnsweringPipeline(Pipeline): if kwargs['max_answer_len'] < 1: raise ValueError('max_answer_len parameter should be >= 1 (got {})'.format(kwargs['max_answer_len'])) - examples = QuestionAnsweringPipeline.handle_args(texts, **kwargs) - # Convert inputs to features + examples = self._args_parser(*texts, **kwargs) features = squad_convert_examples_to_features(examples, self.tokenizer, kwargs['max_seq_len'], kwargs['doc_stride'], kwargs['max_question_len'], False) fw_args = self.inputs_for_model(features) From 1bbdbacd5bc7281dbcebfe4330a464a7ad1a6e72 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 16 Dec 2019 14:38:20 +0100 Subject: [PATCH 379/505] update __init__ and saving --- transformers/__init__.py | 2 +- transformers/model_card.py | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/transformers/__init__.py b/transformers/__init__.py index 0b343bed2b..44447c5495 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -19,7 +19,7 @@ logger = logging.getLogger(__name__) # pylint: disable=invalid-name # Files and general utilities from .file_utils import (TRANSFORMERS_CACHE, PYTORCH_TRANSFORMERS_CACHE, PYTORCH_PRETRAINED_BERT_CACHE, cached_path, add_start_docstrings, add_end_docstrings, - WEIGHTS_NAME, TF2_WEIGHTS_NAME, TF_WEIGHTS_NAME, CONFIG_NAME, + WEIGHTS_NAME, TF2_WEIGHTS_NAME, TF_WEIGHTS_NAME, CONFIG_NAME, MODEL_CARD_NAME, is_tf_available, is_torch_available) from .data import (is_sklearn_available, diff --git a/transformers/model_card.py b/transformers/model_card.py index 3c775ab7fc..baec7e8622 100644 --- a/transformers/model_card.py +++ b/transformers/model_card.py @@ -67,14 +67,14 @@ class ModelCard(object): logger.error("Can't set {} with value {} for {}".format(key, value, self)) raise err - def save_pretrained(self, save_directory): - """ Save a model card object to the directory `save_directory`, so that it - can be re-loaded using the :func:`~transformers.ModelCard.from_pretrained` class method. + def save_pretrained(self, save_directory_or_file): + """ Save a model card object to the directory or file `save_directory_or_file`. """ - assert os.path.isdir(save_directory), "Saving path should be a directory where the model card can be saved" - - # If we save using the predefined names, we can load using `from_pretrained` - output_model_card_file = os.path.join(save_directory, MODEL_CARD_NAME) + if os.path.isdir(save_directory_or_file): + # If we save using the predefined names, we can load using `from_pretrained` + output_model_card_file = os.path.join(save_directory_or_file, MODEL_CARD_NAME) + else: + output_model_card_file = save_directory_or_file self.to_json_file(output_model_card_file) logger.info("Model card saved in {}".format(output_model_card_file)) @@ -139,8 +139,9 @@ class ModelCard(object): model_card_file = pretrained_model_name_or_path else: model_card_file = hf_bucket_url(pretrained_model_name_or_path, postfix=MODEL_CARD_NAME) - # redirect to the cache, if necessary + try: + # Load from URL or cache if already cached resolved_model_card_file = cached_path(model_card_file, cache_dir=cache_dir, force_download=force_download, proxies=proxies, resume_download=resume_download) if resolved_model_card_file == model_card_file: @@ -163,6 +164,7 @@ class ModelCard(object): ', '.join(ALL_PRETRAINED_CONFIG_ARCHIVE_MAP.keys()), model_card_file, MODEL_CARD_NAME)) logger.warning("Creating an empty model card.") + # We fall back on creating an empty model card model_card = cls() @@ -171,6 +173,7 @@ class ModelCard(object): "model card file is not a valid JSON file. " "Please check network or file content here: {}.".format(model_card_file, resolved_model_card_file)) logger.warning("Creating an empty model card.") + # We fall back on creating an empty model card model_card = cls() From 9c391277cc380b1d1eba17fd7b3337c90b35987e Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Mon, 16 Dec 2019 15:19:13 +0100 Subject: [PATCH 380/505] Allow tensors placement on specific device through CLI and pipeline. --- transformers/commands/run.py | 3 +- transformers/pipelines.py | 77 ++++++++++++++++++++++++------------ 2 files changed, 54 insertions(+), 26 deletions(-) diff --git a/transformers/commands/run.py b/transformers/commands/run.py index bcbb87391d..b4951b1bc2 100644 --- a/transformers/commands/run.py +++ b/transformers/commands/run.py @@ -16,7 +16,7 @@ def try_infer_format_from_ext(path: str): def run_command_factory(args): - nlp = pipeline(task=args.task, model=args.model, tokenizer=args.tokenizer) + nlp = pipeline(task=args.task, model=args.model, tokenizer=args.tokenizer, device=args.device) format = try_infer_format_from_ext(args.input) if args.format == 'infer' else args.format reader = PipelineDataFormat.from_str(format, args.output, args.input, args.column) return RunCommand(nlp, reader) @@ -31,6 +31,7 @@ class RunCommand(BaseTransformersCLICommand): @staticmethod def register_subcommand(parser: ArgumentParser): run_parser = parser.add_parser('run', help="Run a pipeline through the CLI") + run_parser.add_argument('--device', type=int, default=-1, help='Indicate the device to run onto, -1 indicates CPU, >= 0 indicates GPU') run_parser.add_argument('--task', choices=SUPPORTED_TASKS.keys(), help='Task to run') run_parser.add_argument('--model', type=str, required=True, help='Name or path to the model to instantiate.') run_parser.add_argument('--tokenizer', type=str, help='Name of the tokenizer to use. (default: same as the model name)') diff --git a/transformers/pipelines.py b/transformers/pipelines.py index 7e2b30ba3c..5b0e81957b 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -18,6 +18,7 @@ import csv import json import os from abc import ABC, abstractmethod +from contextlib import contextmanager from itertools import groupby from typing import Union, Optional, Tuple, List, Dict @@ -152,11 +153,18 @@ class JsonPipelineDataFormat(PipelineDataFormat): class Pipeline(_ScikitCompat): - def __init__(self, model, tokenizer: PreTrainedTokenizer = None, args_parser: ArgumentHandler = None, **kwargs): + def __init__(self, model, tokenizer: PreTrainedTokenizer = None, + args_parser: ArgumentHandler = None, device: int = -1, **kwargs): + self.model = model self.tokenizer = tokenizer + self.device = device self._args_parser = args_parser or DefaultArgumentHandler() + # Special handling + if self.device >= 0 and not is_tf_available(): + self.model = self.model.to('cuda:{}'.format(self.device)) + def save_pretrained(self, save_directory): if not os.path.isdir(save_directory): logger.error("Provided path ({}) should be a directory".format(save_directory)) @@ -176,11 +184,25 @@ class Pipeline(_ScikitCompat): inputs = self._args_parser(*texts, **kwargs) # Encode for forward - inputs = self.tokenizer.batch_encode_plus( - inputs, add_special_tokens=True, return_tensors='tf' if is_tf_available() else 'pt' - ) + with self.device_placement(): + inputs = self.tokenizer.batch_encode_plus( + inputs, add_special_tokens=True, return_tensors='tf' if is_tf_available() else 'pt' + ) - return self._forward(inputs) + return self._forward(inputs) + + @contextmanager + def device_placement(self): + if is_tf_available(): + import tensorflow as tf + with tf.device('/CPU:0' if self.device == -1 else '/device:GPU:{}'.format(self.device)): + yield + else: + import torch + if self.device >= 0: + torch.cuda.set_device(self.device) + + yield def _forward(self, inputs): if is_tf_available(): @@ -225,14 +247,17 @@ class NerPipeline(Pipeline): for i, w in enumerate(words): tokens = self.tokenizer.tokenize(w) token_to_word += [i] * len(tokens) - tokens = self.tokenizer.encode_plus(sentence, return_attention_mask=False, return_tensors='tf' if is_tf_available() else 'pt') - # Forward - if is_torch_available(): - with torch.no_grad(): - entities = self.model(**tokens)[0][0].cpu().numpy() - else: - entities = self.model(tokens)[0][0].numpy() + # Manage correct placement of the tensors + with self.device_placement(): + tokens = self.tokenizer.encode_plus(sentence, return_attention_mask=False, return_tensors='tf' if is_tf_available() else 'pt') + + # Forward + if is_torch_available(): + with torch.no_grad(): + entities = self.model(**tokens)[0][0].cpu().numpy() + else: + entities = self.model(tokens)[0][0].numpy() # Normalize scores answer, token_start = [], 1 @@ -352,18 +377,20 @@ class QuestionAnsweringPipeline(Pipeline): features = squad_convert_examples_to_features(examples, self.tokenizer, kwargs['max_seq_len'], kwargs['doc_stride'], kwargs['max_question_len'], False) fw_args = self.inputs_for_model(features) - if is_tf_available(): - import tensorflow as tf - fw_args = {k: tf.constant(v) for (k, v) in fw_args.items()} - start, end = self.model(fw_args) - start, end = start.numpy(), end.numpy() - else: - import torch - with torch.no_grad(): - # Retrieve the score for the context tokens only (removing question tokens) - fw_args = {k: torch.tensor(v) for (k, v) in fw_args.items()} - start, end = self.model(**fw_args) - start, end = start.cpu().numpy(), end.cpu().numpy() + # Manage tensor allocation on correct device + with self.device_placement(): + if is_tf_available(): + import tensorflow as tf + fw_args = {k: tf.constant(v) for (k, v) in fw_args.items()} + start, end = self.model(fw_args) + start, end = start.numpy(), end.numpy() + else: + import torch + with torch.no_grad(): + # Retrieve the score for the context tokens only (removing question tokens) + fw_args = {k: torch.tensor(v) for (k, v) in fw_args.items()} + start, end = self.model(**fw_args) + start, end = start.cpu().numpy(), end.cpu().numpy() answers = [] for (example, feature, start_, end_) in zip(examples, features, start, end): @@ -374,7 +401,7 @@ class QuestionAnsweringPipeline(Pipeline): # Mask padding and question start_, end_ = start_ * np.abs(np.array(feature.p_mask) - 1), end_ * np.abs(np.array(feature.p_mask) - 1) - # TODO : What happend if not possible + # TODO : What happens if not possible # Mask CLS start_[0] = end_[0] = 0 From bbc707cf394776634bd433c895a9223fe9b256a9 Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Mon, 16 Dec 2019 15:49:09 +0100 Subject: [PATCH 381/505] Fix non-keyworded varargs handling in DefaultArgumentHandler for pipeline. --- transformers/pipelines.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index 5b0e81957b..ee551893b0 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -55,7 +55,12 @@ class DefaultArgumentHandler(ArgumentHandler): return kwargs['X'] elif 'data' in kwargs: return kwargs['data'] - elif len(args) > 0: + elif len(args) == 1: + if isinstance(args[0], list): + return args[0] + else: + return [args[0]] + elif len(args) > 1: return list(args) raise ValueError('Unable to infer the format of the provided data (X=, data=, ...)') @@ -240,7 +245,7 @@ class NerPipeline(Pipeline): def __call__(self, *texts, **kwargs): inputs, answers = self._args_parser(*texts, **kwargs), [] - for sentence in texts: + for sentence in inputs: # Ugly token to word idx mapping (for now) token_to_word, words = [], sentence.split(' ') From 46ccbb42fc89876461995e0ba553d72cffa700ce Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Mon, 16 Dec 2019 15:49:41 +0100 Subject: [PATCH 382/505] Make CLI run command use integer mapping for device argument. --- transformers/commands/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/commands/run.py b/transformers/commands/run.py index b4951b1bc2..8c203699a8 100644 --- a/transformers/commands/run.py +++ b/transformers/commands/run.py @@ -31,7 +31,7 @@ class RunCommand(BaseTransformersCLICommand): @staticmethod def register_subcommand(parser: ArgumentParser): run_parser = parser.add_parser('run', help="Run a pipeline through the CLI") - run_parser.add_argument('--device', type=int, default=-1, help='Indicate the device to run onto, -1 indicates CPU, >= 0 indicates GPU') + run_parser.add_argument('--device', type=int, default=-1, help='Indicate the device to run onto, -1 indicates CPU, >= 0 indicates GPU (default: -1)') run_parser.add_argument('--task', choices=SUPPORTED_TASKS.keys(), help='Task to run') run_parser.add_argument('--model', type=str, required=True, help='Name or path to the model to instantiate.') run_parser.add_argument('--tokenizer', type=str, help='Name of the tokenizer to use. (default: same as the model name)') From 43a4e1bbe4091b11d7926b93250cc87fa75bd545 Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Mon, 16 Dec 2019 16:00:41 +0100 Subject: [PATCH 383/505] Adressing issue in varargs handling for question answering. --- transformers/pipelines.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index ee551893b0..2a8f26b03e 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -293,8 +293,11 @@ class QuestionAnsweringPipeline(Pipeline): def __call__(self, *args, **kwargs): # Position args, handling is sensibly the same as X and data, so forwarding to avoid duplicating - if args is not None and len(args) > 1: - kwargs['X'] = args + if args is not None and len(args) > 0: + if len(args) == 1: + kwargs['X'] = args[0] + else: + kwargs['X'] = list(args) # Generic compatibility with sklearn and Keras # Batched data From 71b47505175111dd391a5b9de9514fbe50558bf0 Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Mon, 16 Dec 2019 16:37:27 +0100 Subject: [PATCH 384/505] examples: add support for XLM-RoBERTa to run_ner script --- examples/run_ner.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/run_ner.py b/examples/run_ner.py index 1ab1236d94..6426a6d1db 100644 --- a/examples/run_ner.py +++ b/examples/run_ner.py @@ -38,11 +38,13 @@ from transformers import WEIGHTS_NAME, BertConfig, BertForTokenClassification, B from transformers import RobertaConfig, RobertaForTokenClassification, RobertaTokenizer from transformers import DistilBertConfig, DistilBertForTokenClassification, DistilBertTokenizer from transformers import CamembertConfig, CamembertForTokenClassification, CamembertTokenizer +from transformers import XLMRobertaConfig, XLMRobertaForTokenClassification, XLMRobertaTokenizer logger = logging.getLogger(__name__) ALL_MODELS = sum( - (tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, RobertaConfig, DistilBertConfig)), + (tuple(conf.pretrained_config_archive_map.keys()) for conf in (BertConfig, RobertaConfig, DistilBertConfig, + CamembertConfig, XLMRobertaConfig)), ()) MODEL_CLASSES = { @@ -50,6 +52,7 @@ MODEL_CLASSES = { "roberta": (RobertaConfig, RobertaForTokenClassification, RobertaTokenizer), "distilbert": (DistilBertConfig, DistilBertForTokenClassification, DistilBertTokenizer), "camembert": (CamembertConfig, CamembertForTokenClassification, CamembertTokenizer), + "xlmroberta": (XLMRobertaConfig, XLMRobertaForTokenClassification, XLMRobertaTokenizer), } From a096e2a88beae06c3341bc502b122d77be72571b Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Mon, 16 Dec 2019 16:38:02 +0100 Subject: [PATCH 385/505] WIP serving through HTTP internally using pipelines. --- transformers/commands/serving.py | 67 ++++++++++---------------------- 1 file changed, 21 insertions(+), 46 deletions(-) diff --git a/transformers/commands/serving.py b/transformers/commands/serving.py index 0b47246ead..a35dff0ebe 100644 --- a/transformers/commands/serving.py +++ b/transformers/commands/serving.py @@ -1,15 +1,15 @@ from argparse import ArgumentParser, Namespace from typing import List, Optional, Union, Any -import torch from fastapi import FastAPI, HTTPException, Body from logging import getLogger from pydantic import BaseModel from uvicorn import run -from transformers import AutoModel, AutoTokenizer, AutoConfig +from transformers import Pipeline from transformers.commands import BaseTransformersCLICommand +from transformers.pipelines import SUPPORTED_TASKS, pipeline def serve_command_factory(args: Namespace): @@ -17,7 +17,8 @@ def serve_command_factory(args: Namespace): Factory function used to instantiate serving server from provided command line arguments. :return: ServeCommand """ - return ServeCommand(args.host, args.port, args.model, args.graphql) + nlp = pipeline(args.task, args.model) + return ServeCommand(nlp, args.host, args.port, args.model, args.graphql) class ServeResult(BaseModel): @@ -53,8 +54,6 @@ class ServeForwardResult(ServeResult): """ Forward result model """ - tokens: List[str] - tokens_ids: List[int] output: Any @@ -68,19 +67,18 @@ class ServeCommand(BaseTransformersCLICommand): :return: """ serve_parser = parser.add_parser('serve', help='CLI tool to run inference requests through REST and GraphQL endpoints.') + serve_parser.add_argument('--task', type=str, choices=SUPPORTED_TASKS.keys(), help='The task to run the pipeline on') + serve_parser.add_argument('--device', type=int, default=-1, help='Indicate the device to run onto, -1 indicates CPU, >= 0 indicates GPU (default: -1)') serve_parser.add_argument('--host', type=str, default='localhost', help='Interface the server will listen on.') serve_parser.add_argument('--port', type=int, default=8888, help='Port the serving will listen to.') serve_parser.add_argument('--model', type=str, required=True, help='Model\'s name or path to stored model to infer from.') serve_parser.add_argument('--graphql', action='store_true', default=False, help='Enable GraphQL endpoints.') serve_parser.set_defaults(func=serve_command_factory) - def __init__(self, host: str, port: int, model: str, graphql: bool): + def __init__(self, pipeline: Pipeline, host: str, port: int, model: str, graphql: bool): self._logger = getLogger('transformers-cli/serving') - self._logger.info('Loading model {}'.format(model)) - self._model_name = model - self._model = AutoModel.from_pretrained(model) - self._tokenizer = AutoTokenizer.from_pretrained(model) + self._pipeline = pipeline self._logger.info('Serving model over {}:{}'.format(host, port)) self._host = host @@ -97,7 +95,7 @@ class ServeCommand(BaseTransformersCLICommand): run(self._app, host=self._host, port=self._port) def model_info(self): - return ServeModelInfoResult(model=self._model_name, infos=vars(self._model.config)) + return ServeModelInfoResult(model='', infos=vars(self._pipeline.model.config)) def tokenize(self, text_input: str = Body(None, embed=True), return_ids: bool = Body(False, embed=True)): """ @@ -106,16 +104,16 @@ class ServeCommand(BaseTransformersCLICommand): - **return_ids**: Boolean flags indicating if the tokens have to be converted to their integer mapping. """ try: - tokens_txt = self._tokenizer.tokenize(text_input) + tokens_txt = self._pipeline.tokenizer.tokenize(text_input) if return_ids: - tokens_ids = self._tokenizer.convert_tokens_to_ids(tokens_txt) - return ServeTokenizeResult(model=self._model_name, tokens=tokens_txt, tokens_ids=tokens_ids) + tokens_ids = self._pipeline.tokenizer.convert_tokens_to_ids(tokens_txt) + return ServeTokenizeResult(model='', tokens=tokens_txt, tokens_ids=tokens_ids) else: - return ServeTokenizeResult(model=self._model_name, tokens=tokens_txt) + return ServeTokenizeResult(model='', tokens=tokens_txt) except Exception as e: - raise HTTPException(status_code=500, detail={"model": self._model_name, "error": str(e)}) + raise HTTPException(status_code=500, detail={"model": '', "error": str(e)}) def detokenize(self, tokens_ids: List[int] = Body(None, embed=True), skip_special_tokens: bool = Body(False, embed=True), @@ -127,14 +125,12 @@ class ServeCommand(BaseTransformersCLICommand): - **cleanup_tokenization_spaces**: Flag indicating to remove all leading/trailing spaces and intermediate ones. """ try: - decoded_str = self._tokenizer.decode(tokens_ids, skip_special_tokens, cleanup_tokenization_spaces) - return ServeDeTokenizeResult(model=self._model_name, text=decoded_str) + decoded_str = self._pipeline.tokenizer.decode(tokens_ids, skip_special_tokens, cleanup_tokenization_spaces) + return ServeDeTokenizeResult(model='', text=decoded_str) except Exception as e: - raise HTTPException(status_code=500, detail={"model": self._model_name, "error": str(e)}) + raise HTTPException(status_code=500, detail={"model": '', "error": str(e)}) - def forward(self, inputs: Union[str, List[str], List[int]] = Body(None, embed=True), - attention_mask: Optional[List[int]] = Body(None, embed=True), - tokens_type_ids: Optional[List[int]] = Body(None, embed=True)): + def forward(self, inputs: Union[str, dict, List[str], List[int], List[dict]] = Body(None, embed=True)): """ **inputs**: **attention_mask**: @@ -143,34 +139,13 @@ class ServeCommand(BaseTransformersCLICommand): # Check we don't have empty string if len(inputs) == 0: - return ServeForwardResult(model=self._model_name, output=[], attention=[]) - - if isinstance(inputs, str): - inputs_tokens = self._tokenizer.tokenize(inputs) - inputs_ids = self._tokenizer.convert_tokens_to_ids(inputs_tokens) - - elif isinstance(inputs, List): - if isinstance(inputs[0], str): - inputs_tokens = inputs - inputs_ids = self._tokenizer.convert_tokens_to_ids(inputs_tokens) - elif isinstance(inputs[0], int): - inputs_tokens = [] - inputs_ids = inputs - else: - error_msg = "inputs should be string, [str] of [int] (got {})".format(type(inputs[0])) - raise HTTPException(423, detail={"error": error_msg}) - else: - error_msg = "inputs should be string, [str] of [int] (got {})".format(type(inputs)) - raise HTTPException(423, detail={"error": error_msg}) + return ServeForwardResult(model='', output=[], attention=[]) try: # Forward through the model - t_input_ids = torch.tensor(inputs_ids).unsqueeze(0) - output = self._model(t_input_ids, attention_mask, tokens_type_ids) - + output = self._pipeline(inputs) return ServeForwardResult( - model=self._model_name, tokens=inputs_tokens, - tokens_ids=inputs_ids, output=output[0].tolist() + model='', output=output ) except Exception as e: raise HTTPException(500, {"error": str(e)}) From d3549b66af6f225cace48f8462ba715508f51b0d Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Mon, 16 Dec 2019 16:38:39 +0100 Subject: [PATCH 386/505] module: add support for XLM-RoBERTa (__init__) --- transformers/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/transformers/__init__.py b/transformers/__init__.py index 740d2440c2..910ba91457 100644 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -49,6 +49,7 @@ from .tokenization_distilbert import DistilBertTokenizer from .tokenization_albert import AlbertTokenizer from .tokenization_camembert import CamembertTokenizer from .tokenization_t5 import T5Tokenizer +from .tokenization_xlm_roberta import XLMRobertaTokenizer # Configurations from .configuration_utils import PretrainedConfig @@ -65,6 +66,7 @@ from .configuration_distilbert import DistilBertConfig, DISTILBERT_PRETRAINED_CO from .configuration_albert import AlbertConfig, ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_camembert import CamembertConfig, CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_t5 import T5Config, T5_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_xlm_roberta import XLMRobertaConfig, XLM_ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP # Modeling if is_torch_available(): @@ -119,6 +121,9 @@ if is_torch_available(): AlbertForQuestionAnswering, load_tf_weights_in_albert, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP) + from .modeling_xlm_roberta import (XLMRobertaForMaskedLM, XLMRobertaModel, XLMRobertaForMultipleChoice, + XLMRobertaForSequenceClassification, XLMRobertaForTokenClassification) + # Optimization from .optimization import (AdamW, get_constant_schedule, get_constant_schedule_with_warmup, get_cosine_schedule_with_warmup, get_cosine_with_hard_restarts_schedule_with_warmup, get_linear_schedule_with_warmup) From 9ed09cb4a31518b13f2c58c057e43e029c32611a Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Mon, 16 Dec 2019 16:46:58 +0100 Subject: [PATCH 387/505] converter: add conversion script for original XLM-RoBERTa weights to Transformers-compatible weights --- ..._original_pytorch_checkpoint_to_pytorch.py | 184 ++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100644 transformers/convert_xlm_roberta_original_pytorch_checkpoint_to_pytorch.py diff --git a/transformers/convert_xlm_roberta_original_pytorch_checkpoint_to_pytorch.py b/transformers/convert_xlm_roberta_original_pytorch_checkpoint_to_pytorch.py new file mode 100644 index 0000000000..888adf4819 --- /dev/null +++ b/transformers/convert_xlm_roberta_original_pytorch_checkpoint_to_pytorch.py @@ -0,0 +1,184 @@ +# coding=utf-8 +# Copyright 2018 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Convert RoBERTa checkpoint.""" + +from __future__ import absolute_import, division, print_function + +import argparse +import logging +import numpy as np +import torch +import pathlib + +from fairseq.models.roberta import RobertaModel as FairseqRobertaModel +from fairseq.modules import TransformerSentenceEncoderLayer +from transformers.modeling_bert import (BertConfig, BertEncoder, + BertIntermediate, BertLayer, + BertModel, BertOutput, + BertSelfAttention, + BertSelfOutput) +from transformers.modeling_roberta import (RobertaEmbeddings, + RobertaForMaskedLM, + RobertaForSequenceClassification, + RobertaModel) + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +SAMPLE_TEXT = 'Hello world! cécé herlolip' + + +def convert_roberta_checkpoint_to_pytorch(roberta_checkpoint_path, pytorch_dump_folder_path, classification_head): + """ + Copy/paste/tweak roberta's weights to our BERT structure. + """ + roberta = FairseqRobertaModel.from_pretrained(roberta_checkpoint_path, bpe = 'sentencepiece') + roberta.eval() # disable dropout + config = BertConfig( + vocab_size_or_config_json_file=250004, + hidden_size=roberta.args.encoder_embed_dim, + num_hidden_layers=roberta.args.encoder_layers, + num_attention_heads=roberta.args.encoder_attention_heads, + intermediate_size=roberta.args.encoder_ffn_embed_dim, + max_position_embeddings=514, + type_vocab_size=1, + layer_norm_eps=1e-5, # PyTorch default used in fairseq + ) + if classification_head: + config.num_labels = roberta.args.num_classes + print("Our BERT config:", config) + + model = RobertaForSequenceClassification(config) if classification_head else RobertaForMaskedLM(config) + model.eval() + + # Now let's copy all the weights. + # Embeddings + roberta_sent_encoder = roberta.model.decoder.sentence_encoder + model.roberta.embeddings.word_embeddings.weight = roberta_sent_encoder.embed_tokens.weight + model.roberta.embeddings.position_embeddings.weight = roberta_sent_encoder.embed_positions.weight + model.roberta.embeddings.token_type_embeddings.weight.data = torch.zeros_like(model.roberta.embeddings.token_type_embeddings.weight) # just zero them out b/c RoBERTa doesn't use them. + model.roberta.embeddings.LayerNorm.weight = roberta_sent_encoder.emb_layer_norm.weight + model.roberta.embeddings.LayerNorm.bias = roberta_sent_encoder.emb_layer_norm.bias + + for i in range(config.num_hidden_layers): + # Encoder: start of layer + layer: BertLayer = model.roberta.encoder.layer[i] + roberta_layer: TransformerSentenceEncoderLayer = roberta_sent_encoder.layers[i] + + ### self attention + self_attn: BertSelfAttention = layer.attention.self + assert( + roberta_layer.self_attn.k_proj.weight.data.shape == \ + roberta_layer.self_attn.q_proj.weight.data.shape == \ + roberta_layer.self_attn.v_proj.weight.data.shape == \ + torch.Size((config.hidden_size, config.hidden_size)) + ) + + self_attn.query.weight.data = roberta_layer.self_attn.q_proj.weight + self_attn.query.bias.data = roberta_layer.self_attn.q_proj.bias + self_attn.key.weight.data = roberta_layer.self_attn.k_proj.weight + self_attn.key.bias.data = roberta_layer.self_attn.k_proj.bias + self_attn.value.weight.data = roberta_layer.self_attn.v_proj.weight + self_attn.value.bias.data = roberta_layer.self_attn.v_proj.bias + + ### self-attention output + self_output: BertSelfOutput = layer.attention.output + assert( + self_output.dense.weight.shape == roberta_layer.self_attn.out_proj.weight.shape + ) + self_output.dense.weight = roberta_layer.self_attn.out_proj.weight + self_output.dense.bias = roberta_layer.self_attn.out_proj.bias + self_output.LayerNorm.weight = roberta_layer.self_attn_layer_norm.weight + self_output.LayerNorm.bias = roberta_layer.self_attn_layer_norm.bias + + ### intermediate + intermediate: BertIntermediate = layer.intermediate + assert( + intermediate.dense.weight.shape == roberta_layer.fc1.weight.shape + ) + intermediate.dense.weight = roberta_layer.fc1.weight + intermediate.dense.bias = roberta_layer.fc1.bias + + ### output + bert_output: BertOutput = layer.output + assert( + bert_output.dense.weight.shape == roberta_layer.fc2.weight.shape + ) + bert_output.dense.weight = roberta_layer.fc2.weight + bert_output.dense.bias = roberta_layer.fc2.bias + bert_output.LayerNorm.weight = roberta_layer.final_layer_norm.weight + bert_output.LayerNorm.bias = roberta_layer.final_layer_norm.bias + #### end of layer + + if classification_head: + model.classifier.dense.weight = roberta.model.classification_heads['mnli'].dense.weight + model.classifier.dense.bias = roberta.model.classification_heads['mnli'].dense.bias + model.classifier.out_proj.weight = roberta.model.classification_heads['mnli'].out_proj.weight + model.classifier.out_proj.bias = roberta.model.classification_heads['mnli'].out_proj.bias + else: + # LM Head + model.lm_head.dense.weight = roberta.model.decoder.lm_head.dense.weight + model.lm_head.dense.bias = roberta.model.decoder.lm_head.dense.bias + model.lm_head.layer_norm.weight = roberta.model.decoder.lm_head.layer_norm.weight + model.lm_head.layer_norm.bias = roberta.model.decoder.lm_head.layer_norm.bias + model.lm_head.decoder.weight = roberta.model.decoder.lm_head.weight + model.lm_head.bias = roberta.model.decoder.lm_head.bias + + # Let's check that we get the same results. + input_ids: torch.Tensor = roberta.encode(SAMPLE_TEXT).unsqueeze(0) # batch of size 1 + + our_output = model(input_ids)[0] + if classification_head: + their_output = roberta.model.classification_heads['mnli'](roberta.extract_features(input_ids)) + else: + their_output = roberta.model(input_ids)[0] + print(our_output.shape, their_output.shape) + max_absolute_diff = torch.max(torch.abs(our_output - their_output)).item() + print(f"max_absolute_diff = {max_absolute_diff}") # ~ 1e-7 + success = torch.allclose(our_output, their_output, atol=1e-3) + print( + "Do both models output the same tensors?", + "🔥" if success else "💩" + ) + if not success: + raise Exception("Something went wRoNg") + + pathlib.Path(pytorch_dump_folder_path).mkdir(parents=True, exist_ok=True) + print(f"Saving model to {pytorch_dump_folder_path}") + model.save_pretrained(pytorch_dump_folder_path) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + ## Required parameters + parser.add_argument("--roberta_checkpoint_path", + default = None, + type = str, + required = True, + help = "Path the official PyTorch dump.") + parser.add_argument("--pytorch_dump_folder_path", + default = None, + type = str, + required = True, + help = "Path to the output PyTorch model.") + parser.add_argument("--classification_head", + action = "store_true", + help = "Whether to convert a final classification head.") + args = parser.parse_args() + convert_roberta_checkpoint_to_pytorch( + args.roberta_checkpoint_path, + args.pytorch_dump_folder_path, + args.classification_head + ) From a648ff738c88a41cfae4f915a4391c7d66261b64 Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Mon, 16 Dec 2019 16:47:39 +0100 Subject: [PATCH 388/505] configuration: add support for XLM-RoBERTa model --- transformers/configuration_xlm_roberta.py | 33 +++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 transformers/configuration_xlm_roberta.py diff --git a/transformers/configuration_xlm_roberta.py b/transformers/configuration_xlm_roberta.py new file mode 100644 index 0000000000..1633cc18aa --- /dev/null +++ b/transformers/configuration_xlm_roberta.py @@ -0,0 +1,33 @@ +# coding=utf-8 +# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" XLM-RoBERTa configuration """ + +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import logging + +from .configuration_roberta import RobertaConfig + +logger = logging.getLogger(__name__) + +XLM_ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP = { + 'xlm-roberta-base': "https://schweter.eu/cloud/transformers/xlm-roberta-large-config.json", +} + + +class XLMRobertaConfig(RobertaConfig): + pretrained_config_archive_map = XLM_ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP From 69f4f058fa5ecc6fea8c65ae59694442bba795e6 Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Mon, 16 Dec 2019 17:00:12 +0100 Subject: [PATCH 389/505] model: add support for new XLM-RoBERTa model --- transformers/modeling_xlm_roberta.py | 293 +++++++++++++++++++++++++++ 1 file changed, 293 insertions(+) create mode 100644 transformers/modeling_xlm_roberta.py diff --git a/transformers/modeling_xlm_roberta.py b/transformers/modeling_xlm_roberta.py new file mode 100644 index 0000000000..8402be4b5c --- /dev/null +++ b/transformers/modeling_xlm_roberta.py @@ -0,0 +1,293 @@ +# coding=utf-8 +# Copyright 2019 Facebook AI Research and the HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch XLM-RoBERTa model. """ + +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import logging + +from .modeling_roberta import RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification, RobertaForMultipleChoice, RobertaForTokenClassification +from .configuration_xlm_roberta import XLMRobertaConfig +from .file_utils import add_start_docstrings + +logger = logging.getLogger(__name__) + +XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP = { + 'xlm-roberta-large': "https://schweter.eu/cloud/transformers/xlm-roberta-large-pytorch_model.bin", +} + + +XLM_ROBERTA_START_DOCSTRING = r""" The XLM-RoBERTa model was proposed in + `Unsupervised Cross-lingual Representation Learning at Scale`_ + by Alexis Conneau, Kartikay Khandelwal, Naman Goyal, Vishrav Chaudhary, Guillaume Wenzek, Francisco Guzmán, Edouard Grave, Myle Ott, Luke Zettlemoyer and Veselin Stoyanov. It is based on Facebook's RoBERTa model released in 2019. + + It is a large multi-lingual language model, trained on 2.5TB of filtered CommonCrawl data. + + This implementation is the same as RoBERTa. + + This model is a PyTorch `torch.nn.Module`_ sub-class. Use it as a regular PyTorch Module and + refer to the PyTorch documentation for all matter related to general usage and behavior. + + .. _`Unsupervised Cross-lingual Representation Learning at Scale`: + https://arxiv.org/abs/1911.02116 + + .. _`torch.nn.Module`: + https://pytorch.org/docs/stable/nn.html#module + + Parameters: + config (:class:`~transformers.XLMRobertaConfig`): Model configuration class with all the parameters of the + model. Initializing with a config file does not load the weights associated with the model, only the configuration. + Check out the :meth:`~transformers.PreTrainedModel.from_pretrained` method to load the model weights. +""" + +XLM_ROBERTA_INPUTS_DOCSTRING = r""" + Inputs: + **input_ids**: ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of input sequence tokens in the vocabulary. + To match pre-training, XLM-RoBERTa input sequence should be formatted with and tokens as follows: + + (a) For sequence pairs: + + ``tokens: Is this Jacksonville ? No it is not . `` + + (b) For single sequences: + + ``tokens: the dog is hairy . `` + + Fully encoded sequences or sequence pairs can be obtained using the XLMRobertaTokenizer.encode function with + the ``add_special_tokens`` parameter set to ``True``. + + XLM-RoBERTa is a model with absolute position embeddings so it's usually advised to pad the inputs on + the right rather than the left. + + See :func:`transformers.PreTrainedTokenizer.encode` and + :func:`transformers.PreTrainedTokenizer.convert_tokens_to_ids` for details. + **attention_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length)``: + Mask to avoid performing attention on padding token indices. + Mask values selected in ``[0, 1]``: + ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens. + **token_type_ids**: (`optional` need to be trained) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Optional segment token indices to indicate first and second portions of the inputs. + This embedding matrice is not trained (not pretrained during XLM-RoBERTa pretraining), you will have to train it + during finetuning. + Indices are selected in ``[0, 1]``: ``0`` corresponds to a `sentence A` token, ``1`` + corresponds to a `sentence B` token + (see `BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding`_ for more details). + **position_ids**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Indices of positions of each input sequence tokens in the position embeddings. + Selected in the range ``[0, config.max_position_embeddings - 1[``. + **head_mask**: (`optional`) ``torch.FloatTensor`` of shape ``(num_heads,)`` or ``(num_layers, num_heads)``: + Mask to nullify selected heads of the self-attention modules. + Mask values selected in ``[0, 1]``: + ``1`` indicates the head is **not masked**, ``0`` indicates the head is **masked**. + **inputs_embeds**: (`optional`) ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, embedding_dim)``: + Optionally, instead of passing ``input_ids`` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. +""" + +@add_start_docstrings("The bare XLM-RoBERTa Model transformer outputting raw hidden-states without any specific head on top.", + XLM_ROBERTA_START_DOCSTRING, XLM_ROBERTA_INPUTS_DOCSTRING) +class XLMRobertaModel(RobertaModel): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **last_hidden_state**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, hidden_size)`` + Sequence of hidden-states at the output of the last layer of the model. + **pooler_output**: ``torch.FloatTensor`` of shape ``(batch_size, hidden_size)`` + Last layer hidden-state of the first token of the sequence (classification token) + further processed by a Linear layer and a Tanh activation function. The Linear + layer weights are trained from the next sentence prediction (classification) + eo match pre-training, XLM-RoBERTa input sequence should be formatted with [CLS] and [SEP] tokens as follows: + + (a) For sequence pairs: + + ``tokens: [CLS] is this jack ##son ##ville ? [SEP] [SEP] no it is not . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1`` + + (b) For single sequences: + + ``tokens: [CLS] the dog is hairy . [SEP]`` + + ``token_type_ids: 0 0 0 0 0 0 0`` + + objective during Bert pretraining. This output is usually *not* a good summary + of the semantic content of the input, you're often better with averaging or pooling + the sequence of hidden-states for the whole input sequence. + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = XLMRobertaTokenizer.from_pretrained('xlm-roberta-large') + model = XLMRobertaModel.from_pretrained('xlm-roberta-large') + input_ids = torch.tensor(tokenizer.encode("Schloß Nymphenburg ist sehr schön .")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids) + last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple + + """ + config_class = XLMRobertaConfig + pretrained_model_archive_map = XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP + + +@add_start_docstrings("""XLM-RoBERTa Model with a `language modeling` head on top. """, + XLM_ROBERTA_START_DOCSTRING, XLM_ROBERTA_INPUTS_DOCSTRING) +class XLMRobertaForMaskedLM(RobertaForMaskedLM): + r""" + **masked_lm_labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the masked language modeling loss. + Indices should be in ``[-1, 0, ..., config.vocab_size]`` (see ``input_ids`` docstring) + Tokens with indices set to ``-1`` are ignored (masked), the loss is only computed for the tokens with labels + in ``[0, ..., config.vocab_size]`` + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``masked_lm_labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Masked language modeling loss. + **prediction_scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.vocab_size)`` + Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = XLMRobertaTokenizer.from_pretrained('xlm-roberta-large') + model = XLMRobertaForMaskedLM.from_pretrained('xlm-roberta-large') + input_ids = torch.tensor(tokenizer.encode("Schloß Nymphenburg ist sehr schön .")).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, masked_lm_labels=input_ids) + loss, prediction_scores = outputs[:2] + + """ + config_class = XLMRobertaConfig + pretrained_model_archive_map = XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP + + +@add_start_docstrings("""XLM-RoBERTa Model transformer with a sequence classification/regression head on top (a linear layer + on top of the pooled output) e.g. for GLUE tasks. """, + XLM_ROBERTA_START_DOCSTRING, XLM_ROBERTA_INPUTS_DOCSTRING) +class XLMRobertaForSequenceClassification(RobertaForSequenceClassification): + r""" + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size,)``: + Labels for computing the sequence classification/regression loss. + Indices should be in ``[0, ..., config.num_labels]``. + If ``config.num_labels == 1`` a regression loss is computed (Mean-Square loss), + If ``config.num_labels > 1`` a classification loss is computed (Cross-Entropy). + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification (or regression if config.num_labels==1) loss. + **logits**: ``torch.FloatTensor`` of shape ``(batch_size, config.num_labels)`` + Classification (or regression if config.num_labels==1) scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = XLMRobertaTokenizer.from_pretrained('xlm-roberta-large') + model = XLMRobertaForSequenceClassification.from_pretrained('xlm-roberta-large') + input_ids = torch.tensor(tokenizer.encode("Schloß Nymphenburg ist sehr schön .")).unsqueeze(0) # Batch size 1 + labels = torch.tensor([1]).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, logits = outputs[:2] + + """ + config_class = XLMRobertaConfig + pretrained_model_archive_map = XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP + + +@add_start_docstrings("""XLM-RoBERTa Model with a multiple choice classification head on top (a linear layer on top of + the pooled output and a softmax) e.g. for RocStories/SWAG tasks. """, + XLM_ROBERTA_START_DOCSTRING, XLM_ROBERTA_INPUTS_DOCSTRING) +class XLMRobertaForMultipleChoice(RobertaForMultipleChoice): + r""" + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification loss. + **classification_scores**: ``torch.FloatTensor`` of shape ``(batch_size, num_choices)`` where `num_choices` is the size of the second dimension + of the input tensors. (see `input_ids` above). + Classification scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = XLMRobertaTokenizer.from_pretrained('xlm-roberta-large') + model = XLMRobertaForMultipleChoice.from_pretrained('xlm-roberta-large') + choices = ["Schloß Nymphenburg ist sehr schön .", "Der Schloßkanal auch !"] + input_ids = torch.tensor([tokenizer.encode(s, add_special_tokens=True) for s in choices]).unsqueeze(0) # Batch size 1, 2 choices + labels = torch.tensor(1).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, classification_scores = outputs[:2] + + """ + config_class = XLMRobertaConfig + pretrained_model_archive_map = XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP + + +@add_start_docstrings("""XLM-RoBERTa Model with a token classification head on top (a linear layer on top of + the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. """, + XLM_ROBERTA_START_DOCSTRING, XLM_ROBERTA_INPUTS_DOCSTRING) +class XLMRobertaForTokenClassification(RobertaForTokenClassification): + r""" + **labels**: (`optional`) ``torch.LongTensor`` of shape ``(batch_size, sequence_length)``: + Labels for computing the token classification loss. + Indices should be in ``[0, ..., config.num_labels - 1]``. + + Outputs: `Tuple` comprising various elements depending on the configuration (config) and inputs: + **loss**: (`optional`, returned when ``labels`` is provided) ``torch.FloatTensor`` of shape ``(1,)``: + Classification loss. + **scores**: ``torch.FloatTensor`` of shape ``(batch_size, sequence_length, config.num_labels)`` + Classification scores (before SoftMax). + **hidden_states**: (`optional`, returned when ``config.output_hidden_states=True``) + list of ``torch.FloatTensor`` (one for the output of each layer + the output of the embeddings) + of shape ``(batch_size, sequence_length, hidden_size)``: + Hidden-states of the model at the output of each layer plus the initial embedding outputs. + **attentions**: (`optional`, returned when ``config.output_attentions=True``) + list of ``torch.FloatTensor`` (one for each layer) of shape ``(batch_size, num_heads, sequence_length, sequence_length)``: + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. + + Examples:: + + tokenizer = XLMRobertaTokenizer.from_pretrained('xlm-roberta-large') + model = XLMRobertaForTokenClassification.from_pretrained('xlm-roberta-large') + input_ids = torch.tensor(tokenizer.encode("Schloß Nymphenburg ist sehr schön .", add_special_tokens=True)).unsqueeze(0) # Batch size 1 + labels = torch.tensor([1] * input_ids.size(1)).unsqueeze(0) # Batch size 1 + outputs = model(input_ids, labels=labels) + loss, scores = outputs[:2] + + """ + config_class = XLMRobertaConfig + pretrained_model_archive_map = XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP From 59a1aefb1ca51b183bffa2d355bc2a22a7c51274 Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Mon, 16 Dec 2019 17:00:55 +0100 Subject: [PATCH 390/505] tokenization: add support for new XLM-RoBERTa model. Add wrapper around fairseq tokenization logic --- transformers/tokenization_xlm_roberta.py | 154 +++++++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 transformers/tokenization_xlm_roberta.py diff --git a/transformers/tokenization_xlm_roberta.py b/transformers/tokenization_xlm_roberta.py new file mode 100644 index 0000000000..0f95397606 --- /dev/null +++ b/transformers/tokenization_xlm_roberta.py @@ -0,0 +1,154 @@ +# coding=utf-8 +# Copyright 2018 Google AI, Google Brain and Carnegie Mellon University Authors and the HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License +""" Tokenization classes for XLM-RoBERTa model.""" +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import logging +import os +from shutil import copyfile + +import sentencepiece as spm +from transformers.tokenization_utils import PreTrainedTokenizer + +logger = logging.getLogger(__name__) + +VOCAB_FILES_NAMES = {'vocab_file': 'sentencepiece.bpe.model'} + +PRETRAINED_VOCAB_FILES_MAP = { + 'vocab_file': + { + 'xlm-roberta-large': "https://schweter.eu/cloud/transformers/xlm-roberta-large-sentencepiece.bpe.model", + } +} + +PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { + 'xlm-roberta-large': None, +} + +class XLMRobertaTokenizer(PreTrainedTokenizer): + """ + Adapted from RobertaTokenizer and XLNetTokenizer + SentencePiece based tokenizer. Peculiarities: + + - requires `SentencePiece `_ + """ + vocab_files_names = VOCAB_FILES_NAMES + pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP + max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES + + def __init__(self, vocab_file, bos_token="", eos_token="", sep_token="
", + cls_token="", unk_token="", pad_token='', mask_token='', + **kwargs): + super(XLMRobertaTokenizer, self).__init__(max_len=512, bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, + sep_token=sep_token, cls_token=cls_token, pad_token=pad_token, + mask_token=mask_token, + **kwargs) + self.max_len_single_sentence = self.max_len - 2 # take into account special tokens + self.max_len_sentences_pair = self.max_len - 4 # take into account special tokens + self.sp_model = spm.SentencePieceProcessor() + self.sp_model.Load(str(vocab_file)) + self.vocab_file = vocab_file + self.fairseq_tokens_to_ids = {"": 0, "": 1, "": 2} + self.fairseq_tokens_to_ids[''] = len(self.sp_model) + len(self.fairseq_tokens_to_ids) + self.fairseq_ids_to_tokens = {v: k for k, v in self.fairseq_tokens_to_ids.items()} + + def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None): + """ + Build model inputs from a sequence or a pair of sequence for sequence classification tasks + by concatenating and adding special tokens. + A RoBERTa sequence has the following format: + single sequence: X + pair of sequences: A B + """ + if token_ids_1 is None: + return [self.cls_token_id] + token_ids_0 + [self.sep_token_id] + cls = [self.cls_token_id] + sep = [self.sep_token_id] + return cls + token_ids_0 + sep + sep + token_ids_1 + sep + + def get_special_tokens_mask(self, token_ids_0, token_ids_1=None, already_has_special_tokens=False): + """ + Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding + special tokens using the tokenizer ``prepare_for_model`` or ``encode_plus`` methods. + + Args: + token_ids_0: list of ids (must not contain special tokens) + token_ids_1: Optional list of ids (must not contain special tokens), necessary when fetching sequence ids + for sequence pairs + already_has_special_tokens: (default False) Set to True if the token list is already formated with + special tokens for the model + + Returns: + A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. + """ + if already_has_special_tokens: + if token_ids_1 is not None: + raise ValueError("You should not supply a second sequence if the provided sequence of " + "ids is already formated with special tokens for the model.") + return list(map(lambda x: 1 if x in [self.sep_token_id, self.cls_token_id] else 0, token_ids_0)) + + if token_ids_1 is None: + return [1] + ([0] * len(token_ids_0)) + [1] + return [1] + ([0] * len(token_ids_0)) + [1, 1] + ([0] * len(token_ids_1)) + [1] + + def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None): + """ + Creates a mask from the two sequences passed to be used in a sequence-pair classification task. + A RoBERTa sequence pair mask has the following format: + 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 + | first sequence | second sequence + + if token_ids_1 is None, only returns the first portion of the mask (0's). + """ + sep = [self.sep_token_id] + cls = [self.cls_token_id] + + if token_ids_1 is None: + return len(cls + token_ids_0 + sep) * [0] + return len(cls + token_ids_0 + sep + sep) * [0] + len(token_ids_1 + sep) * [1] + + @property + def vocab_size(self): + return len(self.sp_model) + len(self.fairseq_tokens_to_ids) + + def _tokenize(self, text): + return self.sp_model.EncodeAsPieces(text) + + def _convert_token_to_id(self, token): + """ Converts a token (str/unicode) in an id using the vocab. """ + if token in self.fairseq_tokens_to_ids: + return self.fairseq_tokens_to_ids[token] + return self.sp_model.PieceToId(token) + 1 + + def _convert_id_to_token(self, index): + """Converts an index (integer) in a token (string/unicode) using the vocab.""" + if index in self.fairseq_ids_to_tokens: + return self.fairseq_ids_to_tokens[index] + return self.sp_model.IdToPiece(index + 1) + + def save_vocabulary(self, save_directory): + """ Save the sentencepiece vocabulary (copy original file) and special tokens file + to a directory. + """ + if not os.path.isdir(save_directory): + logger.error("Vocabulary path ({}) should be a directory".format(save_directory)) + return + out_vocab_file = os.path.join(save_directory, VOCAB_FILES_NAMES['vocab_file']) + + if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file): + copyfile(self.vocab_file, out_vocab_file) + + return (out_vocab_file,) From a701a0cee1ae6cb7b93b047cc3ffc06b01157955 Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Mon, 16 Dec 2019 17:17:56 +0100 Subject: [PATCH 391/505] configuration: fix model name for large XLM-RoBERTa model --- transformers/configuration_xlm_roberta.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/configuration_xlm_roberta.py b/transformers/configuration_xlm_roberta.py index 1633cc18aa..dd03572976 100644 --- a/transformers/configuration_xlm_roberta.py +++ b/transformers/configuration_xlm_roberta.py @@ -25,7 +25,7 @@ from .configuration_roberta import RobertaConfig logger = logging.getLogger(__name__) XLM_ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP = { - 'xlm-roberta-base': "https://schweter.eu/cloud/transformers/xlm-roberta-large-config.json", + 'xlm-roberta-large': "https://schweter.eu/cloud/transformers/xlm-roberta-large-config.json", } From d064009b72c4a29cd66b6c633dcd8c3ad5ab6dca Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Mon, 16 Dec 2019 17:23:25 +0100 Subject: [PATCH 392/505] converter: fix vocab size --- ...onvert_xlm_roberta_original_pytorch_checkpoint_to_pytorch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/convert_xlm_roberta_original_pytorch_checkpoint_to_pytorch.py b/transformers/convert_xlm_roberta_original_pytorch_checkpoint_to_pytorch.py index 888adf4819..6873f1d0f0 100644 --- a/transformers/convert_xlm_roberta_original_pytorch_checkpoint_to_pytorch.py +++ b/transformers/convert_xlm_roberta_original_pytorch_checkpoint_to_pytorch.py @@ -47,7 +47,7 @@ def convert_roberta_checkpoint_to_pytorch(roberta_checkpoint_path, pytorch_dump_ roberta = FairseqRobertaModel.from_pretrained(roberta_checkpoint_path, bpe = 'sentencepiece') roberta.eval() # disable dropout config = BertConfig( - vocab_size_or_config_json_file=250004, + vocab_size_or_config_json_file=250002, hidden_size=roberta.args.encoder_embed_dim, num_hidden_layers=roberta.args.encoder_layers, num_attention_heads=roberta.args.encoder_attention_heads, From 855ff0e91d8b3bd75a3b1c1316e2efd814373764 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Mon, 16 Dec 2019 12:42:22 -0500 Subject: [PATCH 393/505] [doc] Model upload and sharing ping @lysandrejik @thomwolf Is this clear enough? Anything we should add? --- README.md | 41 ++++++++++++++++++++++++++++++++++++ docs/source/index.rst | 1 + docs/source/model_sharing.md | 40 +++++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+) create mode 100644 docs/source/model_sharing.md diff --git a/README.md b/README.md index 214f61cc0c..a5ae74a9ae 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ Choose the right framework for every part of a model's lifetime | [Quick tour: Usage](#quick-tour) | Tokenizers & models usage: Bert and GPT-2 | | [Quick tour: TF 2.0 and PyTorch ](#Quick-tour-TF-20-training-and-PyTorch-interoperability) | Train a TF 2.0 model in 10 lines of code, load it in PyTorch | | [Quick tour: Fine-tuning/usage scripts](#quick-tour-of-the-fine-tuningusage-scripts) | Using provided scripts: GLUE, SQuAD and Text generation | +| [Quick tour: Share your models ](#Quick-tour-of-model-sharing) | Upload and share your fine-tuned models with the community | | [Migrating from pytorch-transformers to transformers](#Migrating-from-pytorch-transformers-to-transformers) | Migrating your code from pytorch-transformers to transformers | | [Migrating from pytorch-pretrained-bert to pytorch-transformers](#Migrating-from-pytorch-pretrained-bert-to-transformers) | Migrating your code from pytorch-pretrained-bert to transformers | | [Documentation][(v2.2.0/v2.2.1/v2.2.2)](https://huggingface.co/transformers/v2.2.0) [(v2.1.1)](https://huggingface.co/transformers/v2.1.1) [(v2.0.0)](https://huggingface.co/transformers/v2.0.0) [(v1.2.0)](https://huggingface.co/transformers/v1.2.0) [(v1.1.0)](https://huggingface.co/transformers/v1.1.0) [(v1.0.0)](https://huggingface.co/transformers/v1.0.0) [(master)](https://huggingface.co/transformers) | Full API documentation and more | @@ -446,6 +447,46 @@ python ./examples/run_generation.py \ --repetition_penalty=1.2 \ ``` +## Quick tour of model sharing + +New in `v2.2.2`: you can now upload and share your fine-tuned models with the community, using the CLI that's built-in to the library. + +**First, create an account on [https://huggingface.co/join](https://huggingface.co/join)**. Then: + +```shell +transformers-cli login +# log in using the same credentials as on huggingface.co +``` +Upload your model: +```shell +transformers-cli upload ./path/to/pretrained_model/ + +# ^^ Upload folder containing weights/tokenizer/config +# saved via `.save_pretrained()` + +transformers-cli upload ./config.json [--filename foobar.json] + +# ^^ Upload a single file +# (you can optionally override its filename) +``` + +Your model will then be accessible through its identifier: +```python +"username/model_name" +``` + +Anyone can load it from code: +```python +tokenizer = AutoTokenizer.from_pretrained("username/model_name") +model = AutoModel.from_pretrained("username/model_name") +``` + +Finally, list all your files on S3: +```shell +transformers-cli ls +# List all your S3 objects. +``` + ## Migrating from pytorch-transformers to transformers Here is a quick summary of what you should take care of when migrating from `pytorch-transformers` to `transformers`. diff --git a/docs/source/index.rst b/docs/source/index.rst index 84012fc6cf..48282c1c6c 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -58,6 +58,7 @@ The library currently contains PyTorch and Tensorflow implementations, pre-train installation quickstart pretrained_models + model_sharing examples notebooks serialization diff --git a/docs/source/model_sharing.md b/docs/source/model_sharing.md new file mode 100644 index 0000000000..b9c722b10f --- /dev/null +++ b/docs/source/model_sharing.md @@ -0,0 +1,40 @@ +# Model upload and sharing + +Starting with `v2.2.2`, you can now upload and share your fine-tuned models with the community, using the CLI that's built-in to the library. + +**First, create an account on [https://huggingface.co/join](https://huggingface.co/join)**. Then: + +```shell +transformers-cli login +# log in using the same credentials as on huggingface.co +``` +Upload your model: +```shell +transformers-cli upload ./path/to/pretrained_model/ + +# ^^ Upload folder containing weights/tokenizer/config +# saved via `.save_pretrained()` + +transformers-cli upload ./config.json [--filename foobar.json] + +# ^^ Upload a single file +# (you can optionally override its filename) +``` + +Your model will then be accessible through its identifier: +```python +"username/model_name" +``` + +Anyone can load it from code: +```python +tokenizer = AutoTokenizer.from_pretrained("username/model_name") +model = AutoModel.from_pretrained("username/model_name") +``` + +Finally, list all your files on S3: +```shell +transformers-cli ls +# List all your S3 objects. +``` + From d8034092153a6850052862f154a398b88b8ba4e5 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Mon, 16 Dec 2019 16:31:38 -0500 Subject: [PATCH 394/505] Fix run squad evaluate during training --- examples/run_squad.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/run_squad.py b/examples/run_squad.py index a39915ee8b..34c31c3bb8 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -223,7 +223,7 @@ def evaluate(args, model, tokenizer, prefix=""): eval_dataloader = DataLoader(dataset, sampler=eval_sampler, batch_size=args.eval_batch_size) # multi-gpu evaluate - if args.n_gpu > 1: + if args.n_gpu > 1 and not isinstance(model, torch.nn.DataParallel): model = torch.nn.DataParallel(model) # Eval! From 18a879f47576822aa1a5c49aecb27d89bfa5fa69 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Mon, 16 Dec 2019 16:44:29 -0500 Subject: [PATCH 395/505] fix #2180 --- examples/run_generation.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/examples/run_generation.py b/examples/run_generation.py index 2d917660cf..fa52905b7e 100644 --- a/examples/run_generation.py +++ b/examples/run_generation.py @@ -247,7 +247,11 @@ def main(): out = out[:, len(context_tokens):].tolist() for o in out: text = tokenizer.decode(o, clean_up_tokenization_spaces=True) - text = text[: text.find(args.stop_token) if args.stop_token else None] + if args.stop_token: + index = text.find(args.stop_token) + if index == -1: + index = None + text = text[:index] print(text) From 3cb51299c371f67b4da40b89c59c63e9405591f0 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Mon, 16 Dec 2019 22:32:05 +0100 Subject: [PATCH 396/505] Fix #2109 --- transformers/modeling_tf_pytorch_utils.py | 13 +++++++++++-- transformers/modeling_tf_utils.py | 2 +- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/transformers/modeling_tf_pytorch_utils.py b/transformers/modeling_tf_pytorch_utils.py index 9d2b663dcb..d885fd23b3 100644 --- a/transformers/modeling_tf_pytorch_utils.py +++ b/transformers/modeling_tf_pytorch_utils.py @@ -143,7 +143,11 @@ def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None, a name, transpose = convert_tf_weight_name_to_pt_weight_name(sw_name, start_prefix_to_remove=start_prefix_to_remove) # Find associated numpy array in pytorch model state dict - assert name in pt_state_dict, "{} not found in PyTorch model".format(name) + if name not in pt_state_dict: + if allow_missing_keys: + continue + raise AttributeError("{} not found in PyTorch model".format(name)) + array = pt_state_dict[name].numpy() if transpose: @@ -250,6 +254,7 @@ def load_tf2_weights_in_pytorch_model(pt_model, tf_weights, allow_missing_keys=F all_tf_weights = set(list(tf_weights_map.keys())) loaded_pt_weights_data_ptr = {} + missing_keys_pt = [] for pt_weight_name, pt_weight in current_pt_params_dict.items(): # Handle PyTorch shared weight ()not duplicated in TF 2.0 if pt_weight.data_ptr() in loaded_pt_weights_data_ptr: @@ -258,7 +263,10 @@ def load_tf2_weights_in_pytorch_model(pt_model, tf_weights, allow_missing_keys=F # Find associated numpy array in pytorch model state dict if pt_weight_name not in tf_weights_map: - raise ValueError("{} not found in TF 2.0 model".format(pt_weight_name)) + if allow_missing_keys: + missing_keys_pt.append(pt_weight_name) + continue + raise AttributeError("{} not found in TF 2.0 model".format(pt_weight_name)) array, transpose = tf_weights_map[pt_weight_name] @@ -283,6 +291,7 @@ def load_tf2_weights_in_pytorch_model(pt_model, tf_weights, allow_missing_keys=F all_tf_weights.discard(pt_weight_name) missing_keys, unexpected_keys = pt_model.load_state_dict(new_pt_params_dict, strict=False) + missing_keys += missing_keys_pt if len(missing_keys) > 0: logger.info("Weights of {} not initialized from TF 2.0 model: {}".format( diff --git a/transformers/modeling_tf_utils.py b/transformers/modeling_tf_utils.py index 6fb4850b05..6bbec71cdf 100644 --- a/transformers/modeling_tf_utils.py +++ b/transformers/modeling_tf_utils.py @@ -297,7 +297,7 @@ class TFPreTrainedModel(tf.keras.Model): if from_pt: # Load from a PyTorch checkpoint - return load_pytorch_checkpoint_in_tf2_model(model, resolved_archive_file) + return load_pytorch_checkpoint_in_tf2_model(model, resolved_archive_file, allow_missing_keys=True) ret = model(model.dummy_inputs, training=False) # build the network with dummy inputs From 3f5ccb183e3cfa755dea2dd2afd9abbf1a0f93b8 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Mon, 16 Dec 2019 18:20:23 -0500 Subject: [PATCH 397/505] [doc] Clarify uploads cf https://github.com/huggingface/transformers/commit/855ff0e91d8b3bd75a3b1c1316e2efd814373764#commitcomment-36452545 --- README.md | 10 +++++----- docs/source/model_sharing.md | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index a5ae74a9ae..c33a65bdbb 100644 --- a/README.md +++ b/README.md @@ -464,21 +464,21 @@ transformers-cli upload ./path/to/pretrained_model/ # ^^ Upload folder containing weights/tokenizer/config # saved via `.save_pretrained()` -transformers-cli upload ./config.json [--filename foobar.json] +transformers-cli upload ./config.json [--filename folder/foobar.json] # ^^ Upload a single file -# (you can optionally override its filename) +# (you can optionally override its filename, which can be nested inside a folder) ``` -Your model will then be accessible through its identifier: +Your model will then be accessible through its identifier, a concatenation of your username and the folder name above: ```python "username/model_name" ``` Anyone can load it from code: ```python -tokenizer = AutoTokenizer.from_pretrained("username/model_name") -model = AutoModel.from_pretrained("username/model_name") +tokenizer = AutoTokenizer.from_pretrained("username/pretrained_model") +model = AutoModel.from_pretrained("username/pretrained_model") ``` Finally, list all your files on S3: diff --git a/docs/source/model_sharing.md b/docs/source/model_sharing.md index b9c722b10f..95baafb575 100644 --- a/docs/source/model_sharing.md +++ b/docs/source/model_sharing.md @@ -15,21 +15,21 @@ transformers-cli upload ./path/to/pretrained_model/ # ^^ Upload folder containing weights/tokenizer/config # saved via `.save_pretrained()` -transformers-cli upload ./config.json [--filename foobar.json] +transformers-cli upload ./config.json [--filename folder/foobar.json] # ^^ Upload a single file -# (you can optionally override its filename) +# (you can optionally override its filename, which can be nested inside a folder) ``` -Your model will then be accessible through its identifier: +Your model will then be accessible through its identifier, a concatenation of your username and the folder name above: ```python -"username/model_name" +"username/pretrained_model" ``` Anyone can load it from code: ```python -tokenizer = AutoTokenizer.from_pretrained("username/model_name") -model = AutoModel.from_pretrained("username/model_name") +tokenizer = AutoTokenizer.from_pretrained("username/pretrained_model") +model = AutoModel.from_pretrained("username/pretrained_model") ``` Finally, list all your files on S3: From f349826a57e6f7f1eb5c28ef3b3ff0ac6884ad24 Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Tue, 17 Dec 2019 10:36:04 +0100 Subject: [PATCH 398/505] model: fix cls and sep token for XLM-RoBERTa documentation --- transformers/modeling_xlm_roberta.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/transformers/modeling_xlm_roberta.py b/transformers/modeling_xlm_roberta.py index 8402be4b5c..4c833c69ff 100644 --- a/transformers/modeling_xlm_roberta.py +++ b/transformers/modeling_xlm_roberta.py @@ -111,17 +111,17 @@ class XLMRobertaModel(RobertaModel): Last layer hidden-state of the first token of the sequence (classification token) further processed by a Linear layer and a Tanh activation function. The Linear layer weights are trained from the next sentence prediction (classification) - eo match pre-training, XLM-RoBERTa input sequence should be formatted with [CLS] and [SEP] tokens as follows: + eo match pre-training, XLM-RoBERTa input sequence should be formatted with and tokens as follows: (a) For sequence pairs: - ``tokens: [CLS] is this jack ##son ##ville ? [SEP] [SEP] no it is not . [SEP]`` + ``tokens: is this jack ##son ##ville ? no it is not . `` ``token_type_ids: 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1`` (b) For single sequences: - ``tokens: [CLS] the dog is hairy . [SEP]`` + ``tokens: the dog is hairy . `` ``token_type_ids: 0 0 0 0 0 0 0`` From d7c62661a314c631b3bbf6405143934c8c3e8b5f Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Tue, 17 Dec 2019 11:23:39 +0100 Subject: [PATCH 399/505] Provide serving dependencies for tensorflow and pytorch (serving-tf, serving-torch) --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0b7e512955..b3b6e2e063 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,9 @@ from setuptools import find_packages, setup extras = { - 'serving': ['uvicorn', 'fastapi'] + 'serving': ['uvicorn', 'fastapi'], + 'serving-tf': ['uvicorn', 'fastapi', 'tensorflow'], + 'serving-torch': ['uvicorn', 'fastapi', 'torch'] } extras['all'] = [package for package in extras.values()] From 2f1c745cded91b2f6cfed5b502ea5cbd7d6b9ac7 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Tue, 17 Dec 2019 11:47:54 +0100 Subject: [PATCH 400/505] update conversion script --- ...onvert_xlm_roberta_original_pytorch_checkpoint_to_pytorch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/convert_xlm_roberta_original_pytorch_checkpoint_to_pytorch.py b/transformers/convert_xlm_roberta_original_pytorch_checkpoint_to_pytorch.py index 6873f1d0f0..884c273d2c 100644 --- a/transformers/convert_xlm_roberta_original_pytorch_checkpoint_to_pytorch.py +++ b/transformers/convert_xlm_roberta_original_pytorch_checkpoint_to_pytorch.py @@ -47,7 +47,7 @@ def convert_roberta_checkpoint_to_pytorch(roberta_checkpoint_path, pytorch_dump_ roberta = FairseqRobertaModel.from_pretrained(roberta_checkpoint_path, bpe = 'sentencepiece') roberta.eval() # disable dropout config = BertConfig( - vocab_size_or_config_json_file=250002, + vocab_size=250002, hidden_size=roberta.args.encoder_embed_dim, num_hidden_layers=roberta.args.encoder_layers, num_attention_heads=roberta.args.encoder_attention_heads, From 2fde5a2489bc4aa7fc42ab76effde241b2a0b919 Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Tue, 17 Dec 2019 12:16:07 +0100 Subject: [PATCH 401/505] Initial bunch of documentation. --- transformers/pipelines.py | 118 +++++++++++++++++++++++++++++++++++--- 1 file changed, 111 insertions(+), 7 deletions(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index 2a8f26b03e..6dcb865c74 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -80,6 +80,15 @@ class _ScikitCompat(ABC): class PipelineDataFormat: + """ + Base class for all the pipeline supported data format both for reading and writing. + Supported data formats currently includes: + - JSON + - CSV + + PipelineDataFormat also includes some utilities to work with multi-columns like mapping from datasets columns + to pipelines keyword arguments through the `dataset_kwarg_1=dataset_column_1` format. + """ SUPPORTED_FORMATS = ['json', 'csv'] def __init__(self, output: str, path: str, column: str): @@ -138,7 +147,6 @@ class CsvPipelineDataFormat(PipelineDataFormat): class JsonPipelineDataFormat(PipelineDataFormat): - def __init__(self, output: str, path: str, column: str): super().__init__(output, path, column) @@ -158,6 +166,11 @@ class JsonPipelineDataFormat(PipelineDataFormat): class Pipeline(_ScikitCompat): + """ + Base class implementing pipelined operations. + Pipeline workflow is defined as a sequence of the following operations: + Input -> Tokenization -> Model Inference -> Post-Processing (Task dependent) -> Output + """ def __init__(self, model, tokenizer: PreTrainedTokenizer = None, args_parser: ArgumentHandler = None, device: int = -1, **kwargs): @@ -171,6 +184,9 @@ class Pipeline(_ScikitCompat): self.model = self.model.to('cuda:{}'.format(self.device)) def save_pretrained(self, save_directory): + """ + Save the pipeline's model and tokenizer to the specified save_directory + """ if not os.path.isdir(save_directory): logger.error("Provided path ({}) should be a directory".format(save_directory)) return @@ -179,9 +195,16 @@ class Pipeline(_ScikitCompat): self.tokenizer.save_pretrained(save_directory) def transform(self, X): + """ + Scikit / Keras interface to transformers' pipelines. This method will forward to __call__(). + """ return self(X=X) def predict(self, X): + """ + Scikit / Keras interface to transformers' pipelines. This method will forward to __call__(). + Se + """ return self(X=X) def __call__(self, *texts, **kwargs): @@ -198,6 +221,17 @@ class Pipeline(_ScikitCompat): @contextmanager def device_placement(self): + """ + Context Manager allowing tensor allocation on the user-specified device in framework agnostic way. + example: + # Explicitly ask for tensor allocation on CUDA device :0 + nlp = pipeline(..., device=0) + with nlp.device_placement(): + # Every framework specific tensor allocation will be done on the request device + output = nlp(...) + Returns: + Context manager + """ if is_tf_available(): import tensorflow as tf with tf.device('/CPU:0' if self.device == -1 else '/device:GPU:{}'.format(self.device)): @@ -210,6 +244,13 @@ class Pipeline(_ScikitCompat): yield def _forward(self, inputs): + """ + Internal framework specific forward dispatching. + Args: + inputs: dict holding all the keyworded arguments for required by the model forward method. + Returns: + Numpy array + """ if is_tf_available(): # TODO trace model predictions = self.model(inputs)[0] @@ -222,11 +263,17 @@ class Pipeline(_ScikitCompat): class FeatureExtractionPipeline(Pipeline): + """ + Feature extraction pipeline using Model head. + """ def __call__(self, *args, **kwargs): return super().__call__(*args, **kwargs).tolist() class TextClassificationPipeline(Pipeline): + """ + Text classification pipeline using ModelForTextClassification head. + """ def __init__(self, model, tokenizer: PreTrainedTokenizer, nb_classes: int = 2): super().__init__(model, tokenizer) @@ -239,7 +286,9 @@ class TextClassificationPipeline(Pipeline): class NerPipeline(Pipeline): - + """ + Named Entity Recognition pipeline using ModelForTokenClassification head. + """ def __init__(self, model, tokenizer: PreTrainedTokenizer): super().__init__(model, tokenizer) @@ -286,7 +335,7 @@ class NerPipeline(Pipeline): class QuestionAnsweringPipeline(Pipeline): """ - Question Answering pipeline involving Tokenization and Inference. + Question Answering pipeline using ModelForQuestionAnswering head. """ class QuestionAnsweringArgumentHandler(ArgumentHandler): @@ -341,9 +390,15 @@ class QuestionAnsweringPipeline(Pipeline): @staticmethod def create_sample(question: Union[str, List[str]], context: Union[str, List[str]]) -> Union[SquadExample, List[SquadExample]]: - is_list = isinstance(question, list) - - if is_list: + """ + QuestionAnsweringPipeline leverages the SquadExample/SquadFeatures internally. + This helper method encapsulate all the logic for converting question(s) and context(s) to SquadExample(s). + We currently support extractive question answering. + Args: + question: (str, List[str]) The question to be ask for the associated context + context: (str, List[str]) The context in which we will look for the answer. + """ + if isinstance(question, list): return [SquadExample(None, q, c, None, None, None) for q, c in zip(question, context)] else: return SquadExample(None, question, context, None, None, None) @@ -352,6 +407,12 @@ class QuestionAnsweringPipeline(Pipeline): super().__init__(model, tokenizer, args_parser=QuestionAnsweringPipeline.QuestionAnsweringArgumentHandler()) def inputs_for_model(self, features: Union[SquadExample, List[SquadExample]]) -> Dict: + """ + Generates the input dictionary with model-specific parameters. + + Returns: + dict holding all the required parameters for model's forward + """ args = ['input_ids', 'attention_mask'] model_type = type(self.model).__name__.lower() @@ -367,6 +428,20 @@ class QuestionAnsweringPipeline(Pipeline): return {k: [feature.__dict__[k] for feature in features] for k in args} def __call__(self, *texts, **kwargs): + """ + Args: + We support multiple use-cases, the following are exclusive: + X: sequence of SquadExample + data: sequence of SquadExample + question: (str, List[str]), batch of question(s) to map along with context + context: (str, List[str]), batch of context(s) associated with the provided question keyword argument + Returns: + dict: {'answer': str, 'score": float, 'start": int, "end": int} + answer: the textual answer in the intial context + score: the score the current answer scored for the model + start: the character index in the original string corresponding to the beginning of the answer' span + end: the character index in the original string corresponding to the ending of the answer' span + """ # Set defaults values kwargs.setdefault('topk', 1) kwargs.setdefault('doc_stride', 128) @@ -432,6 +507,19 @@ class QuestionAnsweringPipeline(Pipeline): return answers def decode(self, start: np.ndarray, end: np.ndarray, topk: int, max_answer_len: int) -> Tuple: + """ + Take the output of any QuestionAnswering head and will generate probalities for each span to be + the actual answer. + In addition, it filters out some unwanted/impossible cases like answer len being greater than + max_answer_len or answer end position being before the starting position. + The method supports output the k-best answer through the topk argument. + + Args: + start: numpy array, holding individual start probabilities for each token + end: numpy array, holding individual end probabilities for each token + topk: int, indicates how many possible answer span(s) to extract from the model's output + max_answer_len: int, maximum size of the answer to extract from the model's output + """ # Ensure we have batch axis if start.ndim == 1: start = start[None] @@ -459,6 +547,18 @@ class QuestionAnsweringPipeline(Pipeline): return start, end, candidates[0, start, end] def span_to_answer(self, text: str, start: int, end: int): + """ + When decoding from token probalities, this method maps token indexes to actual word in + the initial context. + + Args: + text: str, the actual context to extract the answer from + start: int, starting answer token index + end: int, ending answer token index + + Returns: + dict: {'answer': str, 'start': int, 'end': int} + """ words = [] token_idx = char_start_idx = char_end_idx = chars_idx = 0 @@ -514,7 +614,11 @@ SUPPORTED_TASKS = { def pipeline(task: str, model, config: Optional[PretrainedConfig] = None, tokenizer: Optional[Union[str, PreTrainedTokenizer]] = None, **kwargs) -> Pipeline: """ - Utility factory method to build pipeline. + Utility factory method to build a pipeline. + Pipeline are made of: + A Tokenizer instance in charge of mapping raw textual input to token + A Model instance + Some (optional) post processing for enhancing model's output """ # Try to infer tokenizer from model name (if provided as str) if tokenizer is None: From 55397dfb9b7e61001e10abb595931d4a98ae58b0 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Tue, 17 Dec 2019 13:10:51 -0500 Subject: [PATCH 402/505] CsvPipelineDataFormat: Fix for single-column --- transformers/pipelines.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index 6dcb865c74..dec7843baf 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -136,7 +136,7 @@ class CsvPipelineDataFormat(PipelineDataFormat): if self.is_multi_columns: yield {k: row[c] for k, c in self.column} else: - yield row[self.column] + yield row[self.column[0]] def save(self, data: List[dict]): with open(self.output, 'w') as f: From 2cff4bd8f3ad412917f4f295b97b952e297fa257 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 17 Dec 2019 14:01:04 -0500 Subject: [PATCH 403/505] Fix segmentation fault --- transformers/file_utils.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/transformers/file_utils.py b/transformers/file_utils.py index 81c9b8002f..16010f7e0a 100644 --- a/transformers/file_utils.py +++ b/transformers/file_utils.py @@ -26,14 +26,6 @@ from contextlib import contextmanager logger = logging.getLogger(__name__) # pylint: disable=invalid-name -try: - import tensorflow as tf - assert hasattr(tf, '__version__') and int(tf.__version__[0]) >= 2 - _tf_available = True # pylint: disable=invalid-name - logger.info("TensorFlow version {} available.".format(tf.__version__)) -except (ImportError, AssertionError): - _tf_available = False # pylint: disable=invalid-name - try: import torch _torch_available = True # pylint: disable=invalid-name @@ -41,6 +33,13 @@ try: except ImportError: _torch_available = False # pylint: disable=invalid-name +try: + import tensorflow as tf + assert hasattr(tf, '__version__') and int(tf.__version__[0]) >= 2 + _tf_available = True # pylint: disable=invalid-name + logger.info("TensorFlow version {} available.".format(tf.__version__)) +except (ImportError, AssertionError): + _tf_available = False # pylint: disable=invalid-name try: from torch.hub import _get_torch_home From 5e289f69bc564c94132f77c89a34e5f1dd69a592 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Tue, 17 Dec 2019 14:17:11 -0500 Subject: [PATCH 404/505] regex 2019.12.17 install fails with Python 2 --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 9c43abc6d7..32edee0712 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ boto3 # Used for downloading models over HTTP requests # For OpenAI GPT -regex +regex != 2019.12.17 # For XLNet sentencepiece # For XLM diff --git a/setup.py b/setup.py index eacb5ecec0..bf09a7d48a 100644 --- a/setup.py +++ b/setup.py @@ -59,7 +59,7 @@ setup( 'boto3', 'requests', 'tqdm', - 'regex', + 'regex != 2019.12.17', 'sentencepiece', 'sacremoses'], entry_points={ From a4df2e011367020253c8ca8a714c4b4855ff61bc Mon Sep 17 00:00:00 2001 From: Arman Cohan Date: Tue, 26 Nov 2019 16:03:07 -0800 Subject: [PATCH 405/505] update roberta conversion - update to fix conversion for the updated fairseq model - create save directory if not exist --- ..._original_pytorch_checkpoint_to_pytorch.py | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/transformers/convert_roberta_original_pytorch_checkpoint_to_pytorch.py b/transformers/convert_roberta_original_pytorch_checkpoint_to_pytorch.py index b4dc1bb61b..be3460a86f 100644 --- a/transformers/convert_roberta_original_pytorch_checkpoint_to_pytorch.py +++ b/transformers/convert_roberta_original_pytorch_checkpoint_to_pytorch.py @@ -20,6 +20,7 @@ import argparse import logging import numpy as np import torch +import pathlib from fairseq.models.roberta import RobertaModel as FairseqRobertaModel from fairseq.modules import TransformerSentenceEncoderLayer @@ -79,15 +80,18 @@ def convert_roberta_checkpoint_to_pytorch(roberta_checkpoint_path, pytorch_dump_ ### self attention self_attn: BertSelfAttention = layer.attention.self assert( - roberta_layer.self_attn.in_proj_weight.shape == torch.Size((3 * config.hidden_size, config.hidden_size)) + roberta_layer.self_attn.k_proj.weight.data.shape == \ + roberta_layer.self_attn.q_proj.weight.data.shape == \ + roberta_layer.self_attn.v_proj.weight.data.shape == \ + torch.Size((config.hidden_size, config.hidden_size)) ) - # we use three distinct linear layers so we split the source layer here. - self_attn.query.weight.data = roberta_layer.self_attn.in_proj_weight[:config.hidden_size, :] - self_attn.query.bias.data = roberta_layer.self_attn.in_proj_bias[:config.hidden_size] - self_attn.key.weight.data = roberta_layer.self_attn.in_proj_weight[config.hidden_size:2*config.hidden_size, :] - self_attn.key.bias.data = roberta_layer.self_attn.in_proj_bias[config.hidden_size:2*config.hidden_size] - self_attn.value.weight.data = roberta_layer.self_attn.in_proj_weight[2*config.hidden_size:, :] - self_attn.value.bias.data = roberta_layer.self_attn.in_proj_bias[2*config.hidden_size:] + + self_attn.query.weight.data = roberta_layer.self_attn.q_proj.weight + self_attn.query.bias.data = roberta_layer.self_attn.q_proj.bias + self_attn.key.weight.data = roberta_layer.self_attn.k_proj.weight + self_attn.key.bias.data = roberta_layer.self_attn.k_proj.bias + self_attn.value.weight.data = roberta_layer.self_attn.v_proj.weight + self_attn.value.bias.data = roberta_layer.self_attn.v_proj.bias ### self-attention output self_output: BertSelfOutput = layer.attention.output @@ -151,6 +155,7 @@ def convert_roberta_checkpoint_to_pytorch(roberta_checkpoint_path, pytorch_dump_ if not success: raise Exception("Something went wRoNg") + pathlib.Path(pytorch_dump_folder_path).mkdir(parents=True, exist_ok=True) print(f"Saving model to {pytorch_dump_folder_path}") model.save_pretrained(pytorch_dump_folder_path) From ea636440d1ea3497785c2682c410da478f8b1841 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Tue, 17 Dec 2019 18:06:42 -0500 Subject: [PATCH 406/505] [roberta.conversion] Do not hardcode vocab size and support for fairseq 0.9+ --- ...t_roberta_original_pytorch_checkpoint_to_pytorch.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/transformers/convert_roberta_original_pytorch_checkpoint_to_pytorch.py b/transformers/convert_roberta_original_pytorch_checkpoint_to_pytorch.py index be3460a86f..fedfc1ecb8 100644 --- a/transformers/convert_roberta_original_pytorch_checkpoint_to_pytorch.py +++ b/transformers/convert_roberta_original_pytorch_checkpoint_to_pytorch.py @@ -22,6 +22,12 @@ import numpy as np import torch import pathlib +import fairseq +from packaging import version + +if version.parse(fairseq.__version__) < version.parse("0.9.0"): + raise Exception("requires fairseq >= 0.9.0") + from fairseq.models.roberta import RobertaModel as FairseqRobertaModel from fairseq.modules import TransformerSentenceEncoderLayer from transformers.modeling_bert import (BertConfig, BertEncoder, @@ -46,8 +52,9 @@ def convert_roberta_checkpoint_to_pytorch(roberta_checkpoint_path, pytorch_dump_ """ roberta = FairseqRobertaModel.from_pretrained(roberta_checkpoint_path) roberta.eval() # disable dropout + roberta_sent_encoder = roberta.model.decoder.sentence_encoder config = BertConfig( - vocab_size=50265, + vocab_size=roberta_sent_encoder.embed_tokens.num_embeddings, hidden_size=roberta.args.encoder_embed_dim, num_hidden_layers=roberta.args.encoder_layers, num_attention_heads=roberta.args.encoder_attention_heads, @@ -65,7 +72,6 @@ def convert_roberta_checkpoint_to_pytorch(roberta_checkpoint_path, pytorch_dump_ # Now let's copy all the weights. # Embeddings - roberta_sent_encoder = roberta.model.decoder.sentence_encoder model.roberta.embeddings.word_embeddings.weight = roberta_sent_encoder.embed_tokens.weight model.roberta.embeddings.position_embeddings.weight = roberta_sent_encoder.embed_positions.weight model.roberta.embeddings.token_type_embeddings.weight.data = torch.zeros_like(model.roberta.embeddings.token_type_embeddings.weight) # just zero them out b/c RoBERTa doesn't use them. From a0d386455b347508ea31fc88dd06cc5555255c37 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Tue, 17 Dec 2019 20:07:39 -0500 Subject: [PATCH 407/505] Fix outdated tokenizer doc --- templates/adding_a_new_model/tokenization_xxx.py | 2 +- transformers/tokenization_bert.py | 4 ++-- transformers/tokenization_distilbert.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/templates/adding_a_new_model/tokenization_xxx.py b/templates/adding_a_new_model/tokenization_xxx.py index 3d6b4ad9df..7a10a41e5a 100644 --- a/templates/adding_a_new_model/tokenization_xxx.py +++ b/templates/adding_a_new_model/tokenization_xxx.py @@ -85,7 +85,7 @@ class XxxTokenizer(PreTrainedTokenizer): Args: vocab_file: Path to a one-wordpiece-per-line vocabulary file - do_lower_case: Whether to lower case the input. Only has an effect when do_wordpiece_only=False + do_lower_case: Whether to lower case the input. Only has an effect when do_basic_tokenize=True """ vocab_files_names = VOCAB_FILES_NAMES diff --git a/transformers/tokenization_bert.py b/transformers/tokenization_bert.py index ded5072e58..7ab8029da8 100644 --- a/transformers/tokenization_bert.py +++ b/transformers/tokenization_bert.py @@ -113,12 +113,12 @@ class BertTokenizer(PreTrainedTokenizer): Args: vocab_file: Path to a one-wordpiece-per-line vocabulary file - do_lower_case: Whether to lower case the input. Only has an effect when do_wordpiece_only=False + do_lower_case: Whether to lower case the input. Only has an effect when do_basic_tokenize=True do_basic_tokenize: Whether to do basic tokenization before wordpiece. max_len: An artificial maximum length to truncate tokenized sequences to; Effective maximum length is always the minimum of this value (if specified) and the underlying BERT model's sequence length. never_split: List of tokens which will never be split during tokenization. Only has an effect when - do_wordpiece_only=False + do_basic_tokenize=True """ vocab_files_names = VOCAB_FILES_NAMES diff --git a/transformers/tokenization_distilbert.py b/transformers/tokenization_distilbert.py index f40bf2bd77..2f245d71dc 100644 --- a/transformers/tokenization_distilbert.py +++ b/transformers/tokenization_distilbert.py @@ -53,12 +53,12 @@ class DistilBertTokenizer(BertTokenizer): Args: vocab_file: Path to a one-wordpiece-per-line vocabulary file - do_lower_case: Whether to lower case the input. Only has an effect when do_wordpiece_only=False + do_lower_case: Whether to lower case the input. Only has an effect when do_basic_tokenize=True do_basic_tokenize: Whether to do basic tokenization before wordpiece. max_len: An artificial maximum length to truncate tokenized sequences to; Effective maximum length is always the minimum of this value (if specified) and the underlying BERT model's sequence length. never_split: List of tokens which will never be split during tokenization. Only has an effect when - do_wordpiece_only=False + do_basic_tokenize=True """ vocab_files_names = VOCAB_FILES_NAMES From 8ac840ff8758fb242e3e89cbc809366165ccf960 Mon Sep 17 00:00:00 2001 From: Antti Virtanen Date: Mon, 16 Dec 2019 17:08:25 +0200 Subject: [PATCH 408/505] Adding Finnish BERT. --- transformers/configuration_bert.py | 4 +++- transformers/modeling_bert.py | 4 +++- transformers/modeling_tf_bert.py | 4 +++- transformers/tokenization_bert.py | 6 ++++++ 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/transformers/configuration_bert.py b/transformers/configuration_bert.py index 9072820bce..b1974966a9 100644 --- a/transformers/configuration_bert.py +++ b/transformers/configuration_bert.py @@ -45,7 +45,9 @@ BERT_PRETRAINED_CONFIG_ARCHIVE_MAP = { 'bert-base-japanese': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-config.json", 'bert-base-japanese-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-whole-word-masking-config.json", 'bert-base-japanese-char': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-config.json", - 'bert-base-japanese-char-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-whole-word-masking-config.json" + 'bert-base-japanese-char-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-whole-word-masking-config.json", + 'bert-base-finnish-cased-v1': "http://dl.turkunlp.org/finbert/torch-transformers/bert-base-finnish-cased-v1/config.json", + 'bert-base-finnish-uncased-v1': "http://dl.turkunlp.org/finbert/torch-transformers/bert-base-finnish-uncased-v1/config.json", } diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py index d0f35272ac..d0cb5ec617 100644 --- a/transformers/modeling_bert.py +++ b/transformers/modeling_bert.py @@ -51,7 +51,9 @@ BERT_PRETRAINED_MODEL_ARCHIVE_MAP = { 'bert-base-japanese': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-pytorch_model.bin", 'bert-base-japanese-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-whole-word-masking-pytorch_model.bin", 'bert-base-japanese-char': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-pytorch_model.bin", - 'bert-base-japanese-char-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-whole-word-masking-pytorch_model.bin" + 'bert-base-japanese-char-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-whole-word-masking-pytorch_model.bin", + 'bert-base-finnish-cased-v1': "http://dl.turkunlp.org/finbert/torch-transformers/bert-base-finnish-cased-v1/pytorch_model.bin", + 'bert-base-finnish-uncased-v1': "http://dl.turkunlp.org/finbert/torch-transformers/bert-base-finnish-uncased-v1/pytorch_model.bin", } diff --git a/transformers/modeling_tf_bert.py b/transformers/modeling_tf_bert.py index 7cc71f5063..20b5895dbd 100644 --- a/transformers/modeling_tf_bert.py +++ b/transformers/modeling_tf_bert.py @@ -51,7 +51,9 @@ TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP = { 'bert-base-japanese': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-tf_model.h5", 'bert-base-japanese-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-whole-word-masking-tf_model.h5", 'bert-base-japanese-char': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-tf_model.h5", - 'bert-base-japanese-char-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-whole-word-masking-tf_model.h5" + 'bert-base-japanese-char-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-whole-word-masking-tf_model.h5", + #'bert-base-finnish-cased-v1': "http://dl.turkunlp.org/finbert/torch-transformers/bert-base-finnish-cased-v1/pytorch_model.bin", + #'bert-base-finnish-uncased-v1': "http://dl.turkunlp.org/finbert/torch-transformers/bert-base-finnish-uncased-v1/pytorch_model.bin", } diff --git a/transformers/tokenization_bert.py b/transformers/tokenization_bert.py index 7ab8029da8..6f6a4d6f19 100644 --- a/transformers/tokenization_bert.py +++ b/transformers/tokenization_bert.py @@ -46,6 +46,8 @@ PRETRAINED_VOCAB_FILES_MAP = { 'bert-base-cased-finetuned-mrpc': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-finetuned-mrpc-vocab.txt", 'bert-base-german-dbmdz-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-cased-vocab.txt", 'bert-base-german-dbmdz-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-uncased-vocab.txt", + 'bert-base-finnish-cased-v1': "http://dl.turkunlp.org/finbert/torch-transformers/bert-base-finnish-cased-v1/vocab.txt", + 'bert-base-finnish-uncased-v1': "http://dl.turkunlp.org/finbert/torch-transformers/bert-base-finnish-uncased-v1/vocab.txt", } } @@ -65,6 +67,8 @@ PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { 'bert-base-cased-finetuned-mrpc': 512, 'bert-base-german-dbmdz-cased': 512, 'bert-base-german-dbmdz-uncased': 512, + 'bert-base-finnish-cased-v1': 512, + 'bert-base-finnish-uncased-v1': 512, } PRETRAINED_INIT_CONFIGURATION = { @@ -83,6 +87,8 @@ PRETRAINED_INIT_CONFIGURATION = { 'bert-base-cased-finetuned-mrpc': {'do_lower_case': False}, 'bert-base-german-dbmdz-cased': {'do_lower_case': False}, 'bert-base-german-dbmdz-uncased': {'do_lower_case': True}, + 'bert-base-finnish-cased-v1': {'do_lower_case': False}, + 'bert-base-finnish-uncased-v1': {'do_lower_case': True}, } From abc43ffbfff69dc91f354c34f1c7c5b48a5c1502 Mon Sep 17 00:00:00 2001 From: Antti Virtanen Date: Mon, 16 Dec 2019 18:08:00 +0200 Subject: [PATCH 409/505] Add pretrained model documentation for FinBERT. --- docs/source/pretrained_models.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst index c6b990f213..7d037da34f 100644 --- a/docs/source/pretrained_models.rst +++ b/docs/source/pretrained_models.rst @@ -79,6 +79,14 @@ Here is the full list of the currently provided pretrained models together with | | ``bert-base-japanese-char-whole-word-masking`` | | 12-layer, 768-hidden, 12-heads, 110M parameters. | | | | | Trained on Japanese text using Whole-Word-Masking. Text is tokenized into characters. | | | | (see `details on cl-tohoku repository `__). | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``bert-base-finnish-cased-v1`` | | 12-layer, 768-hidden, 12-heads, 110M parameters. | +| | | | Trained on cased Finnish text. | +| | | (see `details on turkunlp.org `__). | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``bert-base-finnish-uncased-v1`` | | 12-layer, 768-hidden, 12-heads, 110M parameters. | +| | | | Trained on uncased Finnish text. | +| | | (see `details on turkunlp.org `__). | +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | GPT | ``openai-gpt`` | | 12-layer, 768-hidden, 12-heads, 110M parameters. | | | | | OpenAI GPT English model | From c5f35e61db8d7286173515071e76612e9e5f5ce5 Mon Sep 17 00:00:00 2001 From: Antti Virtanen Date: Mon, 16 Dec 2019 21:06:14 +0200 Subject: [PATCH 410/505] Uploaded files to AWS. --- transformers/configuration_bert.py | 4 ++-- transformers/modeling_bert.py | 4 ++-- transformers/modeling_tf_bert.py | 4 ++-- transformers/tokenization_bert.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/transformers/configuration_bert.py b/transformers/configuration_bert.py index b1974966a9..c2ccc578c2 100644 --- a/transformers/configuration_bert.py +++ b/transformers/configuration_bert.py @@ -46,8 +46,8 @@ BERT_PRETRAINED_CONFIG_ARCHIVE_MAP = { 'bert-base-japanese-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-whole-word-masking-config.json", 'bert-base-japanese-char': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-config.json", 'bert-base-japanese-char-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-whole-word-masking-config.json", - 'bert-base-finnish-cased-v1': "http://dl.turkunlp.org/finbert/torch-transformers/bert-base-finnish-cased-v1/config.json", - 'bert-base-finnish-uncased-v1': "http://dl.turkunlp.org/finbert/torch-transformers/bert-base-finnish-uncased-v1/config.json", + 'bert-base-finnish-cased-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-cased-v1-config.json", + 'bert-base-finnish-uncased-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-uncased-v1-config.json", } diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py index d0cb5ec617..4e034f4b6e 100644 --- a/transformers/modeling_bert.py +++ b/transformers/modeling_bert.py @@ -52,8 +52,8 @@ BERT_PRETRAINED_MODEL_ARCHIVE_MAP = { 'bert-base-japanese-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-whole-word-masking-pytorch_model.bin", 'bert-base-japanese-char': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-pytorch_model.bin", 'bert-base-japanese-char-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-whole-word-masking-pytorch_model.bin", - 'bert-base-finnish-cased-v1': "http://dl.turkunlp.org/finbert/torch-transformers/bert-base-finnish-cased-v1/pytorch_model.bin", - 'bert-base-finnish-uncased-v1': "http://dl.turkunlp.org/finbert/torch-transformers/bert-base-finnish-uncased-v1/pytorch_model.bin", + 'bert-base-finnish-cased-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-cased-v1-pytorch_model.bin", + 'bert-base-finnish-uncased-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-uncased-v1-pytorch_model.bin", } diff --git a/transformers/modeling_tf_bert.py b/transformers/modeling_tf_bert.py index 20b5895dbd..5a989c299f 100644 --- a/transformers/modeling_tf_bert.py +++ b/transformers/modeling_tf_bert.py @@ -52,8 +52,8 @@ TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP = { 'bert-base-japanese-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-whole-word-masking-tf_model.h5", 'bert-base-japanese-char': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-tf_model.h5", 'bert-base-japanese-char-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-whole-word-masking-tf_model.h5", - #'bert-base-finnish-cased-v1': "http://dl.turkunlp.org/finbert/torch-transformers/bert-base-finnish-cased-v1/pytorch_model.bin", - #'bert-base-finnish-uncased-v1': "http://dl.turkunlp.org/finbert/torch-transformers/bert-base-finnish-uncased-v1/pytorch_model.bin", + 'bert-base-finnish-cased-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-cased-v1-tf_model.h5", + 'bert-base-finnish-uncased-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-uncased-v1-tf_model.h5", } diff --git a/transformers/tokenization_bert.py b/transformers/tokenization_bert.py index 6f6a4d6f19..c11c1b4d3c 100644 --- a/transformers/tokenization_bert.py +++ b/transformers/tokenization_bert.py @@ -46,8 +46,8 @@ PRETRAINED_VOCAB_FILES_MAP = { 'bert-base-cased-finetuned-mrpc': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-finetuned-mrpc-vocab.txt", 'bert-base-german-dbmdz-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-cased-vocab.txt", 'bert-base-german-dbmdz-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-uncased-vocab.txt", - 'bert-base-finnish-cased-v1': "http://dl.turkunlp.org/finbert/torch-transformers/bert-base-finnish-cased-v1/vocab.txt", - 'bert-base-finnish-uncased-v1': "http://dl.turkunlp.org/finbert/torch-transformers/bert-base-finnish-uncased-v1/vocab.txt", + 'bert-base-finnish-cased-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-cased-v1-vocab.txt", + 'bert-base-finnish-uncased-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-cased-v1-vocab.txt", } } From 7ffa8173905cb6d0819fc424a4806e81a44dd0e0 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Mon, 16 Dec 2019 18:55:14 -0500 Subject: [PATCH 411/505] [s3] mv files and update links --- transformers/configuration_bert.py | 4 ++-- transformers/modeling_bert.py | 4 ++-- transformers/modeling_tf_bert.py | 4 ++-- transformers/tokenization_bert.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/transformers/configuration_bert.py b/transformers/configuration_bert.py index c2ccc578c2..7b495013ff 100644 --- a/transformers/configuration_bert.py +++ b/transformers/configuration_bert.py @@ -46,8 +46,8 @@ BERT_PRETRAINED_CONFIG_ARCHIVE_MAP = { 'bert-base-japanese-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-whole-word-masking-config.json", 'bert-base-japanese-char': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-config.json", 'bert-base-japanese-char-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-whole-word-masking-config.json", - 'bert-base-finnish-cased-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-cased-v1-config.json", - 'bert-base-finnish-uncased-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-uncased-v1-config.json", + 'bert-base-finnish-cased-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-cased-v1/config.json", + 'bert-base-finnish-uncased-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-uncased-v1/config.json", } diff --git a/transformers/modeling_bert.py b/transformers/modeling_bert.py index 4e034f4b6e..afeb9d8e21 100644 --- a/transformers/modeling_bert.py +++ b/transformers/modeling_bert.py @@ -52,8 +52,8 @@ BERT_PRETRAINED_MODEL_ARCHIVE_MAP = { 'bert-base-japanese-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-whole-word-masking-pytorch_model.bin", 'bert-base-japanese-char': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-pytorch_model.bin", 'bert-base-japanese-char-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-whole-word-masking-pytorch_model.bin", - 'bert-base-finnish-cased-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-cased-v1-pytorch_model.bin", - 'bert-base-finnish-uncased-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-uncased-v1-pytorch_model.bin", + 'bert-base-finnish-cased-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-cased-v1/pytorch_model.bin", + 'bert-base-finnish-uncased-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-uncased-v1/pytorch_model.bin", } diff --git a/transformers/modeling_tf_bert.py b/transformers/modeling_tf_bert.py index 5a989c299f..b4f97c06d9 100644 --- a/transformers/modeling_tf_bert.py +++ b/transformers/modeling_tf_bert.py @@ -52,8 +52,8 @@ TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP = { 'bert-base-japanese-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-whole-word-masking-tf_model.h5", 'bert-base-japanese-char': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-tf_model.h5", 'bert-base-japanese-char-whole-word-masking': "https://s3.amazonaws.com/models.huggingface.co/bert/cl-tohoku/bert-base-japanese-char-whole-word-masking-tf_model.h5", - 'bert-base-finnish-cased-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-cased-v1-tf_model.h5", - 'bert-base-finnish-uncased-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-uncased-v1-tf_model.h5", + 'bert-base-finnish-cased-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-cased-v1/tf_model.h5", + 'bert-base-finnish-uncased-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-uncased-v1/tf_model.h5", } diff --git a/transformers/tokenization_bert.py b/transformers/tokenization_bert.py index c11c1b4d3c..18b96c99b3 100644 --- a/transformers/tokenization_bert.py +++ b/transformers/tokenization_bert.py @@ -46,8 +46,8 @@ PRETRAINED_VOCAB_FILES_MAP = { 'bert-base-cased-finetuned-mrpc': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-cased-finetuned-mrpc-vocab.txt", 'bert-base-german-dbmdz-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-cased-vocab.txt", 'bert-base-german-dbmdz-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-uncased-vocab.txt", - 'bert-base-finnish-cased-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-cased-v1-vocab.txt", - 'bert-base-finnish-uncased-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-cased-v1-vocab.txt", + 'bert-base-finnish-cased-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-cased-v1/vocab.txt", + 'bert-base-finnish-uncased-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-cased-v1/vocab.txt", } } From 94c99db34cf9074a212c36554fb925c513d70ab1 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Tue, 17 Dec 2019 20:34:22 -0500 Subject: [PATCH 412/505] [FinBERT] fix incorrect url --- transformers/tokenization_bert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/tokenization_bert.py b/transformers/tokenization_bert.py index 18b96c99b3..edc26d88cf 100644 --- a/transformers/tokenization_bert.py +++ b/transformers/tokenization_bert.py @@ -47,7 +47,7 @@ PRETRAINED_VOCAB_FILES_MAP = { 'bert-base-german-dbmdz-cased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-cased-vocab.txt", 'bert-base-german-dbmdz-uncased': "https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-german-dbmdz-uncased-vocab.txt", 'bert-base-finnish-cased-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-cased-v1/vocab.txt", - 'bert-base-finnish-uncased-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-cased-v1/vocab.txt", + 'bert-base-finnish-uncased-v1': "https://s3.amazonaws.com/models.huggingface.co/bert/TurkuNLP/bert-base-finnish-uncased-v1/vocab.txt", } } From e347725d8c2fea7a89605d02d1f465594cf7df85 Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Tue, 17 Dec 2019 23:32:52 +0100 Subject: [PATCH 413/505] More fine-grained control over pipeline creation with config argument. --- transformers/pipelines.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index dec7843baf..bcb4d9e054 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -497,7 +497,7 @@ class QuestionAnsweringPipeline(Pipeline): 'score': score.item(), 'start': np.where(char_to_word == feature.token_to_orig_map[s])[0][0].item(), 'end': np.where(char_to_word == feature.token_to_orig_map[e])[0][-1].item(), - 'answer': ' '.join(example.doc_tokens[feature.token_to_orig_map[s]: feature.token_to_orig_map[e] + 1]) + 'answer': ' '.join(example.doc_tokens[feature.token_to_orig_map[s]:feature.token_to_orig_map[e] + 1]) } for s, e, score in zip(starts, ends, scores) ] @@ -612,7 +612,8 @@ SUPPORTED_TASKS = { } -def pipeline(task: str, model, config: Optional[PretrainedConfig] = None, tokenizer: Optional[Union[str, PreTrainedTokenizer]] = None, **kwargs) -> Pipeline: +def pipeline(task: str, model, config: Optional[Union[str, PretrainedConfig]] = None, + tokenizer: Optional[Union[str, PreTrainedTokenizer]] = None, **kwargs) -> Pipeline: """ Utility factory method to build a pipeline. Pipeline are made of: @@ -637,13 +638,21 @@ def pipeline(task: str, model, config: Optional[PretrainedConfig] = None, tokeni task, allocator = targeted_task['impl'], targeted_task['tf'] if is_tf_available() else targeted_task['pt'] # Special handling for model conversion - from_tf = model.endswith('.h5') and not is_tf_available() - from_pt = model.endswith('.bin') and not is_torch_available() + if isinstance(model, str): + from_tf = model.endswith('.h5') and not is_tf_available() + from_pt = model.endswith('.bin') and not is_torch_available() - if from_tf: - logger.warning('Model might be a TensorFlow model (ending with `.h5`) but TensorFlow is not available. Trying to load the model with PyTorch.') - elif from_pt: - logger.warning('Model might be a PyTorch model (ending with `.bin`) but PyTorch is not available. Trying to load the model with Tensorflow.') + if from_tf: + logger.warning('Model might be a TensorFlow model (ending with `.h5`) but TensorFlow is not available. ' + 'Trying to load the model with PyTorch.') + elif from_pt: + logger.warning('Model might be a PyTorch model (ending with `.bin`) but PyTorch is not available. ' + 'Trying to load the model with Tensorflow.') + else: + from_tf = from_pt = False + + if isinstance(config, str): + config = PretrainedConfig.from_pretrained(config) if allocator.__name__.startswith('TF'): model = allocator.from_pretrained(model, config=config, from_pt=from_pt) From ca31abc6d6fe35a39703ed36775853595149e956 Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Wed, 18 Dec 2019 11:36:54 +0100 Subject: [PATCH 414/505] tokenization: *align* fairseq and spm vocab to fix some tokenization errors --- transformers/tokenization_xlm_roberta.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/transformers/tokenization_xlm_roberta.py b/transformers/tokenization_xlm_roberta.py index 0f95397606..d8484e7f9c 100644 --- a/transformers/tokenization_xlm_roberta.py +++ b/transformers/tokenization_xlm_roberta.py @@ -61,7 +61,19 @@ class XLMRobertaTokenizer(PreTrainedTokenizer): self.sp_model = spm.SentencePieceProcessor() self.sp_model.Load(str(vocab_file)) self.vocab_file = vocab_file - self.fairseq_tokens_to_ids = {"": 0, "": 1, "": 2} + + # Original fairseq vocab and spm vocab must be "aligned": + # Vocab | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 + # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ---- + # fairseq | '' | '' | '' | '' | ',' | '.' | '▁' | 's' | '▁de' | '-' + # spm | '' | '' | '' | ',' | '.' | '▁' | 's' | '▁de' | '-' | '▁a' + + # Mimic fairseq token-to-id alignment for the first 4 token + self.fairseq_tokens_to_ids = {"": 0, "": 1, "": 2, "": 3} + + # The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab + self.fairseq_offset = 1 + self.fairseq_tokens_to_ids[''] = len(self.sp_model) + len(self.fairseq_tokens_to_ids) self.fairseq_ids_to_tokens = {v: k for k, v in self.fairseq_tokens_to_ids.items()} @@ -131,13 +143,13 @@ class XLMRobertaTokenizer(PreTrainedTokenizer): """ Converts a token (str/unicode) in an id using the vocab. """ if token in self.fairseq_tokens_to_ids: return self.fairseq_tokens_to_ids[token] - return self.sp_model.PieceToId(token) + 1 + return self.sp_model.PieceToId(token) + self.fairseq_offset def _convert_id_to_token(self, index): """Converts an index (integer) in a token (string/unicode) using the vocab.""" if index in self.fairseq_ids_to_tokens: return self.fairseq_ids_to_tokens[index] - return self.sp_model.IdToPiece(index + 1) + return self.sp_model.IdToPiece(index - self.fairseq_offset) def save_vocabulary(self, save_directory): """ Save the sentencepiece vocabulary (copy original file) and special tokens file From 01b68be34f906a210b0c0a5e2bd9bc605c5983f2 Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Wed, 18 Dec 2019 12:24:46 +0100 Subject: [PATCH 415/505] converter: remove XLM-RoBERTa specific script (can be done with the script for RoBERTa now) --- ..._original_pytorch_checkpoint_to_pytorch.py | 184 ------------------ 1 file changed, 184 deletions(-) delete mode 100644 transformers/convert_xlm_roberta_original_pytorch_checkpoint_to_pytorch.py diff --git a/transformers/convert_xlm_roberta_original_pytorch_checkpoint_to_pytorch.py b/transformers/convert_xlm_roberta_original_pytorch_checkpoint_to_pytorch.py deleted file mode 100644 index 884c273d2c..0000000000 --- a/transformers/convert_xlm_roberta_original_pytorch_checkpoint_to_pytorch.py +++ /dev/null @@ -1,184 +0,0 @@ -# coding=utf-8 -# Copyright 2018 The HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Convert RoBERTa checkpoint.""" - -from __future__ import absolute_import, division, print_function - -import argparse -import logging -import numpy as np -import torch -import pathlib - -from fairseq.models.roberta import RobertaModel as FairseqRobertaModel -from fairseq.modules import TransformerSentenceEncoderLayer -from transformers.modeling_bert import (BertConfig, BertEncoder, - BertIntermediate, BertLayer, - BertModel, BertOutput, - BertSelfAttention, - BertSelfOutput) -from transformers.modeling_roberta import (RobertaEmbeddings, - RobertaForMaskedLM, - RobertaForSequenceClassification, - RobertaModel) - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - -SAMPLE_TEXT = 'Hello world! cécé herlolip' - - -def convert_roberta_checkpoint_to_pytorch(roberta_checkpoint_path, pytorch_dump_folder_path, classification_head): - """ - Copy/paste/tweak roberta's weights to our BERT structure. - """ - roberta = FairseqRobertaModel.from_pretrained(roberta_checkpoint_path, bpe = 'sentencepiece') - roberta.eval() # disable dropout - config = BertConfig( - vocab_size=250002, - hidden_size=roberta.args.encoder_embed_dim, - num_hidden_layers=roberta.args.encoder_layers, - num_attention_heads=roberta.args.encoder_attention_heads, - intermediate_size=roberta.args.encoder_ffn_embed_dim, - max_position_embeddings=514, - type_vocab_size=1, - layer_norm_eps=1e-5, # PyTorch default used in fairseq - ) - if classification_head: - config.num_labels = roberta.args.num_classes - print("Our BERT config:", config) - - model = RobertaForSequenceClassification(config) if classification_head else RobertaForMaskedLM(config) - model.eval() - - # Now let's copy all the weights. - # Embeddings - roberta_sent_encoder = roberta.model.decoder.sentence_encoder - model.roberta.embeddings.word_embeddings.weight = roberta_sent_encoder.embed_tokens.weight - model.roberta.embeddings.position_embeddings.weight = roberta_sent_encoder.embed_positions.weight - model.roberta.embeddings.token_type_embeddings.weight.data = torch.zeros_like(model.roberta.embeddings.token_type_embeddings.weight) # just zero them out b/c RoBERTa doesn't use them. - model.roberta.embeddings.LayerNorm.weight = roberta_sent_encoder.emb_layer_norm.weight - model.roberta.embeddings.LayerNorm.bias = roberta_sent_encoder.emb_layer_norm.bias - - for i in range(config.num_hidden_layers): - # Encoder: start of layer - layer: BertLayer = model.roberta.encoder.layer[i] - roberta_layer: TransformerSentenceEncoderLayer = roberta_sent_encoder.layers[i] - - ### self attention - self_attn: BertSelfAttention = layer.attention.self - assert( - roberta_layer.self_attn.k_proj.weight.data.shape == \ - roberta_layer.self_attn.q_proj.weight.data.shape == \ - roberta_layer.self_attn.v_proj.weight.data.shape == \ - torch.Size((config.hidden_size, config.hidden_size)) - ) - - self_attn.query.weight.data = roberta_layer.self_attn.q_proj.weight - self_attn.query.bias.data = roberta_layer.self_attn.q_proj.bias - self_attn.key.weight.data = roberta_layer.self_attn.k_proj.weight - self_attn.key.bias.data = roberta_layer.self_attn.k_proj.bias - self_attn.value.weight.data = roberta_layer.self_attn.v_proj.weight - self_attn.value.bias.data = roberta_layer.self_attn.v_proj.bias - - ### self-attention output - self_output: BertSelfOutput = layer.attention.output - assert( - self_output.dense.weight.shape == roberta_layer.self_attn.out_proj.weight.shape - ) - self_output.dense.weight = roberta_layer.self_attn.out_proj.weight - self_output.dense.bias = roberta_layer.self_attn.out_proj.bias - self_output.LayerNorm.weight = roberta_layer.self_attn_layer_norm.weight - self_output.LayerNorm.bias = roberta_layer.self_attn_layer_norm.bias - - ### intermediate - intermediate: BertIntermediate = layer.intermediate - assert( - intermediate.dense.weight.shape == roberta_layer.fc1.weight.shape - ) - intermediate.dense.weight = roberta_layer.fc1.weight - intermediate.dense.bias = roberta_layer.fc1.bias - - ### output - bert_output: BertOutput = layer.output - assert( - bert_output.dense.weight.shape == roberta_layer.fc2.weight.shape - ) - bert_output.dense.weight = roberta_layer.fc2.weight - bert_output.dense.bias = roberta_layer.fc2.bias - bert_output.LayerNorm.weight = roberta_layer.final_layer_norm.weight - bert_output.LayerNorm.bias = roberta_layer.final_layer_norm.bias - #### end of layer - - if classification_head: - model.classifier.dense.weight = roberta.model.classification_heads['mnli'].dense.weight - model.classifier.dense.bias = roberta.model.classification_heads['mnli'].dense.bias - model.classifier.out_proj.weight = roberta.model.classification_heads['mnli'].out_proj.weight - model.classifier.out_proj.bias = roberta.model.classification_heads['mnli'].out_proj.bias - else: - # LM Head - model.lm_head.dense.weight = roberta.model.decoder.lm_head.dense.weight - model.lm_head.dense.bias = roberta.model.decoder.lm_head.dense.bias - model.lm_head.layer_norm.weight = roberta.model.decoder.lm_head.layer_norm.weight - model.lm_head.layer_norm.bias = roberta.model.decoder.lm_head.layer_norm.bias - model.lm_head.decoder.weight = roberta.model.decoder.lm_head.weight - model.lm_head.bias = roberta.model.decoder.lm_head.bias - - # Let's check that we get the same results. - input_ids: torch.Tensor = roberta.encode(SAMPLE_TEXT).unsqueeze(0) # batch of size 1 - - our_output = model(input_ids)[0] - if classification_head: - their_output = roberta.model.classification_heads['mnli'](roberta.extract_features(input_ids)) - else: - their_output = roberta.model(input_ids)[0] - print(our_output.shape, their_output.shape) - max_absolute_diff = torch.max(torch.abs(our_output - their_output)).item() - print(f"max_absolute_diff = {max_absolute_diff}") # ~ 1e-7 - success = torch.allclose(our_output, their_output, atol=1e-3) - print( - "Do both models output the same tensors?", - "🔥" if success else "💩" - ) - if not success: - raise Exception("Something went wRoNg") - - pathlib.Path(pytorch_dump_folder_path).mkdir(parents=True, exist_ok=True) - print(f"Saving model to {pytorch_dump_folder_path}") - model.save_pretrained(pytorch_dump_folder_path) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - ## Required parameters - parser.add_argument("--roberta_checkpoint_path", - default = None, - type = str, - required = True, - help = "Path the official PyTorch dump.") - parser.add_argument("--pytorch_dump_folder_path", - default = None, - type = str, - required = True, - help = "Path to the output PyTorch model.") - parser.add_argument("--classification_head", - action = "store_true", - help = "Whether to convert a final classification head.") - args = parser.parse_args() - convert_roberta_checkpoint_to_pytorch( - args.roberta_checkpoint_path, - args.pytorch_dump_folder_path, - args.classification_head - ) From 8efc6dd544bf1a30d99d4b5abfc5e214699eab2b Mon Sep 17 00:00:00 2001 From: Lysandre Date: Wed, 18 Dec 2019 10:47:59 -0500 Subject: [PATCH 416/505] fix #2214 --- transformers/configuration_xlm.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/transformers/configuration_xlm.py b/transformers/configuration_xlm.py index 0740cc4026..6839a45746 100644 --- a/transformers/configuration_xlm.py +++ b/transformers/configuration_xlm.py @@ -144,6 +144,9 @@ class XLMConfig(PretrainedConfig): self.start_n_top = start_n_top self.end_n_top = end_n_top + if "n_words" in kwargs: + self.n_words = kwargs["n_words"] + @property def n_words(self): # For backward compatibility return self.vocab_size From 0c88c856d592134ee5a9a636f9b73f40b91784b5 Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Wed, 18 Dec 2019 18:18:16 +0100 Subject: [PATCH 417/505] Unnest QuestionAnsweringArgumentHandler --- transformers/pipelines.py | 62 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 2 deletions(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index bcb4d9e054..a10078b027 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -333,6 +333,63 @@ class NerPipeline(Pipeline): return answers +class QuestionAnsweringArgumentHandler(ArgumentHandler): + """ + QuestionAnsweringPipeline requires the user to provide multiple arguments (i.e. question & context) to be mapped + to internal SquadExample / SquadFeature structures. + + QuestionAnsweringArgumentHandler manages all the possible to create SquadExample from the command-line supplied + arguments. + """ + def __call__(self, *args, **kwargs): + # Position args, handling is sensibly the same as X and data, so forwarding to avoid duplicating + if args is not None and len(args) > 0: + if len(args) == 1: + kwargs['X'] = args[0] + else: + kwargs['X'] = list(args) + + # Generic compatibility with sklearn and Keras + # Batched data + if 'X' in kwargs or 'data' in kwargs: + data = kwargs['X'] if 'X' in kwargs else kwargs['data'] + + if not isinstance(data, list): + data = [data] + + for i, item in enumerate(data): + if isinstance(item, dict): + if any(k not in item for k in ['question', 'context']): + raise KeyError('You need to provide a dictionary with keys {question:..., context:...}') + data[i] = QuestionAnsweringPipeline.create_sample(**item) + + elif isinstance(item, SquadExample): + continue + else: + raise ValueError( + '{} argument needs to be of type (list[SquadExample | dict], SquadExample, dict)' + .format('X' if 'X' in kwargs else 'data') + ) + inputs = data + + # Tabular input + elif 'question' in kwargs and 'context' in kwargs: + if isinstance(kwargs['question'], str): + kwargs['question'] = [kwargs['question']] + + if isinstance(kwargs['context'], str): + kwargs['context'] = [kwargs['context']] + + inputs = [QuestionAnsweringPipeline.create_sample(q, c) for q, c in zip(kwargs['question'], kwargs['context'])] + else: + raise ValueError('Unknown arguments {}'.format(kwargs)) + + if not isinstance(inputs, list): + inputs = [inputs] + + return inputs + + class QuestionAnsweringPipeline(Pipeline): """ Question Answering pipeline using ModelForQuestionAnswering head. @@ -403,8 +460,9 @@ class QuestionAnsweringPipeline(Pipeline): else: return SquadExample(None, question, context, None, None, None) - def __init__(self, model, tokenizer: Optional[PreTrainedTokenizer]): - super().__init__(model, tokenizer, args_parser=QuestionAnsweringPipeline.QuestionAnsweringArgumentHandler()) + def __init__(self, model, tokenizer: Optional[PreTrainedTokenizer], device: int = -1, **kwargs): + super().__init__(model, tokenizer, args_parser=QuestionAnsweringArgumentHandler(), + device=device, **kwargs) def inputs_for_model(self, features: Union[SquadExample, List[SquadExample]]) -> Dict: """ From 41a13a6375817ec3836bedcec67d12ad32bf0956 Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Wed, 18 Dec 2019 18:20:27 +0100 Subject: [PATCH 418/505] auto: add XLMRoBERTa to auto configuration --- transformers/configuration_auto.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/transformers/configuration_auto.py b/transformers/configuration_auto.py index 9fe58f173a..29e1a0ee1d 100644 --- a/transformers/configuration_auto.py +++ b/transformers/configuration_auto.py @@ -30,6 +30,7 @@ from .configuration_distilbert import DistilBertConfig, DISTILBERT_PRETRAINED_CO from .configuration_albert import AlbertConfig, ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_camembert import CamembertConfig, CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP from .configuration_t5 import T5Config, T5_PRETRAINED_CONFIG_ARCHIVE_MAP +from .configuration_xlm_roberta import XLMRobertaConfig, XLM_ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP logger = logging.getLogger(__name__) @@ -48,6 +49,7 @@ ALL_PRETRAINED_CONFIG_ARCHIVE_MAP = dict((key, value) ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, CAMEMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, T5_PRETRAINED_CONFIG_ARCHIVE_MAP, + XLM_ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, ] for key, value, in pretrained_map.items()) @@ -66,6 +68,7 @@ class AutoConfig(object): - contains `distilbert`: DistilBertConfig (DistilBERT model) - contains `albert`: AlbertConfig (ALBERT model) - contains `camembert`: CamembertConfig (CamemBERT model) + - contains `xlm-roberta`: XLMRobertaConfig (XLM-RoBERTa model) - contains `roberta`: RobertaConfig (RoBERTa model) - contains `bert`: BertConfig (Bert model) - contains `openai-gpt`: OpenAIGPTConfig (OpenAI GPT model) @@ -91,6 +94,7 @@ class AutoConfig(object): - contains `distilbert`: DistilBertConfig (DistilBERT model) - contains `albert`: AlbertConfig (ALBERT model) - contains `camembert`: CamembertConfig (CamemBERT model) + - contains `xlm-roberta`: XLMRobertaConfig (XLM-RoBERTa model) - contains `roberta`: RobertaConfig (RoBERTa model) - contains `bert`: BertConfig (Bert model) - contains `openai-gpt`: OpenAIGPTConfig (OpenAI GPT model) @@ -152,6 +156,8 @@ class AutoConfig(object): return AlbertConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) elif 'camembert' in pretrained_model_name_or_path: return CamembertConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) + elif 'xlm-roberta' in pretrained_model_name_or_path: + return XLMRobertaConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return RobertaConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) elif 'bert' in pretrained_model_name_or_path: @@ -170,4 +176,4 @@ class AutoConfig(object): return CTRLConfig.from_pretrained(pretrained_model_name_or_path, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', " - "'xlm', 'roberta', 'distilbert', 'camembert', 'ctrl', 'albert'".format(pretrained_model_name_or_path)) + "'xlm-roberta', 'xlm', 'roberta', 'distilbert', 'camembert', 'ctrl', 'albert'".format(pretrained_model_name_or_path)) From 036831e2791b84edb5a89db7a92af7c69f4ff37e Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Wed, 18 Dec 2019 18:23:42 +0100 Subject: [PATCH 419/505] auto: add XLM-RoBERTa to audo modeling --- transformers/modeling_auto.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/transformers/modeling_auto.py b/transformers/modeling_auto.py index 1a30ea4623..ca6c4525b8 100644 --- a/transformers/modeling_auto.py +++ b/transformers/modeling_auto.py @@ -30,6 +30,7 @@ from .modeling_distilbert import DistilBertModel, DistilBertForQuestionAnswering from .modeling_camembert import CamembertModel, CamembertForMaskedLM, CamembertForSequenceClassification, CamembertForMultipleChoice, CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP from .modeling_albert import AlbertModel, AlbertForMaskedLM, AlbertForSequenceClassification, AlbertForQuestionAnswering, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP from .modeling_t5 import T5Model, T5WithLMHeadModel, T5_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_xlm_roberta import XLMRobertaModel, XLMRobertaForMaskedLM, XLMRobertaForSequenceClassification, XLMRobertaForMultipleChoice, XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP from .modeling_utils import PreTrainedModel, SequenceSummary @@ -52,6 +53,7 @@ ALL_PRETRAINED_MODEL_ARCHIVE_MAP = dict((key, value) ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP, CAMEMBERT_PRETRAINED_MODEL_ARCHIVE_MAP, T5_PRETRAINED_MODEL_ARCHIVE_MAP, + XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, ] for key, value, in pretrained_map.items()) @@ -72,6 +74,7 @@ class AutoModel(object): - contains `distilbert`: DistilBertModel (DistilBERT model) - contains `albert`: AlbertModel (ALBERT model) - contains `camembert`: CamembertModel (CamemBERT model) + - contains `xlm-roberta`: XLMRobertaModel (XLM-RoBERTa model) - contains `roberta`: RobertaModel (RoBERTa model) - contains `bert`: BertModel (Bert model) - contains `openai-gpt`: OpenAIGPTModel (OpenAI GPT model) @@ -98,6 +101,7 @@ class AutoModel(object): - contains `distilbert`: DistilBertModel (DistilBERT model) - contains `albert`: AlbertModel (ALBERT model) - contains `camembert`: CamembertModel (CamemBERT model) + - contains `xlm-roberta`: XLMRobertaModel (XLM-RoBERTa model) - contains `roberta`: RobertaModel (RoBERTa model) - contains `bert`: BertModel (Bert model) - contains `openai-gpt`: OpenAIGPTModel (OpenAI GPT model) @@ -175,6 +179,8 @@ class AutoModel(object): return AlbertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'camembert' in pretrained_model_name_or_path: return CamembertModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'xlm-roberta' in pretrained_model_name_or_path: + return XLMRobertaModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return RobertaModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'bert' in pretrained_model_name_or_path: @@ -193,7 +199,7 @@ class AutoModel(object): return CTRLModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', " - "'xlm', 'roberta, 'ctrl', 'distilbert', 'camembert', 'albert'".format(pretrained_model_name_or_path)) + "'xlm-roberta', 'xlm', 'roberta, 'ctrl', 'distilbert', 'camembert', 'albert'".format(pretrained_model_name_or_path)) class AutoModelWithLMHead(object): @@ -212,6 +218,7 @@ class AutoModelWithLMHead(object): - contains `distilbert`: DistilBertForMaskedLM (DistilBERT model) - contains `albert`: AlbertForMaskedLM (ALBERT model) - contains `camembert`: CamembertForMaskedLM (CamemBERT model) + - contains `xlm-roberta`: XLMRobertaForMaskedLM (XLM-RoBERTa model) - contains `roberta`: RobertaForMaskedLM (RoBERTa model) - contains `bert`: BertForMaskedLM (Bert model) - contains `openai-gpt`: OpenAIGPTLMHeadModel (OpenAI GPT model) @@ -241,6 +248,7 @@ class AutoModelWithLMHead(object): - contains `distilbert`: DistilBertForMaskedLM (DistilBERT model) - contains `albert`: AlbertForMaskedLM (ALBERT model) - contains `camembert`: CamembertForMaskedLM (CamemBERT model) + - contains `xlm-roberta`: XLMRobertaForMaskedLM (XLM-RoBERTa model) - contains `roberta`: RobertaForMaskedLM (RoBERTa model) - contains `bert`: BertForMaskedLM (Bert model) - contains `openai-gpt`: OpenAIGPTLMHeadModel (OpenAI GPT model) @@ -317,6 +325,8 @@ class AutoModelWithLMHead(object): return AlbertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'camembert' in pretrained_model_name_or_path: return CamembertForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'xlm-roberta' in pretrained_model_name_or_path: + return XLMRobertaForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return RobertaForMaskedLM.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'bert' in pretrained_model_name_or_path: @@ -335,7 +345,7 @@ class AutoModelWithLMHead(object): return CTRLLMHeadModel.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', " - "'xlm', 'roberta','ctrl', 'distilbert', 'camembert', 'albert'".format(pretrained_model_name_or_path)) + "'xlm-roberta', 'xlm', 'roberta','ctrl', 'distilbert', 'camembert', 'albert'".format(pretrained_model_name_or_path)) class AutoModelForSequenceClassification(object): @@ -353,6 +363,7 @@ class AutoModelForSequenceClassification(object): - contains `distilbert`: DistilBertForSequenceClassification (DistilBERT model) - contains `albert`: AlbertForSequenceClassification (ALBERT model) - contains `camembert`: CamembertForSequenceClassification (CamemBERT model) + - contains `xlm-roberta`: XLMRobertaForSequenceClassification (XLM-RoBERTa model) - contains `roberta`: RobertaForSequenceClassification (RoBERTa model) - contains `bert`: BertForSequenceClassification (Bert model) - contains `xlnet`: XLNetForSequenceClassification (XLNet model) @@ -377,6 +388,7 @@ class AutoModelForSequenceClassification(object): - contains `distilbert`: DistilBertForSequenceClassification (DistilBERT model) - contains `albert`: AlbertForSequenceClassification (ALBERT model) - contains `camembert`: CamembertForSequenceClassification (CamemBERT model) + - contains `xlm-roberta`: XLMRobertaForSequenceClassification (XLM-RoBERTa model) - contains `roberta`: RobertaForSequenceClassification (RoBERTa model) - contains `bert`: BertForSequenceClassification (Bert model) - contains `xlnet`: XLNetForSequenceClassification (XLNet model) @@ -448,6 +460,8 @@ class AutoModelForSequenceClassification(object): return AlbertForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'camembert' in pretrained_model_name_or_path: return CamembertForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'xlm-roberta' in pretrained_model_name_or_path: + return XLMRobertaForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return RobertaForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'bert' in pretrained_model_name_or_path: @@ -458,7 +472,7 @@ class AutoModelForSequenceClassification(object): return XLMForSequenceClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " - "'bert', 'xlnet', 'xlm', 'roberta', 'distilbert', 'camembert', 'albert'".format(pretrained_model_name_or_path)) + "'bert', 'xlnet', 'xlm-roberta', 'xlm', 'roberta', 'distilbert', 'camembert', 'albert'".format(pretrained_model_name_or_path)) class AutoModelForQuestionAnswering(object): From 64a971a9156788ed6d95f850453578ecb74069c5 Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Wed, 18 Dec 2019 18:24:32 +0100 Subject: [PATCH 420/505] auto: add XLM-RoBERTa to auto tokenization --- transformers/tokenization_auto.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/transformers/tokenization_auto.py b/transformers/tokenization_auto.py index 173dee0e2b..5377bd48cb 100644 --- a/transformers/tokenization_auto.py +++ b/transformers/tokenization_auto.py @@ -31,6 +31,7 @@ from .tokenization_distilbert import DistilBertTokenizer from .tokenization_camembert import CamembertTokenizer from .tokenization_albert import AlbertTokenizer from .tokenization_t5 import T5Tokenizer +from .tokenization_xlm_roberta import XLMRobertaTokenizer logger = logging.getLogger(__name__) @@ -49,6 +50,7 @@ class AutoTokenizer(object): - contains `distilbert`: DistilBertTokenizer (DistilBert model) - contains `albert`: AlbertTokenizer (ALBERT model) - contains `camembert`: CamembertTokenizer (CamemBERT model) + - contains `xlm-roberta`: XLMRobertaTokenizer (XLM-RoBERTa model) - contains `roberta`: RobertaTokenizer (RoBERTa model) - contains `bert`: BertTokenizer (Bert model) - contains `openai-gpt`: OpenAIGPTTokenizer (OpenAI GPT model) @@ -75,6 +77,7 @@ class AutoTokenizer(object): - contains `distilbert`: DistilBertTokenizer (DistilBert model) - contains `albert`: AlbertTokenizer (ALBERT model) - contains `camembert`: CamembertTokenizer (CamemBERT model) + - contains `xlm-roberta`: XLMRobertaTokenizer (XLM-RoBERTa model) - contains `roberta`: RobertaTokenizer (RoBERTa model) - contains `bert-base-japanese`: BertJapaneseTokenizer (Bert model) - contains `bert`: BertTokenizer (Bert model) @@ -130,6 +133,8 @@ class AutoTokenizer(object): return AlbertTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) elif 'camembert' in pretrained_model_name_or_path: return CamembertTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) + elif 'xlm-roberta' in pretrained_model_name_or_path: + return XLMRobertaTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return RobertaTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) elif 'bert-base-japanese' in pretrained_model_name_or_path: @@ -150,4 +155,4 @@ class AutoTokenizer(object): return CTRLTokenizer.from_pretrained(pretrained_model_name_or_path, *inputs, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " "'bert', 'openai-gpt', 'gpt2', 'transfo-xl', 'xlnet', " - "'xlm', 'roberta', 'distilbert,' 'camembert', 'ctrl', 'albert'".format(pretrained_model_name_or_path)) + "'xlm-roberta', 'xlm', 'roberta', 'distilbert,' 'camembert', 'ctrl', 'albert'".format(pretrained_model_name_or_path)) From 04b602f96f91529a1259909466705f3f9192113c Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Wed, 18 Dec 2019 18:28:39 +0100 Subject: [PATCH 421/505] Put module import on top of the module. --- transformers/pipelines.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index a10078b027..92c94268a7 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -20,6 +20,7 @@ import os from abc import ABC, abstractmethod from contextlib import contextmanager from itertools import groupby +from os.path import abspath, exists from typing import Union, Optional, Tuple, List, Dict import numpy as np @@ -100,12 +101,12 @@ class PipelineDataFormat: if self.is_multi_columns: self.column = [tuple(c.split('=')) if '=' in c else (c, c) for c in self.column] - from os.path import abspath, exists - if exists(abspath(self.output)): - raise OSError('{} already exists on disk'.format(self.output)) + if output is not None: + if exists(abspath(self.output)): + raise OSError('{} already exists on disk'.format(self.output)) - if not exists(abspath(self.path)): - raise OSError('{} doesnt exist on disk'.format(self.path)) + if not exists(abspath(self.path)): + raise OSError('{} doesnt exist on disk'.format(self.path)) @abstractmethod def __iter__(self): From e778dd854dd5d1fd29396d214577ddbe0f854247 Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Wed, 18 Dec 2019 19:27:34 +0100 Subject: [PATCH 422/505] modeling: add XLM-RoBERTa base model --- transformers/modeling_xlm_roberta.py | 1 + 1 file changed, 1 insertion(+) diff --git a/transformers/modeling_xlm_roberta.py b/transformers/modeling_xlm_roberta.py index 4c833c69ff..abace25d5b 100644 --- a/transformers/modeling_xlm_roberta.py +++ b/transformers/modeling_xlm_roberta.py @@ -27,6 +27,7 @@ from .file_utils import add_start_docstrings logger = logging.getLogger(__name__) XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP = { + 'xlm-roberta-base': "https://schweter.eu/cloud/transformers/xlm-roberta-base-pytorch_model.bin", 'xlm-roberta-large': "https://schweter.eu/cloud/transformers/xlm-roberta-large-pytorch_model.bin", } From 128cfdee9bdd13f2f0bbef977907ff014ae398ad Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Wed, 18 Dec 2019 19:28:16 +0100 Subject: [PATCH 423/505] tokenization add XLM-RoBERTa base model --- transformers/tokenization_xlm_roberta.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/transformers/tokenization_xlm_roberta.py b/transformers/tokenization_xlm_roberta.py index d8484e7f9c..93b1c397e4 100644 --- a/transformers/tokenization_xlm_roberta.py +++ b/transformers/tokenization_xlm_roberta.py @@ -30,11 +30,13 @@ VOCAB_FILES_NAMES = {'vocab_file': 'sentencepiece.bpe.model'} PRETRAINED_VOCAB_FILES_MAP = { 'vocab_file': { + 'xlm-roberta-base': "https://schweter.eu/cloud/transformers/xlm-roberta-base-sentencepiece.bpe.model", 'xlm-roberta-large': "https://schweter.eu/cloud/transformers/xlm-roberta-large-sentencepiece.bpe.model", } } PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { + 'xlm-roberta-base': None, 'xlm-roberta-large': None, } From 3e89fca54359bff728782cce4157118caed33c09 Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Wed, 18 Dec 2019 19:44:23 +0100 Subject: [PATCH 424/505] readme: add XLM-RoBERTa to model architecture list --- README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c33a65bdbb..a9d0fb3ace 100644 --- a/README.md +++ b/README.md @@ -146,7 +146,8 @@ At some point in the future, you'll be able to seamlessly move from pre-training 10. **[CamemBERT](https://camembert-model.fr)** (from Inria/Facebook/Sorbonne) released with the paper [CamemBERT: a Tasty French Language Model](https://arxiv.org/abs/1911.03894) by Louis Martin*, Benjamin Muller*, Pedro Javier Ortiz Suárez*, Yoann Dupont, Laurent Romary, Éric Villemonte de la Clergerie, Djamé Seddah and Benoît Sagot. 11. **[ALBERT](https://github.com/google-research/ALBERT)** (from Google Research and the Toyota Technological Institute at Chicago) released with the paper [ALBERT: A Lite BERT for Self-supervised Learning of Language Representations](https://arxiv.org/abs/1909.11942), by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. 12. **[T5](https://github.com/google-research/text-to-text-transfer-transformer)** (from Google AI) released with the paper [Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer](https://arxiv.org/abs/1910.10683) by Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu. -13. Want to contribute a new model? We have added a **detailed guide and templates** to guide you in the process of adding a new model. You can find them in the [`templates`](./templates) folder of the repository. Be sure to check the [contributing guidelines](./CONTRIBUTING.md) and contact the maintainers or open an issue to collect feedbacks before starting your PR. +13. **[XLM-RoBERTa](https://github.com/pytorch/fairseq/tree/master/examples/xlmr)** (from Facebook AI), released together with the paper [Unsupervised Cross-lingual Representation Learning at Scale](https://arxiv.org/abs/1911.02116) by Alexis Conneau*, Kartikay Khandelwal*, Naman Goyal, Vishrav Chaudhary, Guillaume Wenzek, Francisco Guzmán, Edouard Grave, Myle Ott, Luke Zettlemoyer and Veselin Stoyanov. +14. Want to contribute a new model? We have added a **detailed guide and templates** to guide you in the process of adding a new model. You can find them in the [`templates`](./templates) folder of the repository. Be sure to check the [contributing guidelines](./CONTRIBUTING.md) and contact the maintainers or open an issue to collect feedbacks before starting your PR. These implementations have been tested on several datasets (see the example scripts) and should match the performances of the original implementations (e.g. ~93 F1 on SQuAD for BERT Whole-Word-Masking, ~88 F1 on RocStories for OpenAI GPT, ~18.3 perplexity on WikiText 103 for Transformer-XL, ~0.916 Peason R coefficient on STS-B for XLNet). You can find more details on the performances in the Examples section of the [documentation](https://huggingface.co/transformers/examples.html). @@ -168,7 +169,7 @@ import torch from transformers import * # Transformers has a unified API -# for 8 transformer architectures and 30 pretrained weights. +# for 10 transformer architectures and 30 pretrained weights. # Model | Tokenizer | Pretrained weights shortcut MODELS = [(BertModel, BertTokenizer, 'bert-base-uncased'), (OpenAIGPTModel, OpenAIGPTTokenizer, 'openai-gpt'), @@ -178,7 +179,9 @@ MODELS = [(BertModel, BertTokenizer, 'bert-base-uncased'), (XLNetModel, XLNetTokenizer, 'xlnet-base-cased'), (XLMModel, XLMTokenizer, 'xlm-mlm-enfr-1024'), (DistilBertModel, DistilBertTokenizer, 'distilbert-base-uncased'), - (RobertaModel, RobertaTokenizer, 'roberta-base')] + (RobertaModel, RobertaTokenizer, 'roberta-base'), + (XLMRobertaModel, XLMRobertaTokenizer, 'xlm-roberta-base'), + ] # To use TensorFlow 2.0 versions of the models, simply prefix the class names with 'TF', e.g. `TFRobertaModel` is the TF 2.0 counterpart of the PyTorch model `RobertaModel` From d35405b7a32eadf4fb1200249b2bbc4c12fb0340 Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Wed, 18 Dec 2019 19:45:10 +0100 Subject: [PATCH 425/505] docs: add XLM-RoBERTa to index page --- docs/source/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/index.rst b/docs/source/index.rst index 48282c1c6c..cb34c5c7f0 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -50,6 +50,7 @@ The library currently contains PyTorch and Tensorflow implementations, pre-train 9. `CTRL `_ (from Salesforce), released together with the paper `CTRL: A Conditional Transformer Language Model for Controllable Generation `_ by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher. 10. `CamemBERT `_ (from FAIR, Inria, Sorbonne Université) released together with the paper `CamemBERT: a Tasty French Language Model `_ by Louis Martin, Benjamin Muller, Pedro Javier Ortiz Suarez, Yoann Dupont, Laurent Romary, Eric Villemonte de la Clergerie, Djame Seddah, and Benoît Sagot. 11. `ALBERT `_ (from Google Research), released together with the paper a `ALBERT: A Lite BERT for Self-supervised Learning of Language Representations `_ by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. +13. `XLM-RoBERTa `_ (from Facebook AI), released together with the paper `Unsupervised Cross-lingual Representation Learning at Scale `_ by Alexis Conneau*, Kartikay Khandelwal*, Naman Goyal, Vishrav Chaudhary, Guillaume Wenzek, Francisco Guzmán, Edouard Grave, Myle Ott, Luke Zettlemoyer and Veselin Stoyanov. .. toctree:: :maxdepth: 2 From dd7a958fd6963d09850ad4842307d1d1064d096d Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Wed, 18 Dec 2019 19:45:46 +0100 Subject: [PATCH 426/505] docs: add XLM-RoBERTa to pretrained model list (incl. all parameters) --- docs/source/pretrained_models.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst index 7d037da34f..a359990f5a 100644 --- a/docs/source/pretrained_models.rst +++ b/docs/source/pretrained_models.rst @@ -240,6 +240,12 @@ Here is the full list of the currently provided pretrained models together with | | ``t5-11B`` | | ~11B parameters with 24-layers, 1024-hidden-state, 65536 feed-forward hidden-state, 128-heads, | | | | | Trained on English text: the Colossal Clean Crawled Corpus (C4) | +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| XLM-RoBERTa | ``xlm-roberta-base`` | | ~125M parameters with 12-layers, 768-hidden-state, 3072 feed-forward hidden-state, 8-heads, | +| | | | Trained on on 2.5 TB of newly created clean CommonCrawl data in 100 languages | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``xlm-roberta-large`` | | ~355M parameters with 24-layers, 1027-hidden-state, 4096 feed-forward hidden-state, 16-heads, | +| | | | Trained on 2.5 TB of newly created clean CommonCrawl data in 100 languages | ++-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ .. `__ From f09d9996413f2b265f1c672d7a4b438e4c5099c4 Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Wed, 18 Dec 2019 19:49:33 +0100 Subject: [PATCH 427/505] =?UTF-8?q?docs:=20fix=20numbering=20=F0=9F=98=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index cb34c5c7f0..0ac9c740a5 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -50,7 +50,7 @@ The library currently contains PyTorch and Tensorflow implementations, pre-train 9. `CTRL `_ (from Salesforce), released together with the paper `CTRL: A Conditional Transformer Language Model for Controllable Generation `_ by Nitish Shirish Keskar*, Bryan McCann*, Lav R. Varshney, Caiming Xiong and Richard Socher. 10. `CamemBERT `_ (from FAIR, Inria, Sorbonne Université) released together with the paper `CamemBERT: a Tasty French Language Model `_ by Louis Martin, Benjamin Muller, Pedro Javier Ortiz Suarez, Yoann Dupont, Laurent Romary, Eric Villemonte de la Clergerie, Djame Seddah, and Benoît Sagot. 11. `ALBERT `_ (from Google Research), released together with the paper a `ALBERT: A Lite BERT for Self-supervised Learning of Language Representations `_ by Zhenzhong Lan, Mingda Chen, Sebastian Goodman, Kevin Gimpel, Piyush Sharma, Radu Soricut. -13. `XLM-RoBERTa `_ (from Facebook AI), released together with the paper `Unsupervised Cross-lingual Representation Learning at Scale `_ by Alexis Conneau*, Kartikay Khandelwal*, Naman Goyal, Vishrav Chaudhary, Guillaume Wenzek, Francisco Guzmán, Edouard Grave, Myle Ott, Luke Zettlemoyer and Veselin Stoyanov. +12. `XLM-RoBERTa `_ (from Facebook AI), released together with the paper `Unsupervised Cross-lingual Representation Learning at Scale `_ by Alexis Conneau*, Kartikay Khandelwal*, Naman Goyal, Vishrav Chaudhary, Guillaume Wenzek, Francisco Guzmán, Edouard Grave, Myle Ott, Luke Zettlemoyer and Veselin Stoyanov. .. toctree:: :maxdepth: 2 From 8bb166db5de426ecfb2c3bf1160307324419bbda Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Wed, 18 Dec 2019 22:53:19 +0100 Subject: [PATCH 428/505] Expose more information in the output of TextClassificationPipeline --- transformers/pipelines.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index 92c94268a7..809096a30c 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -283,7 +283,9 @@ class TextClassificationPipeline(Pipeline): self._nb_classes = nb_classes def __call__(self, *args, **kwargs): - return super().__call__(*args, **kwargs).tolist() + outputs = super().__call__(*args, **kwargs) + scores = np.exp(outputs) / np.exp(outputs).sum(-1) + return [{'label': self.model.config.id2label[item.argmax()], 'score': item.max()} for item in scores] class NerPipeline(Pipeline): From 7711403bbdefad62e7ee88a88e04ec08b53412bc Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Wed, 18 Dec 2019 22:59:51 +0100 Subject: [PATCH 429/505] Expose config through the cli arguments --- transformers/commands/run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/transformers/commands/run.py b/transformers/commands/run.py index 8c203699a8..44c1127803 100644 --- a/transformers/commands/run.py +++ b/transformers/commands/run.py @@ -16,7 +16,7 @@ def try_infer_format_from_ext(path: str): def run_command_factory(args): - nlp = pipeline(task=args.task, model=args.model, tokenizer=args.tokenizer, device=args.device) + nlp = pipeline(task=args.task, model=args.model, config=args.config, tokenizer=args.tokenizer, device=args.device) format = try_infer_format_from_ext(args.input) if args.format == 'infer' else args.format reader = PipelineDataFormat.from_str(format, args.output, args.input, args.column) return RunCommand(nlp, reader) @@ -34,6 +34,7 @@ class RunCommand(BaseTransformersCLICommand): run_parser.add_argument('--device', type=int, default=-1, help='Indicate the device to run onto, -1 indicates CPU, >= 0 indicates GPU (default: -1)') run_parser.add_argument('--task', choices=SUPPORTED_TASKS.keys(), help='Task to run') run_parser.add_argument('--model', type=str, required=True, help='Name or path to the model to instantiate.') + run_parser.add_argument('--config', type=str, help='Name or path to the model\'s config to instantiate.') run_parser.add_argument('--tokenizer', type=str, help='Name of the tokenizer to use. (default: same as the model name)') run_parser.add_argument('--column', type=str, required=True, help='Name of the column to use as input. (For multi columns input as QA use column1,columns2)') run_parser.add_argument('--format', type=str, default='infer', choices=PipelineDataFormat.SUPPORTED_FORMATS, help='Input format to read from') From d0724d0794e930eb5821d4289130beade3359e87 Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Wed, 18 Dec 2019 23:27:26 +0100 Subject: [PATCH 430/505] Add PipedPipelineDataFormat --- transformers/__init__.py | 4 +- transformers/commands/run.py | 7 +- transformers/pipelines.py | 154 +++++++++++++++-------------------- 3 files changed, 70 insertions(+), 95 deletions(-) diff --git a/transformers/__init__.py b/transformers/__init__.py index c474696062..87ede9f6a8 100755 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -191,8 +191,8 @@ from .modeling_tf_pytorch_utils import (convert_tf_weight_name_to_pt_weight_name load_tf2_model_in_pytorch_model) # Pipelines -# from .pipeline_ import TextClassificationPipeline -from .pipelines import Pipeline, pipeline, TextClassificationPipeline +from .pipelines import pipeline, PipelineDataFormat, CsvPipelineDataFormat, JsonPipelineDataFormat, PipedPipelineDataFormat, \ + Pipeline, FeatureExtractionPipeline, QuestionAnsweringPipeline, NerPipeline, TextClassificationPipeline if not is_tf_available() and not is_torch_available(): logger.warning("Neither PyTorch nor TensorFlow >= 2.0 have been found." diff --git a/transformers/commands/run.py b/transformers/commands/run.py index 44c1127803..7c00c0057f 100644 --- a/transformers/commands/run.py +++ b/transformers/commands/run.py @@ -36,11 +36,10 @@ class RunCommand(BaseTransformersCLICommand): run_parser.add_argument('--model', type=str, required=True, help='Name or path to the model to instantiate.') run_parser.add_argument('--config', type=str, help='Name or path to the model\'s config to instantiate.') run_parser.add_argument('--tokenizer', type=str, help='Name of the tokenizer to use. (default: same as the model name)') - run_parser.add_argument('--column', type=str, required=True, help='Name of the column to use as input. (For multi columns input as QA use column1,columns2)') + run_parser.add_argument('--column', type=str, help='Name of the column to use as input. (For multi columns input as QA use column1,columns2)') run_parser.add_argument('--format', type=str, default='infer', choices=PipelineDataFormat.SUPPORTED_FORMATS, help='Input format to read from') - run_parser.add_argument('--input', type=str, required=True, help='Path to the file to use for inference') - run_parser.add_argument('--output', type=str, required=True, help='Path to the file that will be used post to write results.') - run_parser.add_argument('kwargs', nargs='*', help='Arguments to forward to the file format reader') + run_parser.add_argument('--input', type=str, help='Path to the file to use for inference') + run_parser.add_argument('--output', type=str, help='Path to the file that will be used post to write results.') run_parser.set_defaults(func=run_command_factory) def run(self): diff --git a/transformers/pipelines.py b/transformers/pipelines.py index 809096a30c..9e7051b70a 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -66,20 +66,6 @@ class DefaultArgumentHandler(ArgumentHandler): raise ValueError('Unable to infer the format of the provided data (X=, data=, ...)') -class _ScikitCompat(ABC): - """ - Interface layer for the Scikit and Keras compatibility. - """ - - @abstractmethod - def transform(self, X): - raise NotImplementedError() - - @abstractmethod - def predict(self, X): - raise NotImplementedError() - - class PipelineDataFormat: """ Base class for all the pipeline supported data format both for reading and writing. @@ -90,12 +76,12 @@ class PipelineDataFormat: PipelineDataFormat also includes some utilities to work with multi-columns like mapping from datasets columns to pipelines keyword arguments through the `dataset_kwarg_1=dataset_column_1` format. """ - SUPPORTED_FORMATS = ['json', 'csv'] + SUPPORTED_FORMATS = ['json', 'csv', 'pipe'] - def __init__(self, output: str, path: str, column: str): + def __init__(self, output: Optional[str], path: Optional[str], column: Optional[str]): self.output = output self.path = path - self.column = column.split(',') + self.column = column.split(',') if column else [''] self.is_multi_columns = len(self.column) > 1 if self.is_multi_columns: @@ -117,17 +103,19 @@ class PipelineDataFormat: raise NotImplementedError() @staticmethod - def from_str(name: str, output: str, path: str, column: str): + def from_str(name: str, output: Optional[str], path: Optional[str], column: Optional[str]): if name == 'json': return JsonPipelineDataFormat(output, path, column) elif name == 'csv': return CsvPipelineDataFormat(output, path, column) + elif name == 'pipe': + return PipedPipelineDataFormat(output, path, column) else: - raise KeyError('Unknown reader {} (Available reader are json/csv)'.format(name)) + raise KeyError('Unknown reader {} (Available reader are json/csv/pipe)'.format(name)) class CsvPipelineDataFormat(PipelineDataFormat): - def __init__(self, output: str, path: str, column: str): + def __init__(self, output: Optional[str], path: Optional[str], column: Optional[str]): super().__init__(output, path, column) def __iter__(self): @@ -148,7 +136,7 @@ class CsvPipelineDataFormat(PipelineDataFormat): class JsonPipelineDataFormat(PipelineDataFormat): - def __init__(self, output: str, path: str, column: str): + def __init__(self, output: Optional[str], path: Optional[str], column: Optional[str]): super().__init__(output, path, column) with open(path, 'r') as f: @@ -166,6 +154,50 @@ class JsonPipelineDataFormat(PipelineDataFormat): json.dump(data, f) +class PipedPipelineDataFormat(PipelineDataFormat): + """ + Read data from piped input to the python process. + For multi columns data, columns should separated by \t + + If columns are provided, then the output will be a dictionary with {column_x: value_x} + """ + def __iter__(self): + import sys + for line in sys.stdin: + + # Split for multi-columns + if '\t' in line: + + line = line.split('\t') + if self.column: + # Dictionary to map arguments + yield {kwargs: l for (kwargs, _), l in zip(self.column, line)} + else: + yield tuple(line) + + # No dictionary to map arguments + else: + print(line) + yield line + + def save(self, data: dict): + print(data) + + +class _ScikitCompat(ABC): + """ + Interface layer for the Scikit and Keras compatibility. + """ + + @abstractmethod + def transform(self, X): + raise NotImplementedError() + + @abstractmethod + def predict(self, X): + raise NotImplementedError() + + class Pipeline(_ScikitCompat): """ Base class implementing pipelined operations. @@ -208,18 +240,6 @@ class Pipeline(_ScikitCompat): """ return self(X=X) - def __call__(self, *texts, **kwargs): - # Parse arguments - inputs = self._args_parser(*texts, **kwargs) - - # Encode for forward - with self.device_placement(): - inputs = self.tokenizer.batch_encode_plus( - inputs, add_special_tokens=True, return_tensors='tf' if is_tf_available() else 'pt' - ) - - return self._forward(inputs) - @contextmanager def device_placement(self): """ @@ -244,6 +264,18 @@ class Pipeline(_ScikitCompat): yield + def __call__(self, *texts, **kwargs): + # Parse arguments + inputs = self._args_parser(*texts, **kwargs) + + # Encode for forward + with self.device_placement(): + inputs = self.tokenizer.batch_encode_plus( + inputs, add_special_tokens=True, return_tensors='tf' if is_tf_available() else 'pt' + ) + + return self._forward(inputs) + def _forward(self, inputs): """ Internal framework specific forward dispatching. @@ -275,12 +307,6 @@ class TextClassificationPipeline(Pipeline): """ Text classification pipeline using ModelForTextClassification head. """ - def __init__(self, model, tokenizer: PreTrainedTokenizer, nb_classes: int = 2): - super().__init__(model, tokenizer) - - if nb_classes < 2: - raise Exception('Invalid parameter nb_classes. int >= 2 is required (got: {})'.format(nb_classes)) - self._nb_classes = nb_classes def __call__(self, *args, **kwargs): outputs = super().__call__(*args, **kwargs) @@ -398,56 +424,6 @@ class QuestionAnsweringPipeline(Pipeline): Question Answering pipeline using ModelForQuestionAnswering head. """ - class QuestionAnsweringArgumentHandler(ArgumentHandler): - - def __call__(self, *args, **kwargs): - # Position args, handling is sensibly the same as X and data, so forwarding to avoid duplicating - if args is not None and len(args) > 0: - if len(args) == 1: - kwargs['X'] = args[0] - else: - kwargs['X'] = list(args) - - # Generic compatibility with sklearn and Keras - # Batched data - if 'X' in kwargs or 'data' in kwargs: - data = kwargs['X'] if 'X' in kwargs else kwargs['data'] - - if not isinstance(data, list): - data = [data] - - for i, item in enumerate(data): - if isinstance(item, dict): - if any(k not in item for k in ['question', 'context']): - raise KeyError('You need to provide a dictionary with keys {question:..., context:...}') - data[i] = QuestionAnsweringPipeline.create_sample(**item) - - elif isinstance(item, SquadExample): - continue - else: - raise ValueError( - '{} argument needs to be of type (list[SquadExample | dict], SquadExample, dict)' - .format('X' if 'X' in kwargs else 'data') - ) - inputs = data - - # Tabular input - elif 'question' in kwargs and 'context' in kwargs: - if isinstance(kwargs['question'], str): - kwargs['question'] = [kwargs['question']] - - if isinstance(kwargs['context'], str): - kwargs['context'] = [kwargs['context']] - - inputs = [QuestionAnsweringPipeline.create_sample(q, c) for q, c in zip(kwargs['question'], kwargs['context'])] - else: - raise ValueError('Unknown arguments {}'.format(kwargs)) - - if not isinstance(inputs, list): - inputs = [inputs] - - return inputs - @staticmethod def create_sample(question: Union[str, List[str]], context: Union[str, List[str]]) -> Union[SquadExample, List[SquadExample]]: """ From db90e1211433ef99f952e60fbe5ea578391b86b1 Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Wed, 18 Dec 2019 23:46:33 +0100 Subject: [PATCH 431/505] configuration: use S3 location for XLM-RoBERTa model --- transformers/configuration_xlm_roberta.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/transformers/configuration_xlm_roberta.py b/transformers/configuration_xlm_roberta.py index dd03572976..d7a26538c5 100644 --- a/transformers/configuration_xlm_roberta.py +++ b/transformers/configuration_xlm_roberta.py @@ -25,7 +25,8 @@ from .configuration_roberta import RobertaConfig logger = logging.getLogger(__name__) XLM_ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP = { - 'xlm-roberta-large': "https://schweter.eu/cloud/transformers/xlm-roberta-large-config.json", + 'xlm-roberta-base': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-base-config.json", + 'xlm-roberta-large': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-large-config.json", } From 5c5f67a256b558b470f03cf36edc5ea35dec2fba Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Wed, 18 Dec 2019 23:47:00 +0100 Subject: [PATCH 432/505] modeling: use S3 location for XLM-RoBERTa model --- transformers/modeling_xlm_roberta.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transformers/modeling_xlm_roberta.py b/transformers/modeling_xlm_roberta.py index abace25d5b..8095c46a16 100644 --- a/transformers/modeling_xlm_roberta.py +++ b/transformers/modeling_xlm_roberta.py @@ -27,8 +27,8 @@ from .file_utils import add_start_docstrings logger = logging.getLogger(__name__) XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP = { - 'xlm-roberta-base': "https://schweter.eu/cloud/transformers/xlm-roberta-base-pytorch_model.bin", - 'xlm-roberta-large': "https://schweter.eu/cloud/transformers/xlm-roberta-large-pytorch_model.bin", + 'xlm-roberta-base': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-base-pytorch_model.bin", + 'xlm-roberta-large': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-large-pytorch_model.bin", } From fe9aab1055604e772be05a1cbbb36a207c177055 Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Wed, 18 Dec 2019 23:47:48 +0100 Subject: [PATCH 433/505] tokenization: use S3 location for XLM-RoBERTa model --- transformers/tokenization_xlm_roberta.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transformers/tokenization_xlm_roberta.py b/transformers/tokenization_xlm_roberta.py index 93b1c397e4..453c4375c6 100644 --- a/transformers/tokenization_xlm_roberta.py +++ b/transformers/tokenization_xlm_roberta.py @@ -30,8 +30,8 @@ VOCAB_FILES_NAMES = {'vocab_file': 'sentencepiece.bpe.model'} PRETRAINED_VOCAB_FILES_MAP = { 'vocab_file': { - 'xlm-roberta-base': "https://schweter.eu/cloud/transformers/xlm-roberta-base-sentencepiece.bpe.model", - 'xlm-roberta-large': "https://schweter.eu/cloud/transformers/xlm-roberta-large-sentencepiece.bpe.model", + 'xlm-roberta-base': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-base-sentencepiece.bpe.model", + 'xlm-roberta-large': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-large-sentencepiece.bpe.model", } } From ec5d6c6a70e0bcdf31e737206caa1e56e859b2d3 Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Thu, 19 Dec 2019 00:12:10 +0100 Subject: [PATCH 434/505] Adressing issue with NER task omitting first and last word. --- transformers/pipelines.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index 9e7051b70a..1d8f226b13 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -318,8 +318,6 @@ class NerPipeline(Pipeline): """ Named Entity Recognition pipeline using ModelForTokenClassification head. """ - def __init__(self, model, tokenizer: PreTrainedTokenizer): - super().__init__(model, tokenizer) def __call__(self, *texts, **kwargs): inputs, answers = self._args_parser(*texts, **kwargs), [] @@ -344,14 +342,16 @@ class NerPipeline(Pipeline): # Normalize scores answer, token_start = [], 1 - for idx, word in groupby(token_to_word[1:-1]): + for idx, word in groupby(token_to_word): # Sum log prob over token, then normalize across labels score = np.exp(entities[token_start]) / np.exp(entities[token_start]).sum(-1, keepdims=True) label_idx = score.argmax() answer += [{ - 'word': words[idx - 1], 'score': score[label_idx].item(), 'entity': self.model.config.id2label[label_idx] + 'word': words[idx], + 'score': score[label_idx].item(), + 'entity': self.model.config.id2label[label_idx] }] # Update token start From a26ce4dee116a1d5d9099c8a94e22d1e31ad631c Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Thu, 19 Dec 2019 02:23:01 +0100 Subject: [PATCH 435/505] examples: add XLM-RoBERTa to glue script --- examples/run_glue.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/examples/run_glue.py b/examples/run_glue.py index 1a51255c11..954a8fbf0c 100644 --- a/examples/run_glue.py +++ b/examples/run_glue.py @@ -52,6 +52,9 @@ from transformers import (WEIGHTS_NAME, BertConfig, AlbertConfig, AlbertForSequenceClassification, AlbertTokenizer, + XLMRobertaConfig, + XLMRobertaForSequenceClassification, + XLMRobertaTokenizer, ) from transformers import AdamW, get_linear_schedule_with_warmup @@ -72,7 +75,8 @@ MODEL_CLASSES = { 'xlm': (XLMConfig, XLMForSequenceClassification, XLMTokenizer), 'roberta': (RobertaConfig, RobertaForSequenceClassification, RobertaTokenizer), 'distilbert': (DistilBertConfig, DistilBertForSequenceClassification, DistilBertTokenizer), - 'albert': (AlbertConfig, AlbertForSequenceClassification, AlbertTokenizer) + 'albert': (AlbertConfig, AlbertForSequenceClassification, AlbertTokenizer), + 'xlmroberta': (XLMRobertaConfig, XLMRobertaForSequenceClassification, XLMRobertaTokenizer), } @@ -304,9 +308,9 @@ def load_and_cache_examples(args, task, tokenizer, evaluate=False): else: logger.info("Creating features from dataset file at %s", args.data_dir) label_list = processor.get_labels() - if task in ['mnli', 'mnli-mm'] and args.model_type in ['roberta']: + if task in ['mnli', 'mnli-mm'] and args.model_type in ['roberta', 'xlmroberta']: # HACK(label indices are swapped in RoBERTa pretrained model) - label_list[1], label_list[2] = label_list[2], label_list[1] + label_list[1], label_list[2] = label_list[2], label_list[1] examples = processor.get_dev_examples(args.data_dir) if evaluate else processor.get_train_examples(args.data_dir) features = convert_examples_to_features(examples, tokenizer, From bcc99fd92efe03ff332e5b26a342a09e9d709cf7 Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Thu, 19 Dec 2019 10:32:21 +0100 Subject: [PATCH 436/505] Fix wrong automatic config allocation through AutoConfig --- transformers/pipelines.py | 117 +++++++++++++++++++++++++------------- 1 file changed, 78 insertions(+), 39 deletions(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index 1d8f226b13..ee3aed2c65 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -25,7 +25,7 @@ from typing import Union, Optional, Tuple, List, Dict import numpy as np -from transformers import AutoTokenizer, PreTrainedTokenizer, PretrainedConfig, \ +from transformers import AutoConfig, AutoTokenizer, PreTrainedTokenizer, PretrainedConfig, \ SquadExample, squad_convert_examples_to_features, is_tf_available, is_torch_available, logger if is_tf_available(): @@ -264,6 +264,27 @@ class Pipeline(_ScikitCompat): yield + def inputs_for_model(self, features: Union[dict, List[dict]]) -> Dict: + """ + Generates the input dictionary with model-specific parameters. + + Returns: + dict holding all the required parameters for model's forward + """ + args = ['input_ids', 'attention_mask'] + model_type = type(self.model).__name__.lower() + + if 'distilbert' not in model_type and 'xlm' not in model_type: + args += ['token_type_ids'] + + if 'xlnet' in model_type or 'xlm' in model_type: + args += ['cls_index', 'p_mask'] + + if isinstance(features, dict): + return {k: features[k] for k in args} + else: + return {k: [feature[k] for feature in features] for k in args} + def __call__(self, *texts, **kwargs): # Parse arguments inputs = self._args_parser(*texts, **kwargs) @@ -271,9 +292,14 @@ class Pipeline(_ScikitCompat): # Encode for forward with self.device_placement(): inputs = self.tokenizer.batch_encode_plus( - inputs, add_special_tokens=True, return_tensors='tf' if is_tf_available() else 'pt' + inputs, add_special_tokens=True, + return_tensors='tf' if is_tf_available() else 'pt', + # max_length=self.model.config.max_position_embedding + max_length=511 ) + # Filter out features not available on specific models + inputs = self.inputs_for_model(inputs) return self._forward(inputs) def _forward(self, inputs): @@ -331,7 +357,11 @@ class NerPipeline(Pipeline): # Manage correct placement of the tensors with self.device_placement(): - tokens = self.tokenizer.encode_plus(sentence, return_attention_mask=False, return_tensors='tf' if is_tf_available() else 'pt') + tokens = self.tokenizer.encode_plus( + sentence, return_attention_mask=False, + return_tensors='tf' if is_tf_available() else 'pt', + max_length=512 + ) # Forward if is_torch_available(): @@ -443,27 +473,6 @@ class QuestionAnsweringPipeline(Pipeline): super().__init__(model, tokenizer, args_parser=QuestionAnsweringArgumentHandler(), device=device, **kwargs) - def inputs_for_model(self, features: Union[SquadExample, List[SquadExample]]) -> Dict: - """ - Generates the input dictionary with model-specific parameters. - - Returns: - dict holding all the required parameters for model's forward - """ - args = ['input_ids', 'attention_mask'] - model_type = type(self.model).__name__.lower() - - if 'distilbert' not in model_type and 'xlm' not in model_type: - args += ['token_type_ids'] - - if 'xlnet' in model_type or 'xlm' in model_type: - args += ['cls_index', 'p_mask'] - - if isinstance(features, SquadExample): - return {k: features.__dict__[k] for k in args} - else: - return {k: [feature.__dict__[k] for feature in features] for k in args} - def __call__(self, *texts, **kwargs): """ Args: @@ -495,7 +504,7 @@ class QuestionAnsweringPipeline(Pipeline): # Convert inputs to features examples = self._args_parser(*texts, **kwargs) features = squad_convert_examples_to_features(examples, self.tokenizer, kwargs['max_seq_len'], kwargs['doc_stride'], kwargs['max_question_len'], False) - fw_args = self.inputs_for_model(features) + fw_args = self.inputs_for_model(features.__dict__) # Manage tensor allocation on correct device with self.device_placement(): @@ -627,29 +636,50 @@ class QuestionAnsweringPipeline(Pipeline): # Register all the supported task here SUPPORTED_TASKS = { 'feature-extraction': { - 'impl': FeatureExtractionPipeline, - 'tf': TFAutoModel if is_tf_available() else None, - 'pt': AutoModel if is_torch_available() else None, + 'impl': FeatureExtractionPipeline, + 'tf': TFAutoModel if is_tf_available() else None, + 'pt': AutoModel if is_torch_available() else None, + 'default': { + 'model': 'distilbert-base-uncased', + 'config': None, + 'tokenizer': 'bert-base-uncased' + } }, - 'text-classification': { + 'sentiment-analysis': { 'impl': TextClassificationPipeline, 'tf': TFAutoModelForSequenceClassification if is_tf_available() else None, - 'pt': AutoModelForSequenceClassification if is_torch_available() else None + 'pt': AutoModelForSequenceClassification if is_torch_available() else None, + 'default': { + 'model': 'https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-finetuned-sst-2-english-pytorch_model.bin', + 'config': 'https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-finetuned-sst-2-english-config.json', + 'tokenizer': 'bert-base-uncased' + } }, 'ner': { - 'impl': NerPipeline, - 'tf': TFAutoModelForTokenClassification if is_tf_available() else None, - 'pt': AutoModelForTokenClassification if is_torch_available() else None, + 'impl': NerPipeline, + 'tf': TFAutoModelForTokenClassification if is_tf_available() else None, + 'pt': AutoModelForTokenClassification if is_torch_available() else None, + 'default': { + 'model': 'https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-finetuned-conll03-english-pytorch_model.bin', + 'config': 'https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-finetuned-conll03-english-config.json', + 'tokenizer': 'bert-base-cased' + } }, 'question-answering': { 'impl': QuestionAnsweringPipeline, 'tf': TFAutoModelForQuestionAnswering if is_tf_available() else None, - 'pt': AutoModelForQuestionAnswering if is_torch_available() else None + 'pt': AutoModelForQuestionAnswering if is_torch_available() else None, + 'default': { + 'model': 'distilbert-base-uncased-distilled-squad', + 'config': None, + 'tokenizer': 'bert-base-uncased' + } } } -def pipeline(task: str, model, config: Optional[Union[str, PretrainedConfig]] = None, +def pipeline(task: str, model: Optional = None, + config: Optional[Union[str, PretrainedConfig]] = None, tokenizer: Optional[Union[str, PreTrainedTokenizer]] = None, **kwargs) -> Pipeline: """ Utility factory method to build a pipeline. @@ -657,23 +687,32 @@ def pipeline(task: str, model, config: Optional[Union[str, PretrainedConfig]] = A Tokenizer instance in charge of mapping raw textual input to token A Model instance Some (optional) post processing for enhancing model's output + + Examples: + pipeline('ner') """ # Try to infer tokenizer from model name (if provided as str) if tokenizer is None: - if not isinstance(model, str): + if model is not None and not isinstance(model, str): # Impossible to guest what is the right tokenizer here raise Exception('Tokenizer cannot be None if provided model is a PreTrainedModel instance') else: tokenizer = model - tokenizer = tokenizer if isinstance(tokenizer, PreTrainedTokenizer) else AutoTokenizer.from_pretrained(tokenizer) - + # Retrieve the task if task not in SUPPORTED_TASKS: raise KeyError("Unknown task {}, available tasks are {}".format(task, list(SUPPORTED_TASKS.keys()))) targeted_task = SUPPORTED_TASKS[task] task, allocator = targeted_task['impl'], targeted_task['tf'] if is_tf_available() else targeted_task['pt'] + # Handling for default model for the task + if model is None: + model, config, tokenizer = tuple(targeted_task['default'].values()) + + # Allocate tokenizer + tokenizer = tokenizer if isinstance(tokenizer, PreTrainedTokenizer) else AutoTokenizer.from_pretrained(tokenizer) + # Special handling for model conversion if isinstance(model, str): from_tf = model.endswith('.h5') and not is_tf_available() @@ -689,7 +728,7 @@ def pipeline(task: str, model, config: Optional[Union[str, PretrainedConfig]] = from_tf = from_pt = False if isinstance(config, str): - config = PretrainedConfig.from_pretrained(config) + config = AutoConfig.from_pretrained(config) if allocator.__name__.startswith('TF'): model = allocator.from_pretrained(model, config=config, from_pt=from_pt) From d72fa2a0f67f9c55877826f767f01007507fd8e3 Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Thu, 19 Dec 2019 10:54:10 +0100 Subject: [PATCH 437/505] Fix inputs_for_model call in QuestionAnsweringPipeline accessing __dict__ on list. --- transformers/pipelines.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index ee3aed2c65..a220aa7e71 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -504,7 +504,7 @@ class QuestionAnsweringPipeline(Pipeline): # Convert inputs to features examples = self._args_parser(*texts, **kwargs) features = squad_convert_examples_to_features(examples, self.tokenizer, kwargs['max_seq_len'], kwargs['doc_stride'], kwargs['max_question_len'], False) - fw_args = self.inputs_for_model(features.__dict__) + fw_args = self.inputs_for_model([f.__dict__ for f in features]) # Manage tensor allocation on correct device with self.device_placement(): From f516cf39564bd33bd26e6a51d6cf9f589e600078 Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Thu, 19 Dec 2019 11:42:33 +0100 Subject: [PATCH 438/505] Allow pipeline to write output in binary format --- transformers/commands/run.py | 10 +++++++++- transformers/pipelines.py | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/transformers/commands/run.py b/transformers/commands/run.py index 7c00c0057f..78109b2a16 100644 --- a/transformers/commands/run.py +++ b/transformers/commands/run.py @@ -1,9 +1,13 @@ +import logging from argparse import ArgumentParser from transformers.commands import BaseTransformersCLICommand from transformers.pipelines import pipeline, Pipeline, PipelineDataFormat, SUPPORTED_TASKS +logger = logging.getLogger(__name__) # pylint: disable=invalid-name + + def try_infer_format_from_ext(path: str): for ext in PipelineDataFormat.SUPPORTED_FORMATS: if path.endswith(ext): @@ -51,7 +55,11 @@ class RunCommand(BaseTransformersCLICommand): output += [nlp(entry)] # Saving data - self._reader.save(output) + if self._nlp.binary_output: + binary_path = self._reader.save_binary(output) + logger.warning('Current pipeline requires output to be in binary format, saving at {}'.format(binary_path)) + else: + self._reader.save(output) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index a220aa7e71..b766549121 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -17,6 +17,7 @@ from __future__ import absolute_import, division, print_function, unicode_litera import csv import json import os +import pickle from abc import ABC, abstractmethod from contextlib import contextmanager from itertools import groupby @@ -91,6 +92,7 @@ class PipelineDataFormat: if exists(abspath(self.output)): raise OSError('{} already exists on disk'.format(self.output)) + if path is not None: if not exists(abspath(self.path)): raise OSError('{} doesnt exist on disk'.format(self.path)) @@ -102,6 +104,15 @@ class PipelineDataFormat: def save(self, data: dict): raise NotImplementedError() + def save_binary(self, data: Union[dict, List[dict]]) -> str: + path, _ = os.path.splitext(self.output) + binary_path = os.path.extsep.join((path, 'pickle')) + + with open(binary_path, 'wb+') as f_output: + pickle.dump(data, f_output) + + return binary_path + @staticmethod def from_str(name: str, output: Optional[str], path: Optional[str], column: Optional[str]): if name == 'json': @@ -177,12 +188,20 @@ class PipedPipelineDataFormat(PipelineDataFormat): # No dictionary to map arguments else: - print(line) yield line def save(self, data: dict): print(data) + def save_binary(self, data: Union[dict, List[dict]]) -> str: + if self.output is None: + raise KeyError( + 'When using piped input on pipeline outputting large object requires an output file path. ' + 'Please provide such output path through --output argument.' + ) + + return super().save_binary(data) + class _ScikitCompat(ABC): """ @@ -205,11 +224,13 @@ class Pipeline(_ScikitCompat): Input -> Tokenization -> Model Inference -> Post-Processing (Task dependent) -> Output """ def __init__(self, model, tokenizer: PreTrainedTokenizer = None, - args_parser: ArgumentHandler = None, device: int = -1, **kwargs): + args_parser: ArgumentHandler = None, device: int = -1, + binary_output: bool = False): self.model = model self.tokenizer = tokenizer self.device = device + self.binary_output = binary_output self._args_parser = args_parser or DefaultArgumentHandler() # Special handling @@ -325,6 +346,13 @@ class FeatureExtractionPipeline(Pipeline): """ Feature extraction pipeline using Model head. """ + + def __init__(self, model, + tokenizer: PreTrainedTokenizer = None, + args_parser: ArgumentHandler = None, + device: int = -1): + super().__init__(model, tokenizer, args_parser, device, binary_output=True) + def __call__(self, *args, **kwargs): return super().__call__(*args, **kwargs).tolist() From fc624716aad05efa78e4d65f214fb978ea0ac9e7 Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Thu, 19 Dec 2019 11:49:06 +0100 Subject: [PATCH 439/505] Renaming framework env variables flags from NO_ to USE_ --- transformers/file_utils.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/transformers/file_utils.py b/transformers/file_utils.py index 4784681fb4..c586c57a76 100644 --- a/transformers/file_utils.py +++ b/transformers/file_utils.py @@ -27,8 +27,9 @@ from contextlib import contextmanager logger = logging.getLogger(__name__) # pylint: disable=invalid-name try: - if 'NO_TF' in os.environ and os.environ['NO_TF'].upper() in ('1', 'ON'): - logger.info("Found NO_TF, disabling TensorFlow") + os.environ.setdefault('USE_TF', 'YES') + if os.environ['USE_TF'].upper() in ('1', 'ON', 'YES'): + logger.info("USE_TF override through env variable, disabling Tensorflow") _tf_available = False else: import tensorflow as tf @@ -39,8 +40,9 @@ except (ImportError, AssertionError): _tf_available = False # pylint: disable=invalid-name try: - if 'NO_TORCH' in os.environ and os.environ['NO_TORCH'].upper() in ('1', 'ON'): - logger.info("Found NO_TORCH, disabling PyTorch") + os.environ.setdefault('USE_TORCH', 'YES') + if os.environ['USE_TORCH'].upper() in ('1', 'ON', 'YES'): + logger.info("USE_TORCH override through env variable, disabling PyTorch") _torch_available = False else: import torch From 3b29322d4c197fec46ae48f62aa2870d00d0852a Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Thu, 19 Dec 2019 12:24:17 +0100 Subject: [PATCH 440/505] Expose all the pipeline argument on serve command. --- transformers/commands/serving.py | 38 +++++++++++++------------------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/transformers/commands/serving.py b/transformers/commands/serving.py index a35dff0ebe..a7321470ce 100644 --- a/transformers/commands/serving.py +++ b/transformers/commands/serving.py @@ -17,25 +17,18 @@ def serve_command_factory(args: Namespace): Factory function used to instantiate serving server from provided command line arguments. :return: ServeCommand """ - nlp = pipeline(args.task, args.model) - return ServeCommand(nlp, args.host, args.port, args.model, args.graphql) + nlp = pipeline(task=args.task, model=args.model, config=args.config, tokenizer=args.tokenizer, device=args.device) + return ServeCommand(nlp, args.host, args.port) -class ServeResult(BaseModel): - """ - Base class for serving result - """ - model: str - - -class ServeModelInfoResult(ServeResult): +class ServeModelInfoResult(BaseModel): """ Expose model information """ infos: dict -class ServeTokenizeResult(ServeResult): +class ServeTokenizeResult(BaseModel): """ Tokenize result model """ @@ -43,14 +36,14 @@ class ServeTokenizeResult(ServeResult): tokens_ids: Optional[List[int]] -class ServeDeTokenizeResult(ServeResult): +class ServeDeTokenizeResult(BaseModel): """ DeTokenize result model """ text: str -class ServeForwardResult(ServeResult): +class ServeForwardResult(BaseModel): """ Forward result model """ @@ -71,11 +64,12 @@ class ServeCommand(BaseTransformersCLICommand): serve_parser.add_argument('--device', type=int, default=-1, help='Indicate the device to run onto, -1 indicates CPU, >= 0 indicates GPU (default: -1)') serve_parser.add_argument('--host', type=str, default='localhost', help='Interface the server will listen on.') serve_parser.add_argument('--port', type=int, default=8888, help='Port the serving will listen to.') - serve_parser.add_argument('--model', type=str, required=True, help='Model\'s name or path to stored model to infer from.') - serve_parser.add_argument('--graphql', action='store_true', default=False, help='Enable GraphQL endpoints.') + serve_parser.add_argument('--model', type=str, required=True, help='Model\'s name or path to stored model.') + serve_parser.add_argument('--config', type=str, help='Model\'s config name or path to stored model.') + serve_parser.add_argument('--tokenizer', type=str, help='Tokenizer name to use.') serve_parser.set_defaults(func=serve_command_factory) - def __init__(self, pipeline: Pipeline, host: str, port: int, model: str, graphql: bool): + def __init__(self, pipeline: Pipeline, host: str, port: int): self._logger = getLogger('transformers-cli/serving') self._pipeline = pipeline @@ -95,7 +89,7 @@ class ServeCommand(BaseTransformersCLICommand): run(self._app, host=self._host, port=self._port) def model_info(self): - return ServeModelInfoResult(model='', infos=vars(self._pipeline.model.config)) + return ServeModelInfoResult(infos=vars(self._pipeline.model.config)) def tokenize(self, text_input: str = Body(None, embed=True), return_ids: bool = Body(False, embed=True)): """ @@ -108,9 +102,9 @@ class ServeCommand(BaseTransformersCLICommand): if return_ids: tokens_ids = self._pipeline.tokenizer.convert_tokens_to_ids(tokens_txt) - return ServeTokenizeResult(model='', tokens=tokens_txt, tokens_ids=tokens_ids) + return ServeTokenizeResult(tokens=tokens_txt, tokens_ids=tokens_ids) else: - return ServeTokenizeResult(model='', tokens=tokens_txt) + return ServeTokenizeResult(tokens=tokens_txt) except Exception as e: raise HTTPException(status_code=500, detail={"model": '', "error": str(e)}) @@ -139,13 +133,11 @@ class ServeCommand(BaseTransformersCLICommand): # Check we don't have empty string if len(inputs) == 0: - return ServeForwardResult(model='', output=[], attention=[]) + return ServeForwardResult(output=[], attention=[]) try: # Forward through the model output = self._pipeline(inputs) - return ServeForwardResult( - model='', output=output - ) + return ServeForwardResult(output=output) except Exception as e: raise HTTPException(500, {"error": str(e)}) From 5664327c24730b30b52d84e71e9af7c1d32b0fe1 Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Thu, 19 Dec 2019 12:27:54 +0100 Subject: [PATCH 441/505] Hide train command for now. --- transformers-cli | 2 -- 1 file changed, 2 deletions(-) diff --git a/transformers-cli b/transformers-cli index 39b7f5816b..db2bd0e2a3 100755 --- a/transformers-cli +++ b/transformers-cli @@ -5,7 +5,6 @@ from transformers.commands.download import DownloadCommand from transformers.commands.run import RunCommand from transformers.commands.serving import ServeCommand from transformers.commands.user import UserCommands -from transformers.commands.train import TrainCommand from transformers.commands.convert import ConvertCommand if __name__ == '__main__': @@ -17,7 +16,6 @@ if __name__ == '__main__': DownloadCommand.register_subcommand(commands_parser) RunCommand.register_subcommand(commands_parser) ServeCommand.register_subcommand(commands_parser) - TrainCommand.register_subcommand(commands_parser) UserCommands.register_subcommand(commands_parser) # Let's go From faef6f6191a8d319b541396a0f850c3d6f15f5d4 Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Thu, 19 Dec 2019 12:28:17 +0100 Subject: [PATCH 442/505] Fix logic order for USE_TF/USE_TORCH --- transformers/file_utils.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/transformers/file_utils.py b/transformers/file_utils.py index c586c57a76..92cada85b3 100644 --- a/transformers/file_utils.py +++ b/transformers/file_utils.py @@ -29,25 +29,27 @@ logger = logging.getLogger(__name__) # pylint: disable=invalid-name try: os.environ.setdefault('USE_TF', 'YES') if os.environ['USE_TF'].upper() in ('1', 'ON', 'YES'): - logger.info("USE_TF override through env variable, disabling Tensorflow") - _tf_available = False - else: import tensorflow as tf assert hasattr(tf, '__version__') and int(tf.__version__[0]) >= 2 _tf_available = True # pylint: disable=invalid-name logger.info("TensorFlow version {} available.".format(tf.__version__)) + else: + logger.info("USE_TF override through env variable, disabling Tensorflow") + _tf_available = False + except (ImportError, AssertionError): _tf_available = False # pylint: disable=invalid-name try: os.environ.setdefault('USE_TORCH', 'YES') if os.environ['USE_TORCH'].upper() in ('1', 'ON', 'YES'): - logger.info("USE_TORCH override through env variable, disabling PyTorch") - _torch_available = False - else: import torch _torch_available = True # pylint: disable=invalid-name logger.info("PyTorch version {} available.".format(torch.__version__)) + + else: + logger.info("USE_TORCH override through env variable, disabling PyTorch") + _torch_available = False except ImportError: _torch_available = False # pylint: disable=invalid-name From 81a911cce5d659ef66eddff288489f25fc195f16 Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Thu, 19 Dec 2019 15:12:06 +0100 Subject: [PATCH 443/505] Doc, doc, ... doc. --- transformers/pipelines.py | 62 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 58 insertions(+), 4 deletions(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index b766549121..71e6d0fbed 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -102,9 +102,19 @@ class PipelineDataFormat: @abstractmethod def save(self, data: dict): + """ + Save the provided data object with the representation for the current `DataFormat`. + :param data: data to store + :return: + """ raise NotImplementedError() def save_binary(self, data: Union[dict, List[dict]]) -> str: + """ + Save the provided data object as a pickle-formatted binary data on the disk. + :param data: data to store + :return: (str) Path where the data has been saved + """ path, _ = os.path.splitext(self.output) binary_path = os.path.extsep.join((path, 'pickle')) @@ -222,6 +232,42 @@ class Pipeline(_ScikitCompat): Base class implementing pipelined operations. Pipeline workflow is defined as a sequence of the following operations: Input -> Tokenization -> Model Inference -> Post-Processing (Task dependent) -> Output + + Pipeline supports running on CPU or GPU through the device argument. Users can specify + device argument as an integer, -1 meaning "CPU", >= 0 referring the CUDA device ordinal. + + Some pipeline, like for instance FeatureExtractionPipeline ('feature-extraction') outputs large + tensor object as nested-lists. In order to avoid dumping such large structure as textual data we + provide the binary_output constructor argument. If set to True, the output will be stored in the + pickle format. + + Arguments: + **model**: ``(str, PretrainedModel, TFPretrainedModel)``: + Reference to the model to use through this pipeline. + + **tokenizer**: ``(str, PreTrainedTokenizer)``: + Reference to the tokenizer to use through this pipeline. + + **args_parser**: ``ArgumentHandler``: + Reference to the object in charge of parsing supplied pipeline parameters. + + **device**: ``int``: + Device ordinal for CPU/GPU supports. Setting this to -1 will leverage CPU, >=0 will run the model + on the associated CUDA device id. + + **binary_output** ``bool`` (default: False): + Flag indicating if the output the pipeline should happen in a binary format (i.e. pickle) or as raw text. + + Return: + Pipeline returns list or dictionary depending on: + - Does the user provided multiple sample + - The pipeline expose multiple fields in the output object + + Examples: + nlp = pipeline('ner') + nlp = pipeline('ner', model='...', config='...', tokenizer='...') + nlp = NerPipeline(model='...', config='...', tokenizer='...') + nlp = QuestionAnsweringPipeline(model=AutoModel.from_pretrained('...'), tokenizer='...') """ def __init__(self, model, tokenizer: PreTrainedTokenizer = None, args_parser: ArgumentHandler = None, device: int = -1, @@ -312,11 +358,11 @@ class Pipeline(_ScikitCompat): # Encode for forward with self.device_placement(): + # TODO : Remove this 512 hard-limit inputs = self.tokenizer.batch_encode_plus( inputs, add_special_tokens=True, return_tensors='tf' if is_tf_available() else 'pt', - # max_length=self.model.config.max_position_embedding - max_length=511 + max_length=512 ) # Filter out features not available on specific models @@ -385,6 +431,8 @@ class NerPipeline(Pipeline): # Manage correct placement of the tensors with self.device_placement(): + + # TODO : Remove this 512 hard-limit tokens = self.tokenizer.encode_plus( sentence, return_attention_mask=False, return_tensors='tf' if is_tf_available() else 'pt', @@ -488,9 +536,12 @@ class QuestionAnsweringPipeline(Pipeline): QuestionAnsweringPipeline leverages the SquadExample/SquadFeatures internally. This helper method encapsulate all the logic for converting question(s) and context(s) to SquadExample(s). We currently support extractive question answering. - Args: + Arguments: question: (str, List[str]) The question to be ask for the associated context context: (str, List[str]) The context in which we will look for the answer. + + Returns: + SquadExample initialized with the corresponding question and context. """ if isinstance(question, list): return [SquadExample(None, q, c, None, None, None) for q, c in zip(question, context)] @@ -717,7 +768,10 @@ def pipeline(task: str, model: Optional = None, Some (optional) post processing for enhancing model's output Examples: - pipeline('ner') + pipeline('sentiment-analysis') + pipeline('question-answering', model='distilbert-base-uncased-distilled-squad', tokenizer='bert-base-cased') + pipeline('ner', model=AutoModel.from_pretrained(...), tokenizer=AutoTokenizer.from_pretrained(...) + pipeline('ner', model='https://...pytorch-model.bin', config='https://...config.json', tokenizer='bert-base-cased') """ # Try to infer tokenizer from model name (if provided as str) if tokenizer is None: From ed6ba93912d223886fe0b88dd4ee58b20774beaf Mon Sep 17 00:00:00 2001 From: patrickvonplaten Date: Thu, 19 Dec 2019 01:26:01 +0100 Subject: [PATCH 444/505] corrected typo in example for t5 model input argument --- transformers/modeling_t5.py | 4 ++-- transformers/modeling_tf_t5.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/transformers/modeling_t5.py b/transformers/modeling_t5.py index 263dc33b70..9baf69d02b 100644 --- a/transformers/modeling_t5.py +++ b/transformers/modeling_t5.py @@ -693,7 +693,7 @@ class T5Model(T5PreTrainedModel): tokenizer = T5Tokenizer.from_pretrained('t5-small') model = T5Model.from_pretrained('t5-small') input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 - outputs = model(input_ids) + outputs = model(input_ids=input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple """ @@ -798,7 +798,7 @@ class T5WithLMHeadModel(T5PreTrainedModel): tokenizer = T5Tokenizer.from_pretrained('t5-small') model = T5WithLMHeadModel.from_pretrained('t5-small') input_ids = torch.tensor(tokenizer.encode("Hello, my dog is cute")).unsqueeze(0) # Batch size 1 - outputs = model(input_ids, lm_labels=input_ids) + outputs = model(input_ids=input_ids, lm_labels=input_ids) loss, prediction_scores = outputs[:2] """ diff --git a/transformers/modeling_tf_t5.py b/transformers/modeling_tf_t5.py index 1336a1c30d..e803e00c8d 100644 --- a/transformers/modeling_tf_t5.py +++ b/transformers/modeling_tf_t5.py @@ -610,7 +610,7 @@ class TFT5Model(TFT5PreTrainedModel): tokenizer = T5Tokenizer.from_pretrained('t5-small') model = TFT5Model.from_pretrained('t5-small') input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 - outputs = model(input_ids) + outputs = model(input_ids=input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple """ @@ -701,7 +701,7 @@ class TFT5WithLMHeadModel(TFT5PreTrainedModel): tokenizer = T5Tokenizer.from_pretrained('t5-small') model = TFT5WithLMHeadModel.from_pretrained('t5-small') input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 - outputs = model(input_ids) + outputs = model(input_ids=input_ids) prediction_scores = outputs[0] """ From 284572efc05a6a8d9e351e886ea3cab0f5f2367a Mon Sep 17 00:00:00 2001 From: Ejar Date: Wed, 18 Dec 2019 17:47:47 +0100 Subject: [PATCH 445/505] Updated typo on the link Updated documentation due to typo --- examples/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/README.md b/examples/README.md index b6b3908810..fcd2fe1f6f 100644 --- a/examples/README.md +++ b/examples/README.md @@ -467,7 +467,7 @@ Training with the previously defined hyper-parameters yields the following resul ## Named Entity Recognition Based on the scripts [`run_ner.py`](https://github.com/huggingface/transformers/blob/master/examples/run_ner.py) for Pytorch and -[`run_tf_ner.py`(https://github.com/huggingface/transformers/blob/master/examples/run_tf_ner.py)] for Tensorflow 2. +[`run_tf_ner.py`](https://github.com/huggingface/transformers/blob/master/examples/run_tf_ner.py) for Tensorflow 2. This example fine-tune Bert Multilingual on GermEval 2014 (German NER). Details and results for the fine-tuning provided by @stefan-it. From 62c1fc3c1ecdfab787ee3c34d1ec1eba65c18877 Mon Sep 17 00:00:00 2001 From: Francesco Date: Thu, 19 Dec 2019 14:43:10 +0100 Subject: [PATCH 446/505] Removed duplicate XLMConfig, XLMForQuestionAnswering and XLMTokenizer from import statement of run_squad.py script --- examples/run_squad.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/run_squad.py b/examples/run_squad.py index 34c31c3bb8..1ff6983f62 100644 --- a/examples/run_squad.py +++ b/examples/run_squad.py @@ -61,7 +61,6 @@ MODEL_CLASSES = { 'xlm': (XLMConfig, XLMForQuestionAnswering, XLMTokenizer), 'distilbert': (DistilBertConfig, DistilBertForQuestionAnswering, DistilBertTokenizer), 'albert': (AlbertConfig, AlbertForQuestionAnswering, AlbertTokenizer), - 'xlm': (XLMConfig, XLMForQuestionAnswering, XLMTokenizer) } def set_seed(args): From a1f1dce0ae511ef7766c6b6a8f5ebf9118279e73 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 19 Dec 2019 12:25:55 -0500 Subject: [PATCH 447/505] Correct max position for SQUAD and TFDS --- transformers/data/processors/squad.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/transformers/data/processors/squad.py b/transformers/data/processors/squad.py index 84aa429e26..8e72bbbd6d 100644 --- a/transformers/data/processors/squad.py +++ b/transformers/data/processors/squad.py @@ -571,7 +571,9 @@ class SquadExample(object): # Start end end positions only has a value during evaluation. if start_position_character is not None and not is_impossible: self.start_position = char_to_word_offset[start_position_character] - self.end_position = char_to_word_offset[start_position_character + len(answer_text) - 1] + self.end_position = char_to_word_offset[ + min(start_position_character + len(answer_text) - 1, len(char_to_word_offset) - 1) + ] class SquadFeatures(object): From 33adab2b91697b3e78af618a21ab9f1176281165 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Thu, 19 Dec 2019 12:40:43 -0500 Subject: [PATCH 448/505] Fix albert example --- transformers/modeling_tf_albert.py | 4 ++-- transformers/modeling_utils.py | 5 ----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/transformers/modeling_tf_albert.py b/transformers/modeling_tf_albert.py index d1650d41a8..ac55a73fa3 100644 --- a/transformers/modeling_tf_albert.py +++ b/transformers/modeling_tf_albert.py @@ -587,8 +587,8 @@ class TFAlbertModel(TFAlbertPreTrainedModel): import tensorflow as tf from transformers import AlbertTokenizer, TFAlbertModel - tokenizer = AlbertTokenizer.from_pretrained('bert-base-uncased') - model = TFAlbertModel.from_pretrained('bert-base-uncased') + tokenizer = AlbertTokenizer.from_pretrained('albert-base-v1') + model = TFAlbertModel.from_pretrained('albert-base-v1') input_ids = tf.constant(tokenizer.encode("Hello, my dog is cute"))[None, :] # Batch size 1 outputs = model(input_ids) last_hidden_states = outputs[0] # The last hidden-state is the first element of the output tuple diff --git a/transformers/modeling_utils.py b/transformers/modeling_utils.py index 9bd99b25dc..eff54f71e1 100644 --- a/transformers/modeling_utils.py +++ b/transformers/modeling_utils.py @@ -327,11 +327,6 @@ class PreTrainedModel(nn.Module): model = BertModel.from_pretrained('./tf_model/my_tf_checkpoint.ckpt.index', from_tf=True, config=config) """ - if pretrained_model_name_or_path is not None and ( - "albert" in pretrained_model_name_or_path and "v2" in pretrained_model_name_or_path): - logger.warning("There is currently an upstream reproducibility issue with ALBERT v2 models. Please see " + - "https://github.com/google-research/google-research/issues/119 for more information.") - config = kwargs.pop('config', None) state_dict = kwargs.pop('state_dict', None) cache_dir = kwargs.pop('cache_dir', None) From 3492a6ec17e207a2830e061528eae9c53639c234 Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Thu, 19 Dec 2019 19:06:44 +0100 Subject: [PATCH 449/505] Addressing Thom's comments. --- transformers/pipelines.py | 70 ++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 37 deletions(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index 71e6d0fbed..e4bf9e0894 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -30,6 +30,7 @@ from transformers import AutoConfig, AutoTokenizer, PreTrainedTokenizer, Pretrai SquadExample, squad_convert_examples_to_features, is_tf_available, is_torch_available, logger if is_tf_available(): + import tensorflow as tf from transformers import TFAutoModel, TFAutoModelForSequenceClassification, \ TFAutoModelForQuestionAnswering, TFAutoModelForTokenClassification @@ -79,9 +80,9 @@ class PipelineDataFormat: """ SUPPORTED_FORMATS = ['json', 'csv', 'pipe'] - def __init__(self, output: Optional[str], path: Optional[str], column: Optional[str]): + def __init__(self, output: Optional[str], input: Optional[str], column: Optional[str]): self.output = output - self.path = path + self.path = input self.column = column.split(',') if column else [''] self.is_multi_columns = len(self.column) > 1 @@ -92,7 +93,7 @@ class PipelineDataFormat: if exists(abspath(self.output)): raise OSError('{} already exists on disk'.format(self.output)) - if path is not None: + if input is not None: if not exists(abspath(self.path)): raise OSError('{} doesnt exist on disk'.format(self.path)) @@ -136,8 +137,8 @@ class PipelineDataFormat: class CsvPipelineDataFormat(PipelineDataFormat): - def __init__(self, output: Optional[str], path: Optional[str], column: Optional[str]): - super().__init__(output, path, column) + def __init__(self, output: Optional[str], input: Optional[str], column: Optional[str]): + super().__init__(output, input, column) def __iter__(self): with open(self.path, 'r') as f: @@ -157,10 +158,10 @@ class CsvPipelineDataFormat(PipelineDataFormat): class JsonPipelineDataFormat(PipelineDataFormat): - def __init__(self, output: Optional[str], path: Optional[str], column: Optional[str]): - super().__init__(output, path, column) + def __init__(self, output: Optional[str], input: Optional[str], column: Optional[str]): + super().__init__(output, input, column) - with open(path, 'r') as f: + with open(input, 'r') as f: self._entries = json.load(f) def __iter__(self): @@ -321,11 +322,9 @@ class Pipeline(_ScikitCompat): Context manager """ if is_tf_available(): - import tensorflow as tf with tf.device('/CPU:0' if self.device == -1 else '/device:GPU:{}'.format(self.device)): yield else: - import torch if self.device >= 0: torch.cuda.set_device(self.device) @@ -358,11 +357,10 @@ class Pipeline(_ScikitCompat): # Encode for forward with self.device_placement(): - # TODO : Remove this 512 hard-limit inputs = self.tokenizer.batch_encode_plus( inputs, add_special_tokens=True, return_tensors='tf' if is_tf_available() else 'pt', - max_length=512 + max_length=self.tokenizer.max_len ) # Filter out features not available on specific models @@ -379,11 +377,10 @@ class Pipeline(_ScikitCompat): """ if is_tf_available(): # TODO trace model - predictions = self.model(inputs)[0] + predictions = self.model(inputs, training=False)[0] else: - import torch with torch.no_grad(): - predictions = self.model(**inputs)[0] + predictions = self.model(**inputs).cpu()[0] return predictions.numpy() @@ -432,19 +429,18 @@ class NerPipeline(Pipeline): # Manage correct placement of the tensors with self.device_placement(): - # TODO : Remove this 512 hard-limit tokens = self.tokenizer.encode_plus( sentence, return_attention_mask=False, return_tensors='tf' if is_tf_available() else 'pt', - max_length=512 + max_length=self.tokenizer.max_len ) # Forward - if is_torch_available(): + if is_tf_available(): + entities = self.model(**tokens)[0][0].numpy() + else: with torch.no_grad(): entities = self.model(**tokens)[0][0].cpu().numpy() - else: - entities = self.model(tokens)[0][0].numpy() # Normalize scores answer, token_start = [], 1 @@ -484,28 +480,29 @@ class QuestionAnsweringArgumentHandler(ArgumentHandler): else: kwargs['X'] = list(args) - # Generic compatibility with sklearn and Keras - # Batched data + # Generic compatibility with sklearn and Keras + # Batched data if 'X' in kwargs or 'data' in kwargs: - data = kwargs['X'] if 'X' in kwargs else kwargs['data'] + inputs = kwargs['X'] if 'X' in kwargs else kwargs['data'] - if not isinstance(data, list): - data = [data] + if isinstance(inputs, dict): + inputs = [inputs] + else: + # Copy to avoid overriding arguments + inputs = [i for i in inputs] - for i, item in enumerate(data): + for i, item in enumerate(inputs): if isinstance(item, dict): if any(k not in item for k in ['question', 'context']): raise KeyError('You need to provide a dictionary with keys {question:..., context:...}') - data[i] = QuestionAnsweringPipeline.create_sample(**item) - elif isinstance(item, SquadExample): - continue - else: + inputs[i] = QuestionAnsweringPipeline.create_sample(**item) + + elif not isinstance(item, SquadExample): raise ValueError( '{} argument needs to be of type (list[SquadExample | dict], SquadExample, dict)' .format('X' if 'X' in kwargs else 'data') ) - inputs = data # Tabular input elif 'question' in kwargs and 'context' in kwargs: @@ -588,12 +585,10 @@ class QuestionAnsweringPipeline(Pipeline): # Manage tensor allocation on correct device with self.device_placement(): if is_tf_available(): - import tensorflow as tf fw_args = {k: tf.constant(v) for (k, v) in fw_args.items()} start, end = self.model(fw_args) start, end = start.numpy(), end.numpy() else: - import torch with torch.no_grad(): # Retrieve the score for the context tokens only (removing question tokens) fw_args = {k: torch.tensor(v) for (k, v) in fw_args.items()} @@ -812,8 +807,9 @@ def pipeline(task: str, model: Optional = None, if isinstance(config, str): config = AutoConfig.from_pretrained(config) - if allocator.__name__.startswith('TF'): - model = allocator.from_pretrained(model, config=config, from_pt=from_pt) - else: - model = allocator.from_pretrained(model, config=config, from_tf=from_tf) + if isinstance(model, str): + if allocator.__name__.startswith('TF'): + model = allocator.from_pretrained(model, config=config, from_pt=from_pt) + else: + model = allocator.from_pretrained(model, config=config, from_tf=from_tf) return task(model, tokenizer, **kwargs) From a305067f2d6ca74865c6d686608a1428e476a32f Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Thu, 19 Dec 2019 19:41:48 +0100 Subject: [PATCH 450/505] Removed __main__ --- setup.py | 5 ----- transformers/__main__.py | 36 ------------------------------------ 2 files changed, 41 deletions(-) delete mode 100644 transformers/__main__.py diff --git a/setup.py b/setup.py index b3b6e2e063..56c4d1733b 100644 --- a/setup.py +++ b/setup.py @@ -68,11 +68,6 @@ setup( scripts=[ 'transformers-cli' ], - entry_points={ - 'console_scripts': [ - "transformers=transformers.__main__:main", - ] - }, # python_requires='>=3.5.0', classifiers=[ 'Intended Audience :: Science/Research', diff --git a/transformers/__main__.py b/transformers/__main__.py deleted file mode 100644 index a6e9ae65e0..0000000000 --- a/transformers/__main__.py +++ /dev/null @@ -1,36 +0,0 @@ -# coding: utf8 - -def main(): - import sys - if len(sys.argv) < 2 or sys.argv[1] not in ["convert", "train", "predict", "serve"]: - print( - "First argument to `transformers` command line interface should be one of: \n" - ">> convert serve train predict") - if sys.argv[1] == "convert": - from transformers.commands import convert - convert(sys.argv) - elif sys.argv[1] == "train": - from transformers.commands import train - train(sys.argv) - elif sys.argv[1] == "serve": - pass - # from argparse import ArgumentParser - # from transformers.commands.serving import ServeCommand - # parser = ArgumentParser('Transformers CLI tool', usage='transformers serve []') - # commands_parser = parser.add_subparsers(help='transformers-cli command helpers') - - # # Register commands - # ServeCommand.register_subcommand(commands_parser) - - # # Let's go - # args = parser.parse_args() - - # if not hasattr(args, 'func'): - # parser.print_help() - # exit(1) - # # Run - # service = args.func(args) - # service.run() - -if __name__ == '__main__': - main() From 149dc376aa5070db04cdf6e3358e87cab6670251 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 19 Dec 2019 20:34:28 +0100 Subject: [PATCH 451/505] fix tests --- transformers/configuration_utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/transformers/configuration_utils.py b/transformers/configuration_utils.py index 1aede2d6eb..f692c9b132 100644 --- a/transformers/configuration_utils.py +++ b/transformers/configuration_utils.py @@ -57,8 +57,6 @@ class PretrainedConfig(object): self.use_bfloat16 = kwargs.pop('use_bfloat16', False) self.pruned_heads = kwargs.pop('pruned_heads', {}) self.is_decoder = kwargs.pop('is_decoder', False) - self.id2label = kwargs.pop('id2label', {i: 'LABEL_{}'.format(i) for i in range(self.num_labels)}) - self.label2id = kwargs.pop('label2id', dict(zip(self.id2label.values(), self.id2label.keys()))) # Fine-tuning task arguments self.finetuning_task = kwargs.pop('finetuning_task', None) From e4baa68ddbcb28488d7cef44ea5483a955d2effb Mon Sep 17 00:00:00 2001 From: thomwolf Date: Thu, 19 Dec 2019 20:37:26 +0100 Subject: [PATCH 452/505] tick-tock cc @julien-c --- .circleci/config.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9a81eea902..7a64eaba7d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -80,17 +80,6 @@ jobs: - run: sudo pip install pytest - run: sudo pip install mecab-python3 - run: RUN_CUSTOM_TOKENIZERS=1 python -m pytest -sv ./transformers/tests/tokenization_bert_japanese_test.py - build_py2_custom_tokenizers: - working_directory: ~/transformers - docker: - - image: circleci/python:2.7 - steps: - - checkout - - run: sudo pip install --progress-bar off . - - run: sudo pip install pytest - - run: sudo apt-get -y install libmecab-dev mecab mecab-ipadic-utf8 swig - - run: sudo pip install mecab-python - - run: RUN_CUSTOM_TOKENIZERS=1 python -m pytest -sv ./transformers/tests/tokenization_bert_japanese_test.py deploy_doc: working_directory: ~/transformers docker: @@ -124,7 +113,6 @@ workflows: jobs: - repository_consistency - build_py3_custom_tokenizers - - build_py2_custom_tokenizers - build_py3_torch_and_tf - build_py3_torch - build_py3_tf From 3376adc05157ba826acafd49f07f9a01ae30eb07 Mon Sep 17 00:00:00 2001 From: Stefan Schweter Date: Thu, 19 Dec 2019 21:30:23 +0100 Subject: [PATCH 453/505] configuration/modeling/tokenization: add various fine-tuned XLM-RoBERTa models for English, German, Spanish and Dutch (CoNLL datasets) --- transformers/configuration_xlm_roberta.py | 4 ++++ transformers/modeling_xlm_roberta.py | 4 ++++ transformers/tokenization_xlm_roberta.py | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/transformers/configuration_xlm_roberta.py b/transformers/configuration_xlm_roberta.py index d7a26538c5..5b6955f4f8 100644 --- a/transformers/configuration_xlm_roberta.py +++ b/transformers/configuration_xlm_roberta.py @@ -27,6 +27,10 @@ logger = logging.getLogger(__name__) XLM_ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP = { 'xlm-roberta-base': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-base-config.json", 'xlm-roberta-large': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-large-config.json", + 'xlm-roberta-large-finetuned-conll02-dutch': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-large-finetuned-conll02-dutch-config.json", + 'xlm-roberta-large-finetuned-conll02-spanish': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-large-finetuned-conll02-spanish-config.json", + 'xlm-roberta-large-finetuned-conll03-english': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-large-finetuned-conll03-english-config.json", + 'xlm-roberta-large-finetuned-conll03-german': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-large-finetuned-conll03-german-config.json", } diff --git a/transformers/modeling_xlm_roberta.py b/transformers/modeling_xlm_roberta.py index 8095c46a16..0bdce941a5 100644 --- a/transformers/modeling_xlm_roberta.py +++ b/transformers/modeling_xlm_roberta.py @@ -29,6 +29,10 @@ logger = logging.getLogger(__name__) XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP = { 'xlm-roberta-base': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-base-pytorch_model.bin", 'xlm-roberta-large': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-large-pytorch_model.bin", + 'xlm-roberta-large-finetuned-conll02-dutch': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-large-finetuned-conll02-dutch-pytorch_model.bin", + 'xlm-roberta-large-finetuned-conll02-spanish': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-large-finetuned-conll02-spanish-pytorch_model.bin", + 'xlm-roberta-large-finetuned-conll03-english': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-large-finetuned-conll03-english-pytorch_model.bin", + 'xlm-roberta-large-finetuned-conll03-german': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-large-finetuned-conll03-german-pytorch_model.bin", } diff --git a/transformers/tokenization_xlm_roberta.py b/transformers/tokenization_xlm_roberta.py index 453c4375c6..4397e7b031 100644 --- a/transformers/tokenization_xlm_roberta.py +++ b/transformers/tokenization_xlm_roberta.py @@ -32,6 +32,10 @@ PRETRAINED_VOCAB_FILES_MAP = { { 'xlm-roberta-base': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-base-sentencepiece.bpe.model", 'xlm-roberta-large': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-large-sentencepiece.bpe.model", + 'xlm-roberta-large-finetuned-conll02-dutch': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-large-finetuned-conll02-dutch-sentencepiece.bpe.model", + 'xlm-roberta-large-finetuned-conll02-spanish': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-large-finetuned-conll02-spanish-sentencepiece.bpe.model", + 'xlm-roberta-large-finetuned-conll03-english': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-large-finetuned-conll03-english-sentencepiece.bpe.model", + 'xlm-roberta-large-finetuned-conll03-german': "https://s3.amazonaws.com/models.huggingface.co/bert/xlm-roberta-large-finetuned-conll03-german-sentencepiece.bpe.model", } } From 9a399ead253e27792cbf0ef386cc39f9b7084f8f Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Thu, 19 Dec 2019 15:45:48 -0500 Subject: [PATCH 454/505] Revert incorrect #1778 --- transformers/modeling_tf_pytorch_utils.py | 3 --- transformers/modeling_utils.py | 2 -- 2 files changed, 5 deletions(-) diff --git a/transformers/modeling_tf_pytorch_utils.py b/transformers/modeling_tf_pytorch_utils.py index d885fd23b3..190caff18d 100644 --- a/transformers/modeling_tf_pytorch_utils.py +++ b/transformers/modeling_tf_pytorch_utils.py @@ -119,9 +119,6 @@ def load_pytorch_weights_in_tf2_model(tf_model, pt_state_dict, tf_inputs=None, a new_key = key.replace('gamma', 'weight') if 'beta' in key: new_key = key.replace('beta', 'bias') - # DialoGPT format - if key == 'lm_head.decoder.weight': - new_key = 'lm_head.weight' if new_key: old_keys.append(key) new_keys.append(new_key) diff --git a/transformers/modeling_utils.py b/transformers/modeling_utils.py index eff54f71e1..cce234838a 100644 --- a/transformers/modeling_utils.py +++ b/transformers/modeling_utils.py @@ -437,8 +437,6 @@ class PreTrainedModel(nn.Module): new_key = key.replace('gamma', 'weight') if 'beta' in key: new_key = key.replace('beta', 'bias') - if key == 'lm_head.decoder.weight': - new_key = 'lm_head.weight' if new_key: old_keys.append(key) new_keys.append(new_key) From 1718fb9e7444a0883a550ab90c61bb8bf8a97076 Mon Sep 17 00:00:00 2001 From: Aidan Kierans <31550769+aidankierans@users.noreply.github.com> Date: Thu, 19 Dec 2019 16:23:18 -0500 Subject: [PATCH 455/505] Minor/basic text fixes (#2229) * Small clarification Matches line 431 to line 435 for additional clarity and consistency. * Fixed minor typo The letter "s" was previously omitted from the word "docstrings". --- CONTRIBUTING.md | 2 +- examples/run_lm_finetuning.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8228dd59d8..7d7f2c73ff 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -168,7 +168,7 @@ Follow these steps to start contributing: to be merged; 4. Make sure pre-existing tests still pass; 5. Add high-coverage tests. No quality test, no merge; -6. All public methods must have informative doctrings; +6. All public methods must have informative docstrings; ### Style guide diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py index c4c73e71af..d8127e24a5 100644 --- a/examples/run_lm_finetuning.py +++ b/examples/run_lm_finetuning.py @@ -428,9 +428,9 @@ def main(): parser.add_argument('--gradient_accumulation_steps', type=int, default=1, help="Number of updates steps to accumulate before performing a backward/update pass.") parser.add_argument("--learning_rate", default=5e-5, type=float, - help="The initial learning rate for Adam.") + help="The initial learning rate for Adam optimizer.") parser.add_argument("--weight_decay", default=0.0, type=float, - help="Weight deay if we apply some.") + help="Weight decay if we apply some.") parser.add_argument("--adam_epsilon", default=1e-8, type=float, help="Epsilon for Adam optimizer.") parser.add_argument("--max_grad_norm", default=1.0, type=float, From a5a06a851e1da79138e53978aa079a093f243dde Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Thu, 19 Dec 2019 16:24:20 -0500 Subject: [PATCH 456/505] [doc] Param name consistency --- examples/run_lm_finetuning.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/run_lm_finetuning.py b/examples/run_lm_finetuning.py index d8127e24a5..75848d5acc 100644 --- a/examples/run_lm_finetuning.py +++ b/examples/run_lm_finetuning.py @@ -428,7 +428,7 @@ def main(): parser.add_argument('--gradient_accumulation_steps', type=int, default=1, help="Number of updates steps to accumulate before performing a backward/update pass.") parser.add_argument("--learning_rate", default=5e-5, type=float, - help="The initial learning rate for Adam optimizer.") + help="The initial learning rate for Adam.") parser.add_argument("--weight_decay", default=0.0, type=float, help="Weight decay if we apply some.") parser.add_argument("--adam_epsilon", default=1e-8, type=float, From f25e9b6f771ea1a10f4525bdb212f841efcdbd3a Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Thu, 19 Dec 2019 18:28:17 -0500 Subject: [PATCH 457/505] [hf_bucket_url] support for cloudfront urls --- transformers/file_utils.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/transformers/file_utils.py b/transformers/file_utils.py index 16010f7e0a..4a9de9b53c 100644 --- a/transformers/file_utils.py +++ b/transformers/file_utils.py @@ -77,6 +77,7 @@ DUMMY_INPUTS = [[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]] DUMMY_MASK = [[1, 1, 1, 1, 1], [1, 1, 1, 0, 0], [0, 0, 0, 1, 1]] S3_BUCKET_PREFIX = "https://s3.amazonaws.com/models.huggingface.co/bert" +CLOUDFRONT_DISTRIB_PREFIX = "https://d2ws9o8vfrpkyk.cloudfront.net" def is_torch_available(): @@ -114,11 +115,12 @@ def is_remote_url(url_or_filename): parsed = urlparse(url_or_filename) return parsed.scheme in ('http', 'https', 's3') -def hf_bucket_url(identifier, postfix=None): +def hf_bucket_url(identifier, postfix=None, cdn=False): + endpoint = CLOUDFRONT_DISTRIB_PREFIX if cdn else S3_BUCKET_PREFIX if postfix is None: - return "/".join((S3_BUCKET_PREFIX, identifier)) + return "/".join((endpoint, identifier)) else: - return "/".join((S3_BUCKET_PREFIX, identifier, postfix)) + return "/".join((endpoint, identifier, postfix)) def url_to_filename(url, etag=None): @@ -126,7 +128,7 @@ def url_to_filename(url, etag=None): Convert `url` into a hashed filename in a repeatable way. If `etag` is specified, append its hash to the url's, delimited by a period. - If the url ends with .h5 (Keras HDF5 weights) ands '.h5' to the name + If the url ends with .h5 (Keras HDF5 weights) adds '.h5' to the name so that TF 2.0 can identify it as a HDF5 file (see https://github.com/tensorflow/tensorflow/blob/00fad90125b18b80fe054de1055770cfb8fe4ba3/tensorflow/python/keras/engine/network.py#L1380) """ From 15d897ff4a29e851285844e82763155a3f9f86b0 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Thu, 19 Dec 2019 18:29:22 -0500 Subject: [PATCH 458/505] [http] customizable requests user-agent --- transformers/file_utils.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/transformers/file_utils.py b/transformers/file_utils.py index 4a9de9b53c..611202159d 100644 --- a/transformers/file_utils.py +++ b/transformers/file_utils.py @@ -23,6 +23,7 @@ from botocore.exceptions import ClientError import requests from tqdm.auto import tqdm from contextlib import contextmanager +from . import __version__ logger = logging.getLogger(__name__) # pylint: disable=invalid-name @@ -173,7 +174,7 @@ def filename_to_url(filename, cache_dir=None): return url, etag -def cached_path(url_or_filename, cache_dir=None, force_download=False, proxies=None, resume_download=False): +def cached_path(url_or_filename, cache_dir=None, force_download=False, proxies=None, resume_download=False, user_agent=None): """ Given something that might be a URL (or might be a local path), determine which. If it's a URL, download the file and cache it, and @@ -183,6 +184,7 @@ def cached_path(url_or_filename, cache_dir=None, force_download=False, proxies=N cache_dir: specify a cache directory to save the file to (overwrite the default cache dir). force_download: if True, re-dowload the file even if it's already cached in the cache dir. resume_download: if True, resume the download if incompletly recieved file is found. + user_agent: Optional string or dict that will be appended to the user-agent on remote requests. """ if cache_dir is None: cache_dir = TRANSFORMERS_CACHE @@ -195,7 +197,7 @@ def cached_path(url_or_filename, cache_dir=None, force_download=False, proxies=N # URL, so get it from the cache (downloading if necessary) return get_from_cache(url_or_filename, cache_dir=cache_dir, force_download=force_download, proxies=proxies, - resume_download=resume_download) + resume_download=resume_download, user_agent=user_agent) elif os.path.exists(url_or_filename): # File, and it exists. return url_or_filename @@ -256,8 +258,19 @@ def s3_get(url, temp_file, proxies=None): s3_resource.Bucket(bucket_name).download_fileobj(s3_path, temp_file) -def http_get(url, temp_file, proxies=None, resume_size=0): - headers={'Range':'bytes=%d-'%(resume_size,)} if resume_size > 0 else None +def http_get(url, temp_file, proxies=None, resume_size=0, user_agent=None): + ua = "transformers/{}; python/{}".format(__version__, sys.version.split()[0]) + if isinstance(user_agent, dict): + ua += "; " + "; ".join( + "{}/{}".format(k, v) for k, v in user_agent.items() + ) + elif isinstance(user_agent, six.string_types): + ua += "; "+ user_agent + headers = { + "user-agent": ua + } + if resume_size > 0: + headers['Range'] = 'bytes=%d-' % (resume_size,) response = requests.get(url, stream=True, proxies=proxies, headers=headers) if response.status_code == 416: # Range not satisfiable return @@ -271,7 +284,7 @@ def http_get(url, temp_file, proxies=None, resume_size=0): progress.close() -def get_from_cache(url, cache_dir=None, force_download=False, proxies=None, etag_timeout=10, resume_download=False): +def get_from_cache(url, cache_dir=None, force_download=False, proxies=None, etag_timeout=10, resume_download=False, user_agent=None): """ Given a URL, look for the corresponding dataset in the local cache. If it's not there, download it. Then return the path to the cached file. @@ -342,7 +355,7 @@ def get_from_cache(url, cache_dir=None, force_download=False, proxies=None, etag logger.warn('Warning: resumable downloads are not implemented for "s3://" urls') s3_get(url, temp_file, proxies=proxies) else: - http_get(url, temp_file, proxies=proxies, resume_size=resume_size) + http_get(url, temp_file, proxies=proxies, resume_size=resume_size, user_agent=user_agent) # we are copying the file before closing it, so flush to avoid truncation temp_file.flush() From 9d0d1cd339ca97e6a40e1f898fa3d3350e46e350 Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Fri, 20 Dec 2019 09:30:37 +0100 Subject: [PATCH 459/505] Filter out entity for NER task. --- transformers/pipelines.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index e4bf9e0894..6b805a06db 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -450,11 +450,12 @@ class NerPipeline(Pipeline): score = np.exp(entities[token_start]) / np.exp(entities[token_start]).sum(-1, keepdims=True) label_idx = score.argmax() - answer += [{ - 'word': words[idx], - 'score': score[label_idx].item(), - 'entity': self.model.config.id2label[label_idx] - }] + if label_idx > 0: + answer += [{ + 'word': words[idx], + 'score': score[label_idx].item(), + 'entity': self.model.config.id2label[label_idx] + }] # Update token start token_start += len(list(word)) From e516a34a158593f82b6f22dc1e568ff8996f0389 Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Fri, 20 Dec 2019 09:38:08 +0100 Subject: [PATCH 460/505] Use BasicTokenizer to split over whitespaces. --- transformers/pipelines.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index 6b805a06db..c3e0ad1a8f 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -27,7 +27,7 @@ from typing import Union, Optional, Tuple, List, Dict import numpy as np from transformers import AutoConfig, AutoTokenizer, PreTrainedTokenizer, PretrainedConfig, \ - SquadExample, squad_convert_examples_to_features, is_tf_available, is_torch_available, logger + SquadExample, squad_convert_examples_to_features, is_tf_available, is_torch_available, logger, BasicTokenizer if is_tf_available(): import tensorflow as tf @@ -416,12 +416,19 @@ class NerPipeline(Pipeline): Named Entity Recognition pipeline using ModelForTokenClassification head. """ + def __init__(self, model, tokenizer: PreTrainedTokenizer = None, + args_parser: ArgumentHandler = None, device: int = -1, + binary_output: bool = False): + super().__init__(model, tokenizer, args_parser, device, binary_output) + + self._basic_tokenizer = BasicTokenizer(do_lower_case=False) + def __call__(self, *texts, **kwargs): inputs, answers = self._args_parser(*texts, **kwargs), [] for sentence in inputs: # Ugly token to word idx mapping (for now) - token_to_word, words = [], sentence.split(' ') + token_to_word, words = [], self._basic_tokenizer.tokenize(sentence) for i, w in enumerate(words): tokens = self.tokenizer.tokenize(w) token_to_word += [i] * len(tokens) From 61d9ee45e3f19b9c661e078e7f57dbe8fb8c812c Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Fri, 20 Dec 2019 11:47:56 +0100 Subject: [PATCH 461/505] All tests are green. --- transformers/pipelines.py | 9 +- transformers/tests/pipelines_test.py | 247 ++++++++++++++++----------- 2 files changed, 154 insertions(+), 102 deletions(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index c3e0ad1a8f..4dde62cbe5 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -343,8 +343,9 @@ class Pipeline(_ScikitCompat): if 'distilbert' not in model_type and 'xlm' not in model_type: args += ['token_type_ids'] - if 'xlnet' in model_type or 'xlm' in model_type: - args += ['cls_index', 'p_mask'] + # PR #1548 (CLI) There is an issue with attention_mask + # if 'xlnet' in model_type or 'xlm' in model_type: + # args += ['cls_index', 'p_mask'] if isinstance(features, dict): return {k: features[k] for k in args} @@ -380,7 +381,7 @@ class Pipeline(_ScikitCompat): predictions = self.model(inputs, training=False)[0] else: with torch.no_grad(): - predictions = self.model(**inputs).cpu()[0] + predictions = self.model(**inputs)[0].cpu() return predictions.numpy() @@ -444,7 +445,7 @@ class NerPipeline(Pipeline): # Forward if is_tf_available(): - entities = self.model(**tokens)[0][0].numpy() + entities = self.model(tokens)[0][0].numpy() else: with torch.no_grad(): entities = self.model(**tokens)[0][0].cpu().numpy() diff --git a/transformers/tests/pipelines_test.py b/transformers/tests/pipelines_test.py index ee10234269..a8fe668221 100644 --- a/transformers/tests/pipelines_test.py +++ b/transformers/tests/pipelines_test.py @@ -1,113 +1,164 @@ import unittest from unittest.mock import patch +from typing import Iterable + +from transformers import pipeline +from transformers.tests.utils import require_tf, require_torch QA_FINETUNED_MODELS = { - 'bert-large-uncased-whole-word-masking-finetuned-squad', - 'bert-large-cased-whole-word-masking-finetuned-squad', - 'distilbert-base-uncased-distilled-squad', + ('bert-base-uncased', 'bert-large-uncased-whole-word-masking-finetuned-squad', None), + ('bert-base-cased', 'bert-large-cased-whole-word-masking-finetuned-squad', None), + ('bert-base-uncased', 'distilbert-base-uncased-distilled-squad', None) +} +NER_FINETUNED_MODELS = { + ( + 'bert-base-cased', + 'https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-finetuned-conll03-english-pytorch_model.bin', + 'https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-finetuned-conll03-english-config.json' + ) +} + +FEATURE_EXTRACT_FINETUNED_MODELS = { + ('bert-base-cased', 'bert-base-cased', None), + # ('xlnet-base-cased', 'xlnet-base-cased', None), # Disabled for now as it crash for TF2 + ('distilbert-base-uncased', 'distilbert-base-uncased', None) +} + +TEXT_CLASSIF_FINETUNED_MODELS = { + ( + 'bert-base-uncased', + 'https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-finetuned-sst-2-english-pytorch_model.bin', + 'https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-finetuned-sst-2-english-config.json' + ) } -class QuestionAnsweringPipelineTest(unittest.TestCase): - def check_answer_structure(self, answer, batch, topk): - self.assertIsInstance(answer, list) - self.assertEqual(len(answer), batch) - self.assertIsInstance(answer[0], list) - self.assertEqual(len(answer[0]), topk) - self.assertIsInstance(answer[0][0], dict) - - for item in answer[0]: - self.assertTrue('start' in item) - self.assertTrue('end' in item) - self.assertTrue('score' in item) - self.assertTrue('answer' in item) - - def question_answering_pipeline(self, nlp): - # Simple case with topk = 1, no batching - a = nlp(question='What is the name of the company I\'m working for ?', context='I\'m working for Huggingface.') - self.check_answer_structure(a, 1, 1) - - # Simple case with topk = 2, no batching - a = nlp(question='What is the name of the company I\'m working for ?', context='I\'m working for Huggingface.', topk=2) - self.check_answer_structure(a, 1, 2) - - # Batch case with topk = 1 - a = nlp(question=['What is the name of the company I\'m working for ?', 'Where is the company based ?'], - context=['I\'m working for Huggingface.', 'The company is based in New York and Paris']) - self.check_answer_structure(a, 2, 1) - - # Batch case with topk = 2 - a = nlp(question=['What is the name of the company I\'m working for ?', 'Where is the company based ?'], - context=['Where is the company based ?', 'The company is based in New York and Paris'], topk=2) - self.check_answer_structure(a, 2, 2) - - # check for data keyword - a = nlp(data=nlp.create_sample(question='What is the name of the company I\'m working for ?', context='I\'m working for Huggingface.')) - self.check_answer_structure(a, 1, 1) - - a = nlp(data=nlp.create_sample(question='What is the name of the company I\'m working for ?', context='I\'m working for Huggingface.'), topk=2) - self.check_answer_structure(a, 1, 2) - - a = nlp(data=[ - nlp.create_sample(question='What is the name of the company I\'m working for ?', context='I\'m working for Huggingface.'), - nlp.create_sample(question='I\'m working for Huggingface.', context='The company is based in New York and Paris'), - ]) - self.check_answer_structure(a, 2, 1) - - a = nlp(data=[ - {'question': 'What is the name of the company I\'m working for ?', 'context': 'I\'m working for Huggingface.'}, - {'question': 'Where is the company based ?', 'context': 'The company is based in New York and Paris'}, - ]) - self.check_answer_structure(a, 2, 1) - - # X keywords - a = nlp(X=nlp.create_sample( - question='Where is the company based ?', context='The company is based in New York and Paris' - )) - self.check_answer_structure(a, 1, 1) - - a = nlp(X=[ - {'question': 'What is the name of the company I\'m working for ?', 'context': 'I\'m working for Huggingface.'}, - {'question': 'Where is the company based ?', 'context': 'The company is based in New York and Paris'}, - ], topk=2) - self.check_answer_structure(a, 2, 2) - - @patch('transformers.pipelines.is_torch_available', return_value=False) - def test_tf_models(self, is_torch_available): - from transformers import pipeline - for model in QA_FINETUNED_MODELS: - self.question_answering_pipeline(pipeline('question-answering', model)) - - @patch('transformers.pipelines.is_tf_available', return_value=False) - @patch('transformers.tokenization_utils.is_tf_available', return_value=False) - def test_torch_models(self, is_tf_available, _): - from transformers import pipeline - for model in QA_FINETUNED_MODELS: - self.question_answering_pipeline(pipeline('question-answering', model)) +@require_tf +def tf_pipeline(*args, **kwargs): + return pipeline(**kwargs) -class AutoPipelineTest(unittest.TestCase): - @patch('transformers.pipelines.is_torch_available', return_value=False) - def test_tf_qa(self, is_torch_available): - from transformers import pipeline - from transformers.pipelines import QuestionAnsweringPipeline - from transformers.modeling_tf_utils import TFPreTrainedModel - for model in QA_FINETUNED_MODELS: - nlp = pipeline('question-answering', model) - self.assertIsInstance(nlp, QuestionAnsweringPipeline) - self.assertIsInstance(nlp.model, TFPreTrainedModel) +@require_torch +def torch_pipeline(*args, **kwargs): + return pipeline(**kwargs) - @patch('transformers.pipelines.is_tf_available', return_value=False) - def test_torch_qa(self, is_tf_available): - from transformers import pipeline - from transformers.pipelines import QuestionAnsweringPipeline - from transformers.modeling_utils import PreTrainedModel - for model in QA_FINETUNED_MODELS: - nlp = pipeline('question-answering', model) - self.assertIsInstance(nlp, QuestionAnsweringPipeline) - self.assertIsInstance(nlp.model, PreTrainedModel) + +class MonoColumnInputTestCase(unittest.TestCase): + def _test_mono_column_pipeline(self, nlp, valid_inputs: list, invalid_inputs: list, output_keys: Iterable[str]): + self.assertIsNotNone(nlp) + + mono_result = nlp(valid_inputs[0]) + self.assertIsInstance(mono_result, list) + self.assertIsInstance(mono_result[0], (dict, list)) + + if isinstance(mono_result[0], list): + mono_result = mono_result[0] + + for key in output_keys: + self.assertIn(key, mono_result[0]) + + multi_result = nlp(valid_inputs) + self.assertIsInstance(multi_result, list) + self.assertIsInstance(multi_result[0], (dict, list)) + + if isinstance(multi_result[0], list): + multi_result = multi_result[0] + + for result in multi_result: + for key in output_keys: + self.assertIn(key, result) + + self.assertRaises(Exception, nlp, invalid_inputs) + + def test_ner(self): + mandatory_keys = {'entity', 'word', 'score'} + valid_inputs = ['HuggingFace is solving NLP one commit at a time.', 'HuggingFace is based in New-York & Paris'] + invalid_inputs = [None] + for tokenizer, model, config in NER_FINETUNED_MODELS: + with patch('transformers.pipelines.is_torch_available', return_value=False): + nlp = tf_pipeline(task='ner', model=model, config=config, tokenizer=tokenizer) + self._test_mono_column_pipeline(nlp, valid_inputs, invalid_inputs, mandatory_keys) + + with patch('transformers.pipelines.is_tf_available', return_value=False): + nlp = torch_pipeline(task='ner', model=model, config=config, tokenizer=tokenizer) + self._test_mono_column_pipeline(nlp, valid_inputs, invalid_inputs, mandatory_keys) + + def test_sentiment_analysis(self): + mandatory_keys = {'label'} + valid_inputs = ['HuggingFace is solving NLP one commit at a time.', 'HuggingFace is based in New-York & Paris'] + invalid_inputs = [None] + for tokenizer, model, config in TEXT_CLASSIF_FINETUNED_MODELS: + with patch('transformers.pipelines.is_torch_available', return_value=False): + nlp = tf_pipeline(task='sentiment-analysis', model=model, config=config, tokenizer=tokenizer) + self._test_mono_column_pipeline(nlp, valid_inputs, invalid_inputs, mandatory_keys) + + with patch('transformers.pipelines.is_tf_available', return_value=False): + nlp = torch_pipeline(task='sentiment-analysis', model=model, config=config, tokenizer=tokenizer) + self._test_mono_column_pipeline(nlp, valid_inputs, invalid_inputs, mandatory_keys) + + def test_features_extraction(self): + valid_inputs = ['HuggingFace is solving NLP one commit at a time.', 'HuggingFace is based in New-York & Paris'] + invalid_inputs = [None] + for tokenizer, model, config in FEATURE_EXTRACT_FINETUNED_MODELS: + with patch('transformers.pipelines.is_torch_available', return_value=False): + nlp = tf_pipeline(task='sentiment-analysis', model=model, config=config, tokenizer=tokenizer) + self._test_mono_column_pipeline(nlp, valid_inputs, invalid_inputs, {}) + + with patch('transformers.pipelines.is_tf_available', return_value=False): + nlp = torch_pipeline(task='sentiment-analysis', model=model, config=config, tokenizer=tokenizer) + self._test_mono_column_pipeline(nlp, valid_inputs, invalid_inputs, {}) + + +class MultiColumnInputTestCase(unittest.TestCase): + def _test_multicolumn_pipeline(self, nlp, valid_inputs: list, invalid_inputs: list, output_keys: Iterable[str]): + self.assertIsNotNone(nlp) + + mono_result = nlp(valid_inputs[0]) + self.assertIsInstance(mono_result, dict) + + for key in output_keys: + self.assertIn(key, mono_result) + + multi_result = nlp(valid_inputs) + self.assertIsInstance(multi_result, list) + self.assertIsInstance(multi_result[0], dict) + + for result in multi_result: + for key in output_keys: + self.assertIn(key, result) + + self.assertRaises(Exception, nlp, invalid_inputs[0]) + self.assertRaises(Exception, nlp, invalid_inputs) + + def test_question_answering(self): + mandatory_output_keys = {'score', 'answer', 'start', 'end'} + valid_samples = [ + {'question': 'Where was HuggingFace founded ?', 'context': 'HuggingFace was founded in Paris.'}, + { + 'question': 'In what field is HuggingFace working ?', + 'context': 'HuggingFace is a startup based in New-York founded in Paris which is trying to solve NLP.' + } + ] + invalid_samples = [ + {'question': '', 'context': 'This is a test to try empty question edge case'}, + {'question': None, 'context': 'This is a test to try empty question edge case'}, + {'question': 'What is does with empty context ?', 'context': ''}, + {'question': 'What is does with empty context ?', 'context': None}, + ] + + for tokenizer, model, config in QA_FINETUNED_MODELS: + + # Test for Tensorflow + with patch('transformers.pipelines.is_torch_available', return_value=False): + nlp = pipeline(task='question-answering', model=model, config=config, tokenizer=tokenizer) + self._test_multicolumn_pipeline(nlp, valid_samples, invalid_samples, mandatory_output_keys) + + # Test for PyTorch + with patch('transformers.pipelines.is_tf_available', return_value=False): + nlp = pipeline(task='question-answering', model=model, config=config, tokenizer=tokenizer) + self._test_multicolumn_pipeline(nlp, valid_samples, invalid_samples, mandatory_output_keys) if __name__ == '__main__': From ca6bdb28f64763ccad6c5b0aef36049c797f1ef7 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 20 Dec 2019 12:10:40 +0100 Subject: [PATCH 462/505] fix pipelines and rename model_card => modelcard --- transformers/__init__.py | 2 +- transformers/file_utils.py | 4 +- transformers/{model_card.py => modelcard.py} | 55 ++++----- transformers/pipelines.py | 112 ++++++++++++------- transformers/tests/model_card_test.py | 28 ++--- 5 files changed, 118 insertions(+), 83 deletions(-) rename transformers/{model_card.py => modelcard.py} (83%) diff --git a/transformers/__init__.py b/transformers/__init__.py index 80f140b31e..73a7f5d862 100755 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -35,7 +35,7 @@ if is_sklearn_available(): from .data import glue_compute_metrics, xnli_compute_metrics # Model Cards -from .model_card import ModelCard +from .modelcard import ModelCard # Tokenizers from .tokenization_utils import (PreTrainedTokenizer) diff --git a/transformers/file_utils.py b/transformers/file_utils.py index 8a1c4db201..47fa588815 100644 --- a/transformers/file_utils.py +++ b/transformers/file_utils.py @@ -81,7 +81,7 @@ WEIGHTS_NAME = "pytorch_model.bin" TF2_WEIGHTS_NAME = 'tf_model.h5' TF_WEIGHTS_NAME = 'model.ckpt' CONFIG_NAME = "config.json" -MODEL_CARD_NAME = "model_card.json" +MODEL_CARD_NAME = "modelcard.json" DUMMY_INPUTS = [[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]] DUMMY_MASK = [[1, 1, 1, 1, 1], [1, 1, 1, 0, 0], [0, 0, 0, 1, 1]] @@ -339,7 +339,7 @@ def get_from_cache(url, cache_dir=None, force_download=False, proxies=None, etag temp_file_manager = tempfile.NamedTemporaryFile resume_size = 0 - if not os.path.exists(cache_path) or force_download: + if etag is not None and (not os.path.exists(cache_path) or force_download): # Download to temporary file, then copy to cache dir once finished. # Otherwise you get corrupt cache entries if the download gets interrupted. with temp_file_manager() as temp_file: diff --git a/transformers/model_card.py b/transformers/modelcard.py similarity index 83% rename from transformers/model_card.py rename to transformers/modelcard.py index baec7e8622..4a879235ae 100644 --- a/transformers/model_card.py +++ b/transformers/modelcard.py @@ -25,7 +25,8 @@ from io import open from .configuration_auto import ALL_PRETRAINED_CONFIG_ARCHIVE_MAP -from .file_utils import CONFIG_NAME, MODEL_CARD_NAME, cached_path, is_remote_url, hf_bucket_url +from .file_utils import CONFIG_NAME, MODEL_CARD_NAME, WEIGHTS_NAME, TF2_WEIGHTS_NAME, \ + cached_path, is_remote_url, hf_bucket_url logger = logging.getLogger(__name__) @@ -89,7 +90,7 @@ class ModelCard(object): - a string with the `shortcut name` of a pre-trained model card to load from cache or download, e.g.: ``bert-base-uncased``. - a string with the `identifier name` of a pre-trained model card that was user-uploaded to our S3, e.g.: ``dbmdz/bert-base-german-cased``. - a path to a `directory` containing a mode card file saved using the :func:`~transformers.ModelCard.save_pretrained` method, e.g.: ``./my_model_directory/``. - - a path or url to a saved model card JSON `file`, e.g.: ``./my_model_directory/model_card.json``. + - a path or url to a saved model card JSON `file`, e.g.: ``./my_model_directory/modelcard.json``. cache_dir: (`optional`) string: Path to a directory in which a downloaded pre-trained model @@ -100,16 +101,14 @@ class ModelCard(object): - The values in kwargs of any keys which are model card attributes will be used to override the loaded values. - Behavior concerning key/value pairs whose keys are *not* model card attributes is controlled by the `return_unused_kwargs` keyword parameter. - force_download: (`optional`) boolean, default False: - Force to (re-)download the model card file and override the cached version if it exists. - - resume_download: (`optional`) boolean, default False: - Do not delete incompletely recieved file. Attempt to resume the download if such a file exists. - proxies: (`optional`) dict, default None: A dictionary of proxy servers to use by protocol or endpoint, e.g.: {'http': 'foo.bar:3128', 'http://hostname': 'foo.bar:4012'}. The proxies are used on each request. + find_from_standard_name: (`optional`) boolean, default True: + If the pretrained_model_name_or_path ends with our standard model or config filenames, replace them with our standard modelcard filename. + Can be used to directly feed a model/config url and access the colocated modelcard. + return_unused_kwargs: (`optional`) bool: - If False, then this function returns just the final model card object. @@ -117,22 +116,21 @@ class ModelCard(object): Examples:: - model_card = ModelCard.from_pretrained('bert-base-uncased') # Download model card from S3 and cache. - model_card = ModelCard.from_pretrained('./test/saved_model/') # E.g. model card was saved using `save_pretrained('./test/saved_model/')` - model_card = ModelCard.from_pretrained('./test/saved_model/model_card.json') - model_card = ModelCard.from_pretrained('bert-base-uncased', output_attention=True, foo=False) + modelcard = ModelCard.from_pretrained('bert-base-uncased') # Download model card from S3 and cache. + modelcard = ModelCard.from_pretrained('./test/saved_model/') # E.g. model card was saved using `save_pretrained('./test/saved_model/')` + modelcard = ModelCard.from_pretrained('./test/saved_model/modelcard.json') + modelcard = ModelCard.from_pretrained('bert-base-uncased', output_attention=True, foo=False) """ cache_dir = kwargs.pop('cache_dir', None) - force_download = kwargs.pop('force_download', False) - resume_download = kwargs.pop('resume_download', False) proxies = kwargs.pop('proxies', None) + find_from_standard_name = kwargs.pop('find_from_standard_name', True) return_unused_kwargs = kwargs.pop('return_unused_kwargs', False) if pretrained_model_name_or_path in ALL_PRETRAINED_CONFIG_ARCHIVE_MAP: - # For simplicity we use the same pretrained url than the configuration files but with a different suffix (model_card.json) + # For simplicity we use the same pretrained url than the configuration files + # but with a different suffix (modelcard.json). This suffix is replaced below. model_card_file = ALL_PRETRAINED_CONFIG_ARCHIVE_MAP[pretrained_model_name_or_path] - model_card_file = model_card_file.replace(CONFIG_NAME, MODEL_CARD_NAME) elif os.path.isdir(pretrained_model_name_or_path): model_card_file = os.path.join(pretrained_model_name_or_path, MODEL_CARD_NAME) elif os.path.isfile(pretrained_model_name_or_path) or is_remote_url(pretrained_model_name_or_path): @@ -140,17 +138,22 @@ class ModelCard(object): else: model_card_file = hf_bucket_url(pretrained_model_name_or_path, postfix=MODEL_CARD_NAME) + if find_from_standard_name or pretrained_model_name_or_path in ALL_PRETRAINED_CONFIG_ARCHIVE_MAP: + model_card_file = model_card_file.replace(CONFIG_NAME, MODEL_CARD_NAME) + model_card_file = model_card_file.replace(WEIGHTS_NAME, MODEL_CARD_NAME) + model_card_file = model_card_file.replace(TF2_WEIGHTS_NAME, MODEL_CARD_NAME) + try: # Load from URL or cache if already cached - resolved_model_card_file = cached_path(model_card_file, cache_dir=cache_dir, force_download=force_download, - proxies=proxies, resume_download=resume_download) + resolved_model_card_file = cached_path(model_card_file, cache_dir=cache_dir, force_download=True, + proxies=proxies, resume_download=False) if resolved_model_card_file == model_card_file: logger.info("loading model card file {}".format(model_card_file)) else: logger.info("loading model card file {} from cache at {}".format( model_card_file, resolved_model_card_file)) # Load model card - model_card = cls.from_json_file(resolved_model_card_file) + modelcard = cls.from_json_file(resolved_model_card_file) except EnvironmentError: if pretrained_model_name_or_path in ALL_PRETRAINED_CONFIG_ARCHIVE_MAP: @@ -166,7 +169,7 @@ class ModelCard(object): logger.warning("Creating an empty model card.") # We fall back on creating an empty model card - model_card = cls() + modelcard = cls() except json.JSONDecodeError: logger.warning("Couldn't reach server at '{}' to download model card file or " @@ -175,22 +178,22 @@ class ModelCard(object): logger.warning("Creating an empty model card.") # We fall back on creating an empty model card - model_card = cls() + modelcard = cls() # Update model card with kwargs if needed to_remove = [] for key, value in kwargs.items(): - if hasattr(model_card, key): - setattr(model_card, key, value) + if hasattr(modelcard, key): + setattr(modelcard, key, value) to_remove.append(key) for key in to_remove: kwargs.pop(key, None) - logger.info("Model card: %s", str(model_card)) + logger.info("Model card: %s", str(modelcard)) if return_unused_kwargs: - return model_card, kwargs + return modelcard, kwargs else: - return model_card + return modelcard @classmethod def from_dict(cls, json_object): diff --git a/transformers/pipelines.py b/transformers/pipelines.py index 4dde62cbe5..be2b1db126 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -18,6 +18,8 @@ import csv import json import os import pickle +import logging +import six from abc import ABC, abstractmethod from contextlib import contextmanager from itertools import groupby @@ -26,8 +28,12 @@ from typing import Union, Optional, Tuple, List, Dict import numpy as np -from transformers import AutoConfig, AutoTokenizer, PreTrainedTokenizer, PretrainedConfig, \ - SquadExample, squad_convert_examples_to_features, is_tf_available, is_torch_available, logger, BasicTokenizer +from transformers import (AutoConfig, AutoTokenizer, PreTrainedTokenizer, + PretrainedConfig, ModelCard, SquadExample, + squad_convert_examples_to_features, is_tf_available, + is_torch_available, BasicTokenizer, + ALL_PRETRAINED_MODEL_ARCHIVE_MAP, + ALL_PRETRAINED_CONFIG_ARCHIVE_MAP) if is_tf_available(): import tensorflow as tf @@ -40,6 +46,8 @@ if is_torch_available(): AutoModelForQuestionAnswering, AutoModelForTokenClassification +logger = logging.getLogger(__name__) + class ArgumentHandler(ABC): """ Base interface for handling varargs for each Pipeline @@ -271,11 +279,13 @@ class Pipeline(_ScikitCompat): nlp = QuestionAnsweringPipeline(model=AutoModel.from_pretrained('...'), tokenizer='...') """ def __init__(self, model, tokenizer: PreTrainedTokenizer = None, + modelcard: ModelCard = None, args_parser: ArgumentHandler = None, device: int = -1, binary_output: bool = False): self.model = model self.tokenizer = tokenizer + self.modelcard = modelcard self.device = device self.binary_output = binary_output self._args_parser = args_parser or DefaultArgumentHandler() @@ -294,6 +304,7 @@ class Pipeline(_ScikitCompat): self.model.save_pretrained(save_directory) self.tokenizer.save_pretrained(save_directory) + self.modelcard.save_pretrained(save_directory) def transform(self, X): """ @@ -393,9 +404,10 @@ class FeatureExtractionPipeline(Pipeline): def __init__(self, model, tokenizer: PreTrainedTokenizer = None, + modelcard: ModelCard = None, args_parser: ArgumentHandler = None, device: int = -1): - super().__init__(model, tokenizer, args_parser, device, binary_output=True) + super().__init__(model, tokenizer, modelcard, args_parser, device, binary_output=True) def __call__(self, *args, **kwargs): return super().__call__(*args, **kwargs).tolist() @@ -418,9 +430,10 @@ class NerPipeline(Pipeline): """ def __init__(self, model, tokenizer: PreTrainedTokenizer = None, + modelcard: ModelCard = None, args_parser: ArgumentHandler = None, device: int = -1, binary_output: bool = False): - super().__init__(model, tokenizer, args_parser, device, binary_output) + super().__init__(model, tokenizer, modelcard, args_parser, device, binary_output) self._basic_tokenizer = BasicTokenizer(do_lower_case=False) @@ -554,8 +567,10 @@ class QuestionAnsweringPipeline(Pipeline): else: return SquadExample(None, question, context, None, None, None) - def __init__(self, model, tokenizer: Optional[PreTrainedTokenizer], device: int = -1, **kwargs): - super().__init__(model, tokenizer, args_parser=QuestionAnsweringArgumentHandler(), + def __init__(self, model, tokenizer: Optional[PreTrainedTokenizer], + modelcard: Optional[ModelCard], + device: int = -1, **kwargs): + super().__init__(model, tokenizer, modelcard, args_parser=QuestionAnsweringArgumentHandler(), device=device, **kwargs) def __call__(self, *texts, **kwargs): @@ -725,7 +740,7 @@ SUPPORTED_TASKS = { 'default': { 'model': 'distilbert-base-uncased', 'config': None, - 'tokenizer': 'bert-base-uncased' + 'tokenizer': 'distilbert-base-uncased' } }, 'sentiment-analysis': { @@ -735,7 +750,7 @@ SUPPORTED_TASKS = { 'default': { 'model': 'https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-finetuned-sst-2-english-pytorch_model.bin', 'config': 'https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-finetuned-sst-2-english-config.json', - 'tokenizer': 'bert-base-uncased' + 'tokenizer': 'distilbert-base-uncased' } }, 'ner': { @@ -745,7 +760,7 @@ SUPPORTED_TASKS = { 'default': { 'model': 'https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-finetuned-conll03-english-pytorch_model.bin', 'config': 'https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-finetuned-conll03-english-config.json', - 'tokenizer': 'bert-base-cased' + 'tokenizer': 'bert-large-cased' } }, 'question-answering': { @@ -755,7 +770,7 @@ SUPPORTED_TASKS = { 'default': { 'model': 'distilbert-base-uncased-distilled-squad', 'config': None, - 'tokenizer': 'bert-base-uncased' + 'tokenizer': 'distilbert-base-uncased' } } } @@ -763,7 +778,9 @@ SUPPORTED_TASKS = { def pipeline(task: str, model: Optional = None, config: Optional[Union[str, PretrainedConfig]] = None, - tokenizer: Optional[Union[str, PreTrainedTokenizer]] = None, **kwargs) -> Pipeline: + tokenizer: Optional[Union[str, PreTrainedTokenizer]] = None, + modelcard: Optional[Union[str, ModelCard]] = None, + **kwargs) -> Pipeline: """ Utility factory method to build a pipeline. Pipeline are made of: @@ -777,48 +794,63 @@ def pipeline(task: str, model: Optional = None, pipeline('ner', model=AutoModel.from_pretrained(...), tokenizer=AutoTokenizer.from_pretrained(...) pipeline('ner', model='https://...pytorch-model.bin', config='https://...config.json', tokenizer='bert-base-cased') """ - # Try to infer tokenizer from model name (if provided as str) - if tokenizer is None: - if model is not None and not isinstance(model, str): - # Impossible to guest what is the right tokenizer here - raise Exception('Tokenizer cannot be None if provided model is a PreTrainedModel instance') - else: - tokenizer = model - # Retrieve the task if task not in SUPPORTED_TASKS: raise KeyError("Unknown task {}, available tasks are {}".format(task, list(SUPPORTED_TASKS.keys()))) - targeted_task = SUPPORTED_TASKS[task] - task, allocator = targeted_task['impl'], targeted_task['tf'] if is_tf_available() else targeted_task['pt'] + pipeline_framework = 'tf' if is_tf_available() else ('pt' if is_torch_available() else None) + if pipeline_framework is None: + raise ImportError("At least one of TensorFlow 2.0 or PyTorch should be installed. " + "To install TensorFlow 2.0, read the instructions at https://www.tensorflow.org/install/ " + "To install PyTorch, read the instructions at https://pytorch.org/.") - # Handling for default model for the task + + targeted_task = SUPPORTED_TASKS[task] + task, model_class = targeted_task['impl'], targeted_task[pipeline_framework] + + # Use default model/config/tokenizer for the task if no model is provided if model is None: model, config, tokenizer = tuple(targeted_task['default'].values()) - # Allocate tokenizer - tokenizer = tokenizer if isinstance(tokenizer, PreTrainedTokenizer) else AutoTokenizer.from_pretrained(tokenizer) + # Try to infer tokenizer from model or config name (if provided as str) + if tokenizer is None: + if isinstance(model, str) and model in ALL_PRETRAINED_MODEL_ARCHIVE_MAP: + tokenizer = model + elif isinstance(config, str) and model in ALL_PRETRAINED_CONFIG_ARCHIVE_MAP: + tokenizer = config + else: + # Impossible to guest what is the right tokenizer here + raise Exception("Impossible to guess which tokenizer to use. " + "Please provided a PretrainedTokenizer class or a path/url/shortcut name to a pretrained tokenizer.") - # Special handling for model conversion - if isinstance(model, str): - from_tf = model.endswith('.h5') and not is_tf_available() - from_pt = model.endswith('.bin') and not is_torch_available() + # Try to infer modelcard from model or config name (if provided as str) + if modelcard is None: + # Try to fallback on one of the provided string for model or config (will replace the suffix) + if isinstance(model, str): + modelcard = model + elif isinstance(config, str): + modelcard = config - if from_tf: - logger.warning('Model might be a TensorFlow model (ending with `.h5`) but TensorFlow is not available. ' - 'Trying to load the model with PyTorch.') - elif from_pt: - logger.warning('Model might be a PyTorch model (ending with `.bin`) but PyTorch is not available. ' - 'Trying to load the model with Tensorflow.') - else: - from_tf = from_pt = False + # Instantiate tokenizer if needed + if isinstance(tokenizer, six.string_types): + tokenizer = AutoTokenizer.from_pretrained(tokenizer) + # Instantiate config if needed if isinstance(config, str): config = AutoConfig.from_pretrained(config) + # Instantiate model if needed if isinstance(model, str): - if allocator.__name__.startswith('TF'): - model = allocator.from_pretrained(model, config=config, from_pt=from_pt) - else: - model = allocator.from_pretrained(model, config=config, from_tf=from_tf) + # Handle transparent TF/PT model conversion + model_kwargs = {} + if pipeline_framework == 'pt' and model.endswith('.h5'): + model_kwargs['from_tf'] = True + logger.warning('Model might be a TensorFlow model (ending with `.h5`) but TensorFlow is not available. ' + 'Trying to load the model with PyTorch.') + elif pipeline_framework == 'tf' and model.endswith('.bin'): + model_kwargs['from_pt'] = True + logger.warning('Model might be a PyTorch model (ending with `.bin`) but PyTorch is not available. ' + 'Trying to load the model with Tensorflow.') + model = model_class.from_pretrained(model, config=config, **model_kwargs) + return task(model, tokenizer, **kwargs) diff --git a/transformers/tests/model_card_test.py b/transformers/tests/model_card_test.py index e75716f0aa..b293b5726a 100644 --- a/transformers/tests/model_card_test.py +++ b/transformers/tests/model_card_test.py @@ -18,7 +18,7 @@ import os import json import unittest -from transformers.model_card import ModelCard +from transformers.modelcard import ModelCard from .tokenization_tests_commons import TemporaryDirectory class ModelCardTester(unittest.TestCase): @@ -49,20 +49,20 @@ class ModelCardTester(unittest.TestCase): } def test_model_card_common_properties(self): - model_card = ModelCard.from_dict(self.inputs_dict) - self.assertTrue(hasattr(model_card, 'model_details')) - self.assertTrue(hasattr(model_card, 'intended_use')) - self.assertTrue(hasattr(model_card, 'factors')) - self.assertTrue(hasattr(model_card, 'metrics')) - self.assertTrue(hasattr(model_card, 'evaluation_data')) - self.assertTrue(hasattr(model_card, 'training_data')) - self.assertTrue(hasattr(model_card, 'quantitative_analyses')) - self.assertTrue(hasattr(model_card, 'ethical_considerations')) - self.assertTrue(hasattr(model_card, 'caveats_and_recommendations')) + modelcard = ModelCard.from_dict(self.inputs_dict) + self.assertTrue(hasattr(modelcard, 'model_details')) + self.assertTrue(hasattr(modelcard, 'intended_use')) + self.assertTrue(hasattr(modelcard, 'factors')) + self.assertTrue(hasattr(modelcard, 'metrics')) + self.assertTrue(hasattr(modelcard, 'evaluation_data')) + self.assertTrue(hasattr(modelcard, 'training_data')) + self.assertTrue(hasattr(modelcard, 'quantitative_analyses')) + self.assertTrue(hasattr(modelcard, 'ethical_considerations')) + self.assertTrue(hasattr(modelcard, 'caveats_and_recommendations')) def test_model_card_to_json_string(self): - model_card = ModelCard.from_dict(self.inputs_dict) - obj = json.loads(model_card.to_json_string()) + modelcard = ModelCard.from_dict(self.inputs_dict) + obj = json.loads(modelcard.to_json_string()) for key, value in self.inputs_dict.items(): self.assertEqual(obj[key], value) @@ -70,7 +70,7 @@ class ModelCardTester(unittest.TestCase): model_card_first = ModelCard.from_dict(self.inputs_dict) with TemporaryDirectory() as tmpdirname: - filename = os.path.join(tmpdirname, u"model_card.json") + filename = os.path.join(tmpdirname, u"modelcard.json") model_card_first.to_json_file(filename) model_card_second = ModelCard.from_json_file(filename) From 1fa93ca1eaa249321ef39994e9f022d0799034a3 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 20 Dec 2019 12:34:19 +0100 Subject: [PATCH 463/505] Clean up framework handling --- transformers/pipelines.py | 85 ++++++++++++++++++++++++++------------- 1 file changed, 58 insertions(+), 27 deletions(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index be2b1db126..1c56033f7c 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -48,6 +48,19 @@ if is_torch_available(): logger = logging.getLogger(__name__) +def get_framework(model=None): + if is_tf_available() and is_torch_available() and model is not None and not isinstance(model, str): + # Both framework are available but the use supplied a model class instance. + # Try to guess which framework to use from the model classname + framework = 'tf' if model.__class__.__name__.startswith('TF') else 'pt' + else: + framework = 'tf' if is_tf_available() else ('pt' if is_torch_available() else None) + if framework is None: + raise ImportError("At least one of TensorFlow 2.0 or PyTorch should be installed. " + "To install TensorFlow 2.0, read the instructions at https://www.tensorflow.org/install/ " + "To install PyTorch, read the instructions at https://pytorch.org/.") + return framework + class ArgumentHandler(ABC): """ Base interface for handling varargs for each Pipeline @@ -279,19 +292,23 @@ class Pipeline(_ScikitCompat): nlp = QuestionAnsweringPipeline(model=AutoModel.from_pretrained('...'), tokenizer='...') """ def __init__(self, model, tokenizer: PreTrainedTokenizer = None, - modelcard: ModelCard = None, + modelcard: ModelCard = None, framework: Optional[str] = None, args_parser: ArgumentHandler = None, device: int = -1, binary_output: bool = False): + if framework is None: + framework = get_framework() + self.model = model self.tokenizer = tokenizer self.modelcard = modelcard + self.framework = framework self.device = device self.binary_output = binary_output self._args_parser = args_parser or DefaultArgumentHandler() # Special handling - if self.device >= 0 and not is_tf_available(): + if self.device >= 0 and self.framework == 'pt': self.model = self.model.to('cuda:{}'.format(self.device)) def save_pretrained(self, save_directory): @@ -332,7 +349,7 @@ class Pipeline(_ScikitCompat): Returns: Context manager """ - if is_tf_available(): + if self.framework == 'tf': with tf.device('/CPU:0' if self.device == -1 else '/device:GPU:{}'.format(self.device)): yield else: @@ -371,7 +388,7 @@ class Pipeline(_ScikitCompat): with self.device_placement(): inputs = self.tokenizer.batch_encode_plus( inputs, add_special_tokens=True, - return_tensors='tf' if is_tf_available() else 'pt', + return_tensors=self.framework, max_length=self.tokenizer.max_len ) @@ -387,7 +404,7 @@ class Pipeline(_ScikitCompat): Returns: Numpy array """ - if is_tf_available(): + if self.framework == 'tf': # TODO trace model predictions = self.model(inputs, training=False)[0] else: @@ -405,9 +422,16 @@ class FeatureExtractionPipeline(Pipeline): def __init__(self, model, tokenizer: PreTrainedTokenizer = None, modelcard: ModelCard = None, + framework: Optional[str] = None, args_parser: ArgumentHandler = None, device: int = -1): - super().__init__(model, tokenizer, modelcard, args_parser, device, binary_output=True) + super().__init__(model=model, + tokenizer=tokenizer, + modelcard=modelcard, + framework=framework, + args_parser=args_parser, + device=device, + binary_output=True) def __call__(self, *args, **kwargs): return super().__call__(*args, **kwargs).tolist() @@ -430,10 +454,16 @@ class NerPipeline(Pipeline): """ def __init__(self, model, tokenizer: PreTrainedTokenizer = None, - modelcard: ModelCard = None, + modelcard: ModelCard = None, framework: Optional[str] = None, args_parser: ArgumentHandler = None, device: int = -1, binary_output: bool = False): - super().__init__(model, tokenizer, modelcard, args_parser, device, binary_output) + super().__init__(model=model, + tokenizer=tokenizer, + modelcard=modelcard, + framework=framework, + args_parser=args_parser, + device=device, + binary_output=binary_output) self._basic_tokenizer = BasicTokenizer(do_lower_case=False) @@ -452,12 +482,12 @@ class NerPipeline(Pipeline): tokens = self.tokenizer.encode_plus( sentence, return_attention_mask=False, - return_tensors='tf' if is_tf_available() else 'pt', + return_tensors=self.framework, max_length=self.tokenizer.max_len ) # Forward - if is_tf_available(): + if self.framework == 'tf': entities = self.model(tokens)[0][0].numpy() else: with torch.no_grad(): @@ -549,6 +579,18 @@ class QuestionAnsweringPipeline(Pipeline): Question Answering pipeline using ModelForQuestionAnswering head. """ + def __init__(self, model, + tokenizer: Optional[PreTrainedTokenizer], + modelcard: Optional[ModelCard], + framework: Optional[str] = None, + device: int = -1, **kwargs): + super().__init__(model=model, + tokenizer=tokenizer, + modelcard=modelcard, + framework=framework, + args_parser=QuestionAnsweringArgumentHandler(), + device=device, **kwargs) + @staticmethod def create_sample(question: Union[str, List[str]], context: Union[str, List[str]]) -> Union[SquadExample, List[SquadExample]]: """ @@ -567,12 +609,6 @@ class QuestionAnsweringPipeline(Pipeline): else: return SquadExample(None, question, context, None, None, None) - def __init__(self, model, tokenizer: Optional[PreTrainedTokenizer], - modelcard: Optional[ModelCard], - device: int = -1, **kwargs): - super().__init__(model, tokenizer, modelcard, args_parser=QuestionAnsweringArgumentHandler(), - device=device, **kwargs) - def __call__(self, *texts, **kwargs): """ Args: @@ -608,7 +644,7 @@ class QuestionAnsweringPipeline(Pipeline): # Manage tensor allocation on correct device with self.device_placement(): - if is_tf_available(): + if self.framework == 'tf': fw_args = {k: tf.constant(v) for (k, v) in fw_args.items()} start, end = self.model(fw_args) start, end = start.numpy(), end.numpy() @@ -798,15 +834,10 @@ def pipeline(task: str, model: Optional = None, if task not in SUPPORTED_TASKS: raise KeyError("Unknown task {}, available tasks are {}".format(task, list(SUPPORTED_TASKS.keys()))) - pipeline_framework = 'tf' if is_tf_available() else ('pt' if is_torch_available() else None) - if pipeline_framework is None: - raise ImportError("At least one of TensorFlow 2.0 or PyTorch should be installed. " - "To install TensorFlow 2.0, read the instructions at https://www.tensorflow.org/install/ " - "To install PyTorch, read the instructions at https://pytorch.org/.") - + framework = get_framework(model) targeted_task = SUPPORTED_TASKS[task] - task, model_class = targeted_task['impl'], targeted_task[pipeline_framework] + task, model_class = targeted_task['impl'], targeted_task[framework] # Use default model/config/tokenizer for the task if no model is provided if model is None: @@ -843,14 +874,14 @@ def pipeline(task: str, model: Optional = None, if isinstance(model, str): # Handle transparent TF/PT model conversion model_kwargs = {} - if pipeline_framework == 'pt' and model.endswith('.h5'): + if framework == 'pt' and model.endswith('.h5'): model_kwargs['from_tf'] = True logger.warning('Model might be a TensorFlow model (ending with `.h5`) but TensorFlow is not available. ' 'Trying to load the model with PyTorch.') - elif pipeline_framework == 'tf' and model.endswith('.bin'): + elif framework == 'tf' and model.endswith('.bin'): model_kwargs['from_pt'] = True logger.warning('Model might be a PyTorch model (ending with `.bin`) but PyTorch is not available. ' 'Trying to load the model with Tensorflow.') model = model_class.from_pretrained(model, config=config, **model_kwargs) - return task(model, tokenizer, **kwargs) + return task(model=model, tokenizer=tokenizer, modelcard=modelcard, framework=framework, **kwargs) From 825697cad4907595bbb76eb43d96962d7bd52117 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 20 Dec 2019 12:51:10 +0100 Subject: [PATCH 464/505] fix tests --- transformers/pipelines.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index 1c56033f7c..8b5a14fc56 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -32,7 +32,6 @@ from transformers import (AutoConfig, AutoTokenizer, PreTrainedTokenizer, PretrainedConfig, ModelCard, SquadExample, squad_convert_examples_to_features, is_tf_available, is_torch_available, BasicTokenizer, - ALL_PRETRAINED_MODEL_ARCHIVE_MAP, ALL_PRETRAINED_CONFIG_ARCHIVE_MAP) if is_tf_available(): @@ -845,9 +844,9 @@ def pipeline(task: str, model: Optional = None, # Try to infer tokenizer from model or config name (if provided as str) if tokenizer is None: - if isinstance(model, str) and model in ALL_PRETRAINED_MODEL_ARCHIVE_MAP: + if isinstance(model, str) and model in ALL_PRETRAINED_CONFIG_ARCHIVE_MAP: tokenizer = model - elif isinstance(config, str) and model in ALL_PRETRAINED_CONFIG_ARCHIVE_MAP: + elif isinstance(config, str) and config in ALL_PRETRAINED_CONFIG_ARCHIVE_MAP: tokenizer = config else: # Impossible to guest what is the right tokenizer here From 01ffc65e9b4e74d0399212435da2d46c6edf2563 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 20 Dec 2019 13:16:23 +0100 Subject: [PATCH 465/505] update tests to remove unittest.patch --- transformers/pipelines.py | 9 ++- transformers/tests/pipelines_test.py | 91 +++++++++++++++++----------- 2 files changed, 60 insertions(+), 40 deletions(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index 8b5a14fc56..efb1de92e1 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -48,16 +48,19 @@ if is_torch_available(): logger = logging.getLogger(__name__) def get_framework(model=None): + """ Select framework (TensorFlow/PyTorch) to use. + If both frameworks are installed and no specific model is provided, defaults to using TensorFlow. + """ if is_tf_available() and is_torch_available() and model is not None and not isinstance(model, str): # Both framework are available but the use supplied a model class instance. # Try to guess which framework to use from the model classname framework = 'tf' if model.__class__.__name__.startswith('TF') else 'pt' - else: - framework = 'tf' if is_tf_available() else ('pt' if is_torch_available() else None) - if framework is None: + elif not is_tf_available() and not is_torch_available(): raise ImportError("At least one of TensorFlow 2.0 or PyTorch should be installed. " "To install TensorFlow 2.0, read the instructions at https://www.tensorflow.org/install/ " "To install PyTorch, read the instructions at https://pytorch.org/.") + else: + framework = 'tf' if is_tf_available() else 'pt' return framework class ArgumentHandler(ABC): diff --git a/transformers/tests/pipelines_test.py b/transformers/tests/pipelines_test.py index a8fe668221..14bf07ee30 100644 --- a/transformers/tests/pipelines_test.py +++ b/transformers/tests/pipelines_test.py @@ -1,5 +1,4 @@ import unittest -from unittest.mock import patch from typing import Iterable @@ -35,16 +34,6 @@ TEXT_CLASSIF_FINETUNED_MODELS = { } -@require_tf -def tf_pipeline(*args, **kwargs): - return pipeline(**kwargs) - - -@require_torch -def torch_pipeline(*args, **kwargs): - return pipeline(**kwargs) - - class MonoColumnInputTestCase(unittest.TestCase): def _test_mono_column_pipeline(self, nlp, valid_inputs: list, invalid_inputs: list, output_keys: Iterable[str]): self.assertIsNotNone(nlp) @@ -72,43 +61,57 @@ class MonoColumnInputTestCase(unittest.TestCase): self.assertRaises(Exception, nlp, invalid_inputs) + @require_torch def test_ner(self): mandatory_keys = {'entity', 'word', 'score'} valid_inputs = ['HuggingFace is solving NLP one commit at a time.', 'HuggingFace is based in New-York & Paris'] invalid_inputs = [None] for tokenizer, model, config in NER_FINETUNED_MODELS: - with patch('transformers.pipelines.is_torch_available', return_value=False): - nlp = tf_pipeline(task='ner', model=model, config=config, tokenizer=tokenizer) - self._test_mono_column_pipeline(nlp, valid_inputs, invalid_inputs, mandatory_keys) + nlp = pipeline(task='ner', model=model, config=config, tokenizer=tokenizer) + self._test_mono_column_pipeline(nlp, valid_inputs, invalid_inputs, mandatory_keys) - with patch('transformers.pipelines.is_tf_available', return_value=False): - nlp = torch_pipeline(task='ner', model=model, config=config, tokenizer=tokenizer) - self._test_mono_column_pipeline(nlp, valid_inputs, invalid_inputs, mandatory_keys) + @require_tf + def test_tf_ner(self): + mandatory_keys = {'entity', 'word', 'score'} + valid_inputs = ['HuggingFace is solving NLP one commit at a time.', 'HuggingFace is based in New-York & Paris'] + invalid_inputs = [None] + for tokenizer, model, config in NER_FINETUNED_MODELS: + nlp = pipeline(task='ner', model=model, config=config, tokenizer=tokenizer) + self._test_mono_column_pipeline(nlp, valid_inputs, invalid_inputs, mandatory_keys) + @require_torch def test_sentiment_analysis(self): mandatory_keys = {'label'} valid_inputs = ['HuggingFace is solving NLP one commit at a time.', 'HuggingFace is based in New-York & Paris'] invalid_inputs = [None] for tokenizer, model, config in TEXT_CLASSIF_FINETUNED_MODELS: - with patch('transformers.pipelines.is_torch_available', return_value=False): - nlp = tf_pipeline(task='sentiment-analysis', model=model, config=config, tokenizer=tokenizer) - self._test_mono_column_pipeline(nlp, valid_inputs, invalid_inputs, mandatory_keys) + nlp = pipeline(task='sentiment-analysis', model=model, config=config, tokenizer=tokenizer) + self._test_mono_column_pipeline(nlp, valid_inputs, invalid_inputs, mandatory_keys) - with patch('transformers.pipelines.is_tf_available', return_value=False): - nlp = torch_pipeline(task='sentiment-analysis', model=model, config=config, tokenizer=tokenizer) - self._test_mono_column_pipeline(nlp, valid_inputs, invalid_inputs, mandatory_keys) + @require_tf + def test_tf_sentiment_analysis(self): + mandatory_keys = {'label'} + valid_inputs = ['HuggingFace is solving NLP one commit at a time.', 'HuggingFace is based in New-York & Paris'] + invalid_inputs = [None] + for tokenizer, model, config in TEXT_CLASSIF_FINETUNED_MODELS: + nlp = pipeline(task='sentiment-analysis', model=model, config=config, tokenizer=tokenizer) + self._test_mono_column_pipeline(nlp, valid_inputs, invalid_inputs, mandatory_keys) + @require_torch def test_features_extraction(self): valid_inputs = ['HuggingFace is solving NLP one commit at a time.', 'HuggingFace is based in New-York & Paris'] invalid_inputs = [None] for tokenizer, model, config in FEATURE_EXTRACT_FINETUNED_MODELS: - with patch('transformers.pipelines.is_torch_available', return_value=False): - nlp = tf_pipeline(task='sentiment-analysis', model=model, config=config, tokenizer=tokenizer) - self._test_mono_column_pipeline(nlp, valid_inputs, invalid_inputs, {}) + nlp = pipeline(task='sentiment-analysis', model=model, config=config, tokenizer=tokenizer) + self._test_mono_column_pipeline(nlp, valid_inputs, invalid_inputs, {}) - with patch('transformers.pipelines.is_tf_available', return_value=False): - nlp = torch_pipeline(task='sentiment-analysis', model=model, config=config, tokenizer=tokenizer) - self._test_mono_column_pipeline(nlp, valid_inputs, invalid_inputs, {}) + @require_tf + def test_tf_features_extraction(self): + valid_inputs = ['HuggingFace is solving NLP one commit at a time.', 'HuggingFace is based in New-York & Paris'] + invalid_inputs = [None] + for tokenizer, model, config in FEATURE_EXTRACT_FINETUNED_MODELS: + nlp = pipeline(task='sentiment-analysis', model=model, config=config, tokenizer=tokenizer) + self._test_mono_column_pipeline(nlp, valid_inputs, invalid_inputs, {}) class MultiColumnInputTestCase(unittest.TestCase): @@ -132,6 +135,7 @@ class MultiColumnInputTestCase(unittest.TestCase): self.assertRaises(Exception, nlp, invalid_inputs[0]) self.assertRaises(Exception, nlp, invalid_inputs) + @require_torch def test_question_answering(self): mandatory_output_keys = {'score', 'answer', 'start', 'end'} valid_samples = [ @@ -149,16 +153,29 @@ class MultiColumnInputTestCase(unittest.TestCase): ] for tokenizer, model, config in QA_FINETUNED_MODELS: + nlp = pipeline(task='question-answering', model=model, config=config, tokenizer=tokenizer) + self._test_multicolumn_pipeline(nlp, valid_samples, invalid_samples, mandatory_output_keys) - # Test for Tensorflow - with patch('transformers.pipelines.is_torch_available', return_value=False): - nlp = pipeline(task='question-answering', model=model, config=config, tokenizer=tokenizer) - self._test_multicolumn_pipeline(nlp, valid_samples, invalid_samples, mandatory_output_keys) + @require_tf + def test_tf_question_answering(self): + mandatory_output_keys = {'score', 'answer', 'start', 'end'} + valid_samples = [ + {'question': 'Where was HuggingFace founded ?', 'context': 'HuggingFace was founded in Paris.'}, + { + 'question': 'In what field is HuggingFace working ?', + 'context': 'HuggingFace is a startup based in New-York founded in Paris which is trying to solve NLP.' + } + ] + invalid_samples = [ + {'question': '', 'context': 'This is a test to try empty question edge case'}, + {'question': None, 'context': 'This is a test to try empty question edge case'}, + {'question': 'What is does with empty context ?', 'context': ''}, + {'question': 'What is does with empty context ?', 'context': None}, + ] - # Test for PyTorch - with patch('transformers.pipelines.is_tf_available', return_value=False): - nlp = pipeline(task='question-answering', model=model, config=config, tokenizer=tokenizer) - self._test_multicolumn_pipeline(nlp, valid_samples, invalid_samples, mandatory_output_keys) + for tokenizer, model, config in QA_FINETUNED_MODELS: + nlp = pipeline(task='question-answering', model=model, config=config, tokenizer=tokenizer) + self._test_multicolumn_pipeline(nlp, valid_samples, invalid_samples, mandatory_output_keys) if __name__ == '__main__': From 15dda5ea32655f2ea565f8d5cd586a036399dba3 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 20 Dec 2019 13:20:41 +0100 Subject: [PATCH 466/505] remove python 2 tests for circle-ci cc @aaugustin @julien-c @LysandreJik --- .circleci/config.yml | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7a64eaba7d..b094067eb5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -44,32 +44,6 @@ jobs: - run: sudo pip install tensorboardX scikit-learn - run: python -m pytest -sv ./transformers/tests/ --cov - run: codecov - build_py2_torch: - working_directory: ~/transformers - resource_class: large - parallelism: 1 - docker: - - image: circleci/python:2.7 - steps: - - checkout - - run: sudo pip install torch - - run: sudo pip install --progress-bar off . - - run: sudo pip install pytest codecov pytest-cov - - run: python -m pytest -sv ./transformers/tests/ --cov - - run: codecov - build_py2_tf: - working_directory: ~/transformers - resource_class: large - parallelism: 1 - docker: - - image: circleci/python:2.7 - steps: - - checkout - - run: sudo pip install tensorflow - - run: sudo pip install --progress-bar off . - - run: sudo pip install pytest codecov pytest-cov - - run: python -m pytest -sv ./transformers/tests/ --cov - - run: codecov build_py3_custom_tokenizers: working_directory: ~/transformers docker: @@ -116,6 +90,4 @@ workflows: - build_py3_torch_and_tf - build_py3_torch - build_py3_tf - - build_py2_torch - - build_py2_tf - deploy_doc: *workflow_filters From 73fcebf7ec122e68b93f50fc770f0515502eb025 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 20 Dec 2019 13:47:35 +0100 Subject: [PATCH 467/505] update serving command --- setup.py | 6 +++--- transformers-cli | 2 +- transformers/commands/serving.py | 35 +++++++++++++++++++++----------- 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/setup.py b/setup.py index 6560cc4968..4bfb774155 100644 --- a/setup.py +++ b/setup.py @@ -38,9 +38,9 @@ from setuptools import find_packages, setup extras = { - 'serving': ['uvicorn', 'fastapi'], - 'serving-tf': ['uvicorn', 'fastapi', 'tensorflow'], - 'serving-torch': ['uvicorn', 'fastapi', 'torch'] + 'serving': ['pydantic', 'uvicorn', 'fastapi'], + 'serving-tf': ['pydantic', 'uvicorn', 'fastapi', 'tensorflow'], + 'serving-torch': ['pydantic', 'uvicorn', 'fastapi', 'torch'] } extras['all'] = [package for package in extras.values()] diff --git a/transformers-cli b/transformers-cli index db2bd0e2a3..0a980a3574 100755 --- a/transformers-cli +++ b/transformers-cli @@ -3,9 +3,9 @@ from argparse import ArgumentParser from transformers.commands.download import DownloadCommand from transformers.commands.run import RunCommand -from transformers.commands.serving import ServeCommand from transformers.commands.user import UserCommands from transformers.commands.convert import ConvertCommand +from transformers.commands.serving import ServeCommand if __name__ == '__main__': parser = ArgumentParser('Transformers CLI tool', usage='transformers-cli []') diff --git a/transformers/commands/serving.py b/transformers/commands/serving.py index a7321470ce..3c3f852809 100644 --- a/transformers/commands/serving.py +++ b/transformers/commands/serving.py @@ -1,16 +1,23 @@ from argparse import ArgumentParser, Namespace from typing import List, Optional, Union, Any -from fastapi import FastAPI, HTTPException, Body -from logging import getLogger +import logging -from pydantic import BaseModel -from uvicorn import run +try: + from uvicorn import run + from fastapi import FastAPI, HTTPException, Body + from pydantic import BaseModel + _serve_dependancies_installed = True +except (ImportError, AttributeError): + BaseModel = object + Body = lambda *x, **y: None + _serve_dependancies_installed = False from transformers import Pipeline from transformers.commands import BaseTransformersCLICommand from transformers.pipelines import SUPPORTED_TASKS, pipeline +logger = logging.getLogger('transformers-cli/serving') def serve_command_factory(args: Namespace): """ @@ -70,20 +77,24 @@ class ServeCommand(BaseTransformersCLICommand): serve_parser.set_defaults(func=serve_command_factory) def __init__(self, pipeline: Pipeline, host: str, port: int): - self._logger = getLogger('transformers-cli/serving') self._pipeline = pipeline - self._logger.info('Serving model over {}:{}'.format(host, port)) self._host = host self._port = port - self._app = FastAPI() + if not _serve_dependancies_installed: + raise ImportError("Using serve command requires FastAPI and unicorn. " + "Please install transformers with [serving]: pip install transformers[serving]." + "Or install FastAPI and unicorn separatly.") + else: + logger.info('Serving model over {}:{}'.format(host, port)) + self._app = FastAPI() - # Register routes - self._app.add_api_route('/', self.model_info, response_model=ServeModelInfoResult, methods=['GET']) - self._app.add_api_route('/tokenize', self.tokenize, response_model=ServeTokenizeResult, methods=['POST']) - self._app.add_api_route('/detokenize', self.detokenize, response_model=ServeDeTokenizeResult, methods=['POST']) - self._app.add_api_route('/forward', self.forward, response_model=ServeForwardResult, methods=['POST']) + # Register routes + self._app.add_api_route('/', self.model_info, response_model=ServeModelInfoResult, methods=['GET']) + self._app.add_api_route('/tokenize', self.tokenize, response_model=ServeTokenizeResult, methods=['POST']) + self._app.add_api_route('/detokenize', self.detokenize, response_model=ServeDeTokenizeResult, methods=['POST']) + self._app.add_api_route('/forward', self.forward, response_model=ServeForwardResult, methods=['POST']) def run(self): run(self._app, host=self._host, port=self._port) From c37815f1300519e1a812e1080c46641db6f9f604 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 20 Dec 2019 14:35:40 +0100 Subject: [PATCH 468/505] clean up PT <=> TF 2.0 conversion and config loading --- .../convert_pytorch_checkpoint_to_tf2.py | 9 +++++---- transformers/modeling_tf_utils.py | 17 ++++++++++++----- transformers/modeling_utils.py | 17 ++++++++++++----- 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/transformers/convert_pytorch_checkpoint_to_tf2.py b/transformers/convert_pytorch_checkpoint_to_tf2.py index 4a9832f123..0edac6fb7d 100644 --- a/transformers/convert_pytorch_checkpoint_to_tf2.py +++ b/transformers/convert_pytorch_checkpoint_to_tf2.py @@ -32,7 +32,7 @@ from transformers import (load_pytorch_checkpoint_in_tf2_model, TransfoXLConfig, TFTransfoXLLMHeadModel, TRANSFO_XL_PRETRAINED_CONFIG_ARCHIVE_MAP, OpenAIGPTConfig, TFOpenAIGPTLMHeadModel, OPENAI_GPT_PRETRAINED_CONFIG_ARCHIVE_MAP, RobertaConfig, TFRobertaForMaskedLM, TFRobertaForSequenceClassification, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, - DistilBertConfig, TFDistilBertForMaskedLM, TFDistilBertForQuestionAnswering, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, + DistilBertConfig, TFDistilBertForMaskedLM, TFDistilBertForQuestionAnswering, TFDistilBertForSequenceClassification, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, CTRLConfig, TFCTRLLMHeadModel, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP, AlbertConfig, TFAlbertForMaskedLM, ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, T5Config, TFT5WithLMHeadModel, T5_PRETRAINED_CONFIG_ARCHIVE_MAP) @@ -47,7 +47,7 @@ if is_torch_available(): TransfoXLLMHeadModel, TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP, OpenAIGPTLMHeadModel, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP, RobertaForMaskedLM, RobertaForSequenceClassification, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, - DistilBertForMaskedLM, DistilBertForQuestionAnswering, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, + DistilBertForMaskedLM, DistilBertForQuestionAnswering, DistilBertForSequenceClassification, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP, AlbertForMaskedLM, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP, T5WithLMHeadModel, T5_PRETRAINED_MODEL_ARCHIVE_MAP) @@ -59,7 +59,7 @@ else: TransfoXLLMHeadModel, TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP, OpenAIGPTLMHeadModel, OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP, RobertaForMaskedLM, RobertaForSequenceClassification, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, - DistilBertForMaskedLM, DistilBertForQuestionAnswering, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, + DistilBertForMaskedLM, DistilBertForSequenceClassification, DistilBertForQuestionAnswering, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP, AlbertForMaskedLM, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP, T5WithLMHeadModel, T5_PRETRAINED_MODEL_ARCHIVE_MAP) = ( @@ -70,7 +70,7 @@ else: None, None, None, None, None, None, None, - None, None, None, + None, None, None, None, None, None, None, None, None, None) @@ -93,6 +93,7 @@ MODEL_CLASSES = { 'roberta-large-mnli': (RobertaConfig, TFRobertaForSequenceClassification, RobertaForSequenceClassification, ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP, ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP), 'distilbert': (DistilBertConfig, TFDistilBertForMaskedLM, DistilBertForMaskedLM, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP), 'distilbert-base-uncased-distilled-squad': (DistilBertConfig, TFDistilBertForQuestionAnswering, DistilBertForQuestionAnswering, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP), + 'distilbert-base-uncased-distilled-squad': (DistilBertConfig, TFDistilBertForQuestionAnswering, DistilBertForQuestionAnswering, DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP, DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP), 'ctrl': (CTRLConfig, TFCTRLLMHeadModel, CTRLLMHeadModel, CTRL_PRETRAINED_MODEL_ARCHIVE_MAP, CTRL_PRETRAINED_CONFIG_ARCHIVE_MAP), 'albert': (AlbertConfig, TFAlbertForMaskedLM, AlbertForMaskedLM, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP, ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP), 't5': (T5Config, TFT5WithLMHeadModel, T5WithLMHeadModel, T5_PRETRAINED_MODEL_ARCHIVE_MAP, T5_PRETRAINED_CONFIG_ARCHIVE_MAP), diff --git a/transformers/modeling_tf_utils.py b/transformers/modeling_tf_utils.py index 401ffeb67e..0aa65a9f17 100644 --- a/transformers/modeling_tf_utils.py +++ b/transformers/modeling_tf_utils.py @@ -184,7 +184,9 @@ class TFPreTrainedModel(tf.keras.Model): model_args: (`optional`) Sequence of positional arguments: All remaning positional arguments will be passed to the underlying model's ``__init__`` method - config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: + config: (`optional`) one of: + - an instance of a class derived from :class:`~transformers.PretrainedConfig`, or + - a string valid as input to :func:`~transformers.PretrainedConfig.from_pretrained()` Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when: - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or @@ -236,10 +238,11 @@ class TFPreTrainedModel(tf.keras.Model): proxies = kwargs.pop('proxies', None) output_loading_info = kwargs.pop('output_loading_info', False) - # Load config - if config is None: + # Load config if we don't provide a configuration + if not isinstance(config, PretrainedConfig): + config_path = config if config is not None else pretrained_model_name_or_path config, model_kwargs = cls.config_class.from_pretrained( - pretrained_model_name_or_path, *model_args, + config_path, *model_args, cache_dir=cache_dir, return_unused_kwargs=True, force_download=force_download, resume_download=resume_download, @@ -310,7 +313,11 @@ class TFPreTrainedModel(tf.keras.Model): assert os.path.isfile(resolved_archive_file), "Error retrieving file {}".format(resolved_archive_file) # 'by_name' allow us to do transfer learning by skipping/adding layers # see https://github.com/tensorflow/tensorflow/blob/00fad90125b18b80fe054de1055770cfb8fe4ba3/tensorflow/python/keras/engine/network.py#L1339-L1357 - model.load_weights(resolved_archive_file, by_name=True) + try: + model.load_weights(resolved_archive_file, by_name=True) + except OSError: + raise OSError("Unable to load weights from h5 file. " + "If you tried to load a TF 2.0 model from a PyTorch checkpoint, please set from_pt=True. ") ret = model(model.dummy_inputs, training=False) # Make sure restore ops are run diff --git a/transformers/modeling_utils.py b/transformers/modeling_utils.py index eff54f71e1..3bc407e4a3 100644 --- a/transformers/modeling_utils.py +++ b/transformers/modeling_utils.py @@ -281,7 +281,9 @@ class PreTrainedModel(nn.Module): model_args: (`optional`) Sequence of positional arguments: All remaning positional arguments will be passed to the underlying model's ``__init__`` method - config: (`optional`) instance of a class derived from :class:`~transformers.PretrainedConfig`: + config: (`optional`) one of: + - an instance of a class derived from :class:`~transformers.PretrainedConfig`, or + - a string valid as input to :func:`~transformers.PretrainedConfig.from_pretrained()` Configuration for the model to use instead of an automatically loaded configuation. Configuration can be automatically loaded when: - the model is a model provided by the library (loaded with the ``shortcut-name`` string of a pretrained model), or @@ -336,10 +338,11 @@ class PreTrainedModel(nn.Module): proxies = kwargs.pop('proxies', None) output_loading_info = kwargs.pop('output_loading_info', False) - # Load config - if config is None: + # Load config if we don't provide a configuration + if not isinstance(config, PretrainedConfig): + config_path = config if config is not None else pretrained_model_name_or_path config, model_kwargs = cls.config_class.from_pretrained( - pretrained_model_name_or_path, *model_args, + config_path, *model_args, cache_dir=cache_dir, return_unused_kwargs=True, force_download=force_download, resume_download=resume_download, @@ -408,7 +411,11 @@ class PreTrainedModel(nn.Module): model = cls(config, *model_args, **model_kwargs) if state_dict is None and not from_tf: - state_dict = torch.load(resolved_archive_file, map_location='cpu') + try: + state_dict = torch.load(resolved_archive_file, map_location='cpu') + except: + raise OSError("Unable to load weights from pytorch checkpoint file. " + "If you tried to load a PyTorch model from a TF 2.0 checkpoint, please set from_tf=True. ") missing_keys = [] unexpected_keys = [] From 7f74084528b8f9fb7678b82829366f11326af62f Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Fri, 20 Dec 2019 14:47:04 +0100 Subject: [PATCH 469/505] Fix leading axis added when saving through the command run --- transformers/commands/run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transformers/commands/run.py b/transformers/commands/run.py index 78109b2a16..50c42d3a40 100644 --- a/transformers/commands/run.py +++ b/transformers/commands/run.py @@ -50,9 +50,9 @@ class RunCommand(BaseTransformersCLICommand): nlp, output = self._nlp, [] for entry in self._reader: if self._reader.is_multi_columns: - output += [nlp(**entry)] + output += nlp(**entry) else: - output += [nlp(entry)] + output += nlp(entry) # Saving data if self._nlp.binary_output: From db0795b5d0da0a9968035751b246854240a2c2ec Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 20 Dec 2019 15:07:00 +0100 Subject: [PATCH 470/505] defaults models for tf and pt - update tests --- transformers/pipelines.py | 23 ++++++++++++++---- transformers/tests/pipelines_test.py | 36 ++++++++++++++++++++++++---- 2 files changed, 50 insertions(+), 9 deletions(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index efb1de92e1..33e1ab4022 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -776,7 +776,10 @@ SUPPORTED_TASKS = { 'tf': TFAutoModel if is_tf_available() else None, 'pt': AutoModel if is_torch_available() else None, 'default': { - 'model': 'distilbert-base-uncased', + 'model': { + 'pt': 'distilbert-base-uncased', + 'tf': 'distilbert-base-uncased', + }, 'config': None, 'tokenizer': 'distilbert-base-uncased' } @@ -786,7 +789,10 @@ SUPPORTED_TASKS = { 'tf': TFAutoModelForSequenceClassification if is_tf_available() else None, 'pt': AutoModelForSequenceClassification if is_torch_available() else None, 'default': { - 'model': 'https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-finetuned-sst-2-english-pytorch_model.bin', + 'model': { + 'pt': 'https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-finetuned-sst-2-english-pytorch_model.bin', + 'tf': 'https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-finetuned-sst-2-english-tf_model.h5', + }, 'config': 'https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-finetuned-sst-2-english-config.json', 'tokenizer': 'distilbert-base-uncased' } @@ -796,7 +802,10 @@ SUPPORTED_TASKS = { 'tf': TFAutoModelForTokenClassification if is_tf_available() else None, 'pt': AutoModelForTokenClassification if is_torch_available() else None, 'default': { - 'model': 'https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-finetuned-conll03-english-pytorch_model.bin', + 'model': { + 'pt':'https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-finetuned-conll03-english-pytorch_model.bin', + 'tf': 'https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-finetuned-conll03-english-tf_model.h5', + }, 'config': 'https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-finetuned-conll03-english-config.json', 'tokenizer': 'bert-large-cased' } @@ -806,7 +815,10 @@ SUPPORTED_TASKS = { 'tf': TFAutoModelForQuestionAnswering if is_tf_available() else None, 'pt': AutoModelForQuestionAnswering if is_torch_available() else None, 'default': { - 'model': 'distilbert-base-uncased-distilled-squad', + 'model': { + 'pt': 'distilbert-base-uncased-distilled-squad', + 'tf': 'distilbert-base-uncased-distilled-squad', + }, 'config': None, 'tokenizer': 'distilbert-base-uncased' } @@ -843,7 +855,8 @@ def pipeline(task: str, model: Optional = None, # Use default model/config/tokenizer for the task if no model is provided if model is None: - model, config, tokenizer = tuple(targeted_task['default'].values()) + models, config, tokenizer = tuple(targeted_task['default'].values()) + model = models[framework] # Try to infer tokenizer from model or config name (if provided as str) if tokenizer is None: diff --git a/transformers/tests/pipelines_test.py b/transformers/tests/pipelines_test.py index 14bf07ee30..08a1507770 100644 --- a/transformers/tests/pipelines_test.py +++ b/transformers/tests/pipelines_test.py @@ -11,6 +11,20 @@ QA_FINETUNED_MODELS = { ('bert-base-uncased', 'distilbert-base-uncased-distilled-squad', None) } +TF_QA_FINETUNED_MODELS = { + ('bert-base-uncased', 'bert-large-uncased-whole-word-masking-finetuned-squad', None), + ('bert-base-cased', 'bert-large-cased-whole-word-masking-finetuned-squad', None), + ('bert-base-uncased', 'distilbert-base-uncased-distilled-squad', None) +} + +TF_NER_FINETUNED_MODELS = { + ( + 'bert-base-cased', + 'https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-finetuned-conll03-english-tf_model.h5', + 'https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-cased-finetuned-conll03-english-config.json' + ) +} + NER_FINETUNED_MODELS = { ( 'bert-base-cased', @@ -25,6 +39,20 @@ FEATURE_EXTRACT_FINETUNED_MODELS = { ('distilbert-base-uncased', 'distilbert-base-uncased', None) } +TF_FEATURE_EXTRACT_FINETUNED_MODELS = { + ('bert-base-cased', 'bert-base-cased', None), + # ('xlnet-base-cased', 'xlnet-base-cased', None), # Disabled for now as it crash for TF2 + ('distilbert-base-uncased', 'distilbert-base-uncased', None) +} + +TF_TEXT_CLASSIF_FINETUNED_MODELS = { + ( + 'bert-base-uncased', + 'https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-finetuned-sst-2-english-tf_model.h5', + 'https://s3.amazonaws.com/models.huggingface.co/bert/distilbert-base-uncased-finetuned-sst-2-english-config.json' + ) +} + TEXT_CLASSIF_FINETUNED_MODELS = { ( 'bert-base-uncased', @@ -75,7 +103,7 @@ class MonoColumnInputTestCase(unittest.TestCase): mandatory_keys = {'entity', 'word', 'score'} valid_inputs = ['HuggingFace is solving NLP one commit at a time.', 'HuggingFace is based in New-York & Paris'] invalid_inputs = [None] - for tokenizer, model, config in NER_FINETUNED_MODELS: + for tokenizer, model, config in TF_NER_FINETUNED_MODELS: nlp = pipeline(task='ner', model=model, config=config, tokenizer=tokenizer) self._test_mono_column_pipeline(nlp, valid_inputs, invalid_inputs, mandatory_keys) @@ -93,7 +121,7 @@ class MonoColumnInputTestCase(unittest.TestCase): mandatory_keys = {'label'} valid_inputs = ['HuggingFace is solving NLP one commit at a time.', 'HuggingFace is based in New-York & Paris'] invalid_inputs = [None] - for tokenizer, model, config in TEXT_CLASSIF_FINETUNED_MODELS: + for tokenizer, model, config in TF_TEXT_CLASSIF_FINETUNED_MODELS: nlp = pipeline(task='sentiment-analysis', model=model, config=config, tokenizer=tokenizer) self._test_mono_column_pipeline(nlp, valid_inputs, invalid_inputs, mandatory_keys) @@ -109,7 +137,7 @@ class MonoColumnInputTestCase(unittest.TestCase): def test_tf_features_extraction(self): valid_inputs = ['HuggingFace is solving NLP one commit at a time.', 'HuggingFace is based in New-York & Paris'] invalid_inputs = [None] - for tokenizer, model, config in FEATURE_EXTRACT_FINETUNED_MODELS: + for tokenizer, model, config in TF_FEATURE_EXTRACT_FINETUNED_MODELS: nlp = pipeline(task='sentiment-analysis', model=model, config=config, tokenizer=tokenizer) self._test_mono_column_pipeline(nlp, valid_inputs, invalid_inputs, {}) @@ -173,7 +201,7 @@ class MultiColumnInputTestCase(unittest.TestCase): {'question': 'What is does with empty context ?', 'context': None}, ] - for tokenizer, model, config in QA_FINETUNED_MODELS: + for tokenizer, model, config in TF_QA_FINETUNED_MODELS: nlp = pipeline(task='question-answering', model=model, config=config, tokenizer=tokenizer) self._test_multicolumn_pipeline(nlp, valid_samples, invalid_samples, mandatory_output_keys) From 4e3f745ba4e754e415c184d53c874031101d263b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Louf?= Date: Fri, 20 Dec 2019 11:13:46 +0100 Subject: [PATCH 471/505] add example for Model2Model in quickstart --- docs/source/quickstart.md | 95 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 94 insertions(+), 1 deletion(-) diff --git a/docs/source/quickstart.md b/docs/source/quickstart.md index 530aff8eb0..60e2cf3fd8 100644 --- a/docs/source/quickstart.md +++ b/docs/source/quickstart.md @@ -219,4 +219,97 @@ sequence = tokenizer.decode(generated) print(sequence) ``` -The model only requires a single token as input as all the previous tokens' key/value pairs are contained in the `past`. \ No newline at end of file +The model only requires a single token as input as all the previous tokens' key/value pairs are contained in the `past`. + +### Model2Model example + +Encoder-decoder architectures require two tokenized inputs: one for the encoder and the other one for the decoder. Let's assume that we want to use `Model2Model` for generative question answering, and start by tokenizing the question and answer that will be fed to the model. + +```python +import torch +from transformers import BertTokenizer, Model2Model + +# OPTIONAL: if you want to have more information on what's happening under the hood, activate the logger as follows +import logging +logging.basicConfig(level=logging.INFO) + +# Load pre-trained model tokenizer (vocabulary) +tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') + +# Encode the input to the encoder (the question) +question = "Who was Jim Henson?" +encoded_question = tokenizer.encode(question) + +# Encode the input to the decoder (the answer) +answer = "Jim Henson was a puppeteer" +encoded_answer = tokenizer.encode(answer) + +# Convert inputs to PyTorch tensors +question_tensor = torch.tensor([encoded_question]) +answer_tensor = torch.tensor([encoded_answer]) +``` + +Let's see how we can use `Model2Model` to get the value of the loss associated with this (question, answer) pair: + +```python +# In order to compute the loss we need to provide language model +# labels (the token ids that the model should have produced) to +# the decoder. +lm_labels = encoded_answer +labels_tensor = torch.tensor([lm_labels]) + +# Load pre-trained model (weights) +model = Model2Model.from_pretrained('bert-base-uncased') + +# Set the model in evaluation mode to deactivate the DropOut modules +# This is IMPORTANT to have reproducible results during evaluation! +model.eval() + +# If you have a GPU, put everything on cuda +question_tensor = question_tensor.to('cuda') +answer_tensor = answer_tensor.to('cuda') +labels_tensor = labels_tensor.to('cuda') +model.to('cuda') + +# Predict hidden states features for each layer +with torch.no_grad(): + # See the models docstrings for the detail of the inputs + outputs = model(question_tensor, answer_tensor, decoder_lm_labels=labels_tensor) + # Transformers models always output tuples. + # See the models docstrings for the detail of all the outputs + # In our case, the first element is the value of the LM loss + lm_loss = outputs[0] +``` + +This loss can be used to fine-tune `Model2Model` on the question answering task. Assuming that we fine-tuned the model, let us now see how to generate an answer: + +```python +# Let's re-use the previous question +question = "Who was Jim Henson?" +encoded_question = tokenizer.encode(question) +question_tensor = torch.tensor([encoded_question]) + +# This time we try to generate the answer, so we start with an empty sequence +answer = "[CLS]" +encoded_answer = tokenizer.encode(answer, add_special_tokens=False) +answer_tensor = torch.tensor([encoded_answer]) + +# Load pre-trained model (weights) +model = Model2Model.from_pretrained('fine-tuned-weights') +model.eval() + +# If you have a GPU, put everything on cuda +question_tensor = encoded_question.to('cuda') +answer_tensor = encoded_answer.to('cuda') +model.to('cuda') + +# Predict all tokens +with torch.no_grad(): + outputs = model(question_tensor, answer_tensor) + predictions = outputs[0] + +# confirm we were able to predict 'jim' +predicted_index = torch.argmax(predictions[0, -1]).item() +predicted_token = tokenizer.convert_ids_to_tokens([predicted_index])[0] +assert predicted_token == 'jim' +``` From b98ff8854460a04fe076c704555705c5d5e1b6de Mon Sep 17 00:00:00 2001 From: Morgan Funtowicz Date: Fri, 20 Dec 2019 15:52:50 +0100 Subject: [PATCH 472/505] Added pipelines quick tour in README --- README.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/README.md b/README.md index a9d0fb3ace..1312fcc0ac 100644 --- a/README.md +++ b/README.md @@ -490,6 +490,35 @@ transformers-cli ls # List all your S3 objects. ``` +## Quick tour of pipelines + +New in version `v2.3`: `Pipeline` are high-level objects which automatically handle tokenization, running your data through a transformers model +and outputting the result in a structured object. + +You can create `Pipeline` objects for the following down-stream tasks: + - `feature-extraction`: Generates a tensor representation for the input sequence + - `ner`: Generates named entity mapping for each word in the input sequence. + - `sentiment-analysis`: Gives the polarity (positive / negative) of the whole input sequence. + - `question-answering`: Provided some context and a question refering to the context, it will extract the answer to the question + in the context. + +```python +from transformers import pipeline + +# Allocate a pipeline for sentiment-analysis +nlp = pipeline('sentiment-analysis') +nlp('We are very happy to include pipeline into the transformers repository.') +>>> {'label': 'POSITIVE', 'score': 0.99893874} + +# Allocate a pipeline for question-answering +nlp = pipeline('question-answering') +nlp({ + 'question': 'What is the name of the repository ?', + 'context': 'Pipeline have been included in the huggingface/transformers repository' +}) +>>> {'score': 0.28756016668193496, 'start': 35, 'end': 59, 'answer': 'huggingface/transformers'} +``` + ## Migrating from pytorch-transformers to transformers Here is a quick summary of what you should take care of when migrating from `pytorch-transformers` to `transformers`. From 90debb9ff2636f3f1c8256237deeca3a1ec3c7dd Mon Sep 17 00:00:00 2001 From: Dirk Groeneveld Date: Thu, 19 Dec 2019 17:03:01 -0800 Subject: [PATCH 473/505] Keep even the first of the special tokens intact while lowercasing. --- transformers/tokenization_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index 59e2d05212..2635a38db0 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -642,7 +642,7 @@ class PreTrainedTokenizer(object): def lowercase_text(t): # convert non-special tokens to lowercase escaped_special_toks = [re.escape(s_tok) for s_tok in all_special_tokens] - pattern = r'(^' + r'|'.join(escaped_special_toks) + r')|' + \ + pattern = r'(' + r'|'.join(escaped_special_toks) + r')|' + \ r'(.+?)' return re.sub( pattern, From fb393ad9945f66b081f88b81b90a2974d81e9601 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Fri, 20 Dec 2019 11:29:58 -0500 Subject: [PATCH 474/505] Added test for all special tokens --- transformers/tests/tokenization_tests_commons.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/transformers/tests/tokenization_tests_commons.py b/transformers/tests/tokenization_tests_commons.py index 13e7ae746a..cdec24d9f0 100644 --- a/transformers/tests/tokenization_tests_commons.py +++ b/transformers/tests/tokenization_tests_commons.py @@ -133,6 +133,14 @@ class CommonTestCases: self.assertNotEqual(len(toks), len(toks0)) # toks0 should be longer self.assertListEqual(toks, toks2) + # Check that none of the special tokens are lowercased + sequence_with_special_tokens = "A " + " yEs ".join(tokenizer.all_special_tokens) + " B" + tokenized_sequence = tokenizer.tokenize(sequence_with_special_tokens) + + for special_token in tokenizer.all_special_tokens: + print(special_token, special_token in tokenized_sequence) + assert special_token in tokenized_sequence + tokenizer = self.get_tokenizer(do_lower_case=False) added = tokenizer.add_tokens(new_toks) From 65c75fc58796b278d58b0ce2c8d2031594ef0f64 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Fri, 20 Dec 2019 11:34:16 -0500 Subject: [PATCH 475/505] Clean special tokens test --- transformers/tests/tokenization_tests_commons.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/transformers/tests/tokenization_tests_commons.py b/transformers/tests/tokenization_tests_commons.py index cdec24d9f0..c417d033dc 100644 --- a/transformers/tests/tokenization_tests_commons.py +++ b/transformers/tests/tokenization_tests_commons.py @@ -138,8 +138,7 @@ class CommonTestCases: tokenized_sequence = tokenizer.tokenize(sequence_with_special_tokens) for special_token in tokenizer.all_special_tokens: - print(special_token, special_token in tokenized_sequence) - assert special_token in tokenized_sequence + self.assertTrue(special_token in tokenized_sequence) tokenizer = self.get_tokenizer(do_lower_case=False) From 1c12ee0e55f4aec6d90c789c7b45e9adac0ed259 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 20 Dec 2019 18:28:27 +0100 Subject: [PATCH 476/505] fixing xlm-roberta tokenizer max_length and automodels --- transformers/modeling_auto.py | 17 ++++++++++++++--- transformers/modeling_utils.py | 2 +- transformers/pipelines.py | 5 +++-- transformers/tokenization_utils.py | 6 +++++- transformers/tokenization_xlm_roberta.py | 16 ++++++++++------ 5 files changed, 33 insertions(+), 13 deletions(-) diff --git a/transformers/modeling_auto.py b/transformers/modeling_auto.py index 761b2ce324..6b49efd378 100644 --- a/transformers/modeling_auto.py +++ b/transformers/modeling_auto.py @@ -20,7 +20,7 @@ import logging from .configuration_auto import (AlbertConfig, BertConfig, CamembertConfig, CTRLConfig, DistilBertConfig, GPT2Config, OpenAIGPTConfig, RobertaConfig, - TransfoXLConfig, XLMConfig, XLNetConfig) + TransfoXLConfig, XLMConfig, XLNetConfig, XLMRobertaConfig) from .modeling_bert import BertModel, BertForMaskedLM, BertForSequenceClassification, BertForQuestionAnswering, \ BertForTokenClassification, BERT_PRETRAINED_MODEL_ARCHIVE_MAP @@ -41,7 +41,8 @@ from .modeling_camembert import CamembertModel, CamembertForMaskedLM, CamembertF from .modeling_albert import AlbertModel, AlbertForMaskedLM, AlbertForSequenceClassification, \ AlbertForQuestionAnswering, ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP from .modeling_t5 import T5Model, T5WithLMHeadModel, T5_PRETRAINED_MODEL_ARCHIVE_MAP -from .modeling_xlm_roberta import XLMRobertaModel, XLMRobertaForMaskedLM, XLMRobertaForSequenceClassification, XLMRobertaForMultipleChoice, XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP +from .modeling_xlm_roberta import XLMRobertaModel, XLMRobertaForMaskedLM, XLMRobertaForSequenceClassification, \ + XLMRobertaForMultipleChoice, XLMRobertaForTokenClassification, XLM_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP from .modeling_utils import PreTrainedModel, SequenceSummary @@ -146,6 +147,8 @@ class AutoModel(object): return AlbertModel(config) elif isinstance(config, CamembertConfig): return CamembertModel(config) + elif isinstance(config, XLMRobertaConfig): + return XLMRobertaModel(config) raise ValueError("Unrecognized configuration class {}".format(config)) @classmethod @@ -333,6 +336,8 @@ class AutoModelWithLMHead(object): return XLMWithLMHeadModel(config) elif isinstance(config, CTRLConfig): return CTRLLMHeadModel(config) + elif isinstance(config, XLMRobertaConfig): + return XLMRobertaForMaskedLM(config) raise ValueError("Unrecognized configuration class {}".format(config)) @classmethod @@ -509,6 +514,8 @@ class AutoModelForSequenceClassification(object): return XLNetForSequenceClassification(config) elif isinstance(config, XLMConfig): return XLMForSequenceClassification(config) + elif isinstance(config, XLMRobertaConfig): + return XLMRobertaForSequenceClassification(config) raise ValueError("Unrecognized configuration class {}".format(config)) @classmethod @@ -787,6 +794,8 @@ class AutoModelForTokenClassification: return XLNetForTokenClassification(config) elif isinstance(config, RobertaConfig): return RobertaForTokenClassification(config) + elif isinstance(config, XLMRobertaConfig): + return XLMRobertaForTokenClassification(config) raise ValueError("Unrecognized configuration class {}".format(config)) @classmethod @@ -865,6 +874,8 @@ class AutoModelForTokenClassification: return CamembertForTokenClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'distilbert' in pretrained_model_name_or_path: return DistilBertForTokenClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) + elif 'xlm-roberta' in pretrained_model_name_or_path: + return XLMRobertaForTokenClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'roberta' in pretrained_model_name_or_path: return RobertaForTokenClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) elif 'bert' in pretrained_model_name_or_path: @@ -873,4 +884,4 @@ class AutoModelForTokenClassification: return XLNetForTokenClassification.from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs) raise ValueError("Unrecognized model identifier in {}. Should contains one of " - "'bert', 'xlnet', 'camembert', 'distilbert', 'roberta'".format(pretrained_model_name_or_path)) + "'bert', 'xlnet', 'camembert', 'distilbert', 'xlm-roberta', 'roberta'".format(pretrained_model_name_or_path)) diff --git a/transformers/modeling_utils.py b/transformers/modeling_utils.py index d899771603..e3f4b9f3b8 100644 --- a/transformers/modeling_utils.py +++ b/transformers/modeling_utils.py @@ -415,7 +415,7 @@ class PreTrainedModel(nn.Module): state_dict = torch.load(resolved_archive_file, map_location='cpu') except: raise OSError("Unable to load weights from pytorch checkpoint file. " - "If you tried to load a PyTorch model from a TF 2.0 checkpoint, please set from_tf=True. ") + "If you tried to load a PyTorch model from a TF 2.0 checkpoint, please set from_tf=True. ") missing_keys = [] unexpected_keys = [] diff --git a/transformers/pipelines.py b/transformers/pipelines.py index 33e1ab4022..c3756109af 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -49,7 +49,7 @@ logger = logging.getLogger(__name__) def get_framework(model=None): """ Select framework (TensorFlow/PyTorch) to use. - If both frameworks are installed and no specific model is provided, defaults to using TensorFlow. + If both frameworks are installed and no specific model is provided, defaults to using PyTorch. """ if is_tf_available() and is_torch_available() and model is not None and not isinstance(model, str): # Both framework are available but the use supplied a model class instance. @@ -60,7 +60,8 @@ def get_framework(model=None): "To install TensorFlow 2.0, read the instructions at https://www.tensorflow.org/install/ " "To install PyTorch, read the instructions at https://pytorch.org/.") else: - framework = 'tf' if is_tf_available() else 'pt' + # framework = 'tf' if is_tf_available() else 'pt' + framework = 'pt' if is_torch_available() else 'tf' return framework class ArgumentHandler(ABC): diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index 2635a38db0..d77a7100ab 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -434,7 +434,11 @@ class PreTrainedTokenizer(object): init_kwargs[key] = value # Instantiate tokenizer. - tokenizer = cls(*init_inputs, **init_kwargs) + try: + tokenizer = cls(*init_inputs, **init_kwargs) + except OSError: + OSError("Unable to load vocabulary from file. " + "Please check that the provided vocabulary is accessible and not corrupted.") # Save inputs and kwargs for saving and re-loading with ``save_pretrained`` tokenizer.init_inputs = init_inputs diff --git a/transformers/tokenization_xlm_roberta.py b/transformers/tokenization_xlm_roberta.py index 4397e7b031..57a42dde5c 100644 --- a/transformers/tokenization_xlm_roberta.py +++ b/transformers/tokenization_xlm_roberta.py @@ -40,8 +40,12 @@ PRETRAINED_VOCAB_FILES_MAP = { } PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { - 'xlm-roberta-base': None, - 'xlm-roberta-large': None, + 'xlm-roberta-base': 512, + 'xlm-roberta-large': 512, + 'xlm-roberta-large-finetuned-conll02-dutch': 512, + 'xlm-roberta-large-finetuned-conll02-spanish': 512, + 'xlm-roberta-large-finetuned-conll03-english': 512, + 'xlm-roberta-large-finetuned-conll03-german': 512, } class XLMRobertaTokenizer(PreTrainedTokenizer): @@ -58,10 +62,10 @@ class XLMRobertaTokenizer(PreTrainedTokenizer): def __init__(self, vocab_file, bos_token="", eos_token="", sep_token="", cls_token="", unk_token="", pad_token='', mask_token='', **kwargs): - super(XLMRobertaTokenizer, self).__init__(max_len=512, bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, - sep_token=sep_token, cls_token=cls_token, pad_token=pad_token, - mask_token=mask_token, - **kwargs) + super(XLMRobertaTokenizer, self).__init__(bos_token=bos_token, eos_token=eos_token, unk_token=unk_token, + sep_token=sep_token, cls_token=cls_token, pad_token=pad_token, + mask_token=mask_token, + **kwargs) self.max_len_single_sentence = self.max_len - 2 # take into account special tokens self.max_len_sentences_pair = self.max_len - 4 # take into account special tokens self.sp_model = spm.SentencePieceProcessor() From bbaaec046c493c9bb3bb2b24f1352413c647e3ff Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 20 Dec 2019 19:19:20 +0100 Subject: [PATCH 477/505] fixing CLI pipeline --- transformers/commands/run.py | 32 +++++++++++------ transformers/pipelines.py | 67 +++++++++++++++++++----------------- 2 files changed, 58 insertions(+), 41 deletions(-) diff --git a/transformers/commands/run.py b/transformers/commands/run.py index 50c42d3a40..c2c141734b 100644 --- a/transformers/commands/run.py +++ b/transformers/commands/run.py @@ -9,6 +9,9 @@ logger = logging.getLogger(__name__) # pylint: disable=invalid-name def try_infer_format_from_ext(path: str): + if not path: + return 'pipe' + for ext in PipelineDataFormat.SUPPORTED_FORMATS: if path.endswith(ext): return ext @@ -20,9 +23,16 @@ def try_infer_format_from_ext(path: str): def run_command_factory(args): - nlp = pipeline(task=args.task, model=args.model, config=args.config, tokenizer=args.tokenizer, device=args.device) + nlp = pipeline(task=args.task, + model=args.model if args.model else None, + config=args.config, + tokenizer=args.tokenizer, + device=args.device) format = try_infer_format_from_ext(args.input) if args.format == 'infer' else args.format - reader = PipelineDataFormat.from_str(format, args.output, args.input, args.column) + reader = PipelineDataFormat.from_str(format=format, + output_path=args.output, + input_path=args.input, + column=args.column if args.column else nlp.default_input_names) return RunCommand(nlp, reader) @@ -35,24 +45,26 @@ class RunCommand(BaseTransformersCLICommand): @staticmethod def register_subcommand(parser: ArgumentParser): run_parser = parser.add_parser('run', help="Run a pipeline through the CLI") - run_parser.add_argument('--device', type=int, default=-1, help='Indicate the device to run onto, -1 indicates CPU, >= 0 indicates GPU (default: -1)') run_parser.add_argument('--task', choices=SUPPORTED_TASKS.keys(), help='Task to run') - run_parser.add_argument('--model', type=str, required=True, help='Name or path to the model to instantiate.') + run_parser.add_argument('--input', type=str, help='Path to the file to use for inference') + run_parser.add_argument('--output', type=str, help='Path to the file that will be used post to write results.') + run_parser.add_argument('--model', type=str, help='Name or path to the model to instantiate.') run_parser.add_argument('--config', type=str, help='Name or path to the model\'s config to instantiate.') run_parser.add_argument('--tokenizer', type=str, help='Name of the tokenizer to use. (default: same as the model name)') run_parser.add_argument('--column', type=str, help='Name of the column to use as input. (For multi columns input as QA use column1,columns2)') run_parser.add_argument('--format', type=str, default='infer', choices=PipelineDataFormat.SUPPORTED_FORMATS, help='Input format to read from') - run_parser.add_argument('--input', type=str, help='Path to the file to use for inference') - run_parser.add_argument('--output', type=str, help='Path to the file that will be used post to write results.') + run_parser.add_argument('--device', type=int, default=-1, help='Indicate the device to run onto, -1 indicates CPU, >= 0 indicates GPU (default: -1)') run_parser.set_defaults(func=run_command_factory) def run(self): - nlp, output = self._nlp, [] + nlp, outputs = self._nlp, [] + for entry in self._reader: - if self._reader.is_multi_columns: - output += nlp(**entry) + output = nlp(**entry) if self._reader.is_multi_columns else nlp(entry) + if isinstance(output, dict): + outputs.append(output) else: - output += nlp(entry) + outputs += output # Saving data if self._nlp.binary_output: diff --git a/transformers/pipelines.py b/transformers/pipelines.py index c3756109af..01491cf2be 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -14,12 +14,14 @@ # limitations under the License. from __future__ import absolute_import, division, print_function, unicode_literals +import sys import csv import json import os import pickle import logging import six + from abc import ABC, abstractmethod from contextlib import contextmanager from itertools import groupby @@ -98,28 +100,29 @@ class PipelineDataFormat: Supported data formats currently includes: - JSON - CSV + - stdin/stdout (pipe) PipelineDataFormat also includes some utilities to work with multi-columns like mapping from datasets columns to pipelines keyword arguments through the `dataset_kwarg_1=dataset_column_1` format. """ SUPPORTED_FORMATS = ['json', 'csv', 'pipe'] - def __init__(self, output: Optional[str], input: Optional[str], column: Optional[str]): - self.output = output - self.path = input - self.column = column.split(',') if column else [''] + def __init__(self, output_path: Optional[str], input_path: Optional[str], column: Optional[str]): + self.output_path = output_path + self.input_path = input_path + self.column = column.split(',') if column is not None else [''] self.is_multi_columns = len(self.column) > 1 if self.is_multi_columns: self.column = [tuple(c.split('=')) if '=' in c else (c, c) for c in self.column] - if output is not None: - if exists(abspath(self.output)): - raise OSError('{} already exists on disk'.format(self.output)) + if output_path is not None: + if exists(abspath(self.output_path)): + raise OSError('{} already exists on disk'.format(self.output_path)) - if input is not None: - if not exists(abspath(self.path)): - raise OSError('{} doesnt exist on disk'.format(self.path)) + if input_path is not None: + if not exists(abspath(self.input_path)): + raise OSError('{} doesnt exist on disk'.format(self.input_path)) @abstractmethod def __iter__(self): @@ -140,7 +143,7 @@ class PipelineDataFormat: :param data: data to store :return: (str) Path where the data has been saved """ - path, _ = os.path.splitext(self.output) + path, _ = os.path.splitext(self.output_path) binary_path = os.path.extsep.join((path, 'pickle')) with open(binary_path, 'wb+') as f_output: @@ -149,23 +152,23 @@ class PipelineDataFormat: return binary_path @staticmethod - def from_str(name: str, output: Optional[str], path: Optional[str], column: Optional[str]): - if name == 'json': - return JsonPipelineDataFormat(output, path, column) - elif name == 'csv': - return CsvPipelineDataFormat(output, path, column) - elif name == 'pipe': - return PipedPipelineDataFormat(output, path, column) + def from_str(format: str, output_path: Optional[str], input_path: Optional[str], column: Optional[str]): + if format == 'json': + return JsonPipelineDataFormat(output_path, input_path, column) + elif format == 'csv': + return CsvPipelineDataFormat(output_path, input_path, column) + elif format == 'pipe': + return PipedPipelineDataFormat(output_path, input_path, column) else: - raise KeyError('Unknown reader {} (Available reader are json/csv/pipe)'.format(name)) + raise KeyError('Unknown reader {} (Available reader are json/csv/pipe)'.format(format)) class CsvPipelineDataFormat(PipelineDataFormat): - def __init__(self, output: Optional[str], input: Optional[str], column: Optional[str]): - super().__init__(output, input, column) + def __init__(self, output_path: Optional[str], input_path: Optional[str], column: Optional[str]): + super().__init__(output_path, input_path, column) def __iter__(self): - with open(self.path, 'r') as f: + with open(self.input_path, 'r') as f: reader = csv.DictReader(f) for row in reader: if self.is_multi_columns: @@ -174,7 +177,7 @@ class CsvPipelineDataFormat(PipelineDataFormat): yield row[self.column[0]] def save(self, data: List[dict]): - with open(self.output, 'w') as f: + with open(self.output_path, 'w') as f: if len(data) > 0: writer = csv.DictWriter(f, list(data[0].keys())) writer.writeheader() @@ -182,10 +185,10 @@ class CsvPipelineDataFormat(PipelineDataFormat): class JsonPipelineDataFormat(PipelineDataFormat): - def __init__(self, output: Optional[str], input: Optional[str], column: Optional[str]): - super().__init__(output, input, column) + def __init__(self, output_path: Optional[str], input_path: Optional[str], column: Optional[str]): + super().__init__(output_path, input_path, column) - with open(input, 'r') as f: + with open(input_path, 'r') as f: self._entries = json.load(f) def __iter__(self): @@ -196,7 +199,7 @@ class JsonPipelineDataFormat(PipelineDataFormat): yield entry[self.column[0]] def save(self, data: dict): - with open(self.output, 'w') as f: + with open(self.output_path, 'w') as f: json.dump(data, f) @@ -208,9 +211,7 @@ class PipedPipelineDataFormat(PipelineDataFormat): If columns are provided, then the output will be a dictionary with {column_x: value_x} """ def __iter__(self): - import sys for line in sys.stdin: - # Split for multi-columns if '\t' in line: @@ -229,7 +230,7 @@ class PipedPipelineDataFormat(PipelineDataFormat): print(data) def save_binary(self, data: Union[dict, List[dict]]) -> str: - if self.output is None: + if self.output_path is None: raise KeyError( 'When using piped input on pipeline outputting large object requires an output file path. ' 'Please provide such output path through --output argument.' @@ -294,6 +295,9 @@ class Pipeline(_ScikitCompat): nlp = NerPipeline(model='...', config='...', tokenizer='...') nlp = QuestionAnsweringPipeline(model=AutoModel.from_pretrained('...'), tokenizer='...') """ + + default_input_names = None + def __init__(self, model, tokenizer: PreTrainedTokenizer = None, modelcard: ModelCard = None, framework: Optional[str] = None, args_parser: ArgumentHandler = None, device: int = -1, @@ -582,6 +586,8 @@ class QuestionAnsweringPipeline(Pipeline): Question Answering pipeline using ModelForQuestionAnswering head. """ + default_input_names = 'question,context' + def __init__(self, model, tokenizer: Optional[PreTrainedTokenizer], modelcard: Optional[ModelCard], @@ -684,7 +690,6 @@ class QuestionAnsweringPipeline(Pipeline): } for s, e, score in zip(starts, ends, scores) ] - if len(answers) == 1: return answers[0] return answers From 79e4a6a25c9d9e267a7d7cdb54a97443fd4fc9c4 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 20 Dec 2019 19:33:12 +0100 Subject: [PATCH 478/505] update serving API --- transformers/commands/serving.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/transformers/commands/serving.py b/transformers/commands/serving.py index 3c3f852809..4f41f797d1 100644 --- a/transformers/commands/serving.py +++ b/transformers/commands/serving.py @@ -24,7 +24,11 @@ def serve_command_factory(args: Namespace): Factory function used to instantiate serving server from provided command line arguments. :return: ServeCommand """ - nlp = pipeline(task=args.task, model=args.model, config=args.config, tokenizer=args.tokenizer, device=args.device) + nlp = pipeline(task=args.task, + model=args.model if args.model else None, + config=args.config, + tokenizer=args.tokenizer, + device=args.device) return ServeCommand(nlp, args.host, args.port) @@ -68,12 +72,12 @@ class ServeCommand(BaseTransformersCLICommand): """ serve_parser = parser.add_parser('serve', help='CLI tool to run inference requests through REST and GraphQL endpoints.') serve_parser.add_argument('--task', type=str, choices=SUPPORTED_TASKS.keys(), help='The task to run the pipeline on') - serve_parser.add_argument('--device', type=int, default=-1, help='Indicate the device to run onto, -1 indicates CPU, >= 0 indicates GPU (default: -1)') serve_parser.add_argument('--host', type=str, default='localhost', help='Interface the server will listen on.') serve_parser.add_argument('--port', type=int, default=8888, help='Port the serving will listen to.') - serve_parser.add_argument('--model', type=str, required=True, help='Model\'s name or path to stored model.') + serve_parser.add_argument('--model', type=str, help='Model\'s name or path to stored model.') serve_parser.add_argument('--config', type=str, help='Model\'s config name or path to stored model.') serve_parser.add_argument('--tokenizer', type=str, help='Tokenizer name to use.') + serve_parser.add_argument('--device', type=int, default=-1, help='Indicate the device to run onto, -1 indicates CPU, >= 0 indicates GPU (default: -1)') serve_parser.set_defaults(func=serve_command_factory) def __init__(self, pipeline: Pipeline, host: str, port: int): From 71883b6ddcd14929217a0ddf4ad627468b9ab5a8 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 20 Dec 2019 19:40:23 +0100 Subject: [PATCH 479/505] update link in readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 1312fcc0ac..769b0499cb 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,7 @@ Choose the right framework for every part of a model's lifetime | [Online demo](#online-demo) | Experimenting with this repo’s text generation capabilities | | [Quick tour: Usage](#quick-tour) | Tokenizers & models usage: Bert and GPT-2 | | [Quick tour: TF 2.0 and PyTorch ](#Quick-tour-TF-20-training-and-PyTorch-interoperability) | Train a TF 2.0 model in 10 lines of code, load it in PyTorch | +| [Quick tour: pipelines](#quick-tour-of-pipelines) | Using Pipelines: Wrapper around tokenizer and models to use finetuned models | | [Quick tour: Fine-tuning/usage scripts](#quick-tour-of-the-fine-tuningusage-scripts) | Using provided scripts: GLUE, SQuAD and Text generation | | [Quick tour: Share your models ](#Quick-tour-of-model-sharing) | Upload and share your fine-tuned models with the community | | [Migrating from pytorch-transformers to transformers](#Migrating-from-pytorch-transformers-to-transformers) | Migrating your code from pytorch-transformers to transformers | @@ -496,6 +497,7 @@ New in version `v2.3`: `Pipeline` are high-level objects which automatically han and outputting the result in a structured object. You can create `Pipeline` objects for the following down-stream tasks: + - `feature-extraction`: Generates a tensor representation for the input sequence - `ner`: Generates named entity mapping for each word in the input sequence. - `sentiment-analysis`: Gives the polarity (positive / negative) of the whole input sequence. From ceae85ad60da38cacb14eca49f752669a4fe31dc Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 20 Dec 2019 19:52:24 +0100 Subject: [PATCH 480/505] fix mc loading --- transformers/pipelines.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index 01491cf2be..7188526a62 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -891,6 +891,10 @@ def pipeline(task: str, model: Optional = None, if isinstance(config, str): config = AutoConfig.from_pretrained(config) + # Instantiate modelcard if needed + if isinstance(modelcard, str): + modelcard = ModelCard.from_pretrained(modelcard) + # Instantiate model if needed if isinstance(model, str): # Handle transparent TF/PT model conversion From e37ca8e11a3aa91e27ed659e6d4e01b208aa83ca Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 20 Dec 2019 20:43:42 +0100 Subject: [PATCH 481/505] fix camembert and XLM-R tokenizer --- transformers/tokenization_camembert.py | 6 ++++++ transformers/tokenization_xlm_roberta.py | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/transformers/tokenization_camembert.py b/transformers/tokenization_camembert.py index b4091558e1..4c4615eb3d 100644 --- a/transformers/tokenization_camembert.py +++ b/transformers/tokenization_camembert.py @@ -22,6 +22,7 @@ from shutil import copyfile import sentencepiece as spm from transformers.tokenization_utils import PreTrainedTokenizer +from .tokenization_xlnet import SPIECE_UNDERLINE logger = logging.getLogger(__name__) @@ -145,6 +146,11 @@ class CamembertTokenizer(PreTrainedTokenizer): return self.fairseq_ids_to_tokens[index] return self.sp_model.IdToPiece(index - self.fairseq_offset) + def convert_tokens_to_string(self, tokens): + """Converts a sequence of tokens (strings for sub-words) in a single string.""" + out_string = ''.join(tokens).replace(SPIECE_UNDERLINE, ' ').strip() + return out_string + def save_vocabulary(self, save_directory): """ Save the sentencepiece vocabulary (copy original file) and special tokens file to a directory. diff --git a/transformers/tokenization_xlm_roberta.py b/transformers/tokenization_xlm_roberta.py index 57a42dde5c..adbc8cd6c7 100644 --- a/transformers/tokenization_xlm_roberta.py +++ b/transformers/tokenization_xlm_roberta.py @@ -22,6 +22,7 @@ from shutil import copyfile import sentencepiece as spm from transformers.tokenization_utils import PreTrainedTokenizer +from .tokenization_xlnet import SPIECE_UNDERLINE logger = logging.getLogger(__name__) @@ -161,6 +162,11 @@ class XLMRobertaTokenizer(PreTrainedTokenizer): return self.fairseq_ids_to_tokens[index] return self.sp_model.IdToPiece(index - self.fairseq_offset) + def convert_tokens_to_string(self, tokens): + """Converts a sequence of tokens (strings for sub-words) in a single string.""" + out_string = ''.join(tokens).replace(SPIECE_UNDERLINE, ' ').strip() + return out_string + def save_vocabulary(self, save_directory): """ Save the sentencepiece vocabulary (copy original file) and special tokens file to a directory. From a241011057245211975b4730170815536527d79d Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 20 Dec 2019 20:43:48 +0100 Subject: [PATCH 482/505] fix pipeline NER --- transformers/pipelines.py | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index 7188526a62..f7900feaf3 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -463,7 +463,7 @@ class NerPipeline(Pipeline): def __init__(self, model, tokenizer: PreTrainedTokenizer = None, modelcard: ModelCard = None, framework: Optional[str] = None, args_parser: ArgumentHandler = None, device: int = -1, - binary_output: bool = False): + binary_output: bool = False, ignore_labels=['O']): super().__init__(model=model, tokenizer=tokenizer, modelcard=modelcard, @@ -473,17 +473,12 @@ class NerPipeline(Pipeline): binary_output=binary_output) self._basic_tokenizer = BasicTokenizer(do_lower_case=False) + self.ignore_labels = ignore_labels def __call__(self, *texts, **kwargs): inputs, answers = self._args_parser(*texts, **kwargs), [] for sentence in inputs: - # Ugly token to word idx mapping (for now) - token_to_word, words = [], self._basic_tokenizer.tokenize(sentence) - for i, w in enumerate(words): - tokens = self.tokenizer.tokenize(w) - token_to_word += [i] * len(tokens) - # Manage correct placement of the tensors with self.device_placement(): @@ -500,26 +495,22 @@ class NerPipeline(Pipeline): with torch.no_grad(): entities = self.model(**tokens)[0][0].cpu().numpy() - # Normalize scores - answer, token_start = [], 1 - for idx, word in groupby(token_to_word): + score = np.exp(entities) / np.exp(entities).sum(-1, keepdims=True) + labels_idx = score.argmax(axis=-1) - # Sum log prob over token, then normalize across labels - score = np.exp(entities[token_start]) / np.exp(entities[token_start]).sum(-1, keepdims=True) - label_idx = score.argmax() - - if label_idx > 0: + answer = [] + for idx, label_idx in enumerate(labels_idx): + if self.model.config.id2label[label_idx] not in self.ignore_labels: answer += [{ - 'word': words[idx], - 'score': score[label_idx].item(), + 'word': self.tokenizer.decode(tokens['input_ids'][0][idx].cpu().tolist()), + 'score': score[idx][label_idx].item(), 'entity': self.model.config.id2label[label_idx] }] - # Update token start - token_start += len(list(word)) - # Append answers += [answer] + if len(answers) == 1: + return answers[0] return answers From f79a7dc661b9d7caee867af18a2be009478ab739 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 20 Dec 2019 20:57:45 +0100 Subject: [PATCH 483/505] fix NER pipeline --- transformers/pipelines.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index f7900feaf3..86fd25c164 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -491,9 +491,11 @@ class NerPipeline(Pipeline): # Forward if self.framework == 'tf': entities = self.model(tokens)[0][0].numpy() + input_ids = tokens['input_ids'].numpy()[0] else: with torch.no_grad(): entities = self.model(**tokens)[0][0].cpu().numpy() + input_ids = tokens['input_ids'].cpu().numpy()[0] score = np.exp(entities) / np.exp(entities).sum(-1, keepdims=True) labels_idx = score.argmax(axis=-1) @@ -502,7 +504,7 @@ class NerPipeline(Pipeline): for idx, label_idx in enumerate(labels_idx): if self.model.config.id2label[label_idx] not in self.ignore_labels: answer += [{ - 'word': self.tokenizer.decode(tokens['input_ids'][0][idx].cpu().tolist()), + 'word': self.tokenizer.decode(int(input_ids[idx])), 'score': score[idx][label_idx].item(), 'entity': self.model.config.id2label[label_idx] }] From cb6d54bfdabfb1fe566a2c303fcb9f18505d9b10 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Fri, 20 Dec 2019 15:06:28 -0500 Subject: [PATCH 484/505] Numpy compatibility for sentence piece convert to int earlier --- transformers/tokenization_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/transformers/tokenization_utils.py b/transformers/tokenization_utils.py index d77a7100ab..eda89f22fc 100644 --- a/transformers/tokenization_utils.py +++ b/transformers/tokenization_utils.py @@ -1227,6 +1227,7 @@ class PreTrainedTokenizer(object): return self._convert_id_to_token(ids) tokens = [] for index in ids: + index = int(index) if skip_special_tokens and index in self.all_special_ids: continue if index in self.added_tokens_decoder: From 4775ec354b20196b53894ffeb9af7622a39dd4fc Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 20 Dec 2019 21:47:15 +0100 Subject: [PATCH 485/505] add overwrite - fix ner decoding --- transformers/commands/run.py | 9 ++++++--- transformers/pipelines.py | 25 ++++++++++++++----------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/transformers/commands/run.py b/transformers/commands/run.py index c2c141734b..2098d03413 100644 --- a/transformers/commands/run.py +++ b/transformers/commands/run.py @@ -32,7 +32,8 @@ def run_command_factory(args): reader = PipelineDataFormat.from_str(format=format, output_path=args.output, input_path=args.input, - column=args.column if args.column else nlp.default_input_names) + column=args.column if args.column else nlp.default_input_names, + overwrite=args.overwrite) return RunCommand(nlp, reader) @@ -54,6 +55,7 @@ class RunCommand(BaseTransformersCLICommand): run_parser.add_argument('--column', type=str, help='Name of the column to use as input. (For multi columns input as QA use column1,columns2)') run_parser.add_argument('--format', type=str, default='infer', choices=PipelineDataFormat.SUPPORTED_FORMATS, help='Input format to read from') run_parser.add_argument('--device', type=int, default=-1, help='Indicate the device to run onto, -1 indicates CPU, >= 0 indicates GPU (default: -1)') + run_parser.add_argument('--overwrite', action='store_true', help='Allow overwriting the output file.') run_parser.set_defaults(func=run_command_factory) def run(self): @@ -61,6 +63,7 @@ class RunCommand(BaseTransformersCLICommand): for entry in self._reader: output = nlp(**entry) if self._reader.is_multi_columns else nlp(entry) + print(output) if isinstance(output, dict): outputs.append(output) else: @@ -68,10 +71,10 @@ class RunCommand(BaseTransformersCLICommand): # Saving data if self._nlp.binary_output: - binary_path = self._reader.save_binary(output) + binary_path = self._reader.save_binary(outputs) logger.warning('Current pipeline requires output to be in binary format, saving at {}'.format(binary_path)) else: - self._reader.save(output) + self._reader.save(outputs) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index 86fd25c164..876bdd0c09 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -107,7 +107,7 @@ class PipelineDataFormat: """ SUPPORTED_FORMATS = ['json', 'csv', 'pipe'] - def __init__(self, output_path: Optional[str], input_path: Optional[str], column: Optional[str]): + def __init__(self, output_path: Optional[str], input_path: Optional[str], column: Optional[str], overwrite=False): self.output_path = output_path self.input_path = input_path self.column = column.split(',') if column is not None else [''] @@ -116,7 +116,7 @@ class PipelineDataFormat: if self.is_multi_columns: self.column = [tuple(c.split('=')) if '=' in c else (c, c) for c in self.column] - if output_path is not None: + if output_path is not None and not overwrite: if exists(abspath(self.output_path)): raise OSError('{} already exists on disk'.format(self.output_path)) @@ -152,25 +152,26 @@ class PipelineDataFormat: return binary_path @staticmethod - def from_str(format: str, output_path: Optional[str], input_path: Optional[str], column: Optional[str]): + def from_str(format: str, output_path: Optional[str], input_path: Optional[str], column: Optional[str], overwrite=False): if format == 'json': - return JsonPipelineDataFormat(output_path, input_path, column) + return JsonPipelineDataFormat(output_path, input_path, column, overwrite=overwrite) elif format == 'csv': - return CsvPipelineDataFormat(output_path, input_path, column) + return CsvPipelineDataFormat(output_path, input_path, column, overwrite=overwrite) elif format == 'pipe': - return PipedPipelineDataFormat(output_path, input_path, column) + return PipedPipelineDataFormat(output_path, input_path, column, overwrite=overwrite) else: raise KeyError('Unknown reader {} (Available reader are json/csv/pipe)'.format(format)) class CsvPipelineDataFormat(PipelineDataFormat): - def __init__(self, output_path: Optional[str], input_path: Optional[str], column: Optional[str]): - super().__init__(output_path, input_path, column) + def __init__(self, output_path: Optional[str], input_path: Optional[str], column: Optional[str], overwrite=False): + super().__init__(output_path, input_path, column, overwrite=overwrite) def __iter__(self): with open(self.input_path, 'r') as f: reader = csv.DictReader(f) for row in reader: + print(row, self.column) if self.is_multi_columns: yield {k: row[c] for k, c in self.column} else: @@ -185,8 +186,8 @@ class CsvPipelineDataFormat(PipelineDataFormat): class JsonPipelineDataFormat(PipelineDataFormat): - def __init__(self, output_path: Optional[str], input_path: Optional[str], column: Optional[str]): - super().__init__(output_path, input_path, column) + def __init__(self, output_path: Optional[str], input_path: Optional[str], column: Optional[str], overwrite=False): + super().__init__(output_path, input_path, column, overwrite=overwrite) with open(input_path, 'r') as f: self._entries = json.load(f) @@ -460,6 +461,8 @@ class NerPipeline(Pipeline): Named Entity Recognition pipeline using ModelForTokenClassification head. """ + default_input_names = 'sequences' + def __init__(self, model, tokenizer: PreTrainedTokenizer = None, modelcard: ModelCard = None, framework: Optional[str] = None, args_parser: ArgumentHandler = None, device: int = -1, @@ -504,7 +507,7 @@ class NerPipeline(Pipeline): for idx, label_idx in enumerate(labels_idx): if self.model.config.id2label[label_idx] not in self.ignore_labels: answer += [{ - 'word': self.tokenizer.decode(int(input_ids[idx])), + 'word': self.tokenizer.decode([int(input_ids[idx])]), 'score': score[idx][label_idx].item(), 'entity': self.model.config.id2label[label_idx] }] From e5812462fc0f81e9808ad87a818cd8af26405722 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 20 Dec 2019 21:51:48 +0100 Subject: [PATCH 486/505] clean up debug and less verbose tqdm --- transformers/file_utils.py | 3 ++- transformers/pipelines.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/transformers/file_utils.py b/transformers/file_utils.py index 59938868ea..032a6af63b 100644 --- a/transformers/file_utils.py +++ b/transformers/file_utils.py @@ -287,7 +287,8 @@ def http_get(url, temp_file, proxies=None, resume_size=0, user_agent=None): return content_length = response.headers.get('Content-Length') total = resume_size + int(content_length) if content_length is not None else None - progress = tqdm(unit="B", unit_scale=True, total=total, initial=resume_size, desc="Downloading") + progress = tqdm(unit="B", unit_scale=True, total=total, initial=resume_size, + desc="Downloading", disable=bool(logger.level<=logging.INFO)) for chunk in response.iter_content(chunk_size=1024): if chunk: # filter out keep-alive new chunks progress.update(len(chunk)) diff --git a/transformers/pipelines.py b/transformers/pipelines.py index 876bdd0c09..f4bf3da685 100755 --- a/transformers/pipelines.py +++ b/transformers/pipelines.py @@ -171,7 +171,6 @@ class CsvPipelineDataFormat(PipelineDataFormat): with open(self.input_path, 'r') as f: reader = csv.DictReader(f) for row in reader: - print(row, self.column) if self.is_multi_columns: yield {k: row[c] for k, c in self.column} else: From 655fd068534b2a66b85f0bd05002e27f212ab6a0 Mon Sep 17 00:00:00 2001 From: thomwolf Date: Fri, 20 Dec 2019 21:57:49 +0100 Subject: [PATCH 487/505] clean up --- transformers/commands/run.py | 1 - 1 file changed, 1 deletion(-) diff --git a/transformers/commands/run.py b/transformers/commands/run.py index 2098d03413..df03cee9d7 100644 --- a/transformers/commands/run.py +++ b/transformers/commands/run.py @@ -63,7 +63,6 @@ class RunCommand(BaseTransformersCLICommand): for entry in self._reader: output = nlp(**entry) if self._reader.is_multi_columns else nlp(entry) - print(output) if isinstance(output, dict): outputs.append(output) else: From a436574bfde4f75f518a107f45f987579d813ce5 Mon Sep 17 00:00:00 2001 From: Lysandre Date: Fri, 20 Dec 2019 16:22:20 -0500 Subject: [PATCH 488/505] Release: v2.3.0 --- README.md | 2 +- docs/source/conf.py | 2 +- setup.py | 2 +- transformers/__init__.py | 2 +- try.py | 0 5 files changed, 4 insertions(+), 4 deletions(-) create mode 100644 try.py diff --git a/README.md b/README.md index 769b0499cb..416adcc1ef 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ Choose the right framework for every part of a model's lifetime | [Quick tour: Share your models ](#Quick-tour-of-model-sharing) | Upload and share your fine-tuned models with the community | | [Migrating from pytorch-transformers to transformers](#Migrating-from-pytorch-transformers-to-transformers) | Migrating your code from pytorch-transformers to transformers | | [Migrating from pytorch-pretrained-bert to pytorch-transformers](#Migrating-from-pytorch-pretrained-bert-to-transformers) | Migrating your code from pytorch-pretrained-bert to transformers | -| [Documentation][(v2.2.0/v2.2.1/v2.2.2)](https://huggingface.co/transformers/v2.2.0) [(v2.1.1)](https://huggingface.co/transformers/v2.1.1) [(v2.0.0)](https://huggingface.co/transformers/v2.0.0) [(v1.2.0)](https://huggingface.co/transformers/v1.2.0) [(v1.1.0)](https://huggingface.co/transformers/v1.1.0) [(v1.0.0)](https://huggingface.co/transformers/v1.0.0) [(master)](https://huggingface.co/transformers) | Full API documentation and more | +| [Documentation][(v2.3.0)](https://huggingface.co/transformers/v2.3.0)[(v2.2.0/v2.2.1/v2.2.2)](https://huggingface.co/transformers/v2.2.0) [(v2.1.1)](https://huggingface.co/transformers/v2.1.1) [(v2.0.0)](https://huggingface.co/transformers/v2.0.0) [(v1.2.0)](https://huggingface.co/transformers/v1.2.0) [(v1.1.0)](https://huggingface.co/transformers/v1.1.0) [(v1.0.0)](https://huggingface.co/transformers/v1.0.0) [(master)](https://huggingface.co/transformers) | Full API documentation and more | ## Installation diff --git a/docs/source/conf.py b/docs/source/conf.py index 99b7b44922..41a65eec29 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -26,7 +26,7 @@ author = u'huggingface' # The short X.Y version version = u'' # The full version, including alpha/beta/rc tags -release = u'2.2.2' +release = u'2.3.0' # -- General configuration --------------------------------------------------- diff --git a/setup.py b/setup.py index 4bfb774155..cd64a6ce90 100644 --- a/setup.py +++ b/setup.py @@ -46,7 +46,7 @@ extras['all'] = [package for package in extras.values()] setup( name="transformers", - version="2.2.2", + version="2.3.0", author="Thomas Wolf, Lysandre Debut, Victor Sanh, Julien Chaumond, Google AI Language Team Authors, Open AI team Authors, Facebook AI Authors, Carnegie Mellon University Authors", author_email="thomas@huggingface.co", description="State-of-the-art Natural Language Processing for TensorFlow 2.0 and PyTorch", diff --git a/transformers/__init__.py b/transformers/__init__.py index 1622c3892d..c0c0901df4 100755 --- a/transformers/__init__.py +++ b/transformers/__init__.py @@ -1,4 +1,4 @@ -__version__ = "2.2.2" +__version__ = "2.3.0" # Work around to update TensorFlow's absl.logging threshold which alters the # default Python logging output behavior when present. diff --git a/try.py b/try.py new file mode 100644 index 0000000000..e69de29bb2 From 3df1d2d144dae698b4aed085aec57a2033d608b7 Mon Sep 17 00:00:00 2001 From: Francesco Date: Tue, 17 Dec 2019 10:19:54 +0100 Subject: [PATCH 489/505] - Create the output directory (whose name is passed by the user in the "save_directory" parameter) where it will be saved encoder and decoder, if not exists. - Empty the output directory, if it contains any files or subdirectories. - Create the "encoder" directory inside "save_directory", if not exists. - Create the "decoder" directory inside "save_directory", if not exists. - Save the encoder and the decoder in the previous two directories, respectively. --- transformers/modeling_encoder_decoder.py | 31 ++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/transformers/modeling_encoder_decoder.py b/transformers/modeling_encoder_decoder.py index a91c046d8f..8ae6daa690 100644 --- a/transformers/modeling_encoder_decoder.py +++ b/transformers/modeling_encoder_decoder.py @@ -166,6 +166,37 @@ class PreTrainedEncoderDecoder(nn.Module): We save the encoder' and decoder's parameters in two separate directories. """ + + # If the root output directory does not exist, create it + if not os.path.exists(save_directory): + os.mkdir(save_directory) + + # Check whether the output directory is empty or not + sub_directories = [directory for directory in os.listdir(save_directory) + if os.path.isdir(os.path.join(save_directory, directory))] + + if len(sub_directories) > 0: + if "encoder" in sub_directories and "decoder" in sub_directories: + print("WARNING: there is an older version of encoder-decoder saved in" +\ + " the output directory. The default behaviour is to overwrite them.") + + # Empty the output directory + for directory_to_remove in sub_directories: + # Remove all files into the subdirectory + files_to_remove = os.listdir(os.path.join(save_directory, directory_to_remove)) + for file_to_remove in files_to_remove: + os.remove(os.path.join(save_directory, directory_to_remove, file_to_remove)) + # Remove the subdirectory itself + os.rmdir(os.path.join(save_directory, directory_to_remove)) + + assert(len(os.listdir(save_directory)) == 0) # sanity check + + if not os.path.exists(os.path.join(save_directory, "encoder")): + os.mkdir(os.path.join(save_directory, "encoder")) + + if not os.path.exists(os.path.join(save_directory, "decoder")): + os.mkdir(os.path.join(save_directory, "decoder")) + self.encoder.save_pretrained(os.path.join(save_directory, "encoder")) self.decoder.save_pretrained(os.path.join(save_directory, "decoder")) From a80778f40e4738071b5d01420a0328bb00cdb356 Mon Sep 17 00:00:00 2001 From: Francesco Date: Wed, 18 Dec 2019 16:05:28 +0100 Subject: [PATCH 490/505] small refactoring (only esthetic, not functional) --- transformers/modeling_encoder_decoder.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/transformers/modeling_encoder_decoder.py b/transformers/modeling_encoder_decoder.py index 8ae6daa690..c327bb9199 100644 --- a/transformers/modeling_encoder_decoder.py +++ b/transformers/modeling_encoder_decoder.py @@ -191,13 +191,14 @@ class PreTrainedEncoderDecoder(nn.Module): assert(len(os.listdir(save_directory)) == 0) # sanity check + # Create the "encoder" directory inside the output directory and save the encoder into it if not os.path.exists(os.path.join(save_directory, "encoder")): os.mkdir(os.path.join(save_directory, "encoder")) + self.encoder.save_pretrained(os.path.join(save_directory, "encoder")) + # Create the "encoder" directory inside the output directory and save the decoder into it if not os.path.exists(os.path.join(save_directory, "decoder")): os.mkdir(os.path.join(save_directory, "decoder")) - - self.encoder.save_pretrained(os.path.join(save_directory, "encoder")) self.decoder.save_pretrained(os.path.join(save_directory, "decoder")) def forward(self, encoder_input_ids, decoder_input_ids, **kwargs): From 228f52867c92e21c6e7223eb2d6c7d9904b230e2 Mon Sep 17 00:00:00 2001 From: Dom Hudson Date: Thu, 7 Nov 2019 19:58:17 +0000 Subject: [PATCH 491/505] Bug fix: 1764 --- transformers/modeling_roberta.py | 45 +++++++++++++++------ transformers/tests/modeling_roberta_test.py | 41 +++++++++++++++++++ 2 files changed, 74 insertions(+), 12 deletions(-) diff --git a/transformers/modeling_roberta.py b/transformers/modeling_roberta.py index fc27353d37..cf74c1e7b5 100644 --- a/transformers/modeling_roberta.py +++ b/transformers/modeling_roberta.py @@ -51,24 +51,45 @@ class RobertaEmbeddings(BertEmbeddings): padding_idx=self.padding_idx) def forward(self, input_ids=None, token_type_ids=None, position_ids=None, inputs_embeds=None): - if input_ids is not None: - input_shape = input_ids.size() - else: - input_shape = inputs_embeds.size()[:-1] - - seq_length = input_shape[1] - device = input_ids.device if input_ids is not None else inputs_embeds.device - if position_ids is None: - # Position numbers begin at padding_idx+1. Padding symbols are ignored. - # cf. fairseq's `utils.make_positions` - position_ids = torch.arange(self.padding_idx+1, seq_length+self.padding_idx+1, dtype=torch.long, device=device) - position_ids = position_ids.unsqueeze(0).expand(input_shape) + + if input_ids is not None: + # Create the position ids from the input token ids. Any padded tokens remain padded. + position_ids = self.create_position_ids_from_input_ids(input_ids).to(input_ids.device) + else: + position_ids = self.create_position_ids_from_inputs_embeds(inputs_embeds) + return super(RobertaEmbeddings, self).forward(input_ids, token_type_ids=token_type_ids, position_ids=position_ids, inputs_embeds=inputs_embeds) + def create_position_ids_from_input_ids(self, x): + """ Replace non-padding symbols with their position numbers. Position numbers begin at + padding_idx+1. Padding symbols are ignored. This is modified from fairseq's + `utils.make_positions`. + + :param torch.Tensor x: + :return torch.Tensor: + """ + mask = x.ne(self.padding_idx).long() + incremental_indicies = torch.cumsum(mask, dim=1) * mask + return incremental_indicies + self.padding_idx + + def create_position_ids_from_inputs_embeds(self, inputs_embeds): + """ We are provided embeddings directly. We cannot infer which are padded so just generate + sequential position ids. + + :param torch.Tensor inputs_embeds: + :return torch.Tensor: + """ + input_shape = inputs_embeds.size()[:-1] + sequence_length = input_shape[1] + + position_ids = torch.arange(self.padding_idx+1, sequence_length+self.padding_idx+1, dtype=torch.long, + device=inputs_embeds.device) + return position_ids.unsqueeze(0) + ROBERTA_START_DOCSTRING = r""" The RoBERTa model was proposed in `RoBERTa: A Robustly Optimized BERT Pretraining Approach`_ diff --git a/transformers/tests/modeling_roberta_test.py b/transformers/tests/modeling_roberta_test.py index 4d34a50528..121cb84148 100644 --- a/transformers/tests/modeling_roberta_test.py +++ b/transformers/tests/modeling_roberta_test.py @@ -25,6 +25,7 @@ if is_torch_available(): import torch from transformers import (RobertaConfig, RobertaModel, RobertaForMaskedLM, RobertaForSequenceClassification, RobertaForTokenClassification) + from transformers.modeling_roberta import RobertaEmbeddings from transformers.modeling_roberta import ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP from .modeling_common_test import (CommonTestCases, ids_tensor) @@ -205,6 +206,46 @@ class RobertaModelTest(CommonTestCases.CommonModelTester): shutil.rmtree(cache_dir) self.assertIsNotNone(model) + def test_create_position_ids_respects_padding_index(self): + """ Ensure that the default position ids only assign a sequential . This is a regression + test for https://github.com/huggingface/transformers/issues/1761 + + The position ids should be masked with the embedding object's padding index. Therefore, the + first available non-padding position index is RobertaEmbeddings.padding_idx + 1 + """ + config = self.model_tester.prepare_config_and_inputs()[0] + model = RobertaEmbeddings(config=config) + + input_ids = torch.as_tensor([[12, 31, 13, model.padding_idx]]) + expected_positions = torch.as_tensor([[ + 0 + model.padding_idx + 1, + 1 + model.padding_idx + 1, + 2 + model.padding_idx + 1, + model.padding_idx + ]]) + + position_ids = model.create_position_ids_from_input_ids(input_ids) + self.assertTrue(torch.all(torch.eq(position_ids, expected_positions))) + + def test_create_position_ids_from_inputs_embeds(self): + """ Ensure that the default position ids only assign a sequential . This is a regression + test for https://github.com/huggingface/transformers/issues/1761 + + The position ids should be masked with the embedding object's padding index. Therefore, the + first available non-padding position index is RobertaEmbeddings.padding_idx + 1 + """ + config = self.model_tester.prepare_config_and_inputs()[0] + model = RobertaEmbeddings(config=config) + + input_ids = torch.Tensor(1, 4, 30) + expected_positions = torch.as_tensor([[ + 0 + model.padding_idx + 1, + 1 + model.padding_idx + 1, + 2 + model.padding_idx + 1, + 3 + model.padding_idx + 1, + ]]) + position_ids = model.create_position_ids_from_inputs_embeds(input_ids) + self.assertTrue(torch.all(torch.eq(position_ids, expected_positions))) class RobertaModelIntegrationTest(unittest.TestCase): From 3e52915fa7106a739aa6f9feda9937961ce25068 Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Fri, 20 Dec 2019 19:01:27 -0500 Subject: [PATCH 492/505] [RoBERTa] Embeddings: fix dimensionality bug --- transformers/modeling_roberta.py | 3 +- transformers/tests/modeling_roberta_test.py | 31 ++++++++++++++------- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/transformers/modeling_roberta.py b/transformers/modeling_roberta.py index cf74c1e7b5..b188799522 100644 --- a/transformers/modeling_roberta.py +++ b/transformers/modeling_roberta.py @@ -52,7 +52,6 @@ class RobertaEmbeddings(BertEmbeddings): def forward(self, input_ids=None, token_type_ids=None, position_ids=None, inputs_embeds=None): if position_ids is None: - if input_ids is not None: # Create the position ids from the input token ids. Any padded tokens remain padded. position_ids = self.create_position_ids_from_input_ids(input_ids).to(input_ids.device) @@ -88,7 +87,7 @@ class RobertaEmbeddings(BertEmbeddings): position_ids = torch.arange(self.padding_idx+1, sequence_length+self.padding_idx+1, dtype=torch.long, device=inputs_embeds.device) - return position_ids.unsqueeze(0) + return position_ids.unsqueeze(0).expand(input_shape) ROBERTA_START_DOCSTRING = r""" The RoBERTa model was proposed in diff --git a/transformers/tests/modeling_roberta_test.py b/transformers/tests/modeling_roberta_test.py index 121cb84148..fe6ffe98c6 100644 --- a/transformers/tests/modeling_roberta_test.py +++ b/transformers/tests/modeling_roberta_test.py @@ -225,6 +225,10 @@ class RobertaModelTest(CommonTestCases.CommonModelTester): ]]) position_ids = model.create_position_ids_from_input_ids(input_ids) + self.assertEqual( + position_ids.shape, + expected_positions.shape + ) self.assertTrue(torch.all(torch.eq(position_ids, expected_positions))) def test_create_position_ids_from_inputs_embeds(self): @@ -235,17 +239,24 @@ class RobertaModelTest(CommonTestCases.CommonModelTester): first available non-padding position index is RobertaEmbeddings.padding_idx + 1 """ config = self.model_tester.prepare_config_and_inputs()[0] - model = RobertaEmbeddings(config=config) + embeddings = RobertaEmbeddings(config=config) - input_ids = torch.Tensor(1, 4, 30) - expected_positions = torch.as_tensor([[ - 0 + model.padding_idx + 1, - 1 + model.padding_idx + 1, - 2 + model.padding_idx + 1, - 3 + model.padding_idx + 1, - ]]) - position_ids = model.create_position_ids_from_inputs_embeds(input_ids) - self.assertTrue(torch.all(torch.eq(position_ids, expected_positions))) + inputs_embeds = torch.Tensor(2, 4, 30) + expected_single_positions = [ + 0 + embeddings.padding_idx + 1, + 1 + embeddings.padding_idx + 1, + 2 + embeddings.padding_idx + 1, + 3 + embeddings.padding_idx + 1, + ] + expected_positions = torch.as_tensor([expected_single_positions, expected_single_positions]) + position_ids = embeddings.create_position_ids_from_inputs_embeds(inputs_embeds) + self.assertEqual( + position_ids.shape, + expected_positions.shape + ) + self.assertTrue( + torch.all(torch.eq(position_ids, expected_positions)) + ) class RobertaModelIntegrationTest(unittest.TestCase): From ac1b449cc938bb34bc9021feff599cfd3b2376ae Mon Sep 17 00:00:00 2001 From: Julien Chaumond Date: Sat, 21 Dec 2019 00:09:01 -0500 Subject: [PATCH 493/505] [doc] move distilroberta to more appropriate place cc @lysandrejik --- docs/source/pretrained_models.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/source/pretrained_models.rst b/docs/source/pretrained_models.rst index a359990f5a..eb7b41ffc9 100644 --- a/docs/source/pretrained_models.rst +++ b/docs/source/pretrained_models.rst @@ -3,6 +3,7 @@ Pretrained models Here is the full list of the currently provided pretrained models together with a short presentation of each model. +For a list that includes community-uploaded models, refer to `https://huggingface.co/models `__. +-------------------+------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | Architecture | Shortcut name | Details of the model | @@ -154,6 +155,10 @@ Here is the full list of the currently provided pretrained models together with | | | | ``roberta-large`` fine-tuned on `MNLI `__. | | | | (see `details `__) | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ +| | ``distilroberta-base`` | | 6-layer, 768-hidden, 12-heads, 82M parameters | +| | | | The DistilRoBERTa model distilled from the RoBERTa model `roberta-base` checkpoint. | +| | | (see `details `__) | +| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``roberta-base-openai-detector`` | | 12-layer, 768-hidden, 12-heads, 125M parameters | | | | | ``roberta-base`` fine-tuned by OpenAI on the outputs of the 1.5B-parameter GPT-2 model. | | | | (see `details `__) | @@ -174,10 +179,6 @@ Here is the full list of the currently provided pretrained models together with | | | | The DistilGPT2 model distilled from the GPT2 model `gpt2` checkpoint. | | | | (see `details `__) | | +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ -| | ``distilroberta-base`` | | 6-layer, 768-hidden, 12-heads, 82M parameters | -| | | | The DistilRoBERTa model distilled from the RoBERTa model `roberta-base` checkpoint. | -| | | (see `details `__) | -| +------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------+ | | ``distilbert-base-german-cased`` | | 6-layer, 768-hidden, 12-heads, 66M parameters | | | | | The German DistilBERT model distilled from the German DBMDZ BERT model `bert-base-german-dbmdz-cased` checkpoint. | | | | (see `details `__) | From 12726f8556152dbc6c115327646ebb33ccb2bc4f Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Fri, 20 Dec 2019 20:56:58 +0100 Subject: [PATCH 494/505] Remove redundant torch.jit.trace in tests. This looks like it could be expensive, so don't run it twice. --- transformers/tests/modeling_common_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/transformers/tests/modeling_common_test.py b/transformers/tests/modeling_common_test.py index c84162117a..c03d307e71 100644 --- a/transformers/tests/modeling_common_test.py +++ b/transformers/tests/modeling_common_test.py @@ -218,12 +218,11 @@ class CommonTestCases: inputs = inputs_dict['input_ids'] # Let's keep only input_ids try: - torch.jit.trace(model, inputs) + traced_gpt2 = torch.jit.trace(model, inputs) except RuntimeError: self.fail("Couldn't trace module.") try: - traced_gpt2 = torch.jit.trace(model, inputs) torch.jit.save(traced_gpt2, "traced_model.pt") except RuntimeError: self.fail("Couldn't save module.") From 478e456e8392be1356a795a354215ba7dbf03a7b Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Fri, 20 Dec 2019 20:56:58 +0100 Subject: [PATCH 495/505] Use a random temp dir for writing file in tests. --- transformers/tests/modeling_common_test.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/transformers/tests/modeling_common_test.py b/transformers/tests/modeling_common_test.py index c03d307e71..8bf66c3582 100644 --- a/transformers/tests/modeling_common_test.py +++ b/transformers/tests/modeling_common_test.py @@ -18,7 +18,7 @@ from __future__ import print_function import copy import sys -import os +import os.path import shutil import tempfile import json @@ -222,16 +222,18 @@ class CommonTestCases: except RuntimeError: self.fail("Couldn't trace module.") - try: - torch.jit.save(traced_gpt2, "traced_model.pt") - except RuntimeError: - self.fail("Couldn't save module.") + with TemporaryDirectory() as tmp_dir_name: + pt_file_name = os.path.join(tmp_dir_name, "traced_model.pt") - try: - loaded_model = torch.jit.load("traced_model.pt") - os.remove("traced_model.pt") - except ValueError: - self.fail("Couldn't load module.") + try: + torch.jit.save(traced_gpt2, pt_file_name) + except Exception: + self.fail("Couldn't save module.") + + try: + loaded_model = torch.jit.load(pt_file_name) + except Exception: + self.fail("Couldn't load module.") model.to(torch_device) model.eval() From 286d5bb6b7afb1fcca1923d431f42c716f53a290 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Fri, 20 Dec 2019 20:56:58 +0100 Subject: [PATCH 496/505] Use a random temp dir for writing pruned models in tests. --- transformers/tests/modeling_common_test.py | 24 ++++++++-------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/transformers/tests/modeling_common_test.py b/transformers/tests/modeling_common_test.py index 8bf66c3582..cf36332207 100644 --- a/transformers/tests/modeling_common_test.py +++ b/transformers/tests/modeling_common_test.py @@ -353,12 +353,11 @@ class CommonTestCases: heads_to_prune = {0: list(range(1, self.model_tester.num_attention_heads)), -1: [0]} model.prune_heads(heads_to_prune) - directory = "pruned_model" - if not os.path.exists(directory): - os.makedirs(directory) - model.save_pretrained(directory) - model = model_class.from_pretrained(directory) - model.to(torch_device) + + with TemporaryDirectory() as temp_dir_name: + model.save_pretrained(temp_dir_name) + model = model_class.from_pretrained(temp_dir_name) + model.to(torch_device) with torch.no_grad(): outputs = model(**inputs_dict) @@ -367,7 +366,6 @@ class CommonTestCases: self.assertEqual(attentions[1].shape[-3], self.model_tester.num_attention_heads) self.assertEqual(attentions[-1].shape[-3], self.model_tester.num_attention_heads - 1) - shutil.rmtree(directory) def test_head_pruning_save_load_from_config_init(self): if not self.test_pruning: @@ -427,14 +425,10 @@ class CommonTestCases: self.assertEqual(attentions[2].shape[-3], self.model_tester.num_attention_heads) self.assertEqual(attentions[3].shape[-3], self.model_tester.num_attention_heads) - directory = "pruned_model" - - if not os.path.exists(directory): - os.makedirs(directory) - model.save_pretrained(directory) - model = model_class.from_pretrained(directory) - model.to(torch_device) - shutil.rmtree(directory) + with TemporaryDirectory() as temp_dir_name: + model.save_pretrained(temp_dir_name) + model = model_class.from_pretrained(temp_dir_name) + model.to(torch_device) with torch.no_grad(): outputs = model(**inputs_dict) From b67fa1a8d2302d808ecb9d95355181eaf21ee3b6 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Fri, 20 Dec 2019 20:56:58 +0100 Subject: [PATCH 497/505] Download models directly to cache_dir. This allows moving the file instead of copying it, which is more reliable. Also it avoids writing large amounts of data to /tmp, which may not be large enough to accomodate it. Refs #2222. --- transformers/file_utils.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/transformers/file_utils.py b/transformers/file_utils.py index 032a6af63b..61ff1d00bc 100644 --- a/transformers/file_utils.py +++ b/transformers/file_utils.py @@ -10,10 +10,9 @@ import json import logging import os import six -import shutil import tempfile import fnmatch -from functools import wraps +from functools import partial, wraps from hashlib import sha256 from io import open @@ -345,14 +344,13 @@ def get_from_cache(url, cache_dir=None, force_download=False, proxies=None, etag def _resumable_file_manager(): with open(incomplete_path,'a+b') as f: yield f - os.remove(incomplete_path) temp_file_manager = _resumable_file_manager if os.path.exists(incomplete_path): resume_size = os.stat(incomplete_path).st_size else: resume_size = 0 else: - temp_file_manager = tempfile.NamedTemporaryFile + temp_file_manager = partial(tempfile.NamedTemporaryFile, dir=cache_dir, delete=False) resume_size = 0 if etag is not None and (not os.path.exists(cache_path) or force_download): @@ -371,12 +369,9 @@ def get_from_cache(url, cache_dir=None, force_download=False, proxies=None, etag # we are copying the file before closing it, so flush to avoid truncation temp_file.flush() - # shutil.copyfileobj() starts at the current position, so go to the start - temp_file.seek(0) - logger.info("copying %s to cache at %s", temp_file.name, cache_path) - with open(cache_path, 'wb') as cache_file: - shutil.copyfileobj(temp_file, cache_file) + logger.info("storing %s in cache at %s", url, cache_path) + os.rename(temp_file.name, cache_path) logger.info("creating metadata file for %s", cache_path) meta = {'url': url, 'etag': etag} @@ -387,6 +382,4 @@ def get_from_cache(url, cache_dir=None, force_download=False, proxies=None, etag output_string = unicode(output_string, 'utf-8') # The beauty of python 2 meta_file.write(output_string) - logger.info("removing temp file %s", temp_file.name) - return cache_path From b670c2668426326aeffe626aabac7ee2dff3c7c2 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Fri, 20 Dec 2019 20:56:58 +0100 Subject: [PATCH 498/505] Take advantage of the cache when running tests. Caching models across test cases and across runs of the test suite makes slow tests somewhat more bearable. Use gettempdir() instead of /tmp in tests. This makes it easier to change the location of the cache with semi-standard TMPDIR/TEMP/TMP environment variables. Fix #2222. --- .../tests/modeling_tf_xxx_test.py | 7 ++----- .../tests/modeling_xxx_test.py | 7 ++----- transformers/tests/modeling_albert_test.py | 7 ++----- transformers/tests/modeling_bert_test.py | 7 ++----- transformers/tests/modeling_common_test.py | 6 ++---- transformers/tests/modeling_ctrl_test.py | 7 ++----- transformers/tests/modeling_distilbert_test.py | 6 ++---- transformers/tests/modeling_gpt2_test.py | 7 ++----- transformers/tests/modeling_openai_test.py | 7 ++----- transformers/tests/modeling_roberta_test.py | 7 ++----- transformers/tests/modeling_t5_test.py | 7 ++----- transformers/tests/modeling_tf_albert_test.py | 8 ++------ transformers/tests/modeling_tf_auto_test.py | 18 +++++++++--------- transformers/tests/modeling_tf_bert_test.py | 7 ++----- transformers/tests/modeling_tf_ctrl_test.py | 7 ++----- .../tests/modeling_tf_distilbert_test.py | 6 ++---- transformers/tests/modeling_tf_gpt2_test.py | 7 ++----- .../tests/modeling_tf_openai_gpt_test.py | 7 ++----- transformers/tests/modeling_tf_roberta_test.py | 7 ++----- transformers/tests/modeling_tf_t5_test.py | 7 ++----- .../tests/modeling_tf_transfo_xl_test.py | 7 ++----- transformers/tests/modeling_tf_xlm_test.py | 7 ++----- transformers/tests/modeling_tf_xlnet_test.py | 7 ++----- transformers/tests/modeling_transfo_xl_test.py | 7 ++----- transformers/tests/modeling_xlm_test.py | 7 ++----- transformers/tests/modeling_xlnet_test.py | 7 ++----- transformers/tests/utils.py | 3 +++ 27 files changed, 62 insertions(+), 132 deletions(-) diff --git a/templates/adding_a_new_model/tests/modeling_tf_xxx_test.py b/templates/adding_a_new_model/tests/modeling_tf_xxx_test.py index 912a4aa340..6eba932a8e 100644 --- a/templates/adding_a_new_model/tests/modeling_tf_xxx_test.py +++ b/templates/adding_a_new_model/tests/modeling_tf_xxx_test.py @@ -17,12 +17,11 @@ from __future__ import division from __future__ import print_function import unittest -import shutil import sys from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester -from .utils import require_tf, slow +from .utils import CACHE_DIR, require_tf, slow from transformers import XxxConfig, is_tf_available @@ -245,10 +244,8 @@ class TFXxxModelTest(TFCommonTestCases.TFCommonModelTester): @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in ['xxx-base-uncased']: - model = TFXxxModel.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = TFXxxModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) if __name__ == "__main__": diff --git a/templates/adding_a_new_model/tests/modeling_xxx_test.py b/templates/adding_a_new_model/tests/modeling_xxx_test.py index 30e614b3f2..5e22392d00 100644 --- a/templates/adding_a_new_model/tests/modeling_xxx_test.py +++ b/templates/adding_a_new_model/tests/modeling_xxx_test.py @@ -17,13 +17,12 @@ from __future__ import division from __future__ import print_function import unittest -import shutil from transformers import is_torch_available from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester -from .utils import require_torch, slow, torch_device +from .utils import CACHE_DIR, require_torch, slow, torch_device if is_torch_available(): from transformers import (XxxConfig, XxxModel, XxxForMaskedLM, @@ -249,10 +248,8 @@ class XxxModelTest(CommonTestCases.CommonModelTester): @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in list(XXX_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - model = XxxModel.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = XxxModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) if __name__ == "__main__": diff --git a/transformers/tests/modeling_albert_test.py b/transformers/tests/modeling_albert_test.py index 1911d244e7..b726fd9278 100644 --- a/transformers/tests/modeling_albert_test.py +++ b/transformers/tests/modeling_albert_test.py @@ -17,13 +17,12 @@ from __future__ import division from __future__ import print_function import unittest -import shutil from transformers import is_torch_available from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester -from .utils import require_torch, slow, torch_device +from .utils import CACHE_DIR, require_torch, slow, torch_device if is_torch_available(): from transformers import (AlbertConfig, AlbertModel, AlbertForMaskedLM, @@ -230,10 +229,8 @@ class AlbertModelTest(CommonTestCases.CommonModelTester): @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in list(ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - model = AlbertModel.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = AlbertModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) if __name__ == "__main__": diff --git a/transformers/tests/modeling_bert_test.py b/transformers/tests/modeling_bert_test.py index 0eb7bc9a14..a5adff8f68 100644 --- a/transformers/tests/modeling_bert_test.py +++ b/transformers/tests/modeling_bert_test.py @@ -17,13 +17,12 @@ from __future__ import division from __future__ import print_function import unittest -import shutil from transformers import is_torch_available from .modeling_common_test import (CommonTestCases, ids_tensor, floats_tensor) from .configuration_common_test import ConfigTester -from .utils import require_torch, slow, torch_device +from .utils import CACHE_DIR, require_torch, slow, torch_device if is_torch_available(): from transformers import (BertConfig, BertModel, BertForMaskedLM, @@ -360,10 +359,8 @@ class BertModelTest(CommonTestCases.CommonModelTester): @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in list(BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - model = BertModel.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = BertModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) diff --git a/transformers/tests/modeling_common_test.py b/transformers/tests/modeling_common_test.py index cf36332207..2116651f4a 100644 --- a/transformers/tests/modeling_common_test.py +++ b/transformers/tests/modeling_common_test.py @@ -30,7 +30,7 @@ import logging from transformers import is_torch_available -from .utils import require_torch, slow, torch_device +from .utils import CACHE_DIR, require_torch, slow, torch_device if is_torch_available(): import torch @@ -753,10 +753,8 @@ class CommonTestCases: [[], []]) def create_and_check_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in list(self.base_model_class.pretrained_model_archive_map.keys())[:1]: - model = self.base_model_class.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = self.base_model_class.from_pretrained(model_name, cache_dir=CACHE_DIR) self.parent.assertIsNotNone(model) def prepare_config_and_inputs_for_common(self): diff --git a/transformers/tests/modeling_ctrl_test.py b/transformers/tests/modeling_ctrl_test.py index c7de49b2ab..ed0d62d1e6 100644 --- a/transformers/tests/modeling_ctrl_test.py +++ b/transformers/tests/modeling_ctrl_test.py @@ -16,7 +16,6 @@ from __future__ import division from __future__ import print_function import unittest -import shutil import pdb from transformers import is_torch_available @@ -27,7 +26,7 @@ if is_torch_available(): from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester -from .utils import require_torch, slow, torch_device +from .utils import CACHE_DIR, require_torch, slow, torch_device @require_torch @@ -205,10 +204,8 @@ class CTRLModelTest(CommonTestCases.CommonModelTester): @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in list(CTRL_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - model = CTRLModel.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = CTRLModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) diff --git a/transformers/tests/modeling_distilbert_test.py b/transformers/tests/modeling_distilbert_test.py index 82f71c40da..ac6f5d248e 100644 --- a/transformers/tests/modeling_distilbert_test.py +++ b/transformers/tests/modeling_distilbert_test.py @@ -27,7 +27,7 @@ if is_torch_available(): from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester -from .utils import require_torch, slow, torch_device +from .utils import CACHE_DIR, require_torch, slow, torch_device @require_torch @@ -235,10 +235,8 @@ class DistilBertModelTest(CommonTestCases.CommonModelTester): # @slow # def test_model_from_pretrained(self): - # cache_dir = "/tmp/transformers_test/" # for model_name in list(DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - # model = DistilBertModel.from_pretrained(model_name, cache_dir=cache_dir) - # shutil.rmtree(cache_dir) + # model = DistilBertModel.from_pretrained(model_name, cache_dir=CACHE_DIR) # self.assertIsNotNone(model) if __name__ == "__main__": diff --git a/transformers/tests/modeling_gpt2_test.py b/transformers/tests/modeling_gpt2_test.py index a82e39c261..ad2ec1fd91 100644 --- a/transformers/tests/modeling_gpt2_test.py +++ b/transformers/tests/modeling_gpt2_test.py @@ -17,7 +17,6 @@ from __future__ import division from __future__ import print_function import unittest -import shutil from transformers import is_torch_available @@ -27,7 +26,7 @@ if is_torch_available(): from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester -from .utils import require_torch, slow, torch_device +from .utils import CACHE_DIR, require_torch, slow, torch_device @require_torch @@ -239,10 +238,8 @@ class GPT2ModelTest(CommonTestCases.CommonModelTester): @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in list(GPT2_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - model = GPT2Model.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = GPT2Model.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) diff --git a/transformers/tests/modeling_openai_test.py b/transformers/tests/modeling_openai_test.py index 7655e432e8..1880febcae 100644 --- a/transformers/tests/modeling_openai_test.py +++ b/transformers/tests/modeling_openai_test.py @@ -17,7 +17,6 @@ from __future__ import division from __future__ import print_function import unittest -import shutil from transformers import is_torch_available @@ -27,7 +26,7 @@ if is_torch_available(): from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester -from .utils import require_torch, slow, torch_device +from .utils import CACHE_DIR, require_torch, slow, torch_device @require_torch @@ -207,10 +206,8 @@ class OpenAIGPTModelTest(CommonTestCases.CommonModelTester): @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in list(OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - model = OpenAIGPTModel.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = OpenAIGPTModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) diff --git a/transformers/tests/modeling_roberta_test.py b/transformers/tests/modeling_roberta_test.py index 4d34a50528..299cbd01ad 100644 --- a/transformers/tests/modeling_roberta_test.py +++ b/transformers/tests/modeling_roberta_test.py @@ -17,7 +17,6 @@ from __future__ import division from __future__ import print_function import unittest -import shutil from transformers import is_torch_available @@ -29,7 +28,7 @@ if is_torch_available(): from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester -from .utils import require_torch, slow, torch_device +from .utils import CACHE_DIR, require_torch, slow, torch_device @require_torch @@ -199,10 +198,8 @@ class RobertaModelTest(CommonTestCases.CommonModelTester): @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in list(ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - model = RobertaModel.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = RobertaModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) diff --git a/transformers/tests/modeling_t5_test.py b/transformers/tests/modeling_t5_test.py index c337163375..9fd9a4b304 100644 --- a/transformers/tests/modeling_t5_test.py +++ b/transformers/tests/modeling_t5_test.py @@ -17,13 +17,12 @@ from __future__ import division from __future__ import print_function import unittest -import shutil from transformers import is_torch_available from .modeling_common_test import (CommonTestCases, ids_tensor, floats_tensor) from .configuration_common_test import ConfigTester -from .utils import require_torch, slow, torch_device +from .utils import CACHE_DIR, require_torch, slow, torch_device if is_torch_available(): from transformers import (T5Config, T5Model, T5WithLMHeadModel) @@ -175,10 +174,8 @@ class T5ModelTest(CommonTestCases.CommonModelTester): @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in list(T5_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - model = T5Model.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = T5Model.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) if __name__ == "__main__": diff --git a/transformers/tests/modeling_tf_albert_test.py b/transformers/tests/modeling_tf_albert_test.py index 93aeab66c2..ee71371a18 100644 --- a/transformers/tests/modeling_tf_albert_test.py +++ b/transformers/tests/modeling_tf_albert_test.py @@ -17,12 +17,11 @@ from __future__ import division from __future__ import print_function import unittest -import shutil import sys from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester -from .utils import require_tf, slow +from .utils import CACHE_DIR, require_tf, slow from transformers import AlbertConfig, is_tf_available @@ -217,12 +216,9 @@ class TFAlbertModelTest(TFCommonTestCases.TFCommonModelTester): @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" # for model_name in list(TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: for model_name in ['albert-base-uncased']: - model = TFAlbertModel.from_pretrained( - model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = TFAlbertModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) diff --git a/transformers/tests/modeling_tf_auto_test.py b/transformers/tests/modeling_tf_auto_test.py index 7ab6eaa3d6..2ad39ddccf 100644 --- a/transformers/tests/modeling_tf_auto_test.py +++ b/transformers/tests/modeling_tf_auto_test.py @@ -46,11 +46,11 @@ class TFAutoModelTest(unittest.TestCase): logging.basicConfig(level=logging.INFO) # for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: for model_name in ['bert-base-uncased']: - config = AutoConfig.from_pretrained(model_name, force_download=True) + config = AutoConfig.from_pretrained(model_name) self.assertIsNotNone(config) self.assertIsInstance(config, BertConfig) - model = TFAutoModel.from_pretrained(model_name, force_download=True) + model = TFAutoModel.from_pretrained(model_name) self.assertIsNotNone(model) self.assertIsInstance(model, TFBertModel) @@ -59,11 +59,11 @@ class TFAutoModelTest(unittest.TestCase): logging.basicConfig(level=logging.INFO) # for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: for model_name in ['bert-base-uncased']: - config = AutoConfig.from_pretrained(model_name, force_download=True) + config = AutoConfig.from_pretrained(model_name) self.assertIsNotNone(config) self.assertIsInstance(config, BertConfig) - model = TFAutoModelWithLMHead.from_pretrained(model_name, force_download=True) + model = TFAutoModelWithLMHead.from_pretrained(model_name) self.assertIsNotNone(model) self.assertIsInstance(model, TFBertForMaskedLM) @@ -72,11 +72,11 @@ class TFAutoModelTest(unittest.TestCase): logging.basicConfig(level=logging.INFO) # for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: for model_name in ['bert-base-uncased']: - config = AutoConfig.from_pretrained(model_name, force_download=True) + config = AutoConfig.from_pretrained(model_name) self.assertIsNotNone(config) self.assertIsInstance(config, BertConfig) - model = TFAutoModelForSequenceClassification.from_pretrained(model_name, force_download=True) + model = TFAutoModelForSequenceClassification.from_pretrained(model_name) self.assertIsNotNone(model) self.assertIsInstance(model, TFBertForSequenceClassification) @@ -85,17 +85,17 @@ class TFAutoModelTest(unittest.TestCase): logging.basicConfig(level=logging.INFO) # for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: for model_name in ['bert-base-uncased']: - config = AutoConfig.from_pretrained(model_name, force_download=True) + config = AutoConfig.from_pretrained(model_name) self.assertIsNotNone(config) self.assertIsInstance(config, BertConfig) - model = TFAutoModelForQuestionAnswering.from_pretrained(model_name, force_download=True) + model = TFAutoModelForQuestionAnswering.from_pretrained(model_name) self.assertIsNotNone(model) self.assertIsInstance(model, TFBertForQuestionAnswering) def test_from_pretrained_identifier(self): logging.basicConfig(level=logging.INFO) - model = TFAutoModelWithLMHead.from_pretrained(SMALL_MODEL_IDENTIFIER, force_download=True) + model = TFAutoModelWithLMHead.from_pretrained(SMALL_MODEL_IDENTIFIER) self.assertIsInstance(model, TFBertForMaskedLM) diff --git a/transformers/tests/modeling_tf_bert_test.py b/transformers/tests/modeling_tf_bert_test.py index 20073e1ab8..abf20b1514 100644 --- a/transformers/tests/modeling_tf_bert_test.py +++ b/transformers/tests/modeling_tf_bert_test.py @@ -17,12 +17,11 @@ from __future__ import division from __future__ import print_function import unittest -import shutil import sys from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester -from .utils import require_tf, slow +from .utils import CACHE_DIR, require_tf, slow from transformers import BertConfig, is_tf_available @@ -310,11 +309,9 @@ class TFBertModelTest(TFCommonTestCases.TFCommonModelTester): @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" # for model_name in list(TF_BERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: for model_name in ['bert-base-uncased']: - model = TFBertModel.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = TFBertModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) if __name__ == "__main__": diff --git a/transformers/tests/modeling_tf_ctrl_test.py b/transformers/tests/modeling_tf_ctrl_test.py index 0876582e57..93b231e517 100644 --- a/transformers/tests/modeling_tf_ctrl_test.py +++ b/transformers/tests/modeling_tf_ctrl_test.py @@ -17,12 +17,11 @@ from __future__ import division from __future__ import print_function import unittest -import shutil import sys from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester -from .utils import require_tf, slow +from .utils import CACHE_DIR, require_tf, slow from transformers import CTRLConfig, is_tf_available @@ -189,10 +188,8 @@ class TFCTRLModelTest(TFCommonTestCases.TFCommonModelTester): @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in list(TF_CTRL_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - model = TFCTRLModel.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = TFCTRLModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) if __name__ == "__main__": diff --git a/transformers/tests/modeling_tf_distilbert_test.py b/transformers/tests/modeling_tf_distilbert_test.py index d9e971c2a5..f28b5c397b 100644 --- a/transformers/tests/modeling_tf_distilbert_test.py +++ b/transformers/tests/modeling_tf_distilbert_test.py @@ -20,7 +20,7 @@ import unittest from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester -from .utils import require_tf, slow +from .utils import CACHE_DIR, require_tf, slow from transformers import DistilBertConfig, is_tf_available @@ -211,10 +211,8 @@ class TFDistilBertModelTest(TFCommonTestCases.TFCommonModelTester): # @slow # def test_model_from_pretrained(self): - # cache_dir = "/tmp/transformers_test/" # for model_name in list(DISTILBERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - # model = DistilBertModel.from_pretrained(model_name, cache_dir=cache_dir) - # shutil.rmtree(cache_dir) + # model = DistilBertModel.from_pretrained(model_name, cache_dir=CACHE_DIR) # self.assertIsNotNone(model) if __name__ == "__main__": diff --git a/transformers/tests/modeling_tf_gpt2_test.py b/transformers/tests/modeling_tf_gpt2_test.py index 3f30b32787..90920342ba 100644 --- a/transformers/tests/modeling_tf_gpt2_test.py +++ b/transformers/tests/modeling_tf_gpt2_test.py @@ -17,12 +17,11 @@ from __future__ import division from __future__ import print_function import unittest -import shutil import sys from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester -from .utils import require_tf, slow +from .utils import CACHE_DIR, require_tf, slow from transformers import GPT2Config, is_tf_available @@ -220,10 +219,8 @@ class TFGPT2ModelTest(TFCommonTestCases.TFCommonModelTester): @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in list(TF_GPT2_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - model = TFGPT2Model.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = TFGPT2Model.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) if __name__ == "__main__": diff --git a/transformers/tests/modeling_tf_openai_gpt_test.py b/transformers/tests/modeling_tf_openai_gpt_test.py index 863dbf1bc0..065bf2acde 100644 --- a/transformers/tests/modeling_tf_openai_gpt_test.py +++ b/transformers/tests/modeling_tf_openai_gpt_test.py @@ -17,12 +17,11 @@ from __future__ import division from __future__ import print_function import unittest -import shutil import sys from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester -from .utils import require_tf, slow +from .utils import CACHE_DIR, require_tf, slow from transformers import OpenAIGPTConfig, is_tf_available @@ -219,10 +218,8 @@ class TFOpenAIGPTModelTest(TFCommonTestCases.TFCommonModelTester): @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in list(TF_OPENAI_GPT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - model = TFOpenAIGPTModel.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = TFOpenAIGPTModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) if __name__ == "__main__": diff --git a/transformers/tests/modeling_tf_roberta_test.py b/transformers/tests/modeling_tf_roberta_test.py index f4ed97c44b..93c478ae28 100644 --- a/transformers/tests/modeling_tf_roberta_test.py +++ b/transformers/tests/modeling_tf_roberta_test.py @@ -17,11 +17,10 @@ from __future__ import division from __future__ import print_function import unittest -import shutil from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester -from .utils import require_tf, slow +from .utils import CACHE_DIR, require_tf, slow from transformers import RobertaConfig, is_tf_available @@ -192,10 +191,8 @@ class TFRobertaModelTest(TFCommonTestCases.TFCommonModelTester): @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in list(TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - model = TFRobertaModel.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = TFRobertaModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) diff --git a/transformers/tests/modeling_tf_t5_test.py b/transformers/tests/modeling_tf_t5_test.py index b905a9875b..da9ce6f89d 100644 --- a/transformers/tests/modeling_tf_t5_test.py +++ b/transformers/tests/modeling_tf_t5_test.py @@ -17,12 +17,11 @@ from __future__ import division from __future__ import print_function import unittest -import shutil import sys from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester -from .utils import require_tf, slow +from .utils import CACHE_DIR, require_tf, slow from transformers import T5Config, is_tf_available @@ -162,10 +161,8 @@ class TFT5ModelTest(TFCommonTestCases.TFCommonModelTester): @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in ['t5-small']: - model = TFT5Model.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = TFT5Model.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) if __name__ == "__main__": diff --git a/transformers/tests/modeling_tf_transfo_xl_test.py b/transformers/tests/modeling_tf_transfo_xl_test.py index 746a6a1321..8225c09275 100644 --- a/transformers/tests/modeling_tf_transfo_xl_test.py +++ b/transformers/tests/modeling_tf_transfo_xl_test.py @@ -18,11 +18,10 @@ from __future__ import print_function import unittest import random -import shutil from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester -from .utils import require_tf, slow +from .utils import CACHE_DIR, require_tf, slow from transformers import TransfoXLConfig, is_tf_available @@ -205,10 +204,8 @@ class TFTransfoXLModelTest(TFCommonTestCases.TFCommonModelTester): @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in list(TF_TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - model = TFTransfoXLModel.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = TFTransfoXLModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) diff --git a/transformers/tests/modeling_tf_xlm_test.py b/transformers/tests/modeling_tf_xlm_test.py index 228e436149..8b5ab6d742 100644 --- a/transformers/tests/modeling_tf_xlm_test.py +++ b/transformers/tests/modeling_tf_xlm_test.py @@ -17,7 +17,6 @@ from __future__ import division from __future__ import print_function import unittest -import shutil from transformers import is_tf_available @@ -31,7 +30,7 @@ if is_tf_available(): from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester -from .utils import require_tf, slow +from .utils import CACHE_DIR, require_tf, slow @require_tf @@ -252,10 +251,8 @@ class TFXLMModelTest(TFCommonTestCases.TFCommonModelTester): @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in list(TF_XLM_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - model = XLMModel.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = XLMModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) diff --git a/transformers/tests/modeling_tf_xlnet_test.py b/transformers/tests/modeling_tf_xlnet_test.py index eb66d92793..15fd917481 100644 --- a/transformers/tests/modeling_tf_xlnet_test.py +++ b/transformers/tests/modeling_tf_xlnet_test.py @@ -20,7 +20,6 @@ import os import unittest import json import random -import shutil from transformers import XLNetConfig, is_tf_available @@ -35,7 +34,7 @@ if is_tf_available(): from .modeling_tf_common_test import (TFCommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester -from .utils import require_tf, slow +from .utils import CACHE_DIR, require_tf, slow @require_tf @@ -319,10 +318,8 @@ class TFXLNetModelTest(TFCommonTestCases.TFCommonModelTester): @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in list(TF_XLNET_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - model = TFXLNetModel.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = TFXLNetModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) diff --git a/transformers/tests/modeling_transfo_xl_test.py b/transformers/tests/modeling_transfo_xl_test.py index f41d50a3a0..acbe95fe4a 100644 --- a/transformers/tests/modeling_transfo_xl_test.py +++ b/transformers/tests/modeling_transfo_xl_test.py @@ -18,7 +18,6 @@ from __future__ import print_function import unittest import random -import shutil from transformers import is_torch_available @@ -29,7 +28,7 @@ if is_torch_available(): from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester -from .utils import require_torch, slow, torch_device +from .utils import CACHE_DIR, require_torch, slow, torch_device @require_torch @@ -208,10 +207,8 @@ class TransfoXLModelTest(CommonTestCases.CommonModelTester): @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in list(TRANSFO_XL_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - model = TransfoXLModel.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = TransfoXLModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) diff --git a/transformers/tests/modeling_xlm_test.py b/transformers/tests/modeling_xlm_test.py index 7cae6c848e..fcc2f4699b 100644 --- a/transformers/tests/modeling_xlm_test.py +++ b/transformers/tests/modeling_xlm_test.py @@ -17,7 +17,6 @@ from __future__ import division from __future__ import print_function import unittest -import shutil from transformers import is_torch_available @@ -28,7 +27,7 @@ if is_torch_available(): from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester -from .utils import require_torch, slow, torch_device +from .utils import CACHE_DIR, require_torch, slow, torch_device @require_torch @@ -318,10 +317,8 @@ class XLMModelTest(CommonTestCases.CommonModelTester): @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in list(XLM_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - model = XLMModel.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = XLMModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) diff --git a/transformers/tests/modeling_xlnet_test.py b/transformers/tests/modeling_xlnet_test.py index 6d901ee699..6d218d6ef4 100644 --- a/transformers/tests/modeling_xlnet_test.py +++ b/transformers/tests/modeling_xlnet_test.py @@ -20,7 +20,6 @@ import os import unittest import json import random -import shutil from transformers import is_torch_available @@ -33,7 +32,7 @@ if is_torch_available(): from .modeling_common_test import (CommonTestCases, ids_tensor) from .configuration_common_test import ConfigTester -from .utils import require_torch, slow, torch_device +from .utils import CACHE_DIR, require_torch, slow, torch_device @require_torch @@ -385,10 +384,8 @@ class XLNetModelTest(CommonTestCases.CommonModelTester): @slow def test_model_from_pretrained(self): - cache_dir = "/tmp/transformers_test/" for model_name in list(XLNET_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - model = XLNetModel.from_pretrained(model_name, cache_dir=cache_dir) - shutil.rmtree(cache_dir) + model = XLNetModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) diff --git a/transformers/tests/utils.py b/transformers/tests/utils.py index c950ad8f17..ba0e19f420 100644 --- a/transformers/tests/utils.py +++ b/transformers/tests/utils.py @@ -1,11 +1,14 @@ import os import unittest +import tempfile from distutils.util import strtobool from transformers.file_utils import _tf_available, _torch_available +CACHE_DIR = os.path.join(tempfile.gettempdir(), "transformers_test") + SMALL_MODEL_IDENTIFIER = "julien-c/bert-xsmall-dummy" From a4c9338b83ba612b5f5aec645f375d048d9a7647 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Fri, 20 Dec 2019 20:56:59 +0100 Subject: [PATCH 499/505] Prevent parallel downloads of the same file with a lock. Since the file is written to the filesystem, a filesystem lock is the way to go here. Add a dependency on the third-party filelock library to get cross-platform functionality. --- setup.py | 1 + transformers/file_utils.py | 89 +++++++++++++++++++++----------------- 2 files changed, 50 insertions(+), 40 deletions(-) diff --git a/setup.py b/setup.py index cd64a6ce90..fe2e1526bf 100644 --- a/setup.py +++ b/setup.py @@ -59,6 +59,7 @@ setup( "tests.*", "tests"]), install_requires=['numpy', 'boto3', + 'filelock', 'requests', 'tqdm', 'regex != 2019.12.17', diff --git a/transformers/file_utils.py b/transformers/file_utils.py index 61ff1d00bc..ec925c6160 100644 --- a/transformers/file_utils.py +++ b/transformers/file_utils.py @@ -24,6 +24,8 @@ from tqdm.auto import tqdm from contextlib import contextmanager from . import __version__ +from filelock import FileLock + logger = logging.getLogger(__name__) # pylint: disable=invalid-name try: @@ -333,53 +335,60 @@ def get_from_cache(url, cache_dir=None, force_download=False, proxies=None, etag # If we don't have a connection (etag is None) and can't identify the file # try to get the last downloaded one if not os.path.exists(cache_path) and etag is None: - matching_files = fnmatch.filter(os.listdir(cache_dir), filename + '.*') - matching_files = list(filter(lambda s: not s.endswith('.json'), matching_files)) + matching_files = [ + file + for file in fnmatch.filter(os.listdir(cache_dir), filename + '.*') + if not file.endswith('.json') and not file.endswith('.lock') + ] if matching_files: cache_path = os.path.join(cache_dir, matching_files[-1]) - if resume_download: - incomplete_path = cache_path + '.incomplete' - @contextmanager - def _resumable_file_manager(): - with open(incomplete_path,'a+b') as f: - yield f - temp_file_manager = _resumable_file_manager - if os.path.exists(incomplete_path): - resume_size = os.stat(incomplete_path).st_size - else: - resume_size = 0 - else: - temp_file_manager = partial(tempfile.NamedTemporaryFile, dir=cache_dir, delete=False) - resume_size = 0 + # Prevent parallel downloads of the same file with a lock. + lock_path = cache_path + '.lock' + with FileLock(lock_path): - if etag is not None and (not os.path.exists(cache_path) or force_download): - # Download to temporary file, then copy to cache dir once finished. - # Otherwise you get corrupt cache entries if the download gets interrupted. - with temp_file_manager() as temp_file: - logger.info("%s not found in cache or force_download set to True, downloading to %s", url, temp_file.name) - - # GET file object - if url.startswith("s3://"): - if resume_download: - logger.warn('Warning: resumable downloads are not implemented for "s3://" urls') - s3_get(url, temp_file, proxies=proxies) + if resume_download: + incomplete_path = cache_path + '.incomplete' + @contextmanager + def _resumable_file_manager(): + with open(incomplete_path,'a+b') as f: + yield f + temp_file_manager = _resumable_file_manager + if os.path.exists(incomplete_path): + resume_size = os.stat(incomplete_path).st_size else: - http_get(url, temp_file, proxies=proxies, resume_size=resume_size, user_agent=user_agent) + resume_size = 0 + else: + temp_file_manager = partial(tempfile.NamedTemporaryFile, dir=cache_dir, delete=False) + resume_size = 0 - # we are copying the file before closing it, so flush to avoid truncation - temp_file.flush() + if etag is not None and (not os.path.exists(cache_path) or force_download): + # Download to temporary file, then copy to cache dir once finished. + # Otherwise you get corrupt cache entries if the download gets interrupted. + with temp_file_manager() as temp_file: + logger.info("%s not found in cache or force_download set to True, downloading to %s", url, temp_file.name) - logger.info("storing %s in cache at %s", url, cache_path) - os.rename(temp_file.name, cache_path) + # GET file object + if url.startswith("s3://"): + if resume_download: + logger.warn('Warning: resumable downloads are not implemented for "s3://" urls') + s3_get(url, temp_file, proxies=proxies) + else: + http_get(url, temp_file, proxies=proxies, resume_size=resume_size, user_agent=user_agent) - logger.info("creating metadata file for %s", cache_path) - meta = {'url': url, 'etag': etag} - meta_path = cache_path + '.json' - with open(meta_path, 'w') as meta_file: - output_string = json.dumps(meta) - if sys.version_info[0] == 2 and isinstance(output_string, str): - output_string = unicode(output_string, 'utf-8') # The beauty of python 2 - meta_file.write(output_string) + # we are copying the file before closing it, so flush to avoid truncation + temp_file.flush() + + logger.info("storing %s in cache at %s", url, cache_path) + os.rename(temp_file.name, cache_path) + + logger.info("creating metadata file for %s", cache_path) + meta = {'url': url, 'etag': etag} + meta_path = cache_path + '.json' + with open(meta_path, 'w') as meta_file: + output_string = json.dumps(meta) + if sys.version_info[0] == 2 and isinstance(output_string, str): + output_string = unicode(output_string, 'utf-8') # The beauty of python 2 + meta_file.write(output_string) return cache_path From 29cbab98f0b36a3056e2982bf968c8370bad3838 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Fri, 20 Dec 2019 20:56:59 +0100 Subject: [PATCH 500/505] Parallelize tests on Circle CI. Set the number of CPUs manually based on the Circle CI resource class, or else we're getting 36 CPUs, which is far too much (perhaps that's the underlying hardware and not what Circle CI allocates to us). Don't parallelize the custom tokenizers tests because they take less than one second to run and parallelization actually makes them slower. --- .circleci/config.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b094067eb5..a7496c81e6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,9 +11,9 @@ jobs: - run: sudo pip install torch - run: sudo pip install tensorflow - run: sudo pip install --progress-bar off . - - run: sudo pip install pytest codecov pytest-cov + - run: sudo pip install pytest codecov pytest-cov pytest-xdist - run: sudo pip install tensorboardX scikit-learn - - run: python -m pytest -sv ./transformers/tests/ --cov + - run: python -m pytest -n 8 -s -v ./transformers/tests/ --cov - run: codecov build_py3_torch: working_directory: ~/transformers @@ -25,10 +25,10 @@ jobs: - checkout - run: sudo pip install torch - run: sudo pip install --progress-bar off . - - run: sudo pip install pytest codecov pytest-cov + - run: sudo pip install pytest codecov pytest-cov pytest-xdist - run: sudo pip install tensorboardX scikit-learn - - run: python -m pytest -sv ./transformers/tests/ --cov - - run: python -m pytest -sv ./examples/ + - run: python -m pytest -n 8 -s -v ./transformers/tests/ --cov + - run: python -m pytest -n 8 -s -v ./examples/ - run: codecov build_py3_tf: working_directory: ~/transformers @@ -40,9 +40,9 @@ jobs: - checkout - run: sudo pip install tensorflow - run: sudo pip install --progress-bar off . - - run: sudo pip install pytest codecov pytest-cov + - run: sudo pip install pytest codecov pytest-cov pytest-xdist - run: sudo pip install tensorboardX scikit-learn - - run: python -m pytest -sv ./transformers/tests/ --cov + - run: python -m pytest -n 8 -s -v ./transformers/tests/ --cov - run: codecov build_py3_custom_tokenizers: working_directory: ~/transformers @@ -51,7 +51,7 @@ jobs: steps: - checkout - run: sudo pip install --progress-bar off . - - run: sudo pip install pytest + - run: sudo pip install pytest pytest-xdist - run: sudo pip install mecab-python3 - run: RUN_CUSTOM_TOKENIZERS=1 python -m pytest -sv ./transformers/tests/tokenization_bert_japanese_test.py deploy_doc: From bb3bfa2d293589af0b3141c6f7235beba1c6bb44 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Fri, 20 Dec 2019 20:56:59 +0100 Subject: [PATCH 501/505] Distribute tests from the same file to the same worker. This should prevent two issues: - hitting API rate limits for tests that hit the HF API - multiplying the cost of expensive test setups --- .circleci/config.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a7496c81e6..f9de338fa4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,7 +13,7 @@ jobs: - run: sudo pip install --progress-bar off . - run: sudo pip install pytest codecov pytest-cov pytest-xdist - run: sudo pip install tensorboardX scikit-learn - - run: python -m pytest -n 8 -s -v ./transformers/tests/ --cov + - run: python -m pytest -n 8 --dist=loadfile -s -v ./transformers/tests/ --cov - run: codecov build_py3_torch: working_directory: ~/transformers @@ -27,8 +27,8 @@ jobs: - run: sudo pip install --progress-bar off . - run: sudo pip install pytest codecov pytest-cov pytest-xdist - run: sudo pip install tensorboardX scikit-learn - - run: python -m pytest -n 8 -s -v ./transformers/tests/ --cov - - run: python -m pytest -n 8 -s -v ./examples/ + - run: python -m pytest -n 8 --dist=loadfile -s -v ./transformers/tests/ --cov + - run: python -m pytest -n 8 --dist=loadfile -s -v ./examples/ - run: codecov build_py3_tf: working_directory: ~/transformers @@ -42,7 +42,7 @@ jobs: - run: sudo pip install --progress-bar off . - run: sudo pip install pytest codecov pytest-cov pytest-xdist - run: sudo pip install tensorboardX scikit-learn - - run: python -m pytest -n 8 -s -v ./transformers/tests/ --cov + - run: python -m pytest -n 8 --dist=loadfile -s -v ./transformers/tests/ --cov - run: codecov build_py3_custom_tokenizers: working_directory: ~/transformers From 80caf79d0743a354a43d2aac5ccfe58e0ac1b80a Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Fri, 20 Dec 2019 20:56:59 +0100 Subject: [PATCH 502/505] Prevent excessive parallelism in PyTorch. We're already using as many processes in parallel as we have CPU cores. Furthermore, the number of core may be incorrectly calculated as 36 (we've seen this in pytest-xdist) which make compound the problem. PyTorch performance craters without this. --- .circleci/config.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index f9de338fa4..812817efaa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,6 +4,8 @@ jobs: working_directory: ~/transformers docker: - image: circleci/python:3.5 + environment: + OMP_NUM_THREADS: 1 resource_class: xlarge parallelism: 1 steps: @@ -19,6 +21,8 @@ jobs: working_directory: ~/transformers docker: - image: circleci/python:3.5 + environment: + OMP_NUM_THREADS: 1 resource_class: xlarge parallelism: 1 steps: @@ -34,6 +38,8 @@ jobs: working_directory: ~/transformers docker: - image: circleci/python:3.5 + environment: + OMP_NUM_THREADS: 1 resource_class: xlarge parallelism: 1 steps: From 343c094f2156962a24bf19c5fbd771d01c81caf7 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Fri, 20 Dec 2019 20:56:59 +0100 Subject: [PATCH 503/505] Run examples separately from tests. This optimizes the total run time of the Circle CI test suite. --- .circleci/config.yml | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 812817efaa..bfa3b943aa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,6 @@ version: 2 jobs: - build_py3_torch_and_tf: + run_tests_py3_torch_and_tf: working_directory: ~/transformers docker: - image: circleci/python:3.5 @@ -17,7 +17,7 @@ jobs: - run: sudo pip install tensorboardX scikit-learn - run: python -m pytest -n 8 --dist=loadfile -s -v ./transformers/tests/ --cov - run: codecov - build_py3_torch: + run_tests_py3_torch: working_directory: ~/transformers docker: - image: circleci/python:3.5 @@ -32,9 +32,8 @@ jobs: - run: sudo pip install pytest codecov pytest-cov pytest-xdist - run: sudo pip install tensorboardX scikit-learn - run: python -m pytest -n 8 --dist=loadfile -s -v ./transformers/tests/ --cov - - run: python -m pytest -n 8 --dist=loadfile -s -v ./examples/ - run: codecov - build_py3_tf: + run_tests_py3_tf: working_directory: ~/transformers docker: - image: circleci/python:3.5 @@ -50,7 +49,7 @@ jobs: - run: sudo pip install tensorboardX scikit-learn - run: python -m pytest -n 8 --dist=loadfile -s -v ./transformers/tests/ --cov - run: codecov - build_py3_custom_tokenizers: + run_tests_py3_custom_tokenizers: working_directory: ~/transformers docker: - image: circleci/python:3.5 @@ -60,6 +59,21 @@ jobs: - run: sudo pip install pytest pytest-xdist - run: sudo pip install mecab-python3 - run: RUN_CUSTOM_TOKENIZERS=1 python -m pytest -sv ./transformers/tests/tokenization_bert_japanese_test.py + run_examples_py3_torch: + working_directory: ~/transformers + docker: + - image: circleci/python:3.5 + environment: + OMP_NUM_THREADS: 1 + resource_class: xlarge + parallelism: 1 + steps: + - checkout + - run: sudo pip install torch + - run: sudo pip install --progress-bar off . + - run: sudo pip install pytest pytest-xdist + - run: sudo pip install tensorboardX scikit-learn + - run: python -m pytest -n 8 --dist=loadfile -s -v ./examples/ deploy_doc: working_directory: ~/transformers docker: @@ -72,7 +86,7 @@ jobs: - run: sudo pip install --progress-bar off -r docs/requirements.txt - run: sudo pip install --progress-bar off -r requirements.txt - run: ./.circleci/deploy.sh - repository_consistency: + check_repository_consistency: working_directory: ~/transformers docker: - image: circleci/python:3.5 @@ -91,9 +105,10 @@ workflows: version: 2 build_and_test: jobs: - - repository_consistency - - build_py3_custom_tokenizers - - build_py3_torch_and_tf - - build_py3_torch - - build_py3_tf + - check_repository_consistency + - run_examples_py3_torch + - run_tests_py3_custom_tokenizers + - run_tests_py3_torch_and_tf + - run_tests_py3_torch + - run_tests_py3_tf - deploy_doc: *workflow_filters From 767bc3ca68d7f1617e2edd31374be3e2f05b27b6 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 21 Dec 2019 08:46:26 +0100 Subject: [PATCH 504/505] Fix typo in model name. This looks like a copy/paste mistake. Probably this test was never run. Refs #2250. --- transformers/tests/modeling_tf_xlm_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/tests/modeling_tf_xlm_test.py b/transformers/tests/modeling_tf_xlm_test.py index 8b5ab6d742..065d355b45 100644 --- a/transformers/tests/modeling_tf_xlm_test.py +++ b/transformers/tests/modeling_tf_xlm_test.py @@ -252,7 +252,7 @@ class TFXLMModelTest(TFCommonTestCases.TFCommonModelTester): @slow def test_model_from_pretrained(self): for model_name in list(TF_XLM_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - model = XLMModel.from_pretrained(model_name, cache_dir=CACHE_DIR) + model = TFXLMModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model) From b8e924e10d283c095e5aca3f762d812d5106b105 Mon Sep 17 00:00:00 2001 From: Aymeric Augustin Date: Sat, 21 Dec 2019 08:50:15 +0100 Subject: [PATCH 505/505] Restore test. This looks like debug code accidentally committed in b18509c2. Refs #2250. --- transformers/tests/modeling_tf_albert_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/transformers/tests/modeling_tf_albert_test.py b/transformers/tests/modeling_tf_albert_test.py index ee71371a18..374417cfe2 100644 --- a/transformers/tests/modeling_tf_albert_test.py +++ b/transformers/tests/modeling_tf_albert_test.py @@ -216,8 +216,7 @@ class TFAlbertModelTest(TFCommonTestCases.TFCommonModelTester): @slow def test_model_from_pretrained(self): - # for model_name in list(TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: - for model_name in ['albert-base-uncased']: + for model_name in list(TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_MAP.keys())[:1]: model = TFAlbertModel.from_pretrained(model_name, cache_dir=CACHE_DIR) self.assertIsNotNone(model)

wk{B}sNRrUY2nC9Tc6xd&5{@TUQ<^7cWaLqLna1Z z2Tt`gc$(`K?tXOJ{`uVSQ>f%biJ7HO2e0CR@d%T3v=109nJ0eNiHR#l^~`ai6|z(S zvo4nOXNUmCHyPEM9F0-$z>OCS(KgwhokDS3`=uL}Bt8)Wogd;N@5~EGE|B*9TAX>2 z67;DR$2VY#{~69nozs0pN7(36Zjx3z$4dLPB9hH_#^&DA>!#twpthW>;kna=K|g*I zH{U=)^9s+_1inPJ<;pgo(bgnaqpTf}?)##8{@br`e?{Qx#lAh1*zIRD$&W`@J#h!S zBo&jDJ~z;_Gh;G0Ry$3(o=kh5nAqwRoc{l<%9G*5`uUTshP1khcQ7W+4;nG${UROh zil?G!#)o2(W{phvr&1bQ!@Uy+Qij7SgI34kld$moa_>#$|<1|{krW0K0dIqOW!N&7%g26m-c1D; z+3&Ej=k1;|*60O5bJ{bo3Wx49s^mqYPFH`p6g@Ah=FiYB|K3H-qg*EM`fJDuz~S6|R5tB=9!5ST}Z(-8aN$5(A$qOgmM zc$oG^>I3!q6bdQ5*O=My%7ugV_;Q82Mo-n=j)yqAh)Ujm7BYQU*?BXdoDbJt0?IyC z60&b{_t!-3%VR~-Cc#c;vld**$)HZ#@-x1cA5b%6l_Ec3G2!~=Dyc7av);Q`V?8&G zV%wCM$ZNH%8MW50zUNhrrcq2&_uh4_?-7t8NZ5(h`h-t-g*(?ZdgjSAdrup){;>h+ zrYJvG?Hk`8UjaGRV;P@ziWDdRQ0Epu^isQ}R)pmaaQ>}19@j;^obvxW*ZhtK+;L$* zeoVl|SQ$LV{kC?sH@+uq+G_tf_9$;+o}9$jzI> zTjnl&r85GhgHGrziJ-*@6EaQmf9yO)g+(xZ3+PW0!FVDDtbx7&Di^}1LIULTLBW(C z|9z8gwarl8i!_MO3D`VH2{fteN%VSG>NJgkK8>4ydjWhO6rmfJ$N}YFpl(wzV+C?^ zTMm*mmH)Lme#4vG=9(y05)EcGdRLvMWf4=UPNz6X+)pNN%H zKC)^yNg#%;Cs)hbufsw6W|0}G!?_sOx#V7-X$epA7-T8zIeenZ!?Fzl;SGa*Ly0hbLzjM^bQKDDU_M&`XzsY&_~=> zDPk^K6>QXM>W|4&f|A{!29sZ+QOAq9P()^350;h^ zH1ud(*WwF(cW{dM>Oz^8>?IR|5$nXLnPK48LVW7x@$)^c| z-t%^_<^`>-tt|!CdFBA@JtAHqfnsjWH|kEZ0BtY_Lt%3lkje*603yJ!w7KGk=gI7E z*N@S<`TF0`5bHL3OTGhgN|Xl_g61m!d_S;%5w)0w+T}11g#b%TY|vTIa}9EAL-V_L zYk7RORXzyi_Ba}ThI*N~Q3hz2OEtB?pXn~E?)&n9iiBdoJ-yr2LJr_#Fq%7_@GsAe z3nz1P%xk2C@L%j?z!iQhi1979Qh3M}?Y^vo@Y`(DwI!pmRGNJ#>mKe;nzK8|hMM&3 zJuzqKm^@<(ZHn9RMOEi~puH_t`U*RmYfpZ74+|VJqHvk6NPN*<350;z5st7e9gf@a zeQnl&IU9eA(GjF6M*yWjg=~ZD7m=;0@;le3(tm8RD1)7C(AEVlFhyyE6Qreaid@%P z>UUv0%UrMLgBBp=OP7c40*7)hKYtdIWSpm&tm>ciEb#dgMo!*hUA?+oJ#59Wd9p>6 z1OQE8y~|kqSNd@u>WRPiidlcgnXYzybh z$9i)C;9~oSxbZJm3nt*-cxY6H`)pTEEcfiaGo(0qbNjHgCn14+1O@!yQ~WG=e_ng( zC{gASae8-t9EX2Wx_s5rCz##~*CI_X575R-J2d7h{J`%ql^Ml}Z2N^tmsXgxsf+0`;|w zAK~}!r5T!<+ePV;ve!rIFLQ%Ln8?y(1^M!}n4QVw~fNqpdd>l8oh zthXmO38}sG#?t5SH%Y>kZ3%q&b+R-7(K*@1*{M&m$)*7dkpRJQGsaDhF^0Zmmid$l zyzmjInKMg(-uaRXww`o*i~Xw)P-R|ps9gkFhyXCiUkN^`RaFYR_qSsWkVnjK{F;lL z-~XlmffhVd!_2(SZ}(z$EBbBtYiWFs=L+kC5L|P!GYs-Yy9QdM{5Bg@r@C%G-|u@= zynFD%rQpGHVEa^ z@4>MoD#D($5}sWa$AjUGkQXnl%hp`je4C`c^n8+OzjDCqtAaHoaQZ8Y(tNv;iE3Id zP=gi^9>anUh^<6J*&$_BVq@=^FPs5CVrjFaJi|i$s!FBvcg&0Ap?X(3(Ui^wR}=fP z=5s8GBNc4#wsCKADd*p_%jEH4n*ry$Z0jmdW$Him79071<@p$d)V@^P16CFPhp@Md zs`8D$KH&gT0!m9tBaI-9bV+xElpLhHq`Nz$yFsNpr4cx!bW2G}=Un{$^UTci=9yP6 zbs?_%a9^>%dw+Jh#@7Kp9RC0H$o|_b+5LdAO$=aLpHNUCU@wS3_`&+$ULK7YZYMPQ z!=%5grEL6=;V)R*L*M(|eW9V=*QQ>S#@9UC=c3nZnJ8|oe``ijZfQ#{R6~k;f8!|J zF6cf&=EVA2Wu5;-HsPz!{nlXI!{gSlHhqEO8$Jr}J6Kf0?|H}`GI^w}7h3V=s8(E=YO-O3Yi>hU4AejVmd-6aOkj#Oso@)%Q-%1nEZJB(QA$9AQnJ% zDdf`oC=D!^%tkBcP-5RL&DGSb4aEIZ!p#mFGb94@5QwJ1uCGF-%4!(hgwgpP_VN8C zO|HA+est5XK7B?Zv*G!*@^95~4c@x+5$3Qw%AHwOtVJ7qt06*~aG~=|$3F%8(YZ5~ zjy?azELkJvacT9Ue^Pe}Dj6?pvf$x_68Zr@c{gCI;QqMcq5S_mS!xD9P*~Lo)?S=C zH$4{cH`u5kdd3J{LjwPN55%qUX9BDy9a#Y>pyRO9<=?09+bee$%>Et->x){+rhg7nLfs< z?>*|8LGPPLckiCzWZ#~>QBufyVGyrUoHd>;_&!D^3ca+kKLggxWx=Vd>{PoY$U2uz?Skr?*>pHA~|IODH@-anskS}i-4g(2$xCgLaL zL4w~W$Un*OW^+oT`MIfcM^FEl74oP)f6h4TN|wMUj92hnQg?I_(PS9ym|i!f=+B<7 zzCjpki|l=*lET&cF6`MoKQ!a2@984T&bK#}lC|-45Mc8YrvYzrPpoeWz9Sel_=8$k zZ{C;t5Y%6^3UUFpoW73_nzvql;;jM51bM3jtKSdbG5$6^Nt!W6uDE22$1W z9=ZKi;Wksc4+@TKxF-;n(1Ca)8MntwetmGmMfbk$#X!btUi;0CG@79d$8D^tABGM0 z6O?WrC{?DBx_+wZp-JJ%5k+b#{Oz=}>MKPi-j29eH_;`190`9+Kh^955(+J-Ce;)wYcuH)@nD#BDJDwA$nadQi-A>ggxv zHyVtm)T-$#MRUQ>-ytaRa3l8)Q>a@&F@74a#V*29A-cjx&N9B$(1x{4%Xu~xBV?F)9_nuhn5~W;- zj#SBO!7)0pNU{<5@j7m5h@3fiTvQmK>}R_^wBrEbX{7jdef+k_rRw#U5$eG>NDOefJ~ zmHKpgC6@b>+Zoc8CQ|z>hhZjSvO>C;=0{|oHyCP=| z#+R>jx|n=yhVki$Jcj+GnQoSWH=)TfVoGRUd?(FPWwokB#XeV>D3Qrx^apYqamPOe zB0X>S_nVsow%Z}(&EpJ?FP{?!mz(}(hmUx{);}vA(YbE*-{=3o-4y<}-$cFuFXw{C zbNSc-64YDlrP3MK!Hm?D;k`-nzjKWp`C>cbaZIX_M&oA_%X@<>@nDx!rAZp z<}ViYeAUYv4HeFAgtt6*3pw2`wC_Dc7IFcb8V3wdwO?iWv2`Qf2CNdG?beq=$Sd)@4qPUXvtv2LF9-#H5fG7oHW* zYVNJ0!USrr?r&9*6F?lY)SV~hVMBt-YBTZ|i+ApqsE!TwuDAZDRd-X4gAopM|0~Ot zCPPg0x=3V#wxo|$G(xnT+JHS}_v`Ls!e4IjKMR!Xml+0kH3vx4%DX?EXYH-DJolOysy&A9w%;1KM+9d|exj3f&BI#~0FQ@ZWHB zEp9j>9F1gMXTlBFOc2>#{Z@KaO7?w{4meDpd>Rt9Qu4qos%q-hQxSct;hSI|wjSzQ zduW!&mggY8k(D0SvTzW6McDOstXg}I?ng1#GE?*(-_{!#Nz(9$mu!VQ9Um4)O#ZHW zjFUd6DlztcvlX@AUQ8Y=hFE4nI`xYV#1>-yNs_kHWREGe5F_VAjO?rM+fpipFrGg4 zH;3z1yqRdNmsuAL($%=}Ni^BeoG}mt|8p;?q^I?byx-OqtC)=F?-td=t~g`;bvE+} zuf_0)f5m%*FLxs@t9%ZGr$X9HBgu0WM=G>2i6zos%-*7RqpA;w=umx@j58vMeE!9Qbnt6Q)!&?U(k*jf1XA)qh z#4uir97xgOe7I)@e$Ic}#Uh`}E~Xu~n?LqFKe)6#AOvE4C z$A|kXWoL8YtBMMy#qgscxJM2lg|aE#fRyUH zUMz%7_JX+C5VXt&foW-Q$sy@qm_scyk)}X_?O{WitKa$ME(|*9Yw2CO# z-dt!f7*8{OYglik4FwDjkH>Fq7dm=xRQPSSfS5pT*ZXI1{S4SDw%1pdg)qqkP@YnH z4JkM2dk*(E=LmOEZckkgl~`xb$@!e5&0O}T-h5v;umfLEex3`CDC0j&q#CdPq~|(p z4N2xrKNoX8vYkin90mbs48RA~{VR59d2_bQ=J-dBWaqwU1_n+%Yp)A)J;t8=mrks0 z`Yh4UL6~&)pzxu6J)F_*`}G*q5%BfdT>fe;Vy?^nt}(|l{q(8qj{XotBTCKASG?nX ztG>S@t+W0~rvmIaR6J`}WKYV-=j9T?!a>8bZ$Gmp^6C>GN|-rn^10&Y(|}D563Scm z?u;ouzg1ON><-*bbLzR{HVIK#K&;rr-tZ6U3Ml3)dQZIU{m=o-d79_$rx(SZZUBt# z=~>!Q>XXLr%3{NOS~?VVL*VhgB)d$fuIy&E-OF(4Jgo03Hfo=bT9J z*>Yzda6wDA@-G?yfAF*ZvCjpB^42FN+xupaPQgQ?ggn1~TvbxhkTybW*l=d-iE2>7 zjMlHrFL996VF|j`548g4D^`3iGb8i4{+5+joyI7b$sH{6AGIQM+?_`7y?^R8RP!l| z`Pl&&3FA$U=5~IeY%x|-{rYxbQU~v8V^H==PT88;RCkwcic-( z33~*Ucj62N*qE>yS?7$toU;bYS(cpoj3EiRQmZq)ajk$pb!MH~P6FLbZryLs$%MS0 zqC=p$2?R(2C2aEe+o)jRTf^tzYSu_4Au8h7JPRWA$-%*kbNN z#9vLH?VQb9sWZ=yr_(e+*yLf(n_byyA>$wsHp-D=;1xaaBlT1XFZM}MPq_a-h+VK`}3%V_k`%=|t4+7%GKMz!rbGN{$XnE#Z2D-uz_T)h=t z{XLo9W1ii#wp$zl1{j>Swh#Y}p}q_F(aNi=Zp6rTMsfei;i9@)ANQZZBy~{wri|SY zqTPYD)XMu{yx9|icUP2(?z)tOhC^$*cBz2C80X*NETRumBN3}LO34IyF zL-)DA3A#TUlmln~(ehMyx7O zzKV0ki+Hl^d~(KR+`&SFmyyVof%SNo`$EYbfc^0@BHt#GW(;5#1sQ{{q{rwrq6XSz zPvEWqV8%LssZb)bGycKD!trngV8?hF&sFW->I}%1M`k$! zRbTV{^=3jNE*9O1=qUG=59qNJ*-Fn^0`Uw+`ZNqa{UOHbM>~>mDaeG^aTjt#{!bZ2 zBqRrTBZ15ChYiHan~^-Nyeqy8dIE!hydUdq^oeSA@tTCAI6?}^jP%}b+Jn87bhmkr z?pFqCLPbs2>dvg(Nzj}UL6VALD%h=`^;X`j5uk7Ia8KRyP;PuK50`>0a@}}#8e3UA zdr8pHWCHDnhr}M}Bh{6tH%Zm?eDYC5w4PD97kKU(&VbEzxmaLIGlyPN(O$T z5D8_->vE}BDVxr5N<)=g*1CZn(?&BvFL014?u@P@ciyuUHUBh)FTKZ=-6Zlkm~Yobkq6%I`((4N>mD;w0A`&ufa z93@bPqb33FgMo;>2Kd%|tGtqNxo@H4Iib-;^s4j~YIa?L{_A)>?zsZFs}dqcKl~xQ z+0|dSd~ZtO5uTNd6^7)(9?zJB&c?-p`!&w=ChuSNDthS@kc3AETI7@{aEED~3ZLY$ zWmCR?D}*btpFGG9$Fi!dq($F`RqdqIS=kD7Ml`hawtVzG?mG9pE9E;a5-gTbrH6mn zkM#7^xWRb(ieA{^li?f3kA8VP?-cTnG&#N9H{+ym^)gn;)kR4s`KNId1#%Nv4VT$0 zeIBOBCR)lhd{_d4MLmt0AFSb7%G9e`4*8G>dQp0Ax5J`Ku?$+>elYFE-tEqHNw@~D zz;-e-ZnX8B)H0T%xR>df5LjlIFmWQt0TZ~4jl9|(zAlq;RPr90LCzYIr@mF$u&g3} zWTkUme97_opM!%#_bA`AUlLDT< z!x6&0U{uR)-EVTuxZ@Xqhy~+*%p6R55WL=Oy4V7gu^v)^9W+)d_SRKq z51wc_n-H)dohr+As*;GP37%LL&VAYVPvPx1W?L$qdb#Hhyf}Tx0<2l)14&emMYu%j z?cq-v_D4a8LU8~S>#{M`3`wi0mQts8C-&+nFi|H4sU8+*aEY5x0H5q*WDYGBI zpsex5wwQvk&#QUEe}ByFyvIok7kgReXASs?-v%hnS?FtaJXmCKzmr-KbCf<>{Y4 zb*JLn(jWh_E+&{^Gh-MB;L+F^X#YC4^ESX~?{@J9islB>$r2U)p#1*q4r-i+=#?u6 zC#BOVYy1)>-eBHj!T3eNgg=(ZG%} zo_OByvCBUeKiq8=2O9~Kluu1$*d@)P_mrC1mz^R0k|pBNmdG)Fd|tY(zWI$AIiDg6u3{%zI-0-JU!3p{7za< z;XbZ|Dy~e9vy^}?EEIOy;FFjvL~C>kPNQPj2g9i_%HJQgcTwF+zkLDc5GR82!Bg=E z@MBFd5ajT0hcq!TAvYRfM<1T~pj>8nj}qNYtcl}hwiQIFjbYxqr!Tk2I{s<&W)d{!{5jhx_UexFy&Un8Ot0Ydo8#s9V<<|o z9_D?byYnJd)-aps-6mvtzV6>~rmTUYb-iBi2Z+N;eERLqj&ehMGOf8{LiABPvCfKF zeSEfuQ{!!M~GD+wDagbJJth|*3j z=hHWN)E)=j^nFGtigO%~QfEF!BKFVFp0z~sSzq+K6yKD_ zWMjbIk!Lba`b9RKp>SX)x9L0fd@!>3nr z3$>hwEq#0*O&HLvO~N$%!W$gUu}scV>m7^Tl#WCGoQi!n^tc%(=-7La1_SSf;MB(XNr>%vRp`fDt{p@;Rx=w#&we3QEWix55 zB6U2%yv;edXlgWQCn27kNHP|^?}zc_R5tQE?;+M!deKMEi{ko%QRdP!`gIJN@K8N5 z-M)$Bwf)&|Sp4tjB?;+9{IYt*o#>zSABgdl=za@Pd0)#@RefOvled@a68xxahvOia zci;^pvZJ3vZ4BO{ur7>7w|XvQ`w*9Ng)$y*@`Kx2QE>s5@5^1lQ@SD=5$21J7O^4s zlN5r&d(;Q+XXfl_;0!w&DN~r6+fJqd$-oWMiwyLyTeRhfWEP+30n38l%`YVqoxwm|nqVTXFld_rdrP)0bk#_-`&jZH!2NG^r721brD6KQC(> z6Y&u)D1oA1=DZYSseCoG#KHsSr98|t00O~gfmU`EbO9mi$su!kn-dU2)O^T1|FU;V zq&4b8-Oe-G&vOdhgF8zj!d&Bn5*$>-7(vAj9<~SMP%_zoGfl|wDD2kPXZNi zKCVs;+zOu%bu>mlz?%!{?BHNdOuXJ76>0aT<#!;xMmQDvVwhU~*9Z&k{?}@$x({$~ zueDT)rODqdJ|p)gd}QV;d{uiKR;rrmtM5Oh{>`SR>DhH#oB^~Or%JFTsOdwguHpZq zhf&2QnFWvuo-KU2xofb=Bkw+L3V@DbWXMt@7xC0^*68^~8 zII*h(p|XmK$hCH+)KGgu>}R&~wWU{DX90oj&^X3bT%FuN=};mwQ7UYTJ>E1{-Xe__ev7R)Muf^RB@=&@V4;j8s6ja?Co}DP^1&ykcCtPo!OI< znp1c|OX!+LrZi1lMOJ>Z?~Dk1?t+Ln5W?5hT6$z$J5QaezKn|0wQ2d*{Bf6uKAvj$av zLDt2t&D6;hFX_X2um@rO2-uwC9IvAD-}lvw2L@_V$G(P%@kFnqx{4qsMP$9JeMX|7 zF=bh5kgINe&W*||ikIMS9(dzIBu1S_^%A8H-re2Zh#=;WD(u6X(T}8Ira)XlFzJte z=kz9OY$%Bl89JxWaR;{I>E3s1EW$83@$F4;)0~np@VKWx&zuU$LlVs-9LvmNHwm6+ zfg;hyWz)a+I9zexCasjk$jq1E&1( z`b;|3@~nXaBU(#ak(2bD47x9LggiRm`~pm+H6@0qe;4rd;H{|ep8<+PNCNYup6A() zat;!rWPh_7)==Ub6P`y-E}Aam#t2e+5u*Kwz^h=64ZPT8^EQGBx6FyVFpctQ*IH1+ zWf`2G01eVhz>7lK^@>3}{+lP;>9}Ixb*{b?sG_P0Um1>|b3B7_3+_7saXfD*_r`X- zjQ6HWn@?2WhC<>_0$fdNLq~VGsgL&Wd^$j#SeerDicsPPFiDmBU)rr;h|2K4{074j zblE)}OD!^$v4*_7+APf!MjUI7CdCM?w7dVz^Da-QQ88OLG7ea|;fjxbDtwwXSf$r| zzpBIGzWFPaFxGa+a1T#YR3}2?(@>eB!Dz+_?-?&JyL2M`ew>oXPI&YIH>$O^g@|=I z7oo)hI9kP-EG80mN~9I9+7;Q#&bXhqRROnw+VZ87|F33EJ>KE7x>yz3llE?rHKLI^ zz;LdbOHV5)`lw^RP``E{Xsx#Y)ja#L_J6ZPchp5sTfV#7mA-de4>ay;K@eOw9oJm3 z`?y%=lwtU};gN z{`j}@b@(e3x;;gqdo?#+?T38S6h7xYIGao~>f^Wanl;uiiQoAt3=#l2Q8;6p_G;hJ1c1L+aA>TSg5{6ArRy`l2n~1NNg-wNRFVEszfO2Gz;J zA-3Xbh(x=t$C_PTB#}w3;BIhU83>FK`8~`=M&R}7?av}NwZDkwKRhPEf3*Gm-RQ&? zhLy&14@4zKzQUWenQS{i{qK5G2xr28L{|Rb^k{#pqt$4an?}CAeYE2p3yu7#4(eP^ zg{-eq1JS#*Hv6oXbHPR$yq?Wz5pT4HBq4pGatCt(pbRl#Y|dBF3%%u(>>gZ#l%Ln- z&f}Qihbk=NJL4*<*6@G8pGXb|h~HC7C-GvN$2Oh;qP5Zj(H>Qa+*IGhI!1HWUb8g4 z^cYWAKM+{U4H#v67#^lBV!y)-@1#Vt!Iir#(vDF-2`j8`XO7BEFKrObA z)8~XmaM-jddhf-39AU-;od^c%x%77$2kIG$a!=}Tl-@WLA0R)$r>Gc;`QJWaHcSw) zGnB7Ogu1M^5V{kP?J2vgiS;7aZ84G&gG}R6D#TT;OE3z|1N%s(>WA@HW05qgQ!hZO z6K%4#jP88tXHQ4fc#UKWsgf6TswD14%J>J^fJ^VmqsC0pjY5U$xV!4^*Uvc@|P!?eC-;eUGC%Fj(>njI3@?!B18qo%zf!vFmxOe3zGr4oY-Ob*>6LKKOvJ z+s_FvkZi86RZv=V!UB>8a_x(WlKftBouggqfkJ&NXoYgKlf^roy`GZ$;_cwm&0k@s zpTQmK+*!>aCf^>5S>e=`Q?ls$L-ftCRuh2dYE8y-Pm?y=PPHrDN;qdPSC$$U>3@al zNn9#zN{=XR_Sw-a;?6U|#lxPtK}-!&1+%d|utD{{`an9m&9!=ngp8{7mWWrqwa; zFwJ`|P23-aBQ?H#%9VLLn;Hy+-q{N>F;dC?hjU*t*;$#1}463Qr0 zva^)^b(%w{E+SI+NzcRH4O_S?#ot)YNRKDCV)bVnUCE&j?u$1(e)&1{qJ-p0 z8zLf^$2u9mCt4Mbs1-udHx8!DQStM+-e3 z_)=miPRQ!a^n^gJlCTJS{F*83bE*G2y2TgzO->R`YJ|L4$y7Q9!|#|#4rt+*fhWqD zIebC7K6{c88gsZAddYnO>&9Z|;n)+#hf9Y!csxkG!D!(D)8@o<7kId~jP%JW3%HMR z`8&#KLPu`OBl)WrhiKteiC{pbE@G7Azi`JuZ>9N7DmTby3N73>&|V1WqVPudf=W3I ztuy?JXSbm8PSq}c_R%5R#KT3olg7FMNTMPq?PKq%5f+!VQdvLGM)C|XhO-9RE4h~_ z*&?}p0)eIG&pM@0#`>6q!-t^`-EDR3@OxaF0(!;Kb#tM@$j6?r%juAUo3 z)Nn_n8o3fr(JQ*T+$n|7(?E11TwE8Zfg+u9Ep_X|{}LjTL=w5F#)fOg-h*!|^nS17 zyifd0FZZ<+5qbD99&Yk3Uh$Ox+RsA!+}~0ner5i7_3OvgOvgH$D<@@ zOMzy>CyB0XODptxno6mRkr9MzqBKGCOjOZ*gpIF15E9f`HhvtH=G-HcaPqmkhmMFJ zw2b6R8|1P_JUVBYJk}@QwK(q=t*$d`C!(P-<5fOH_r23_p%+g&?GDOA)9K;X6CmPP zGZqY*7sYZsdOY*C+$0S?%)fN5UYU~GFjfib)6$9V>uiiwB%+H_jpQqvtiH?nJuzM* zwt`r@EY!Bqj}#+WgCA$t_*zBlef|bS4@%ccMj#?Z?UO56{pH%hs6F)EG?iXPQ1D$w z8ccN|EW#hRR+u?5;!gQ5Yg9m=89gq6K}5IK(vJeypN(+R?ti|T7r5s-D1VLO(er&h zlHdxKe^+~Nv{|=H`oi&8D(r$iuWY8Q{EvfPV!!;#O1}q_S z1@La|K4yP`hB_2jSYI2~@WF2@#JKmJZSnm(Do!+!-$%U5^!RI_x_v(;w3OxyN>?4d z906V@ZEe{te|&1IR_D)ryph8ni}63OVRj2cH`b}G>W)?gIjf3U=$aKkS{5K-PV4Y z1$m)6f#P1h!tKLcgQ8(JZrq3@PrR<}J%7R5kRgWO`_4wlme4RZhrOh zxE=}=~IM05aYgld=Y8|1XVSVcZLcX?a# z7GFhQej_%@!J|!s7?f?urm#$ZXul#6?AKW*8Ur=%O8M3T%pc=WX{1YRe*(=9Ftf_8 zE8L6y@Rte?!i&G0ku%@EvPm#IpzjsSV-kGQEu-Ma(=3(7$eE$XfBxPRSMu%Us*I_& z$oU%{SqgU8(;=FHi(~Nm31%QZe9(z<>y*LIp*T2&S}ngJ*}T?|nO4fgf~p?I{MgeT zjUJE1y}G&0KKJ{E-8`kF1iTz4$D6Dw5Ceu9Zq1_=>MGIw)+}m_d#T}EG#z#PJ{feZ zQmhEWxAv@g6HXPMFF%zP)u;vqGe$j}cNIvY>;inQ292I{B&|B_?B}Hv?KH`K= zVhc>>y%RJS@08u(sJ$yEP(N`eQXTGlRr{h}dWq7n*y@Yoy1q`N`4p`CCAQD4aWi~o!&*;3j zY0)m0$dFW?@&1>!b@em`3+oe+^!nQ7&W5--wa#Lag9rIjn~TG)%FQ3#G&^36*(6pU zv~9%o$My!}Ae=hA8=?FeS3~Tw-c53Oe!+mAnc_G4laB@8^UQ>#Jhm0$u zD$KdImqDRATmfFE2$IoBkj1eVmXP8LQQ3P(KJe5W0+_2=5?qunyHv;zr)+bVzUK=kf`<|IM044r7k46^MD$*O3{P2`1 zz%}5op63*ng>^e>oE217S;#t2VM=&)S0>$mlYr3@T5l+qASk}SF7UzKV-H6TeCTq^ z`>wo%rnRsAFB*$kYVQvzlE>%!9PhzUG1(AbV;7@w=|pA1AE*MLq5}h5VA1!kJ%Rnj zez?)2dlAeT#deO4WX4zy8&{7i$yArDmYVC1Gm`1Tz2FFJPy;~4p5zWWhGMM}IyC70 z>%DZFX4DdcXZv<3NI8`qz*kxdS1Ehzc}tRiHM{-WQCN3uV?Shj{Me^CxE^xp+9-|g zad~(E{^j(r5Vimck32SP({tcH`79i1zb2;iL$gZn)%XNeB9r#pRX*HHv^iwNsT1Ld zF5x2)$$l14-yCam>zmw^3z~BI-aI;{_4(u-?7;Gva(1q>eerLcS3BW%|AkUYG?uuCdpu*S7d{;7WQ>B_Ubs<^fm>YPh^aae=p>=S5&YnWvmv*1XbJ z&&T`Apklia%wMjeYZ8w~VJ-F2{bX@&Gds6vEWtq_%1@}Y-zMdM= zYqdZ!ZZG}$w{4X&Bvwaf`B%UzLi^DJLSO(5xi~n#^_?znQslb-73uJ- zrp6isThcx;6e}XgWs&d%Pq&6D{%5vj5`QhY6igIt)%f@AiY=351V}-!OkribBU1Q-#Lq9W{-hrEp3fSRkI)47s0h*;<<5}^W_N|U3SQI2G3f-Kf(3@oPm42C zuGO$ygIRdEkb+Krth%H!4~Zm!RvwMlfJ5J{7f%y$$vp10>Tp~RapklH5-N}DMilq6 zt2kt4O4M0pL-NS+D8%>FcQX!Z-y*|Z1y7uI#=HJaZu?LV?cn6fV1i-^#M({?;NUIjdWus8BT*nAVl#UBX*EF_2M5Ku<4g@ig|1fT-ZEypNhyY3UNl#3RWuL+si z?!bqFaMxfpN2zD`!E5~q1l^6iL7^M^BFsFh?+>()KlN{kQE6pj9lT^#T@H?1JbL(e z>~5L*Y0Lw4yw%eDrK;6nJeNG5PlCuRbRHy$s>6gXbnA%v2e>BHl|WT2%;1h@11z8V zCX(nnD-<7t1;S`o8w9$eU37B;FA5gKCwKG%9XVoA)Pd;C>Px=|+J9Y9Nt}*}U)`$? zLB|heKe0=V`KmH;*Xs1@5vupZ-8cSblsd;d+ zP#{xlC+#V)L(EA2UpBBp!gtwB=$?}pqw?C@oLwh;cM{!bz7~?-C69A>(7<7@+-^}# zH8Kz*)Vy}s@-J}Pc$fFH8ro;C+I{4CrNF}_bU5zPNV6FpX4==(Ir_RmDXGJbpp}*) z0;CqG;>YcIn??=8k8csAZ z4b$2Txe6IbhZLE+_`b@LVN~pU%2R+Lx3%YqbXPgo|!DG=W|BWM;zG zj9HZE8~#wa0||)Rql+AC4}qSYpV}9Nb=zuh!U`&QzY@iz4xf(ki6+>U47K9=>BmgD zJBgE6B%V1@;z}owZ>1H;EWvlPmxUU+N4X6#4P^4zLrT8QI#l-x;4!GddD!&Wn@TtU zXHz$Q_knF-9GZYx@|IbPC`um6F_C~9+xRFhwbCu{&>Mj{Cc`!WzYD#tXh0S;)Ngh= zxnOu;)`im`5k3w$%p9L40gNOA zj@oZoUZX@gn?|@IK+pwtmN=$PQSwY!g&@>bK$LFDW-U1i9Rrp2z8%Uj0296%_({{q zE)H)QbTw!|3o~5IQ6wS{Gwo`#-gzJ=Fn>8!Fuc&^`ZM7FGYV%Qh&6VJs&QVK> zezHHmM~IEwIDNWZ53?R{x>jDH6&$z_A40!k(p!%^Ei^Ee(9usl@1grD%!Hz4&{;x7=FnyeLOb@Kg3vV-i$S7J1->C%t==i_;RLcH;9mqqkb9|0UE| z7`^)yH7Twllj_j)nnz1&9U+d!x|&2Wf=1L?Q{P!#I^whGR{od3&qxZiqEwcJVLQ?z z1Z9?sjqExmFBSDBAVkb``?k_wsLD)q>;XlwkDU4oUY$f9Pxnaz^D#vik@k0t-a2qo z)~^sYvwU(aEC~uGNRns|;quP=FYYuZRnZOq$+I2?2{h_DOzGLLQOH|w^e({+-76jr z>3^{R;Fxi7d0*KoYkehWwxZY{q$C*SX$Fb&JbLH*#i-PTM*)ZR#`+sHe9@CE|R$)!rn`Hk<>Sn4D$MvYit4rzn!0viNRwT<%x9bV&79 zM)1-+4GtGp2Qq5_qiP4!fF_KtZ8GZ2HeYNsL{pyVNqz{BgndZfc=yuJb;Viq{^t5S zG!7lfbay;Y;v+dU@2`VKs_z>VC86<_hWhdQGYh3k?R3hXjIc)R5`f@$UCdpo_dfQ! zxRudE?+p7d2Sh2JW22AkeKh>+i$2yoUQTC^Fo-x&EJZ_TzndkiDP##)luVh1 z#a5iae!h@DoGRBG8+X?6Gu8DX&Xj&qjO{FKvA^!2Cba#dsR=AH_e!ZijlUA!G@VHl zH>$f! zH1fQ$HZ_B5ZXs?NPA<=Uuo^WQCJoD@C8fH}qgxLz?fw!VU!c&TO@F{gmh(Y~%S;GR^`G3yXgIMgaps?pUj6cJ=sd~wn* zF-uH}!AGD;qZ*E=K1qtJj^8OS_=)>Y9FuP?Vn%|D`_4Uu7 zIkzh_p#m{&(jxY4aDpUL$#@_b?1NR;-hU$?b@XJ+!{!yjp4P0?!JE%f@&L6+Y#wJj zp-dk_XZS-ZqF>R7;Ce#O%OFV+gU?=l^v1GqLo;7lQh>H%M^#IndW ztXt-W>xaYEhg1Rgw*tUwwkkdun&?eYXI^WIjk@Inl| z3I*4w@Q*)Fa(sW4;)2M|w}10bOI)=&?_2>HqDlHCfy>z2k9@OdZJ+Z9UmlI^XW~a9h2_l+os6ghkLZM-GmVS zqajzhIDKNp$KtG^&dZUC*h;K^$|Fvs5l$t9dc`%BwCl+)MBQV zj#Cd@UvFmzX@@7~gj2@U4!i|iOV%%(8yAYhJNEPo1`+GYOHUvdNn8OdA#G0_nZD|` zgA zmg9X@T@;eBVtn4#HDtXFIn7>9hc=+apHya{T*tmITkln97eBkIgjubJwk zwFc5D>X<+Vz5a%_z7WX*3JAokxC8yowB<8*GgE*5%Oi>X7QYDezgrB#pN?9n5~QCS zqk>g=Zg)~j7w)ZX7EXA0g)+H`FLwzwKC8JD9}vchfL**YDLmDTYNG-94S{tn9L5g~pILit*Y~(jCQNuUlqx+1e}W z$)-=gNT3IL%cBnuEfA@g(<|n$dx$AXsMH*O7YC|F>pw?gUwpYP#5q@QDuMl&=?+=2 zRSa11W<+IePlb2h{~SXu3c>M<;Igt?tMXr8(x!aTwnh(ThoDzw@{>4MZ?iL_aDM8d zE77xIc&ZP=Ey0qUr2|s5H6u83A2?sqIhZza;2(nfSy@o^c9y3?owv*aKh~?a)PD<* zpA){0G+AzS7xnQ`ocGtE_HP1)eAr#zlKr?fga(5+q z*1pBCa~yiz`@qj;s)@)x9ZE>P4=3WMjgnDBFT)gE$N|?GKkn)=ri~H4<@4u-G2Wxa z1=B2I3l@nF67er+Wa9F=j-}RaUoc3>B)$k24VA+arO16wqpU{sPBG2mOFzfm{&Ksj zxt3+7(GiExnDeyf#k^(f<-F&&y4kkFe*Oe7swZZ#Y3mW_IB7jO!>I95w+^qhb>AaA z>iC4KqP{ov>5w>c=u?4~#Jr^Xtx}Bi0%6pJM7V72q)5|fMQI<=>#h>HcMou=FBlF! zWg}&sF;Hq%WT7hE&Yr`p7wTcZf0&elUlDtS=rN*SaRh5r>6QAAsx=OA^njwC!m{Rv zAW-8V+nAnpKyGC{eOGrw0cZj4&N`o}g9U>_R64FQt3@J#y$KSp0{mZpjI>PXsbDr4yvxnm6{dq6T z^{{Es-1`?kM&(N8#1oc1@)mS3WIB2qpIPLVb9v-JUD58fq`}yQO6akl%8*MgX0cKp zSW+ZFwp7oTSCT5vGzhg9331;Y2u2aQh$=MLl+Q5;fg^3dW%J;Cn5os`b*bd*RTC5MQXUn0 ze^?|?4QJcW#yTz!?GNG7=qUiK{$+qw^UvL7FLwy_fybYnJg2scA6F!a9v;8k%j1Fr z$&%L&KZo{E$FW)<+ky(CG4RfJlYU0~gjyl??&k7YF*c|%OF+#z1@K({r+E@y)!r`7 z8Eq1O5=DJZtq}bmyMetb;JkSsbCFXp^>vzCCRdPpX1w=-7m+aWEJ#Ci6;mE9{XHpBl8iW?O>*_P0`Aaay#Jg86bC4}aOrQae&YZ+JdNc9H^iG_bI z1CMd2?(2DMhM&nF2llg&6y=kT)_zE2_TKbW={Lj&MdM_w1J@EuFX`>SD&>uqPVKF0 z7x?JT6e`9c5fC49oB-eGG*Bj&jBnpvu8KsL-@k06aSJ%Kw>jqkSj-Ic!ZB6(w60j8{s#8mSD?E;ly9>z2v}eZ=gvg z=$sqKuu&K{6UPc;TdHF^ro@2$IaI+HHEQ3wbQf)!lq@Bqy*xFvl2Pk_h0oRx*$@|Y zF51u+M9~b@`|z{CUDlfsQ}?^AxpUBjJqMJ!tn>UyNhS9dPCjhRr&tr=9K0+ zvE7pR^7Uo!bSrEFa)~^wKD9*kF{!&2guB9kzSG4~jb5NgnVmT!#)N_4m;cCPVP8Zi zjs5o$z0b}!ukh&ueihl~ag%QHJ9Jpkjb{e&p~w*`?@x7V|uAWk-5k34#W!{LtA3TXiaF{}y!}m^`{hVbH_9^!f^>UcCI>9iF{0h_*Ngi*+P{e%wNZT17h7Vd_-oSx3&GIMf@hR`wQcak5uF|ZSZJ{?Wr_W80s`<`O81; z;6TQye(jxsD1TNI#N_d^nv<6-`^lTFo4kg$gc#+nOJ{dRn%}voQ&pxNh_8C-GFHQc zi;Z^g3fAcHOJF+z!2FUB1Qjy%T(F)gRv}-f+7%aY?=}m|81n;Y>JF0Bo`1qo%qPg% zQa~U}Mp%ul{6rFdfy*M9ey6OHr#vF^-OI{DRu4S9@QZj`(QOad%pDjXCHSx@x1k@O zJhJV-qmFw?az74#yISWn!J#sPAkb;zrTbe}ZP+nH%z+zt|JU}tbU>SH4YzR{eE*Yj zn(+Jem`JTBKi}i}OlFtrvUct=Ui3|#dr zNk3OY+QF*3I^Nwbv%olSMKL*qW|%sfT7`YSpH|Lt9Qe2RyGwt(BJbyKgqe~H<;D~p z=bG(=>)%M6=%yLjxLv?U z1dEqNR}7hm-SCIfc=pe|%9w;+z%NKt}x_Q^EcqFFe z_YgCth1AE^yL~tW-`@-EdzDvQTn&H%8+kn2cG=5XTfYt>a{-sQILfiS4Z}YALZ2~R zS(6SL+iQfL&fM#b0b&reLQ|osco#6XFJS%G*w+q2>W^OfRjIB49bY~iGPJO_5w~e* zs!uJ-4rQzjAnbL>Sw&;|%zZa5Fk*FAiSyle1Oj7Ck#s29 z=;_%c4s)ls0rLPV=RQu_GAPvNVA7Eiwrdc@+&8QUlsT(I0@&}^|9SMl;H6@LEB+j& z`mS|r=y~N$=sOd z|8vYDXjtk8c5meha!taSrysX+n2NBYmEWFQ2L6bQtrj}ryZ>iF$Gi9qW>HgDC!2#` zJ}Y9$ttjGbUf{!s@R5b6{&@K?p5p>U6$T9CR0yT=^F+nK7EIc%>JtBwV^7L5)Bi=b1#MA#}_>x zk#ZhpUEuNe;J4LIALdKH@Y{?w%sR#WG!nF}daz(gKmI_Jy_k;Obr4My z(_2X)DXQhoIh)jUe78A79cQ;QG4Q}jp3OO2Jb;26P!q3+rHP*$g4 z4;vrc_y%q`dOI@dJy; zy)U`V=v2IBY-;L)d@F0Bg6?BnXku`V6v{;A7zEHZ?o^{Mu29)M4j0pOYD~CUJDCQm zXF1*?vuTo&kx|)qfgIW*+gu;k1Ap1Z{Uxz)#e9$CSN#xD)`0>S#tK6vb;?{C$I(Sc z%3;!v5DBLLJzh$_gZ(O)0sz3qlrSC&1{@y&TP`XiMAtDMdGYgW6_DoCuKUXE0si{K z`1p8JmAR@;Gw#So%5q^mJ)h-7GO{6pvO}QL$QJiJV3)R;#9^{Fn>m}5F)jNpMdcm( zR1-Udlym6?`IlGhk7f=$FA6nk15}_dSlU`W597;X7sd^yU3ojy{s{afDsC3g=nW?i zCv=1Uuto$7k62iw>++%0>^JBlR($2bSrl4J9<4P-m9BvPNpGFF+Hh7cjT`5(ir;`c zfx#)!qq4Zz8eXmRx#Ox$HBBcTwO@Yia@F|FijQI+$uR$h+07qM{!80aCp}p=D<#0a z?S*t~yrX{kA=7uD5L$a|hlSD`X}!zo2oAL#ND&i?ki8Yfi;lO%aNDtOoQNrGN0H^7ZI-;KN)t~q?kcl-CYR~c~;d_);(K5CE;mC`#F;a|Dr=0 zV|I&i8iIUB-ueWEF{LCl72w+mDD2zc7JVt40Ujci1Zt5Iup2LdAXXv0m9! zWaV1MtE<5ZX{fWlbre2rij8&CPjl_n(BIp7Qke6D&F7l#X2cA^b*dGa`&lrZs@S2> zH|V54y(xxt!!Ym)f!3VL@*lop3bZ|gMTP~wMdi%H*#scr@Nl_(4vKAGi18sbr6xZ4 zUr6k9KY+Z7gnARCYz)MZpl+~P2HPKy zV?8#W4fI|%Zy*EwLosvS-0WZAuUmI$F)@^yJHcv9q%0Fx>#bER6O&GQfns|1 zndAg_0ij-po=l0;l>7reLCJgcXSy3_8h!68*C4Gp5pb7G4DFxT>YcKms}A_I4Ct5Z zY*7E;eT_=n=wJ?O;pV1TV-`6)NPz$% zdiX}DbyBq z-|}#!B%PXwL2MlFeh^^o*tT0}v7TdMybsbR2B;j%k0*dRLU-TbGz%^ljMc$f{mU{* zij3HYJnm4)ekLCSj|!6E<53Q7s>t^^j|{DU@p68B0-ShKas^-WV3v*Ios$RL0NGsA=@qPZqc+E}!E- z1doTW_(L8Kbmj(mJ@gziV||_Yamh&<((Lb(O-MGJFR}{fU3{6j24y|QKtR5i{uX%v zsRw~8BpqaRN$Yx?x}YQ-M_ZXU#ryq&MzAFH`jVFg5wLc#zc7HYK{GP?UBQ zovnSs0=c_u4h#!tey)ir_x5rpZ*;oFl|_EHG&wfz&RwwH?;de%dAmbMZj>ws z0wWc@mW$`tZ)G4#NuIC<8_%{QaXNR!QeBs7I<%H3pJd1x;`Z-ZG4|(N$9$w2K;hZ{ zR)8u5#9>@p3oa0{+*3GE3u6lD23Z$_KIPlLIt^;*{qxt?cbxk}oY4J38P2g9?uLNJE^WO|vnj|)YPIEij@ z&q_ckHKZ?Qw2EhysQIJnH#%9{MT#Tb%!VOlE6XRl*|6&Fhx?l}k0|>K?&c*6|IJpw zEDE>r-*Aq$qLgrs7GK16e1w(MlkH`!%OP^?T8|a#$f&nh zzglj!;)SgJ_4R6uKxGaIVJ;VP?;~%O7c+5hy;ib7|1y!e{wVvz=*{(Bj4K|=$$Q7u zqr%ldElaM*yFO}D`;8qd^c97J6rwq-%`}kXN-8hh75+w);reux=HyQ39tS}rl1L(e zf)(HMoNXa|hlY4V_41=B zT$bjYPpTp`-QA|0Kg2ojLM2W8T`@ck4P+gD>~?$kY?AIfJ8I?$e}enlzQk4hDS;Rx z-tXtB!}y4x?(|KAFkrUpd2LotMnd24(1GOc%a7SkHuje9VrsP6=zt=3vX@^J~d#rdEp)QYPu-K zX_>S;d(m3`#Y zXT5cz=MwNc(X^qdw`(cYc6<-t%R@ffeIKGY>-mm#bmB7H-UAofiGH&}Ir%&MWof;S zmxcxrn3YYWROkztd@d4robnZm?E~~R)}uwsPkmrS3EOIc*tg@5aS`&Ym=to>bO>;r z@=%qv&84^o@^+fp(e1uzv)j|W2vths5Ej4pzAH@H&=HNTvdBA_Iy<>v#|`T-+%7SJ z-@j!ej_klWNbI;Zf`-e_cB$t1q4vLHS9l`lz=3YUY(LE$MkNftG;HboAgXb1Y-IR~ zD(8ckR+9S7AIiS7XsS)Ch>=!HiPj{VbRC%n8(KS2OjaHg?=Q=~4$qN%bYBfYW+!RH z&s0rJRF^8h2&tfhe*}8SKyo} z2ClDpl1Edfl{c~qX|?QcD`cQItARJ;g_@$&4dAkMgc)l|MpX&7ghdjkkHIgJ2eD#2Is^|WMhPuG&j@Mwn{TYtVZ8RBO>_%g6Mc^1hh}-e8A92M3&?%;WMu1Tb zd91E!%8Oq3`(a+bv~8MK2EcH6YjA}q(7q%f*(*MNk1cs~&``YT6aTcz*K2e6pHXRl zrBecrQXu>ve#x@o@GaA`#2>V1KfzYxYPar^z#9(gAJa0Y<>BSP(#6j01cC=vevm7a zbZO5#AtQ5DI%kHqs^Il=caioziu3*Jfbu2OuvxRgb=9xDG8%h@d8jWYelNZgP$N1` zdG6Rt>V1(AhFvpZ5-=-=kI8FP&KLTkZbJOc@V6YbMfb#}%3_a6(Mo}Z(UI=;=Oiv> zOGNBEo?00^EEbkE&ZZcY2ZzlK<*u7E2mP*?xtu)^Zw6Sa+p%j=rZ`wRbYU-;=pRo$ zdbZY2kh4Ku+4eRwj>hYQw@br+7itgW8>Q|1-rePB^0#)ZukVF$_$`!MSxk#={iy+M zhjKlE>C5Rz&_n0bj_#T4b#7ih1B~>)9mZ_m;f1Vy ze6VuRz}?3QEg6$t31?Iht$t+vw=9)uJlW2QdFk+HGc^uSQm1|^QOlu%*RN z0_P|GFedWg;9w(IKkv0FqwI`jS+diPLZEMdk*iS!v52WJ0u6M`VzVnd)U3Q+spFlcb zumOvXv8n?@m|<9<@S_neYm!i0L3|Y>G&i?j@0c1!-pD-Aw7_y&*wJbzqmwH5Gt(yj z@zaPPWiZV6IX}wmwqwU@@cqW`Xt})k#h-s935bJFU&6?kh?|}-WRm#!CFMwi2P$|& z7LEXqtgJUXa!mX)_KVBJk5tbrKWqliWT9_s$lilC%%a-GPVVA62M*$9sC^xJ2$n)h zWuNzAgjVdVxtG+-H5qXyfAJj&Ss{JMR8teztLJ~KT>b0cKfx(kL*#O#;I=Ydw~T}|BNJv z-SNud@+d3vx07~zv$Ge8GLvHnkdf67+;n^zO|a#gx`&^NQQI-mn-clk=Ih z2ZJai_;jAXChvQXi`m{C#vilM?iOhOM%pJtua?E*A_*q(3^ce#P5fTJnXwHc%atXn zjRd(BJ~Q3j{yE~)6@628rFr|S=5M!Y?b_TY>b_??W4Kx~3*<>IJSSmg)>&qHEm*1| ziRS%_tXF6OjDMeyMl-?a1RbE3!((HG0ZDXH15PPD1YrQqDH3}PjuEWt6fl|=TBl02 zgTH`rY%RKsBE=xbIe{{twS_7Y+ZH&7aZju4UAdM7;3ovw0dLYm)qr$Q`>Q;D9&p&E z1lyMogd2shE5P4stN?Dpmb{(lERt&}a0bnQ#`5VM)sg%9_9Kqqf2d7S{o>a~WiMz& zU3%a*a*~*JCN>;}pSYYv%`3Xl=MG$moFn9Nhy$<`-kg5|Yb2tPfZLNVpN4bK%+Nv^ zq3&3T3iGvQLtQsV5`tezc(|h-E5=b^KN~_Fahkd+tJ`7?8ecJZ{?3&HSco?hVmsf! zaaIjKYQSb^p9g1N!W#f9p;EB!*jf7vB&Y~sZ8X-kijz^Wf$f`U*D6)B$gp1%)^=tr zbQGjnben$87GY#kzeeO1c2BNxVbZVE0tj(Y**Q<7JDiL_@z@t(g1Tuy3rvP!y0Sx5 zons%*IUbGrF)`SFCB_E^f=*thnd<44Eg>aWxhH}y$s`u*TJnfN5QEr-c~U{pxFxUGHojHrwLh6Km~BX84@%O772cinh*@ z$fS11D6J{xNO+7qva$|Xl?60}=2S9n?4^TfcgOf(O0=tT5@cUkbsA+fjSZf?H-|{M zjghtG`oHBf^6VL*H^2$>2fIUWSKLOqk=J`sWYLgY6>WICZw2(%``!#h6GGQ#JJBZ4h-v!r zeCPp#9ik<7KineJBzv%e)|TqM)eS?c-T}2)VGPmjqvA!3Y?`eQA3(ef3&@;iVN`1np;dVypth%%&%F`rSW+O%7G!>yoaZ@HOig1wix z0<0v*&|~JRUS{%z{5e|odqPdvcgz!(?dQ!MU6K=cy@$572p~%u8X8kmg=@=5*y}d9 z)Ff7(U@|rm+GGwD@;Y1;g5nQ>$hrf*mAQXb2hmlKS`^G@EL*r??zt518fVXS^X(}C zk)%*2f5#WT#6QBxo&}l-U7r@icWCp1zThLzcJ#-JVMd>?OcEn>1}!JnjRT9i3x4?m zXD9~qmjI@01ct&4n%)WilokDVEt@d{d+nAL)d*tK4f8eQ2lla*wm!`izMq63NMya`y)(afgf^QGe zVa=(yJkU};_{(-4u;sn*G47|%z_3bzBDYCLaKui^%6)P+z3ZEulxg3cGxeDop&!l? zyvS^HAZM>%&wEx+C-I~DtxgW9cX2JV5p;&exvgBm`y19lzjnp^zy0>G6T!WSN=u|` z1Vsq5;#%W(`R-NO?MC?y=<|XX?e@H;M_`SLi1q#O)7(t%j1x82`es9)IPXaSJ{JGr;1h2gJkuBy2b}w~?JNuFtz} zbj*<+iAcNT#m?hVNBZMmBon4Bjp>x|c6lLoP(NrYQqiKB2r@tMPYvdTXUXAFVOu*E zFdTz)B&8~YxUQsgft$&odnTOE%(x>vU=ftEjUHj!fc>#QiXdYfFI0<}nx-hr70N`2 zhMd;+UKA=cav$p{4L*yF0NWd#9e=s9JN2=6uLi_#m63R_ zzm{#1!#?UA&y3Z3w6`4xC|a0sOoT(ctE~JVk>r_z9_mCE4mzFqZ20CYLPvnbqQF;0 zcD)mk@_81$eY4n3b5ARdGWFi;2xvYIh#zX|8Ba3Z**Qsuu*RnN5XqE~GAS~&6GKxv(ElWlB0@~r)alKzTU3QOVGTRwhmy)IEQWT-f#)ITq% zI`D|e^XAVFZeNUax^BW|y`L^l3+$g{ZN|m_RTo@n*B6|m)T<~rH7?!?S|wKBPrg*H zYRAkHSK~1yQ$aqs$nPHNHBY=W^)H~yh|DGVqWT8yv$I!BokE??A^mHLC(2!)vv&_S zum7ut331RkbQ%HPHjSu*ado=Z2Y_KmG$XJjk1iB3s3%{aulW77!29u_YW2nMPmA4X z`T%vimCfS75?fU09`);$gA%9ZSad|f^NzO{NNOxkWgpPh(nI@k$r}VH)<))V`OyZ5 z%gV~D^7CaqIEs${9R9>9zXUUAd=mJ6>eRS&L1~&tyj$wm==U4 zM%crAfDrzsXiCoLi^q3v`xhklyvMnIM+G1RPUYhgC>p_gn95WOnB)cZE?V<*D0lV1 zuBbUT04t9ey?nI@Zrrjlx&Y4Un{A*w%pe|z%^x_3!X+%jdTDQ zFjjbM?H8u7pcw#ZoReWM+64AgSOw$FSGGLmyhFduBeLcpEpj#el- zfNCoeyTePbtebywcdjPyiTJPst%PT4?q<&w0}?NWzzy&n^`Azr=U_r2=FcH+A4eEz zCCX3ZtVOqJ>Td|$$sL*TGUBngY9BMC!8NU~)~F%`yBn_GS3tJrU(r1Fp`y-|)46Q^ zH=2$}u$S(w@H<@V4I61KQOAf40~@o*u7`WS!G)l(cYg!F3v<0x>l?pXY}zNd5OP=q zakh61;ur9X4HzSd_lc3m;aU6Kz-24;N7Ire&Yw3Tf((hv7LKNT46p!^kBbYbKTW!b zm>D?eY=A}s_fzk(C;K1Z9ofzuDpB*mR;S*-k$v}sMg8t1N`*_l;-pOfyV7iB>2^xI zWzKOT>&aM}tBy4g+a{;XNu*I-43Hd8Y$prb&DU~)d4*8S&NPNQlqnesfmfB$(T`LB zCakZbjy@H>3J;pjCpWB%ev9?5x>7%6;}#oxS-Aq8*yd)p76nk^Wvhr@>uwq~GU= z;&8<2p^g=@sEoukaR&xgR4ZsAC4|;GXfYlvn%&eInw?-gR#^Y)u2U z3ZK!fpJd=ONt4r|l>cpFm&EN$#2b8;`{HEX(6) zDXYqbjdS#;jOr}rnt=yE_uWs0(M@KHB(0qaWg7l^8ikKOKDrO#&>eyO#(CCBR1wWV z`n*N>H7GjgUvvP$HGAjzw8}IqJ+D73)NzoE4scp0gp_H4reE@sC7M|vOx#T^b=DzY zd7(;QQ2(USewM@X`yfy?742w;ICoxu)n zct(cE_U=$0Gb2uiSMstnz{oE=w$w^`L8<$`M`2B9&WP#a zNpKzW3|igPW22`(t*FgGjNtR_PKKFV{?8Z9V(#07Muj)oT-8A|3$BbC1^a&9HR1Lx zJD7M>%n6Bqhd?3kG)CRW!05Enk(K}G)1sp{*a(uWI6Eptp=YC8NiWDO@zy<={eU`0 zlr7UQRMhW}ZRMN8awdUdA_Jzytglxv>HW>TJ3@3bVIX>b@)DF#5yVn<8)dm#(7#a< zHlHh|=8sDknd7{OX8Qa2y)oVf?8ws9Clpus;hofd6*fW<*|2=akeU~!51gpIi|P0~ zb2VOZc%=L3KdNcP_vA|tkdaUN9)?oK`53pDsM$GaF(jM1?-apcN%F$O|=aY_i|)iuW^H5*<$<41QyeIhOm$G+0&p z!q`ar%RcFJT=s%&S2|k-oP<6xjS($v4|eMd1&3v0wn%e)4gQ9}5LyzvZy(s3&ME1UB6^gLVN z1AcxuMqdpjDuY(>F=ih!WE8f42LuUWWT;!TEkEF`-jVgS_65SPwuuyw{yw+SEBg(` zFvB=`(8b_2NYe)FQPP-6kbX{B#J?hXnqBW(7uYG`i5$SRF*Jzxl)9Dq-?oW@bw6STP8zdc27-kZRFsYVb>m+ZR^kibOZKnw)!9#UsAD<(~@2*12Ogi059T+f0hfF zVs*Sidh7%CHFDW&cexNMF=VFT{BlOCE)b0|1Cprvu3)zFW(!5nN&5VyW^obEv(}fV zz~FYXGqRPE?;7l&)7)M3>E1@lMbM%-EQmlPzpSyKWd{?`h~{&nAmkh6idzVHsp?-c zUFELTjQoBK(F^V%5L$uZo|%so=XX)Z4q7JCzvvWIdj;}9KdutO+C6vkj3-5((tG{< z{Edu4zOvP$&d4v(=e!<^Kz>#M;UOW_T+o6;4L2PBvT9Cjk{%8oHOo^Rc%w3}odmdKl@5ZYgj&D^k9^1?& zId<7r3x5!Yqk-7K$J=2n)4Se@9e19ez0(J_v!fb`X@((1{3z>fWn`n(8xqJQ%9@`> zA3bV`$XCSz(jYE{{Ltp37`FOw-y$1A3Q3&AUa-iHmo;Gcx5`#s!rP3slsf;)=H9Pd zkl1D(cHjP9oFxYF0942`kGphJV=oYzmTK-dhb$I&O$y$LJVbrzETPTXj6W7$ppjSV zB`3y#fjHy(T|LYt8oNv_NB)GiRk?_lC^V$_{CABS$K#W`>Z$G^GLQX7{`g4(x>eB~ z+-;txt>^Nx4;S2>wUE{(JtGu-)HoRdY9aenv-9p}VVDHM%9dbDEcN0lBF>l4G2czh zbwgpVj{)vb7aZPl{~3tpi&cytTIwy6nQL7t8TEaq%Z3?p&kF7%{yA>#h|6w+ zapMN;Rff%KEsRDEEgnA}Bd3v=rgYt&s#IuvjBJd5h1Pe+F)S%C2sS~V zDdRd}XiAbT(oq6@)tpQ4&FhcGyG-oZIGc0&B51%6s@H?2LfSl&O)Iv!a zkA1!niM=7T<0Iqb0g#=*s?c4DHW??cP$&CS zI`+XkH^k~3>{sanJU{ZT8RJir3=v&L;c3tZoWDIaj(zgB%+Gf-xDFRm!hx=!z67)) zzJW^TB2&j@+Lh_`v{2p6yykU-Vd2PvAmC2?x|>Jg{Ph6LyvXQ(IZ$q@O=e;RQ0-q) zIBd}8;=oq5^#S}o!PW4ui*5v*(ygT!@&Qn%BB=z0?T2y0hXCE1`b_OSghRW6V$(DJ zo!8Nl-4X(Nu%REZ6V|=8y-js;ht*DTiCAceQWM`}eqes?8sEx%)D*|lrYt^(^0+DR z?t18~9Sk9dGRdiXJr5VH4cs-;xvF^HQ&cwk;BL1O(HQttao9%ce>}5d3_)I2hJhh0L{!4Gq zUI}C245Sn4>{!sMzDjmK$*qE|o9eD*K1EhxAN#mkUOPR0!hdP4qJ}2?`t+_Au^Rhc z=qa0Ij!%j7(T`6~1@J>t`P(F83nRP+FqHn1kgcHaK|Ro|`JU=gfIym&#I3Mw48p@2 z@y<`?n^oFzDPJzyU$v%7IZ_^n`A^L@RQn$+Vjl~YDLg&gzQ1WG+WF2x7#AROUuM!s zQe$R6#pr=0lsL;-oFQH*&3XJH(VCXZ)~6=rRUHG=r+X6>H4T$tzYOeo%`A+s?<%_*qVIBEc1H3H|zn_ z)&Hbo3Yf^rxswvSPhB|AsrACrx;!d-II-yM#Nnmi-LEp(628a! z`|hKn+-kk~({?@g1|0QF&vjanuoZtnLw_$G;@_|Lz4*J*-W=$0DfhHKs>Y1?7;#3R zBn$2N*aB%so}rY}gtxj4pd>b@kU|*rRCT#oeJVl}bG%jqbeER8U&N3=D?kenAM&A+ z={WRP9J(1NZjm;jLMO1-B$BDd^PrZJD|G~ zAV_IL$nV-fC-N44_UgStKWfnyAS3*Gbr*Vf>}d0D5S1Tq)3#$3ZeKv572=o}v`)26 z>us7>akfC~R6?`rhV~BvZYfw~UyUa8dNLr~R$@B`dSIDYEa>{^{5J+IQFLjL)l`1Mn<|XLJI3$SdGP~Q$mUNYSy}s+?NR?3<{8Qu&d1mD zyI3htgekr{cAG_;chFa0GO5$eLlaJ9y6DTQz_~u8q*Z1(ihIV4c&QHZzMgE;prfPL z?>_4%B+#=Itl2S<^3}ZU>GWLonDfCrK`_fhDMA2z_ZuR+VPg6n7qNDTNTpi9Nawci z2qKW$I@g$~peEOF3=i8RNGw0GmDV|bY#(bl5x>mt3K@zhOZ6n6gY!!N&471xy!7T6 zd35x6wj$7SNEEktrHU)S^v|S3TH?;6*W05e7cem|5YBsl)k0hRORrn_*ll}TV`xDL zwO@?+2qHUmvgi;Op%>P{v0LM{@^v)I?Tu(EQ8XUP+YUilsLv)|)I3GjO18a)dqkmD z?C2JmgPYpAMm@Z0D@xi#;?gMlYU16y^FEp>xioj{e`}&(B9NnawA`Rd;_Y<+xszh6 z49XG^Uiaa92I(M;YYMhhFw$Rw^~v)7-6GF8ArL1FC06=mun>{|G!lAT(H#Xvc>G@4 z_~~3n7&2^5z*-PI?pg#1zh}{ZFWHG|8gMb2yj&I*O&Q-2ht5>?%h-3XIMN~u*Erd4 z+@RTMxe-TKrySwCk7Z4jEV356@B%f43|qoQosyu^u$c#>OSv>;bu&IcRmW6MH$2Dze2YtEmml|gyH0@%$LlXy?{ln9jBqO0V zxVdO1{wIq4v`tz5#8TWYp$7Dj4jcY6Cr*ePDB}f&F>Hsj;S0rA5U6={_`sQQV{OT+kQ|FYruU>$A_w6xFsN6 zq$;YZN{BQ?o4qJ$pVubOKiBhtiNc3`Uvl^cgW?yY zT7#jgNqDk#$nyW61@IbbQ4)(sEB>TNI^vH-KSxKwz{uJm%8SptH#!u;oWWIiv+HJ& zKF(#?S#qrSN-Gh4;*WgRr_j#);u&u7Onl(k1b3UKt{V)sJ`I5dAZxD2q1s-N4=95` zftn-}!XYWlvHrEar_*Io;@9Hu$|5Jucj|w=?sYx%Q>1T$EY_zz!$0=K;U2Z~YDs1n z>r;m8enfQIss%jD@pJnCq9p=u3Rd1#Y%kR&mOXJ14A5D=wi5h{$E3|4=hc4whr579 zn@qzjw6H13$0|L(IOj zY6c^TgFano0BB%?85Z0Y4uK3_IhebLrzt};vmW6ESU7x>M3kT%5!48IZ_ z_Sitm@&T~c64r8idp4O6^oj|b^v-CtSeO9*E0I_mSCXhf0+6LLQGn=Rn_P&PY?uN* z=5a+xs)~JHrnUV;0*m<^U!GC6L{78K7HX>=H4uvo%z!Ml ze`=?I@aqtSnHXihu@>}1p>aEaogWStavY!lEu);gJ&1W$w$pq}yeR4S$MJ+8jO(&6 zUtAVYr8b}759kqCu{BV?mFE^o^@d42Yd@wNxDhiV^(gu$#!RTAe01omv#-7CZ< zH!qN?Y|IMxIR!D7`H1cZlNPybyYrv0ZdzFn`ta9|DSs%S|OA{rSS}qBp`3 z@7PjqdIIx=7uBV5PM!7-Nz&~x#nDYfC#QX|-H!y%JWQl7N9Xz507(z&{ks4Y?Ha zXTpPj?oO%F)SX-CWoCWfBU`2V@aA%v#bUVvHs%#K_`X%kyBn)J`KQxpzL>jP-=Zi* z;NuVf{_%fhOfu8AvbTZnY9wdtU>UyH0bH8V-EQu$H~bXv0kX$`XSwe=7DmqVEUjO zB8y4Dr_<6d8aQg6Zg9%XxPv{qwNW$8x-Jfy)nvEw&1aR4;G#3S%4u+23d&4lX z5U-QZk5>lt9%WPI6H?;{2wZP1*yhn94S-thDRAjo$hM>*$Ap5S2RdP~6#hHp<*QtN zl4?xljkgpl)~o(!2?Yc3((yqp+q*UQ2%+;m*jMMbGc0M}gI%Id2oK&nR1-ePsELjPHCc`UVxDdfRV=*Anq!NiyC2 z^uABh6S!X^_XW1Mym@?bwXUcqhB96i1Uq9=jD}_Y&Wbwv**rO)jFiGty$r-&j^{Vn z;zI_RqzXa37}|&%JoEXg5I?*JS>Yi+Qgg?>Fj=_}L;pFgn)+r3TXM_urCKDhtyGkg z1jXDUCA@5C)XrCRw=y#1xvtJe#NrPH#jd%^rD*!v}6MV!^_Iy$X}&A4(C&zJcZWSWU2B=d^~UM5hJ$KZ|Zv-Fm0HAKv-+JrWTSAKyMVWLQQwhW5L zBd%CJq)%pNFl~R{juu)^{o*6XC_?8u{8+<~h7Po;VDp7KdJOwF2>cZsYHT z*Z+!(960k3k$w59iH6L2TrwQ-%jSPy=l}kQiR=Nfny+@h6(G;{Mbc(*o&-nJ^gsqE zk;~XE!{~4;?vFq84*i=%{`>D`iJA?X7+DB`c>Ve2m#8BTY)T;q9cJZyw+Q@$L3XY& zmCAAS7cTvhuCHcb9Pj^Bt5=L+Bh466;IOe+W%+z7Lro6qHmxqxs=WJL&%(If2EyC4AYaDN@lr4S(L~n&y+tMv z)em~AUQ0xOZ9cvt^w#L?lW)Fa)lhZc8R@R`5n`y2k$BUi$Ccf^rAWFpGa}i+`=)%> z(+Tf?O5J~N+yB0l0v57=RUpXGMsWi=!c(x&YOMt@0~!34UC@hq7F-mWhl8T2 zj~54Z8Xq;^S>^JPhj8{m!ee?uP_NNE_RGE`Dq}Ux$vn(-=bf`0FSroScdxz#e2OyN z8jGi5tr{O0sh*UDpLy#Jo1Z#~l1Fa-{}~Zp2#MOaB*j7^a1Km1D-blMW2M6A zVy0u1!vyMnHp7xjZANi{b)VA=iP2UtAE?lh;m#MkfZuJ97`Z>QcpAC;JP?Px4Oe99 z?XR48yS2iz4AU93@^@;C6~+=C8Rj&Vy2`wUI(Lyka1bwrqNwP!zt9l=LCs^~aB<;r zTFmx-`WdAX)RhQ{0dCv8l=582zHIfv!cjR&_RXmEFTCpZLmf(CbY4-h25-QC?Cg1fuBGz5nL-^#P) z-Ez)%{y;TVbal_U=9ojTJ0dpImedUn_+)MbTgWFJPB~M@X^Nq zTHA}Cc((ZEYCb*!&WDp*hs>{k|F@5}AQ$j_AfR1_C(4SbYQuxK>XRApMnMr5wI8MtM4>3Ej9#1-DW560Pk0U2zy;TLI)IY6`#*T zJzZ^@Uw4#@tl_VYD2opz8AXRpllJ%oxXHCUimVJjfRF41|NAQXr6c%w?fK5^xF)M{ z!v%DMpFQTM@JPIA*8~t>(t)@TLA65Xuoxh z{L;SZ{73hU|2UawGN2%6QDv83LyusC0v6mbVWt;gzy@(&z7=1Bb3hak|x`0`YK(q39faV_cOwh+M?02haa-9ethr()KS_%fBr$ z5|9#J2HJZ-Ewty$^|Y#G8qD-en-Ub9P(T4b!A|D>WlF*ch`fdY$;azI5Q@KyZXPGp zl;S4xeo3@0ap=MUN;CmMOx9?*>-T4|51}a(IP_4An@9ycytWw;05Xg8E?H#3pd0GB z1gLlyw}VJkH^+vuM;$HlCspOw3#`_TbChzg!xa=0dBAP6=Y>O_FwOf%--N1=jRpuL zMRa*%Qo|1* z*X_^^pUjU?(#@d;V7ie?W*H5<$>uT36?{>=8|e;|ff*vd38S=YEUw)jQ%-$Lt-Lu_ zqMK=z+;*LJf?LhQ>5wN%C3Pf&Br&*9xL^CnmeGTRx)zqq3zvc{&G6Q|T@&n{PGz_* z<>6|M{2(<0nlQY7d~B@n?BY5z;??ADprr6%_66k1H0wALROvqEv!tvkcyP2T&(U@dDM((a? zlCl3Z?O)KJC`h%LW{skhwVT9NUp#6#fjX#}D_J@MDv^=QbN81h9_P)UmYZ3xmw`Nf z0U^Ki;ZAgIa0+8*pokH+1kM0D1~(hzDpZngaf4 zPW{cWeDD+C0|0-yplhqGMms=;Oa*Aq6)T}w2>#m}0bw~%`=y}IvCzOrS*S|VslZ1N zM{$={+l=2KLE>k*U6jC|c&NEUCewC_I$)xIfu+1SeyTXzHE_RTS+G31<5OuaIK(!n6o37SJv=dF>8{nL|w)BfQZ~oK*x73*% z6(TXpdgrverao|z#?Zlol-4u?DppY>B2)ZF!3H1-lK6XGj#a zj&b_lQ~DSfdX8LTN<5A_rBIZ2PG|MWWqyPpT7c$Z$O$x6WIbG^15#a`uk#ho zVYcoyi~gTC<$&stoLAvEaCx%;&=1xk@d(-BaTH$^+$7%6zsx|406ZYbaIWiqTwI+> z>w1CRX>f^-cM8*ID0bWRRi}GE9y^)iN z(((DC&xQRFdq~75^UVkhUS}?P8fsgi7u(Lpk2Sn}eQ8z(+$2EU_ff4)g&Nm zA7nUArn{Pcl8`v=h#l3TMibd^>lMzG;Qn%DdgIAI&B8<&s25pN068ahVEv^mpac@kVd9K5b`+LinCV{3{abIYKWYF6_Y z+HrpX)+P@d%8etnTC3jhNxk!E=HT!>9n`@sRWx*3M113EV-~@~|)a zo~ugCD#0P#)}?#O?YDhscGE^RsWN+DZoCh(p{HaXQOBy`f7~#{duUvjeVFD))A?)w zne$KccHhAGMEtHUTL+L7ngs|~alQU{aVhY8fCYrLzXLEAM!#$<^bBpV|Ju6(^vPy~ zposajw824iz;&7N530A%y$OPvF(6ZfwJD5^`sO*`DB;2zeFknhyba#U{uqeo50j7&#m7r`BwG8#Ip6!}-^2ub2l#jXbER{1F|6ZN4>9)OR@7Y`=XpGi-=%b=QQtGn@KHG1uRj%Ew5U*Z z=q70W$&a77A1nbP^s*?L zGOVN8N@)lY=Ydfi>9rPYc!%GREl=lATCE(;164HmTxoxdj=vq6M#Jc;V~vgO*d}>e zc+gwi4JLqXpYg$h~t3QKpM4AGnZ$1Rl;B?Ktjh%)2?0v00OhAM-X+$4YUzz&X%#ehOZT=GzB z){ZxQlh%a`vj8CLVMz6z4{4$OnA`e*OO zmnXO)SmgP*Ve=i)c|hy#15&TQb0dU>5$+#nz5bZ^0I?sMH>b;&F6>7c#(2eYtYU=hJ*z(${@>CEXCF4_-E26DIwkvpc;Qnur5& z=!da)!Z$6YLs8HH$ANC&GW(#il0H)o8?bvwpGtJdZT@-Z|K0Az2*IlWw1d3mG>2j8 zYrrK{_VMxgc*%|Nul8}s&rcp;1w0U!LZraLDxYX{qzRsvZaN}`Z3n39Uvpd2s%QKH z|E^TfXdw{FRxlwaCxfThe~gnW^FSubD5vRx*RMxBHs8m&t%K>i+}-Z?fU=z_C~p0| zjq@T*K${4cl{r^rFZ33!j}Ved`uMb@^c29&o#Ku6i* zCpOG%sYZr9_x5KDte5vT{tEJcB`0LxN zkybes$D$BL@hz}U)O}9|n|)7|oo%p&U2{(KS*>%VL?HKmP8;q-X>R6t&&!e8aZeH| zk5W_UyvD=1=4NC+Y~yk}5E3maN^BQIq{S1}|BpEMpFy2@xDXX+9cXZy0Z;|ODgDjn z;TW8$|Id~9gblf7So02M{qg1|(?Zi)G8MrVegUUi= zH<2O8e?ndz@R~2NJhu~kb0gZJakNJW=}KXLe2F!flN&mw)6Cf*{#2>A`2;g;TdoBY z;t9>><@V(~t9>SK8qDpOHk&1k?tQJaE4J@(V+Dx*0y3QL6^f=~xl_5dN7FK_RJ1aJ z`_kn2ao0j2K^VMRW?+eui4z{LlK=W_>?S^-}4M8bv++u)34VH5xx3~D~`2T?O zPi7s9q)uU*!u*mO**BwLrgU@s=z14~fRbH;_mZ?#Qp$4JxZzOj?)cmUblzlh5fMsp zEjQV{T#5^A7;S5FGNk{%rTlvwf`R7$GZGIeKb#iu4qnu9C6Iv&Jm5a2T}Rfh^eJ=S z=wGjtAHd_7SVBiB*$I&)9ePna(V%2g!2^rwwzt@XLX;E0NqL_(TJ$OXSIg=tW7@MFkAUxz%C&Q0b0wKR)3G37lK7=8;m12> zjts9C_j@rcP0b-34yP{&_2$a5Z)1Zj0jbE4>GL1t&DR4&?rg>%afVc4ddCuXFo8;U zCt_nX-zPB`n6OCreM5RJ{G74Wn>#bZ#sJHKKhLEfJG|FzGWWK5soEv^#s0y{;TUJF z6c_fV_i>A_PaA{FVPkC@-if4fdpBoZx0-P6PZd5d1!csUAGtwp2^kKFad>_I@qCNn z`U4HAGr+C;rSO9E3V^{(0T@Bj*+QI|KV#~%l=F%J8w>Ox)+-QZbHnBOuS=g3?l)fV zl#s!hU9K(>5@|8_sgh6#y|RwYYudkI5{pao{?>9=plYL#+yqP{)*$;V5~dJ<#s4)J ziZKf)V!yFi)9RdM0({kOYv+>19C43d5+#^q1S5&CI?^ll1Ing*C6HVAXuP1?b7v&i z4G)L)=etxlrTz+Polp4a8Z9cXUJ*uf(*<7${5g43gBoccc<+9V1eR{25)r;bC)ezD z{+sc*e>oBUVj>VWDypUltXiENkB-*Go7!a>=oD8%s#919z_^h(47mMh zCgn6jOUdObLE!U|hhlk33L1(mhgDUS7n6lbf?7MQMLnrR9*Q||5U&^WdHfrl`fL1F z!slF}iNoBxp1`~x3o%ghU8yc=n4x$;RiEK^c^dACtGruf&?M&WIW3HFQ}^e+8%eg_Cbz(dQ{9veun}*V;rmTpG6_|M{~PTb({cWD%2^pdxSAl zzGJ6JmFJN{J?}+L4Chwco%n6D#$vwJqr-E6{{YB=p&3OR0=?Y7Sgz?eYli}a3|Fy- z0&o^$|7Gd^w5I%f0?7b0{oelzh8rH+q;Ey=iBW?QJeI7RKWOAo!_PqRV1x5;(LUvs z=Y$T=)&GEFja#9lxWBYEFTR5;}#t5pqOMoPy8CtREu$LCB) zxj~CrF5+*{h7n!6LmoetbzMymbK1SOxEk2^rZL_@#!M&B%S4(vlkAk0tf=cBRokxX z0Hbq{$8QiD!gjMmrQwI`imfh^rt)Q&$C9#r89a>qkWoew_NE7j4nX~ryacZ(1pY@e zGQj5%m3sZAS=A+F^b;#|-7aeA&xsZaC$>9`NnQ6C!S}zb8No1q15V z-pYJiY)sxqPHTc1Ey4GhJ(0&H&X;wmWlnBEUw>4ukNul=W{3oqyhc zf2DW+^%b!waSCYNASlA1B(vtRz#_?LEmP%sDO6x=E%>!2IP*44}Q`ZAf}~7+9sq8c$~@LXN`5uGQ!FG=)daJqA-04aO?)$a)qP zDRxtEAehVmvy@VtS`(=UIiBZIkzjJ>RyS%j#S0~$aQ39szo(Y|nOFbU75wXuLV56F zv0X@wgFHqeW6VxbZCv4l-d0*7##3Uhx&-m8APN>q*JI^fWmu`ITnS#)DN!Do|YyG*PmkNwu9)j;A ziRiNdwn1cW+YIFs_;N=^l!EH+@Vl@~;ywVwJse;hijrzOc^xDT2C|viR<8=<9K;op{?XPj8{+UJBgNQ!9VGQFGNCImH!`(^S>r84L~A75PdlD z+;eG2su-CU;n>zH?JX&Gkcsnl2x;ePzfPcN9**FM-u`;MVp_Tyc?`OQL|55UfkmGi zOD#WB5DhU_S|U!|>Uffdxm#X#P__Oh(IoGUyO-{1W5L)OCk@}Gi9A7>dcO5G5~meivQ3wpTMXP5OTX%O0B}UQnUO=dGu26%41}^38Q3K6KOOj- zzG)LWfy zUj`c8?=g8;I`9fcMJhTZ(>3EU*81}Y^i{&bTK{|Z-Qe#6XvT=Y49-;-fOYNegjXpq zIal~v?NMJ(RvxR+0A#sOq(p(L#bF@o4_9HDHH0=6nDw%sxWDLeB|-C*uI3xalo=mq z8_Iozd7|bHaxmXOy1mz^{h2a8;Iyxt;m5Tmf=F9Mp*L7Ko7R}0r!h4tk@Q0G;0!k16eMSAp+WTKi2o;`!rXw_YU%j3w zS2BHSY`VLwba%dY+>a;>ReS7J=bX9F+)V<5E)R*QQ0$f#YFT2V@wDZXL4p@T6hNA+ zV-*rw8m9Z_V~zKFqWr+UksBoQ;F^^zDs+~BYd2XR*gnT|V6Z}_GT9AA9q^!^q0qGz)q zz(=xOq~!5KRv2=s{Z6auiB2sGj@s;01CCy=%vZ8krYnuR85p&3(}1N;QIG*x$jw2I zca2f2%Xv~Ein0u%KzkV4?rP0-7j#pUrs1f0;kR!_;HFg_ymUwT5aI_@guM+ufCl7H z=i|+-l!#BGulGqQR=Cy_=4xHG{t%+r*GA=P&rIO{z@6#fSlJoIVmW(@ zuoGwmKU-c3Rm{*4#UoI~}>_w>IrmuLtj zA6(ZCxOY1-K;r@{6gWdt~eJ*9zv@FNV^vWw( zaaOZ(VoTIoyv&7WYTmO`;AtpX*$Zb?P156d&Uv#4W**jzT;Cb$(2NID&_QKXj{?+f zpZKcyXr+D*o9lj9vVMSTf14ZRpGu7pf>BrmkLQTro^?B z^+D7DWL}1^>z;4QgeL(BILWKI^;m^eR_o=zhb#b}Xzmx-n|_h=%~PDIVd_zh7l!6c0#WO#?KB!nThmLq|EG z04hSfQfEB*zdY>!G+%k5LXsgtmcq4a1>fX~nu>Ab7Rnuw7OMA0n6)P~3SH0EL}6@Y zFQ_>>H=vCf^}8v*p?aZ1Z2eAioP9Q*qe)bhtnCn7vf_BpDxuKeH*N6xGF1Pz-a0h` zh2U5J;uDpQvlzH;=wjs$ScbiJ-1a?CaIY5KEX>p)>l=!9>BoK-&{pUJONIF>tWLa3 zw8aQU&lz{IO89kMr*4p3hbd8UBtblH<$8;HYk(+Sq**P(Wj~=I)f^Pc@9n=>~6n_D4#4pbt`ae9IxD#k4frgC-Aiv_S zq-1F|LlLn*!6*|?qj}}S=N1D9aO}RH3GtNciHv5x8cog*a8o18V1x-kb4?l-e2wY) z4Uulw1Gz=xBe-_}E=0t>ok=z$%nCt9omN#RDDVW+XC>;<6zKcd!(%i(Hovv;<`MwB z)}UyJ^WF^pawX-?mq$9Dv%1|ZN`d>V7v7)1ysd2Kfr@&rp?K=Ww-`Af-yyy@2<3Ud z_381^@(OYAb^qrpB@5J$eJB}AfDY&XaiTurfwD}CYI;Le&3ce*=TDQiL+JJZBHhNo zQ}0g32iNZJxpv0$yonB#XAVGJxabcty!{OX##w>flqSCI6iHx<)70Hwv2yzL^WD%) z<`#>xS-qvUC}=7kz0%PpF+y6M0hKBPp$#&ok8QIRe0_&n3jTTRkSF%JtPAfqOND-= zFRq)z{iP&sx~m?#Tv`!jqQGn0i`Pnn3sCYM++YlZHF;<cq;ng25O=5O1f zoCbdol^)72_Gm!+?YR?aH9YtgzgNt{mSn+XI1b}+6WUQsZM1%Wlv)N**P&EQ?#~9` zQb5k$Awbbot}}^aOs93MfimMrm!VNn+GP@n5i4R0N%Al+7G+Y?i~x(~2hgV&Kq6el zm<`B8e2qku-}3I2^L;%l$RQya6&3RNNf7`{7qVF<1pbVo&u#@Xkra281f zJf6P)aqF@#6mY?Qp!@HQq|bIdqh~YNble)+G9tgn>Lc)bFr*vFHW_(iF)j9t4D*y3 zXpPUJ{LU)GXZJhZrR-=nrUIEm_|bYKCyrz!n^i$Y+PoHMwtalrf$RU;QlMRf(Er!o zTtu7QJImldJug;I&v!@lewr zF#EIGCpI8OYqzj=x2{cslb@pwr`b+|9B5cQ{z=n+q4_htQnrDXTQYqooymO333iX5 zQKV4SU3Y%svD9^S1!H33J-izhDv&VsE)Qr&YEurlcNicb2c*y(f>-uNbb&Ea50aPm zyA)EO6>kX$2Lf8lf!;f@ZwS|5Zg2y6k8R9Lf!xf_LZv>%TyZXuHl>L6k9HIo7#oiZ z;Vaq71wC5PNW2r(-143h2NPbo%nZi3X33<@m?-e;Fu7{xN0{FSS246 zGU(P6`8@BY#I8CM>gyIsU z&;eLT!t$T6W(;Lkae|?$Gve? zKG|&H8es>T9SUfv&C_aT@h~Q6I`=vJ@VpJQx}O)(EEau7cpV*%e}t)3QQM@~t>J>G zDwBvDg55X=ZZ3#$*UsmH4qpX<Q@m zWrpGyO{t_prN8#=^g$AlG%T~K-lDy{Ee9D^Ek^~L##a{(a|1TZY_Sqiqcf6-TXg5C z!9Ynas=>E2LeH~2X}N*ls_4`OtVaxgozLuUDrDmM^9@}67kiDSL81n#(=6`?5g28lt%YQdRrGlH^wN^9CZrKln#Uz7EVf20FVOQA2|SC+`g z1%8jXulCWVo4F-Sp_DLC`;xz}cp+wiPtM!ur1;mPtOH-oI$zIoq6Oz;1<%_#V>hLj5Iwhot zW!+V3ox~E^TUfK#N5B{PbpdQ>fZGizPfz_R58mtL;XwZKCHx|vz-&D2^GtHv(QF+s zCej1ITTihvx&P>NO#-m8QKk7k$wkFDU`%0|I)mgZ1UfO4m>)bEB#98{UtXZ0fG{w2 z+l96wFlMSa%yUdo4<8solf~9orkHiscy$3o|V^1K_KSbNXjqpnEEHLO`lk%qH|qu8Zg5@#I=h)p5t zC<*ObyFFQx&X(kxoV)Ox++~c4(fj)Nov+2WF7yh2x!OOs|l#!}@Ms+X3dYKNJKs9L<2jd_q|Uhn)dpUY?u62L(c8M~BG3!NJW{N%Q*m zNo>@x$FrX~W(9)yJ;^8&Q1@VsW=A;#Y_-i+gP8KR4;;E3m>h0Okg?)>Ld&|qg121;T;-0m_OY@CoTG#UJne4H&0RmZd_8HJ)5v!aJ zV3}OQugroN&zgEOzFj>k7RnPO%E=qK4zyV#;WC^$Xb=C}uKvple?@X)$IX6$t?Od< ztH&3JPJ5$t#jc1Ot7T(T)KhzEFVSCVmEB&?77}o|TKsU2JU6WK=H2{IR3Sp>;A*4a ziD1bnz9b5Mp5?n8wb`88Gtw-nSf|A%B|5G&!HvE*(6WGLArFCv&3awe%}EYG$-((W zBxqmmFU4GaHR<(`qid3a-z=owQl^Vv4oc|i@ZP>c(EfG5L$-;0N{H~hJZg$WW;$K@ z9lG*LV69o%XPw*jGf`mAQU*Wr#d+)ZX|UKT;+l~?+P5E&>CK_2(SFQEeF0cysGYHn zdn`S;`iJpldet=v4053ayt_|^yC3%+sOSBeXub?#K$q#!3uYLv%~h{-h({P^D`s!{ zn9WrXKq?3lY$@yCo+EohGV-j*XQaU*;k@;FUq-xgAf1=3;n53A%J*QLhtaR}PK;Ov zqpC_t?CJn7c}pKrr@6;hZ4)W!ZX-IRCpgg4`Y!+3WW*xd#hU(fxxaIRsp@iQBr#Qo z=jL%5cEGw+b*82pX$nBw!BQzpAYs!4G3XC;Oz}XR$?+im%Q5_y4g2Q-wZRxn*Jj9T za*<2MB|7Y)MFIp>UXTnmUuAJ$gLj)*S!%x29QfU5gN@5(0?a_hA_7kP{BAL?pN4C- zcAJ$zLGoQS7ssWrOmE}3Lyd3TVksOq`hO96%KD6ZC>LrqFcU7G%wl<$Q(~pguNB*c z+eLJRVg9;Ob6Iayg)|v<0NLUAk49CIZ8{5pTux&)eCy3~q01vG0rV-o8sG&Vp&B9L z&Q}#5>w1|BjGj+T7b$(W3b(?IG$xl%{(iOz26g-?TB7MWdy-zeapX9R_@v#Z5_2Ycb0)Et}>AF@a5ej15 zKiOU~Co>ZHl|oYbSiQ$4$N}MtLjYT_{+(q(9U161V3nL$6}$1P$f+@Kxvb}PFb0W< zhk>C@Oj@O9EZ9eNS*1NP&1rW|Drb_LBy|HV0TcmzB12fE@8WH=KsYGP-!H%yq3uc@ zQKy4$1GfgJ&H@|KwzHhjU;SzyU9l|bZUC9bDwFqMdp7|RL1ei=PNGQvRiMryzR%Q6 z>43ipe&pr(7bJGh8I=GN$WD=BU0tWsCIkt0)b6%y=jOl1Th~Wjbpx0`Dwb6$ z(%SShX7d}`bl}!g6pG4|HB0!-&2cC&Fd_=cOxXP@z7X(;b=8@#^M;JlBHK>dcdjQ1_a0@&{3;1%piiIZm^_?lo_v}u zpGQ*f2%nx^QKjRWCP2SX6Ogc5;NE7&*d-#nHIImWbtP8FXjLp2)tjX3*ud~GbeS&Rn5D(!w%xF!CY7$1k9CZ7 z={}o`14iDf_i9DhvWU3QzzY>9x+|IWP=li4bxN}L$_?u-=~EoB5K5avxZ8%wcQ5SO z;w#0?sY-paGA$EB-yfK&ifgmQhnpv_j!hyk^kq8l`hz~1$NV7qGU_|U>zcB4IiPHC zX^6E4lEnAi^n2Ih%K7mQlEZbo@1ts%$eyt%>)e+*p^c6qhP&W0*IKgy7RZfWM~MW^ z81r&1@t!#9?I#s=Yo*pAEM3LWH&VfSBasy?Igi))+`oO(Rro>i>=)POm`~}6Abhp8;gRs2YU>Q_7-e5VH_ zQ{eV;1sNK#Uo0BE=W+B2PwK?vUZR7xn1k zhFd=SA1r|Fh-3HQ500CWVUSdcU9YF7Qn03DZubyeG~zeA9e8i=e`qo2Y-Wx$$PLGR z*b0b*@orJr94C0%3CDLkTgny^#la6Q-x)ahd~>W4Ef%HQa=7Sp>yJ5Ts2hQMQVwl7 zz4xQ=7__53Ga$aU+{RUsq-Li-CW=3IVkw0H! zbhqRH8t!=M<$COBUX@_IV4-+t<{5r>*4ptSjRtGLi9hCY=?FDQGbm@1C$ zT?b=8mJRPKZ&#*}Fto=GD0da1h7IHxN<6(&879?cs5Y)-SRi~)+ zH?4*`+;PF%B%BluU=9!Y_Y>EOOT&aDAtZ-t(|W`EstL2qNCOf*gL8XWAzU#{p#v}i z=Cr0ytZLs%dFnXWrrUBpY0)o(?W1g5{Tqk}K-y{ZVMrU>r`T*fX)_Qt$zy!PFQ>(R zXK|1+J|vBsh|4{L|76|$WHGW&>_(SUy^^Enq_WV@iYdiNaJEA)?`z1|B6|L2{!F<=zh9=gmN&`Z$Yh&TMW*Cgo&YCE^j(Ge4> z2j%4CWO>{imsk_AfBUc4@9&0oON^LC8oJpL+A(444$%8E(O=H-QzF~!$N7DY$+H$T z5E0O4pX;ZIsNHR`z4KvfExLTiHs4fdYhp>EurtXLox+LZoTz`?JtxF$kK_ep zCK#a9$|jj8*V!6ZxhFCkg(fm!_YaHxU!NHK<(Zgr;$f+-%Hz1|#&Qiird=N~W!2B+wR?kMR z+%DWVr)Fl=hFVlg+ucMjrekesoRwF1ZMu+tKa;AY{t)mg4{L|9fjffWon7;0r_eJk^$P z)yuU{@<;~uM&vH~aT8)PVOeex_hK}u-U>Q_4u4_8cdd`|sAgGyqdQ$T2@wu7=8lu= zrEWSHlQCmd=sy^(Ge4Sa_XM#r#Wd_IwfETfkpN)1Y(OXPgCQ$n6hhkkGY9>ep%cYqVIRza;iZLq*_2dhP;+c>?K zjr2SeYkERddHU9rP_HTr=R%-W2$Ejg=fYfjJXUf9)^o?tHj)I&F|fM_wmxO62PR@p z_zB!8cd?*v%HbG=aB)|xRl!-VEy?8ia*;uRpE(jqNdNM((liAkl>pSK@xn==Y%sHI zIU1FeB>p`k^AGpupN2lpno_WM4r58K3|C1ffZ$Ht-hazp++HvmhRgi*Q45H$=B1@igHzalKI8p zgu@&5d)L{@Z!kzZb})EaB0zxuLl0n=}C0aj}fu!fV&s}OMQ3yZP;?53Li?yc=yk$ zaMQh|>fX`YY?y#&f-Du^;52Rk0bVO|*WPY6~9 zhGF*G5RD`xi)eMFavd2}t+tqpmxL+!-G3(dM>^?saoqi#>*`u0D)Thqhm7tA-YB+>mT zH;3a{v>Un_!z|u-L-9e{8X6wnjVurXPH#Y~Yk&UkIPzo@^Q$Ki5?s0@Vf34(>b0n# zayt%#>G$%fQnl)ea2jl*L)7({yIeQIIy1%Vb36h4Jc^S{dIb?E4FV*k!5~92)|tbc zj|gD?AFLYz~N`8R>RQHJx;Ubl~j{37_izyR=(f&4{i8T4a&L`U+0FAz;hNm7pnBn`=?SssRJMpGYdk74apd2Ak8+q3&`+Jq* zM}yw@Z};ypQjKS*-X`NAx|w_)!g!GZEpb>#vHq?`o>qH~n@YQHl=Np%==kQWrI;WT z2>VMN3SiiJp0><9B47eqX&*BjufOCvwTx%8qr}P=wEt7)hz^(wKnZzN7 z%5E_TzA|2UqD6*#9!MQZ4<)WJ_2Ze$7&6^$zlExwzM4Ih`S|aAZPT^IbQ_g?wtlPt*F3 zZ`?0!GWgyDJ~NcSZ>AJo+U>3Bd@H8UB5%T6P2hCVW=zN5FcuyV=e%hyu~!5v>40dq z$gcyW4-3pvfvIfY@0<_hA6TKKrsBjvWuTs){&#>PLY2pj#XW$XPm<~Cv;r-%n^KSO zpBQ}L#CuF))EA)AtRVxK+=v-IkLFas)W`rTy-;K6`=xjxKlOXZmsi)T9elUi%z=-_Ia(ojD~3jpL?0ta3Z6v zN#Hk!W)H(-2~t3z5v92JHsPTOh^?u%>TZxZ@Erjv$2`aO3!uB0o+giHd1`oPG4M*W(6aF81)oXhoe@x!V3cM|c|X%!Br zGQyokgpI+x#ly4YpTBe)cn&%EoGTIObsllp5`5qd3{9m!?o0In`ekJc7z5TND#bko zvxbgjO8;|_px}?kuaI&5lFx{`gsJeQd^?O*-K`j+P+oHpNgi{yqFewHzOx_Ike+(T z?lt%OJI$O73+SoS=zv%FEWf+jiN3vDzeYzh`xe1Zb{4nxJJ*wi(f2S+0;cD4BM{mR zVlXBi^2yMQo_#146$mez`(PIO)1{)znCGXM!S@P&6eL!9v=4aM`Xhi z*)H=>V=C*F9q%DyDJ2nuXZ($^&YjnQREE!UTceaj;t{oSoVzU^)x7IDmp;;3Ij`?a z7vgYiEd6LYM>4mbCu5aMqpkFA@5&&HNu6RvTGZHMc6IoxB7{a!4D4{i9CM+K$u);` zUHotiCYSqkAFScm&YfLtWeY0It}E>R*2Bfjc#%NZ`!+eR@_u1ytgn8IpxNirF7XM) zBp%y!zPGHVZ_)(3wUY4qv+qMmHh|KP)>ZZ(7*M>q{1_A}vIeLO{FI6dhsmJ=8Y~NX zINW~td3)@|tEX|8PvrzlLg`_-_y83mCi7`B%hz;7V{|I~ww{dNI~xGb=JQrhu$ny` zq6wfrHDVJt{ENepwgM1sQe$@BAI+7F2J$(cjHJhaE+Z2lpLh=t;~@xQ37~&wC*i)j zfY*@$6eV@5J*~mo0iXy}W0+$;-~rtp^>hp%aq0;r+gg-#A_v|w>a;i^0Q{x>xS*9y zW}=F0=aYE@CN9BQKrHg8f*Z$u0XDbWi|`pN5qW0hy~r#}FQlSmyT8|6CEBhPcb?`(jzaIP%aj zN}C&)JashAJ>PHiyWDQqN>g=$cOlHn)2KUYYt&G@4D^OKE{teM(pI|bn<|&``_^S= z^Nu|OLN1_;Dmay!VgdfoX30fcLTfNn;XsT_eyQC4lZYwu-;=Fbm0GeGFR$_5P07wf+RU1%;Y z*V~WsvDU#xn>lrb2Ggjr$#z0pxmf#r;B|#P&_wKH$HEH-?JA_Gu97fE9|0O)Z!i+B*wX2?%E%*Les4@LE5f z=kdC?%XU*&!3plCmPaZv15NYw$5cmk-QTcP}SsY-SSM6 zgSMBCqc`53%6}V8`&|h4o%M3vkR_zqpx;GS_F+k6fSF#Gk$y3g>$#4N62P`P_{l4= zT^+uy**0IMRUaNHdwU8;gxFfBkcpBnlh_dM=1$@|!1GrT?|*Q^F9iLk^Vvr)Wpq&G z5z3qnS2bS(j4!vHDB6@d^zxrR(@wa#3FgtgfR~U`m*`ePV}Ee-?KfIgl<}CF~eLUsyP16P18ZrGC)P@f&E|v4y`|fYw`zQ$%x+3nr z%Opw!uWm+{sMe20a;3D_elr9gI+bnsuT&py^EiD0rE#H)Zm)HHHv1_iL9y<_vFXr= zuv82EhG^VH?`!Z(;lqNc`kDb|Q;lYH{{po7isz3G6B!-=!B<3c$?zf|5jq?#7-d>3 zI?%h;y7YOa0c7fc4Gr{cXM4Hp4(i_VEm3OHgFC#*)J83y@Zr9JSKp!LOT%2i3W>*% zlCUY}hG-ZL3R!_NDArU_y>Z{PLzM}3ft!WX%;+iyFR{ub2$;P#xglhWLuMn{fL)w#pr z0ED`+^Wj+9{>d)@xU*Ep)K1LIJav+Tg3o2(OeWy_(zaj%^6agvV=MiNpRVT)DLxEP z@OXE-KeS<;!hZ#w;c?m+>wil;{pC^XaCfd-+O%rF(q`qx*z($fcxa>D>|la+rmenW zCNGKp3@DqzEAZc7t#)|xI%3dc{$>>c(yOZKJ{E@ES7kbBHZ(Xl;7*+4C&SC_?W<4K5}<72-VDK0i#cps$d-WMGr$>i0~za~ z0~v#LAB4RWh4gJyl^6oc>{Wf&WdYZUuw=3!$J;a7cT@cw&R7>U3+zH1$rcZn*9wV zTT%#4wx8rP5$|Sx78m1G!bQZ>{(p46WmMH`7%vE$1_5aS0YRlxknUExySux)8$?NI zke2S=lpu{DAq~IbZzqKF{;ZVzWAw+s7Ar?Q?KU+HYz- ze%$Rh=1e%o$Qf<`(2))o$XinMSnBHLIdIcdJUFIS#Ni&yi=KBPj%8!USzxcVp+FYu zIPIv??CHqKz24MU?tWA$)6-dgL_i4+&{J<6SJhud9ZaZ%HuN5Tm5hIh=6CbY$R76$ zCqdH?(q;m++HM%>79mtTn9UVI$V0H+vdi&3`*11tO?r!V|6TOGZ&o}x25w6P@I2pB zh;Q&Cd=508ATsd90X3cMA1Wh0B=Dc1wm}6UJNB3^XsSkVAsvaQ%X`MyfW|4FE^K&z8=h!EEoXo*>R<69hrZWX+BA z15;T3G+UBUt2UHj==fJ5hRa-{(5LPo_?id#_+nRkOrfftVU7tGc_Mtl&s0-%y?sI2e@L;r#P+w{0Ud6wP^i@<&{Q zV&nHKzE^ZVAs$EkeGyeD;4k@O>uB%do&miwBFgg4+THbwxh^4+AF)xU$JO;CMpI3V z1(vWdBG+oeplo8{Fo+Ab7ayO>iXbj#I?S+Q|H08-;k zc>hD%dd0N_g< zR>&2f(__%q;-N`Otm|lb#an^qLYLUu%9uq-b_;TTPu2`)GpjAywoTBp-P7!}@jX}x zpUI%hf9WRm>sNH0CVMlk$toR*ad*=DreIygPidJ43({?1o9ZTYXkSy6==8nL1)W?6 z11E@__d_Jy{&(CN!De_*lM~c8GZPS-bb^)z?&Q)f)Pl*uqUarH5xLf=ojJ&Jt{AJB zce2Mp*M;zMa$)nzod@G?uY*ieFA1Fl75kom#DyjY3!zM2HVv$YRQAR;;MO>x98FLU zV#yAeO_RXVFFt9#0KH#g7kkD7>EIPz@MbRh_HFR1h}Hk{{{7E?3Cuy`zf8N^>_6QY z>Y4x+MZpr!`FD7W?WMjQAM<4&Z@J%$Uk_=2qh=KYvR`lq>ao0er4Txa zrd9IjxW6=(pa>IMegMaG5mSr_yTkz8gM4w|tk-p2QV2>2j4|v|VF&LMg&zGL1-heB zETV&wfe|>n@XVKEutjSJI`! z5J;PtR(Id>-TCGXaymzp$eBRk?G*j0%VLRW7`h?|FdndMb92SG-z!JK?mWbuLywME z%JjK9G4cdae7WF9!+osK`;wy`R7j|=tw!O|tb&ilgR!U8|6q>o62ba8*XYt9XcL)3 zBXwFaTzI(IXSozJY*br}87l*)X^3PT2|Ri#xMHzEBZT1;ZVIk?K~h0gH3SK7LVyG* z6PUf(n^axBByrezrBy!RL0uAwPCUE!gPP#U?3?Tsw(7k=y(ozF9+E$8j6B~S7t7$Z zYaZ)JLs@cx9n2%d5Hi)avTXmLA$UaMCprNaF~Zh0sPdgx73Q(~>7L^$aIn(V1@ubK z|9h?ny#-w5nN&Fn7VYC|r?CW+kI^fCr(k(mYxlK=7r$1Q+j^nqHUy7dFcFPGuf{s) z(a-N1I#IaVw5rm+FAwKUu=bMLJu?$tP&OTlrS3o6Z;qcP|Eb#^`#HY3-rvudY%P{M z%yx9$+c}h`8Tnm*N-%yV_~R>Pd0ju{&)jj!k%5uWQWf8tGHAKDs_a}}0x1pADFI!+ z5mu5jCzP=;u+vY$AzDmiT4{jBTwx8k2Bk7Mmz2ZNPo9nM?R)f)o;7CoGV zDs*@6@6DE4ot3(p2H>MT4E%Z0M;#9`O8zX|=rlxEM_Q{9_k5O@}BxyXn zCd6b;8kM@~HAlmupa5<@iT<|2&SjG$CjY}CSAxLhi>8mR%#s6U4JiC6lUW3M`CfS- zeY+Y|zHw4bUUaV8<@{bh@f22KWVu@JiCNI*?j4c{HqyBVS2bbu)nqrxQoa)60sU%2 zDd?l5rUHm@*hKLcug&)BOc6)if-F-jIGHmc{JdKhl6ER{YMEgro0CLe3 zU{;jIJK-iP7LeZ*U2wvey(C2t(- zx;94?mmKOUwuLOuR5Y+UQK1>y9_*+rA%{f!P{fbwlcC*S_OmDY@v}$9{I#}e zr^{$W!-gQChc5GwlmES6h_RiAD^f@>YEAK^)l|G2fp%3WI_&O<$5QDlLFY+NTIIlf zB=+jf7&m@@K06~z>Y=9y!t%ew}s;gN;v1Frzl$bb7MHrCkqM+T6Xb&(!Am&B4X5p60Duc)Ff<1 z?Y^F8eK+5*1zKhg2BvgxgCBInG|U!{m9F+jLjsQ$NI7i{3*xR>wgf>|752x1@BedP z{x2enz|Ww3bQ&&W?&AwxG@($^R_UET?}G(|a4mS9dU}wNd|>NZx@E_7;eG+4Lw8y; z!LEfv!aa|Y*ZQ={tM7vdnf^jTF7`uwp5@V6&oByPl&LKDi-bIL+>8z|_>{#*!%VTD z(Oi9nl-+)@C)>dKnqXrM!98wj*2cC;o}ruO6`o|?>N~m*zKh6R7t@dioM!(e2<+%o zhaegN8Q4GAt=3=@f`m;zFufMVcrO}}D_RX>O$V#-A zY)yLT>v`EGv=M1~tTHthzlx1i@tQSxI19JD=lY0$8e&Y@|5$`*=tog?cbd+oFf$2W zrf<{Sonzl)eOWDdg9d+^_;#Ll;=@kbFb0?lCMs5Ot8ZHfU?7RJy0izKXS2iWoI1n$ zh_Co+1m|#}DIzp9^hrJqCR8=v8AY+D22Qy~)fTTu{%aesb%kcoP*eBB60!M$z>w6)VXW8HxhP%0$`kmr{L z>@!ZnaJ-Cs$gZg%m7CLtdt==$WF!2sqF~tZc)6pc6a|~yw17Ndtw`z3e~?LFoUIx_ zXjzk5PV&zsjq}g2k6bo?RR_V&wx1dJ@7~4sbaDJG%=m)4_HgR}uQCPK*C*QOq$2WK zg>pEiKotqKnefEBR9`XIu90e@-*%3*#WGM}$YOl#mu<$?rpgvHW?j#94XxOaznKVX z$+A2-YlAPZ_bjG!_21uGz>c4-2-V)uGS#DHa#MD0ZD@GVh9%)+_}-LG+N}M|m!RhM z)m+}C6r-B=p}XdZ90A&v3^(@Z4h8otuA~<#I=x?@S$yU@zZvGY$MWD99hN_R%ALCa z>M7RIFc>VULTC6`>yX~X1gXCbeARXEa6y2uuq`nm`Q)i?@iK{l=CNZ}l89C&4X;E? zW!^+^#HqB^$8B%N@bC|}W-CX$uE)}xirLWM@&3b&j6cP$Btg>K7+l5srKy<;oq3M8 z2SJHq!bWthdS{45SXyD_v^M~Az~}PYn=W1S)G>XTk+TCrw(Bd*eb)zNT_65&jDWV$ zwx~h9=@jCyAke^71OAARRuOJOVm4I~gk}iWNsW z1c;XN6i6MHqXkBRryUCby4c1hCN1-R0^==V%KxWtf-q?BcR%*gWXCQPzUI}B3eXpG zbD7-5czY^^r`x`h(~2bbS&~Nn1;P_X>V+%mi;B}P^YL6I{}F99sW_QFDq-b(n+b5= z-9q7#3Gty)6a-CpA1d_i%!4r=BWIj$hGU5OSysWO-Zc8?qkK*Btj*&OK^cPzmaEtK zLjUB)6EtaGgYR)|-fI8zk(ui<@a=#(8h7V<*!7{lcIp0CNle@9cP!ANiM8Z`^6b+} z#JqLD`KUCly6i>Oy4`5uIHNqUN#O4aB(!;Y!%}3G*6k@p20_UwGW6EKp_hG}J0ssq zhj33$QMSW0>dyPG-F5szGGl2N0mHiCIZ$$_4R`Ekm=M=zU!ZbW<98zun|Am|s3d?| z?!XoQn2;Q}R<5Nyd z{j2n{HUCUkA%R?AsA##Bim4oo{zX>>i4f2PWQz`H^xhXARQi`2H$xRIAd@(!p%SDQC*{BN zkE!k>E()UvxWoDNYmtH}%FB!SrzdkhLKSVUjl;)}(JId3;)s!zc=G9QacBoA!0E{~ z07`tImd{48x$O~XKmV69NSKgXv7`0wbT3=r!OJn>NZM>TdL?D8AlPc^?zQDVSC3-5 z8v?*wnl3T6`dLK!?`bK-Y~T*7KHQ7FBCo+m7Xl3$`%QwY7cL!9dfSx-wH10yl2q4 z9Lu6k{iC#wL*Td4108E<3qu(;vx3GcUA91JPHgrAmX`R*OIT~FMNXs+;FkD_S?m`=WDlXBt5#ZkZsj>R=efM+8$?jc~h@$0GAdLLS)jiV%T1QPzU z{E%XkoyC^A>12ZbJldCkkqWO=OB93RRNctSz$>7}?RUdbWu5AjAoS2Y%wPKql>7dr zK7ZlNj!cUyHINr_3Mv$u-(McDP?(QpqG?y`DTvwM{>Fc*1ASsTRlZDQd4oYGf0~av z;&XdxS8S@Wy$&$vxpZQ!KDCMKmhy2$^0Ect(3A&Lm zoU?Dc!c5rbshO;D39c2EfvyK$vJ*2`){yHNx#q_;X&XGMq2!3Gyxx%cmt_k>0b9UPW_!D%%j$ceRM{QCS|FoS!uMqvhU$6NQy;i?+>_@SW9sl@dy>p>3#<{Zh3w0qVbvepV zF*}N&8Gd^DL9+|+=^z^$i#p_Z{n8lP7lQcR(t8}-Rwb*x!!1#lL&@9ETnrj}Q|O!IiV%A~LRyGoP%NR}%#HsA=(aJ!Pp()n+2 zo$QjOq~HMUl#WUea^n23EofpbTi(# zx*dIwjdXhOMQA?_hsi^ao=eOwVGBl@!$wbSs}Wj?TYF>ABGGa^I~5LKNEMa>C)MvRd0!tCR<4{k;Cg2y`QgBq5b}O!|+TI zfH9^TEn(M#uM>o8jQjD8LSoC+H-&OtA`Tnd` z$DmJ;IbR&j9>VYPpXZ zi;1f023z$>*LBcplbHAPM$?EB1k7J~(o@ugU-4=qV3I^8tbF^1lCMd?@q6zXr$aWm zqU$RRS}3otW{b;MGO5mF{@O*)pXyfAiyzorqo6TdztR892Fqj25M7@G4jalp=>p)(J(?QzktRPM{`OmY4ou0D$d_h>I1~HtZQ;yfY17vcWZ}$gb}9wTWjS8*e1@Y@Fb`1> zj0!?0*m;oH)uSYp^}3)(^Gx2#QX4wIx9>^#R;kxU&Di02y8fJ)xO8v@uuv@*i=r9m zMDS#6I*qowN&7>|44@kXx!7|6HIc7`0j9?&)rS%m>D{4^!#U{rBbGRRw6U_SH=-$T zP$V@qm1qD@_WTTq;ppmgL)}A9>z+aVJKSVvs-2XTF$5Hsg@RsoUNhDg&p;)Z5O+VH zWbRHj4TOm2)_nP4YcwNWgFR(9nIX02dtb0Ll%dQZinsLHf4>F7^11$g*RR!GkuEem z4;xTFS?vS0A{eQN4-&1XJCxF2c_;rI&MhNQmN~HpQ06-Oy_u*jyfa%K7eaZs!f`7F ztnxDoW{s143hCKOt7sk+3paQN)dCucwvhD@*;g9Om>CUni;*hDcd(_`mC4GYiNFbX zW+Jg#TJW{9s8B0yXA~-%OwRFB1*TUp3GusmF&9YVlYUrz=fCskj zi9EXKMqi-*G9M%4?I~o^`DJwNyz;CE8S9^4c;fl?=tQ2_X9ZDX4Zd2HVrjrMz+3H@8K&GsGT$Eu7xn8TQQ}p=DL@;|xOYe4N zYr$T~Q$9ZYX7z%YlHfmoFom5U7kfXEJ}op}E5K>xhZ1J{$m>5!Q4ptpQCZdhmVtNN z+p!h`e|Yx~Gegw2YZl)^9*JJ+$>Y?I{z%rf{BM29<25j%Y*xL;BCm&*D0N1$+1lO0 z45IR5%s%rR&kdm_D@=vC7;Hj`OFtO3iK)q zz^FTzn{%_RwQ76$GO-auFo?&+cx_T@*XLJ`4`h7Pd9!|^#^g;b8R_WAn~l}6tOIqU zha|e11K#|IgZbT;ux-IXUH493M{;K`Ffnfn`>{lV)XvgKc1uc-$(*vp2%fw1-e zbg-EcK-~7NMPB>6kLw!`BL#LklkloOe8-fa3vyMGyR&?g#`1vCx?cFoXrwqc4l%we ztN755TCpY{FcHs>3+gPG98Yos5{wjhad7g2KwR2$GNu*lyDQf%!M|-+np@Be8f;ylJ0y7Y6&8Dj7i%lps-@S4j zV$oFW^`EuQvs8+6BPLTe5j!Y;mhqA-nrOCGtJuv}Jzd&80T(ldXrR_coPV(^Cks<^ z2swtZzk9(!N^Ghq@y(FO`PDxu&h$W{{nteL0gBj<@6_&NHo(3mvV;(2kHsf8pzFGO zw;scC@F!-WO0xz6YV=Hr9CX%Nj*Y(#N=!Fq9kQ!l&Hc$x6Iuy)^lNtA#T`jw>qYAZ zPd0p(0pQ)Xk|c4m@gielLY^q%ry|HEw$>4#Y@v{hePQ%}|F5)RM@x2Ep*i{c;&0k| zm9@WTvs=j9^X2FJ=46NIuu9N+ zyT0K&4p-j*)mxGv3{=FIi#m)qBBvc+$Mi06z7ggjY_9dL-=g66iA%+Vt#`li z+z1bg()l&Vh;Ug^qWZJr5{UjOUAJQ%LHb+oz$b&40Zn{O?FrNS z8FXxdkjlu`^X-GV`KNE$K2ihBS-fV(b?>jna%G4*OeV=IfIVuc&vX4&D*@eQP`CEm z4E-{bp0=P)Os{$UN5pnWuDP!^Ook=VG)U~m2V3iZz_<;k8_Qv*ZbV@#{dsi8= zYNpj>BiS$2LK|$ZEE=O3stmZmu1E;IMU$2JSSyUqzA68q2JRsxHVU2^ZblQucD)06 znVookcZiy#cFg&76%RUMUob&N^jLpT3p3O*xfd5G;`=*47Zunl; zKl)i~5+5xMH1|fIT_ptaL6&RH?lpvxUuGlTJx+-_|ndI$SI}m z+Y3)t6a*4A-9OwOw?LMGF6HV;#}>^3h=~mAyh0Uz7STSMAiew3@fFH|hr5SLs=vb& zT=p;9FJ^QCjrV_-Yv_cv$7hj%d!EB)2D}aQJ89zk`p=;*lScyU+{FhloZ?pW4ZK5qW;Wc=CXHje}y4f6V1R8iIqGP?i6oeSRALuGei5LOyCY9@dWRlQAH=`zJ4u^ zT5rcdbW*9Ah;`gGeh{dNuOA=~oNke3I7NMWUaC^;Xl;rxlDWII>XCVQ_TcAKJh?=t z8ywbA$ayqxHA@DTMECvrd_hQMU&}2lBmI+}-(eA9X#{k4im)`WL6-P&sqIP>2oS1G z_%N9b8d;F!9w)VyvrfkNB)MG&Meg^jfvwlOJR?VpqXIYR(_2H1{1bunj;?(Gp;rHp(C@c4C>Qt_i?wCzq|#|O;52`YX*EPvKnb1*r?5~SKq2X z8HURUPol+J)yh?U0r!##96+{ZI7+G8d26oJf|%{bqCNJ2=b-)(Iw zZF%M|+8!6k1qclQ4OCEYOdN6w0*%1ILKAkGMwLBLm(zyi6(~A9Tz5nof9)ZDN2Q_5 zh!D2m1@saU5@2E6mm{Oqy?h_J7?Di7FxxnJDgIwZ1;~e7FBc}C%h#1o_uhDTCpX7R zw&z>CC2BGl0S|P9>{t}`fio2a_v<^xMwf1|WiPT5InTvFqvvW}H7}40OX7v^Z?p|f z=F<^yx~y(tx3dyo4a)fmbGpXT3oLrj+z(C%+I29lE_LvL2OW#0ef9d92ILs@#NeY* zz&qR|h8X_jcy07#KpO5{c2-e*|3CZ^WPE)qXtjs5O#9Ed{dN6gSDztRKLc?jMx!L# zjYKS!PBy%|Zu_#z+g=AQvz5G@w`JQaWY&Kr?5H4|kmHWjA#<9Ooa!<4@p>yOR>MH@ z>79q@{CP4yUkg^xY7d0IPm@b+J8;>yZfEsuDtmQWb*NXn#oXmn#mZ*NJmOG22o4TO z8W&5-Zqbp>lI_yc*cV@bJ=CnYCn~ma`bqip#GTR0z#*;GI^MS)YroXGP9n@k{0jmR z7A+E4kH88$(zpmCWhiWm7|V&L%Jr_Hs!zNa%c$Hppd`-tCwu)~ zOnA8c)B;>Ul*K|afbPK%yWdB6NJ~J2Srb!0aE$hZ6SSQIeM;6k00&m_BXIKH zy<{J#zQ&s%$Rx5~YVD7?gV!e?;6}g-x?lsEkO&<}r|^HDi?0}PEPsik za0}$q+Rhmot1#Ta;2$D!J|kC$lL^`eAz|E|%U(Rpd33wpyb*tIgH*!;BQnbMOvatq zL!NO0)W=M_h-;-wM_;YmfyzK)6c&=#OuuR zo2awvs+|1gu}1>$PsQ@Zw~lwscUO*^X9xBIAGb0DmuT@<)XV5|yl1$`{mA23AbYTxI{87@AA3Wq zoQ@0UBvv^7rL%x+QB6%Yfow}12mEZ8X>jCrq7?bsO!nu`kbwIOA}*%1L~sGD@wzi> z)%I0t(e8qI1nACQzqB4oqI=~mPyWsn zRkz(Uv5ic^TQZk4gXmeVB+PTi0j=-YCG*gm&PE-s#1I8s_-*dADu&CU3?A>8NzLgg+;FN?qEBHUn?wY4&@GHc2^QkPE1*7mSrP##Q-oLnx^Xs^<*rzgVi`{J(*5!cuM&i?uK z#w@1|>AQ)lk|J8kerhU0G*}T`Bs|mXRyzH#T0**_dq%diea1>Dfr=`^X13C-k43LF zNr2X)lkL3Ej|ih2R8vc)rxPiz`U2hih*JG`T*!4=>dm(gUz#gKxTe7S?L0;MiyZ=V z)ZtsBS$b5UkR1s~y+&zmXDjvGw~Lta00h`5(Ig0vo}Z&Yp`mZz-jtZXufN>x6(%7m zDd{|w^!#KQf27YdORnW#}0Yl&CHP3h!d#{SZ+a6#&KF{eYbg}L9CE@(^ zYKW&hE^VqFpo{f7~*G0cE*E0Mq90m@;Wz$Un;vNn2_@s zz#8{oxh`i?wV!$YoRs0%#krP0OF0S-9FhS+ro$3f<8d(Rv%V<2YEZDu z?Q!(-Tk)-X;&zndOdbRv zuJT}DF{6U~$58)7fXd;(95CH#9479R|2%i5!tIrdNuRZ30w?|1*&p9+&c)l6R}cQoJw9(mPTWLS<6_!rYEH zjiyU1x26x58fqL?@TEUGH&MgR)T%0hRl>x$-R6r=gCaB4Do;?mR|EKaWMsGf=d0t< z2!$8$^b?72LKlBEfhY|nwOrRK5ERm5P7Zle9orkl5V3^!^@-bmElvcSZ5Zf^5d-G1 z^kL!H3%pqxh3o?>C3Pe7JkhYsMt~8fzd!m_C{t}VOj!KS{qT>-<-vT`x*3jl2f6j{ zvi{LC;C6G~5Qq~`Tv1UFKmuaWuwWz%?Q8yG!t+FMf$9B&) zhwCO+H^cg)#2-LWe3c`q2W}ERO`8hE=6`z6bKM~gHa2TGQQ;R*cns1^2ErlQ-(3oV zYvVsm7#@FM2EtZghi_ZMhzS=-dGJSOt3(?WiiE{W-!-o9vWKEQT?i0n-JOiF#ZbVIbswg-zeCZ#w9MxDcwHwI#ISHOz}YtRPfq8FFKR^tPL@P}An7v+ zC*_i!%RTCNkvT)gV{CM+)$gWJUKFn}2r6d4)@TSPMtu(wK1gVldi1}C&u-S59d6im z^d48O^=`rhftSH>QUyNjJ+yC3?JkV7wub_MQRL|50z=XE$HseYLe(=4`MrORGr6bNB>Q4lCTB*#hD| z^>H-mlUly3!nM2?p~RbN;ljEvAG4>_y_IqAA8KoI6<}y38?&k3Ix2LUwux?2DOuYw$G+pm#1bB6BQ z8RCgLB`P9VI8=gj4`7++`FrUL%!H+mx0i>pm{Y%gt^WP_#rE5xgYu-|59E62+d$*f z1i_og40m99*qJKE=8B_Qq6zHz4W0Srp>Fg10A3&>N^J(KCzt`Cu;i2FucUzFdLHjc>@6McM{`_bDgs|t+ zG6)1N&hUT7h1fG*2t{~bxSou#pAB?Q6&Lr`n2e3nF4<|H#g-bH=63;)Y)B_ofMH#m znUdheb76RgZ{ltBW-;k4sa^q#S8c^61Cq7G_kUS_jb+w=$i#G2!_SgG5!=HsY|=KW z+L=F0mVF#zT>TA3T4I8ls@iP{n=?ea0$IBeKLED6U*fiZ_qg=h;ca1ABJ}d`Y$IP1 z^SVR))eVPr61?}nYL8{Rd1VZn8Sy$8|7B04^Cri0oHbv)<=TEkmM&XE_Sk5h#oVBj z4wc^#qi%YODj$DZBYMN+&Fic_Vm-jYe~?3Z!Z?4#n_F zKU-xGCZ=A8_xp?{pJc zjEmjkoN(|uxq#(F01|?X84NHqNA@bcHbU|E@mWUPGxba14{X9QimAv?E~rVjMOJ%SjuR@ zcpNEL@Tl5i0VW;$f9aTlSZPicU5s%E-2M8|TQnu<_$dOur^>5E$$tw$@qv9Ft~SAV zGoFdgwsa#ra!!_P{Vurx0^wHUMK2JC4-1STLfLvj*dRhkK~NYJhEXub=v=D{71yTb z%qm}mXA)SV%Cc_JWV1+2YgczG`pTS%aNZxlPD1qio5YWIOrcotuDK~;IsVGWUJN!| zh1aRjMmuLEo%@^hV}!@XXRqG&L-g8;05o+uySLooHe&YX_*v=^yohpiQ+)r^{-}Jx#XDey(Q=02T0ScC^(O1mPKlx^EV(O_bgVp zZ*hVGaiHDi=i39d^D=iZfrTmqbiDnkdwZxmy;@&3&uXAP8>;gvL0uj`8~nBb`+B>` zhi;_3n&dLw#)#-#fLym^D*^9i}x482T` z+F!pn+66P8?539pTL$vz5-l>hK>ZEi?K+cCX=uzqyJbLgSzW;60~l!ANtk3!lfPl0 z-cH`?NIZAyFo{m3S*u*F{Kuie)`(~3=?lZ`-KpX_t73j1t%&%=IS|zLUzkN?AUjY+ zx7fTZ^ zmOtAtUlVofdf|7##suokY@GTTs$P7Hx7|626vb15(#7;rzHqm1*bncdL-RZg3>`%0 zJsHwxtI#qD?i1e(6{8Fe`??z(C<`6tm&tXvJ6_glEtxSpi^4N6QDY$Je0*4gVJ_>8 zW|l?}v*&+5O4VuNHOM7HYLCFqn9og8HT;5uj-Wq0TNxLe$>mHB}7ttk(rSU%lw zX98dC(&zAx>*ewpxol<{dTHs9KNp#hy>XFM6!6i|`#hHF!cMi>{M%!2jDz<;w_(q( z!euKhM2T9hvVuGbj9FM){vjRcS@|m5>8s5-@h)Yz>Q03n{V$)Zv^j!%Qb*BMaf7OL z^K&3~1=u?aqC>6r{s3BFYVwv-Ut`C2EgG&Ym)-yg03*|O-W z|0Z5`f4+|9yKc+p)MsEdjsQ!r)BiY*2Ax5>z+xOg_y_ds{{g|{w_8Nd?O8y3wp2E^ zw;#=f8ha;JgG>YV39yF&P`tgZU+et-Y9ryK2ZN=$G%M4AvtS;t9DJ(HYRdp1?H%+x2IwZ#j?>B*|F`kn9&$TkJ+=SeXhB4oROGD>4-piSq5v?!4?!kg zB6QRG{AKm1TH{r|De)lZmdaOi?RZX8m4*7rAY0zSrM6MWy`(zFk8)&3vn-$GrWgbG zM#wuvQ<;&4i}m!m1F|U`hDn*R+QTbI?erjWW4h0^Hcvs81S;7Rax&-h?$kp8eaf*y zmpJV>dm=BTLC1S7n0PD;&tSU@WRxBULu0hN9tFv)r=`PQOOZELxLRSdn?#!*+fkIg zX|!_@%w|yc^~B0WZPuz&ZT3=W&%q%01^p4Hq*%Yxd)U`P2MAg$_%CQ=-@gRO)$hA` z{!^-OHH@4^dTgRxj_9~Hr%$4DNf<@WL~nAqb$9T(sYe5oF?=Zbn4SIo_I?PHBFF91sGl)ae@4_!c4p@PZjUYH3Nn+qtq1A353fs)Oq- zlP{L%tTfQX6Vrd1TVolCQG}6!2tl8&)Pm#FWKtMGpzmFx9QYuC4I*H1Px&WatJx7d z`mW+P*j9#dL?kw7y@_iaN%vw;NCk#-iAvI96A?^O&QM^>;+6EN*Z(GsllI;w#W4d| z(&zV}QS#Z9J%H(-i>x5DC8_aR{GNEgOdw%{pY$4F&c~=^%Ci+Z1CsdTij}pDn!jXN zI{G9{g?y_+P$!K3DAjA{i6UJZmV_&WDn8c+i4OL}v?&q;i@;F-xy55JAadDMAz86K zZWbp|?1}ZC$DSNoY<>B9p}}VRMRHB2?fg68pkv_WXJ3Cw#H`oaC+X7ZK*s0<_Pbb9 z2fZP7^uc<^e>+NZT>)KFD`2C4w|VBv7ABCjl^o-gjb27((*+_as%`OCLHyhQ_BIB6 z{{X6o4syh}l{K3u*k^SH4wv)-{^DT9lDzdW7fqc^e67bPy2d{1b6l6+$)P=XdCip@ zm7n_K=qN&Qm4r89D1Ppji4V#726K-OfjB zqO+?&a8TJG6akq(#E+L(%JVGdZZ1x33Flwz{#*TM*u6s48(t{7)&omhQ^aM|HZ|<= z**?$En2jSfH+M}!RuD)Y428n9e4GpkS&AK;HU^}`I=s(TJn~73WIu#_p;_8bD2W|8 z1$Sv0pBE$Mw~=RIDPqykPn46kgpSwj=mxz1HdB%J-TG4BS?W-&GQ;DZw8})Ewa7r* z+Auw2G*~I*;f3t}aFnw%R%zl}bzJ_jdGalkXw^}xR%rjGPZ4w4Z*kd9V}Q~LJMHC5 z#C%^D&?kPu1-$bw_&uv>=MS&0t~|}BmV=hK#}e&(UG|dgUrCFn${#!ep0az?Dehib zB0`3g2p@O-9eOfoMYh|l0~!>~I&(}9pV;Zz2oIoN2m|jX#<^nfHliZFfXkL_`Nc5<2($UB|mCF&e%Bbvckw%+m4A1R;h{EYG1#_|VkTpre<7L+;v1_rtT=q+Z z{%EUoN^Sg%c=FtUfJZ)UY8r$#7>D*_^=aWpt8_<$fzS?;%WMq{UiXaf6^GeMEjY41 zY;q2*u$ug1{v0^dKUe)kuhZ@O_y#6G0J`9DjI3M@v;VCf@XG0BpcY#wjN7S3wV+)Ickmm;oEKlz~~>Pn8pDfc0YN_*I;&8l2KB0(v~(a)VPG`DIM${Vgv_{p zSsmov&GyJ)zvPCy;_|t)H6mB)0j`?)ou)^;w<_(}h}N@}m|(NyeHzp9g{y2UZ5|Hg zk-z8?sTQdMxASGe?OeMvT`o?m+st|gb`SOLI{>dXORVQY7H8Y-5Fg2~uk!xeVv*?_6kdU0k0c2_X6&jm7-`+q3z^>GveVp`+;3Pt?M<-VrI& zQs1pbxLz_2K0*c~Q87tR9)PJamnf>$~f-#GT7HZaHnkx0C<}m)g z;8VJ9ak)F6{Q-!SZI}KOLi!v}`-Na?4f`kCAc)1Kl+IP8WO6~B4wg=fstkQXLmave zACI%2_oZ%rJ+O|xzPY$(@t$v)>RX19q^s=o?%Q!p{lp~3g$J(aS}pnO*?GH#Oq4X$ z?<`MDgx3sARV!@-N&6V7p;FlesEZuN5}G_Rj>d32^X1P1Yb~I zc0;5{y=% zLD5+5CpA3vJpXmf!vDqrTj!TsYD8!HbMsNxiNYA&$hv^8Fa~GY$8G#K{%3-wIsAC8 z649mc?C%GKK&U5&Eho)28@o-4CjNeox|haY8e5|655eoYv;Enitk*l!wKb3*ev< z2I0O1K>;F+8SsljOx(_WBeH`Kz7wdB0GfSiX(w?jO4BKPNfz)k24?n`olL!*?7~-y zjbN^&ZM{${`P1xVGFNYf=lggM6HM*jcLUxG&&e5Xpn8-7!<6-4t_I;Nr`?{)DY#Vy zu_**cB+a@GRkZ4BY#=qZS3^(A+mWlS3{X(Ql!JIW^7GseJ2bXaQr zP45q|u;Q4-GgDW~8=9;wJrE{w{mW^PTAr%!rvnLRq88G=zbrNGhh_hxO%5oyg%Cq1~coIJO|Gx#?## z&qA zXtt7k`iY$;CyWnHGC>|`s&_b(FGKn~x0+WlyV5{eI_ZToqIP4fX-H)HMQ2V=CJ%Dj z`{Abj{G6|Z@A`|rl{~^awT-N6f0Yn59ECo&O@(=oiyYbh{QR~ee?cq)O9U({bC!zZ zMGiK#FQPsjcRCJ=F9z|5Q|VNY_hze(o4l1yn!v`iZvb56{LYL&F5UO0 z(c)k77`xpv8g#a={P}>ifCnl(i6cXoV8<&F0Qs0i_3|;bueulV*@oXzopfCLlFisA zL-fr<0tp8L5fzF=vM6fAmambQ%S z0P0&ivtzlodAVAPY#N2nd@jZ9O)ESul2PSAzvDd2B(jb7QM5em_(Svxz`;Q}BuzAt z3K3F=_wKL4{=sUThi-JBDIrq9)yMhQYK;a$ZWxP{bQtu*{X)Uchw`?sLb(u~-2cj` zD@{oI`}E!jH|{I=|NlRcT#(w31{VuA6qVgjO;r(I>yE2@tp@JY3FgcF?Z8a`*$x{T zdDnKVEt{%{>oOa)Srb!ZT+1oR1nb{PVGtSxQCYvTymyq*FZO>=VQe5?+#3)+POI1( zT_*l6y1z~4V+g@Xlc?eOYI>-7FDEb9dPZQ_y1tkBc!$#kQtVxKmUQm_u)A4TDhl}P z41JXnT`jMg9$z|KC_fgHf1=1 zK#GUxK0(L5PU{H>44Lfszxevb=sMSK?Z&okJ88_uY1G(kgEqFYg2rZJw6Tp9yRmIM zY3%RW2k-gD+55a>tRHJUWAsP5?q}ZDob!U4TwRnLoZ&9Zk#mWWb8rVa6he@K*gh7; zPQ9r;%p&sP?kjKKmZ#kj6vmS$NOlm9O*DxCq6(P3`#-G0u-qI zfM6h@M<(0_z{r^~Q9Ej^u{1Jr0Rs)R&=7})%;PXHJpjABcZQ7%ku`eh&FAU%FKWR3 z)wt-(z$W0*5fK@Q4D=Klk*t-6Z-bv}04U!*B+*wQlTY}UIMtLem1d#cr}b^c9v~oa zgmV!96f-=uqFMV_U&HbVJpy~_&yvuS!{>fTlp(od;bj#CK zJO(sH)o-jR0|X$KVAG~C*v>xXbcsq zSa!&@A`H`hYB5NJ4;#Rwz8~PSj9Co>2qm(#vY@Be!??*GTK4ZBa!N?n6lLV-(IEWR zQhZ!ycBDAF$x*3Rr$Ri%w1v3hmGKR?zU65j@{gy>u^M(J?>;@&H=5YasaBIj$|^v5 z-MXs)L7bD5isAXUXZ5h$yB^Hw@oN4&R_1uM3dKlDY24GBh1rSIkwg~A2bLsAry3=J z`xCwyP4zZP&WF6~A<8fzhoxx{bFd@H!j21g%KSbsyFYnES| z<|w;EadC2b$3FNySHle_dj#eCX#zgO?+j4y50>W-+*Y~M1F;Zr=lVfVB&mYDRh@L) zIPUu-T()iX2WQoYrEDp2Vs!}2`UlviOOaI(7>y664O?G;_>-mT8n;TJl%JoMd2T=; z6RomPt&^qM(7Sau8szA1QPCkryVWBMs4ruTYl0z5Wx$AotL?j+5^6nj0UcwtSK9AfPEU1i+NdCM_!H#}$aAgr>C>b$n@-Mo=-(-C7D7^guImGnAKI_hop1UKGz zfjyrc16c^^6Gf?`3IeTIvp2%py|n;BA~hsn6eL*LoMPZg(AT=JvI+UWtMPx-XBK!_ zi`CbcS;3xlPx)08^$5?M(fkikZRn&Wf|EZs{Y}Q_R%XG586MWiLNXE=wF10wuiTNE>}^X0BIdtdZgvJa(}L8vvV}h8 z`lIQThYpwdLQLSn_0aSp!ZL=qdUKeq{9t+@RJOM@erl;6bhk{a=q%o8`^&xYz?CUk zT>(0p_!5!VDoMrT!^|-Vj+r9D&8(r`im1xg08zkI-ER&d53w)n%dy@o!1!raj)6AAHJB z4Ok9uaq%!j3TBz^rvp()aN3`wK62#F)ksHOi`x^nw_1HGK{fJLK~xs`dJ}Ow`#Zd^ z2@%IxyNMk^tze-SnK~}J@b>&}>uS8dwc)o7)?rSF_Rg3rRsKMtPO37BOfVC6d0j3hGLK+O=6Ej}9u5dABiB-entOx-Hk30srZ5 zRO~?3-Ica>x#s1cqBoD>=D*jm|9DbIIZ>5M%A!l(5pV-fGD>7u)VldqBjv>V3*61J zUAjtx^={qO$rRk35H#_;rcF!AKUMH|JWx5n%fj71ovD$DQTE+^{d(zg zLOX`bn3Y_nz4@2hmDzkH+VfS6GW^nt_{LU0*LaitxUf>)*UvIHM?Ylch9sYEQ7I}M zdRWbo&lacsAFf6h+E>G;3vKRSy_jf~c_ft7SnfWQ2V$c!1^?NlAf6iqKrV8r##2qw z?_EbsdR9ogwq5kC#fxl;5mV;r=Z9a$)sn8@gBt;b49mWw*&*?n%I6U^{fxxlrxRVF zEPARzYHnWwTmJ05Em@%sS_ra;1Gm(sWyM)Rr4;{2Ahcs$F@cyaYw@3^kaeV9nVa}EjJ?tekhvgPGB~D2?QBt z%%1_Agna#G$9f$y(;EfbZWOMvDD9i*(R}J$d_{6&GN$js-mroV@)bO1JfcEq=N4$ zI2x18g`o(@MCR(mMHHo!2JUo9Gh+-@q3Hu#-Q*W*UPt>!!)(09>mp>FeaO=c3}S7b z0xOOY5aS|Nn5eC|o7Z0fijv$y-4Ly0J1+P>kHF`NkOGcL4#TLfXQ@6IX-v>%>hrNC zk>`zS$9lNyOR6XWHV(w$wScSj7qu>VP=?T6J>qCO(dE-Q!&V0Ra^5r&Yb+FjjdlYw zK@yV{T>`(`SO~vUozt!N9ziK>MW?Qf(VhgPl$;nU;=weXz&>Un*1;qdiQkLHy|9!@ z1J~Se{62Re#p9h@_1K4E-^0U?96G}Z3A5V-@{qw86K7*T0hMm#U)OthT&Dv^1{ml` z?VHE^F!()zu*@=o2-?3vdf*bFXMnbCRb1`5Uf|g1LF2BmLWDRvZ6tEITYV5bQ$w~; z8K^Phz#Fd+-NbvxSYRwki3#K_y!onDMMdCx*dKiIb_qyk{Doy3g}XVUUB!L$#-s)N zvxwg?&}p=hU!$?^JpL+O6f;KbV>oKQOlbreVABK;*%%&=mA59NUvT9x`nURymLJ3D z6nsCNk+|VgoMQmhF7LApR2nxIphs+Dmtd;wdk=^8BT^3x{jrJM#i2~P@H`*@JV4|!f~<%OXr@G2`@!v3aa&8 z#oXFt3(hlS4B~VKH8p84K?yhm{HKsqnuG+g6%6;MQ|h-DnfHvo_f^UVGE))W=q~S< zFh_-U{aInnfF)}&TDP*&URyDza6dgpj=xqIa!SHGgkEkCAl|66T_x;zO%tTicMSxH z4}kzV#{6xU_kY9W{{+sGQU2J7Yd(!FOeTwWiV14NVM>IuHDA@*Ls z6LMm+Z`nlHzA)g?aDqxJICC66?~+I;;XU^g9gRg-4;HFW2^)IQya!1NDWVy~HY;ma z2>9KLSXA$tBf2{oQy;`se~zLzGHiAbhx@LQuOA!0{=Jv>V$p*1%NP2_%Wx*=rk(hO zQ_f__oprx)oC_eyXaHE;LZ|JDHTOkQfbOX;`iYb_LrGqRQ=Fl_?^b^nwV2Ca9yZu8 z#In|?S{h}1Ky*8AVJZ~?QdEMbekGk-^pqEZzbmB|WImk*Ay7oKA~kL~G-Z8hv(41w z>s;or)i2<~4c{jaK%ML}NGjLN#CKOm6cnx;`E6TId4%pa0uQEs zMc~^Q5CGRU`ZTQ#OC5?bP|I?Mn`8e8{Q5)ZeK6jU1wwUQx24JEDm2{sBc$7YCL9!I z>&6PVt@q8XLPR)}y~2tVx@OKmAbkl)CAU9sJEFW^tX;L|n-Q!w&7*)}bQl+=y);x= zHQ_sF{Eu5A!)hA#ek|*fcVOXy6W?lXZ2A4Gk{IL^%4M9)jMUNbyjO5)=|QC^La1Ot z`t@G=Q9wt#&+PbR-)a+?-?ca@Zt*9G`wb{AfV~r5qOsU#u(BMyi^l?L^AdKy-yCGd zC{(J?Rm$8r_v*IE_3j6jtxT?`A8}k7$Z2R+GP?DYlikkSrW|--uJ)2JeflS@YO&SN zmy!fK$-#sc<+F9ge>wFVHBt?UX3f8BA>k472t_2bj3dB(6*Tl+1OdSumnXHi?zXtc z72-LnG{VWua$_D^x$*DIitg5&HXrtz-{`yYp(-(dljkTxK@>g0#5Z24Cf+Uz#^2oKwdW-@T8)FLgQ|M?BERLrq8 zXiNRB`=W7z#8wnxRac?r>WgRabKqpr{B8YXLH=SyQU?iYQ=^?qw1t8==E3}U(NUhO z`iTIUA8|INSo0gNCAvPG-e}rot4F3)`zSyZ7G8A&UWzn<1I-?@_D+ zK7Tyevsv5<#_nW#}V5 zLM7=V*W7Igjtw%P^PEeV#%Qap0sI;- znPU!bH)^&7YyhjNH%;{9{s}Lc1_46gBioPiDW1L7;tzr}5hX{B>$U)`Lb}QO$YrD= zM({M|B>S0_0y;jyrRmAm1vGzLi5CMeJf#>LAHOSUQwuY8c(=YiZpPQDF$S~?kHynn zj%tM`ea}kvkdBC3h1r^-jq;+AaAY$|{)rp1+Z7(Q=Ed3R%Z>Bf-AJ=iYkjK+5Kxib z9`lsM?w?z;`bNWc=~zLB)0y{Yf;XRa%5QdukAbx8c-{D-{v?*t0Vp)>m*}QaExL$( z)LQ$==x29U4MbCOO)B%zn$3w^^)kH_%AhgaL`EEswXS#kUUMNZNOc%*LlD`*o<&BL zfyHWhzX-T!u`1h3d}DwNTG!3G5du?8n`Mi}LMyEVe0by0*V4mAEwFF%cC@qgI%zhP z3U_cEU9CZ@cUngN)t4XkqPy?R>&4&&b`d03Y7N-+_vs%HwxsNz7Xiz1voAG481U(k+#Mu6^3BX?oo z#{tPODdT&4s-Rmnj^{rSc%m=Z*_qiuN<4>;XEIj3DU?*NmXSxbwa#<>WSbyLR^Zc% zBA3A`@%QkgnY5uMXez)05@Iy1*bQ#%$2O4B*t_1IX`KG?YA=I-yQlTMB;xfuss>p` zuE($LL14Ul^THd++>D6Oivkk%5^~>(8wovPf~iPi!h3W0iNzZ~Z)DedTuFz}W%HFH zO};CVfCrh>x?e+!;8TVECU07CaVJg6fy33P<3uijU55=;ZEeBE_2HCu4K;y8h;GnA z*}?MN5I_a`B3bv4%W0GG;O=ap#b3r`{tWrP!oEkYlZrX|$u!F7&B9ZOo5(u8d(jPwm{YioQh1 zVjOqgib6pJaSEc=Y5!ixf~h`iSdKgn{=OH!qxF1D%+vF}{AhWeI>R8J${VReQ^$(O z+S31d2Zew@WUveF57>_L@Ol27$7@U~JC32feLgvS8%t=iR`oU$hP(K{VZKZB&-eV0&t_-h_l0Z02AH%%H7lhV>7VD}s zmz$?UtSY0jEmo@t2{u2<#4sYcAGQ-wtYCs$lveH;&UI+;k<*nRsnN-x0kV}cV7~|Z zmwh`bnMg+Elxmfr>xnipKD$_?BZ4U^*xTv~?Vnz}cL?ADD^Ew$FUT!;D-VlO1?jQOW;K&5s~MI!uhv{Z8Wmm)2j!mW(>26ni^+I>xEA- z(oxP^R2?_nlPYZHyAk78aUazJ!~eK@7pV=fjt%|ZuMNYBa{n{5)w~a1f}tP@Ns|HO z&Z0(=l7K_2$yxaaO@@=PIXXe&WdYWdFCA``G8vyWq-BFX0rV7ehtu6Q9m6lcHW=u_ za!;Qe3u3eVlyK*LJ@NO^GOE|=?aL*s~jY!TJ*O(Au4$AmLu6gs_hXD6~7J@9qIbe^~3;$e`$5qzwCYZI{ z(*gkIzYmY8NkM>uWwQ$|SSnV$D*W!b)lE<3*T+leIx(84(I{Nc>$OGij|OV7{i+LWoptem-|J_wV8mek zh430xDfKxSvgXOcm>R*K&W$ULmG@`7;qXuq6rd&=@=RKQ>&48;*=-d74l!!3N#r{Z z3(WlB1gTcIblC~zs`zKMib6+78OCjO_Gc&G;7@~Ar!9Upp9vL{(Sy}xNxZf0PQZJG zL{O-k#wpdN%l2yUOzpK2Fu*#m| zp_t=~{wDOjdA&WW{zu?d3IPnDDws(n->66((^SDkA9+yydr%!OI7tw z^T+_DtHgPk`fs+KUB~u6Z^2q< za2b;`LBa82#2B$q1+b_i@OogTbkB7k^KiHPrcr>)COQDCcRi~`M8C%JPF`TJ(tv19 z_%;BBl~^FLY5oqf=;O}%cN7$$_fYL7;Sm@m<7QMtmX63xUp#oP#;x_B6#n6d|I!Jr zV>u!B$A04vX5x_uDPoCAR-2k3$8rDkHvHH7A-07X3t56eH3Jh1a8#@P{3hLmJYz_L z&#xIoy|<-OIt9MGUlW`YUhltuJ*nUFvkd{HXF8N?is|9)?GKFoqEgbv8)8)P>gKpf z-L}nW15Usn{JC>my}|CnxG)uix#7-dAe`+bUEHXcQ$dR-Vt=%|#Y!wkk2l|_sW>^Q z2g3P~varH+BSCqkduS}mBFiCHk2l1=aui#fO0Yl52^i(`6-gBqQIBZK7J`$(OYak#TtQ4#6ni_Y=K#nM#1rDtPt4(C2j;zwkO_4Pl1WHRtbeo!gQOP zXBYIdQ@!VF77|+!tFESoLz9oZV^X0q?_XFLL3V$sas3t^B5?X#U(gqf1dLe7;cTc? zftwR>yVft#n~y%OYWUwYrtMAzK1zF=5+-se;qlznV3OQdwKQs%#p*>z~hAiSdYNpqQde7v^WmWFC)d+IrrZ7Z zStvirv0m9SI`!;m!XQ!@(S8M2;?AggWQbnJ3K!LdCDOBuMjXvj3BqE9$(Y#WLfM(9 zHz^=Nf~h6~D6js$fd8K-AarxR)Z(RLXf4PC_s8h?^!t1BwA9`1VgXu#>#+Z6&3-6m5BfYe`X z?b`33&L8ThZRN^!Jpo5Ft}!=gUDD zLlpV9pg4m$#OhL%j%U-Z2)49|=2}v--$5a=Ef};kT^+u`h!;qKHm11l&ETT$TnQ1O z$08(dD|19V1v;fHb-W&2jkW-Wf!#l00;Smzx^)vs^WHaCZtP|pz>?Yj78)?}3u9}C zH*%*Bw#K5Bw9xf9)g#L0|Cn-n-j3u)v}a0aZ5H8!4}*FI@#1Tqgx`1ITrvkS~q#A7w zmRfPqxxw#a&hP&`E&8whDY*iqvd|`wZu~nlVv6K)fQ-d*j@eV3X#SRkPw*1L{r3E( zQQW9r;5XYVk5NQ)eLG~k(|*;2E83ZkppcQiiQmtOZ)-xesjfBfI9kCPLBn|;jkl%Q z(s(q^C-fUe0#Hfv8sDVrK^~mhe3M}eYBPO=Q*rKnh=8IhEb6ue?WAYu?DV-@*7sU& z!Si}(n(~zzLd+EnmG}UA|M&vd0NG0Lts~`=evc&y}kPsO6kO@?MiAu1l!&v$tcUy)wu1YwJYlu<<6~uXPxtikrt(` zWh%k-aZ2>vnp#IzvDqrP8RB@30Zm9lFU8?f#&}hrF%{*2Sby`Jvn0xQ8-?hsZ9XrJ zR)zBE&h6eJB0Ra)fztllWH;;DG>(W~vYdjWN#im(TsK#&SP^g!pd{f>|0DkVuX5!N z3q6)`8CKlF6^t*(Uz^Nwqur|MMaMF}9uoVy_nxm^pISqoAE)YuEMP|kZTH#jd^s>d z*zWeNvF#bRMbrmHvw|61OKCxKDG?s?3o9R6n1h-Xlx|y7dWHrBF<-8~ed~C`kd&;N#o1e|>la33w<@wfx46^dUzuI6v3bcCKa~9=tGla;@Y;^#O!|G57A(UT$aH ztnnW;it~Ep*?vqEDqEu}#pUh6qHsE#q&hNHwrR3K!(I9*Y1&eJmL>N^Kqg58E!Jk5 z@NOh&S>p@0e1RY7{{FKooUQH4Kpduc}I)hlHR{OqV6H3W}rR%AbO4bFcF{Z=-fVOTDjH98F&iF0GbA25(pA z*KuhS#e%j6kYbVoai*twuPKnJ(P2Sbei7KxzkoT!zpjwKf9Vwj&8TM{ywk*M`gl2M zksZuhrfRw^?1QLoDx>}1UI6=szJzo^T!o(xyS=vznxlW&gm{+2wJyp2DM zrX7*|kcoi==tdNo;t_&;n!bmX0lT873~MZBQ5X_He!w2qlSQ(-mZp4*MB8f%!PS1c zO1t+3jv_ayL^)D){&4D8SEV{MLqfcBY_y3)2;t?JATE%;-6Q(8@O*kaJ>c)S+g%yt z{M@#UT4=x6@Fdl}xsRR6O{ZXA9PVaT3=E5W$D)-+Q<)HA#mFH6#tDao(*4#dk4>a0 zyYv>)Fr))>xO;PX9lSxyg?Cy13_S&+>s^=`3Lhy0*) z*8ebGnqMAh$v*UloZr74m#PQ0I}$cvGyj}FwMr! z9D5zw_58M*>iXL8LbeNt3C0j~TLNUqv@_*izTU2nKdEyxn1pR@9-&B6J%yd8;+gv; zc7F>D-pXOTcDyYCwG+3avr*y41tSm^<~jK@jAUHo_>UldNJ>P5mCXv!0RPxGn!id0 zwhtey1tr#`o)rO*3k6M#n$1UN7IgYvx8*&%ond_Hi}=F|3Dl@UC?ha+>8rP@CrFis zu-h=mQ_37X_P1cF z`#`wa>lf8TJ6lgh_EL7m0IXO_+=Ql?BE=BBiVV~kVe>z&b7za6`=);3j$~Th`r)$i zY{!U}@p;XC$0J(`n{njPcm2HGlyA;{@JuPIJD2PEAtSvh+1OC!_a_T|YiZVFG2ZytBSbXH`B+%koPMRlZct|+ zQKpj#FHJ!J{MN1$`aK|hN!`_1->04gAc=VA^=I^7gcLw~-f-q9k=#VL%O!-*$7v43 zqLEtnc7i~C$9F5+y>=Ziek0#>WCnh-y(}&sw3pJ%HH{u0R|@-5NJJ5KxgpE};IL2$ z78&38+ONfxdn+&E`T~CU==0rM1f6+fq#o%aPNTN-!DLS;K+jtMFNOoDB2~Jzp97x^ z+m<5wqfYOiueMqnfHJ7p)K#2q^&ORaD_%Ze{k5w;s~9LBlr&X7tump6|;N#fPl?@T=@Z!4dh3Ob78xdkxDs?*~Anyvm@Yw><-*2l2-Ad zKrVID&75?oOjU#M zcOuWfo?Oa{DX9Q=+)?KF;UO@z14+f6`LqT*ok8(==+W_M`UD&AZz~eoqg|W*?s_;@ z=JVKR0ZhC^$Uc4EX8QAz;&<&yQULUHmji*vK_Xt7N`i;IMJh^snPKyrE~Gh_HDUs~ zN~t+#CrysO@Q&wbl;~Pk&eH+pw=#mgH`};&)CoX@)Alo^C#WnRuN??Pc~Y%Y%o@R( zA29H;bO$XiE?fDEZW~SQqWS6Qkx8YR)nam&FO9$|7_j;>zOya$GY;2ziO;C4E>wmB zuBpEI2?o|?DKGH(>hX;r#D3`~kSGo#dCs&SNaH>QS~dqBe(3|%*+nV(PU9r#rQ*v2|yCQl_W*Qe+Uolu}@w2KV{+@y@wWXeZToD zMVvSZM|`%y?r!U>HJHRMAh_;5z!rgMI2_A7+Q$~SfPFV;sR6kpR9%YOCHs$rwtqe3 zhAh`3vL9wSs>sUrJL~$h8|+35kW!<$Vxs$p3`sWbJ>@344Rq+#5=xCvr=Tpdy|D|= zPm#_yJFc{5O+km4yTwz$+r$hFZH^y!5&S(96>-3DlKVz?IaaV{ zsS=;9E|B85t1cLV&6O?_&DQ)z%o}&IqWmPvIUFJsEKG}R-l9rsfvF)CC{qw zRZeUkQeYa`pA6nBTq<(^<>#JC1hRB662jW#L`iJRiET zES9uw1KaJcz|2@E@0o6=@gMm-Np-r-k-#eA6CJx(16N*`qSGEVpbfPdJ3rz!v*rOs z5h3}Z4o9Qy=?Q7(N=1;9r3DPJgQg{qI*)N;(m*SmSolU^c1sJ@n#e@B-JxaZVLrD% z?)EA7(Rr|J8J$_52$b&Z!5C{f9VAc0!j_K9$vAw^>FuAfVB4 zQzDK>;a8Pz{dL~-9HzbJ9Jdv?>Ab!3Y>`na7~3pm9|EJS@fP)~Fs%|Xq$>OCAQdYG zEw?4@n#;l}K6OTa!FVZ!Wch~@-EI7@qiLQa=@uCK%t#eYl=ru+I$Ib}>L+}T)Q^;F zKs3PivR#rCG-e03JsNS52p@ekFIK4%q=Wp%y#pJm&kU*S%P;zJCySe>iYl)C@L){) zR-c6y?mtfpR+ZeH=18wCs+MLS2fJlS+-V9{TdCH~X`tQ2d9Lc`t$}TjXTZZ`K9|FF z(PNvo5Gd)aX13(=`Dfkej>`t{8=PPMda{*sS5n>T*^R@>@#l5?J_7Vd|AI?@`$9?k zsWOWcO|)#StIjF~O^uGEtNAP2hUC5`-_-_ulkW=yxW-2A;?)QJ2iX`DXHubmUBfx1 z`m44G1ziP-u{ymq&_8cc$yq>))UpSPtejeeGE zQq{6&rgIG_P=E-j`{TqoFx+=A1eG#A;X)K^9B)P3xIdVs>tgmxfjXl}3^_tM|3og= zA$YIjdoYB>Ni5cQLgPyiR^`H>2DxRxjHDuL1^9R6^Z<4!;`!1`t62LsfLdBa;Roic zaU|HIulwAf72N%SN1)6!*C%f717b!dW5IBWfwXHJC+p#{6B^^8*Slrqe_(X}*F}KN z1r3~uAOV*dsaU4<6;R}Z(}u9?`x;OA$%+g;bsGF{GXc#mf=Sp9`GoH$o}|FMYxvKl z`+X6@O(!)9-j58c?pM>1m400<7Zo}Q+Dh73bc7DWEFcorUr6U}>r#!UzIe9P%d&R{ znH2V!TUKbFZJ;ptocco`pfd1z$9@*OyLGktZI*K5T>gIU2bILeUaCGjtozVD-=^be z8Q`qcE4p47`L#i*!sogS4qoIclZBM{izV}$1R$gs5&}S=g6it8@5`o)$aa9_=6B2l?{~WOVNm$sgM+=Cx-sY()Jk$H{gHI|;|j zwlOdNN5a8>PfutOw95rHujxGZHM;$jd3qAo-U3-^G-#4|REE$*NNdom^!Z6NE9&l7 zg90-;!%bwu^TwTuYBZ4$%<^5LS%K+M=Y=*RR{N$P zW_uTgHCqU4*f0 ztfh-pK~DV6gCV!4^@4i<8-iwqch-QM1$QIUjXBoE3>bm}Vx3xZ{%VuJM~*V}ZCXWS zfcoU%&xMTyV$thfw)J3Qf?G&r9d&Xtqy_oP9CZ$ZazXGZ8Nf}%YdjSHr9)QZSQFqM z2MzyvxyqcoQ6E+wBULFU99RAp*Jv4lW&_PTP^Mtp!O|?ioEe?)T$*#}dF##!x08z=2Nr5hy`%*;L|eI<4G4q3mvs;z;b4W@&kx zRGl^yDOrg!E8fA@HYEgr@*4*2?!>&d3?i-5`e1ieUct-;sy22+pV9Fe%rl zImpF@4T`E;-3IQS!hxL)449GVWrd&Y_DrK4)lxWC^%rC0M`K9uBc+X~;evm&eC%Zd zne;?9ZGP`I+Drqc$rVCywZyzs+e0Cni_KJ9-;aJLN*t9sefqYr5PHR!3R*0~3iwiM zG)OdA1k|+VCu=r6U4fc|ryTB85I0Aeey{1G>wy4Slfo%3 zpd*@ugwIvMXBLSq5j7Lnq}$JT@yLoZ85mSALA#z@ghF6!hTXHZsk3Exl}}bsUTINO zxmDsBqL6+d5pQj_`1`+}TqqHaw>1^{`UAVS;(CTq6`=3rYoi{Yr z*LOSRJVaCwqX@q~2^FZLrqJ;~oJobX`78?jtI7G>7Z4u}xfGw%UTR#TvoDk?^8Q3+ z_kNMJh^OzDBKaVeiu(CX`bJ|IW)?emm($p(MoW{<^A0@#5lB;TBKm49>ioGJAk}va zBLO;lY*Pl;URdSI1GV2iT8Is^-J);r)C#||nyU?SX>OU1a^`E4cCNCa3GZM&x$GdH z$m5AE$u8O+;xNH$$IxSHx32_}3KQQSP;u2|MGFeAhNSC}-L%HHU4?xL;)GVr(d8C? zyyZrWtAP3Xm8a}wgHB)bcHjFrhHw{qzH_6}2sH!^Xk^hJis@k`5gZv0#zks30+7U; zs`g3=-hfs^GNE&!`iFgpLo+XY+x|+kTrAWket3+(CKUYP1pXsWnJUd6pSQ}6nabSU z`EV<0NKlsWsCLay_1tsa?yuQkw8;G`02R-cNcn>MxS;A=>{g0=-`0L2EEZpW*0+b= zdSb@ZC;Aco?~3jyuF*887`QO&zM6YqMZx5@T($(66g#uF9Z(9MHT8f-(`N-$Da#nOV*LLa zP_8&9c;fXUu{GIWTW-0O$mfm*z~BPddo7MG_|R>2xA$l^t4hub*ylT~e$!rBLG}p_ zH`(vO*$(X7>a*=2dZ5eJo&YyDDEW$1OWYnhU<%$9Ze-r60oG`buS8_M;g!jvK6Qw2 zV{=1V5UnJDFFyoBA}aQZEt%(?t0~}|I^PU9Sz+zGBY*`Mp$k+%Iyoz|5$ShNt@pct zva0FvThzWsjIbn!QNBTIcZc$@tv)dDOk}I{HOk0LUT=n`blFJaClSED7f92i^h=_K zBoRmj%q=;8Cqqh%Mb;#?>}ob+h!x2lO0E=zuu#>G_Z7A;E)CXvVp@tnt>{VDOlqE zkz1(=wTD|C``#Y^V}}UduGUM?jsuoSk#5`IGBpzTQW~nkx_Ya)P@!s3yfi@OI^XX; z7cRxfR%U<9)%(t1hvYbS*1tHAg51D;HY0gFD|q2mJSo#3a26&3L@Hg3z+M$HmaWBJ4w3jErU@rX)E4{Z@SPDY^H_}aTz+dvE z9Bp`oH0s1BlOTE4$?Y&Y#Fg=oI5e%W41ZiyaEf8ci;Li&b_VXm-01Wpb~Ob|VS$tUsz}-7n9;ZI4V8+Sydn zX2T@wa+-85@!!1+n9-eUZU51$|KK`O^tw{y0z%;({VtmgE)ZHpGj&4gN2^)-mm0?- zCi4wGge4`y1C<7~CxmFSG8?0~_{3XvOMPbe>BecJ?JiNu((&j{F})1KB@NdlU7W#g z_amBst216~`4{zp=s~vy!ou@UpRibGb%3X@_5cwvEm7zCXPjOM8O<^>i0Ys)R z4(F9G4 z;WY(6Kb9j;QFnkhTD=X?I7e{xX3Yfrg&ga9Lj_=2@K5T(|9mK=1Q-Ph8~Pbs@4><6 zOY_H_qUK@ciTF5v90BB!3EyU?K3|$ftXJ0oM?)YA7{QtB$l{e~j`miWC_Ufmxknd3 zpmPaGLaG&{8mmmk3`bXSe9oZQBNQE4uIDm-yzU%013`(@+1U*gt95x*G&R_Kw-{&L zY+vT}^{*WUDw9rvHy=qrr)tpX>de0x3auCofcMwZd$|qdNt>^DSZw&v&$!CE-QAv! zb)UW8dcgZ>9JvbMJ?2#3fwXdZt!?qT<5+!m#`QNqC$jZ&=VaizYD@Nb>tcGgvSRk2 zIe1lQDrLOLDIaaGB`tTiT5r`RZj8S2K&9;ZEC+0)q#*O9e^3}S2BO$36FR)TFj8d) zjD{Y*rvjo-cLJ$)j1|6Lz^fNyu5p>Te)YPAZm%#e+;tnG&6z&z?NlwrN6!o}ADvlm zl~v6YB*N-m8Ap5D{rTSCL+25=gpA7ii z+5afrUs$Pg&J&BS1#z0qEc?E_u<{XyF$tw}_AwWjp{!tlZ8Py!TLsArkJOPw3ZbHL zZ#!HE9`1hZ%HdCz;+}9G8wOh{2%^!O=i0DL~*{cTWxw+-RdW~lf24Emo30U)O z2+bUg|K+1$g8q$em3fBxXwlMzXW1r(omwjBOQia3ENS-H&JZ-!jTIUND8T zfx+V4$!4ul-Z_$I;#_`YQLnq;eHjm@riyWJvN z3!i@imYWU=KuS(VD&J%m_`M832@nLYtu z<)Gd$F)(%tpaGVFFn@`czfq3ivLJU$3F?IQL9Ww5>#0SP;GmR9-JizH)#LN$V$%gz3> zJ2@R({OOt^oMFAq%`n^VyIDtYR0d2S2)d3u=PE2mN*d7C{+ofXY>p+o&>I*jMrccw zMS<-R1CV|NM3W%lJr-3E2gU&c?~$xiT-!ujVu7ElR;Ng8LH7IHp)qT3K8L>q1hRvh zUdA(qFpocexGqpMq=>vV0T!7jw-cO<8i@DXXiOy<@+(U0Lmyi{*ezkZlaX@vl4oQi zW=lU^A9VSz`A@y^GQA&EAi4UBWyesk?Xukk7iCKRz3P67Z!r21*o<^@k06!yMHu-i zcAKTpoN*L#h|HeQuDu8o>RXFD(br4rP^j4%l@d^;;L$vO*-{0<|vk${?V!gf=L}WX$_EmW7qW6o7+u8L- zUPKV&`+!fm0jM%D$aP7Di59DdpIZ z2k_y=i^^V{WWW-Mfuw3`T$`%yH*+~4AbiRWw|4)XBRMJ*f}}V=QkOk&F!;CN6c$ny z+xfU$iCHJ_1eUUtrf|uyV=eqU3YJX9uJdE>bvG%6y_3muLnr0nC*Bz6N1MedfAi66 zK*-T%FEA7Fb%G%vo|uc%d!f3AuF31l+k2s79j?4g+_qztydaKEzmD*zA#(QG@9nhD zWuCvv%?_A?%+T}SxW105k5~ISsDlzQ9 zael2K$q9}}US|%7Chp10Hvnfnc$rJ;^Y6NV7>A`yOU*mC=azFse1&GpT)-E_7sKv? zKQnuZQYkFk$Naz)MlQdgNr=^=v!@Km;U$bYct z^-2&(XhVxU4l=<0`Vv%cd;9V2&QCLv;PdF~upRIZ-Y{Zod4PULLWn3ec$+;N0r3nU zkT?R}OAShEhQWSeHrN+bnZZyhgT~1`82c7T*V#truBiZANdGtL;rJj%w|C?P7C-6^SbgLF62 zT}pR%$EHiVTco@DzkGi0d-!z>&KY~(`(EpcIX@E>aC09skK3>Hdy(sHVlHT#vizGm zzmo;&;iofxPne{bDKGDyR@)zMs3c9)eJjK}Rg#<}+p3t;Y@I15jj{*O2Y7tWcMzKW z`||cKygkY9KuwE9p}$*I%%J_aLx~oMcD;7D?}t?s(BxtGx-|{aWlJ&){dB?hrouHM zCux$_)lS!wM=f~!#tsA7aZjnG_&d{)!Ey6g$gH08X5AuklAwUviz4HX;6K*6IM1=~k=Zq_XX{AOxQXEK+NwX^vAVQQDu11a~+l9vTG^AoNhTEzAJVA97mu zY5(wKhY+waK#COQ1JE@e8; zfKe_XGt~)%XBjGZgRW? z!-}_8dVI3!ieumQrWi1e&&NzdTn?r1C<>qTFXQovxM0CjyFZ%gS>Hd^dIl-Tmi@x@ z@cOZj%o%a}jA473$a)GXU(bojyf(IBS#UTlF*K7?h3VapM6C3Oj$*r?Ob(3XJOt`Z zQycB0bZ->2m+DI2bMx>8oXmpTsAe@28_DE959=FrD4q(&3He60vCe1~Fv76&6nc!i zN5+%TX<$NlLkLXbCHBNGTLP@-g`sux*_{Q-f8DJ(XCeWaK{wht4}BuC*Gnm38eL=& z&shwMuhKv!|G!_4H$Uin>R0RxPuBPa;1L^F?ISVRP5(5xynPe4-vioCaJB-sPUGqA zHt8q$jSWN-A5EvZp|O{EFL3DhzzylA%_L+O=(tPG;HA|QUS=2Skgd>k_ouw_!5Y0c zY$8DA=_>fibe`u;`<6IjUrur90c{Kg%x!wc(RtVhxwMn@8(T8jH1B6EnVt0ZsgtMO zfcL7Cd9I%@vK+!JRD>u9m{d-(Kscc0MY-G?q|#EC8?H13fXnBHav?Mg6pH3e7ZTHW z^ugJ-a=RczxxCKb)_C@X97Lmq`w6Y&@roGt*(KgUcUIT@TCTD<>4&gLuB67sa&Z~r z%C~x9T#yDB2IlQaT#?(Mz7{vYJ`U=F*TBdI0|RdY7pu%V(qQ`+_E4c@`YO!Zy-Y$@ zkYuLmn;Ng{I~<(1B1eBxx#R$<$%W=uVJr#Y5A`&|5J@J^JQ2KExqWAmRLJWVrFc*3 zwG>Q~fzIrT|A`zHI1mC~gh~8!7lw-f4s~p+&^_Nz8pkmdp>dYAiypu;ImF?7YZqB0 zRVh7&PU-_xA5m5r&$p3j)GOX+TQJTK#wK@w9^=%0UanYZw{XL#RpObinforJO8Y2N zr7pv7bYH6;<4Fm&%ttAe%9%?(2B*U9|m z2h?`ERyJezn_E+&z1e(kD&_BzyXmrIX3E_DJyNm1O5!VY)n2iiHnCW!Cj@K0mKfcD zLviq&eORcaT$NR9(aHagD&S_PI78g=@c!>l_dRBk;PUITpV0^|nO-ZQL8O@3m-t=O z@=X_o!SE)Y7YdkG%@Mv$6M8De`CdhPg8Wd?rD4^P1 z)_RlHXvO5ypZQ>SzQO#jYa94D$ypqW5J2haxeR<1L&vgO$SMqH6s4*p04`wwI%S*8 zS_E-j_Pgsu=t+ul$qzWR0s1iTgn%<{Ld^fAr%Z=)IK#3NfH3?pk0tRrQ~z>;kIGW% zYB0M4Upgho!ZvWNT66@Fr93hXnH{q3rxn!P6Y)0}=4-O`_CP_=t&s3e?cyEJ6y90y7%_9EUt{}@95z5WZ(S@0ch zeq$l+O)DL{M<2cRgX1|ptfwlbd@n1p+S-h9++x@nP08v0-1;(--#{b`*EMO%F-Kuu z_cW$ZBT1RVJg@KZn&@=i)C7QUP@0_=PiC}mFZNAEYR%T2SuggCvA?(O?U!rs_S_#Q zhL1WmdbK^b5y1JKt&Fz~dsUuYT)Ny{Dz7i?`OgwH7Kk*y>lF#}Ub6Co=VLYf4v(k( zqD1@axJZ&>McbIrao5iH#3{#q6ObCdv|K1S(0|Fk_UsLtL+i8`t}N}|^r551cyI@p z5_q3qe?;E6vp{fV5$X6K3LPu4Efge|f}M76&GL=yKcZb6HZ%c2^r|^f``xZ&MR?Cr zL0qD($NIIpi9eBZzB*Xk_S51wQ|i#Vxw_fHvtGw|&lI1Z8broK02AXU(q#nos-T?& zse�UvIoOtbv~}unhebxSAEYdZcBc{d9AN=d=th`S_tVZZ0C;U|Y2m+2u-u-SejN ztu}(?%^xBpY8keTUZTrSvfr1R#H&qQ$F%_Mkhfjx)Vw^|A10?~%;eFP9(1zdqypgu z+8R3Kq-KvMqwkQobT4arzdLR~LSn|}nMks?$JB67S9CoQOh|V3;W^C{h`=b?zfXy6 zuHcyJ6pKE95(uGBuNFz;m{8USjc{O?YmSRj`RD22=zb$|X1Y(j8RY0iE8s5NyhOl?LML zsAz@$E_A%yJTos{v`R$V%oZTABmhFbU#BkCtW{S%5wh0v+?5%))!!>l@qFcl_&(S z-MCPC9+f7D0%}$gup1oz;d8Z7d5##q{*p6w^jI?aP?~O;)%oD$z0+KOq=W8!%Y5-5 zdtj>Cm+2gv3I)o?XF=X~Rwd%G)UKcMH?lzKw{v>4Ne!GWXQQqcv?ptRvhDlU3Nxh* zgL{*GJI)^nS%i5H{>)?*?OMNMtbG}z(rLPe|j@0EGP7!#~+8#xD9Yz{6L*4)Y|IYe%rdS733K<-Ix)pupQ|8pLUN zG1o9o=y+Pj0IFp_o21$G^D+CL<@w9C|APPtufrk}ekuFinE09vS`)!V2soSUw_V1F z=V#UyLOzf)GVZGv=%#NM_%2?odUSqe;)qkown10UR}HAOuUyE2uk1VsJFC3ITZM9w>BoAzFo)d{%Ls#RG3dTQ#~? zTE4n>B}gIu1eTS9wP^mWYoX`g&uKid&ZYfcL%TD~4Z<{nC)V{vDvkvTADMZheml>I zJ3&MZ-(K~bX9~d&s3MMP)j9mOPaDZZ#OI%*%k!?Mper;w$UnnR$ck=0cIu+68RANO zT8YfD+^waxwZuq_k0(cno;ld@gfMh87XG{b^~^E>d0F5eKUsa&R460nvxcrjq!Fb2 zi0I+bb&B9Qo8PhuWNRv^5^Nl2CMXyr({^915O^bd;jaym5@FE{hVp zL8D1OkC4BudQ3<(7!{J|Xqi0oO;lc2726SiTjgFti+v(=UMZs$UDWe1KIQwAi8t`F zM@)YZ8;@Fcm{9t(pHd76aO&<)Dh&nOY0vXB?b6eM5zlZ-QYr+a1#hc2UbZiBh7U1l zIYK9h30cG24dAgFj$C^jnhEE41IJIm173@fx^Ovp&&wX<6Il#4ym2hhkwO_Ez;rD# zwWrgpC|_@Nns!cE4a3OiFek|y4vDConH7fK-GZiSh=|&$1 zkEws(r;Mc9H@4oF{k_(PQ$^cWRas5ej@vB}Zb@nMKHkz$fSO#)Ka&1mE->MPztYbx z0?2C-kH#*h+y&0BS>0T#@{nRk&``G28mLTnxNE|`Ga8R!e0_6wSdUv{rBfw>E+kO@ z%2Cjb#S|vZE2HXiJ{ric1{(kwPth=id-1&Lc|O z`b+iC;8f3|K7n94NoQC@7Rs3D2!z&t{T68`zsGo;K!YYv5d>D`+g18 z-&L_R9^9Tw^nz!LL4ZN>bP)Y27M2BJq~W+nLf0#HHSG_YIAPXsoj z8MU|Y$mV*I4}a#`T1kcy>Dxax-&?`_6;<9{R)5z?9HN0Un|owuqpQ7}R+GW!2!+{+ zyRm6(`18vru^H-rOMqS=ykUaksl6NF;sbLkJ9rVN{oG(6C#D}v%qNoP#PY-=f!gH+ z*GH9oM2Q5rH`*6v?Cb{S#pNQ2;I^HM&n$!K>lwP=n@+nARXu;|>`b*z8Ehnn>Go4Rhu4$TH`+X}d-Fa|+-iP52G5&nVQ%-?_zAd+zuxFTH zj!17}R1rqtNe;Vn%0<8p6!KU1jV8VCx%Gbm`pn+k%TS>3dx;Xhs6} zICX!F6T$8VR_n8ZwivW0w7Xk3$v>sXm`UCa7@AE}zMPiCa~1D(Aousjf)Cq5PVHFC zqAdt5N{}y;t`{s>T3l$jOxT=ih@X#jGgrqQ{tU-V4jituZSfP9DtSR33BHX!4$d%H?|#5|k=q&rUt`0tW_3F~#$ zG*3id^<$LZ-26rIx}SMHpUWlKsG)|^(NXBsQoe7xxjIbs(et|0-%~c%Uezn8jkT>5 z{d@1W%3tm^LQl;VeLkHQt5U!`9@NzFsMY0m*bi{w@*+8I8Vc?swcm0R97;9VS0&@L zy4VP%mz)!Q%?$gbN(CTLZ%M^s{zYGUyWd%4(^H& zeELC)@agkW3P*DP{#(t82aW0_V5#wTI(#Ruf<4R+_w0t`GfsU8FA8Ptgq0`jc&{@o z_%uhc6z{@dY9DfS4NumCEP5Vw@-mbMUCYf1c+DLGv~ZtwP@n82yD`c$E%t)18NHfb10QV~A$yS`hNN6J&MZ=eAAX_*M1 zO016TUbgU~{i{y;0gaB% zdQ_spN$i}a#HdolxS>CDN6=#-IB=A2*f9 zVVAS#siwx4=tnJ+St4+<09N|aaqnb_&YRj82Yj?DVR9_2AHOrmGrO)11s@)&@uU)N zrXX_GYe0OFsmZny`2mA++suvsIt?j^#H6cscEPSTm_f}49nj&|0t&y`U!h5H6=rx} z?__VvQIK-~Dxq8R*d{60eBFnZ9eur$+j!32VNE&qKS-|sw^J`5hb82Ew=u+Q`DI4h zt3y~@L*#%+Q@7HW+sB+P9jpGCPyu$;J>CFcH(34XhI1hj!`GtHNNXc(C0FqM?qe24imj znq7ofFu2!VJPO9@=Gtj=>|fuX?w9L=G6)M#r$+6O=|B-H%BfI=>B`__&7kEX2C3TA ztwo{rMFg{nAIt!0@S6}ud&U26gZ*}ty1C}D5!h7vG`@?$tG!<#8V;5~i1 zqlYKBP`C9mK?p{&ZgpMw+V<2`zb0CE^m4T?aLozcKQ&P&At}jbmWlRS6%P)@?){Po z0h`oXA6Y99 z#CSa(k)~hyW4cpitw`d;m3-tg9?u^2nBS|$)J$m9(!#Iac0aovt8m33zljosrdNeM zeoT4ySt#p`N&O;P<8>sS5k*T0J~nf3wN<9zf=5RP7TvGNzVa;VeDC_KOsEo}|NWH- z2T=YhNyY%>)5kE}Q+Tou@|>2?vfQ0M*6mxuAA=F}s^xyDGWyv)R_&QMOsg<$^%8kS z$#D4)N3PwP8y{fawXme3;PcG{ij&pCiey((idlKAzm)5U;Ht4O`>pFK831H?f@z*j zSk2Dxk)CG~VRn`5`7z26)+*)8!hBu;PO+kGbrOl<^t#fZPt=R}5p#=18c7Dj-qL#7muesnwwyoPFoR1D-O!h#(a4SiU1-+u3#7$-F^6Y^(%4?3c zH#J`fYjPhKJR5Pi$m2taH#XwD>Ik@HNcCQ^VZ-!_$wnIq7wm83JBa4b*ld?~!VchrjOtUi?}EH^vQiD0Qi7W#y|64Ln&|qeiDP+AGZrfl8*X1^Oe3B$xu__ zzt#p*(plW$!#OCNzL%NhEbyJ~TJ;bs_N`$;#Oo!)&YG{P%B6%0S&vONKgme=cv(F8 zMco#1F0RWox0HsY`Qlzs<6k%J4>vqIg+2N3vVt9AnA2e>tGQ~9OLtBis$sACYs=9n z4`?~T-Olo`7tJGK>y!ucV-?}{1*#)2FbE!2pT8kq5o-2+N+tm>t`K*zC+uC zHtQ?-4lUM&SXM_UpquGpm?43F!5h1)6w_K!?wi;MP2)@`Tjr+r9r;d{(0@cnuv`oTtaSMw%ds7eA)g54-E^RvowEaS&Xqn8)W z^6bX@iTS57p|%bP2>bTM7~}2^9?zb{nhEXS?j_RUzPF&Mvx;EofP#OK>1oy-uV&WF zD@xiUSBMFA|74_|+_V`4uK6etM0kIqih@T)`&m7Bb*_{I)La0&M|d5x-{0H$c%^E) z+NDxW=GR>D5t4IZTPhZxQ(R_n!-2UTK9A?48&*9R!zLubJ9nEU82@BpbKunGFBokeZNManUf9q}Gwpna~pE|NZK|K*;&@LjsEu z2EsX5!gOgVWc8Nk^xHZ#-~^=&-u>0- z?YvssX*ALPy;sU*hJksZ&Jm2?upb?vfi>$V3A8F%Y=L8HwCLpXK5 z@tto1%Oqbn=78)Q1VfD2qlR^@EA??H>A#7>Z#<~xW$%SrrDw~;g@js|V;D5-V$Wr( zmhv*R0JX?zk9+x4Q z>^erOhDKX9eXnPwHBDV1!!kYWn|UTjqq;#t$iH^ApUAUe_hvZO_R9~s_ZevZj7A8d zV+>x*n+m8m&qUU*)@3PDq2Bp7oN(~L%JPl)Ygr*Mos%-dW6)mgZ-d-2r~ayw0CfJL z#NN`tPF-yK%vTvgPji8wyElZq8hhGOFcf>T-prHfxZQ5H8KBbLA~%=OD#(ggdwsCj zx)?br=WH+#v-ZOr@Z@XX)RL$avwzZcgW{%q|o_1ih{2Xs*F{Vl15n_!K4VoJ3PpT%fmoHRmf zteU=NHYFl8_0cGD_73QP022A7#OBi2RVuAYikp^vz!AM;y4AVA$z81-iZmX=yk7~y zI+mc}0njK;VlVd#SgB-ndaU1gOyTJn+|^zuK6uafdO1du$JZbLME~4Gh|?1BDc+rV zk<%LWNKGOB1>cRSf<208IAf;Yu}>C--<}Y(N-^P--ofnNv=1<%`v`^f$+y8G@4*Ip z^u}2?Ch`62w;m--fJMu6wdsqL29_)y80FY9a8!9R)$ifz2tJXg!h*f*cb;C5{XR}Q z0dh_JPd6>+86=Z)Tes)eX-T=LO+52V7!~Z#Oj)l3I6coo@Dg?Rl(S-g;XckbIbxun z>bl$h*3Wv1#-R*M3j%JT)2+l#^BiQ?(R6dQ(Z)g@7#f)~FqPFd^XBAi=R0CCoMS-> z7eg)RrH;ETr7-(4TZZ4$1LwetnDqQH+U^Cj#S;ZZxn4Uy*UFdP0tK_*ng7>3>I?at zk1mE4DlOj&(cGIH%t->@UJho%7Sla2H$knC7 zd{=OcaUO75N>Hg`S zGtE}4CfND+CoZdYfhX&}(>@|_$O;rbe25nPL-#?~7Dl&u=6bbQXpe&c&ncA`M`W0O zbU1>L;8%m~O|7vMjAq@tq9^F~a zL9dQ2o;NEv&XfcdGNLW?viTna8%}ofD;IOcQ1plK>6uzsp2s%bd0|>~g+Lm>OGvZg zQGRA6PK%4$oYuw@x^L10$jjQ?gb{t@W!A8GGU}rPQBnPBcUn~P-9E?B~ zpG?n*Hq{~P&J>R5*pRR=ANw(|YlWy%1%E%N*Rcyh_!!i*9u`{?T=4ub8Khbo^ISIc zyUX?R#}o1nh|@j=1rg9Y^$o^*_QQ&PLwS0+n&_Fgp}b&>0XwuBDHPZk=dMxq zU&(gKa&{C$t;KDaRcz~>Wxh*2v22Mr*`@x|!&O+RWb$EQiF zp!v{3f7_%jKlVZ!ApAX|B;=7B<%CtW7b;%Qse%b-T_R}ZLg@{zk(LIH>gCArRy_I6 zzNv5aicPCc9eKB(eE7{8O(l68J5TH|BCv&x1po=Y(UhfNPL-Dti~ad{Me=4llfwP6 z7KOP*zY~QkX_{TYP#C);zL-Bly=AU^;uK8T`F#dTNqD*Cb0t|(g<8~QvYY^jb}5N# zte9HlEagWJ+l=60HPsBp(m!>;Z{&Z0n-(}AN(KWap_LA7Lxb;z)|NYb%Dh=d4*&Sw zz2bg|Qh-*|{i@m}b{MOeX2aCb5u($4_Mq+1>#WF5L%W7^Lmdl+CScS9dU()>HL!nI z+r5E=T(9|s0gk;@aG0DZ3g`<}{ZiMVW?p|BZrCe=1mM`|fboL#2 z_KkC=H4jF0fM%Dr`$+gxo}PN3C_{Zo4Qz|3sC;pLk_y>tr_-EOhGO;jj47rurkbwe za#wlWSbl@}pYLH{nw-%>-NYm7iX_?rW>sC{5vMRDkCXp4M^q5q>c2l5vf@ASu2Q{YPV=Zk09>jM zWE^?{(|pG+ag@<8T&o9x72!C0v+#n?OM#!eOVI1z1XfW2XCB$}eWp}n-^=+({Ldc` z7jTM6-l}m5i?ryuuPY6~oO88Uj0yjJGEqaYsQfGf5+gIE^X=9kGV*A*(H)galQXQT zwdK}?$n(CG&gKo)m$VeMW{EDzv!(~A z{K>rt9h>TU5@R3s;iE!ObKF7KH6}x4mF&NnW^d&p66rV@B4jtG8b@wuG<&r@c9w;*K~cEGNG|pai8>? zDJ20_OKw8#_c6(|2qU^lQh}QE-1Br}rbI7L(?g=AKh#@^A>+%lE^hcDNkggCNyHlH zx!Eo`X5=s@{8gQg0KoG-rkV39kD-A4 z#Yr_skr4u{1 znkSBpIgDdBw30K~l%RRJ>;jJ>IU1n`A`H;H^!2M$K-RBIXN1Q+qJ769$RdL}Ek%Y% zq9-mz3Rcne{k+LStY3^|HRMO<+<5W0oq0MRydR%KGb-k|2a#h>;(Im?19~RclR^6T z+w|4*k%_uhZ-e$i=SlO^4161&Tkowrb=j*Ozq0_vv`?svaM~#!u+bXy<;i%K=#f2p zl4Chw*?t@r5tOF1hb81-D?s?DDxIt-kxn+v;_iZBCn0~^jYj%#CS`XGqvv(lsiqe| z(`Nh~S7{L2Ckm!~>3F(0u~7M3C|%@b3oy+#pCE?RnmOm$yHFrEe(&uBTz232&y9J}sBUSYu4LHO+?Ck0aszEEp8q2HOr;sD%ItE%};nzkRPF;%A!%qwY!TFX^m@tjDeH6O8cn@xwJNM{_V zJRba@t=ARsOPMQtB%+8#@Osv1Mqc1aIMz)ad6E2z+bj-8Dgm30%eFZsPG4bGD4Tce zVyNyb78u)-)?$WY1rcv_^haA+ET=55$~-E@dm#H&?U%dMGP#WoNB{Hw3iN(8eblxX z5yhUX+p{+X)%(46J)E6Zb<(isaR4J~i!PFX!e_d;Hi#EY41+bQEm)E<@!w09z0wFd zymjMnO7oIbUJ&(w|1lWVoL!gS+_`2U1J{Usk$q=bC+)CWe!N6q%nKwpJx zEm|MxzC0G1Z*JoIKF}h(eeH75vHx(p;=Kt{k#w=~_GeXzLwdt2v&_|WkuJX)a|I$t zW_-)T%5=vhX0RUbLPuA@!`a&YmP)5t6@DD`+6qLOb*4MNb>sX$DGyYwQ3FwaW*Zk3 zrX5+Oha1OKL9{^h;dbUw&cM`QA_(-oAz6qNqD`Ymi5+dLhBiE$EN9p-kt9b@bZR@{yDBc~|{5Hq% zGiK7e>^e(xWu`_>Z5to31Cn>={qjo?ELUZ>-=dClt(N#pF^@zFs-?Zf{{3gepdB*> z4<51&3MGWi-g(5Ti~&{2J1-9N-pr8s+F}g*ZL!ewEv|J4h~MRsTjZ#J4+DFYzdy@W zfM5qEmvK(;_oI9xW|%tbp3b<8XX_(e0{B8J+5S%<^>Hv8iX2Mg&Z85$*-l7CI*Eo!R8Htn!9Y-4ml-XIoK%edBQyygq3wFWT{p=X z|7NFo`B$`*_|KJ_CR=Jwy5A+N;jb6F`~ngTzYi6;sIiqObpL?EEd`vqX1noeQ^Io0 z4PE*&5uGOP@s>syu9Y3dBS+nSebX{HI<0cu(O0KntB~jI;(Tu6)$#TEcPz=5r3UUb z`4ljXD@&1sGf*p`?Yf-RqFot@?tMPuV?Ju6VOTj_`cRh5fzMijp*pa1=R$iUmr<5@*O>`v6v{0wCUVrZu+Oom^r8X7mX zCh6|MCNN*agTz!2)EFYrEg?|5VPTD_z3vgoGIF3hP%ofwAw9>&?@t6(W{2pdPw7>JWoJq6S9>iJp z0rYv=>+>a#JoMs+5UXq$>#Q97$bTKHpU6YKyVucz&viX6+b=(68;Dp}6rLK}*0vmS z+Cvom$5|T#U@<9wwRuz$y3CUGN9W?VKTIS{4zchGX8J5UuM!tD6snlJJU>vLv@Ha0 zf95-#f_NLz&eX5E#(Pgkl($Wg&Pl)H)9}5x*^J=ceWSEi?%@U_op(kid{s;Um#Dqb zl_RmfKld|;kQ{nNPUy05OWMQ7I0maUA0Kt7GE&IueYHsliZpc!=Vq?B{=JjXckV8q zT64eIFxN}(p0KbfZ`a$*pxtQ(d-1R}UF`-Sn#_XzT!=8krbd`2^7qj;z6ZAS+u*A}~qiBG!OV!hrk>YXgJ?N~TzZA)IC5Z-CG zHoXsVL@7{wxyO9dR`BWmy1}q-d2pP~y*)veCz7Yvzfg%DV7OO`7`YIj-_#%%9;I9d=|#68K<9q?K1HKn6IpsT+Q9bZ_Ud!YEZg3zkUNeY}*~v3E?0E&A!~VBV?!%~mM7h_HZbjh0MK)IeLv zG&sFbpz;k^acFaZ59oC*_5O!{ow8dZYY|-hX7dVop_}D`_$pPv25E*MKyPFNINv~w zKCRa2Q!Nr%u7!7VD&A~&KG=Fb+HTEUr{}`!p32p=R3{+9v8QoHgsf8hb7uQozanld zCZ!9Q1(%aJ=&Q5PF)SSRjFd;0dOOIC@2|v&U4Hx=PV8;j!p+B|59}eQ=)jphG4dmx zElxz(sK6d7rJKLN7pjI8DexHOk#)aNxZ%&>pD#+QG(ZD#g6jtf#y2P2hwrj6D}r=7 z&hz0I{zD2G1$Ahc9)jemkVZ;4QFWE6N16G@UIyI*m7%8N&ICrfCCHm#Y3gng7yVx- z;3zcgpi>j6rx>+dnk#tnEvJ~{=#Nkc>>gs~i$7rrK{ZW#;B)3Il<-iAQ)fCi4v#fn zv25GPY6Ul%W##7|-qdEg@Ma7J=o})ntj3=+KtD`?#;@NwK*Q%q(>Dm)oy>kXXv^u@mOGSuoH<*5 z{TcEKP4`<*>#;;=$^=>1xFd;@#>DSX&RZ9wWrmv4(m)gIWFG}<46%C8>o&{@b8jaZ51Ws&@0 zVPfTdo7VbjHT;!5^M`3SGPxWxjfnF}IbHr!fAPlnZ|f04qsgiSB2#WVKgDXyj5}Tj z;m3EZqN87is*6C#cl|x?A8z)Ug;dKfKJO~F+gV=God&8k`sc=iDy4!U(yUA1TAiH2 zC~)Zwi(L+gav>$vKVDw+55bBTCI1Kuh2c3}s)gJ?3*LrI-iIUdv9GS-Kqm8SxA!+)|z0h6J=E>X;~wVV4#?XD!RF zYulZZyliQlXJWwl*LvYM6%Vebx_eq4$EY?)ZgT@eTQ5`AO(v4n6Kpa$K? zvCANuHy4R8H}(62UD%&+8~4=)*17wG6JbXg_}UNCpU2i#!(8D6)rf8>ENCu%eNf1| z|0~UD+~zk!pYtQ(8)P!ac$h6V^=25MH&`iB@sna2a>||<2@GF>AZRDVtTf{RIU5?m ztGM=3Ep{zc+gT!(juP!epiqzLS{|Mlu`mPZv7XaDzV#=$cCCEW3L^=Hg{+g|1U6?y zzJm&bp1$&>P)pjHt*`b$Y*OHCii(5Qs!49F%j|y1bh< zqOVWM*#JmamkJXZ3tPT1Dvb2V6cA z=%3x|YL5d3E*tp8_14xMT>+v^8UySVhLMg113wpLHB(c#ZnhHZIyf4^yx*#cE!pbg z3w>Ej)j`^lBa(_31|<3#Oc0=)jcRG5t-wCJ0LSkyj;Gp{DJpZD^JPoZ$ndLw%@Bm4 z2Qa}IgI3lwn7hxQ$fiT^3mtw4OfI_(2F8MpsP?QmU$FJ31)AK6~x_Aro3a#(yV>hb(2ZnP^2e=D9F>eAaYt+bS%_P zV}-uGPvAZa!f|NHhuNZOu3{C3hWpl0&Y8kj40c+3=02P(H9WCI7j z&7vif`>CkG7$I<^ZbWdEa354W7y>z7Ayy{Yzm8gOShkPCaI9ca^Z95ALRuGOXSq1@ z1J*~C*k1aUM=fBmYj%mM!YW;^F@yJt&n!<-(bgiJdptCZnMtU!r;SUuYh_#fkh$%r zs^!$(7fz=YZuFzfwxC8Gc;b`0oX30n%aIr=L15>jsc;8m=+8fmKy>=NUC83`OS~kT z+bt7^<56>^s)s@Pwy|wB)GAr`yRSFSpB7FZud#^2nak}Go!ermzx`^im(nprQEyaS z{Z`U4H1zO)pXn{C4`?7X-+JU@zRpwIUT!YVmW+SEV9Hlp{2lqUL&-HNsKsa+{ek;R zJvL=2-ztN^&evPIsYEds-*Z~je6kmmSl_H^P|jzxNU0PfkNxkblyGaCls95MjC&%n z2l z{!)>Ot?&b%QwhylNtv_7>x1LP+3;Mcuqh+W$B-IX8k_q=t27&5EU(Ah<=#dJbz?PS zqU31RKfdc0OH zH=RZ7a!!XIS9*#-lQdQJL$yRtF9$NtS$mL6uu+nZ1~w%aj$t2*akT2NKkq3M8Ff@B z=04DR39>L1v&p9Z>D-JI*QEizxgzcRxdd0sUll)XpKI+$(;f(JW{UBy|Ia3X4d1U z=N?64yO4|Bzrq8%iIJ_LN3#!E@Gx)wJWv6x(!XUC}Mmwd-)sYt7l%4P$3oTS0j#IM})o=d&`N@!x)M{pI=g~*P zXI)p)mSR;AoI$6Fuik|8&eF*>UYU)Ki;u_WhY_!Cnw>a6@ktMsCZ7;o524g3Y)5%^ zeoTo%CI6PsN3}U>qy|}PomrwzTL#Jq)@@&v{K`#~%hHI9hmAbwB{hc<27WZq=lG#1 zYmM@KRSSf?B6d`^!B2)>fw-{Z{`X0R=P`E1DIZtQw}$5Hr_5MepJHb!-L49r!ZF#| ze#1f8uAv0v$u`8L zEgcxfG!aSmR%<}*+jm>AW*JUavMntpdau7aFsdbvwNCg_fLGkCY8XLU?hjtQp=V-Z09aC_XAp4YAIjE3yNVq&XLx`&}&SD98>pq^N*p?}u14ruks zpqI)zbU#MG=ohWBhz&n1y(!B8NIO*SZ3aDE{|KU<^_ikFB-|}uiw}UmO$)PH642!6 z6D!9FN9?oexmcu~cqmz;gP@c*$6^#L%7ahktTU+jKG{a|Zx+gC%X!*dEq$7F7%m*p zy*@dTyTh(J?qNR&x}VGRV8B*P|KBZe=GXBvC`CSB)|D%wSgl&FaC%|popq|y;h=7& zWbx7xo(zb)7?%d&YleplH``wp>jo~7@EFhDQ)2{m0nHXTXgvwBs9kLLORc2SzVsMI zYQ&@*kPZ&5dt)Lu1MJDeo&N^zpn0igpfH)Psv6zfiSp^4aZ-o3Rb6=LOTnUBy>+{tIwc&0#BOSa_9ITZqyZnWo+Is4E(_F(?JnLK zD_dx=F}IB|f(WD=7L)KR;YT1-Fhm>y)h;2r|97JNLuww6COkj*=F=tnuM4t|L8Ir} zWOI(!BPV6Lxd@YUn#xrzKd3V->wUlSqq_AlJq9K zC28Mhsv8WeBeI-)N4_p=myg5`DQRO(3WfxD(!5s1`4 z)WGn0&Gjk{JBrg%5~{pLa$vx-QnL_*;qu6&{!u9w6}{$wP(4(SSrn0N$kUHO2B}&s zrwZ2IY?)@(>aTxr)30j2Ft4*l>l=3i!hsb;nl&71apKPqIiY#m0V>($?NF_HXaQI5 z(}#0aZATTWOUs*!kH^T{p8x!6eM&ICjCTc(3DwirqtowAOED1*DvF)I{ukfE9Jg5di)u^oq4T4$6|b%Vlu~dDs#|q@ImLa z>~Q$J;<$EueYLp6SPE8Zzm7`qlo4e2;rf6)#f=O%iuQVn4<2hiH?y#%7jN$bZL3g2 z_}iWZWpLy~g9h&Y?}{}_6Yl#y?%Qa@rqRAquIixw%t~7A@UlhIqt4}78%!E|uSp`&t{%s0$eLwBoWY zFn~L->e-W@&FLaP=ELDmYd~wRiHzTdh0>Vsg=k|{%-FPAv(2hUS(-2AJ@Q~7MHS1x zo38-bA6{P7gd%i`qxmLDmzj;?YwC5wkzveEKMlo$T)&X*w%}t#lbxk>WQhv4Jk~kg zuK;mR@$-ZCnv zZfgUj6cA|%rArh9lpe&c@MdmN0x82;Mq zwf0^Ya&Gd@^cd*)ot+{xE|)I+S~?)lH!R7#tHKBAD0}@uX4? zN)vnjlnKX<6+_;o#F5-#eSbdmgLLTsU*g68`#&kf=%JAQ*~yiwTgeC8lTd)Zvj4Ni8l z--$~mlv%5`hm0m+l8tN*_nUL1O)rf5d?G1b0Frv3uL_GFqZpfR^(mjq^QF=oBO~4> zWV8B}4#2@(HjilXWx63x@t!cv|CfQq|Jf#T2ct6Io0Bx&bQaFb7YOV6b=6q80Fjv= z04=z2rLj+woA+nb$!deRy5BQI=)Tw8Y3B#yhE8|iDnq%0=~XSOolI&Bg(vn?r}^o# z^UwSBnh^JMm4@W^OX_pE`pp+7tVCyu5tUJ;i-)`Wb*gt7D@#rm`-%y$BM(qzsRMNmwyv;n8dMD z0$1eH0F-I6$?* zcyB_o@E~w=B1B+XHxM)?VfDu~2nUC8e+lDkKuMo0#=Jir!Dy{!EAER?6e28cibm1S zm_YlUS0Bb>;Wk0!4K~KsT&#nw*guYyg$V;-+Iw_nXxZ=`yMOjZh7WDu9l1PHuY=i3 z6o zgzd+n9L#JvyrjQkG2w4I$iH3PxgF3^uokUbe<9wR1az5Q9iHC~`uV?E+ZZ~CC6XHT zsQMay?D7mSHhg;oT+J3wHG}aSYz4IqeqTDo=4^zYB%K#mC( z*Fy$J^x5pbZcn_yT#zs0u4;CTe%zVV#Yv&>*-e}Yw1 zgR#!)r(0o2Gj@^re+~LbrPN_Yv5reMu!{~VO~pfkm`Uj(xIBI%)r-et>cw9>KQ{M@ zW+xwlj+ym%zYKqW_J#9pE^^7CMl{tJXBE1|G6vQXn6oX9%Z8&L_r4|Koy(Oq`6wSv_J|E zw8gjb6;L}3LrE)y=4hozM3EUyl=JypsEud8aUySq|FM4&QV;lZc1r~Zjg&|%3hJ@7 zbNw69@n0-68ea?!Cc3|V<)0S<(=-*pF%;BZ*rL$dbhhNBBH~qZsNo8Q8(iJ>ZP$(L zbtyhMfZVyopy>1+Blm32D^^OnJ~3~`tiz6K>$L{u$aJ`es;K0{zvlnfFb%x#jjt&` z#?B9;cbajUE-H=vI7U7{qJ}x;tG*^e6J~KyZe+jjC$W}PkPzq*sU%M0xc)BmeIs*- z^DKA5iMY(+d}RBvwKsBh>ypT48$nZl@chN*YNc5b0aGb|i==*nW~<%O%t*b1VOO+R znF%%9)r>D)NR96ef1$LqLbbOsZ4O#Vv$ns%qjOj_Dzsz z!b$GT2F{nUYBdJa=KCeN9IK197$Ct2IgWv{v)M!9liD8+_i*Qr4*XeguL%x2Z!aX9)%hkbRt5oWCzE zR-+;^7`D0|c350l!NPyDh5~?1gjJ+$4T->S<_Bw%b{Bu5G8iP*5B4s=bKp)nVMaCf z-QW1YHhPWFX@&zEJA1s&$5P z_f|Hu^faz_RHqI@yAvmC2q&IMqfLc?9f~D{M@A`^@rawI#+GggSDEhiG)_(+^0s|o zO6zL_?oOiajYWC!13SpIn%hFFZumatwTI1~GxnF(bO+z+0pKF4 z2jc`>r#e!20F|TO3W;4I8TyKMBQ2i)^JE_K5J3}>h>apeQ18?dl;?uFAte|1T;L#4 z!_W73+M6=Pig4&5I)lnG8GI#LfxWcx+xsB3s==Ztl1WVVb_oxazm9idLN=J)@vj90 z*c1buL1C?p!?ul3qgbNk-~L1ljp9^ld-HF7Y)l2xj}YW zm3xZd`VbDxkDaPzf_JW!N^~v~Aji(zmv;;i&IKXj&&Nt#qzLlF=3j1P><_`k+tm&6P`2%70I;IY2E=o>F1L zCkf>iAeB}XRY-P)h-URf!KJgh6GJRj+QKTIRJUa*4Qf_SGTa(CZ*tGO*FIT2c1XwILcgQ6W7JKf7yEC=+HcbD^zS0JNAu@@>;vf9f3iT9 zpgo3T4$It*9SpbX?@Mt?_euF?(1Ev?+4p7{$}P&mPq{YX~bXffJd$^aJ%+nHoE zYjc9t@FYU){NN*5Ak#6u6>NR)he}f2qa^?)uIZizs@(I^Ver?^t_;4rtw6?zoCX{v z-;yWg(7jR#TggrNOIb(W28& zIoe&_jZQnK@NHP?a7FuNdg9h@#MWZOhjnFoFG7Yg+?gVtcCumBw4NTS;LbK@BG`&B zhB)-4hqCfx82Ym0Nx&_t*w*?$+2WSd3_%svK?9TsLEsa{&Jt%>k4A3rKii27msQkC$Jt7BOR1@JMDbLfdh#4NU6ATHww-( zwM@Z9mwX8%v}HAb2>6{dM@|n-n}Ss8JIt@w7I|cn=?f#0Yy-Y`N659RSc;KWS*&4D z>W`S4Vr|8Qf9?HV2fB;Lq-CpIYFKh(!DQi8hHrL+UXsiwe_fUZ?sK0xv-?BaXzS5+M6F!in56AB!LHUw_VW@nq%<7lYI)QTwe z1g0azPGTmEp2|eMUXKq0fN}aWuZWRvUVALD9HnWqra|b|31A5P*%CH^ ztFP$r-Bg>9Ym8$F6x1>wj49-AbB)v8nJ^65|G`l0nXje~?;BcWXGZ6GHxT?$Hs5fW zZbvI5ykJO*=mZPNLTp9pk8;vzmZ7r~SQzl(VIj)`x@M9Ipbo549ZioH| z#?q)#HKneN-o5qgyJXbuv^}?tck|4*20o69(VYS!a?6EvpRhl3FZI~3^3$lhan#bp z{djq+ubyS461GAiu2@9|3hCI*XO6#{AK@tX zg`|DdkrNQ-r&ns7+sI6Rn{yn+YwLtKMSFI%Ie4*}Xm>vIbVu_nRZx>&Io(*Noj@}0(|j~LWI#vCts7&xk$&?K*PX?~{)a6uriJy^Qr#W+Ul)e?8m}7O{AcL({{jO46DzPx;J!M14@~0lnjQ_iz(86m zS7!Hxdd(<-Vq>Yrn%1*d*jBPrB%ids`KOBrhxJ6e+6aUwMjsG|L#cE8K3?<6S7D4zEoj0!qk*0L;80os%pw70OMyiS^v!Lhch7`xq{@ezCbx_#Br7~~( z)natGJK_4cxM6@(qWHYHQmDX-p((B`Rvu(}zFSHYxFsIlPH$C-W7J)GFGf;gcgoJg z|1*>C?(!v=Kc9c4Mh`5iAN9sk_f6V<|Kl0c;(GE*mVQ&}&Sage9H+K3RN6b_bo4`V zaAK|8wg8zpWpLAwJF{BkSeT2UlhC7;giw?z*(M6TqhrLT?py}Fu%;8O!BeGF z5HO3EZA4tJCJXu&n{6g3n!WxyrCvq3vkZ8%T<)(NDer`@MQfDb>13`&KhEbX?Yiub z{*a3pDr(&A0Jpv5fOH|1FK&%$;)IMseiIKAw}?#DUo?5kKUAFmfd@A;(`2q$)D{Q` zZiuh^Jm1O$gW89FTdb829XMW;v-IdN!pr?3O(NIUi{f7Ee1<%fE56qvU|tWGq(g^r z9sZiadB7X(YY;?)2GdER2=)5w(}()I!|9(l_Fbf9i3$Tc24ATEc|^y9g0dA1YVq3L z(LrxPTVa#|Q@VP?cwVJz0|6MzNe|*7Z9uHTF|QaeCfqOc?2>gKk#txX3==ZxpwOy^ zw%lYVm3Gc0jvRDS$Vr=Qd*+;6ew%hYhY^cKMZ}?y6U?M6Y=aEv@+`RPc}qoyPU68! zRQw~H>+=__y~*u&J->5-o|AFPV|$+RFbV+JaJ~o}|KdtL&uCwHSeuOn1l-ig*HVpP z90O#4Uix6@>2a@%R@oXcy4){2QwSJ3z ztw0No)9TY^U_C2PMaIIMNX8}5G-WlpP)cUsFzjfGnn+Ql6X!10@Al*pe`^XO8};?q z)jYcOnp!>c>AWQAk$gcQHj_kC$j!Jcet$gHjhZOw1OMuqLnivLSvsVDez)G_=F^&j zVCL9ydm(VWPlrQY*YEX5PPU8It&xUM&}6z8J39%aCKox4(+1#TPZHmbisJiP`OAaz zF%axJTg5iZZOxp^_Ocw71Ill;hnUO5m&Z5g1ca~0yNo?Hav|!A5gF< z$4Z6r3Tld0(EwL01SBH>n)|kq-o@_opyLe&*$rNPH8Y+3*>6$wB4vyP##1)^Qj6CH(#>Mzd#`P51WSi>0GRzvIb zY5guv4b;iHUG19m7xH&V?Nmj00kVS=86i#24nhQamLBc3aav% zw^LpR4{xEypHdrUKo?!=1e{V+p8bV`=P1IoC%zF@6BB@4>^|G5J7N#Rk-6rFU2IRK zQd*VGyW*!K0A@$>SALC0CS#qxKt2bCT;z{8RJBpWN&dC6M#FwXezXsX#yGa%a{_qAW06&S8hJ-x;KY)3qVF2P@-KBdL)*nd2Noc z*?$6{?6#$X(JpGWU4@$(;1(OUx-xq5`#}uSth62b-#m~1{m4sMh6kgm4J~b?`-7P^ zg+4jQlNMO@{+N}HB#s^hji_^Up5lAI_(33xw>YfHYSs`SSjJ@?_r_x@j}6BCAwVmt zZI~UkxHwB?wBo145ixwn9Qa7~*evlOnuZz%+;lw8hPFb*p26)DddQ}CF9<$2W5k?G zIVL;1E<+*=a>g&EDjN4{sd&Cz`ATjyxvg>4n3)bN> zMYuZ-g?O4zhMp=G2|Xu7S<=Ms^<+|+{*YK2#+~hc zjK(uE8LB4vVEBr@J1Jfuvf6()p}<+eo{TdP$1=XmU1FJHvodGaaO_AK^;zVlZ&z>- zuE9rfx%R<_yR9hmB3lzrLq4`z>k$IIFvt!t7Fwy85((;$D*V6|&i&#&5U$~@b#*mj zCV;|vwUn$fveZH(gnPO4EdU0O1JQU9#N^?VHo%D_)p- zZ(Q4hS5pj$j+1%WLmx9d7wcK*9I|B9@p0aR*(nBjGVX)1pw}dD29O^zNw{7M**;Qc z_C97)I@!q0PHR$G!G`lagawvT?^E?C_@g;xKhoj-Q$qoIjSrupR{JTbL(S?n+d6{; z07D|)@)v!&EP1+R#n*z;rVm(CUHE6q)oTfcJ)K(ThX>!%}ejayPZrSs?G z;-33zU|N_vJ~|w0R)*;bw54W)v3pQcl^OHqaK(0PN zT}vn>IfLla8d#`xtCy&7BcR(=1d8641EudqDZ4O39NDzHIlC?x#Z>dxyob0 z1a;P%;wNt`cExI`ItgIvA@BLQhAjfbd_xkr=-KSsE&+`G1xraYL~Pa!0AwpUae?oy-=S zs)fCKQ%rfKhyb8SuQ&!-%KgzySKr}_Rs=OjyG4@V51jxl_-v5|Yc#cs%$gBpbQv;R z-V<_nIEHUxHVn~Q!uaJXbw8Tf6{(be|L#a-q{>Xoa{{!haz8cnzyBD!&b35_f5Qpl znmNGCL&=f8wvd*Y(=-9Y2fsJ9-+)!ODO0|{JZbe?wXWzj-O|6?2M09aDUT}lz9b`u zp_tSKF42s7>*a}T(w}3RZD~q6A7xiHM|HXpDVU8LXG-VvKFb3U%gL@ z4rKV1l)my`_}4G@id=ykA*{=7^>BGE-vb5^*(G3e%mxADi?nLyM+h(ZS>1-CK{mDN z8dGbv!QU{6ITs{4w8DptFa%GDF1~S2O`qH>BZ9afYN`b2CS{^MZR~-N&Lk#37 z>Y$+TvX2ZjT*ZMy(a$AWh5`av#NYjss7kuvb*LCTS?kABU@7)5VjZa@ce_H#dnmGS z8Q?1vu0Y*|5k=s(|FzB+yk8;vDK%<_SUT1uIEze;SYt4R_hK|zC73GzaqrTz85wY* zP&C-zzR91&5uq9hpu+z3_yDo98nJ^poRdYTO1Ki6om*zyW&q?`%v(t0E21{xZQMv~ z4bG0vj-+!amc4g+dVZza#z73d$=u z3H+T7zpR~)1p!U|%KD|y8`O!qBS(udaZ_BUd|4_6jlj!;fG_4LvMg7%;I_oHm7p$Q zaK9P9_+aCEvFU{#D3Xz&vC5G?;tu5BtCe$TlpqoNb~rNtno=Q)YZ^U(s49%R5DjQ{ zQV=R$Cv?8jPt@&4xH?eOrCB#2dx5Ywbyh3aZ8A{%yj{5)QOxz5uS200zZ(t3Z^?yA zjK!-eHZ}54X-K|}o4k6?gffiu+ipBA>3V0e=-_ujBAQL}Fx=v~#^%lVLuMLkk?T?T zs2=fT+|mTGBg~5Zjq6>|RT~jz%_VR?%xW%s+h92mztx5+#5~S_Hyw_(&a!AVq(qq4 z){d}h|IiG*^MrlU`!?ir)jwH1KR+oZ{CJ)|9J3hbZ4O4VvzCG>2s&}Bz&mOpcyn>N ztIOLL^>U!)=V%8yQ=dS{e9hHhKkC^ESG~^(+)Y}i8BeW&!)O-&^ zKO3;%<0O#6Zs<|YR$r>g68(I%CQRi*I zz+9-)o;odVdkp~NtU{-E=@})+8nGO`Gg)~s0Ne$a5r_!Kh6D}u5O9anBEfJ0uMSMVa(%7wluwH+j-p;a3#^1 zp-^y|$3>GRiQI+elJI&H&$$>99Xt2}nv-)E303W)Nb?M%z3&->|uwkO3^2y-F}_w&GrlWy>{=IOey-IC!% ziu}&Uc`N57PA;GgeAwd-O~uzu$$&m;^&Tqi zNF)-A5}H;TP5H-6g)}{$0?SAjk0G}ws-p+RDhZWk%GO1xd74IFeCOXSjQ~@H{9=|a zD9E6Qfm@8W_5E%2?hS80nWE@5ZbM{4>6}CWAP4SnNc* zB=kDlAJKeX)4)&ziuXW>0OM-06;Eg4eM^J4$oDltu4f4IVow{nINXl)!j@Ja zculx+6)tvH_Rv5U*8FHb&hy{3jtuET@=!V}Go_N5>=2ys*(yt@;3fJUK2?^8HyIX?=(duNTHPVlffJX_Q@f1A zAz66&Q60@K8}6UI@m%%p8~b+D+zb7W%ZOp?l>cQ3z~$5;04;l7Z$Jx6Im&aP5pTMF zhTs~#MQ*#n7$nsan4`cq-a-@0THHl?n4?DYqFpsT9U|XDp1`t+^?u6H8%U{Eh>Cb) zQB)uG=(GxNr@A#Dg(w(UyiY(1Z_>9l6)r)mIfz?l$>&d_`bnv&i{R=*w62A339r}O zYf>pVnDtt#(XUA0h|$14T@8>2taHafZSQ7^=sl@)!D?&bZoR)jK#$Y+q42Qkb%`J? zzN>E3w0E`w(V@v-WEO=U4V$4C-5iPvFH?gHeYQeW{8)jN3N$S2v=|0#iY z*QJLn>mfY;f;mR)Hs6>L`dP!#5#V{%TBdjN+59{Gm7;^od6E%j_XCU)2x6)0w zJO625BxV(vM6)5oGQcahXD3RU7S4lR*AR;=`)ofA!Twgl6!*0f0y><@U1NG*OvBM@qH5{zTQJok3PL?-S-hOJTCx zCC^Kq@@EU0^8*$8HcVPVHnSu|K-u_)RCk+J^-;E-`H&eFDDqOjhPS_juBf&ofR7;e zC{TF-3`imF82ln!T5MMH?<+1>YZFWwRbw7Z8Y;mX@1I-VRx)nb#q&8^@bza&1RYVt z8Xy7Stj1p$R}qKtny9sLwB=8r9f2#eRo;+q@weNpLo_7}M>oquRTIb~X`#0h{JbQc zL1V6%L?$Gr-F`4TOEl`ZVrH9SW~~n~Q|f0qN6OW>qw-YB*bymHC@6kvH#Hw{`rlkG{gC}iB%ZB#5L#MZ=#UX36)X-)^yEG^UFUawT;D9os*rY z=|p`twa~kpcjjwsH$KNr7x%Y%yISwAH(xFJz+PLHsDDC-H$Vlo_`@Ow`H_99z9dPz zLp;$H{CEy(<77ZO+rRP12quqy=HnA88J<{DQ7m;LyMZN? zcUSnd%gO0R7&1S<0WiUF1tSDxn#S#~AItJufw zrob5hcY8KuvOPv#0?DrwcjASf&`b590y0H4m#I(s3;h1{O4rA1=I040@?;T>kg{ie=_*i0%_G?mxwDF87f|n)L;6IYIvqjsO3g(s%@-U^Dh3TKt}P)%HtC z+VY2OLA7_OeGji2#v&TEtK{r#+d(w2Q{LF>FHB*LLhH!EAKhuqLR!illtEHWqfM=K8?8S&k^4)I|VDTeg0rl;4 z>Ty*or)T16Xwk;~tF-X~TUnVOF4@kOC;EFW$5t^E>DG-f|0gWNUdX zd94to!9j_HX1K%f@L0Zt^dn~3KWu+IGFjXov{kT)DLz>|noB&KldJk8p|B;JUB7(s zF8XT4hx7Pu;L5=uA`l;AcSBdu9rZg5HspW(Ma;c?yaE9hyyO&4oC_eGZwV%eKTuEU z0<7Jez&d<+)~LXR8DK6D;&B;()OfE+Gj*`vU2YwgMA9s=s4~5 zr7x;uJ`kpm#eMc*dX-`c_t!&s_@W%!6ZaPC4Vuw;z@UPRIz&$GLCw#(D`uG@NTmL& z38%)fDE3!s1i0)u+)YiMHxh}z%@DGbN=YHyo=lRcW0KrR_)^q4$OCb(>RMpw1AgOa1M7ft zW~#py3*x{0iXjF{UxLQRSw#z-#)I^#?z<5K10= zH+}uD2dDv3AEr|yX;;$YBn}++t0g-j-H@`|LVUq@>*u?9tB}zEA4Z0Uf7>qOVtT2_ z1H>k4c8H0rx~`R`qwVk#=)cw@DJO@>ZT47GrPBi+s?Cz2thRdhKx%-4vCNyNYi*|t zNlHbFcPvnCK6UQhizUUGqR{61F(+ph1~oN5Cy#Cpu!J8r1BM;rsk`LzUwi)wdXrU0 zulX8g@SbX3ug~ufM6kW{2ud!f#nt9V1)Eq=5CPFF>X^HkAGXXnI2;O&1O8STL322_jX@YRxYsH5`MY=v{G8vjZCUvY)qI; zF28WN*@)Kci=t9y3dCc>+?_1QJJXL`3lqZkkR`3jDETOx#eK5c!Jw|L3sd^iBZTu7 z4wFVbpK`yX3*S@4tU+!|ANGH&Klh^0_jb4w*uL+5esGt&dGIY0&QGGd++*3~ybX@( z0O<h1NjZ6KQj^}_8YZ)JY|FG3V}^=abnt^)xDRl)o7}Yp}^g^ zdI-fhn38C5-V~FM=+y=F;jb-c3FCW=>4%zHWH9m?O`WvFQg(4l-Kwo4U?lg%yNvpCO^)qpwjCBF8=}Ie7j$EJSb^q_*T1Dzt3tK-2;^`!K ziyAgP*^b_feWuZ<#Jo$itN&~qgHTSVq1ZQ&yHCu`9v;Fe;=WfT32kTHNuQzMvA83a zeCPu-8<&NG^H!s68-XvLPSTf^8r$V{Rc&@f$DAe$em0MAWzMBnLKa;X&R!5cc1DGc zmrzz0=Al;%<9>jK&4SEgw6?kW-vqu5rT z2<$}=pAXWGe**oF--BI9f`oY})8#+HkCW0=s!WySUQqk|k3)kHmI&BIa^kL=_H*O0 z(Bca_L{ZT}4RH}5gI2j6xz4nhH+Zg0$RplVr0oj!y8IspXpd9e6q_gUgm3$2&yY$P@-xMv8b`bE9tT~8U9^?32))}Z~cnRHM{Mu$N`A% zezh2YBwOI>LJkfUrlzr9b1GN+@XfFUzH;Bdgh~iQv5^1?Ywmlacs$f~;x|_U<(g7x z!>P&e_c-+C#2yNTzvF)shrIp7c3!2y70pVAG!V*H>FNTpIdBG|aQ8y|`l1Uu-all) zqt^p5t5{%@zv2FqhFv21HpgqkjNnS3YU6_{s<@)R#m70AUEi`7&b~aDf0>)?2C;R! zfP^LZi0%w~wWZH*DKPi-OjTN=P-l>5844+Vw$9FMFjq=gTnA=x32>50 z$&*3H~CD948S9gvHmhiPutFthV^^UE#+!`D_p9Se=}XO;FvJ(U0xS>PP&B zjP76OcTgr=%#^)=@PXN#k<~i#MDryQzPfoy1!rT=a;qG){hw zTI2|9R-=ywMQJ~k_HtQfAQ+B+J|tKf*WHtUQE6IX8`Lu&-uf}k-N&JIgB z@vLHlJ>l~n$}up}RCPXtuRiW}S?01P+%Yf4z!-bua%p5JTrsu2&Jb_NO)PgN^5_?LSBn-Vz8|;>ZhL6sXR?=P8i4dLsLC zar&RSUL_K|V-)k0%Ul1$o94W)4YN-Sa_n6V7FZEsr*XFO)fXTGu*f(1a&v>idJ%yxo+f=Fsi>)Pk|5AYzud6Rf)<5aZn3YZ5#nQV zL-NX6rm+BbODfXGF-{Mw2G*ZWJwJ^L5U`T!P_HkhG^})>4JZD1(+CIb?pC@zxiY&w z)(>iAieP+@17!0dP;}~0n60-_&bLRPv*WArFgTF16i8@33ec!jzAkKfbo}}U*f`Nh90#Q$t)34Mr`Z7G?pf=g~q*E*55mo)BqsC*d6=koY;YCg)Mt*nKh zbLTmK%$pxce^qX>*9keSPRO9cC`dR9=X{lj{M?zzWjA2>2N@8^a{8pTU-}2*tF-Hshx}B(!EC%_??DcC5nZmYK15} z_|xFG{j4vlmZkMvrG&v!QnH7qqQEIA0?p>l<~yShR4V>=nXynunWC4;$#WF}d|w*+ zHyX}=V@V{I5>UR0PIi4om^%LMgpQto#wP?&0;#u$iOw2db14f-yfZNAu&m&;mr9h zRMssEZeDJoHW=>)63dM*9nopGCwC~Qxo)>T$iSZ(0z}b2kwjd~TJ_d@fjy~?%^29d z(pf+u4JGBkgt`=MwL~XKVP`UdMEd>3;(WzZb>+X>gmnQJ3VCZN#Lr(v5t?lDt{7H? z29;Q*3NKo=lJ{!hRwhq9VA0wJ;>gcN#ORxzs9iDjE*7>APGjkUdO{cKx_DF*u3bFA zdhCp?KIqHeg^{QY*V%QJ2)uLT`(?jecSIW;R{i;U4n%T$Zw@35|B=Iv9iSMP^M(_P z35Dp3%1pg2F)!QAmui9R3VusEtg}!f%Tp){QA5x<)L9ePy!S~K6D_W4l%C^$A|uHq zWw|))5(?%%Wf%p6rdJ`mhRA#u<~juT<<&06^23IHn_~3RJ6L!s70)vJuWi=#VW{09 zaP3ot1fXbP+Sj`Da4Ap&wsvVaVuYT$?{8 zcr?$H>3gp)pj~6Q!xcKTOcrU-7{4S4;8mRN`Cm&JAF8mZ>@wHW7x^W?7lKb@J?Y9e zx794uvnaOEMw@qQ!oj4b*OYuLa*w2h9!}wT2WAKm54Rh;Lp&?ooepWf5Fh=6aCf|? zG?)}TC}h?t9I9Mh){rBl^YMLpiD3*3mEO6b-lq(_s#k(x$7=iQ83+a3eI0>Ub4!+Z zX9njFhdRW|kx$`7yBeebrjwtW!Iesh4g@8d;x>w7C{Y5wKq7Ej(&Y7C1u zL?-@{j6kXU=6H`6G#jcgI{asP192PkB6In%`9USDmnQNd;al7=n*;IW-;8MlBomDT zxfzL$=dFfTd{>ETM=2Ew^UpwI`2W4|6PghN?Jy^}3s;+mdiU!<#pv&}P(Y0S;!jQ(0 zuhN%_$rO89n2Lx})bm8PAz;x8zpBYaR8{!<{#x%1IP77l_VJ7ZcUM#(4*VCDNj(dU11vBg#ra-O6-DM|f z53Me4cOPmOvjy)ya%c3zXvy)c(aCVSkRIdK@e01d+$5m5GN(!rDVb_N#nM5j)mCd> z?2=Y*zT?~s9PuBVjMfPD0al=*`iZGZ*=T4GQJ=nkgP+W(HrC_c#{bfKt=bHuPWOgn z`mnYT>5a=4ItWSqU_VuWDQHQP1nWVb46P3@zk&g!)AIFICksypbM|-}luiOEEHNR; zv9z3aYU|F~jXCM+dptCWZ1gxrlRsx=o~Yf_&r7F>i#6Z5&+Fb-5ddx{7YBMP=Yib) zs5y!y#KQzQ|MO1u$lrV*|GgDPn=FCt(AP8GwD*pnOM3wt_ZlOH^S<2lz^l=P!+hDj zA|x<_u*^`70yslL<>_`fl5eD5XYfv>`{1}fOe7rhj46!Km++@)_SM08Nyxgc@GhE= zR|WpHAqcMsLEFVX$Q%~StY2{%Nx_n$*;emzWO;87As8Z2_R8-^?P>>J4u$_d10x1x ztwz)Uztz=(1@Ob~^pg2C!V0zItj(TMEEK{_D*rBARWv&5U{&9MxX-cSReKjs7s^Dn zlO2}z_OfTDC|y!KwK~l9#2&>>rMadCN^#LOj5bRe=zSt|bHGwcVCduXQ3mkewtCO~Fw=6ZD zhR)?x{fD6)6&li<2Kd@`s)N1P%>>si?C*_$^=r1mNC*!EDOd3s8vdcCj+FZ_zrs~( zu>T!~ScqHDbchm0oknECxBL9$tm@KiAmXozpWXue9 zev0?TBYf^}B$!M@e!4#7qa#Tq!E7%sK!XeBuQL6OJ4(ljb>W(~9wu{8!nKlRVvC7% zbc{DOhZ)T{}c&j-Tv1ZCKk(>_zF<-Pkli!X?5Ik7(x32y-Jn{MzzT8f6~?-YLW_zD zv5t?CSwkd5`4Zp3)e;72y;I7!j}|r7w{JOS@6~#O@f5z+gAuGo)3D=7=@+PU!M8`@ zHhWOTnw|u^lQzU*OsMY|(2kbAf6OU@>x+6@T>ga!-QLrn<2YKl{sq3Xm@@Bdn!~6` zy$9G)r?618P-fxp#m#U1wv+$pZY-F$A>Kfno9Y(wa0@OEP`cRcO%^QM*4f;7zL5_X z9{d^fKMK=-+PLl&ID7WZ-MR+H4xWeG5x1_$wp2fd3*nBfJ{t6uG07d0+KbgJ@iV># z$K22Q!6&d%d#EhB>U_X)n~P#6E0zLYy3Vt=E|HDSfvM;MTH5LNZ(2@?@INuUVzB=K z_6TuMg~hC!p^>K*yr>igAw#(EX=%xD=1}s|vc=Du`(qb#yt99-ow4fK0t-G$_aQAU znZ;%g3#qdAT+nkZV;>HZI3TBMTgX@6W+zTa2#F7{qN7f`o2_ z%R(IYK{!mWaBwIirQd#}mEVBJ>5>p^hVcO4P+P8Swso$7|7EFe-|l_QNK2QN>AaDu z{{PlMdq}}#NnW*$q@RYUW_&n1a>L1s_Uc^Iy@PxGZoTy?;4AY((MebLaCb(h(^eur z;0WP`^-39tsSo17)t~&r0&!L@HoG$o0pkJ;w&%;#T=$&aBJ-NP6E*JJ(a??jELc0IJ_6u<= zyo4_xh0|eYbiz*4no6md5cTEjP%bPQb>sS>{wr65oqAkGsNia5-=;=C=B6Qp-kP#i z6I6Q){{T0Z76X{V)e*|qn_jgRc%Ztp)S-3k`LRoJ1{E7kQCC}T_!nQ4$h-~u<`FI2 zbsoJC1^z9_b(fx6$*4eI>&a)l!!h|>8a}_l-pox}TK{{+3H3<;tMBCD5YMi?a; zZ!zc+J^qR(N)<3NpZJ4-^iKW_^x44@!s61D=hJqw!c#Er%cl}9$RSteXc=1O0F16C zG!%t`m!$L4P^%|2NPC!tH&3jPk8Q9vGl{x=o_R7DiAz^qoNx(`Yzs$ob(AhF)bD3zthNlY7Hl$^eygO z2HjS%?d54;hiLM~AKo!ry^KKEGDspde?#$l@ONgjQWh@h{P-(0(;oT~k>mIdixlu1VKUtsxY<=hkxx7R@pkA2MPYkg7P8O;mJUtw_X)e^-P;71uC3oRe zk;*wlMf`TU+=pMNv+LSHHR%t@q3oFFfw5|Jy*b@5zrVJ)MmU1YMdV+m_A=WMjiyGN z$W!S02+5i=&15*<&7MQp@ioF_g*NqG>4hTUdPUGsSpq*7s+dDvEIGWQ32j*6Ue zY^-4jRO@lZ5m%B5z1z%(uLr4|O6h`#7gh2F%I1I~#7+CPwSAa+%>@gOvP^%dGyW*? zy4enD8&}{VN43UMDwX#|#nZgY)dIy}SVH>tJvWST!GdKU6?5&lecT7u5Se5+hS?|Z zkO(%p>umrGZpG4YPQ{P678@>++)`zwhSR<^T59C5KA2_DyNZmZ(AD%Vv}n3N4WQO7 z1Nk+dDx+Ng%hS)<(&9v7ckxjI{udjqxqhpD$vyhUt_pX&ky+Ncs`! zX6qzh{Hp+f0Dw~n%YfPr=d)F{PS?OR5?CR^RZ5J_}$C& zC3Zye{Nr?2|8$|0%jn^{e+)Eb;a9eK=DCC455vN*SVt9_w^Sy!-BJ-1TXNT`L~;=*n#JUuLrTr1y*c>{gxffH5`@Kp=e^UaBS+6 zpzX&0Av2;xnkhw8p=P*ZI0`kUs=67)XditugzlqxurIQeN#2nh^gY=;bR#p9l9zrN z!NjO|<=q0(Y(cLUghZLCUlCzw6&V2bjCi3Y*-z=6EpoSSKrkxK#irv$bSC=tGspavzz*_#E@@{$(WK z*e39gvZ)`t1-)<%7uXs8nvvsMXdmVi1e6Rkg zGF@fwCuWMQX~$x9GZBrTUhj@N-CnxDitJ8=1LelyiZ%m1XDK~&Ev=wF73%$^HaJ^Y zhtJF2ZN(-3F{`U3G@;qZQWGMb&JNMsE9No48|q+tr2pbcsIL9?K#PF7vs^v=P)d%s zu+twoiL?{C^EX$xG^@S5xjT#4ilMvbtsZZt%eC04*$=~qY1L0VN_4T@uY_68u#&0a zN9u;3tmaoY4d%aW;gcqMFBG2gF?AKh$Y4-E|imB!C zw<7TPF33aiv=&&Qn)U2jU&fE>iflC;NWp$7D^keitjceq*jd#dLHUY=K%QL5&aI3JmRB}XT1CNdj5t>m9$hrgFPGkyBb(^1<|vwbG)y>@L9^V&^-Ao~$_P`)xy>>&sWdo}E@_k*k z&%bbK1=L;7*O{w$ur+5Z#ZEZ|SmLIt1!I@`c{aQYDZr|l&G@xbE>sx|00+i_ zO1|Jq#lv9V^U^G8i_AQ0Lpb{Fh#Lz0+B}xLHtGF%SX$zkbycxz%#m7|I+KHfE|lDJUEt$+5O=?N zKT50rOqR-LABukEW`eEF>U(Px$qy5dI9$w5D#3x+s!z0%AOuS8tdU+~V=^+J;PW>iT8NtmVTW`a~f_$;FEc z(IrTJ*S~lm$$ToR(Lbb{P61oI()?eb;Jl=wa5-Fxp{uAf)I|iq_&Xlb?47b2vtfApK)?H~(2~Kaw#XW8=b!OLZF)~HcN-9HeU)1pmj?^N@;w8u z(779G*CF8}3X? zek%V;AD_80WqycH!_Ojo+5Kn)G@`P7DwVSdD@tw?6@Prfn@kHr?9Jf(gYLlyQa&d* z03Rx4n4~P9mr0ScJm@~oc3W4Tzc%oF*d>{c<%Gm({P*MeDX@3%7jzcspl0=Td8sBc zO^2%+pe%4fnIiX<{^4e`fEQu`>O#ixzK>x=+wgj?sG6Z?yW>hXj1s~3)|K(5-C8dAxgmce4P-b3=0o;@MqEb9 z6ZP59vlS5NEK`ln-1~Q#0jfV!y`S!+#~tunKFXH_YAlbRzap}H@)U-9<^K?E?jp8% zkjvnBJC+1iZ!nRO5}!q-3<A(C`*$?tch<3~=l4t(b8z-kbAA=6Qq7Dwa8} zDAjFymjih$-*Q1_=AFU+iB}@i{=$-PaQyeV9e)Dj1b+R-x-rwHR+XIZZgXu@|0eOY z(}NEdtJ`GuJz{wFv|UdW9U&72#0?@!RPj9+!WG|5=6R6+{Ys_e>;6Rfk1Y)YABY~h z0H#?{vr0!8Me>@Y3&nQKLKv@n5ExXBnV(d$(`?OYKJkM%lm#o^y`1$Hn4=$WPf-DK ztX>CR&r_`$He)Dbq)1(EfwlTiF;@_!JlN1=1<2CJXVxx*Q{cSHQO?@@2)f5IE8{10 z2KycI#wMBA2KNeh-P&}f*P_nRg1FU~;nXgg#EqgFnF^o9``ho zQXmH$kB=&pPW!-dC7h;!K1$@${1$e;WIHDeeBxZQj&>TpSBl=xG~SfGlDI}`no-qr zo%&6;X{yx_PGdscwo-KL>LeKOlEs8TD>zv1lLJk`Ob?c9XcaLPRHpo{Aaq);? z9HlS(32a#VKD)^pn}7_cK_UTzIVtQAL+GBO6Bz&V?N-Kl*YR^akLYi_o&O#vuRMrx zPWgJiz1+?3z$Xui@tBTY0ZU-eu97g-_SYgwc6=7yGE}jmtJ~9+<7aKeiaSwYlZ2jk zYXY7b4mcxhRj!4N zHY7As&5n)Du}DIDId-!7zoHXF3ZP2zQ#XN}XkQr>E;HEQY4_^&1vS3k^KUCKc}w0c z>lOsTZbXN&O*!?x{+eL$z6ij8*ZT2Z_%aApCYj?^KRerHGTM}!AJ_j^=AgT(R|Fm` zF2Q*EFb`TqHUN!{C9zAsUVQO+3i|;B)L}ajS2fu_TtQtK%@|IS`MUB*uDmWl!$xfs zd%rYg8Ll^!qT(~l1c4-KQX-Q`sLUhp;+q6*dtS4|{Qd3U^aBX=YTO-Cn5frFF7-tR z*XGK}ci^fXVKozRNq9R4l*_&^FzYHlY}&To_b2*a|3Ynv%;Gf{(0#t^>1`Q=VMnLm zov7c1YE--p>>VqPWca+~_xHP@^|~Y9_BKV`F=;92D4+RJbVl9DslLKO zH|p$@{0XqIt0jZw6Oegu0@CKl!=Eip03j6QZc^}W6_)pgQ>{#?kFKraQCgf7WJXap_A@uk=0~<1Q@?O+`eZFAMQafaw58$z&P8R<7fjF8>&RMY`AAfii93bXHx{OukQmsMXk zEhbAmj&l1h5S|xR1f9cbCl1J3#5pV{kFTitNu!WehVJg5b3dl_-S9J=L3ih2(>0)1 zIWk07u|OM^=1WER21L1Zlq?;#mI6JzzkZhWC}!ZU8wxl4Hd_H>rQr5SNZwJ-k(V2& z_#O^!Upki5^d@G0-d6M5V`~NR9p2{cl_QL_D zISWPZQ<~TM6o*F}Ik@DyyP;dX{p*l}@?y(fL;r=!D9LJ)qn zh|*#ceA5PqA$L?_B~~6=6+rpGPM<0R=zGu~6v1UizD=DHkw2b>gKW*K>$Ap4pf~b` zDI87|zx3Zr%J*T_64o}CLqF@&3by*#Frcm9I;CY-Eg{&_A! zit;MRX^-nZPB-KB7ncL8m?;hPg9Nw@Mk7v5_fK+*_!i|A_00j=rM#V7?3S=IluA!p zLN}RinAp3-^KRkRj8|d}gzyn%4wZiSmvgChi+>7MhgT9_$J`KW4r~Xb{j-vnUk#3e zx}L~S0dmN=;QdjzHP{*i`Cdo^{trkEg1B_a3o)-R?&^4|p+P46E9}$X`QfzG;n*aRp8gW5@#S@_2!dd@Bx^+Bp}uzmK3OO3LaR7pdSWEPpfIF92WiFFB!s z026*7E7tdLOq=;I8!0%TDGQHqOM;;x_S02Z22isCTx=@gJw-BfAc|K&vN_e`=SQK^ z64fYXDwm${@JWs#Z%)4&4bVVp18|>9B|`}R{u>FPN>duuf#qQ6#}S zEL}B4p!@w?zwPL=*b345cU~%;R874p%sMEQ(~Xv@dc@%QD*(u9|LaVGUtIbrn%iD zUq^>V&xnA$TVy2v4Z4h(wfnxnvu0+m)Hou{M0^gtBG!8LeRbqW&s^-GdrA4>p)Ob^ zQ5?7MD-J!IAFwj>QcG5_opk#=Z2~}CC;;ES`ow-0iN1b1??D7%TSp^+KfU~&<&ToC zc6)t(V5m!2F6FkOEyuCn=q9yVs48E}Vm{^)8I0j@n9LCj9{vEyNpd^PqUGd|o#o4_ z*>V1LU zTf%P-82|Krneuw2U#UrR0XZz5T`C_K+Iyi&tNzy^yQLq_gfr=fLgV0?x1tDz#iq!> z+6=C+;74M;$|}r?;^gmj0cYjXOSG&5W`8`3sMbLIUfW_Q7i4p0PW7I|MAM|62L z<7(!e?LbZE*wi%>Mq+mXTwYMw;}%fqC+$EZIX};In?Hij_y19s{WC_GAOU*gfqf*8 zD4ArE`zVuB%EV77_Te94PFD;dJEVfksP;Xl1B5bF$s&hBk0YT)w&dNJ5yR=iy1W;B z`@U373l-1lM{90fyw$af^+znxeO^1!?pfKvqeu@8!v{Lm)Y9K!YRM>X|Esdrithbg zXL)}rD#sLE#1c(3xFzSar&}UDHX`rwc1_NTZiTy;c9t4Ruy0Pq&cEfgKf(X|tk z^|rn>M7%TC7YC>=J{RTQ50g7x(F(dvrYGhJ4ajHFX?UW6qS|fmtC!Kpz2p2hey3p* z{h&_RD565F$QPodwup))@}{lJts)^&CGmL9C6vOk%!r*SOg+qm81 zu+A0pwhh0#9mF0|Hl0SQHG-V663jXIQ_}3(3et{JN9U7k z<`5V_{LY zpP(xhXu(_vza3qA@?=@9~o(LsPn_lrr~3>qL*g(6mg|NvYNTp4ZF@vt zNGedVw{fWi;S-5px03*-#yGgnLdTU5J0!>jWsn@4gMwZDt};y3mQi{tc}R3nXW)4` zS~@U(sDPg`UpiBwwHqKK;lZ*qn$r5Xdx?DU_cT#W0DP^5pHDujf>J?&V*J5F{xh{Y zeE4a9r#5W@HXfB9vhlR9xM)I=W%p z!iO-_7f^sszVYlxTu~*(mK;&+L^>~TKb`pF$9Bn}6mnf5K!xi`v;=cHZ0Nbfnk`nn z8EK+w`uqEr8F%Tb6bPl6Z@b;-GTu??M-{)}&*UiCJqNs^)VA>8Evr{@-}}#+!?3*G zc;Jf}jsub?IHxb`O0?OEN?Pjq#$1j1eWoN_g%sudT#;KG^E_!9Pmw`J@X7u1ld#S? zy?s0=6&WbTcr3b_55&98CLva9{od*mZ_KyzdETb%FdIN}O+NT~EbPIRD9UB&1 z9@l#z9~|=%ME5@{>tAhclENbr6AtA?&&4Xoud^+PVQ6_x03+Oa&`rKqsWh572j-A| zqlI!&KHvXfcz`n76p70=;<}+9;LYzZbC@bo32@0^ht*;9jDO#D6HzLW@a8#02b{%(EWGCz?VV`vGfX z^8X_w8yQ6Mekdm1faDGwe|3JN+hgs~e6mZ9+-lIhDU^DVajL&3OgqbD)!j6w95>XK zbKjGL>M0CoFEuAvCfBI>L$<0>drKdTdc30A<`h9K4)j)4{un;Kw$9xNJZ9}4xjRhp zZ;G=x)3q-#&!ZUX+dwB(VHQ_T+8KtG`G!D0_Hg~^%UJrht-;ZJKoNJf)v>wKrAP8E zl$F-?fMNYsn3euO-{YLykth@vC7K8|IM)X`OX9DU&$8S^zeL`}2toeT*IKACJ4>F* z<5Ycb1Q>$26njHUR7Wd=+?=)Cpok{;Gx7okPeK#LjXxFIuo2xa?7T|lawcb{!eOf( zW2tjpsow=Hg@br}IN-1C*25wII>-?W;q7)6%od(=JWSsIaB*jlU-NetMl9FgDEAkP z#?A4JPd&WKALgmI&#OIU72&fhR?H5MxVkwP1q<2d3lbGoaH33eZuB)(pO*2tRQ9P{ zz$%PYkV^F_<`?QgA*!2$=EQo&@-Z!TmD@!6a}YAD6xEBN)*u$Lc*RU{*(w0{lO{`0 zQEH157W^r+oR*M@%-=e1-9d+7#o&I9uCiIG=Y{lh0KHY!Y=pLAS5Xd`3*g^lxbPj$ zhXoAm2~Jgbln@gYA)C3fc(#Z8%N=H8#fV_g%_kM@q(K7w2n7{t8a%z~_$w;x_N{ph zgTd*zg}DjK3Of}_>SDEc<)`Y56U|1i6UuZ2h%%L_Vs(53>kVRtuYQRyNk-@e_3jur zktVo(HkHS2wTvwCFy=i=Tz!MwR#K9!MzuD)+lEhzCECA>LEi)Z+zT$GNC&gr6*U1x zuc$y?b+G5FS(t?#*X8A;mVC0DZQO*EGt_HEG zrrS?pHPG+c10CvV-U1^Yy7YTIyU5@M$DN@W>!VZyUqaV-1E2tg4LHceO9#`10TbUq zh1XaKClG4)>02f88)Fqs8g2s;68v_0pY$_FOEakdvrr+C2LR@m?~aA9d|dD2?Hm!& z=Qj2cTCVLUMDoBmwB3wy{Y!@1j?H9VF8!B<_gMg5(#X|0c>^|A{!WmrI*bCX^0P*V z2hV&qp1ftbKJ%l?c1Iu8!5sZQqqgPo#BrU3$djgmD!AeQ%qh1NY;HA(<`r^%PnQR& z!tM?*T=o-_E}u;-@o`4QGw0qxJi%J4!mm)Tl%YW@SKkt)Q2?<5*2j64aK-oHn@RBq-@xI9 zlB&G3fBV&=k({_p8%Hd(?Ks7X+2i5`!`#i?GOQV5!^m1zog!;2K+|T_d^DQSoU7q- zk|u;8XDxO$n;+%J>v8BQ-$lQR;t6CuDtMnsB!5aa5r^tN;EkDbi{Wsr0wuI^X@(iXM94~ngi0S&2_kNVvpK^YEQckd ziHa54SjKRWm7_~CT11`EQeGTfZN|Ye#X;cY267Q!98vNc~8Mc;;oVW?&8!Y2OMwD!qW()~HW#iJt z;?DSM)?s6)H_AWPySzf%XHqS{%c?K`11krEJ8}=(#rGwur(dF{+nk1DNRJGWY=yK- zU!=v|+H02Bi7v#3+fTe1iGP17eWx(~AFl3?Ih@dgIO_>!#PDg$@Rnf9Ya1UI8teW$ zxb1BcHgES&C+peoULFr~zx}iVC*}e1T>(HC#zl6fV-H*8;7VaEl>Fdw+gBp=*IVh* zN4(@mK*mPL6^E1zM16{52|cWp`h4nerthKbKQ;f4ksC0MNkn$uuX9Tjbr3iP%OpGX z1|ctW>qzNpvLJmomVy#`(`CXA(I&r_4-TKG`yCyLAI^@EGT%`8V*cB3{p8Vh8iWon zoscJ&9z_(VK%fAXvBjooHA{ou1EI;s%x!XG0b;^JUta{>bH3wmY?B-0Anp~WcT5x}x9&p&5ef!qXW&bUSKGCsXaWnuh!orie&wXd3 z=7vY^B}bY4vGE%wLHY8duIw2PW|rwVasy64>R3wp+Yj~^8Y=!D5H|IgIrH`-u*ITQ zE4{;CBup%R*31UO57IVnncu@Lg}qt-67%8360gb<93{4He>iMD^YjY?TvLjo3mo{Q zgR&x9%B<$RHZ0W5r%0F48HZoD>f$+GzIF-cx=a5`2tx;TTn-USoQ3%mG0-)bkZ_2K z=K`z#Sc3Emhf|Q;1Jiq?(yWvOx_@xGH(MH}oYhk*F1y4Hh(k&}GpY)FLjf-g!ozuW ztpYx64gjc#)bM;2`LKzoVk{>X>{bnYyyYPqjXJr8B%r8-jPE0Rlew%;$KLh6R(GxL zr%j7NGU-4Nb&wP}clYZN1Ku`8vemSs=+OiR^_V7I8Mr zH=M-hyz4IFBjBsZ40^3{0^Gw8ZJq{xZ-9g29c%JuAw*OAF!(3SNc?~bvg{BwXCMJU zmC`jD;?H>W>!n3S((?{t$Dj9M;stMi8&9>nSWnZal9$6qBFI(gb>O3ueX__9&OkoU z53D)ddQblxne-=I?G*q7h(n5?3B?QW3F+Ba2v3TlKfg$bOH&8(e#wGJo?_Ay&E^pk zYOR&o6?rlebxQ1y8x2N)8@lheBPmo8N{w$46@{?YC&Ax>Z78tMU+yAvQxEy&D!Og{ zqoIMKG$e;su}kNhO)KLwLGtTtjJ3JlqrgjJ?UDPDZj7xASzHe7og%VdDiE2>A+uOu zLWAf>3)hG2?d<-h*c@CE9FD!KKpBlB(0K}z=Kesv$J#CJ54gmXEqZ;0!{n`btrWy4 zK79DP(drC-j;1V~&piGgt4c4@A8I)Dn!j5DM~rk~xS8r*my#U5vS0YMFxQj|JMYm2 zT2mhG(#^N?mGXbyNu{e^X!9ar#_kQ1?@p!^4a`>SoKaq2;{yo>j*?SUK+0s{#>pDv zp&`tL)ttk2RSh^KXlBaB!0$8^kot>EhTCjW0ugnNemc!^>ju?HW9F-vN!~hsq2uRW zX6NUpmkx*X1e@E}16$8=UYpLDLk`X60@4pXJRkhU^(h(u*6#O0+A5T>FRBb+$2{t; z)bmTFQ1PJ9U_)%2si;Kz@(08|5k`&$U&l3%_+nO;Fb_4m3({7iE1DNCdB ztcqJJAeGw#1_obLD*yO=*`wpP^bfT0571&S!fbag_f3J}PsPMi6W9M3Nw}(D)dNUq zP;1N{vp;#%Z#`rkQMV)D@FmsMZ4r^$uXRKs`i>$EY^9i{oC8_{{{-rR4h(7=sTdM^ zrv55}8DBioM1bZ91}-IJV{I1|gCxBH%4pjr^rRZKaupHS5^)-p8N6cD8+kx1bo5Uz6qDTh5fT@HQjMiadId6f<|f^dI^gTC!INhKqT#T=mNPpIt>QXKH_Cpl!@K~}h1nZ$~ z|Bvc(1d8Q!fBD(d>r1*nYk#N@Arh5^(30MqU(A^FPuc%{yD?qhfU27hj#%dN3elbx z+=hVr5$K@!+7~OY+y*+z;Qpr;misv{2@go^z28V8Nx|R5qt^uihdfTM7HJ(FzpRmZ zt#9&=m9pvik0&eUvAVJhHIew#{c#_Lie?OryI&S!8_wM0!t83ZfY}s2D|+Gmajq|R zE^U?3uz3Pzk#~n1)Qf2_wd)bWXxCFF{_Yxux-u2Fs>VD~$-^&fD%UPpmF_f{SK-^k z1W&&g_=Iyz9Cm5T#TM%h7R4hT0};TQx#_TUt__d)rb@ut!@Hfi4yW*g%i%=2gysq3 z1zP_HU$?@GoNr-rWANAD%*6H$>c5EDIX9Ki-s_uIkF|wLAEaBGR3ANCQ7;#H0$Pu_ z`}{;^7iU!ZI&3)sl`hBXRNKegs5z}t1yYh?2;I@?GGO1}GD~IAGrZXBn>m#vwy-(` zG`WWfNuSsL#ReQde@4V&0z>3Y{wc}cT}NDnd5iyE0a_YM`wKl~FW^ZVcF0zA}A z0v}K4E)oJAlmz3XOG_1V)#n{2v-ohDL8cZGvE;&J?G-FI%B_Hal)q=;v+5ea6K`nq zV6r%`YSsZ#vwq;!aRysvT=A^Ci|u0!f0n7UUZ|Qjr8WTW2Ip4`KETZU$v+IYwA?5D zef)^Ul*s9F6J&69la(~mHA?z!ive1d@82?q?{-_|K9$1en3Vl*YId0VmTiQ9PJ)!{ z`&iC`ibYKeaAw!$P2a)}wMBd8`I$7oi=-tXL^N7Vi?;rXNesIaS1JCgRyN&cwpgPl zvJZ(LjUi~S$t>TD+zx&`aQ`rWy6K4hT>Mo`ovRl2sFEi!c6G4KapLL z!4^L$L9?>&{Mx{;DTd2)K+Ith7ANw!6Dy36Xfk^KZ_NDm-t1>uennjelh7g-mIl}R zz=>2+e&@L6VBB`Fhn{ug`C%4z`E*nAT$uzcu0J zxsR>DSqCe6ZEf2fWk$2$gQFnJL;&o>w&NZxK@{on;FEf;t!Ztw4&KeqX3yKHe`&7k zmWVOIgwWs%DCNa+qZ-_USbGdk}X7NY;q04i(ig%8;|C9_axjTF@IJ97@e z{|tv%`(Lz}UBa*rhV}2^SNIDT8@VbuLd1!oyj)v$n&91pJm*;E2Ia!-2R-F*wFD5mplh`Y+pCGZ=~?+6pbvYu;vMJK}6i0K+caoE*y#-*4sqJZYXU| zp1C*8K_nK(ZNin~?if0+x8^&pF2sEE!v5d^ubPP$!R7ZoByHT~lRirK#f!}IzpgXW zdNY5^e-dmf&ifEfBK?uU;VlkRaj0Pb?uL*tqqvn}o;#(d?SX=zeL%d zr#YLch^)RYn`QPOIU&30=HRg%Ef9sHX2^=!4x9l8ToSN@IK=oHXlr*@N#21ac% zfD|QFVwCkksIP!2zMraSlMvZ5NDV5yjDdq$nLf?zdhZu6k+re|_e<&=QK7_h(P678 z)mkF0Vo55!HiZ*+d{vWVkt?Vr)HsM8Yw2jQmK^VjCBC24pu4IPB|fQ&?rAK>M*z;0 z5IM6XGBtkM8SkUxNDO&{j|KLEb3;_zZ@_v!+i8Xz#wIoV2O!buMQSLB1$wy(b{F8lab9)O@XogXGzpulz4 z72AFzQ-)6QBi#uiNy<5-l9*8v8f8#{LpKP7SU(P|H5ZnWEb~;rc-v- z7JBikh^7$Uu*PE{>p2Ty6?0#Ys-;nt+=>oHlmo$BWZpwaf6SzvOx7l|l!5ee z40JAq2Kn`B{j~sC9-J>f)7)v!_eQ!hIT?#JHu3y=C(@htwf_r&v=Ia-3cH9+=G0N1 zV{_#Cs6+Ln2fLl;dc`L%dar*Xk01@t{W1oG3W#A`Ue}UzjWnm>xbJR|f*AR%zY@?p zLBK7(FBh>L<~iY^<*GO$0#gce3E)I;-tpS3B$ly(?ssr@U6Afv=@kH{zkC2u26}_Z zv~ogQg2JKb)A4;Tx#^yNzE>-of2WN1<6V|Op$6lNLHj+z&=20vyyc7v1%E2D_|Uos z5Q87o>gf{{B=+=r5GK~v;3tGVg^PS&3@*s|sAPtO&|tu8XK*IImYq|;1vadDkX5Vl1jGl zmmaS7CG1GJ`6LbijE@rdph&bblbE;uR`mmICoKSuS|Y3^En?P$8$3pJqy`?~_CHwU zH#PCB92UX~Gd1ZCCG(Wl(fxhAbvE!F^+%LcEExp3x!}FoHyCjbquoexpVT9@IwOe> zO9BGOozqJ3#m%D=FF?c!r|pC~YN~trX574HA|LYm4mf@C_3P0Z_Ik~wcG!NpPuedk zuCH-7$vWiqz~452Hrjdfvj462+TLAKUA;_V739rXdKvp{tYiSH3W{WUrpn41&@ zGS*vc$<7b7nTzz1HrrRyF16-9X7rO*Gpq@ddzs8SJ)xx%Cr&VSHF4;^Lr? z<##6^@&@XDG{(k)&Llmhgox}r%mWf}ZMgmaq6P1v(O_!3GeR2LOCriBGLp7xg>E%r zd^YRn?1?n=KB41{9h)89&=St~;;K zKCn**eB+=LxP82H>wA6(SLzM)zb#YN*Zl6EJNoTqbi#f-WoU9TzVmo<3(ECYJk2tk zBGdcMCvFZNoCJ@e9kVU$s+RSUp3nrPb^G1WdMnDJ49>sw1C8B^_ESeZ22s&WbRb+w zyhYxH;vr(r{SagQJ?extqnPNmdKrut>JJ`k^oc-sNz?Hl5#6OLOh02(3K-`ytoY@p z740AADM~F4e71eM`NHL1togzRYBMUuCXO0iA$dsBeSMv-zIxnN$caFV1^(l8y{cqu z{>K&Nt;Ptf_}Gs&n3cf&PONS`j?~Ff|89_4Vi_>W97Rxw9swHIitEs0I^EXqY>f0Z zrpcot8R8b~Dn$+2JunF!Y*@%CMq^250Ih7^BrSVr!tmXEGKc#otH4JiN8R`8a6AcP zEcBQOM#toS&-oR9r`0xkfZ{&3%BZknc@j%v)2Fx*1&-Awm+$o~k*3WU)jv{-`D!rdvBIxqkuKXm zkh57UN$a68%x>~_3_f0b3%gr>vA|bs?)owi(l5_Z?8Pq}K5B|3CR4A~C(IZA0YlK*;iI z+=tkoI!+uwNwL?LyR#9b&c0i$QRheJo=1(^9M}-_VrWwq@4bDDjn7d(eSwK`^B0fh zvMPCZ2ZwC+hC-_!V1IA4=o_viBL$M)D1?NrFNqX{8&%Ltzx)Rxdk+nj0T0lHYWF>F z*FSJ$_+7H@V>CYKf4onC2lELgVL z^HT9RdTuBNvXrRyXnd=qgd ze_8dJ<8lU_*eS}0_FHp1%v(a$w4+gUuitD|Ko;$AxqUVRH3ivgRPY?$%ZXXFu$k9k zLybxgD9TTpLRhk99|B|gC%fP615GQv;IE>Rn+IP2^48A)hQcyfO%F%AIV4EEC9qO* zo#H*5WB4DQ!`lvLIQ@v!<_iH;m|DP)xVh4;rO_GrXxNm&?^j0YprBCJ!Rzf{F_Xr` zNWeNoWv777KzT9EHK`8vcU_iJWgo}YqRckak#u4Qt$E+fYl!yaH2DF-5!Vc}wA&U8LEO`gKikmxXV4jH zy7m^J(8I}K+TL}z`$7c|yn|cVtd!-E%Ing|osxA8#>DY%esG8xQ_Np_}19)u;yj2baxP*|3Q4Gv;OR*QWV`d-!Uy#?>YmV`=u;`6c@ zxiqD5{Y;-eMM#*SNKmb0+O>nFXcWNUKKgp|oyaWc^cic762)8=szeMDUZqT~S$;hn z)3KdtA~j5=IHb-3FAhRJ;k4NQSoeWNnSv8W_}Nf$b9WLg9c%o>`grjvo+-oQ!m?4j z$rv#ntCP*Ch`{qELN&bebeZqcBSiQ|1lxKWXCA|;?`81hg>f?->%&Fe``4FqclRUu zXQ2V|4pX+e4Hy_tczqM7|Emp|!{YVefbx0L4@hQ=&K&?Tr~UnI_KP9xF4CKv?8gj{ zAo25k+vC|tJRqz4!GPamy-1WuvC_QT7sj!=(ir|tA`(0QtGO{Y>#u~_mRi6J8ZN!H zV{8yox?oV5f}?bd*ham~tg#sGO;&SB+=TfDdu0P*^bQUAGL4o00z4mJRS(1s)xLuG zxV330C;98DEQrf59`DZQgOlUK=<^#)=;MK#180(2RbAB}d|zhr$nkh59AV%d3xuo#TZ@$)c{Cd$;n&ZnC0>GNu|hD-OE zmFWKD=T`Zm=dG_4Cvrvs@(^Mt1ji%!*HMIoA)&%?m%l*Dls?$Zh2zToF|3Xk)dTE} zO=o{^1|tbdSwmThii-nmbS&DXQ^%9&aV(K!AHvf>DnkF(R<${-a0>>N#KN&@=V@)7 zI-}r);u;*wP+ZVuSu7(mL3WB*jSy!yO;gBq>hK z87wv?X0=@Z@*B8=ZGVR?46KF5<1#QdNXLWXgQ7jxJ0*d#NiUP92n0-%(rFiZ-9)T+ z1JUL@xmC}rPMyDP?JXWHZsa>&R6;x~){NS+NOu!aYNgTKmm4ih;D8XO(xXj^e(#NL zAD34p#5Ji1wIX`?=>iF{M9EGj)ns!R`yW$IQ2%eV2=ET-?e>Ar%9d1^X+PsfoU0|z z4hVQaS|C|diQN zkkRr#z}B)mX{c^YSUfQ5_ywrFQfgtr0f{+!zWI=&d=9TGHBSj8iy&iVYSh*5j_9rJ zArgK9uJK83NuOd!C2txZCswXWR= zmW7V4N1b+tlX4Xd3DX9o5I-rcO_O>9!N=7PmNw?|QKRDKbFaQbvzxhECLWIR>Cms} zL(=AGt_)DUc?FqF1E7awG4E32thgr@ zz5FZSRPA^Ab(_kT7_KVG#0k1cz~0XFdo7QYdy2%RJ|(*Ob+_S1ZZ%hypAh>m{%4ik zbths*l{8xVxbIWF@$^xhBAg+XWd~pb#uruE^~mHGt0Y%ZsR&w zHPm7+oJLteGAZeF#iDtDE+zf3MR~nSU2J|Kc&_gykadqk?(6bpfo<%(3y~%^L9v#U z$x8x)THILGLcLe6IU=CXY%Wr(=2r2cZSqXFUGF3Wen`oR+}NuboDHEz%0wNPfC!g7 z5dU%JL;CBZ6Vrc2Uy21JCT)}6?H~R29nr$~L!RU&F3%eJm;Dpjk9WR(o>RXsFL!n> z3-eF`7+W;Z64Yo*yhKaee&fA^M$ToGgurcG(qE=bmNhE<{Dny~w)0*#2;dv;M2clB zro5Izq=n{Q$a)Uzm%N}+`-CT5VGY| z&6;T#w?`@z5ho3@=dIjGvgTmLOezcg#Zjfx&6;^`bZAjgCs-~Duf%nSs61`PBvU-k zgj3RgeOfNv$k(Rlxt<_~Ec%rbXq1r@E^&-(u(O^m8F~T6;ESB-s6sQnr&cOx7@0A= z0D|85G^)Q++s-3Nr`x|UEX>0bluY_Nl>UCRjGHQf&Fgmk4Y?#=#YmMDhTlinKR}ri|XrPzJI-8$&{kFN-4D zqE+PvI@_a_KhD~2C16QV$9KQgTd3nu@M!t1Em0pSMaCV2iW_g@|Y+Uq%wPD zWyS>D&gPmz=Za+k>)wa}cO<`LMoItffA=wqV@6w@wBch#Mri^z z^e1ha<4~c(4>{8es}wO+`UqMxJbJBhzc}#>`*g4QlFYAZ~Q#-VNtPC zJdb17rT_`+@ia;Pxdi;;)@z5ypc}nqUc+Ahzg+e3S^^T{Mq2MGeF@Vwxi&|w-V*z$@P z!CdOe9YGcPM;uiEyB5$x(TgE`f6b(P@w*NR$OsSCu>%jJcgz^B@{*GX;ID`F7!5*g z7!MAdt$`gD^|t(K!)-8}&N?IfQ+j1japibZG%_bojs7!~8in+ht3s2IxWOaV6%$Sb zVpXI3Jt4#>l%ga~V8(%|?KzWRFP5wMi>JOJ9||%inGEExp6U(f)HeSVIyO_6 z+dbO~fE7FYpC6wAb*xjrXDjbSw)L~nht*cT4+e#jL#zTo!C-8sO=qc-==ilGX%=>| z9<_wSDzx)U6)2_Iw;o;k|H5TiP~+3>x*yE4H)6yV@F_V!^x5HI(wFyp zwMG(kM6KEU>BVSLqw`E9HXTuM+CbsD7QT)zVL%IDN9^b2*miGB)+sk=U9;089ZO}! zK+lMjFsgqwxof2mRRZFTzW_@eko&V*bf$@(+spu|D5`j#Id59jkDMh+(s&7?w#6W# zM~V1(REKLx++annLJWwIsI3@&B=pLfSjLA>5EeL}pkf55Wz=%XNdEDNPX@ve|4iTs zrDK;Z8cRX;B>ew)D69RJhK4{&Iv-SjG zQA!>}q{MrbzXH?gOz|SW$Lp%;QJQ-jM=Tekeeqw_f2SwO-70G=~p1-^K!p<@7IpuL>4cggof&T zs=*L?nG59vfyK0z9S-6G|ET9aO5sQ~TJHStn&5qo{&rmJCVsO%trA?2WO2R@<^9%0 zZZ>%fq~1YyV#F}9l`Er>9*Ke^KP55I`ymo&(Ps(tP+}64p6h7T&?&VyuIFylZQpYK zCze?q3uw59js27m@nagv9*G%Wv!w>7M1eWZ`n-_&*(X{dU20ViAL+`!?($6D=b?VnxO3{!$5(O6k2q9af=>l^B>!ErX;&+>xbu{wK$0P8c3pcRl*J_xf1-AJO^h z(-F-XY6FR@@_dbd%D-Pk6_GSU2j`UHhr4_2o?fP}1RZIdQD1*(JNK9K9o#{TZW}d+*Tn(qN}X&4J+(0R6rMwsu;obW~ATMH8?a z1KltA@{BsEO@m<$^r{=xK$Pqh*H4d86O&&SSYH=v20vrN98W>dPrrO4cq(r{(0h#`O_@DSe}|?aHbn_f1-;CetJwK&Fi|!Ke~Yv+ z*foQ(M;?7)Lql2u&|h9Q32H@sVf`{zz=C4gVlZ~=0Zf%!0IU%)_BtN^*{n=w4;L%;&}xEswrxk09Oh0Oe&UIgNgd5Vzrg>;)msK+wXV_H zbT?AcEe(RC(xEic-CZx;-5^MpbcZz3oq|Yrr=)a9pU1uS+UI=VZ%}wAbI#|EF|L8h zZ4>KflaKs|PJuhSxk~E|ecPieOxuHFoAjm+C=><7Bz)NTn->he{vgkf4Pgx5G?-JXAYdLdB3okYFC@iZ7GC?t=#q`JRBe0dia?b^Gnb zU-)ovS{TiR2RaJ>D?_xgd41nPDZGtCsWT(`lojQOUe!Jua5EXA_%$$lqkg>OJdz;f zd;MzpVZn`f(9XP3l>Y*!ac}5Do3S}&-=M&Hc@<8M?;{tmVZV5sQEzwGV}GtRE<6H| zL5a82pOYMGnSQ#=5oHUw{YE+A%l2}0{jG$$GyFbD8m;IIJwt%$Yg}Eb-fz`Z#ZBk+ zFZsdv6}jW5N{XB1w~CJYp0C~lciF)NaYo_DObiVDXR*ab{SSG&x^5p+vY;;MRE&WJ zruYoXU5y1%Lij-bz!Q;I(Qqb`FOHY-HSG=xn55y`8tKFR<>1Msb9JMpee*FZcGvXy z`GuzuS2E=qNDKdxaWb=~Wun8izyf(`vr!kIPqRz1a8Ur zS^I27rQ80b#9m5iqvV&a`=2~(NI(86Gr_(?c~(98MP`skP^Z4%DDdwYQ<|G_6a{}NU7)50Y3~u5;;pkT=QtQ5P1a^Zwyobs#06y12CAt)B10?h@?^?@|iw5 zg7FgL5PNGGW4XE>pI-R)SJbPE{mXWpiH1z+6J2(?j*gx!n566m{?(ECK81dCf1wiW z70jXB94q%j$3B(=p%&?1F@y8FWNzH!(vNtfiw+8LDX;|wSv21C2}o{hq+ur_qSTTN zvW+oLyLU5|LfZ8ZYHUh8MCszZ$ZBFd|MqH=2yzT^I+vwW!MqUd){9*{)&9MoJnbMg zn7B+;rr&%^Wp8dQDA>$T<$Eib*8kaJRS}R6yjhhoIJavOh`Lx173S(r7Z&PMla(L& ze<~h}*(kR)7qQ=q1a*mR+GUBbC!bMsV}q{lZy##yEfC@`g-otij#k&WWFq3yO0!?? z{H$6kQ*!~O^$SS zH2WpG_uA};lfQhTWL3_>#;2bfvs`~(I*w_ z5F%!NQ^so7_i=f7CMGg%ZI`%3kek1BF;Rb?@<VsZ>(xNkBnJncRSW06mYfitV0e1>-3^3L;?%p zd+nTze^~%gdxA7g?=Dl0(M!mqzJGJJQ7!#Q%jF^dqgIpb6Bu^Ct~h!cm4`Ql(-YwM zznLGgNl4==k&==J#Wo(WAl&bau9oxN5N>TKW0uNju>AX)bK)`tH>C zT}z*q_hZ?>azpf;t>l`VJOV{w2BT_*Voe_Gdyh}PKT?YCKic!xldyJZFb>saBYMEsLk#qFY%#d|nX1gcL`9 zL-n&P9xtC1_e{{M%nAG`F8)HI%@w{i4=InU2DN0JEfDY2NJK>#?gc6eBdLUvitxUK zu;*@KqMsUjtjU%j-d4v#FS_>ru4J1ah8>2DCA*%;HA#M>p{*x+vBdJ`M}c%8UC z`^6No47sYkd4BcV&H4<-1TW8^5epF!ad%NDoVA0QH{DO_J=7E?x$;OW7HUA`S1rZ~ zgZdqbbo-s*V)(tMs})elKaV9oS8fJ{8j>r@joB7$q-uHg1S3xtr098;a*^%oBr-Tb zR)8iYRj<0#xgA2S+B0@BCZzBDNkz%57$-dpC-~kRspc>4>>SVH7J1N6CYU}_Y5j|;&P*cTP|z5&c+3nV`mPz`!w?`23@v{ z?|@I|3GWW@ZuD#9!JE5TUaX}Eguw}Zj79w((8qg(P*}E#5!-(^=eD!(4Zff#F9Bkd z$wMB5ofnDz1hRE~{v8<s`e|5y_vCrtIQ;UH&V+vkiht!RQkk_VGh7>E*oJs(PYB!~aw-(A0gS9a zG7?OaAd6<+u?DN!r;-l|r~I^pHjX$T4fg{$%~oeiu1b`MIMS>dy**IhJyiGJBX0C~ zo$w*=L?IV%;>%}HP{wx&k4)w1X&OpI#=6f^;Z#kwP5Ex(LLY8@t8`~kH^?=WDNi+8 zICkL}P$?lv6|Th!uW1yn<%&A&hWqE2peTalwsW4SHsmwn_X?5bDk_9GL+JdR#u*h! zB%w)#|Nhr@)lB)y<uY2_@p)(D_c&DF}O`B#@w@cP}@fpbI-iHZjxcvk|~( z*-55&%+*(`Gi$cEvBS)Cklg-cJ_7N~PMexX9F#$mA7Aq=_vQ2NZjUAo8{}xG9rG2? zV)XrHViMmjB4#;r;ittK{h8+V^I+lRjE%WEJ(7S+x6je59aMFVroEQWWD~SAtaa+; zA7}?TTQB{i7vsx0K7QlQ;caJe|u^ zv(`)U4+Yj^LWzyu9DE@ZO9?5&AMhDEW2C#nLieF59h&&Rqm_{@jA@^i89X(CHi(i%DAB20`jo`V@0?Ax|DmkB z5(k6HNKy6$bPsJ3tUX}r%I%OuaSwPA=TS0Zy8Biw3sn$`(dq4Rd6Ue`HiTd-?C!hj zv0KRa5|jf6y!y?KrX!2+^yfecJ^N$)ld?cx2t*($M1zElUi^ehM?HmElS)sg{lrDT z{ddFC-ewdVcX+6&OSps%^R>)0wZ^A|@-Na9D3uG%*CQXaRZ*}P zEOB+bqOFC0ji{gc&a2_%uvDSxR_u~gz6*+6^AE7RnM`2!Z|D)iikOJF*icst_uqYw z`wpZPY~t%-nB)WM7~lb6hoSRX)+P8a3d^j-w5e;C0R+0;AV zCbxq{6>Vc8D{aqP(f1v~sS_}rkmFhChcXfTg<^-f5pN0?IDr2n9zycZlgb_-Dr2z3 z-I&U^!8=p#Rv_Hhn#U9%_)v}HvPaPYiN$e0s`1`w7tE47?hv>h6Pb*J(Covt;W29T zQONk>M>_K~KY4y(KTx^|g1y=va=cKeGnN11`Vr~#g*<=Q;yR=qfXw_bWLx*00-yIW)OpZ8HGHtq>n zde+6oIK52x*0!uU_Bh5`w@@I*A4M1+J9EV-=5_A(7_p4lw#CnSq=J8${N?_cuY`SntdarMut zAX`yr3EUsm8Lz$!m(@w-VSUx5iBVY)htO2@Rimb-nja3r$PvfJQ6#N~Ol8u;w&rOj z6oyVLxdhU9Tlz#Gw-Zw269BwMbM#b<9ZVa6Qf3={PB|G#2eMg+CuhL7Rk;MJG+Pfy zcijW#eMG6mry!KCYD`y03(8vN`UTS7OL-Eg`th;Zj7m}Mm>_^Bkd)i@nKSbXMrSv)|a1sb4?#S zTo;jCg{Y_>wi6kY^Xy883tz(QTU7EXsWT%G)dS6M`VX-777wf3n}`-^0!L3DYCVP? zwm^nUIS6JW4S((3bSKD=HHcc7+kM<&@e9az36r9{c}v(|!BsX+!0|S3F5{E1Q~Mv> zK{;^gG@EYxC87>F%>xEzDR7YKIS3%|h_HVGTKv(>_8v>kg^0-BH^g(r>sZoVQO56< zi4asUX6K(x=6h$l%{E!5z-g(!3X(8P4moax^~1w1nb$^N50 z1RgpNk?oU|k=cN${_HO5&)Xg$Wo7Yw<)ogsJAXRVgRt7f5Wf&=WKb66jo=5_bC&80 zwGtLooe(O|@Fi_IFfF#XMi(gv4H=E3qH&a@w_beo_)_Gxa&dY0>})UC`1zlwGu1y4 z%FGu~;8UBj-^p$b?nVfzSk^X4_7ra>C`3*ssSn)f3F<{d_-E|C)-XEtqolBxygp#? zz7;x0YCXBr0Q8^}y({i2jhfSU!~^loD|We3WD`*n^m36pU8py!@Tl@FlSQqtpgOtp zahPMh^8LD4^w3vDl|XYwL{OhH+t==cx&r|5y7=-YS&%gs+ho+C*8(m{W;@to;~R^I z_;zJal=d5MpR=)j8r5P2oV0#K8vb%X!A}ZT+O#1q1}>4GEt^(buQ^79Ztgv{1>OBk zuq~+l)|6NZ)ca7n*tL>U}vpyWZOY_Y!-L5^;5KhlN z-%(GzJA6_JXX4(QNxITuWn{IH>_XyMH7Vh0 z5MggGjl1lzn>qQ%UA!mF7U*7wzf#ggaxrVwhXd0D>iz))cLsi~ey-b6WpiQOxn&Kj zCYn@0EqxmtJ$aFMOwuWolPsgwwfjFQk?a9q(Xo9UzSVUI(JPTnUy#>QqqBx(W0nyW zE5A35TSfTZWXdSt2aER$dYLVf`2tWrrT&ym$B0?MvKX+f?Nr7=OI!iNM{v=PRs%mZyxITZl7S%q2dj)O)5u0|rt?X= zM;;rW?+rPSC8?=x&Ae$np>utnk@Ghf4UL~4E;6u8Rh5#Rtv~}2+}K6DcQ?XiNkX@M z3?uLgta5J9Cu!yf*;9aI`rxFtx!P3HKrv{IY{F$NaeaqgXc_bNnZ`pt{5FFMjf|uC z=S1A+4|5Hw6_VGBR|P*_vn8Vvp5yx|WM$w^x{Qc>1}jeA?vhUyG+4bUIOB*=$>4jd z)Z){hXLtSb+t6xt1Z8=feJ4)L;F|RA(Qr#%F=cxi(ibcRbJsm8x%5sO>>b~WX6Lkp z?_2h$gNkbli+;3H2&VczL4LA}zF|=-p!%S*UGsaoAh-0ck7IfqI}h{6(i7Qq}XY3G6$x!$~1ZL>)s%B;s#NQkc6oPQhw1g@qpvNHjeI zaSb6QO8KD-B2D)13R2+HF7lW&nK6{Vi>SZMsknsxbQE3dO+@N=2#HtE7nykmxE~rUj7|lMP#$KYw{}LHTlz z0~E|N!pzD<5Gqiw{im^j7ha#!pE==yl-kS;Ye9Y6q%xMEU>l?6FSzK-F?NaBqahV+ zGazlXnyLM`)`4+%{)3s{o&8J4J!?Xi_oT@0rQ*sQf;Zi^SXDCUBIkQ1?|ZTTtrsVgpjHW3bq`+Lln(;FRj>1k%b?}c z!*^X~2=cRq3OZ69hr80$wIeL0RbJHU@AlFITEJ=mky?PkN_!IqK| zlB=xv@@S;s_w%UCH7}@STCII{zwg=ig5;g3R2G>{I1h2${mL8-sc)iguuam~>o0Qi z`^=g~t56@1TnUqDKD<|%-H{ZrTg8-NFprst9?>dfIS*?Y6g=(XV!Kp!9Am61=wdP|4Ssn}9qBmuDy-ejI)?TT& zY#BEdnY=~n*pC0cK3*Owr#sth~A!3pxR>Iu0;m}IIPr0QsJ#N)78(5f4K@|ML z#ev~2`~ao}r`WFkipa`~#$Kf`I>kSOHNIh+t^i^AB-I8|DL+C1(;tZrP$+Vl4!`J3 zozmyXya!>aj{H=kYt=&KeLB3!8@@z#UA`#TG=f2}eXSDCV(3BY@3+?G!`6#P!5518 z-GuJ;@`o+=dRdBdrpn=Jc*9n~Yyz&^j|O`8WK1XpGcmB5jLJ!OZ>}oa`K;b@s?WK6 zs_=(%oCPL=>36uYCciEaYa9x&bP*W8!+Ke9V7(|rlh9Lv*P>nh;p>ljpgdX)5UJ$? zR`7^<52CB?TO^O~FqKzBSQHwAIdSXkUty2V_Hy8TR%M{1g?Sz zAIoLlFv2eT{7`fjlc^gi{O?8hjVaE@&1#I?zG*%_gm7Qy21*okIVzHG>AhaNRUz2Su!8{(PqSB$-vt$aQksEH(U* zCD}B5Nbno&<=-6KB^^vP;F8OQkbyu|$+OFx}9VsHByi-y$4Ec~OQWtAHSUb8C8wzwK2 z!76Yw?ETqpe30h6nQ7i&Q=)HnqFB~s=_!1p*Zon4l{u)1?|CkJQ}yeImhDue_W@PMb30!RhyO*{Kj2^FXSzl?{u!sh{ zp?Qhrr0SG4;x2JHHDD+(*tGdLiJW3y+h3Qe$o5HYF?V?*9!4wB-;Y(eIeBJzpf zSkdTJqrJ*W?(3Kbt8^W{b%a6Q(cZF)+%j&$kz5=EF|<;gy8B(XDLEb`FlBkjm-iXG zrP5Kl4#%eMiyHJOK0kfN**Cy^eoJ*5BZ3fwPkrHQ6-SI)>R`EwG7jj+PVprWCBwXqFot|1-%0 zPe80Rn513C+T9Ho$u)aXCN-Etbf52j$Wcp{6h}$id0sJpShGYI0mB_`B?ZLxN7tfM zXcp}@;-Mlu;3qiWM&tPMSr<*PQE6)cD+5tE{L!9+RTE9R5h(zAm2bZPBCJ=1wvuV$Gx8r}b=6LvcF&O&v1N`sQ?w2u_RzD(rBZ2kYG*WA(0=Yu;PH6#*QoujJBz11uKrx7 zAm{)u`tO)}v|wsXM^Yyj`WbKEfI6q)LpDC}0`F zJ1#=B|1EM`?wibd^Lgai7?1SG<;~8BT9X4Z&*En9VDO6(&yYNcs9STNgZml|>iyGo z#P7`uuSnh|?X9tq6%)SUyrAZvE++7ZktrI$^YV`SVuz{6QqG&BN zYUs!IjP^R?@7I0#`L8|r-|qRM@G9Ta4!4|lZkNoBv(dG8mg#5$^S(S*{Qy6hfQY7> z)*EfKwY}ZIIE$jR=Upy?OAPI!g2_`(JEYZ#q-^rf&l#>Ey#2l0JjK*zJ)Q#%WCikS za4x{TFMGzL^Y@u5v;94`3)Z;7mbLV9=`=2gmoq0|TJd+NJOx5`yp^D{4e(;V5C_M7 zv!28F-g^VPNrVK6+4J0el6z}DN>UXoM=Lr|KLwF_gA1g4baK}K?Gv)of~fvJiIZm0 zY@iv>&M({h9Qlp)?5B>~U&NN}57Zp(?P)ny#!Wp&iSXD@)rP$V{j9nziNKvBEtAX~ z2mUg_ga@D-yN>kDj0_Owl~?T%er_{*B<-;vA+CiAcnxO<@lJ}6e4Nv4cBJh##g>n# z1;)RzNeKd78f_JeEyv(!lu^FV@2*pm*<$dH$U&!@!2iu?(IcFd+GSgI0NtP4&>uNN z)6@{48S|!^zU@Jg=f0~lUI7_zUhrV!>C?!}WWPL~D^?2RLc=>9@S2*MJ?;Cj4Awez z;IQ4t`xwUGxTS?MQGW0c2o1w1Qc%Tu_Taq{LeO?kH@^=9SHv4uUfEALZ#=s}I`?9x z8Oi0(<&*-lD2OmNKP=nU4ZSlMkFqjAI?mVg+%GIa*b4b7o{CJN*e{Iu?C*mjYVte| z0u?qK@mJnGeh0}_FzV0VK5Ob#dcWHTgDv=*Z1i8Q(ZkI&r36&#y4Zr&<sePkj;+1dHoIX7)>m0`_-c;{1`}s^4cxxzbCp0-RWV~!KmzX z>xYCcuMiVE)VZJU*E{ad`<${lR)v1;q;U3qRV$sd)GY{?K&!2f6d89td!h&`lK=T4 z=($e9TnD?V3CI;|_sXkY_oHbNT`oIeaP)@ij&vF3msi$C+`y?B;v3-IK-A9UyAND` zhB&cqwz_`v?C*ly{hk_>S9rIBHP(S~#>ybp7B%z&wxM?K&za4JRfZf0%akx4FbQAC zBw1}Kppts_3Fzd6CWp?(%{t)`3?d_aBsg>Xw+UeQVDJOK5i9oB#CR@!pVvixp8YY_ z8+%`sOZ{xDlaNejSt?tGy-+>3Off=kJlvSB(#H%0x2^k=v4l;Xj%Ailt&ZZID(eDP zmPT?ktDKm`bZoHYhU?@ATMqA;0~D)%;;k%1RG7R>3^kt`&7ARhk%y{*1~=Y`i88=$ ztCizrTZZh;V}Bj)1}}cnNj@L5?t1q;+kASwO8UeXImY|sobac$T}zJfX%E8xT2`>X z`C@Fw`Ly9VzFmoP_-H)s$mM11Hyw`?;ZpP@YiI1$;_jDSQ z9RF86N3@-N(K9R6I#g?F=i%W~F5& zRu`<-+j|u2VK;T*rpe{KvAkA5xPTFt32b1F7pU0ulwL0ixg4kzVxh$hG8$Ip!aPd? z{PD6tA-}j;?f&YT$1Z8r_*HHg84hON&-Uewp2*T^N|ZtJzE;SQDm?Fq$OxR&J5CYW z>CpE`Z`X6s)J2ixDTASlYm5h_>q0G>fF&^06k;4ajT6<+a z#B{lq_S6$Pi1hkSbqSWXD0(_e&H&Pj;NzYl3Q!nUK`q208Cw-uZM6r-2v_vAW)@_i@4brek z21juf-fp-|^j!~}VdPR@+lOOqd_tmEk=6$_wOR1xT{D0MZ1*(LeTup9bb!=+)sI zu1=K0!%$^mhpdB~`$1{3w~(0cX2#P9=h!$vNTXRg9EsTg4UfH7e<)VmW?N7<)O;wf z-a1C)VOg0x=;b#z4`QNtJ;oMDJ{C&7 zZFqx#CvwSv+H{;RNLaA7wE8DFinoF1Y2D(hQ^P<_$RS(CS^~p*Q&{iya`T-zIg(jE zI+4*j-SJY_G01=lJ%ZRf1A?39Ng8Ceb1yFnBt=QDUI;coS8mM<2|hnKQKl*i1qqMG zc#j|~2N?+yo+rVFHwoPzah^OfUb7t~G#%#%VqIzrrNIrD zDUUw!c|4peK-O=4@FR&1IJp1c9Vij<(egh2qnZ2!;sMUdiNcd1{~s0P&8u4W`e7U&JL~XI+Zf)79w&o@Q(fkq+G+42?; z$PaAk^n;95;aJA0xR1)aavnMBB>PLxV?Jl!5I?dIBdqI-A7TEVPd9V#4altaV3jt&z)B2?F+BcDYbwzpSOyfML6QFhUo6x>-IgjIu2 zeELe{l@+&%9t0cD2U0Dv?rG(tG`MUpc=MDQiMk~->t|m?)qad@9Q{3kf3Y67XX-}Q z6e@r*BZ$S-39+J7R*WW)caE{I*^bg%Edk#V|80Ht$Kb=&*0A%A z=YjiO?Xjccd{b*i4pH8BVJ}6ExqDx{N2G+wP%)*PforRIfzXSNABcoRZUSv2_7FnM z{*zCf!mK}O<5!hTmE%LNYp6deRaxtC3ljxuxEC2X82y_Rg(&C(ng})*A+X|o5Sn&y z`xcgqI&b1839C5&N;8-4FRxA7P(gGPNo%cTez&rMloh_F|PKfQc2%w#t}fvM_`^wJMYI*ge86}-`{3EE#J6N)=! zTl+>RvC+pXAZ7CS(F28%v1SULckzBy<~!;QVvFr7CQJuH$o1pR8mc;aa|Ob+!pKX1 zA<)#|0zk)BsTT|9A>`%1r#QTX!4%!CfQ7yf2#$T%Px3LQ>uigRLnbUw>T+$?#LVdW zYo8Eo0f}rSJMODsIFp_T7t-wEyTFDAeHAPw#e8+X+IEZzrdq|+jltWaLCst9NEiO?Q%FXYIJAIq+h)?4_=wub@bRf8=NuCIc)cP zzFa5z?8Cb15qa!&w@dE9fEDQ&jvmlpdv)Rj`ILsvY2FeXr(tj~jeDA9bHdi?-WJ5C z)quvLH!vd9bH-J@@%+@|l>!O!jSBukT>M!e;K$td*-TrZ#mauygoI~jqkVdm=HLom-#^XnFlxc)$ljdgtSBs8Q1`& z*`0%Z?HVByy=gwP9m@QX2GWvCZSuIQkw}i8bE{2!Ne`VoL-CZWFa&v6%z|h3{5mRI z*earv7C)4Ch)c1Ej9xZH4^SDDxi(3PdTU^9>ef?gKPteS&BZL|UZy4*A;_g93RurZ?K}A2(=J>0Oz^5v3J7S+Mn5FG ztxNLW5O=(?s z#URgO(=nP}!Z#IqsMZ}}?OKsKTgJ%X0rAu@_cz32V|(D%>jezD zkjr##t5a2f6Sm))=}AYqjR~#Mp|M96N4pcWs$~m$7$Imo2@r_~DF~M0Z3(R6vwE~snu9GsVta3BGa0%_cNw#u}6fv zP8*Z)od-VO!JALK*+#vRiZ$H|s7elJ;TNYB7Uglp3_mcoNu)wsIV>bR5K?U<8r z@(g6+PD#b`83c%CRU`N*VSv8EW28KsVbpZg^sgsZD>xtLRFV-bR2VamP$4~vO7+mnipIldAUQ@V1MdKO zt_yG=+(~(QZE;a5m5VXj!N9kuQ5&el>aSQBLcin*LK>rJgi8=Owm>CT0%ZHE%oM9} z@JQZ2;hI>#R49Pp89CH(CH3%0ip3yQqXOvPL8mxD=-#$5TvhI05gU8hIv*wzt2I>x zujP1)@}lQ=dEs!nYTkmOa@CYG%{`9z?!2VivV)YtfaOCu$gQ~q#UK`fnYfp{;LWmA zJB&!UX}(KK2%+;vPlz*+)JRB>2ioxt-|2Ve4YT(ooiy)~o#Z6F7w_BYUqaNe-)JHo z@+H+VMdQ_~H9>Ycn&)s=(&%)^hAG|wVRF%-7^E*}TO#B$XtOw+kGlXC(0ZgCZiAOs z=wgI3*xHk812_k7Fl4*C3N2ukpEr7BeEF~QMPdBcRw~JR1Z-=@TN? z<4dI8vS~qI)X4I$APjb9_j7MpM1vi=&s!tNTARm!vumrT!0^GxaR+zxO;mF4^@4mU zu@iIV^($B8*F1# zYl<#vR0X3*P`EYKFS#-mnvg3BrIl-*#7aj-ZWDc0DoiRg_`WjRd^z`He_L`&7)4; z!kC*4DM!G0ZyoPlg-&yPeD|AaW`It+_X?rVZE;q@;v>~<_jK_DVGNeQ@T|u(KP8j* zjG%PW1{fWFkXOuBldRCkg^;)4iE-L~v**f)C2t0ba!af|CUL{buBilm> z?HFm_tkPOpyM0_iW=1_&st2ymu9;$sdZm?M+%qrR{)C&}1yIIZK!C7F2y-OF&?B1L zfQChIZ-hU<4AZQG9>j!`0Li&w>h79~qcdg)UO*dw72||-Mj?B-CfIT{K_?6SnOpP~ zS;T!F>$9^K0ybj?C8hshjfG_5VyQBHRXB<0n1w`i>GxVNx#0rLcVPXBR{yRIwsY0# z6M#HfbxH(_FrtGwVit3_0emU#yCfvLwHGPOY$WPf$D3N*U^N;}YHjKvg^OrzMAC~j zp6)PFq1)PUyg|pKYzU4amDYU`D~qCJ&Hq({xq8 zVBEj}ksZQh2>mihIqZS~|i1PMSdf!L8iRXEisC>U+Bv5LC#_)8oY} zHU<5F-7KeY^WEtKo2ka|6!866GBB-f#8v1Vsns;es@rYzZ2XEjVbtRjYmL?lWe(Ixy8seC&MY3nA-vXWRPm z93yZ9>g4_2nYl`pyRVwcwwK|*+a2;f?+ZT)B;;1grSIKbt@)zre9^z z!?*IG_Cxz~l6TTs#+zfmsjT;FO<2^kzM2llwM*Jox$`WLXuk7k4%|K2UTu_uEFZe? zH>j0}bDxEQ`)15x4a$MFtJ#Hf=easnXLHI&_{bU&Gj{inQf;PE+*eTxn{uirMs++t zbB`M#&dj9^mD_n@S*OJ+Mlr*ClBq!-bxkFLGxCLh&O;J-;*_v7lWKP;kYyG zp&QKBv<}m9+Jl)Jh}3*s!Em7?pK)~Ne**nilnty5QHpYQW$3A6(# zRaMb{g0~F0po0(qUfkD;_^$HA1h9@9Z!qr}mtQ($ zm8eyu-;hAputm8l*qb8tZHSVPtPhYfJHpOcP8E54Ce6U4P#g3>zQ=k3Xk7fO_2_la zf?oR!dv6KZ}-fPW~g1&hV z$&~WxBHS&(rI*Kwq0abnia6UO^y!-4sz{EPt>+DK&{Fkp2b+!5p0i+nk(|!~=n;zq zKc~~C{@F}I;Ua}W!D*Rid5AU-yUZ=@1!2BU`r#pRj_uDmLz}W~#dMwhtkM5yTU6RD zz4*zU5}c=&G9V6nYxA9y!375CF#ZYdH0gXm=`F#(M{OULf6!(K0U6%o``i$Xh*G8P z!ki_@d^Ph+u2;&bRRwEr2w>3%KgE?m$pIp<rh>|S{{Ati6fpHH@N|8Rlvl6O3} zLUgW@PgL7+v3)kB3263GlladUf_!diHgQ>hFD%dpi{0!`p4yq*I=1WOyPfEG-rHVX znLa<+t8k-+3JGZ`~bB{{80Fj{jh_0nJ)ZSV2D$3QrVt#+D} zsnfUwlyg@q!~0a6&yda$X8&0rNU_TXc;l@44~*t|mYam)q8Xq+zZsw?o;QJxI|l4M zf-fVy1poG^AKn9Xu19fERhfCFX$NjkBZeP73AFS3cbBm(#-r+W9L6Nft5@CY3g920 z!C{bJ6N1s=oY7e&G~U14YaFb-r5karaJQZ_P--ii_tpN$v zRf+DC&J!H@Hr9jb4?is)+>1Z9MZNXBX{d82pg`f}vdAG&tkT%PhgpTGG)aH4R-K!5 z6^ed$BZ`RO-uEd<@{IMgARhK)wjvAajP>BT7t;=GO68)7%ZW_Q4%VWOh4QKcTMDXc zH~Rk7vZO-kD46B~=5(z^jPZYEWT3eb{)p>u{m^>SVO>JK+?HT^8kg%x@~@=dAGt5M z>-zV(JfZ$CP&387j~?S$bDkvk+Y?x`YwD&m3;5{fUMwlooD6&>S^)2aAJ zRV}gj{o-@z5u=v6Td+njHN#!NB>Dc7BBFt^ZT=(KlW1$Q@Xien{^R}&mU=U}D8jDt z%7=ky&spFSKrr(t~11hsk6wKk{FnLXIMK_;f&{ zqR8gJzo)JN41<^f$e*frIaCw#0Xi~QF_QoOG5^jT_9gsD-KWgG$lmVd(q}2UspS_+ zL}D3w%3>58U7B-)&yHogXmV6osVFjaI5z(8LHq(MfY0e%EK(G;#ELQsFQIU`r zq?4eyQg`Z}jC72lT%GuY?SXQ|vpl*;b|^=NZRW?4@AHb-NgK5~Oy^Te0w{3D-kjcl zn_DcAME7|Lus*GF#|FAERMm>Ij?RZODV`0Y6tPRV+FE9nn!NNYa4ANoip$|EOKI1W zP?OgD);?f8mj_yP`Lnglll+$1dL9x(daykG&tAnpP`L7xXRX2ZixGnqCo~Rb0IL>2 z+dzk?qE#>ems*8>=W6vQ>Qvz}oa(67WAFkmw7UAYpEZc@0NdlOdB6b-6ruf_8vO_5R*QNf0sZl=^J5ah(z z_iyxG>?dp6nABx<2=Td&h>g7~CL2eJTZyIIXGv7Ul^LZa&@sTvv%Td&Yy5S4p*x+A z_=D8A=BiA+aXg0f{6?sDfydiaR2-Oo2FzjC96_Kv$6l1otU?m(Nq3O=R8eyupF~Gh z|6+i;o^o}X?mFMc*T$S>AONj{k@@1!i1820vb>&j6! zC+m1~=%nnx=uJD%GUJ~PSVd*JB;T#b>yb@#_>eJb&!(CUpb;20GZrMnR-F+I&Q(U> zW=ca>#X?)cA|M9i{$YNB!6N_TzYp?%O&$N-b`(F6k5=2iRCe51ouI*X%Yjv|J>C|II1MpIvmefQ5;c7 zFDDD_Ho}t?-dO)qY_#L6w}@*|=KGbjg_b6agQd~FI4|?l=$dApT?hW3R~#j?=`JR^0}Q6%w6|Z|_&EM|tNiCp{6B4pC}3vm%7B%5 zK5$~sC>~bowodZxW*CR>BK`A0z;{wSB9XjfJSjN)RR{q9N=evK%(E#J;zk-EGGbOKggS51rxeS5oi&u{+-;J%lD@p~{_ zp0l3WJQQKHld0tV`+HHWAZK`0vRHmWqFEi>L1A<;L2F{C}`F051AC&AWU zK=7Qz{DI3bPzibuqG8d8WT^QS8Eg6Rc6Z1tnZr1v-mjD2;r?39YNZ8oratvPi#CZ# zyHu0&>fNsZSn)M)<2E+AEc^m_IZh-ScLF+eJzTuXFrP8&B5b(~Krium1$?e74#`?+IPU3#J z$zYP9|KE-1pC}A!-XW@0is=gNlu`aeW{|Y_2Dq_58}W18dKfPJK2y#hvamSqeO)qlK;9Oo8Y(6`IN{mFz{o2><;F5tNX#evCk*%kkX4MDwCV zI=Qc#S)2)#oq|EHIZ=17F6KEfuMU!=a)a&N41@Xd+Y^-AD-Cm>JPYW~>%xau?aAw_R^^U$L4CydTPnO9;cJj_@_^IoP7P+%=3plCeT%A zXoDO9FDPRq0UFMBa4fwi{Fh)!R2vn%F9B!hJT)VP~{h!l*VKRDe1%Z0uawFC#ZPj(i24% z7QDUp+|O#KX->C3JEl~4Uj2C+pRS!~{qyhb@ZKIqB1)@7AzOB1HeEXhokc5gFWvg} z-t`jaULfj6tt=o;@VVOtql&xxW4o2FqTA(W@YpNwZszwl!e%x{KMM(#vQ8s6X@9Hz zSS-&B6+ILhEk>uVZ<}w}JH6yS%bbF>=M}7&o!y?-tzVgu1<)J=TUVKGq9tw;As;2W zS}hYEhz7IwXDi6=uC;kwjtJd5#@l)7A__JzrWLcV9;|!U{xh2k$q)u8>h?;L5E57( zeG!BG$-=(_NTTGYU7_b(5D-AL$rt!vW6s|q2wrX-3|hH$L%@_YB4veIws<%A|9e#< z!X1uD;#0dKaLG8F0$cKNRl|rEPB)G^4wpW?R;OI6g$N4WeU|`;!ASyrlaCQNcQNCT z^PPHG3L>P3)1aFC4l7mED7tvVEePp9FLhyQgS~O)^w+f6-1ZnO(A1sC*?{BY@dSWRORpePGf}2bDC35tn$=t(gEiS%!lA?Z|b#S#+4>XEo*r zo|9ZLO;!RJrvTOgike?PSuQWeB%v7?L*4Kp8$RpMu1YaxVo5CJ`ao#VwNtL%+AkZE zJ62lyde7^^?Dh$7-U&P%|K}U`u}|)gZMqj3m{84b$qV}V?4mSf6{rCsrXswIOlvv z^Zoi#B+nYjUcpw>=WVAU93GebcqabDt0ju~HV;xd#aOsx7Eae`9=nm24H|?hXM-K|~2@l#njzSkg#$v*_+_&ds~`yW<<@JL3%gTEoFu zvext5&z#r1=I`RG4}E8EBsXT>^+vTLGV6eSfWM78-304z7J%g5%+{KH{o5HNOqQ`G zpWSp=N#1CDY3aoT{ZpEqNu$xIDq5zhPsJL!(f6K0zWj%E=V=~&oGKyk?CEuB1CZ;` zhO24^zMA{!Cv0eMvRoEFS}cCJk0!G3b8}mi&*@SRcLm4ED`Y_rwM~%r5?~?^)4EN z$Ww}}pt{_L(l`A@A~JtT_qe%kjc(k{DNYKU?f5Zwms~ zx&UmVf*(6Qn6QIeBpf+ty?@V$)9$BQx!;HKNClIVbzV|_j`;Nf)>l3k2?Pu%GCw^I z6q(dNl*ZoAa*07{=fY1myKx_MRy+UH+^kzO!HaIE~znLAgsEMPGd*Ee?B2 z%YabQvY_l(r4{Tkl%gxRBdinG2J%Cg9f1y%psrYG`bkb<2ax$Op_U?DxPE!DDYAsbWl znJ`gIOFVFS4Epd-B?w4ZeR=h6#jcZv&!2Y5P4+|f%;1dR>}F@IJ0i`-3MZX7sb!qn zh~I1Gydtuh8GBi*H@(?-vO8A)EA7>nMW1b)sm3TOKb{On_AK;8At$pBY!Rl@GQ>-w z(qf*&WuL?IwfZF|jR!9{xOjVrkn`3GORI;&s0w0dwT<1>!mt`eXy@G>nI=QcyOhOl z29j53D&J2^qj-wlPtnJ$ZSGh0RU=0)ih}&Xyxviv!|nV(iMqlx-Na5T|GJ+4eI`8C z;QI*fyHgT??S#9JW7!z|8Hn9q_m$Sq|LP_FI&hw%RM~u`6|7rJDGc8al%U(1E3Yp7;0J~N@#?b}>EVDt;6ct)o z!RKNjMlj|}6Gc6EG_W6Oj9xRyatB$^9QJWvonEd42?;CiobUG&4aQw7KUmlxPlkf4 z0w!0dL~Si(((ZUIkz%Z-v{RJh_19%*@IRxL`~GI2FHhUJ|2rZ&+`$?LnN%nmQGvlU zQ;ae5c)n@Q%&+3*`Twi5*qUgVKug}gb`bG?R=`dV>1@|aZXzn!b^(4PaAwPTnH(b@ZOC*fV} zT2HXge-&t|7-2^e2q-(Qe+`veXljp5DEdI&^*_qIedx)$! zpOWRX;cRy7dqtL@F3R4?NRQ!Z9&a8Gjo#`%K)k>b;(vbyHy4%R%_P-lZ~onW-q{)| z3dLJ_tBf_ADRGt8-qJ}x-iDwS49UwNH5}ZY_*Le~_kx#dvH}JL@UTwCG>))ElT>Oc zG03|o7vZJ|4CwGts7-MB+O-Y8F5LyY01{~Z=As1lR~GA|NY~0r73c|KQMVWkO__OE zihZ37`+`hoF!F30(Seyr;kYCwnch|y)#=5t+zxtD^l|m94#UPu99jmrdGcS=qTijT z7$C;zY8ZKS$PbDTrw6+*Rf*ubRn4+~c{;SV)}wGDaA-fhoC`c|0Yx3NK^GtSQq4?c zPB(Vbgw}4pMk687D5k7u!@?Jtx#PtGk;|<&0kg$kSW)_A+RERL&j0uX|EsABL?uvz z+l0JE)a;6366xy7fD~xje-oa5R(8UERllRGe6>T>1y#ZGJP^qvl8I;NEUB6(wlhOp zXIUD{H5{M)eKBdgE+lwXccjMf$rEmKEJ2{cW4m3QkkS90W5iE!$lCGHRC&-OA;!Ar zv3w{O10iXLn+2S;T&4~u`Y$Wfyci0Kv5K5Dokp_RkI}l!C3Ot3_gAtR4k^4^ocwIZ zuvjnNePI0Tb0~pqX&G7*(~Q^%unPMPD(v-?VhdY@8VuQNTRJa?v$wUtQR_V29--_U zcBF%D>evW}WYYNHZ;R0AbMcM%Fk3`DFWvW`;%%&dlU#d8XARa7P50Hm<*9!aTK~NE z{{4Z;s{m~(-X9-P!|`xY*zT=fZqku+dw91{Cp?-$CDpr$TI*}qrpqUz#(yN6ytRgd z66FOwp4Og1a3kR#tqPTI1ad}=z#hYGt=zb~bJODk!uL2rL9x`9z)kqAi%*LXT*t4a=8EOnis&^eAO zxih_{;TU+B1npQn!x49)SiEC1F>8Tn%py-p%Ffg9A8n}hZ&w|U<8nehFS>CfXM%`> zORy-ve9eprUb$HBw?|`L)ZEC@YgN%^;vygXzu$-d{wx$TaqvKKzrOw%$EEy|PS-u^ zpxI@Z*mzT@>!1ayJYL}Bp?b(09KOBZaDp;gQXsj~NmTG!svk?e9>^F*`&tj10Ra)kIT zD(G?IjeL~+*iOeyT0^4qGhV;?s)mm3w{|IdP}ilr^y8wqD+5slCT6+@TH>+fcV+3n z9_gyEweY~-krfT%lN;K4MbhCG_@KgG{t@kK6D@={=S%Ym0nv2_KvN0{)Xw1jT>ld) zxY@g$^S@sX|N0kkQoLucCFgM?t?`%KTb~M`nCRIXK2!g&IV^hyr#^qZ)wo-?ky_=L zWpwL1X?PuDcKC#UNSTl+U*2?kk&yXxEq`*IUNLhK1^5}W5YeM=_CtyFetJ5cxn(cf z^wDz%{k7^GmB7wN$$kIpJxZ?MjR6REf$InlczK)lmyi?!Fwd?&$laj@Gie?(pbL_O zz76}6Cl{ysT$!yan(prAggZ_^uP3>{!B}XTXQgYR^kQgtx-;l)&-AykpM(e{>@7tB z8`STV8W9@BZWpt^GOH{|ePw57p7xzvXo{PUQc6REE%i4fI{XI3O8xvF&C0(%hW~ND ze8PEVsK?$X=&1_TecJQ+`?9!6CkCS_nyeuSZ=Xa~b)Nm5t0AvTZaigx0Ted^>$g#p zBsw}!S6#a(u3e*ucWTt@o!8xSc!Qp+FIv2@mAcp`JqzI!=mJy^sl}^}vq?fK6h&re zg=I|bhsvSuu8>C~gq$C61o`C4#v1(T8Lr8(R*%9#c(aM0Y{1l@5?Acq%gS4z^ZFs0 zSwD8=a~RB0oZ87Mp#SuWmZ@p-FAdQr?S-Y;M6Y9{?JL5SK~xSqZ=TIj1>Z9 zD~kPHmxW|2qsHgHcm@^x$4CEP#l-)7A^wz6wcraSBnbX`t&!&*!GJB+abg@j6bPKE z8KT~%*0=Az`O^E|6h{G-AlBfitNLRhXE#=ak|ziLC@WiY2j|+S zTZB3L*xHtXoc@%v4emZHF7uZRmTyxUy0FHNSGuHfs4LJk=O95&YWN@6($(8Pfnc1i z47-+_0i{B%W!}lL5lZk0aOxapOeUNwDz80NMfW8ifF*#Nt7x$BQ<|;wgutiD-XX zA+fTgoRH-!2F+5+hMVvvAE^u>pYy8M#d_t;+`37`o!(XUcHepJPG_CxY7a}hLgR7* zQrxf_@3&g7>}$I2McNw<1h?8FMt_AHLCCIeo$9L~4p~Up8b9&;Z_a$&O|EXE{y%OQ zITyGbg&iR>9f*FJkhpS8Sh}wolyf>e^y~eNaTBxc?`oRzoVIz#)o$P4bnL5D%^)>E zDLqwU$yHF0oTxxm!f!4KmzXXAC9xCX6ajD=8eSO&h-7G=u6kOV(v`UtrhuK_Dq4eD z3~KLpLyHJ^2LZ<`Y;(8&=ntvm5ISQ$MgT=lG(hC8DwwT~uKEI3zvctcA|`43enZ3@ z)v}@1!~L~H0+^;{``t($5FeGiH60YA#AUz}bRmT_ zZAbsx`S2gD%TriIy8`rGyu9!D50<6P+duYZ&(b~*ifm9T2p737LmQJ455fDCHY^Qg zU9C=#1$*`SU#&%l8`Oh?U;A$8=^2+%E5hfGFi6&l8TJIUtpc&gNOZFk%h%w_z@<(7 zQc9(bU-1nY3uoTnhth5H6D|3cj z8tDkQ%Mmkn-?s%=UtJ=`(8TOGZ9_}iw1(&8dW^~#1m3w;>iAlAoD(nBpH9FT^u?SJ4${<`-4Jk~?Jlzc(ZQb$*C{W!e&A z0wh1^@NbkA*~QJ**dh;&M)YIl-@gjFMHV+`-s%ywq%PIpOz0~Y7(YmNKlk(dMZ!PC zhM*A_fgVYyPQ{lci0rp^!iX=21Q*YUtgNzAs2+x;RbY_Zkl-hL8IKwWNnqXoG*CzX zD(UUnUX8uY*t4+f^ZxI5H&_0>!UD=bb`{S?Ez(`deOJ;${}!d#B2AFU`(i#i37f5N z@M0(pkyC3TXO&sMuSjC<4H z2TAuELPc2gRzw z&CMTXI|6y30YT=9`XdhcJJAk>zeL3U=Y2%L1V_lM7m^ZD-@~$~SsAhMSt?AJ$Gd95 zkC=^z6UmtBg2LPF9()!umTOMC`twuKJ&NmTqr8`s^W=3tKW8?23f zgg(3^ky(1TN@mSS@FZ>ZvFGfK9VKLgOW|v<9-*sSoR`f8WyDf5$Y0{J-qk_2d~=-; za^DW~B}?iAfMVk%jpmRY|N7zxcwoa9N*cKmdF@6r;)^6dp5Na*0lTZd7S@GYTL))h z7)wF2h)t#Kg}2UYEUtHF*Eb`m_@|Q_p5v^9CnEmBje1f^RNHmbcuR!^M75KD`Y{bi z(P15)Z7iV?vbf$~EoHROxGk=6Yc2Aw8!J-pp@0?O;E7UW6+pz{TM~Myq79UGMHh{$ zG2=J9XFwT9TZZ?e88JN|Yn4?W|C z4MAx%uK{J$rbH*}pgYm7RF>-3XyxEb#s!(?&Bf8sG$D~zrPmoRYed0?9YM|EU1hNA zc+&4ExN^sbI~&+5M*-Isw()MBYN`2VNGh+i7aOccp|e+^4!>qAHYz4=zEYp6Ir#C( zqdnXgFbb7%5GPA0nlszQR37c0KrRb`sbXmI*+?dR#%J>-xTKcG|8tdbfF|@}vhZ!z z{`?u-pxn&h4jp#T&3P1!Whk5^+c zjl$E;YHo@NDaB&gJa)~ebcfB42zD`slHPlbFt65K;HMOF;l(u~a^vnUgbPk?gE~X>lKM1aXTM<&EW4QLgsSUqk$WOzI)^{8yqxny(<1f%UBum=qQAM3v39_a zx;*@3pjav_ol-9T14*{MTJ3&==Li2w)38mx!O;Y}ny&TB6uv=L4Cars{J?%A?1$f8 zgi2Hzco8RiT1kx+vI!Y{EgH6L)JRtaQT~jX;N9+nJW1|O`ou-k%I)gC92u8|OJ(L4K{v;_eH= zo`)uqDPLL6Ulma|&&iOw8@H+y=VHjICAZ#vt<*xy+_z#3?+i9@9 zrCF@^VuaQos47mW8#R57J3(oh|mc(3_WhFM`e@qco@$&V?UomU{5qgVF zK{Hd?MQh+qyOX3!^auS5LVAFx@s2ZmSPEmh9CSdQAO7wzSU-unwmoC^#_&={nw*rU zY-sFfvg?3AIli#eX$d+RpLfVfHX9;1h7c9R*P3TR>99TValTxwtax=KS=<%{4QFO7S1fY{dvUZzUQ>w-Ok#hCWI2l-(o>@CtD*X&ia^I3OEH zvok7>us1a7{OdgW0&y8!8&zNr^-pv^>d@kx68FE4sENR$v z9Tc%~T5FF0TXDaLLzovs`5&UWS!R5wzeHT%m!l+E$`WyyYgEBrsMiN_!klgEqQ0NQ zB$K2&?**QmJ0IAVAF#$(*32--K=}TtL@g$kQSZ8im*7dvYClzGYQratX#>rYLbhan z3@bGt>Uj@#2Gkm60O711{gA54x>dK#(3NjEi{Hq#&G` z5Zmv}3fLZ>Ln-b5s}vEx#>+Rn zT-e?vmwqix-IFkPv%AYdVvAq z1E&4;D*ja_raP2%%C0Skkt)ct*WmQ1>v*L@Tfg=ofhngY(Pp@HyNR;t^s^NI+4Tg* z0e*17f>#%P?$1Gd_(EmR(&Qu??~^yp>mjnA)qy80llBmlg{_cK)hvHMbfuJp=j}e% zC&u}{lB``OYMkrFq?G*Xc zOy&js-pbK8=$B&N$S=#>YTA=-OX!;Zaf1Z!PE*tad;+xmSycZY&VU6cI zN!F^akJLI{wbpjYbj2LGJxdaMbj?un1SNB)g`B*~hmq|QaN!E&uDws^*ARV^@Niw| z`$=-Tf-be++~UqUmnJn=9F>6G9@qZ-`eqDU^wZoTkyUYg;bD_cKa_0FBO{GTOW4bo zQ4$gr1Yh86_H^;Z3M8tSFedO~x02Q3*GqS-Rc>u(3NiyS1{}^H+U1|SKl9SLel^Q@ z`uW_8kd6OT598v%2$TxK-yHjiU#wf%sfr!$XGYk3e+{72Y^O?S2HqgGdn<_`>iIj% zEWlIJX!LZNrumo6@sk$9`R+8WU?%b~*wysgrerB5?fz)`FKZB}UgQ+ub|ER`DZl{8 z_gh|tn8R&T_w5$qWc?Ysb8_!mRx-EY=~5Wm^)@rHgfGDkjOiZtAphpfJG&5%l_RvT zOki_HwJWAa;c|_*Zb&@+n|ZSMG79A9$-Kwf9n!^|Ib#Z|f0xp-UJpHI&=d}+WaZI` zw+A<*KJd2^m3@6~j$7$&sR|rH-sjW4K;Wh)91j~sO)wH;sIgr(cZgt7OJfLR{$Yl_@VA^Mst!HF_mX>j~d4mMO$ap7;6*gaQg>ND7SWaRC7py<4Gi21E zv-Xvv?GeW}I;%o%1@WEHPKS3za9ndm{ULbt;UMGbG}H8J*+imt6;@;I1H@Li00$Jn z`MQ3-SdD<7`&=lc@j|NL`f^#S|Idm7Dwj3jJuW}Jwr#6$q^0hAfe*fF z#sg6`N~yJM3?}3S@H?`}+@*gNp`gA8P0?dD2+Va~NWZS>{kMi56o5MR^ZUW&?GC#O zg$M?ZARNxosx?MwB*9Zs!BbaRJ=Yw1_dVeHGvX^f+gYh(H${U2a@$aG&5SZ?-14+! zcrY1X^~gw$v&8Stk2u0!twe4el>=bw4+ggO;E&*vlmBlR*hfTUD&3;1x7DFZ(+!(+ zx#P%3p0NT;LI#**9^0gpMF{p$@xCZ^s`KKWtJ+x9PRXs_T1JLa66v;Z5VaNPKE*ge zQ9Tj`gOp+DBvYadDSh{3fRM^RNdim|JtW81aQC^A1GVK$vIQSL=p!pj&VkBH#~N>rGVH zF~Rpx1gk8B*k6J>dfPe?IN6t(H1fp(Zsj}3n^Txv>2Alb1@8T(*_T>e2s7mL(F|tq9Yp=+l{3^0if<6cNHZ|*71j!Mus1`$4|K?+L+Wik@)>k+#y zii$?z66IzQo*w*=Pi|dgOriAYV+&!5TPL{k7F~*D3QJ%;n-m`Rew<8*ALHvZk-qOD z7xd+KHSCHV%-o-+rK-iLM=(cA4!jr;+B4#FGPjA5_kf!;E+yVjPR@3BR`TG@nt9s^ zHfMVCjawk*`E z|K=6G4Gl4i4z-~kh~x#2uTa+~vj4bE0#hFiY^YdUzXKc^k3RS!#{A6z{Oe5{|3+s{-tzigTNAl$nbGdHMPMnV6vRFmvt0W#R3Z{l!kwsHiyBS-Tx@W$7O>DE zG1sk1k^06{sN##q?fBoKDqzNbM1Y^}9xPYW!r@^zwbXmcrCvOZ+ zit!~f6Dy%IPGYX%?LM&ON+G*k2%l{yD=-%Nt-kx?yuuF(XtZcZ9IfmR%Nl@xCLN6H zk^rMK{T-EwzrVP3pF`Nu38fMBd8z^2}O!Nd4;+0b?BxAj)j1((&uzO7yvb<}-UJ(Z!bm@2fGxrYfuvNXkji4YfXEBM#fuUWax+askqHakva_()a$ZgSX=s+ zw z=aig9SZuMwA%Ku?RPPPA+Brx6;Zpj$;0vdRoi>qKH3mHuXR);wOVsi|_VKLcsmI2; znYA1lbsBja{}7I9rziDAU0=kG%ycH|!if>HyAM0GggTH0DnroO%D{$z;AR&U_f|$g zh4r5=ic79Z2I_0sCpwd+b)KA5+^BQ&wfQ&*>HdNl$H%J?NLY$ZCsB>`A zWEyBH*24ch;l!z7!5yr2f2wY>VZAtW%MAPlRcN3IEzL-K^)<55<=IHROoI{sfGqG5 z1JkA*kDoWv-h2wHL3{-moj$q&SnaXGhO4o$A_On;K9Y%4`3X_+4Lh%&eP=AQPfJV?J`x1;roeYG$d}VRr~U;_VB>9t?>X_AUb(JXVZ!)MAS&av z-%W+}`7Xxx?4+_|7bd-8L@rXTYART}hnF$HYKlcnmkM zV40$Qhe(29XcxH{Wp}Q*tN=RnJV@V(=i=q<=^AZMh9b2R>Xhwuyxcclp`tgIZ{9GC z?j$_djx>YK^{|^PN#hBVi6W9;>6Zp^TN%f9A`1_OJsX#^pucs>;*R+oUw8iPIkq(s zn}Hk-C2|;$s-m8=6d=f+IQ(r$e$;LLS`HHLL(NBM1PVvAmu~9gnw%mv1*&-XGb8Qi zZO3b!9%6sm8zDSFz~}u+ThRjUV&4BQ`rufRSTzu}#twZK0lfsx=(S{3gR^ABD ztp0_~9)9Duq30-yc-3I1eh<;i7ynCgp<;lA&1-fATThiUmr6F7oFr`yM0YvNop_mZ!-j^F0*-Hx4Nl zMezZ5pSOItL~eR(7e(9F(C(JytL<4<&~_?IUO`g!u^K%~ad&)LboOIYAD1J4Nl%XxEv<%klJ&HH4z7SRW(mp;ItL} z3P2Y2e|RUL<{v&(dQh0CY3N>HVqknQoR^W2_>_qP2gio?48^Q972ZS-me7q5J~9fn z17=+_`q{HcR9)%!78v<*M7T225{R7l^=up*ePaPCbuT6~pf^nJDjtIk&`D$MChsGy zadfvqp+o`LG6_Z`Na1KqT>>g5g#Ug-;g&QOkrbRFkuWecQynA6a zliT()w_et1RBvSF+oxd~j7UqQ+ecmR@RDj0uCF%sdiHD5t`FPvY?j+;&Zf*?j8S-u zsZJW{=d(4I)eG(vZEugM7Zx_1f0j^Hm1}UnE>|w(aT&IPSW&;@Df4zq-kx!b(rcK2 z+7JCyfAPZ0Ic~;Y$nW9PekZH`xvXJ>TVXDipd5CAf`^~t{q*-Be3daVbbQ9+6X>qB z#G7{&UX?N~aLs+{%0HF;f{RorZaJ9(FY0%yKU0L3#tIP(Y(3mS#jas5esOe}!ZsG` z5#}4Z$0>kr!w8y0tdGlUJC(WMy~BD2=#C>6CsNMXNBByj0o4+3IX)APjFy8CwAmIx zo~RE|*MJC_ubd`0ZCc^oW*iiR;HKN?S$+-h6yw|Dg?muo`?XWuRIYmAPh9`~k?vGi z!vBfrci5;DR1;P%krgiS0vhljqEC^z^BW5-?E~TF8vqgE$ycdP766fh`ez^l;3q(0 zC9Evf6%LZ|>`oU9fA+g%xO!2-yzAn|{KJy0Iyy@G8xN zcR!Wz_<4u1LdpAXLh%=<5&@J=<%RV~dxYMS@kOd7ejxD$US~ke^BT$EBz!E$j^%tu zWz2mq>!#G(zND-H;S)s<$4qkvBW0Q|__BwMOGe+>b(?+b=3^fNO4?sle8qs7oxSS@ zYM1@R(WU~H!0IQ}L9f($O?SOh>;a)jPN%g$UkMjpM~8wJH?bZB2Vqq%uWYdYNQean zlmc$`%kdJ!Su*k~a>W61ol;PIwl_+;az9M5VH|G*7yY{U$I48l z8V(+bKOO$G7UdB#vJW4;V)Tb5gQjPgt!4omf7d6}s~e4n%Y!aRZpcAv5BF&!AH}(qt&gJd% zXncS5j1L>Uhsro$^g8T#f!-8NY&FQA$d4r*%&Hn@2UP{#q6A>Jg9N$&s>}4@(01jj zK)aOEPi{dW&2u^8oV%ur(nrIJGSFsDXROLv{pP9^M_>ux0UCM{iofXv_)5P$6#;sWF3uPnAlztv2iuHWSSO znIvWCVcAO>>N8~;HY^L-TjnuVj1o8jsWJqe@Nb(w1f0?iVP4A>O%BPLrF{@!U0^T5 z9--gFpK~EA>Nsp`988{FOn2L#n>J!J9n};((5|0JmTva6KQuBYcbvGd^{6c+iOnSL z^WZ>QtaF~YAEIdej#l%s^V$ty_*HuF25$fzcC6NEWmgpaj&YxHUEi~MC~R3*7V5Sd zC!4#c+jVjR=1sGAwKok2+e<}m<>mX}-mSSA`SM})HO#`hQ%+C+xmH-uqkgmR9`8ix zB?FRI`CxsymnROBMwjkG2>T+2Y94Vs?$8g_X2mfVa4W(y*zC_XN6TKerp|{C2ZA4Q z5=?Sm#}~g$g8DST(P3F28++QH*gE!tYFYHCo!6`{ULQ_fvZUf5TT@WHwqbnecag5; z`N8#$f)xw1YLmBSp2JzFg;e+cMt?>SU!dklF@}pR&!;jZ`{ue3+XUa-U?fXMoVe0r z)EA_3rAQ@>&V|{7yn0Urs~CzK(5jp&_Lc{Ph=)8r!(EpIeLGrAZNp}tfV;+kcc`A% zRL^jI2a`e+6^Vv2`dR;=L7AOd>&rCRdbjH$GYD1r?w}%L6Gp*Qb0_v-8`rdreRB#4 zdb69xVO;FNI{nE;+@~pzcM}@h={~34OwX@YL|l(u?$OcP z05julq8b$~i)w1L;e*(n#ZL+k4C5GP^oo|PQHMF5@{eW3CdFd@!Y%0z3&QIjXmHTg zAD=k?cmvKI3s-`_9}CCG?}bI9bpV1xBjn{foJV6A8PC!~FtDB%fsiSKBG^oqzF8eiXF;2d^Q z#^(sU=QD<@)2W*NA=$K?>U(?Cr4>sl!U9);XcJ!}%HNe57pO=Ct1GE(xj$EDTZa&W zokkqwR@8F0=V0rf&luFt0(Vk=)%mlFF~hetK|XT<&2snyw%r~7JRVLz4a6qYcR8I! zrwk&v0)|WX#i7K8BqjQ+j3!E7xPS4)`>k`JN8TP8Z-FM%v+* zFC=z#3D*QX+!F<*PS-9}I4VQf(C_$Qv3pPCmjlJyCV*^Vi%Y`iXsAln-m)CY%mSA| z{~A*W8_s8DP!2rdq{b1?e&3OES%jZaWlJikq3GD9`5yP2*M~K*tC3vqWl4oG__utv z)&J~;O9CYo@vf3FnI5_r7=GHx4p*R;q%VzIq>$E_8c>8i|GxPKtecc`s5mWO-i_u; z?|{kzOQh5(aYv-n#T#H8knVYd-0sV}DBtC1rHFF}2?1q3U(_eJ+uu5kam$ zL&@U-bZ=!c$FK9oq;4@F(V&7pfK8H3eF@1z7lAQ3(AR1}e z+GOO1-EzZnw8Gx3!@zd=vjQZJ=KZR8VXcE|eI6m>vFjw)m+;e>L85+<#oES7cL`rG z-J;Zx`;cuW6QC}68Ep>|$adBKcQ1t2OI+4(hxw=+1Cwi{>n^TRbASBkoAG*kRac_x zLhBbb81q(uY`<;R;o(TM4tn*j^z31MC_A)q!EWr?T!XHG!ApKD0WpvH)RkoJnhvcj zXiW(&t=bKUQid`OU?;L+#!5Ky!R>kpFt)c^;&MNEPEYDMnSuprPO-0levFWkw8#RJ z*?9FqBy0%-rY$q`)kY^h;-Vh_R7#g&i{o-Ehkkub+bmmB9U+b`2Kk@XW-t4-jr}xv z66D<+TO;;{c`2o^H4z~~F6(_$7dY3mfYVX@EYo>Y4|x&MGm$%A=%Viio4P_9#{Hzz zW#C9z>qDkN9@c(<83~qQ8Zwpdr~arU4CYmO*szErR@Vmn=5tZ?D!3mrzGC7GV?%CQ(b+@bun+GlKqduol06iq*A*@!Kb=yU^@~+O4KNNdV@kSaXz>I45JDp zHpvEPIN^knL4gf&bTTwBJ-UM7c_ny^-c|XEW!RmtfF_Xmj=$4ROhkSX_LHL73Awma zFfw*EjPjR1Xm~LX6dlizv^NmnX>VVc6HrYpG1Pmt&$n=Crer-)NaBsVKI^ZNn~>SFR7r z!hm}2C>G6IBA}}Wd?(sZ(_J!PF6pb^vdM399K*FI1SttiAiB^6My6fM+%d)q5P7{o z(JP90%VWZ$8u|hTl4DM{bZ=_K`8gWtW$Z+7&bzTGD==UX{;taP;$vd(SMB#dBzcRF znyL?)Q;ImvEAYF0hGnjKMNbq9I62a_lx)5SnUHQSr1`F)9YC2&lQy`Lh?I(aCt)BW zK6^RHp9zTUs}fjx?l_Yh5`j@cJm8e7iz1N)6-FBw{|(P4CDO~?O0^1g2fC36&+}MY z>rtTtTiQm&vO3QO_ml`%1|OK%hwDkIMFDPyo&-jwhHa-m&pVb4h|Q-~dyaxKrpK2z ze|c@?j2&QuUW9WI?H8c?snv zuo<~}zdlOWrd8R;MBi=XI5*& z$S@|Y8Ft$lOGWierPl0&zDFsJR3TsfE&09&vs4WKGD@{hZ8SEsRCMH}7@rbohLO2lh~hc&~MG|nH82IUjl5vY+oWTU_a zjy-1n(*Fcp#0Z+hU)}(Y4<`~Yf&MfWxxnw{+y?MTCF1Guuw7!oAmrQJ0Gr8)R7hi~m#N!OO3%BmQ{O2RwGo=4g7i}i_WQ2rM;I%K>`JG@BD`*6T%qSaA{p?U zV)VJ*K65cM%F!3?IK|;dVm3sp+ZSSyQ^JI>p1ORerd*wO(p2xysO)CY;;V!o2}5rO z#W?Ld32VPIE*95=2vnN4p-#t(!Pr^|2-SwOvIF6G{-6VmOe!!RXH>r0QjQWJTesJd zrK9)60^=PvpT_oUA=KYGvb$5~cj4clKv&zl_%PM?{Y{pY%h!42p*L}IUMBbAWDeaO zQ!O+(gXhw$dR6(V9SN-ZTKpc$DAI+%g0!BR@bDd*2I70=l?v}`yhOK2PVC3C&IRK0 z7AsH*U+%7Mm$hN3BP3vd9gyZF!@Cfly!AxAy)9 ztbLAaHhBxYo%lH=C)lu4Lb_V5@1LuNhXO0zS?YZ@^U7tHwJIh;Y=by5g24B3>H3Dp zPS1UuPmNY7J9QLz^KxvqH5Oj5USEwW7pA+|aS`&8%7r1>*Jeq_scYFy+}#it0Drq* z6K^m}bFJf3s0zQP{Un>%>|WL2Eg7cg3*cU)w-8d_-m7s5xaalrem-4LJ>=o*z^pzv zTg|FEnn(OI%{@}gL8V_ruRDzDenD)nG8Q}a2tX( z^IEh+LZ!j`1Vl)sOCvd!Hd7pT9yn|@YrSVKglr%AcG&}lFYMYoR+P*J@A%dI+dd!` zXtDbFKTr08C%157yA5`V-KN?^&V5;%iM{PU*+U%tULsVyn#e#}$lC3}~(NtWd!)VOey<)-Wb6*gY6`3p-!B-V<6fluRZ#zZ!APN_bI1$kfK~LNs zsqEcC*q;=5D%Cut9hgv!hHA>Mc(m zRhXnsAJ~$T-!D72ev^Ih!1LP$^mnRfqIOd#XBgf)L>mU%QTrgzUUm9>Ym`w{Ka06` z6Z0hIJ*;3i34-?6f=|#1$jf`CptAS2O@%xX9+}}6yCUdH+j4(q>Oij1Py+NIDMHH`zXT}I?_YWO$v_MLdg(=%cmHY*j zyFGlTOP}?L3`DhPH1D z2zC>&<2OCofwr{NY(}T5qzfk(W1rT4Ruz&!*j?5(bkn?w zgRhWcM>m8w1?gB1Cc&~D^H|ZeIQoU!rx)D&HzF5v_*G9B(jUuL!tp?=6tMD7Hqlud z+qvA~T`g1wVbM~3 zu=GtW@_2P|-|Eh(>|6j|dgesYuo!cqL2G3Rm6yf#x9!W7yS-cg3v3pBfu|hJ z+ZY7xpUiOIB_TSJlvgE?u}9E%fSLJMnLe-T8%;3|lIUv3`5i_W-rr4G8jQ8#A9}Zd zWv}8rEY=pz%jm3TwkD8xBi~{(HMs)gxwPJ#pC4|A+!`@VuSzi&$5ywB$15hQ?Kp}R zO!9{zH|zKZOfFoR3wR#c%m*CQK>drN4vwRf;LSotCs4(01DhNj$lLh#2_bClWtL=3 zsn<7md1R2NYgmo7J6NmVF`Hm@8J1!FdA zx#F=k68dovaamw7y1ikqT7NNiq}}oDI9TT|jvA$`)uG}v3z5OlAiPGXt?m%Q7Ei@b zyg~j{&4sY$e1;0|8Gr0}{D@6;>whshweQFWqU{$HVI%=C)xj{}?o11ev!}fIy>Ozu zy#%)wfeG8FTyqamR30h}1{G_pP&vn$^mG3gDNsdm-<( zsWUZ@i*~hZ_#wJ|%&?bnGw>Ia63|5cvrjOMX+j=vb|wlmnRkHJRTDlwo}OVkijf`# zO5n-XMtvLXRsi;-A|c0|H&<5y342vucv7_@0SqrEj%P#fF&FCqfTm)wP^0VSpru7P z)|JA9+<9I962zDY&5GJSbLw)5acbd@gK0F-Kf<~IruxTTIP~pX&_ka$wYvmNTP9w? zLEcBh?+cpEyDbJGSM0fNhj}b5g;7<9|I9Eryj4plr}iik0zv8x$#7(f@6pzOjtC+L z25iEZ(?DDY_yZeZ@$P|?%>qM{bMiPCsy1Dtvv;FT? zK|{k1pE$X!{Fu2ZJ6Ru965f+V0XS+>7wwJvr6=vePu>MIZuq;%&waOnL4*~Ltu>B?Hk2T-{j-WMB*cr_!a+T+w++uZTtctCq8O7)2yMDBXoEeU51SXTgz;lQ9~ zzWN3Id?2jw6kj-HD!%~iVyF|PI80aO;yrRWlF3x2&{tCw#CSRqsrjqgSW}o2{4ZO| zE97lDd|;=cWp!=RG(Q1@gM%Ll)>Rtuu(hm*(iOqlDvViE z1SIo3);TceRJu3dk9tLP$FEV?73D*-i!OKcYQ`naRVyvEj3Nr5$h@w7}&=+5YUK?{LOW zpa)C~f~B~OSoNflPx+Zd1_F#puFU`8=`7f)>e^_%HknS`zC@JxB^ z`a7KpKkMN)Fv!(kgIboa{KF}IlBjBaa^kECq?5zADabG2SqAAG`VBE;!(NqF&uI#k zQwtOgeQl0bJKjC|EgK-dqGADI?Fi|knU62yID`K!z1< zJW|dBZug-k(5sARuoh?zHv5&t+VOkA3pby|UWjRDQ)RgY_SdD*jn;GxRY(v0%{P++ z>=0`=&kKh!k7`8w^vU+SD8Y`Di*Oeuaf>d%br)HmTnN0hyU(htE3t4QQ*6^k zdMUn*sjgQWw{qG1LQ<1C#nFutIU`yQ+OBQM%yxh8fns{hJg%kr$$p&s*|A^3qsV5~ zvZWZ=X4id`sea2`ftgo^v(9Vifld!i4OWGjo#%D6pq8--uuvXtd+*?X+|d3>a_OG` z{m8nlst3i@lchyGgrjxoF-eOQg%{G|kZVP>rw0w;^Z(v-drDwk?P40(W-Q{N$Z}hL zWA}Pgt5W(kjnB2sLmzIgDs+P7kLmW$d3{!X>~>k{J7vxk*M1gyttH7mqb$6Q7wfk8 zr#B=u%aiX<l9;z_#q#jy-0Y=}=u`j+M zj%6)vKI|Z7KPvL6H@f1c_MOo08mk$pX^dVPa@#KZbg^T7*e<>l>`B8#Tabj`=9j-0 zYP|}}`8X?|gjbYksAN7x8YQJcNBA^i@q!!>ggIT$!sqborI0EGr|0SKpQ#&I%xTC1 z487O7KdQRWVQ^X45xo(XGfal9@=Gu&!j^IDL9%czMO??tCPi$^{j;dWKy*Cf6Udk$ z61v$h3%w8rUtgXBg&||e2@OrD!3BFQiyU+FH!dwr%Srhi5!!_D>z~PAZuQ#~>2kA-Ik3EenR^tySO}6F;s13T`d9+j zod7Mz4n-qXm^%%=c!=>4Q*#>>NI~v@I25yJBEwCt1vIIwqp7^L?`G#Lvm(P;-0rjB4^%zOK>LbgLB2WV6hlk{+~Jc? zdg?wolU@DnmwV#A5%Vl@{L1KSc!AOB&u3^p3=~m~cU?4zqmUu= zx2rUIjQNqnccPq;WiZxD@AURa*MC~-HTF7%3wYwdvRe)ni8N=HUq#Yv5I53COjwKf%{+Te@-p@2&I$0I&Wp znOR63l}HLY%)GyOkF$YF8u`BI4%<*Ups%ccnXPW$AL9{b>?;py_dJK))TiYNkYP9$ zWokptz<;Fdx-{^hePP*dCX0g%I$SE2&bAARE>U#L^-#&t|0}E^@?bQ7aY7Q&a00&7 z%9(3U9XYI>_LQLG3H^{6+!M$v+|BUEJ6&?C|DjtS-1cU$=UMva!_CySZ>!%eR~8$K zLL}ULI_y7u|3F7ViA0AkzHLb^JB=y}WR||YRb+Cu9q5z5Q%YxRXC8U_ZIFwqKUKrG zotxGt&yCqO@b(mrq@%@=w^=J=}4mh;=lQ&j>p=lNhi>zsuaQLpQU zBqOdWVTl0Fe{!G-9I(BNjEuZ&Hp)p{Tt?Sx3zUWDtb#f_qh6wtej2WGweg4ozN@y( zo12qyX)4!N+}{Ux*uBi@5ByXDYGZ_rwp!ls`%a7(D4UznR?OaAtVG{Sa4r1MBshk26K3=Vqe}%M5dM3gpin2?L=|^c}(5oQ@#&b`ImA1hfBeO7$({HZt@Nl zGqQvodgY6<*7lu`dYZFRgACHs_erbNM?dPUEuO$XlvC{4M$7h%u1tjf>q9D8K28eCH=H=puBBuKp^exsx4c2U$+kw=-%!{#iUpU2 zJ|&QXxRHktGK6%Af(EsQwQ3knLCw%w-v;T`efaUbWo67y67HVcAZY}4bHU2^2N#{9 zZQBK@Eo>H}Pu&iN3VM5xfa{p1yhe_RZWV5GTwGE<&J>!2QHXco*Uqava0>9$B*M+5 z2F=0|U@;X|lP$YB)BYm|wq~<3+mBkIRU46OfQjJi^h}DS17VNoW zRw$25Noe@K^E`g#gQD_O*-jUO-A0S764rM97R$r%GVpO9Z}H`=k;AKvhwov}r+Klu zqW+9ZQ9;kOeCzn9&RzG{-L|qri(T(yi@b~n`fc%UPBk9R6_`Px{L?MR3f``@d(%PY z{?A4%xi9>-(hElW&b>bDBt@dHDr9(JK+e~Pe&77=&UY*54r6-#>=s5On`b*01yjw( zvN*|nWg^jU1S&0}eEAhZ@jVemMt7HvKD{)47}+v!_(2>q{psMs4KL z)8W1SinJ3-K_O!8Cx-23_3iz)>qd9gYAsV3wEDbD-0j~>FtCeddICzx923g=L}Jxt z`S{rR6!L>YvFL<{tesK5RQ*(;duT%X&>?-|*zcV?hZMe-^bmO1`v|}OOQXggiNH{A zUazMZ60z$m2z_tM0lee#J}A?1m>Z#6Jqc9e*$>}&3d7#~*Gk3!i1;S|K`&X=fbqxY z)zC}Up9j!8OZ5gL!^a_1uaIhX_E}CSxojxJ=>MfEe#T_T&azVJ+o+&FqH<$>P2tb+ zpem%=ffk_Z&OOYY@cVy1puRM0`u0R8`(WbO^c@<9)W|99VWbwA8{gsmFDc3WAtO#QQ#tl8~4D$%_)_L-#0egmIQ*i>Rx+Qq4A~SX}WMy|Azyk z>G0Fb?+;4CmWHsf&s4LFUm;M1h64#C!-hHbYOS#uq_gPB?CTVOkw(jXGzYs;S$n-a zMOme^?LXwJJwa5E6gMJL)ERJj2h;eTmr8s(c`Cv-}9>{a!uce1l1gy6xCy0}|U)oolH(sf_gI<%1IOIm>Pm*ual zTd0ysSTCSWVuvf!cgLE)F+~2PUN~4fo4B821(o-a3Q+;GLIu1Y@=WG z`OUU{l>yN4mM2}_@zM*{)bj%L=Ze+=_h`hXb@UgTO)MT>cwqJ4S{cwNGnyeWXNG!U z-nHZpd^ehhEhnFze!Zb^QatTYLv{KLg!gpi|3qPpaqlZ0#~{o01mCS$blsxxu85zG zz0k<{A%nkB_@q>}q4vq95NeIq(}M<6g|=rXm5)9zOT9D1i+wfSeF-*_Q^1Hd=czS4 zpIH{k9E0pQr(7V}^LkrvYv3~d=6%&4NznT!v`F*!R|I$UE=y@Mc=YLYpMX7uh{AY_ zVxS#ivY-8P!}tbc8;WVF-vliIZ3rPvk3jac2`0%`RDNthKZBsvkD!zc+v6qCP;{Qd zixeGM*4{i`uo&;-#-nypa(}4ulTCT$>P%4bS17&i&R$1KC%rNe0!r^ zRy~=ZQ}8RKx^bkyHX+x)^}JZd)#TE{u;y#z-p!RmyW_`gT$+K&)98RGo7tqxiso8)G*QZ(yPrmAwq4_UHpukw=wNKLHNcnNnO%ikT| z$J1vJoI7rf_)V7BE}_&;s2rd4SRhgFK0k5dr2-e4Oq9w=cAhL5LG-5{^4l&w#(5jK z%$OKi@yUPfDc(ZGj6Gs#UAR>jEYKh`y`gQ1yrK#B%6LxgFbqqt-gYk!+eIl zSjR*oN84|X6&3WM^RPxiMsX$RfCQZsKKOSq%NfXw9yBgjA04|0#;c%JGzY!O6sv=0vW%+jx zXPD3sIEdF3#xKwsea_HzM3@Vhs3vtE$EZLk1oXtKK@^m43{d+g!kxf&jO88DNoDZ< z#|rVT{iR3UBSNa0`AQSGUTec43$H=(;};>79wZUv$8gpgK)-oE;Yf!jl*3L)XZvo< zIrzqRHdmH_vw?850QD`H6W-HbT(YE7jJ~A9tva#5gVK`uG)_0krH2CFJ%B92dEd5h z8GHa>2Cs14j(?x5*|i5%zpIei?@C>AUfo5D9pY%vA)CLCf3OStHBRJ|-N{e#UTNbv ziYi2smWne66A|@2c7Tb|RZeIT>-%Eh!iQi7pRufGDm7#G(34l`6Cqq25yW!FGjv)E z=iWk;J&@Gr9^3WpdU+(boweio)q~O*ndnEHOe;*@qRAo-ImtbTD4RszSFHkV!}k}L z?SotrP<@?_<4yyI`JvHnqT`$FuxxS02*4X=o#dTCwYpDtooCITpVcQL_1-s29N~p_ z|9%aqbNDsmM;}!)jaEs89AOE#(@7zb>F1uAkaN;M#(wNe9y@;nTurRU`kx&i-Xib? zqN1E<|8`+{tiGO2qn53FM(AWv5lQnjamjl%>itL*dL(*r20Fgj*A$z@&u{UOz4Nsl zk>AxHY4Lhbm-1a%aM!m02d^1U5tHoKcRFG&#VU2l^jC&j8btTc=)mUMW@p(;qg@zH zfto*-_Q+@80em&8=v~~REIbfnaP*E;XOiw(!Q1qkD(R4Uu7Bw?axFUPE8yi`bn|dZ z9Lo)t+RCaY=v7KfCGkVcc>*@a{T6tUlv)Z5fyd1}Zsl#GUx(^AjSOYZKXZPZ<9p=1 zS+)So!cTJA=&Og#@4dnNT2f@-Hhkyl>0&*?E(=c=>$mzKO>`qvs;N@Oud{P?)_y@0 zh%xip@7C(?g#u649G>xC(fuRe+dAUUsf+buW#Cy#d3k(#eS5MnbTezZ@Xkcgb;V%C z)iLO|W=W?Xcw4_wl?bQ^xXG-zkEtJn1en$?$sgTV)cd#Lxx@H!R59)V3At)_41^N|PoWS6r z=6AiY(vpOdso3+PlF<>`Lq5Fcrq`Vn`8>N%NRFHTpuH^+m$)Kgaf37`0VV&(R%_VN5R&+CAyjDhOYD8PgHOn z?2fk(+?o>yc=OmLT=fJ0u%Wzk8^b_!9L5ffEWXGpBqS<)=q#JYV-j#(b3>t&`b>9W zkc0`m@kdS0Y2iq*mlo$ z@+(m)I}+lg;9;9G{))%g}YsISx^|;&CD7c{lWaQ8?|z#hs9$P=Jaoq z_t8>yVS5aonfmQ*zP`+7g|j!(n-qE}!UX5a^tI=tFJk-i-(^^e!W;0VV}XBo!D2mD zH>dLf{up-TBg2kfVj3jsA4a%nWQ2~PhdJHF`Lg4L3%-mJVoSur1n~s)tq$uVDJUd5 zB!}dwO$(m31NE{JVbPGA5P4+$ZauOJa&(~MaP#o(<@xXk)b#oOaNKD!Lx_Vb9I+*a zt`9mc1#**giYM#>c(JO3|DrlVhLdkl>MqTaX3fPbsFVGGS8<-tcx(%yvDEA~Q{cp< zfLH*~a}l*!zXQX7ejI6Xsh`rAa&YvmI?%DqNZ@SXX_!|HH&4ly+xC@({e|LanIVf^ zT&xCJYF$)IL-(I(T#7=t2RU2AsY#Ay;FREi=RVCjRbdds&*s1~i^CfwnD2+hEsDi{ z+MO!tH5{+%sKUA<5A(DJDF9YA9;U$TX;mcV5^!jHtq~#89FldH%pGpU%b8)T-7Q31 zlRsbilUR#Wi8-^lv9fpOwefz~5P_t|cbp?GbMq<~5K+49awlh5`vdvz4#EP0*0V7- z<+N~aAs%}}k(f8?w-u+}J1sXB=e*J3hVz_Ny`~4~RG_#Mu$?+f+ZMJvK z7J;NV=;bK0(r7b?>tT>0AhngqI`jZWk)>znylw2pPL(QozNl37NBYc<$DPs!Z{>(R zc|HCqBYl)d@Hk4SqV(zZfS%oUlrD3+8SYf<0Wy)_QvEZERSZ5u#Odjq03oE!W9b*p zYZ;mZ27e!UJ@_hQW0p(zMI>wzNo$tP-lfk^86%Koy3+dh*K_*S&zZ`ss$kfS;ihEa zs>5`v3~o`_(F4G%si|;g8F>9~r;)Z2)WmR4HXzSqvr_b)(X1?1CK&nDw zZ(2mImrV_P;@Z$5;P;$n7&5ClgQjPI=SzF&r0rTEEN{xMOLJ>vs*m0A74DYNox1d_ zxQ~>e^H;xRb%WiR87XFB1o>W< z8OrX|z>AuUh0Le*yWwS2DM0zB?l+g;pUoI){D&GwYWy_)dN2MZpghz1d=CgD5@hX? z&~;)wm+1eBYRt!MGhg#F;6-Sy-HN1hinTICC$Ko`6gw&La(Vn4Flc(70VxR;E+gIEV>gTi-r*g5a&1tdd0Z*b zU3@@UsCB~4rnfb{4&AtE;Fs@AI7bJo9f1a{)w_g<-3F&8mfs@Ejk<5tQl2Opk5{Jr zvubuR_0RpC-6NBw6!?H-x_}=k+^%q~_mbiMmd8YtFN#B^BtpLS0Uo%*e$N3xM`#w| zy6d{b33*sw28SGvco?j@}fgA47?n9#X}A*O}sOa z;KIkI3O6R1axpq=H!7H}sF)9_W)i>=~$@e0W*bpw64tf{$KD>7CN)t^1%r}ku|FhMw>KX zK!oT&7<`Ev%mA}!!@P+A!rpj^g5u=hVj{nh%l<@TrVR{fbu3UNgpgBkHm+oWmcQ=B z=zCR@A|sj{OFg^0yRqpE<$<9AQCVv>lNg&9MC&rC#%l6=0dn0i`}3>2)JunpmaBC# z3F62EUOVv=0n1LlK$Nk^{)mPq3o&aTp>t@^A$~YK(>tj9?jZ}K+n7a#jmKSCy3H9s zUx!*|kq)4vydk%MI+q|0fq>*0&u$Rgk#ThUYvYU$LwJQP!zX%wwP__9JCW&6cWIr0 zeQL4&%K4&`s$(Xv=pVLh4(r*gi}riYO+3&V%wc?wQ{Df31|6%C(!ksXM9p=gFVCs4S!+#YJ_7;Kb(9N_J_1!aasF9;+w zvkAH>|Ad30x$zbe#y|-6WsG2R$C&G40KnX)KpN>MUODmB?BAfMV%_V&r3bgz&=Pda z#OWZ>{Bm3B2l}^Z);P$S^dE1W#g8%j>~R=FNXbi3fN6} z&mfa|7bZ=k`wubaqg5AYsgDb#v}o)X0?@APm_z)9S5_j6m)ah`P?&rY{reiNKEyfo zXz|auM~kh&L!Xu9=RR7MXF%*FB^)aw%LhA*jzP>rO_!^B=VgpIH`k9@U%w(S(Kp`T zOsI0A3EdU5*(}Hy;Slc0u;VMOo-Zhd$@S8K$7XtgfhL4gaS5W^}GcYZ}&d$A5ne1m}g1XekM_1U?UF1({AJ`X8( z1-44_dv#*2yzy8s9!GNhUGlx&uZh?g1!beQr`adEE}%@~#|vHI{^Eo!v>xGfNE4?0 zf+A5=o892;MJ-?nEc?9m#pNM=uf!D{GX}%0NkeEQ#)%*p+S_y8YyGUQ*tqmAO8OR_!pEJb9fripK9ruygH zXNsG}Gh8qIcHi4dc|I~)1Szz@yXCa$m3$40ssA{|_M_|hI4LT~DTsroaR(&s2!;>` zNk-!{aqg};JmCsXCk)T(o0PHScdM#T6dhF_6mY8F;~>4$&R&g){k}4Sgn79@>+OK> z)6yp$1>v-)dmYIbG~jYtY&BEPaa~|n@Zj0_=vej(Qr~T|bl-6U*QOJURMso6NEooc z+&h=vy4Zv-gvxA7snLF$o?1#+ugK>1xZFt6;On6#Pi=My@DlX7I*n)f{a(yvCm#H+ zkevIs)xk3`?=dEQ+G@&W_KmbNm_@R}hDo@`%s%{E^=nXy?APpk+|!+tyV?7;pm~2M zsrglO>es0jS$VO`c)lWE2%W*hOXvMae(de$M0Ik58m$G6$`TqEO_!A2-6)rNrRVxV z*V7Bf+bLUx8I=}1<*B&2($f|5a`OR#FBrN>L!+au2iZ&1xXhVCjMFB zf}R`Hm6i@foyjjM22g(7xH#+4rhoJ2*_AriYT-kvwy>~Yr?)@OX2^+6BE zXe8h5OGN6Y$^kB;8K&)9x!$MPNGgiOR`4Ry5GXZ*9IFquJ290Zq&aSb8P=twJ$F%F zJ=iND*ZVC7>nQ#D@&^qZ$!9S{u|Oj++Y4Jn3lrAK10vh{r~M~hBsz6dqFjGDK6)`g z+%3Wkj9)Tzn6(aBE;Xmq@~f|<+K)TFUIX*oS1?2csk0-p|Kia~E~1yyI1TY)mmjE# z{0w>>IpuqXqg90YnZ5Z2Q;RIl*eS(ppS$erJMOr1cJA}sG{GGa4-g%sif}Rkea+RQ-X_~FU6^rm!}8`-a7TzhOn}f2!KLQva?#f z2VMM{*%?bsT2HdPrgq|l?dtk`+3h$$jW%LBv>(!BV_8U}z|+h@ud}V~i(At^JDs?p zR@SMlXir=rH)vugkD{njzFni$BNa`?V_0g|$+ataMM}}VHNvT(qpwzb)hIo{_xV@> z4joBem)`iAzTgxU<0n>!`!ZcQtSMk||H`z4DYMH&6&E2ZcbRp&5NdCFeI-A<#v(e= zWUg53m=RmBGcvZb)@{!?oB*07fvnHZAE+CQj2%|#n&DXM@N;4wu{z6%9`wvkx>U=f z@`*^0TphKj1TS0}$%7qQt}fWi|J`}{IdEUldLKReD*j8R4WtkrAX?P&tG}c(76DkI zg-GWMkQl9&PIjJQRJk+qIZJ5TOtJc9_PMS`zS=#sC=YfN@*~l2aiA$WVd?%VYALzI z)bO;z?{@GUm6ut3-KtUvZ|bwmN7F67T&#j!*xS(Lc3ZMMm>yPCFH6W{$iv*Lwm)9)*Q=^)iKdgEI^11u71qEGMSF*K zKZ$srez+*|iM%=D>{n%Lj(&A}oK5>@;#=cCnU+u)4%wwS$1cn9de-1(L~4KZEdZR$ zhsfqG{Qs=c(OF0~Mx;((<4v_R8oFI9H%GNF5A-y!Z0w9XAC~*QFHnxd8*K6(7R-fJ z{QF3UJ+;q*?Yg^IjFfyr_qmW2p4nK)rV)TE|pYpX~r>Wp&s6y z#+lpom43X{7dr^==WiAqD~R&l0|ktR(eK>`+3H=JE>S*`JrMe{8Yrkan$a=M^tkMDZ z)|p062a{#zFJe-^!k(!A*Vz*c`oy#`G4!}*qym(Mv60Yina+{zaH?>{bDm^I(KRk1 zJyas`K*NF+y6xD=C#@Lri9~|jMYeN)KXrEVFdM5TnEe6-R|@RNwr4^Ln&%=xuAmvA znp;pZ`{*lI0|}etA8Y7$uRdm~ByiHp*fQjf3Re2VvZzi2w;#A`xr6wtO^8Z)8ps?g z$m4G7-*^)&)L4=%yc`ORi%mIde#cNo8FpgJq{eIM?YtFr`HNIX-z4z9gSdUcwz-iz zR7_^Y8MyL`fG22X`E{0}{#?_urGVeEGa&;l|9H~DE9lnY^01MkmC`r0Y#5Ie6j}wX$YNyiZc~mTxH;zO z5=8?!lSiHTyj(YTa!NuQ6R#;L_+HAa(zxmh0#~sokH>)(vIhA9F@yBENEllx#ttJLvTIE_(HQ2Mi%+7r=frIBDS4zx%k{*6YxqD%!dC@uL@L1d_4T zj&hDx?6*2XMd%5HlJCA_=2Wx`5U){6bAk^-=0`v&+F5Mm$CHtrRkEc>6BRyEOTjRv zrqDdP2Df%=re{zr92!!^$0dz}t_TGJ^}wyURd~hQNx?<(=IF)3 zSC8GY`U3qEoS0+}$aWYNNxFZG4H-du_UIK1cynp&N_Ga0KlR_uC*XED%rn3yTx@06 z;>iU#mrW>&md*xUA?|r}A$^1#H7rmVBX0xIphCGM1vS(!Ix7C}a{d(zd^O@XfW5;i z<%$o*Vh$incDKL>{RK|MI=W<`@AADhI_vuQQ*7b2-$|`|DFBC|5iNdxh9g#itW$P0lmVy~JHT94%#$JV5G18_IE~sCAK9f<7GH5kRUJ5p zl-tG*vD+FP3${+)Pcs$AvZQ|N+`-yFO#q1#4LlmL$>s0i9z%ddUd*JLp`?Vhf;yZd z{%BTpRN_M4>!b&y#`PZ^NX9q^icm1eErxA!&G@7ZJ5jkwwDvxc{iHc&6< zuROOK%knoso$k=J+G+v-jrW-+&&{Trwws&5ELz`s=K=ns*&hVhNdm88W2Z|6PysEw zQiWS5qqFW;+XCGum>tUX3K>7Q_jmn6^j~#Mwj1c}8K(dvfQ&%MEQ#W34 zXIunjUS70Ego(stZmA@{R-m)EwWv2{{GtmY;bs0yuI#VvcQ{)szyv*i3o>s_>1eA< zy^l(s31xlQCAL;b=FQa|JoW-Pxg7+&m}nVs2yZR39Y?jYziftyGw(OsUCnF*7dhJR zt(*w1;~kz2>X1@!7Ax+z-`~}k79`j-1pX<1^Ub34y0;QJACD4@06U0vqo$0J^H(yiK`ezj=W#uuq0cbe!3sB@mdfZoN8BQSn`Wb=Y>!Vb=wl}cZ zT|u*4d}Ad7?4eC)Qst?k*M6<;-X^&F+;B@ofRp2wA<86xe_v?67Tx(Tit~M%gG}%d zSp9y+^|aH(?COVMP#Z^Yv63f4meinzxDD^CacpuH#}t$tS@q*1ApW!oqVnLKNZeGa z;W$a&0JwXSa%#B;ZH5QrJR*5*-Mr{|wo?SWYkCHb{fw+<(zO3Vp6r!>v|Ts^RY z>&VJ)lGbxwrWiIw13yCr-iOg1SNLkUZ_giFL{#LFA)=OY+KBk6zzmr+4;s(khP6+M zjkse}v#%o}A~ecjB9RUwa>eLOO37KC7YFH{{#cf?mDv(H|4`wV6m(^`Rjz{=N9DQb zr7oh7UF@x4tcU0ssJJd|Zg3&cb#G05#Cy0p3tG>mw)5gpik|P7{@KULGJ%)l<;jrr z|9f+yD4__A3d3L^cbi*~@aWO_dGiFyQZbee; z&ry1U*I#~|{S4J%hAM7GhQ4UX7zT~oQAC#L#7TMy+Ha-vNmtn{pn!c|f$GR|^ZSp9 zT)@ZRbUvpSX0K?}sV1wHTWcV|6E3;6*idXrpRc-VcEZ1=VY?`=p;&=_q-@_4$w%+T zqLyrhMv52;O)<9W zE8!*07qC2btW9sQrByrGno+;r>$(U&s2NV(z3w7q=jc>#(#YGIZ7?&l|NhZsVXoKgT81b0 z&%QOuT9}sOI%|Sa^!NDa&sy)q3&Hjq@gZ39)~e?u?2w+50;Pknx9Uw|eEe+4dD%F} zOBQ{UM9w$WFXcXeF%JK{xs~asl7|8v7U1B<%b~q6qko7VH=fYUmX`f+CJ;qp$%D~n zZz}LYeES$+OCaUNZD`d6hhn1zg$^zx{E`S`{g-mH{m)KpI?PZ0;h*l7)B2ngcvtXy0k9PRZ z_5?(ni$^7|hcYgP)1M}}ZcA8v+s%oNM>c<`5;%x$EWNA!)g`0jXqgwxxf zv?rBC8im@JCI2xWToyIoCUY}0swHd1u{9U3KRIhpXvN3Oy$8^`3E45L{R?8M#7g8(bE>iO z8^XxDGsI5X`Hc5-+i=%*t~xl9B*2B9>cMMy7o5 zfUwEX_{79wAVi||cYgoBn^U=F^bsi|J8+$?RVp(arWzPGY@}u}KC$p&?08_tkby4d zR<4rTW@bvQJGAV@?KJwK-qtg2ac_=qS5;g&c~a{coYwnwF&dMBm(3I!lXmd6-F%@}9y|~P_l6Ua1zjN>TD5}f{QQi(}D=)3D-IlZ2 zysa%(GBH)0DCh?Kvic9bauq6lr@1oGo7z#cErA zhCDw;mK^p%82igUI37{Q?d{k5g`-y8{|$NbkIi&i`Y_&g4Hu!lM%l&1Eqx*rNv9*n z9ONp?u@SLZFc^kW|DUlLyo77k!^B%0*g$J3(IfO8JQVmq_`|=@cLSblaesF^$6 z$!CV?rHma9Q5atan-o>Qi18taxB8PUKc>~ zd=?wqA)wrSZ2^vzKOfi!b^0Q|I3?j+OvTy1tyB<3~PH%L9oS zI}osqe+#Vg2G!}$zVZHHdjtQi$Ws`%-n3i(BzN9xXMnvTZ^`kRrE)Zw$a0zd{m8!n zZ{jNgmlt{r2o=!W{)QW7G9S0=&@n!O)l-`%B=_Y{;Dmj(iTZtNNfQPO?W$L4XyqHM z)u%iI!XGE)G^u+Y_QRjIZOdvte@=sl;s;jhNrXgFRJpL?KnqW9F8 z8lenpDYdq-^SBx0LME9Wn_pP5ctZb-h&GCH5r@p>g&c}5Zd$F!mE3ad*#2{2(6tMU zRhtWz{Rm3jSkLAZ(bM#`>;=PPktX}AS3jp)=ZA`qx;0d%;KPhU1XsO*dvq0?FCKfZs)5#{kg>@Z& zmL($*((nVvaWm3FKZ);pJaM zPvo&hR!1hha1q3j(e|v8u6w#ac)A)toNkhy-_$MG=F+@~nlpoxx!=}=wf3L^??WPv zc53+Z)qfgog*iO{XoXnwf`QMXs|$TIuEQEo*b33u%3e#L@EFAoN%rA*I97v6)1l)b zUKc++Cg)@@M68i?tV~ZB%$&GKT>4Tkpk*|3n&ngCxN;4y-Zq@I^T(mozlIgO z!1FEN`fa}R+YNn>_{b}+o??b;dW2i!d(0iIYf1Z6)O88-fp|xoz{g4O^h_`a4B&j_ zR{c&54fR)LKQ8w0r3f~+Buuts^f;e;KL!4^eSM@pJ8Z+(!s5?V&S@!54i#{P_a#PT$sJ07rMiTTtD*JBu33ig8pF6#)1CrN z<##GiL8xgG=RrW!Gu>;qz1h#Lj~HH4!efLC)}&)RUqzWtH>nAdOVJ4TzA8V;SjXIK znXTDR0B{R7>#z81$rlL5G?IyE`?tI%{|4lQe1ApmDJo-`Pm40BFi%r?^QqSA!Qc{= zbfiJ;=f9L9KT##5O}58}DAvYe)7`jU+bj&M2({hKt6R-jluQ1pwQ}qu!{Y$UK zueYiPvHMxdFiy=B^^_zc(PPJ@@9!8t&I{a2f=iAS@v^j(HiK57_3-$mw3BWFW?f>-DMOebQ#@)Z~pwU2q;8pfX+Uo0i0 z2$skh`1$Y@+L%CyVg=6`p}c!e(>;I#mR<9GoRl%1Ih)rg#1^;i+eu%Yq-dv;gkb%Sumb&J#1 z$Uo{z^x2M#SYaE@Vi#n&EMz;ue z$}$yBen$C}(KtwxEV-ZIwQB4dVe(AJ9mc9dxa|~|)?|>CAXI^xW=zuQv6bFyfK458 zUHh&ZTgZ1zls0vi`6$Jzyvm@-=?7RpXu-xty8E{(pI?x5QV*Z&OLSoI+?Je+$q3S$Z!*|>=zm_X`C+;XbM=)+_%B7$MuoYCFW*umfT@gvboP|Q~ycc7pK~BF@*Hd zPi1w`Itl(NA?c33R8wNnci&1^pYJnJ;%MraQ88#nv2UUANWej_Hmc5xoY#@oef;n% zljbNYxc8Kz?*PZpp^;BME!?T@r)8=8m?Zg1#O6p@{`ulbw69?Q$83!v#cMFT0BU z4?L1FW?$$oY3Oq{WglKGE`<9zjlqTB^F`n1Cyn%dE)x%}8?cn7pk=)Z5p-94Bu*eV zM$YNxPq&e(BeA_9>OeN~l?vQ`5j=GEezY>9jOAS@GQ++T>-49uAC&-p;tU*LL^|Db zr|4G*k=>&N7p~S^_lj5)I)%fhLSKSN5yz@e_iN(KK$(P$=pZ-7>MPpJ^^-te6yyy< z9*CKknQa#U=f*obeM_+L>2$);*_HWa^{)=knh9_X`KTT-5i0#8u2fDr3GO%g4<}|v zLscUGXte(oxj`OiTO~hGPrzKe!W6jq@;)Pc$#?y8L}^MDRmM+To{#{SOcL>HdEmgo z*>`}Y4u2%!LtVoW5-<_pLR`+4hC0xsMps%^f#t?};?_FSimGTtHxQsGtM<$-%b133 zkOIR|{??n4{>e{ZLs;q3!ed0v!54}<7v9yS@q^>@w5!`dwc&D2&ndk%B~R(Wt%RYR z5LQ^vR{GBd7kC7|NY6b)HC_rUh1q_4D1AE8gLZJ+qyIl7JwIu=+$|>5B42g1489Q9 zR0U&3IV<1Y;z8UuhUl6;TbpV2pDXr5Rt`Fl&|i}q_EmVKC0{J#7{9rSqEYc^JiafK z+P0#McTvoYXv7vuPvkV$pjiB9p^h*5vK(i}1h_F;s4mrjcw%1D6W<}Zm7ynht|Vdq zSW3X;t%<-KlxvvsWYW^6Y*NidqQpzgRzIzlmFo8VXKBh=| zS;ix*SM!z3N2>fwr}q6I_pvwtDaUzm*V%lVj1snG9Qr{<6-k|^jHY>t^bh{HIYeK3 zebBMX3$9_j6d3Sp^rFykQ-K&9O6Y?D@OtPJ(%hwF3O}~C2ltGWn#b8;EiAidqe+C~ zr)}3C39XuZqoNkI%TU7HaLz7n(h8NSd>(z7{aaG;%sxGju>dtJnD_A~9CJ-F+$Um$ zh2JG{5~V22k*K>C7(EzrJha_J4IdKb(X#Bt)`2=^{;Zq@hohG#8H1=l8PD|>1H!E< zl%OKZ4!T-3^w=Ew!okn7qc6=*j`iU7Uy-9_A_c|;<=)^WL#)XSaM)@XOGL0Y==4R9 zz*jeFIzwqd^p}0}em(+P%2wZ{~@_}8@DrL7 zIMr(n=1hHmO>>#N;u!Ir`N_lF5m_iya_#v9#-mbJ~_4})>oMY#a|7{L?O#R_-+aGc&NnU9596y$~; z`C(z?>wh}JQxEU|A%16`c~GNf<;&pZ7JB>PSJ(PPkSG0c`|otp_OKOaa<)p)#cYJfe&hs8vwTL_}%H>y-yQY4+L3SVY zmM&9Gfrn~BdZ`KRw7jp34>EisJWfgjw>#NH`~Jz0-SiPYPZE`IdEa@qJ6S5DK^n1UMGF6dJ!r=@_d12g_4~-iCK^g?jmb8ReS?d<=c& zAXcvQv+-zfL>~*v(Gi~TA|a!tb|DywBYtmQk1KLVp}9l4C_S%KAZxIhWOVBf;-wS; zSwG8ceFs?{h|M@@{puCI`=HtXVeBoVvW(VsVR&f?DFJCgK&1ugR5}HuJEWvL1YWvJ zy1S9??(UZE4w3FNU1#mR*IxU4V}FA`FdT&Qna@4%dEHkW5c%w6#rDn;$4|ic{eGX{ z)@=>!THMYyDKxgX8*s=c39&mbk^?t!w zVQ)~-gc7mi-61lxaHB!J!NBjYETxXXkK0y;=~xJ>41N7!)!=-0eDiYK{8)aKi4i&^ zj*PeS#>`a41!7=S2>|kMrm$JAYe1F zG5Sh3eg#C!MVZswZdtD3(|YNQMB<(~_^khWy#Ht|5WY-+d z+191I5ftWwXuXVYWo^BY&S%@@{mf}Gn?Uxg$EeG)3gTh=v``7JfwgEp@MEeRx7D@S zD5GWR zTz-!^-YQ!Sf*jsM;2bYBs=$ z1@2ye!(ra{<8GEREMFx1`}IKefHxnt)ejvJuYWq2*E7!S zL})A9A2u4BG&WttNQ@OY`02@hvFBkdCKmiV3nCKJI!QILkCXbw!}F?7KV{+q;KPj@m*)92Me}WcQoD4OA_Y2B@58}uXI)h|5yn}Jbj>R}@Uch`@8lsZ9Pk+MZ zCN_d%BpO9H=@i6h5O{}L15+3vrgxhB+r^Q>;OdNqP{3ea0nium#Cq=mZ4yX;b2KIW zt;7`p8bn#X3)n{iUx~k>_LmDVd0Ine_68Xm=@9b!yD$7;7g0LXHe{w;Ne+cYyuQIm znF78?X#|Yhdh6pw=`&pp*dj;y)!n1M#I}0NEwmoFImeWN4d;mXg3ow!GFiwPW*aZJ z;h|V{Gb0F*%W}|y4JIpCjn|&9!31QnuMif83#kvG%xB>o5>bHxzom>tGRXkdlJm5g z6QY3CWr`(mc;Cvs;X!f+Od03PuvT18c-eTw$W>}i(OlRG>n8WNzaKkUhn2%Z+z6IY zWMI-AT^&}}H@^{VD-xqJ60nTZ06S`OtI2kbAWnKV)2~Tl5Abb)^$hW3PCL{xNvomy zKx;c+`Da2+0rm_tjmbJ1 zfq#niJ5f^UDIW<}Vd`~$cDo+EVDj#DrYT2UZ4t~MHnRz7SVK5dUotc~r~2o~(qRnF z;2@%rW|I}y(vX&e*Ot($*r;ziT;x0Z(|hcgTZt{1Gqd4j!krh`)neuimq%YGi7%bq z&&SF9gv>>@^oQk2#M6#F2=)=*HH}w61_k02gg}K>r*qAb0s~+Xz{|hcoNP!fHB3ia=p(Z6?EOY;|MspoWTtWOhRP)e1O*iu4kwzW0-rX7 zLF^8ZM{V-xzmr>>#D;K&L^t#jRVvszDZQId%^d8P@>((+bFgH<3Vx@mga40IEzs<0 z*pYdF-3bqoZOcD58geIqHQoYI5t$<=NZaqoeD3}dQVIicXezLP9-QC43rnEU^Pq$Yk8rkAxt9D88M?kc+uV^V)>O{YK{WuI&l928^>`$x9K$yLB7 z>DZst<;~P;ox_pZt(9#w7&%DM?s)?>FF$>i4}%K_?cE6H zAE6dhwN40cC)h@y?`Y;l8rXnhnsxgKz)-%%e94x*`k+;192t@LyeF58so&OXM+D+$ zdnz5T1qV1}Un!wK6Am`!`bq?t96^WX`D~+E#?F>cv!QI3$+2nkhvb|AjH_K5;-Gp=cn+G7hcaUL z%3ZIT&F)yn<*jeyF~;B&$0`F$fxa1e&;2i20%Z^jTKLyO=Y;B%_!i*lja>)ZABEAA z!JXmRVVy<0ExY~x7eG;7EK+8SW#*yE#DppRG?B~Ilo$U=JZad6?gg8QPNjF1Lw?~I zwzxL+;$MV29ub4Lipv>eB8lj@LGt%0X>mYP$aH?BnMOxmv1QA+x9K1GwzJR5b5impB zzT#2la*%DU?@T(H(^==XlR|Y)&qjQ}&}MAAja$Hdoc|K_JGY5^0e}xzk9OvDI)DsB z5w7u&VwN~*O+w;+2h#>a@g#^S%%))_f77-rzoUr?N-qBeh^A ze|6-zI$yE(bR94HR;qp{7i%Oe?+7vQqNXxK9jVxAK&mG#(+3GCo#h4WB5c`CZjyyO zQ}iv0N~;~D4WEVy5l(~MYE@t>4rziCLnBWDP2SjpI2W%ZOk?gCJ}?ei0HdpA)sA3_ zT9^Gn9<(V~`GE#7Z=mR`0lFwdADAS(nFDd^2LH}s>LuUm!rV$&2()vcz*UpY!Sy^= zj&MnOM!reQDJ%O7+Pm1V9iJN4w(rutF`jjKAy{eoW02ssSthqB-hw!0g29OXEg~wk z;$+EKI$aN7<}MsfH<{=M_uJ-y5UDOyff*wnm9knL25)VDCv3wJZ43o%JsCIZgpwd? zyas?Q)^Y=#s}O>8KY+iegfu=q-Yvk){1H-hhiQAWEXTxb3sIg22t*c1{E1B*tq*M9 zvn8DGz zmmbq7%;y(L00K}qWYHc;{2&P&NrFPtetHtb&w@X3JzYuIi~cM{?$ zb6gh(iNTpJbuX-?=0+;J`TW#lYTK4K&@B?(69kOsWs=1}w>6fjrGD8O?9vBxEoJ+? zFWXc5^X2q!gwx+Rk>xj=NDfHq4?us0(r3+6-h^8@%?~{K`_pYXT!-Naf!>h-{n}&| zh;`3ET5?+nSu$~3Y18+*<-(X{p^e_4xaVAbLPS2grA@{*e<-Hbiz$dpARpt%}gYLC!lY7Z}*H4M3 zhWZ{1?)*0L_`+CYy6yWuK(O> zY+uyu%9qPwjWtu!{+^L0>;7BNWp+=MU(#2O)IybW<5ftBmHsotQT;GZkq@R*`N^X69C-s1XcM0xeH02VlR$rcQts%2b$PaosK! z(sT+Jfn1!6z407oNKmkFnM;V%Uk+upG=aq^_qG3>GnBgSvVrmwqYfJ4Cu$ldjE*w+ zEY6U@>oo!$T7q!?r+eGX{8Yfi+CXQhCqaag69l(+z5KG8>U@8FN@onyV>v~HsK3ZK zg+h+H7zfA(6Nsdg^e}=Jum{QnBD#+AW_*ZuF9gEJ5P(xO%sG-31Db1{hf_FAGF%dK z8;ukD+8+(1ASf*M>NCo<4=^xjFv;>Op2V={A*k6GGsVEo=CU!vwih)58W%W_6u!cl zN_8IUL^BHzTM^L}z@=m%Lfbs2g4yWMK`(>2@6xm_zd_OHkO84x1{iS@iVhshL?Pd> z=Po**^H+KD!hFiK^mhLpmU+^}{j9dnNfh?a7Ia1Cj_EO-U+g1OOS_t{haa1W0=hKE0k`^onaYURbg{1d7Tn*CjF27{fhSk#Qk_>Q`zJrNQ#{KNedATo zm(=v8frLI9D*UoR5%{pcpGDiD4HXB!3PTjMMwo>31SO2$?|p86VyGb(u< zHkNNuWFyCOxy0{Oje^nYSY&lPI?---<_+6uHv%&1? z4@DBll|RCAjJ{05#J43Hx*9_)!_v3#E|@O?MK-xO0^0i8QZl5t65b;{HLU)thOz2S>?%gT3*7qkpD0blZc z%04Yr=@^tUv~93{cao-HmCEI{Y}FK59>uK>Wfc^Xqr#@qL6FP$yl0eyLUsS3&6kUtF$NYSrFJ;iVQ#)^0KIN!J2CaM((NHT&_}x%E2&i%t$p_bUOp=gXm`& zBjyt6A*s~zjCwFN0lvY*!76q4g+F2s;LY?7KG7vb4uDSC*Om10@w%IaO*XhqGGkvz z2iWhxo1w&n<0dRxRNiFKgCRpjbg_JAAhUQe%f2+sLDkd=utpt=m2B(l8la-A6c)fU zcXl65vP*#>#`Ao@F8UT9V$1@+Kn>yZ?eC$0YZf-MH`9`D^!AzD#tMcaXY8-zC$QD1 zfU!qZO$q$(i$Ursdp)$;u~>3*3?lzUbai-QG{v-RiplL4HPQr8na`LtE1X40_&97K0o2T{cMpco@Ctdg|@s7j5G}|o-bk5@s`_U8cYFINV!G6_Qcu#UrTRz zt=ZUqUIoDpV+F4*cIDtq%BY^deC7#W9y8qbkqBQ$SeI( z0*6=x4vDx-dRBI5M;%P%N7kPNzU7i-QL7MIaZGl#uuiR6WF0hKbwrPj zj#kY}{@BNn#)(t{yyOgv>gtB_!^YbVN)L9-m9?9F#lL!mAimG?q?X>S$6h{SY`N}~ zxw>k_S9iOkX1~lBee>?eEGs(E?dWu_)Gxyi7R8ouv;OvnbqQv39#!-x=2s`J-5}?4 z^E^r@MT%4eE@7%o({a3nY&g07dsg4(QlFV>r=L01V$x#TK`QD=^~2TG$=ya31NYjP zfvOd1yhVEhp3oymTq!I5c3efTUQ_kUxh+-*(Qw}U$37XzlzywGguRJGO~rH`+1``l zcBnsvH1rl8GblxQ4y}#P9^B6>``c^~{a*GDXGR5}xdlP7_JP}I*-NsN-*rzq;qbPf zf!LGcD}o<&8EVw&2YDCAOi@zjE_a7T-5UoqX%ptKKSHUvmcCtoxQ}FcR3gh*Xymo(Cn8RAHn92XwYttklry#XYgR10jm{+4ZOW>Gz(vO?B@Hsa^8~|0Z~s6|hhM%wuYe`O6o@#S`*uSr1|!tC zLIbtk%+))Dk>vI)>NfI*T1TFRM`4cYefGQx*1&Bt8sj&=7O%R6Usfz_xh)(XA*9x4 zBJ)n*HktX#q^8I4s<~I5q`mB}9f@qtY*Ie?%^INna@Pwam0Zq$D}B>n`3}A50RE!N ztGYFgb7mnP2y$iVRmub|juvJOIeIHMUkv3LSLU&EI3NHLWPm9Qt10C5 zBP+863S0T+9X0Mun^(!3pl|Tj(!mXxUaFtvn4n^~nChbor*1C&@{I_t;c_W_{}f^n zV6L_jrsdJhk9YUO8!_+w5;F1vq20}vuilQLQLrh!8hlsdjK;vQVr~6i&uHpOj^u?t zT3!m|Q1{f5dX>f#1Rd$k3SX$~)@?x66SB-w8Tiw>?$q4eE`PL8NqXtU$y~fJpO5Y) zaha(x7c9S1_nZ?hZnNfgxT}t!3nfjFKHJ#|;xx0rzWlcMn9BB3=x}kcF}-f9dnbp~ zlkmg&1~i3x`bc_zT&$EBohX)mTgJxMcWd{oSM9mMCd){l%Beh z84+1Qr&6RW??v?64YDVuT8(I7=#e4tHqrW(!Nkv`T4$v@73J~91GyDw?YkDoIRg4Vmvlf3g*v|I z?uGlrP}xE_6760OR%3FZ98|3fUjNDOn#24ocpqxMR~|#txQlXr_)CE(Kq@=8xou0Z zs|cGR(+w}yL(^~bm^A>4ybx+kYVO2}L6HRBTOpp`53g&GIo~FxV{fY=*IS2wEG~S9 z&mP##8^mN44$^DqhpAFW1&PvalmnX)%ii=@{$(-R(mDjBl-Io`^#2*aj3-D2qjjh! z`VUue#y8tIHM!waYwiLozU6Q|l(7o*@1e$QsTdn5$k|^oP&FNn7Yn$OCmvM{U?4TnK}O~LRZjyw^)oe_0}fjqyeGBZP9Rew$L?y$ zQIx(;SVeE=*%@csiLa&mPEw~nh&OjeHZVf!YMk{}|5Yge(2Xz1yzPj$lwTtI;Yq9c zJZjIfxw|;??b+5Kdiifjl8+lbFIc2=aWGKT-@Rs=`%IG>ygm3d>G`|m ziBGl(brdg=(qp0c!#A%giuMq-HuMUJ9v$3_HA{!vXar z`9Mw`&DqdSAqF~AaBhWIG#;@Ps#fME4N8o*`^W%Jur#eqd~vaHQ3U3TCO|ZcPNDZ0 z=K@^I$#qE+L{@%sC+r2wL^3HYXDl5`lCriMz`1e(cniu*)V&q*Rmo|yp#*H-QN>Op`r=<%l5+c*P zb2bGU-ndnbG*qlZz|*JTCByroc;t5uiA%-Q0+X9QvnN0qwyu=EqPX<=jG>nVrjQdL zLo_RO(;?xoE(H-5i+X7sDTDD8SoEOWf&5@86`NmrertIjRzAb*J~uex7{G=stJDJ{ zf_SB)?wYOb>-Hi{;yu3)?^V3RzVP4`HRagUk#IY(A~^1P0Akw-R}p5VARUO4;E#7x z1_O`*P1LnmDFe6+#^VepE@bz=G8B_&V2K&bX&6H)g(FKR>q@`U8a1%Am$*?QLHvqEb!prIpu*n1?H2fd)GW?0Kx8| z6lV^GixprF^8mRvLkLk?J;*@8Nv)zMyn^oe?)b=%U`XTzc{+XU3?mXj5MbJ%w@`{sTUSH?&uC?(NndS+P!BtK7Zhu zE_)g=5+><&qEb$}pFT=Jd$5aEkMVe&kuw(5S7 zV0V9d8wk`%MW5@xi8PFJpGiHi4j+`_=MiCRpWSuiyTA$y2Syz#Z5X^cKSP*1p>^c#ekb0#j^SEoDTLidYXYeJJf;Jaw*5$dGnQK5mD@bl~WQQ*&JfkA%6byZ&P4o$jl#gV#1y zCdK&&x=Y3&37r)L5YM-`HP&BzsA=`f$NMN35O-<8rW)s z2ank?k<&K!S5GF!I^4IIc!An^tOKH>K&i60LR@fiof|G?)$#vyz`M%7@N8SQ4DRpm zz+IP_B12AVL%c9)B}n-6#&=#I3j$)ILxnkbF4Q+KdnI;o64DrrNz zMKPJwMjxFFJ1hBb;T^uhmsLhw6o2DAS#O3f>KNeP!X!EZgk#yYBm4qj!iJ7<+R<|4iff1HM$$1?asyg$(r$sU(176 zR0VncjuKvS{;50KVx#9~y*c;0sdBMDE~4VQz^hHqDx+=~`CxEbfgGInhC4q@c$Lb0W+(WYnHm!ZQf8pud<%QpqD0EW)_=bd zE}20webnPso@xv&v|uX-z=@nv-(xzA79jL>|85n=OXVqDasV4p~= zotn3-O(Q8R#`o96T$gimHHu4(JHhwxU6AzMI-lIMCl*fw3Z`#Lf$^dYpQ&Gdyj8c1 zA*%bL)rIsUwoTjPr5=7C1tE7$JZ?>TwDY%F25&rukm2h$CJ0RuYTu!Mm$D5$9tKAT zA_czitfCofa<)}Se)$X%-1kN4@YCqllN5OhGRTIRjN)qv zGXo%*5i8tZTyMA(T~`Q<$?)D=u`PYKRT@H{I9xVI_C`YLVvuUGl2^hzjN^lExv%h< zuU4p~eR_0cTXJLPcg>Lv<%pl8SNGBr?WC{`d3)$rZ=>fe&FrWoPVg$IDNRlocVUaT zP~}(J|9Aw{(3F>YTRNe{N;RLBUg@O~s~j4D1sC9eRXM5B_-TBJSpiwI{LjZm{u2cr z0^9LAe#Csycr2dz+1H%o4qP6G?iVhSMN!l-m~`LE&qP3$S#Wn);Qlw4^S*q{QwDjv zDY;veR{p0i!7J08>y!mDtG@WGFVqa&R|VYEf9!wf@F);XFB?k_X0V}I?lzPm^B7v! ztgY7R$9|~GlqQOetoz!(b0Y~8tY`;ZPOUs!sG7W{mrGk`X3aI{IN02qT1sCe#mYrd zBki^$H3vH~){#$Yv^fpZ|6;~JpsM=3fotKF7{b0%Z*^Cng?7P4EtVXJ=U%5)LM#P4FHV`n;tY2YIysV zYKpRMR~8S}e2MPx4u?!#HLIp_3e7IC`^%au+qq;PMy%iC^o_FbHl01&ZN8eiSRNpQ z$DhpIlFyT&ldp{DaAVXgB=e}Y7(NQb`NT&`1ewe`IM{w^DKiJ+e5QPQdvXR7j~ zSufAHne;x#r;FMnmAL0dMTg1RhGF_AQLF1gR=C7&7YWFHgn!B)DR`)IS!k!n;A-*vm$mpu zU5I{d1$YW$7R;3O2L0IOy!{8PHk)D%=M$2?F(n?NDJxI+JNy$&ptqsAFp{N?IEK%a zP7(oTrs{9~02!$Goc-zlR+xX38v#~$aD}t<#m6hQXpZqMc=-~R@VyWtbqZ64&rC_E zMg*obC-p{knmb|kbkQ^T%=n`Ak4#k;@VPvak+aR^yft)#$L3MIX z3ld<(WV@YzSMa-X>F#&mATKRn)W5ATY#Q=Zv8n9Ozh zIAY?EBzDVOARcdFwJZx+0-Ugkr~kHN$1!&oIiqebCS?gbD@bb`r5znFrd93cbWNXO zQlq`>$n~7^83+T;H>12Ry4N>5M)K9B7%cT)1#cfwA$lpOJ%%!$@d3+K2a-SebeE>} z#cnGGBj%mwS+BT0!~uf&`(js4pwf_&-KV}@NFjH#LSPtUtoy|N0Bycn(v3oWMX_>? zvYd)ZUj${ZN1&{QDB{hjgv7v<5$~pJ(FVANiev$;GZFzG?=E z&B^N33tXS@jSJELVmUpFXdXN%|l+E&L z4Vz@@gu#>{5Bb!#`x`;ER;?wGR2Nf35)U5Po80fhV%}&L^SHPtZ~ypCvlyK+(4Bzc zCxhTv?+B46mE`&Ng{rm7E+RR;1xAQW@U6?mkChV#)FvbhOnSBK(v6C9zVpwKn2!Tt zPG^8tB4F7R`6*e9w=0`xb z$OM{aiSPm*pk$?CX#(b`VzSQbfi!QtjRbPc=2&=9aD&+wfJXqM`BFnQZW~zOJ)k0b z`!LG?pDhP0^(vC5A3BTi987ucIUJe^FlpsvetlO#~oXxg+~nd zyNo4%7;#(7@c%JK$QSoTkB@&cCP=s4CV2$ZTJxZ==VZYoo(U||_BfEHRNgIn7CVhC zd)Kp2BJcD2v)h)Lx3e*{IH80w^z&sr4d?O?mEj61?!-^KS=e3#tr9WU>Q5Jx55$N# zzoTGKogT6iT6EhX-y0ng&aXE(?fw^qaXLcJ5vK{%aw&@=m6^aCLT(Xa|8!XDq zq}T__dnc2XW!yip(S5a(zBxl%!GcBx}A@+=CuDZ{6{nQ_gCmiua-1c?8WP!NA&{U$ZP7R>8Q(c z8GQ<$aFk1r2T4m`$6k!ggOam$9u*1$4nz|I=1eh_p7G$;<3`z04#fUvs5U2BZiKYT zGb@a`uoa>M*c?`E>EcnFC&CeXW^yTv_1WvbAK$9OB$wA=#np2ylf3aZPq>(RO~6chyXjlK3|iPgbBTsAAjEmCHRhNDlxxA}PG`uv zQXs+j3x&4E6|j1w-B^VKJuga?$)w-ki+9k9KPaF|g_HH~t(Fu1*(!I%3utKx&=Cxb z_xwncwL7Zz#YHQmIj)Ay?rVSbsUqLm_Mz)3`Yn}L-tvH1(CY8#&`~~GEOqt$pCA=L z&ChA1*r|Nuc3)rHCcU7+gE6 zfNS(SJE#5Qu?%l#hT>q*flCRcqA$Io0&;#>=~B*u&1YwfHJpHxFh~B1h4gDCXsy6;4|(>&$_MeTk1~*%`|j?< zMNWLWkS_HZeWTq2zjbDdw825+o*Iih*S(l5qS! z{m%RHNFi-J7yV3e5}aPIF+-v3teSUMQJ6D@zWbMFNY8s99~;BZED!14o5+X7m-b~T z{lq>=^@P^SQ(>z&(`d~n@BMix)#d?_*+A2!4FGFq1zFTncEYlP{7Z!cH!Oi$?7rdQ z5A?eLP71-`+{DxtfbVKi1hO7{#*3ZL%`O+;QKH~<|MSKDzu!&n0+HnlH2-?Bc7LEP ztmh9CUb%Zq9(8OS$Bi)-QDA_yib0^AO}!M6jnEZrjv zE?D(^@K0g;H31@Y7PSGFjKak76xdKEw;r?$UIwj#0O9dKxjdXmvb?v}D7biptU3a) zp#*Z9b-y9?#uKjvQ1Q1>P%r4r18Tm+F(}r)MQ*hJW=Kr0-?Gf@35SGsyN+tV1Xy|b zpO7r=ZI^lMos^#G!hQ$cG|__zeDt`1|kC)?Y9 zyvm%;cCN9bU{dkzF}M-S^&u~$+teAA#|yTu|KmOTkGEQk6QZiBdJaN?K@?unZmR2X z24xrrMJCTjw8c_w{!=egHlRkP^s)NI{LdcXKl;rG4tPSgq4Y?~3%t*|Bs&ANs0DQ@ z{Qa>U^+D1SJz>MS`r*^EuOCm55*ETNf30gdVJ3mWTfQGtJ-3d)D{#abeQxF&-w5Yc zXA4u7sj--zwKUTm2}1#)M^aRTpB9W#V#NJPdhC6_N!Vb`)la5R$}*;-;IPIl#OBnP zvlzE~@pKS)X`aI;7yqb0Sl)eUi@c07ltMJ6wthTA?`>n#bm4fmy$7aicj&gFsO$yZ z95&~>W@@zz;b171R8iTS(=AQxp2FhcK2}wc@+NrZegNOU6F&1*8=g~EM(=D-Fig1J zhH&D>zwaIa$#hWgSON22p^qC_d2(GX+U^0OTP^@zHC!}BMEuGm?5fJrr+mS3a{k4@ z<{C_Y+-SSNy1xInMf|)5?}sr=L1T^FMfIG&80gMBSg<$XzmyQC&uR@~L15j4)N2T) znq6Vdmy7Ys_CJX)XL%#O7+uVZ5jh<(s24b3KwaJ3q#lm4Bj z$JzjkmU=Pt{ikqX8InHAKMGEtNpJkz8b7qRnMSGBcrMS0`<2nwDKT{PJMpP`Y5#W3@UF0CA3sJJc2pH9Y@ZlKG9KrzYc`W@; z)hJQv#Qfem=_Jzh>7ve3?FLp^9$O#nt4Ucw(`AhZIxjwlv2;2>X+F$W7l4T+Vk4;f zrc5<3ymdLbQ#jC0_14pIg&wcb5$~PC(}X0a-NkfMRvtDcSpcf%@8x$4<2JAV`%?W~ zw=4KUrJoY?YE|h#)1{ztU`k0{zF+1v%ggug*Q3|Cp4-DLUB^T&lNG`rD}mt#!uhQB zTGS~@OacLo;aIv^!#1H&k$w`}OS^fK8@(j!W`s%t8g_4^Vc8&@^roSMD@f1S0KTBLEGKfXFr64Cr6)mU-|I7QPbf-eeyVb(5-W%;MRumOkU>k-;?kgr)IkyLEw{6>STD$7nq%}$g@9<`@<)VY0=rFIQ=Y$8>fTK#-wZ#tjbvt3W5p#I_1Hcq|f?yf-2 z^?TxH@N+iVE7iLF*lvvzx#(B??kqdMlTp6k7P!dal<=KN?f9A%s%#IVKEK-tmo%QQ z`tw+?m*uB!(Vv;-KEQ(SK3y1-d^AdvM%>ifXt(C7sz@`(&x>-nSfGuo(d=Tge72{W zSdjG1qF$lajMM0UUli;g0wN-9RV9vnuWmP+L7pYh5EjsiP47IejwdI>G(WkQ*rBeD z9iTo7C~2;VxV@6)J?PeUSN%w@zCC&PYfBhE1MN2%-W1=W9w9&cFlAp zUHD;K1HAjgG(G!1-~x*61NbU$(`CQldP3@0bh-ZJ9l z6%hy@wAvN4Jn5n~jbm>gP`+YvYtT|zE{Z!-k*OBYdA!FicS@J4^VcNSahtWlJO6` zqV)gTE&u2)#D!Pq^jVC-S<}n<{H46uihB`$3J2nkO`0zS*y*x-{W&6;vZ@k&}5m-G!LHI3>ZvF%YYFHHY)(ew*lXyhLe<|9I^UaF~JljB@?py)b$>bRz_hIy^KLzp!AP-{+0+YH`2+p@wFj*liVu0W z+`$t|6+x`4?`oNHu)xlIP|Gm#=Tvg$a zLTo zk!`-k(v*o~%Qc0V$Aj=xjk(`kAO)Afznql_23fH)fm|5}xNx(LZ9r-EmHZv5eVgtjz# z%({1LlJ^RCBF%%^ANe*f&CBkz8-XGXp~@_wkyW`6ez1`f%tY5MT3ei~BkXYcFg1xC z;vcodD&7tDUx>lrog0o!p*p!<8N!x|7JqjkMlRMO!R;vjp2#Gq?Qtajl8r8y?~x-e zjKgbzmpe}79$37jJG??;Bn?Ec$CwNR?f|z4hRrU*D=WiEj!a=OwCLqoqNJfjai1>s z>&Zm9w_eGGdHm^buc?)cn;KNlWZJfS&FE>qFf|(GPbxv@BD{^|Mx1i|e`A`LiY1upR{Adc6LwD?ZzdT)vYr%|iKJZ+NYa29zatjj1B#^9!5tOmF+A^GTKh_lMn1 z+qA~(R1*{@&`6aVeDAk4>le^DP^yN4_)=2;*PmYIej-$;^_TFU*x>p14)R~7O#iG+ zVz7`DZlekB=Z4(d23-LZE+@Qrr`wyvRT}qWpMHgP7;%3=hU*P^o9l;u<~vsn<-5)I z(K)T8SOQb64n6(yvYI4Wk2`g~=0_KlnB5EFtslVr7>Y;NmvwoBqh6*%fGC|>JUXXp zbXuiZ-yz+}Pv%ivO6eYuFo8*H#3=%QdC0sAT#|P8i%e&z?23oIi5ElU1K`A!`#n~y1L7aV-8HhqnUSVx4vwN(%(P^YY+QLe9Pzkko^ zX(L9-z%o+E;*C6af zn3{>>Nm0fQTtYEj=bDI?M>XEHw6TP9PHV@d80kvGtsk?2qR0k-5$-d>nFj?q8ZhX@ zDgu+aF_>Ldr&g^`7RW_|-BYF7uq5_Vu!KzMp98F#x~rk2emT~BJ-Zo``l z;(b_gxqx&*3Z14}y23ie+Xj^qaT@O@-r7sXMt>fRy~(7X4IVj?DihP~|G{HAPI0Ew<7KRr(es5`K_fgk*V zB!vI8$%7vy!xGkfAvYlq6_wYXSEu>9@#N2L2kOPT(F@NM?!caK#*F1=)O?o!8p$r?!0RI9ke0`&I8wg? zKb9L#swqUmg$wKu_is(Z*%iL1tNa4+p~m>A6}OQQPWX!J6=9WOXhX3K53_)?vj)TI z8`Q9@5KyaJPM=;H)t#IWvX0Gyf;D~7N+;ZZH>me32X(OZM=dVrRyoD$>s2FI%i=!U zIxUpn%eY#Br#Jum&Wab1LMC>1f>O(lYvbEY2A>VWwrhTR5 zE3?nE>lOCHO`dKKfcai^X$mT^`slmYA1|@^ta6BE(>Mz~^SIX?b3NJfeHW*(u9d%z zFNr}@j=Z`&Sr6I+rW4*J&hH^8T|Z#vD~d|qVSdzb)2z0HaZ0&dFviEv*sp@Tob@@BZdt-9Vb4{^x6lsk}nL_(M!PL zBDz{wRAGE=Wt2f(?09ja{f2IEk7j!UMJMJREJ&2HH9nf>X?VP)8|W85UYyMrJpbMK ztK)~ImVA(7a1ffyhwar)G@|B6>Xc=sVy-s`a4RS9+KI6mr{XcH@d^jq#7AYzT4S~R8 zMr5JZGEXArUEmSwL3|ppx0o0gC8JkTv%x=MalrTo)8v2di2xSZ05)CborkN7sCytr z`S`&UcbsMPb-nzwQH%Y)!Pwd8&RAchDyxGTbMx7Foy3A(B5cjp%ux?!#BG%3i3bZm(XCYr#TR!Iz?qNDZGhPfif9? ze${-P*+N!hRIk+xEo(^nM8fTYv-+!Zb>$IzOm%g5n2KvJd-F*DfvP*BJGd?6>lQ-Z*I%(2kYaU?E!7%@K{8$tzd0r@7ZcqUzl z8!*4q?E-9HF(VM%9nbH|rt?0keWSdW6i2MHAd&pPbB(a}8MNgwrixAHUr!iB1>jSLLNkjTCZ{FPZ=^6>Tjore5Q z0yY8%dudPid8~6tq)yPp58tSH4V}Y)pcao#MdyM33XIrCKLgs_z|AQcwGLJtkX8GX z>HoZ_J$D_7XP(NJ%lgMGH2ayST$ac#y)pF7iZzgVQro#~$Aq#bH zmEUy_5!J^jb*VWEcB~YqpTf;9U&qkd4FVn26QUCm7Be$DK6;b7 z{jA+zz~PwH7=$ho0Idhwz{vKWKd>K=)_%bBh}iBFF1(;K+n?%BBDl!U3Lx*IeG@^W zH%WcLOWsPs`Tgqn1E9101LP9M>6bH29&o||+roU2L*V7hMWmiaDGokP&uq-!4B9Qp z0O4pZBepx3p&tt8jTd-_eGLX5e?cx?RoByQ{3s>hbv#ay`2MSeN-Ft~(O>$BGvqIw z4!BFHOMP)uc!aKNGZAD1C4-E0bn`#Z+H%S=u)ceBB+QS1AZ`@`%&%JHwVwJY!#T7@ zX|fqTftpSd-#?UqnZhl@YD6Ki4Wz=os-bwGqYDR;L6kH7@HwouYmLkQL)Uvp!yUeB zzllhg2nI1kjUJ4agy_B35M>aZ=q-9D2%-hiqD3!JhUnd>(WCboMDM+w$KLz*o^|%S z&VP}$u*}T&dG2yupQ~{L_%A|qvdCWE?FpV^Lz)jjzgYts7QcybX`yq$kH&c9A2I(7 zro)$F)d;PAg2XA!RnyUe=M9I7A!kpAyJOvp%aB(`1iaQ3^Md}YwJ*SCyoVMu;$1tS z`j<|~$PJuUPFFYP$ht|zM7kZ07}5t!*Enp<5dE=RvRoF^U+(yk5cA@P+Rvly_*fFm zox3q{lFiaduVM6y`MdKdj?22*TIO4sY`kM{8fBK(${b>T+m03DGqKW_E`)_3Mu?PEUxO9?H?;r&85w5h|ur~!W^a57<}lQzKc|5si=kAesf@_4xyn}?hZ z2puzCuL*@>YWLwjLs3(c=jSNAGW#p-S!Uk)`MJU6?6X&&j@pUn4Bv3i#@>+mcV$?P zKjB>tljprYlIJ~;wiU(pQNf#j3&tE!&*L$d0Uieb?{C&ZmM7xGKL!*}FH!LPn@MEZ zq?3}N_hA|@gs4PP^oM>ZOr~&o?+aADbcl~8@vi8}==DX~4mEVT`Pt52a+JoklXpj% z%Wf1I7OpozbarO3I!i@I5^-H?g+;80T0xfjoXREZ!VEaV%1kaK;GL8MrwaAmpO)?E zo6&SlB|RVg&-TdgF^w-vOGB_r)F4<1d>M_45?D}2Nb(r|=cNDNRgf8`@Lz6gtzxJL zZ1Cp#J3)K8&sC2}@kS!ZiZXhoGGh~%Fx^0DCPW7R7+ugK9*L*?$5u!x@m*wz2mpwj&B@G9MY-~TT5l;9k{kR#gJ)lvh0EQhTyQ44 z=h%bYyL;njw*R+i`Oin`zx#OM3<9%x!Qt|Q@hV%t8l6H`5E}9X&(`-5&R=sVIm09e zru%-)-{=D=pWE3VMcUZIpC(fU9&JyGsnR~_CAn>4=BRutw8j21@I5k5J6F*cEMSfU zYgZV_ud2+HAqojUs@TWtT~L^T#O~^$&p0a*nEmBUCr?ry&UMEcKe4H8jZ;1fMDT` z#&5mP$q)Yf6CtgPHk_kCK=AwkV-naF|6HHWE8QUz9$}Ltz+3>pR|H7TvDAvRI3+^t zD*wMP-TzTYIx^UuOeO;16OZGlT3|Lx`58gIO3lrMr~&8L4{aL*zo@V%jV2OKv;bT_ z0l#Q`f15^4Js~P@k28bfc=zaahojAs<0Z7YB|)zZQHd4D>B{HF8)^%(^&V|_#;iH( zfBYXB5vu9hEC(X+v20a$zP)?-g2w;SiIDQ-G!0zpbW{5tCAv{qR-~XMFhMLR&|$>c z5x;ZZc*;|yYKU?(Kubq=gt0%-aNf?z~MDhC`6PZ{i$ztp4+8C_@qTuBAsC z;IAppW&Jg`$Cd&(}}V!^S{3Ihbe)`$}p*&C!Ye zZ#mLo{cKqi9`|Ha!qkgseXafe@3&v_V>{M|M`0hMhF|pudm0E|U$|4HCULPQnhhk_ z+wJ4RgDIR71b2UbV^d@%*bBV}J1Ex7^rp9*;}_P{^xQpbPihTG1v-pO(=ICTsfXV8 zyGz#F0YDriGvEVEunV2&qpi#814P?UAuiDN%hxjtL<#)wedB)~p|2U}HjO+a{iKd<+Z6Icz8nV;VSuaA9bVS?{$pRZO{j4mG7O- z$>^#iABa;mqs@skLKHCwq+g+Rk~zqKUrG4YgI-TCfpy4^e`kRTVe|N}w>UaYSTVx= zr2x_8B=Y}mwkI%W+%35XVffgT#YFN;_y~#ipK%Ony{p?9)=$oJeb8@!7N>`eEo;fp zDE z`_%8j23fM5s4or=?MIOqUSq-N9d!Ma<9@yrqGrdyz46J>b@4^>f`I6^*+@C7}WP-4QdK^c@|>!c?bZ%wsjqdf_1jW4+|?lZM#!kTUW#{->W24ud>J z1%$Pc1HBNw-ihJeeu-8Htk{y|UfsVH7QQ}EaL4<jG|upfp=As#IRTLY<$sq2D=|^JhjFx7&Wp zu$v8rzk#Vl`Fo#!dxv%e$U1OprQ{5U6#QU9r!htCitdG*E=~o(Ke%YGKK$#H0i#;B z9Lqsrfgn&Z(K_!8p4NE(eMMo=o9ya*xUUtNY zce75~A&F~-lFni>+YofpJow=W3GkJh6Z_T;Bd9QQFmV-HlO2=3`5{~cC-}re2KTcB!&I%a+1yxbm1d} zEfn9tf*N)$>MBpT>F>=HwFAaQQK{khj4Y+aj5A!F-`Rr(Kr>PgxjUagbB$wCop64( zFV*^Vzdx;{vX@}@$iefs6sX{zr+C=h=V{mwI-QXx;bvz(f7!(t!Q;nic{K1_A*t4E z&@fdt;(07)Gum}9-bIZ=eQFXvPdwOzwzK@!Isrv~1tut4r=tr^lET(gIk9%9M;Lei z&bgb|=4JOyvd|mVD>!sSFwm z#XIz6iqn{;YQ9Q#!cd^uGBA;5MG9U71Ja3YjtNMjEO*(}{d6pzl@EY*C|a*q7ogR) z$;l^)D{q?+`NnD5PxeX&q+AkGDVzy6C&k3jP;^AR90n0k%y(HbK)_64X!+DYjQ=r4 z>0c1K*(!QAJ%97zBc=e-ep%9eyJEc8(FVkd6<(ZA{*8-|mzv;h1YbLX*phc?f?ofC zlmw@NpX&Aw0t~ceB}2e}8fX%fhUw%qUaaE?rU*&rAbf;T2QaTc=yY{|nJ)_Ycne^z zW&l2VXS+IVJO?$Hg5SdpbCAT<$7>Jbr8MN_A%3}|} z_LIWKOZ1e^0OY!*e)ns{ol(*093WmfcWVZD^;2w>o;(@=KCvnfwJzHcU=+MTARZ%( z#;kvOXHuA$nkKD;ylnRZWPXi}$zKd9AG;Y1sp*{AFuukC40qx&i4!((8w?2b-P|(? zb1nun{}B*7`AZRqwCCh41MfYJ+L`dTvi+*3%i=)<&uYRD*S)4pCqO$*nJ74j{Mhott4e!b{6^2`+Q}UhGiKmT z{2(+43Sv=@5KllO3Xj;9sKI=HHCgZWD!l>6DX#=jwy=&P&TjA2E5P7uFir_5(5u&Y z&dn_kjxRxa9_Ma5fL69xtqpdF`l@Pf4VaCd_owhBUCY-wZ}*l4iy@WCJ6tiADZ_Z! z=Y-vX|Jgi_Z|Gr8P>9KB*Q*YpBJHXi5QdyU88%7n^g`%c=ImnQ9pGwPm;UFS9U!1K zxSUFHgr!{R&8#EA9kXfXyg;a{C+yiPv-5+M>}$R$kubeMMH-*#NcL5+hW6^m)=Gt) z=>f49RwAceLJ>XeNR^Z(xZ1Fs-{IzK^l|G)(*IV|A`~6nr#rq8fPZQZq%o ze(h+4nX|L{QBLuXsErVgNX_69hgO*n)8}~QtUcj=?^~}w6X`*DN_3$=b9p9A9P+ZG z8k@3Q0jM#*Cypzo@FrUOiS0F#NnVCJbC0BB2bW*BOi*~4<%zjTT8NNb(-ME?EmzXL zGz>6_#o8gi>cqIl*r4dn27ziHk^upxI-}39ll=GhO9EZBpye|j518v{+gG-Dv+uGIfxZZ2)sn_nn zRd2S^9lKfwnTa9(X(S(JidUWlrX?IH_8rd*QM6nGnXdGx2{I@2W4vq54XU9}_7BJ| zq>fU~e)J=Z9`OH4PwhZ0x~;})=yF_A{G?dW!JiXac=pKp8i3oST-6=4s|=(1%~0YL z`}MddW)J>wX>a{`r2qay#r?cP@Q88l`s_(*_x)GB@YM?ad2}0dVg@& z!hRCKh!TdJ-oJR;3C1aLEJkmFG9yzWFGPTn_d+D9buaX8;lZI(qI*;R2Akwx>tsF8 z`NMh;QP|*eY~kl}=`03+!3;e#+Bh&9l^qx?;cZ!D_e0>HT^*KPio=vHPu>rhyq_rf z1i{49M%D4{R6h_fpI8HlzpM&%32;u}D&9gm7>~Co1#3Q`>Su4uk}xOszJ`v;g@NOu z3mQXFJna8ER?jSZl;l^R4d*JIl9<#S0*2^&^Lm@^1Hj_I zuB#La!dXEPEr~6IWS&=TZMG9N>Vl$8%HsehU;%#cM+1pNDa@>JFz|e4crfY2`xfJl zl>X64zbzQgV0>9HU%Co7%a=i7;>&>SHe`8V8TK_>8?*z2mY8DhhuLa23%mk!N-<1d z{#+?}by7e@uPQOwr85eJu^c3?zRfv{tV9)wUGDGt+)fL7#RO&_5}-X^23kT#(%2(@ zDjZ1YDVWQq zI{ZCmLkUo#V(OchNt5`%Kq}H8Lo%9WmbR(lC3^?_3s|Y0Fi9j-qz004 zL`L=f_1mi12uM*q#6HDujluxC>~*2|B~{yZklj{qP;ND;(%s!1xkPSI5>NSz$kKnfc~)dd1^dIRyLp0MzEZYE9EP3T5nt^L#@(qMYcPU6D3%O1 zf0cT<7;mBDWYe~F41)H|2>Hrx+j796X?qvB#!Kmnb6_W07A9C{E(9YXkT&2H)0aw5UNj_6Sm z=H%k*|LTl{da8yLeo+bsFvnmTqGs#AAyI+HDj1^gHH>j`+T z?3{Qg-TpRCNc2;^x)1T|2p{z#?@%G-`X*g`uyeeg+ukimxF1FmTfI9myIN_|;}36` z>x`i%gvL5GHV;G2P6dtE_qT|O2Z%=J-G0+&Ll{zp)mJn3#4RISFMHBb>Wt#cC zuUy9oSjT7sj>Bby3?>KOJVK!g{6grMucR9TD@cvNwfY$uy)0K7z)AS-Exd3>BDR=G ztsr4h|I^%02mHOqSW-C&PYF(=*jqVKUrPNT-QxBygn|pG`y!Eh#+aL4ADpkA?k%M& zAjn<^m}CYIe^`N_u>^Ng+m}Awdn^|z36KZW{b@+v512n2qKtpeaK19_50fS{-v`sE zpRW3o`kxcc(B{BcPl(ArHCC#;1%we=@z_LNwV*J9-xb=?TuZv!`Ls>9Yd2p4+`9OT z?bM{`ljj7IHH*0fdxrMm$ibajek$UlVF*^(P<1J02OC3Si0m?S%t%Lq3ZC-}8YVt$ zn*pbie>!&dzJ1*>8d+)iSbWQgc4{6_76@XV&gs%`pw~mKA)z@cCuQY|3G5=@t%D5$ z9tw^xcA-9}AwA7zJP-Rg@KodoC@_?S(Ccvp&-8gJ5oXUFP_h{+-~OHGMh9A+`a>1f zy-^qDHGy0wmZCmbn++WxgF0RQC)_hJ4H;n`tRpGvU$S=#I9Xf)qsEL3nu5t!&%d@E zrj488u%YSNean8Z?L&(bO7)y9HV`5-;wPPq{6q?)U$P%&RZM)@?(2$(r7yXG?x$0> z{mPE<03fLB{Usg^`VN=JO3qgR#u?jjAFsL}5_*FNaWSdiyINnPrq-!7=>%hGs5@5Z zkcwj2RltQoA)4+V?s|;sl{oDuY-me^jsZ77AF;rs6}0y3s5O1D?wqYqKc4lqtm^%e z3G!xWTqaF-+LxA54Lxgpc>8~D$~ZV97tp(<=2g-c@;USCTg~^!xcye|-&bS{m|II; zhR=c3(SqM!qAzVpw!G~@=ij%ioJb$qvw>LxgC9fci>CvS4c?%OvsT=>HNR_lnjk^} zzY0sa@)ggMN%&Ai>dT;oxMn4x^-aJhx;b|GslL8U!aeAd$z{WHWZ9MN>yuH$CQ3dUdO^#_?~X?jypw|$B+o2{Dn zX2up8iLPd%Kv)Zaz{YHYuA%J?Ty=^}d)vnI4{g-Mxku?j?cAuMVeo5}2N?x7xeNDy z!`>}=9~pR{A!6%c=qHUt4flm!F9 zaI%5ae$>tPO$k8s$^G3yn0sq67B<7sEwqSN)BllPY_?j{1{-(R!9C_ZrXW+xQHNMK z{EasNuoU{u`ZA%`(d!XeTP7N$YPzKkcoz-QWQ98gFnIow!)BznD+qmzqV0K4X4i`S zMtU=H`plj}{5KMw7L2C*t*P*OFIs4{=AX+ulU{t~hfASg*deWjmJlF#bGG-}e3F*y zDnigbZU20&A1gi@hRMLsgeD;&!XXvS)5RL?irvg^&F!!ZP0Jz^+zWN2q5pRkAAU$qt_$rH*@nI6O8GK z8>z?I@fm!PQA1oh;a`lE=*`clO!bpBDJ$+fBS+Rhyd-u)8!N51V!{2BrEv<{XxZKH zx9Vq+7h9Fv?dM2$4CdT@yM5>$N=py;*>(0sXGJF$DZZ9IIc`0NV!7Ef-7o33&@)RO zyZw_XndK_=i+?9 z^I*t!11-+|bST!06&hEqR2XSP`Vk>h=}_)%gr{1LgT-TPWOU?DyvxIb*mbhZe-;{P z*Hl}f!sGbwZSFrK?<wVxM9JpcX=|g<=dswE<>o|}25sWmp?<#J6RcF;ARetVRt4p_Vd|)FQm-beuJlCdB zaC$hjT~fDg2s%<{%~k{s0!f{{rM;a;SkXo7j`PGqJOVvGBAe^*G`VXVd*r@1q+Ws@ ziOevK9L>cmyQA8BFszLK-jzg0HBfs4C<%$@U0_Vyz34Sjbly3;BwRS!3*#x)L8-p;Uip*b^G^ zR{c-T;+Xcz7*|nQDsLE6B6{#ag%Z(&5tR zl6Dsyx!6{n*i?Sn>9`w(4Kt2`zO^}JZ%s$NMqZ5!gQS`GUj+{r*)|>o$k)nPWle`7 z`)u#C73wYCR2>+3iP3(RnTn#3B0+ws1MQE|{!;rRdexGeRT3!!(kW~<9PL;w`<_Os zjz(Zp4hKB>c&!6*?Le-_)ZA4lG-fL{^U#y`m>6oJ!ofy=V1|6!Rs_~WN(8CkbzoYXf?#3orlI}#0UWN@zIx)}Iq17k;{&gu1xHq`$ZaP&MH zG^kF%cCXy)!B8j{uJwaLEiH9Yd%eZmdbh*gqm2v2zt~<+;6!!yFYnuet*C%@M;fQ4 zddel8`EoNZw<}?cQNAHQ#KDs{JfMA%@I56UAjEtyLfEQ^#6ino7Ko=)qx0lQH#kxU1|t?G9R^YbvJI9S z?q7JoN!qw6*0jvOdU@ysgnM?OO0SU8WZN%0R^!Aj!*0-btySadiZ-|BbvxSC-~aJ8 zy2|xP^9`hS1ucjMzmFE596$u3p4NM`7^>!`P`phWj00o)yY&ps{b0tDT3Mk))3k|J zTX{pxw3eos5m|8}8eaa#apKI5X7?nNZ1V z)b0ISY@yljhibJ=#lxDK%eE`GNrA8YJj>s59{7F3-!sIYkr~9~2(=k{r0vD4AUVbS zh~8u?=Jm-u)ImSGQxIa-G3(5R>D$6PY2*)_fY~_*8n_~)zonVevQzjjRyoT**$#2aka7%u7!pVRJ7jKPE z`ee>*h2D2GNT0h4k?cm=QSxaXNxV;Z?PPhb8c)#_nP17&Azn3Ii^)-9k)&zoVyO(x zfFd-j;$yGhh?CgR$0~n7Ui~D<5%okUE7IK@e5XLrR~?q#O3t-3=g&$cLr;^zoWC16 zm|6&AZfXa{Xu2M?h4To{Juv!7q-GDEd7el5+U%=awIQpcHycn!%d&%xN}UjMT4Pg;-3>Ai%o`$SQ0;y{v!i zN)-?r@*oOUwJwEv4!YRY!mH}iDJzqwPMBCpg_cFTVD5~_4e$~R4s!0}9lr_pL}^t_ zd!~o&d*1u``^0XmX_ntM1+1BdHZ#e!^LN-r`#D`7{mPxa2p7b?ZqUZC&n$corRcCy zT=uNmuNF{Zh^{yYwylq8M%_zk6(ty9yL394?@FUP842_^g3eEsRNQtr_{V$PUzSa2 ze~;*LN@4}Jme--CY;QtqKstnRHF;}?c5`UyQ<(X*n|9M5$;DKsp2xe7kYCaHAPDfc ziDCBscazwQp6Z(MHtgZ3D4ONRfs44ZjjH+RclE%|X`((-=NP5kJzGj+=J*fFs{eUqkxmXZ z?Yc#5uuDMUliW$Z?i7@xYm)-&blPQYa#4vyUwiQkk`?AVUX<6$M2kqkO7TVB0J74` zm^tWnY}A&Rf_d~OJAT%I-uc`5r$F^LIB|}u>WmyA1>YxB1H-rd)nZhUHX9wEqDk+w z1;`-Lmd8XA@|PliZ@TjRRjw%Ff={@X;)g)gQ=lUSKW)TgphJ1$4-@tRj>*|@V%(7Z zR@qcXbs+h;_)};$=%mEi2y&pW{vG@sD`Rh4OXn}V1>(~*X4WwoEFxCbF0Bs<6N)-EtpAG%vRE#{xu4bZSXB zn@&-+a;!Dt-8f@)PB_$-p$h-rRCZTtCyoUgk^8$!ni$~S*r3lspMTD2zu0wp{~;TT z+CDg|$Zl&M`ER|W25%3gk|Epm**>y&szjbR$Qf^Jr@j_wVTHd-{T&ZQ{5mb4%hZqq?o8))c-{GPOuGii=9^hTFN!GZ}hEj#V8jdcZ@aZ1J@iZ1G;zeBG zAy|RGP!{%lQTNE(rmeEf*WNZimF78x(?_Uj&(PmWf%rgNyd1bZrkEN|+x%~Rj|=Z2@rf0Or=P7VsXj%=sZ4X?Is-Tx%1NfoHiGC0@k@7qYkX_U2N z%ekswM|mAC`580+X;awkPZ8M`<7@J{ua&1Q$UrqN<&;-xR-EM28*(Rs2 z&wI@vGhQxkSS9IB2T@X2L}7E@b%OoIL=WgOG9xfct@G&&$Ru$Bd2ou)KiSQL4d##o^9BZd z?Ds)%xzv*L?i@?S$@>yrf$W5*`@(Mv@CjOFt4ZZ<%Z@T^Rjd=ki`r6V6yD6zt_*`^ ztF?DBDsV;jnaf#C>Ay|dg}~~epZ*IfarB7V&=HX>R;=OBF1Dd+BR3rhOxGyNQRn%8 zay0fJ**&Y=vlv8|TH6sTOc}RYUXj zODDoZ#hZVY6$${T!W-r9Yj+8s22K+6?hQ*{nBNEGyB7}%yvs|y-_XV^_k5$iu;`PC zA_E3hy+wUcyG0attS|*^R88>%gc$2)=7_P3U2PXHc?v@l@#l;O%%C9SKQF<$IyMxj zJ}+%L7m)WP)%EO@#Em#S!42JANjR2~Le`N}=Gej(&A}jEkEkwqOmMRg+ba-nN77S< zw;ra#)1ZR?T`^{%-A9xHN5G!U&Y+CorH}lVfz&>N>HN1hHV4R$CLzjAjdi>#+1-Mr zJ-Fl8Z{D9LzeMwYv@Rj}($dXDGe-)>dfkdN;Nu(rKcdjHuKFKt8- z(L`k+!IHs`$y`rcnT%7XF5fYDw!hqZwM`+~C1NNiQm3(Q%ON8xnvA%Z-^9(5lXv{A zLtj}=PbG_eSmlA~$EuJfIJ={Ep^fm!1S*Nxo6`faV4wBH%rktS|F8gpDC2ag;ds%| zWtqsDU zOAN9)kFR}yVyQh;22&$z4HAlU1aaunkvwi+Ac$AiqW4}1sHtgWF^I+Bn^QyCiDL52QK#fL$u zRDcu0Tx*cK=zX)AWQzi?0RVbLUfZtQmaYh-vxJO5@X#R!wMQSRBB;wtey@K0V9>7q z-KrEb5GWpK1y(x-3l_Jb9<0u7&&*B@e)Q?5iXlyderH%Oa3-nEcut zA5!{ug7NzDltInoOU$2vFGX{$#jTwGJOX^Fw-QKkj}(AlKPr`}kYDziMTx_kzVwyv zan(!}@>5ZtQcVnV>Hq?6h^jYP{PIxRU@f2D zc-e*Oo`&GNEuh3uUF31)uRfe>F!Pbr9QMqibvySyEH{&HV0cPWTg&hArIq+jQ`sBqmu-ASXtgVGqEo*xHpwQknfy-Ym~-AK^)Zyj zb#Z_kW&*D(dsd)fxUBt4{37iu4EYXHXds3&ZIRDq^@RB(8=6KO z$BJ!1V=+uG=8+6)|0c|wmh&(UM$dfqG;%hm203D-za$iC;xS|B^ET>febC?C$0 zk)#^pUY9V;6j9CMJJY#b+}}U38XscaK40-x|Ith|jpMFRrV*$s`*lu4AkB}a=R*#} zXzKIxcT#!#AuXXEFR($Pg+GfwJiL%X*hDFQhxGVH zd@|K}4=xir%r$Cz9v&XrN}F%Gz{;hb?*r!#e8_0-1%){>euuomZM zN_%gv-{ zVQ+(On)9!*8|d+FLak$D3c0Z4WP#oZLVaVFTwL<0x+lC#jIgP|Ry~E1`4%8a69ZyL zkPB1@eC?c|b3QlWU}gt121iA@oOXdMiz>&($;eqC;|OSE8WE9>Y*bkBzS@_HuE4@% z{w^UOj2rq>HPqp`N@z#AQiQk_O%L;s-woJ$=E&~S#lJU<=xZB0SzTg2N`ks}z6;Bj z)qZ&)hKVbpmUp?}kDK*rYVdIDR-;HuLN1=Q!XvSXgPuS=zV(U)cjV!y!#!Azp~gy(ONPFv_ZsK=OYdUi3nr#$@-I_L2o47R&;VZeCkGmN zF&zjw1K>HwwBgXDi$qW8_=7J9Rw+090*%a!#8Qf_CA4n{;)zT}+KN zZHf>GM5f@;+nygY)-JnGfZ|B{K@)4t7D5w4%^ z&(_fEqY7r9<(4PqzqBf4kBd@zm(w+$SOWKxzM zm`aV#Ugd9)oS34)?$=Fp9Q^XpC?dHjp7Krx*p4c6^ZNvfMSYWp2=0B_c~Tfb=@smL)sgiqj)a<`MQ>LF5|dVgbm1wqwUsb z>J^UEy%8#*Gpw=y-i`TBd%Nrq|A2`8b4H?_NY33%7JvHB=M`LPj#>1dy-fIMEq|%t zZj!)}x5H}=1zYTI7uu->jv*}OnPs0%11ZgG3pY*X?{+c0(-mpmUkZCK5;uhfQqKS{ zT&~0yibuUHO&$;c%@W@Lmor_%QP+WEWoA_(h-SZuqtbL>O4H-1uD z@%qo+(K|j!l7~n4%aZYRRTx*7WxMcE`bOCH5bE5##3b(Va=%dE ziCV*_=V~ANhP;cC=Al*vg?23)7G*qyY)23#zpQwE(XXE`6<4WO0xKcvQ~2%q0a7oq zd~09Z&QvQiZMC40rgZs$op`zXv4#0yLAi{QI=Fi>@cTA9$fREtLNiMO&BU#S6@Zw9 z9!~qE(k7k!Z3Pc>QGA)2urzPz5FlH@{;P>XIU110XS4&wTkrhQCCAreLiJ1#r@`vQBylL>0_MUZ8MNkcz_ z2zFGusV~Vd@T3KjgUkoLm;}!7ckSq1uls>HJ+S(2pJ|LHBS6wgOW%h3hg+V$pc1Nw z)c1RBhIsWJ5G#*}=`FAA66|6$gIm?JL119a>&L;KfV#*ku&Pn3uOqkGcy-Vd%rcmd z`qoZ!7^8{P=C+?KuL5xe^L0(Cai;%hl*ef1ihY0l__6#x3%p^YFzzzeJ9h8kG=0xhuk=!2 zm+-27w!&yfcYleQ5GJ8Mqw?yN#RK-l68vn5sYm)AD?laF6|F<%jrd2Sop%qb0$KlW ztN|oi4|%l7IA%>1P{ch-;hppy8K{v`g`rU_1{q5l1ua5~w^u$kW=cIFp>L}zI1&2pG0#X2)$g7;S|2u+4h?vl#os)@A!*rp1oTm8){X_2y zJU)(<;X+M|#TbF`iBz{m6o2W9V1`fI@6W2U8EwA03 zZh`qkhJ{EvGup9#qs-)ULe;yp@mkLKitjj-K(=X`mfxvLY?B{FN7iEM>{E_$k~JP?qpd)mrQ){z%>x z6|dj(ZrmViYTTC8w^TMIu08iyqqqrustgRGWmgS({@6z1#s%Ny2%j=tZ-2;WI5KX> zMTGGJB|V|k=-h;%`s#D7Q-^3}dPU*-s4u6+=FXc$);gb{3Uv3ztY-`Hjq(b;Y5>my zP_v88g6i`shXWC0q7I0^WP+a6nkT%vfBrt44(LhNvt~m{^5vGPs^J}L#p%8AhDQIc z468(sw?WxCL=<2PUm^CEM$a|lbeH2&a5wuKJWk`tqrY5bF+5hnbQL2<&jt!!`#hJL z+X7$*!BZ1pA@U6sDySp~b)aSxBT}|dI$sQ;l;3PDi!pZ6a!&2VH58kyoGWuM@rG`6 zlMTH#Y{NuxjZVmVfH*wE(7uMUcav-#=yoMzdCD|!7$P^Y-Rup{_7$-4v?^|o{v>Ka zEAG$V(GZ&E9KA01q8&dc&ilFY1K5DdQhB~UZG0yPg1=Pr0}+Yb;}ozC-ZFbXy3{t- z$LToB4zr&a5CZ`fgKrsr;gh*5!f$AbkmZFI!!vPM;$<)IQp{Hzoc^67vsmOX+FBH^ z(+jR8$w0fto)-Sx)f1rj---5;h9B#&=WCRt9!WH5gGuAwwB}g0$s+Hl*r-D(<-mqC z?$lRR=R1kJxsy(u6vIiGpaKqrsP`TQ>_owdU4 zQUdDqR=j_R&&tPccT)`h4}sUG=1>Pvj*Y%6rES~}rZv?atPlqA5X(fj7}E^l!t zoocgoo%2?=SUiqk2m00Xy_0E^(*Srq7F#d=A?Eu|6RQa}draEn7uf1LS9UB~ltM%`N7ZA}4Y;p0IL37_LVI6g+>c39q*Ix#v`%uR`r$^iU`l->RA{z| z|H-dfdDZb4;ug!(H1#4Fl^g&mJG(w<i4>3n|8rw7NfaU#Bg<$@aug-1fT%xaxa?}VVFx$EwM3v(r#PC&XJq406jrB?!f za?X1b!gGVhK@m^@)CDC*RQO9~RwlSTHw)J~%QF_CrqHoLBItHD+WS~3uW&THy_au>ODihng(|3hzl)?Zas)vRb^W9f}f z638PeW_4u|!rZeC^|2p#X*0vk2}UEPIKh{8a9RH|AtLKC_0@N024xA!W~vDU|6rUO z&SWyL`;Ac|gONOM;H-L1qiP!$Y&aJy7^<$5%c-5U!qk%~8Q|44N z&zh^q8cOFRci`@6kM2o~ICh=k(`R_Wi~syS{tI!m>&y7ymM3++F9 z!b!KU4e1Z#tY#>E$dnJby?!PMTvly*4hV#fT<+#p(zJEQHnsbAxlD&UOc#AUr&C3L z{*jVz%vYK0J_nUSmRhpIP!~#jJMnC4~2R+aM1jX@sZ#>!|gwBCMo{ zwDpyGMm;1VF`~Uj{LrF1e%tde^X&F(CpIE$^9!%uqnAA+#VRtC>fiRzc%GSBG?DW< zO4EO@Hc(g+NDk&t2>5DGqe$iZo5bOza`sTFy!!RkM&@A2Ys)ctKsjMb%s8WfEyYoZ zea-{UWqz9(*K;i9Z)eRF*3)B?x|b71kxuY))PY%9|4664r77a}csN~WvkjHA**#Ex z{iWyIxYg->MDN!yISt2Xd99Go0vmamWIahc4@sd_qW>JbsDcogB%p;qss2rEV5L7i z;XQ3##pwBR+io_SFID8nLlGYF-m^$t^x9A&HfMCyBz{%SbS3hIDTq?v0Asa?PyvfX zf-#;F(Da!qivg$B(~D-H>DldRk zxy_wto~lc5F@S*o#s+uz`0;%b!9?e3UH_3veHKD$eownTsgi4nP*}+E!-gNM!(sCK znch^*E_{f^S%(3E*}48*VEJ;p0#>QIQCw>8l-7RioQGK@4JY5%o|ZScE{CnrhGMi; z@3@Q1!rxcFEeu~L%GS(~JA)SY0kf7HYI!P*;cMsFB^aQ(_XqwXm0N`5Ziyc}&lH)@*t32L7k^3n>U z(LzG+05hod!T7=ZD=hT*2%Gob5yfW^96zWk% zv3#!TLd7yodcD~HTijS10Us-3#!7u*C!I_pHifCGk@Qgfcw@5vF9*JK^1TcGq|6Fa z(iA)LB+efGyx%0w74zQ7kpVv?Ta%_;(dNp^dcUk|Rf=oB(Zfyip@@2TyO=NlV0lDh zz)+ag5m~}k*F&fG|Il=nQBl5M7ghu*rAs;l6o&4S8bVT~hHjDW?jBk|N-064aX`8| zrMr>t4(a#i_h0WPKe!eySaZIkv}x>c;Z)w*anh#_ah*zU$@0;cu(_ z`XHXLi)Bqu1tQf(bhmgG(bjLkGugeD6bPf91g6xrSQ$#t>HJ_e`ykaOzbr%LJ$BjL z%Yhu*bUlASF0PtI>!b25gm7OqXtEM1Q@81Qzb>8nr?4B+)8@7>8co-a$`#R!ooE$Q zD6QNAUcQM~$KS)$c9QH~alxUS4pK`1JICegWZZ?2f^E`jnj z2L8F#(-*bpu?lZ)&~{{I1DrKrYxRFa$l`kOh}^}6L$GJ?p(dM89%iP+qBF*X@4{Hm z3||(Z45V=7s=qcHOfRNZDq}GM{Sz7lM48nTML3<>KSC|N!B^?N%I_vm}_VT$G84)}jT;+c!X>!dCxf-AvZtcRK$+cz({apdWQv(<;PsNIZpb zR2*UE*o5I`O%vd#z_=u`3^(G)Qd8!!v0}^3CqQg6k1lJmj#pbqwNkzO`~)dmdb;=k zG%$o^n|6y&B)>#|DMkI8&iI4oTUK^Gqe{R;s)>L0T@gvoiYA;m-^L8@ytd7ByJ~CA z>)nazO6F|YEYTZyR~gNcqSP}hyn#!Wg;^lgu7&)Ht+XhcXK)-&KjfTIRcb=1*Xq_h zM>d9q29oe4xP9u;^9JaF1}Kk19G-d>hR48YKGVVT6IC zZMCDrRA0I11n(E0S|*C0ZZ%s@W=pfeLwNexTN>3MQX&Ccu zJR>_8A@<(lB+jPxXt@aQWWu=;?gkQ*RHNw~8-C(7ZIPp~8|sp_O2eLpZt^HB>wdvh zS9gwJLd2u5d)S0tg}f4;_?xo>mKw^P93zy5;U_R^Rkbzm%pfUjoa?&zw~T44=W?4D zpwM8R5zVrGy+dbz~*xW8~;3%%Qts#ciV&tB=I+uX9p}ns%0JUFr}PJ=U(LRysb@5~5enJz6*7RFej+2qr}>NatM0QM{t z9{t+-_4tf|4`jWeNxeP-@!fMoP5EgrE%wHUs;-lJwm)ypIAl<*FXBw5w3{>BtM-x& z9_usN|Mb8@X7U9!?G$~QI9q#w2A?mTGh{1r*!*N5Vhxv08n$Bit#bK@{24RbeD%(2 z9V}r%m)%#hmZ!%*p0&&Y9d!RF#$yvI!|wtz%+i4*_ZuEMGMx&0uXslvW{o0?w%Wgd z=T>f1HFa2d02*QH0QZ)e{CfVMn?y^(XMUS0%amjE-7r*!wTHbS zqE*ptpm%7bGF0OKi>%DO`2-ji_K!k8jW!JZ29S^fFl$t_f*@42Ycep$7Tf*vN1?Jo zHM9q4{7`yu_!RQ~-(lF<=fjD0fUlS%$8(Wl7H+@PM%JunI{uvaBC6MnE&J%$Gd( zlkEtec|lrX*^=(EpSrgzRB8okg&5OY-#@sx9g$+cWt9$53VOtUe|PasEr#;jSh;x* z2Vf-NO&jJl!dRZ|6_!JBI26RthmdT=m>Ghm!;NJm(kX%v@BWm9HvdUE=O~fEjy9$I zblQ#PABMkYR9jXS7x{yX%v|{K!A8dHucWDJhs)bGY|B-Vd0PSgyMK6H$2PZDF8;^1 zW}8ibiM{vF++Slbys?FW-A1Q^B!+9F)NzA$x*~ohBLyc0IZW(R&bp7*k82Q=uW&Z< zpn((T3!?ThPa_(Q@cL4YgITg~3F5bVu|#1;EKoBhSK6#xsQXrtQ-{!`iTUK%!b*pk zkc|h&HWA0vGTIJ@&I!)-j$ULyL=UHV*%}Xr5-+7_8y^_9=k2>FqANsTi=KajiOFf` zAT|s;2KHE4SmnxA%0-S=qk>EyMUOduzdb{n*#!I7fdIuU9)jQ{T8#@5MGZSD!FIac`~>^Up6qo2FfH=GmDCAl z$q3wiCyL05z-~9i34G8;qfJ{(PM%|CJY;Sq?$Kk-dAE6|g+pEouQ%d&Mo|b=s#l6$ z?{djRx(|t?jvsf$a2R)fD$e?f9J-HYHObuBXT{$HO$=Kl4Cp%<5E``bpCiDY;X|&g zJ~wP3nDiMtz7@T{R30^daS`gQ4i}gY(0~zykg!fDe&v3Bo_nIpI03#G=~8#5Suo=S zqlOW|hg2u-@UQlJKbF~n3#y@xPOO6`$OmcianXg*Ktd$uc7q3}sVoCzJq@9d*83Yx zhuCw!CU+FoAQie8`R4IHcutfV$ezhR%pfS@kJ$d$))lL!n~ujyOBF|t46b?#5v$2?n6!0 zvpweWRg6DF0(4@rHUmS%s;x&eFHq_)hsi(omto?W`}2uz3p~qt#a1;Kb$x*y;5W%^ z@SJwGYVmHb_RZxgp1ZV4!S;lra(#ruHsmmT)jt|pLdIZK_P{2zG?;`Ne&JXu{7+;G zSz=a;#>vkBtF^`{?JbF!$K})aegiEtlg2S3KZU*!imEi)G@Pk-X?;}ZeD4>QjQsVd zO9Dog>N01AeiKW(_^JUO%0U2Qn*07?7JiYlHf8PyTLvAUPE$ELHD9zh0B=jhk2Y=4 zfitlMSQmlT)A7s?J>g@eV>z=%0@~K3nw&>iD&H8`G**uPG9u$9>AIviu+au=0nG|T3DVTstQs_U7qvzU2RC)T9NH&?G= zS~A`pf`7~r2CpZ!ig+Y<*IhmlgU#UAR}oK_Vo97sJ7wl{e_#uBX4Prptoj#Q^V-YrU~r^p#2It331#8uKtGnDDyQ1rIG zwN$1(TY%o+cQf;^7uB#=xfLq>VZN89QCihQ!i+ZK1+p%t{Wq-@^DfMI-wQcem#=Od z9Vr7n89Y!f<{p7`q(*XHr0t#<2~dBqItp2DvTG*r z9v#fTB_2Oze~rFT!C-h;`SoOz@)7Yr;og<=K?d2LfAmNChW zVs5d_sP<@;WtCyBG_?MvMppMWq|Az!q10L%R=1(7UaU9NPsxPc5GUe3hH7};)a?p5 zQ)=q!TE9``lK{p+ZMtm6q90f_vgW_DQs+fGEg#IM0rJZjxW zjwBoBkrn_bsCz3k+n5xZmb}s-_~jwK7IijrMUm(Z{&s6Lm-!3viD@<10lY2$Z4yIN zA0o2ZqQ&>aW)dWY-{M!GaDqYpm1kx?!6+I(ucdWq4Qy+P-{$Bc;8KiB9a+Rxe#8+i zjhE|W%9RhLn)*b?wRD~QqF{a42sF+!TYVg0{l|EPB`63*e~kVrTEaO-_!v$q>Qi6( z4{;cNB{|zEBFTwarjkM?_9Z%IfR@Dt8?HCm`Ha!7*~$B*U4gAQJ5^i#pO*_A-s9%E zY3pcS!;JKTl~P>E7}uLMH_8tm_rVgG`b~J#Q8|t9H-%&Ue73P0K|M)cSEr=Y96vq~ z5x=g6~4q>EKI+@wk`{3Ncu zcj-0GD-ZEpI zs}b8&mO(P4WjBIW)Z=q(RNF2Zxhr>jT5!=&qLcc+t5UZL+zB1k6q2YPF4mbhPu6p3 z-hNqxyw;m?oM35aU*cxMtQ-FdJrP}zZ78l^o5_0ql+yvU@AWX)KEDV43d@*pGuga$ z-6Ej*gUT9R!K&P!CEblC;+#O5n6SK=X$?F)n^q}aK(H>@O#D;JDK49XUnch1!lTD_ zyUY2AaoMauXw*4DHgy`yxVLc8`_a@6vLu3%jF7>!ZZ|~%yOk%+b1Nsw!d=qt6bz=5 z$8Ff~HU-QeVgHV{v7~hmtI!-+6*bFOU-%cpeuAuzXuS$6CfxP=O>aM?hr<+zeJgLM zll{fEfc49@T_48%lsP_>+yTykiR1SMY!sa)vZ$x+0AxRSa%!Z{B=jZ62S z7I8QWo`~{L2k!<8u7s=&fH27gcA{(%vRlITZP2Ob+J4=0>I@%pnqPrcmVn*KCbXPU z@oV`w*@WXfR-&h(dN56*qYy+m0M55<&@;b>1?UCF-yA}NU(J_KVH)Yem1UyIx6oAL zaSv!I&@P--grjplu;JKcS=peL7F}z(gA><$OX^~+r8XJ6;vl#_3Px~Hvxc})kkbS} z?~NQDdOf$^EnG|b`L0Cqk@kg2iR7)2*;I=c-R&BuPJW|hKf^tm`EE`bfWqrjt-3#c zE6c|l4PZt+5i|6qdfH8r&5`a*GaN**DeH;a%68r@XSR61x{MFP!$sfvAiB@TrKiq96+%Rw~{MpfguB#d#i1FsX@#f7pY%BP|cHU^9%yNDsR9GvR1!_R# zh;VE^08GMLB3(wdg^ttx08BX(VQaA?n>(J^=A`ZkjUgtBr9w9D7hlc_)EUV}hJsU@ zcc*#Oaa9J>*UetgunGRAVnP!4Me5Ii;W#anNV6b%2w@G>*HGDjR zZ32IVX1(mk2y%NRG6j0j*GK8{SX6qZENz+MsyW(^j|`U3Ol*Vxhz-Y2kBlS=7cp(= zfXJcq!T@hx?GrcLI=jCT^WTg98G5RxwMD#c3i?s|4|CqOYTjiue9S-%QvxcQdTr`L zYY&p7DrC>>Q}kLbmEkMqwLUHY?80xhK4>_vKKDwO>Fj4VQi~}oG3w^=mjWt?Tx;2C zmQ+Ng7T;(`@LuRVmeCS$1J*R8Ei%x2=d~kO)~UlJf((UVsMPVEJ>E=W*SlQ9D8}|% zlR)#B!&)wZS(f7%Z$Y0Ugp(t=qV^c}d)m{KU}#cKSOE>L3+^&{ZfV6rsh=|xUoCI^ zOL+Iu3kwA4;voxV2lrOepFO5KGhIZ-uOen!kSxa(Q#k3boOkXU zy{~O<_wSHOnTU9+bya1fOC=dGLI_WwocE9w9#dp@rm@A z%>NX^N9tHW;MJfxNYI#wb(gfD|8f!QZ~&ug9g)){(Wb!+<1N(CFOJg{4PfDxEiUiL zsQC0g>S}h8$+Rk7v1}BY^Odv1$OClH-(3^;A?i{9rmVs73LO2gvTXz-@w4C?m& z?Af$E)jQA@Ka$r%u3TK08^AfLS}ZHhLE(SIIQZ3+3f1$pJ9>QdQKkWrgX|;BLJ(Sk zECmNo(nWXcX_o^Ze!T5?fx(40%;dc3ejJ}7ya8J|r}y=eVle3-I^N^C`xJ;3T?Xu> zYwzpn9ovBR5O;Jo|97DJ7l&^txDC9P6+EBt4kZXoJq$NC`p95~?R|-WoImpqfV);0 zENy%+-o{euu}eZwnH`*Zr&(+Cn|!ngepco@w>oT3Ol7bX3{fFJc!I^>Jk2^u@i7oG zo6cjII4iWaDA@9tHjutd!XBi9GAr{8jAv|AJp)?s+1dh^sb$4Hy@2>U_Fz4AsH6_? zGw3yGm@?_>4J8xXoO)I^feIi52J7rc9=bDG~qp|kB0hT+!oe8SRsADnh3FyfpI6@60jk`{0G zU1tR3iu{Nt`;+v2wt5oZCLf^Z9W~!Fue(3Q+1EcU>)}8}0akqmAF-*FWNW=7d=gpQ zG?ldnI8@uwLi_KMC3993tU2`V?Es(9JpOD&j{?{DNP=}9l&@j+*X^6Qy1x0EG!d zlcZKNicEtZGc1rrP`07{ifghs0zOnJcedZ~mdM%qgX@B3j)4dZA?hr?vhe$DxU1*+ zYQ(%M$uYSv+7#rlw~$5Sd!9WR!6Qo5c;=-H~cc^Lb$(Om2pY&WH{~h>#KM%X*A)p zFZ1xuX8&r`!9Qp5u}B;r@Nkg6rx&-(=+?45J8V_0nt!EN{J?-xfEVR3%+`K7A?LcPO+hhwa1Aw>RZtfL~|bQ9PSdsdGLppTG?kgN6bNwF(Yapz$v_W zCHm50^ND?jp(Hy%Uu>H!Ps^v0Sxnk>Su!_ch(edIzf%8Ls5w*;bkA_2k@Uvz>3`H<#fa8r-OtLT)}Esu_5BIZNBv<&z?xVaWQxJu0w zLZr-KAzjO@05Zq5V%_Pqmn}LLricGp*(y0x=eo8`tU9s-1t_fN@Z|T{Fn%+~WXWm@ zqOtF6p&c$&ZM3N%D_GM|(WH15&@8?eKb&L~X)nYcLV~Nr*OUhgJf!?~nb+Jq2jG;e z@OQ>)p zYXq)N1Gc9ETMMF@4Mz5otZ89F#{^?n<%JGQ@v%9#$UJ#kM|m)WZ} z_Nej8v_0Lla96)5Om=Pd>$jLfFjaWf=vK$;71Rh_N3dB0EfP8Z`}<9G^e3T3_h&iW z`JZ~~)BLi44)=>FZcwx-`utL9U#{;9W~`rLcaU@4H$GHt9aXq9(FEr~ZJ}nn#A92S z2EO?1l2;9$YOYRqq{_^3;#)!Gsct=k(`MMA*R+wjlKUi}5Jq4=VyB6lU2HmH^% zlz2lBHNpwFl}9M#dJn+WFTdjUWzi;(1;2$K$#%q5b!z*OG!*Rvda?b%f7Y9*wH(=$aC`VtOKIW&0(#8fl6dMoik-u3+aook>dZg2~>Pu;jS4%wj z%dwew-TSXn9%Jy$*-7@YwH=jlUj+=M@mM{!cJ#ejp{mFtN_7N-4U-jHT0xsj^+VSC z+zDO^Xv4W8PnF^BPFQSbZ1sl{#Zw=^j8FxRwAmG+hkoPB63%Wt*oNN+{Dmq%D0cjn zS_3M6MsNOrfULF#o7)R)nzw>j6Uh%*OKSqeE7*q*us~WU8Ukb8W$NiIKp+&(3VjEd zFBd>UW+32=_t&bj*@Im;=K2#?^1crhs^725_ipR`HPFJj2~Mm|p<~un^Xm|&AxecI z$L`NGGaM+b9fankPM9Ev3RbM z6SAQ`Q-Bzp%|PVc2DM!6Q74&mDBgKn799jWA~34c{ZdehJem2&3VX>EvC#fAa$kvt1$T{DAvZ_g!ZT-si z9qwcM#Vj#$RrW4R)n$&Eh#7JB?h^RpNr>}_EG3&TJX~kYvv-fH-p<=e<_H(&#m&DA zU>vyGuY*H^N)k8763X~(;&*-2XoMcw5HZ)Nl~wcK-q=*7ebCTVlj$sf%6Xn7ck`y@ zJhztw6TMq~*#+&5wV;`DkPK#kpEa|)yDl}5Vhy5T!?4b98p~O262EF}vI^YrpYJ0dM*;-Z_ zQo9l-A7nvZ3adRd`TG3o2gWn5s2JL=3qtrBfa$0AbnTjG%7ePQ_NcKq!sg}-jeU&^ z7o4Oj5vk!Ujm}u=ZZh4KTlxI3x4*3^Y~;8S%3B?Wad*0S14 z{~B?+?V^&aqz1?ZE}n%@bw5hXgo%SH!MZ6hUAIT^F zv*u*HQf@v<0J*87fQ0=tnx_)lu=?H4yT3{9&))tmzZOUaN}d)Q18EgPzdt?MNXgA- z$?}y|gG6o6O|>*v?#R&Y;W&a?r*9r`$<1gET1^_J(+P>5UJ7{6w!43qUX^WTHG(Ud7$;g)X&3#Ytm5{y@A0l0YTUz25 zDP|b0;4Y%AoG9$jf#>}W3?XYW!+Y}J5Gmp@(w>D<=vh3c6o#c#o&>GEnLjVw|7Tmf z#gOu7tB`1-n7UeCS{99iT+hqrxQB87N9rQt<4RX$@=UknYS`Y`dtX7+$GIdc3n$ntXuSD zZUNXvhXiWcLwYmS2XA@FWDZy_KN0+qOuYroYcKJ2Gg~Dx!>rStu~z&&gL9`P@Tx0`?iuzzt$;#bTIRc zGu8=8JP9(U>-mDG?Y#K_tjg=sVUOb*BV;;@{0Nzw_i+r%ol^{jUGw8IYcqM^ruaQ5 zeDbH-yKVz|B&Poh|2{ydbHoiRf{{9L#QP4Ur#pliGNY*+x@l&PE~EGs4(!9#6q6A}#b;13b7esx|dow(|$w!{F7xw%Dni2QFa{um=tDq|ovhqzf zpm5X^%J{`*IpqIm0d$N^IgJz(L;l&v@N8;;@9lvplhgwk|Ce4Ys%q>JO;zb1)wUC7 zPtx0UDfu@hv~8U)@2>Dy%)ZwG5n4mr=?`XH4RN$|fYf9%nV77FV_3N)q@7E%&l`vs zgf5@op89+QX8M8p{o<;HGE-*Du1frPYeCD2?njVKBSX6~EKF~3x!J51;-_zYGhO~7 z{Xc6*0%#uZD6`8_UC`jh@wd!}yktw15>NU5`29h-IS*KjEO4KL+sECvaXnA#xQ#AC z&R=tR=krw(6uvhMXSt$k1(qfGU+ldi0A-W~?4|`{$sB5B*@g+m92Y?Tm$K7liI^*( zz&&&vRO|U2#=@C2yPSHlA*vBB=KdR^REUF_O(w;!LnE!i+ruBuUuZ(99*=7o)8Ika z#~P4RhAHRGaq~fA->{6CphuyYW5SE!k$r6c`)14RHsc-~P*yNS>b3RZH%X@lE%th< znDApZmc#Y>a17cf_>Ah>eVXd(Mk489fjG}*i(-Dk{_^M~g?S-!C{1SDGDe8r|FNfr zJ9h|=GaR<#pEs_z>W|I)LaZ&E$#7#aV^TyLxNc1;?C-a zOzO_%eju0(12|Y$UzmqcPkMjdsIvM_9-CC#S#82R(A^$}FS3y$d|b>qSk;2AiiKo8+;QplI1?z8*#7s+iDo*GR^{q+2F(dwQIgP5TwJ^t zD^uX%q%-b{jv|Y(@+ONVKJRn`1U-Y2)cVhp>nzhZ-Reg(ZdulHkmu4So^#d1;oY3H zv!|phGr)6AcT6j4BaB8(YhW!*nLye4Q89BoVt&zTuKx5k54SzT3F;C5c`>KP& zct|Ey_tGNheska(vQz5n?msdfQe+(OcoaL&j(;$cgjMB0~QouLl7 zEBg=gcYXIPklILMQ4H*BXuKn8t5QIbi=wU2}8)_;=V2#18? z?!Br-n?D-m-hZ25A<9BAw9pv%yUDMV+n+$Pf1=!u0^C-;JJvbQ>mbw-)D2bwn(d~4 zA|U@FW>KY1!mdNzR&1>VI7Fj2pr3yAYtvo`*H)Wyg!48|dG#2*oX^(+p(!#lGo}^l zsjg1kYYIo!(~2;DKpy2f_nLLZ&1S?a=81%{E4^v}#ux3CQi@bg({OtqR*5}0Y(NPf z3m>Bc(GeMF(C&<|arBq|onh__X?K8hfOV-z&#g?QLU<2T@vTK^8>5AR{}z3o*^g$Q z*=o=_>u6bY{WeSxH;42D?x{JKF-_k4Y1`81BqLcE@ z35}5`EXuy7@0HPg(6TA@!}mM2{}>u*$5LGV9?C=~;@Qf*|I`Qa>?`*)PR-46f2{jS zfl6TiZ&~V4WZ;qP^#COs2j3i%g&5M5k$)3vYzC2cHP*3U~utEP6e zI2&BC`^VJ|ZE6NwbqdwjKBlXyf6S1-KbAMtqq>+=+1lchj#(-*%pjC`amJIbNP9 zbqU@}QN+?7Y{gLK#8Rg&Xe)#}%W}b*~trQQIBK1Oa&>W-4bAu036~r7Tz_}Zj zA$~?!o35dxaYSA}g7`ncJeB75jsM+(_6H_dJbvuJgfNXQ!Xikxg_8<4@Rdj2wRQy+ zQ>UA+a{A|W!lUgo(EF=)ysTZP_^J}^>2DP!0VzYv1rQ=Fl#U`5uvSH8GwLviw4&#o z4g3vUfs_&Y>0(4NjspTRh7o!&|8W8Lnmp)_P1tzQI2#N$>pZDT2x(|B!k&pkV>A(} zqOTpL5VazXDmF1RFZ)bUgl}B;7n8MGZ1(8`ya6Rb`d{=O zTqp%>OM%2E&a=J|I9;WA%vn!M@f=b|D-F>xCY+=)e>DDE#@u*-15q?3lA8-m=j5rGj^oMafR2t8Gzx&TsrSjXqL>XzEHADpfZt6yNa zJ%W7+a)!bL#{_il5y;`z&A3GB(Zi_CcrQBNX__pbK7P1|rS1Rgk-*QzmdsFpwGW03 zby_W#nfT&AwJfdHf1SJ7i9bsb43Hx^XE}mXM=civzzE(AOhxaYCps7s;u{Nb~14y=v!US0RKgn@cls8l+{;0;D$6=EM1^Dl6tIT!skWYv2AzO(`xnGB{+XdlOj z zTgyRRXW^wpjwn@92UGk8-Y;(XM*0%l_yauA(sJZr^d01QiHr94f$q&pLVK$w&nFt~ zZ;jqOkzY$r+Sd78;{9jbS-s5oB+8!aJt{9^l_erHGS`n7BF(si=x>1YroEZ)W}SI4 z5|Z;V@+dDUuR|{qJCAs`=x9tuS-`&A_?avC{V}O{m%&k@E{cXDm(mGWm z0PF3(*OCuv9k<_b#iZWOA7wj{r3Kusp+-Av13p^2i8F7vJHPYG2uo4XRLsKaFhr8n zm*_Q*G*b(hkj#C^4yh2MfznG_qm4J@j<5cBn#6{hEyV3N)P7TAYO==;5lFBx3kF6K zA;)VVC=3qV;$A)af{6tgZY-U}tM1FP-{XJrqfvnHjd1A05YsA``4)e7;KSr4L{}?Y z&#P1m96gFZ4sb7y22~X0uaXgk$B0xyZ|^SX2YlywoAcZI9A&9zk(V`>63&vR2k=Jw zRFEhHr&tbIwc0C*Itz&FK04@|mDG7(uq0l5c9CB+gz%k^Ly#hf#$mID8v|0ujZ+J# zrUlSqZ93hP8~2_#xNsL+z2%RR^@P?zQL_Rf5;pXIl1rBxg$c!Jt;^@JGV4Eo9pwf^ z<}9HHiJKMu1|CRL1msDCz=tq%UxS2~?=64Yd^|3D#c9(0x%|X{2^He}a~b-IWKt{R zb-&x)aTW`be0Er^L}@eG8jZvR8cD}Wa<@I~V@QX%?v%+41kw=hCPfy*x^Hp={qe$| zzSl&jQEMrR6X$&sThaUb4B!~yWyPJ6itn58g?;?3iKOszzPc@ZiN&zbdl)I=RY$4C zZggMFB5>$wsu$-;^$}bzvSHT4RiJUvb6c{@-rzt%X2S2SZ%nwpwQe>6zhRU$tk zX**_Km;LD>*QHN1BxUm5pdMnfT8(yy$BGY0u&LXtv~M$BfYA1KgO#`3(9oPaolCPr zVH}CA$h5048(K8kwLV)O0p+H^cqB=wlneK31xxe0ikF=P1W4cbCc95~PAa_qa1e_3FrCDrB3I51%uUQx zEheMVL+WYkk3#0lj&qVm${hYygzqN(ySr|f&@tZlE*ZLBLb2eQAz};q1C!OwGQ1oj zgoz787(fl)xyx0b53_0brC6p0TbxVrW;V2z@Qd^v!Sclx71&Z{-4ya0L#tmSBQ95Cxk%h2Srojj1r z(n^feWc0$fp1j<|?2AS$h43bp#1kLNRkEnj^2%R-4eO798qFy<%~sCShi=-@;uQNztk>4VZoq>j40zPul3hI=FvIrFStJP zPq4-d4&#w{=hnkDHb-w+bLpoesR9TEU=)FCT%yH%Kq>kjEhQydr2dgy;whe=n)2+F zFA$Bcd&v?Xi~d`BvgVpGTcpXgK^lZI0;OZdjXB`+HK3W=2-3`G(8qrG3d!wR18 z4ymd@dT9KblIxVhjkGU-Xs`)QblgT#ouI(Z11s!m|KO4U_a4u~S>j@1rH3*IWdU3C zW~lT$dAFwW5MlAWu%P;*!(YTTTG1ZkvQOgL5aQUvG5JaJh)|Ja>lJ;cVHLobluayG z38iK<0aLdt$lqPR$POZue|S3=VCa}N`GUI!e)EQS_K4d2-(xR4Ss`H&nOZ#o47tyD zX2@K&5Sj$hQG=%xw@mwK3RoBH2&?_jYtV>M%x-%$`uEM=fY5=$W%~SJJu(gJ7^6!< z3s<0<5@ox`Ao3Cf=UeWEp7}7v) zF}%Q?6J=ON&>^{mlnQ*otjpB_Z2-}PwsWYp6p)K$$^XRFm!EZ!ShNWG_A#8tg$~A& zu6Lgr1vjMEZWjoajZ#8(P5IcSWuM1jpgw1cwXaew)}0k|d0xbrc)OEF;i_> zfLoRt7bz!4s!TpyV6{LE<|-wFbcch`{Ht3^ODhzOXt|ZSf5f(tIGTFvui(ho#px^;) zkBv3H%AoNY9l)T(KDEbrX*he}L~*5Obs2t#C6TO3URg!hXSWTS6nVqb@WY6+%u6u} z+dt^*UD)I3iGOySG7O~;=pcC-&<2W84luHq;G&r0;&YE^3VRIGm#ZW*nAH!*fqL}0 zd~-gH7`_gieR^h=hv$2gh}%Y@(XG$_*8om(N{g`rMm$!3eL$NTKV}PfU8xw! zUWd(CfgU0DRvZoc6b+6Tje~F9G`vx z`foy#a1-kfv2uiMx3jq`=89`9Xmf+lmH#mk4 zkaN#hA8&b;_HNsi>WYSaAr5f;qiJe}PuOt4p-TtT`3)2|XqIzprfnjH$?Ergst;NS zR9yD`(2<5*CH^+_rv&nI1GE<=#8REoF&ieJ$uRz_sBdHI6p1s#c};qf)JfMpFrvPE zmxoNs#~SC^gQtn6UGQ9*NT5jNX8u5Kbs@JzSfiK? z^O7|DeYnSQQAu=t+xJg=PK3SB7@-Ci^ZnLjS#rSB5g~SczFrwJ`OeDLCw4$jn9AFP zh>M1p;PB%wRt%n;{%>^0O0#oVhS>$m8FO#r&S(1O4!#d0LO3==GIpGD)9Q6zE%k4Z zV*5^qJTELGTF%Q3?--KUzaq6F&Z^fb<$z<4hp41x8`9* zdOH2o3R+w)wUoz_4dW^zWSyq^Revl8)oYhRSeMo( z=+D;^#FoLMx9rxwLO_AnJ^XRgj#tI2e=NbD?>6Tz6-3tJa`Gvu`l*!Dwv~4Hjyxn) z3V~9Jc1{N%a`qY9R>Xyc*toOqtsh69c!#68r8X)d<7(N|dbMF|?}R4{)2h#U^&fAK z7t)mtQRiFRuc1`6`AnCYHm-NXbg!b+CKghGGX-ub6v|B^F2p^aFsly<67&YARi&=) z2lic2i$+__X`dV&O5nZarnr4kF9u^^?<3(uOaY}&9O)=j(x@uE8o}i;wYcQA`{hg^ z%uU&1fi7__)$(8#RH%bf(Vj&gU4Zxw2{))v*>eZqKM3A5Ct;38V6eGF?-w=R5f74# z1(X12`}U{ao^O@Er3H5Rlh$~4nMp7s~(*74F#aL|PfJQ-5{-c$>s6m&mp#vh_6 zYxuvE33nQ1PM~%foVY!IhpNg9i;M%&O@0z`Qm}bhpVGMT#k1cbM1LrYUDnS zZ&oP+RW0tXgG?w0F_+;?;q#?yyxcMQp5)li*h*yBtU-h+1J$uq=p9oB+qtQ5m&;NF zTch1R6%=~#`nS*`NfTfb)M}kD;vQQJk_ovKu}nu&wmC<7F>rMy=~B|t&P6RvM-B93 z*;xzNKOY~n!*rzJx{7^-{#ooD8bn`@0*oW_s|AI;GSMQHv?D+JJ7w+d_x2p-XE#3X zTto}meBSs$7~99TNbY{M#S)8xSs+k?ob#8RN|WAhJO7(WiD@J2WYJ$Q=0d`_uCRHZ zI&bF7Bz@LD#j9;PYb2A3w=&t%WVGYsyP~Z+O@`gq?P2Wm)|740f(5j>9eD)>v1UKk zZ_YMjXK&Up25v6L(>->^3)wYS3$T)HmXUd_3&NHbh9@TLPw_U`!|+YJGx6+H5yu~B zZByYGX~j{7QT8B~CD*PKEhPvjF%A=NSe_eoL!rt+xBm&n&?#+{2Ux;#c2n8|`P1XPy)tfTl!; z9aQydCvCHzaSVfRQy)P@wkz#IP}FdR7%1OAl{CUsFSmx##7m*c>D>rwf6V@c`@YKZ3Nv9g2`$A}?>3-!=>N*2 zef{Y4)&Tqhm^VZ#14BEEYiE!d6p}NefvgU|E%M0Oa_s?>_X_TrQOaMr-vSqk?T(iu zR0m@=y{^mUW&m1u4@9FeU=bcWUh5<56u#+#q4v3Vqe5)5p0|W+muU~0g@2nBjJtOt_p1CIz(O==ZI-&Gr*TtPRN(*OE)wh3xHhG zoM)!8G*yB;z|XOnUBpfaQ!4`izp33}4fB=lA7guK>1*5FNc_E~BdQCDG8~}0 zHk7XYxi~O*CVw)_3%~yd?{?amdo*ZqnEI5++HrjGuSd%*`~K)ri?B51YS>!HeA=~h zL;0dk$YJC0n7)Z^T`iXzvL$=-~A6D@KY3$JS@Dzc@H`&oE|D*mPiY z#(L0&Y^ceC+R9CrtNWLwZ}IuLk-1W3j>@sZ8m#J``eE|@AeKO6H$b&y|rr{N&RDQcsWK+2vuk_=oEMa*@72+r&9=u#h1ey;d- zOj#wduX~oVGhYYWcih>*9!Wy>qP1H~NGMbJ=Kiei{;qbrtj}Tgm(wncW>EHwIJ}#C z?CR{fcNT8?Cj;v$&EMf#FKnjEWQ%oPGsJJnQX3!xY>r%-eTe&A*{qG6_mR{S_mbTT z2n8aGUFf9&{byGsJxg=<|F%5|*+<&9oi@x=ECH}xb))A3SMm42i0=ShD*8QetP_D# zB!T>;I9Nb{Nid!sXs{15Y&p8v+-o6ux8;W>qy(7d`M=S?|29XkJU)2%ePfBFctk-x+OPf=<*w}r!bd4oM5O##o z2<^AB<~aAF1~(D4?HFI~2t4U%h}uRf{%&x?K@qcRt6k9MF3GbQ*jdu`G4eH)dt|L-g1nM4cfmPyA{AkFmFas&ZSy zg#{N#N{5tmhje#?fOI!VcXta&H%NDvl9JL$w}gb0bc2-K$=+w5yZ0Xd9se0)4Oxgo z*7tpLzVnSI-l#Q5M>fpho`I0=GEL}a-^vR8BVZhLl~CWj?EMRHSKwHnG!{pL*<}-7 zU?(^~S5w>p_rX!tpsjV}1{7P{gTdAz@+VcH2>g%t=dr|=T?L!*()0b;%iqV(Dh2r; zxetH6SY&eyyGiH2yk|OOl})Qn=(K|@G}PRAa?Y2*^#4WgTa}&QKoz;ZBXs& zgpjYlm|4Xb;fB-Aj_^Ob)?YC+53o)RGeOt^|QMqPeCW<@$ZZPPhy8h*=P zA~s+W0See98-jbi#!qQzO^R2bE>iGp+Pb~i+de6N_D67ej4w9OkLus$YL#UNG#$W1 z_{T~9e}4#m{>N*BEC8F~y-Ml+jkabf6JcfEZu2-NJ9N#vX6I5p=c{fM8J{ybF?Ji7 zy@=9@kwZ<{WDQKY_@uV0q8yqQb|v@HUX0_*qnAjE)^k;tRwP;Md)}4v$qW(G!)f)h zp}w1`#(|1<^-g{IAyUNX;U#EfNQrCDI4>9#wEi5iS4g74|+-r3) zKeS}9nsP!VP?!b-45t*(IjOdC^S+aV9z^YI+2dT9W`Y&s>&+8E0MAc8^Sw&Ac)}x$ z!Y#8Knu!n|64D7O{1Pm>VH?L#!bF};%%NVw#K2cT8-*AO!5stAYw%d?7?9Zm7>*SR zXu1ShKKecwpYQ?WUbG8f9AX}XjK?Omr=S2<|30d}%RU$(&LwDczeApZ!O>a$+tq+a z?037kG7VtZ+KbA*h6FYME#99^^!OY8{)(-J@HoHfsJh?8q2Ntl}B9`xvuWWS^A3HycjPeFWxu2wfIZ)dGY!oyPM5vcLB5IL0p!>N1)uMsIA} zJTIn!T6dmcT>C$61-LD-#Sk9Xx(F0(MtVL&#Vj?K(TJI<%(hAfefe7VQ97Ccq)yV@ zRv9)P1otR&_f9H6)Jle>wn$8Zk$pE^U(PL<&Pk^lmTacoc*&B+JEvDI!2VJ~6xsp) z?p#@{?N{FXS58u{qgEb=b;BpY?u{6P2e%qvq(M>xdCXU zvd6(1kXSDP4tXWbPTOT*fL?mBH)n;>=W;#Q@mtj^GV4DV{>B{g`y|GXt3}#?pclFh zjFt38GPso)^jbCaSax2*qP$=c1K5z(@i(=t|GHN&@|~a&Y>v%Xp?H7e5U~nvN7;_q zVFMJf^xpUSKxM8;B{NJ3s)ti9R%NQE;_&x9kgq-qICX5Psj8sYFvf8J;M|x~u9gN6sedsYk{Ynm7w*KZZ0ddQ0pKtOb!S3Xd z5b9NA<4Wc+@Sy_8pR64K@g}T__s!oW(hT{)z8N(G4RK|!{n}8n4F(z-hYI(u_-)WC z+!h!)yDdt$RpR$NKmH?M=+uV>Sh6x;>>fQ1XOu5S5G(_>j2Ef!i+BGy3s1A21UEfS z56-?2aA$Sj{Q?&85jc_k|2;$gb7T@9!8z}n;}Fwpi7A>ct9nKwOjl^eS#wrS7s=fq zd0$g3e#m-Vs*H;2J+3|roQ%D`C@$!+{jQ>^{|Qs$te!$)U`10x+qZ^Lx90*~F^ zpus6=@f_R>rA6!f>37KWXD1Vqn8&k}UHZ+$ue{xp*ymRsvXmt0h>4IW0t})ua>WK- zCx9^4Sq%Z^ghh{ec|(6@c<&d_ZK`bVJV4fGzX0vna)5!Mu5!moc6*GXE+g!78b(_r zAV3PjcRIS}9Re=%X7WmSml4;WK{Jpzc9;u+_))m=lY0^IYEA6FmI67dXpWCpHCAhC zHxP~b^oEv$pd&$u4_aJY+$gwfX_XyCU(~t=A#sX zvLjMwjF12!2_G690?Od@a|&(+^B@|tJfP#S3nZ0PUi#hDf-Z{s*OOzhKX3`;qt^1ZiMn0{b;K?W}d|2jj1 z@V@MRrK98CBj>V!-u?2>W^}GdPG-4(F=SA4z-Tt%=hFPpMi7c6kbOEd{+K0AsbLS1 zh-tWD?s)Jy#)ik#p{VVAL9%)f#~P1B_L$-abdik%0tCR7LfE}7t=UZmPyrn!w4B3c zff^3WxeBxv%K#%_7l6o|=non7+jYJJMvrAEq3bwI@k@{lyn`oj=t0}9>3*&Znm%n* zg<1SwWq?$~^W!_A+tP$n#$RXQo1c(Z+k{{yu{)j{Bss7M1b+dNKc%ctHl+-luZ`o1 zwRits@j1?u5j zTD$44jiYn@IbFK=SO*}x5N@Yo{KxgV@iYK9x0VCam9>>(S$R1qgMA7CWQAtYL8T0c z{c~p@fGA~!>q`O_KR;C1;YmRE1wz~efNhl-5aM@{{h=uOSVlqd0%LUdF8CVY$Lw*; zt6($#vG;`}^Fx$b<{S}gO#ENF;Lq(6E98szbF!B%FPwn{(&}@oRD(JcyrN82b2jSg zu)R=^Hq-87>2OxDkNRuv*{3Fp+k=CP^h^SlByS3^DH5+0zD|uG1qyBjan~eIPC= zG0;Zzlr5=Xn^UwXZSxJ%!(R6AuQRps+pqgFIv#WMFVE7e4bRfR<*yNU z&Jsp@8ZrL+y!)$A{MS{S#y$~Cs?<&oam)7Aa?i`-tJo64EcNyA;;^38Juo{pZ-7gs z4$U4Q5%f?0$J=KP@4RE=XVYQ; z#c=$t!SQM0$yXV#w?8z0)@P68&l4Mz=SSgxihEk5Xbx$=Wz5?eW=7O@`>tAArF&qk zftqU5s2jInC+H+FSE0C@A2Ym(B=UkKF7)?(+iQWF5{XK^5f4d9cUF0(035v3~<;08_)&jQ?84s|tb zB8TtH{ol^C!l0Ky{-W56x2eIWSXe1{j0;aLvpS_H>Hl&Uuf=op+DJhyq{-K8PXh48>QUQ8Y%9K2*n`B9fMj4B)I7+n z?1EuF*`p&lllMudZ9%X9@Yk<&$Pcs{c=w0`?#n$KA_v`SyR{SyB_-&J`y z=Uo$Dm)jvGnMy5Hjl8@J6_G5e&z4ELk>XHx$M{0!YHy*K93SS*E_t5&!l!i;qowjT zN>pkZk1Adr_I{-e zex3dh;jqGUPy|5PX{PBS?V0J_z2AGxaqn-oz7@$=D(d^`YRoClRrQwX^4hJi_;^NX zHBZvluqkh#p@%6L0C6rD*)htwYW;9phKmMSWp_ZDPlf+>1l_H}vX%LN+m!!0eg5^W zhzN1Je`5_qoTRj2+{9_QHLR?H&8*SM?7hjbbD&d!HYZ%=$aNZjx)CocA(8aE&b}zu zz0@c?8kg08wFCloOO*oau64ZT;4GinJlfZ=Mqyo9o@MHhgmD(nDv#flpo=EYG_Ewn zsP_}B+a!?Ck$#eehR1+gG5=scf%_+p>Hl#|{{4e?3FnB0l9s=+n;>l&Cm|>_VRRB- zt!&)a&Q(ZL%Kv>)&0yILi_q(&Dj8m|*yIp5tMvAKiQCC?I!5s0^L)Nvf-cZMF*xiv zvi*9{D9UVgBR(v7TaL#3>?x~F!(%TyhO|C~{n;De9jaamm)Pn|zRDWlwp=?VOTBvW zf-4`9Er2Ir^!tOEEx-)#0;T?A6PNKE;a$*fI{!6A<_!T3sIWj{KHHPU_fgIWq*)$! z-K!@+E4}nlF8rhCAVo?^_jBIib%mHa(wes;crFv-I35TydE$r{K%UY4Zs$>G{!z5& z24GwHgdfQRGi_dOF@les=+pF0@JmZT4Xmv|5{isp${B}m1>ODK$2PSA zH4Tj-fZ1RC3LRVrlW#TUqSq%T9w2x=D(j#AG8&12b?y+6gqg*z*2iLo5&Zd;}%CS7Ov24 zpwC&@j0x=XZ-+psY{!Bi_Z)g+@$8}dG1gs+-zt2wfDz77D(3$23^7Mr%y}^ zi*p_|#{T?U@>m40gghkId|UL|JWDf(nw*HqXZV%<{W+x=lWu`(x0&UT>w-n`W}P`I z762>XX7N3upNKkY+!Iq$z;Tr9@o@;|0}=W8-;*DygeMS0N~ohbeMvEEOtc9!7-S10 z#-=GlRh(Hi3jM$XO zrIVWCS9keuWjO6$KF{_jk}*6HC21B8I)D8L1O8$}3y9|8Pt-DPsuW({a+&d_^h*&( zLcaaz*036OQiB`L7StyM;6Ka9Jr8EegEAjp2mj;4f<5|CE=TK94QTj3%M6JCd>iYA z1g4+o?kk$(GhZBNbP7IL-afctxYFkpY3@)oTe}s#WgC? zZz{H$8Yhhy1LRwI+ZD$ZR!ttCueYU3)`;V)%GEy#E_PGXwZ>?Vjkw-252rI1pxY4f4Iw}+yS;^B z#Q;_G|6Qg5Z_v%>M(HU<0+N}E!&nlr(z{=)tBRWxPn?UHqwu{cWNc4Oc6ZW<$|_Lc zm9wK^#k2ng_^W()C=WQ4kD|T@#TE2-O8N2eq%Oa&T}e(wQLywnIL%|T1~C-gbZV!z z1w0h?wE}$HFti^1TdQThJD3D}d1+qjQI_?d^_s+3$N;6a zO8G{YP|>c<6#IA9{*#40xg{bwgSwT!of-n zVd1QBRlw?0@HrOtsrU7H%5Esa?{c$4fe_d#w#3k=QfM>6 z8Rv{kDF9{dvQnO+d8o+1!y8rwrr4U$FTLO7``%4>&jvihF7JzCc$#!Q-AOp_b7kZ* z%#00Fz45F47GVhCD4G|$bUk{C)#Ft9k|w~ahh%s>gZr|1zy0<@e*-FZiY8@1FdR7- zmI5^+EGZOv7>Pa5&nyKPqn=-PH}$}sjY!b*v2OK`>TCl(v?w#S#}JFTs?5LQ@G5BX z0G|atSsb8v;Pe|Oi{j4$5)KAR19!Q`Fo>!ec%dBsn)p|6^ytoPV@1%_4bBOXX-3`;eJi# zr!Ian-|TsJLhEvW;evI8#e*)D2*g~zf3QW$klOW5cU{-{ue1klY_+I%%)qu2ml2E|3YL>qXl!2So=?BIeT@?&~)!s<0VaMM5()Z0SlV7O{m@kZB z2LIM1eX=(w8cKUHrEmk8t5in7W@Q5Me*#Z@q=iItT6F|}iv=YoZKlM6=#LHtFL57u z=URofiqacOQuJsu1de{@%VJvhfg`)}LjS>TspOFaYBUh-BV)sImxKi7dkML{H6bFz z9TKx5rR>QG{|>SL+QI+%mY4~CzrAJtBVn+o;Da^xi%kktY+5S_V{<_jnGzjCJr%dB zb^QmPnY`2b?9qGyw8vI;u(NBr4Jfn=8>|hp*Dy|7W43OO$Sk*BfAYu48$JAfrtp@{ zxkNYi=5%O=HtXlBY&^;DpT&2ll`njT2IBT_&q}MD6q7>`M|@s`jL5P$bNFmkXCuWT zUWfy3V`bn2vgPZ4EwTT1j>{s{38b(b%hi=5Q=s-EPaoh3+;qb6m`S4p)w{qz0-FW# zNdQ=R`MX+mq58fr6uU!}?Q#6e5X7;Uy+3Kqq}IEX0TEJFoeaMppIJw#6;Jw>_o}Or z#0hVb1_PeT)+vspD(1J}e4Ui4Gn+_&_);7L>7a}6Zg252c{cSQj2IaJcij#wuKOsM zjz^FY#B2n|rN~=dL z@C|JnXZ%pCZTCHo!^WrsZS%{+TQC~DwEA2zygWP1{OlHvepJsXs%;DeO+1(Yr`#sd z=QL_9`#%_L#6HJ6tRsX5r9APOB&l&O)xG=vncXyU=XkSujp7X|Lq}Pe%o+x!x~^^| zEiG+eJfWtrYJP0->n4D$G9%UMdl4!`*`%%pYb>rATb0Q*gSy*|mgZr%P)B2LDx{B< z*zogRP`mR`@}0H~Q6e*@4NHm3kuBoR@4kAR0uR~Go8kO|Q*&_Naw{Pto3e}I_b^d* z?W2f+9I(cbLp^sa->ulEd}K(khn$H57pf_`-WhfYKe54`a~I$S1D7$b({PNTzvn@y z*3c&vPn0_G_iu($mhR88T~zmGU74@RVj$au-tG1Zg#_w)n(A1ZWe7ff`7s}>lnk}a zw_j(N`^ist`1vBZD~@ev>r2*~8#c$150B?2ozgS7ALX)4$QxQ*ry8GM+~4e_P2ia3 z5k#_TKm;C^eD8bG-5}*`!^rTzBs<%KdpevyVq-vrEBai~=jL1yjFFYVR8@gn9}wB2C4Wr;MKr1?^MZlyZWO_aEWh6# z_15z&+acexybeowoPXv-^XyW)05#td(3BPrqB{2jY5T{_uW*kr@sN*X%k@@sWx&>l z*P_vCP98{TzsUr0_+`RDu<*TUQ9?jzBp1lzW;asSyPy21Truu=G&tJ@O!xN}j_r+> zGcu1JR_h%=pl80zyuQB!=w*PC!S(D{zo-8S5CSxABZvkPJ$?*=cfe~Z$P>tfPd`e> zgA>|mJLm@HivRMPC}kQ$hAT)*w#zf!mgk+?N1yUZ#1K{Ld2C`)y$o{$;!pBA=42CN zKtjZ1Ch6^b65}NKJu&*1i!)$$OE*0Mq(0WYiKoBR2RQ>!O4%a@#|7y6l>!CLvUkWi zD1!5UX#wC7(d7~ue& zSfh|LVf;=Pe{TbNNr6!y?!3KFtz3NR47*M2T|pO5r7lU}AXcF+-yy0I*p7U^1T^i6 z$9?n8f=h_e=3Z{W|8hQDn%@~>RuFng0J+*s(Wf1@Ga}rD_K0)=!#`zEKF`g1ytuJT z%mwBdK#`5447iBbe!;kTt~mN=pinngs$9%-_gF`p@ktT^SM?u}7-4@{nr(5crdmA9 zWn5s~GQ#c8UuWribDp1ym}}Sh08)*`L2cT5n!X@&-f#wt8`^VC4o;`7*We%&oIwb* z!ndKHUE|4MvxvuIw@lQkG59je&At^{sol7|Fh&USb9rw2DAP&9_8<@3n-VZumdb6P z5X@caYUSo(>g2;agRbi~+5VU|+(B~)lAO}hQ^(@5-lZ4or~)NqiHJUYH*H{~$8}G* z#hZ5c)czru8)^%70E5f8%74%H5&YVxf8ITSsF5!?`mUS5FO}JNda_WqQBVI-MXgWc z_=^ixTc7>+_20Cb#rT4+I`)A8XJrYcd`8J5P=XzZpcgazxHD((eC zd<}Fu-TgVDtq7J*gcqynyomGkyO)ZAdiwLL-}kq?XZMi2Dy$7~Tis3x_bkvO^O>f; zH?G|{#C(31bze7smp>U&pmZk z|7!d6fxSY6muh-CuUpmBQO`SJ!I;{qgI9g+=aaJXX|Eo-@35{|a8vBR8$^nJpvtfh z>`F}<{eEELYIN{p|fK_{Wf z4P#N333|_(eeaijaJbHUt8L}ub<=m&-m${Ha#h*ZX9f)57QFcPJS3xs^DcDj2gX{C zu&40ygvXd~1ky`vUMG3ZRAt$M(|tr1&DG@&YG7oo0>+Uuu$U5y zblW^#EHcb}NG1d7tTn%z8Uj7-m^#bGCC3OheNt}O^-$O!cZ)-$dCKOP>wss^dULU7 zh8jmoU&YVhJ0(&G;0P!|aJZ(Y@b@QRwc&$ zm;I4N)zJJU8YL1b&#x*`)EB60;+}e8Cc8XdT8SKCSRKudz0=B*YS*fT+gjPZN)@e$PGo+9ooOoXg$Q)yt?N^0@;Gzv3 zR1dn?If0?o({+hA@4QBV2$UV_mk9@cdtW{86Y-#dcZD-eAWU{mVua_PyYqFdX3Yb^ z(jci9hAt|GHYij|Wy1KJNTgG}pJ@7&75K4cVV4ri8rV?7kHI;8F%%kupMsZV{8FIaVOWYa?nHo4NK zzV@k~55CElb2oR7KTVZi(8Na*prpjyL9wSFjTffPPP90hT(gFd|ioOem z3cP;5z0Z~pFNq=>*6uovyAUPDuMA1M@x`FsLO!X!To^}BVurs80ms$|*TdbJ?hN7; z_0n9}Pn%G^Tl_Omua{Ix_7->s^7!AnegqeN;ePBGA(Kib7ftzm@o@&>J9_(EP1F5y zccH$#*5}q$_tLOG%)6qpBdOz%b->F{11296z>gKA7Nc%OE9kYxGV89@7C45n=)3EK z&FqWEJh&cGaejDA?OhikV@-Oly&#V_v^6*iYPu9bcG?4AhtHN63xeLi?4ORQ>mjWN zwCVjixa1L=FA1e*3SQI2vUz=_Bre|#_X;}l30dc#Ut&ezz~f>bd2vPeaM;Ev7@{>K zCv5^b&bmre4$CdyiJ2;XsdXl|(NN&|U9FOqV{&qO9d>l>Da!G;xZV3sh1Z(ad|Rqjt=U{Sp|ceC23bEXuJ&mZFT*Kh-=nwP=yX8D490W3O|!dvEH`Bi z)#2S2nq-)To|JjQ+r`1P7I>Td(nNZC9japPz4iq9EEz1190YuX)0*O z5KC*CDU1C#B13$u=c+=uv1>9>r7~SZfb)uktV+YzsB>vrk9X>)e&o6xg%Ox?v3I%O^G&rc69O<#W6t-gT>r%gkf&bVU1vfv-~mpPJ#B#6}2nhs%q8Wtlz z&a<7mys^ zIVv^KK0j=`0wVMQa>??qC0Bc!$xs-h_ku~MSXh_Tnl?9-iR#+&2*S%%kkmcq8P$=SPwY+MRGD~Xr?Un1DB7!>mJznwBK-{ zB>d3~Ab))c7)if?EVuh8XN&c=5j&vbvPvlI?tM?cHO|YZw5l4+HO~;-HVjQ8&)4WM37M;Sdu4c>#Lh@Y0?F%nGBm?! zXFDX$ChJi&caYjbbWAWThCK3OQ$Y(J70fECc5IgrC})&dk!*w7zIBBl_Oj3EkcHK3 zQZ(=2;rb*DJTtUy`~Fr6g3S7II&`4gHi1gTy9uTP%tm0A*oGpdjsvkbn{G$>#*2Z-L_8QDmRyY4=eX_t7;t@9vm;LoK*paH5g1`~uH2-3WeJjXsyX?TblZWpty)lN zI9Z?bM0+sIN%IUVj(4nhpUI7_7gP{5rAV}cBsGaJ+6uQcC>}94g6Wsj6@YIvo7Vbvr*ck}x35cU8eA{RyBE!sD4c0WkWnhzgF_YKrJwWCn@^SVo9Q!^S zYh&R(9U$uk^~dX0(|+#AzpZ*Gi5zbd`;9oylWjr zi1#Z?xJcYM3Kq8F2)&f>+e5lAt?LYjQ&OQ1ct$5i7g`U7q3~LvvYwuC`54o=qLh^n zKNopC9BY3a`3?1dn=1$>h;Z4PzVAgFCn(h+6?)#CbWlWLtXTN|^wmoPh!P&uwNyH} z9IORu1S9eOH9PA3&D9apQoOYA$KJqi!)dI9svs0ubUJYU@W+kpb?K0J9~_vdPG~j- z7#S8i^eo20nM+HuF`=;WyGfm-iC=}XU=3lit>e%-q1WX&zi*`b2f_>i3||^do?YCn z?pT3X`7E&sP(m)BRp`&=x+=)nNv(4Rr6asjXmqL|9QMy*dH6jv)<_2kDsK(5|MZta z3T=})P}j=F3_;hfL7B)=Z5#~`zZU+gB7@$MBpBxM6c!0J2p0KcppIYOiht`)urss> zQmGBR(!_LfQ0a3%nJm65FIHfm7l;HX>+;YACe= z`szrG@~3p_Rcz7;zVLD-OgD64G8;zoE*H|dZw$KOPRCy~lXDL4&d<+(HOE~PDuIN* zQjDDbKKxtw>s)kMuoq>soGT_yoRDodu6!A25{vPY!m)2WUv^+_dR9_}Z{71m_zt*s zIVHZN5#jw{R;f`d606lH?eCme%Ya_>(mJp6I?9AA8}RNhaMfMjog^k|`q;=!ZC3NX zs||0Zg&VFkmDr8g^2S*w;24eLe5T39Y7JVWTT7#K6Nqz6qfr>>uLf3{sA9EpKMKr3 zp?-DH58K`%a!g?a7;zI->FaQZ=RSSXEwA1M!IgbrqDIVp3bN}g?zcQ?q51NuI6wOg z5<_YUg1xKpQ74EB`kgcjZZT+z5tM$;TuPSTe?tP27J_@ldkUv`M)4+@l#p7j3#4Ex z$XJYg_-H?YQ)E7e5i6!t@;9B%KA=-*B6nzB%Lsy6?M&;k z1r05!(~DebCYtLRK!|An0GLgL(_e;&{J$`c!fVpd(i!IeIv)NyEQtM0QDbpc8tk%j zd3x3&g;wD*>_tA8s4ILY8C;`n8(I^1!F5c;QlU(h+4G7GN$=~~W=cfw2lm&kXHV>8 zsGvO5i=)8MdWV!slK*)D9+!eZS|vqyejCu>8|N*WetBy-mz;eC z#Y_q@3=PCyahj3}bix(DL3>gP2XZQj6T?`bh><$lQCrSdSfKwJU~lQ{qLb1n6PGLb z>PqFpNo9hBh)2x~AokyGUR=Us4S?3j`*+Cr2&gCwr;J~r!=L+6qtGh935$OLNBG7^ z2ehU3Yw};kvK7Bh`$5eJFgUu4xC(LS~7KFUcv&sA{QVtVidta z3lQc*7E_~c(|p%!m&Ss(0D~U)eL%N{pPII#R^$C??pHiZV(F)LsH}tRWztP|pm!rD z-Um`yxzg7VX!t6OSn`}DEK7hhW*@AM!ed7_f7x7(z1U#Gs+vNnkVS7VU`<0=;xWM=tJNn$)r&ILOB#G zHRl5IMF6cG_}JZWy|`q6J7E!@zrS;gFXKVn699*X(#@TOHZL^Xr4j0-1_H&;5FLJH z68t&A0YhAT)eX+rqTD5JP~h~-fr`9p%)c(!DQpjF9UU%~*m{4tO}iD0wTlq=I8cJ| zILZd36opawMTQ(s361IcG_bhAmoPUF^En1RL21d}uf2h6YJBc+x7J>&{5GVEX&{`3 zK69G0go}8vbb+2uwB8WTAr43C9c_2h)F-pCNiWEN_bb#Wp~;XgzMDepXc%^pC3Q5E z^rsnkr(JEb?Ac-h#76=92=Q4iD7%Hk^T3l7eR%eNAsHkH=o~z<13AeM;^R1Gm&~>6n&!Ij8 zKDt#tLuCREUM}X6O(P=nY;`RQhw5H_ge!%mqgSxn>1H*W&`xxG>VsSMokS4!CBHSp zFH!s_Js4Z5#WKpEm_GX;*C$}`XGlTndZ_(1Y zTUfB%RtVZtWZSf3)(}Nv&nc~LIZz6pG7R9lZ{^7X$?XS40+Ii z>13<<$#-HG4m!&o1*LYDumY>`t|m6Yy6gcYv@>A{sRHd;jeS9{PY`wtvJwqD^P|}w zvQ{u>*nrV0Zu5tvWbt4T(`B(bmat(GIUGExV5oKQFVwe;XV9|6WDw-=5{|@*l4qkZ zBXyG$5br0w{hi3kW=t&>-dKm(TdXb+JLa#+!fFm ze5fi>;wbCHr+$3&Abe+pc9R(<3maEa=kX2Yhj)B@LY)AR55Q1hlR6m8dVP}~rU=XM zO7@HtIwN3ju{76jvJcimuLNhbG~om{HHwITLFhVHLvux%d7K!!HQy3#P3l8vQp$z2 z7)?+ZE7}=(SF&({hDBCq92`2@nGl4OR{AUz7-{yLi*?2q)h`(N#M2oW11{gfom9Yg z;=o`Q!38d3)CuYIP#4tucsRqatzqA6Rc zVW}ewVm$u#xy8bGCy@9yzhyw963v&qlZU)9Z5uD17?pbGbp>L1*>GrLLA~ZUEW3pk zX`|KDTFxPn?V6kk_0x49ax#%zZom4;c|nUH(E|Ff1XNfC7A>(%rAPQ;^mG{d7f;W$ zmh28LYJGVv(rN9O)^N*Jwj;wnz}h<-iQ#dMb5^*16Lc8iV%BjIGf~|VO?O9HxWi$l ztBa88?E_FR#*v7DJy^7#q+7<~ou86+^`fn^;MSw7Jou_n)AY5dHu6E6S|Xw7lW~;Q zJ(!Vjnv)d?_ZS9+#lvpZyAz4b*VPVNs--D>Z`v^-vNeURaQ69f;Dop(bII|86K%bc zG~su`z}h4SdM>-I4dQNd0tce>;#ratvz*32U4gL4;Ojgv$u@R``ne(6W=_wq+QpZqpT z0Ey^n{WjvZ%`Clu+3z>#fpl|Y`HfJx9E%>8z#3y28 z9%d!Ae0)`|T3j(wstBczEJ7Jco8dK}D>@h-93oD7Y^10u8A&0|6EyA8bqX{8e%iX= zLNu$uo*-EQ%fHuv5N)taNL@j7*|$4LODc*o04G?em>d}uP6aUvd-PXNSDSLH@ZFOW zTDG=}Ow22SOz7B&eTWeqiV~6IYmG((GVnhqZWp*f>j71-N-TEDY(2PE;=K5U+&bh= zP}y?8F6_)Yi&>yvOP@@rCU?dsk`!K$&(KL0SYeisJu{$JiPKj}qSzFF+GAJ#GQ`)Y z)zwHIw39M+$jc|Wi0R6tS__~x@D#2nK0x$h3#CRB)VqdNj#|HXNyH!({X)}nXyI4N zHiuAaDjajFo12^j=_dO&U3S(VyV_Qw46PO8>>Tu6jCk_S!FVolu7cK1ryta@1s|iz zoG&INwRoVoiHP(3XqsavZfu9Tu4=Y)zfp{Jh^#Z z8Ykn>Pk_Ikq-+dbty~Zog){AB2HIOyxqvL$T*l`C7kkN|^Q^%3h`QPw-JN-BIFoly zkslR)ZVL>rs;m>Ht~krKEeQi~O#H0wYW&p7)kZFys@El)#Qt$G1|$mQxY|&174(F^ zj+1zuN=#+130b~cF=H_&&9p>Ci4ow*xU?n6Fz z7>G)Hut(bPSld@v&OGg_?kGB}XS2(~3%a|TwsG?~IGeC_tGC>}wps7nbgKyGI0R4v zGU7R&V$~26mL_wGBwi?-V|g3Uk%hKJFooq@i$h|3z1FV%wvLNZ%23Uf+6Who_;t$W zM{ox1%&oq~Jq%fkRyDQXW9)`yR0(gwNsh!0ljHnsU%5B^24(g2*KGL&`&Lv)*f7Iq z=N)wPyx$0%rS>?4gHm5q7MVn*Fr;<(?vA^2g$q|W?Ur?Jbgy7`!rx$Fhp1G_UCr4? z9QK9Tn|ziESyX93yIyiGwMJ1WSh0tZMcg#HIp3MI{6fgPw3?S`i^OQZR;BvZe&i0Q z&cZJ^6Z2Le1>IUilBK1k89J=C;7 z?*-@sDg6h`12IDjna#gS9}dmR#`My~!im3qDcNTF?iM3X&op$HIh_MXD&|WYXIx-G zze+$V8G1mUb1g%}8NeD326lIa#nQLAO%%Zji;*f8$|TjWeSV_jSH`W`dc(FWW_k1( zu>ZPSgH>9)yU<>N#9i1#a;J7&XeL#NoZ$o**CCwRqT%uXa4YEMjw{TMGBE<(PqZ^Q zT4*$#LQS+dHBY7_iQoFfr3(ldWND9JVZ*A;gOe~5$^raf{%?135UsNXW!+DcVwhoj z?pwoYlie?U&g0(H3$j6>IroCDooJ9u$9h(fR}g-;57d=LnxVYVhWdPNm|@*XU#p+t zUHpq0USK6+5(?>iF)V%XuGQQ)tFv`D%8=-t!;ibOF%;Q%+?S8SoY(%ObkczKtOL?3(5ZQyP@+0f_*(TOeEdBZi6-?VbLfP|<;yMMw6S#Ma} zUWOAtHT>r;!xSZ^ll@K{6VtGZbPY#_3I)|`wLGa(H`jE|>lZEv9r`kh5;Q?r!`@&G z(vY=r#@rop{H|QX6T=T^cIuwn7ag*mp`|bv&y{n$yMIr|BDOtxdi6z(a)Ur|$wmEF zi~gSbN59=Ig6GM8@om>S5B_MxWdW1s&FN8S)QxcFj8Pk%8vxVjDpw>jBt%ScL2g0T zad#?f;ciY$lK}5%{1VT^_nAY`^mJEOCai&BI3`!>V$0-XSGw55|ix6VgP)!vht- z%rOvi)Fiadkn0)oR86y}zltAZG#`IYzCcuoQkC*cSBJKd7!bB7xaKC24vZ84Jp~I- zB|WT-OD#`&CE|e&PzNST0Ne7p9;S{vQhw|sp;$NRiD{j%)tTLPgxxIY|s#2@$z6aPO$# zW~apFW8^brJaT{}sH+hrMqT6U8Ir1BIC<+o&J&312AaR`V3T}TnjDOG+-7i4lT7Z> zgORYp6_Qf0~vY1^-p+T^BWL` zn?^?6yd5Z@z+lr{&`AS!f zgiL#1^?5r*B8LfJP>}C6ZwL#1EVp^3{B}JHX*Go7qgy~e6Q1FlCgME5Qj{d5c%^V* z)Z%)8PW*fb$q9lQ-G}fzmAp%^UgU6e$c? z%&=z6tr}Zo8>KcY2NpjUZK6IErH9bD@|ptT1Ea$)_#e7+)}H)h8$jVhzh13}>JnUk z-w6e`Wl!3|IAFirB4Dx zzE~U~qhsSYyltz2h+Q8CMnISI#z1b+89$;Te?C50U!6feDO!sZ3+h4kTPBqj2PhM7-M2nV8i? z40YeiV@86+;qud z{f#(B&6*SS&AnFL7QF8s)(4_=bQRi>+!K?PTt-?l_k;+y6A&Q2#&bOnYJ4)o?OzO) zS$f03!5r{wcP3V?XbF*Y3pl^DzTKnA#KiF%svnI8kvD?}JK>0$0^D%2g zbZ6)%0J}m1pXB>Bp)d*UEtcn2us3^}b;EW;By`%Iredw77g6!+SYK0Xa6kWYopbjUwL@bos^p^J`L|Dt^ZjQu0%a!x*Ue70cY z_-XhlKe9Dqk7!KxqnQP#slCoW)@Q(JfWL{LhDGqd#9bxj7qBlXrU_Dyn*eRUMj>y zrwe-R7_B<+qD1RfDhItzNkAFYIx|?Pm8+gmd#MCwr}OB$WiPL;s_QqAV`PTu@W(h` z)RkO43xt!F*ACJ|a$?bw5qssoN7Jd*Gx6@zM$~Oqa!0C&N@s9u-e+EYoK+@pxBWQy zSzkP~R<~lGs~ZLn_4q9k_)#GbWawBTQQapsCn#z*xk&>!3H7$7Q0D%UlIBg2y>~`F z&mg)}UDLPcseGI%^+ZE5r!9LWS-o3B$;GpXL)gPze0(tiFfM0+nll@Sqi&oCme$!X zpBjvkVV!COvN#`iV0xKyMh+@sV5t2-wODJeWGPmPCc~LZkS=5ND8ib4H{2T>vQbzsty7Ok zqvNs|f+mG>dQ*6JsyMo5K;)T?Avmy*jj$`2p4Ku$-e-S+A@Kd)D3CMn8xqy|+fzbe zbpZ7_>p|gz83UH`$Hyd41sfiR&pkXpZa2bZ;?2N-o104=tV?fo6;{rpx!8k!%c2^n zE4*PUZwZijYUD*mY1h<|FH1msBerl`XE8*$$B!$N@sCLb$sWJhGOO(P-50QJlTV}= zL+V#;SiQTy)yQqdc(r3VYod1%9Ul}F|dy|Tm2}AI6EWTheEdnnM zu{}nfpBc7!2ORtfi#mRp~D89%WKg{34q$!*GC;}W81 z4_KDaa}IBylAzL;>$!^T=V>y;x!1@VTF$Qn{t&(0Oxb#ixBt!GiyS2=`G@pKZwSy# z%69-oE0E-nDbG!2vLI_VKUi*fNT5w>C^TGVmbMu`j^pV#7jP(Pw6kae^1tfAUDSWY zE#`I{3a9J|J1khR*TvCD8Y}@A2D8hFQ{YV&_v^fYpdV0_x?(l;r`1D~QW^e0;7M;< zTP#%OMsqF+nZq@HIzwt1hNIDmtrAoFeD;oPK19#*wQpP78%ZDq^cC7ZqO0agD9wKv z^PBAOAU|w1Uk+{l%+^cR2E^4i6Q*e82EW&w#-}!GNoUR&7 z4C?FOag$*o>zKz(*R_qA=4|7Qvn|{5N5`~cogRa_+285Idjqn#f&ZG1Bv7F5E}^TM zI29`vHj`jPJ|Py9zSCAe4!J&@RI0!b#rRp~s$I@&US}(*yu2n$2pm2Ff~j(|Roe`t zh-W($H4eSDvntcxm1@I7EY(Z}XtoZw|LFgmSV%Ov7H6AIetxSt{XOYKyGTCg#k9KZ zmQB)sDpkgd2!oCQ|5cWk3Z$+oYZt}{hN#}&-YP*6!*|tOSidx+v3l8OE9p{@Aw{?w zM(mWivS_V@Xis2Ker1}WZirb?ofWDT^fJ$CP;n@Blfjo>UDVs*cizH_$~f-b~- zDId$3_IY!lOYp%Gb40iOvGch2GFtknmQ{eECW}h;OnK|*v4{fkt7*EEM-q_C^6*YT zhrxPBd~61Amc=5@Yaaw`q2|bgYH;sxJ3qv7eMMbaL!GRyqFCj%&b)rn*(IppSo!mT zbmvjt0py@`yMtK=cFdx+2g|8O2SZE4_WNslgdoCwM#vbspxZC(7Av#I z^3Yy3)V5@ZRZBHaBa7=>pb&QC{7f(q&1KlA=JT@l1=fxvdJnVoPb7P~D5w%tB6NU2F>NtpQK@wPo-FDp-;$iGD zlzwqblbPtg>%R_5C;AV3Xe%?nc`bJw4(C0&4zkS3tf&P{mc?*`9$;hl3iN86Uqw$) zblknK)o(X?IfWHFu&RXzyE?nrxvLxbcilL4O`ud1Iayd=^=WDxPl}aP`aWHZy}*7h z1?pR*P7+JuvYNoqlP`vRfA4t_M*GfWu|?RUnwA>ZFNWYVL&6LLTt%{PMS0mNl|J^} zq(WN!ygA*5AGj+vy3l9^Qo)c7NoUL^U9VUtfyR=1vUc~t^VNhMOG?KNdWN-d#v=M) zpFd<}Rcc+&H93GZncp_0eoSnHQTW4$?MuAqVYn0uQ=J#ZK5FO$1`#uB1qh0jW{qza z&FiMHAn15%xVMg~Yh}{|ioqV-x16sM@q#P$CU#b97Sd2cqJEIhh<+V*Ib~z85P3mH zfyQW(v4`H;bTp(^e99bMfrXSm74Q0m`$(S_DUR+^Jy+ricOrKEdr@dBp#yw~i-9(vg}P#Gs0>C(q>ydQQR&cX{bCmOp%WDPg#m zZ&68MZdO~1X1A0$)1_B!WD9o>&PK(fh02Ff{{0ki%@#3jwhiO6AdSAM!QHWFYA@ie zUKhngeo^u1`cB)}7rfc#$V525;46n-GX3S>@VtjFe1K=QEc~Xsk@UZIpDb1-+q0tW z9S~vB`jz^t@9qB;vi(QTCc}ytcwf%^{PvHN-Km@n&20_=IwDv;pCfmijhTmz(K*Li z(cnp@Fy*CG){7rmkeEoIP9BpGzJwLpF>35Ky60X}?+5N=I5`M7qZ& zM!M2*b0~fSavie*p*P)^kHWFeV!O&w4imV-1t37{!d>@^B_RZ`wv-$-TkR-Fzyh?& zKu~(fxKxMW)!Iwq!^lF1j{Qb-r55L5+ASRt*ep9ubrEEt89>=9=`=pER|0{hBdo)Q z@^fSI*m}P~G3O6ZP}t~2zK?j=FOSLVXPvz$p|FT;YL+)c`1K>++1S@SnRi$3e^;vie_1n|k_)uQfEH>B=A zBVhH80uy;8M|G2|4A=Zf^Fl{pN9U(eP@OR!ld1RUUk=a)qTjTknP(55y?_oMKpHqG zHYt?9|9X2!iF}_OE{uG8Zs^3MD4pt$?P6Gv)?AUyci6Ov>mcyXtQoW<->oGaKE>Sd{;y3qq_44sFD8r=OhPs@N-`K8d z7%KNPa@biIyKmGHofPH(7?Aq*xm=HmN#{u@fSb zw=_qmIgL12tsr#qxz4}r5{y_h*%&oZae_jgu8Ras1-#YWDh`v<$Kfk%(;Asg|Z1)5wTBGmXEEqeUM!hVduR?yYhQr6Bz^_R!LgE~iI+e2@!h#4>(D^#yJX5b%W0DW4 zZfB7dxAcg(yh4kp^V#d{^9Wx~fuu*dKM{PN5N&RI(i9$~U%ih6A@VYhEKaV=YZU^B zF$9k(a1CU+flf^*y?Q#QNTWmqJhwK5@W5X99sC4R+?j=WY59h7h8bn6m|{o!Df7Ol zl3`FIyAC`8h!r9M-#uCztF=4{m?`wsvPCAfxB8l{{%yub2O(a(46ML8=_kKPFgAq; zB-+gDvd7u$uC9&GDA3Wkwb>MA95{Ttm^17H<26EubdkXS>L_5ukhKBYUm@fb-UC!b zqvG})()Sj~7U)~Q$Tw;S&Jl@?TYP-SnK#ZViTRqEgTT@p@NJ4WC_yx8Bf zjru*eH^Ahu?Uz8HY=(TUr${Kw`B7(pUd)K~;kW4np{f5x>P|uaDY{Oz@*!Cp=~~}1 z_pl}#x?UNy%U-8@s(g^E;%uD~1q_Ty0sshuDEoThnq9{}vw zeIP8ES(23P`HDzk0AD517ZWjwhogsk4bLC5yvrg+mC0JpIw~8AVb{DfjFu&sX??N8 zsKq1lyysV0Hk91ju>4$%x<GlL4quU~brq#aaXdrz&y+=@5o;MjmHZgu!Gq*zqE;b@j(fv(R zzh_6D`^0`B1j;zbUW$Jvw@JqTyBU+cYz>kFMas|X@VDixXLYIPU z9(fymC)+Kq3B(i217DYV+UecRB^kH`cw*C3Ott3`21n|6nRXj#0ljpJlS2p2h_edr zM*)eSh~~Uw<)8aR)4sF0bi|>MOPg?SgKD1#a6ajBy|>AE0VYxNOZqdZ+haKm_8HJi z5;`n25Y$_J*Cf2vnJ3*wg2c8ZUbj`^mIpW>5WQ|8z|}FG<)y+KHWTc?gtuqoF(izl zzgf8iuDz`BFZ|YH0t4@TeM?^Euva&4Q@A1x1gstWIR5Vu0jv)F-7=h6oqii2mFd?U zR*dMa#yCl%NxNn}FxpH-(*HUIsuBU@h$6kbZNb|fcb|@Q?lCbAV+%`a6~jhtpnsu~ zed5nXn0N-S;OmWKsWbWUY=-cGas@4pB2}HHea7Tj$MrD&QJWs)!dGkd7aN~m9SPp< zR}dzqmA(vI@m8DA&OaNV4f|0m=l|(`KsxC`O5D9Hb8xj-mAUFCQy`_zPFhc4w2Y!6 zg#)fKQH9<}O@1sTjvKdOGv8(HZZ5`{$G@ yO8dBkppOUIGpI?Pr8R^Euw z7mt61qkJL%XW`4lUI6H42m)MU|o|)t+ytX15+F!%Pd0wzqH}==vjl|~ z?=#XC2epLFX~3&s<1$frm#p}5L+0E+;ny1CiX&u$%EIKYkJz36S|`FJ@UNbLkXA)C zL^C&_82RoUB&gI`8rAg4<}4%v$DIHos3_z8bknL;P2p7Ac=`lhI`10XY6b4>e$ni+ z$J)Dc|8WJ-1O_GD-CA5vveG7A-oAC~i6Y^!(YujIE{Z=24c$+($k*QjB?KU+MghO% zsNWpAWY`hh9&7UyI8{TVlRe-R7T| zaADd%bffpL!cTMxUx`P7WbtID1!j@M;`p6LY2W6qr`cgGvron_4it_1SjoEW&6cq zWJB+1@{nm$nyGiRe{5u@p&>f49{xD@1g_!Cz#pvWmK*W5p*T^C&^?q%U2AAa0BCgk zfFr!Spxn7X>;pL45kR&NbXE!<;3nxETZy;9S=MFxGxMtCaG`!*qq8Z}|9FpFk-?JO zOsb}ABh?7kLxN7>ShhkgOqP8)px$ zDl(89TmyX@R=48wn+t|R@IH7UkhPW*7y@fin|1zX!>)^mFN(!XPkJTKZ_hlz8{#dp=2QPYPTJv_r>6@q%78w{;98M=sv+EE7et^FJJWh2a>19DQF8p|Z z|Kgxtf_wM)#b5JPzryzMJO{O@luzs8N1R2ml=~-Cm%Bm^-OYlhON2ZPfe(oJ6NU7= z^bnI@UI!Yj^&kBN#UA|b1cSVZC>fwD_!Fiizi@EE|MZ!i0(dulY^=gxCe@D<)wBCk zn=cV9xk(ebO^{8>U^&j+7*p^-;A5L|{t9dT@Q1W^D>m?YuT+(z1MD&e91Duci)7Y# z2dEhO&rcd6iCEZ&Nv0{;&Sz_b@!F!cRy zgk66`O1p&;k1$?3M<`OA6R;ixIae)U!%pyWWe>t)5#a(Sy%Zr|L0$W!!u}q2|Jn!K zIBKDnzpGnbVgpuvEAV2Q9WFH^-bs;`>~GQ;R0vu07MK{J0)t~ODuS2kEAP|Y>C*Ta zeuZ~-{l)*&`q+ui$Opzn-q4>qIs1qnDws>XLRWxT=QhUC>V9gFxc4eag@AfhSK|O3LeVLVn^CD4eQ6&~I zO<~%&{(ZJb-6Hisz)&Y2OQFU;QJC^iDI9x}=)~$fJE`GUX2(EvzYwC9u*Jsmfg8Ba zw6c2kkIZ+L_+Ed1%DNRQX2>kY&s9w;00Wz-)L8l}MrIA&)hE>*QgQ;0+1U>DIn4Lu z*2pgKUwc6Xy3Z3(?pua@^= z6i0PJqq2g{_Z z5y;8SI7rx&@|`MrF=bvSLox>;V0y*8tUbyrT8;r|6f*d_96#1>a1wc+JCBIsHllLHAPZ4RWnCfdSjt zXVmu1+qe1ufK8!za#K_zL3dDYO>`$)?(lkx2Pc@S>ztG*6q9+(FKZw(Z}P{BU_lj9aJdwhjVj5!~L? zLj&Anv}$CVG@113r{Is7S-jfFUbY1vkj_rQc)_}=JSqAnSufa*3%v|Fo0@SzMVE{D zo34gyv;`=F%>P<9%tw}AD?*XqFOfZxHHa=j8JSoIb(#d3S;^>siUI;Q%bb8);o9W|%UO4qn?D)X%6a%9k1jK3!k#BIO=_u zDjyGzj$^2{3M09ABlP^JMvp~FE*vMh#)U}?y~Q&v!lVr|Wme+g`7@e83DL!pcf_<2 z9EfSAxxlWruigG&gRl|I7XF0DOEP;3U?WttAjajOTb(j@M1+JTzkc=V7PTz!{_Im7 zdnBlomcLG|15cb;$pDLyB-2dA|Nijvkqd8!E07={& zi<<&+37Zy&oi3{)K*g19m3)5UT!Wn0H>D4(xThwdJL?6VlQn!}wp?ofTkCsQJAmVZz}dxPik@CfIZ98l)yeY~6TNst}Ey5-S(qIwqayXq@)zxe<5 zM9>k#G^8RwZ7?a9KsUfza*(ljLQMS1MBiq@&93&TgfHN7QaX2wNy!blF#}PG_9y@1A}CxxtZ2*d_~QI3xlsHSSs=(AFs~i z_%tf@2?FVl?(b0BdWK5{u?6$-9E#_^noSkG^O+ymlB(H}nw$bG$lL@p!7*)oycYsX zo5r731BF9g2IdC-@6NScz_;=lrils)7!V_Q&4I32TB_K`M$rGHjCS|{m}fmJw>VYtNN&A2@MM?I zI$=n4`x)}CdkADOu{s-x9SrVFd|vj5k9=7m-}SMEvkW&1mApHjwp5;Y>MP{9WCLH# z_1P~UZhiiKh!cIviyq(08rXh+Un9V7ft>gwgwb8z;+=jo05KHrIrC_j52+BbA}Yl^ zeYK6Qd<>*J1dmqgrX9uUJXRdDf7{=)_7r;ZQA&YZebuuTgk+>`BXpOIW(qqwI@;CJ z%C$MRUCi-a)DO6vQo4O0Gx5q7U_b2e{dIq3r`3Y6ugen^nrD|V6 zt)T4#cdWW46!XtZ$1qwnU3 zEb@2Fcxmix^EnNx>5{i~{);%2SKiaitsTP1T*676;gBN&m0s~9;^=+@XH2Ug@&*c)iY0gMfyJ6?SH4gnB+IU3uAnTcE*O^U8F&>R?mj& zHMSJvDbv)Ws?>NQ2E3zmGjhbOC&13xe^DrSg;(K_a-Zs-GClp1srE5;y}0ETEn~?I z{{t{ORBmF|P)o;uZ4*yLko472@|o1ipI52UeqQ`%mxy&=m2BfG7+33gEi#^_2Tp!I7!E02;6+lMV>d}o;bQGhe5I9B3 zm93!2D3GaW7!=KNM4oT9fP%F9XAtCxzoHccU)h;jwolpWv8XMB<_8@>yF-`XFFL4L zHi3Zgj;%T?vtF@6o(l9@IzkB3c)U8GM(wS%20q*mS3*95ob8IDfyqXo(J};O##&*# z6uZvB$k0$eby8cl6%nZu(1myZ7Wwd|yai{WPA;xnW(gUl zXGM3uH<`SdUZ+R`mk3-CPurBlV{&IXqEb2BBJi^m zdR09`DhFsd2=OGh)R#)i2J>RuLyO>9N>S}74ZI!c``dr0oR(}7=$#Xr)`I-BYA+|1 zu=;8`_J^z;RS>s@SF?FQZblx?p`#`$QQBXbtdQ;g*t1bL!k=-saGG+O zP&JP)`?;aE8JZ!cQTJFp7}uI*?k>F!G=adaRk@JqCuxXP16!KG3V&(@pNR%*_{q4bz}m8Xw1`mX4al&AnEwoz`MLmC5yyEYlf_boVOjg>~VcF z{_kXEJ0F{uAt45xag~}80jk?9hUSgV9&3VA#I!8it};1p@9j9@M@c@(BsL4`Jy@TkvdB*sk! zXeYzvkPMRQ^)|xFoR6D~G!Xz3`94#CU8u@S@VwA*`CIBMvadTNN8UkgQYL|gfsbW; zCe@`G$(2GT9V0$tCon4KlJW##VMw0qd_8$mD|4vhVAV}EG?eaF))2fZ)7rC5n9-N{ zCPGX`{ZVUQw{O#vm-~pbv87t#pm)fHiIuC|Jss{e5%%q}M@5g}Lz*<19k)1X3(Nll zX0^%89bhjm5pU>a^&#r>=%X41OQ`4h-(l@z7yK+b?Oeqa^82GsG+ICWl3}Oft}Bqu z+<2jN9qHRKI-!`Vhxqz-;`vg&<+$z-oBYi1uoAq}n-?rOMN6$A*2?dsPR*WP*0f*S z{gRw!LogB4+uEV!vF`Jx=wBAqEhi+pl%KuBz5GTefm4L3yM+5cU;D%Sh;BQi$>BAF z=o9wJK1YcV%tjLB8O8xdnQV8>Z)Mu$oL$?Wd#2=*naR%`#Z~<-bQEgsbWkSqk!oG2 z7J~6eqW~yCDOdkBlV9G^i3^AZYCJ>!<}n?Q zQE%1vaIO5TiKu{W3#wsHe$Ab)o;8YwNAq)}MFk64mDdd*+won^MG2kNu9rneC z4bY=;tD^|lvJ-|0bUh%}9%CYQ@sCRsMk|V2ok?;)ZTa7O;oF(8d&uE$=h2{GXFrWl zrmKG2B{qq@Z^38SfL@3uB}Tx;L|F|c)z zaeS=iiw6h=@uOSVlQ^{*A+9kNV0uNsu8G~B;x;B4{{Fp2T#9)^6j+aMyURDz|>oeophTT?Lw zwS0u*wXv>lU8v-7CsI*6RMo(N6$3CJn@dK#Z9^VVWWP`rbpibYQ_AAoSqTLH87$FL zu-v96leCTm-+FKDxLB>?y9h2C$UBq{jh9Yn&7a$|nkK%+jL3e3KsE@{8Jl{ohfbO2 z!=Om1;K`u@B2d6lkR`g}s}?YgcEm<1zm<0vGqup40P5;d7smw$8oTq|1=Lz$%T#BKg|I>Zx|ldYS7Y*F{lGQg~S265>1?JIF9t(S{RH#eWKlQQ-DMTrx7q4DELlPl)`{r~%vI$@(>KQg zr>MnBp(f@@A0W@#v^bnwx8dWj0^EDT2n}e58Du@T8t2QCnXSGLKT9k{Pfa81%YX6V ziSqsKC-%kGpLv3JgZ>JsxjjU92%!u^Q&F-xEfPNgF-{ z6@hn6QoryK=*5^?31B$kXa^DzP4Q1+L)o!j1-`rN=mARuzwgn z&qs^Mv%+tNWTdq%OW*%9w{hL`)LD+ue-?YhjPH%lCi8k#mAko@E>*?hb7ALTGAW{LfoZwCKUZZh0qAK~4|HiiQ5R%~-%Q!zXBcLVbJVob$I?5;xp^rA4G(Wup=kCStV4-H-7-0~gny`(M4VI?(yyEm;Pl3{0Ya8YlvGxxC z=Zh>d2chQ^XpzK~qtWQjVbir?Mtv0WaNTVMtVPsv0}6T9>A7gqW*p6PLFH#g8D!N; zEMX^$`+?syG}i@`i=kP+0v~QnfgGD+_!jW<^3`d5@P7^M^Mk%(!e;6Sta_3KxY!(2 zHd`~WQ1ESsQ`m-m^3|I>MQR`UUS43<8w}OaG)*$mD$AVHn8*D9A40w#HHMlX7SD7NHL94)C=q$`JHv;%m)Qw$iH^e33 zi3f{6^qLj0_6oj*N>+6xWHTK?y>)r^kK0bOk0H4N5%|=+8k)JnnSek$iZvcAUmc$w zPXbdu9d30tCVaN=Lz$zdi7c7D@%_8b`p{XLn;k5k);A>NXBydZzW&EqLYBR)+`2G` zq~{bWiR~g#!C8^~x;hIXdYP~tYtNNLz-yX0H*`krK$=nl2p^6`W?u$Qvgv=LHRQ`> zjfs_g`#Pa$$`Mtlv7padg|@Gll|uEaEAhXXHJU`Q)>gvJqrg>-GldJ&>5}IDT*+%s zjhB-{-?@0N&$}dE8|Qlb$I&`2T`_WZGC&AsPRidVASRO$Zga1j`}b%2US>I(>3^+m zxz??2UEN(2Ue#}J9Xz?g^$i8D>yVW@5^^NeEy~%7-y=t}BM;xI+x%`#KIgHTN!O1N zjNQytjk% zYin(IQNL`cmSTUvW*-??bnPx127cG8Y+BX-8V6DUcKctL_UW0h&BQ6~?cK!Iv~RSh zpR;6A;u9dW(Fj)2d=6!J(#K3+6#<$P(ObIEPA*9TG-U0M5dthW&lLX0?~4#$z{y zlG1O~md9YGCCEe3RnflwmEXp~idTAKbh|+%Q#I}gnyI|yp06JECv=Ro+#cBboY6OcsAuSR3)dgMiu zvn1lZX|0`6w)7`-T+Ej;0N4^sem;Kw?^2it7Ao<|=xJ5*sJthQpTk@?Y8v|R^$U-P zu8e?RM(H)~6yz=OW(ZM_NVt6|5U*&?N;RaR_cZw4;nbVVT@5scs9Mip4HrSfgAwb8 zpOd0gxno&&fG@+!lrzNnGHt*OakfCW5vv`W7*SzmT0bv5tG$u`ElL!Lp#4ByFmThye!3*q+qBdB zYd?3{yqRAoZM7Etjq)@D{vDz@Hi2KN%K@5Aa%`s`THyP|7S9^GDT(H0-%)pajb1g_ z8GuNsEH%{-ljMD00{`z}ba6##x7%>b+aW4>e5QfWkOVerd<2PF0R99$GmBh-G zBsFGqxw``{oh;=KIyC@Hxu$FnPhGxKvdqs4zc*=SJ%>^(n%eeD=4w9AvcSHmf!Bw&Ns zXZ}mS@dvsohTFZQ5at>Zzf)!9qgL`6>50%n_1~Kvr+heR2a%&wyXpKg)KBMubK}qV z9qeq41*rKPnn!n2Ih<%)rA&kK2<>l}j4=@n>hq%{@@lp+Z=%U|V$ab)8wMp?+ESb+ zi2jO}yG|mW&`u9K`CVPbzB}x|{IF)>{lRTKgn+~Ia8th0N2!r|4K`n@7psmrXE41K z<^N%H{{PbEE4eq<=Vm_DI$v8{Eg?rAU3Y)Z{j6^kD-Bj+*H&gATmM-?c=Q1QTB}o< z=fBy&ZSV$uxnSIdB)ETR z@DYEcE*Q;v^7>6lnf4Iv-&Bs$49B^3ySpJ4lr}65X1uD$<$IKEPSLwPo_2kC$E9iI zSiRNUy7HNhW88q5GJ1m!Zuc$HT$7dw-v%npB5jV%a@9?Sp)eWX8FIh-A@)|80W8m$ zdD7LmFkp=39W?LtGwGD_w{Y}Uu*=*_;SdMUg$8yUN2}a z1(I3FCKs}`SC?dw2D$byjV<_>%ayIYo>FW#k>pb;5k|B4I&IJZyH?T3hfsR zQgRgD_-ZUQg?osXazen_WEAL+>K&xPa3Ahx5_YxM2}9{+D}Y4D5-3|D(`)(>(a9FD z9UCMhROrXh#og!Wzc`#?=nu`L(q4-|=8RQ6)T!D?{WPh)_61XcI$mT0-IhAX%&afi zm3=0?ay9?c33N??GM}Ep0B_l|tZ+Dn#LKcQBplKfYh)zeZ|KQ=FpvjF4;}Y+Hf2fJcB+vnk)=wp!(A!cBM|qD6x*x zcM6sT(#Jjoyptb--Z4gpA@U@eA=%>**D;jiS1c%p=r9a^3;Bz*V6(3L(^N;eW8Dpe zJ_u@{G3ww$q6f*Sp%cuVKR2CJr+A<2jX`6`@U|UGlHi6pGiX z|!2k@6N&krTb_7h)?6y-$gS6 z&k2qp_(I4jTt{5S1Rd|`#cx`=conokL)|U0veWu*-}6C+BVM~NfFN}N57w1}`UT6L zwEOS328lgeY$TyHZrKjaL7G|b!5c9Ty3}&xdz}s;{sJ14PvrbbdM67g#8h9X7}nm6AXl+J-p8{t zu3eM?GK7e;s;-|nL|s|Y?fjt0AqFJGf|jVMP4drY2!=JF!>8s@QifBLa@pEXtwCz4 z-1*cMZi(VQBcc)@b6m5h^z==DWhC?`;*i>AI^ww#yb`j%)CO^-wCyrN=8H9>$IesTDZ4F0&Trb=`4Hwi<2Whzroc_U8AkqD+@|H&0`DewYR2lb2jyaiOI*Q|bpt6seH)>-ghPzjvj z=U=-e@I0c!*QI2$ru|58`aD^`#!-n*)$YUFI)OoBPMh_@kj%Fox;5|=LGY*7Ps=o8 z#g@4j225|{R$*c_i#OCMCiz}e4QYt1|9Wl^o?n`Q1)9J(iG#{{k_0CX@hPtms^kL( z$Kcy1hguI;yZpej@FO8f?x7H1$&8!b#Po zK8ioc4J;xMP5AHDwiy3$CEQ9=p88~ctwc3#m+cP7a9UxOR{24DWNt<{y|-0lF0*6_0k<)7c{zBK)2g zxB>V-bYEM1hViW#!{5e)90Z}1-4)3reOaWD{Z5ww zMv`u~b9J?1r!hu5v}sRT6}dB!*+?Po)I7jUV(XYd_Q&sZw3NF>F;>7}HN|&vxvp4V z;#da3dtijYDd=izs5WBmpQ>81u}9px(r2n~?zWLY7XqEB%;3oN#T`CgIdWTPa#pT; z<~+icvAv|KceLop!8rb9NyGi=N$r**o#+uKMv$nHK#%`p1`!H6mPr-ZuL->N8e!xO zn!sexF>t~5tNVd%@hT+)J*@8v%uzGA-{pMk$EgxlMl$s~L0L|*4?|_?_2fJb&ifOB z4%z>(g}ZhWQs!Bwi#R|2pMCWWlY+VF_a`o2!M36V>l!r0Q+g(mDd{tE-=)go(;%XQ z&4OG1&&fI;)t0a374tmtmeWW$Ebrfv+jwGDIZm7~e_0e#b2@R*&|Q~|>(7nxJA{5_ zeDjI1yg}(Q;eH?dA{lNs`={gU6f}pskk4AVNL`mc}TcDJ)DO#-Ps|CA2JzuXGIPe%`;H*}v2k zpD4z}rd&%ssP+>M`fe6o>llchyS+OKywqWlj&$Kc9ZGDK z6v=$}G1C1-*)C{H<;3dd*?b4Y8CY3``@rsdPhn!HdODo>Nn}c^cLb0UU}}EDmLyI) z`tcc1`Du~)9{TRkpC(#kk6ZjS_+pos^*MD*>_lqyQf=!3a4FKyZVK1rM#JBlGvpGr zCcKi1mIPay?^mI#ezIrS8{)Y+H2&wW+>nUbUfv87o?#~Bo@7VF%9;S%=*gqdJT3|F zDOX-e#1WH&DlI1F75hME3GOVjvV>>RLVu;URNB&iZ9jr!0LN?VNU*9ESRlz*a)_5L zm)rg8cYkVX5SnzXJOZ+JyQ|>66PtEh_8=q`E3>$~=sNFw2f`-m**~cC>vCEqN zTEH$v@uE^@e8`7Wb_8qRce;d}u~!o&bW6>CQ~C+H`QzLrp3YS=GfAY?_fwni2l%+~ zCUmC>x>#i$QTgmRN7Lz9SbQHIOw40u$A49-tGE1LPG(ukQf0=qUt#CoG=tOo2$eoR zi)$dw_4@OL-sNKUYNM>s^}9NAO{=EbCE`)WR2^Ytm&JsSm%_Jji`43A?{VG$wiXYL zyzhMbHwdg6TBTOeAhFmtuwOCZWzOl|N&<^w3@ago@Y*FFiTA9s{?z6Vck_$Q6Y=^X z)6=)%)Jto{_CwXis2GJ_S$KP=zszG~3h44WI9OD;h?;2hvr%C@#$E5cM$(*aAC=#U}(L)QJ$6=ocx-uLNfg0D4J> zkpKSnOhfC1X6c&kIzrN>vFEa-nIPCy_&nQ;6Nvr&9%OW$n;nWL&rs}Le+H=d$P-{> zsgHfXLuv$>EdAeVnw;Smy^#2ZJ_ zV6kt>ICf3K-!u3JsC*3@YzQ)QYK}Tdcd)bM^KfI%j*6so^P$ElY$TgJ+6Q-z2DPU+ z8)v5@Ji<5hLeV*a_gB?*^3n*TyH1Y)xX8o`Y{o>`Yy<(rXnTazo^B$U+sz}BM?w#X z(A+$78E4sWWZRuO?u59e{8gggQ!4;6luEq9b&xNb_6OcH>Aj+4dWHB!7bEH8xMPqB zGgYu*sY2rY8U?b{_UohJZs>1jx9V5ZJ0;5R=sFbxg2LN}`t&rZ;pNwZ*uIX~reA$))NXq%EG}w4NKVQ5X_xc`+GL zd5sK^J99@q|CMCtjnJmY%}nw>P_TdrQ?ufG-j{gH1{bFA-plj(@Q$g1Pd=mRvOaiK zpX_Cx(Ow#jIIjX3%qI!T?a8**bfF`CrJx7k_zP~`{gr`yu;#Wym5cz(avCf0(7p2z z3h+zx{>~!j+Y}e`!4P>W;-HV8>uZ_qh8zMBRwmQ}r5Wdw^kgFe=jewRq~b~-9naV@ z7LK0Cq$k>+0E)Eg>Al<(gOPyHfrtv?U@^j>EhPd$>R*=g$vySg7qyhj5?JN zHido(J&R0om=X~NEOmx59*;K}kR`~&WIl}5>`pkzdXxIq? z_I9$dg>&l~anytH)h+!XT+bDe&lA|_n}ml<+e{y=x|yJ6p-zj%3mJNTG{~ z$vzM{!`g0#%N+fjD2Uyi7?V6T{*sk6jv|k^^@|d#P|f9>fj}1S?aW|OqAu0$SE#$k zy3c}eX2!HB28IS5Wuf>?lEMNt_rSHekw!B!DCLUjgzRXxwRW!va@N%4BXqgT6~f7B zKgZ?U&&^9t*`nvW3j9ZyZAg^QQ!GI;EvpmCHz`6nMPK9&Os^M&G(^Z|#uF2m)Yd&~ z%f!}N>eT&yFTD91oog%1GYPO=t_Cr}xWDV?#U(rH-1k24JsO4+y=QO=q$5TWfGfoO z5pFQZpYnD9Dqe`wt`7yEED^AZ_jBs141`)BRVCG7%gI}_l)4bwY&DjP&42MOD^SaW zMU=KV-1kk8A);Iysqu@$W2c%M*WJ&({NRH-$zry*Qow_WX!!zd)q%=w z5ZKMJ%9#F}dztP}yY=h8`J~}`urxrv;+M4^xAkASL zHqeO7ng4<}{WBrvH~PdN1b8w*MuN>rImh!hD(i^!)L$#nDgxQVSaZ4F^z>1|OzjgL z%`RB>uR6o|@J`6dU`AB8z86_fvl)`2V0h-{@@4QB*1;U1Y~zJ&amrXqA_zzb29^uN zt)v1_>p`R#L=*+U1BTeVxshKoses&PZqJ{)JEWhSUN>;U+=b`H!4%~cZ^K6f+^jEx zbQ(VeN%?_qo>J_)eayoFOO96D7XexBlEISi#xJgN(h2r8z2}Qw7qTV4QMk zh%M~N&W&RA7Jz79Pq+CZ@6($QR)4+k)=pnW^AEo2Kwvt*-G-Ejis(Y2L{uV0^=Cf0 z-4k97l+nKj{9FQt{)pZ**x zKCn>dM}PEm)!dn8Fa&v;mPu|nGlbovWYHYbASd{FbdmS*fB@o~80T|=FxpC#?HMLH zz{CT+6&NAiX4xD67>}m zc|5DvhVhoYTsBs4m07S*4Bb56_ft}v%uHBIi|Dqtf6rTq-1le04 z#OATu$&9(BJM}^44d1ldj582#|F=&3Uo&g|@szwAiH?uk{j%~>iuQ>G%S4@6l*@h; zaqmq-W9tDsi80X#8JX?!RkdUh0?LkGaU*0$s`@KuX~B#Ec=D~b)eSDy^>tr2&yL$i zP!nKj+xM>eqQ*L=OI3vA=}^u#G121=hfg0v@yUer=txP5G;;?v?d$Dd3n2`K{(N87 z$o-gz_gk6S{56!5>QeWUHBn4MWe3N>1ofCavr?tb94pY%SuvQQpa?Dh!Uik{vY#wx zMIHB3T3@gC%h5m=AE;bcF-_|DcoCC#zU|9q^qsEGegFmV0)wa&WUIC4)s<_4@JVCa2HIWlRckIxP7zea)hzv%u z5sr^HB&AN}?ggb(^Ysp1Ewy<{L$5zii+3mYhCD0kJy|Osa0)9TbGyR|fTx$lp2)L$ z8eZw>{{F&Ckm-Ismuc!>X8hk@dA;eW?gKpWP5@&XQEbhRM)UJr0XjX^!@*@g5Gsnz ztOy!0k{^qIUio){^S?j1faj1}?kXf7LdPSGji;p1-EKQ9D6#V#wA?c73+a3gdMV1q zDu}Cuf;jC)J3+rDv^juXZ_n`t+HVp+$Qu#2t7iqRAbpxh7R zKI;izs5b}#YzA0$VZ!)NFvx>xQWSB&0shW6VHiK%1g0jRzwoWO*@P_B6GK`mx8=`V zX87e&XG@7?(U7{BQelyd!nPPNAT`e?TLS;1pFHz1F;_iF@y}}{yapBzjPXYMKwl<- zbI5FYzCF?w&K<~VILtp@pfYB}iWm0(JnC`)1X#aU)Q>If{8;pghf_jN-!C#PO_4#Z z3!k$_P8q5ez9ZT3RgdyfbMhN11#7KazNej{#6t;K^3LbTdV#cR2-bp{LhH5M47NF3 zr{;yKM&u(-+ruEu9oj}*mMR7?zB_g&j1@O}DY66i_pd`lD534WSztQ)ur+%^mGgyz z#XD3CGJ`=0((#k!E4YMuWLDyECuSE=ocz-As9oUR`Bo zaIt4tkYVg}SW+O$+;}Mz@CzZK0e7~Rk#STgTY8#EzIFcNcdXCKH|Jw$B(eEo38_wJ zSL5%OYq!dbxJMkq0r{OIPcj>eYW}DE2T!<$Tvz*Avp*zr{E#G;Kr$v_aseFpg8{oo z7`US@11b~6Z-tCeMUh7Re{{wvzZp&!@tUp>lgbltQ>_%oKA7wFM=)9Wl&@I#Y-cPt z1Rpo<*(NhR{fRUmkA`H@eiM>t41!{L778X6>`0M@*Dz(~>>Q-E{8H14UfHdrcsxCQ z7u;)ybJSR_>ewFFUF$*iHylOicJS8$NM0sIUbFh^ClI8Is+GPqD_;?2VJ+WwH*gbQ z?xX75UHatJd)!-XE7LR!kkx^Lj=|2Dbt0R@Of`oH2C*-VGp6{l-p(Y@ai5{l`w<9@ zQQ^2GK(gT{?ps6qUx5Qer;83+(Xq^wr*1COCGX{&c%$6{`-Au(MF`@)4DOA-QE!?a zt}uylelC8m*2ba@l|YP=n$OkmtIO~&5qirQUcZfPaO}=D2iBuEs7gV)&Cbb_!4daz z!rR$*k69WJltG@}%#>-eG;g)rzXk>1cY13pp+Im6cLb%uz9rHYKrKm4QAYU$MNJe0 z4NoMDQbHVL8KeIFVbS0Se*JsgK$gb$dI$IBdeZXpa;B%m#6-Phrw^E*le8yVYy@d< z3h_Ih&yki1DekDM#WGM(Am8)D!a@)eLHTHp91I#Ac3g%DAm*yf7;vEPR5GI0*jT zjWdgFC;3eA*jc(JQ&KxJB49sTN#>X>eR=CG?wQx7Xmx+h?Vg2IpM1Ui+{gX4_gzfk z(MY@SA;KWbjDwbrhzoWI;ZZ6k^-JXrr5@?#oAGr2c;ks6jhG;3MqD6bfUWLUg6|{?wx&_`$GZrUQ}ZGI3F8q zk0>?UVB`7pZ;>J8vaaFMek!l5`13RfIoWf!nJq19 zax7cM(n8!LaSu0H2J?`VF`6zIml$@648&uP$;qhHr{_^w&gj$0rP!VJmTIsl+)1c! zfKYHFPb|8MCX_n+D0}%4H6R{Da^6aG&6A_Bmx`f79A%lh-af3Cz*i@9`6bv*c$Cq= zJjm0U&MQf^v$Q0|^kscyh!1->OG;t86Z>Fv(%I-&Z~ZP(04PuAQLV_?~sGBV`QY z<`NJ_p`JOm`y*$AgtSgc3b?=7pMS0?3giyy@uV7r+&;$MtL)^%b=#S<>&uqsy6|bc zlVqGLzSBE86FdrY@bLJ|MzRF>4hd6=(H>9uW=Nk+x|zwq1SF-K@Mg8S90%i=2kIuZf2adm*6S)v?_sfZB)vz*-n&8n!P3H}VwLD%koWbKGk}Q82z#A{i z9835%k`)p;TV;2u2lIj>agwCzBSF;71$bmV#yWkJ+3e+qkDh0@)4G#a2Vs0jqq`FV z=D4HS3RRbgG26LvXFCCJf^s`n?O6;z!d+=dTeD{*=a`yhq>Zz=G(jk0q{$-x^W7#Q z_X0@X$b7BMjl}9sh1ndd*{#Plb|1jUChy0{gQlNtEug;nFbpEtFiM4aU;g>r{&k`M zI4}BQTKoxNw(rjx*Q+&ClTOb9yF&W=sa6EWvPy+fwvvy?W2o)xPBYy1OX=#?(XgeG z3Fbn~+Q06j90N=-{6pB~*m|#foAvO?S8vF)ISYp|d}YpGg*Pqp2)o(Z%&HpV4K1k87Mdhe4ilzT<_I+MY$ zL|=sQ!2o0J#)_xM&i%UhSjTD;C5l4jPyFrvTY-*jmy|IWPZTA3VxA!%Y4#B~@P+OT zk|@mwqY6jx+$w@M2l}&rvgl6XT@}I8pPxWs>{YGXbLBYOy_m>dOS!#B9Iq)Ivz0Sj zobP=|j#OH;{dqFON=(Y?wJ!#8ncVw(-!IhFO0uo(Q(Db@?@;x6mj7Bbrvd6K!bz<3 zydo6HNu)onoRz79^D-q0Ph83=^ZNEf;Gg%6D3=#-k{bmAM4gvTGCUXKzF|`tV(2)m zrmtwVni4nxsAI`nmi|BPr9YPJziuodOsEo9c0d%Q6(n#kXnrDpOf2`koGT6}s_p7e zlWauSxOU%h4%08xg{ukWAm{3ILkZAu&@jcWilSy}cWm{3mS__igT$E=o8&~!&#QHB zP-><7Vjw9|#G*+>`{nZLkicb50n}sx+-U9AtxgpeJ7+p!_{SRnIW4CFMUeIX@8K{o*ta7@xT9Za)-j7}xyQv>CF-~8rK6Mxw0yQKiID#O#|@ox3Vx5H?Dptyzi0BP4bK(Y^p zYZSoo6B{4Y*{e16!D$Vskj{|k6w(=59uF*3t%z*%{J8>VO3Z zz;Jrl?aqq7gbtmcxg8YyJkqP>)G--{FY9c>Aji1Z`?Vb$F01YFQH|ejhziObkY2JF z$qESz-?mj4Z=?WshGfnbm7ChFdk`C+49sRuM9wRXQ4w+0Q=8ltR%`P4D$9fgwJ&yw z=oOAKLvZWN!@~dY0w|o*o~uH_dXE|4snlLtCiHYFv7`6nsJYslo|vHB-(|a~zChSM zwc_+ACuA@d^mQEN9I;7Ts{g7pU9wAJ7Hm~oy~7m(naj%|6_0Hh z5y3|9n6G*V+^PI_jwOaW>9rcn$02D zn_iB3LpzVCvj>*y^4PA0kgOsad_TN;4s&g|ul&$xe(#8Uu@m{QwLW9;V0*LSEODDE zxL|wKNHQqt;L^8wzcq*$ORc`$S2Lw4)cm9=N~d(&Wml&w(SRcQ)f3h+GL_=)|8al( z4{B%GC%D&GYSv+~H+xK4m(1d2d!3zt^NQayspP|KOzM3OrCp@tZdEe& zK1BT86@`jXhbFu3nCl<`?KWRDEYFw{74iMsZ?D%tzN0uKG?|3THIjRfj2V-* z83mHd4BYJXOufd=?|)_TnOK%Kjjs!q(_8KoiK6DIPAAl%d-fCkrJi^d(l;LOq^X95<#$n?bWZ zTN~*p8}(WG>#U7Tn%kvSEEtV&yyep(|K*!HlvjNBvHY|`H1%&+=#O*qFRx)v4wNkAI&?a@6rSxM2i@6GPI|(+Ejc}RVC4@x9~h+D zjhKD=W$5@A%~QEB0_V9AP?}X3E9hK!TJ9EBd!O&-yUUT{m1IWf$hH3x>#_aG~6zV(?h&a#2*hSA>_LZ*#39T4EV8%?C^S z7RxeyfU%Zmo?_G`Mz8|o+>$F3xbb%==H{LA<3EH-x_=jE)cJ*W1~Q=*QHm%;)pDlQ?zP zBSX~{Hxp>5+UzV@G|vpRwrbp4b#v^DN-Wrd0d)eN#g8AaxEWGHT|NmcH|7CMaLQ4H)H#%K+oh?ZXKdc{We?PX~X@g=75R+^= z4M4t96f#KoB$+M}GhDp{qhhMBWu_{x=l1=jHOS}DFQKn#QEfQyqjpYxqi(O|*eau> zb+Dn|7p%OkCh4$rz~})g8v07b5F_dtBH-F)LTDLSNCz-i-h_2!Gk)@kKPx!>+rt`MR{kXN3 z4BljR+!?Rpi&O;{eHW|c9MF`Ja9%3Y^W5kech<9yEO}TA3nJhAKEii8(1TTo_Otn0 ze!Au1_M{0z?9Ply+#RY3H9FrI&Q?6@e6uL*^v+zR=3S32PBKR=9Nj^zWVA%*Wi(-&pPQ6~QZ@+%rpN=naj2a?tUT zhmb`hW=6a@RVu4lkWsQ*(DrI@#sFF%j}y$a}I{AZ3alvLq|nH-VN2Fp7BHt~7|bRGf2C zh#VZrm0V`H@S&RS8Ab6q9e2Lsif=D@91x}|(PPsyB+Jy=^2`+{W=K+a;hG;GyY)L; z3x!)Sx!wNgU~wW%)GrEf0{Z2P#yZ8q`Mk=gcjzoPd&OlJDrKbn{)C#6JkHmKH`XRt z`d54t~s0hhr|BBf%B~19Kv%)sF@-1INg#UmXq1(717)no>Bb_3TULu)-&^oF{Pm&-wQDcg9wr*O|@h^MgsZip}Q!td@NMSypAdPG9Y zO@Dw$W6P=Y8C$wK&_x{>HI=cC<(*}P95IUrq~q~p-uQP76q zD99cTwxwEkoYm?2^RgUTp!xC_p_Vj7ZmODPH2h!I^(B9bu1E~!>3msccxQ%_ul}JN zew8_yeHi@cg2P>|i4y%mqXR07iJ}nb(KBUuOR>~D(PlF03VPT5$GC(z4b6v>O^IWGS zFP|ptfDj}B&ixV{tk$foL|kse*F1H;s+HdD7B>o(clN6c;&AvL4Kb6&^v!GJ){l>m zY92>*cDv(G`#}|d1<@%B7|CdwMx8(`0KowJOB{4Zm8gck5Vw~Hrxyt-V zfVhX1%wH_J7$}D0AmE(OS#%6m3Cz}vEGzja7PBh9!(Hd|nJi@LQ&^Y#$WIesKO&4> z`)kVE``2j%e2xz4^%0+sd0p-}87C%7zCy6!(?jBZ-$9BXvRxU79Ie`J z%(I%+m)Nvg1R3XrRt)31u zayGODZ0-(i^^)b2;@NQJi!~!y!{{^;8uFl!J4j)@@JK)*ipQxUEC0XtegE^WdIptG z=!TJ@QVzMfMBl8Z*dv?yGVc8%ryU-Y!D8bj$>ZDuft5l&+QtT`=m?8DibJ28(ORwZ zGBVDGj@a53(ao03hpWu=I1g9s=2%_nLgvac!cm-3%3Z?{5obab)enk9lJ8h}61s-V00UaCww;N5 zAMQAQ`ei8$f-Xvb!ov(fBrDm6TO=$`PQx&sfN7ywJfHg;3e+wPlPQ>;{9^A49pN2i z(ug&DyC0E@&eX}qli!M$d>wDTF*R&LWeA1F@46)v+_|5BLcTaU60@5W+hDWFO0IL5 zl6kegv@|ey?us5s1Z5on3R;_OJ+V)I{}X`!_Xhl*4Q@eU?&Z_O3eva=@GZV+=;OqJ69Vzf5Uvy<^aZ6Ql z`L;U|2)*t{8oI731ZU=W+=_<79AMC52niMY{Kc1;XL54#OwI#8iCD~j;y_pgVBQ%+7ruJeWUWq?;T93l@!7f~yBQw_h zX+e5-!AO0cjL^zA5&*1XX0z-wjnhU&L7rlAHA>(?D87CB&`$dHSfjEpWrz=XG)rpI z&13tm&x`sUFdh{_nahuvQV%4NZ|a%aO6pJVr|ak4TJEv#GayEh=7(xnVO|GG<^OSo zOXLsN(uycSKa@=PPQ%&Z@YhP8-scguYWr9nB3N%uTg&fbMluPJehLvJ8?avdEh&4& zS0j-!UDX-4BO+Wzs~y~6SYd|20->`g9PFJ?ovi83pXy583plW2+4A3i>+J1en=~x73p?864L&*l7PbXMnAk?Rnp_h zjLjg;2gw@HO6z}@ty$Vrk4g1%?|%iKkla2F@xuRUX8U65K&!^(K$zKD zj{uYgmtCGirn4!&KYeGM*#_5P&P^LB)8$8l#MgcsLa#GLL+l^nf$l zG_JYj*jkuDSfoMTZep*i4##8~-}S@Q`=D+&*2s&S&3K|pGm{Yr6^x<}h-oKlXWQQ; zMk&yM@8Giq>)^1aJ861(WO@?3TZ5s&G3iU-;7ue0LsfNf?6T~IRCi~Noao{1`rzUi zCx7H|O)RS0eY;b;b^h?gG_*dl5tAJTY&9dS!uT}bvn}Qv>Aw4NcRAPAv#?%{(&G$C zvOI7rM+%McA8$;4SW=ngB@e_g9`lpcXSAB?RNEolC$u+cdJaf_1viFslVTzSJEZK_ zVCE5QJW)pP`Rj02Y{xp}8Pqixlw@8lAH8CZMwtW^{#ItoB!#;a9lU2JFc?eO-1{&z44gxXI?7f`&i6 zNs7kxz9W->9B*n=(t14*{UHR8*+P@I(lEk(OQ8m=bV@RCAtY|rl1nMs2Drb?t>Fr1 zm!1frhN%RC`bU0d+Z=!M`T;2dncFV1?I(XE+SILNJK-SPId|H|CO zW%Cn)@oYr^^iF+R+9DWCb+2Po3Xu5nN2B+^K^y^Zh`#7UrnHZj+ao?8W{m(dc{hOQ zO1y53*<|t?f%A5%JYm`BI-BYAJCF@0Ebj&k!&=!Uu|A!k5kvh~1J;YKTyIZ1gwQ}n z_2#^O+oPXaPlAJFj|(^?Jwn{2_ZVto>AmwH=HRAkN)wSxuP-tYxoo`1SN}BIu32EhABJXEOR_ky^eF_Nmc*{k%9kRv%)`s=FeZlkf$Qz*I z$wl88NW|5y7N&ptfa#vGaFkfE;(Soa&ACT|dUBXWa)iX55};TlUmuw^U^=HLI*}*F z7vC2>TAm$~B{g=18Kb$YBk;wY3OTN-p+pBru1??__a$O`%R` zHVZPIiACSNe_Hl2M71k8!c=S;}k zO%}pv8kM$k^04yC;N9!#QXqb+x-3QrJhr@XNZJJ>Ox(E%4ZtX;Rj+wR)hMov6(#zQ zB_<+<1a8)Hi%MKzqVUl@@bCxiMgYwd2;GQsEvi`!>dA;&J!1p5yz5p#0;UvnD?v6U zs6COVa8J}BCH0e8YzTsMN?hXERW!h>0}ZyThp1qHn>N!*Dm=O8EFv%|q5It2;~7X! zvK*2@lKeO4AkimCB*mx-LoYrGE~Ds8@1v6Dd!6B{r8RoP$3o@LDA7ufS4%=+KWZP0 zr^q>%pHVz<^fj=WZ1%*Fbux*_geFizRxc>(s5Jnq7`mU_9P}IJBiV_YY4lBRQ_6=f zPfUC<-}rcOt&zoC8_qDQYmvnE$*c8Ntr55sL_#+hGf>VVbToo4@Nu=-bnQpqX@YxQ zzii80A2o#Y`6nOasX_Nc0Q+Hn>snnu`5{gE*5ZV&Z=y6k7~kkjs`X@uZ?Au#ovLp2 zWiq8JaCbb%Yq(BZI!*A~UC51${4z8MW=Q>4^Qq?jG1$YN5M;oS!hB@E&JG6gwd{M{ z?hjlq{reI^m0*KJLS{9PCKv`ey2(~GFkc1s!$C$_V_bmvgoj-^ngoLn$7yj@Wq(h{ND z;X6=HaGB%<1g8HU9s@6(()f|Q0x$*>`dg456C%JFm&N+@r8-^R%@<^z2U}jZM<KSa0+N{?^%iR+VYDqhvj58e#7(IznB)XriVAEv=}6JP3SqmA6#y5w-X^! z;3uBLs73`aYRLK$#a zWUnGp?~~f(fL_ew#5{)|8XD;*J3|A3 zLNbM2j__?NbFQm!{PxV42#wSjOwG}v3;hxX8E7BK!qJf$a&_u~Y@`nGON<5+IKD(D za@vV2Oh&OL%kbX#TF|(`9=f9aal-urqYeFS^EJq1_X4GB9W=XCaIS#%x#{pdjPH>> zYKA;5w_?K;Gnu>*J-pQR*E2F$G_Ue3=QKu%zk`WkS|WUxDm6PSsk!A`*L@% z1_0zg!#y2YzrTAlw=IEa8tNZDS47#UF<> zrM{L`XBS9U(_#=hvz&p5z38wc_4Lqv#O9b58>2*BVU7)k0Xh2p`{r|TykXY&`{W-f zp&B$FFWIcuMR6E(pjs~^-k(B&EEpI-*8y6MgwX1SKoB ztfg6gR#bnO1`<^wmI&t}NTa%|`rt_}fQn)a*P`Kblw{SP!?Aw>DRN4nxC#6z&vVq+ z&e9@Z!mCgzeuhK9RctF7nVgt&u}HgGq}EoNKSifzu{hLwVZyfHkbLXF-f?w!kad4Q zu6~8W=m9zbaIT8tvf+TqLGCqSTPY4pPlD)hL0qJgCSBsk@VqtE$e2QVSY}hh1( zExO90;+LQK+)D~~LPu$|5*aB%IyG#MmrWW_NB4sv9V!#ONA5ET!iXs24;3N?6n z3IIHE+XuP8>9h3|&s7i9chefA41_@R91KWJnp95H>5>l0-lR`Z1n$?Lok1c(+A7)m zzn(n4=rthjLbw4JNoXmUumyzufFC$KYuWczx!mm4lqs83%9FKd39_d|8)l!nGcQwNBFP7NO z+JecC{P+jaL>Cl(e1HORCCDrEa;+SDcPXeeG@q@QhFq*(d^sG#c568Ba~hJCCt_M{ zxo@sCJqAG5or6tI^5tJH12J`{NUmf%tfvcQg$D&WhjzgX^Cur4KVU(m(w#^{F3fCB)#RLt0;>^ZdDd*QI6;uJNa#&HpWr5Hf5eQmps4eTVL0%Re2i!9tguXNGEF@{{mJds{8%<9h_za=8M!~XIgn_3wNMIOwAK~HsZHV1Ab*H_ zHKDu$Za-HZgX(cwz9j*+CA{Qh^4p30;!ulZ<$}@LhcAlK%Ui^3OBlZ?->5Fuzr) z0JYMtpUJ>a4p2W#>oh2Li%ncUTse%MZa5Pk4)NiR=ZqYtH(qIag7QORuIeDPzoY&c z_Ur!gVl{)~(@Q5x`uFb)V+Y$zCSO$;MG9Er(b17xpXT~(7bw!cU@=-vd5&bMxUIir z^;r0Rt!rj1y09;n51-LQ>p&7qb#*OFb(@J#n;NI6S@t{5Og~f(0ZfNUGShry8LIh%JPEG)@ zGAK`dD+>b;*?BM3dte{$fz_)g_Z`a8txlcpjJ~&k>i%&1ZiJjHBj(wxUe$u`2qEV3 zT$V2*l=$P58RC^A+z3?53thq;wzL_cu2&7bI7Q__+`NMLjhYsUmt-JyfO*ECUzw-O zP5yky5c}rlMn!JG^s&g-9n;O2YR;H7l&dTY*yjE7ZTw3O;Kc|<^wzIN9vhg*(6#FJ zM^6rIeoFor%h)LcYC~Oop|@R&CXLQWyQ7_AV8EZoYL+mb;^mX4M6sGIxKrFL?pLPe zhM55k?|pt)53$;PUe&G}b|W=1eSOlqhfs2Uw?6|xC*m1R16PKg*{CyKC}F#uD$Th9 zh5f{#Vz{$zkThujOuZknuWfnEtG?cL9z6EsvQhFW~4*HJBF|GdfzrJ)8!& zrimjzN&l?1e+8bRi4wL#est}r3WL1a-ojjy=hb?ojZDkl3A?1gMY6~HmVA;DF7rJ_ zA5EuDnIT?W0OwX8zOg^5yu-f@sb1{M_Ce|^x?4F!>*FF0$MZ_~2oLNXLft`5c@A*# zoBhG%)%EG*H;e7Jm3sYZGXU^3j@p+%2v*#p$J4{{+p~wJyTg9$2Lb(6aD;N&?V>!Q z!SzSeri1RJqX2ui3fNAn0;Fe!SgX=_!VpNQH(x7P<^aRhh9B)>n_6kz;u)?HcQmRM z0WV7Hvv~`&nt2%mZ*G9KyF8=WOc~ikv6fbC*g_A8e;iBTurUP5$mbw}`M97`>YPmm z+q>p>XwH|%fq1rT5LWaN(4NPra5~zzka@2?El?~*u(e!UnZ(_*fv9}|BVd`BV^BeZ zz>e^@Op}PyN1FSj<697w`^^S+Rgo5^AI<@S8BN47NYsh73!9(nm4lUGU-9iRoJ+>0 zQRYV}_xImV3^}a*Zug!R&5TH41^gnJx^6@&h*|`BV6OV1 zzuv?RZR7qwspE$(WOK-2+=u4{TzKC2ij%eVH#%Dc7(Hpmr5*Kqv)Of`i13<_l56KvGn>#`1HVpiV-@%ipo1R6ji`xW^0Ptsz)P>s{ zVIID`El{szJg@pi-9?gj37&;sDm=U$fL`d9IS zj7eu;R6k(FSL6>~yeZSW)FbihP!}L38Us==^X zb9FBLHt4Gw(+%b5x|ha|li^POq@;co0nr_h@b>u#*KGH1$D7c0%diixPa4s5q2_H$ zMh7dWG)>$^0)EL|ygqnlmjq6NB5o_%l;o~!x^w@d%UAj0e^cN5*NMu{4pD=x>IW`3 zWiz@fvloP>5Akc81DO|2J7j$HVTWr5&6hzvP5e+I-Mnt|_qb`19%}Ma_3myjOr8gL zY*~+lu%YqAFiBE$kEBanJII+tr+()?ff8zfkxELyzOv9~Uo0QGg>!O!YHVEd3&nNr zlnJoef*IiJXzeUhW5YJVp9i#nTG~yGI(d(_6(gG#h}|feDNI2W_^gvYk-A#oo}-8~J*Bb4DsP+#6CTW~l2G zg-QRr!+PXKmGLEDdXrAW#a9q2oa)@j1=d)TB(x16rX=K#GQa==L25N(J zt!8%^c-<(Y_Aqqkp*Cn;U`GNIyl%e{8N@~d$?`i#)Q6CJjt;)B!<#|3#S;Q1m`?W7 zx0*BUk+8A`kXQVTt9TeGl0rstGw`;Ih~WoKkrGxL)fq56$v$vZZm|GH@;rw&7{*+V z`lx;%8o{vMmyypi^@~bkwfqZ2--H}@kxqYmEV5mKJ9q}1CRe><7Jf0 zMIQ+aFfP@q7*JLyk(1$3a8k#z$o)PSI#*+r4*rY#={~`U?j`Gf+WJe4_Px!L6a}Ym zDAZU{@oukz&+QLl`q+LuVV_>(bJ_;ny)%4)&t)&a>3>@yDm-_US%#PDU2fRlV34we zcv9gkTn8`|TrT^4SpCl>HDa|#!1T11I%I*3rh-XX)L@y{Tmfl<73&NK&@U>yP6Wx@ z2iTShXzqa5ShQuSBZ;3^Da3A0UcCYy{3_ykJq^OQy=(5z!cGO?)aEO4EkG+LQgA}R zqQ&9nXfd8*oU?q2<8`ZRB{A1`S9A^KJ{5g4<6a{9dG+4&E=l2L-&7Y>U0p=~Q#MF0 z+rp*62-%ya$zY~yzPI-2bg{yxM-Lp)T5ZC%z*iwMSyt_Ih?9STW$<%JW|daX5f>N) z#(SL^uz^5E@**2NcbW!5PTQOX!5Qgpx^}TBxQ)>TEyvSfMvJ6`>U>vjj?R`>5er}& zym`IM1RoayPR!}$UYPn*u}<8i1f65OzMP|hJVxK{(w4az5Z;E6^eoDf;kUD={(}?a zaS``Obtoa<;;;Y!aO715BgZv9918clOxN%MS-;(u+0VNo5o@cEvqN(JxO z>`iw)mf-QZBBq9T%jtk8yyjYA_5D3pM3Z#H1qfo&S@S3Se%DBa0*f=ms|M;Jlc$=i zdEI0)F|pxSOkD=!MP__YJnK#9(JICejQh0~6 zV%@sK?tL7Wv$?8ME+;R^ zKkB-R(V|;fYH93heX%wZ_~<%2^VNn|$2Tlle|^!Zx&}`_66GUWNfJ({OdRPCG*4Wa zjCL|St^*kMtX7p=nPgsW_&t~U`UCN9gCSdA-#fhdw&i)H*KNPzn53%HK1P1xhr8++f~i^E8q!a)t)&T1?gb zEQX0aSr~@>EDulP%~&pP`)6@%`A~zpij@-@P1vWJy$6s$(YV)mh^UUck`=hO9|erb z?(qD!;2oIPJ4xPp@0k@mdm%G1I&Y~WtRjo5tS(v%*p~8&mdL!onKFHK48L_%%{!k4 zUM>*2*oqM80LgsDvdI_MNgvMghXFC{$^=!5#u;$+TGzKXEh^WIQo*TiHM#~vCJL;o zw_%XopUz)cewKFNZ{~D7)-F%(Fh|)+ao8B7#>Y-(IhH zzLsgR(50slt+Wqt|cq0c}dU!Or*WI>b}}Vr#;&2s@v#ZQ~VP zou%tnUAG4PmqWPuCfW_1kmEI7w<}Kan^%$KQjAJ0{7cWXjq1H9>W2%f7mbTfy^o}N zVZO;Zm8Cf~x4lALN?e7rv;y^S{ea6g(C}gp6;+M3HR3$z0BgFnTe=Vz_FMUhD~xYb z$2`zZvj-Mdjo)zD1{=s%GHCx)>ipYHN`$hPJ)k_-A5Vg3S=iR*2l03?+24$=l1a7j zRk#iqco@ehCJtEw%{ zyS-wV;+p-9rEs?fNV}UIRnWH>C#rv9sEVd)j-@%(Bm2(z~`5zy8fUfMz|Sow?WbrA&Tn_@%O3h^I? zS|!lJ3b#Fvo~S@S21c0KVA?k*eHv?2B19dX(D}U&lY`&Wt`!aNF1)UF=+cf zHG@OsiWpceuitSxzyD}P7gsc!)W;qWO*DuHh6Em^68ruLH0?-UgR7mvk#sn>sUT{Y zTKFnZ4+)h*dFUq#b;LsX7ed%N7$iKypr54`E@co~2m#Zx!?vKS@07)k5Ymt_)~2!d#RYbyI4PRYSS73tG(!C~F6&@#B8`>qYWF z70;l(n}J3rCK{BqclCjTK%v)Zd(XHMXOOhAF()nNR1aD{oX=?_U9ZvhQC~3|qI@QS zYBC|mA(Ga2u=Y-28}T%OS&|y5`u^bV(_s130adCu=j!P8?gsNhQvO|k(aC+w7M^>+ z`E8LY@pR|auj3jGT(gE&D0bl0LcTf6Pa^iW-M73kq(Z)`65pIVE0|xHpyEdd8ziU4 zuB=BUnz}3Y45=(IjCDmm(*b&z{_K%-o{q3y+UFf3G3u-Of{2aB=egPU_cV2}j>O9S z4YG|9*fD8?iH){X8oXFeF-N87h5h?8lOLWZas>-feyZx-@22(}1$2?p;nBs)uib@V z+^btl3c*H;4IN31U22OrXN6%Pmb%oh|850~jYMy$E}!+N5| zJ9AH4N zB_+}g0!j)x?8$MT2hei?(XjH?v73KEuZtf*Y(CZ|MVAo?|a>I%{k_X z>-2v)tG1@Tr|><`luHv0`brCi*$l;H8D(9;zReAyuENYq%*Gr;-d+>3a_*qaML zcW3iAC_8Tf;uvW$$7H%OufiJ;+CF%upP#H^F!iUgdSe;g$(%>%w6T#l z9}MPSmEZGF!?mHZM@_MO(}DsWbr6`F-~agXQQ5j-sS-d3@`8dsM&C3kttI>ytSMkN z>}r23B)(s>=SG?#qK=wFwZ3Qq?bIQ4$>iRt$6J}lJj5IRV4K2}X0CJR1i6r4SVwB& zRfKimm7)=GAbguyr00*q^QdP^xFFNmR zBGxzhA3LsGg&y_AtBVm=oA1=S*f`w!$OZRM8nS zc=T7IU6(i)0rZoN75+@oa^ZlR-TeZOhlZaHBzyDyk%FqcJ2}To9*rTI_P65>iz4Tb zM;yLgsdv>O7uM^+(<5wexQY)AB42oBrhFuigtRQB2I1BfYkn`C`yD^bJur}Ma%X2= z0ymupRP)2nwHwLP_=acGj9P^Yd{Rc^Bu}?Js~7m2vDhzp+uCGhd3}_6J?QrXvWEsn z{)Ws6)&y03;;GYN%$P0;Rh!t%Dkwfbt4IPWGKAgC$R9C3L?2@ujPTKyfJlD2@WW=T z>e70B!QXdL*ooecsa%2YbrV&CjD=x-;n8^V9ZAT~0M2 z!!s~aoT6^v7k$B{Cr2zj7!Neq@Szpa14qfyP~OoapyORX90KQqf2SA*s=?e%ytZrK zgFiV$4;!9&Hu|%1H2i<3v9-`BAxl3o8Vi2@)u)f{UQiE`Enht(p<}QGB@gjV>g%Z zoRB5*Oa~zLxZEZsX5Xmn6eEx$=MPkDS%k-eni9&3W6orm-oA_gfdTD~!A z-Sh)!O!#W576}XmHD~xf?6Cw!a+^dGPWGYAXJRt;AH6tGv>DgLB37!Xw zf$TFWyG4ST6DyiWJ5cHB*7L0|huC0vn)>{F!&sDyH6Px;MhN{9ru2~n0v12lczN^Q zc7f%7foB^%bnu=Szewyy1=l2op!v0O6k$xGG9P6W;H?Fpk=l5uVFX1&IK#TE1w_4k z+<-#SGk<<5+1?JgSpANUHJdKX`x<@xWsz0xRpDYfaxOYimMr()y9?JgTJk?`j3Ns+ z$KHMH^%6MNgkl=rZUsxHz(-Xh!e}(-k5@-N(v7z76WWGMrTo_*;pYHs)Q^c>l4RjW zcnFoIV?t_`YaM~{YA`Ph#sT`tOv`Yrkk4$$YcP@fl@0SI;?3MP4G<+B7LdOXCB?;8 zw2AuOE>LjC9Oee%a=v@w3X{SB(qP>UiC>@7Zl?XZ`ByZ24hRtlhGKg77%q>tLk?fx_~KrZgjj9{R7-=Ce0t~ofHU8R{8G=BB2@x zzw2$$xpQG#TFQoVT3p*O;h&Bu-;#sBzyPH0k6rc{d@6bE;Z!Ic z7bEw#9A7-z$XXW-A`j^snkTWX27J-C&#X>mJjGTkJWyNC#Js(DP)})_80#-&BI~?p z`<$Cz9Uxu8YI`oWwlq;cy=ICvl73e*oTBwW^lc^HV*afc=?Rpuvz+LMHDIQp4w>&= z3p{;fVMijzDd`R;&St%In5mnim%j^CZ8UU)r*6$CEJO#MucWR8xh;UC)}uV9_T_CA z)A>fAOZBzvCrlLDn4T?!fL8*WRVfStm#nI@8H1su(}ah|-+4+DsSI<-j9jQ?Q;BBC zRaLVFX&&5taZz|*p3#BS-z8*2t`7}>_&`lWH;1a`*m0CsRk{3;6;3muDgE*#5q;Q*1;*%gRpzM*PRDG2 zoWDu`W5-3i8%{VL{W*u~Tcc9430=X&HVPYG|Ie>vpGqAHMX<`8mDI!>$0`Qj>?F|0 zzlVyZsD3mm<(y`TJPi+yqm1bRzP3iCtKZZ}9ECYlTX7`wJ&f8S+OWRIxQL{(;&gq# zXnW;|{7JPodoa}B_LXd0>7qa-rsYqX=JywZi64tUjCi|Qze91~?>t@hS5$^Xgn9 z8TrBP>mR~=p?_9vGsHHdy4^L9sj=NbVS2=+PoBQjtV}o;KBgPElUu&#x-G$=+(1B2 z&#F3!Q)Lx|A39zX!YszvW z;27E+JsorFjU5U>jaiUsW$VSF>C*Vl@xQzPUZ<*N4WS#Lfzg+T_CimvafyEXR|HMu zhXL$@$PoJ1p;F+n-yt;W69t?Ml->@9q}0e;{T`LYF@@o>QW^Rb)%zHnwjD#4mTMP^=w5bQ{ zZ-jG5s&>SBJgRMLgvF$@ylMTGs77k5)QW;Bx+|!2L&p?F#9_z#v(xwb)LrOgj^V@c z!>^@3jc|<^m8Ir6N9#hSdQ_e2jm~97jkB1NeFQ+4Wo}X$no#5}6ijwF7OYKvqu0|Y zH(uE+q>E|Bu~t$8_!37O8H|lYXQN6>!7-K$$FA_zmH~@LQ!Ra%y@`R4x$#F5&De z8F)=yd{Pi%Zw2v`Q_1mRn2zzmB1?Jt<>M^ih^-eR`*mPAI`nN#sTWF z$z`+N>0B-)G5<$6kapo^zpn(zp+4!1&Br!r8rN<(N*<4Y?7lw?3cUIYr#~uCN*sxL z(6ZMg7*4c|6nVo9i11C%B$;j=!?XTPHT8jKkLF$^F%=RYvZV}Yn= znq%^R8$Gic78^&n#)MT9ld#}0<~k9j)KkvI(DlTGP^8Ya`@m&`^1u}HvakMIzW+Tz zMQ#o8a#wz^g?JZUt-_?o(@{B9?s`^OM_G#xQ1m#Bezb_}yG}@3TzqfCTe!PCE1&zB z=G9WEB9H^#rQ^D;?qn{{r=z}R4Ak!e&x=-zE;%!R603egpk^H8x51@MQMkSY?uH`x zaex0g8llLV2yEs@s}^|aF&9I^ihKNwBXZ%!Zl{$wj&TJW_f1Umwg)AZIgX#-w$k!A zcY)0Yr76ge|A;1_>6Nmxb5aUqJA9;b$!H#CJHw*gSh6=^fETl(V!XfDXXF3b1*=yp z(#CT>_DHa*lTBbV)Vyw9j(83`O0^Q>PIWtWiF&}j_zUT5%DAT4A|$018YND`951PS z#OidcsbsH6)W9R&MGiaAobi=D{~KgRv|KlGmaktkgN+3>4$@=WwuM=mTuM8b#374a z6n1~wf$s9bzx?2jx0_*W`OpIyH;ei${(x!)C%aja+1`d+2Iyh6>fG7mEV{Y$*A(98 z9jVaY^Xaw28M)%X{WFw_T@Z>o3=s#;#vb$+AzSrWNUG*h;>Dd;eGr@_ z67{$Kv=R)!_YfYpJbUWXNN+(@+wlhGR64yK`-I+?t5f4!axU{zFBO(5@}Xevfq)8m zk`Us`A*w;O#gfI;h%fX-MH(a|1Ab12vt!qK@{mm@kSjmGEFLT8up-V~>EL*%aFh{l z(DtUxS*|XnOTGWbBFwOiq0BtL+;rDUZSe1+F?KN3^4yh1qMw%QP*#5|;WKF;EaR53 zBRjQYqB0~DJD5EOx!%BkTz~JL?yC$W-4i2w>wJs>Awv07P7w$#+>^nfFGQ` z)A#k@FY*l@pcCrE!pc~uq#PBVR=NcnkUZv znsb+|(eL9UiGGW?=*$2GdtRY za{qHz=n@$S)}GtB`eT{sY*g}wfPb-`cc}rW71{3HS>z-8od#sQe*$Y84a&oTEMU2`>afLryrJvZQ?D5Ugf~m#g#=B|_NCWnWp&Vr-fV z=uSKIzZG~s)^4Bq8+nh$lwvE%TxF>t0$}iH-c*XGrp6#N{@R2%h6&a)*WfF>545^C zMiYgJzG=}7dt31rH!e2hK$F<&%7rd~^0S#NR3`c@l@y|(EK@wy=Wl9ZV6+f7>KV4~ zl#!4>bEE+3qT;Usfs5G8njmHY3F#%ETUfU@=eG_N6=V!_3VL7l(WJ!$qj)%j0QlgP zc_GpbSN|IoXz6`kPAV;~AgUj2BQHd;4n%gE>+clgIxE46kVUNxvW|p>Q8@Om3l8AQ zpXCSDEW1i!ODn@_2c;_d5Wmd>Zf^O~)$3$pZ4UX!jh~8X>%S(;6v7MA!l9o90MM1z zNkU{YYw*3;W&kST*S4Rub09`Snp?h>*29p<@4S{#5oY~Y$ou8G({>aj*k78+7>65A znGG^yION)j!5!mjvMC9HDg41p_8a`4L>j^7RVJS#|DGS&wiO1UQ;X!VH6lZj84K>} z5H3wFb*m2R_+^4(CNjp;fyhJtWt0v0ymVe?OB^io4}%fO^AZSOyf17f<;6vhQ1s|P zvH{_2*)PQmvzSQYHHHc|5Na2@FcPt#$-`Nzr-2NBQrogl?YyNL|2b5ad<#PjZ zyB8gErCCqC!@)Q?&~40%B(zfh4;z%+ML2G!>NrRv7|fi{nhU)Z6%9}AWbZ|7N|nvK z!+3!sj%Re+sx8+nFI8lC`lY{jPy{af1kEZ#?ueFC-S@S2ckWBMgA3!tv1bnM6g2K< zcMc%^)7_rSk~-uCPLCs4(*q;k?fE4C=v1U$mB#G4pKlXisKqK(i(+}JEwuWOuqS`e z?FgE+Wa17SPxu!XF^DH%?%w#oV}pq@9-5~kKq?@9Jpl|w=HAJyh&A7|*9&uP;`>_MXn$DN1$!R~A6R-twajEqlveUe?n3eks zQbj@Pqe~&1bwvP8#iJMeXL_aFRk$}N9(0@uii)yZv;Ki&jg2Nka~&=yGD;Xx?Nn$P zQrcoFd&7yNFy$6BDYH{D%nq=10Py13J_bR-%Go|Y_v;Ue6(ALaNN39+HU=k_=m$ZH zn~vi8J4!`;jn|Tj?V3}66d_F*&AFN)x zf6y)JpfP>;+8E^mWd>S&%_>FMl3YJM%w1tcnMyYE>91eX@#3nEYFJH%6}gE!ikvrl zqpC0Dsr-W0vXda+EHkj_loMnu`op=1^6b8iOw;~B%~<(iUP`O!c2t#}q}6y_p5M40 z>}z)$c=$UHncGBJX}*_JyE?`7`Vb%A^1fofIWlEuFwuY~^3)GQbVJ)2;LYdg8mu>g zNnP3hxNDYhdh*o>oa_b14L9U3^a~AP{S-O^P!)_}a+FgHOidA%n)L53_x`-A*0F&o zVtpEoD_mR!o^tugAN@SYroSrw4EZ6v8|;{g41hQVQ4iivIi_cNmMB(MWhY z2O@z$t>et1wy=zA`Nc`=H9m_7-Cc#-3r(hc304$b#!b@U<|*Ts@!Co}0iTy77juU` zTD-htUV7{pPN1VNkc8r792*`x?cEf^K~D^+9oOCoy+xDvFl_yco$`~HrD&g(I-Cdy zHS?w8Y%p35V7u)ByD1G)!A=sfmYzcP| zr1A56W}7j%S89}qOguhNuF$I%3$+=L$I_24(2+v@tHzobqLzZ)nQq-gx@$5o zI=(BLpHT;i@aRVE^u)wc*}J5NEL0v~?u)y%>_ z6~%2L$s@Jh9)1NF2lNW>5S`Z6rFxXg1sri9Y{!+7bmySg>S7Aj4%7(pC* z%cY)&SL<>?VK$i?WHz(hC$#v@X_4C@Bv*~~?}zMY+3VjuAhFNyxUr(s9x;`j;q%2d z{B2^OAPARre|)Eyf?R_&;F$|_%1}NTM`B07=c~VU`5CC?f3e$vele$8;(f*CPe09I z=n3zRcJXy*8#K1^8{HIqh1RG-w2H@Yo3xUYwkd?OAaS~r6>WHg>r8||UepNHMwUeo zDyO~$hv!LzC}K|$4g$LZXrynNAbA=~9jAWif9EiZW%G@C)>=^lc?!~P_W)Jb0O^PN zlS#dom^6PuwCu1gO452&++ZE+(ZU8*Ro}Gz+tx@nlqtFq?hVlL5`vg;OcY(*t z39avR9fT>~97>M}L?K{yGN$TReMCqeXT}MAZSCB{IRt67y};uUc@=bEe?%U04Fue1 zO$-;m^Uq!-OFKwa#m^*Ue(m~olWs1xnj8|$^IjeBPhty@bZz60HL2vYaUHE^28+C7ebvSGGT!#Gq80&U0Z>L!h#W;9CeYD#NWO{6JCUr7%xh*!7vNtHqWmT# zkm#6C_kx;``b{BRV|}cUKCyU}8PfF}W{IVA;mbb9|6WK(MuVj3UY0aOb172W$-Axwa!*K&uV^5NXR~4} zol0JKH<=I-KQAkCXPw5lJr{1V%>x@+owg zM;Lysqus!$?-yB#eZ&r%;N62=?EyhHp7WKNujg%-t{us`l0U0Tw*@bY8%jCz+p`Nv z1c4mKad_NXXKdaYt`EB7e|g!@`VBfF+#Tr{S7TRDzTaQd(vLG3cEcB9_L?w!CRwrK z^}GKDTr_Z?+-mQNjswffp1x6&|1^gyH~$-BeyJY=1=6y~Z10~XU=ePC?)3@?-Gi-Y zTTCaEyHOEH-L1b(LBLwg_E{BHJ0%@nMuJ;KdCP~MVy!SRa4`zz*tkHYBqB`a)&%ib zabV#I1Z7?R+y0QqQmvvQAu56)MG0u+?FM+2V6~H56an3XM709**vT5I1kd?v9!b~d zfQO~E={OujLxd@N26W5;=6puAV6LLJ&?peLeb5pnNA|@s&Hn*$(WhrZkKYP;^Np%s ziUU1pimY@p7_V&dp*M3oUelAyB$B^QDU+rsOUJo_TgO0;MV%vyYTaPXz~gTs;dyfcpM^T9D6iv^ou|C zRU9>Z19xI<#7gjsnJR_KcmRJf9n&*Irnd0uSeMMOc9~YqQof8LIb-!nU1P%jH+md0 zI9d{){V8I=Zyy_p>M-YaF!SCXLH84ny}xJBI7D)udl_;EttpnR!+m;2IGm#6m;cad ztxCX;waxy|wg}APx={J5#+|jm$HN2ydX*Rq8|ePl7Si)w$GNQCCw;$ee$|#Fi#`h2 zseCSHcwzJsGSx35ywN?~%oMAGn|nTlP|tg=8xAW3hrRVp^U*GuZpd*G@8!dV3lNUn z9@1;J4!8T$n=)>id{k21)$B#9hkrN}NF(v$9Z7y`J;VlKpR}uxNDEC1=emV@aFrzT zme^X5zkr?NQqL#bc_ow6Lu3~=U4_DErOMA$>Bl1SUNz)mQbCq2QD6hVL@1W&%P9qBF}o$Je-F zh!(3ArDjs?$0dyUF=E$Ir2f+3YF@1J<{?}E{)Tim3{Y{U^O&ACc_>(QW-zrpjK3x5 zc$C@to6zgC_NzcWg0TG~{SpaX=dBCsjh95PKEaw!IDVX*Yj+^PUs= z@B5WG(!QLJ4v}{GS~?ys$gD@LbOb{or8L{sv^)B5T54p_Wy@~zh659Xmq4oQG zjOBemr72y+3vq>BpIr>Wf(!!I47vuiMqKUPjJvC;jES#ldDllN51IjAcVu{s_U5M~ zFvg8(S2bIp+WZZsce5Y!3%7kp^UiId{B~;h=kL=-Oyr=)+WKBB1J*}X5F7&Ih(x0` zLjz4XWMm>>OANU<31x!y3x;EA-pvts3Zf&FC?u21e3j_ovQ&?Dfj#58RByB)v|u(p z_%*HMeYFTstN+voCOgv1qrHcz9@lCU3<^7p*r_Dw@6I>?&9$0jWCvfd=e#hOF|^T0#z%SVJj+90neM&8e@ZU7Jr6hCrB7uPA7X*jUk9ReFCV%MZgI+1j1eZQMm zKj?$JPZ8pmz_u~#>su@}InD$SFB-FIOh7{R>s5~`MIyj+0#1yZ1ASFTQuzQ!78@ik zdwr;osY;)g^vrPYhMr?9}9)ntX-&_6`7nd&O5Bc9RGpVRs zRS%i&IqTU{JwH!bnr;*UM!#M>npO0zK;c!OWHDZblFPNTr4Cw# zf4ChfZ0hY2&6Ml;MVuZ)_A?g5XQ)3iPJ|EHyHLG}WXCkOctfXCyKPX2&9!|8k_Cs5 zxT2yP^|xpiuHabMvPY{Xg0X1cJdG2-*=rzMm3A;4O&83she|pQ48Ssm8Q7cW9WmLIEsb{yUF4bC6!UEwRD_T00oL!uKo@z@;!*e!z^ z^pt6P-dhJc9&C|}O_M#4-$nzEz?YCc=WGI``m`j^XzgSIzxVssyuVO!N(q=Ls;~|y zBgxZPh10coaMw=8#Q7|yeH?LDWBp%NIumY-^y9}=9w$iR z64`rCkZG1{wbE>~-XZY$Awyk~(EXo~>h$k-j}k2Ut~c3T8t)O?#fTs2ouu~wJ<&F+JWYx-;&Dyhn?`Co^dI-Mi*C$FG$b=JhztKHcuz}UPUGz;M-xOu zF1thm?tA6+u_}a2=v&+DuAs zzwWHDVu9w}L@2U5P#pcWKs8KFLA>v5d;J0ZPQSi)LoY6 z?g%RO8${sGeuFZ)-X-`{c+z59wEcirA0`FqC0C&uFH#D;rcM6yJ8Yf# zQQUW18zNE)8hU&bl_rr6AR7@u=RJL)_rZ`JH@SCwIhce8>)y!h`^|7@1CZ1cichEdRC-<-Ec!NZE2UDWtTDiI2T zvgm^Hm~Y`;BYu&2^A-bROGFOtT_)@RGR3R*?Y-ywOYVjWJ|*I8H)B`VmXjS;9@Z)B zP>x-C9K}i`~i~`HYy!q^~PtXMR|#x)CEpW9!=Pr(YNww;;b-PRqYhKJnzM}Z?1Hxd=Hifvc5aW zO%im$aN1fz%eUm4NZ+3yX|g-el42sbjBW3xYqB@Zy%olpJek=7f5uD73N+a>id26s z{@7?=f4iMEP8I)EIwzTh&Ewck;k;_K&t+a>3gaSxQ05C=@hnZ_0&zp8HU^#(?2^~? zr{gc7JVZVbAP0*=3%tp|_gQu@vZ9uIC{3U-I;YmO8IbXmUbbN#2fLW3UzND7%G=nQHN%&&o7w>y%cl6x!^T>C=t7mj?pon*5Pgn2`_g)mBpzEoPaA`+60uHjG)z{kN`U9NT zrW-4{$xUDm$*Ilz4)9*f!M?wlR2a`r46f?!rVIjc6N8@W3u$N8$puFhD@gy_>HN{cVsk2=5(`@F&{$!rV-Vc=Q?4p{~*k{Lv-0{6J@e%3!qZzgPrzgo~5&;g6 zBlGZ%2FxJdr<=Y^U)>K{45jVJ9$b_xWp`|~nO?V2RDpW;c6vW0oB8D=fOTSCGs5$A zjbuoPvp_dAkrn*OSleRSW|gN~zSN&_<1rsRxZ>ZeC|uotle)^vy{ddT_qa75gYF2l zFo_DR3)Y=^jK$N|`PKxdAz|RT`3!s7Mn2xzd!SQ%!q<^k1H6sl0_E}WbY08UvPA5< z^M1O5$f=0^?0?TZDY26i6>c5#P}I_2bI0@355LS?&O6+uUD59aMb!k~h&f;ztB$~` z39md-EViC)TGgOTp7IOUMIox=KWslV9JfW&an49MpzE?qaD|)_mEaVx!n3p-=d#p4 z*o>}bdxg;^%8%VCmAEQZ!EOL^wA5z<9_A)!(+NY7$l~STMNrvD;}(NeUcj|^PN?oI zh#OMksjcVJZ!%l(1i$K8`nMw%GcTqB3p~JeTk01|1TUL@4(^n-XVstU0IbVZ#52>B z+PTe_q&J_hF7{~zZY5~*bAi_jLJs0_53_BhI#EN&ytNRY=tZzzO5q}L?1e1PQRU@T ziC2>=NSJO%xme0p*?x9@DzZ4s>HvzEa^cjT-(tPXZzLe!j=8_p+ z&(um-P`haB0X$T5w!O~^-TAEABzv0AH%LcDW}><6P<6KlC0!R5w*Ov?l#srQ%<9={ z4`oXIUGhCL(z50AU`r-LDVYcl^UfzRx4t-W@WGC;F;DyZt6gBgly85x0y=-iGx+*DX%WcD53WaTCwp1*#$QJc}cXt{n z@bG7C7xQV(c_X|)hY(V$r|$K5RiI!2xC^TTI#0H1 z%%`B+mklIwHGT3OMHef&H#%7;_b}xoKG0;yksG-T;K?yF4o%%BgpEHp>vY=|pb@vnNn90$!yC7!{e1@@n=T6sfjH^3hm zm-V`vX!Fe%5Km|-EO$NXHR3oF$vxv}xs^WJmAX7B^H?|kM?oiAzsNmI_-MFE{VJvj z8hD{L*}0Ar@vT~*6H(y#c4Pu9A8;aga0EK}@q1D-R*fya^xZ7V`>3bvihVAMP}yh6 zV&ThpT!xhva9f1ME=L|9$zF`k~NQeAQt@t|m)YJNIt) z7ZP}~@tU`&P`k}jQ5a`mu%~XUv)=agtldfNo>>DI*JhkRE*N&AiB?O-xnN68bK z_@5f@gZbukk4vbuXpkK?)a|fL{BB>R2MC^8hi-n3jJ2yjx&p#Cna=7| z6}OnG-Ev5dv~boBCS66JDGzVwMm(zthC}ze<27w1fGd5+Z)9&V(=!yE8=;(fR*DP! zp^cuj=ZBqk2h*(^Ob(5=RVQB0_rkBlb(`MXtFlLuv-ZC1ALdx6E?xukvnhVlCyUDK z9GOF@4x30Cj2ON>y}}VeO3ugWn&zrcjf838@`)V$JQSBtK*6C@5Jn1~#h%uQtI9l3 znoD&w=qX(%@+8|8ugxZ9v-y-8%&4B^6czcG{XaMLhLrA-1I?y6qJrHY-|oi<@o}q~ zrNalmF0R|}>fyfC;X9o6VLZpCMBDBgggvs}PLEfF;>Pt{t%Mr8Y$b^a{LnhluI1v! zasgZh^Tw0_$dafqW0So$8^-ciXD9vTMM`2Kok>;!%zyo>&oY*&JNvlX!Qz#9<+voB z61ctB!;!`1hd$-EOc#_A;`UWt!}r5ZG`Y&VE;-CdpD8W8LU5)Wd_#WL#?Ve+q*P5& z%3V8t+-fhc<(mGuD=r{e)fSd}r-t*MBmuJOcA2w#w)L`v)MYbjs=poc+_IWry|wbJ zi9TI@VW(|k^1hCK(g9fSGqBGAQG^_xA%4OBh|yXNm{L8;JZtU&1D;>iO^r7$dP1ZT#lO?;GK16_ISPCW-*IBHrRPm#}T z5anIP&{4<5T=~iVJGoEC|E_QLdDBcFX>r}l5{9i3O%oVkY_OPjTJZ?{s>7asIE%w> z!x~>*`+2VV@N!39?P{ZZi)ne_+LLa9ru&ZQGK_Wjy$AP~ML#?alCBwz<@2~tL^=N| zxW9D=-wthZq7O0F?(}1&0iCJX|8vnXVh4rPo22nkiOQABhaxDYx}P^{l9?KFZ^gLO zk%^l4Haab<3E%%7=a)H1*F3iCdxTU|+DPXjcmUEMD>o>3Y#_g#3U=Z{hZ0M6~d9?MJ>J9)P~>$PKhqQXMJ& zSktfRQKH`}X%;3{p3dZIs=RG2K|K|NAD(B|YxgtrZgIRZeIW6v&bS7eE#-nk>?ZbP zJ-{H=STn?gW3na0t2?Z6eAy_ET9tdX&zx?m6&g^~e7C=Mk@oC9CbjqP-oi{EEdqWc zmF=RjM^gTjGx(g-K6Qw{#Qc~3UAbMEJE7`_JHSxbZcIvxnVw&E<08FU)L~BMmEy)l z-Qlbz&Smk@MC*wthXyZLt$nw~zC%>Oy%K0|z%;g~wglnD-}_@e_-8tq2{0z#auTav zyC^CSwWKJ~S6Dgd5Xg}B4+{?5eM&ZZZSuR^uJ~i{b5i4;v=zayMZp9wP|#=jJe5PQ zN^yqsJpRLx74I9VzNS;B>z4JPhIrNR$3SDvmXHHp5rm7chpcn+)` znHMMOEa`wMzun$)>l4lOo|mfWBBi|{wAwH2KV$Fb1K98pG4-^Z4`y)dW2Kno!N|%- ziYcL|LcG;=h2f4}t>vC!J8NfOE;4W;>AZIT1cqo6OqT*DZ*Ebt%);@5Qe*DHV6ZWb zBv$~OcF;+bbMw-Eo~b5woc-fCs^4-_yAcukNYb(h7?|GyCO;(^;M*bhlJXds#k>?? zf6yj$Cr%)n&MI`>ZRHGa9!ch^Aoxv3U{wn}rC_ggg}by-sqID`Tbl=AqW zzCMBBk?~rcuBB5N)uvGs)C&}vgYA(=#0p6LUbg=uN%%$&_|E4KoMqIhH0-&7fKRHs zCjAl@unybH`iDD0+FMX)Xm7}?zbh6pnWen``HvFuq460sc7I(ie&73Q$ftF?&Q5Ds zY0dh}97~DH65*+uW{za~oXy{AJEG0^J(ANZ_b6bjcemWs$%hg;_6v;ze-7sms2WdE z;kwPBUG0wM(Pd9%IDG&01YlD#0mm4-aFfways{&(PdUF_Sb1XadT#slf1kG(KBB;` zm+ym?%hsgkrTJK?n81>Q3KfteCReTnRE%@9aB(~n=QM>h0yF;>*b`@zQ!upXFpkRW zlx?y|KL&TNPT{uHGNss$|5zuF$b{mr6Ip*Xsq4>QZ5T{0+ z70USgHyZaBXQ$RNr~iDEkD4$r%WM5?3CFpa5&6o)9(_{U-#p;1!33Xp!s}{MzZO79 z@Kx=3E5&jC0|S*R@L)2Q`*WkZ2-SO z4rotursk^nJP(sZW35W+_w)A~L))10+lS|H&3o40#(52B@Te-p7>R!+JP87eVNbdn~&|uQ+66=Y~ z2=0ee2PM(H${!bxh0a$`tD9$)%V`|d`g$=XZU2KLY$&B^)ZMQ|lpboyDI*f)CESuM zt0i>q#^bq}@8y%k=bGPgwAt?`5X&~azf0Eno)!B|&AT@Gh0#55o#Tu9XS*IjzyCK{ zCVyG(Eq3|_C-#n)jqfT90@2Tq{2APo_}BJ-v34{usTs{h7cwj~JH%zvEd;_14Cl^j zrZ*TqNZ#k~(a&Joh)13V8C|jJpk7Z3tg6COi*7)5)jWYIi zsv9i`9NWoOQcg@v%jcvSF=(4m)i`tCjC#7v5r~z=6r5FQaeWY#%a$&%N@NlbO9I)! z(dW?<)XMj6ZSYq$XFAX75~Pp2Kdhl$xHF_qA6r9plF!+m^KMUuP~uo`*aWct3*sw6 z32UjynVu;-#T;f%a?D?2-|q%AqQn!^de2_bZa^NX9Q*5_6Di&uuhbbKlkxjplX|U{6!PeVS*0brGG?%alCD zlI4{z8fUMS)qA^^=4zSZF#VyBzzu+5fB!Wby?-%vBStk?85kGQ;quISbX>M#%@L{| z+~hgUb*V+y;(TTZJR)^<#wcsSVJynMJ3ha?o@;_GH;fgEUtVQ+P~zYz2?B!Qa&`j(#ap6Y{etJl0Jrw^(K`+Vcna=%30p@DBb|2?OQ<*wakV@w3IJk-PVG;?L)aWN9UuzKDMB_41Y z#~v0*8yAhe70Js>6tQvZGJ)7jiD5JIR}QHRRyn^rrH5N&=p6i2Iju6OT0>6jr z-r)k@PRKK>h-y4?~Vydr&AML^LpcU%SC zo-&?p_+F;%)?W#Oi=g0)bVEgC62y2k?UsL#rh1j-X=z59699$CeFtD0ixh1_F%CgK zao`}&t4fWwf0oBzSMHag$UDR@lTf4I4v|$KEi7MGlQTW8@XRWD+MN3c?#Zt^scsN(RTC@Yj6VE31fLAL!}J&qK;;~X z#Nqy~Iw6LAm61PcvluC7#d8GOd&XpUFyJr{Ei=5_2uR3Ccep%R^+ku!{lq&tD=&3E zOzNl#ma~-qeE0)}@rw)^$2;9%L z%WsI4t|`7}Oje72$z!vu>geeR%>80hHQ45wF6%}-7?;5K;GY>!Y>q4rG0??}J(@u0 zyOg_2d%jI$-%cH`X}&5Nx84GH&RV-m{1Pt)WsTb~_Pt4>%V5i#U9rreMp>gw4dh6L zX1e0A36I0@vB&E{Tq86nNqarJJ43lT4JX_j?|-kO|N1vw2@L1ui@R}XivXKWbmFh) zV!lQ;&)q+g+}jB*To}A`MNQX(?hNiSV0+?HcOzrdHwQX2Pw#(iWZIu$Y~@_i)$rB> zXEVAIUUCF`nw%bzg3Y82+^fg6!21bteij1uwm-}oe-%hd}a*mc!tkO6# zU7_5~{Fm26C=*etkr^&x>C_}Nnv?6yLng^KVq;RMfuctJ&dILw8lcsh04J_FBVNo% z)vwUZ*F-*?0hXc468J{v5-Z-gfpq}w^CS2RL@b_k;4&A9fk*Iqx=2G$bgFnsLG)bg z)KqZ1JHiLmT$&+lat3!fDn1phLmYfnNH-~qFD3~F_AMIU2RxZ%^r`eyc%}1>N}dBA5UtCVGA0(NnDwi1NAhO^*%zr1VJai zRHM7EzVs(QY1!83*=Sna-_|7J&(BQz>|q*C-`*5xHll*yW^V7nzmgJMyZ_5y{0O=4`^=fS&&*uc%s9}A9Xd7KeG(B02H4p5&$Ap8qNzdIT%X6a2QfpK zPNAp8Z7$eb>N5)Rv((c!Nji@(53VZ+d1+aCi~!j&rkbCMG`YM?@ol9?LqC~q6vs`W zk;rQ6uqPzZa`gn@zHOl_<&JPc8F_PO**I4~zp84d*AuQ+uFWdsrMTwDmusD^0@``0 z;K`1^*%PIOLZG0Ia>1*lq|u$}&emmT!F3!587^LuCwrB1q)kBp`X*$`{`;8sm;Kid zf@IlkzqHxTs^yO6oz2>p1!7W@0f%|3-o3Bgar-DigcV(09i_FIx+d+qGTMTlI{W#- z*kZd_@9|}5{qEMU%Zkj^W=X0zG=w>xWu!>D;BmS@t$m*9YyiLNA?pVmipxe4DiO6mGI* z8x&+u64~0Me!`$T9^ovznt%a4u%%s=Khye0-k&_doFQk$I)I~_2KNOs`B?ZFm&N{a?8iJW% zZV)TORqi(omRM_JsuJ#ZhI#_K6trZn4r4{lmRVz}S~v0_1;~2_yScrY zdS-lcy}-3exnmrOXB3lUbMMxAoeS`{8eKVru|zrO=_iAteMBj_827=Mh(LL}7?0ez zDYJK6cTM-6>$|kN%06GTv zI1Q=uLZ+H}#c;XKfFEgt`z%qDFoTy{(j=4WS69}bN!U%r*u2)&^AsG0TMTU#HVyfyZQ?K{T2R}JbO_V^TNo45Tj?5?G#oie=15PQrhMM7uVDiVsnlTkT_UjC*WR#Zb|GRZbJf_6mVVa3$H8%9mTZQ!QM@@b1f3Z@ZKX zO0bL@3IgTr0f*1@xls!>a#CWSoEUtfr%c!!|Khfv%jc+=!ofS?0xGZvMzZ8b3y0;lu7+7%mvVea zdnL=SMJ!QiNd5#m;hqs2q|pYnjTJ~%AF^e%BvKehMOHeW-VvS$Q{U*N2}i3lQP9Xw zliZST=4*wotE$5qL-4X`{}5C*Ce8;?FsksLhKFI#!s5wN5Gsq%wDA z<=FL0^iqm9kcLHXPTpHsxkG9uM@&fP&AWOoi$wImEQX4G`L>x5ON8eS#XHE09`$Xy zzvZiXfER7UEADM1==QxcC6tR7q>Fwmi?Y#~U!}8P*A#%-@(+4Uu<=7*0n1rn-7^=1 zwuY?MZLvjK8EApoMeDO?dc!gdPX;}59f~$t!_Je6KU{#GLCO=CYjTN#?w}ljpU&Am&mpeVQ$6Px?3J=ttpDLW`<#W|BgnahU(H zu*iajxs|p}#e*Z>i=iLahwRrU%X$)H3cA)aAGCc`cigq>+U`_fQdb7fVE0-98H_H@ zKFt#P6^{Cy!B%MemgL}~7w4G@R))O2mALv}XADe0Ew(fR(}}^nMkboD=z$BF$+wXo z&1%N7B;B`XGW8G8fIhSSVXet@55knz1Q$FhQs~I?GGQzai&oY-9#(i$CorM_G)gQr zQeFpEl@%15&gFpQ#n3%|eAdcv3?d?vW~3+m6gl#!D+}7&YE{AA-Jj)RVd(zeYwk?U zZ5&d!27waMB3|Y#QH&P6rbc@d~dolP;tdj0e(|J|lb0 zM2lUFS6LA*{NRFfmj=~&?q*ZtBbQo2t(0>GWW$B3CZF1WUIKZZ)0aGT0w23Ru2epE z#*hO+BrxDFXekRt3KJN{@m_Cnm2_txyurOjwX*@n>YeqL*J-k70_U{{mh>^vxSb1; z9G@n(=nxgoJ0VGVv%ZQ7w}hmnJ~dW;zHGf*i`x9Pzzo&JyS{jDaH*3I#WU#p_UHj@F!n7U?_H;%1^b_LLJ=AI*1kL4jGcd;#=^ zO`h02L5>S$txQ7+Uo6+6vQ8J$I*rWYg4EzmoKu! z*!|ctkSH#**VX$}=^)wk12U&t-Ed4w?Sq2CZ_{B_tE=nzo>S|f^6Wyx0e3iPo zE1A{N@1JS!d0UzrLGrc|qkBwBg#6XB_GBH6%=c>k@Gy0SxY-pO^DWD6n()}1-gbSG zNR*J4ks}Z1?T)$Ct;H1-Xtegy|AwF7A1+H#ym$P{pIOhK{Swrl))znz#?Z?VoaR+4 z=3Ty+mm!>8OzoG0rf34yF%=1Pt_Eo{cjMNx9pHS2+SkfroLz2a%&fOEbR4xowqF@M z)xmw(h6WA0XHgeoEzKtqhSMxjKe(z!T zcC39;d|R59i$YuVY8y74Pr{+p_5q_`D-(&YYT(W=@wh`cn`Hi^ycy47f0jr@lU0mK zcK*|cTGw0JV@~o(sEu9|3Rh}pXbbw_{i1*%qtDNm>hw6cg~TgYR|_3IT%g1!hgXS> zF>!V?fb_kWMad@cJ=As49p{`&c&MztE)=XAq{hl9k=5C~mM(ldU(uisZDQ$d5RQr2 zI*MnSV_(WH?#cF>+^5A8+@M1G*jg9gbS-8lWVO)~5OMC_%`NU*4i~Dce0t#SqgC72 zLJBwvGTtN$73_J>==YvshNhQC?(K_5RXV&9IucxhRn;&a{T0V-p$Nr2kOJ2WB{Dib z46u|K8??P5fb;{*1YH++$>OEMvBR^QJCngGY)$B#sf#u|9o~rx9>>VN3-)Z8yW30Wjb5+ZZ zH@N4v4uUOL&p4&nOm2d%2v?-Q^D)b*XU0BDtITDLusA%~X)8Aj%Eygnq0)&}W~dP# zf_Gn(WE3w6d|8$cb4+&nQ?v9i-#oWVIhifEykPAJiCB7cADrK#W`Re@Q@KuRy|c7K za;{V=mj}AdwrFl3Ty}2?XCT&zJ(j-hsiPXzidiTX*R1@U7Jhr4W}!4|jBP1KqatgF zHD);d8XoQ-k^J1HvMODZHe?f`GaZ4h@+jVItKU&AV{z)2KAfWxiaIry_6tR|jQAghs6$`Cx`kiXw=DvWU^F}YJUCdR4 z&Ht5B2+l~SG6M5Xquj15LzgeA;`V77Sp&M2DvdbTX9nib!x)i1}Zj3#B5 zD=>^_rH!bySvJi+V@y0PmV@fiGlf}K>!Cwuhw^-mi45S4P4{dgI0l7M4FE^1#?9>s z;mqUKA+47te})oYu*I*sW@~P-jy*p)XQ#cQ-5T4f*+zkZZ14EF7?M`mJJ_~K+KWm& zrrwVJev-qrCNu6(fWu1=kGKKmSDkgn-B?dS^CbWaSc~%^$-AyD_4w&b7EGIFPcj0r zjYUki^VN??CGRzh@Z}rScg`coL&UpRX6GKCj;YO?#@Ao&P>1=g&im}TSl0!~TqX-Z z?`=0SO;6Rgt>ZISL=#nUy||}cQlk>W;83|eGx;oYa?W>MfCcl3`Kt#gpSxdG?Y`KV zY_?z0Opatmp}utN~kDKTa6W^d_@@)*XbXBY+<)$^yJ z7OoaSgh~SF3!A@vb0okr&kv*P-X#Shc(-0tkJrcjmLJlb9|e&VvBQU zHqpJn+q6}8{b*){m5%hax+G&b3OC?-ccHxcG)YN%b_cXLu%9d1q~n(b^INn7Zw9ma zgkL1W{7HkQSUuyiG7Z()W;!o9LNmRpJ{hnd^GLsMb{;W?b6&eqB*YFH3aKBiI)ssJ zbfKj6Z|DCi2461p-)lZE`@{v?+PUe)ci9X;7`ce6bk?)Yo;Uc~7S}4Ps_nMhVSdjh zgf|Eyfa04ZII^0$sZ(K?pCvKW-<^^_($2VUmMhhDNT>zhLt1q%l@A!A2j$9y&;uM)+}t8JQqIO4->i3M{vq2uQk zQeCIc-+TpBB}skQ1WcoT^Bhz$m~7nKJAsd>N_5i>x3xyV>=msp-cGJ*p}m!v?4r`D zJ3lykHGd>IQg-|E>=i)THv805tEToJD*4x-8d$1 zPbId>?cbFtw>$HkBY2=wnyZlB5>0(v5iWMEcKD<0dbZ|kz#XV&QFJ_YY^HR^wtx&m zHGNvoR4`76C`Vn8ZERRx4x<~2xOJ225ZAjDjw8lWD*8LA90A5GuZF;_{I!%zT$fMJ)^*pwj9Z& z5{#_6J`F7M_<{y&KvkPieJu-rn3ZD#3u_8im;h=IILOtx12}4H3=_mGn{uq zF^Ql=yN;zY9P`4=&bc;X(^O}KLZVzT-D@A6=7^ER9oYv0f6}1VB~Ryt7!4e zaM$-w?2A5ZnE>9b)T-&yl0y}j+q_t>iL3hh){y@dc*Qi9lZ zP@Q9*qklVG`E~+CgExcOTyE~+Ho7hItR3n-kaMP3+#GOyu$}Ex&Ie4SMbAMk!reoX zWDivo?}M?OPt1#D)vfg3C_%j5;@1jwvv2O|Fm!0rh{ytL=l_{}!+ z)TadjLMgd}iqWnr$DU|>5L=0wFCI+r)>&@%zt(}e$QbB@5$De7?ob6a07IZ%w`(Ve z>ZJf;^2TS^!n#i5)YU$GXA3LlgQiV3h$vc}xp2Q_Qh)bTS?2+KekbZ^AxX7gBGrn#!Y+k8PGt`aKxUMgZCnSD3ZXZX>oBO@;IlR<$`lVT9gF`xtgU*0A zn4!LUWP~1z)+zunE-^pvpgH$x>eB`()ay4a#(L}b+-0wNm9prB6kc<9N{vuxxLBv8 zTbT8ZF#rvPj>9R%3+jMhDW}h8c?_;7pYsW|1~scltCIm-DXhIC0gf}+G{9FbpCNUC zG+0p|#cjsk{`k_?Gf0#%UdH>L=x0XG80ytueb({h_X@U>ylV=gi_{EJI;fTwq{&4^ zs;VgQi4e|o3d4*j7hAx7sdn)2hjxr@=xpXE^6AUgI-|!4sUbMMz)us9fl@o&O;Dr` zZ`MCw?kU|Vk#o6)Ai8}D-AhSavRH@J-C!W3IgNyPbnI7DKfCd^?>N`3Q|L!qX9Ciw zOBKfCc;#F4`8d<>Ny_?SU7STu9mhNLX zrZek4i{b%9Hr;b}alhtGkPN+uqTQ!z6u#OhULSS~8aOy%!o9JOb&f^wAx46WjU8P@ z_z1=gkI&lNOAc3hC%3T~u-XFmR}}XU7ua|S&>UWcnBD;23myH9cP;$GF!vqH`>k&r z7;BpGF6nBvF&dI*(mWpG%UQ>@j|Ck)u*=!uJ%Q|=VMG^f)qiNH(qrqoLw=0e4RFn; z9#Rq)E1$D12lyonOEyPsMD6lHU$OR8Opwo6>AsilP-^7gKDC5n02~?1`UzLNO45$) zKzZ>SkVnk%8VXAiJ+#pUFHkb@SoUL)Ll@=t?iKWf#W?>EGY|j1yxg=~wXZR)z!c5{5J&W8-jUU-vR3=Kgd25nc{UHVDFRs%I(!toieXva!i_AIs7jXxX8%9GqW=QNjBaqP%a3!ISU zHa^TH-1MJYe3#|>`aH~h0ipDyPOmvsZ(!+CUAUJpM2|<#z3}NGvO>;ZV0*9Oz%FqH z65g`f^^WFEotOE5@8-dPVaM`>=nOTw0d`M!)vqgAprKwahuIzjcd=C-15_T4OMA_s z^k1Ag&L-OLmiyha$cr82FVisxMAfati?6&ve8nTPWkKabVj`Vs%Frx$rE>S(xkIIA zm!HPHS{vxqP^0Sse9Tb0hqOV^hm6)XI$y~Q7LTd6d8;Bm6$`|I)u|>ofQd74dZzAJ z3SV2Ge38O4uLhlQm{_NoexN^m12?ZF%Ihrpetvn-+<)u3|2B?;kEVWN6VykRi9%t% zu|~YFBg9gp0eke?fY0g?iApKXypqTUsCQ1rZ(m%YVa^x(VIB^LkpQ+nLMILy-vkAE zSYg4$#W9{64fZGhNgRdeXl_i6&|V2CxD_uRc!yHMu{Dmmj-;^8z{{Z%u~#}qGT(dZ zl^+J9a_!944Q<(#{7CmUx7G(<0v_+}K)4M?gP7>ZExCa`$B2?N!?NyW;96yleC0wHK}sq`oRj z^wXoY8~cG4x!oKCgd)=c>mX6S5l^64P9N#KU8=8 ze}DYDUs~$r1sA{2)a1`o#e!VZS|iw|8AIQ(0~{wt9F5vSzO&4~Q-;s%zx*;X(jVIs zxGzBPgaH6~5@%6>r2RY+v=qu{5k2{hIelN&_m`Wj#opfsgE?wKuHSE9rV*hZlcAK- z+>JAhjWx7R5MInfeG44^{dF4*sHvADA64HydG=!oV0$7xya2`~aV~kVw>OT0>9_%+ z%xel!2yE|r8HfeyZ^Zt?nqSn(oAXp`r^90?9~m)xURu%wE-)++u&|8WvAUGsD=|(*N{3iPAcfM3;#eYPac3u8AU;1S~^8DF`LGQtC zmjA2feG!3M5!f0g7#OqmofiG0+&;@tG-EKMC*%h?{tzwx_qs2_fjDa5xX&zqzrjCV zPk!kL_43BZm9ig75B~8gru;x0C|0H3e_Y7-uNp5+`TG@{<=LWK|9$L#Ska#!eZB+4 zAvhvlt@ywH?|<+3(R=^?T4aTK9^F4KIVGd)O;#l#d#@xbMMCz@9>+d7 zzw17#&-?p-{IADDblk_euh;b&&)4(yyq{^PDV!mpC4nI5jN)xsO$Z`qgCNoc3L@~z zcoj1<1f62FmXXo0wy=Pp+s|TR9%@ACQ@5_7MdyPh&q{}A$h^FI#Xp+DfW0U0jpXkX z4LKWB1^%_?D1Ny}IXr5^7?#I!l%%17_*BkMIPu%79M0d7(zhSGkg~%dQG-qJ>P8+p z&5k;C@7d!qqn=ncc6ga^w2X{vs{5X?@d~V63ki0p)-4sv7Iw`*0HR z()wVgqN2uek0a0-N?)@4*{P(iKats?LLPx_dZ?>cXgJO{rZ@51&tpmXYdz=wtEt5< zBj58R5Ix5}9U1m1)@k**wou^az?&~NIqHtuo;vZ&uM*07PBF%HY*JoUXknH0)mp>G z957@nJjgAOwR3D|m9lYG5k~uw-Q?rtKVkB4{qv%7#6@|mU-v^fliF!dXut_##?*OE z#6Zp~_>%kIb=7Sv3!=Rt;gz);ClaupZ(`1zIq8r8Tat(!|8Dvz=KusRL2Lt+5JWwW z=PgAs0PQjpUB<`zf?sJ$%nr$!G6|i0C&^_6sivP#CQ2C>t-$~AWN4h5hA1L{LM1Rx z^2BHW;mN>4iVLr$_#)3{kUB+=gjep@+HRY6AK zcCzJ}1sNgVqkg;5oPHW$X4c93HJ(|8{=tnL8Yi0Gr?@K(c9 zv)&@Y0i`9yr7KGmu4gff3ZFP8Uf73Mm>QntCf5yr{MqQ!nTZ^GQg??M;Zgk6fQSZU zQ`|V++Q_cRuHpf{RA2*(9>V*SLl8p{iS)GzTC*2xPj6p-N741n_>Aen z*&^BEPpZ$Xug1QUa!HrhK)-E$qx<$8e^8FnRL^cnN(s@go4+nI$UOY+u_@$x_LLTf@w8pZW9|qKKCb>}JUiUhs-5aol#uV#(gVU8;0Ou}ir_aY#u) ziCU>#kzCm?+N@pZ%cqW|xcO)gme5yyv0JaYUWu`Isk-Wfb3f#0jk^{t!nJdoF}6Q; zCw7d}Po>VpS54|pqvA!m49$smP48J`zUy39D^gj1J^A9SYMTLv&RdoFS3RUTm!q0h zOm%p@5?CxfEFaFCo)MnGn;|?O`#eo4RVk_c+}*&d_iiecs+20+&C1D|{Sce?^Y!*^ zhJvp;5qBiAMsvjr8hcKD8_56u_IlT=2kUyb-c$XIXVveOzj3$N7+pstADVx@z(sHI zgTQ;+*9@6P`92E7Kd<*a>$~;-Mg>W2yZ7Cmcl&oUt1bh7%CQSwsbG}pR^?GI&{~bD zOY|)~@;wwk>?UF*Iz{6;?x!A4N1po8S<|hDy$f3@U@dSpthIINU+DiV{Q2c) zg%?cm-&BTGU6VHBz4>n5I3Hh@_%;4*LU3YVeAvf%8+F&~n5VaX4S)4c+Ven@Q{_<2 zQ{{3Uef&}SqTHq|*|^!@TxE_~S5VCJyYY8ZV!kC>Gd%M7VPKVMbMAo_65XlOC(u;( z-Sv59%d@HE>6(u>3eOFc4s|(kRottHTSj=XVWcs5#P6P@$;32kDV%S$YNb+m8TG99 zL2k!atB1OChhH;A2^rJ+Kb@SocVh345NkBOTTEaGu~C@f!-z-vx()}E^$`5mve zUwy~_Nch*)eaTIg7pf{Aoi?0;WmmUdlAR1~hrMeQ4%Tcu8dSP^7D&c2BU&3nfLGjo7 z%JO!4kqtO&>rcnPxXwxsU^6Qb46j>Z2QVG-5|0UEBjW~T5bLl-VOnSW#`AH zkLARIj5YdphQ?kfd-kcw!lwM!d7Al-CNt)DRr>5pja=@9d3U8I?lmJqLc#|at@H1` z;d*!I&cFTpiEwUJxM$e@hDE$ZLTjqvuHfe3r=z!T@01n2S?^uXnrb>X%I2BWnV6V3 zlca1g9{VCTA?hOA>X8lBNjJ~#u@PNn{%=a57>9)xo{ahz$*y7}PEUPuw%+d#Cb}AU znGydedMoqeLNCV>$pWqDoa2Ywp^w#vy0s<|X}aPjK2jUfZE{U52cgPwj@(^FYL%j1 zSN7XwyFaJ4>Jk);u5K+bRLpuVe|BkI)Wrz-#M$YM2>JC+&Q{)Ax|dR}igepd+APOD z*tm!BpOAkt>soas|8Sv3=>84onbgDD#T}0wBzXY?BmFv^2QMw(Rd0%Q(|xn1 z=KcsiCN9xY@u!T}yjhO24&EO0?vIuFBF`m~@Vpp`*pWbcY`X_atZzh=NKmn%)1uPi z)Hzgds?w>Zr8y*5V$4TE+u}HQs2`9XMei2R#?0mw`wbrT@sLoe)=||M?^o}JbtX+A z!^XBdJ-&+h#O>Od)veWbV$LxQFzt&Ic_$luG5Ex-o6SKu_O;rf;WQ8FAAPCwAFZut5AQxT;S}=tm;dlt5wok6QBMN>&WUj0ri^Z)| zdOO45l~eY&^_?IHj}!OhgqjDa3?g<`RFxy1I*osVg8TLO9a#t>are~LbJjF-W43oh zKCrg2WOnwjw`9fvHND`JINMkE-=2}f7L)Jhw_<1$+z%8fO@7EDh6}!JT zw7DptrXR>tUoio)6*(l6dU(ZhW>a61X z{aXe}>*qrr)Tilnjqd#1OVFLw$BM67yxB&63EO+tLPbs?@#WQl`MN^Z#*JO77J^I7 zyxQD3zpjxI$tm~nxkL>e84T)gx-M|iiqg0X>Y)qS&zw~)8cy@I!JQRMQIn_p=kh9OUj%?fb})0i699n@i_ZmKj7OwSy0OhqPs=9Qn; zB!04UCPFFcMaLbFlSSu08r#>KW}XZUc54@@;XZfC-SEXN_j~C-19L5@Puu#An>*Bz zkAWmZ(!uW(R2RBGxnf9aik1kS?Gwez#4{#v|9m|xOvO{6)Fx_-dGut3N0!Z! zIj3IsbW0IcDi?E+P|s@{Dhj>(R*2Bx#P{wVv*gkv-&&qVm1&eDRSl_(Vq(b0PWVPi z@kkbaSdwfEc}P4$R_1NJt@{A#zwS9|BKJ|y)>- zTdQdIe&x_#F_E@k&E{NZ7xIJ9{?BTYF5)Iw#?K|%_rRSro4BmdNh}^rh*)G{nGI|%&!hn2Im49Q14{UzuQhaey7S< z^mM$-Hm>WsAdz6|QgOQlGqZ;GT7I?Ay2H-oiCsy9kgMYR z?44&Bl&=w6l;5s&cYD@WYRv6l)vpv8W>p@vmNZwC`N?QYCi_!rQ(l8;fbeaCkC*OJ zj8A{FsYm6EwZAjS`zb7+AC-3Pkr;88{`t{&_IGW6K1L@#3T=(O<0avIHvIGQuWrx2 zyTnl;ypj78FC^O*wVXcx=&!k38FeuE(lzg$_64F%p#|3k8!0rg4OMuu@$xe+`BS2= zy{_6wua7g^cA96(?PpB+%RU-WO}ETTddB=({1N&=-;7RcI#ckCTEjNPHD%ht&*eMC39 z8kUeR^;I>qRP5d*O9qqqmj>lS`w7l``Db=LRQ7Ka*nY|o;=g$(u--1}F-i`(FWX!* z|6M>C$6FrwkA&D)8(aF zk24Kug=~3Ms@Z!idqY_oFF!korcz1Qf8=njec;XD5&OnV@(jXYX*nL^Y-&G^tA+($ z6dG3;m$E`WP(54tVwTIW?BsIw;3bi!yv!Mn2⪚$9`fN??J7i&t$^M&sypR=EF zO<&87DQW-ovDHJG-0T{4(cv4$cakI-Po&JqBzdpK(Q4!p7|=~4%#5D+hm95~SqygP zw9nGCOQk#t&?U5vv3$bqLGdC(`|Q=!&nE)4y5s9Y8u?g>CN6sDx0(+cyr8i-S8Dqr zwCl6~r-Ls-?q1G04#Px0_&@AcU{13$5%Lh=rS~HGWoSAh(jnfBPl^Z*pGLX>~6bt{GEeyp+`3>vGq2H9kEIKD_2ar&^VfHZL3hfh zLX4*na+Y899p$0ux%b_^9IjMhrSsv&&esAaVz<8Dzg{GiOLN~oSL)Y;U0OHISZi*R zb8pi=*{`h4<6EpPNg9RRWHTRCcU*b#)P~mQMlIbk?FdniG+O_s%2RL2Umf2zbUBIQ zByVkMN4?He`+SnlE{*SUtvs>iiC)x`t`z~n@V?YF?xbLXK5QH5^-@BMGdw-_H0 z2W3bvL&)(NTKg$D8S%?fB56)AfAgE2mC7H(^r_vpSxHNpD`2%xa>F}8dliZ;Vi8W1-+=@Jb%3|OqC^`I= zA5nMfg4J)s!xTCNzcZ(Hty{a@R<;;ucYi@ssrjLjRF~GxY(q94s$5A4au479&APr zTUfXG>xZSixqlTTlii54mnacsvm!TlD3;GJiTo9pS8w#%G=5zECx_h!~ z&Z*_H(e5NZbor%#sfZZWfW&1=O(W`_AVY!mrv^Met75axr_itZ>DWz#d^Y2st$4|1 zn6M)@`GT_aBo)6SD4dDQo_ckkd3kQ*m3-!{gd2Da`D9np@rtwbN;dA@{K6K@_bk07 zL*8!8A)@(Zw5{wu_43OTUzGEomjYA^DauM|yZ>4`l!!_kto!Y8^v20t*nH*FnG7_( z>v@kFfgI&&fnQcrv8JuNcQuw>R6ZfJOZwbS2N+8Q*4Qbh(_MGSe-`*l%oX{@@iUFZ zllBWvY^B|#743-(-^(-E%&9FJlsHuf-pZ&dP$xM9DA-=EfR3i|f zE$R;|TeUB9ycvE}N|e{Z#^V@q>hHV$``y$nDC+OM?Mh0w?P$IAcD>+mBWu_bfw;ph zQPD0Y@ZanI{_KxArv`Jz@ERA(>0Mn!AV%zVJiwtGEr{;Khbxk>=K4G4_EuKx`5a#c z{zb)i&!P83fz{zvZj#&4VqT=jIeO$|hs8Qp8gj>7de`@;)SwpQelg`r60*XE&{@ zuIVeT72HOz`iG?5=lO!ydg=+F0YP+ov*{lj4LM<4A~*NHEZ2{fIU7q<`W#8XccWzK z1Z-#%@z3!Z+N!GdE9nt;3i|y}11F+yosMaL7>7V)e*c|^LOt60&iS7$l6Cjt7OSC~ zSH{mxpYRS0zB^SDAtWB{F=NelyfyzYN?FE;)~R(Nb0K=Nkmj+cuR>Q`Z{IQ+CF99e zE+B@WhM`kzfBiX`zHxf$$D)yF80xoks_`@<>0f(FI@EBPx?vtyaBx`-S@F|`eV*mx zawmn78PQ7OPd%>Su9J&3#@;S)8a~?h(WJO#l)A8ZrTj@d>@s<_+_l``vpD|es6w~4 z_L+Q9sCc#df;mRgp?)jxo6k`+r-`s<|M)uS;E$L82%^v;O>=i;8^4^xpBSR&Mk{T% z)trTY>!18<*|mBkoCa)ef_KL%+=PT}$FtVBpxna<{8}yGKgRvNlO_?0`t6z~)-6Wm zT43PAABv(0lQ)r*)xMf$G|PDI_~XERwkl^-t-FgdMi4Hs$Kql^Fr#`1uV!-xlJuR8 zp-lL*@FCU{{a8(=@?iWbi#cC{ zH=^j&|8fXr@3Ozq{yr`WukQRFpBg3`%P!#ZCWWdkjN7jNImmIDtbVzw5xerC^D5kl z&b|_a(Y+pi5G2IdmOs^Oi1JPR-InITGZXobx0R{Xz`SBW8inajz4%ZYtS)yTu{ODy|` zbKQ;+mef~cy18}ZVp@!j4nYNqKvjSMO7>ckV^0734#=h-x0NJ+BLh?){`*27B;F#6Nc38OHMv)5P8dqrju#Mj^2AbYrgl2#3cl*bp`G;)xBBzyC0ysXQ@b9dDcA)5 z_-h{s8B1kY%kWV-K!iDsvZ9)yL78+ehtLHt{bc90jEc@{xKi+(t2VU<*k-w`UstUK zb~7Rbe14B+7|%e^aM=Ic`}>86`p5FJ{7EL*>pOZYulqbVK(f-wQ)OYH4J(kZY^6jL zfr3kgxXu`nB*+tS^NnG?c%q!F3TqS(i-6QWUm5?Aoo)mSv4cq}G};%p7t&ng=1PBj z(FlTLE!o>wl_}{*47=c!exKXaiz#shJ?NQf!~qnj^1Aj`P0MxShQ>VjZ8=! z=RXGm?z-G@9W$BkCYPaGz;);Ur0{{TO;6?GLEyOv^d30wImYvHFTb|pGQnI8;QCtr z<3i3~zaH%sAd(u8b~77KKxX}JwFy8%{;|K8U?c1ijplEG8I8t0Wre~*%lW>YXiHK86?BIA!;dRYC~ejRzZHJthS4X%`i z#RFrv$x2>;{LTMUZ)iE9lbhNqvrRYd^X-8%1Bt0aEXU_|<20ql3dmXDKNy;@x9y|Ej1K5;$ zvL!lm9sK)V@C=^O(u`P7iqRm>|&jnO< zyK1nFsaE@zov&J)q!lOK{6xz1J0#`CT300iooLABlIOHerxjR$>^`3LaEt$r-}?OtRt7e1^@ zb)wJz8AXjiJZTJNt3*bz(Ba|`7#d4~aB$gjy0sI(^18p;ofh$ZUbOa$7}Xz}K|;3H z7yE{!swt}ozHkw)51j1Mw`wF8=?@ZC7v~r8ti(_P>uak^gJuf@Yleqz})g{K@R7!68Xf|1_@zEYp&#lQAoUUVNCFb${WR;B+P$+&9nSpOxK zg)NDLNH0zVqGM(Zo9cBsWa%4zdsJIm9SWG1QQh%>VNBM044{xtTye26&B)NOl1=F0 z1<*r(s1+Pd*T5Z;MF1l#{4s(d4sDFqh!cdQK*U|-rpw!^gaas6l7RhyxhH8?I%~<$2 zu#KJoMgp$?T*h=x^P95}ec2@z;BrwX8>LLW{!>)LGXY#jT%@eG<%mzlZ|8`-#}9ip z0s(t@Wvk;CJL1UPUTdGN!@FzlMS9622UYA+%lrj{K{0v7DAmYYKDI~y0A&uYrx1{; zK1Nf^(8@d9d82@d7q=WEh?tqsO8%JZ$7r=ojyJ*LGH|U1?tc(tNtJ}Xd}yo&v~+S> zR+QrwegBQTrnc%B{K~nxdbs9|X2G7#GP$wAlQzq!{4%@MnuH}+(y@y35seqE}xpD_@Dcs{qgWaEW&ZNwnpcVNwDHl$e z!y$i5Zo_D{y(0-nuH?8Wulw1eQ|kD`ckeW{;e3rke5BMDy9XS=Zf6iZY~d00f70ni z!Vj%e4o+KJS5e8Bbq_1JKqHb09W%ev$i=h$&0=y_tcW8OZ~>CkfkUBA5hUYN05|>Z zzG@;&`Eqd?BOisspunO4E?xOshaS7>+c?;jcrj&^WwkX29l4dQAC=1q+U6aLcIP>F2Ez48igP2uE$#}K z--y6LH(>atACpwTUEHLnyW$aMpFcdj1N_!Mj>uz0rGrNmPCQf=VV75 zP1>+Kva$tk>-_&Jgg*mb29Zx&-w`LvOv(0GvAm@a=%*PyYe81V~d-Q{Z)K2#>y+ zl21$mUSlWJ@Wf+57WmUj`%hhrKh4CQLajd6fd`EnDRq<%6W|GXJOOYsLQmi`Xt7`a z_yCYO>djry+RJ7Yn5EgoL6nW=TX;n?NT5Tw8s%&@q%PmV}3-55Ge#FHuuoKYS0;h5`+W-aUP!!J&c#0}P0<8;#`R^eWc=QUhh@`p3=$pSzz5j^hK8YU1b zPch1KttR!phx=QL! z$JZCX1#BCk_aQBpDf-NkduCu?RtWa*OE@|NL2fJfxI_49eKL%ZY>l7e=Kr`VK zK@=`S!Y0B5f-cD$_;KO+s4Y4LCK!`^+8FA47QATsb*dOFl!-3~sK3=F##rp&!kKsL zRsann55xxoOMGg1mYfe)wJ09diXD9%9rfGW1U^qLIOON7gDiX+VOD)2oq{%xn#}FI zidyXA)Jcde3bw}X-Mokgz!SzWa1-?mWmR79k>It|$~}-t!g&yc}OlXTtlfcE8c_lsWrv|tO_7}>ObD9!N|}kU;N=KDuMid z-NlfTLwNSo%@Yggkoi3e@v-H(k$_1M;N#-QxF_Ac$D_*CYvdw+*#qIy)Ef|l)HV1p z!+6p=(q-U*zLCj_P}a6UwkHUG*-sS!$g?Odc38pHbcOC&(>>yNUwz^#x0$FAko~C< z5^f;SzTS=^>)NQwMpu3qIc{0Eh;H zhJ8I8eSLiyiwq;kN3Dv4@nWU@tIG_9CO$I^0GqrP>)LT*(uLLn;HH89@YMd0UXjDdI2^q?!&vpL*ccYF7hfp0AS-ZP3V6HbZvYG4WRs|sk>{pK7$4nac930gdAu# zxZ}9uJ*$2WwQiJ3G7*{sZ%1i#F?GCd-+Dg;;`^e}G~?9xNApb?Nbpnp_vR{FPrzjs zh1&D>y@0q*8wRQdjA}3khJUs@?lX(-n^V&K#?56<0%3URKZwu8r$+o~yI@Y`=FV?_ zazJcLs?EmOAM{L#5GL69`C!Is)e{SHIvhPp>(-5S=;p_CVM2Uks2p157Z9LH72hTLF23w_n%ub$lnXsVIzx;YW5|Ve{cY5E)yc`E>!h{ z9#{%xB_b&z*AY;7uEhd2PHd=HePq0^Nei7b-t*3NQb@7~w2d%Pn!D9Ev$IuwAb~d% zq4idKbg#;?9($khm|S9^3Qha!7%^|uvzyp8>xKZ- zNFJC(oIpmE^?TO(o(EksuCc}0$COP|;|;Fm0vR1}K*7E^!fz~S1?CgJe-RTNHv-9k z{VZBZNc1?FGS<)Y5r(bOO4k)ozE+&7Eb#TK_1(CaRp>}En$@xC6998c&zTu`a?!RJ zVKrh_V^e0+*AgCOarCcU7R#UPPW;$0E?N6@IirbVA*{Q4j#Nba+$^Za8b8rab zGq=M-_XDFFERzQZ2d%&>Kq!1&Zwi6=L3)IP?-7RSk^q|PuWQt4ep4GD^Jh^aMGAs= zD8BF=u8DS_6F>qGEBP`(S9(3GK{-)E+EJ3I9!@vBt)R@}bJYW~$c7l8l%kdj9U7NFt9v28@7z)#Pxz?7-JuV1NwYU{Si@PThRhEpu!(BUv& z!uLz}Oji$D$1_#^YE1h~Qk~zBx47RFdZe%KuE*9+)6dl&0H`)*`;R-ozw?=pD};(q zo44kapm8Y6Uz=K+pfe(9y%64$gk@2YG9 z@K2*@q?a=|f@G2cVkw6#^dk;a@fY(IS_qW2?PocSo)t} zN=qJrBj#op8R!K;UJZZdab-!((>uFRb3YNIh+Zv8djwx~31AYy=j+T1{aBRJ5QTFs zBPk%-3pYS9u{S>vH8#KR>Mq?gXddZ2H$xM3l@%jSF@*#UHK~0Inffb5t6GT< z&)~@3zw{P+EM-|9X!1QrV4K7Zbvxnsb->Khv%zKnGZY(jPtlujxN*eST$`N&+h5Wo z`B*lzub`p*l^hSUkFw{(6p;;`WE~6h-*C_u&|&NRINuKfXXReNYmgf>tP+~yJHCQ5 zo~_$}hG%yPp>dgP=m80h?|awkL$E)X3#%;9dI`pDQLn z9e#r_Dr2|DW6@ch4D#<~P42W))j>jf#_vaOw}&7!#Lwb;Q@#XfndP24fjA^61d!0~ zZP0$XrD6SGK9S`$+$}p)1#}Fx&ul29l>gqrj0{^4yT`TN-;jfWf@$vDL%*XvEQWc< zWh7)MV4U(c<8iYLPX)|(%6Bqu14e#sUhWfXR&rp^{I;;bzsXJV2g**0GJ^5wGa-2{ z**)kCQ)+J^aALd#K@l$*1(!m2$>4oN;=i94)!V0KuiLBwv%OqZp@bBJ@WieMH_SP_ zU3vQPW&#AK8Kbvc>pPw6V-`Tge1(xuxBp)R+pBZ%#!Ck=T_cPhTHfwRGC`YZQXHR; zo{%`lw?t)_k;e#(A89tY@zeSwcyWfXi>>C2uO36ci>3k75mDPb$Lth#*$=aYy|feW zdXX@C4nFnPza36>ADqlT5USvf=TLecw|CUO!PQ`P;6KJ!5VWIg<`5Bac=j`>;ihyo z+$Tbgn}Fd=ASV`_xbJGd=ehN?$IvXvB1ZZLJ!*K!Pj&p6V4)HCfRH9op&MdP6Fp%Y z+P%dw-M{RbHdm(>VgvX!r-*>5UMqh2+yl(u2g-M09f{c^zfAYpaF z%5}zKzn#^i5%S8`Lgg*b!w?h-9vuGf{)b*lU$&m;fu~bDfB}M@b8xMtNV|c_nh-r8 zswtlUJUR?$)0z}XWkwF|l-VtCK5G|`sY-ei{1CR3a@RUqXuRYUTd`4C9~~4+UX$TW z9UHYpdJsYXg(1-LAwc9e$kMsjJ|O`p7dlxflK@c$!Uf1y5wFwP!vy9w?2I?eo;GI; zh2=jkHQ;eP7^RG~Els_@cetbj69#_<7qKk2hxXv{0EV^(@(bz@;D?}Jem<2sBf$eHA$;UQ_s-Z9scgpHf6{F;g1%EBO# zb4YJ5+H=ra0}>oRIK}fE@tpQ$O!KbB;vo1BhchSgi9E(4@4SP#hZVYjCXgKeIcGL> zzpAEy5(0e|57O6XX%@YwBmKR1z9bb>W$*o{^p_6fm|Guk)XIVz)w(zy32{-MMl4Uc|8DisKz*0*Ejqm-@PpmT8T zu6d;u-Q*m|>5+{Agvmg5qyvd>&bR{!a~h6qvBm!~FT|BX?Q`$sEguO&;$CbugMs)l z_i}JV$xO&zRWhcGQD|Mauj%myv}P|$a_6D54%_XxGsKj)0GDzjB&fpjqvxg1d^?Al z)b5i+Lsy`X`;%e$5L*`;Jm|5&QMNjftD^k$B1lOG~v!BLo z5%z!0MgqvU1-H#oz!AyHMt6{xc^-%XifeXurpSPkDhBvE;EXzbQiK3j$}R6mt6zph zLV$5{V)Aov!0M9rNM7F%5IP* zVa$SLNNzM-1dP4EcjvWRR|0EWUP=1`Ut=K@)%tdW@D*Io2Zxi#YO1bodsfYR4wH%H zPD%S4YNAFLinx!@ zygu6ZSa(`FZqTIS)KBXY*luk1lhfroFogrlchIOQFJIxCF-kL+SL<ql3&RJp*Y_ zaE!+VHnq}jsGz4QgVg$+U~#~cV~LYv#95&S))LcOEJ8-sM-!s_P=hr)SHB~95J3^y z6`r)@w6rt@PU;rliTV<<;pkKV-4zc6qP_bUlcc{draRs-G8u~l6c`m^3#6nrv3I@2 z2RPaaEhesX*hhM*e@H9{K$PZ<4jwpVVaM(r_xnoZwUYReFoEUoqlh;S>p~W}eSJyx z!T`2}?ce+XSj5$Q@AX~lahu#0;Q(y zq8Jhv!p}v3RtC4m31A8Z`=%hta3Z&l#&F%PFysqQv1(^Jd0PS|~ zJpYc>Z)csw+EbH&)Oq>!UXF%im8c7l2f)3X4iLk%zJWCdU8wbn0S0DMx2rLCbr(0l za%5X9#Dg?0W8=Lb`dCX3m)B)$jngj7m7C4N_Qhl*euXaX6_h~XLWw8_*d$y}oEA)9 zhx<3mKdi>BU)}T;v)jHDT;?PpexkUt&Gl5nMSTfm%M@UonMq+U_lgblD*ZLZ#8Spo zrJB*TzFW<6N#Np)%+0j!yhVv47itNKU=&?g!M3xHPGeh(`5k?edUR6UJcfhjkVs<5 zGM#u9$Sz#;_A%s#xXH`8F)~g7tlJh=wIcWI0y=f_ZlX{?Fom4i&rYL8@VL5N)1mFPg;pVW?!;LpV z6W+=k{E9FtV6)00G30Ju-Y5+i{VED2R4Yy-wzixmV#}r!_Vwrnq`B|W9z4SN)Uv<* zn{qRZ+}g0-q#j|~QBHt-X}2?soOSebiUNKs!*^bdNXRPr_sfxbH zxt^)rwaT`6Vl4?MF7jOdWUGHQKsus9N1Jv6$5b(?WD zWhb<=<6%yt;;&_BK(GfMRv(Iv;`EOzcO2ezUNiIRyw-KiYz)XjY?ZqKrm8$0k~b<( zLV&iz9f&+`;#GRfBr2R-OYg$CPtQ^F5p@MuS2xiU7wXfHf2KB$>}@GKaCMP(90a=9 zso~7M{2pN|EzdGC)^fK4qMDY4p!SVW92Y(_*^hj4?Rigk-nq}&Fy#vdeyzy8D$pfe ziY4EuyN-$46#9MK#sv?9hJZfWg%=$M4-e|G5g3ixZpZ_sf4?kGH2dw}`?S*ZjGtot^V*tD*0+h78541z{(5Dzk$GnW6CWQ2`493oO>GoKgCCBPu(S1hesZ$j zJ0oskC=_nl$AA6duW zUZSDsZNF~X&bgexFaz7I%1p{cWzv8#vhM8Fd+Y~Yn(o)QULwk$E??FpHjfb405qCCnwCq7oUQsq4?er$Vq%G3!~%Df%6kwNWd$0uxNeG!$EI?z z(D_wUn_$zIJ`@0{2V3I45;ck>4mrGRt=6JPA1(l8pzk5bs+FXT_gR9XkVz^>o1opq z>8Ac_lIVq{5NdZw#Iya4NP4qZNu_6EaB326WXM|2m9=};853>yOO4|WX)&h8Z5(_B zoC0X23JJ6l*|R;XBK`9W&_LXTeP5JSdFWu25LL?8M@D0mnnbsQ76YnjL3wLr{yUWd zf{P(wh};=ry)r}M&gVJoX2q|^)Y81gV9MpX9pF@kGjBes!A2r7O7-`jcS=}{dA<)j66K1 zApb~ls2lIwomD#mT$FkvZZZnW(ysd)Zb{#v521YLb+3ZkCvECg)9vzRH;}U9iF4VP zv6{cRgI7(d0p?vkYysfETk06+Fpo5*#&w3C%zL|!ZwP8JUfo(wjvJ~2?h?`QV{ zNE&3lOREgIS9YX_H_XK%lpCj4bCEcA-|WLOoJry+EoKkCgahuTZF%G4 zWa5kU$oP5%=B)I?3jRqG|B8;ux|W$K=>sVrD3LzQA=K>cZ&J#GF9`)$0`Kcj%0W$4FQS zr<8bf*@Q?h1t;_9#C50@;kS3V_`iu>=`Zr%5I8@3M5Q(jnxBSn6&)UL&b&P8aSO3} z12wx^=8rSf3IceueXB+vc$AC5jN_~((Rd+9uKpI$O-})J2a5a_H=t34508Lk0_7^o z=*LiWYBQ%}eiKvSu_wT><|XnbSvr;DL}!R1-VZQ+Qu9+Mc~owo4jz+tO9V5CEhf_Q z@~kkB8dkg!Idq<@4gu9nM`X!TWDu5&_Gu&&lFx6!C^{Q2;J zZw%GINMYC*?aS>-vEu1XcsHKn zdy#&~+u}T-@7-t0l>L|Wx61skWul+oLZ4qb36{xG)% z*;ByIeoFD;)Uoylhk9%GTmb1a6LL^PmoeihD|)10oY${rRC+K(#V3@0coP!WEMghhS8f>jlEhS;pq-;ghU>4(P{U z-Wue8weuB&nG8xKh8}(xr6_^tk1DGHGeUMtip*N77tD36_ytIljB!gImqe&=LhsZ< zhU)dQ<+ZKF{sNQwaG;V0m!fyu%u!DC3 zcE{IJpL_P%uqMeBYEc#X!@dCfx9MhcyX^rZYLu7Pv=>AW&F~>aL`kkFeLHFkzB=1 zM^Qo|n7MlSlb)kL!D9%&=2q#*LV{0Y4tKDJepr?rn(=Shtu7?)Ifuw9m9dBQJ!rG< zX+T>HKX?M3%2=wCAa%WBn+C==k?5zs^;pgw08nvDh?0u-a!AQHYm%oUWT@==0&SA2 za8fpo;N>3IMEn;VHCA3Tk$`92BKfu4Crr9_z4KaIBS`D_f$|?@5eRZbsBjD5Sqd4? zuP=F1FipDh?$u39Igh4HMb4i2BU=O$=OK_Q+J&=Xr60>_c~*E5^k=Yl=^`UJ6D>(* z8E)?uto=00wY(!8v% zC$%0(yFjPWa1STSKwiu89)cf*`GkCu3LazvkM~gBsYze0Te_B>Y&T|TR$s!975Te4 z1iM)daMq09UN81atNS=*mMM~?KeyG$lRkC9*R5Zm+pAfcIM1s#G#4i*lzwV;%eL>) zR@lW=Sg~>3rF*>)3~QZwuyXy65jC8xCo}^@TsgQ@gy9yqk@bFDU70;c2q)n({A@z+ z;EC0;(n6l;NNG3R1ap@0Oh{ABD<9s1>G%X)#}}Q~I0EN?>{rXNdJ{l4%ECvySdr8C z?e^Z972;xb6GnE#QF(pK?}>*WkE&oY3 zk6=<$>WPT*#Pl-zZTCWL{WSNtLU=M@Xuo0wNMiW~lb=!(7N@S%Z|&#sT9U9!7}RDE zKp!2!z-oAKIdTIAtF?QFwqEycmk%E7M<5WOjb%fmgGNWB(r1h{SQMy0R3&uT>7GA4 zr(q&S)(y1kH>GcT4B@D!$+ODo`a`#glmT_YB;(c10}@b zKk5cBOTmO(vYi51!78{yH59aji}{a|9JyD34@%c_c*ijk+n?;gpW_h1TQxUs6M(r_ zk1MSWHD}%FTFIe?vaBgGvW{i{kK_EDPh45|Bl|c�h;65d*g z5p|2ZsNF2%pIivZUD0dy$qJT#2}G7nlZ%{-Q*<2025c()^{$)sS!G!#e( z3u|`(ns<9>+;$NcqGeo!BGhmwei5S;kM-xHPX)Po&TE_-XX;-@ytf9ox*yiV;^6R5 z`;E8v02q0*Y(pX|J7{l**4d?y+{H{OF>5p!@xvtmpTpo}6Q{(De zk>K&p-`k(+6rVp?71lcq!JaK)&Np-mT<9__WcXJFXIj>y`5VNoQ;|g(n=|(!!MkM# z%QBUO_)3ssmfXTjfN90CaNigi=;;JQ6TG%A9nu5BKi&6Crw?(Sbcx6}^_@QzEMg=? z*p4X_>33K=oW0^a>CJ#9L`@4z#kw{o82t{MH2q`K0aztvhf<5XYULkOU_Tam#B8W3 zL$c2CY`1hx;(N?~6>}Z0ZN)DX&W9HA%A)q`Z~Kef{#w#PzH%*Lo6_|nME2gwgKY9JbQ5u3 zKqg(OYpSJ_mcj07g1jyYJ}j!X?>!vZ>sJ9{)R`Efi{3fAR1jM4^nH^FgVZ-a-9G8h z5(lt^k{Rmrg9d zM)?jE(AmxIgQT4fkv?3nT0dv_P`KAixYFonA$@M{JzL&)P8M_p8ZvBs@#yo;NFnOZ z8Ug%Z%Dkf+@9E+NMB`?6rO4G8!`Wj!qU_rHeg2SrNUl_9)?L$BaJ3HQWKskIZdPwH zzw+ApBmeo$A?zM2?2PiD9vN(5pS^dWA9xU{in0W&+LLn&gs2bxgY09wL+{3Ed#yKt z_inRR`==t{&s1IX!R(r8SNEf^(3ZgKM3L!@9`}aYNZPHse?mT=Xqad93r0>+`l0T+ z9LOAioRm&FZJpf_kMiw*Z|6an?@MNTfZ<-6yQrMa}b?-4wqs11&vC&u6qT za>#exGa?rJzqwtJzMOU(dgc^Fli??w$FirPU0km>3^(dOi_?}(ylN8_mb#bjJ17Ag zP%pXv>afV!;sg`1_Na%b;ekcYGRfB9+&o2B=?*fr89XS(i){@9$UNeOH|L>K~)alLQs)?dGQe5 zWjScspUgH5g^yrzKi0}B0nsVE|6aJ(PoG{19(D@|Rt2-yP7q%s#?{LKE5>Z!vh+!} zAR~#GFaK0YoRa{ZCudyDY>nYzC?HixuxIkdlsbeqE`dw`1*fOmQrjc87pIJD+H#fNB?F= z#By37dpIhx-&qvuMQW=dT9X3Gg=YB+i`Uku{VqX|MaY;EoT`y>1}S)+&fqT3s6tny z8xUexrmU983-#qv3`$Ib_G{=FV_kx*6d|e1Q*OWb z{dahV(1J^L5df7sazPU8wXwep{^87C+Tp-v%wuqBCYyMx>O6yvIQzUgv$_jQdUGa= z5jlKk&vQHS=fg`oMiY=;1R69flLP)`*P}d6NFz;jjX|$?vz-{oH6{#a8Qz|2Bxx z=10j966%K8RO4?Dvb^<%R&&p_L0`~+ybWBfV`v~!L2Y%p=>Hqn#FqcYa-bOwHrExqLVp&9%81W7cI+o$F@1~k{{+$mF`fMYtZ-oez zP5p6x>XAO@3fI5{@YOy(K*0sy*e1lqIEV^)Cy|P?OXS=>ocKX=cyR}&n2}e-ZkAf?aaGrHP&J^? zDPbQQYyEnUUuBjtFyRhM@(zULg|`lbl!bRXaUUqqJPY!usmu`V%VpKkph4s-9G+iU zRrRCAJGkG*y&Z{oq}o>fH8Q~B`CkmjNYp;&z-EhzHs$_-^ll2vxAtAxk`#EB_An@T zzET-+AJ#j_7al-#N2md(d$SeD*@PuK7j~6$Ovz2&O2ZFYwJ!~U$Q&zhiCYa40E6u) z*Y4FLbX0eoSFGK7$@{L~UDK@%a#F-nOdTn>G2gMzBdR#z=KhyY{LwkFJsFVFt$ zeTPxEcFD-^^fuF@V5>@cC%gRDjQrnJ)p|^eMrFm9sZvDiQ*y?l%3F8uVtmd2$XuCN z443X=;73&Ag|M&t>??81bY|OQeImBSymqi;&ew|(-^K;yQb)Sh84OzW>G@APch36< zg8esx@M%|)APZ0dR^Y;}Ar*phTObDCBDqp?=_;na8lX3!n<&O zd$~#|Rdp>c9x=m-Z7Rbnfm5OAt5s>TQ_Q%j8-BvGhs;A5febu0h&kd_e}q$(9QxI_ z)c@MwKG&;p^a@8j@kVW}T!(i?oxx!3oRTkP=-EtN1|iMXxv_}`WBVaXN__&BanEz#h%mK`JsO|!3vwNNxVtW23(3X$jAwk? zUcR}8u~{DLYcP+IP4Vh=*$oZ{UGs* z>Aoj$IDw>LwbqeaNz@C%ytK~Ocn}$BFy{YuG2nC)OKNKeyRxuOL~%c_WK*>Z#ypih z_t6Kl;Hy>zA~EoX=T572b1P4#@Pc&1L$60=@`Ey=HFQFYBsT*^r?f95@VAXBfl~_FE6qU=Hy^p@ z%NNDqLHUXjQ+mQfj^b)OS9N0Jea7c6Q+q@$B9$H~hG#g9ML21Y(;$Gjo-<-G687!S|tC8Um+$@<@0_)fsn@!PFmo-tMc9F|Dyi&v@s$^&d&RDotg@Q!5?_uP`=_0Q zi#eG>gzJgvT-H7yg;>ZS8eZ~Wt?8TQ4-DcFDn`prfi7nMiIc58tO)f)=qL1#-ZH5C zLSYgqNxEalu3bdT1Un%WJEJIPx<$Yn!>JJQ&Q?|1Xz7$wSZ7{6uJUSKM-XBJ)j175 z*5ALWLlw2EX}O@$uk8!ZRJA1k4+vA%(jS+Omy=tY9X}E5oNNg1*f5wfvN(PF=u!v# z{n8)Tc(^t`yHG7F! zkG7LTsXvytcVZ?Ir|{_PS}HR>UGfvvEjM#fm?I1_pF2ci9W( zK%(|{`h1EjgG5#I?k4nHNTyr8?vr18cCxR1lEVBF-SXg&dfRG^%Ia#j(JTcwJHLGQ zDRsv}gZqkmybxjHyMNu&(KJe5|DQYKOK;}>6!j}{3f*NDj^A&U%5@#HS6Dt|w|j+> zh@MkJs-UBF1rg`Po}N02w79zO#d9Y3;Y)%cFo7xqXpDpeioIBVTf4WxKN76C>gitU zeUerL&%pl<@0;5DaNBq;G>${Al_7baXESN+{i2XUM1H45ez5#rp{`}F8W8`AzN^_~ z0N(0q=mSQ`yQoVOciFI`yooE(jI+eY;}yHQQwvo!r*rdiD7RX zG~ebh=qkR+3uA>f4WkC_HPGTA)1s{~`R{(_h;}hzvLZeR6A-bI>YFjf6qF!5|MBqb5b`DA7@ERW>}HP`7kE z*NL@vMAvFKrk{+8vb?{i5yX63Xj!*R z;%e%8O^fE6T^$uQ z`lI#QD%Gv#hiUzid1sktb&icQ_W{ki(lyms8}}#dy@YHr*B=>w4!6uYPxUJhRt~CX z6ii1{+(NC|=Y%*UXG~*klBei%N^;Ufx!l&77Vq95wXG7j5=IY?S9H)0#FALTxzmV- zfNm~+!nOf1+nvVw#>z1n&~yh+0V0xgUcCxg=)b1EuIpI_i&|l~TO2%jN!iR^M0U3` zWl~+8w|ywABX-KC;kq{_{D)6I;$gR=Q#^7^FilIz;AR4d!Ny0Nh$<4Me|0X@7~7nh zj+Ry_<9ytFN@@mfKzC7aELD9z{9S^kH)W?$zS^|UEtL&b^Tc^^YsJXz&wR)4fZK z7I-2}s?MMMsStdloL7A!ZNZ0X)9>|b;Y%S|hRpWsDPNXO)4X?ePsi$<%C;+WS}wBV zZb_IVzd!;BK)4tO8qd%M}9NTImzle|g$>~l($TBtRnQH&92d}us zWO2HR1&p~rys%lIlGn!j&uls1@x#PkL7i7tPA*trUp8*S#Qa3>bdf%wG~}iW5#3MV zO8V{6+jSf7w?2J~LD6=HlkPlm@5cY_yqr-@sJR*_(jV4{dpPJi?`OlaR}@}w893wQ z3$F{pZsDPk-~F7~hP-HYQ1>;T zu}Q@UPF3f_ky)QfiJf7!Y)vyf#;HVNg9Rxrc&QAkt?KCy47C3GC>MTGGOk%*=X;)C zFT`fo{%Ec+)-d8(l)wcwyWQsv*lt`l*>WNri`?4|>vGZmSP)xu5*8xLPkkDv&Mp3w z&tbT%tMpT&p&->yoJNYr@Pf1M#X(ns?q0o70F5=Z__MRDAuLYI-})&cK$8AX((&`l zmR&-~3kG$=`U?i(@M7y&SO0Z}iB^}Np25i#`+aMuz|GwDZbZ zR7xkZfqB+-w)p95iP4rb#f{B5=C>+B>L=&Q^ag94!0d%CReBF#F;^$xJ8bSCW(0IIc_?B;coe<;qjT#8Lz$VOeK)iRV^K#Xh*nx;y^wccf=DU zTR9d=HD?z$u2=X4!<$1XEZ1W9)kdpqu?A0zhh7CCOeldmA7h0Z&(W$E!Og zEpnrW@L|mhxi#nVN`LEmAq}&*00oBWD|E^3_guxm!b8?0@kM56bvt6+b&tsk`2p36 zj#vj8h;D!IUrSgHKLTqBk02n53UG(-g>mJ|Lq}eH-;t zb#PemkiF89%3Qpq)sBbkn=PF1r`gI|oGdl@q_Kt8-HSJb+hjh|I06#N=va}$#_!+n zJ}ZJqo`62j&r68Ic9m0#BP{TpHH-$6ex5uR6P9?^J;v_2JWU6dRcIM8OOL2I?$?i_ zOUbO&aN9);S%{^elP18v269)7erVT5LmM2?0L+74#;}NBY89II0;3si!he(m%#u{l zC(G(W?mSfJvlPGrPs~ob0Eud!h}k-n^1pMh;L;(DTa8RBD?hBpA7_V9+h?j%;_8JZ z6e=m$Z3kQ{n`wFjf$Q}@*Tv6#Y$4@4o325THhc2XHTjIfuJ9?S5Cv&Ma-Hm%?uFRd zo_DBECpUf-Ix!Xp<#uFKnn%$@j5<}!bTb}h<=|3ZR2K4hD%+(S%x;pNbEK<&Dbrh}j;5}7yLrQIqD#MC% z5kJ8;Vg>5T+8%Q07XGMkTHD53)Rp6m7|@VLZ|}KeVZS)*KJRyshmR)9@2V*c@f4VJ zT+N^twuPpLxbENww5oG!T=Yje!=q}u7_p5}n9OYvQcgtVNOXi2w$5>+*?sbKj{6U) zWvC0oUgPniW>2azOmQIju8UHA1b5z1g0BLe4c51Br*>Fogm#l0tkd$VI*ptnmb zXa4feVcJ)vO=>nfC<#r?&F7Y^!9URtiO9yAEfQ_iMh(aE|DBHT)0${Ir%wopkjFY= z5{O4oQ2IgoWpF$$?(G=A&HVCTMO5QFfnb5PbWK+r2MzZ>C;sNNELT1@_7sGJYmnfb zHs-*xtNWOCQ!qYS*^{RmF!~J{zKEl{QrF||Os;EW`1)nQvED%ekYNYf|MO}gI8uBCl;Eyoc_GJSd$Af zS)8z}LFh30OdokDxHd4sF{MZ%L&6GE&KejQdiFIiN^Y`kPzF+ z2DZ;L(q!+FckJKpf3vt^(okZZoTY0>y z`+S~N+rgb6235}~Xp_M%#Bz6Q^J28_s1EPQo>XuucQy5gqNQb_>8HYrhq{(DHgC|| zI8i7csKXrK7F_-naF?2fK7L(6=8RIMRKwu~s>+o}it9L4B%bT?tzbatLBpRR0cS z+-Q_C*zQtwHk;&i{S!fZpLoYqy;;#9Xaik-~!EcL(Z`RZl)Zu*lcayyC2@Dtv2NQE-uruuDiwyldnAF-bgqZET0 zBJ(gOT66i$)TV`qvlKJq(cM}aEzQe!IV#^3i{^<9$?9!cw0X$G&pidCI)^&!^x{Z` zFmt9QP~Czv4$rrYTs$^?)5}3CgNV$TS2LtmBU*28tdTSR*mMTl36qLx!fN3fbPt$)?1u?NieNms1Q8q67+5B8?7=CLkzC&=aPE(ycmz5@I;R`7_v)0_ zyaW&R&rPZYryf&_&UsS@ezby9W2m$h(|tauBb zcNQY@2$Bc$EuRh5x)_enWHf%mlspZ4_O6pmX8XF^*dn5D(~2)XtMzV_tU4cddabMB z^N7FZZfyR-bW2!xOW~g6R*}{LwzK zXsi$+^@*4K5`KQc`XYIdS`w_ZFnMS)qMdf<~ zF0yQf@Gf$95(STPKDdMUvh8*|3L~a{wC43>_K)kuX8?a+31nVsqw)SRUPoIa+~Dq! z>2rf~Xnv9t3rxx4Ie0|x=gcv+5n2CGA*OH4U-$I~2G#io(>pI6>Fd8xoU@~orDudX zze4>HJ(BMH@^9K0&99F{e`03wCE$c*;jJ_KkUBD0C9EKpmY{v=c%qFNpCQ>6>#3k( zJ*)Fp?;FYJ#7E18pZlKcSx1F}?!?4r>X8)-h^|rFHmjgS#cSK1ikJz#Ft}AZ704aI zfii+dHJ9^}uNUf4JPM29v^V+k51+|5o2$0(_E9MgmAr#J$L}SSb-b)S7+b=dwMU{y z{B)4HBOd+TKVapC@D4Qvj1)hnDTUoPqD;7t7n8f=){o%pydKV$+*j~m-P%1U*q8N^ zHqcWLQ#hK(wEnRMad?2bd2PcOlF_UTCu);&@LaQOVe_lYc%mP+Lal<5Z<9!!3|)_{%7)*A;s6W(<;{NdBmBCC3xYwqW@}wimHJqvkX7 z3xAH%x4S2`58d{AOyBM@zH;G}J&_!ZRgxITIW)us=uyY$?Jb~8`>3A$&_ro@=68h@ z!*8mj|1AOKU4Ufm;#vK{>FjA*I%fy#lyYe-6tw#Ul)GdZLRY+=f5|W zffe8#*`DEF4W)%o6u2 zz%k?*x#!GY;ht4*RX-8pq!&6u0UD$`-*#^5I`W>-zeb)f_ugeN#j-eF%!}bxUR(T9 zzg!UCGQys&yHOVqi@AJfzK&k9<2rZml^wf6QcWU{%PdMa`6o+z;>;mFmI5e+I1(oh z#3m6uw%kYX0X?ZcXUB-Pf+m?1Kk2?~vgN#B`)UN^T=|e^m9-*VdEI{v4s#@^d(M}+ z<)ALFEQdHRO{xd~k@I2PA~(F}+rE;$h7VJjW~i5<6>2DiVCr=B@wRrA)RG$W@EttQ zGaH^-wTS$96*#=`-uSK6tg%SGwN_Nv2QPi<*t_-?s5U%^j3|4nSS6fjjUfyb01*380QLuNELbW`_?na^d)=*rHhTt5jTZXF}JrI zD8CRpMjjl^UrYa<81!Y*wd4KN2V5bwy01^Km{;MAsLobm{e`lYcW-zlZt->1zW36a z$O-Azy%Hg&U(Ee1adY0-_=w|t*k$9KnE;R4TmMoq4N15KR@HoeB#^^ zqDf&QiX0Iy|2u(JiSy;FzA>U^=IfB{%X#ylK_gc$dNR%8daiK*#v(i{;vHUdVwMgu zujI~@CB3icgGs3u^E#bsxqvc;JR#5H_-E5tDe>Dc)2Q`(@Z0S zzG#{+g?SKFZwuN6)S}i?4o%lx4`!7txfaC49r3vM8(#BlkF$2}pgi``>?XwJm#j$twXBBp=!`;iySt1JSCIe?hNdU}j4d})X` zdG=^lM>9kDGRlB8$>Ge-kvF8B>AQGDARL7k*iJyhXiK+iL4TY?Jz(NkCfau( ztz2l^Won8_Or@Z+87y8=I2_&zq1EKyyQ!2;IwM$ugn%78bAG5OZQWU>4@is}e#bQ{ z3ocL2riJHouJb3K8Je7}tji!ydr6g5`kz`XHg7}hGIoAMrW+ul$b%n2(6X-H-ATxf zomDBi8qZwm6CR&my&zv{&(VXdjfpdlnP&|KPu-j!bHE|3-Z1z=uQ7tU&A^dE0_4Mcz{gm^rey6>qXw&H`#y z^?^=&RfWD6(QW=~1K?lQBZ`qhKMnH%PYhz)9HrhawaLik7?A3<7EA7k<$pMas1cC# z5;5q()duXmLQ8J0m;Km$2$c%Da_;+7R$1it{{ZQcW<*Mj^JYjC1mRXQEfU)z^t|z& z&jPw)SiW0!Nla&}MnR5^%Q(GnZ>kP=MNY`9Z*{{rDua%w>_N%liZs99^SlpwjUJ^= zjE0X=nT%z6YHF4m!Rg%K-R5(D-=29g}4)>@VlP4_=wE~-qD^S(-2=h*pMZ#&1X$jO;yD3*puqN=~yzwRP*4 z>XAi%0_;k>Zl2$FcaztOZ>+iEVjM+k6gjIk*@6y-PbRk(BC2rKxSR9Uv5j}v<{3v=RoI<3~LAJY`xNZMTtgTmN5gD>{ zCKn19zdjuWiHB0KE|$pYCC(aJL5EkfK2-!!qD&R{T<-(5FY;(iOV zO_C`XYsF=I5?=VpYuT=2T_L^KJng9MLqz5Z0g@4_TO+LiPeg^pbUK%>oobpIHQbQP z2U=;KCiKgF;}QU}0}S+7e<>)`VOpivMO1g4waa|vG`eO;DTi}pA5{E8V8D^uI-+^P zlr@o}?M_A&Qe8f=zo6jtyN_p7M6cvq;8nQBLZK)cd@qi#>5ct9NrR48juDl2WsXap z@#UZ#^eGbJpWu`m)aC7Vj~$=6ScFufbS6ya5JMn05mZy76nAKAQ z!#h@+Q9zgfgq$+4ba)#pb6+7#!D`SkDF3T@KK?(5N2=`L@X&zY9uSm3mG41Tc&gHc z%6GZ}Pt*r4``F#Z$RT6CxDtp^in|c;WZmDJO$0YF|RA!$UkExRA)H8nowX1mKZA zBI7n5ZlDKVI2{7HxX$Ty@-qVC^P@4qDU%S^T+PjqyFMP|=bzCd$Z#)k?3_bb?Efp~ z0pVXxM%pQKXegdv&03m!LHRf^uae9}5a=Kn3C|Lr!EfK|JSMWe`UJu_c7w&la><-B z&Za&3k6tUpW$O<$DX~A4$X347z2)X8G%jBnq-jvOs)LG15W z>dp`NKL^CLFE16%6+xv6THR6ZTxUGE-bKw;9P9tVXZyKhV8#=j*a_*zXY~hyq%}I!(-S&0*@4e$t!_)kh7Q#+MA^*) z(#&VKa}C!S$DMye^CQJSEjx)df(DvSkWe@y#g7`|bzy_*kLo)jFQ7(jr$Q&ra=%b- zR*Np%E#6~OJuKUf8+4lrtsk^}@x`*BpAe*mckkb^qQ~u;7-Sx?JH}Dts)Z&=j?hYD z;Hg6ouTq$Aq}W_ew#CJ!--`Cu1R1K6xr?rYb-X@?bJA34=}Qj9!=LJXp`1QB_O2nI zPpNWMnN07H01VrRCO}q{{@eS_a`qbHHhnFFr72 zOvJ5%mWER%TzGJ^u3KIUsPI#b2}-5jXl4%T(LeiFd^Wv|-k(0XB1kqc}dRs3_$eZykjlQYkvT0`us)~4QBAqtRipVN|@g6*|+0n2XE@%%v zc4uJF25{KOL?OI`~YXugQC>Ycewa05lcxEQJ+Ov?M?E z7mvG6Kx$_+PIX8@DQHlmYwRTcY5DUx20jLCzir})$b};^!y;_W-ET1z1;MQRPyF0; zJEZU*N=NQ?3i1}pk!^pCEc@AyuS{DfbkWk=Nqs18Vjg%YKrgYzdx^WZ6`A-Tp0zQA zvEJ=(mh>NbR4`-EO1=^yg8uvi`2tenZIZ~mssQy}{Qop(@LOH8U4lC$8^8(zaGgT! zqWhjPM4;C?kY!OY)Cp3BXPBgkiue?y@BcM&V|0#v#o^F9a4V}14rl3wMu+bfLhDYs zwzd^wr{qhF{&qpQA;$()oTVO_$;rR5Y{}w$=&=*m8)LdXeEOdufvNqR;ZZ+TGAI0g zSJYS#+&ac@mFG8z{ zj6|faiUfkwdN#vCF}&lG$fX=^)r@+<65H+m7qT$q3HkY^NDieO5B#+~IpO~*^LD%< zv85k{`_mmFDaceJT5EQ|@p+k5kKm^9KPo!z@}#l)zq=E{j2R#Yvi;%5DA1JXTHeoD z6Ct_-GPQr*)7FpSk=jK$NGUQWWJ;WW&t@)LgQ^j5unu}0AT~Xd#wQ_Eiq<352wRJa z5?jaft!jL-i z;FhJWyr7kEMr?a9CIbQa7_FP4$U|s#Gw{kUk4i8G_Vzvh-x-Ra9?=zo>#PR*3OSy34pb#vKLV`i;(P996irftXK#SUFf4B^Zyjk7q}(xE&hm*(M3Guq(IAkWs zolT!%AcK#X#4$x;$8@3pC;+=|s~6nLYq=zq3MATqax0bv6Zo12?6V{8ONSsS%WvzO zql2}tid&+b(vMb2N_Ad?e;#5TXu+>P znTsX^1twSW@d^B?WAYbrIk7&RC?yLxpIKRRh`2s6pOx>N*_f?!F?VQTl24H(^xAEl zH!iOw3;Dq#P{xJ2x*P8~uJD#u0x#01vUA}Oy}nGEQ=xCSuk?G%)z!h}*@dicr1rJy zgWH-7@Mw@aJNK6&_*#&8z zv#x|WL6+0f}SbCY7&4!wY z@*8nu^}{niE;BEcA4rSYal1DkOzfI3kg{(pV@)kuA1N`Kd|Xk=W;*;Jxtq7I{tW~F z7*J*OjY215lAdICIA>Fib2fJ2b23!Z3?%6TFsZb@#D+HA!$vWse-i96Y_tIy9uO{T zEw2Z$b@=UBd^GZ9HEHs2&QdBZ&y^isD7lj8DzJ_yi)$W^+R|`+N;2FV;YENKv3#u# z>aaNc5Hd9&W+@o61-I^MN0iCl!FhYgkic>qk@ru*IMMhL;TP0|sj(8PF4k*d4pE9mF+0XMa;9ycYwvxl*j|6ymJ3J?BTf;O637g^!^^Bvd^`qa?@BHN8RlW3u zsmf^<%REASR+6rZ7g>N?8gY#zgr!zbc~q){1S% z-DGL-i2}cCnYfFd;|tTCD!O|LHZGNc`183MhA=ml6=s0ef?^lsmfSOq0&otjB?6+p z1iV25PH0M2m@6mbD20WqcK&Yw>#47E{(1AVd7b;Qy3pIUX@Ma)FEZIO(5X9SN0MZ? z4Ae~2%Jfmu#>N@_dZjE{C%1QjJ+Ze;HTtYEeH#7EKe$HvN6^!|7cDZ%Wz~@yw5y1q zL1S51J|(d+3_)$ceW79)K2cGQe&EFV(h(8iz~nYyj~$w+sEnfek>wR~_c?JnFwUOr zV9k1MlJLA8Fqo7hHaj^P4E>n{t=Cv8>z!Z08~d11)*bFy{lb;;j5S}^aWrG&G(+a6 zed%>84pB$N{%;Q@4jIR)k%LWl4;L_{R=S5)aA@h8tl3C?HWMCG;7Wo@4>393V zmD3S+Uwb)^!!yhVAPnLeVDgt{kc4&%9tR||v{(E0T>;rSF z#5;_oG=F%q?s`E1**L5N-+1bX+oH+GBUVALkx3Cv67m69qyVodS1$9&OzWx7ISQ6L zjb&Jh+F>%Pf@PD!9 zgc&$4lIq?5y@>0hCtZcx!b-T4Av(ASMPuTpn%N5YKWJ0Wn*8z6C~q4g?#LXq`ZUt1 zg8zR3M*ZL}1KFh7@GK3Zb6O7t6FIj&Vk{Ir)nDZ;eAWpP) z&ZV&3sE8^_16DH=QjTYl&JmNQs2ar6+ccGdcaC_%byvV}{a*CwKT#d)OezVkn5GTZ zb>5DV1mXaQD*sRHprlcn?4=4xI*|TgDTtN-SB2t!)92lg^93jaa3X-&-+1HRdS$M; znxsq!5NZz+hQkwn>Ex_U6zAR7hm-cqyAd4#U^H>bwlUC44tL{Vh0m_U4!e7C`eo&6 z-0o|l!pNnQ^_id~71pdZ^X+yiboDA%>T$nNIYu|QF;j=kS2U2Xo7a!+*RQWc#zv9W z`puM-&)K%1cZFA<-8Va@q=~@&;C2z5F#Qpe6YKLsq~peN&=2QwnO0K+nRr})W^DSH zkC*DSUey|0MEL)cQn=PghrkfS6G>*;8ezb-LLH8Jf4jF6W^=Zhm|NH<`+)TTM$yAC z><6S(Ayj5JZ0VaOuR};)A033V#rRI<3-A8@46;gq5!R^k&?ZYi(_-SPMb-M=n)6T8 zw0w1iSmqQex7)3d z&l;RLzPC51W>Uo{XaX58p(tm_^pm$*E3wL9Qdwh`>~^0u=9s5(9Q@xy|;+fRhi!?;ON2 zY*tc#^McY5aXTOexMsB0MF%{1!`yR2C|8`mu-);sXiR0gj~;cvdAvt3_CaQe#u`re zY|se^WK*=<$8AB&7jAU)&t~93CqmOV#mKIl!hDI6@hhQ#gJ+sA@I}aM83!J-k`fkO z8TK4=z>iGDhh`3glU+C5LQRkXp!M=QFlUDO`I)A5N6=9xIZQ}u*hiMh%h`I(vsM5}c9jcC*3J&j zmk`bZ-#7P7e?=f+LdB@|&bZ&caDWiAo&n$Na6h)p$aeN_7*8^wPK4pIAplKL<3-f) zC?Q0(KTY4y5(Ru#PhK%O>#tEREAIl%fi@ca3`a7;OSyybU4*m-Y)z-_QUeaa^l<6i z_q>uEKDSoW&RHo{?&un9GJPEO4DrS9Nt@@w5?a-F45$bNKOU#&E_E&a_8W`S%~L#p zFlOo#5C6hR>p56ovc$eH@>*752Kx5}dsDgQJg^)CuCSQ3)A;oFy+8B($b*f^t^f4v z(K|esnJs(}(759K{H{bVE~0L5JAjL|eR?g^Z90 zE2o1Gd%8h8ylgrS6hjyVTA}XqW!WgF9$HOr*Mb*T<&=v(F3fphuX?%tZ2D)CPrv0j@tL>#<94NlSedg%JiFd-ZoRUT@nhh7D0?+W+~a6-^1+GQ^(UhLV!ZAi z%X;m>h81sl>cBj?{2^qla^K_lcAjmUMbADIRg7z_b*Mi7;AblhhNn0JIIx;fa9NYt z(Yb~fAwk<~mfr6}n>q8`ZDuMXMPu!|s%A#m`@GJ7XQoJneM?JOQU--VE5d_AqJ?Ri zF}ks`w=8aDcif~k=ThYS{3qw6vc-IU@bSM-(XCV$u>i0jpoz*EbAD4-*4F_1H`?%R zrJrWJ*kPzD+giasL&aa<-keL8X&)~>i^pmu^RckOw|t;TSRb@i68}Ccb#V9^j~N?E z>+&Q2T3OfGU$@8oNX7S;O~ht9dhXW0t33a#t7l?tpIu#`u!5P=(lYbU1*n0Z`6iyW z8r0S@y1++CcQoKT{(2>1z^$FZ6jfN=7lLE1(5IK!T->qR1Anq(?Q2(OG{ryd-lndq z%CKx=a#T;4{m_7zJlILoGtb6tc@2iB!Ao0O(VSAy<&Hel1lZo@OFA>73IOr-UsAIZ zaIBfqQ>sMUenKkHB$V_UJ{$f% zQSGUkKAP?%+8%UE3Fmf^&GCP47yeM8`OF=TV?N75MWQhbrh?w~Pu2r7TQnzzvR@d@ zA|rvK=es*rUU_OFB;^?;y25Z`6joEpwhcJK~&}2YG1wE@?xp{9EEAH?5k2+x4N{tSyN3L^_SE$1DMivy6L)m z9`DrM?`Af5re5fK#^y4CsAXF_>Nh%z5|OV7!36%ddx+t;g%Xet2$G8l*Py-sT53@N-a{dFK zCBg@B4I-nb2RY+!y=`^d&!XbBsBgak2VI1>=72NIazOoYg;3(ByI_~pBkJIGrR$g^ zn&s5V0d8dW!fQvi4zliQ)XSw4Hqgk}cepud%;wZ&I>G4RqX3O562z&JS^Oy-N^8vA zJHZQUh2e?)f=7cgQZ`5Tw^ftx@MYdJkt&rQ96rG5&2^UsuJ?C?QDbyQbkF;QFEf0M zl{(Bo_heszOA>oIZK+1YxDG7AkHHdL(tGA~lU=dB3t`x$^43Z$xLjpnzIFU-lV3iS zSJO|QPEAWoxUHJ^01({M;^MNhtHG_5IX|JGBUE#mfB4HJbtn8Gf7{+R% zbKx83TQ~4!9u(8cQj7nOt1l0Pa&7+~N{S*K?Us;GqSNWMl&#Q0A(E7}QYrhs&$Nis zLRp$q){2y}XJ1CtnUGYnHP)DvP$^ z&q=~Rm6i{f!TGTY4!H+U+lx_8HPI6mxFBY(jZavH5U=A~kq^b;sJUp~(;Jmth!5WH z$#~`l(pp#m6LliT*h}K1(X)^35~Agqls%(iDCwgzk3#MGWSmQ1CzOE&LpPb@dOzM8 zU!uv^ow3EHdt{Us5YC37#&2~F2yT8c8p{&d8+0}cM-34u8SJI=Mc5O&H zpcT^Vua=P1HORGJrid5I6Yjcan9~+4nJ27eXEQUHVPy(BNUGU(QTfnPq$($(TM=l{ zR$KtSV<+pdWlWJm@9Y|M&&z#3*42?_C7=vlsTHj;!zo<kH*oPYH#n-o{*Ezd^qP46*EX|j^407H8$Yd6o0 z{{*VB-BkIgLq5|JE#sx>)zpawB7YOQKw@HR*RJJyO@uI}ECT)#c-gVitSaH9b{8v5 zguJ=S9ZW@&tl#AUKX#6;4QDbBw1`QOD-X}ZzLJ7Z1#k}1U?|XAI&d>rnVFCjE@0;S zk)j&e`nu>4*5CH1Wp`BnVhCO1ZhLsS^~Y2ZWNqI3Szp;j^u_Dh6&bt9vi$fxEaU@i z0ymmyja3giq3VN|si)T74qe4MPG4_4HAv?k=Lo~s+1ky${uxZ__GmX>xl16fVQcte zfjH^?@Y$032UXVGWa?Cz3Uxfmb!Ff@cY@qr*!S^rcF&gOpqZu{4H$b>;$UL%&ifLU z|BOFb95-^#p5?7!prqVQ(+-vt`&p;J9hbIh$~mr^G?4sfgt(EEh`m;mwlMY?;rVYa zYT-9Kcl$c)p(H=;c7LzmlGJxEwuxSyKzl-d@q*6@@T5l_of4}4Xt+;pXS11~b1>`= z(^NyVxaC{;YJxF)ky0j}OQ0neT5cKHv43D$px2Gmvfn0a0At5C6 z@z0Jo=VrN+vZwt?AudA@k5{ZQ3yXMZ3sRIBR!!q>?5Y&~_%zNCo4@Y-`JS2WHM45; z10aoumqH?qq4B|6=H3;Db2ILRNo%3G*$AZ)F|_8dzf#v}SXrehG6AGclZ?^~criVA zFV92vW@4-Q*G-@?W~`M*yE*)UAX_SBn6;VK)9~hUM6B_n*4<<3Yh13@bu4<^gOb(j zDcc-^ZKgp0^lotEnAUqO4NxB@d^#xeA>UZ023h0m zY;ht{6|#(`Y`g9RD9UK{*hE=bGl)=(y3;>+{LO@!`pCT^_M5M4KZbSh!(k~adw}#` zQt%v^ubj(gY4>iwl(tnNrr2wX&cCT%eEOmb5JQBXu&Sv&rCjeBBEvY~0jN>^NRXap z_$}!N4m(+YqVM@|FbV#WlT2cwHPdYM=y7NMAf~X5WBI`T4YNDeJdF%IRP(`US@1b8 zYp4%A?zAxYTt*JvGg@6lYir0~>=S*K)ITMx9z&bDN$2l)ftV`geZBWc@1TYg;u&*$vc3o15_UtC%4! zrfT>T9@h5;ANIJ0U7l&LU*%U=733ga8rDG5`{xcU)AWwy0t;6An>)yy98q)P2D)3DKEiS79Qi-mq%=nAmZ5YpLjdVbow>{ zm`fBPlvb;g#u>ZtYfK9b!A>9<&ir{R5gIq}$ye$_PCy{N8;Q9ypD%qFIJY0YCzk2( zk!2+hZX`rN&@UlOLl8`_S^Axyb!Z*p)xBBVD!3IOM5oz3#Or86x1x7<-$3W_gtc}? zk2muvWowQS?40A|wYCn_S{50LR6BLF?ip2SeML^+?AUampi9)g^#Cdo?+`2e2)T9X zzI?JDTl(3jYu@GZ)=j!=n(Rw1pM1jMQU>LdU}UEx&hoD$6Vi=#fsuN*wPv$4QL*w| znUQ;LZQ@@WSBHeYNWEx%r59+~)X-Txs3{*0*ctv-V4wRrEh+9P1A~I8v-gA7X!tpk z{+AHy57K7q5=pn(v;NST8o+W#(sJiy&S(341eeH<^gdC`+Q7SUsmmuKg7hX}#lqId3M=BfBP_PAF{R zbPj^a2)`qYH}M#|>r$&-*eO%^_uA8?HGXy5Ki99yyt=i%GDUk$YoWDGSjDm}_V+e# zJ{gRPrt1Y2Dg91|+=y52GXJtC%H0a`q5TD0^Y=T}&4!p^Xb^4iWlz3M3YYgNs=|rMd|kFmD#}*|3}xxNa|!+0PjFrUjZ6fIVJhUqRVo5R z(3^AE5vie#R#&Km+ZoJtM4lUe8@1@&)m%wFTy-s6vlq&JnZ-qyH%{KUDyr-Imte@J zu)QblfOK!?HL_<(@ks{z<7Bmdv{T@$w{rA(3SB8Y*8BNwxX{~V&w;g$^}HkQ*BFX7 zT_ay=FLA?l|SOP&SQQ>CR}lUsF_iB?@M9g-G|D*O1VO4wzi|%SDa7rG_*39);CnkE zU0YjveiKPtxk_rcjy9(OG`kDGpsc-FT*t)TpR-FlTUd6m1q(?_N=hb)&j;b&phXDv zCY-`$rrGsWNOxSm`#c>98sF};82tc|#?w`_`iuFI6XY4a*Ci70D@g_i;);>I#GYCL2TOsk?NFm2@iMrXn>UMpl22LsoBGA%^ z)LpQbuOCLx`~DmdnUW0Z&D(sry&(fF&ywg%6#93sAedZmFZiFNlMLwab_(p&`sFY! zSN?+7FKX9)y|f`eNm~u=rg&~sC^t70Cs8VngUOc1cfQzjzangnm}gKB8nkh=LbFlZ zV*Lc0vsNrxT0Cs{gOd;ifV&*2@xd)hqxVflwsx67{gn26khQ%_M=AAPV_I*;9YykH zx~lo6saVt5L0wtDSN*hyXQIh1F;cpN?_)jwdky|#~-BfX@FJ4$= zzM%_5lQ|%JPoNxZl0&rv7)uWS6Q?epm}q>c_F%_Vc{Bb(nB;Vknk@U(ygFhVY>2^- zG+9)8sqoCR2Jmim6wR<;7-vNhBa)!}x0msZxk(_lnYy)Zf^)n>D#K`J(<{Md=>~Fw zgO_s=d&Fps5dLg3$wTgbacsjkkZm5QT7`S-t`#=1>-5_9&mRD;5E~iU+)SETmnXa@ z|87kLi0)Frof#fIyX<@s|GmDYz>ey13tJnFgV(?QsRM6c^YM? z{WHxsr(fc7-M~aaWMph?EEjBJjK#tG7ma9FUe2C^G4_G489#r!I(X@)58l-LbQ0NH zU>(&DJWib13ZyQO#J~o+9=qz{rQ)}@N{6unrVRAf6Hq(gtf7JLqWVKZd!F1GJN+|s z59_eg2Y>=f&j%ca-ZIF2yBN(@(No^%3r60{`t2QI|M00w))}BG-cZkAC$dhYjY7{F zXZpv~8R*j9#?g2cv0}-fsvOL|hG^hg%ySib%BTLXZ{EBi8EDP+%gvpZYR3zOMq^BqDq^7;tcKOVC4DQ1Dx1l0aRS!B^$KpM9lC1LYw0t}@S^Ff>D7Y3=&=}~@y(vN&BG6h%JT*P zHCBN0!?cf~<+P%n-N=7XbjWZX8=nam-C9>h%dc9ILp-a?ucxB)f(XW4^zxHUF6Epy zX<0v+w`W%9VP_;0U*6psF!RItb-&;%aHS3@J?_XjVKGiXWS~Oqp^-nciTbfc&wbbu zWq|y${}5?7J2q{Nq7DZf6t?a_lzc60x1O}g8 z^FrxZVi!4~`izhkQSG0^*6mI!-WF|yz_b{ZvkOEPJ6uY6$W zkQaM0Yzh?S+b{Ks5-#orowJyIcN7#9xU2Jo4Wt0A9d~7{_X-%MYr5J$Z!uMXKkm$c z8C1UdfIga*aT^TvDeh(S?AkeTlR@Cf^>53&+iKXZJnAc+F%q^sz0XM^0Tz)Jg z&&BT>fOqwn{4sIF-w{Rm&7IBYU)a~w-H%gok$?HtRic4u9#c(mI=?YstMSw53NR9S zS)2Ltbrnywuk3!-IzOEy#` zeKx_6JaP3+!zWizCeAxwtvgj1^PjZa^2_2pZl{{s(T`~4wW=J_%hLg)QJS%7 z4@F-6=%;E9wOyKF&L`hNGIvvYF{zd|LkGb&>*0Mj{a2K^v0{lzw-Wc&QOw4j43nil zhtc7lJ$=V{*1DZM?&#QdsfhL12S4c=th>#1hh}BAdCW?G=YZLoBXPsNb-zlsr7o}j z_u-PbSj>+0FH@2Zp8WN&rgkK@=<&-Wy0OsNC+xv0Kr`K?`hzr~#(qeKnrY1DU?<1A zJq5BPW{XArcDgAbU*pZs2Sr_h|IJ%ueZ{4Otv^cysMOhIdhr3B{FAxo3T((QnH9Gw z2d=S%h}_BT)tsr8jN7%m zzC69xRcQMxb4VctBs*VIo6(AFSPSu8sC;CiNi(_Fo_lI~I$GVaDf3!PtawybX#Uh2 zFQ+(VpM;|5$Az}`c42udV(~V&Ke%S0>G7QwasLdehUjITW%nTy+Oy{u!E5Zi40U3< z&4`=)^yB2P>{kzWQo;`wTUC}REftmm1ldhZ-f;AqL&HcYe~RQpBuBUK3`lsHbiId0 zh?FDA^8yU3p>_V{;q5FrhCu^jVknD>{E=fkw@JMLS#90A)`m6m)o6Gu%nx9RSC^l# zIv(pQ@A2ZrHb?FYK|Wln7N{+ddZM4UMZY}5H+&?cKr_tsOOBq%l`bA``gf#l?~NFx zQ#(cu7RR1E#BW;XB*Z$_=oefy!-P(G7k|xc!Nre9MA>dNex(Y6Wwt>!5y6TLoC6bbl58P;$eekU$A>!V?0vWt zh6vV1+Gx{TS<1tvK{Klk<~;eOu@k{%?ODd3cE6!R>4dkcXCY1o;dW0;i^oUbv$pZ$ z$mFv8FE6$=WC5nc0!v(f293P*=5y+0K4@(G8A|TIA=QH@4jaWx&OPoEcx}G*)Q+4q zz68PI6^`-py|Mhk@ynQLxy?7>v~-8}s-&R6QF_r8k}HW(+yhizR~EP1uKUK~&r*r> zu`rR)@?Rw){#N(cP3_VH=91E(Fr^{lX!7W!WW367*^0;a+>$))ohv7P!SX47U#3hM z@7b7{PEQdXpP`0{aHblLyLO=-fs6oBk9q2}8LO!{UIkyK%jtMFvE9SNJit#U1uy0~ z$wwUs${j`d)_#M;#2H$XSj)1l(!m;fx^(224X zTB%CJbG1$vOK(ru)dmMOId~0IfgS;`RM1(Ft6NT)fFP(V$l%~_L&iR(Tr_g?QOsfn zj*y?5{Xz}pvINSZzUICUp1Ut69C|nb5?gk2(*dd*OG%#3OcA-L^8}6>P}s%o~l?BY4|#a zDq`DxqOR4F*w(r|UfAbJE-fFNysU~Y2L<8t4Ry5R5wl> zK((-Y3b`B|_u3W;?dML{1C|t1dty-E%{c2C!(Z{}0Kh*EEY@HgYAnS*5f!OrScdw9WF~pV9V}oE93nFG_ z`+rUj0c!Zgq(KUolPO+Oh0+lu(oAz^+|Pa=&m4r)Xdwdmw9{*qmko}=_6IwFUNN@_ z2?>pABXWx%ApxXpGFQBH5eVP-lqd8HG1-D%6(@UNr8(peM!+A3VJz#7l*3)?W&QPb z6rZevYfiay()!9@KpJw@yU^sFPOuTmPTxey%eU?RkmpJ~P(`|4c_Ch|-T!_tK3|n- zESO%j!m+LDzDY**|vUOQB0)!H}Z($v-XC#^2{pv=qJJ zmY#Zvu9~3xqSsFgU;6M&@T}Jn`pXThoZ`#uXb6?Q*V8B3+b#uhM!yVEINrOD^i7NJ z>t1rJhf!SY*bjfmr}Ptx(i}G9RRy`Ed_@mm=P4n@XmU>Sel#xHejxeTa&v;;4h{VJ zJmGaufal~6gfHQ3s%v_RrO#x;@cPufpEH+OSp;afHnHy&MS#5(1PNVQs6C5RclJ|o zq-MuKN{7!h-3qBD;Lk2w1EX<^ewD=RE0f6ZVW%X|QZYVqj;?`Lci92~g0S_FWBJ7N zj_$R>I6bM>_|efDH9xnC>;-%>8BTr#h++X?XM1q{rbCVsCK-w3XW*!M9cR25e2Ra^ zT!*m_U;WxAO|#nCaIyAhPDkVP9hVJiYx-MaN*WSL!y?ox4t#OAx|NX+G^ntp<=CDKAIZPB# z7-ac-dV1dV-$LLZR5ZF>ASk%9n{>e7MH}hlfqzv}%TIE|#l#>6Y9kG#$U<|F>Yx~Y zxg;QqL7|}&G`Al{>n2jX?AJQMxBx#in?nRRq`!hho7Y1ld5-~b-RyPWvP@Z|H>W>F zMWSSbb>CY_NSN@Zas@2jelRv zosnTF#~pz7zBN5C@&i1BC>rNA>HOMFcRnPow&dSZop9f*~#0% zY9(0b^mr#$WQm)p+-UTi_ym-OlUa?Pn$0l!7N$R%mv`1{zs)P1aROpw{U5WF-{ns2 zUC&{jf+`9%`83%tzp%1rwy08?Xi4W)+*>IhPP3EyXyQi3dox@Q<}2?lecYqt3Zg*% zB_YVPoJ^+4h1Xn!`sd7XnLpk-1e2y99;?q{($0~r*=b)%AD+MMUp>P7hycI?ubDq* z9Ma83*Fv1GAidPm>zIxajYt+6k}A8QB^5AhF5(nyS6e%&nB`qevm?~T)?`709#m$u zZizb_aiju~A#W<+QrN^eooNrf1_I(N>ev{GyNBhQD~dt(r}3}8(6D$dGD7&+6L!bY zkbPK16Kj36J=FKUgM5Baq-N=T@nrCGO=^Q}6VEeXJm|oWwhT5$C3lw1){Su88JNUB zn&aKHZz7EoD)d5FY3^QCSG|Ce_{G(pUipDNYg%@uxOdyfi#P~-T=>Mt(VatASi3rj zkAFz20Tfe^y;vm4m+F|8ty~P=)vZ`=&CNzno2O4+svM>n;0UiAyhCa}K9%5zxP5y< zEBjTUSy{zC|H`S?W7md+{!-Lg>a`vTe@z52p7qvVEcBXqRB}HI?`?DHvLBA%H9gq5 z2uCIEBYGwW$HW9|ys^!{cW4aFyD_l&_uEV6OP~+;yMBM(`ejLq`K0s4O+g#48}EHY zlg2+_$NV%YGVYeQO_ABLq_~wboN_V6E$s3=?ZmPb+x}&(#y`{+P~4cHt5+9RIkva( z-*fC7j=VrD#wl_5oS8WyHk$%36(A)BVpiKRD1X(mebDr zz>mLPyH<23zN}{g4C($(&McI@-pk6C*&0tJ{+Nhv!z54>W{IlVNu;0nw6afO7omHp zjj4ZNwTTJ4QhYhLx?e{6P7Nit?%engeE6u9=JVA#w(;dXI7Ko?dtvNBkm-B-i4%rWJ&RXo;R>BmNK8EBy76@Rj2dfmoHf1eh~mIl ze+3?Anc#ZWQKXx6m*1=5$akgixO&D;frKKymIi8}YDw6!>DfYy`bNLH@|PT$b)BaU z)jT7PH*PGx$e~OBh}ybg^l-G5bJD3Xhi`eA>6^aOTz4=JN}FCxanYyVK2l=ybi5>u z^I2uI!7gyN#x;#jRZSqLH)!nHv!_8@oXzyBOkkX+v^DXGms+XL6xva zDDy2nueC)oNK?N)>z!7LK zM5FWzN;ThidX#7~E@3CXG=pENCZIwy>Oxr$!WKGp>b0vsUX1GxZ#!vu_0j>^@CRO9 zFb*Nx?DL+MRthC!GwXb0uI*T`do8OvDV`d3%((B1zBMVsH{17|(&ub}tV>|5LZl9C zJRG(yE5jw@fIOYpeZ3&i43M~O6yFX?u4cqD9_R^$=d4yDu!K-G1x;X@*u^?b9ta|g z&Gw8mREClAc?P9{xUUb(pFK5!Mc>Sx;ntn~xF;_$F=R*D7ZdnF&0Es{iNn z49uH{h@lBVJzPmKPtTtzDR!Iyp(=abBR#4~v(|651wxy{v_iSR#y_!EUnAW|!CeQy z1$f*4)aF(nfynmkDr}yTCBwNyRnSHd=arYOrHuFkccLdIJ%{|%$=qR^q7-y)ovG;fKduh&C;hnMcLDWXe2` ziW@RydhZoI{Cbi>a5=>Po_&0eB(M=8ql+^9XLZ_YNYa&dV5}meQS8+m`Q_o_43@I3 zY9W2XiNkeszxt_^wMgq}@vDJgMs$fh6eX_y3b6d6NcSM|Idr4F*A~zdfl16sVQkCS zDlUp=Cv$&Hl#!E80pqAi`YRjSb={@T%pzNIdhL$N+QjHp0ZZK6cdV-jc{`G_?@iMC zHgKw8o3-7r*#B#j{#()2k4RZBs}2(78WU6wJoqciG@w}ATx$7yV5Rd?38*lz=$!@g zJ{9L-q2xh!Ja}0EM&ZNjJTuL8NP@n;Q&^~O6O5E0F@nyu-z4!~0)_py$VO1Rd>`+i>u(nIi z+9qMHfsl8KYqq*b?L=>xgV(ua(pk&0K~?pnJy{zU$B7zQ79SvRjQ73~-+9O?@!0{#8`Wy*SZa*AjSn#ypkzgw^m%N*muC~s8@X=q+D7#>lI$T&7B=EeRt3}p6Q1V(=IVl3Yd*K z^d?1hsiO4tXhvDpB8?47A4)Ka1iJ3^+7_!@XcPWbVnOyYZ&{&SeNZBo-aMe4&{!4t zt^xEMQo5;J7?RFB^L|-H&xKvPqbJWS)eH}_7@Z{A!{9y7iCv*u+Eq`Qe}!(i-K64W zU$pZed+H_Q1ct<`C?{o)gz8F3+z@B#U;v)%lB?rHF)`XijvdPIv9(n({3=f54J!Ts zoaSJ^TXG~NA`@~fk=t0nagD4H3FvJ3yTJqoFy1t?Rm|*$!X7Y+;ct)109@%L`fBM> ziR*NFfc6gJKIf6xeGTp;9RZ_~1Ujdb*$80&H>qy{pM@OLMDuOW9WA2gw{GoFo&1-F z=$|rTH;*@}_A)D@OlkSW4+mV^$1kMQ)CE>HPkii#gVdF*^e&`!!n>nVJx+OZsl8GK zEhy_+G3zEh=!9bFfEK>)kxP?3BSbS>JKJJKVF&Z}RVgbzH^p!XikQi|9MJdOaD*7%6JviuDwyelNK z1i;ZaS?0J$8EYoFMH%0Jzlhow1|UqeEnNSi6=~W<#d~u*`_4)ItF|jX_Ou z*2e=xSXMP5HJjP-`&!9TS|JYZ`o610C3H4WsE-xt=(rxIO?=_|@htPw<$m^ci%ZVj z3fqi$#%z#sc20_g!~a4n(wY*?FYL$~508Mxkt0CoVk%iR&xX)n?`@$?qR6LLcLa{TWV}7%6B5;c5 z4%#QacF9Od-j2N~%Rh-m(DnxGN=WF}%h<8aObYIG$-5!6SvYmX1QANto^G-gZjy)> z|Nj1ErQ~z*FfntAs$&v6&sOKwCwF3H4CkwZjkw|6;H4!Qt_0O^3*G4{n1VIswDvpu zLb+~0@B^J>lj6%1@E{pOBs%rTZd9^Tqi1 zDzo?71dcXy9y4qGxDQ&Lm^ty^rW$G$Ipf68L(z{ZgfjY5=mBBVcb{nA;Lv1V%?vuI z8NNKQc>m+R6`xo;-~uzo*-*-I&-Bm)@Xk6)lP^;?%lJvNJE9fDBiCqjAv zW#<)e5%$^n*MdWlNCsPYLfJ1vVu|2sL7Y?qTuWdBglt$rhMIM6Q{hR#K%SUwyJK1= z?FZCG&qZ-ZDj5M#UvOeJl?rMEQ5+6r{Rx{chMXjaAh_g+W6nyU%Lm5Q(wIPoI!f9O zL{nAPv6gYQ;mtcr*A_JZH12o*+cZc-Y;>-)XwY6dY*ChyPRP1LcQkS&iO<}mt6s2p zcRqMJf^&*{-QB08sw>f+J(JLBg_HIQ4fRFElyzD&<1=T*)s7|GwM}-yn=1a#1 z&gQ!hd8rpsj^YcXOJ@SuLBmI+GfHDmx_gtdR#P}kf67F2o(mq=k?!F5e8R~9bqPBH z^LOWvt0^SPkY!6vgq420U^6_#mT)4JBG?(reSulBGmtZ^(*vqjFN=Fdj;bC21_lg= zg05?j)Pmc~{m{1dO+qt$XwZtQiyOHor3!fgOj8_Tfk-GIG={HL4kA)!OmJ)Br+(8i z9Tb5@c9ns*<}E_`y4z|a3jMiDUHEX(jb&pytf7a70b(7U`-{sple;jvg5-?aGe21B zNsoPW7b`wL=j|)iZdqrfN?GR^|2{*GwgU&wBlTP?&~8h=GEIcTvx@Z z$0O=Jq$19&-V#b~+W|`NAL`?3x0pe|s+0SAk8L=6jXq|QS!UZ>zsgH;t>MuDDgItD zThWW8k+|AiTl@4K&dyT!ZCfg(V(b*AzVm{C$S>+paoxaBC)bY?_dNSCP85u_Q}1GA zG&AGutf*n1n9QR=5{D_aLfyh!7M=BzKQRg99a#3Vvf!)_hz&qaKeGH!$eAh^yR%k1 zWM_(htHS6|iA1?BUAGPUOk})pZW(sm07*|@+p(0C{ll&D&Rt$F%=OB;5Nn2q=0{FU zTQxmjIM!~Mo<#x5)DVxUw#s*R0t><@tzESG+Hc# zU52K-R);QCAQ}qochM&oUUK|9TBi);^Jm7JPYP|$;DRV-jkKAR(yy15c;WFGJJ)jE znF|{Yf;aB>*49?0Pz5F5`2`MN-UR_Ug>38iUX-oB2?{=U_a#u;4qa25l5Dcq`Rdt!(QJO=kA^_g?XQ(H5GmpI2Me>=?)FoY)eS7|-RBWo3E3ln?|In3Kc+YGD?yR@=yTG$D)GvOL6B(BW z($K+Sw zg4vYV%yPz`OS?m*(7P#UBMfxqJZu7mi(uPiXv3(G5aWd5OQJ_C#NRl-Z@61DBEAxz z0%f8gxX0jZ-v{!Ev`pL?<6)5~E$S?w-jVWu|GsL9($f6Pb8b@zoXj@$k-vaKfKgLq zF_&78Hiz!kUA*F0gs;4rCAizhvQ=YZMT>9_DNmXkJ;zY7Lg{k3F-s2R**K|vbgyl! zO*og^Sp5dS9{3cI4YP3S2Ba=>FDgkCuW5g_m_l{@$9;B`S*!{KL6foDKh>%WSKQ^p z<(+c(1;ec@Ta-Z@zgXe9MZ1c$qT|W996r@#3Dl3*6;Z5RigOy?_~9$N+y}of%qKPI z(LF|#14454+%lgpqyKv27_#BhSm{R0@aV4MjEsb4b?ZeXZJeV@Ck{JPV0gOO3Z)DFUNNka4O%aN zdIl?Ue=U1!^j%-4|5;n{Tb{8Elz6f2xl)&^K;!eq>6I|GFo$bt5o8yie4d(*W~ooO zv6XJnfAU|}1qrC3z>}4Xk~lLI^_*lVm1Mn?HqHT{(M*M#7!0M{Ll5)XwH{`t{9s*wk;kPDTFwkaHeaa^8+uRN@BV+}FoO!2HKIFVejIWltT`P=OR$^PT0A@!Dfg@zWK zU=WDAUs2V}HSKG_`f~has5Y2OKt!5~@o}XR+k1h0QHDZ`?QLexqnR){Df|AWeWgF-qc)Nw4kuXZ0r>U;C>)`a ztC*7iqS@m6{{;TP`ao_1G6DyYZgY4ec{u86U$KYBR^#j^pF~ zv7K#;amT9JHHkegCf&)Vd#UBwB=eP;S&ysmO>LERq2bk%0kiwoSE%|d#~qLfpMKD& z0QEF+@TiI75CHqVmaeVykH2zz#!g7kzueUR0Jb4rRQ52NS@xWlbU%`~ZI?`<(YC}M zQk6Ze5pvhpj&P<02+@TAsoEY4gjD=iyTEl(+n_`sIfHYC4sS`vuDM)A&%{a7Y$wzl zP{b`PSf%R&skSFzqrEcLR#JjAqqn9d5;#IcTuly-WRyIZac3+jX|N>LY|BvCPSp62 zhxH6ZKlUpHF;E=i8EHztyn*fvO&qnJK6cLZIh= zR>3GX-LrlrSto^BLJOu09e2&}FUAp8jm-$)dM{4|H8ECy=qkQ>lRE>Pt4UksDt8_q zx=*b~L0~Dy9b-9deaxd`#m3?}Qz~3tfgR+2tH6f3wnV7EB*~1v)J~@h6TA+-8i4L4 zDL8`>@l|KyeP^B9LmBQJ#koz2oTtc zEtx-#1kI)t#@6Xp?pRVT>R9V&0DWR}>QZ7f|(&7s#y~mV){>n2qhaWn_UObaP zCl_ZFhaI7^Ss^n=7{W&B?JCAuepR7zuf8LBdZSH@>>gEt9z>oj5Z*#sQ3LIK_XB77 zaii4`v_2CN-*NwN5Y~wl9GU_%ojPYhuS?SWU`fmLSVW|4>hzNvk2ES9+jQ>%C1)4L zt0l1mgbbnZfzPY#6#2f$jDKR9OoOZq$N{D@)U^rC#KF?@ABSReJeBN z!!5$;BBy{l5VZ_ftq%YgY*NcT{vfz={=VAlwxz!NQ|$1PD8t8j^mL%J8A(%tvr9O< zA;JDw8e!}}kG+kJjX_A<`xIaX{EQzT*~9scsoEBCxP@5I$7@(3Y=bi2GyZ&N@&|HM z5a*KiSAV{tj7^ZWoo;Hc8?!eP8v?I5F$zXF&z(KMeh@fWBV8wWCU0z^b3P!ocnoFFce*KFu5<27k2?vbqHKd2_EE7x_{83 zyKLRL(75W|G%JIu)8#rKA5vAPy%AU3HpiHqY=VAFkR^e>LR2XzJs*6~+14j$nkX&j zM}x=UErJ&)NdU9_G!!(CKksZF{eV%r>VEGy)$!mfZ`j5N82XH~%_Y7iFJbK80Tooj zPUy7f{g~dn5Y#y{)_12Px)ijK(YiG_1Q3;b9!ZH%Xpk~7c|{j*1y;o&RhA2`vj;_E z2CuzJXQ0LOxXv-G<8L73~-%fEZ{D$mWN+Z4zrv1 z{)ZDW!+JnYm)e9us}AIjcY~iq?83003&q@NeFI-9;=B(hWL`uq8mAN|+Lw0n;n)E8 zCSq+`?Sgzx^@;*90r9>>GAC!~Ote6TU`hwoqmr#%U$KOy<>E{%PX&-;AuaRaFKK0Q8@jnu1s- z6pZ5oLgo$>&h0sNk2)=IWW@Rms*HT3c;gWy7}4@@26&9`W*i%McLS%fhZl`?XmngT z#*a&_2YN>(D{Y%pCYZ7Mwvg0!KlCn z_Yq5Q+61 z3B>mOiOm&^4+h|kr~ZSz^4ZoA7Nopat~jw^06KRT|1G)i4$S$QI#LA<>6X(gKJq#p zX%o}2odvi%_CYoWWvIL!PWCRW0S5T&_(+V=PH@AwpNC>V13irnZ8i|kbv9RfZoL7g8qPSV<)i5- z*xFPO#OVhVlGG#pEt$GyHk#Ydo#YxU#W9RCn(ucH(2hS(G4X0D6SZ7+ZPBQP`3_Bi zI0VOeHN7--uH2jomzKB47cqxq+wR6wkF&UY3KABd-LTE5?Igw@g$QIRhdtt$?%ua# zez5YVyJmxV;gm8~djE(aLyTvN3F56F$NI7h7Y*IF&9o2j%(KhpPpa7}L^((EOHoQ3 z@|HV+=is^aV*CVl%$2)JshsnzkW`eOUqXo3i8y~CsmUd;6;sEDD3|}548^<>YCMO& z=mlg!E6s;6gHZ;;>64YHhr`d?ln;08vah`KY1?Hjcdt0WQ8)~d7HA{>gSyYPPV;&y z;YEYO;);ZG-1Wk+Nd<7@%E!w7`0w2^+lk$xDgDbzIK|WoX=LsbLTrzUXP?K3@UHwE zl(ai3C!_TImkcNw(;_1=r*A#K#e`5ErxW?EU%z%C6Q$^7Y6!s(Y-y=5xb%wXxyOw~ zbq9}kJf7zX#B2C8{^d+8e;ulxtzDa}z&Cu8Y=I+OTseObQqQ8&h%A~?eY3Nm^YsK_ z3>B~#_cNciTx@h7^QBFcB?lkJ1t?=dVkGiO7-OKx8;%06d3&DYN6^nl4&D-a_X3nL zi5COVS@A2E%tL0@7kytJy?~Mx(AC#}+P3ZpoL@m&b*(x2KSxBNjSF}t-WGbLSa#BB zk>`$GHA5_BmqPO0{dt22%=X(wz#olJNSr%^dWR5i0b>T4dN+Cfp_Cfp$87AuGzQ>P zACp&yfp3PM58x%<%4bD`jtlBCbLU(D}+&Lk>EihDBSwju>7q2ksYVW-=7WzziDTJD|uin0%wWOv=b4 zW2j#*Fi;Zgg;etV#2uWxx>PiG`=cPo>i>T6%v6LD9<8LT2g&DXB2axm;fluqWIE;Sp6{Yw1LvH3;S4?p2`T&3JaPDvAnfkwQ0e!2XR35B}5On4ui?yD@{+Q+3d7Gtez17J^Z zA_)uox*C1h{RW_x1Sw4EO^j6?3BrN;*KobixO3Z*u^yM23BY!f9N-n=$Fnn0XmxGk zkNL}{RVALkM-wg2R0qZukk#jf2+fcWoUqClZH*@%Isvpx4L_x~w?iPl1E|bU)xhbs z_@Z1v7Gx+Py$AURBi$-HQAJb0PN?%SAH1-!PwcLPy8>C0L3s-$6 zKl^M79MDs|BXNao#5x1CLx%ZNb%8j65SE;_{d%}fKLgetG#S)Rxm;U3cW+Bnqa$#* z7C=mS`=MvJB0@F!!0^6$%UFH_#`}^z4J6C^!B2J`bR>fhnJCw>2OJX;uhgwD_6mlA zrg~YXPgtNL*EPc2=ncKnWeLY92*Pb+adq?p(-+~8r7f@AguI?MP zwi>&Gf~ZIlp1z!t-hJ0bU)KgV{f0qi5S zklbu_L;YQkre&&m*$v{S-D(f#i?#%A%{Ms7B8I;^ZX+YZxKj{e(625$^ZjLMmI;fl z*7O}>Q!y+GTA0xSm0|$p-)c)nJObGzcIahcX2aaWpRgKR2QT3PjvuKd2;owNUU2)E z=HJ!B@yOAY&YUsR44wT@TR${Kp3w6uv#aPv8$E^ipmQhCK^H(Y5|na|KZo)T)}+^L zn^p?AYtiLe=RGiHVcIJ>xQHBAQwP+F(gS>itAAk(_uvZpL=aa>EDst}nzSjlX#}sM z;*Cg;y`^~}*TL!8DP2HX4UdmP(FhvY`O~y+`8dq_R0raEJ>dEOLC?Pwpj`O*|GEl0-oUY@4K+NnZ`n8c=Zws!w^sl2T=--U zFKy$cp-qC-j${AlaU4WIs{mOzb>d@<+mX;1Y;Rjx5dlfB3U{C%sKjD$G*EHOQ-h|! z=QQ_Sa3U$Gl1{uCCo0g8bqsv2lY{ND2!$@dM+p5GeXy+ePcun7L1Ve`gG1u^j)&82mj7!$j7vn6|q)5PG*WjbRbTgtM`8%|vVip+p;~Lzx&4ca9bHvT2j?bk2v|O(X;=K>D5&u#qBNuTL_3$S zGs;)q5(0{dG|u{#&sZLhGB%YGw1DqIj|NT}r8 zMo@I#o$E=$?jQA=R}|P{Kyn*83%U#+e>~x;(qxLR+~YO?Wql_?kdsCI|`?D|hI2*bx9@VZeeRp{s+$GmyAK zAq0A?W$UGv94;2dndvx7-u=%8kq`GS(-3#Y-{bci`pnm>c@`^tj@0-KX$^p}kr1L8 zMcwt4_je?naAiQ2iqq=|N-p2+01IoVx742QL^90g>S@?@$g^wiQbMywhZT9M_bVjAX$O+-xGc&rp6E3o^Q$5 zp({5Kyq)Hj*7)2rw#&%mpt+XS5|jB19_N;hWDdY*U2H_NVn7lIv8yKFHXRUJl$@CO z0*iE<=hPsIkiLIIAcSay^7TL?uMDzV_)3}kdZYAxv~7qcP^W}*>~qITh=Q7QGh%}` z^xiqaJ2V^&UqUC1V6gBIjLib!z)brgCUzcEqQgX4IzXa66=INQ{Po~$PP3a-^X>NH zs*_+AHmYZ0n*$POzqdsq1q%fIE|+8X*?+Lv8MCjY8R55C;j~YIz0p{deG7F1x~a2v zRm>)%zUw}BLaQL;1n$T9_0(mftYanf^qavRf~E z7l;FqudOU!e%$Z@2pd~k#uvVl8CI7#NjKptuk_OnQHUh#*z|~4YQ9$53GRT{Nbl+9x4@x}e7b&N6hA&INn8;QwK1GxTc{#l)Z;pa){) zubE25dYZN=xaz)@W^>>HLEoFQ$qvi=%U9aP@NXL$$1^9?IaOLwG|FCH^;3G$!J+;k zS`W*&dNwV$YhILOQjTUM3&Fk+lvrrql%3f7ZL@6B9ZtE6B_bmn(j8yd%zL3sT*-h7Wf3K;A=tx+?mw&g;JEzVf3bbI0QC z(8Dn95)iUjNPh`}Em)rk??wRs529|A)WQjUb4C!`{(S)}aZ#@xE{=mn7LY95%~4|= zL?;Z^`n1v2$E>YugJI;l%YoHkxt79v>BDUR>;F=`dVJcUu%r@#f#aCpLQbjVS+v(e zYd6_RBjl$R%vDB#YT4f2ZG2Se?BpI0*qz$6_Z%*@!dy=?i{q!(ARo#zLn%&`bQC=vd>)bEJm zYWF1+ONbwMxr10BA-PK93U%ehr*_I!YzFcmSQZ=^~t-B|7m4&b1pv=^d z%aaO?NR%w^_d6pT(N@P#bGGA-HBG?8AX(V3fPmKc>M9KOxsUKiNCcDG z9s1E%M^f94EhW6+egqx86xJ%YB#%( z{{$GB5P=^Zd7ECc{g<1&}TSuFJc6e1~j4UF4b_FDxw;!rzJqATqk_J!+F4(}n2 zzCiY8b~$VZE@1iZk9$aI@rgnL2+#y41uuavMM;RB=6)>pTcNRAVeaU~Y(+Pf@MGH? zgjro4n7DiSoCo5E=FtT}lVJ4xq+Vd*^w8$H1Qnh!*H%=~su8vvz#xoh`5RIrZw9eQ zOj9A+Fc+CM#d6HkdYirq1lfgz-iK+vA5o(=sAiq1oGrxi z8hqq}g#@8YN2(Pp>AZFB0LT+yq`La9q1)W%ez&a9pNBIDM|x3guNxC);Il5LCmO=JZhiNA|?QP1IbidOac%HP6(2Qi%Qi0&*205FysVfxq!*A;N-@EE(rQjqR)OC z7ORD`=Wz^!e&#~@iU#0KQqjhnD5WNvAZ=50JJ4;JTQ>1Een1$|Nf8En>!A*T;8g^_ z&Sf}e`-gl%G4?;#FADmr!X5-)_eauj*k%-?^gHs0K%4@SF`U15aKuTdKC;Wtkrz(P z+Z6cCX7cl2IY2!c?BX3>I)|P}A{}e=LK9_WS;|(t`pY>GV8OPfI8!hVWJy6-1Kl{Z z2>mhp{1EP7BQ*E?@KN(JQ4Xce67Oum%o*Bzk-Kem0PUgrHg83pL{%ZNkKi2d2H$(W z=BD8u7926LmKBQ82pja1S;&ifAI@Q4lx5%-Jl3|x>sexH7Xm$`SB1EiVTCU;XWb7Vu_Lz!wc-0!h#LkB#SALKt=$~08L(SE1|drZ)J4^MdWjb zqI3WMIkk*z3#Fg2xVp8N`)#(MSonZwIvXLERw|J1$bzY=(FZk);?6ZRUzDY6ayYo8Jw|_sM?6S}b3O zARJ0m5Is+IEzM^)yq!%UP{QA9gIr<+#sO&&R&2u3b`(MX=K@DRM;ILk3YQp%4?!0` zUUUAtrYq}sl>1U#wFy?g0Wcx^ni8Ml@!3n5rlB#wcj8dhL6OhPs14C%?!Dl9ly%6_W85!l zG8|9dlG=~u52B=Hgt3x&EvClOgZ zl6UEf&GtxyLub^NqBn-=AIgTYQ3JB93aR~ieDNS=7M{WoQR_}_d1p^hCG+1cxKwi< zn+vLYf2Oy=kAA9#9#HnH(T*wf z)jIpxX{lp~%_#0Uh1AjHPP2K;)l3Amc{i&}9S*E;B~kS?%!$n1DDyF0?8pzIX~fKdgsj0u{^W@XJ@WDE@_+ zy6|{vm~s{ePr(?j^WoO9Bi88PS+)Thi7Mel80jQDAKs|TdGPC*UUe6ln5EI1Dr^AB^OE_Pvr z`xZkKL$O64NGWpueVZ z*lJ%GHx8mkg1-5@^#k}3@PCYu0mqJg2g7hs*Xw?O7LgBtUyQBlhtGiKd}aj8sxic! z(*+nz?-s}+XXY?1unRg_#}oS3W61Vh0Bxd@+EOwbX;av2L)eoO6Ker|`|4E)VN7WR zpOIs^ZV4u8qx{vYJS9@qeUgi(B!140h%3&(D7V+DQ5c~|MdT50h-y6E!hkd_DPXWd zNz)gjq&@2?@%Hm$LOV6pEAaoZ_2uzUuJ8Y&Jt-+#MQO7XjuvDYg*HWGN!AuE_9bK; z6>(b75h}}QRm3RSmm!r9Dlr%gStboLSqEcge%JksI_L9!{q8?Hee^N&+|P4A*YdvJ z*ZW#F<14q!TsAIEF>GKCN_9k`+*vtO;g*AQxfTwhy-P_qgl>2 zJWvXr$EM+*tPhydj+c%?7`>YjTE2M9_`!#zq3|9Rq7m7k+9CFB9bg)?e=SK!+E468 zp|UbGQed@e-T9?|Ue_fz>0UUTp5DJn1QW%$ohK0eTWW|tR-Cfn=`K4s?h!RxfnqYe z0MB|AjJ0?Kygsq5wctF;9kEgsPV>6>F&OnzkBY7{XBSn5{#cD_rvLB5^gqt)ZI|qg z1KNs!lg4&V+T>W$XCK>=-q@!$4fk6c9V`1N6Py=%QiSOVfgL;jYH@chjr*=R?rj{5 zGO&%phblTD+5E!cp;2$B7y7sTz<;eDxh(Y5yCtwKdZ^c@X@Qu7H8FkDiO$~??RS6Q12>mz zHT@$>uz6W%WwCG4UJVXC;&S%nY!%#L*R|V@cA13y0Qnb7y-(7UbVWg6UF{<9*0LSb zi2-qF0AQ?bg+Iva0F3Yr87QE#BV}In`WBp@ibk0>FmjxeT@4XKYUZ`koHt*4o|-G= znh%CHv$U7E9|tR03C#d0CkKe=rd(cN>pKf(!Xt)eeo>7blxRC=m-mG8CaIn0pfl`# zAFeY_(1wURe5!QeN;_9zM!*eW!-m$gMJyT_-t)C*;UZfx12(DqPEb|&Z>G`tSt5-6 z*KElc>;U*|D4OxZSl+R&Y(>6?fD5p&(y63Fk`m|m$E^W+R`)!&v3e*{D1>u}jEC=4 zMpd#9tLE#9$5SR=pY5u_PUtc*BUP_PWFs|Y*6==yq?mr~uDn?}>5((4_REX8Xvy?m zq~k5~J-+Q4Ts%Pp${tXOyN!}*2o$032d2TI4};ZNXOe<|#G>|gc)I~`+8iyeUl5rNdN2jw7*q4i za^WC5RnDEYK6?fpm~lvuTR*>wv~Og;=b@XwUM9Iu>PJP#yrcm5wCyP($9Bi3AK-n$*ono<%o<)0SQtlaKa`y6z)YtG&(ls`v=n=_J~fTM%Wz7vn7#K z>8eP`<5i)*?Pc~kBqBD5muC1(r(lc%i1e$UAi4pnTR@cq=ET03T`h15kYGn32@m<- zP7rt37Q^U^qHWT9gSud!$;bu45}-EW0C!YhSesQ8_-di39dG#Kl^QKTeq>*~Au#Du zb3VGRRg5%zMkJ0Opcer42^Y>qX^RZhQuPD8g2#^{gUX1aDoqYW()>v#AJr|ll@jQb zMus$hBapbzP>=M~N-1+Drnh$j{t;rS^sj>wVo)rDtn|p&DR1y@H$jl^aLz3bl*!Or z(dY0%S`u(;vnu=4qm(0es-JvOgp@7FD z8{?AC9HxQ{cy>gM*{*9<6z+zUX=pc-8WW-msc1x3uI!Y`Wn}AGx}np+$->$c?_qi; zstcX)Ff)nS_`9%b+93GAd1lTBP$FVz~TKF+;mrg4qwW;6Uvi?c|yHNEK+|) z7SR+}Ngw1*yc(Upy87_e!TE4Xk&*ehHS8ZiM1idYID>R9ajL71Hi~-^O+7p8&$&Y( za{{v`dn%!8p#Z`?QZk8x&wR(4{MsuCcvN0|-re#Z!1~2sqC^%#FBn}4ge|aa@EQsd z+EK+nKE=b^b!MNq@fYwmF~)4l6qEL3!7t_kUr$rK>w)747HZr@Zoi6Q2(LL{VxEV~ zdoW4A{PG?Krnh!TZ~FD*?EKcPP&B<4ARn?ZTI#%Zt=58QD#(J9pNpA;69_r*292J0 zwP-U?4QPHqGq-07(N+a|{AIr}*4DH_KC)4l0(wbEfKBLLH)ioQR^lJZNg= zZl&|Sr+KA>(NQgadi@~{V&t;H4b)98^F97Ey7YYhvDENYM&d6YD z9c%h%OM>`r+l=6;_R0f~<0<4(VrKG15Z)v+IMDF}qqM<xxmsBPn4 zr)UC!G*^oYBWoKb{U7`po)8iqzht5n8HOgsv+;Fsq|g4Hbiqv~H7D{(l&7A{zukP2X_iT)UlM-t`vf_rwcT4C-}C%J9yBlCT6qB4W$ zr8P`w!*6Bo?mrn1mh)qrw={fLo38LC7=yTQ%W=OXkS`hRw?OEa+B~Fdjd9kz^(K>z zp7?=xytIRr_pe*e4-~JqejiH+m!I^gd0hRuI7IRx6`K;R zcTqpLP?xfHhg{bBWaR`CN&WQ#^Cl~vwK{vPd0Z>4IaIa>Ebj`B$w2@+d!?T35wxQj z-<%P@4rmyE3*j&hH(-arCz;bI1Y~b>w-A&;pm9fuDvGjL&8t{XK?HP~ZUcTyKwQ8R z-v_m&8r0-q39&C&1!g&Rhv9S@3O7ULRy{;PU?n9Ly0F_X_4NATEWFWttKj;~oeA#8+n|Pz#y{#(~@S$lzphTZ4c4t=2K;^hU8Ro)M7_g74r5(!dqBx8{765f^5NO-{O7<)!Lx)DH;8S$%h_( z_|CFH35>%CM6sxX^H(@nfd~m_1>QmF3@>!7cewXc@_d>vcpnK}nxJM`v-B;l>#MCm{P zltC3L>mK2xI--sj#4UFi8G`eoZ_Rs61EF`2v0DD7<^mAsBBHd>h}DRR96&?7>=sIV zyN@Ea8W@o6VBzGJ-1*df+~EPgs>0P*HMZ|X7(5@_QI!A6yxWTIaLQW0L zPf%L9fw-LbS_C_2%v9bk`L6+?_^J4|jU^302yPi4g{(vd%m;pgEZZ*d`u7%Kgwx&m} zA@a5FO`8kL5?~D59Yx2*lidhVI@r<3?ARp>PUcV-;?%UwtVr+BAZ z_j4j2s?9$5FxJT|^|guVHSYcU2?m+?b06}q(EB^jDXvyZSj^d>!vUt(3OFmxr+~`u zq4#cRXh{CM$_Zmi_sluE@--u&9obzp^n%3%R&GQY< z_byt9@0FTkU-^`NT+APRX7h%I{x`>YbTG1ke~a%g6^WC8yzN}J-wH8f5{;iZ{g`IM zNyi6voAF2yHD#kPfi5n-(n5(oLdj^O_APzF;RDGX+9J=5bOHAjDqLO{lR#r1ghWK< z)q6&b8UrK%Ep$*o^@K3p7z-aGfp!=pd>Bnpex&xnaH?~^tn+47yo-SB#P z#Q|&@fAVJ{m)8)JPKZi4L>+-9+9*V&uE@r=$oX2RlrfTi?$u>4G%8JtbgpHl=#}zM zqCNCSQ-NoN3(RPgZJM`cNjOpO<5eOwKzb8A7d}(_ zTtQ%fxN*{Fi^oG;>`=5wZ%mUu%xj{?UoDdp;WFC^e0uF5k1D-Uadx@>(1jJo?Z?_& z4&|}@W!2|CYw$se#815K$uxYGR!!v-+G7a1>`>(rTgQTG&%Lixf`q^jo2(-EX(xNZ z*7LdP8!E;wqk>Qq3jrnUoxUl>c4<-(nY*A+ruAsNp})U>dvZ#td5*hh0a*~M50HC3 zTRv)4n8ACSqFaETKO%Ye1FZ-_4Gc`Nml<>R0UTBbLOn#YT5^`tSj^nYUrAUv&VyL!42e!ugJ`)`OJI+ zl?g8+??rhG?D#j;Ko(xDRh!y~=5|=>?R1@7nMdXM;d;Jcr9HJOtp#PfniAl{e`x@> z(Rh+b?n&G@_ZnP?w3l0qyaDw^OW?!{{4iNY@tO-dF%Tl2Y$OFIwV5|>pL+)aB}+ZM z$6wTxlz_RVZ~UpcfL}1JKq&QDh{F+gNkvH?(36yw2I$H>ogM2hodpfFw<6T=e0JO8 z-#bG89k}wlkl`L-a?mX&%?E!qDxqGLZjhHe$9`>SjPLPm%W*)%n+;&_#DL{TcW&O% z3;DU*>@Ku5E$Hb$~_4d+vKloG4sw;1*l7$twLe?R&$!%7h^iV}k%~_+4 z^m(}Tsr$P*5RAFzaez3r6M)etc9x1XDa>R6cyPTU2I>hjXA12pq+-h@pikr-+)yIk zi4HQ(r?RANc9wKgP3`<8O?Y465Hc^8mJ808gIYnFV*~j5$Otwdr-X2eH|bqYxL}uD z2y_a@P>KEk*S`<4OFg`vOH!|>(EdwI@1XS^$?k}RB$%Ca+tl;us0nDnqk+v`!TeJlvfV=VAzaQ1mnFN8f$+!H?ng-Dk84aAI4B{H@orJrR}NgqO!W^RY3K^7sl2tn zo~S>eHG`_#|64lBVjx6di9rcd6LP)4lK1=EX5l7+P*A}+s-_;_ix4ZR@5~G%5W!8o znwO`AQjZy{XxGd(K$DBdMf4NSc&k{+|iYV5GC4?E?*D#MoW%S-JXZ_cKml z&P7Jb5L(->3&kaf>bbp$O1X4SZal8$HLg{-2S=}J% z4wv9yv&L$B`@wiL+yv(5lx=rfxRn+ACOAO}nvr71to{7N4r@=oqp=7rjV}qMtpG z$Lf8CLOr0%-WJHM>KwB3qFtwvZAgSMF2j!S4zF)Lk7aTKwCT2~(rBS?=Ee!(CbY1( z^WCLAHu~0?MqUPtsj8k{81(DP9>N0Be|K+clx>+jA7o;2DX05JUdNXi4*4$!jC85t zmh`a-U>yAYTx%Oa$PbO^Q=PaX*T(jOi0oZzMUaFs=lJSdinFV6xR$$J-QEFU@Hc+1 ztnKwL0MtQjD1X{3io`D&yyYu)F4D73{Dk4DUX>JU14fnjNyH3n_Je7UJHOw2n`;$2 z>wJc}lZ2QDK#$_DG`iZ1!tN-MgGcby_$i80{QT{6i}5-5-3g+6DUa1Ah8$LDSPtxR z_t+E{B%OgW13k%WwCc6KlAt@j{9I|JD%bemE&lo2JMd2c2!v;YvAKEEo@SwNEtoM( zx1Ph3mk;C0*v!H?Q?V_dz-z;boN)gNHdBl-%Dq8K%`_;d-=q%xKx^3N^($V}*!cDr zf)Qfo@D(984Jec}ZeQ;qycC?D1vUO4DH~b7PLLXBqHqx8&@HSNgy=-cD%wEl8al z_7iIUO9C(T+CeTPkN{j~#cOY+>kx`H#YlC_Np{g*72u%}b~pVYM(Yl7gA^tl{GpcI zJ!nD=E|?n2eH!g0JZdYtDK$ie8q16qH`U%VrNobLC&Rm4n8r)q0k0b(>ik(DT&yp4 zMlMJgH(u*#HomI)&>yk|^8sYmuS-|#$2tp0IyDXg4qdQ+L-F4(&$Av_?ClJ2wMa4GKsNB`a;!_WZsbn` zB??>i`0Xo-JW9dsq(RLW9Lw)^O8D0hgPPt`zhB=+4})EW?#Y{gl-jMG>6_2Rk%RT&gwjK z(Bp-C`2akl|}Z`I%4x&p^3>+BOs{00(=)R#vXR4Ln4k z8E(y9-sE6ECWAW>`-^)1iTl~Xa)FMFtEIo>vq}eHqhNMt`L+3C(%MifYmKcrkPv-d z*wohAmNU}NvJ-t~7aDiv=kopSkwehuXsvA*EK5`h`-fcne$NOb{X-rDG>`MPcgDPG z=G%Z(?i_j6V38CN@%Zs$kY!(WI~iZ2)!=e4Z{u8#_?o-}v|%q3oxDazuBUgvD`F;I zs*MKW%?khuWT4V2Z3LVNv#AmX6d?S|mlHfW;0WfO?B6l)`kYq-$Pn9;7jCUs_dK@z z<4OAH=o^!*o}r{;^VGY$fMjJo?-m6GuywIZJ*R?&6KTrL_j$ZA`>AlJQ++p6h9g72 zol4hK9H)-ib4%w2@34xGBHrva+=Dfpw%r$>1?ALY(2l|M)?k z-UztRQs)h5L?yYh!JTvI912v7WjI2{#3lQg7Vs zw)3;Tl!!d0tC8>mX=!Obpj+4-KBA6Aqk59-)^QKKOVzEu$aymO;gkWVb_m`+saNEO z9#DH}xDY6qKOKxHuW|4c#lkrOYyWp^ZEcRd({;GE(%;2=N*ZPCD-?VFb{~GIaIGo_ zWT>3Cqy!$%`JZ=a`lRq8QqAXI`b!H$HjpZdxclnd-p1XZT|ZDDZa9d7PV4I*8@jj6 ztug|q#MpRhxZe61o)~O6H7R*-*l+SQez-W!Y>#0kp;_|4naaon#%N%yC3Z_Oo>xel z$tqC%ino(doNXPh80L0-DZBg{zng$`5%65^5iU?RZkg7j=ZGpY++~OfnxzD?56Wy2 z=^daU_P_S8eUm-1BxN=yGOB|YXgqEC2C=`|Cc;N8oP_bbPmbL{;kb38a?A7T_!qGc zds>YW2It2fo41T27p-Xpm~h0sUa0}sr__STqwcvr&@U%utbr69A};I#0EsF0;^nB? z{kvDbiSIA*DKPBni5X@kS##6}6K;E~R#766MEaUasiF6~&HUOd5j18kS=q8&?gBq= zcru_rY+Sw|AHOCc#XG}0I2e?OfD@$nao8^xAxHD&!fc>0T{DdMF|C^$`qK@WtDl!b zZw&wdEzagqpd7^t2$I0r&)>C%hIQOR`~{3mC1Ovq+;`7*bs`yRFaYWSn)~WMProyF zLG!@w4d_6mIQESkjfdRkd1*kw{FKuj&a-Y|51bfq?`Qhc&QW6ZdW9a->ypQ1Co8J9 z{J1Q3MEe*+Wj(j+3oPSzQ#PxUdzGK2TrQA)N$8l9OQC69{?%FD8$7%rh>wC-8|Hz6 z*}45w6j5MdZTen;-i_`EZ@<;O#RJcWc}30qIWY=@--k{}xm9pcRy+N{{?`D>4Fk zSlv8}Y3QE_1!~AcfGpSd?{k(H^F&e2@6{ZM94Q5fo|I);Wh7Rv4xwgSOZh~dr=SOg z8t}0!L_Ox@<5suPCM?_0%Mg(paq8!#uZu75xj`ci6ks;<`tjq(Vxx){bUpk0@^f#n zoNm|3Q$54MykAH}FLTcOhTzZ1z`$)6{gzonzarZ2vq+Y8_hx5F7RT^#6xq9xMfR*4 z*bgoi5^ZoFMX!-dgE9=S7Y1Q{`7sFqUirfZM{KqK&KJ2i8<&-^<~FN-Bk({gl@Og9 z!k6V9lL700Z%y!D-xsop6d&mY*0bCbBrSZbN;27ipdAd2@#^0NeW=V0+ANur;*BF6 zlj9&Oh~C~68eaIb>x_jW`_WHp0c1AzZ83w08L;Z~aw(byG+dourqFB8S)mUhm7$^MK zUx0-~qZIysR!GH;f3r}f4KOCSXoavgOcNBC&8yL8WB0w1HgP}xi@jfQRyE%FYSD%_ zR!4h-gJ1FOL^mypeZA;w@bm<%6Uu&e8T?#z?eOjq(l_kJrovZ+P+y-cA)&b?Mz$c8 zF|R%uv<(}@-#@n0dxtc2$dD<1W3WqH%gdn{0&Hq919?t6OJP|1Omc6A1t^W&AQX?o z>SSk_^R$4D0WBnuQ^P9T{gj=$Vo}h2nVJF!!{VeL6}|w5fxXX1JPfw3qWHX$s@ZM)7Z|)0~@Xa`qQQe$uii zSeNag32%x$=z+OKqB*o3N*3hI2jcsomInMMI7Rrod`5Ug1b@no&oG85Bn|56!-Hy< zV}k%qMB024{=el=KWSB?&>ncT6%QS@V0QwSWe`{6 z#IJ*Js}xj)(9eE7HTkg&=ekeX5C4tfK$?XMxwby>lj;jCMEH+rtvI@rrMvo4yDGd~ z7{hsS_+e$gqB(bW7+h(A!iV2-!|OJ=JS3AdzV%+9*g-|gfgF^q9e9V=9@12w`Xst- zc1=7Xc=^f9rzCL99B$#NRmOQ4fg4M}hV9(LG7_ zM%k}5uQZMdf{4JPMRMchqEh(R|E$jRT?uu|$)cf(Livlah8%kG)dbW?g<`?BBdb3Z zvE4^{OXu938}njTc~9}2yLm557H+be15>&0K~IL#U-%|qRfuA+5!>_fNf>Go!iwa zi|N1EpQMii%4tD{7X9fTM5~Mrz88pPYrEN$BpRmTWBG)w_zMsEfOdjB)c_P-po*yq z72i}qMGox>K5D@D1Y>2vfm4GLe^;EbyA)!z$#em(XUog^xJ_6OK_A?nVJtx|h!4Px z>j}&9FQNH_#!drGiz+X61Yob>K4ta$X5A~fC(}jTr6O*AerE5v`ALYIuw{4(YcN08 z20z^fWyh;k0o`TEd4$KB!RA^&IC64#16eIwR@o^iDP(KQQ(SW91Dh&;Hltoz_0)uO zWfbxSJsk@?JP;DEld~OA5E+@sDE=rr;^(|j4mTC*WHFx3h*U;TGwk)ii0=0O`-sE%PPwhPm*3Es} za)8n2I!KXI3uK}8X41=>@}-eQ?m=A3S7UAZOSR|T6*YU$g!djdkt1%XrLD~im|1EO&peox0dbxuD_ z)Ska$Q)UT9fzY+NF64y4c8|uVrl+ShL))|+24-)zx^uNyP-amMG!&LSz$u%3&yL}* z+%mF=^IYT#iG_&i%v0K|(^{hj1yX4r{Rb0l>AwMspNO^1qvcLZut-_Zz|cF^=MD{K@`2yG-|!0K-@d)5^~Q~D;TPbTgrK&; z3=RDGb`lPfGAu{_JqgP?!VEc=)dIx2{a~iBim&l-H^L9)*pF5p}p{eYp zR#^(=J;S_%43aG6XaFf^5?ArwKTCeew)J*Slj~7HU>3daIR<9{*_iCwn=B{u$d*jL zkuNd^CP%o={3w|}`lNa!D4lV|&efu|zOs8|)W$S2A9TYYfQ0~q0q32#QISF{fR|7B zH!az;!7)c&2|m9Ku`FOwzd1{NoFoZ_&`xYy!qDV%(!P9#e^=9cTh()CRuww!e3c6p zoXR3!R)>CD5rXsWg8jtaDG+QIylKBt9csuPBsq1SMDqhHm2mUfq zo_qpzJo!=0S$Qu+Xy#gpwgG}07!TI$yhj0V&rUf>o7k~;xfN9f#PFLpoPFVvmV85T zsTdGSf+1|R=#6{(3NwEBBFk7efXB6yEt0pvg)ROv^RT@O?WvBpS)Q+Oy2a+KCAhNq zLyQ_IbfI2PfU<4>;2~(Mu*SoIQXrYxYr+R|nh!L-9t+|u^FOk;MJ3VN8)4q>9WIZ= z5M1q;M7K8`uQ+3Mc`)(PO+%z`qe=c4l$J;4F$@nyu@TN#6gX{BtFJtXESZc znjK9e+sO}2zHp(*499Fjcs&};)?l|WZTXQ>sh=(o5`u709s-~cWb5(t>RDn;E?ZEE z7YW}QAQsZG3*@GN61$7NjeLIR8m$_HPuY>EgXTYjhTng$9cnbdw({o>*GFHP6X^-< zyMM>P&AZZL;ox2aO5jO&lHl&k+!hYB|2?n(JgS=D>t-giEc^#?BRkOfQ6K3Z+ExtWPSlV z3Jyid&3s>^v8<%zM1t3%o}^j&wRsOeWT8xk%Qw=sbcX|{`54Bd^2AJA7fES`0CJz z>dO{NKr~p4szzdsR_CG2^Z$9&ol-c-WYmgWk8(9!;7tgqanDo4Lp7_kJ|_QgvsJT1 z6QvJ_dZ&gRT^2==19F0;NF6!Og+g=jC?=>r} zgg|(4^B3#F$aVS{sk?XUz7GvnYu1PTQ^QSd>Ik)^e1Qm{Ag|8i6WL8Mxw1yb|NVk3 ztb5zTZ>6u9LZ*%S3QHzH56TX+PJsW4189lb9o4_9q!hcEM8BjFrm$iy&pClYV|epv zVq!D`pG(EaDn4VcZOMH4a1?JcAu}Wc`V9GN#GopP>HOp!0AmS8OzS4XIZ-{NQvuQi zB9cIYCXBypB^!vS=zB`*G=X3QufhMU5F>>fulYbJ*o&f|kop%A3GP2KPZC{by)_$a zu^*;5x-dztsN7W~;xBc>+Y3MIEz#_{8oqcfSe~|uz><@#Mm0J^B@{X=u^gZo`eLx+ z2YqEttEFmlYeYA{voU7eVA8K5M6#^wBvuoJT05WnlTxr4xp+_>Ch)x8b%ka@5`HL> zT4YOdN?;KDm!a74oO)m9xvyMQqKD3tGbmY1kB4aUx(}zaK~O4!g9Ku5P-Kb!twk2% zXt`}ST@-G90t_O9@3n_axE)0rcCJ{nFow3nR45TsJ114z<%17v$D9F%_c9u>%9A?6 z3l$%I`4l&pT$E3iW?t0s!|EW625?waTA$+6Zv%x2~B^yLGZ4X>QDU;>BXR ze31#M2jZpRaFlZWO0ZNTUP#5qO4!a~%mbWMGU~$*AyHiO02_60grFGx09&1T=IOP5 zuus(Uhkn-l%)jJ2ozds(rv*}NM#j#Ce*umUvgVz6akjFhciwR2*VJV^JvO{gW=SFUli@sH9wwntzSzjbQAwwR4Od9TrTU=3ho#wXepwyrw7i z!e?Pmvr>?3Ubj@%$rZ@7PK?FK{5u6PdLhy9x~V*%d2^y zT7QU1ts6YVF-Bw=re)^guJKDCnC}YWKu~J2-i1+1{*=xI`xN}?w=KR=yIu>|2<>(l zH$4A^f5|{5Fz%v5V_H^m=0+JX(gsCuw@Pb8& z*DK&_Fw}0a6CGM8g!3`LC_^yvtQ!O~G9MVGCZYcH|E_^dtV7HbOq+9sV{b8sQ#Q32 zsuoag-tcmNY-~BL+92w&O1`Cp+$L*AVsf&1|C|(u&*QeM^Gl>OOYdof#o4*-tHM%| zRpiCt&qxX@s%=!$bMS#9gO^pI@L!NRk@1Gu#<`eHAb9edS7y{M^24%bx>R4DW6^LR zcePnSTZaK>EZLa`ZLr#T+uNfUST0kFP{3aqCbE7ie)o4BYjxJvyon~U;`L0G-k6!S zyZRQ_GOREX3dR<4)5uFjNPikTKf%z>TSk!$!U#U7W&TdU^>j?19*Xj4xvM@n%9nz= zMgmZQ(GRo_To%PGsl<4-(2)M@C6o=2ylL5XOPVymaS`6{_x0r9Zv%zSdX9xcs>s16 z#3^cVSpR0cCg<4j$~xk{npopk(c^h_b~!k@{)wVCw?~gQbZHcEQDb}C@xXVP^V1u6 z+UXw_(N?m(8j~qLM+@x~d;+z_a~?fnfK|cCpUd&L1F#3m0#157h_`Z3lqNgk@OAuf z7u5x3k0!O{B~dZIjsiKm7Zi3V6SinD#eMQ+f~U#ehUPC&AEV}sT8}>rhr|=&;4h$K z2kjJtmM_g4zXCB#*{S7*bx+gEb)QF{G}9#$Ou+#2gVgT0_O^Ffp>Q^oAc|((YV4*Rb-|?5AXpsELkMDD1KQ%4sroCaw0ceHAjXe?DI?SLh@W96ePJkf(5qU8LHXbt zyI0a*yWOdr2KtdL%QQFj9>xvI-gB%Eat^1%__KW6Y3zUnQv(oE7W)61nY#FUl3vW8o=9=1hr2O z=v&&PRhPbEKfkHd&o#RipZ~nN`%T=e;qX!-?U*o`bY6lpe;4&6&D255_rgQOFDm^! zWQiNS{cHu=}YK=ai{Y*Rx#&k1-V$m72>UGPvQji*_l&@ST~%g;BG{| zwb7Drjo}p-AqBVu=M(LbIYDe)w8~sgZqsh~%YN(5H4ag;0aD$9hGO>iE*a0LS#PtQ zJ@4VN;N?3%Z3;Wy9mG7O?V1F$S#9R1(lFlj#WaPv3KTAc*Y#=#%_^#Td5N6Mmo^O6 zix!8xZ-4P=WJ2Ccd2(vx^rU7MPj6!4?O2M(@Q+r9;TPE&1ZFPl@{y_2S|d?VFYo9W z>(lPA34KKDO@Nyck|;>q{Y$oz@6uWwl?yQEDF|}ExbmMTS_y&w04DM(XZq&ZT|HcL zj0Xu!n}_4pU&oYIxbaqbtQMJ%OL_2>bsKUB0#dBC^Y*OEM%9t5OI*bY(DOsezd^Q>ACDr>#w*EYDw38r~K)nc`q5h#@vh?eGZl`M6inycI?mhn2 zw^el<7nS!YdA`E;E z1d=pCTVjvl-tJL$bq!6DoH_qCd8d1E_ulS|$*&v)e~rJ5wI8mLq!8a)pg?KC!xw26 z5vWOBv=#OTSUl8PY5LHpOC!=d=CzeVzmpnPhVGHtjJmQ9x$U6bL`>CRb8kGlrw;;} zB+sa23F^BbD@otw@4?Q&9`$6e=`r(KonexF!a`~;92i5!G$hKzVEE{9(C6xOVJQU)h zB4NoiZAJZvLtVP?pP-)MqWPjynh_JxJnKSDq&O5Y#)XnZp{J#GCZ*TI$tH(R)QXR6 z$YCc_;6A9$DY#Z87abTJ%p;P5kLEmaAIfDB#ih>RpK{ZvLDqkpc=&5uXLMK$1sS5D zaEy;oSlFzsN@t_H_}>)*o^>D4C2-@=+_TnfQQtoY8n51vSbW8Nv+`OJv~p{tt-Y7| z03ZOEDE`R5K2M^B=!EL+3pTjtK2oXgzL+e2OmTMQz~7^ReQIDn%4}=a9Z|#*c?3A= zdGE8B233=D)Z)^~z>eymgn|I5aooegm#P>Pz5VJ%<{0CFVv!H3s{BWIM{u>iPTIb< zuDE7jed9Z}P%N?pdt?4ZdD|~BIcLi|WtUtwD09rJ-Fx*@RAbTx@GD15H?6m_LeiB? zIki=@-nTtGxu>I;T?~n4nKoP@6El|gYI#SM{V)jB#g+yTumrSam5}JAgKbF|JGh1( z-u64p3M|0rs`-xEM4-+5_a@H#-QDYLm9y!#ov1FSqyMFl-hhEcPUC-j}r&XKw+OLZFZ#a06yv z&2WMtT}@gFbz^KR8v|cM4+01qA!+}z&5Y`Hn5TmaG<fPt=nPJSU2*skl04I0ZrkESyNiBzRMm5K67x2$Zq#2;R;wfs%*e zDcI{o^I_WhYri;rIcTP$QAQU81@&~ODP#Bd&mU#&Co)DcOSl9=YI2;$lS8&le$+{e zO>Zm{a#4jUwa;>geo_YV{JA{&(;Rl*ujW(zNV{zZn&}t=LRdm>z%rBLCVzG1Uw#k}w}1?c ztbdo;s}pJ@M263@f%*wjBY=mV4{FKx%{E_{HywE+H0BKU#x3w^-{Yk=`8}V4Q0q3U zYuEl764G#WwYXZfGEG|=K0zWzzm$htVY1&2GZZ}JAXQUS3;pP(65#m)*bzPOa4C4 zWF43r6e~howp*k(%AeM%m)`i7E%2)0=CaJ&3rRS7Md8od07k0x(V zw&P25`zFN8&!*Xf9`0nIqMA?0@5A`J=C4g^cSWpcd2;2y_X_5Ydc;3Z7C>Nz(CwY2P0zHg^+<8&_Msu>-Wje=5wn+EkiyeF?nEPRFmz9%P^ zK%eN>slROcw)_^CbPiu`G8^Z))^lve3 zOt8 zO94A9R*!`8a|fplJR4dP79s zUcTiRJ6j50N~Nb-3*tn?Iujh*ChXX{H(uJql|vkUw0?0O)N&_$AK-93AfAcKBEB5) zp`HdiI*ROQfLp=KgBIW_^LGJAu7BdS%1K~Ik~mN=Pi9RyVY*2r3)Rd&6w4^NyZGVl z=u56Fx-1piR-Zt=6F@eCjD|-biOAY%lD#(O4Ev3$?(Ol^NL3y#34qvUow32g9`S?Y z#c$S`sWajGBwvaj0bLfLf`hrw8!imBSD}IAxbv8ID0E6Ea(2fUVbi6co&C4w7d z&o@gyXI_dMmK~A2T9ubB5-Q!sOMd%<)>|-jQ)d7_P=7W$x!y>{hCik=oB#m-?dST1 z(_@K@J|Il$y{kaHZ=&0`XPrZCP34rxfSgf2i9b0y#xSxjPiV`_l96Bb!#1AdJJ^SZ zmnVcIbMKQP_=58&@$9K+`oj?fz5n~gf&ZL|1d*&wFvFHb0r%4bJ#LiWG3x6&P}Z?3 z$OF>Jf0mB-HwGkcN~+l>oTr2YeBw0#ZA#VyQ+!odPn*fape}RoVY$!j_3ZV|X2MnD zBay}i7D zNptFA`p!4z|#@u=Q=NMojaceCV zRq5(3J97hr&Co7=u2O{HN!|lqp#9_B!{yp4vbr}IH$tzbFEg}F{e0H-YIa__d{@tQ z7MPdTkP&sb)@mL-AufsPPqy^R;kt+9!>((9H-e^*G$;vbQ%R1UeIW`Zwr1RiMF!sjP zsy4_aUNtVg-?<9Be{oyA8|Kxlz(QRF&I@>3Bq{}Mljw%PaiY=Rg+nkqPayi;Y0${1 z3ooq!3Z)VD&s=DQmoQuSL0Cj8?nda@m&C;t2m&iQo^MeS^-UzWAA(z~e8vO%0gLkrB&htTtG^5l`F7cBY@)9Fcf-{Lh zE9iA3o5i{hJC2p25pm=dGYaJ?kLCb|nZFo(1MZj>6df&Q<6xS|9?%{exY=tG4)4YI zZFd#Ptg+(L&T=v$nyQWnjg*ZjgiY5 z*a201U_897nr?+zh@g#HUJl|1S?gTi7iz=Ph02C$%_VXsX8AcyUb2o?%ga*Lq~Lek zKBF=&nCfuolsh|_@>4+UPTd3H$syDxhp*w#G3a(si!~ef%i-%(7NBq-wgnib;Bq>1 z7be5TP!teSe{pw`uLSz}xY^@}R#0hkTe#IiFEgdjo6!Ohzo(4n>s1y5*PbHhQ?E5g zgA^!bg-?XAv8}@TFDy;bEnDa0Cxr5zPM@dteZXR!+?{Epd|&5b^HE&8BXB2+tWx>?dz z0z>(jaz2zPXo-COd0~&zxZ7UK=2B~h0yY7s^Me4`LbY8NYofi1+1-u&{RDR!@LT);H?K$WUg2%V@*~fxVO|0ZKVCmK z25Qs|iC$P-j>tl+^jSDl9n@xKow?HM_bA;Ie`tf7nYD&B@!R7qAsBLB`4U6`AYv>G zjp+)>NMF__ZrHb_8aIJ)@j-gSDn}nO%X9hP{q)~vm@Rz6dXY28DRR6YskD-9(6|p? z9~i3qbm=+3JbQmqeD#%A2RpqwAxTfZ^7C*#G$~hX+snFS4a0!zJ>7r@t03n* z^XIx#hodTvG~tD@kYz92#vl|9F?i;cP!)JcS${=9879hqRQjwR8ozD|AKBPRI% znLk6?vrniIwcxEs)M#42-!7=mgNHKv3sjyzYRxHI(hrYl5?X~F1#OSrIDP%AwGxX? zdf9u1$50*ATDs9kO*}!7A65eyva!XAq5I&D)B|7vjk{?iR3cK;9W5C3FNw#Z1k}0h z3B}J#dj^pl{J$&M4kgPDo*UmHaxpIONcR1_<+EVM7V0`TZ73A^r)l5k)w#9ZtQ~QX zV@bTi$e}XdH?eid=2-Hl_oT-9WbgcP?VffiMpfUnBRR!q+@E7It;PYxey1*J)s2=H z4*O}HL`9Z>$L{Wmv%dERFMtQCj~K$BSe!~qZtop~x+utIKb&roXS;+ef%f_!$rNV! zNdtjCwZRPGzNMwL45Npv{?SIL(qp|YZp)FfRWJ*ZUKnzMCE|2VO!r{-e|8x{+nQJ` z4W-`6Q$iDa5VZ$Cy(Tsh_^+0rp#2kV5MCuBwT$1QU-AWYmeDr{0x{k#|E1Hg;C8K zNubJt`8F0GTTsu&;R?j{|AMmM5#}g`TmJxM=wJr0wfn~D5mo>cp)TyK)>>?*kd%UN zkAhz~VZq!DE-{1~_m)u(~3?k?bt<`T|+FKZgm%=`bcE)BS~b+ zcul;Uy>^Y(!JL<S+bT=iR% zS=BWj%qwXb=FU>_jBR&fBfKYzZ3FTb%2uuwa>%`aF(JiJ@xtxa&C0z`pcNIjf0kZpXTVH-$YiRpB z$0W3U8dGjKzgciEus=JGhx)K}g|EySNDx;mmD1sTqi__pKy2?)VXg9mMh z=yH(Lf`uyusX{ncnT#S}2yh`zI|lg;+5Gx`%6SL(H>VvPWDCYUL2v-*4Pqs=Yu_j2 z-5t68s+8>aIBP{m3U7;pth!6(*r8pszsGtHseN>^QIeitqasO#QB||HwdD+}Nl1H& z!lQD3ZlaUj&<`g(saPs{R#o@V3FZ4GFbyYrPANheV4eZ@4D)E&5}2JBwGlB|@Agg| zcucQ{=@I}%C`#9QVkiA8(0f7Nek>V-uVGZzR@5nc?|;Hzd!>b|gowOji6hw_C=26d%yR}%TSbK78?K^6&Vg)&8IQX^zM`M^C6HZM%=9w)L zULVZ8j<=Hwi`fx=Mu}Oeih8<@o$l64XU)y=)asg?1Mh~Fc{&@EnxUk_PPe1wiXQ}& z2#@uV?4)nc-oGmvQ+EQB3V+vfemkWr*9IStUea89V+1$NXS|O*~`= zFf!5UltZ)Kta0>9AQVsl55CzAAerky!j&Hq+N|cNJU{$A9}^20Dn-IrqS^XI%Z?bb z4OVZ-eCb)*>YI?|CG16i9c(f0jC34L?M;YhD^KoLVkEn_n=5l+N{yYuc_Ghu={>1w z5pZH&lIWm^j?~2<>HFGQu~NR4r~iU4R)=OZ=#NFr4d$>vh$ZkPH{AOaY#{WyNI2RK z#%ShxY7L}_0#oWhd(}N<}Fuq_VXu zvXrdDjIC^;IE4md8%tx!I>s3DyPx;yoX_|7{L$;Y&MTUEzn|xR?)$p0>$+`xbzdad z_#UPS`IzT1blP!Qj3NryoQ{ecW;{Nq-;sq+t&(;x`R;HyXHA&6q2b>4DRjc;Io?8@IhChqPDIPrz9U>5Byb9(<$t3v!0hDoQUu8|dYc9z+7%u4J+ z)WP_PM=3qVOBI>y_zCM~hITOdlIBMK0bCZlNDo-|@WA?>o3y-JqxfNo&pRIlTk=HutNADOao#L=;wWVQO-n+4CzMh-EO)21|6823GS5khDX zf{Y{tJok_R4M-Qc0ddXISF77c{w}o-S*nD!X~4>6BE0tQST3}k!1M+9^tHuDAZqV} zAT?|C6;Dsv0t4_Yq~pBk$XfE>l_9R$CRSGMu8D41zmr)$dbETnbKytZ4zq2SDxzH^ zCI(5Q``WL=yJ;$?5~m=lLknV+VPB8Qit)W8dpCwkjI-|s-ZkL~>4VUyW0}4+F{Krk zrIs388AwfgIZR38gO_iqtyum}ankZnh9}V|PBKGRMy0GKd1+z_d(dmsBm@4vfRs`E zHVSv(2QAQV`2L+|-BD4h8*KtVJn~8`Ygz3fZcMh#D4(YLCm%iB5YI?f6EnFna6KAb z1H80TCWHfTOSkqwh_5Rd>>L*>oI0 z!r{~gkxti<5rc(GdPrg~RCunT&~W*sRd#{B>H5itb6(_=cD^qwC#E)f&T^E4mXkJK zCkGbSv4WdCYG(hY6o9gcmoBsNC04fmJQDx(j52@dqjEA2ZTjb2p1|V;w+#7sYPR8( z>04>BPrn5%dH4!9xjR+LaNZ179&1c)41+B-Sratsxj*X>lDtIxgr zC8L-JUjFz9dL1%O`>;fg;@F)%)_y!>ic@c+uNo>i3rDv zuVNS8uF>2zSn-eH3o)L5C~ippEQw>6DEcjq|Gi>(!oI3z%8elPN6z&Tde9|CUMTrW z-{VL-He{k7Z!v<_E7%_{c|7H-{d!5>ISlKEj@8mfqd$7yC-`!ZR88|-5Mtu_C_J_S5pQyj|ay~wQPa$9S?+x$PYy4p%xb@-YtIYPq2*(&}D-lV`&v?wZwF1fh$41DgC z0yib;$1XxHS|Wq~#kA4n-h}7ilk~ZG2b`+Cyq*#G!~BbCbQ){@IpylP?;5j1c{@_7o=AgHiOjD_yS5iYyPaR83R;MMQ88QsGL&Cpa@JhN+YA<@^ z$Eam`9qUq4vkfKWr@Ix}jV*3SX4`y9r&B2Gg*vK|!>4-c)>{<3%(CAB>bZI_>=Ae9 z^bbN)Zp0bU8qcm2oM*4yOBI-xbG zKUFntWuj_2-HU!mrzr0mpnEXtC-L)ycjS4_E;oOxaDRWoVOzX1x2=bP=l-A@-^@uXPsRZ|YGAT2AO7khP**3lom3_p@}hsR_*XS;%4F zXI+KmC)l6p>l?z>J?zPfWd{wD>eibwK0l>g4$uL|1=IumoF8POP(WVdpCKE&so{{6 zW8q?EqfZc}zah}!Le8|8l&~8+L_i}e?aJ7srl0wZiqW{U-M2_OGA>EJ&G@(mio*(~ z+>5!Knx4<{QNFh88vOi*#~{_MpsqHZ%t@TKhs3-fg}zq#jEFMatv5(ckpGIvRRP_Imsh`jFhPNlR9WTgAZ0;Rs9UT(hDj=OYIfs`{(y8;{D6Y_3rwcatmN@7AOK%ur zhEm9F!(7EOoyem9q+?QW@e^sBCVYFJ!)!rwW+swi$;Kd3q@`6M7|G`*~mP$SE9_5PcNn|h5F-`9=YNSXAr&-*K$?Li*e-=oQU zB8Y{S@#biiNUq}b37(Xj6)y2T=z~8Y!|_`{mrh-Fq`|<9h8|&V^yQbq*?|0Cy+33V z=;Z6ZJ#r7NZkUL5#2aj!v!S##oK#?23baW-NYx_}(^Vf@v75HOfC$zT!nq$ApW{78 zjs680;A;8=p>J(-As!l|;yEqRh!$@r200PsfYps}u9gT~#q^R}VwUX5dC^k_LxD}d zcv%7$CL0-<2T7p8EySe2cpTx5#7e`xU%O=Et>RTU!=eK9^0PNjiN?BF^X1m&-Q*3(AI21Y~BbRTe#3o7p54Q1S_pNhYda6&$ zK-c0{uOfuN5>Rqdk3E?+$^UCZbK~yo?ZB<9SF3a7%}Hxe z@KL)DNUj1Wj-CA{4)%SDhxpN#rv%+E-BDF&N93dOcH5bt9M${{m4VZmk5SA2z_2xB zXNTU7vT<~qZRN#bYYZd5124{6Dl0OXSM#>>5_9j_pRLw&bI5lIw%ePQihggSfY)D0>&E@5ahZC-T zJhD{MmODS}5@s;topa_T(~0<3GpKm^4FHGFOSYfDd&V?^?H-<$H&)O8%M{D;gbI@&cgts>yi<5tYh(Y z0@@9-LW;vo#knN@>awUTc4JQ~Lnh@2zZjNd~UXnxb<{T0E%Fs5; zUbUJ%7t-JPZ{EE)xplcDxBxh;$lumgmAY;JvmCR6;tuDlU5qf8hI2nq(~~4}*R8CK zor?KoZtixTAh>a9DubFf@TPL_)QGhas&p#tC-@d2y!^>-9&`+PaTVXOiIeWmn$q*e z!GL9MQ@*}7qWi?i3C)d|I}~VL2QhouM5R9QqbV!fM!tQzOuoFpb9Zt_rLN+)_j-eTG<3kq6WwOw~3&o%Fv}i zZ;}4(;?zbtGz+e%)rJ}5Do8dCkT%sn!?YGF|wibBUkWW-t94mz!hqD<7JIa_H(D1O6e> zX^{dRk&}~rPadB11r_v`_73kyx#mA)5ZlR|GRj;x6ENs;3N;KdCq{J9*h#)XBnixE zA7^`XtqXqpntGMW`NNxj0rv*Ha1o6#GICLL*9y$)55NYTbJRkSHM<`M+}z%_vdAi~ zwCIQD#KAB6Px7CcJlrMY+VR9DrQl7BM8#WO>b2lBvhDC~O!EKF0|)m>4kQcuZ%5sf z3oLU!$WpeAhMBSmljA=oSlQzZKrV>Capa`?+|*foBn%X;SGUcSoqf`%To?zaPuK>- zNI6}%UQU=k1D-1G(uocgU0DlgrR1hJ4Hh98e+R9aA%iJg%9mvDxs%Yo#LVu%g%lVX zt)He|^_f7kvefFv#jp99Ho?jUjb1c}Qp55<`!zHiJ<4TYBA zAq#UF*+ky6G>(kvpUd5?(~E3*Cd-SWM~@a*^SB$`9gGQBwOM|3K9tots;3h7W#-t{ z3@4OjrZ?zBrLZ@pE0L!fIV>e7w(TOn)+K{uy))LShoAUa-)P6{0C`<6Qh2W?Y=ujL zp^gna+zSprrfG#ClVOffXy8b0SW<^tfGqV)jSccm9;-SnUv3Nx(9^=a$NLiZ!0F6tcx)dRrscqE$(^LMVoN{FocIFi#H8M zCOAF}jk$Uo5*sl51ZOwQ1q+$JFj%^R`F|8Gqw+?8XC*`1=kCi8JN>u)q^W)0OFwz<3u*yt$Nl#G7lbbE{|<$Nfv0l#8H8ha32q<3|9;6X^uRDNa_ z93&pFdS}Li@nDc<{~ysL@R@A%m{YMb(m`}+6b^xw=V`hEv6m|QQ4AEWP8o=&DBYE6%ay>Zv|5e zXqo7vs&c~yeK79;KP%>%&Zfq(&l8KBqb~mezn~;|rC~aV_FmEUi5$U4s0AIVWLuE;Ay42 z{Xd8*S2GkZP@Ax;&_0 z8@8F}Zwvc}jTKGzM*X5ApQAqC-#FRWB-t~N`E6NH+{ZD-RATo5iXBn;+c_H`x71Va z5}lV`_PA>DXf3!LM~p^I)2O zNs(1^h0>;RTI_UlW&E(u7{PM2Q<5V#7GU~NN|XO;5BAeai7e?Tuz#V=dO_Y(GS9%& zK>wAj>zm^vpC=kRA~6aFMYvOdWCH6LHDEV)Q=8idSuUD7tcF_W}u3-7;ra_ouCEgv&PFGg^au83vWr`^aPY3jI}~BWF38dsaOq{IBcJu zlQ}m=8Jk_kZBt<0I4d99V4kC(BI~N>SC9c|!y{sd*D8vZUXeFpW{s1!nKI&&xc?X7 z8eAv(zyB`Dh()u1;he$0c)l4Mj)<`CqSm7`%8N~eLLC*|XLrFpJbsuljPfFS9(mit zHpG(-2U$c^ZKiHAPUtDZq=5H`jxRKW%WihU)D=X{D2ZEHNL5h`(M4ur9wSaIe*vLh zHzuL14M)9i(`=_0h^ELD~aYZvKQ$S)Q=uz6V%bv*iBP` zh~)QWtn0gh(yZ9cYv4H-vi{iKO87SEg3-|svhZ-jAvFUz9rXE#H`}eM`Iy}Cz&pX9 z(z??f2`LW}*a0LgcBR&wJjkf7kbB;k?lUZ zw`7t9lQvugYpWBgu_bJG2{O5Krs#5 zU`{(dbuiC<(y6)2 z?0bH#AgrKlbT1rl^J86~Bw!jKPi73-g31E7%{ao*-8MVoY7e45^jxcYhQucQ|U)ENu#8SU~TYxilXfAhRG>l~{V$5;GkJp=$Px2ynDA*QG4v{mrX1 zTU`_{+}pQ$#0a-hl5w~FJR<6+BTx7hZS9hZ;nZQ*`a`wBM;LRUDGL6~h#Lgw^OhS2 zpX=aNwO!sRG}I!_Q;E$A!;HeOeamO-EGMa?A7_?!6xSE?Rc%`w!;a^iwHAzN&dv;c zt~G3wQwmj&lQdyt-t-fZv&tH!ol~@$;U66hV%=7Y)NEa@-p+}0P+;w4=}c^7Uc;^S z-Jxl9jHY6A#)>rG@235N@J^<D~Q`TF394$Q$?5J;* zm{>pIW#g=oravBrA1oa_BR2Aiv9}i}uqEV`-R(O@RL~E96Q3sazbG%>&@>;tFCuf;NY+&F`lUMt#)W2j_07nwr;`DK z#uLZbrQWNK8uE?o7dtQDTvsK8V+lGHGhx()ZB*@L8`m9-!;lDh{a2LcvAlrp0K;Ct zSkA74gzUeNJvPF`UUg*@LUz$e)?IeA(CYK`JhlCsLaLTkcVjZbQFx&fl+h#gI0p3z+|eVde3o8!0$<%Qg!T9w9P$9k2>?#I&`ano(V8p;Y?i10m(mTSxw73#5py zV_Lz$(F;VN8~OOs=nS@HfPhEub>AxG>o=bhbjTe9(gXe*4lTdr+#iyY%W4K0*dI}9AKl*Tc0SW=+hF(&ZY~3( zGEv3wuZ%B(lH7RPDFjEu0QR9mOmPLqnm*Fv9)!7+bQ>a)yrTNKNM_UPrKTTvXun7j zhjVG(CO{{u5_K`x^IT&Pn%gg3OK-e+gZJ3-H&4MxUICvUo&{RT`b#fp{;7k@0=_;Q z`Sj16aVp*k)(ud|Ipy!bN*L=K3I&fC07+W&|GM1BIAKF+F{tb7(;-EMk`#*pryUZl zgf>NRl>2zb;irxHRCmu|)UrHuAWh^Y|F`I}Pm1or;ugxB$uWutV|P+Xvbnl)7XpY$vdXmrk0~V!A8} zAvG>!Gqx}b0(c-rMXg42s%hWYFoQ7d+iuJ|fWO9Ad~gg#1_vurF3NK^K2oEsfadLy|dQNE8jBegif<(gDqwDH2ei#`XHDz@7If;>nmbXHMcEk2>WB z{VwArO?5)vCSUTu=fj-^9*w|6&|uzyosjCp8=SLNhxE;E++mOBTqe4aoR1+#Hf%JH zcYJf8zm9C%&oSv4AB4PA(0eA1O=0F=$ws*RN4z8_Uq=SX#GuYWD@41y;ve?jCxW=D zxvI?ZII9oerHbR@`m!jf7un*>Kf*Awq!LG$%>M#DMOcXI6QeEl$5_;Ckad$Al zBihAkbc!JrWvf$Pyrc6AqKaT~kvsF4f+*TBZ+eu;wFthH#^)0f^ld@r0fr~~Fj^en zx6rH)DgMpW&VM7(ZLwXQoAJ+*9=;G4XkCb5dGJvN<%7*(hS2megewKaF)D&7I!5i0 z1>|~zD?M~iFEZKN2`~6M7e>@vAMnxW|Jy<1B^#qUR>5sl`42KQI(xvSXvv*)Dng^R ziNVmD^`NiUXYFk;^Jl3tcR)AN=iu|Xl0#BD8NBi=Xt05$$XpH&*XK}XGy7PDL(6b3 zB3EJ~mX2P(iQ5>)=<0S6iUGc-975h9B<6Vbz7^aAh=OB*Nzg$q?(wlFX$Cd^QxAEbq=dBBD5#3nDR3d(2`?C6xJcMP4fCo^W zf8NPG1??RPQGwJ9p{S zxH-&0NbTsmUT}ucx47`c$X;x(;K)3Ws|o{Bi5}2g09&Vz zyL{Jb3|KHubxmHiX(Z13>M^itgBPp9d$D}wz!Y4k)j!^G>`Dnh0r3W*6A)@F-DzQ50hb#J30XyH-?VD>U;cm!2jZU zo9mW@g)&RjX#OEvo!{0G8%-jpxeG)0VAS~KHSg*|Jdx4hor67^-QzRM+mb+ErO2H~ zD+4QuUrwxKnK+#9>;Gj8D09jeq=+No@NFc!R-ojuhI0-jk8dPC88G%T!prYp^@r~P zPTnxkCdd@c6NzJp;!qLz9(~z6of`O@GF4LPf|*knnGmSSQEr0M8|n+CbfT)V4Pk_&S`TnF<#EHfK;zq(}0 z6%q&N+q_9HA^YtYc2{O>8JLylrK6__@>zGwx9j4UaI%L7wbk zKn=jG3Tq(2^DN1z-e3&jDKjH@Ie}Dd-^hMI0Z|&UURN7YzQCqm%__%YyX(~Db-drR zBA;q*2);nh0B;5(XM1uCP6p>rZDbf=9|9g0oU?c)%xDE^5Qb~waH@}xrwqvAHpq@^M5gA?)tx&^)z6L zkZ=^FbmnS)-GaLt0iOm9JSCrku%AbbyRskRDAEoG<9SSa92+_q*jEVIajdHxYe!>G|D~ujn5EI*mMtLt9`$HJgpuY)zsxCZ-Ul_BC3vt7?4yajX z%sQALz4{x53Sz&k<-Npla{8~cOErKLF%OrrfGc?Fe4f4X4RkA?iWFs%KSf%2Ep~ID@VQfabcf+f2?DjD{%R5uxnLC zpk~h$8c@^dAA%E&L|bJ@_V+u>wdWDf0WL47OOC%V>&vRSt9Q%g#^(nde|Rtrt)t72t+r@!D#$NMT>AVCWDt>0*VH- zMy0ppZ;v6QBi6vkPBWajC)0rI z1)lx3DY5+YZ0u>`@`LtmWn&42C97>Mt|arc`G_P!2Q88Al#~b_;dXII3Y!-wWqh9U z-}^aR3>z2;yRqLBq78%^&M_iNZwC8TVy~g{hFpmEBOg5e-0<0g>jD1PzE1;a>|+~35H40DsWg~hU1%NMjBFhZ(p+v6T!fzf$$CrAD;XdjU9o{RSOrF zJy5U1%F%KCm4jVVpx>Zf{PzqSuf~RTz|Xhc$I5Z)vQ@9Lx_L@1fm|iuhiTv-brU{G zbQ>QyG2%JDr#9q5)LU71-b;zGh4(C~4*lz~RARcoj^&5Z;^+Zo>1ldjjlgA*ip27% z|H(*9k*p7`lI5r+!1G_t8=El6f}X`>C3@fa)`Oaq;SkS8hOWPqM<4^-xJZp)_zl(; z`God`>&Vsi-w+A0i8m?hzkT-#yMb4K=L^~YU|#_Yw?Kp2ZnPDFeu7lX&}Q%51Nyvw z1ytl#OnL?xkxfTm1DMkO&lQ66?FraZ07=ui_+~tsG==5}G1h5<>JN>H#956OE@aH? zh9#-YTCB-~CP0XO$6KIXS6AQ0=PC6W7wQP-ktgw7J2M~7|yOqs=-AFokW;G%?oe&IlPH}sw^MkYP?G5 z%{!muInPGd8^5i#@7`zHA}(b-GNng^%l4wK_UZmnylxN+va4bn?rv0S#=39UW^{M& z+avwtTFpvhbwl2zVmCF7t>f;13bwC>s^Zsv)rSvz^g;HqK?W`5PkbxHJ>1bUMjtY- zYmVlF<_K-%4f%r-V!gIDI~B6_VX`0jR&pHQ6Cf|V9bgN_l&4-DD(M^aHcZmrzOROH ze^ZHwf}^pN&k?eyvU0~m;YCjlq<;JPV&HOkcz%9<7OERzbtXN0y$%;*x)W~L&i5i) zm&x;&OxL1-1%Rr4=5mK<81QH3#rt?`ir2J#XQAa^f!b9^|0F_0)yD=Mq2Kjb^@)$S z_hgzWq^lGz%(m?4D35qHx;cv~DwZq~VIySMH#0K`B$el1-SlpSzc#~5XZA4r{8C!y zh9Gq%apv^DMlEjp$C~K~4wpI4K=ZdOFfj0?rp-An_Gxh?oHuasUCbHC0g$vwiuVW; zoQs5URQb?+2y~j#OM77=Raz*vo;6=3p?1Em?v4N`Bo%J`&Y7D3aaQO0RLCj#nWulx zjt&nG4Xu2RUa)A|4>FFrxpE2AiqFHRgz8D{lF&81n3GIg1sZDt$#(K@{^p!K<}b6S zbgJiL&xVWUGWJe}%3%MJKilG%@1KKk6L=T7>_^mMZtEA6TbnCJ~ zj}4Ks{3U&<3*V+Q0}=8X8nC~83!C-?U$`VtM9@~o+NQ%)P)s{9+EJgT@x%)L;wAk| z0b@g{kPV_l4p7(AriSbjDcrd^(;afpzzJy8JK=5nA8nA9zU?{@G+l|CTk-ET6SX9j z4`;Es=O};>pV1(pv3FT=IOk^~Tl(h-h&k~$`!WpW>eA`-(FDLxTU!ZxZZ)#2i+Ot( z7xV4o$M|C8mC7Qq;({;7CC8Pboa^h{u^xEUxU5IJ;ggC->~zj#;Zyk3)z!5V6>BF3 z0-)CBJd=e-h*qw(p>r^{Y$ksILigr2IB?DYdm5?|M1_K?1pa?k+W3sV;1#das1xy% z&E>GIG`9P;ySJKhM}jJCm9a(Uj%F0%{kanifBdy^i|d4rCDVK*ebpZ4lwrn+y{ckB zdVcyG+j?qV2p2N^$M3kk8rTxStKSvk&O*5vCnuNsO$>T6Ch!^yf@bpvF7suie4TkA zi_UNs{ORqz3%n`uL{rPk(S{P|^l89nzNn^mI`@vF=w}8 z01{KcveP}DPi{SKzXTU^-GhE!4MNHHmk0$Y*8j3Hr2RqVDg^#~y9L}V7*}!*yP?C; zwG_4a6)MB52|#d$g#II?VYDIH{JWj%kX+^=TnnLI0D#)Q+2_v|T16r6A08*SgBPR^ ziT0T?*{zHp?d%*=KqG{LbhwS#*#2A3+kft`{|Hjz9>MdexskNwp48{hjsDJ+NuIly zYm*53^NU(rigeWIukT^Zb&Xc{QpMzS-1u&JIGgm5Nd($`_07_hfQ79GW$GIkT&?vvW=vT8#lew$JMEJ~8B(f5;a)sy&2>*9uBK zHFf7;=S!rcEu?!^J>;3g7sSe+nelrU1WX<{4dMsujXeeLO=yM;Yr3zd3V}?itUA6f zUj+6%%DT}SdJ8X;gJ$!R?iKnpNXu7()HK+qY&HkQI(R!=Pdb)lO@5Cf~ML#CBB@2nDD3IDMKtXo&{U1rIWu=YZoyMkNQ^R0P;R z$>m+3#q$xFo8{{3$*Gc@weJc%i;_xzQP+tG9@3{thFF`W2Wu+-cV<1WRwM`~d)D3r zoW15#zGpT{NF45MDi-nKcUQwcuyRL~O4vP2D#|;&j$@&?l&NkrVo*Jf$a&zd)3xfIidw@KdKyt%S=hk)QdLEN5JDf0*MaHSfX zNp*;K^37z3M5?30XNR<)q|Ahs#3xWs;E^7o{)0w-EsFBMwE)m^_bQJ06s;S&Lx-@C*$nYtgWp4 zXpr>4DtX*{jjhFpr{5>aLLgAf*sg(2*^ug-|3U`$1NYX*mB|Z!Xqryf&TcqnnYNV3 z0kk`8*GjeMM6QB_tBSelY7yPX53PI)ei$b8I}`Oeh{Bi$bElKwBFpseN4_$b*u{#9 ziXKEMfGaqIiYoTcTxh=*Q*g7mI1CzNC{rcPB@lxB{}pXB&Yr9FT$wL`Q@`WkpoNmA zc)8H0{52)H)l3JeTe7ev$275~{BsajLPTDa5i)#MC-3wS5tko{Whl37mSuE9I=1IG zBw8XmfKI-Jbqh?!)4gMHtr<6Rkr#!+v#bSNmG{^5j3pCSh<{}S&F|WG_>t$OUF&S$ zcK1VJ-VfW6_fCPJDBKo%%Uk{B)8Dwl z&U9<^O|cl9Wfzuk$)`GK9<$+k&G3m>_Jb0+Ca z@;R0^4i{gO|K4hD11`iMmX9uSr_Uwi*^aO>6rgt90`>?zRZsz-YcPMQ9iY4dCDS#` z{nJeP7xXO*zh2^lka6mjmwx6#6_w`WJvo>uCnp%H0)kF2QBfH<Tc2dmTDVc*+Pk8jO3(&RDM>hx6pWLGRI z;b>yZV$oeMMG--FHGjg+C`l8;i@R}cpAcp)7}4|BTd*Hivj=cQ{CDKDRM=lFWk^or zWg-9oq~-n`{%){Dk6-`M@B4okk@4b^R~P6>9oGYgUihltE+$&d@21kR=nZjuepH!8 z2F$S)tVB}0#!->fmXbi0FaKEE#M(}?xjf)!>bdSNVtYL_SZgx5o1@v)Ywc}nH<#Nr z_TGqcnc93dnAXwJap#~~9*1{TJ@G@T6z79T-jtNIrdXZyoR*w;whjj*+gA_)ZlLX( zuB?3C+|Mz@XpdCB(iQs8)B1$I8fq~gZ^ToWo7D_8zd@2T&zzqesRfJ9aaO|kL zT1>*IBM_cde+R7eO$0&bR3A1A;Cd8Y<${TV6jhv{ag2y!a#E6de{F-@h+YOMt*0kN z%4Fog_Nx=lVWz`jYFR)^A_lP3jeX3o-;E2cq^epsKw^EiLhfOaR_Up(UR6s`DcjZR z_03JG`>G^*UX{|@Mn3EH0aXFk+cJAq?FkctxP+;Us_s623>O$*6RWGq9MU9`w-a&rM)O$ zu8@08fUM4}H-PRl0a*d;=z~4c!SA9ZeD9Txa5%{vAE?>)Q;R$LjUvcySyaz6i5L}9 z|KVlUbVilkV4q>0#RK+gLJGGb%LfAsdA};=SeQ(zT1_9;OzCargQ4na;fbSh&?e2La zM(u6FQ~KpJD47wP>rOD%yuBITLJhni7q-07Ie1!Cu6uHl?wB^?k44^he!)2Ami<+= zwbtB+--mk4BHRa^*WELi9{4KD)q9rD3Hou{-BXXN<~&EHx=R)mSMLc_#!{M)sp=AI z!BMb_N0-6*e%%BX0RJ~V=oiu5@mW}*e549D?zvGbPU<256KT%`?20(^^-?+7wIiYM z{XiUq%Ub&T-L6;Kuf=AwDI@|r7pp$R6YHxXTVJ1L`E$yaumMI`Pn#Bp(?GD-2-!W8 zGUEU^K#5DQ=1zecV3R03%OAeDTQ7K)BY++~X}y7BwL2yE*=@!RK{hxI!@$vMZJh5FCh!OCA7l6)LqxiU_xW0AVijc2O8 z51+m!;Cq8sl$=jJwJrAO%Yhub(`kRPdu?Rg7KNCuG$zZG0;Xg(7B@Mz+D~ zw!pF~E$wkT&(K(==xFiC6JaQa4oJ)5gQVWgu-naUkc2{2h*GEv)TlD6x36G_(0#8* zdo6<7oCEIZ?>t@Nt!6)?8CuEOZDG?#36-mR_19yA4=dgooW;ysAdxQa7`lDdu%!iq zG{0Ul)synBy7_8C9fwo<6x;kaWajQ_kgUOqpaM{DQ+u|zXgEl$#HVJs*iH`Gn)HoX zA>(4JfT>{@`#X(MuZJS?D}j{;sfoxCSn2}O6b3lWJ&`V--SQs({*YfoA#Ar(^OnUT z*HJ2qm39W?^;(QEP!2&>1OHlbLtJq!!{ARyr{vFx7m?Edfymn|2Pg|uZq4X^JKrUL zWSR*ccWhF2eI5~^u|6TBzCZBBy$OFZVSNo$vSyptP#=kW;1vh-=E2XAzs%FJH_ww~ zHdb#urfgsjUrhH?EX@JFVb(hMW<~Uk zGdFxVjIPjiIk^35O>}i0n^_>E1yf0EE9i7ck0FJD$0v4iM%C}YzA2Zzl0BurOc3wM#+A+jDIdSxH;@A&msYy%GA4_zun@2D{76h`i^!Jjb=iYQ)|B?Pv(^TTL`6mFalXwG z*{!E5z)04STP7qscZ{gfJw%+VBoZdMrx)EnyS}$l9n5Ex&c)@bV9zZpP}LdDlCJ9; zDjM0E)LM6wq&AZ!9Sm@2FbbFMvYv#3OM7MVBSf|-s}yJQb#AS%Kc6g;fn|mRIh(Nl zjKUL%AtQa#J8AKi9&sZipZ>^{54s3M=c+O!bT9pnY}T20|64S*cZ+YK|9v+RtkJ-SjMt!h@P zzo~Vvf~=j*RpDErbr;?`ijX66hu>#(wtMEgYT;mL@p*)!+r@h7w4$VLdiGN+Y=8S2 zrb=z?#lP@CcVT8?!mHMPr)KueGr{4vL~#xu2zoo?;^k;=d1tbXI{G2OvcTFA@Xv%a z7Fw=#+_c|AN^r44M0R#2%sri&RUs>T>OxE!sl>2Y{lhS+@~s}JxZ{au&TN*!3RI~`6ZMsgN#Ed2fw|K7b*NPD#L0Vq$d3YoG`nVEy@1i)Vt<^W*H8eOn z8qy>cs=w?P{HZ#0_ZXixE~MQKqG-uBj4AbASV2sXbyKoMnnm71tHi*mE{+I}e!r_+ z+V{k5X%%4XbbZm;gNv~`9YyV(_htq8B{?p_g~{TBbEM(XQSQk2Fsa7QQ!wT%M>R%=tvN9idPG07 z$z^*OWW-e(oS8;!0Jhx5Aljnn%<*=20F@f*;ZBhRMd2T1LjSZ}&W^ft5LIgxb$Nl< zH|PD~;Om1{t}vVKqRmbf_2?tYDhP`<+bMQ|OsmIvcGNrExX*`sa{KqKc~m~MTFRK5ZrI-o-6eqe;ruQ7 zwZCSiAq=sb14Y}#sdA66QAr5mTnsbBVSn;E6}v^@IkE?fh)U4;r`CZ8N${ndU67 ziJ;@;%qvt&(JhX`niLx|mdDEkWqa7=-3;ay*<=xUnnr2wBQQuF8X6ydk2iWMEEVns z&>Xam|6Au#fHofRaZG+)O?6)d=XZ~%vKq7isM{+~U}=#g6n>6ayQBw&gAkX^3k?F} ze33g)GP%jQ$_7)ZfqvphCaZUAg{MG`k)~9L=~@x84s-;xwbN{EGIkG;-txKyetuwk zuL%jR|JSIGyAzYhbKAkCYi@|Xjp#~_s%HMh(i7@%gN~cr8oti=VfWRpC3JO)Y{_K* zT_p8W7witsRE|=C0qME+^hT)+H?5ciYy`mXuN2${S9OJfAWkT>ReqhaGU3@3{S~ZV z?$gz;ii-34Yl3h-&Hw+Fts1KWXjn816{@h(e*114=Lc{kZIPu`kSWAutlxNm7n6-} z&CuX33EB+}50KA@2-21)^*5XE5TKy+ZJ}&I=__;-J_eOJGmKkk0?Y`@a$kR9c62sL*qI6?&9Ee{j;MnyI|mS|H%i)?T7 zdXY{{PV)eI?AzK|LTM#Xlai8cq-VeJ?y$G}PWH8vH|{|+?fVGCMvQ zwPj*WPRSn3jx!hengzvwHoEWud78+*Dju~FHj(-)R-)l`3-TUFkbLbGc9lQPy9~I< zT3{B<&qBXzRf&Q0hM?IrG-a{&IjYT-Z7K6%#5ZW)PGM+;#kCkOe_Qr*5qZbMTIDqb zvOl~X9c{mD1{R$yAxIWaY0~mvgEf*BXUQfX=m2ax)sB1g6vGPFU#lfh>Ok72Ae$+& zmB##_&eF8UU8eR%l zoZK{%N%Un+8#r8W*|iRg4Nf5Vd6vxAc0h2AT7JH3*#vIxu84w4Sgcxp(KZBIw!Ry5@L&iMO7s z)M4e?iF>5#=Hc!ihKchMPf;H3}-)NptatjZ~M* zyX70?#6;qX|uhie{*k7;@&Ab?l0_-@cy8yBkf-`Gqk3^U(erTm6T2g zH9n|pQ|Oi2+Bsib$*i^2XFaJI-a%SjtWHg9jt}boHKU2gOomI;VQyFco30`LJA)V0 zuXkBPCaaon;jaZdO#Fn#UmJ06>j;$jY49V>-1*PRY7R^1iB`*BVCS7{QS!g7e>{U-AkB3=T!5~=6CLjE zdF*a*!UQ(K$DldFXjtxv8%}Bimfi|lp{cB~A8;pw1G_ zS@NttLy#S;P%zNHNv*cDhrZEM8IlBtdzqByu);r1xS03SHt(;VNP}MfzpF*Hb}v0U zyR|5|pv}8>Zx21w5vR37C`wH9d@{je-WRur_3dJ|(Lhpizr5h->(Yl{THctS73yEZ zSpm})z6{1$$Q2J31tp&R(2;r6*XPQ9Itya&nxK9@JB{Q0rM*szb# zES>{E>DLRBo7b>&-oJ(jTsIU*TrBhoblWF? zfy3FqEA1ZODN5@)!KTRL#oaEnaA?y^}Nw3Be482!pdLAH+oEQ z(wZ4?$w5P?AkO|=CX?0F%1ONbLh-rCx4v&0{v-vh_9KG2nHrqxJ}hx!iwXFmB(ZyJ zdqjesyg;?XlFM!+=Hh!34CR#v-Ed(mq3zIrNke)h}*Y966@ z_{YSKyfi6eIQ;*u5SM?P>jM_2q_CElxt?=FONG~>YVAMa%UJaW{Z*2Irj*_3ztt}f zP*p|(e}0MgU)UnCCkyy-X{u;Hi|cp7+rJJ(9lo3ZToz2Op0$RhAZH^Te zhGu(QV;V7H9b4f7&^Ralc1KOd_4_C1H$c+AqkEn|BRgu#A*fZx=v4i7+IyYR+>f5Z z{w$vp#L@TFI|+JRAALQsK6AB^&(i<||F?iRfiPe%we;^p(KL+XvT!cm0-tiv&qS!A z4;?oG$VT_v?t;uW``uDZk`4Z}U~Nm0UW^-YmN^=%EhZnQ|8jTy#lmp{z0<{GKq-}}G&~^AXvjfPgYg@BQd40~A;ds<$(qO))SBHQl%lsgn`W7nk z=+ZCG%AOW*Ym7aWegEp!BK;f``m{<(9Us9`ep0BsuR85sW)zg`;VTZ3GulSL1A%Tc zZDZ@`5Z$!G{;te_4+s9sr~2c^q39b?GP(hi11G*-%GMHvkZZpIMUdCfcZa|7L;()Z z#GY}I-8kGaW?~lstCJMZ^h`C!b$*v=c-0uJjRq?{Gw)+>o3I-vg?N6G_y&5TV5vXS zP;9ggu1DD_61K)i-69*|m}hs+j4PmNa80(W3uOKmZ+V!tV=MPtFPcTm5lZ21l#`ne zm;<-k(1}oJP)u;U=b#V^r^phyVPtL)Uzvzt-}tKs4L1zuS=D%1@jqD_Te+XaG8&A; zplLHbIcmVQIZ*MO`131%wfG5+IgGc9N~i2pi`gp}q4qkVmbo}K4HZHN87*dwPlS`g z!g*9R-YE_4XQ#r}9NYFC-UQ$jkrsG?F#BXFac$C(itO+Z zjG&-lkrhcJ4USLU?RhU9<)GkiUf!J-+fWYUr2umsiJkm)5c*UeqPrm%!!ce0NSA&Xb=C;MmCb+TN*&Z@oERv-zY1 z^ygXjGiv*vKr2{R8`ORPm|@uyyo?tUKoEp*t;vqg#@`IjpSdJS+yM`;UlRS#IY}|@ z$MEs%v9v%+-(I?i=J}bOZjJ+EL3=C0xOL^+t#Z$%eJ>ILct2Bb-068i%Hld4@4)}Z z)|G%mo&SGblx)(LQlgZ|ol1>sI>=T?NscfoN69&Ij4(>T}1}WVG?a2C+%#F{83Vqv? zfP4H348%TC7{X7@K3LT$a-b8sVijILC%Sgjmd?x>yT0#$kz2yvP02k>Z?O}kBy9?g zdSejwnm=AKHb946<^=o_O6uZ%Y=Hl@YuDBnFHB%gGjK>FDE6Y~za0oaeZ2&P>t3g~ zVV3#qcXCHr8iZUORTecG8bqCzCwY0w;K?a&3P z0;y?NVOX7L6sYZrcv{=8GJ`~ITXJ@}pSS^`^ONT`%xMDY6dtzeP7w1nqQTe-=`?{1 zrI7)Q^!ykkn(&&~Xb_094Px+6c72Tr$y0b6&qjw^+Brn$?FN1+tHm{3pbn~ zrHTA9N}V7LQxPNRxk=`&XuNf#DZI5y&fgg3#F?)G$EIWbMM=^G0~b$l@3Hw}N-A{I zjuL$$m-P5@+8i(21 z0*=;43%(sZ6`2oT36cwOVfq%h^N}(tp|N|5Dg!tQqP&H=YH6|4oUv!sQpGFRI|?Ye zyHkc%jKwX1hIe%0TEP62)^-|=h5irG9h>VaV&bwV-Yonl|~j8P&I^X7BgQ>~X#C%Xmygc~cQ1yKya}ws!p2VqAq) zT440<1UAo$-cSUW0qtJq==hHHD#+82E1RCWJAH2jA$QV zMrle$RVAhM%Cqz|AW)#OMDD1uv;R@sHO{ot&9&4`^I!&Dz<2RPLCJjm8K`$d@T>vq zov!{vnDiAbcAXUsm!zBnFcv5p9Q|jZHk&D{<(8HCm2C7x{iO|@+&E7Z6x@713Vl&b z&6aiWLd7z2xlNPWBNK*WO4`dxo|=PTjQiXkT2@`JL(I_clD@RDnuA#upqG#+)@p@0 z6Ao(KGrn5GoYdY{%m!^+(r5iHzXPh1mrBV)wrgaOW3(m^+UHV?;97J)6`o+Y=I_p{B=%m*5)gRjR=1QaD&-AT zxn5=C;}Ph>IWdFZWQc*LLPAj>uS9g{3Uj(k)*n77@lLs})~TT6oSq&pVgU#c4sT)s zu8g)>p*#4s#Z`xCvD=GjMc97vbGNc8DurI{ z418uNcyZLJ_?9MJo|i6BLsG?+?%}!7P2)$;w=vjNlE3+Mn4^Z7d+LU0x*PyX0-8EY z@>Y^v*@3Sa>KWBH8)8p?8O2t_P-v_=^WPH^mNkFBnTl4c=fmVy;wfzvp}YK2r|DMA zjRKa6U)TFT9}h|K_lnq_*N^wzh->ti!wPsGOmW8gYn(r)ZeeSa{Ji{Xu5K5qJkSr( zfr_;sN9D{GHWAI^MnMp|NzBO91N1E1MbH9{_yhxAHVSU!VD*|p+|bms?SEnhs72yZ@=GtuijO!dU$@jS+a8Kt@tdK}Cs)u{$&4#Wc*vqsib z1r@2KshLbSavZ93je`{9j?mJ&tEH0KEu$DWY32;0dYL%Gua04Ms$A#1u2gZr{!ufk z==TeU{CubwS*!hlIw@IRb80!am2-UW*BL;C{i_(k@Zl;&rk{pBy<(y^{AP}*EsPX9 zy)x#7N|I1#4&1m^wjY8S0TpK(Cjd4qIwEnR#e?CxC?n((lz{2T@HGbas_~#E1v+x9 zllN711J~kExqZ#p6cxqwm64gl=kbU#G9<1X{3RoPp@wERP-wAcx2jM);VSB{RHE<1 zB_!`P$AE7W=+qR{duGj6sw)R^xgzyrrK;!u-XQ0fZ_?kO-y$B z$yOGY(s>dQDY@Xgw1{=gJW!V#^M?j;ahW%f#W(_ztis{`7e0z+E|$ z6#5I5e^o+h>>O9?LLMWxUazuMmC4P1do%7zQ}&@ij=u5sN-O%!Q4Bz2o_^dHD&?aS zl-#rZP1(l^wbl1S58C?;wOUk>+cPE>Tz@X89ZaqrWPknIs3vofPy3?9izL(!`Of9! zfVswV=0NL$qh5VzHRr3Zn3w^6?4!H>xSNSuNC03i*8Tgse9cU7sPW|$w12fjh11$b z#YNHdD`gb1F~l zCJ4Un+;hOoY%R)tbe)~0TPQ(TRTX2DOsB`J{ozHUX3F_uJ=+&p-zci!a%vt8(;n5l5Mge;F zV*kE?gJcj0uhy;xxbGFG@dJDjO{O0PDIJ%Q|ubXtqRWj7}}Yn;VeCA)D4uLblt zoCJ4*)fZKO5oZ~lOk4uRHI9YO7%iXDrW)ej=@Juq6dD3?9fR1S7>UF7nM(20>QT>V zYkwZXdHnEn`i-c7OFTWrwIN!y8V=`%_X6Gc8M3ngJ!^$ac=Vr&WctpF1eD9+2sM;fh5 z3k6(mn|q(C?pbNR>Q#)u(A`*??>3EMv-V@0$1+%8&G@YyR}?rmOUhBJ-rA!G(GPbJ*_sAS0A%~{G^iLp0K5s|__FA$4<)UX zt1E0Q(a7OVf5L-UYUV)8!Ydjt<9>^^@apDaOM1Y(`byuP5<6V{TyEx*LkcNZzH3t_ zIp-0>2VZhEhEFh2NADyuC2m%!vVB_|7W!RdIK+T-r<%LX@fUi#I}Z13nKQ=upo#50 zD!lY^8!M9XpCapDa<2UFeZwDuE1)XW=zo6nai>|5P&9Q#OiK3bZH@+a03lf3Z8RZV zb26n?T~Ax4MHpW~6H)~kx%{6%vke&hxZlI})};zGt>1dkkhps5Qs0EZk}n7P++}21 z($fM*%@POii%2?ty_=-HIzDO$Yr-ydYGgc`G}|sOPyDY4Qu@G>Uc@##IrE=bJonqu zdt(DSJI5wEAxHgB2F?Vz=6a-&Pd(rAu;hM^EHR~|WO9reRCw|>3IHnfMb4`Wh*0uq zqkw+nq^DHW>q))-MWOe1AAP_FUi{{cWTPY&0$_JW*e21he*>Ky6x_>vtGdr7yJw7? zeSy5U0j=7-x@l3Tva3=*6D3`U@ZlR4v3^94G6L93*slW(1qvp3^U_WeaoqZ8#0Sby zcPNuTn$bd9C=*$&5*kO}oK>M9DO-7>e7#P~L4+wi$f@Qj^2od){07fCX`TVE4LwO! zwM`h?6JI=03!&)cS4#Rbs2SDy;CMx6|V70xX!aJ5Z+`BMG zb8w473Wy&b3uSnpFBxTw)9G{r!%$lC8G?GEk(dp8Uyl0nFMUJZXCJ#8jHM}<1+OnV zBN&TLk22qd()U1VzxE_=YZr$@O$K;6T1qZoowjIJ0Mb7GW1zqx-1@Ei3|};woWso= zSQdSRoL4Mac;2i=g(pS1KodFTFtF}j-ggWTkC>KzOgnJ%o=qTogux)WyFSH$Z}NEc$A;x)e~56Pr!&31f%(X( zJLH-}+~G=*VdYTf7X_ZN`R{RQ-cI`&fwup6Yt3K~%>LC7jW3oFpnhKCpDn?&9Ui-Y zifwWvm;7$NV$}DJi8{RRhAjhu2uBy|lp14>vEA^SN@qt>jmr5oGNTi{q(uSRcbme* z4T_&vKsbc7!=BeX>#;WcfDU-ir^BvX#W~M&%pSL9_vZ!d2NRDzTDWGHyz_PMxD(zQ zG(XxH+b7>y7>lfgOKzz+NSkM0%=zAAt6Anpg5QjN`U+dGldDxraU|BBsy zthTMfa?LnxD1&IOZ;{kNhP*3eEs+}mKZ!qeYDcHnpa zuUHiALi&(kqJ9bc?)N_1cZd=OFu5Cpd1ilurulTeWL+MyDZ;rN2!=KChQL$|!A$ba)!`ZK8wZMHd@%#mz&8kw!S1PhrVFy^QaUW&M9*uS=mi(>7L$YFf2C&rq zK5#pi{8^f!H*TzkYPcgIjCt0ZuB5soJetF%c@|Ep{Lj~i2DPAk!9QQ$qAE8f>_nhO zhdGS~6uLejRNaU~%p&EvO6yatJEl5_ zt2tijLfvI+P-iUKKTh^eWw$_cb)DbBX*J!BO?UQV+k2H89v4?2Fm1 z9n_13b|kUFO=2k}lfmdrIND@=inI`4_nLxHz3Q=-w;&Nod03)Uu{(o+Y~ItPOjfSF z+Mc70NB*#m?-}dv{>pHxrsuLS3q!nF-Y2|-=RBH6Tqr=Z!8V7K#BMw$wl{`TTUk}m zBbGUjwN9bCGkhZdr;8I9(1bdLA<>e(HjrZv9d-4`I7`2O^$H71$>2BWr$4V9&O7_O zX$>#z@DYINAqExc0dE2G#r6CgQDGs_( zg~CeiC;d>;y!nb@Mj%GDf@YWMA&K4@(%2&&-aKIn#hQ)n3kP;3VyXyh?j~V6`)%uA z>eqX0n(SP#s;@mHK8SRy!LkFMa5JKt>8> z{#$y$ce}@9Bb@v62H|*Um z`IM?=Njh=PIiX@*@C~A3hNKe+S^r@Q&5c2`5AcX$Av`9Pt;L&dEp@M66^n^<-y4=X z)I($aW3v12V|z#=+hS8ocGq{?0**U3V+p}nh~=3NMr>Jnm0Tpy?(#I6q%OM49EX;1NmM=1_)yP3nT;3BBP&M3LBO+H9)jea!02h}zMt_PUmJ#Idt%j<59dcOj z0Z8J-Cx?%>U2QFVuHctS^eWe0L!NAV5@3V0<+tpxmc!INtWGwogeMs6{jW&RK&%it z&>x;)7R+W2KOm}U%m-F6n1RVaDSp`l0-m`2v1uOadRJz0MEU)#-Zet9RoqyjZW3>6 zD2nF4M}g^SBHN%3JgW15z&nw>fQVIrO5Hanj~LxAtco;U>JnAqmSYZ+WG(K1ilM5~ zT2~?Ca2}%2skE;crs^z%J565Td71FT5TY?(^v4)f$8b0N<5h>KA-3+ zx=UKxpfW(BZA?F4ru)NK%yGSE&sWNh!WI!XyKQE*X4~k5BvmOYP}i~WKnk0lh{M@$vXjS4Ey6?C7S*IyCSehu zjd0?Pda7L`Xonc{3XTGq>lyyH{Hqp#zk#%n(f?<&jO7`QhTDqS`5I-sLl~5&=8?NQ zVY`Ux2+P&I`nSjsm2zQUc9t?sTX57^F~xaNDRHJyJNZ<(($nYS5=!a1y8M(DR~f0d zBcDRevl*2UpZGPYeodKt9GhAkjfd|^KF(@l6kx4gN~Fi%J3E&aH6y&Yj0~bw`Q~>) zMhA6DZ{#MrL~1Kk257jWW|Itx(wUgdMN-#sz-AXfB9>4ZT z8M<*z@zaEbxs1q13wNqHzPuq`vr_E*xMjWly20m%3!g~plAJBy<(*=Br*cY+dI+x1 z1JD`f?Xjw&jb9H{)Pk>Y^*ExER^GVOjg?e6Zp zxZ1D$dNl87hyn>omjw4=JtrMiWwH>ojvv|ql32t!j_7Vajp`G2 z{JRc|Ke&iPWvkYX3-g*gn6pXwT?heS%!4wYvv-9U?MSV>s3^QkHLr&~*mkthdiX;!U(9CYw&+egYFA~vF7OIo%t4cgd_oj` zUMajmBGWEqonZ^tJ3Y=@JgVZgCuLk=`31AmlaYi3Dr>lt=sKT980DXW!$HGLxa~y* zT%z-CHZJy(le4X8dH=0H-D`Gxxu@F{Ya%H%)b3w$x>E4esc?GUwNcX)hr7kerg~?D zCH$3+1ME78uC=%8oOb;5=ch4HgV=JT@Ed0ekco|Y4-aHKw+x;>$Dp4rwZr-!klH)( zxv#0AIIeBn`^?nrmG_$9{Em2(_+bMClk87vP7ouC?sAeZZ<`&4NQ#XDvj}X)oEew@ zCI6jwhB+YfdGRfzmf@ke!^0x+${BYWJ8`##`JO4U7t8H*G(f^V*xFlNrWslBy5{q^ z^O{BDC`pcvx6SCPIKoB&RW;^$w4A#UxS0N$c0ZPIVPokCn<5%sML6r~dQ1sq>( zS-L>C&JDD{ys-KaU+4f}q5wcH{~GBj?;rMCk)!x5kR zvvAvu8dNN_NOC5}DD)kSnL^BT5@AS0*6CnU$LrDag3-)eFFHlQZcC+VV!(HT_0At8 zX0U=9pe4(@cYpYm3%TIa5FbV||2(I_493dVm1A0RO4r>yUm)TYRLNJ;rCINvX8I3P zE5PSrcL~m)HtiaYkJPYocrfF7Q-is?0jlE)O6N7#*|#$%yGpuUj_LU}dD$pMYItnQ z%=nkpV1U{!rvz**uaDMqpo5N|1WzO4-bNszjsKW)J*k`!ZReQ5D$w8-8+d1?#@9Jf zh(vBa*E{5QP{1GRKJ6FW@j9=U>@Tjtiol8MxYyaN++}*qyh`=8i83{gL#TY8*Oli< zG|;6ooHx-cU?r+l6I0azm2~)N-wc%AiYZTecnBMyE-JfGvZhLvev>S5Mk?{yjPb5sXRY0p1mYz=SEA}~bdeMnf$ zXhFOqD&JW;vpQvE>KbE?Kbx~JTe+5Y=1{(cgp$>ojkJlpn?< z;rcsnEM-5eWnX-2>F@;YE5mHC-zct?CE^;lf2D7Grd7I8BWFd8Z-HWFDknyyXN!SnhNC%N zFP8IJl)Q^*idxe$d8zwVUC*utCW@pw+Hr)o18SE5s%uE&_Y}ZSrHepmj0dAMMTnKn zTzgT)k{~4UIWWf5Ya%Rq#X)I)uUWjZ_l0Pq$tO8brmo%N+LX{K!f-sOqrMY``QmaY z&`4NR&Pi<_uYXJ7#|Gx}{uQxa=!(aKpCt)8iy}X@m|tnpJw)HyVWYrGVpGMa5ur=3 zF+`}}3@pmRKXmVCIrWx5?ktfH{V!)FtJA@^t`X;>v@DQb@ymgG26A5IlIL1z*@88U z@l)v&6bH-5Xs3^R>oUw6Gc9h%LI~BE+bMSi)(#0n70N=$#M8pH2UA>8cT-Q&g*p{s zVz?QrdDyJ$NTWc_3s{=(Ej&N-Gv_9d%!n_?1!vU*EIfX5?Q}R;rk)J^+vAB>XX_IXcTwfU{Ek66B;(U*03gLrIW*oNRw&K#Oo5GylB zrIS5{L9l>xkRX0}-dl}Cr|8GBxTm^@nfqBeg-Znm#NDxu9 z)V(khk!nTUbRJDw0ZFQhaU^_@RQ%U17;|GgZN&V^&pF))CZ<`^f7|QByAK}#22llH zQ&{zEM3!{S807)h{P%k+Lb{ION=I{xuzQ?X{i>KKi5YSUeoXiLJtuERCtP8CPXgOg zV2ML;d2ee|;Sr1R(0)1M0|`6vT@G6r?B`IX;n^A$yNOzqli_jnON`t#gm&o!x+&vx z%*6A}q5>9oDgw03L9$?rMpdp^gdwOV;~680!_ zbZDy4CHzeQcf$rlV>XAX@_XuiL-k;*-PeQt+}@`cS*C5D_-NxRJhnfDU~U=G_?W?D z$M)ZBq@9cRG`7UJLrA4{MPHDZC({nAiG(&CQIp6V2h)lBz6;#I0{!}-27SUy7Ojma zl0O5ayk0#}6%ZH)60CZeP1g{M$@au1AIgJt3L%1@qc*!RmZfyK3<3-wB>?lwTE6tA zc;zO-n7p%_T>nPBmgz>d%Z2EgLdwL5cO3(jPT`tb&YW~1(rSon$K1f>hLURW#73sL zhLX1vgc+J#Pj)~$=HZkZ-YPopf-e7uQNSRD#bQC%90O#|`YpphB)xYP-6u?mL@cN{HmzxiT9 z6su($AN(m-dBq##iZNGs& zu%LA!sW~uc908aCUi;qVgAwq{h~T}ftyNu66}!%(<(Xr+V=oID7J(nI(h~4hBx=f% zOl|*EgWY@Wr%7w2J-m=}R_vbB9js-%nuigx!a+*1MZi<=56YLmARnm!6@T77?#0q-RaZ5fTFOOp zHQdoShh9^Y8aT&{AeRNxPvgLRGKe2qCScyE_GbKYU_HB<{n>0jN1xEarPN%OvVj^D z&|8Qojf9Cgp!U_@ba`*&lNXNUOlvT5c_$Jm0cqkJ1X~_FlF;r2%ix zP6)T{>9Ab$#PPL4NWR41iVba>2LNgw8M}t^usv&n^LcitetvwQ>i1ap%_bn_*$}ks zHwmR??uvZVFHv5VH9oKJxJ`i~`JiP)3%-oMpk=x=c*la)R-d<-en&>Kn134%9#qJ=}_53(5k#4I^(aTHJnYkN(~r5&3e zmeVH$>Jhh0j$7(gZ1u!qr>lV}k@&6Ufg`Txw13yu^1hU<6Bwn=K(ixmz6_UFxkvor z6SLA>o8NTnq?s)d2s-qIK=Ir!MT|379pR8APnG9-?(MrKS#{Nl*NJAw@V@dkwarN{ zl<>y!464C6S9+lsKhEP+EBY6dLU096`NL$QhEr$K=v|&wfw2mnY6e}> z2&)sfGuWs!*ABZGiEOBsLB=Wm@PI{)aT~XBWniLdbV5&s8Y7`(H&y*F@doDZS9Hb8 zTB1&*)t4RZypO)f-LmCG8oI?DVd5-%ozIT6=AF15)2#xXKtazd!H&+qzIrfnke5g- zB=y6!2AA0}ENuy`-zEI5Z!0A_6792qt9S@TBi7S08Fde@g=iZ4%RWf zDa|2`?xRa+SzVYV0%|ZM?z`I|)guT=6h?wzClU;~|M#$#7T1r8Ck&e$&-}AeOZ#?#K7P)Yp@U z+BPxK{7#w9blob|A{8Ue;b8*JrmFgu<7~%r2OtK8hc~N{ubwxZM_dg=BUS}4&`Gp9gVKE$!N(| z`g84;c0Q#wakPprd{c76tl5eQl1?0f$m(3}8Jnjqxk<6yOVPNgp91Pyivx2(ZM1{5ChYlZt+kbr4Zcnuil3gWw zZUj)=Ojnz`r3X^xd^Pc@IaR5`Ws(}Qv6_o;Prv}DmVyMHhKV zeMC}eS4F1F%Vgh4#RgR^KSC%PBn$!Vx}FzjWz6Q)_1}?}O7O~kOAgRF-N?N({rt*u z4a>G6_4{rRx!382#*6XC90Y;L;XhUa3WuH@QR|TTF7@0{Jb9z>7m$;In-%bA$P8Z5 zFYIrzA!JW zQ#GWlp+QU)qjH2Op=7&eixY_DdG_lj2Scf%GId77lP{k_^VdUJGFn;1MTqVl5WMvn zl9sG=Aw#-FYWsbM%+xQ6Kke5QKdvbjNcI%)-6T0sPQuP#T)-y%Q#7#<4 zQqo$urlsMiKAK)Qb{Kj%R!Et{8VD3s=6l4qnqYw6X;H`0Irk)EZ4bNy2+sgwmk7i35&LJYa zD*ERTZF1_8a#j+?W*rnu#pwo+WNrS1b%5y{K4DtW!YD);lP8XCu0+ zP6ImVAm&KuQHsP8>X!lVKn=CI;ObSDEk!osX` zwlED-8joIEHa)^BIRiXKRhW?MYP)!irY^;Xvz{9czi!sROBvxWDZIJb$gLYDWgCWW zNRCpJYGM&m9Fu!;qFC_k$Yu&$yTYP)(O>PEiGmO*JnLp17;W;;|I?;`SgKV9Y*=n` zQQ`=Vu2(;P`Q2PdEG>!FRQHN1R9C<;^$58#vb-(C6ZP7@5~=Q_orr1a-s zV)V``h~a1JK8%5Dj&^-naFcyfBAHRDsGi|`9?ca~eOx@f-B-Kp9Th%~lw zL3h&nhNj9M4X@~Zku4PKb={27*;g@f`uHo=@v>+-M4?!K%^X6^B38Y1|IGJ&Eb@DB zEj7m&6^?@AV8>hB#ixXUM0&JcpEzZGwOy z5O`$%p-EqHeDA@Go_6n~^f;xz)^rvRE}=b9Pak@6Vt@PQm&r3b0YLUS3K8JA1UenI zVmL`uAmtM3YfP=OK?F6&iu<(=q-X;(>K_wsRBKb7L1wgYksAjL?FO2SakDp{{2J9Q6?`k$$_G};^N?Ko+j@Y#s!uHhO6g3s4?T%cL; zdG;5O0YiWYv%MyNdasCCbeuz?CFGF(T=C6s8XSI-6nZoVsg%ojn1sX6$fptQt={pa zR~@ER{K#1Md(=$?oy&Vma2;jz1R1H0=s?uN9B39b;{AS~0ek?br-!~8b)V&;fyh<*? zNHyo9RScJX*3utmFu;bR(Ie6YyWj&ssyTDWImY4r7mE$ioX=vpVYTL{&tkh#?HTuDS_lR5-*iQ3EO zdEG{@GHSA)U$sM36SUzX09k0BqX`@FYuPODZjy^|@3tDYjQzqNWiS=^Lk0+({e?#l zmb<)@v<=wUJM1{Jz2ijNjTc6%k_%6=zJC2$^f7KAF#7=nax3{OlT<@K#w9@D$a>#3 zcS#)FRbl^Xo5>-~v{02C!U6lw5QZ|bEi+(8dD0@RzSb^*le(9xr=?cZZZ^Y zz4crRU9}&?wWsSLXzAJe)@rcp^ZkB^w^Y^N<_6sgjvd_i*=br64dOzcSGe32vu zStfk|ZvV=ALH>5ReLIwk82nq!VWnaQKM89w5tg6pEOwJ@BFKJj?W@-PKTA>c7%X8b z>edG_;tp2V-aH*x>-NwVcIAqGd{FN^dru&eVXRMRl1a2q5{eljVX9@jjeE^3Z+4Dv zT0)~gayfn%h^;RBW=AH?hExr643uFa$$HhOQb)Tdk_N3*`pEsNj8d z>z|A9xr;fm8-qVQh3geE;H_sK6iVchGqX)LAkocp^GfvK{t;$sYcB58N7sqABs6J31y2JJ33-LJv~oPop34o;u20GCm+qfqM3RM zrCs^yZJkGy6x>;E^-p)!_6bQCrVCkg7V4&aDEazk&Xq=OTevAhThLfWp6>hYX=RVw zHKSEc&1TwwGMB}O$U-)LkWUJ|)dag(^a$jG!w~RKnOJQtXIoqI-}kx)%~041!A9{0 zaiHXn9O=aR>}@^5f0;?Y2so5h@;>+@xwJCkLx1uc9QlvADjJ5icPv0Ns!v!gu;@mD zImu#ja$+2g7=Fen!XYS7Eetx5i|@YX`Pa+L&hgXM7ZYRjS1Qy+?S`fnk^KET)au_w z8Bn+aA)Rnx z7$$@r+(FHF5oNh%ms6Q!FgDo`TA(nu10T9eK<65!N(y^)R%YRA=*x-r_OP5&8+k#2 z$Q%oq<#B_dv9gH)kj4N(INuUW|NYR5KOkysAoVXRpST25A;N>!3Kv~i^4|48pw7u7 zm8&Pjch=QETpfAN$u#gBqB$KD6pFoXIC7XFv}iwZIG6LpfdzL**G_)4qWG`3sb|4n zg#Jglztn@rcy#6Vb@f!Ua}t%-Ys_2;&6;aOP0n?ER>JJlqCo4z;)t;}YHA}|r?AUN zC=gvsK#9UV+fC(daAk9yY**YH5-tO+Qt-sSO?m;0<9a0$lkCxN-@6#-9y1oM@G-k} zo)e6*^`Q0uM!;Gkw-ZS=E{PeMl6mP{`h*J;KL~^sId2^pUkciK$IMcl^ zHaBPB+Z&K0^<}{^Sxl&eWI=!D**@Q=k|tk9&ff3UKOHsn)`uv`?}H4^j%;_>CX@W) z5SXryK^9$y-s?qvQF9UKuHjN1++1}0CKw4mR`*D%rEpf89pAKM{AQ)XMYz*dskYCw zIo9Krmjx0VeYF61$Kuf22!C6_-0A%N#CwDT@X7n4?+^38aD61n@}*GR-po`8i)j*8 zC}|{bGz6G)4f6yOZaz*~7j7*qk4_ZI zP?QK(makuvlFygqi4z_e?Vm_-hBRmk;v#c~OmqTa6<>(DOHHuU2HDQB=-$0(D=Fks z_rxEj(6u0zOm(h*S&<6m24s%3yzDfWWmV6p3#{Kwh*TKk)MO2FS`lpDVw7{L^q!r% z02=6{N9QQ(&yo-uPw~nY_&|S<#SQ!UBzhZiM%!-u=_4W8_Jg|Xw`%I_H&eU1PtKJZ#`%<#jA|b!vl5@p>2XirrYJ~LZC9*(Wlg_7@eH)vZ_S;q>RO? zLt-%DV1R+`q;XmiOZnG@m=23IyDEgmgU8C;Oi;W&?yqMZe~nd6Oa%cm8fXg0#{1z3 zb_JkEW7^qU=hr75gb{qhiNpBTPii?|LpcXrV=tX}Bl3%T+Rc!+6ouy0x%DT;Cy`+o z_18$f*Ql%I6I?3bm$`&M`<1im-gV?uqc5xZOlzY9aJs(>xx2(74n2HOM$wiL(0`pL z8y^zx8^;*}O%L3(ON+P+g-U+)W)mKJc$mh@2!%(VuqEY`nnW-tE6+l?bQ=;eZ<5F3 zp9p^6et7@%F<|P12e$ES3#hcCV20wk?DJpW2EPdJ&B0|>d{nzaYYsTk;`&P~>x9Mz z`~5jbeqWdD9@QUN>P7zWDWHNgW3OR0KIM&Uruv;ZcLBC3dz|XW86VG^b0zT*0qVDK zaVU;ElBB)=;EP}NYjTtrm;G+%p4*URoff^db4|muC1g9)Q@_sqz}qoGA+?z>at3Dp zjqUirf}21&FWzUni2#KpObEL5_pGUtfjmIACc=en`Mih>&LqD_5S(yL0_rZNX0 z5oO>!+h6zP%!4fHdz{nHxrdmAripbrT?{*hS+?lt@nANWFkcCBE18RL5Wf^*>%+E` zQ_8&x{?I_lC!_|$s^ItX{wBCFxTp)BIqmFE^Cw}h$|(Wg1cQWdow>kgBE}Gf4Vo5^Yd*BPti`CcH~r1-m2)=O?mOK;A8o!iT#KZZ0T(Nm#$$ z6}Wha;h`+~19=}cOm81TLj1A<4&T+@+yhV+GVlJuz>b%(hjmXe+2^NPE#sXG+^=hF zuO@Vr!AZN;0pXzCZ4}gldj*7>!TK*R=FF8JaErm)`Oi{M#NRLx1aOiRRMwIICSknF3~} z&dUU1Pu`A(EW7HeW|p)hG6o2&ZxGW%66m!RMDY=2VXX?!hax*iPp36s_YDkM_-xiT z=3xmYuuzt#D*oW|y;>2BKsLS5pS0>#Zwu!L#XOLkTNLfFcfDNe!pImsoAar6!SUl0 zL`n-g*(JSF`2J4wvECF(p;1mt9X!a!luN9%!S)7qaK`d(zV3H-FP+!2eMT)CTd-%B zy=P3&h8)eeW`uC|B{j)3LX}zq)!*&dEqB6d?L4msf(pi<{uJbO)AUVn;udg~eBAJ~ zmO+Qik&M=ORZ)Po*9!!UQ4~Td%#910cRg<6rMLGd=V+>)^oT1^ z=SP%zR##(-&zoAGIxpKTy6=EJ`N)ly2@PtL@Gea2HL1kN7~COtJ80iU2{AUNss{rw zQ*L6dDrkU0@@#J<9qAtF;y)mgl2gN989 zAm3hR%^U+i}s%?@B@?MCtRDT zz&gx=wjE0%|6uXCiI7#2y|N1LiNT2Z9P;lTu&E=VAE3dibun#);pMe%YZ1U;H z4eyocOUIkLhs<%h5SG3@vr_RDliN51-g<%Hi?n(edW+$H)}Z1St6xYk?2lQpMZQj3 z;`8nZZOwM1>^vZP!$SvQm!>p}*oD{Q4}=t$TdpypF>(*ZO)RZY!U!=-Hq&p$)32vF ztW^x5NZ1JmOGgjZvnDVaW?xRh#IQkoi_id@Rl5ic492>`l?`h{`n{EH&@<$y?(~zwU=h9 zQQOlAe5c3}Hx5_H`V!=IPrQ6z_5Pk6mdvd=-lnMTn`+uvySbTopuJ7$ifPx3(@8_G z3YpBx^ObIZefBN}U;9*Wo99m+NIr<;%Zc?CuOaX)3pScyyO-+Vp6)+-x5w++G0o-u zqlNd=APUB3N9z$)p`z4|3tNy7u=)?UbPhYuen26F!sZIBEnX=@a>qY;jP310zJ#7B zQjF2vh6`B9SA9yMkqLE06uQujvGYNJv;}6Mqj>>lt8EKU6m8G%?p|V1k9zrBXEAZ( z>2Ys7GQ3Aj3>;l$m7u(w<@_^?rna^gS1*%~;A<+_sslFZRA~`GF!KU^1yllHt;c|c zM$|=2-g+4G{lgQS21Cedp4n$UmycuO4wlo%~6nKF(358sCOhEBXG6^ z%~4&5aId~IOTm|a+@DllmiSWEE~R(sGS%!xLC@_QnIc$xaw~PQ+7a-wgu3(?a?KvQ zRdi5bZ#n+;o55^K373zNt{RJNxKO(PH zrni`hE^_NkakkT(y=UAb5-w0d-@znQ&bRQ;2mR)a>Sg{4@bFN39?*jurGIBq6oXc% zI>JJ`rFdSz{n29V?AbhNUFH96HehT>LdS6KZkuksxGydWFyhC!yv%{T&MIKz&3zq{ zRkQZcWXiy+u3{$0+?|i^EAXICu})y1qOf>b3p~)Tv5^h>$P-jdHkiv*BMt zy@cYBl`QHEBgC1v2(I?ZctrrX;J^=HL(@X4R$8X1Bl=hMHT0KS8Hj`x2s?^lABDLN zV4Dsz+{Cuh3wr4`tRlZGAIFWsj@uS*$h%tk>SoU10%iPr`kwQIpqRPvt5(!1lhb4Q z`k?LH#8MG#>~x2sbkl_u>bTH&ptYoC^=(4-*GmQhoXu|3*Y7Z$&`;8;YhqocY}&0t z3VW741pntd^q(X7?etsH{{36qPAZ#ivh?Rgzi~T%!+b+{AW*H+rV47 zuFmC1`kLgOdNu(y$HKmVzR;X6MmI_~<1+5^hk(z30pd-26nn? z?X$L7T3s_P(NTIbh{6;~;UE(pX=-7!f5U;a6}W>)@8iBHF{xJ84+i|3;cCStXJInj5L@AJqwGfagDyxb3^T9DZxU-e|K+9yW14lHY|$c9LCwqnOG z#t8Rn{|wtvdc^jzdFC#@uyN37G6YQk`j)e32U7z-AhfJ>RdUu|R&jhDO*y_{KxF#n zp!ztMWS-2ei6majG=9LUe!Ie$q6f;c_#{?X04EA*RbO~7VsgfOi1{uUY;suOUqA{# zB}u1S<^o#Cgv6Xc&WUz1`fMcNusAs7fvGAR>T)t!2kyL&ZG;VuiL~s4N~*t?{9d{BsDDGQQ?anC6A0GWvF-wuf7K=?)(5<Tei(R~KgDwqwy<Q0}6L4F>t z1+-(!-9PW1h{lY*Pps_?eBdh?*&zcg0cnz-xtsp;+^qaltc{{BYP*rk;MEk7%-82k zq9*f^j9qeHw;VepZc)s=n@=jeQ<<-#t49_3O-1j$tdK;ASA9uR#Bu{2@Xul{ve!pB z&4mB~42*mrE*WWc51s~9+NfwWK1N9X^m=SQB*j}ssKJT6-HB$kMh~g=$p+J%fKS(U z07L?co-Rg+qZ^c9^!0h}lIx{J&3nEyjmsQNzq6`St?59Oy6D3~GhbR~#@aEvQW^9n z1Ae(?14Oh!SBt?_PALora)UT z)Y^dyWZx5DG_A9$N;aHAFx28xRuq3;MfhCWRafUMM<5elM*I(|rXZP#OuXM5tnRhU z4cd4_wVjF)Jjt6;xm%aKxzD7b@U{Nbu;1b8P@A4zVSS(2L?uzLRw=DlhGwe*yprQc zsNI<&-^EjDRiiz?81}npw6!3sa|P*+KfOD^vH4e3FlrVAV}Jim*3&5>?HT2_p{vE7 z21(;{TS~xgLlE8(v0nl~`uSmYoiv_iMDwvj1FL3V=dDqohJMMHzT*%z_|^IY za8tYT9ttLh{Hh({AlqNH_q~=8qZn3X`ji;tIkI?VWrZ9IV0>pVI#>lzvwsv~dkrQykQ{d%W646Rw7u&}Qza5BS8XX-@A}<_O7ERG9zi{~O z1;oSNVA+~|E;}I?Bfyb#N)-;)Q0M>-%uW^R7e?4a25@MS>|ECcdw;{%a)3Wb!IpLh zH@9(wkS5vD-8AUp0n-zOBIKH_k>6A~dTvYvvu0VyDf0L?B*RUtML5a zGyez`JXy!`jh|NR31%BeqjO(Gu@-i6ibj$>-)PvgLfOQjAL8C&IOIn)57uf_lv^mc@AXJ9f?k&aOuJ=8{HVM&IbiU|!*l z{dxyMlhfnltw4JC^XFb6&!B`Ek`25HzYX$6aVzHqsiqHxtqLL><>lKyKK%wr)x_mU zE$Eqa3bHA%u#&AKrg>2mU$Vie*nko_UgV)?AA~|)>bhYJI0n59_W(AKMJV6pSF2A+ z3x08$#h4&Ds?N(%^AYm4a&(Ck^VqrvW7RcV?;{t;P2-)NYfbmfmlK84AX6V2Dxohk z{z>fUUVkI`QCsX6iy@Wch1e0F&Iiy~E{O`Z1k=xWlQlB_&2DC7crH0T*KoBX`ox$u zDy=@_&(#vyDio+k!dV z;0z3J7s6xg4T4{cfX}7g1#1{EnB*6$6-C8Ay=fXpmbK?(IS^$UNG_WcDy#N75sM=Z z<_*{Xso^W8N%D!OvUu`JzK2**UC@L+mF5vKEGA-Dj^z$r!0CJeSw}#N=I-t3E>3Zz z8mevl190?1_x=ifayQW7P3`J*_lToU*Rdtbin8=^K;ImkO=rC2x__1;kiiLmw5p+_ zNC+V=xMce;c01HI_kOf$!Is7MwF(euOUr%T8rx9wbiX8-AChUb*zJ}Ckm=o5BR{4j zWKdMJ+t+5+u8KWuh$~lpkQw(mDT6P#o5Y5jYquZ6FHVmSk@F^5H5dssc>~v8DWIrM z!}tx+7WmOZV9T<+yw^J0Fn(O=)cKPq0#kAUTsh?=#b6HAtsS#z(szX%)C$WtppXRG zmga<+K)}4&aj34z5Smme8A$Kfg10sehkn^;p!9SJ|bpA=QIx4qjxm%Bl5BDwls%HrwKyMNEIv zwjgaTtD|MeLI2>pZvGXC;+<{SNIv&r2++?Th>ZX1eKr68yssLmxw6S0|CnqW*$v_C zwZ7@PrxsFZW60RwwAR)mJSyQqZs1AgqLClrXz&~^9l)vbH9uVCUoMkWqoz^QFHwxx zOIL4ZQ?uERyq2O_bHZGCNdj+SniF<=p{ zdpt+A6W?m-Wa%VzC)3OYH;%gwzfyTKhMl1iqKDr!?Ao)O*>_Z*C$KYaSzP?gr2h{h zmveWu;yeA+=76zx;b&5%2C%0;RBaHvi&uSQ>(}&KBhw_R5Ymw@abI@?e;o0lq}@H8 zR`ZvMTfjTf^NHy0svocu&X8Botp#7RSe6OgMc(?ZOs+I&Ab*FO{6O{#t&-MC$}Y&I z(!wGd_PIV5#iuamm%-s;?v>ZpA#O@TY8rOo&z_XLv>j{)M(Du4968O?2dn~vji+8A z(j29%22*15k)?Xa9=AA-pUV*^PuGrBE?NG_$8y{mDmG^-D%o z6RUm9fT>N!mNQwRjJ>T@p`o|l304w4(maXf3T!qyGpZif_C8!BHI4K6?yLR@ti`U` zlsTx0Djj$gVUqS|3HjV)Zn{8VRPl>@b*p0Xr|{r4NAHCyGKnLu(l5Q4t(0_BY&-vs zrQ&mh?eig&&yheaYiR-CDJp$au=Dv7%C0um}tT4J$;kLjF zPNrXWW!AskstewV`UrM37=h}g^L;p99^XhltW-md;ndYuiMdF(yau)uK?{dEKKJ#7 z@#sq9ZtY`Fr0-MIaG17I&*X< zjP);5GMYxuHa18edaTF@$!_hLs0(EZjf`7JB$AFKj7uD~CFo(qVf<&F0EFYYl&s$W z?Jl4$#ohYBqC)&8W45=@yR%~?v6Bb87A~IWr*&n~(GH-uej_+6^kW?&;IEc6U z+@HN4&PXyOWjim(ob8H#pQvhAFLRrCKr}@{;kjBXc2mUp@n~!P16(5}T<WAmXAR28%A3h-!7`67; z{Rj-1%*_^s!t$q0W4~Na0fPlQ$ zkwL`n_B~-PnhbZG^7!dkYhTL5;tZGQ$wz8r&nkVy@-i6!l zO(I&|+y+K9hwE_ze*%ljM(zN&K01_52lg^EGcRJ!vn!#q@vqvgP4UKvXOpU`ZMYK`Y6Xg!foR3>VSn=h&1RL+XdyQf!{Jh@-sAHqT*hc3qQGe z{C!aViQ!%NP$tT6RTi&A5Fu=c;lw0{vBo{cZ}*amp~g|<}jR77<*eR2aMfZ-R)f5O4=v? z@|$XJa^C?eO+V(VFDBOpG*Gia)5wuSd0bcT`&S(rO%!rj%FQIhyQ@Rrm~~BFh`%r& zu)s7 z+i6+coT1|vBX>yr-1eeDwf}v{-op>s3+$h{zC$YLL$Vjos2x)?Q)_sfvRYNU%y{HB zxv6&LO-HQW?&crZiEhI2%7%CU!5e?>zqRvDUGl?n__!V( z7jVn$;_7V(aU}JN`!s3_^h>tUl3hjv&Ki{D?Oe?9$_S!gi`9wn@D7Pe4 zOG@rqt%63%@7;5X+9??jzP#KM(s+zidtg0#1}?MWhgb_XnkIlZ>_hg|-+ASy4lpKAv<+im5*CI&`o zwPvT|dS%IsK$)o+UIvvHBLp@{EGkb&$OJo7q<)e{_0)V?S}q=@6N{G&bTI>b_uT#c@g357RxFS`)(t;|FD0h zulc$=s_a{%KcB6R8RD|B{<)v?;O4zi^m&jypIB=<9(Q?jTBe$4 zvc$#uD0NDmSCZoMOw1jH8O!r`$P3D&)6-|~4|Hfw^2$5`KOH(e&lutYA$o9pLnqFM zuD6H=_frXw(xU`i5LPZ>^}^p&RaM!tYm08kh!rA=E{Z;s0tkT4<63HJA9?pIY5J39 zK7uitH^Q2l2gbQ-B`Q+EgyR4^cXF&XL&P2#*S#=2(=+Y2#fm`QJyu41ICi>6V75Rj zU~%dOeK-;VQ^`3Z7{MANjhzl)4<+_3+K$e>uU&6yqXu2uuO z3eC6Qd%!5k(=c27=0Uqpn<+Izxo%=lw=33xB{^q4%7RQ(z}f}6VZx1qjhOA&&3sjeBIHBGKZLTTgx!E!#|$%O?|Q>xDudD6I)R3}iNvIYgydv7p~7fmT)Sl!Q2X%{ z(33|->W-{5$CsBpt>$}1CcqA>H+uNavKALz?MLQohiK}GWCjTKAQX0t|6^YvEu|07 zYi)o3Q$p_cQ>7`V#6IQrrcSaURgOI27G$;*`HAz)b$RI{klH*;FfU*91T7W$(VQzH z53xDaTXN_lPn^8eVFTCIZ=GHsD|xfmy}+7_X;d1xHA+YXtCo7Ft`2$D_mtJos8`sW zpIm?@0IZ1N%PRs}X4~a>5x8M;NFElP`kmofW+pYx>xUa22lQn*Bcyt#%N|pr$rdO7 z^an86;uCM?ROR9_U}mUBWJ-*WqM zPj{9_Hn$x;>dfCGKl`uU3`#`leBN)Ojkf;ws2m6J z3&koPoKg8vIl;Tr-^%g7zeJt{6ITF?`y3ne-?;@88_oxhZByXRP*?`|1v%k#j>p`) zeTFq{u~cNK&zTWF)USw|VP`@TCByFN0nvRs=LI&Y&3*5g3eN@fXt~x2n*Hx10PlpK3*ppMVg30g z*X>6bZ0=u})tBAwQR&ChFJw<_IN5mm{Ho;7$%>XICosf2ZPqC=4HQ11=V7a(7E}|S zsYab!2X&vtvA@;3zb4mBy>&!DU%vN-QcP7BeH|DoK*A~@*AptMqc9K9bl7ag&*?Rh zYz7~bkcK#rotZgpqUl$U8zm)QQ(|T4NP;Vl5rpw6&iuaCq!%J&prLD-J&bX|#b8Ez zRN_b%u!ey|sxZp9WpYx=J-B4cKeoQedM ztYSVd9vfBYYGogeuC1u3*mg+vtNpo(Ihq#~`$6wW_=1lya3HTLVazW_3nu^@$Cu3^ z!|%EY%3G`Ez+(d^twO0gk;zZN*vXXBt(QXDGKT+xBD9ng&fgVI5yLqc$y4PA3gX&& z4~jq@H$kqxVs##GCm61mJ*uy-4?wM1HZX5(v7V-wXQV^Z0mM}4>K`#8TJk^U!N4<&G$2m`J zEXu5^h!jfnZ&S9i_YbVe-i}&5aKUHY*)I0H1U5GGe1BZ!fq@bU3P_^k*RS2~w=yDk z+X7!aSuVY(R~!`tD{x3Eo&kc>k;9+}>fI|}Sy?GyOpOingJ6FrVaLXpP5u2@wrR)HnzfkS5bp78{wWA6kAEEQ-ZHJu{5&@VgHMGROM4&R=z#5q#P998D}V_I<>eT@n*38x z4(rQHMg+B?nnEw1 zWe09sLiF!7#SR!8NzXm2SNE{Ft)_!^_vBCyIBwFKjVS>4oIR~=dGF+;>J>!qctBdS zHcsCcxC8e#bU{#uh2BmXiO1qy(Gx)5@#I-<5>&btCz~;7(=Q14T&dAWroD2@L6Zxl zQ-CTWzdYe*^u~XjooK1g!>>90bythMR&*dR@43Lfta<6g4avhzVZI{m zs1*J56i@z2WLCL42&8L|y^5gI4|loDZHtlXS7Xxw=yUCGca`!?*QW!b$?Bp7JLTDw z%GHtT4yd%jl(Fg^3B@8%^<$7`4n;XtH41(>8Ux;MlpR|eDpV>l7zjyq5_fHYZ9dWv zW%yjn*EKaYcI;X-hB<6fq@&S|vl&-)ll4P+5P-EXT#*^yTXj@c1i}xv;)81IOEMpK z#2S~ULaOJoS`|W5PgZ9H8nAf2k%Yc#pB#|Je7-V7Z=WJ@ML-Glk1Z%P@<+dE0{xD% z)u$O}&s@|}@b_asd9FurKzM)U;T#d3AQ?XaOt_%Qq#F z9xIT(kR=j=a!t-{5Pu{NhVz?Q;ph@|ri$oh7#esREDI$bsvFF$M7CxD9cIVY&0n(zFKi>{bq^(wxQdgv> za?c}zoZGW0Bi~p-U6jWIC<4>|fk?Z!9|65Kgm=PR#osCD&lV&i=4)wA&uw1jZ7I|% zqh09IBLh8o)!w0MX0FBgtonhJ`LEdSMU`e?IFr5j_oAhQ!CD4j2KMxlJ9ugjHXUgH z_$=^Ey>^ewSEQ3xI8_ty6F#iLU({QF9mlf)Q#rGGsIrgp;ni+OysuY5&+@(5?SQOr7%_|Eu0O`(JNUNAaU?ILl za)^OwA{`&cHbEemMQ&EiCF}B4-HFo@gUUP!$R!oL_~6BaChCsp>vyalTuEuqpf^1$ zwth}u_3*BAfE*C@aE`QQUw&hHk3|b0kb7kAib5k#0z>BUp>q!H^k(GQkYj6`08Qhx{7m(FL$NCjw};&dh#CQ&Fc!vzsTkFr6h z3FzckR#n|fZmlEU1sH{COL>IA^BbkSVo^6Xx2sigMsCf8wfPB41?Ujgfo z_=nBUnpHEiO%r(hOW@H}Js}$d;@R~DTpT!igQmlPoIAHj z(EU}(Ob-}sz8Wq|nah)L4|HhSRpYeRo;Sh4r%B3X`TF?y^faJtS9u=(mTW0a$nLG?@zRo> zBJ)+zmyX3J5>=_fo2zfTeLg>+U6}9NAFT5)#t5e8RY8rD*TP7AVOp#EBJ>=GH)lyFezUcvU(l^6_*DK9v0P z#VV`BdKJ&tp)*y^!wZDwHk{KMh8j~Ka27fX8e*7yA$nBEjuBW2l0U~Nf~XH***voF zyKs@Zq%+L(lNa0-a;ZW-H>QqJ;dD|DLu_a`b_a$aNk4j%mOB%zj*k^Vd>3395UWzoDrdX8-qE8lL!TxVDJqe@8!xpA0&JhM%b@ydsdbr zPQO(&O%s9qbCI{}y<7sP*j&oq)e54C+BM6OVJ?AP@)P-%2xoJ*4Ebf?+;)7^*D?~H zhT#f%?n1jc3@C&z+9E!W=bP)%e=LbYE>F;5th~B?EVbO}s;L*NN>AZ*#z3#h==Y}a zjPYu}dq*e2&JTAU&_KzmC6x=`2}d!4Bssp^-Yor}Y@*=ERgFu|F&y+&cg4{c+46!q z;V2VejZ9ZJq|c**`YD8Zp?-1skXIX-eHaQhCFNTlV3$Z>re|h`R6N`&ZzHcqKi*GVO7)h@0_@UzA>W^% z^Bcb@_84i}x^&vH@?@6)bkYm=BL*9um&T5oRXC*iWo= z%7cHy#Jj18Hepg`cu$L_IaN<+Jx}%J92b8Rf%1P`7fku>l?QNo#Ea;l?H;m!6#&r1 z(+E5Blbv~Sd#X~6Y?H0CueqSf2Tg4Y<2>H2cvpD)nbt=;vM*a~Gik|&`&b>6xHn@%cdwpZQ1x~61J1Y+|n0qmkoCB>DOBFIP?Q_HWe!W4xiba+6UK&L60LQx2 z)@zGZn2_-0Bkni8lBvH6Inz^PKPf{#!x2vCwc$ZQ^ZN2vqoVkZoF!X!ssWDWdVN6{ zBnxC_|7*npHqGFsv5)Y{Xjz&Aw6%|;0!D+VIC-HX_CxI|<%}s-RFpIR8On8CkneB2 zH^>`4r(8xZBgBTcy4Dr6FOHn(yBDBC5P1>-^YA3C5gVyrm#%KM2|)Q%HTpI>A&#qG za-{XX$zRN6#QrEC^Ot+zfg7LZzK#!12OilRV`^6%b0$M{Ry6|J{ja2GWMsgWjf+Ak zs0<=iKRAjW4@^PF+sz*sQlfaa@%Ni57w>$vBWZ8+d`4|=$kd88N&=vGWk(kPI7mH- zsLx>OUgwEG-K%j8-GGZJ`Pro^rTq4UqeehgoeSim9mqS(^xT%lyN0SipoB_S8F)=+ z-01RqnG)*wfyZQ#4S|T*?s4SFe?qh_+lqP`R;5U9-W1`5z3+%nbI<9&jJjhtcxb77 z=63RF603s6uSP!Io))S8zD~8d2Z?gsL9<&>>T4SSkfOlo8aMkC^O>avq)D5|TuhQM##w&Co(cc-s?%@b3Q;^=`;1?7hU zL?hhTxTKfMFesr?V-}-+Z*q0{*lx{8T6LUNpl&*$Sbj`$BL8{Qoo&g@8|=Z7>|LX9 zcqh(b!sC)`XwBA38zI9j6T!z9d=os0Ar%bS)j7(6ydaS7(WZV=U`0M2Yf7#Y+{G9r zV|NFt6}Hmh@i%~ufr&ihOt3ma?*?XZ8_oVVFcqHiN0&c{^F$TdBI5qKR+6v3K1%Vd zK@NM_m9ns#ToL;S1Ick8-|JuvpN)L$TMA$qNQti-lmT4=^9`Eb>GDbQHzR$`gZnnR zH7%c@mXz^*JpF@bW}=2-g#L$~O9c@N>|a~s&lPc_+$#^neJejwd^kZ*G+ce){kFrh zyWkdH!*)59VXa(8099nc{XjUD*#FhO0~GY=p#iAqA@XhKaSrOv1KaOCv3HQ?+_Wyl zy9TCQMXS6mnwU!!k;3sNe#sYT*d zt70O=UKiPj9lXe5CcYCU3qkw_T3n;qJx=Gq;4Vil48=}UAM_I|LU+4EJ9MnJS+{-SZq*RRe3z;Fw# zqkvI~UtIO6UY%os>-}_~Ucy`mY< z7&^Zvs5IZk?*47as8l(AV`9FZfBj3fq%jsBI3e9SxZL zVQ!02VkhIOZkgLQT8x*jEP|5VX=py zYl@bN)FCZ$9duaZde21Ig{+dtFRlQCIH%#~(R#9uK2kvjIk59s9v>IF?w=({>8bbC z7udIpYUjx4kuqsr9rw>o?^GYH>I_2hIQe!W@m@QzKXce;)oXKIYYqoC>7t>ewR_s#D{|+@HETB$CG*YwAfp%=MjfWmA-lp_?31 z2%0HN5k%T7GUt7b#s4;bu792FT1uzK1PQz-jxIS7`wsjgYD4FXa-CIt)>2cYt1PsKgH*t)VSj2 z1JozYQ&UXpco}B?7N~#9{T)O*>~XTy$Wwywvdo@f-?0+QoK!S04@>c*!|+!^!p9#; z#FB`e(4}p9Q}=V5Ll6Frvj@0+iIa(p?8!*3&yf8wt&)-POg<^rH7)@DLW%n|a5XmZ zDTPMp9v9jGqO0iuIRq`5Qc~x2C@B|EAKw+i=DF1wQ^CE#MjvnuAwyA)oy%WQ=GMF9 z>KMX8s%#A-_(uBQ8$|i>BjY#oRZjUFj6b-h=i!NNeF<@1CLvEOx6;DwMoMBlt1t(J z0#y)!fAApH4R$7+d0A!;ivs8%I+5@%XQ35`Z9ypEKas zz58wn9yM=3-UUbjq*_PH_iK6TX1UBykWvSfu>@vgVFdk!+qTTiKlTK@X~|y)cE#}6 zYK0&v9Wf1>kh)by{H_0P6&``Y+@B4cG9ZP-!pwqm)-f|D#}qR z-(i<~9fNka?(ds1t9K)L=W&1Z;BnNiBLetlcttmtuqVROAMhLwyLhufwY2YQrK-VU2}{M9w)*jC8{QM-L*HdVGS>)K@7TV( zC$9vf*^Ra0=bD&oA2LT=+uj!=a_P-9Q!bUyn@*C6tbKN7QD-MrUAZ;*L+)u(pUIm2 zRGWp`7rM%3R9Oexe#FJ*zi@6Kq(D6h(=sAo&|3i+X2carYmhle$%*IO$#O1v*-QeG zpwt+-fa_c5ru$}`3FSv7Y)sQ8;4hm`0BxQVy(r^0WZ%@SLip+4Je4Vq7$ay zL}pTKWhwSo5_A% z5s5*E?YOiH8?k)l`uS_k;SP(oPAjDx54Y)3&QLN~33GB}!|zynXje^4jfY8=TtQs> zhonpE8io^@@?Zjc1vqebk6Cv7S5L@q14LGc$Oxv~)6~+%{Z%Vjz^=6o0@_~?qvm;> zjt5N8`F;1!^yi=r5~@gBy1Ib#DC2$ow8C;;nDJpySCZ#NpyJ{Z!aA}~^lp%Y5`iP+ zT$(*()j%ncE^^?yB=+k*(%y*~>hJHry~Ew#_k^p4 z--A}EH%2%?4RRlG U)%%3Sr+MskKGE!q=2fL6gx|?$8fbz!nlCOBPxWLl48x{BK zeozmE;d*Kb@l}O)71dpHQ&Xm^unf6bmcTE*br}HR=u^19%95XIqN(I)zqve~3=8K` z0g#_6*ts4)E`YQnu-Knn2fD?3L8(fh(1*7}c$2PZSAV1J(5nGpXW3&bIWQnu0#XL; z^Sxq)YNK{Ew;EuM8K5%g#a5y3EUx7 z0DE!K7IX!r{T&**st2lcMBHeV&E%fBW8iQ3_i{S{4p8Kh)mD?&OGFjjM;BG%Eik%1Tt{eECJKL|w$?v#2G$_~s7wyr%ai}`?vAD>{^>MRs!zBkjo>YH z_>{oEa{sHuGP&@+tX-TP@5m22 zAd3Uw%^VSw8G3+y*z+;ZBix_G;Y?0`ejy1cY&#{b7tg_$#v>njc6;W#1Ek&kByX-} z!gJ~FRf+fKH^DCq{zGdZvBS@FTmUvoHHmlef?DC2RL*@qsqz^wU;>qvtObR`6}<-zPBFr5sojL!W|KJ8o|s}Zo)TG$q&(v zZ<(UWg|`7@Qx;|aW$_ptoZW;u4Kt%TWfQ%w3**710gBQy{}rhJ`G6NOBnl+dJw$Yl!F2;)A*!=JU2Ao3VyxdoSO&@|UNWmk$$ygsm4!^0ad z;lXpN38G6IDlF2`5DkH-jq5R-@Z41H2Z!f>ufpTt2mZ1>GgIWm8jj6Qk0+312P|G7PZ^Yte;+x?<@YXFV zCNbd~TtNbcq~U;T^CvjIBwTC$R0(LC;8Qn6jTe>m*`{Hd+dDq_3^2i74JBNQE#rg# z=nHu0fDV<%-iDnG(AuOYl#A_289S&q9D07_?+pXP^J^`9*EW+hj*k7+JR9?k)~bHA zT6&dX#GQZPm+_)PRWvvsaff48&J+& z08mj{%2nk?Jp%)SjkJT?$A{|JbV*4tM+sn=E&x2&lIc}d*&FxD-+Y(dSOn%!JTkEB zZi2xf_)9su=1|5N!`C)#vdTtQcS5O=-e0pQBtLfmB+ZyOP_x`!EOAn~{Pj?5>P2z= zVzt8NHb}#)S4_oOsviCu{%>ih{}snOy8T((zR$tg+qLMXR0s=sfNuYdERFu_&*lg|Gc9h1%Q*G(5LLh`u8S=>JAT_7%L$)=NcB%=dsO7T+AI z4*K;g40Wj54zb_eKhXXdZu8uvd5g!^x!xE@c7Nn;5Rt)US0FWIlcvj>Ts=hN+6PY# zo9orov6dH)iG~6qRMOr&BYhx@g)EbcgoP*rqBP3pHbMRSTE&876kEmS0|b9BS(_Z! zSrpr&fcewUuV9S|>qlJi{R}|+J>7Ck4(`nYnnT96c#`e=+kWkz3?-1ut02Z6Kw{Z+ zJ#D{3hR-WF^6rpa>7&z~cDg)rM6`#Nx1298!p<^At~}TMFrjO1-z7>u9*dq2UMV)= z$##&Pg))b!*9NgMLI8<_p(ZK@o*?q5RSipz6R}%cF|#rUa&O1Ry^YHSj0(;|`mrc! z$wM*%P;1iR*w|EdFRA&!mi&$vgW86A>|~bjCb&!f=B;^L8v_l*s%=E^FnhJcARA6# z|GgfLjMt_bose^KdwKsoV22&p;rt5#>n$r}CD6CiQE7L{mlxe6W+Kw`)&Zd)SDEYT zWsTVHytosZoS6Fv)bU$xtw3l)DnSiv)+koF9wj`6u#;mjkR>T;YFg6RK%rB*1i$JM z0BKFVYYK$K^aG|_;O&&hmlzg|jQ@#@-n16}7PHrIwvh;=2w8>Mm5(^E;iTQcDsd|d zT-De(0|3>Ris(1)7n(sM0+yhir){NH7)NHj1JC9Et=Jwg_T_sc`O1~cQok!Ht%tV1 zz#2U==~Gu)Nq2_4M2l|BeO{@>=SurN`bq4f?GS4GS-o@RvHQ}Xwe@K7xpo4r&V0Xh zbrNWM9hH=XX;0t(Z+z(o+u5K5{$&b~v49(&u}Y-hb7drL@4ckGjhsA2MUCg{W|Cfb zO7lve`RsT#XJO4-(8>mVtEJTanG|f z@SYqPdivWB6uZa;lt#$K`gQ|c6eWC96-}XJ>~i$4iOxv=Wc1z*wY4JGbt}}-zw{$- zb5y5QO5B6@a)3()C`cj~kE2vEvRX$qYQpUCRx|puQo-bhm+FH3BqeQc)UHWh`*!q? zc{w)F6Lm@f+$4Y2Up&nrUv|~R`5Vb1`E5`d@^`xs0Cl{%!gF<6nef5IzlJ{_L#5Rm z0NKjS4`eJn*l;%90wGvsDBfW%FkwVddNBG7)$nc&ae9MJuX^8+_mxb=-xCMF5A-9(!% zY-c&AHjPh`KZVX)i!WNaNPz#Vi@bYs=cq_k>ews2#>UsLUk5A%lru-iDhou~b?%PR zXw-XkDG-!KsCR0MzOuNb3b0>_)mFxs=f}&qchG`{--hk| z+xODEH8O4!tU+Bh+U+qRAbthxYyF}KEMtTYC*8HSs~~*GEa-XbUv*H+DEJ1C3U(*0 z*{(Ry`p%8;u(rN<0OxZ3QPcRv9hd`VnVONrXUgDLXgKrB4<>&|{tF@yiOsXFj;R>8 zIj*%EW~J9LfPKu2x8&5eNW3 z=Q2W4d+0=b6BL^iT>k6K;cY7?U)*2-cqECJ zB4DS_&ogQkA)sYQcOoy}G41(rq`IItVh)rT91g{vjpKtU)pEeBiOp*}?rE1F+#{Nt zt^enx=b!xTbW1J+Q37i~oRxxL-wVa14b3W&)q11Fkt0Ag`UX2az z6EIVr{~`lA9KF|XB@TW43ft9Qoq7N4jrRj#jQ{yR`$Bfg;kmRin^g!-CfV=;20@-O^d;T?=N}RQN-#r(t8_{qm}w-Gvhjtf=`uE70QBbM+4# zew-%uHE3{8{GKPIzoKzx1D#yXw;>_x8X0k{GRPjTbu0ldx zSJpt~4_zt0k*q$@%BuCTt74;P!(cEh>XsB9lTAw+U1@d{t@M{9U*iGW0A>~k`%O&W zQE%V8mv*Ln3vm2hXiNmxlWIex#y9~cj8zqw`8^!|=K=|emwCRVC>+n`CD-=;{LJ_k z(|A$~%0IL6lY0&oee{AL*@AGk^CBBgw-r$jRu zWF)a}1&bwqJMT=ZIYBNx4pK##{Jd$?31J3s&G`3EpR+Z(m>ElUqyL&&76e4Ld)GuiI=9wBf98dOXVX^#o^p zqE4d2Rg?P%2vbY0?Q^BxenEp}AfEka;jRB_ zZ+e3EX0Uejn{S`0R%i`(m9$s4H<^1LwXjs!zd6wloetjGiaC#SqrtrD%jf)Fe|1WE zE=0^9z^(T;!}>FBxdYe8^Xzuxt*E7=w2|?@=+?vih&#P0*+T67z)o80z=8JIAO9jv z_uhOz(8rmQfa&xlp$t~-!h$IPGrGFEq9eeRXQ8nRtHeA!SF)FVGI3M~4BQ{YU;#1c z#mZ=qB~vg3BZCEW*pBS?x7V9^u0||<_2x9)qpcTVm}Vk0Zb^V26pDY&${u7Poy zq9H2Q=KA?=zr`1R>r3l>Fm*qhTZ2jT_G!zn$Fd3hm!2Adh{cNi-COab;F;qHap%xo z%iYje9->;kZc5j-$DH;#ql?_tTFPV9YrI6W0!qskcl~GYW3^Wjh@7A3(t8@y@VKq8{!;oRYzBA$EcNzi5*2t z=Z z%Gd?wRmmKr;TDHp=LWU5>i(F9pR|{@=WjeLjBDe3TaXwW4jwzzJ0y2uS5oay+{Mr4 zY(giuKHziYJ9GAFC$_LE>e#G502A3Wuk`(}5%QP*MwpD%R$gJB*f=66`j@@e4UY21 z3u{noHw+quvU-~5B;ep%E^$c}oj(OXe?1Map;Gzwoc9vB@K7U1SVNxYCziEP^ERUl z4}a0Ju&($)fBM%k9vFUur~P4cVlTa&Kl;w(^O@(C)mctj_VzIxv%lgYn3ILs*piaD zAAhhVL4J18Y&(NoD6}~^PWa;!+T-WqAD?cooZ}e`7MQMOGX({3P_M>e+JK^U1Dn+C^$+O#~BIc5~ z?)Kf)(q%#dzj@lP!=B+@<9v}&Y=6l_&*Fl3UISw;c+9eC`U2MsCuqkm(265=X&miw znV9-i4}$r>SMV~N{^kU?LxorCkXMf=fk{ZYi&HsmRIYq3s8avH9aX+kUU`%=#RNhH zYoE_?yc=)CpvIT-R8!|etoH{&mp!e4Cf6=@5kuVfPj>PrzxN*J+TD0dK-(mrk(%C4 zXUSuF<2{vC@h3{yqo7CufSxlOt|^OcP)>P(_y345D9O)ZdE zH&bn=m2(n8`#j{1nU6*FK=qy)c3K{fCh(yqxCa;+ENy2k<=^wl z)AyZ^emeEryxza}6wqdqLD`4BZ&tRkMB5U+bhz5Q-AAyg~y% zQ*NZ3$qUs6p?6?({ZIWGkFlZCKdXO^;@s>>QmwB$>*2zH#5d+`R+E{V^H@=R7>)hRA_44 z5!_J57=+44W+(GWgU-bpPIlekl&&w}^3Az1S}vU0CM-UF_eA(O4(N6q1t3WfCVL*@ z;EI-cYYpu$h$fXg+laSpx!x2?e%>>sC3Zk)T32YhayJJ8Z}NgMQhK{R;ayd#2)R@(_ydL zr!BQpvBl~Pg0xjliPZ*aMJOpjM1IdZ&gkd+$M5-rCNI4De)5d_zV7R~?#cD2kwtUW ziyG9#;S0Yk)(*s8is^_fwrm4QmV7(%CbDQssU+~s71t|jpVlSS5nTL%rSyXE7{32PcLgo>1z{pN9LjB517UzVpg zMPVAr6J2Bmx8LV?=AFf}*?s&lZ@TlszER2cZdTjuK3;2ofo+$kV$`IHQzJAf%{nHVCRC1KlUc-U0rwz67`;Bka{vlGnkC1L&k2tcP5-`M4_vl9(TH zvAJ7OIyV6b($Ca6=1Zn?wn?Wj2KITdnhg4TqnNHA8tQc2*alc(1ortdAT#%TWuo*< zglCBz9c#cb{Col#sqP_Xjc%F%0RX==@q#LBr5I;X#!=7{8IQz9nQl+O$xDc2A7$NO zZU-%;+0D_;C1#QLH8AS_oGP8U&`slvGCI}EfuvgnKS@*QcWD~4oU`ioYtZ&CGaQ@x zrSdT@195)=*uw?_)ik+$V}lh}U%!#JKz4fTj_E-;fIhd$a+(4PNFa@*PU`YY@gXWL zgf92-azvt4!sv*BO#Y!+yAo@K-C3D7{9mfGt7OmG4m@}-0vnfCYLp@}YMKua$7(=~ zxc}s#3s5q87q^+8N70VC-BS47rya3rRM|12w!HTN1#rjy@ECwKxWBVgh3y!cXk%Bo zN8yHq;?doH%`G~6ZwtQLF%Yy;JU;2JtHnA<;#WL#a3V67V18P@%#Z<+>VHT1IUuZk zJ2G9#f!l(~ETagEufve|uKBLCMT0g~}>0j2yg>LRT61@wW& zj0tL9mlW8%1pM~TL?-gqbX1^Nx6Gmi!i|Ne+uip146L4mz(BT|`DyPc#8Xr0s~?Hf2j% z0h|i(dBdDkhgEN`6gy4Yud4lKWZvpnrRm)Oa-)MM=|Yn?m~+2;;)rM0?aRW>Sq^YH zsrNa!H>@tJ==}ZUvC#y&jpBmF;WCHjD!P6>9^;d<+>O*r8-I&@nO7J28V4Ue9X>aO{2HDl(3n zn~vOH1tdSKDw5Zs2>E*rh{X5I|6o6!$rDK6bve#wGkYL9lYew%1o$L4f1F&p!y4HG zhtX51E824k21{D}>c59s#N92A5gE{<h=RY|YVKV> zY_g!;(m2nVyhTRGYVQcp(Ti4kcZ)aqnhbTmhGU*bBGU5R_55}Ih%Z4BNA}(ogxybK z7qrNCuA2mEH)!=2af z?INK{aSa9n4gWU+V4?uOEPAqbI;LxJ->lEH7Mm6<$#==M4nRKuP+^cD{Bwl%dl>4= z5H4Y8l9o$?P?wz{eeQ@d@kce9A`-VCMEEapp1>0L0fEMw@$vAhV|!cD3FLahz!=vK zmNz{|W(;)k1Ev?>ss0W-B?>Yb((1iZC|vejSety}h3Ipo@SLq@|44M$pO%Zi5NQ9( z?^T3(Tw*!t+5k-%b`!J?>|}g0qFfquY@S_xwX7W2&0c8sDVl8fRK~(Z9)O;FW9ZcY zfSYh_B|X^{NdK?nS|&MOBR&v-`tCOk}_kRR@5|+b&zs3;@H8cogu-#|EHdl~57qms7RgF%eWE{CnhyIb}tUK^@di zfnj_EFZakY_&o!i!{xd(| zd#}W)CV8rA=JC^477l{fH%qSBbkF3xS!;SMh!8u7dGt`f#^CP5V5uU-du!pDM+-w& z0ALP^K|I~T13=1>$rLlxVQ3@e)>==1_RPMEbFQ$zXs(unpdP4xTBzn0 zX`G(>rd|HTOtaqP8hb!uTg-|3=@i42b*s3v-Xr0*OV6+zAfh65u{EhhW%>_c7QC+; zALh&!w|rmKB)T^Btao60bx~2r#}bD}Tf{p8_Z6hDjKo}<#>_EMKQlq0wv6%zAS!g( zTjJ#wsn)lr7egR?H(Yxz+mMm>%S_$3;Bx!AzZjihgnO?IHHtp_ zEqLzW8>NFH&h~3ujxk>MXNalh?5h)o`JX%n-+H`-WQS7M!#_XRo#c^+eM%=I4qsLV zjEk0iyQ=FJGuuHBP9~egL&gCnkgf&FUn;tz^HHSXj){j4qn3q^WLq}R>atBZl~KS` z@op4rY+Zpv$W(A;hAD5+$LtCzvB|?@p9klQTgz#<2adsRYZq)?1zthjHn`APwKgPN z)#dqdM4Wr!8F~K0HQSUSr`&q)NcNRsIyM88(p;7W#Aru-_o+50(;p(q)#wVsRK@}+^Z^>e`l)-N z|B^D}AxHY}qbZ+jlf!tmU5ug>* zc8QN4*r7-v@~6v;iW4{lMd7i5>TW^ULFA z=v*90MOsl~-y5GuGp1&^=Pz81ypg#>Q2m=GaGh`XsK6&5Ae3;32zQK4wd$`}I>HhH z)%F&0quU({V^zb|(cYy0s5<{)P^0y>+-_P0M^OEO{lN(Ax;{ZoifLzT^Y_H%Ms=^0 zTzv!f=#8@`fa?esq#79MxCc~s89xXU*oa0GWVYxFp+2&8r+?S(w(f2(bDiqpDWOFznM6_azW9*wT$i>h*Sf)b8_b5+OZRl zP#O5J#nr;lTt1jND(Xu!>r_pS(ykzowLSh7up5Hcl0oUz^7RaFbgfCb+b34?`<*ar zv6@+_9z)?;;(Q`FtE1Xd`$5c)hc!n)P_ zw$1L6Ux4f%=%}mAZ50*QRtD~kGy{`%P!_2jtiXItB+REu@&yZtS)0;Y5GR>|HZs+R zQM>Nz+w8suF1(&g-i|6qPHDK*>H3*K^=JSa5~J3+wDZ4_E&tBZ7hrP&GX(4w`qs?}&Z(w~~2dyjT{J^bX-)@GsoHrwi_ z=rVC-_Z{WI0Q3cop}e&4<$o7p_0nU6gVxS{wbmPkho1rHgey<<@z`(%`a9UvJBT_w z4i#g|6EiJX!2JN!>`2^#YlH=}IlSV8o)~;{RJ77SAw_3o+(AOH&-PQO?d%`c7|2wV8PP}kMDeDN|w}7THrFTl86QbMP znvjY~-+tHUS5zg?gVa>lq62_r^P?I!HFXBeov(w0zm7uVc2F2(%>!T&a6eM?e&H55 z4Xgpj3FsKotJ#YxoM|LK*!_V5Xy*QJo+I8j2ej{=2KIlIpLgn3T{Ji9=e-w?D*dSs zAOUxfZQKXB)bVG);Y1tYqIx*g;cCH6_Dj4k1_ewN+YezS6gwPHUs(10pWY*yI6WRJ z@&ff$oj3QwDsF3ITqVrxoZFv5Uf7r$`?QuW)<=I99vj|STD6OWFU)$d7b9~(5jxOKESB6kpKL#+o$)`9Z?lgJaJau!kUI8d{~MHV>X0pYX(!|~gCFF}#`)`NL;Sq5FxlcyT_g7J$F13M+hpl?Yq9Ko;vsV~_eTRlK1{*Xl3fluy*s@j@LHlju|UU6z!pj8bid z%d`GX5rrO+hO*Q@o%U@_XwIX>CYBR&@lQoI9qZo|o6G|nI*f^8gr34@+KV%G^8S*7 zMAt*2I?GS`Mn#I#t2k)=c=15R|~@%N{;)4NDGobkSu%^5E-3oEEMwy-yY(0+Fv(eq01aI_I!a$ltb~HZVL_ z$Lu*%$L7PWZB!nJHv&7#%+SjURj`~{}@+<@7-pcAZ~?Fe6%kxbTyPZ zy--yTI8GROo|{i*?x=amOJdW3Xc-NB-gse* zYQ&4K(gPo}YGPO4^qfhcpvKMN`?7lZm+J&4Iy*YUTW0*%rQ!s9OzHRzC18{p3?kIm zwWmARSIcSz2K4#`8=cpdSB})oQ)((nFunzYod20=fEJL#<)ZzQOv(C9+ zkJxv@8~TG>Mwi?qMM28(FR=oJDPcw(uy=oU$gvLMy8$np>~K`%aP9aP)A>kK&5F;R zraSjdhDQB1X~oI8M$f^19PDyWAMWd*%|AgLvrat7Jm}&KZIVC#wXD`+^@HLTaE&|# zafceqH!MM+l7BM^m7k%jxB@6SfFuU^nXSNfw^-?K)~?t(Yk^_r`&(mWb6>zL-Nsm? zC5h)K!$p@jdNRG{T5s+^Sfgls*UR~pEL`bmWm`DKrY4;frGa~t$dlEHLoTef%uv$> z4>Wn{dBP=|weF~O&UX#B2dqp1ldM1tS<;!6&bWhaO*UlQC6pcz%xASqNQ+6f*X6OF zx@%!)*vEkT_7MyzU{q3kENJ-NKn6*60KE6l=x6~+D>;@oeA7$lt>902X;+qiU}eHY69ri1``Ds!9l4A+PFA0kg< zIie-K6_1iT(XtlPdDL+ZIO$e^obCMSt*fk=G*V?`S#OI!#k-auQ%%syu>kI zK}ip76Q{PIJp{#<4Z1?2pu(%bVEv8*2xu9PAz`{X(kHOXzZ~%0IAXJp57Z(ZT);Mh z1$NWe)Bb}PPE$ckU)MaLrR`)sM9cqs^kk1u6rYH@E6?Y=!?>{WX4KYKj7y~-iYw>JHZLVyfI^X76#Qa_sQn`GTI2=;p}MYaG1y9c?9)aV zxZ#5ETeMoE&tJ>+HhCQrwZkvER?rzWB^8Z`q|md0vBC;7alh?xHf&<*CAK3 z8J01|Wq1Evbs0o`nVq2`I3c~B6AsgZ3cd5K5QJu>x>IHh<32tVICrF@tjiAo(8ziQ z!vViFfE6%|wjdTDEL9~mG6_0~VGiwqgF}a*&_?L0*M%}ekdi;i#drF}PA|uxcT{!z z^@z^(qL2!|06xR%U`Ax((w+z|ivW*YYaPy-)sA~FCS)5kc@#H$pN4kaY^h)~`wD!_ zU)O^hECX0v)vz$Z(b(WEG9h5#tnWh0&rpRn$m^#q%*mV`K#f_!NR!#W#?7X6#s(EK&%jkZAwBv$ zQ|gK2uJh+$PUsQUP9K|z5F;ej^^3QTLforaErD!4g-~F+vWcT0&i@5nO6S8oTG|?s~NY1LBpHl-v@-2{X zJ|cJe<$OJNo?clz-+$@oyAW$1H{%ywH9`L)m)l?@aRvTRt~EU1An>xmNcd$hau~bs zXS>bSwJC5Myuw4;5`6=k#`^;)R&^kkZ8gw<2tijmAO2aqi3paFhO>l&L(F#IXvUBL z)oCmzn>}cu{Ahc&Jil1PXv^D`g@|_fac>(uSxHY~jAU_o2Txf>p#6FkyK1JO(c=!T z6aCO#515(kx8S>6&OXmj4Zm0D?0tl<1?#Zu8@P9ZGf9pb>lO~H8Haa?I|fgldXBdJ zZ$>0G+_1-k#EcAjA{sk)6Nlg0K0~lqg9Z)SM@>a)*?)ohOV0R&jn=8u5LTPP=AuuhYU9GK%W@vb6!{8CoowcB8p z-OiRK*A^y1=fI38;5iG;CEf$kt@WF)O)l!oX9ng5j)JZNl$Cn^CFRDy%9`63bLGQm`n6%Gg``&Jp9c%Nzx@})WJ$t!WEH>rOtbFlu`k#e> z{dKdgrHU}0AR14^R}@bNA_GN#Id-j)T=)iP!T#X?qGkbZ?Z-`@J9p2Ny8&**R;|di zV}i804E;|z22-am!;ia0FE~6rAGHQ7)c_iC0jhYacfA}I5pAl_-(`~pycna}X9G(~ zDl$JYoMs>&_}pjw+C=wUZvJ`gh)^cUHKtNf5~>`))cJxfS?&p3z<6r!*I*wLoT}XO zck7#AXixV%hP$eYmNJNag8+Ei8Ll;R|Sl-fMpYRFEx9_Fzc@-wM~O1 zM!O3H?~wk{L1*dRxN`y1y~*bXE}A}&!MX&$t$GLSCg}@jjz2lln@rE?;DK2m8f>?? zxkmCAp5vO`e4&>i%nK-SMnYHGj8&t{OJ8)O!$HXi`TA#QEUqhdRb!o$MpU1q<`Ud@ z2I#cO7m8Twi{VA?j;f)-$3E|n7Juap80(y#2J_dxde z2DBDhA9?%s?fKWsEu$%YZ@QkI{e9jjqm3E~+}9Q|xg3SLeTRXz=GoY}v!33OhdT}w za;xW(84Y@VWPv#%(SNg)tB<0<&FW#LlAl%jXKYHsfIHmuC|np*N^OTsCGSr0>&&A? z6iKyCd#Pt&)~7{YXUCX_+xE>Gxv(yxBhpBQgyI8Fhtr%oS`s=bRuUZ}6N>=}oO<@E z6f>R{o0$_dvb83O9*}UOR&L7?mcwAe*imvN$nI;(6W!`P<7?W}7`wc-wqcgS{$4m| z>cvS8jvDRYvSy}}(lypo3>i}8()h#xZ9=2)e>0>~uyFv1P5GK$A3q~f*iv?{_tK?? zuR^G;{k{7X=081W|6aIwX7T@XMLQ41UBN1v3J*P-2Gk_+Ba%x?5JMa6tTS<`%D#Hn zDkg(SdiQ9Jnb+%_uf2hQ2#`UVK|Y+1*T?0(pM3|uO*{vV#SA3VQz8|a$^6gAVyBv> zl^VT(^6bfMpj+JVn6hu+8&|e`r7bKtau$8aQm&)z8^b+;JVkw%q4&19lXxj*)w6uMs@t=20XF_+!s6@4FF;nL0UMv70Oo4&u$Mz2jo9_vUuR|dZ?V1SfI zN0@I%5`ixYOt?tXJC*WfB?~O3&1Ar>!i4>T#7A%4C=2(zrCf5u2|YnQ{LE^p^XX10 zl6uT1DG6odBOoI`k=M5L_(9fz+REGBBmduU?|ApiH{eRv+C|vklFT2f9AbL-2Mt6+ z$g^h!;bRfN3ioet2G=ehcSY&cqscV7HT0cHN(tXbV{0u6TkENP=u%B3cbKy)@&aCF622QN^@bB&QN*T)F!d#QFq zt^TyXeIwv`W8|&Q&FYGXCR5x7I#4&U^^1mcTHYh|bbB>`Oxmjjt^-iRIk>v|B6`@F zW8VIF1nW==uX4qGjRZTf@Ov5M%fNhoaRNipZoscb)JJs=y#3F(516jt-_e7%)v(F0 zfTHPN4avp89T)Jx(O7ane#9kB&)!aXTd3~;XdyRDl-TP8Ds+~Ux?Y-yWaaV?XWr?U zz}fbVPar}9L9_XntvT{d&~D)8wWk= z^LEBFY=WAKZ}@#>4-o2LLY*{wQ$X1Jn|HJ4GoOPY1=Q4yE%~T&hBOfvAn&S5RP^9Y zaoermMYt9a5TN{(67(i~FVM-%__yFR*GE439d?n$#0TVRVD2-}|4bC1%P#@aH+ZQi zZYWl0pL@A&w`2H`P`Xv0)Uk8AYmgrE$!z+cCw3e!s=N=SAm+@W+AfI6y=xZ)G7|u` zxoPDMXt;1wWfk0mAjdd={v6NFUCFI=h)rz>tZxXMcUwzi<)omo@J!t>S0UmeKAgfb zoYh4tfd8_fonjcX`{Q9e zVPQ6}7!+Vz9>pNkyBN1|sV+yzr9fl2RN+Q7WY#c8IXYD1f|+*)kDbO2JkFn!jmq5midYNyR zp@N3c1ojP}*p~hJg-W*RLicJ%x#}&wc>}sx-->!4^=!w$l4|DA6PM=F(_OUs79G&O zf~)BnLnj1S;oBXoA9HaLKYX}g6%ehU7Ff?RlDd?K=%;ZQ=kYwcQ_gRd(RGYoxReWD_7q_4LllUn_ezw=u&>X4tzsADs&+99;Y;s42azf_ET6Ud zM>#G!;oKz@YXHu^L}tu8kOxIgx|STqN9d>8T#v+E3TP*wkS3%<_TBzn6g5_nQ7evG z#ik2h?1i#Op0a*jd2&U9oCsRBhl8{g?9laNcwE3> zt)Dd5qOTau+IaS`)GxU&2|^m7#b><>_0D&mYc+k33CN6E+mCw7#B`~j;7h@CmCo6F zvclaYR;k_)HXXmwF#`C*bOd6ZLdd*qZ`nB6Ty~SeTtpKhOzZ7+b3PR(;h|>Tt;MmE z0pt$3ta!wh(~sn?Y-yImMk@=MwIX{ZUR$KH`kjTdY`S<^A{X;ws4L6$F+l^-EKw7$ zB~Q?j86Pa~U@~Gf>nb9V#2SNE&e_)!@+oet&=JnMyvLc9MMw3Mdo5DgOi)1q&THVd!Zr6$d*i|R^pVy~ zQ>Tm)Su*27a?mtAf2eDefvS=axrEz+5C z=PkOVvNCEd@yN7-G{@r?TbO>jqbtW+KCte_Y^k-pO-Ys4ixd&h^q7MxXY%Egvx>a1 zimt-5n}%rh^kl}?;Q4nQPX(h1>^_8{^5l%Mu)j;af4eS_eI{79#(pDciVd1KMWJ@y zNVq>z3yy{)QkNGFS!LLU(U>iXQWYN*>IyoV+1soYQ{PEZCF6@bgP2t@ooMx*vY!eI z%S2pv_U@VNxu{tyv=}m-cqgcfa-5u3EPA0_p8RmeWXe1*s=cI*H>_-%QaR=3ox9~(<^8M6>@^3x_%wr!HH_n>_j%mlPDi@xL z`?G<7c}rb&DP$PY9&w_~V7J!u7UtAWORQ+VVO{!ZQhj)p8uV*?yO;*4jRo~mha1L2 z`%xGdD1Jp|PMYp*Uoje?{itTarPNio#NbMChj1IlsqOEL(W*HyDfGtXyxq43FLObz z{*H|LF2;>~g_xyguZ~LY)Q(R8pYTINqZqel?H$;$^gGp=4#;3sSjwGon+nh1@|XnD z!+;5D8by(J5jf3jopv~IAsZu@QUaH4@M?( zM$ntQekDV&6261VP~|^r1uR*YuVx@}E3ZEjME1d^ZrQ>ui5i8WS3e=scimIxKZj{~ z^27=l4}dlptm%Jfj$fCTih9k z)!{eX%w&cbu&Ua7i?nn*zRPKVTgP+|KI|$Ukc+KkQ&Fk>pKDIHRYfI{-Z%YRi9jZt znOf;H}wD=q*Z>K{@ zU5&&QbRB?5N5-JoV`FqEZ3voSMpxj%QNTPzYxn~E%>E6w1TyD%*ap`agjr*N&S3SQ zZ^fuj^xMwgzuH)z7C0^^pzp{?W_6`X<@G7?OXUdoJS(od;U$!B9Z1;ii*tYBZj||A zk4Wc;B@K*SyC8=tpx5`+1w1RSoe4DQS)h!&p5N}Mi+h9}t9Z9`-8l?|eBq)e=>>A- zEp0>?D*IU6DAu75HJdpoKM|OuH$?V3?T8unQNC+YxT)YnEo>4h$lN72)C?L^9h8jb zOeW?tDIQIEfe*=}x7Op-IvK-|KOLwsZ^Mps!k*XzJG+TXBQ$Vzc1(Br)wMW2o%K_j zdZM`_nUc8zw9fyG+AiCTN+s;<15(Y=6GB&82p{^xN1YTycKLC*ZJow008_P_#e!8O zS~qV2g~sn4Bu`XK1?PwETJU-v3EYo4gVhDKcKt&q^=0T8R^)BGN`@k5-v=cK+{C!^ z43qtb0}kGw=C~A^EpgobKs5GTch<^}88B+vk)J8J+rvQd7~rCxAN~CK^WCg^(qLj| z;he?Rwa60D%+H`W_orfC?M)HUB!++pB8TMz7koaHEqp4JRJiC_kTZno3sM*xn_*6{ z+P={~YR^L6zSoE3Cq~ZqdDJG;8do}LneMuhnJXFo*6ySjh)LbXkXJo~C!Jdsd10cdRyj<+Ea+VJ$| zk1m@rLfG--BAoL136Clf<{ZrWK>I2%Bqt!GuZ$QEMyggG+Huqh3Gt}~wxweLGMb3f#L;%F1&&@&o zbgsY)w_=|gjocY_l@m}#=?A#T#%UZ9Tj zavsrnlubiN$cOojZO%7ExprhT%uXw9k%@a^T`@ctkc%lLb=62VKF{+Pm16b_zV5LZ zTC?XTMA(W^(cyF8i5(*HLqW${_ZDy>@TJi-b>JZQzu?PI5)PK5u^C(i|=oT zcjgToJ+;zb_TmFNlE{dFUz+65@X&R3xs3FN8{Qt$CHVPK#_1W9&*(AFzOl&=I(C$~ z<=j?21M3mn9nluaQlm^IuWS14p-p%62b7l(HVURUv?t+9C1Xn-{)f*o;OavBibY@XDb zSH?~grFQu_P;M1u1`9azgar$!7^+^B|>vO1%>Gvqz>i{}O9#)*2Y?rF@@(XmX= z2JrhI1792IQ~R_kurnr@X}>GmW4mb&+_jQ9b2?B64&3}}CL8AcMYS_m!q-LVX_R`6 z1RPg0$`*@Z!T*Z_tdDL$4}qndjwkwTIolUKB4{qg@m{c)oG)3M#==H}M2sbay;p}< zS{|Eas0^ET|6l-icCZj+ap?+#3Pwr%9vPMD$@~xY*=AQFz7_DVmwzH1$v7M;B@jzH zRDXuN%}35dZvfj>c)VbLP3??+QNsS4ZJzN;orf&-kx~c1w#qq41}udO+(Zgvi-PjNd?I3$zr0ZDpjWA#&)K>n0of_LV{&&=1x(EN=KdGc&p&oL-KZAlP)5h>hN}< zIjiZ2+!usIDaT9eKVCh(?G2Z~Q`X9q9v=O+s(+7(`u<8koRs7KC0_LFOk!65MT zj=Y+Y8?g?xU64S%dyVB^M#UU+Wv+B~JVYrtJ9FIPTWr0^7UhNYE`+5@FV?=^4AuZP z@#}B=e@taBuJq3Ld8v5m?kcob&^PM^BwFFBKEIc!ML*i;C+zf>~M;!?6MvJzvgfAr&7#YaA!sbvr z=lZ*>?17l#6SuFB>|3p=0<;$wSdsUUQ4-(0k!kzvgP>n9>juP1$%2| z!1VZt16CN(2b|YWN?DhxGGSeBy8VD_Rj9Ued!UWE0OpkpfiZ#=nLFwYpdhg~cFefX`EU>2@DZu71FR5j{>)L6mA0sm z498u_2ag56$stjfG!FbFC`pX=*O>#c-IKGcbkPlC{S6OU_69kd8~Z>eH`9y2D9|m2 zQBQdJmU~JnI(@=Qy!R4d3dQeojM!J*dT1UJxN)i<-(6-D_^dY}8c|X18KQxmZY8=Z+#G7e5)#xhezMW3+)f1XT=vXLXK%pYHBe`8xWF3H-jLM+Q2yIEYH%UM`UCmq}-L(Fu- z1$8=o5o|_gL?Yp@WS(y~L$_!N)f*-_y5fN9D+qM+7~Us3XpGjDcSj!QyBmPEL+3rB zX<#CwW~pd(^I0nN!Tsg9^?n*P{oGhy79PDAt`0aUyTms)LD4Zm0ng)aFM;3cKimEBv@C-{*ZPcS`&nyM{7G)g4cd=#}(Dc@!W)n0x;D`tfDdc=ayrT0l~|6{&1<%>_f z(a65VYVk9S9$Kh;l#kdP6sYgo#jP>&CW2Xs&Y5h~-#D=MbK^+ONbkt!4*u&hcmmhb zL&ux7ruyF!Vjkf_!j^jQ1Bp1$iS1dynwH4UwF<*EJu^}sMyu-2|f*ayoa0VVPQw0o%6|G`w0F>8D;Xm*#60D z{$jFtBEhG?QJ#TjD*P8qBh%&g%Rn=tS>Emho>cPJ{M71Hry%3cs?C7OxT?T{65%%?Cn{Hc)A(y z>uGa>v)#Ny&>5CJ2|2NHG?6BFRv7ry0Q&5?z_rI~!e1{?rrOd!%MY;n)B%KAr4uB+@86}; z;lGiG5=sQ++XN9+gCW7YSJb$^ma_7Ed6|}QOI`1q(o04~v;%af-Evv_FFo=ueScrS zS(>`)=uM0{4ay50VSi0s9K%i+Jjy9~|A$l03a@)J=e zOF911Cq#x6Fw`o6i+#~Q>m8WT!CEd$jVPx~Y0O@;ugUY*K0RkhL|28kRWiu`&I_#E z*cW0MBmVl^<%^iJxbK~Pfe@g-+4L?x_@j~<*TA@{d}N1wT(D4Df_HjDD(0IbTskx6 z6##?2ldnI^|1f(_q$7`Iaka`7egxABG4Jly)y;=!=T}8}3)Lw7CKxJ&W&_>k7XZEo zx`$o4WFc&cT$IQL64v_*?$a;S)Q=ob9-*t0OTffjAcN}b=pjsw^Sjgc)AeHzHBVD*Dwe$P6&3=<4VAiCtQ*2QM0^L{ ze^?$r@1ojvxfMGCW~^d$rLq*OtBIktvqm4LVhC)hxRl=Zj)gA9AJI@|>NGgncp)JU zRCMg^BWB2T^b`hLqpuAZx&N}5m?hijGq~rKOg)d7XYTemXQX+~7?91Wm93^Ry zq?pcgD;&@|g14DE1NC*jdl$ltbS9X<5S**L7Wqi(K~9XIVgm1ExR%@gaJ2+82RV*Z zvW9~$gjYG1`bM=XIiYNZFl2YTq>tF0-(G1OonP#fkaO~z-sTh&3!BEI)TkC}}!lNk zH5?c4ok!yg#vS2p-nHdt7ndPBH_O){pDR+Znfi?i*rPJ_9owM+565qCH_I0jS%F+_Zt}GiKdobJcsjR`(VlWCqDM5pFr@?`d^>4%<(q7vyN1VXFdUe_5 z30b)+o&R}&Xt@lNBdFr}sbcP{bGxD|ttL9ej_b&#@|$p@jTlp;Fq@|WkqNm+-F4Gy zDx)?PcT(h8QZ?J;tB^iZXze)NO-H$ReK<1VnN?=3p{k-*=G(6Sa<=JesM!n{j8vZ2 zUdoXw($};jZO(sHLEK$Fk|=ddDn~dgcc@t$R*<6K#oT_`7=-ae$hA!=x=8MY5~&m} zv*mLB!(oLCl@SgGJAYm45NRYzoORhAV0_9&w$zA1&o8jCrT_m|7TpIc}S zvuNP_H6&btHsdJpr7L&%THwVO!qa2!i*$Szjy?Qkwv6(AwVWVLBy&@tK@afs(ePbid_PZ8qo=f)pzy#p_z%l z+)^1_c*EKS<>YM_7l0q(@?yD;g@4{1af#d)*fQF)r2ycr3t|=Smwc?xLr_P2OXP+Z zA5=_&=xpV(e48_lB}rk__QJ9&&+)5!`CQ1&|I}I+>>6c<&>w9-ehPh3Da%7ib5l(^ z)rv)W=0+OvR-eHK9s>3v-5CA?72ZWl}fEdEz_-C$qJfZJL9 zrqU^DrhYO(vf+ZF3Q)CvTJX+GpHo1TdzvUY1rHE|1H0B;6WWRQ7~GVh!*gQoMnZWa zkU$QgbP9AYfSmnffV~c1dVtPwgab5J8-sE9pKwNMe-3?Zn!X&qV9^h*>b85DZ714e zLy-)tE8-@S;Xo>!ao5YkN$rO;<{Q7vEBEKzIEhs>SH?uzSc<%Rb(`W-pHZZ%rm&=*)tcM;ZvEZaA@)|D zZf1Wyjj?7w*F5IxM91>ak6Iv~O)gvXqn1AVmytY|DdWYmOtxtRhc!Wxc`1 zMgKB*pAb+O-5wtSO~A_Ks5a2UMO!c@0Dilxe+9>7w0sWZvJBUcg+Cav>E1c%KikFi)rO=5eu5Gw3v9hmX;Qxh&DUT<8vNRFyqrLiPKQ z#vlhOoob>3cEp$^#&uztP}nbM-bY6cu+s+1^2Hsj6dGE{B}4hoe?y0B^l?ArA!%y5 z)Yw#hLW66EKd{HHv7o-;%D=ON#s9Nn(zE+J**1lL7sebf%Do>KlnECD2A!Ea-HH3n zm9T4ClxxaIJnk_KGF~qH2J&jI?rAiX8<4O87YjW0wKUXLvEB~nn1eORRwscQiWPp$ z(J}F+Sy#TGr&HS|d+Y6X1cZ<>7$%+kb+4?8O>u=vfZ?>zj&r$dP9CXx5W-=tV$;d#J7_w2`ZN430{ zmJC0$AA5R!uMdFT$0A5^2z53Y2%5h`$>{)oZHOBJ(tP(1DujQdKH{&kYAp<`KLWu? zi+=siPQD2N^#c+3yZ}y5$DDoDbR@xZ;#9C0r!SRI89GYPTM8Nr{1gt|i@W-l#JLYT znZ$fU4PkLhtYGj{L`71+2-ZsT7u3*~i$YXF-_|1e*( zAX`jTmg&uI5B}U^n)ZfOpBVx#<_ei09QfbP4qRz&c{yZ#JS4t1DMr#JIzGN@{I?gi z4FRDC=6`KI<2~<Gf`b-^s)RNORFtkjN3 zX6oI~<4TrK{ua*sdbr&+DvHAbK)Dj^{w=OFb})z#n0}?ey~EV30&}4y%8_Y1eA#Is z+_@umTwyxxiQJOzl$*^e>i*Y|+wp|oGv7H6j(RZXifzX;96iX3td`i*%31380n19? zzO(3a<^3;u|Ok;MC9)ttsm^02{ zDj2o~lY;2TXVuJW>O9kQl>8Q0=gdl?pC^+Ci@O+Z{e9)!M?|~@&CNnz-xBiyCFLr4 z#>$#Twlp=EYwTGZ)2)G-(lP#bE}RB8N&z|m# zwY^C8YFI%OPrFGV1;yk18iaI?@L0dF@>>iu4i@j>vYXGSO{hJLJOeVOMHK&@CRfXZQFC|h+xy$nb)`M{Xq;V z^*aYWFyY{e7W1@=`pXRqJSolCVcbu(svF&$o7p$NG96yG(&9o`SRSDZJwMnYyJaR-{^VDXYBDWXT z2r2rA=ay6Ppyf?;P`nyYJA2rnq8c96Jh-3w;;z?K(tMJBjhE0&D@NNG`#fMICQ$^I zOL^**&Q56_C5k%s;S4rOGo%M+r$U-(#DNq#RQ;PDC|ddwxe#KTv0}h|s3}?Mxqy64 zDq{g;p1o-dUUXD9x0rt`zi=(+zv{6bWn;f|F1#VE-rL+3x zsiqaciRwcSh0$!N9A!ISMK8*ZDFqFuz8=Pm4X-^$sIIR~atd18SG*7=48F1WU01gs z3|3Sxdf3|dNBIe%aSMx)ozLFUj7uNhw+Z=@5lXk)B2BJSkm1DcJ(z|1%kv8GW-k#E*pDz>>3L+Tj?udPnA><%dQ zh9JH`alYpV3?kZ81D4`Y{Z-Z#h>N(gv4AD2;a0s0t>*y{e!$%`h{w5eUPu={b6TV^ z3YaXwGLN^n7jL>bwSu_#DEQGKN8*xd`7vrK#3O#$(sAPu2xP}1xdo4CR_97{ZbRSv zYIv`%I)-#vV=RN=$qiFMM|RMY{Zs#cw@~j!)E?4;DU-txkydt2S|dT85ni573lppa zx~ZWGpBWyH(d7yNud}4*fSHUCZ5;tCmx?ntP#7<$WP*bBljg*tvqkXZnO_Um`=eG( zpvXmCou-qqdvYaCuPWK4ZWkWQt6RsAu&CO1uzjm&Q+#+(#Sg>-Y;$?PuBesOfPoVfAi0tuPMJKs=h zK^q@0xyM~a-lHumgG5W&12=Ux1-w7I&9nNl&9{duz~Voc5cwDN;4zsxal5kc{Haqf z%cSL&8>D$^R$5WM6I@+CzV@<_NEBhN^=ItZpn{K#es_x@Jcg8j;U`B zT1v0TjsJgaeFsnz*!Oh+!2*i9iedpQC>^Cr4cO2Cf*@UrO0SAi0wkgQTv=TOC4ka` zXh6DBLMLFO6N(V2!KFmN5FtPaA<6fWxbFOCzBi7$?x?eQdGEe^&pqedZo8!7k0XGq z=S1&?Slo@;SWn^(F`9pzS8stQL5rB>5b&n%w2ClJ6kBh7uQ6(2E$B4kaaTR34KsmZ zm7`V(bB2FD>&(K$s`$j}i>FKVoY$MWTO)5AW)wNiIuSRTeSTeZrIl$Sk>R`-E8MLW zP=8VshT-O^W}=p*4|D%+ISkeS%US=zY7%T#b*t43u6}`wdj@hkecJ}SB(wdI^T?%$ z(}p|>H5iOrLe3!7)z;QNeBNE2f(vvTn9{B0iP5_WPSRZd4u5r~^Wv!+yVfRuxI2$q z-vnM&FFkd)7cs$FVw09bckn(6Oq4NxIrMVr>Bs`{STX&xXuRPV{d{Jp> zsfih}YpCHrEtNhWesH^^W>jOBT<{~xIzs;2^t#IJh&r^@KT2QTC0wQOXB^D@(oq1Wy|yUR=;&;sL{C;GVrj9V*A)`wascv#zFr5&E*uMHg_ z*49hMw4r50cW&MG@^YD5%D7xdBEgSt?AE(fGl+zI1QAahc)u?{^Orj{M_p+V4;b zBN_r@?7)24`>C()*76YStCn?QW?7q#B)E>3Zd$8$4bW+1I9CcY>SU&$?taq&T27#s zgVCQxXlc8G!4|8lE(jNpu-hLF*!sEM?LhM`dQ$TZP=|&w$++3jW5A3c;U5I-uWJP+ ztJ^lk^_-V%66mm;L%c-`q5=XoH{Swf3T!&D*wEL3623n-L2%81GZGH12*Mn z10wl{!!kF6+yQU+cKgiV&!5nN-U>Gwq&ZZ%;hY`ZO>#>8kMZ2er%x)8>;_T>y7LAk zYz5j$L`gHj_ad?HZHGaW@9l?GcqJ~aS)6c6#H=AliEgLXG{8K^4SN8{fujzB!4svP z-W0bYew=@~X-K#Iy|RN*WAWhNQFM;Hg-B+l=9bqLGRlprnDhWVs!z!wyjTOfXW6=X zhd#+WwGa3_q7uvF9rgp`{*j}Ysv&~kJgut2fzapKyTsg^U7rD&_9vw3K6y%r>&p_Q zClmfUw2vU;VO&5-A)-bwdG)S_oGV}*Z5OX~MT=cg3AxUd?}M3#9Smv)l^++LFslN_ zE;&uD*JhN0uYO}_UGMkFFyfV3mKat6`Zk%)4GqYA?0s_K-2!*jU6Y*a0b7y2ZcF0h zVJ8wn9hUR=Mb{v%2nEHMwI28Ko1ZKnDTOGVlMqvFIAy}khULU)Mz(6Y zNd-ycAVTR;qHxJ32edpUwPV|x*!Wq7tz)FJ2@7_R)o*xnFTAn$BGy3L%WJz=+|OYN zv2<3W(tdDH3~k!|Z|BF9F<1bl`$oaoeJT?|n@R)i@lRX_)4~v{=2l-}6bABH)FkV? z8MzIZt~oIm^EW-_rID7ghe(7w9AEAAd+gGRmFI;=$c?e-VNey zMT#kZAMk;dA(b>An}Rah0H|JF#x_?*Wwz0GDy}nLti4gVYB28bW6)Kg{gKvwu^lsu zhAbqfoEHmP^KQ(>?2?SPin!g9QydX}w(8*G3Nb|b6SJrov3zw53>wo{RXUs zWhPO6;N-DmnRHH{V2icLr?wI_dF9ID z5XBCX0@7^_8^5$VcenO}DkKcV=LnY5xgep@c4}euf)YZ4$~w_`J<|x>~8m~t8}_6VG;p& zjb0%rfaUh4$fSvVms5`RbSO)101k&5BmQo|PNFP#H|uCJs3Z9jKjB>TMTTi+`lfbp1cmZ%L#=K;g36+)(dD6g3DJ`bo? zX_AfT!%pPKob`7O(RE*_nj_3%R`P~{XJYVd^Ve&hEzat4DFZ2l3o*0FQCD!-F}}hU z0J`|=_&}+*Vmbs8&t?SPXhq!hUwOtq8N0;_b`NI~xiClXkgBZJ)Sb`LHob^RyCmgI z6Krw?+4zKU+vK01S+AXb=g_nclyj50I)95oX;=p%qPgLt7W#(~)8iWzgf1JAyN9Ht zq(HbZN8cYehN7lbOd(+7YAPZdqai_Sl6$EGDANwrH**-UV(P(zCkVII)V||Ise+R z*ecLSD|v|Q>oW-^J^Qk4GuFuteuN9iWe zXX#8un;fZZm!aDBvkYvzpz<|OA#Y*nC=%$B?Q{Kt*h4K(qx9BGCrfyTx(-F|Q?kY> z$=R$UGzl#>;!P^%`7z3HQr-OQU}CR~z0VNp)cTe5^+sfYtjI2vDBKru%OoHnMJJmIlN$DRMxyQsN^06|gV!mY{0CBvQ+vZ6&2 zC8@sA)Tgx5>B)!6hQs_AxeJ;>E)FfVa5c^31k_!pr_v7a$*c@0ZT$Cs^yu^V<7UWU zVIP?Q#`V>WJ=LB}C{WIrB%IiF+e^~n{K0!yxMTk23?IhdA*D<__`hc&hg(pLVYX(o zmVE3Ye%_LB+tq2=Cb%H^yk4O=ZS|8_P+((}1O2p;g~W_J3wbZK-ooSd(1R^Ey>ViR zZD*~qZA=f5xGu*58MGA_!5l=0CG@7m!Y>~2DObv{pQBOoZCCD=KCDNj21{e;YZpbF zaK)La^y0o20~oUb%|@Uu;hfav$`F(0R2@&R2U#LXh9L#DP7#Ohmc+a^W-t5^P0;wsL6H2BLic^1Ioal9a>}aoY z4KSv-M=xcM_1=;6*Fz3)Z`tD0zE8zHi4#-QEi;XIpf^DTFf9^~DwWfL%)DHFPnWE| zL*n8Y6KP8RWeUxOaH$H}#5)5O2B+XS%RL?)O#l4m%KYVt0-I!dTi-?bV051H{lz^G zUp+1(y`6hCj~#P`Kktim*|##<*;I$qPwnTTI_GN$vwebFOzX0w{Is@&+N@g)QJ)h- zY}Tutv#7&c#gkY~z5@^M#9MVSkw!yH>s|A;UzqE+rfJ#5G=6jA=T4oVL|SwzSRiz) z{7Kp2d6e96mvd1>UK%8OqijC#!`&&jMSdz!1B_XI2(W`!@H$-DZ zla;;t{R3?+f<2W0D23u$$qA zU%;y(5@T3#%9IwDG~#eT4*VXx_?RrpPx8>>SJF*+<7*UsQj|b?@Bg0?+@NAy$U-#`q|G%r`d0Y zpXp?6l5cssyX5D+&6_c7fJk^RuY3jxY7o#9{GW5u6ew}UK~wVUi{?#|_izde+uryD z-EPZyRhFViAmTrLJZYJlcEX`r=M;Ie|wPq}@!*(d+dFzVzVYu;vF3Q2+`9p-tW76KW`gJzyPPA{57e<&@YMD7^vd{n z;NWMGwPg7Fun^|hW)MnhaWf69+JtF&HTzpOf|sq><7%hi1KS=&$tC74(`qTUvZol_ zL;TMyJ+Ci9Ish-7iX5L|smDE#GtFnaKAMqHWM z{@?RNh$J!h4 zO2Y*!Rkjzh=-4-UGHRmTgL^0RgNCwHryK1YZwLDji8&quWGkT<>A8G?_90j0O6iFW z*CKsYUYVCW`61viNzU1YqqD<5WH;zGx{JK5!Xk7OF`E-I^Z4i`QjRQo*F}eXFqr*z zVMVjvsXKkD{u>`>c15rJCn!G!Cc=ZA^2R`|(-sXsCq<8^Uyfc{D_Hh!4&9%S_HHk( z9J}c*2r6KbBNX1wbhiw~lQ=~SgiSrjp#ph>ugYu# zYBjG)X!OfYb)?prykfO2@>C02-`_4h4=O8D(e!V9R6Ly28G9qZrhR3}9{Xgj7-o5W zmf~ldbT!EKGVYHuRNuX!#*(_s%!VGZH%jGp>*t0VRb{_dpxulqFT869Hi1)Hx*Rv{!EqPL#?ZXWr>Y8Tjp z0CCH1atxTf#a;GU*MoHRP|s;A0qF_ek20X0h?6x5?e+hGp@rvp#cz*PQa0%h2Moij zUx<^pL!z{$gZ%yd&;MO|LRQ7ZTs7aN`p=B2%!SwY=G_KNl@oYSz^{@I%>3e)!rW~B z#)6fTa=E~kfFyjX*p9EDC~07k3r{#XIh^90GnRMQIh{`;&Uk*ba~)vKc2xPsw867V zYWoBq8y*)*Nb5a_s27`ZTygQAg^5O(fo`%MNP?_llZ0PijrC9UxUE+7tWVH6eHhs> z+3wcgH%+UXrC*@D40Wi4OdqPM3yKbYqKaN;$&CULqw7q#vrACG!72qX6En~+GCDYm zJbanAFK?sDX#{TNaym0rF2tpl5}aw*tysb<_voKqg#8XE*^hq(ly}y?A7SN>p#`w~ z1XnIvc)tGHaLBF$f2=i-aJ!2WH2IjozpW?33#LKaJ&m7P#ok;{L_w1Vr`k6QPf;0- zz~|0|G&1aXUhfz(^_CW-zBs0>TtMm34i|aZiDG3sd5Bhh4ekPJA*t?3R@#A4Vkn%Y z?S5@)Jb2lCp!?wcuX4sCBR-F;h7r%h6e;wt%VckydO9+L4m$JZhoOoAp#>H5#VV5D zhf`#h#Npo2!AMoC@Oa7kT44WFIBgm%3J~kmOS{M1wRIY&s=v)vCi<=7%NzLB-xpP4 z-Bdk`A8E5{mFNE%xf$B%!9_%YzgAe7_Ius`D7eGM{L^#bR&$%EreR^u)xbY(e29xJ zNyVv+bO7~_?j)04{Y}pQIx77X_LSPu91>>*)WINmQ#4pRjcKH9Ll!wn{AiVl<_hb! z7)LrlprV9PJKS#5ttp7w%E|=80_J|C9ARD`GqTQXo%oZ>z#D<8rn!BX_BG9R&lj+c za?Ou%qV2%lk^0c@Q8(nRSt!RmiCwNMKjjBnnjZ87iQ;@#OrK}ou&xT2Te!gJx%19y zOxs9@s)!Z#nqj^A!oKb_%|H`@IC9A0f3alqr^4N)7{blQr3ht;&h6AF6%F^5KX|9- zwrTgtnGF`$Ne|G`JxZkYDAh=*(cig)Yz3+nz~btZ_^UpF2fSxTB^i+xpY9&$Sawca zqH>2W9{Y?j$GK?9LaO&{*F))z;m~YU5$NH~>W}A;fEHCT>@qJqzWV3N9}#}NL$RU@%a^lYW9r5b@Cb|u#gD-xir z@YO7FufEH;#iqn}?Jz%*W+9P&gRwz?O!sN7G`=9+m8azNwH%v&M)4MCr?OpO){dya zfvGdL68eNYXdBn4=;`p^re_imKxXRcRDjD;F!@1@GStX z&%U(EH2$|+osfa=hCc#H8HPsn`E&GSB27*5bq3R|llbqK59vA1FNv2Z8n0f{!(!S> z3$4l->p%JxVqu9(?u8!bkgk<_g|LDIS1tefSvMQg_qh^fJ z>jN%+Bj{nI{k7ZXxMR`Gj!?&Z;~(bMj&*;-TsC7j*h_Fb-#T57pYBe>So3*Rw3S;` zn(%b~COJHlDfjqJCqP?pj-S*H69!d_A3!|DvC05OA^h@2?}vZPsA`p&&IlR>S0i)=t`jp}crDo+LATMMMr|6XDa* z!ce)y2;((CCKFCNDW^*usxPA6Dt34(m6^QV%+L_Yx_tiQr%!ebpc_}kOajwq8XD{{ zMAi$)u%f6o8gS&IGp%B`sKv|2xfc#*oR##^^1kVE-xUTWNSBN&0_A(syvns}?eq*U zFL^oz-CThc`I5Uau72i4*)*+K@)ew(`U?JN+mfPz(eAlK;f?1BeG47eV1DF*G0yWU z+$M8RRJDA$Ew=IzY?i1s2IM9LoY#_}11x&&2l=$i^;~^wSrdFDi-_Hs-C2j0${wu< z<-@I~pfv||h=i%ATP1V=v6_!etv6*l(Zr}PbtM0Ppa`lav-ok^x zVL)tV_q)Dd9chYP#3@_DfOMA$eWx1^kXKx)*q-;#lZ_2|Jv4HjRwT^hHWg37>(XAGgkD=m6o9P~KY6Ag$KcW*?%xc_$I$fa z%EivCBSVH=f^V^Y?X1cQ+Ih<#QPbfUo>WyfNIxfz{V2L){W(CYpn|-cq|LnELBzQ} zFi)b#9V%VTIFbXOTo+F!)LG5&agyi<^PzQP@$tO5K=FlO#n0-S_{kPxo`@#d5E-vHj7t^ieN8F5N?hFT^d9MNQxrraG$=ml#)l-)83OLx{cJQZipY4}Scg{)nr=rc zb%AjZWL94O{rfNm+jKUQjA&TS&e>?BAx-tB7!GM3V^CrB4-Y)>mMGzNKkBrSqdaA< zrTtEv7+cw9wSR6$LD2A-11t6epv)#{hkyxxIwkp*!F5w1ys;R}%4M>%{8EjQ(Tfq- zc90XF_hE4h9soiigB$$2!>&6!J71{q8Uq({!@cBKJBVa>4eD>OuIbz_u&&aXD$#mD zxI1)^i}3^r%cE=@^E;;ZQ?EHGpmx65D0s;3X!_^*M%@{w+kefpp}h2UvKBGdoQzte z2b>+*LRkkkoHM}7?(c22@3#0lm7&oh@bIRYMcq7dTS; z4$IZxBzk`+`n>DaEXp~x{$1C>iyDTsrX8TCAJeN9@wY0JG{8@Dq}*)=4=0}>?&5a!E}97?o02yFR_r(3 z4Eg*=h&1=tP3I8%ESY6l-;=#CS77SmDHSVAZxrsQ#WYPe#y@NMe($7{kNQttRAbQ2 zFp*OjyVA*5Wu`e$=FPI?>KU1(PrwQSXh=9nCb>mdkC_e_HeZL=*Ff1R(qqWD*A-2& z^7!_oHdaV2`>0cgV*)+NSjoZLqu1ELVW&H&?GR5nndRm2P*#6@adEtAqx}?W_T4(m zm7{y~pPxU=2{ZH(!K8F~s;T6rFZb-g+SRpPshw80%#3{uNwLkF{`LkeULKSEQgmEPnfj^EUr*6Cy>>$>zUuP*1Hhj*kU3%uyq z@jsPmX2(#a(r8EUmYo5(Ll#|b&bH^Cm`;b2+--zK%Y0*&MlD=WD@8ex)wQ4b(%I;x zDczWi&222JK~W8-g`hU#1uEe;87X@xYXHIfHWCJEN?>xxegCT~;slDwsw$q+F^@&+ zB{j`>pICV~}L%Akshn=M3X4*3qRH%)2m7c+bu z9szJcSy3|)``2O@H2q+FWZk_5ieVlGWYKp0PLz zsy(HPB5&O%g7Veg-s%-lb&xyigxdB9dPU@ndo$XjBYXY=huz(Ml)Y?m3=DW{T+4Z4 zv3`WbE=3*YTBZ(pC7@ElW~pe($%&3@u;@wj#SLV7PC&C<2~l+GsP!f3Qt^#Nt15-^1WvsaESXXf)u_p$X;hAVpcivXQ*=KYyvPAp z%UD^gY^!O1E0LQjwH$|GdM@HxDQ+ccacgy75{A zTvFBfk30!4*Ng*#tp#$Z{o*GlCo7A>8C*qN<=4{B#kvl_c5;%)C6lPh>BJ=o*#FN%xK*V+XImth{#4+933a`VM(8WR&egUQV&JuK0+@*4@Rm zsS1(quvV%&YzbrPw+P*hN{m2DRF8!Y@2fX8+)S%!jY>8Q?2Jm*3bLYC9|D`F)aU