From 5fdb54ece78b5d277fe26a3865beca8da0430495 Mon Sep 17 00:00:00 2001 From: mraunak <83710963+mraunak@users.noreply.github.com> Date: Wed, 18 May 2022 10:39:02 -0400 Subject: [PATCH] Add Information Gain Filtration algorithm (#16953) * Add information gain filtration algorithm * Complying with black requirements * Added author * Fixed import order * flake8 corrections Co-authored-by: Javier Turek --- .../information-gain-filtration/README.md | 100 ++++ .../igf/__init__.py | 0 .../information-gain-filtration/igf/igf.py | 419 +++++++++++++++++ .../requirements.txt | 6 + .../result_igf.png | Bin 0 -> 34410 bytes .../run_clm_igf.py | 438 ++++++++++++++++++ 6 files changed, 963 insertions(+) create mode 100644 examples/research_projects/information-gain-filtration/README.md create mode 100644 examples/research_projects/information-gain-filtration/igf/__init__.py create mode 100644 examples/research_projects/information-gain-filtration/igf/igf.py create mode 100644 examples/research_projects/information-gain-filtration/requirements.txt create mode 100644 examples/research_projects/information-gain-filtration/result_igf.png create mode 100644 examples/research_projects/information-gain-filtration/run_clm_igf.py diff --git a/examples/research_projects/information-gain-filtration/README.md b/examples/research_projects/information-gain-filtration/README.md new file mode 100644 index 0000000000..bf95cb8ea8 --- /dev/null +++ b/examples/research_projects/information-gain-filtration/README.md @@ -0,0 +1,100 @@ + +# Information Gain Filtration(IGF) + +Authors @Tuko @mraunak + +This folder contains the code how to implement IGF for finetuning on GPT-2. + +## What is IGF? + +Here we present a general fine-tuning method that we call information gain filtration for improving the overall training efficiency and final +performance of language model fine-tuning(see paper below). The method is an alternative fine-tuning method that trains +a secondary model (e.g., a simple convolutional network) to predict the amount of information +gained over a given pre-trained model. The secondary model is lightweight and trained to +predict the Information Gain measure. Information Gain is defined as the change in a loss +function for a model before and after an SGD update with a sample (Equation X in the paper). +A small subset of the training set named the “objective” set, is used to measure information +gain on the pre-trained model, and consequently to train the secondary model. After +training, the model is used for filtering samples for the fine-tuning process. Therefore, +a high information gain value would suggest a sample is informative, whereas a low value +would suggest a non-informative sample that should be filtered out. Thus, a thresholding +strategy is defined to select informative samples. With such a strategy, samples are filtered +and once enough samples are selected to form a mini-batch and a usual fine-tuning/optimization +step is applied. The filtration process is repeated until the fine-tuning process is over. + +Paper [Selecting Informative Contexts Improves Language Model Finetuning](https://arxiv.org/abs/2005.00175) + +# Results + +Several experiments were conducted to show the robustness of the IGF method versus the +standard fine-tuning process. For example, we achieve a median perplexity of 54.0 on the +Books dataset compared to 57.3 for standard fine-tuning on GPT-2 Small. The code was +implemented using the Transformers library and Pytorch. While the method may seem more +expensive, we saw enough evidence that it may lead to a performance benefit in the final models. + +![IGF performance](result_igf.png) + +Figure 1: Comparing IGF to Standard Fine-tuning: +IGF with constant (p < 10−3 , t-test) and shifting(p < 10−6 , t-test) thresholding significantly outperform standard fine-tuning. The left-hand figure shows +test-set perplexity after each fine-tuning batch, averaged over 50 runs (error bars denote ± one standard error). The right-hand figure shows the perplexity of each +method after 60 batches. IGF with shifting thresholding (red) clearly improves over standard batched fine-tuning with Adam + +## How to use this project? + +To fine-tune a transformer model with IGF on a language modeling task, use the following script: + +- `model_name_or_path`: Path to pretrained model or model identifier from huggingface.co/models +- `data_file`: A jbl file containing tokenized data which can be split as objective dataset, + train_dataset and test_dataset +- `igf_data_file`: A jbl file containing the context and information gain pairs to train secondary learner. +- `context_len`: The maximum total input sequence length after tokenization. Sequences longer + than this will be truncated, sequences shorter will be padded. +- `size_objective_set`: Number of articles that are long enough to be used as our objective set" +- `min_len`: The minimum length of the article to be used as objective set +- `trim`: Truncate the example if it exceeds context length +- `eval_freq`: Secondary model evaluation can be triggered at eval_freq +- `max_steps`: To calculate training epochs +- `number`: The number of examples split to be used as objective_set/test_data +- `secondary_learner_batch_size`: The batch size of training data for secondary learner +- `secondary_learner_max_epochs`: The number of epochs to train secondary learner +- `recopy_model`: Reset the model to the original pretrained GPT-2 weights after each iteration +- `eval_interval`: Decay the selectivity of our secondary learner filter from" + 1 standard deviation above average to 1 below average after eval_interval(10) batches" + + +```python +python run_clm_igf.py\ +--model_name_or_path "gpt2" \ +--data_file="data/tokenized_stories_train_wikitext103" \ +--igf_data_file="data/IGF_values" \ +--context_len 32 \ +--size_objective_set 100 \ +--min_len 1026 \ +--trim True \ +--eval_freq 100 \ +--max_steps 1000 \ +--secondary_learner_batch_size 128 \ +--secondary_learner_max_epochs 15 \ +--number 100 \ +--recopy_model \ +--eval_interval 10 \ +``` + +## Citation + +If you find the resource useful, please cite the following paper + +``` +@inproceedings{antonello-etal-2021-selecting, + title = "Selecting Informative Contexts Improves Language Model Fine-tuning", + author = "Antonello, Richard and Beckage, Nicole and Turek, Javier and Huth, Alexander", + booktitle = "Proceedings of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing (Volume 1: Long Papers)", + month = aug, + year = "2021", + address = "Online", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2021.acl-long.87", + doi = "10.18653/v1/2021.acl-long.87", + pages = "1072--1085", +} +``` diff --git a/examples/research_projects/information-gain-filtration/igf/__init__.py b/examples/research_projects/information-gain-filtration/igf/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/research_projects/information-gain-filtration/igf/igf.py b/examples/research_projects/information-gain-filtration/igf/igf.py new file mode 100644 index 0000000000..99bd8c2d06 --- /dev/null +++ b/examples/research_projects/information-gain-filtration/igf/igf.py @@ -0,0 +1,419 @@ +# Copyright 2022 - Intel Corp. All rights reserved. +# Authors: Mayank Kumar Raunak, Javier Turek, Nicole Backage + +import copy +import logging +import random + +import numpy as np +import torch +import torch.nn as nn +from torch.utils.data import DataLoader +from tqdm import tqdm + +import joblib +from transformers import AdamW, GPT2LMHeadModel, get_linear_schedule_with_warmup + + +logger = logging.getLogger(__name__) + + +def set_seed(seed): + """ + For reproducible training + + Args: + seed: A seed for reproducible training + + """ + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + + +def compute_perplexity(model, test_data, context_len): + """ + Computes perplexity of the transformer model on data in test_data + + Args: + model: Pre-trained GPT2 model + test_data: Data on which perplexity calculation is required + context_len: The maximum total input sequence length after tokenization. Sequences longer + than this will be truncated, sequences shorter will be padded + + Returns: + Perplexity on input test data + + """ + + model.eval() + device = next(model.parameters()).device + eval_batch_size = 1 + context = torch.zeros((eval_batch_size, context_len), dtype=torch.long, device=device) + eval_dataloader = DataLoader(test_data, shuffle=False, batch_size=eval_batch_size) + eval_loss = torch.zeros(1, device=device) + nb_eval_examples = 0 + for batch in eval_dataloader: + batch.to(device) + # pad + context.zero_() + for i in range(eval_batch_size): + context[i, :] = batch[i] + outputs = model(context, labels=context) + eval_loss += outputs[0].sum().item() + nb_eval_examples += batch.size(0) + eval_loss = eval_loss / nb_eval_examples + perplexity = torch.exp(eval_loss) + model.train() + return perplexity + + +def load_gpt2(model_name="gpt2"): + """ + load original gpt2 and save off for quicker loading + + Args: + model_name: GPT-2 + + Returns: + GPT-2 model + + """ + + model = GPT2LMHeadModel.from_pretrained(model_name, output_hidden_states=True) + torch.save(model.state_dict(), model_name + "local.pt") + return model + + +def recopy_gpt2(orig_model, device, max_steps): + """ + Reset the model to the original pretrained GPT-2 weights after each iteration + + Args: + orig_model: Original pretrained GPT-2 model imported from Transformers library + device: CPU/GPU + max_steps: number of training steps + + Returns: + Original PreTrained GPT-2 model, + lm_optimizer: Adam optimizer with Decoupled weight decay + lm_scheduler: linear scheduler with the appropriate schedule + + """ + model = copy.deepcopy(orig_model) + model.to(device) + + no_decay = ["bias", "LayerNorm.weight"] + optimizer_grouped_parameters = [ + { + "params": [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)], + "weight_decay": 0.0, + }, + {"params": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], "weight_decay": 0.0}, + ] + lm_optimizer = AdamW(optimizer_grouped_parameters, lr=5e-5, eps=1e-8) + lm_scheduler = get_linear_schedule_with_warmup(lm_optimizer, 0, max_steps) + torch.cuda.empty_cache() + return model, lm_optimizer, lm_scheduler + + +def intermittent_save(contexts, real_perps, past_perps, filename): + + """ + save the perplexity differences to filename + + Args: + contexts: Example on which the perplexity is calculated + real_perps: Perplexity after back-propagating on the selected context + past_perps: Perplexity of model before training on the context + filename: File to store perplexity differences + + Returns: + file with perplexity differences + + """ + # save the perplexity differences to filename + avg = np.array(real_perps).mean() + std = np.array(real_perps).std() + perp_diff = (real_perps - avg) / std + data_final = list(zip(contexts, perp_diff, past_perps)) + joblib.dump(data_final, filename) + + +def collect_objective_set( + model, + orig_perp, + context_len, + train_data, + objective_set, + max_steps, + device, + filename="dev.jbl", + recopy_model=recopy_gpt2, +): + + """ + Collect individual IGF values from pre-trained transformer model + max_steps samples of training data to train secondary model + + Args: + model: Pre-trained GPT2 model + orig_perp: Perplexity of original pretrained GPT-2 model + context_len: The maximum total input sequence length after tokenization. Sequences longer + than this will be truncated, sequences shorter will be padded + train_data: Data to train model + objective_set: Contexts used to create (X,IG(X)) pairs which is the training data for secondary learner + max_steps: To calculate training epochs of model + device: GPU/CPU + filename: To store intermediate perplexity differences + recopy_model: Reset the model to the original pretrained GPT-2 weights after each iteration + + Returns: + file stored intermediate perplexity differences in intermediate stages + + """ + + # initialize variables to record relevant information + contexts = [] + real_perps = [] + past_perps = [] + + # Initialize the transformer model + orig_model = copy.deepcopy(model) + orig_model.to(device="cpu") + torch.cuda.empty_cache() + + # Compute perplexity of initial transformer model for comparison + model.train() + model, lm_optimizer, lm_scheduler = recopy_model(orig_model, device, max_steps) + + for step in tqdm(range(max_steps)): + context = torch.zeros((1, context_len), dtype=torch.long, device=device) + story = random.choice(train_data) + start = random.randint(0, len(story[0]) - context_len - 1) + context[0, :] = story[0][start : start + context_len] + lm_optimizer.zero_grad() + outputs = model(context, labels=context) + lm_loss = outputs[0] + past_perp = compute_perplexity(model, context, context_len) + model.train() + lm_loss.backward() + # Do LM backprop + torch.nn.utils.clip_grad_norm_(model.parameters(), 3.0) + lm_optimizer.step() + lm_scheduler.step() # Update learning rate schedule + + # Compute perplexity after back-propagating on the selected context + real_perp = compute_perplexity(model, objective_set, context_len) + + # Periodically save the stored (X, IG(X)) pairs + if step % 1000 == 0 and step > 1: + intermittent_save(contexts, real_perps, past_perps, filename) + + # Reset the pretrained model to the original pretrained GPT-2 weights after each iteration + model, lm_optimizer, lm_scheduler = recopy_model(orig_model, device, max_steps) + + past_perps.append(past_perp.item()) + real_perps.append(orig_perp - real_perp.item()) + contexts.append(np.array(context.cpu())) + + intermittent_save(contexts, real_perps, past_perps, filename) + + +def generate_datasets( + context_len, file="data/tokenized_stories_train_wikitext103.jbl", number=100, min_len=1026, trim=True +): + """ + Generate objective set and training set + + Args: + context_len: The maximum total input sequence length after tokenization. Sequences longer + than this will be truncated, sequences shorter will be padded + file: Tokenized data split into training set and objective set + number: size of objective dataset + min_len: minimum length of a context in objective set + trim: If True truncate the context if it exceeds context length + + Returns: + Generated objective set and training data + + + """ + # Generate objective set and training set + # Designate the first number (100) articles that are long enough to be used + # as our objective set, rest (that are long enough) are training data for + # secondary learner + + data = joblib.load(file) + print("data loaded") + objective_set = [] + if trim: + for i, example in enumerate(data): + if len(example[0]) > min_len: + start = random.randint(0, len(example[0]) - context_len - 1) + objective_set.append(example[0, start : start + context_len]) + if len(objective_set) >= number: + break + train_data = [] + for j in range(i + 1, len(data)): + if len(data[j][0]) > min_len: + train_data.append(data[j]) + else: + objective_set = data[0:number] + train_data = data[number:] + + joblib.dump(objective_set, "objective_set.jbl") + print("objective set saved") + return train_data, objective_set + + +def train_secondary_learner( + secondary_learner, train_dataset, max_epochs, batch_size, eval_freq=50, igf_model_path="secondary_learner.pt" +): + + """ + Train the secondary learner (igf_model) + + Args: + secondary_learner: secondary learner + train_dataset: data to train secondary learner + max_epochs: number of epochs to train secondary learner + batch_size: batch size of training data of secondary learner + eval_freq: secondary model evaluation can be triggered at eval_freq + igf_model_path: path to store trained secondary learner + + Returns: + Trained secondary learner + + """ + device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + # We will use the first 512 pairs from our dataset as a test set for + # our secondary learner and the rest to train + test_dataset = train_dataset[:512] + train_dataset = train_dataset[512:] + train_dataloader = DataLoader(train_dataset, shuffle=True, batch_size=batch_size) + test_dataloader = DataLoader(test_dataset, shuffle=False, batch_size=batch_size) + + # secondary learner model set up + loss = nn.MSELoss() + test_loss = nn.MSELoss(reduction="sum") + secondary_learner.to(device) + q_optimizer = torch.optim.Adam(secondary_learner.parameters(), lr=0.00001) + secondary_learner.train() + + # TODO in original code this is written as number of actual batches seen + # not number of items seen but other places it is number of items instead. + # improve consistency! changed this to epochs for clarity + best_test_loss = float("inf") + # Iterate through batches until we've used max_steps batches + for epoch in range(int(max_epochs)): + tr_q_loss = 0.0 + secondary_learner.train() + for step, batch in enumerate(train_dataloader): + context = batch[0].to(device) + real_q = batch[1].to(device) + predicted_q = secondary_learner(context) + q_optimizer.zero_grad() + q_loss = loss(predicted_q, real_q.float()) + q_loss.backward() + q_optimizer.step() + tr_q_loss += q_loss.item() + + # model trains fairly quickly so we won't wait for a full epoch + # eval is triggered at eval_freq and end of epochs + if (step % eval_freq == 0 and step > 0) or ((step + 1) == len(train_dataloader)): + tr_loss = tr_q_loss / (step + 1) + + secondary_learner.eval() + q_loss2 = 0.0 + sum_q2 = 0.0 + predicted = [] + actual = [] + # Compute performance of the secondary learner after this batch + for step2, batch2 in enumerate(test_dataloader): + features2 = batch2[0].to(device) + real_q2 = batch2[1].to(device) + predicted_q2 = secondary_learner(features2) + q_loss2 += test_loss(predicted_q2, real_q2).item() + sum_q2 += torch.sum(predicted_q2).item() + for ei, i in enumerate(predicted_q2.cpu().detach().numpy()): + predicted.append(i.item()) + for ei, i in enumerate(real_q2.cpu().detach().numpy()): + actual.append(i.item()) + + q_loss2 /= len(test_dataset) + print( + "Epoch: ", + epoch, + "step: ", + step, + "Avg. q:", + sum_q2 / len(test_dataset), + "Train Loss: ", + tr_loss, + "Test Loss: ", + q_loss2, + ) + if q_loss2 < best_test_loss: + joblib.dump((predicted, actual), "pred_vs_actual.jbl") + torch.save(secondary_learner.state_dict(), igf_model_path) + best_test_loss = q_loss2 + + secondary_learner.train() + return secondary_learner + + +class SecondaryLearner(nn.Module): + """ + Our secondary learner + """ + + def __init__(self, model): + """ + We use a simple convolutional network as our secondary learner + + Args: + model: Pre-trained GPT2 model + """ + # embeddings are from the pretrained model + super(SecondaryLearner, self).__init__() + self.embeddings = model.transformer.wte + self.embeddings.weight = copy.deepcopy(model.transformer.wte.weight) + self.conv = nn.Conv1d(self.embeddings.weight.size(1), 256, 3, padding=1) + self.fc = nn.Sequential(nn.Linear(256, 32), nn.Dropout(p=0.1), nn.Linear(32, 32), nn.Linear(32, 1)) + + def forward(self, context): + """ + Forward pass through the secondary learner + + Args: + context: Context input to the secondary learner + + Returns: + tensor after squeeze operation + + """ + pooled = torch.max(self.conv(self.embeddings(context).squeeze(1).transpose(1, 2)), 2)[0] + qs = self.fc(pooled) + return qs.squeeze(1) + + @classmethod + def from_pretrained(cls, state_path, model): + """ + Load the secondary learner + + Args: + state_path: Path to save secondary learner + model: Pretrained GPT-2 + + Returns: + secondary learner + """ + + secondary_learner = cls(model) # this calls __init__ + state_dict = torch.load(state_path) + secondary_learner.load_state_dict(state_dict) + secondary_learner.embeddings = model.transformer.wte + secondary_learner.embeddings.weight = copy.deepcopy(model.transformer.wte.weight) + return secondary_learner diff --git a/examples/research_projects/information-gain-filtration/requirements.txt b/examples/research_projects/information-gain-filtration/requirements.txt new file mode 100644 index 0000000000..2aa3227637 --- /dev/null +++ b/examples/research_projects/information-gain-filtration/requirements.txt @@ -0,0 +1,6 @@ +matplotlib +numpy>=1.17.2 +joblib>=0.13.2 +scipy +torch>=1.10.1 +transformers>=3.5 \ No newline at end of file diff --git a/examples/research_projects/information-gain-filtration/result_igf.png b/examples/research_projects/information-gain-filtration/result_igf.png new file mode 100644 index 0000000000000000000000000000000000000000..10bb0b7d681630c668d11dec6c6606b9934f168e GIT binary patch literal 34410 zcmd43g;!MF_XjKuf^>)=-QC^YEge#WbPZi1Al(QIDI=mFg3=7#4GtjPIZA_cy%(S7 z`~2Ru-hbeAEtYF8%suCxv(Mh2{n?*=V|BEYpFDo{_|cASDV&eVi5h>%t50a`L!{MVx zQTM6}vIhQ^d--U21~0Dr?teS`Z98MIuzaK`wRvfS4E;Pq$o9GNJso2aCKQv9ZISic z#`q%e&qC_{vLI;8%g@)>59H?n@;kc>mD;ZMwgrtYK}6ev3QoH3W1%c@P!o`JM12742fl=-=f(^K!7O4Y&0sMUiW)bwu-y6vN z|Mv#7mS74Y7ZxLP&!~lpr@lHF#PRQX-t}r>k?$mG6X?{JU;{ z+H;0pZjv?L91o$~So3Y!`-Y zTy7Wo)n^#5TpX<*7vA6WEyi)M!PsE5MbI7Kd%(_A(XqjF6%>U948YH8@Nn5+YKFk4 zI0b$1<%+QD9K+6ZC0^NgQeMCPh5uWFkv6>G!sO@TTcKUr=_U&2{i*a`e-OHUL#xYS z_wR16&xZ!mnHMEvb>WGU2H+1WH40hOB9)CkJBB9_cqMx?-|34_y5Rq;CvdO%diSi& zp{!il5%n5Hzk{U~y%G)b#*Th8KnJ5R)>`sbT%Q}IgDtTkeP4R5xevC zV~bufB?!82MwS5|FP{KgX<4F~n@Q7ZnO%E--gYm@@5jvs`yL(VMm!J)HT%;CukziN z^>epVJ&;Hd-$2kRsn`_7}| zQnEx~;AzcJ3Aq@zZ@-FnS@#w>8RlQq7Onb>Kxc*Jf01Hcv>agE$r|%HFHRn2fO&d) z97TTmnq!e5nOpljy&-ISWuS4dnTK46u|Bj6vR ziCOg&NI6aI>c}MsqW*OH&=3&`6TsyZg++MC?w9Xx3(Jkace;32E0BsExMcOklW}Lx zyESpHHX%Dtm8Sw0Vo?x$JP_%#GdcId$bxY@&pXKxXLtDXu5Y8)CbE>WnH`1jatv&f2DnYvC01G&#$c`tFT)}tI*4j0WL*&C|MDg@wfNQ=SxALl0154+;t5Uk$BI9?F??KCCqmui=PTZ=VI1w&zHHW!)`p^ zl-9Y-R_Qf**|X}re=YFpM|)%uF{kNdGj1VCqkcNkqL|Nbl~TQr!$S8I$<(12tEljh zo1?yw9p3BJQ-pYgd(>l+VzYX;ElSWi`t+&Kx=d52o?rgkEt6#!&l=QW4_r94Q+{0M#!^Ek6on;kj;s*Q06mT*k{Lyv3 z4hwl#dg9|)G#<53#BQDIyhc?}B;o2RcJs>)WN*v$-o(v#?pB^;tDrr>^yO#xd;Fs! z^s->ylSh>u?}gkJM5+^c0#E+5Uhg-JLlv=4o+53uq!C#hZ_xd~$-}7_4s3z$2y)*a z^-=4=z>bcyRUaGCW4Q0dX&KH+!jWd@ju50EwO6PpuZj=ieNbL&HuZZ5cj>kRU7iYk;>B=>${%!-Z$OyTEs^c&f zkuiihB*wEn$Fao316g2?(&u3eLwS|W@C6jIrroE?ApzWvwH5+*zA7Ce!Sls@$^vj= znn=`MX6C0Mj>CA``Sl&XX=;Igp~wv;_>2`uf~alO3L)h1?C|P441=825)}Mg3bzJX zMdA1uw?Li0UO3_rUN@ffjm~2-uDlfl3eHt!+Z~KN8!Pz!=46b>zPpHnEJC0}tr_DM z%Mko2VV5@jcsoTiC{I?n#g6o(N;-y^Uqk^DUdwMN6M1-7eDInzse@#7^~&sUs@oj< zj%g;>y>&K*iysGdI&KCqW2bIH{KeU7hupr`qU6=@qc+f4W4Y+GWBQp3gKf^3nPCeG zKMtA>TFpSi+16*8IRXR0Mu&#=Bfa`WU5*Ya!K5x<5M_vx$T+txARuAlTc8pW;airG zc861=k)={gqlkj!pGpSSklT^7kk>#u@Z+}`jp6y?K~=+}@8m%hOZ)q()DT55L3@E@ zP|h9CiS1q{IpwI_*&NSnFZ!-WZ&)*>UV;?!^ocCgVBfj0ceh2GnX9cqL)rD@#aKfs zTHr56m0W^OlRU|NNza`v=^5c@oifHvQ8LhYRf=lf5_eQG+U>B_`>ie4i^J8e;EKjQ z?CNtS&7AB$bACbYFNl)2|I8h=_4aID0lG8hw;Yo7jVi`*lESYqVMgrU81N#S?3qJv zC`IiqiYIJRsQBU$Ybtin-{Ujn`nb_*%AnPl6@b6 zbe8Y8#5;zh(N$iER5{ijp+62F1N$ZHBDsZc=HIA4kw%j1qEFel`eRUvc*w9BmcJVP zYFb+s{0FXF!hI*;G}&L?adk55P{=4Ee`F47=>C0s5O&Xh{b)yZXX@+7`bYJwoP)<& z`q}eX``la#-WlUK3VciR(C9qt(7%B-Cun3prWEn9tVb`K)QtXmLg@M)?03hBAJW^58sQ zFo?v0vAX@-Kv7r)IhHGlKl{ex=BQKby+s7da6bTM{OT#+2y(CM@HhKwIhs_)H8F=2 z_cc>$0&wXsIM6vcb~FIonCp4DwD5g z$@TY7r>)$lp0-5N!6FKpwM~OpQGb3SBU!BZ5aQ8a9-;%6O#Gk4e|9Jg5z?hcLJlyv zJ)c*2+)h?t;xUlMs3{U7T@>SXKaQs|q+Q^3UJSUW|IgRsfB`}VNm7JJ;u`*}v>6*VRD zsjT9;02>TK^xy*15QYKff$xH3wp=~*&`YAhv2G{VioRdH&P%ALT8~ynbA>Bo2N;h~ z@aNSUqO<%IA}V|3|%@)Urgii%{opL03#UqXwW`{C7N7q zTg@f2$r&8d29b<9*0tXPo?@eMyMUi+13;R*OB*k6DrhqV^l}Q&`E9STf4Ip zVWSH3ttNRk{i|Qeac;XsPzpskui~p(7*5a{m7_qnDr+`XTFgH0a(ivCLfqa3R7AAu z<$INMaYc|1j=xq~okPntdbliDWlu#H(~A=CDG!b8>stp{q#5luoT`|o&`6y$s-yDi zQMK$BX>k)KvZy|;=itG%<9-uD#7#5nje&lq3e0<@A@o?boaGpc*r0M6EkdM ziK!fih=1)BHrwGCff(Hr#)F9WUMQX0Hi}uTUPRUt(KekGFTP|FP)NyF zDAxc!gDO+IIo69~!=_$1sQcO~_^3zqxcbY#7=swTw_KE(z!sX!`aR1Mc(yxEx_m3G z@Ayc!Sw zQJ^qNb};kogZ~?0p`4R)gzvBMo=M)?kGC$TEDF<2nZvs?Q}t+|m4R5UfO=+)?3c$P z{MFXJY6T91_;YlZGHo&2AX2w@ibrUIcX!wB^f17|))H7+j>(bogYND4)+KjeKYwBX zj)-X0DM&?T2@boo^!~7o2Mzl55%8TgQ3p7!NXA|lsR&ZM^Rq=;$MUL`@v}@!$WsGD>J`k@1XIQ*(T*Ax@AJ1RM1@WceG(}MqweS z`cG<&p%@BjRDI9N?QwI7 zyNto-TXkO43e|_@$9-&^=RGRAt^9dF&xfV?!41SQ3^=!)AZa z^iZK6ysmI(4)OeeEy(^SN)rRHJpSTEYKVPc0i|A@?$z`W5Ul99hQ0PU@T1GVRKNxo zu=F@OM~#vAU4Ve8NzDT{_0&=YmnztO`NLydvtsiB9IzwfK=97cwk4T0o9;|1-g?9FVS zLx*5{-gC}}rAHMDJ~-rl)at&J%X7D;e7N6qz*P0t7{KU3-Q@w0Q-Lsy{9z>)sJQ2x zF{F?`y7}sBwG!kZ(5BI)BSnb^P-Zk#e}WS7?1b#qPHT`bp<>h8?PZa(@k>k%sQ9)y zj1?$Ej05SjW|E7{OVQ-2;9)w?m^DdO3KbHTrR0$20-r#eM>QWW9_enM20SVI*?c_6 z9-Cyw!+9e813_O6ZYmjiR^x*>_A9m-dCJV$4E<_LjKY75(%L0rMvKVq7R~apTdm??N6WdVjOf~3Eyy<^+zWpA7|0Dzp0A=u;|cDa?Sx%Xm6 zAbl275$rIQ&*{BCL|YhYrtSM&1I9%B81l0@?-|7BeX_vc2!$Dq@S>02z5*bFBggY$ zb6#@q58Wdq2N1%@G*53~j};m3iMJCpHvkg!nzhe@1ddL=E-9sdu}f496}Roh5xw56 z9>I%LbOzyOZ}3g*3KPTgh20A@h709$rYg9wh4*}tixa`I`{T5&i75L^{ZO0re-iKn zQj1LHr}D=04j~zAwuuzNLY5mO?1mXb+-(b(pK2{p-;leSo96}K79ge)Bs|v?Y&#Bx z?x8F|8m&AW$MsD}iG$Mb^Nq(V|FC0}nyrFYgk^{%wlsaQsWO%kWiIc(!@3c}rTz(4 zwDgK({L!5Do+Yp|ZzPc6wiK}G}cMABIYZS~p1T+(?X=#xBH!?CWC#P>bw z`7$RJvNrkha8U%A$Ro4R#NHA?37=QxBapwR13*JK$k%5;+0aZ5+wJ6T*-V1Nw zn|8d)sHyxSy`#LM@E#D7`JAcw0yWRS_fhe`QNs~(R6hz13_o31t;~NrFo99@ntIDcr zfoGD({qzL9qS0)ykY#q-F6m)rzcBG`$uyyHdKh)Wg-ji#Mi@M*=KcR`bJMbE0)3kg z-R+dTCX3g6~_j^?G4pQ@zY3*{ldn-rbhKk zFQql8rZIWo(vi8zkuHuEz9VqoP7`L6|o6_I8^e9yyV%t4F?gEd{p$R~@{QS*jmQR1SA4m%04IDVm#5hcA`h%lc`rGHo zS7}PW-bQuyCQ=YF{yHQqUrBv3aD8vmuypq9N_goUa2UR08U-~mT$XKi3y;wxB9D$w zF+3UF{TdXfJI9F%d9_|}XdhY7uEr(y;FA4om=))D)ux2lHvqrJnv={c?#%^B>(uPu z+ML_8_lHn!MSK4?n8UaLNS5-iwsk1zd{u%3mfwYz>7t-<+a1ZQ# zX?<98pj=Y=H*=85W9UpFG{@*JFi5iNUlY^I8QDr0Wl@=ul}naEW%tK?Vp4gtdQv}~ z(v2g~G-7JSqb2J|U=fFO;6o<`rO7Nb+*Pjq^B4XjRAkfOx9ddLRs0FUla#D_p)sw$>;T= zxxCJ4jq&%fUSW1CohB(KCh>MtgU%@)()gIPBiPWPgbW;0j@lX1%HcmM8!}LgQ2Lw7 z802jaF5YruMuaq-@-gNe^YzsYs-9~*dlP@Q@(`jtglsm&^LKo1CUcTsZxzpH25hT5wmXUE^G3Ywwt6AQ#$$bU zlXJRAV@K{yNySo6P2eA^@Ssksre3$N&R|}7!!i$qi{o9ybL2=J+Js__;z0bh4(FFg zE8JVZ0di<8kNQ=~kc?KDRI9Ma+~sl}Pk*&WYThUDH7&UyDR;L&{#LEx3&W{*$+@C; z)aZI_?cZ)>Y40W7AX4lecl;Jtk-x|;Dm<&(cMgNE+ zQ}OE`OUr*T{Hl>B-uV`TLzvi{yY=2Y$n59aF`_So_rjIdm9z&D`1-oh+vZ`nr(4{G z^reK2ysKeyuUy-{BB~0<3J-sd>Mh|?+TU8KMZ>`4Tk8XN?{KkWi=Xib1Z3$cUXqf4O_L#nw#%0*|>FZ1Q48x5j zV4A!rrYbkKq8r~2#6Nw>ZfQfTJw2UkpVo7!ygMmt=35~dm}jlb*tZgUU|KG6Cf8%X zBJ_4&R|SxwQ-C5d-#!2H6ilERH#c!1eN&Xl86mJXj6*>Ceu}G_T}2bzQpn9OhAzk* z^Q2HP3`Lg8v9j_}wG2pU>Wz)80gFS->%%;N70WpVvC{!atby;#+I{DS);PLC_U#3W z@wda7=8<0n7S|Rai|m`}*N8ddx0DV_ah0f@8n&9*2cm7PsIjMkQrPI^Dt5x`+X6h@*^o^r~1A|(ft#z*kM!6 zd1<1A{rQ6dtkwrJU(mG|mGy1o6@Tu3( z7%y>Jt8ZdHs}tK2Ejy6&$*C-gWIi4Q>?TcNoo} zk_^nwrN^Ew10H|tJBbDAk9`z3^iED#GdHW6nQWopomh{ zn~{SsMUJiLu>wO<{dWNK9C~lxVc!3~yF7B%Uo8b zb}Xd${2ldmETkjH)iT}Hc}j8EgMi_xsv)ODi3tdbMn>tQ;{oM2_(`y}~R{j`{Ykw(_$<^Q$YqfNcl<;GmD+#5U zi&on91$@4h`IB!?c~nlaLutTg#_xNdJRWJnn_co+j|)o z@~1GD`OYe-I{D7RYk(1ydv0Af8v2YRi>G6qxW^$90=dohI5ODg#dTV!ISl5|ycyfw zuC9VQTP628!!btdpQMMGDrB8^g2x;%B!kWbM6+)d{eXg0tXnW2A<5zl{d_!EHHOXJ z8(LzxeO*WLDgKueZ@B4qFww z$tfdTF;VsgEVT!rn(@d5>6yK+-L5-#F_r1N-yxCgJSs5&nNfZswsN{HNzN`Mz^rTh zZVehtM4xBK+m`g#=%wvWpYG`%D9-{}VW-pLs{76hSz>#F4AmQDEeor{nuY1?pT}QN zHyo{r22&I0UOy{`(4oelMjidasP=N3YM{04ReElD)!R4>GyVXo>)WC6HO3SnJE;k= zK#ik#LS_5B7!sCO81I zxl5oA9eN#I8rO^W9KM+}vwtE)>B=)};lCOM^1ytnqcAP@&iE;UT%sk?$#Ti_RW!#T z7Cd&~tLB@8b!H-QH#Bt+A2-Gu2&$FFJ$clgyui_c{QFSk>SdYhbo%TQ(NPz^Y7303 zo`k8wjS5XK53XriMtt!KBvcqKxuYfe)Tc&IYmV=yc>xKP9X`_*4#Q)+cpEPrJ63Vx zaRsx;v%bYW5>8oo=@ro#=uZRi;K|v97W`BAsS@%%|RY+*k`~E-;nwIHy%bvcY%&VsLi)$Tnd6 zz>&`jZdlE2007$mD-wDD9QjMihTmmhhP^Pf!tLa3Rom5pdXWJd5f=Y?LU=^Hw5O>rPS8=i|EW`g;fc*`ewKegO8 z9=dU;04Rwgc(-D;VUoPn`?ub6jalZjWMnMQ!A&d&)m~c=2ee>G*H!n6*t~zU^G-IH zNq9}ne!E_yz2eV%wY~x8V$kv>$m;X|To-2bI@=WK}S|{ zO5E(&N7$XBafBTJwuJz)CWVEIi8g9EyK9C;U+LJO&VD z9!Y!eFZ6k+tf@l%Hgob&x3GJvKC^btw<7vJ`O0+&72HDi*i*EQ>(6hgf$*$);bh)l zcPa@o2^_PX&Gw5iMVr|^Z|PfXKjVKG_6|pyqu{@swhDu=8L)Qvv=g9Xh_-q&NLuSe z{rqB-%NG_-W6+&R$H0p<%G8SIf*K96%7CFgLz&&aGrL|7os9TJlnj;NLhYyW9p^dr_W18+vG4ql{&)s1GCP#`C zK{{Q^MOUtV_Eoumaft%WcP!NDSIOBITlTqXjO_Tw*zlm^>Gf)x)_P1%2g7AKPk~}C-DF&MCRoH^GJYDq2P4`t1RLdKY4li1b!i!qtxELTMwKa zw!0oid&?}$WFiYz9UC}j{0q?7U{V)HN)a_NZf%9qsRSSQ2_W+p&$>A?gHM%;&A|7k zSWBLTm>(x`uSS>ao!)V9AH`4}V;Q{%ileN{*Trv1EV@$;V=`pGrCI8ucF%a-Eid@y z2}3an$RkuNU%#493#(f}qSL-WKbp&~AAkdN6*7S{)|O1cx}Y>lWygy8u#io7%3Ln8 zLRJQ}8DrH3lY-Xv^()?!jk)wWe1hH&Bt+bu+SC_enC!HBG$|^D21UfYD@<=t*)dr{ zR(La!K)oBJTlZQ?Kt9lsciyu*l0)}$N&124@k_Ujc}AYD@~PjpJKI?Q%40%P7~ zx+V4qy0gb$?SF<8@=`M0xxKbt5w{sS*D<|Pd%4urNc*f<5sXoLX{scZBX8$o`%B6-f1^y2V-Xs(z|1ie zpQZAhlihHkrssT#c_@zfiGNbq{7&xAkHD%;t+af+eLDPf7jio%O98ikWPh8yCEb9` zcw=l-TujF=>P1eEC@+$8dhs?JV!|>XLv zjU!x)e~5z$pJKXFF{UP&Z>@?JdZY2^fS+Amd^Ui9O2MD&sFsw^v7QGDGcXMgWe1T* z-eSOY)UU0f_K%cBRK}t)ZA|N&Ysq!z#^#lmB5Gb6EF4))F!pird#ijw1WN8e?7ip* zIP#pWSFiI);r8uy9^PFqUn6YhYIR;kyg?tHakYIr;?v$es`+Rd3mb{w3}C@ah8#K) zNO-#qTx5g4KRzt5gnw-QAnZ+X8&`thTi?26{W&1G{=MFs77M6v{pIBRriNwk4Tdw9 zn)_~a$WcX=*1R-Q9(=vHR7IkOe?K#9ukhX1>!kiY!8IE>aCisW1k~?`{o7}82(Ns9 zO*8$>ouN9P%#%1SCa1_@pscPJIMylj;_YYRbW(Wji?a- z!;!fPG+HR0&??Hex_EsPFX)hi+B=486Ymzx4@i}W4c!V6cWLu3%ci#9hAqkb znF=W(s$dB1Y^Li&7=H|)?mF^o%_*=4a5e|oD>aP$dR$!NyXe#E zIG!Ub4+uq0ch0%ez&Alj+q%?bCwz2+!->;fs{LA*YfpiPe~3;qI1-Z_pOY3>TD67B`EZsanG4@p-6;prC$M-^t^DXYNIze@ny4J6 zfst}h1Y?-A9@izP2vcA_oWv5vX*o{4$1mw?7DP|T@CH0`-|Eo=yXQIe2$IDgomUp} zPTM-qcoE5)Q2e@B2}HOkb0{UkbBXsk9DP>B_F6`srZF@1t(1QhrtU{{R}t%x%(3pf zwm;#0<;=E*S|fg)Z(bdVoiVZC=XH6>J5`b+uJN_t_DZ1-%<@rJn`3_pA;+v7nEXAa z?Kf%sVRJwghSU6@ zp>X6=r}iMUJsT`fg%k-ED#s09nMA6Cc{jbgk~UG}F@;5qU-=K4`j`^dqDDNxhZAVL zv2~|{3-7A+)f%t$laQ-Fo?~&iz~Sq1p6+i~_UgY2>`|D~$Ur_-9#qbkAL7WMf+sS~ zVPMQ-3^xvAwm7e((p86&t`b_v4V4!@az0(?HsiURw(0Io(Kx@`3Pnc@& z`lOoV@I!}s+U+~G7jZcdQBJ7(u54MVtVYJ)eHmn>e>QfVZ}rk^-h4 zJ7Hh`sI$ifX41Zj-4^cg{vYZt%^FjeaenN*Y{Wv#rw)P0ESBvR2Er%fbg&=4^69Ep za$+qVGmCV7gfGcnucVk!?UWF>rCypgCGEHvB1qfW%8q^8+^o&WSznL(;uB$W70-QpjI z=4MFq6*|8@#%+J&oE&?TrtV8E!L-*~nRal+?UDQz<6cT3V)zF}m8>Iml_52s9C!kS zd?;r13?uy%D9~@YiDD_mw4FWZ2<@=po7mc_B5ZjpQLj1-y00{7s*t(?zhG9+QFxt(YGiMWpD|g628)&6PZqerJKE%rqG>PY?rzFbvfa3HZvAg zO(G4R@bqQo0J0!x)M%-I{e1(|H2z}sm?Zbgjw5#1dQi}mi9=(dmmO9Uzzjv9UX;(y zDJfB6_hI>J|I#AFw-2RUcsG|3Njf}JK9E9LILN#7IyB9TrVJ><2smrr6~6%Yjs9ll zr#il+*7T0qwPeRgE=b9%eLFay-@(YDg3SiQW%CDwp&cDhq}Oq6QFOS{j+1rjTujny zhvgm{JXSDf@d#h33e^xSlS!Ozxv9HGj&t@JiF12DJwbXWwKXTG@}hu)9muJnm+X)CO!unx)(?WGY5|cu`32?FQ1Yq`9W3RXaNrnH(jOZzVY= z*kC8ZW2tDO-u-Q1QnXcbs+;F3>Ad4SZr@8Kh;aeos=*=ULoLvTI6<}!hjBF5#oODm z*6Brcrt{GSWtjdOaotM9+NC-xAH}^opc4@bQxv?^ zP?D_bJD~pJ-o9k6!;5S~=rznUeZpu8 zMPYF6GlL)h`c#;c;Ys?tBa>CX*C5O~k@=&NoX?d*JsF27O;_Az#>fc!V3Os}k=DQ# zrlED|#0)V>X6C%CR6sxre>I2k2{?zOF4UOV8kLAN+Z+=o)AvmL>$oHz`jetlkg3M( z>ge*BXOA(NkCg%OPK*h>$NDj^{m4&36|m>_wnS@l=_)tu2GN4nc4iubb@VhF~mxPtb&u<=fFs?7_4xt5|{a zks3Ugtd{7cA&X_4)MaI6pP50-$>!bnB^J@UAWM+z88D{MCJs6G=j(Vzb{%sMrH7LW zR7KW2HAdp|R~x-P zVVt#U-bg<-|3$WFq(cV;cVg9Kgq9V*?AGSsucq2o<>C+e+tu(CM!nrWIG@viP<4Ll zXN1`26;V53c)ny%T`Am^Xip(-QbCx}*z_bhTL-E=9K)scBs+d$4k{a{;CH(8lkfU2 z&&%uVQR-HbvJlj)b&)t7m+!-W1{B{_ER6aC`>1oNTbMldU-pBt+$Tj%Jsus{D3m_j zenj-ayAJ@NxC2whdkVjW9W;M4pw#=H395VHP<}-2;C)OKiSjSmAU?ue;7>Qg`3xXn ziVld*2P;(973L)9s@%#57OmE^AaU~1NoYpx(#HRDYufTjKAi+z4^{8K$$krTe)H!| zRhbdSGN8rkuc@oDKBt|nI*)0->SOpN?i^85*f1x_fBWxpkii*kpQAl~5NtOBm55b!)oJ+~BH!9{PKq#E~z!)Zjo=+0qH( z$=mc#{Fzk*U!U$cc$J)pGs2%Fw>-h7pc-2B`lk0#BS=J0iF$S4$T$E}(sAiDUbZ

E<}*{Z8@Nk?`}NtLW+#=}o8G>@q33=J zv_SEIuMg^tB zSb`UO^u45(4&5;X`Y11*gdh38p$#5iZ=`G60UZjxHofZ`rq!U4 z1qa4WxH7e1DeXb?iI%Ne3G%aMfOf$evu%v;WGPOqsd)wOByJ45SdC5$5T85)ME7bR zloFAtCP-U)pFB-(eT8obXG9WxGnk{Q)_gi;wk2^~rj+w1Za-2{-rjJZG6|97%E4Dv z7u4;SCFS;U;rx~dINARiAO?Jd#f6u6%7gtynVpj>3TfSgBpQ&8i?iyNBpgNF0V?qk z-u4^wA~`gDop3dWA|X(}NFbURpwW7=ny3o@s{bj8V4@}H92SI3L`LQ?+J(Y94%lnf zt|{G#_f`shJn5y-nu1u{E)@Z!#wL2fl3%NCPkIf|9sjre^=^wf&=dP_>IMnI*{Pi3 ze)|IaP9SOS=EFBQJN`4Qamdsa}18}f7Tb^)SF}Rpt!#jkFfE1@fQMBa9rl~qmyGsZGGDLF;J@V1e^DeCMOV;I2~6kHog`z3`c?{O3rtM>w*@Dm)tH0= zSp7~ys7p_-?0>G1QTiGMPl9IJ!Y@DJnc(1VmxjtN7t@;OdM_m8ZpEzAbWiepfxdpx zH@ErjY6nE^@gWTO+kr*DWmg69ANPRD2k2&qX1a-a8?)YRUr*5G@uS93Kj3(PS#$jv z0l7`RiIE{8piYX9!KIQg=4!j~T2FXA`<7c=F5Xt*e1GwyOgQpZpG*M9JjMS2*PQ>|K^+;J~6DGLQ#~`|cXs z3TZjVcRzDiirLq8Ut&*ZYg=`xVWk}(xa_2dG4x$j+#@^kIz??%@SC4es_k|>fBdhx z=z>?<$*^ld2B$7$sk#DSC!l zmhtrABaZ;M{a|aVzk$9a!P!0EgC*Cq@8*h|MF8YczPmZmuI2`!*4Fu84j4~CTsBmMip5`wUcql|?mORoYu<6?K}3d0zCz9x@hnf5 z>!#M{fOZ+_p8#wB^-ukBsCw1ipb)N_$djmlf}3JJ(6|IzWXuDtY32$I)J|LG z($v^BlmQ)e0Skb3p1$*j1&>OC1?brOaP<058UeKFbj6VT`Y^-`{Ep&Tw~JpV_o!`F zRPNc}S3&eI2+*U4Wzz)fd-)>g+Oz%dQ3iJ3AoO%u%vW4qe0wSDH*^^;aq-^vch3;q z%=QE@TtRYBGXQnv&bPgWGObba6MI_Q3K+;U=Yyf*UT{)BcJM!u6<9jcm*uW=@RO|?XtE0byvnuOAh&VdiPQK@<7 zS?MzNVCp*}!u<4|>mskhO4o(&MuzT-oMwIp>R^f5OjBw)IG;TAx4c&msdeks?=RqZ zE?Z0#d!PqUv$?_u%nBZRiVtLX59oOjNLW>Vmc7^n#HMLLUr<+e2++D?OCOrCdhAY) zWR04g8crPm^f(OL`u{Hdlvb=T#By@{y5Q7);iPAglUn|cnUD7SIw`Gup>Hnd*WU6B zC&_J<0|>7>lD zpkYI}CjCE#)dsYSMp8J1!8e7qes}$k>mj68|K0b}%(O%FH_+>t>GZQ~z)Nj6;H3q` z4nb+w7TVHuE&vQcER~%v9<4u2g}X=S=>MT#l^G&7v+;E7M$B@sJ{LGEO8*>B+^0P4 z8B6p^)9kPdFCF9d(Kq~miTILfFuyo(AD;5=kg9-QRKKCV$^gYX5sSRAz0nA--;-_Q zKEK5R8myb++@9(F&1lpG-%A7>XS?`3=r^H2OPn}SVbIyELo}y=?%zg`L^^U-SjZ4b zCL~|tsIg^<)X}%H$ulQHS-U@fxqG0HNRIvg?OHbpAIB8NSY*<%;%;qbv~y^BlFaWC z689K=ejG_eiUs~X^);7)%J?Bo4S;XPYZ}Y;#F2ain1QXzs1)i}hSh{k&DaFuMXCFs z7i-Qf2X2?`sb#C0`${%cu6#;MdpwJakNq*wHN zwUfhF{)U-kQEhqAFMmh@SA4drIe5nLq3>&QI56Vhj6*!$?K2H5@|kA7n4qjjc<{(Y z#3O$5-+}4tvwv%u)zG0V5v$f;HeB%RHp{~ORt)mol+J2(1aWSMAXfbF==f(OQsA0ZvoLqF)95TUvy$0gbb)$=xe2ShA^Ru>pHr&zqX~XidvBlz8gXy!!6cRKW1IHN zuEZB8f2-VAZ4Y(Fk)bZSwXT^HnUW(;tMuv97?~8asF7^cYH8I0B0hdBpj1>oG9rQ4 zB0ER;6Nj7W*y6FLT_KSeSisZLIG6H6XQBS){N2e5u)GYADT>FqiM(w7K4g-e*>X9H z#pyEht*B`;#bjj2$RaS+<^Gs1buJ5av7kR73Wr*1AtY*7eK;p;XR_3`n>ZTdVMMpg z6DF-Z#C|L#IVa3Bc`L2Jsu|(d2tI1LyRqDV+vWZtb13`cyxs9&KB#W!nY+`!P}Kpx zSnpn_8d?qDBZj5kI~{ABZ5G0Hop(46TI}VKRMuV~$9T`Ul;XSF>s;H1)QKLXn05NJ zJLA1CFfOANpZ=}KyDhcY1ky#zk0{; z`O;yntOlL02WI4&)|14Mr)Fzc-T`ijdkBDSFDXCbo1; zs-78{1vvGxYCf?34mu<@D7u|TKPGXK(8OG2SA*~ks{RHJ6-{p1^zyF<7MjDuUp}It zC#~rWK;{Ae1wgm?I!&&|4MO^qa`iM2&?R5gdl{R1=JA4G{~z4Yk&}&u4Q&tpZqu_- zH}94XwBbI}1~q41ZuccqA4t(ZjK%f&s5Bu^W+lHva))k8S%P)U6=e@D&1HVpCpdUb#iDR~w-?TmX|r_S9XFz`%`Vc->f@k*Zv z;uEH2QsY$C)MwwjOy^Sx_TsxH5Du%xq7FQ($70gln6VEZ+5HdXv*M{lRmpG|e`}xo z`A?xwBfbKS8EtSsHL(n6;5}XnI#31LG}8@kV+8=q!01g(*Bv9vMA~cfId7m`SiK&% zVyp5?Z(0|K;Yu?2n3K$(d~n6&afI>j31@m})%F5XPbTq+Cq{CA%Q$8+ZXm`Fieyap zI+`$pGrN+i$rd>~Q*pm_0ZdL>d3iDyUXmCCu)fk?4rG7ot63t^i;g#Fnnfs6V1jM{ z01W*!vf^BX`S5$z3OwF%_{$Oj!f>yTYX*u;*~2m-3VuTk1bp8%LcESM^mq8fsVj&a zjm8&6Mq26n?7Q~mdx`HSxz{(6ay<7_yZ<#UVR)@7_>E(6!3fLWuuZmD>pY!ZXBZXPrz_(>CEzrMpQTS+uOb7cKcOsc<&AJl`oY+KW&6d>)m=Li*n{z`fC+R1au+smxwuStvfatNWfwfy?&=u}0>-MSjoh z5cT7!!_kKUOk=TTfLkGYR=KxGCFqn91lj_eSbf0B)>eedNNEvJ$=qrd_1mGQ2-2rxB6M#4YFBYJGcn1T_XD8=&QnUD_;2X0qhMeiuryB#^TT32s`!m97 zs$G8j-t{-HOjL?*z4w)l7pxo_f@gy&0?Pll+kQY76rXKxf?Y1BMj;${1q1M22#?5H zwp*ZQz-zy8d+`45|LN;1qpI4%c8>`PN($1_ASKBe2#DabIm#5=Y0YptkLN1(!z*)u^ax?CSQ?Ph&iIRf2Gx4xFBihe_?k zszuAP`XEb~1$wls)*q(u$c?XHBpx;aSsOVhTkCT`O-`fq1$rs_k)kf#vH1Z$RsMES z1`#o6>|l)*aD3cSITCN$Z$ZP6eZ3-qlmaKyg#_%A_a9j+=iFDvc=D6Tx}F?-%UtH@ z1A>SYR?$0YVL+PB7XA&$lW25Oe0{wh-h)m->ZJ1ZFLH!Ghf$A@Ic(FglXcSf(Q?Vy zS#&G~+U1RR*yTB8Q+3qu{*GH-sheY8iA-P0c0?Q;>L;IlS{MJGy~+o1NalAc{%ei$ zL(Ir$!_HoAV-U#3DU7;fq^c*OdeUJDGd3(5u(sL9=qE0Ll^{AkC9x6f?8- zfzFFjw$M9v{)UX;p9Tc{NEzN<>YF)c{A@h=w(8fh6vGlqXF`}rB{?3i7wA)gC58vq zuPXhpSBq-qhbgsje+oU2hF7`BDn4k5V+_~<)(?wTygnoK?j~c58V3`G!}(XkZ&B6F+jUS)z{-w`+?rD-Sv25NGyZj zpe+Db=2B4&*I)YLO&au8DT7u%<<+Xh^|nXY)v=4tZEX-XBXsd1VY^yhWTz|q|X-EzWT?44v-Phj;M2G}AF=R-etCd=d+ zbiaPbdKUVLj8?cCD4&xoWq8C9ZSYyx0AjI!0P$&*^eD7$Jp5WqoKTx(nEFJ>eabdt z=2|YLaQ>rYSQ-y~FxReqxES)}j=Tx+_+ihz#>-+CK+j32VX50;JL2KoXFn*B1V$Jy z=#{bp>OK;tuqKj}-4{O=g_^zBb=3#sckb`$>lGZ_elVh2vR|vMw!&7kn>_u+>fe!1 z%{92K0?DhL@M>q&syIHTTF`Qtl|u)KQv%sDVqQD3AUvj*tKG5D9q#P?U}h&?>$cF> z#yzu+i|?*@-1%);N;D1~6<*>p6S{J4`qRN>Z5``yg=HDC8RwytWxKMx05&Y3w2s7` z1-no;FhwvWy$og+5wS3N6@mYgPfJtR{XpD4niQ3jC4H?~1f#o#JB0y%wU%z9{ak>B z{oC5A!xsq-@xlXH8rd$ijvRpd00$LE1QiW-M%^@m)Vupxyl2m4joVeiq+-^bDm|Xa zKVnsfdWPA6@2LcO3G_2`hdI#y9wNR!QwrO8k(+pYB3jufkTujA)A(sT;Y}t$!6bfM zQf@szC%6@ZUJuTG(MZ9b zl?x%m;Tk}DDE0>WxTuVtCmc0XEVUaH%kO+QJolSX?0^81u*6UaVofO|4ZBzz7wRZ(Do+_(2?R>G2tV^gYT6Y^$VuVd}{ve#)R73vPA%KeQ+e4Q9N zpIFbosNq|ATx%;d&?Ae~x=RKy7hawjp2MW&6Duif(LdZ>F`?0K&$t5*%W}T_@nKn4 z`T62rU_sw$fVh^?-eTLgr;D$_?s~a^DFujP36CB@LZ6X^&eYfwl^A-Vy?=`w32&$3 zZWA-^)RyaB!Rw0c9B!jJhDi|sMzX0G_#D>y@r(ziKq0+Pg%-MM1#7MNn zNo?!>Y0ELb4spJk1^(8_c)OOktV zy0|ce!)8+)En)}15VUg(<=JEvbf@78{`>NVGh_;xzL5g@L#0Mf!%CD)?-aA*;}t0K z@C)1p9C0&c>h!+r|zU_bs2gQl~G6Kbw@*U;f+gUYbMObc*%M3ei+$7p92fN zsm7qo4tcf^t|>=1u0%quS*}A$`CHdM)bM`_Hu!+zk%GQ9E0%hIuO%1(dr$&Bq*+#Z z20$T&%#37DHs6h?G*uwkHHv---QOkATlrpWI9~%Dm zCB|paboD_s#jFJbOlPnfL6vB`T?^S3q4RjTJ{g`9@3o zmY(E5{E4S*^l`%3nA4TN|~qIB~4tRWcM>J`uX>$ zc8<90@+pS%f7hVX4q@u3>n>qwTy07ymAffwlsSdgasSiW^vv`!ud{RN4dR>kwqymQ zN2p87+6UJ%MOGw?FTB zy!a8fG{e6iX^<6A(f@gQj{s8qAcML#`36y8aa{4cq=AOzqnGCQ>vNbJ6#-XYr1D8g z?5tPg5*#&*z}&Ixh>1qBhD&+HJB*#`{jPqFkhLbLPJtF8Zv!d>3N>1-AoxmGbRR~k z=k*25cKIzR1p?zXNYnV=hZWDiFrW97Xn+5b!FOto<*15QE@08f%YS?oa=k5fYGs4S z>OZu+%>HH_Cll#Y;yMsLZH6BAEBPL-4l4HFNuh={3(Q%7=4D!PX4#jOPDp5u_(A91Tczb?KO+Q(X zwZoI*u9jLvnbgy$rVs?W4}q(4_nf@A+u9E0)f+_{cZKzCY|Ats{JUTj0Jft#q$uOv zT}v?U;8?8JwP0=U<2l*L3{O(}1U#1he9S<_fE)Z)1qFk22actD8HI`LD0D#LOoy`q zC;Q_EXsHxmOl6=Yn|rXx9N|Y)Dr&fNkV3~hD6K2x1k2;j8Bu1Vl?ds;Ty`p^iEnV67d2{eIZpXQUpIBblFR_Tl&rz zQBcyz_6vuH%W{eXYlP$H8xWTGQK_ zec?L@d!YfeW7i=w_u`fPu!s83W;H9!645;O7v4C9C$R8b*6X=LTW&5X#DPWG2`KrK z4y`*c$cVbjgy(3P*2^8j(`XdQ9E|SY?5>5)V8Meb+1E3l!FIbL}UWpxnO4)xQ7w(o}$Da9h0KWN%)tU*Jd< zp&Z5t?&EDJVIsy|+J`bo$in7wvDJT8kfPP^3Hy2|X{zl(R&x}A7~(8P&dvnb6O@{R z{`L-QV&b~4OXYmou#xSO4|(8)%}zvTP$Y$+7q02er_N6=$$8c9NOAZv%@_!zXslj(z-78>TrS0piuK?1CIJ5V z9*XSwfLf6z!2uv*E9%7)m?yFs%@}9Ng1#P(Sh;epyb`D)kDli_ugxm}E+KIF2>YQ+ zy|pn1jQ!SHe`a|n`gQ2kIer|ld3{{{4?QnCRR;hP`&>h-O1TV?CTRluKkq^P1l`YU zoPb0sc_Zym$NLZWo|Kuf$Nw~@rD!ky^g4xq5BGAABUdsrSsd?r8IBrSeV!cZ;!da3d3VyY6|- zejF#*CVa60YJOb1pH-$K?YMI44Scr6cHkZs>^olqlGa^s4q?I|Im&ux2Diz2molv7 zrC-5yZX(O0`O58gdbk=bvf_M@P34N`BL8fl2y@+vj05FK{7N^IKXibrg}-=5St-L_ zpP<_AL$}AoD^bWSu7rDCJq6pcHR81mW`{Yu>aMWeOxw{~KEJkRh$eYSRVu}DoF#Y1 zzsGpY*&6%x?f@*x5Mfk(VS!UFMzdb$Xzx35^SnJeAy0bZnC!!Okk*jB8D78%RHI2K zpSuYZ`k>yI7%dq_cHT2pHK8>(FN))J920BG9&A~I&wNzC4`$ROoi5}t!-SN7-KGi9 z^WT_Ql2XfUsbEmyulkvL!?7mRYzswfQ2a9_8blCr54v^FW-21?9AZ}dOt#20l=oj$!O0J_; z0vLh%@wQKz;+HIsde%P%C7UsP z?ys;^_D4W&SstvKEY}%r3im;bn#k^^#VZkUIoh=>OAtPdz5CI>b41Rs2ZhOB`WK!k zb}G?kdoK-Px^LBAilU90G+MpTBgQ{E@vH0QudCI~7YqSb+0wBq!!j(K zyI;_lUwtvCDggu{MBzTNzQ?MO?lrJQ?_> z0$&q1Cfwhhkzu?D-MO)9H)T{qp0i#`{>q`sV(JI;0&jCUtf*b4sVg}9B)}!=sE)@a zZw@mNW)sV(Q6cY1$iu&2-!l0ew3juUlTVvXn@_)i_T46)PKzrXz=-#4uX_-6AHJ@) z6MQor%J7{`AA&p>TJ?hfOgM)%$F6U;(#x+1*^npJZ;}Ses94}{IN<)GGSaT^%ZquN z)+fN2BfQ#exc%zN=k!Gks@=+;BHfJ2`OB_Genq?CzLQ9OKR3N%tL z?X8d!YL;UcHJY)rpMnez@G!7VkM$yjLF4@q^ipgJjg(`9UoOpzt%YhVZ`3Fb*MetQ z5Sw?jd^=*cx2NrTIom?t0}-tt#2OAvtymsB#(fpeo>U0c4a5738nuzoZ<3t9IgSL?FBY>Om>!M>^aTY|J@P3g@iRe4H7PCkn;aVUZ!ar3((iV;9Gs z`t>{hi`NY8Y`lZQWMQROfr1H2Q134D;Rp%ENip7J^66ciREfS=vP*fkx!(B`y${62 zjkm_%ET)~rgtl2F#&!i((g=k9!jLe5wh)3&y0!#;N^Df_wirTCApAjET3ll>1%*(d zDR{Cypkd*PzMlQ@pEnKx635 z?TldbB->gix%c64%f}efDknCRPB1GWq$AVq0~mK`VXpO^VOgrGN-b_AHsvozU+H>2 zua|_EuxpdFu~&tB9UJePE^B$P~HqmYZko?UC|{=(A85$;rONgHfG$8aeT9!Vd!}_&kXPfRkh(#yyT`l|7-D+RGHh zIsWjsuUls|%coqyC=o`SBN@b05@T~G5*Q(_#Rk4jm) znesPl-}Hhl(zcaPvbsdCq&E59F?wAjr5-L1QrjV_L}gih=?ff_3?2)k%Qt1Fi2tQ* zzT1la_WeUM!;msS-Pj|6iLGn%JWQC{wN!0e0eUaFLK|MGPr13S=9wJHDXddZ3-%A@6vxAl`e%JC$zMb*@ z{OFW#l%)LK%gS_S>g2H4me99Sk#7%RN1t$lXN|TrdF`&sTeb3#uTWwr^x^+`U4>O2 z9*RJ0bx^#9er3s8>PQ=ldl!Nuti;69Qv<3|b8EA<;?Ec53`nQK;M9ekj80=kKc4LGuwGez| zF`WEQyPv)f@kTx${vc+<#~-WnZ$wpmLe*Z(npZO#{Wq#6>9Mwu=zGk!SL#*`Cu+zl z4~zfCb_zIH&4QK)7~V~x`zi0_!MYx-aIh72lNoL#u{~zD(uS4KUsEhFcyK3QFyFD8 zW}T!*JPzH_j$64R?=+htIlA$^W^wCb(IwJkBGmBNiMw?Pf${ybg1MuWKH*&H@XhUB z7ys~7*@P##K1jYWj!p|2s5TjZC3Tq!;B<#Ds5!XoBFk(6BzOMrtMrWBXUvflT)jGb zcSU4%bMGO#%|MtyDMf98yLZdbYLA@O#eyg-$Ab49(e{Yf^Z19wM73DbfElxREj}yt zz41MnBt0%H35+v%9JBqf)TygP7NU+~J2zRBz?KBb28gqqWU*{7V$&U{alb2jrdWp6 zv(hS`pPQiam)amV2E2*;Ry5vzA`mXzh%!!{fdkx!o99db_84gz{6H6Wj$*fC7O|GxYCFMFfK3^Br10U=wDyj_E zXiIm}xm}YOvnO#tsekw*ijB1ZGdQDecRdGo9WD&Hv}Tgcrhqowzj3l#;rfkBCn)Q0fOqGUblb6wg^-lLki=OgocqzkbqFzT2 zlO9G~$z^YJX-}n0HZyxDEyv&W4IDCxrFI|RWK7rC8v`Rnx8CK~5qhaao-dybp$gv} z`@9FN)P1g3tPwzi@Crhd#TRWUn^QkcJW$vcvyle)rKaaou`QJ`7{qv-Jq z3^83Hl@&(ahkMI^!FWkBt7!j&^-a_0F!!qqXakbkeq+zoJ#gC_!NPwIT>=K^;{cJY zgLGlztiC370tyP0Pbs};Ayg1ECsR2(l_2JJGN-}y8eY2Mwy zEOUq-5nW7MW7IdtsYG7pm5i0SR^ok0+-~EWhgj2--@F<7Bb6O1 z&5bJ&;lKv^*N?pkA%Z&3LU)cO$};$6^FBr|?+w*fYq2VY@Er4Sj!)+v2Hz(2J!b*q z`wW#N5grmL$)DS=b7d0em$Ki?7`W`Lv)Y-5HXm96376H?OZqjO1@Yw6=6D+_B~_&Q z34or=y(zrPjaP>G9;N)QLQPTg@Li@8=*Yobj-b(Or%C3; ztS1djK0L_UrrkX0qrPS75Z&+Z8AQ|uIu6$wQ7lb`o<$*r^tXP6B`O-H_dAzFN!PVW z?4&mDWK&zIrVtCdsK{S$_o7D~H$P^V;_|GX3p^p2{mCu9v8vpVB~MS#oFd+1v@<%n z#+sq|<4rw!TGeD$TGrW($2Bt4kr zG?$aY1p$Xi1U)Mu>Pvi=9`7$n@rD?jEE17BG_?hKE~@f7)3kN&W!}^^dx%HY)V!-i z6HhO7o!)oI!!>~8LjE$F=`vORtE?}$8``fulTM?Fg#ww!RJTQKS_ zkNSyw4=ev|)bzOBc@9%x&&!a7`6pn00D^LFRARFdA*Ma6c{Shyv37JW_i9=Jz?9nS<|-%P zOB2x70e?T*+AGQ7;fyyxrZOxEC*o6iI*@L?>X-Di3h!mu83dg1dG)CW9CI3>`w8j(-@tePC6Q3Dlh~vQv|KE{q9xo;dej# z;U=~MD80~74E-l7v*>?B1%b>J<_AyFD#X2a7;AM;pO8Rd>R2N^^Pu2fC&l8egodUF zEbzq3YH`T9Dr_+UlqSYNFo+<0oxH&R`G`F%0%Eu>ojwS+txwz5hC}KI6=fhI?DH}F zCyXk(Y$6tFdpss%uHU-$vk$|q)j|euN)X!g!zMBH8#oJY9tN_7`$W4r(hYBdR}OLv zL(9g}L_4y!6KW2X=cU%pJLFT?@|h{+9I9R?2^D*u$kjj1LofU^(F|n<90gaLE)@4{BCCnEzj3EJ-?c9UYZ^%zH4HWFo68+%Trm7C$lzQv4w8VUo>B%?eYOcT}b`7ZAhU@mD`1&V>j zHYHM00xB8s#FbHw4sUFiA@c!PuxfQ%jTqXJ9g|7vr+sObk2itKRAU9qaZfYxhwT@4 zwnH3cUCcZ1>K~cztZQhLgYJcOsOnIP;9;lDob*y-dQ)%zi)`LyZuA;dV=4 zV&G9VT14ByO&!`XYwPSU1!MJWb&&THh3fLtb*>7Lu z?liuS{KL7T2jveMJ~Ky+`n>0*Wxjf8M!wA*NEs4+>1zb&`P9`3IM{L2o7YOXUu&r4 zMHJrdQ&Rr)A>Q4Z5x+U{_!eHI$I`+5u>YUm299Wn<)%>_iF0C)9hpp|pNu7HOHB2f zn{VCZw72KqgUo&<6JI;se`NJwoajP+-d~C*%9^Xb&1t0LVuN+Q4?k^0|AkVH&i!x# zm{EWNM^Us|uAXwEFz&!2v;5eB_De`-`o!MI*1aX(mV{ZRDS--hNg>J_!$= zQnxA#%Tb=i16al{$nRj5F4z}ei0UT3(MZ*N(#c}X^K!EYbUneDgj7A^{R~#oCjK2A zQ}&WjfywnQpW|qha57oCwg}06Of`Wtr;=xhBfkS5;M}VRH<|hkwfGrjs!)LXSJpmx~)x&QL7GFoWfgoih*N~w|xD3bpBz)Q3pruJ;Kp9 zkIo1?DvKfv{V_UAXNguJ168Y|dIO#Lj^RB`Cna%SUe#*yrzI4$IoH&*bxrggrIxg@UyFNgfn)KRqSx;Q20#q)(1^HVb zSc^lh)8w{GD;qy2IHU5M$9ufF+fSzKdUQ-^hilPi^IjNd0@DwlkW&42RgEU~c9qQZ z49$|vrBEtVX6+hhO^?}75lP#+is9Ivm*u-2k@|jIh`rBQZ)veN1}pxB2HW3LFj}@A z=xnNBiN*O?5;rS%*mLui*P}zar!B3iYeHxjVum2Q*XUQZiTDFo$|kab=%=LD4#DR% zYAa7sE9L5!q|bAn@o}Z~JafFZU7*WVJCU_yu6%Cv+53E@U%fsJ zc`m#h!^0@D161QYo0lOWC=T#Txg@EU2viBSr|R?xUp`#KZMKsKWBcf#Uy7vdo!6@n zt0|K19DbFO3G`OmGiu15`DzC<7)-XnrVJd(eZ|*{0FIUVZTMt@4*29mAwb_9Ajf1JXbYPxDSKCC7If`9e9hjPngM zxLdd8&3bFRHal4gV)sh*b2{nLc^EgAsW$if0Ph)9y+32DkaQrC{JUe-4HoEgWRb@Dr#^IdSz~f zSSPP^!|01F4?T*y9-74f_|H(r+S$LNIFugjkUsYN!}V{>$K zfavxcR=&_a0jX}eWEi6^qnj%tE*6;i2C*{s#p~f+PEpMFI?FU%z%u2CC_5$Mm^q<4 z-etx;^xX1plc&F6cDc$AOfS|pm8@)dH|y$4IMT)@oTZ`}2+3(6nb0)2?l4hK7%K zzD+NYMSDfTR+1~Z7et|j$HTr#!XWmPXDsOhGPyxOi-^+<(tc+uj6@zChHN0(A#Jm; zy@16D^1hz~)*na4EJ~ZR3jz-NYNa!|Nkqr(c^)e@*c_5TJao0NpFjgKMWW8XWmtMn z@t7bI)0K@H4syIT15!7Gwh$cN9UOj<$d{poR28(pT zYwMeWLN|nzSc+%bk=##=fKYS#O%m@%nPNM*o4%Yzg&)V=jv-$ zj`+;0U&Oj3XWv=hFW3LFARza2+n9#7%MN3V#p&Iw8xhpH=2`IlkXjmV?^%(P|K5&N z$4BX;efHXWc(*gPgU6AHga+mq)mfb`RvU*!Fd_;0p>Q8_MV=+7oPH7Y7-RhWREIft zONWwnTaRV9HlZ_JA%J6F5RQm6d}ysPh=nk<6!&xAEmX$I?RAqqoMnvYNJ``^JHjb` zU;?^wly0dVS&IUmwQvhN2=x$k+EJXA73()|s{T7xOS&$r&v4Fvvc)!SoNkXRyj1;8 zg(Mbpp1m*$cD}s1+*^$TJILMjbTS9Gh5DnFowEMNXRhTSEuu%GaT!QI--~lov1M*N zyyttaurp)&P=mNeoqg_5^xkrt`0m29aA9&SFEffdKF63;FWRDv`vJ5f=z`GWh;~XM zqDXjCMicI)T3bSHSK%*4g7xC42I=l=l0EM2u7Ojxu>jVbo!aQ_Bn?j1&W5Kx+hiqY%bxltq#M5S{)*(HU6=4;NXETk*jDi3Ji zpwhsFgM!enWFhZ3JIb%?6nyy4pyrSrk?KVKTjykDmAl5JuJ3ts_WihdJ3JdZTxL!c zB?h;(*Kls)-MR0ToQJJqo?n{Ihp&_H4rk=;SY8)S*Q^9qZbZEuX4D`SvJi{Ac@26G zS(=>loQGJF4x2}yBV{U$(x1pKp@pj%4dfV&GAyxSw+*}-V@rNSs~PqKPry^$9LX)% zc#HOIcChn%MlyEY<#DWzPF?ruyjQ&HK+^}Y=ZbHA#_M(GR$@;(nlK7)H9M(8CkgXd z1l;>Ljy1+4x-poop{-uM*P@3X`)SidZ)Am@-+ab0BxHv|2T?g&uAwu@H19c| zehAIsWGmZojov2sRzjku_5DdPH03AWt~x|^p~Lp^U$`{I(Ei!1yOxryv#NAfvO{gc^yRODE?w#|IV{L`z^z9@y z^_CZe=F^acU#UwU)9FLUdV6hYF70p9Y1$L_Uv|F=(-3}J%eX~xk~_+6{u|ASF{Y%$ z%hHN?Y*)*;zRB1!m3kV(GG2)NqEl~K-rs;4uwW5wfjBe_aoK5oGvUwES`^n)(Qk~pWS4%L<6AD8ot1oDF8BS^f;;~H#5 zA@;jd@Uj*(4_L1*o7UR_n?otXIMPaJf=h_u_+FV8w@5!B8~ca07y_q)nxd78&m~>+ zXg6FLJ-vjqusvE2>-!D8U)cb^FbZosV>Xzu#LA3v`JJCl{FYXC?;x+=2OOMh?Sb{) z%N#Kd@px1}K=E%nY{OAVm}ES-*dFKS=Fz>I7!PubYC*aG9Bg-yslX^CuEK;I&4WF& zsPQu)xo!$&uI-@atrYc9yqQcP37Ky7yRfsq6tuWtKWX9Z zsK2NS=kYzEWlEv)&Uwlfu1qntX7muGOOAJs6?H-Aj+$>b$vVIxeF!uOkP9?G)ljU- z19u)yay#TPG93TVKzr*Bh!d7Di^v5-nQJmgg~HcF!RQt5vt?zg9iCN8 z<$}%?1!&z<`pCpF2T^dj|84@%XCjoyFjW}7qB1-ZIIAnYz=s_-`-hapNgXZ{5vvKQ zN{^|!j~C*XT<%j&{{EJ!!9BOYj~Gk=FgQwSCoN)!m2@J2kKII_)v>S_(ZHO-=dOO* zk*;gF=CXsv4GyN&fDy|I5@u|M0SWWDft!_*c1eys62%;KgvuFzwf zNkcC?XorMH^7JVo&c}7No50O}W2A6`4lTOs>#_Idd<@b82klWVx zy7}4HYye0725ug+UnqhTO_?*|?A}O)kyChDyFjQ3^X;W|;{4ao#9CGIPy13Ch=fX@ zVI)a{(}|o;+cb4`h=__kl^9OzdQ<2W_$ns@D!b-l8LF^sp>qK@-|?k;E~I1Sngexa zYkP`G1y|0-iUOm;&oYrC8e7bdw<6*kw(J3UTcEYZt6vAwMIWsz+OhG176qtCO@JB; z@>?i@iri{WpLB0)PUq=B;&QU3!mFA>spqhYH=>loDxR9Uzf-ly#gk; zi__T5Muc9Cg~mG&MW)IYWM0ysR$RUGc%+|qomd%X*6Mwc=5Ij;qyO>W1`R)Ll`iH>~MIv{!NCEv z37pdYSknBT@e~AD0r!?{g>)e{^D)K|U^QWKTB6%9H9DGoHJ`w0DEq~Cce?7YA`_hN z{>PmFG6A3{MWcSl@4lFIoVJ9ypW8Ww{|AuKX?q|A2gmyK704Ljgl~8Mnq|pdE|eFJ z`rm!KyEj`D7!P)Wfgk~7A4O4!2X(IXCC^e)$iy*5h6`OtO2^P46*-OwJ`U4i zu^@aFDmmvQP;YZ4bUn|F=z$6DPf&aV=}H5r zkET%cGBHr4kmY#u=^_va$A^*d*EZF5+Ap;|I*JdH2BLeg<({`$v?6m?0R(agM%M)% zXQ~O;OoN+~{RoIS_5GVzjEqCJFMIw3`1U`PK?Yxfr5mp10GMFqmeXorbmmSBd=NjO za`SO{usGh6XyDVJbV`z~>>6Q-`<+ms%T{GL1d5Gh6rmaHWe9F=HQoFz*bcl91b*>$T^2!704< zNrvTsK9_hMgv|{4&L4nb!XH`dmoD(~oDhETpzu$u$U@@XwGNw0@Q(}XAXAkJBpyRO z9xK$_d`f&$qVVuLQc8r!sJ=idHG?v?-SXQX zqu`pSxSf@Q{s0o=+wpkLYT%JD7NaILXt-crp0mVp!56hFVj62wi?5c@{ z;MZ`rOz8PX;0dwjI&mN8XQ(1zFtyI1h<9xP3B+B}cyz}&pxToGqSD!-{Ue|+r6&A| z`mq}5{`$V6DCfws&Sfq>#25f^#^VAx7`H%XhI8@y%8=^fhxNQr4TyH^BP3~Ao<|XR z)AAsr-B=OK`};%L7~HcC$|7U{Wx=ZydQs~;bJU z@Zy!ng@e64h#gk~DD>xJTXe2CKqKkt{Z8TAv|%vU+H0m!WtFOJ-^U48J6quUf%owE zb7JTN+E|;@J0n|QV6?!6?fsX0kE^o-sSBHv-|y?R2+-Jf{mDGl0n0oFoImfC$@cg? zA!->zfgXno-&9NK2mHi3{0E0dR&v-Mso?`u6_9W&RmK(M&th9*VfglRuN6G%*0Fyd zln7wnVF$v~y>19phm5|4mv1ateAxgooj)xu(R8aXKhekDf9R9@VX6I*-{-j|j74k# zA?xI5;N);)fFr+HRVKOM@v4k{IhDuOpzY$*U?YgiUSqx1w{%(f`B~}$Q-1M%Okr4% z)yp4orYMAY?@Q85BPk+WCe781Qlpz$v=?~%I^D)GH@(MG0s{F{JoZ}_gaE~w3i{h7 zX%Y)_dp5|0NYNY+)izs0x#toF{UsA87eQGQDISbRFjDMA)?%bvtduQHfx;Z8gR~9G z!dT?6Wsp__;O{fc=fwG)ys|ljvMFW81CHT-7%9Fs6c_ug7;>`Y;g)J7KDiR%r0s^) zfPYP+s>Abv1gj$va6l6fxsmC38}}w%TV_$lP_W$R@XNT^&7wx`u>-o11pZNMzn$suet=eN4d4cawvr8^ zG_owhy)z&mkzYU1nnNG9{#+t!#9>Blrqh5yGTKA#?h|?}`r^x?;%G$fEF=#_xO6vR zAVZ2fsvpto)H+OD{tUqe2Dh@+Pjf6~vMv^URX!v0EUT~)pOZ6fPQHPU6)N~_&t*)7 zUdlL7h3p?3;dBYGD28-uQcQx^-i=(Qz^wEEyDF}@VhrlHi~>6F3oH?^%z2-O0$uG{PRTk7e3C6ZSkKDjww{6ve3 zAQKa8z|57t?YspSjG6@3ziS|l$OvX;V~yvCI`;ysmBUeG1?qr3Bt&hTMIs{W%w$a` z55xzmxYX?6Kykkxr879TQYz;bbc)Jqq0F>can@aKdvuE&E2>@2Bcn&Nl3;s@a5 zb)SlJht~`hwO|{YO|b`B z*o0mMd_DKlNl;VfN}+5$u;=eMOQ+jNL%M<7j%dy3DB}JCQU+{HhFT=hEww3O*j|Nn zPP1JH!EUcQ=(XV(s)_X{YUeU@E5K%~8MWZOEIH>9n<)<#w^&4CL8{APEf5`Wlqm*- z*qfyKPRFq>dt{7#7;D+7$Zv}K@rV~ru%a9>dSNr#BVy zC!+-K;;TB`1uD|;#N|`y9TxLS`nTWcg0O+>l=SPu4TCTb(DBD1%$Y$Ub^b|Q2hyjE z#tJpy?AYEvBkz;x!{?itOmTJBT?&7>N=Z4;AAUL~P_}y|bzHPT5%c+jV8U{tlvW|Z zRNVmD_6V`{@9(jbIay>L8B0T8>&AJ>LLZco^=JN$jyV8fheep7XXTnY8qJ5T20MXw zqG$D4RJA+|wjp=7nm>q-*Y$7vR_KNq##O$HkM}6-Q|jw%Vl7y&t_p+gvWF7g9kWer zR=#tiQ_w9NNNAH!h@+00oz+kw&r3NWm7NhuYvXv5^z~0@Ztq+!0>um zp_$j2z0>*{oWC`0z>)~g>RgEy8^dmCd~0+sIVK~sLHvj_n% zeIsD_2gd@RFhD>5eotk;hN_BOJ41Ygj}(k1UC$Ae`6Axf z@ParM{{F=9wv3eC{{4&e3DB9``ZOfg?3 zo*7Px(uSGK7O<6y&m<_PG4)SJFcohn zHvuEQmsM;BfVc4=gg&8HK0&AG1rWOCXtX680?K?GIIc<}oj}*wBIvt?;?-wCs9ZOY zIn(};i5Xzg@q$!NG#;-Z<1nCw>7DrpNjy0OFn3o`)tuQWPC6PFE2{2{4*{!av0|p5 zR7A7qjUITCuaQr3vf4Hi_oWJ*RIAGt;F(1OWSn3%jo96e`ugPtGYq2d*JY1&`5Gu=dx(vTCep!9B)Qrh zi!_;%cE*62VbRLgN?rVOHP53{ir0I=v~FwPayyf!mg&W5BMV`E-zLr^5M(>0tOMbr2qf` literal 0 HcmV?d00001 diff --git a/examples/research_projects/information-gain-filtration/run_clm_igf.py b/examples/research_projects/information-gain-filtration/run_clm_igf.py new file mode 100644 index 0000000000..f16dee3ad1 --- /dev/null +++ b/examples/research_projects/information-gain-filtration/run_clm_igf.py @@ -0,0 +1,438 @@ +# Copyright 2022 - Intel Corp. All rights reserved. +# Authors: Mayank Kumar Raunak, Javier Turek, Nicole Beckage + +""" +Implementation of a new method for fine-tuning transformer models that we call +Information Gain Filtration 'IGF' on WikiText data set and compared the results +with the standard fine-tuning method + +Steps followed in the code: + +1) Generate a objective dataset of pairs (X, IG(X)). IG(X)--Informativeness of context 'X'. +Our IG (information gain) model is learning to predict the ‘informativeness’ of a particular +context. Informativeness is the change in metric between the model’s accuracy on an +objective set before and after seeing that context. For casual language modeling, the +metric is perplexity. + +2) A secondary learner is trained to infer a function approximation for IG using the dataset +created in (1). + +3) The learner created in (2) is used to inform the fine-tuning process and filter out low informative samples. + +Last, a plot is generated to compare the performance of IGF to standard fine-tuning without any filtering + +""" + +# Prerequisite libraries: + +import argparse +import random + +import numpy as np +import torch +from torch.utils.data import DataLoader, RandomSampler + +import joblib +from igf.igf import ( + SecondaryLearner, + collect_objective_set, + compute_perplexity, + generate_datasets, + load_gpt2, + recopy_gpt2, + set_seed, + train_secondary_learner, +) +from transformers import GPT2LMHeadModel + + +def generate_n_pairs( + context_len=32, + max_steps=10, + size_objective_set=100, + min_len=1026, + trim=True, + data_file="data/tokenized_stories_train_wikitext103.jbl", + igf_data_file="igf_context_pairs.jbl", +): + + """ + Collecting *n* pairs for training the secondary learner + Args: + context_len: The maximum total input sequence length after tokenization. Sequences longer + than this will be truncated, sequences shorter will be padded + max_steps: To calculate training epochs of secondary learner + size_objective_set: size of objective data set used to create (X,IG(X)) pairs which is the training data for secondary learner + min_len: The minimum length of the article to be used as objective set + trim: If True truncate the context if it exceeds context length + data_file: Tokenized data set split for training and evaluation of model + igf_data_file: file to store (I,IG(X)) paired data set to train secondary learner + + Returns: + Data stored in igf_data_file + + """ + # generates same data everytime + set_seed(3) + # generate train_data and objective_set + train_data, objective_set = generate_datasets( + context_len, data_file, number=size_objective_set, min_len=1026, trim=True + ) + # keeps model same across runs + set_seed(4) + # model, lm_optimizer, lm_scheduler = recopy_gpt2(model, device, max_steps) # store original model weights + # can we train on GPU? + device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + + # load pretrained model + model = load_gpt2("gpt2").to(device) + print("computing perplexity on objective set") + orig_perp = compute_perplexity(model, objective_set, context_len).item() + print("perplexity on objective set:", orig_perp) + + # collect igf pairs and save to file demo.jbl + collect_objective_set(model, orig_perp, context_len, train_data, objective_set, max_steps, device, igf_data_file) + + # clean up, delete model and data we don't need anymore + del model, train_data, objective_set + torch.cuda.empty_cache() + + +def training_secondary_learner( + secondary_learner_train_data, + secondary_learner_max_epochs=15, + secondary_learner_batch_size=128, + eval_freq=100, + igf_model_path="igf_model.pt", +): + """ + Train the secondary learner + + Args: + secondary_learner_train_data: Data set with (X,IG(X)) pairs to train secondary learner where IG(X) - measure of informativeness and X- context + secondary_learner_max_epochs: Number of epochs to train secondary learner + secondary_learner_batch_size: Batch size to train secondary learner + eval_freq (object): secondary model evaluation can be triggered at eval_freq + igf_model_path: path to store trained secondary learner + + Returns: + Trained secondary learner + """ + + set_seed(42) + + # Load pre-trained model + model = GPT2LMHeadModel.from_pretrained("gpt2") + + # Initialize secondary learner to use embedding weights of model + secondary_learner = SecondaryLearner(model) + + # Train secondary learner + secondary_learner = train_secondary_learner( + secondary_learner, + secondary_learner_train_data, + max_epochs=secondary_learner_max_epochs, + batch_size=secondary_learner_batch_size, + eval_freq=100, + igf_model_path=igf_model_path, + ) + + del model, secondary_learner_train_data + torch.cuda.empty_cache() + + return secondary_learner + + +def finetune( + model, + train_dataset, + test_dataset, + context_len=32, + max_steps=1000, + batch_size=16, + threshold=1.0, + recopy_model=recopy_gpt2, + secondary_learner=None, + eval_interval=10, + finetuned_model_name="gpt2_finetuned.pt", +): + """ + fine-tune with IGF if secondary_learner is not None, else standard fine-tuning + + Args: + model: pre-trained GPT-2 model + train_dataset: Data set to train GPT-2 model + test_dataset: Evaluate GPT-2 model + context_len: The maximum total input sequence length after tokenization. Sequences longer + than this will be truncated, sequences shorter will be padded + max_steps: To calculate training epochs + batch_size: Batch size to train GPT-2 model + threshold: The threshold value used by secondary learner to filter the train_data and allow only" + informative data as input to the model + recopy_model: Reset the model to the original pretrained GPT-2 weights after each iteration + secondary_learner: Selection of IGF as fine-tuning method if not None + eval_interval: number of batches after which decay the selectivity of our secondary learner filter from + 1 standard deviation above average to 1 below average + fine-tuned_model_name: name of the final final-tuned GPT-2 model + + Returns: + Fine-tuned GPT-2 model + + """ + + device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + train_sampler = RandomSampler(train_dataset) + train_dataloader = DataLoader(train_dataset, sampler=train_sampler) + + num_train_epochs = max_steps // (len(train_dataset)) + 1 + global_step = 0 + context = torch.zeros((1, context_len), dtype=torch.long, device=device) + model, lm_optimizer, lm_scheduler = recopy_model(model, device, max_steps) + + model.train() + if secondary_learner is not None: + secondary_learner.to(device) + secondary_learner.eval() + contexts = [] + examples = 0 + + observed_qs = [] + test_perps = [] + + # Compute the performance of the transformer model at the beginning + real_perp = compute_perplexity(model, test_dataset, context_len) + test_perps.append(real_perp) + print("Test perplexity, step", global_step, ":", real_perp) + for epoch in range(int(num_train_epochs)): + for step, example in enumerate(train_dataloader): + torch.cuda.empty_cache() + start = random.randint(0, example.size(2) - context_len - 1) + context[0, :] = example[0, 0, start : start + context_len] + lm_optimizer.zero_grad() + outputs = model(context, labels=context) + do_backprop = True + + if secondary_learner is not None: + predicted_q = secondary_learner.forward( + torch.tensor(context, dtype=torch.long, device=device).unsqueeze(0) + )[0].item() + observed_qs.append(float(predicted_q)) + + # Here we implement the simple non-constant threshold for the predicted IG(X) value + # We will decay the selectivity of our secondary learner filter from + # 1 standard deviation above average to 1 below average after 10 batches. + + if global_step == 10: + threshold = -1 + if predicted_q < threshold: + do_backprop = False + + # If we passed the filter, add the context to the batch! + if do_backprop: + contexts.append(np.array(context.cpu())) + lm_loss = outputs[0] + lm_loss.backward() + examples += 1 + + del outputs + + # Once the batch is filled with enough contexts, backprop on the batch. + if examples == batch_size: + torch.cuda.empty_cache() + examples = 0 + # Do LM backprop + torch.nn.utils.clip_grad_norm_(model.parameters(), 3.0) + lm_optimizer.step() + lm_scheduler.step() # Update learning rate schedule + global_step += 1 + # Compute the performance of the transformer model at this batch + if global_step % eval_interval == 0: + real_perp = compute_perplexity(model, test_dataset, context_len) + test_perps.append(real_perp) + + print("Test perplexity, step", global_step, ":", real_perp) + # Break out of the loop after 60 batches + if max_steps > 0 and global_step > 60: + break + if max_steps > 0 and global_step > 60: + break + + # save finetuned transformer model + torch.save(model.state_dict(), finetuned_model_name) + torch.cuda.empty_cache() + # Do some cleaning up so we can reinitialize for the next run of this function + del lm_optimizer + del lm_scheduler + return model + + +def main(): + parser = argparse.ArgumentParser(description="Fine-tune a transformer model with IGF on a language modeling task") + + # Required parameters + parser.add_argument( + "--data_dir", + default=None, + type=str, + required=True, + help="The input data dir. Should contain data files for WikiText.", + ) + parser.add_argument( + "--model_name_or_path", + default=None, + type=str, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models", + ) + parser.add_argument( + "--data_file", + type=str, + default=None, + help="A jbl file containing tokenized data which can be split as objective dataset, " + "train_dataset and test_dataset.", + ) + + parser.add_argument( + "--igf_data_file", + type=str, + default=None, + help="A jbl file containing the context and information gain pairs to train secondary learner.", + ) + + parser.add_argument( + "--output_dir", + default=None, + type=str, + required=True, + help="The output directory where the final fine-tuned model is stored.", + ) + + parser.add_argument( + "--tokenizer_name", + default=None, + type=str, + help="Pretrained tokenizer name or path if not the same as model_name", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + + parser.add_argument( + "--context_len", + default=32, + type=int, + help="The maximum total input sequence length after tokenization. Sequences longer " + "than this will be truncated, sequences shorter will be padded.", + ) + + parser.add_argument( + "--size_objective_set", + default=100, + type=int, + help="number of articles that are long enough to be used as our objective set", + ) + parser.add_argument( + "--eval_freq", default=100, type=int, help="secondary model evaluation is triggered at eval_freq" + ) + + parser.add_argument("--max_steps", default=1000, type=int, help="To calculate training epochs") + + parser.add_argument( + "--secondary_learner_batch_size", + default=128, + type=int, + help="batch size of training data for secondary learner", + ) + + parser.add_argument( + "--batch_size", default=16, type=int, help="batch size of training data of language model(gpt2) " + ) + + parser.add_argument( + "--eval_interval", + default=10, + type=int, + help="decay the selectivity of our secondary learner filter from" + "1 standard deviation above average to 1 below average after 10 batches", + ) + + parser.add_argument( + "--number", default=100, type=int, help="The number of examples split to be used as objective_set/test_data" + ) + + parser.add_argument( + "--min_len", default=1026, type=int, help="The minimum length of the article to be used as objective set" + ) + + parser.add_argument( + "--secondary_learner_max_epochs", default=15, type=int, help="number of epochs to train secondary learner" + ) + + parser.add_argument("--trim", default=True, type=bool, help="truncate the example if it exceeds context length") + + parser.add_argument( + "--threshold", + default=1.0, + type=float, + help="The threshold value used by secondary learner to filter the train_data and allow only" + " informative data as input to the model", + ) + + parser.add_argument("--finetuned_model_name", default="gpt2_finetuned.pt", type=str, help="finetuned_model_name") + + parser.add_argument( + "--recopy_model", + default=recopy_gpt2, + type=str, + help="Reset the model to the original pretrained GPT-2 weights after each iteration", + ) + + # function calls + # Collecting *n* pairs of context and information gain(X, IG(X)) for training the secondary learner + generate_n_pairs( + context_len=32, + max_steps=10, + size_objective_set=100, + min_len=1026, + trim=True, + data_file="data/tokenized_stories_train_wikitext103.jbl", + igf_data_file="igf_context_pairs.jbl", + ) + + # Load train data for secondary learner + secondary_learner_train_data = joblib.load("data/IGF_values.jbl") + + # Train secondary learner + secondary_learner = training_secondary_learner( + secondary_learner_train_data, + secondary_learner_max_epochs=15, + secondary_learner_batch_size=128, + eval_freq=100, + igf_model_path="igf_model.pt", + ) + + # load pretrained gpt2 model + model = GPT2LMHeadModel.from_pretrained("gpt2") + set_seed(42) + + # Generate train and test data to train and evaluate gpt2 model + train_dataset, test_dataset = generate_datasets( + context_len=32, file="data/tokenized_stories_train_wikitext103.jbl", number=100, min_len=1026, trim=True + ) + + # fine-tuning of the gpt2 model using igf (Information Gain Filtration) + finetune( + model, + train_dataset, + test_dataset, + context_len=32, + max_steps=1000, + batch_size=16, + threshold=1.0, + recopy_model=recopy_gpt2, + secondary_learner=secondary_learner, + eval_interval=10, + finetuned_model_name="gpt2_finetuned.pt", + ) + + +if __name__ == "__main__": + main()