From 2084f60fc7673426a9c45d40aeb56347d9ccb6ef Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Fri, 2 Aug 2019 11:30:53 -0700 Subject: [PATCH 01/18] [WIP] Text tutorial Tutorial file added. --- beginner_source/text_sentiment_ngrams.py | 371 +++++++++++++++++++++++ 1 file changed, 371 insertions(+) create mode 100644 beginner_source/text_sentiment_ngrams.py diff --git a/beginner_source/text_sentiment_ngrams.py b/beginner_source/text_sentiment_ngrams.py new file mode 100644 index 00000000000..36a799ac0d7 --- /dev/null +++ b/beginner_source/text_sentiment_ngrams.py @@ -0,0 +1,371 @@ +""" +Text Classification Tutorial: +============================= + +This tutorial shows how to use the text classification datasets, +including + +:: + + - AG_NEWS, + - SogouNews, + - DBpedia, + - YelpReviewPolarity, + - YelpReviewFull, + - YahooAnswers, + - AmazonReviewPolarity, + - AmazonReviewFull + +This example shows the application of ``TextClassification`` Dataset for +supervised learning analysis. + +Load data with ngrams +--------------------- + +A bag of ngrams feature is applied to capture some partial information +about the local word order. In practice, bi-gram or tri-gram are applied +to provide more benefits as word groups than only one word. An example: + +:: + + "load data with ngrams" + Bi-grams results: "load data", "data with", "with ngrams" + Tri-grams results: "load data with", "data with ngrams" + +``TextClassification`` Dataset supports the ngrams method. By setting +ngrams to 2, the example text in the dataset will be a list of single +words plus bi-grams string. + +""" + +import torch +import torchtext +from torchtext.datasets import text_classification +NGRAMS = 2 +train_dataset, test_dataset = text_classification.DATASETS['AG_NEWS']( + root='./.data', ngrams=NGRAMS, vocab=None) +BATCH_SIZE = 16 +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + +###################################################################### +# Define the model +# ---------------- +# +# The model is composed of the +# `EmbeddingBag `__ +# layer and the linear layer (see the figure below). ``nn.EmbeddingBag`` +# computes the mean value of a “bag” of embeddings. The text entries here +# have different lengths. ``nn.EmbeddingBag`` requires no padding here +# since the text lengths are saved in offsets. +# +# Additionally, since ``nn.EmbeddingBag`` accumulates the average across +# the embeddings on the fly, ``nn.EmbeddingBag`` can enhance the +# performance and memory efficiency to process a sequence of tensors. +# +# +# + +import torch.nn as nn +import torch.nn.functional as F +class TextSentiment(nn.Module): + def __init__(self, vocab_size, embed_dim, num_class): + super().__init__() + self.embedding = nn.EmbeddingBag(vocab_size, embed_dim, sparse=True) + self.fc = nn.Linear(embed_dim, num_class) + self.init_weights() + + def init_weights(self): + initrange = 0.5 + self.embedding.weight.data.uniform_(-initrange, initrange) + self.fc.weight.data.uniform_(-initrange, initrange) + self.fc.bias.data.zero_() + + def forward(self, text, offsets): + embedded = self.embedding(text, offsets) + return self.fc(embedded) + + +###################################################################### +# Initiate an instance +# -------------------- +# +# The AG_NEWS dataset has four labels and therefore the number of classes +# is four. +# +# :: +# +# 1 : World +# 2 : Sports +# 3 : Business +# 4 : Sci/Tec +# +# The vocab size is equal to the length of vocab (including single word +# and ngrams). The number of classes is equal to the number of labels, +# which is four in AG_NEWS case. +# + +VOCAB_SIZE = len(train_dataset.get_vocab()) +EMBED_DIM = 32 +NUN_CLASS = len(train_dataset.get_labels()) +model = TextSentiment(VOCAB_SIZE, EMBED_DIM, NUN_CLASS).to(device) + + +###################################################################### +# Functions used to generate batch +# -------------------------------- +# + + +###################################################################### +# Since the text entries have different lengths, a custom function +# generate_batch() is used to generate data batches and offsets. The +# function is passed to ``collate_fn`` in ``torch.utils.data.DataLoader``. +# The input to ``collate_fn`` is a list of tensors with the size of +# batch_size, and the ``collate_fn`` function packs them into a +# mini-batch. Pay attention here and make sure that ``collate_fn`` is +# declared as a top level def. This ensures that the function is available +# in each worker. +# +# The text entries in the original data batch input are packed into a list +# and concatenated as a single tensor as the input of ``nn.EmbeddingBag``. +# The offsets is a tensor of delimiters to represent the beginning index +# of the individual sequence in the text tensor. Label is a tensor saving +# the labels of individual text entries. +# + +def generate_batch(batch): + label = torch.tensor([entry[0] for entry in batch]) + text = [entry[1] for entry in batch] + offsets = [0] + [len(entry) for entry in text] + # torch.Tensor.cumsum returns the cumulative sum + # of elements in the dimension dim. + # torch.Tensor([1.0, 2.0, 3.0]).cumsum(dim=0) + + offsets = torch.tensor(offsets[:-1]).cumsum(dim=0) + text = torch.cat(text) + return text, offsets, label + + +###################################################################### +# Define functions to train the model and evaluate results. +# --------------------------------------------------------- +# + + +###################################################################### +# `torch.utils.data.DataLoader `__ +# is recommended for PyTorch users, and it makes data loading in parallel +# easily (a tutorial is +# `here `__). +# We use ``DataLoader`` here to load AG_NEWS datasets and send it to the +# model for training/validation. +# + +from torch.utils.data import DataLoader + +def train_func(sub_train_): + + # Train the model + train_loss = 0 + train_acc = 0 + data = DataLoader(sub_train_, batch_size=BATCH_SIZE, shuffle=True, + collate_fn=generate_batch) + for i, (text, offsets, cls) in enumerate(data): + optimizer.zero_grad() + text, offsets, cls = text.to(device), offsets.to(device), cls.to(device) + output = model(text, offsets) + loss = criterion(output, cls) + train_loss += loss.item() + loss.backward() + optimizer.step() + train_acc += (output.argmax(1) == cls).sum().item() + + # Adjust the learning rate + scheduler.step() + + return train_loss / len(sub_train_), train_acc / len(sub_train_) + +def test(data_): + loss = 0 + acc = 0 + data = DataLoader(data_, batch_size=BATCH_SIZE, collate_fn=generate_batch) + for text, offsets, cls in data: + text, offsets, cls = text.to(device), offsets.to(device), cls.to(device) + with torch.no_grad(): + output = model(text, offsets) + loss = criterion(output, cls) + loss += loss.item() + acc += (output.argmax(1) == cls).sum().item() + + return loss / len(data_), acc / len(data_) + + +###################################################################### +# Split the dataset and run the model +# ----------------------------------- +# +# Since the original AG_NEWS has no valid dataset, we split the training +# dataset into train/valid sets with a split ratio of 0.95 (train) and +# 0.05 (valid). Here we use +# `torch.utils.data.dataset.random_split `__ +# function in PyTorch core library. +# +# `CrossEntropyLoss `__ +# criterion combines nn.LogSoftmax() and nn.NLLLoss() in a single class. +# It is useful when training a classification problem with C classes. +# `SGD `__ +# implements stochastic gradient descent method as optimizer. The initial +# learning rate is set to 4.0. +# `StepLR `__ +# is used here to adjust the learning rate through epochs. +# + +import time +from torch.utils.data.dataset import random_split +N_EPOCHS = 5 +min_valid_loss = float('inf') + +criterion = torch.nn.CrossEntropyLoss().to(device) +optimizer = torch.optim.SGD(model.parameters(), lr=4.0) +scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1, gamma=0.9) + +train_len = int(len(train_dataset) * 0.95) +sub_train_, sub_valid_ = \ + random_split(train_dataset, [train_len, len(train_dataset) - train_len]) + +for epoch in range(N_EPOCHS): + + start_time = time.time() + train_loss, train_acc = train_func(sub_train_) + valid_loss, valid_acc = test(sub_valid_) + + secs = int(time.time() - start_time) + mins = secs / 60 + secs = secs % 60 + + print('Epoch: %d' %(epoch + 1), " | time in %d minutes, %d seconds" %(mins, secs)) + print(f'\tLoss: {train_loss:.4f}(train)\t|\tAcc: {train_acc * 100:.1f}%(train)') + print(f'\tLoss: {valid_loss:.4f}(valid)\t|\tAcc: {valid_acc * 100:.1f}%(valid)') + + if valid_loss < min_valid_loss: + min_valid_loss = valid_loss + torch.save(model, 'text_classification.pt') + + +###################################################################### +# Running the model on GPU with the following information: +# +# Epoch: 1 \| time in 0 minutes, 11 seconds +# +# :: +# +# Loss: 0.0263(train) | Acc: 84.5%(train) +# Loss: 0.0001(valid) | Acc: 89.0%(valid) +# +# +# Epoch: 2 \| time in 0 minutes, 10 seconds +# +# :: +# +# Loss: 0.0119(train) | Acc: 93.6%(train) +# Loss: 0.0000(valid) | Acc: 89.6%(valid) +# +# +# Epoch: 3 \| time in 0 minutes, 9 seconds +# +# :: +# +# Loss: 0.0069(train) | Acc: 96.4%(train) +# Loss: 0.0000(valid) | Acc: 90.5%(valid) +# +# +# Epoch: 4 \| time in 0 minutes, 11 seconds +# +# :: +# +# Loss: 0.0038(train) | Acc: 98.2%(train) +# Loss: 0.0000(valid) | Acc: 90.4%(valid) +# +# +# Epoch: 5 \| time in 0 minutes, 11 seconds +# +# :: +# +# Loss: 0.0022(train) | Acc: 99.0%(train) +# Loss: 0.0000(valid) | Acc: 91.0%(valid) +# + + +###################################################################### +# Evaluate the model with test dataset +# ------------------------------------ +# + +print('Checking the results of test dataset...') +test_loss, test_acc = test(test_dataset) +print(f'\tLoss: {test_loss:.4f}(test)\t|\tAcc: {test_acc * 100:.1f}%(test)') + + +###################################################################### +# Checking the results of test dataset… +# +# :: +# +# Loss: 0.0237(test) | Acc: 90.5%(test) +# + + +###################################################################### +# Test on a random news +# --------------------- +# +# Use the best model so far and test a golf news. The label information is +# available +# `here `__. +# + +import re +from torchtext.data.utils import ngrams_iterator +from torchtext.data.utils import get_tokenizer + +ag_news_label = {1 : "World", + 2 : "Sports", + 3 : "Business", + 4 : "Sci/Tec"} + +def predict(text, model, vocab, ngrams): + tokenizer = get_tokenizer("basic_english") + with torch.no_grad(): + text = torch.tensor([vocab[token] + for token in ngrams_iterator(tokenizer(text), ngrams)]) + output = model(text, torch.tensor([0])) + return output.argmax(1).item() + 1 + +ex_text_str = "MEMPHIS, Tenn. – Four days ago, Jon Rahm was \ + enduring the season’s worst weather conditions on Sunday at The \ + Open on his way to a closing 75 at Royal Portrush, which \ + considering the wind and the rain was a respectable showing. \ + Thursday’s first round at the WGC-FedEx St. Jude Invitational \ + was another story. With temperatures in the mid-80s and hardly any \ + wind, the Spaniard was 13 strokes better in a flawless round. \ + Thanks to his best putting performance on the PGA Tour, Rahm \ + finished with an 8-under 62 for a three-stroke lead, which \ + was even more impressive considering he’d never played the \ + front nine at TPC Southwind." + +vocab = train_dataset.get_vocab() +with open("text_classification.pt", 'rb') as f: + model = torch.load(f).to("cpu") + print("This is a %s news" %ag_news_label[predict(ex_text_str, model, vocab, 2)]) + + +###################################################################### +# This is a Sports news +# + + +###################################################################### +# You can find the code examples displayed in this note +# `here `__. +# \ No newline at end of file From d0b97fcb2bef9b4c29ad685352df8b2c688d2f6e Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Fri, 2 Aug 2019 11:35:21 -0700 Subject: [PATCH 02/18] Add files via upload Added graphic. --- _static/img/text_sentiment_ngrams_model.png | Bin 0 -> 35680 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 _static/img/text_sentiment_ngrams_model.png diff --git a/_static/img/text_sentiment_ngrams_model.png b/_static/img/text_sentiment_ngrams_model.png new file mode 100644 index 0000000000000000000000000000000000000000..94fdf554047d323cc86370571f3de2e940bbbc92 GIT binary patch literal 35680 zcmeFZhdb6^{6B6+sSs&ccSf>HB3lU|dvD5~*~D!%h-{${*_({p9+k599yiK%8(Fuw z-R|#ssowSZeg1;qb$#B~^>&?(*E#38&v`!1^LVMEBzxx6khgK-BP3)diA+vXOG+X$Tf`d^ZX=dvl|-M0=~vce;5Ccmn}t&2)03J_ z<9`^qvPuqfKbO$eOrS+|Ztk?10^S&b90lS;aPnDGegDz6OE1|a5hSfV!L&{O1D~Zo zPBy8HMETdsov0Sdye=a7{!7f;3zttjlTdVA_q<+p{o3_}n-{O^@GbG(j?2)B^`{nI zyr4m2o@E((?KXjwfaXUbiOSS+mWunbA8+#ND$9OsZR*9#I7@9wkuE_JplC@g5<$^n z+}dQ4+u77K`IJK8?boIfI62|dX7*EFRV~uWZke13e^LYv{P@ zC_WH==hYUcRRoX6AtG0+npo~WlVXxf{*n!-Ho?HpW$J;j)gzYqp(sGFAwcKnH} ztr(Mzq6$pP(b*h!i-(`*I+OS*7z`%rY-S;>c2DN_aPTiNCM#Fh$HKh49v&V%9yfU$ zoh^B92nh-CUgzWGLotD z*jgpVS0$n(3lXPH)uD)em@UV)En#WFme_T*b_^+Jado?v(K6;i^}P42v-949QhOf@ z3lW7R9{%qit)v>}?>344{1f_x0e+y_6e%gpACKVIk2z8bmPP*`=l=N!d_nksU;Gv1 z|HBLJ?FutM_*q>>6TfT8*LvOSZWF09cNS8aLBS)sXW&zM&{!5 z9oTh`_W8s&o2lk88;w04!SG|d5+WdK6N1&<1NJD-ef`nb)77h`9rA_`3J<+^S6G>4 z!wV`RE*yXPoLG-WyqWS?7gPyi-iXtT;sy)d$($2*Wnl#k2P>BnmIty&-hL`s$;+45 z)O`IQSy*$S^UdtLky3L7U9b7q$VI=S{cJjy&KrDBcC4(CH5Cuu@JXxX#ugyPZPqt? z#4(-l$%mobbBif{aaW{X+)os+pLFInR%krj3e6a>-rJ8>;9>9JsV(5rDUtNWtOO*N$usIny+)6{wL<6AOxfM}<|dN!o!^m{<%E5u+up>7wivcr%tj50 z#}er_;mH574K(Erh%t+GUwR>T zATj#Iy6tWkLSxPz#CDH_L`+X+ywS?~_ER&QKXJReL(e-gtl|3X30k|Eb=>z*UYoT= zVkl%u`ra5T`#Ds&$UWsC2?~FQ)t{LV2%v18YHntSk zy@_(-#D>VkafE$hE$0RbtfP0d-{aQ~l)1<-qH64g!|qATrkl?+1vt>JHo^ChY<`EE zLEcK}SaJ{9-H3-BoeeJ*f6Ih48uhk>1Vl>;vS{uK+mWKU7fNA~glBIzAuV0tShMXA z?)`|Pqq~^W2uzQ??}yB%&H*CIkqZ=d8((R>duFKV-e^HL4Rb%8(&Kg;Oqvd?PxK!k+YO zlb=w{O1TgL3G*B=X>`mes{9f@(PipU&)G1fi`{bNq#24%f81*^`DTz^Ln*6%2)E#V zN5%X)?~bssKL&VmZMI`++tu~-E`3C7vFOC~DBs%UbH~~V15qH1xCb)+S52EOj2q&9 zi^(2y`Na+GtCegiz6ZIrG+Par^~AA_h#wZpK^L!^_2E^Q;P%t`kNd*`M1K^8ohg%?qfJ!3eP@1&k^#T|^E;Q7-0T{` zr4m}dQQ@yo;zeb(6LVb9fZmNM*L!|tH7LrB4=oJgQ<^>SB993dJHtn3tdJ-!<65EM z6J6cow#93wFY~t|{jj?uwTlxq?N2I2_vj1_KljZ~C4GEKGQz%{7Ru9J&{NRSD^$ah zobEB~vb{9=A$_~^W{=sh^ZImSV`4T^;~t^2?9&b5{SSoDIU!XBX0E*7bbN+T(J>h= z(A@%#R>fRft?u`Js4EjW2niA)!5Bur!#;XNL`Bk?z~GZIiawEpHW^y`SnoV< zjGb+$U+AhZTgfpUFy&s)8Q}a zLqr-qJD@#C4N3p_c$fMB?4jm?Sr`9w;062yC%KmF;vDwBXp1tgHI0`ueZJMsaZrqii^cb5`;(SFW^rMmPWW*7+a710ZFMjprd-8c~LMYa|a z9lz!*Xq*R(+G=o?jZTu+6dnCSkuIr2I>$7hw0#Hj?My6Ogi~%wYGX zC8D^WBZI%f7mJ1&i4$E=!?5mdrd@iKkJBmC0Z76EO(VF9Hb!-mczO z`Z!B>EbVI%U>>g|c49#FV`B0BIVvx3zBy&{icUz)2);WM*dp}H7Z?KK=GkAU9h3jS zq%+}0f3=IHT&CJOx`~jy>N;DBPbpXJYUz4|vF|=H*D>{x#8q$Zx6(^)f-{0$z*f{Z#+3Yx)lvoC!oyt}^>eW&q}ykzyKW z@rgJL{bAMp-U;X{{*nfudv@GqkQQ{`P0ee^+)7f)=1ddgCx8S`4FoU0EH@s2NAQ8# zgheP0DU(Sl3Bv~_b#lqYl%pn5~1HyEuyQB@Z1Au9f$jHe+-=u=s1DvBT0loCz#?R8{?+z`4 zjKvu;KZUo>kSaw9Ua>6RF}(ok4j&Ln>Pnyqy5~+&u&`BbSPK!5-qZJo35{c=)<6{c zM4aHD`i=gH3bm;<=zYgp5LXVx7idoWU-8g{>^*(g&HOh^kR9CbZ^UK%gp z8Ca98;IlOkm{0*YVBZS4%~RljKfx>wM8`WLDIg`tUnvSAof33Q!<`u@ieMHiG+D)6Ko{wCB-0go#hXAI?hr>PK8{y_BNm9G-K+L@07oE%6lF|haD$N| z5Jzs8&dWd}0Y@3>)>i>X>Oe$qv3Cn0GyI*qNCp#h%AKW5NrOmw0?0sp{6+z>V}maf zt-$@@2x4;p^d~NS)r8tGuzJAt=#vnERlopdGR%NkkW%wDxrTl(yiEleKsM+Xs$eGr z?N&-)MkLSTp`rJ{G|c^!8X!bRvr9%Mb^2xxfmz^y*rh$nTleq+PSKduNE3VTC`#c4 z=#YV^RfEY9B_M4!^pBdW+Jo(az#k#1HhidE ziZ{Uh_Cd>Yh;}*PHwx_v3s4&iEMR2gvnu3)TwtD(Irav(pAk~Nlw7ecnEC-Q;Lk_rS^ph9_7`X~W^k~Ff<03Q? zh$u2V>kVL35!leZGUJ@{K zAZ2v|fyZA;`VP4Y6Ie9#*>`@ZpBj*xsluuZwf8~0OOS;GYPW%CWW`5+0ivB1*b5Ik z?;A)QyJXj+4I0U1AzLp2Bz%s}HUQsCf+@Xdp|I2kqC&VuqY5WrG#F_ATz?-wFpGz5 zfTLQT^iM!sas=SWT4Pu8@kD7SUx$WTnH3qJmclzAHk}<^n2JxzFV%!|@wiz!qXv<} zL_~U$BaPPjo+JS`VSA7rMQ}w^j?=(HLqCoPV4nQisc_W%+eG3KW))RHbxzKD#y^|# z9Y|;I?=t|Mfsdgd=UG8;RYl4@f_xY(0Dc5gfbH;Zl<40c4tXHzw3?W&{|zH0x&l~u z#^m+4Mr=6L4uDyKOdNk~6*GXG&wJ>@ONaXsmR1-XuYhB35^%FK8of#K=pF zrqfs@f9|!muBJmZic;6~`D>fC9^cK*f%?ov#S!_5GXiniC&F6AsS_8wto`gR%Sl6; z3qF-Wr$|{yJ9Iadiy&7+UM4J2E3Wif#wI}{4jr20yc)2D6f|z*>)_&4uK$r$uIrb6 z;$|u394C*sI$C0hRA_cm;j*Yi9vgoJR@I(-H>5}sGu(0V${kls(A(F>!0fjY)tP{9 z?wVx2)_$!hh)bkpaje?Rk|u7AgJZf7A#0@00?O(WDaj-~fFO_|@iryReNA)%d1$Pz z36%aT7}$OI(r)0)AENLClio)*2)5Vgs~}kSKS@F}Q9Qy=ygZPwGql8Oc2J&MiVUwE ztg+jPPpQUuj9P2I5%CE3`LV8|YcV!&7Fvjiq2bmW6dp0^OcIj%x^yds4aZVAVK)=5 zrmZ70+>*J>vMahWsE$X(ZBz=%tGz6j5=-~v^gC?wEA!kK zcp-1S2EwmwwMPyu5U1Qs=HfxCA7&@IM}nT6UU>cW$!KHGg_9n*_!K`T?^QSd&bPPJ zd1_0_IuB61l}SkSsO&nn$Itfl*hZ%1+>3mk*ot9Y<*cLqphHpfXuIlNbpQ4{*_(H! zBW7FLUdJb~##~j3>_Uv{O~EZoXs;Rhb}2d^wprHfn~&tqzQeS6f1ZjqB1bMW)UM_+ zIDfZZ9OS61=4Gf_)LOYY2H&l$)Y2Bs)D^6?j7|-BaDr>mG12$vpLK|b>oJgtB8TwnR=KRSVhyvyv~#8Tj?rYi^-8jS@v-lplC*no)VARR>{Jx?tgyjXI4(@o8AtEHjvL*BdFe##GcQ^uxyW6WL{N1fEM~5I3GVvzo{6+4 z+Fuy;wh!gKQ6Fsp3f3VP%$cZejZ^laU8Oml&lp|Fvdfu!Vd5;k%&&^0h zL}}R;ts(k&JH%Qc)C~3=gc-^WSY8lxmb|^1LuKAbNS^QV?OToC^a|G2YQ`eq-5Sbu z_<8BBfJoJ%K5RK#zt+P^R!78jQlIOG!~66s+2!EeOjD^7FT*1BN|#Bv^$@P{iDe@K ztdI83YsI<>Ckrk9NLP-=l8b*jTbrIMwgK00sI9H^;kQYnb8g;TKT7p|h5IHV$VnEB z-Ki|BM`I6A)PC_In%^Utl^z?Dd+zRc`g(7Tcl5N^pelx8%N5W0E5JWR{BdW6oe{f%?mYs2|n)0B>UN8h=FYLcH&jBS<# zTkDM_mOKzV!!GF9ZL)3|&Gl4Q;Po!d99YAX<9-x{GfmK$0$fsbjNYqa5wmtA=>0e* z9sj*zeaQ}rSE3_?vyh2=$5%=ARd$D(u!qNYWvWT*Q#UO;ANS~9_iMmdN!SgJ_D>_m zRJC;7-1e#)d*GD45rpJI@^SjU(ql%84|xr$Bwy4CNg8>5C{FZthug_!-{g(FkOIGJ zw1JCG^4QdLFtJPjis;_C(c*E=k6|ZxZ`!GN!*{Kb<@GJbiM@n;=1$+OiSiBnDU$Pk z4CQ`F(8d5e3pDDq#JU2pMstli;rzHGei19}`3ip1!##i^Rn*tmD(`u{XB2p7YiuTx zJ)pWZzle)rju4Z!lRkwDbv$x=-j0qS*|lOM5?-&45S>Dad!>zFh5f4PX%ENg)+Wo* zHHGGk$!JsN7V_y4b-SYvibDIli|ku|L5%T2@VmQ9uG8^T&Jq{R&dzzM6}d04zXc~t ze@PQ4p?MYk3J__BpJ-8Ha;K}+9=^LNd*7vJa}VROk6A$7sLrk`wtqIP{FG!j<`Fqk zT|3EbFS%B>k>WxJc$dL#)W+~{b^bT4q|!}8VP zb#3FrV&8p$$SvefmG1fC7L_h-@|Fq>#f5DCDE>wX_W0m3u;C_oN7R5Lm&{CYqeXjl zazxt$@>V*l?=idM1sg{Do3b0##_b(l3!loAo#VC3hIn{xqlk2!b2BG-*Y=tFkM;%J z-qW$yVg=$4>c2_NO@`LA7S(R3=l}HC9ebRPUiU)d&wfSm#$0(y(yH2$tcMbwo_&vT z$gh|-dUfz!y?QAr2DNGlp&=BO;6!kHkHJkAc7@>RS{8BO^L{b0+K#Lz@nRU~WL8X? zAvzs%*RQjf!5STb!C+^cz7cE)>ko`9aO>C1)v3E({=&BIk{E8vT!D>8^0(d?MoOOMPuB$cs^$->y*}UT|@|5WDoRI zkHYtdX^X2->=S$*V{OK^2Twhw;~Z+!Ih%7OY5qbJ`P12JT7v z3$;-$5#@n{)N|t1tP9iOyIG<>+q+ivH@?uW$4}W!)m}feVEm2zs*3rHJ z-TiZN68p82El$&6Zj65K_&y9D8T)Mp-q%NsMk`8Gt*+3zjg>JYE21Kf8j*PjE-?5WRY(^iZrFX{_ zhzS^v;Q|wD%bFsx%Kb3(9&WPL_Te9p*PFGbFa8`4bvP-&yT222_I0Gw-m2 zFE~5U&wbtt8oDHna5;PX=l$jFrKjuTt;!NdB|8JTmuw5F!x84?<-_LFcDrX4hKq9j zJQ+{f*;{nPLkL#0Dl9YQXUBGB)9n?N#e1YW*ZFM>)KW?P?J~T( zDu$zq1gAL(55#zT#IZTGS|7SjHeeS}b8cO>-K)9|H-~NO8w@;VlS|joA{Xqm)F&KW zVtB2|p~TRg+ym>3*Ss zN7zPmuIuQHRH9>bn60htm59S1hSlgTznVMM$tK3INlTnmMoS}OYf?|l(y4OCdV_^x z`o2QeTT-b$AY{##Wn~0|gj2Ts5t9aYqTY3Me&0I%fq-?R^b}q{fmz~{60~q3QgZMs zqwm`vPe)fMOTS2G~9(b(7b znC#VvZl$8hb`#m_mZL3@0>rnxQTq>iV}+nXO0Kc5@-wI##_g0O?vFQUtXvA^+CZ|W zWRx=S{eU`prs z0Oqz1VrkB^mCVMW+IOtgkuhsp<2}3HEy)9hlS4_G)*uPf){E$Ww{Yru^rXR^B3W?q zeaO+fyWl;Sx%e)c-hEv2toY7Fg1H1V^AJED`bj~yby?dvED9`gV}&aZAepStI%ef& zbn5sHSI-TEr;q>6A%I6_0{VH9xrL5l`{P`rSp(!FS@g0M$EWP${6-$6Vg|zc&NBZc zEy1}UJU$Q!xA{9RR>^;eTi#Nk`vzsn5cmD#C79er+|EJQ0R=IgB3CvE4$~ulR^s9P zT7Hlc;12ZX+@V$H&sXgZ>(7H5Fb>D}u`hs=z*LIC67`~V$M=$)*Uj6htJiB#b00|< zx$N)ELEtZ}7fO_5oSgWe5LwU^COCDi%%UqA+)pTiTSqFpg}LLIdd>F+5qiA{Ya%^*Wd%MEmPbS3?QCB#F{I?P5JP#f0m3-g$n1sfY1APSYsD0Rz8a=#v)! zpqk&ABGDKg*rjR&HnwE!>T$w%4dT+Po*lwOBB2Ywncz*2X*e4kgO)ydOcO+Wq=^4_ z077{YAm%2(9UqxMoooa{v{E)1!y+UHf5~Oys#&AuFm?z+X(b)yb3_$ZzB6!cV|7Ti zLorXBdmJb1vA()qhWBK3^h4?9S6a)WSt#N6m-Hf5d!k_EFlc1tA}%SS-NlH#@S z#`$15+eIqs)hP)64fp}TJxX6j6+)1sm-tH*rb&m=oYYEPhGy6SNqzbV+aQqIIhL6o z#jE#~RI&#&z#a-=JX|#|`F8QNzNIp>LTg+67U}jAhznH@!02}P0b5TweGwnq?`bd z2azbrzkrDR2SMxQ2h(I#J3+V(K->U;NERajwIN(t>^9w9F#BDAN&E)r{sZm)2f*|f zCi(~V`v(mKkOk2ntmz*lBa#BF_ZykAf`Zxq#+P71_k2PiPg?5zI3phFs< zI|D7c2I5Q(lw;NhYM%yeiy0^A)Dn? z7Hanc*~d>MoP*kdKvcxmw;@w_#sL(_A@Lk|TR#wH%+c9YEC`pHKsY(*)4KrYz6T~@ zOG~t70oqXk>%2&oc@6l5Jd87&*BZnLjW|Gx`7T=_fV>kRK(*j=4nXT!fTDzM7%EBu zU50=aRaeiiKnI*)3YKji$VvZkqyG=i{h!zU>x~q_U>VpeGw9ez!~@8_&3*|o@GpR~ z3yxm^C<`t(AcM$N)P>sN{){ZznwzNGK`)N6?i}D0XJ4m$0^}?cEW3`gTJ@z#_p@6; z-c7+C5Eqqq<~0Poox9pw>*4sJ&kXpYn`jrff@mcJ@K#hY>-l@B&j7rY_pCYwG8+nV z{7(U%J5&{1DL{?T351mF_T5yvZ|m=EfPp;>h`ny4_3&Y7#Cw3}J{q;w05|tQ{Lqio zav+e*T~?1j9l$w&!?%dcZ0KcXCtd!pVcU&d}xfb$P|J{)P(d98k;Dm*3&H&OUJ^5gyMl^LIl5c4F z5qEaJ!!lEQhsU}`{kl6+hQ>)O`Q7oA{y^amyuqo9~DKtsnUqc7(l8 z7I@x-MMy{48N`<-L_GbE9IYaU?lLy^b`i9Y>{K)&(|d~h?b(sImkxJWITvqtd32=m z*}+nVpB&!hE>6&xuCBE`K?7sdH#7PYtG$$js(7XCRz^3+LN|fqVE%J|`_Nh;;8g4Se-wQ;422nU=(my%+MRNZ(=H@M)>yEebLY?yO5| zc@yC^^|TsUyK`kP8T5@SAEAU=B*2;zOLCbEchD%)2-{`EYX^xH#5)Jf%0f0_h4R_q z3mDr**8$m^6_j4b(JU7_<{j^jOk5?Ngl_t}5yp9UbbGD)Qg0TpHA?U3eBid~tlp)* z)Ov;1jW-tEM~F84+1m&2xunjRKriSF;%}=I$#IieCtrV zgm~+avMiuidA4TX#ms)yE7rky#e#VdwG@|pm^$Qlc!#{9II*h&`JmFLZHq2HG0bOT za?;*S->}kWSsS_k&Ou;te8qZxba!9v!=hfb-^%a}HmaMYSnnkP*-AmTvM4WF?#gAZ zYU{V_CA=g@X^C$t`U~0HaI%k#0zwOw9G5M$M3xYN4*O~W#MI4H1VxwEdHQ0hBy|pd}Qxht57NBR%W~2BQBSc?N+O@SDdMDjC2_3T38=H zn$%dWj#zmvV#T1p?4Ic0yOODl7#Ptxttaa`#i@!6(raZg~9Uus( zK#)F$GEzSfG@^(F2y7-`$b+R}DZ>0!eko(SoJXa;c;4j; z9lkX<7`)sy%X`rK9d`RF+%&x_e%V5uluN8QaekJ~+ae%4tY2wRbb4q_7rC&4a&g0v z;_ksk6^y6fNANChX6N^GongWbgo=$2?E9pb(3njGf5`k;W0-3`U1UC$lNdII9_Zr2 znlW;OkGN>{x9sgCiPvEDD|vS+3?0*#3f(#s1^Vq}9=Ul-36i;c4EosJ^P@+0_llOP zvx!;YmtPvoV`0vZmT!H)t;U6YG-v20$>t~tj zBU(xXJE3nWtXqJVn<0II`?61RQAC{k;3BUEiK)lRb_}wfNd&b$@75Sbq&(C`&XG@! zS}Iw(<|2P+>~FbLO};End!<27kn9>}1^wOu@%?Eb;)6;hp_^@b;t=LWy@j_JI#=9J zM-UUOkG_E98F@6u{QXIl>f*bs#mYg^6E;F)xiJkn{aD8(Uox#AluILAYNvv@)fk0{ zcZfS=-9zsf)kt8Uj1R`$+Q6@h5M5f$w(`Q*?je z2x15`p)E1~*}10B2>V2S_YwND*1;$8NP9m)ZVy#>Quc&4LTVs#WoAbdK6%}?%w&wf zoYoZKn!-?a9r1J_Dw*qkCzg03!G*h+we)7^lLpV-?K3-lV1jsfLJ9wu!l=DT0u2S4 z0)SG&zhhqQcr1uAWEHd7Ci)JnSt9M`d~K1Icghs^N+>s12aO{dGbi|Jwn>n78dHpS z5VnVdg9krHwvVrj6kvNEp4IJ{bYLff6*s-SY&B1ImJD0EMfi2HlO}&kRV21Uti!uL zM7%&BZgqEeR&XeSxmJ*Cw>A81|2yniDYY)ctg=)?HKFI zl_%q|G-N-xvF{TZ3&u%ayy(w+5*T)`G2LseD8k~v-vR5oJwS8N++NAfA;A4vCTV;3 z5ykFi#~g&sUCWewKV6Z@6;lV{^??NTJFUo}0K<?syb45$h_J)Lk$40?KO3m2P`fYff5hHhGgHGje&G|&GrM09X z!O#4R67?C%T(=RgZ+%@68dVh9D!^*5^U{vY@$J71kqKKZF7&2)cmWb z$+%E;m&Ps(aj8tbKUjJ(6j5!=QjTOqS>y}ql%`Jg2S2e{iI@SR;2l}Ol~*^pw}_HG zmgrlfC|tY;FZcRVxjZ~t{&1MMFt51s@a%w3{IG1t2xA{wxoiD%Drz?VR}Bqx*6vN4 z1E$Fo;Gs`Q1EQ8dZ@dj_ZA?@E=d|aXE>0{YHO;-TIKJwlYYeLAg6;HjQQ#{^Dy$yv2^mBgEb4>pPTpfWJ7jHZ} zuv6X0^m2Z==cR)kd`Au1;V41?-PcyBLnNJvxq?fFE^xt%Cp?iBT8;v5G1U%e^J>h_ zrj_98R^XSEor+AxdXha$_aC>|Z5xTdsplJ8rfj5a1&Z(I9XvO*oG;9d*wc=;3<@8x z7x7M8HbUBQKh!6zdhI}yc}9PW*$am;PBtHYB=#%>-!&x6;>bP9b=>IN{Y0x`w+i zh*2@=AweM`-r@4Z$AdA7%`+Y6d3IFApD?+QnjcAh^N6It5ce1f$; z$T1G-SjE+f<8<~mG9xb6)nE$kNtO_jdwzuzr-m?P6W=6^A4DwEJzPA2?ChOR!CLJv zhg?yjMTQS5YBS|-<3bIK2e>9tBLgiT4-R%#Mh-H?Wwm*nirEq3W7wwcnyEX9Isj;t zOhxz~rDQi&-X~?zfxVVrZd^qNJycU8vOD;E4QWRN)K(|B){A^&*3k;jov5iPEY@%E z2*om0sOpI9Io4RKuoJpYNDH2Ex6K3&jQVu7u7r1R?2Opdu-*j!tL2(8eo|WNl+?tq ztG3rsJWsR9yX0@~*wA`8UifDPD%(}pT+g2R) z5+~$M4x{N4vAYZ<=b@f0eoVwy5HoeP< z{VAxx*`eO$`d!<@84U=JTBYGQAL(D%Y`X0~-ET6cRj;EVNET8#Fr(8Ln+53wPV0{>YG7#m`nx%Wpny+}>PMLx6pL^e|Ak0Z5fuoD+G9vwVDkM-OX@q)5O7{yyat>I(pO!)Si!fmQ92om@s5Rf?L5qXIAo-~yy$XVp_m!Ed4ncEc`? zcyLAO@Iz=JDX1V4V(|#c|K^L=K~nj7Fx&2L&wU7LD%H=S{;e*sha~{B zFkMj>_}jw*hB1$EYk&f(-&%w{03gBjU3O5n_^0QO1wvTcu`R$TpyH&2gV?aYJ#t{! zeRs0g&_VduKvDqF2|a+f|AQcBfMG1P&*lFyD^QTX&|B31UpVp)&Q>vwx>yB}cuo=oZuC+~&%*%V8c%gBdfSHvU;14BDOZbLN z2DRgwp>~l5wQuVnzv56gnXNA>?tIb50elR$KoCV9?IG@<()lu1{Eusb66SAkbO=2J zPW`HH{Ow1ngTVVUX^x@fUvBhoo#ZdShCpWkPW`Gz{q5Tyz{U}77!QG*zfdl`05u48 z)Jr)02b}s_xc6s&__sXrSJ?OmSpDP2^4t{rBI~a&!+7>3d_I*_Y1|s#T|K-Pq=HhG zh!2uj^>vWv^m62vF5DbxUaS9zN!x@Q$Z0pPVJ{1M2hLsGO}H5ufKzkVwn=eiFXI(< zJgBzHyPACQH5fO4JnpCM_k*b!xT+7McK;hEc;nYx^5B~9gZhI|{rhj!jtdCJ+%${g zPX}JWZ3yqot;~U-=3o3s>OxTYrw34;`&%$h5PYrfsH**UTq$&i`25Y?Kav84{45|) zWlyCQ{T*ix-6fITeE#1=q|MN_qEs(!_*bCqpeWUV#^Ti9n!(^05ImEfXR-86nKZ-7Mr&_R@{?|kuJ_Z~priSToC(!GmwQ-09lcS87kEY3w|ur}%7 z3dN-W`F$+aB^8cyC!t-zRmz(41165$x-z`=1HF2{`SPD>OtNclqYDK|d3s)ORGPex z8~Yqp*e%x)<0$lsY0+QvuEc~f3%c=Z)D5b8!$#6(QE^OEzLd*8LCj(Y=U5kAg?K50 zUrDka9=$w$_54c+G&ieBb3H$1K^Ub#Rb9VxEvep z6tRius`xPSy0>R>dZX1;BztaRN|WzslcTb@a-!9C%}rVOp}HNo$)peqkpGFL64SJy zM&R|yjEYfgZ(Y#Os5_(Mp&2!#O`&V?aw{%zr@y_gC&u@}8*zn<5ovbo+81mM3-V0L z#zw5%JA$vVtlwfBFDWV=ct@exlvC*%{i3twLqjQhB`l(&VLz5VnEbk3GiGTjWMd@u zhbJ0?l-d$*D$jM{gh~(l*~gryoCsqKXfbJ%meZj=`%lvY9<0?l_f!aFJYoy+IY?$! z;JI45Cc`e+RI$x9v(Ig*@i_O3^lJP!#+x?IdZyNn#$5~>Z?Ko%X=uZYB_<9xN5oy{ z82uF5LOKR_l% zrm|_GG5F}YNv659mXVU#QlX}sI9^4c{}{q?r@p+ubhmNQCcYUI^IprAO!CA1R0_ii z_BDpLv{SAYrc$;$Y!mQEws&qUWt)f?@h{rI)!a7XW=-;~h{88x1^ z#D5*kOk$T3tzBG9AL%2mH0N*IU%DF2q>oUP!;ug0=@X)Smg94o*pLgYKUBl*ru61M z7fb6Q9%n2rvkRg8V}0@h(3d>2ZdGj+A%wakjFO_c$1}_}c&t4bR=(PXie{z<>sJ)& z-_5JBsj$`AtIJqz8|j%=I=7sg|5j9Jlv1DlJMta-GP)ez)=wK|uR6cfz%k}KJ1!r# zyeKX#cvw(NFBn{rOCf<>s8WCYZdr|0;FJ%XQFgW7BA`5zzMPBfnd3n*bEWuV6=TRM z_A%xXyRL>G?U?bYt<34Hmt2K@Vxwt_4e6@@*S=27@oi68{YdfjKvHQ3Gbmr*x*eny z5=L5-%KW*Sz0fH-?4thVg7g+`wELvw{8x3g^&H1W%J9<%&16}tNyUEW#`e?7%Z&`@ zX1(*~+7#6JRuLR$eAcYi%$QqAj1t{tv)ItN&V(N!mqvaPPtGraaa-Mq1_fm zUne|ztJZD>`vxf!vo|Honu1J`kSsF1T>Jo-ioS^rBmdD`)Fpb?W^U5j2da{@W#T=%crw>GWVdRRTJIsfIBFwlNz5kQobG~UMyrt705g{lai#?Cc`p4 zfL3bjs~swQkPUam6-^zvx$bn!P}@(>%VwHcp3&llW79n*eU17$Xv69%)L+xijW#Hb zP>uQ3*5G?iSJHdT{rvv$$u!ohq(nns8+v)o%=p2M^vF^Q#iDq_}TXxy- zgOi1nsXsc{e`u)Ho251C-E=T2etj&bB=bmE&)!e5#LQ2w3T5Yr!(!l&BS-ukx#3@>To1xX=?&4{ns?p+GY*>PMxpEH~8$V#DKT4nvcZ$?cK6 zW7H%Hv;}YaRn0f5v6C9I`zPRBZ*B#%9r0{^_YNQ3pRiGhO&&*8S6jDQPQBZ99juba zWo)pK`Pw_=cEQmxU7Eg!#+s!u;cEJqihWYmZ?5O%X)@vr7*H3ia#AXXR1GA?Naz#< zaNZ5wXAnYDVat}7=wwC{N6)wJc5YjkQ6r@zDKpJvKfAUgO87jy@5s$;I-gF+fN2OGjrGO53HE&m`v5#dNxGfGl>hMBGI^P_{aACV3B;jJJFSFIA#6sDC3~)Iho|_ znn^V#fl@`AI?0c|-Bh6##b^6teG&WJadvY*8(g-jXXuM|x4)68ZK18hA7B(F-D+*= zJGVatt}@qV&!57gUJws~Tr{`{+A!GB)6Gk0f?H&IsgeoQt~cf`m5HCgCA3d{40`PQ*T~)r1U(^74)5E&hf5yyA#?_)&|{**>6_J=7)<&OCcD=$ z=`O})q8Zf=-S#InaOoa)oaoapp|*a`zhyew&y5-rxJZelOIApZmgRXcbbxwJUq-G8#FN7nkaKvH%cZF| z@auB@>&z1g|6N8b$Ap1~LNx{pYi8W6;5FL!d)K(&l&ux{&_#vYoR@f!_om=ab=mGA z-r4>6!^7-WcCmp{Ox)Bh^Z-EtY8PC7uSA;)tfgVvjDDDV7|{x5F4Rq0`c_xF^@#Dx z6~oDJeU6yJl_<d4&S`MF=7C990~Ppt zC2MQ3N!5Px>!LSV-LcvSUwRdA>LTAO_WC}KEPKnK4yQS7i!DrDQY%G8xFxjxuJm>! zr2i(B}8;rQ=yh(wdjl5nx z2la20WCQ1jc6P}nyu*@y*sl3E2U05_5cck+yym)>aiM!w+s}%{yUC1<`&Flyy>G8e zI3tcq3u|rL;$GDr+}L33nibBP^dCV_WAd2u>0ZZicgri}&-Rz9;GEX#isR#lP{9t? zdt?T_lIHn03Z~m@x5uM|x0Y~TQ(IdxYh`YJ-1~lM+eWMy%LZRWs*8^s4i`Iwi^TzPv+Lh)6hBkTIdN3*bn}!*)6wS)zhzdrXfV&gu^}(Qw1LXdcd|k z;|lQj7tbF09@5db?GlcGGn{)}Q?05jn_0!szUjp-q;ssNiJZZaBS1 z#(wl?HRq?@XDWCFMEAn?$mifSN%4h-wrvC#Xm!hJ!$#@K@R50QFhg`#G+edALB*8w z(as1KakjL6YN4*YHt+>9^3y`T-V>W(X5UD0HCZaa2+G z>LD{Xn*7!7Zn9!cQWr2tXcXE4-JV*zKB}gY1xC1s4b)Lx=cTK ziP`q_n$%l+cQ%Pysg&1FpBUCneV=j7OmFu4brywl6I(SB+I3VGRUrAlsx{miu4g`c zl?VkWYam4@6!XdiKX~bsYaJhl&I5^}2ifR;)tyZ4Ygem5qIBKq#0%V;uYs^y;>{J# zj7eI}Q4SIsyz@0w13%3mu(g#OsF)}HWz@2*ai$W?e-1L|3v**;DiMXnex$tl3;XUtUTzW2IRGZcFmXm_M8OCuaZvC0Q7q8chdX< zWXQAv-(OulHM*8j=E#n0wyWX_B-pVh5_!&X3~Ui`G|}%7%yZdWxRbSeX5F9K7cGt5 zT(LYrBspey)(f_?$E=om8ne{mtl`TY#6KfO5de6n#6Su^uicTw9rJn-rHRL7^g1{853cg6ltxD-tJU-*Q^Drl_*PTLEO<0 zRS6Kfx!93&hsu22m~65|@+R9&qCDc&&ri@NP7`E;XKy^uZ9Z-9_&M#Lt}e!6MZYS; zDtTRuTZ*LzZ(9ZM9^hEEEqGnau+B60-bl1~Gq?)P#oGm}k}r+*#1!5|Jq5x)s~3C7 zGR<4ygGmiZk!yhr9vRo{P z5_-z!M49S1+pqp@&;?!Y#b@#OTXoC&X64lm3rxW!C1UmpCZfv+*Vh>>+xkM5*6;HN z!i3HtRW7|AeH=g3EFiIeXG@8#aAV{9jVxez4@8O{3>~?A=NhX@_dOia+_PVX<2tnDF}RY$iaih%goTTJ1Zwp{`K2eB*AmbXsW(0e($Nh<`r6$c-MK*iM|FTrimzpsr*o0Y*EoK4hC z8-V3k0|itw)g8mfR&oC7@M(%H7QxUc@W31Pw?Ot!WJrS$sCx1DGJr%-VrsL%QdT2f zAQIiq9Qhb*N6#AZtssnyqDu0r{8>>;K8VtG5CRMYY)(PgTJ)93mr4Np*Y?F3Aj{D$B&ZEuq3Fe?;F6xf@YO+p;f+|DyfnF;?8|W3I-K+zFcv*3{|Vf8a0`^ zh4wS6gR0C}1Hj>2GxeXC3{)lvxU1T|xC`A+rSUWg?xjsh4Uz*gp}<7&QvH`fU4Y5( zOoc%2_G`e@eQUHfP$76Tber(|uPZ>;T)$q(;}LXG!{i;V!a#MbBsCyCy_akds-w<< z;Xj-^wesL?pkyG>Gu7liKRsmOxp;z9?8lFr|1IB>{7#g#@_sIsf|adk+xj}ec>wkq z+)0*$C=Hd^1+gYnd~M zo9gIDtN-!Kdn&2hRP;+Y?iqU^q~VM#$+lrBl7K>=wHk!}HD2&HT2 zj-iy0?(UZE9J;%^lIQ-v3|My;$SToZ4rfefGJ|-WMcmi)rS+mHhXQ zG}tt@%BKB!qX1a-VCw^Uu0MODJRnMuFFEpm-ux#o^&ivm5}2N}5~Y`aj7cOQ>c5S| z@3{o}1|!IjpMLd+ZiS*3D5kJay8S!6|F?Gh8(;v0663^jgI3e-gVFEiU;U7TAv)(3 zR87)M+ouL8r+0c``|69CO9@HJ1?;f`QmdX*B!tImR(I^~p55+C_N#d>I+Dm4(CGnX zOhc(a&-(ZTrfcu;N|rB2#Ymj2OQ>G&Ug5l&0iU!+U0)K1F_7D>2Fa0Fy}_JwNQf+` zbNzXe;IdkNAj*8O*1bdSvSv3=NmHm|VqP@T1lg7@)@Gc(GY#%m?#cwfzO$*D_%2c> zBM&OUhga6k=dZY{bniB7euqa-XE|s<7NPV%!1;xrJuA8? znZENtXXpyc5|Mkvvf8%^r}W5a1#auCDz$ak!dC3%dZNJrP3Gix%>c~Sai)7zYYp&G zPy@Uq+*=2aW&x+$I zL7QUM3GY}%H-Zdu=t)^CYN6*jS1IL<=+&*GlrwOG$M0i*PIbCySP_O@dr97ef2rfq(p1cO%~+q=?e%T_ zW5m$0ltf`3seztx0Ou^rtf9zhK+t41nDe7IJousr`$}o!Ik`;+RR*13h1SB3O?`WI z!DY{&(~go@fdKDjZt0nW{ju9!7dZsa$vu!Np(rMNV6snO%w>P7M^Nwv-NhoP5^^=j zU@;f;q{+Rkcm*I>p$4TUYcKWqq@DdvUAZUcwl_1H&{=_H9D~@vn=k3u+iy)U5G4DEER5@1KeW@9uY(u<+e*J#2L28FKGG zPDYjc*yB*A-qll`rGLvpIk)YbBz!H}-L(GJ{OWyTv5rw`5-k-J4F1x#w#W*MiPGOb?5QY;NSZijC- znj?3WNjE{mE_=9@b1O-jCnrI~P%PG}E|rAKT8pSp9%gxCdw3<^`R1}Dlbd*Gs1DB0 zwJ%ets&+x|IUmiEbRS>6Q=WBoilvEU3!R+fWUfb@Oa62rbxP7r*x z_SBpq+zcrTkF_82*EwS$aF5q)vY}(k>0_eR>8Iw=JO`li(-@@rAtUTbRWU2WhL1Ek zyHVi`w%iXio@w!#teZ;A zCPNwg9#x38CfCe|U={)C`KGtixu>4j_a^0as*Ac7lMQqynnMDBo&1;aN#5F1>Ehfg zK3*_ps{dPMbLj;Ii+m|}IbKC#m4i&w;%&-B!vYV-&2C)b8!I6@%bQUbj;>p$THb28 z9wG+k8}L&5lBm}B77-^@cMijJu-XtJq~&69RBXoM9VD*Ky;J?eZ6WT)1za6wkdR`U zR|-htc>O@#!}SO9qC-kjxT}rDoy`gV^@a<_4;LTz`QT~3FH(;WK9zHplQ#W)e_%pf zNv^a&cF_OJ{ro_k<~+tr{|!a8Qzs|oZU_aavDTPD^9i#{Mbn(bqV{7m=RsA$^<>SP zv(TZ2Db5bVwz1K;0WW=Gx9Y#6;>7{)FBBcn4GL_}HoN^2r1w-^lZfA|x63{g7w_pm zqc;K)dz`%;6_b1{03L5Y~q3~O$(|B$Z^x^deb>M7_*#)SY`kd`EF&v z$&HyZz^ghDZdW!76Ak5bx9aYs zyKVkiI1bwyn~NNCHC+uGP-gnmu$KDZ6d;ev3~*@lE(8n3Qdi+Q>77o^wODm*0>Gz- zYZqs-)(iRbEXgzY!)8z39SAR;e*IBU(?R~F-$BaYVc*8IFwsufbMjJ&=lG7hshLLf z;C#o({kobPI%AAVT3dBB96hPtk)PuZ(Ves1J9_1ykC9*y%1?i=W`5;jN)K=j&VZ_N z+|3)s5}H8L#|w91*E&T#0$q(BwJYC@OY=x{U6gz5W{aYt2Lv*HZruBZ`RqH^d`_Es z3F@KNKF(=g8qMqL>Sa;5<}QO)-s^TM6Uvyj9cWU(r5DoRz0*(d zpA|>9ZkTTw3iHt40UqTyy=j0R$P;z9#viGHlGEAwAU_xq){CYko1a{K>%)v-;P9!O z!>)Yn$i=aE@v3b9RtbbPldZa&z`@^!wQVhC3x?_lQq-?oSvdP@2g@9leB4#Eyx+`t zsM)GpRBw&$YD3sFlp*5KYG0qZ7l|=GnUXR$OSoNL=&e8AjI3`+8pz^b^{J?bSjiZl z37&{~bnS>YrZ6tw9h$Pp_fOr==aE)TmbeL9RSytr8BUY$CEfbQG@U`C^cOkTIt~~> zl*0nzpV;$vD-)p5O841b8v9@-xJ{M^L&M9vpMtXj56rIi1GN`C=J!=glWtWkp#8=y zC?(7iUjc6SNqq^BYX*;Z*wkz&<#naqUd~-_zoRdev$w);@K7=F?3TkPWH-#-Lh(x% zlTpHeTE&Z zy3?TTo;lvcc{Aem)_vPYkZuaTan7z11Ei2~ATMSXTuH<5AbF=4Uq}|le^JQ>E_-y< zr41#LzSZ0yF;xqB=tGkSH!)cl2%EAQQ;{lH?l66B_@p{lv!k!+aD7|hDmYU=yKSzW z-e+I#R#j9=gC+om>Jx`GDyh~Ry)3N%;Sd}&t@V&e()Cd zFKeh$S#lB#?KwF)sn*}TW!Y3q1#7pKZ&6>QsMcfR+Aioz;g+pSl24TyC_lTgtm;)z zf6P=oVP&kAyI1;L0haJA1OvO@u>)r+fJH`jnriLde_s23V5lJinf|VuKn=*ug<`s%0rYz+vBpa$}v4mGb7(;%;C?!=0#1aLndrW#QoG zjG>_|qP}|Vbw4J31JhTaGL|#7;eB_u!ga}mIGM4M)X_#h!!2oOi@*CEKm(AecPOL0 z|ARQ`z%uhpqAt8O&p0k^=l*;uA$@hQmML2X;naW;l$;cu_ZoUUI2Ja1a^NtZ>ePtr z7!7z@BIZB#8f3lud`7c(wlI``K%E!n;V5M} zx^A1at2&F@>vPzv#3Xi*>_axGbJ~e?byQU@Oo*F)>auX-jLhlW1U(;%5p<-Qv{xbm zsSJk)sk;jdf_}DN)7%6H@1x$r;G;4aY+5iC<}?dC5Np~C$Tkg#g!{@7&<+X-j2)`@ zReLPF*V#C)ocpZwn9=fRO1Q|aVm)er?tT?yg<#@NUAJy@G{h)>y$TZZ{uRlKvS&>;hlPYY1%xqA+;N6bY&Cf2VW0pF-@x)ds%#}7ajIWgMd zm*UDfAdFchYvp{mZ@|e$o}LjI{ju zyDF=k@i$P+=#aatW&wgQn5EP~B8X@F3%Ub%4llJKD+a`)DA`q5P@vC_$;slVa_v*-kvU}*!XvPZn4LJvcSE81yaD>S=AB)OMENE|(ngBL%tZm%!L!MI#Evbl$MduPyrQ8UAXyTR# zZgg`|v-QWLq5ozU!PtC|N__#jnjHipJJjfYWcKK0OP)vPcB(q+mfuVyM__m3#lE&_ z3?zD^uzp|J?A9;PX##4&yrB?=!Ecq0sW2otM_hK$@5)8NZ|TsFRU9a5R8SJ)0#L3eyYw8?yB> zhq$F|?(pK-@^`a1dR|`_H|n-beI?0wd8pPzjuuQ%pV4#jsg?cv9bOL4#Dz;X0P&kp z*#*H;mn*KRCM%6M$6Q6abQdn46uXMJO${GC*^qEOF_253(?!NlcG)F!4;Wc(Z&HYd z8y-H0^$F(6Fk+S|0ib)jniFsH>%gUT$$EZ}J`mtk5&<0=RHL#%miq-vduLWoYu-i* zu3=9n>Dhx>*Bwjw=^Q;cCM;@7Q=Mc%5CGgQ>BaJ%zW{XL`)21f2LRyPYvL_Dv8d@Y z(Md?TrEtoLn7nW}B2a?6=FBEl%{Hkvscx8F&v~US7xBL+ybaPN={P_GV(}eDlbPPh z@kiO81Kf=oBMVY1utkcEXa@O~^|9CGujEh>U|%AzCsKyt=K!7;_&1&>UjWFaR51|- z6;sGZ0{>BvP6T4paR4AX*zvIfKd@#fAOH~Ur=6mp3ouInsQuvcpB%7dB2*6Goi{$L zG_`7h8URB70=O^porik{BUt=vNn-N@t!|#7a=>6@9VYqLba4|*ItyVR$vnRTU#7do z5ITVwg=Qpn1?y$*h~LDRoIsf}1tn~tl2CL{>A-#=Yj&PQ6-xOuVC_AhEqw#yH${<2 zb+$r#A&yyy_7WiNj0ZR$=3PaA*b3lx-A^rl_=YPnT7+e_Dgm%+0x+t1=fG>2NwLED zOwG;AFUk)bDPUv|WX}LzOqg}iq_zQfxvtH_UxB1S>XVm{WlGW{DrcB5vn?4jlq0nfJ!si zQNj8<$8s;DoE6XlEc87ze)V?@Jw^CAw&E>XTRi|d>o7JKmk>CRDZ@b7^q0#+Rem}P zxwt^5{&%Jlg$yv>;z=@N29^Vj{x|c!{gVd)cW08=1TaD>;7&7b`7f3a3ix|hG~{rE zpVR`v=9?WzzXu!ZT)fIq@AJp4+845;I_@No8NegH@_21=&w ze(b|efu~ZROU**#!U8-j;Mxf-*vJ?wg_SX#Bfr7K4w5cn6qKIQCqiyO&Xx16K`=f; z^|sQ=+5Is2WurF3QU6=0s_(^2-=5b9LS1_{gL-Fn_Q#Kjp6ItR*Wl1uI?ZiLqW%N` z<_I=X6y|`at#;V9VHYcZElPQ7Kp^3`O8>s_5Sj;o`jtuDaTw3UUlaU)nQ_>X0cM~F z5S5gyG~C}c!`9b-uDt%%lyG1j#3aVT|D!qpS`1vYkyMkJ8n|ur=O?thgW{I2jPIs2 z39U9+@E(tc79?@__8pt8n}zva^Rl0w#Pa}fFT0(=-mk*_CYfSrh-z|kL9HE4fd#ST z)h=-zZSKmcm3;WBw~!Tl$8GYZeYkja4(XM?iIo$wn+kSQgjaV4_g?(5;X?kvYt0im zjRo9k+`rbieB+Z_!&Iudq?BkugYn3#(m^ zowz$n-WxJ8b<%|B*ZbCb&0<ob%1$WVd2a5yg=FHq8 zeu&AmFR72)(icPpVI*NcFA4=+bYu>bKz|}P7Lzye?Wvc)h%Q#4Icm~@R#G||=^xm> zqdh^%Wh}#L2=eBd2XEUTPr#rj3 zj1xwjQnq$9r6YY#_Uvj=w`$R1@Ps*{^ddQ?2wY`V*IlOd^rcj@oZ!)#WY|#zcv$A2Fje5?pmXLAJPH`Z>%$wmBhFSI=OCBm{@M#h+W&+pC|E-hZm=qC?}j_>;!1A z&xXHn`(^j3q<6BDzk%$c!3d2nU34x{ut4zoI}-@^RIIj~eB!wib2rRnaz}R^x zCO13EsJfa*tb5ZyPpf;tUz`qZh>ERGPcQE!GbE1DZ{?>BWrl}b``a^#LaxDK&UT|& zqE~0xzsE6LR(40!3#nIFT5>E419R#|Dl_Pgmy@sl&+I40;_7}pc)_6M~E;w)2 zQ@B)ys5L@GfBNgIIX&Yle)Rl_rj%rGfZ5UxvjXqb$j?CyO6wCY-A-1%55)r&Ya0=a zK=1%F!b`#%b~=@kmA%?Qtg`Q<<*PWA7Lm@2;Kgs{FK5I(K3H>BNfOz{s~YtoxGOET z2T&fbn{BA>;3H8Exqp`G3!t(pUf-FJ@rH-XH+d}Lc^@3pNL`Grr|sV;%jI#`ZK zzsLBF#Crz3+Z9v94>B?o5Aeq!baQ#{A{bqW@A(+w;aoT+2jx8Lv~$^VoN)<>sSd)9 zO}XnCNozQHN~%jtm)`0|9qJ4(eB-};Or5}6ESVDdo#)4B@y~0zbL_okVL?}aX9MB7 zTynB%O*4s7>q`wD^@CqcjMSk>>+*Ptd7kt#B*w;6YTVEDCl$ji&~dNU2`}v!y;>As zT&et4qs!KEk`%Wx35e9%Y&5g9BX7g38d-SS_jC@sPYBApXV3cI)N3 zpbVIr9nBZ{g(k=~5Ut&0-luIIa98Exp3GGJps?2@U=`P{nqTX3WRR+W0K&{4bgXht zsy+zqeg#d!Bs}^$&8*mmS7d+KwusG8%x|^J{=BcTLOfH=h2)f5!~wZ*Y^p$n4@%Y?2G!LJiTLma!o$+ z=wB5XK^A=y{G^7#PQmY38rB+C;p_@xvhGP_j(iU4mZssW@X=RJ`sN@rWinLUAa%+C zXX)Dc(I$;AFBj6^;ZdlDlp{6`AFVZRwSI5P+(coG-A^eWf7A?#E?6+>3%9Fsy^)r> zzr~dz;tE`7|1n^aI-P+7aNDhZ)E95c37qj3Bhj67DJ-7!OBAF@*EJaNZ+Oi`$=VL} zVh!MK>KQ?<=B|=}X2;))%POwSC8~m={dkk}3BH7`Pe+KxbJ_3TX878y_EekQJ@Qs< z^V2?u%-W3JRk@alyNO3iCs_0?6Pw*#(K`!>-A0OWSFyd67pH(CdY$9;RJ2TcOOXd| zLb7!kqy(<%DKREquN6CdVuy(D#{Igu3E8>aiwNI(mWe6o95q$^Q@#88C~_P0_^d?l zh77cxfApGQ<@L%DNssBeky?o`Ms}k-clO8WRGhDxf#j6d9kGelZOi>v8Y)+c6LXX1^8U=K@f|+i^0+g;lO-KPV?159PZlcZWpXoqYRTH zbt#{vbd5f;S~kxY?i16OF2~REq7qPekDMo6?68H=6G2`j7h}l>f<}22o|G|ydVN

2P%}e3mLQaI$cJQYwQR?TQd$YWT&FgQN^E1NrVh&1g6b!% zCa{EVArIcC5Ce;zS(n|m-D*@f;x4B`z)qX`pe_=TAso78_X5Fo zyz5mx3mL1#a8g7`?Q}g~_HjxV)aKWI^1cBd-m*0hKSQ7-dq|^ktmJg5?M9<-oDpyN z03sJdXG3f%0CCxUIH|J%FCGOl`CT}#5_Ji6)^0W&@q~tJ!3$5{fQR&mR4ZM}oNH4M z#Z#S8H0~VV`{M9f+7%RcUSc<-y`MyWtIV281k5aXrku42^NpZ5Dddda<+&jiCfZ0YN)SAU@U8eGcdFwgm{`7-ZIULy;h92QLsJpiNj&oR1yoKKS%%|I@}M@5TD& zr7C&}s9t$>E3ay?Q{i=5M`u!c-wV;9Jhz2P9)zP`7nTfBh+buYxYxE+Cyv)926U-~ z;NxgK=G$x{bu$~wD4ZPvn>ZIg-K%81@rLt1LPa?^KDoo1rR&h1n0cZOi1n#v*KS+F z^dx=euYwj?pRDTjX=hfx;Dn7t?wCHyr>S(@iF?sjg*o|ClGqWtGlfDavV(Y#DB>0t zQ-W^%d2rt$xmZPYZ5UC*244GX5!OSN84v7>D-AhP&(z(!3PYz^(=ke?dclFuGTd z!y#25o)XM9&s8*68Au4mulw2vfsBUqpB@l6ekR( zb&=iR9DLm}bRlSzvf7xfh6NLk^n!o%VQvanNA!(W+1dpcw^h%|_-8)Jffqtlwd)o7 z>Ia$b_&MbXrPS+8ocQmh+4NS04Ewlzx=I|mgwY!tShj3Nn#@>-9m zL)F*GM=#|S$%rEq58C!kVDYj{R zK@VJ1OjI}bYiABCwb?|ru4En7>6HkTS zhkho7&$tR2GIIiMuY69e&V%G%d9tP=ppLUBM@Y@*Yl&hszM!Nn_!Yx4&0|MmeYu!* z%q4O-YdVThYZ5<7K7$n-evRSuc89we>$ycHEAXw~@&{r1ay!>S{) z4SRbWk2J$pg2^ZW+~r43@ANs}nT#~$*&6!wd~3G6?I@4MmRyYkB{fn+40%hRn_j+f ztcouqK3-k|FD$*dTT9w@LW(Qb$nvQllQk0G!M$X8hG<0Xn;yrWLp^#vUz}a+(@bS~ zJ%Gc3mb@6rB>|-Kk4{ zk)I*1o^Pt$`%&w%tkTe<_QiKasu2=##v5THKfSfrP}oT{Z%jeX8qdQTmsAB^$FXu9 z_WMTE$mMCr0Fj)T#`2%@oJorqWEYy~gt|n&&h;lMXgsXuo|ULD6;zDlxqI`tp)CoC5e8Z*Gd|JF#w%9h7v{bO^?jNL~H;_K8{R zV=J=7?07=CAlW=EpS^OPl@2SSmeITjr65Z1S)M^=s1PfBt%*>lIKQ|2)!OivSXl(a zR(V-HiWLGLTu0i#nlY3(H;rsx5AAc|*>ahavO45f+Es5Z^oTu=12qUxGLUJe-iCci zx-}!~v4`i6TFEf<{q&F|zVv_S(}_TFc#@&*3OgTSSsMo-nbvqDB48>1ep2zUQ^PdS zh~lcn;elP|?*}^&I4FH?aO6Y{ee};G#6$sDN4z)e7GeJs25O-o0M>SkasPKHS|P`cGk?7SGSXDtWxpQ7-Xs zH$4zd;NS*p@Y6qZK8uWum-E=uIGL-Vcjz7xQ{J5DFT0f<@3&vmR2GEmQ&GG#iJYKt zZ@qKg=k)U$bo2dqATxK+uJ zlRMdw6I$9o##zf-s6Q()(`r-|aA}lZc4K-az-&i6?w?;>$9e5u2wk?gCl)>KU@%3< z!%@LAZJJKwTdN%{Q!-DzS?l>dei=aH43$jf@c~WXRxzPYaqs%9{Q~unfXiY@{9&ys zm*oLsR$4>_3e+3R9GFuk(fTsT(Z^cmM^e33j<3?Q6?2{$W=0e)1&l)e>2fnvo|8C= z)-zsec{=Q=@l0*0Ai|7y+KxnFrb4;1^=|G;&^bxbh4=s;GpTIQk zYm(5R7VoZu^c|r6W${A!%rdJ5J2%SeT}RcJkp^Hd7chgHa0-$Q{m7h8P)8??i{Ady z)FpN6*l+RIqq5FZ*3bFQh6xo?J(6R+r)K~=a|AIszsEpiOsti$2fxjIJXk4jRR(t9 zjIwu2kdjR*%eYxa(mXmc;C9}uNwm^u;m)m!fs{!2TA~w-l}e+)4KCFZ zqaB##3^&iIh^`{z{}oBUFkQb^Qjk!sR8>5~vggbZL6);7v7&;Jo|7HkaTM6?H>Tlzu~l(H~#s>yiHr_8!j zB@SSb5w1Z(y!e>Yc+y8v48k=oC=z=CQxjHTg_2}XrWXNI=ZV6YR3%~1RG}WTHQZFo zX{$OoOpL~qiZo3LbNJ+q8wj^#np5Dt^VvN8%!kGq_DRdq|F?<(nVPpEYo-1 z+ku7Yzg=>>!o+i5Rm6@w_p;4p>e{=^`P^RZ&>-xTbVBQc_86CObdWx|xylFqwD*!E z8HzDCQ_gXxhw^zT3A05`L)jw7Ee|XaZYCa7r(eSw&I5GZ$O9gr3Z1t`9-RAF`^=MZ zqE*lLL;BiBisW7!&(+5*KYq;Dk&87d-Jupz^P_rq(NSFB{_Y8M==Nqi3ByaUdvoVn zV(Il|j!o07Nh=^ay!2v8gEfnx5{6>wsGnYq!m<%?8ceN~%T6 z$*q^K_Ve*)PhoVqm)1mW9zM^=nZt}9>ZR3f`zJbCj)~jGOji3;r+ORQ@g!~_1N!q! zbmHEfc6L`W77rT48AvH+IF9mi1XCl={F(9dP({yQwv}~lInCnMpZAf@;<`Cnl)B!B zRL&j-CTny&cdXM+Vb><&ETqGY^2CzkGPjzv`Q}BAbLc^3+WpNTr2x9RRI@|V z;CjJ2i5oHIfU6FA&jvTd`K$2@^#$RKtUmb`gG%s9&_hePVQ>79L$|LS^B7TzorwFE zlQS{Yd~k&LRt&a1y?7z-|FSl8o^XO1?rCl%B5OvHL^bEH9|8&pghJR)*o@v84j`vJ zW6gY_YfeJ3aj%#9#zafof}CGIy$z+&?%);sR#=*|SNKtNa%xD6P2Feas32Qmse?P+D&c>!^vH&(8Atg%Y3Z0@Vn zCC7z2g*WU+k=eI>l2K?{LJ+#om2dP168D@*ADmN0RD3kXndixJu!*1QXShncLH8S8!Vywl8djnK#S{+Ni=WzOld)SmGquhhdC(c}jn zER$EYz0>;~4TU%Iwr=eg&QenmAhBwz0Z#tJPun8&t8ld`N-r);JyolAbUS;s3U(Pu6a1weH#1!4# zpWD&jx7mxP!TCuERCE!Z;wf15<~^!tgIH(4$i%S zg2m(dM{Q|Ay{*E(tW)pKtGNR#=WLbEI z?D>N4T4W|^$|lKrcn5L2`Zw+x48nL;x;i7wnZ-F4P|~xK4!{ah`OD_q_GFT zC^pdW(k0#4fOr@~OA%@lBT%!O7Hf;98>n&~J$k$-@m~1j8|FE?r?z6W*kAQxD{Q3} zmp++m7X8xrz17OYPM6#38OK2LTJWsv)}Xh9^p@yXA-Mi?eSv(<5Tw%9Akf=Ry3zbE z%i5H}C|&)AXc@?M={BR#Vo~2u{crf1vu5^sHSBwC67)vKgY?gbAjVO<@|`E#^eJ!3 z9E}v3P_rM>D6+Ip+40vgVIEdDJPff;#mR+74fPO&H!m)9SpX5e>JNP8S~pS9$s|L< z{>zNF_T7_yiM5HPB8&dqPCc5NYErh2)3VIBl;jWQ-)za_@dx0;`h_@_mT!jk7f;aHZ?{d7Zzy**maDt}c1B7Yf2M`Fg}>tHYIqM#LQwR*PUQ z|IMo3anfHYm|<1&%%{co8LW3GuK`c?i#h+cEPQ342QTt8`k>U;&xcoJnAe$Wv8Ty#}JoFX|P4Z4UT$`sR z4A~Z8st1?v?D0rLZ`n!{d%B{2no%o2%|)h2Rl&WEF_2Fbfz%I)(|WYA%d0{48#^RJ z!#1-P(Y*;5jMxCNfK67W2-&7wGBdWcsM{!#W=4#F$Uu={*%yjRl5I;V^S;=}?tR~( zJW-@I)NgVZkZNUNMp8|j0xvS=TdxA}M?c`6K7u2N%Ae4~_k-i~a$I8e21#gP(b*-; zgwUYZmU6A-6P`qGOR#3=zlsRSduA9bj_5tKtRPxoeSiZ#GqcX#<>bgi&NlaHHqyKw z!RsNPE0yj<%Y^WX%2a)!W%Nul%WmSd!%&T|i*4xj+tcK7&*7#f9v`2&YnpsvVnohu zDNQ`R{#8cmqFu&A=UkUC5{VY6<>QKX& z^W^;L$5G}3R%>bp@=^6rOicC@mTx>Ngg2pNm>P^6@HlM4tf{ImGa4Wnb=F74I!aDE zUVm>86rb}38Lx69Od33IQ7?A#Lx2;d$+%YxEwEph!?O}BsglWEL=KxLEkoN29Q{Y3 zFWMjBe%@VNCh*H9dB-TrR~(S&7k42U)a*c}9%(5=MEgAMbXZ>aUbnd5+AP8Kx?d73 zAFsJ6;;R|$~={0D@QJw9q0r*oqPs!I{RX*-{s1$u4Ji^KH z08`@DY6xc(Wyf4BIfmc5pU>l~3EsxQv9;ZYZStgNNaYu9XXnUDe0A-aR7*8C)X*So zjCS@F2*|!SYK(a+k5<+nYt0XD74Q><3SY1ShstZM7}+F}c$e%jlKAZObYSMAXe{*} z*uVvN5rhOzC}${crIFpmkcsDe>;jo3K)ho zQ+9gk{}c~3hgTD1s*Ky=U7+~%-@ZOVC4fJrCVu+cY5OyRSHQva9nrs|{Obc7@XfAI zmO=mNfd}FnptW0yKZ6JA@bG;Uq=$uV4)X*5JVAO8z{|wcL`sDS0rXJ`J>`s$ks7B&Hk`@A9T@n5}8^1rm uE=R~H<^d1zKkostq5nrb|L= Date: Fri, 2 Aug 2019 14:43:34 -0700 Subject: [PATCH 03/18] Rename text_sentiment_ngrams.py to text_sentiment_ngrams_tutorial.py Renamed so it will build. --- ...xt_sentiment_ngrams.py => text_sentiment_ngrams_tutorial.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename beginner_source/{text_sentiment_ngrams.py => text_sentiment_ngrams_tutorial.py} (99%) diff --git a/beginner_source/text_sentiment_ngrams.py b/beginner_source/text_sentiment_ngrams_tutorial.py similarity index 99% rename from beginner_source/text_sentiment_ngrams.py rename to beginner_source/text_sentiment_ngrams_tutorial.py index 36a799ac0d7..4908d4a46ca 100644 --- a/beginner_source/text_sentiment_ngrams.py +++ b/beginner_source/text_sentiment_ngrams_tutorial.py @@ -368,4 +368,4 @@ def predict(text, model, vocab, ngrams): ###################################################################### # You can find the code examples displayed in this note # `here `__. -# \ No newline at end of file +# From aeab17e8d251d8c1efe9ff8f9bd353b5d217e6fa Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Tue, 6 Aug 2019 14:21:32 -0700 Subject: [PATCH 04/18] Update text_sentiment_ngrams_tutorial.py --- beginner_source/text_sentiment_ngrams_tutorial.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beginner_source/text_sentiment_ngrams_tutorial.py b/beginner_source/text_sentiment_ngrams_tutorial.py index 4908d4a46ca..eef22baaa3a 100644 --- a/beginner_source/text_sentiment_ngrams_tutorial.py +++ b/beginner_source/text_sentiment_ngrams_tutorial.py @@ -63,7 +63,7 @@ # the embeddings on the fly, ``nn.EmbeddingBag`` can enhance the # performance and memory efficiency to process a sequence of tensors. # -# +# .. image:: ../_static/img/text_sentiment_ngrams_model.png # import torch.nn as nn From 53959f7e63470d8e423aca8dfb073fc00a62b749 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Tue, 6 Aug 2019 14:25:46 -0700 Subject: [PATCH 05/18] Update index.rst --- index.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/index.rst b/index.rst index d0b20f958cd..bf53a216759 100644 --- a/index.rst +++ b/index.rst @@ -143,6 +143,9 @@ Text .. galleryitem:: intermediate/seq2seq_translation_tutorial.py :figure: _static/img/seq2seq_flat.png + +.. galleryitem:: beginner_source/text_sentiment_ngrams_tutorial.py + :figure: _static/img/text_sentiment_ngrams_model.png .. raw:: html @@ -294,6 +297,7 @@ PyTorch in Other Languages intermediate/char_rnn_classification_tutorial beginner/deep_learning_nlp_tutorial intermediate/seq2seq_translation_tutorial + beginner_source/text_sentiment_ngrams_tutorial .. toctree:: :maxdepth: 2 From f3e0ed04704c360bd6a12a5ce2ef281e9bf6d90c Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Wed, 7 Aug 2019 09:31:09 -0700 Subject: [PATCH 06/18] Update index.rst --- index.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/index.rst b/index.rst index bf53a216759..98a9f7e6782 100644 --- a/index.rst +++ b/index.rst @@ -143,9 +143,11 @@ Text .. galleryitem:: intermediate/seq2seq_translation_tutorial.py :figure: _static/img/seq2seq_flat.png - -.. galleryitem:: beginner_source/text_sentiment_ngrams_tutorial.py - :figure: _static/img/text_sentiment_ngrams_model.png + +.. customgalleryitem:: + :tooltip: Sentiment Ngrams with Torchtext + :figure: /_static/img/text_sentiment_ngrams_model.png + :description: :doc:`/beginner/text_sentiment_ngrams_tutorial` .. raw:: html @@ -297,7 +299,7 @@ PyTorch in Other Languages intermediate/char_rnn_classification_tutorial beginner/deep_learning_nlp_tutorial intermediate/seq2seq_translation_tutorial - beginner_source/text_sentiment_ngrams_tutorial + beginner/text_sentiment_ngrams_tutorial .. toctree:: :maxdepth: 2 From 4f4195fbaf179913923ac19c21f7161ddea39c56 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Wed, 7 Aug 2019 09:34:55 -0700 Subject: [PATCH 07/18] Update build.sh Temp update to pytorch binary. --- .jenkins/build.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index df59585eed5..73be8ee2d12 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -18,6 +18,9 @@ pip install -r $DIR/../requirements.txt # For Tensorboard. Until 1.14 moves to the release channel. pip install tb-nightly +# Temporary install of 1.2 until release is ready +pip install https://download.pytorch.org/whl/nightly/cu100/torch-1.2.0-cp37-cp37m-linux_x86_64.whl + export PATH=/opt/conda/bin:$PATH pip install sphinx==1.8.2 pandas From 96013143d0e7921f6fa6d8bd570945874a04190a Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Wed, 7 Aug 2019 09:48:56 -0700 Subject: [PATCH 08/18] Update build.sh --- .jenkins/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 73be8ee2d12..09a0566087c 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -19,7 +19,7 @@ pip install -r $DIR/../requirements.txt pip install tb-nightly # Temporary install of 1.2 until release is ready -pip install https://download.pytorch.org/whl/nightly/cu100/torch-1.2.0-cp37-cp37m-linux_x86_64.whl +pip install torch==1.2.0 -f https://download.pytorch.org/whl/nightly/cu100/ export PATH=/opt/conda/bin:$PATH pip install sphinx==1.8.2 pandas From 27a7311d8f470419ca8dd85dda4f1f057ce1d447 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Wed, 7 Aug 2019 09:51:33 -0700 Subject: [PATCH 09/18] Update text_sentiment_ngrams_tutorial.py --- beginner_source/text_sentiment_ngrams_tutorial.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beginner_source/text_sentiment_ngrams_tutorial.py b/beginner_source/text_sentiment_ngrams_tutorial.py index eef22baaa3a..f2cefa9af8d 100644 --- a/beginner_source/text_sentiment_ngrams_tutorial.py +++ b/beginner_source/text_sentiment_ngrams_tutorial.py @@ -1,6 +1,6 @@ """ -Text Classification Tutorial: -============================= +Text Classification Tutorial +============================ This tutorial shows how to use the text classification datasets, including From 572f9c2bf273d1da15fdadf69833529e09a1ce38 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Wed, 7 Aug 2019 10:04:56 -0700 Subject: [PATCH 10/18] Update build.sh --- .jenkins/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 09a0566087c..40c2b7bcfb0 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -19,7 +19,7 @@ pip install -r $DIR/../requirements.txt pip install tb-nightly # Temporary install of 1.2 until release is ready -pip install torch==1.2.0 -f https://download.pytorch.org/whl/nightly/cu100/ +pip install torch==1.2.0 -f https://download.pytorch.org/whl/nightly/cu100/torch_nightly.html export PATH=/opt/conda/bin:$PATH pip install sphinx==1.8.2 pandas From 1f63f0fcafc5243a6ed53d1ebd2287e3c296196e Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Wed, 7 Aug 2019 10:44:42 -0700 Subject: [PATCH 11/18] Update build.sh --- .jenkins/build.sh | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 40c2b7bcfb0..f9395647f70 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -19,7 +19,15 @@ pip install -r $DIR/../requirements.txt pip install tb-nightly # Temporary install of 1.2 until release is ready -pip install torch==1.2.0 -f https://download.pytorch.org/whl/nightly/cu100/torch_nightly.html +pip uninstall torchvision -y +pip uninstall torch -y +pip install --pre torch==1.2.0 torchvision -f https://download.pytorch.org/whl/nightly/cu100/torch_nightly.html + +# Temp for testing. +pip uninstall torchvision -y +pip uninstall torch -y +pip install numpy +pip install --pre torch==1.2.0 torchvision -f https://download.pytorch.org/whl/nightly/cu100/torch_nightly.html export PATH=/opt/conda/bin:$PATH pip install sphinx==1.8.2 pandas @@ -41,7 +49,7 @@ pushd audio python setup.py install popd -# Install torchaudio from source +# Install torchtext from source git clone https://github.com/pytorch/text --quiet pushd text python setup.py install From f96a035e002d8d5524c31e7d73db21eacbdffc35 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Wed, 7 Aug 2019 11:12:24 -0700 Subject: [PATCH 12/18] Update build.sh --- .jenkins/build.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index f9395647f70..732ad87eabd 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -23,12 +23,6 @@ pip uninstall torchvision -y pip uninstall torch -y pip install --pre torch==1.2.0 torchvision -f https://download.pytorch.org/whl/nightly/cu100/torch_nightly.html -# Temp for testing. -pip uninstall torchvision -y -pip uninstall torch -y -pip install numpy -pip install --pre torch==1.2.0 torchvision -f https://download.pytorch.org/whl/nightly/cu100/torch_nightly.html - export PATH=/opt/conda/bin:$PATH pip install sphinx==1.8.2 pandas From 1febb83c345e69a29526fa9cdb56b16e6994d0af Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Wed, 7 Aug 2019 11:26:46 -0700 Subject: [PATCH 13/18] Update build.sh reordered some installation commands. --- .jenkins/build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index 732ad87eabd..db3677487ea 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -15,6 +15,9 @@ export PATH=/opt/conda/bin:$PATH rm -rf src pip install -r $DIR/../requirements.txt +export PATH=/opt/conda/bin:$PATH +pip install sphinx==1.8.2 pandas + # For Tensorboard. Until 1.14 moves to the release channel. pip install tb-nightly @@ -23,9 +26,6 @@ pip uninstall torchvision -y pip uninstall torch -y pip install --pre torch==1.2.0 torchvision -f https://download.pytorch.org/whl/nightly/cu100/torch_nightly.html -export PATH=/opt/conda/bin:$PATH -pip install sphinx==1.8.2 pandas - # install awscli # pip uninstall awscli # pip install awscli==1.16.35 From b0d9128f5c96a69e39c24bc6c5c88a71e99867a9 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Wed, 7 Aug 2019 11:38:27 -0700 Subject: [PATCH 14/18] Update build.sh trying torchvision from source (temp fix.) --- .jenkins/build.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.jenkins/build.sh b/.jenkins/build.sh index db3677487ea..9feb787b1a6 100755 --- a/.jenkins/build.sh +++ b/.jenkins/build.sh @@ -24,7 +24,7 @@ pip install tb-nightly # Temporary install of 1.2 until release is ready pip uninstall torchvision -y pip uninstall torch -y -pip install --pre torch==1.2.0 torchvision -f https://download.pytorch.org/whl/nightly/cu100/torch_nightly.html +pip install --pre torch==1.2.0 -f https://download.pytorch.org/whl/nightly/cu100/torch_nightly.html # install awscli # pip uninstall awscli @@ -37,6 +37,12 @@ pip install -e git+git://github.com/pytorch/pytorch_sphinx_theme.git#egg=pytorch # this is a workaround to the issue. pip install sphinx-gallery==0.3.1 tqdm matplotlib ipython pillow==4.1.1 +# Install torchvision from source +git clone https://github.com/pytorch/vision --quiet +pushd vision +python setup.py install +popd + # Install torchaudio from source git clone https://github.com/pytorch/audio --quiet pushd audio From 208103050de85964a6d107d2c474f688bccdf6cc Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Wed, 7 Aug 2019 12:24:46 -0700 Subject: [PATCH 15/18] Update text_sentiment_ngrams_tutorial.py Creating .data folder. --- beginner_source/text_sentiment_ngrams_tutorial.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/beginner_source/text_sentiment_ngrams_tutorial.py b/beginner_source/text_sentiment_ngrams_tutorial.py index f2cefa9af8d..d336b1e3077 100644 --- a/beginner_source/text_sentiment_ngrams_tutorial.py +++ b/beginner_source/text_sentiment_ngrams_tutorial.py @@ -42,6 +42,9 @@ import torchtext from torchtext.datasets import text_classification NGRAMS = 2 +mport os +if not os.path.isdir('./.data'): + os.mkdir('./.data') train_dataset, test_dataset = text_classification.DATASETS['AG_NEWS']( root='./.data', ngrams=NGRAMS, vocab=None) BATCH_SIZE = 16 From d6bb0af5e3ae04a478a8ca7df660738e8f07b583 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Wed, 7 Aug 2019 12:46:24 -0700 Subject: [PATCH 16/18] Update text_sentiment_ngrams_tutorial.py --- beginner_source/text_sentiment_ngrams_tutorial.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beginner_source/text_sentiment_ngrams_tutorial.py b/beginner_source/text_sentiment_ngrams_tutorial.py index d336b1e3077..79ed04fbf85 100644 --- a/beginner_source/text_sentiment_ngrams_tutorial.py +++ b/beginner_source/text_sentiment_ngrams_tutorial.py @@ -42,7 +42,7 @@ import torchtext from torchtext.datasets import text_classification NGRAMS = 2 -mport os +import os if not os.path.isdir('./.data'): os.mkdir('./.data') train_dataset, test_dataset = text_classification.DATASETS['AG_NEWS']( From 05a9a0bf857815c697c97d9682bb7c7acb6341ca Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Wed, 7 Aug 2019 14:32:13 -0700 Subject: [PATCH 17/18] Update text_sentiment_ngrams_tutorial.py Updates for build. --- beginner_source/text_sentiment_ngrams_tutorial.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/beginner_source/text_sentiment_ngrams_tutorial.py b/beginner_source/text_sentiment_ngrams_tutorial.py index 79ed04fbf85..6d011e2ff23 100644 --- a/beginner_source/text_sentiment_ngrams_tutorial.py +++ b/beginner_source/text_sentiment_ngrams_tutorial.py @@ -251,10 +251,6 @@ def test(data_): print(f'\tLoss: {train_loss:.4f}(train)\t|\tAcc: {train_acc * 100:.1f}%(train)') print(f'\tLoss: {valid_loss:.4f}(valid)\t|\tAcc: {valid_acc * 100:.1f}%(valid)') - if valid_loss < min_valid_loss: - min_valid_loss = valid_loss - torch.save(model, 'text_classification.pt') - ###################################################################### # Running the model on GPU with the following information: @@ -358,10 +354,12 @@ def predict(text, model, vocab, ngrams): front nine at TPC Southwind." vocab = train_dataset.get_vocab() -with open("text_classification.pt", 'rb') as f: - model = torch.load(f).to("cpu") - print("This is a %s news" %ag_news_label[predict(ex_text_str, model, vocab, 2)]) +` +model = model.to("cpu") + +print("This is a %s news" %ag_news_label[predict(ex_text_str, model, vocab, 2)]) +` ###################################################################### # This is a Sports news From 198d526f86388fbf20d04830b388a6306dd8f4a3 Mon Sep 17 00:00:00 2001 From: Brian Johnson Date: Wed, 7 Aug 2019 15:33:11 -0700 Subject: [PATCH 18/18] Update text_sentiment_ngrams_tutorial.py --- beginner_source/text_sentiment_ngrams_tutorial.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/beginner_source/text_sentiment_ngrams_tutorial.py b/beginner_source/text_sentiment_ngrams_tutorial.py index 6d011e2ff23..5e47c1791fa 100644 --- a/beginner_source/text_sentiment_ngrams_tutorial.py +++ b/beginner_source/text_sentiment_ngrams_tutorial.py @@ -354,13 +354,10 @@ def predict(text, model, vocab, ngrams): front nine at TPC Southwind." vocab = train_dataset.get_vocab() -` model = model.to("cpu") print("This is a %s news" %ag_news_label[predict(ex_text_str, model, vocab, 2)]) -` - ###################################################################### # This is a Sports news #