From 7ee3ee4328277a92af1ab1f2c23c566706d54c8b Mon Sep 17 00:00:00 2001 From: Parmeet Singh Bhatia Date: Sun, 10 Oct 2021 23:25:29 -0400 Subject: [PATCH 01/17] initial commit --- torchtext/__init__.py | 4 + torchtext/models/__init__.py | 1 + torchtext/models/roberta/__init__.py | 18 ++ torchtext/models/roberta/bundler.py | 129 +++++++++++ torchtext/models/roberta/model.py | 108 +++++++++ torchtext/models/roberta/modules.py | 304 +++++++++++++++++++++++++ torchtext/models/roberta/transforms.py | 59 +++++ 7 files changed, 623 insertions(+) create mode 100644 torchtext/models/__init__.py create mode 100644 torchtext/models/roberta/__init__.py create mode 100644 torchtext/models/roberta/bundler.py create mode 100644 torchtext/models/roberta/model.py create mode 100644 torchtext/models/roberta/modules.py create mode 100644 torchtext/models/roberta/transforms.py diff --git a/torchtext/__init__.py b/torchtext/__init__.py index 8e1a92feab..a6dda66847 100644 --- a/torchtext/__init__.py +++ b/torchtext/__init__.py @@ -1,8 +1,11 @@ +TEXT_BUCKET = 'https://download.pytorch.org/models/text' + from . import data from . import nn from . import datasets from . import utils from . import vocab +from . import models from . import experimental from . import legacy from ._extension import _init_extension @@ -18,6 +21,7 @@ 'datasets', 'utils', 'vocab', + 'models', 'experimental', 'legacy'] diff --git a/torchtext/models/__init__.py b/torchtext/models/__init__.py new file mode 100644 index 0000000000..268ef3b71b --- /dev/null +++ b/torchtext/models/__init__.py @@ -0,0 +1 @@ +from .roberta import * \ No newline at end of file diff --git a/torchtext/models/roberta/__init__.py b/torchtext/models/roberta/__init__.py new file mode 100644 index 0000000000..f03218844b --- /dev/null +++ b/torchtext/models/roberta/__init__.py @@ -0,0 +1,18 @@ +from .model import ( + RobertaEncoderParams, + RobertaClassificationHead, +) + +from .bundler import ( + RobertaModelBundle, + XLMR_BASE_ENCODER, + XLMR_LARGE_ENCODER, +) + +__all__ = [ + "RobertaEncoderParams", + "RobertaClassificationHead", + "RobertaModelBundle", + "XLMR_BASE_ENCODER", + "XLMR_LARGE_ENCODER", +] diff --git a/torchtext/models/roberta/bundler.py b/torchtext/models/roberta/bundler.py new file mode 100644 index 0000000000..27042ef203 --- /dev/null +++ b/torchtext/models/roberta/bundler.py @@ -0,0 +1,129 @@ + +import os +from dataclasses import dataclass +from functools import partial + +from typing import Optional, Callable +import torch +from torch.hub import load_state_dict_from_url +from torch.nn import Module +import logging + +from .model import ( + RobertaEncoderParams, + RobertaModel, + _get_model, +) + +from .transforms import get_xlmr_transform + +from torchtext import TEXT_BUCKET + + +@dataclass +class RobertaModelBundle: + """[summary] + + [extended_summary] + + + Args: + _params: Encoder Parameters object + _path: URL or path to model state dict + _head: Module attached to the output of encoder + transform: Factory function to build model transform + + Example - Pretrained encoder + >>> import torch, torchtext + >>> xlmr_base = torchtext.models.XLMR_BASE_ENCODER + >>> model = xlmr_base.get_model() + >>> transform = xlmr_base.transform() + >>> model_input = torch.tensor(transform("Hello World")).unsqueeze(0) + >>> output = model(model_input) + >>> output.shape + torch.Size([1, 4, 768]) + + Example - Pretrained encoder attached to un-initialized classification head + >>> import torch, torchtext + >>> xlmr_large = torchtext.models.XLMR_LARGE_ENCODER + >>> classifier_head = torchtext.models.RobertaClassificationHead(num_classes=2, input_dim = xlmr_large.params.embedding_dim) + >>> classification_model = xlmr_large.get_model(head=classifier_head) + >>> transform = xlmr_large.transform() + >>> model_input = torch.tensor(transform("Hello World")).unsqueeze(0) + >>> output = classification_model(model_input) + >>> output.shape + torch.Size([1, 2]) + + Example - Working with locally saved model state dict + >>> import torch, torchtext + >>> xlmr_base = torchtext.models.XLMR_BASE_ENCODER + >>> model = xlmr_base.get_model_from_path(path="path/to/state_dict.pt") + """ + _params: RobertaEncoderParams + _path: Optional[str] = None + _head: Optional[Module] = None + transform: Optional[Callable] = None + + def get_model(self, head: Optional[Module] = None, *, dl_kwargs=None) -> RobertaModel: + + if head is not None: + input_head = head + if self._head is not None: + logging.log("A custom head module was provided, discarding the default head module.") + else: + input_head = self._head + + model = _get_model(self._params, input_head) + + dl_kwargs = {} if dl_kwargs is None else dl_kwargs + state_dict = load_state_dict_from_url(self._path, **dl_kwargs) + if input_head is not None: + model.load_state_dict(state_dict, strict=False) + else: + model.load_state_dict(state_dict, strict=True) + return model + + def get_model_from_path(self, path: str, head: Optional[Module] = None) -> RobertaModel: + if head is not None: + input_head = head + if self._head is not None: + logging.log("A custom head module was provided, discarding the default head module.") + else: + input_head = self._head + + model = _get_model(self._params, input_head) + + state_dict = torch.load(path) + + if input_head is not None: + model.load_state_dict(state_dict, strict=False) + else: + model.load_state_dict(state_dict, strict=True) + return model + + @property + def params(self) -> RobertaEncoderParams: + return self._params + + @params.setter + def params(self, params: RobertaEncoderParams): + self._params = params + + +XLMR_BASE_ENCODER = RobertaModelBundle( + _path=os.path.join(TEXT_BUCKET, "xlmr.base.encoder.pt"), + _params=RobertaEncoderParams(vocab_size=250002), + transform=partial(get_xlmr_transform, + spm_model_url=os.path.join(TEXT_BUCKET, "xlmr.sentencepiece.bpe.model.pt"), + vocab_url=os.path.join(TEXT_BUCKET, "xlmr.vocab.pt"), + ) +) + +XLMR_LARGE_ENCODER = RobertaModelBundle( + _path=os.path.join(TEXT_BUCKET, "xlmr.large.encoder.pt"), + _params=RobertaEncoderParams(vocab_size=250002, embedding_dim=1024, ffn_dimension=4096, num_attention_heads=16, num_encoder_layers=24), + transform=partial(get_xlmr_transform, + spm_model_url=os.path.join(TEXT_BUCKET, "xlmr.sentencepiece.bpe.model.pt"), + vocab_url=os.path.join(TEXT_BUCKET, "xlmr.vocab.pt"), + ) +) diff --git a/torchtext/models/roberta/model.py b/torchtext/models/roberta/model.py new file mode 100644 index 0000000000..0656e647f8 --- /dev/null +++ b/torchtext/models/roberta/model.py @@ -0,0 +1,108 @@ +import math + +from dataclasses import dataclass, asdict +from typing import Optional + +from torch.nn import Module +import torch +from torch import Tensor +import torch.nn as nn + +from .modules import ( + TransformerEncoder, +) + + +@dataclass +class RobertaEncoderParams: + vocab_size: int = 50265 + embedding_dim: int = 768 + ffn_dimension: int = 3072 + padding_idx: int = 1 + max_seq_len: int = 514 + num_attention_heads: int = 12 + num_encoder_layers: int = 12 + dropout: float = 0.1 + scaling: Optional[float] = None + normalize_before: bool = False + + +class RobertaEncoder(Module): + def __init__( + self, + vocab_size: int, + embedding_dim: int, + ffn_dimension: int, + padding_idx: int, + max_seq_len: int, + num_attention_heads: int, + num_encoder_layers: int, + dropout: float = 0.1, + scaling: Optional[float] = None, + normalize_before: bool = False, + ): + super().__init__() + if not scaling: + head_dim = embedding_dim // num_attention_heads + scaling = 1.0 / math.sqrt(head_dim) + + self.transformer = TransformerEncoder( + vocab_size=vocab_size, + embedding_dim=embedding_dim, + padding_idx=padding_idx, + max_seq_len=max_seq_len, + ffn_dimension=ffn_dimension, + num_encoder_layers=num_encoder_layers, + num_attention_heads=num_attention_heads, + dropout=dropout, + normalize_before=normalize_before, + scaling=scaling, + ) + + def forward(self, tokens: Tensor, mask: Optional[Tensor] = None) -> Tensor: + all_layers = self.transformer(tokens) + last_layer = all_layers[-1].transpose(1, 0) + if mask is not None: + last_layer = last_layer[mask.to(torch.bool), :] + return last_layer + + +# TODO: Add Missing quant noise and spectral norm from latest Roberta head in fairseq repo +class RobertaClassificationHead(nn.Module): + def __init__(self, num_classes, input_dim, inner_dim: Optional[int] = None, dropout: float = 0.1, activation=nn.ReLU): + super().__init__() + if not inner_dim: + inner_dim = input_dim + self.dense = nn.Linear(input_dim, inner_dim) + self.dropout = nn.Dropout(dropout) + self.out_proj = nn.Linear(inner_dim, num_classes) + self.activation_fn = activation() + + def forward(self, features): + x = features[:, 0, :] + x = self.dropout(x) + x = self.dense(x) + x = self.activation_fn(x) + x = self.dropout(x) + x = self.out_proj(x) + return x + + +class RobertaModel(Module): + def __init__(self, encoder: Module, head: Optional[Module] = None): + super().__init__() + self.encoder = encoder + self.head = head + + def forward(self, tokens: Tensor) -> Tensor: + features = self.encoder(tokens) + if self.head is None: + return features + + x = self.head(features) + return x + + +def _get_model(params: RobertaEncoderParams, head: Module) -> RobertaModel: + encoder = RobertaEncoder(**asdict(params)) + return RobertaModel(encoder, head) diff --git a/torchtext/models/roberta/modules.py b/torchtext/models/roberta/modules.py new file mode 100644 index 0000000000..e9056980a2 --- /dev/null +++ b/torchtext/models/roberta/modules.py @@ -0,0 +1,304 @@ +from typing import Optional, List + +import torch +from torch import nn +from torch.nn import Module + +import math + +from torch.nn import functional as F + + +class PositionalEmbedding(Module): + def __init__( + self, num_embeddings: int, embedding_dim: int, pad_index: int + ): + super().__init__() + self.embedding = nn.Embedding(num_embeddings, embedding_dim, pad_index) + self.pad_index = pad_index + + def forward(self, input): + positions = self._make_positions(input, self.pad_index) + return self.embedding(positions) + + def max_positions(self): + if self.pad_index is not None: + return self.num_embeddings - self.pad_index - 1 + else: + return self.num_embeddings + + def _make_positions(self, tensor, pad_index: int): + masked = tensor.ne(pad_index).long() + return torch.cumsum(masked, dim=1) * masked + pad_index + + +class ResidualMLP(Module): + def __init__( + self, + input_dim: int, + hidden_dims: List[int], + dropout: float = 0.1, + activation=nn.GELU, + add_residual=True, + ): + super().__init__() + modules = [] + for last_dim, dim in zip([input_dim] + hidden_dims, hidden_dims): + modules.extend( + [nn.Linear(last_dim, dim), activation(), nn.Dropout(dropout)] + ) + + last_dim = hidden_dims[-1] if hidden_dims else input_dim + modules.extend([nn.Linear(last_dim, input_dim), nn.Dropout(dropout)]) + + self.mlp = nn.Sequential(*modules) + self.add_residual = add_residual + + def forward(self, input): + bias = self.mlp(input) + # Using hasattr to make it backward compatible with models + # which were trained before attribute was added. + if not hasattr(self, "add_residual"): + self.add_residual = True + if self.add_residual: + return input + bias + else: + return bias + + +class MultiheadSelfAttention(Module): + def __init__( + self, + embed_dim: int, + num_heads: int, + scaling: Optional[float] = None, + dropout: float = 0.1, + ): + super().__init__() + self.embed_dim = embed_dim + self.num_heads = num_heads + self.head_dim = embed_dim // num_heads + + expected_scaling = float(1 / math.sqrt(self.head_dim)) + + if not scaling and self.head_dim == 64: + scaling = 0.125 + + if not scaling: + raise Exception( + f""" + Scaling not set. Please manually set scaling for transformers with + head_dim != 64. The suggested value in this case is {expected_scaling}, + or float(1 / math.sqrt(head_dim)) + where head_dim = embed_dim // num_heads = {self.head_dim} + and embed_dim = {embed_dim} and num_heads = {num_heads}. + """ + ) + + self.scaling = scaling + self.dropout = nn.Dropout(dropout) + self.input_projection = nn.Linear(embed_dim, 3 * embed_dim) + self.output_projection = nn.Linear(embed_dim, embed_dim) + + def forward(self, query, key_padding_mask): + target_length, batch_size, embed_dim = query.size() + mask_batch_size, source_length = key_padding_mask.size() + + torch._assert(embed_dim == self.embed_dim, "query embed dim doesn't match") + torch._assert( + batch_size == mask_batch_size, + "query and key_padding_mask batch sizes differed", + ) + + projection = self.input_projection(query) + q, k, v = projection.chunk(3, dim=-1) + q = self.scaling * q + + batch_heads = batch_size * self.num_heads + + q = q.contiguous().view(-1, batch_heads, self.head_dim).transpose(0, 1) + k = k.contiguous().view(-1, batch_heads, self.head_dim).transpose(0, 1) + v = v.contiguous().view(-1, batch_heads, self.head_dim).transpose(0, 1) + + torch._assert( + k.size(1) == source_length, "key size should be equal to source length" + ) + + attn_weights = torch.bmm(q, k.transpose(1, 2)) + + torch._assert(attn_weights.dim() == 3, "Unexpected attn_weights dim") + torch._assert( + attn_weights.size(0) == batch_heads, + "attn_weights shape didn't match for batch heads", + ) + torch._assert( + attn_weights.size(1) == target_length, + "attn_weights shape didn't match for target length", + ) + torch._assert( + attn_weights.size(2) == source_length, + "attn_weights shape didn't match for source length", + ) + + attn_weights = attn_weights.view( + batch_size, self.num_heads, target_length, source_length + ) + attn_weights = attn_weights.masked_fill( + key_padding_mask.unsqueeze(1).unsqueeze(2), float("-inf") + ) + attn_weights = attn_weights.view(batch_heads, target_length, source_length) + + attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).type_as( + attn_weights + ) + attn_weights = self.dropout(attn_weights) + + attn = torch.bmm(attn_weights, v) + + torch._assert( + attn.dim() == 3, + "unexpected attn dim size", + ) + torch._assert( + attn.size(0) == batch_heads, + "attn shape didn't match for batch heads", + ) + torch._assert( + attn.size(1) == target_length, + "attn shape didn't match for target length", + ) + torch._assert( + attn.size(2) == self.head_dim, + "attn shape didn't match for head dim", + ) + attn = ( + attn.transpose(0, 1) + .contiguous() + .view(target_length, batch_size, self.head_dim * self.num_heads) + ) + attn = self.output_projection(attn) + + return attn + + +class TransformerEncoderLayer(Module): + def __init__( + self, + embedding_dim: int, + num_attention_heads: int, + ffn_dimension: Optional[int] = None, + dropout: float = 0.1, + normalize_before: bool = False, + scaling: Optional[float] = None, + ): + super().__init__() + self.dropout = nn.Dropout(dropout) + self.attention = MultiheadSelfAttention( + embedding_dim, + num_heads=num_attention_heads, + scaling=scaling, + dropout=dropout, + ) + + self.residual_mlp = ResidualMLP( + embedding_dim, + hidden_dims=[ffn_dimension or embedding_dim * 4], + add_residual=not normalize_before, + ) + + self.attention_layer_norm = nn.LayerNorm(embedding_dim) + self.final_layer_norm = nn.LayerNorm(embedding_dim) + self.normalize_before = normalize_before + + def forward(self, input, key_padding_mask): + # Using hasattr to make it backward compatible with models + # which were trained before attribute was added. + if not hasattr(self, "normalize_before"): + self.normalize_before = False + if self.normalize_before: + x = self.attention_layer_norm(input) + attention = self.attention(x, key_padding_mask) + attention = self.dropout(attention) + biased_input = input + attention + x = self.final_layer_norm(biased_input) + return self.residual_mlp(x) + biased_input + else: + attention = self.attention(input, key_padding_mask) + attention = self.dropout(attention) + biased_input = input + attention + biased_input = self.attention_layer_norm(biased_input) + + biased = self.residual_mlp(biased_input) + return self.final_layer_norm(biased) + + +class TransformerEncoder(Module): + def __init__( + self, + vocab_size: int, + embedding_dim: int, + padding_idx: int, + max_seq_len: int, + num_encoder_layers: int, + num_attention_heads: int, + ffn_dimension: Optional[int] = None, + dropout: float = 0.1, + normalize_before: bool = False, + scaling: Optional[float] = None, + ): + super().__init__() + self.padding_idx = padding_idx + self.token_embedding = nn.Embedding(vocab_size, embedding_dim, padding_idx) + self.layers = nn.ModuleList( + [ + TransformerEncoderLayer( + embedding_dim=embedding_dim, + num_attention_heads=num_attention_heads, + ffn_dimension=ffn_dimension, + dropout=dropout, + normalize_before=normalize_before, + scaling=scaling, + ) + for _ in range(num_encoder_layers) + ] + ) + self.positional_embedding = PositionalEmbedding( + max_seq_len, embedding_dim, padding_idx + ) + self.embedding_layer_norm = nn.LayerNorm(embedding_dim) + self.dropout = nn.Dropout(dropout) + self.normalize_before = normalize_before + + def forward(self, tokens: torch.Tensor) -> List[torch.Tensor]: + padding_mask = tokens.eq(self.padding_idx) + + token_embeddings = self.token_embedding(tokens) + embedded_positions = self.positional_embedding(tokens) + + embedded = token_embeddings + embedded_positions + + # Using hasattr to make it backward compatible with models + # which were trained before attribute was added. + if not hasattr(self, "normalize_before"): + self.normalize_before = False + if not self.normalize_before: + embedded = self.embedding_layer_norm(embedded) + embedded = self.dropout(embedded) + + padded_embedded = embedded * (1 - padding_mask.unsqueeze(-1).type_as(embedded)) + + # B x T x C -> T x B x C + encoded = padded_embedded.transpose(0, 1) + + states = [encoded] + + for layer in self.layers: + encoded = layer(encoded, padding_mask) + states.append(encoded) + + if self.normalize_before: + for i, state in enumerate(states): + states[i] = self.embedding_layer_norm(state) + + # states are returned as T x B x C + return states diff --git a/torchtext/models/roberta/transforms.py b/torchtext/models/roberta/transforms.py new file mode 100644 index 0000000000..609195c548 --- /dev/null +++ b/torchtext/models/roberta/transforms.py @@ -0,0 +1,59 @@ +from typing import List, Any +from torch.nn import Module +from torch.hub import load_state_dict_from_url + + +class XLMRobertaModelTransform(Module): + padding_idx: int + bos_idx: int + eos_idx: int + _vocab: Module + _sp_model: Any + + def __init__( + self, + spm_model_url: str, + vocab_url: str, + *, + bos_token: str = "", + cls_token: str = "", + pad_token: str = "", + eos_token: str = "", + sep_token: str = "", + unk_token: str = "", + mask_token: str = "", + max_seq_len: int = 514, + truncate: bool = True, + ): + super().__init__() + self.bos_token = bos_token + self.eos_token = eos_token + self.pad_token = pad_token + self.unk_token = unk_token + self.mask_token = mask_token + self.cls_token = cls_token + self.sep_token = sep_token + self.truncate = truncate + self.max_seq_len = max_seq_len + self.spm_model_url = spm_model_url + self.vocab_url = vocab_url + + def forward(self, input: str) -> List[int]: + tokens: List[int] = [self.bos_idx] + self.vocab(self.sp_model.EncodeAsPieces(input)) + if self.truncate: + tokens = tokens[: self.max_seq_len - 2] + tokens.append(self.eos_idx) + return tokens + + def load_state(self): + self.sp_model = load_state_dict_from_url(self.spm_model_url) + self.vocab = load_state_dict_from_url(self.vocab_url) + self.padding_idx = self.vocab[self.pad_token] + self.bos_idx = self.vocab[self.bos_token] + self.eos_idx = self.vocab[self.eos_token] + + +def get_xlmr_transform(spm_model_url, vocab_url, **kwargs) -> XLMRobertaModelTransform: + transform = XLMRobertaModelTransform(spm_model_url, vocab_url, **kwargs) + transform.load_state() + return transform From 9f4f0586849f1ba9d6aa3b5009eb9f4b38b2aa8c Mon Sep 17 00:00:00 2001 From: Parmeet Singh Bhatia Date: Sun, 10 Oct 2021 23:59:18 -0400 Subject: [PATCH 02/17] minor edits --- torchtext/models/roberta/bundler.py | 12 +----------- torchtext/models/roberta/modules.py | 8 -------- torchtext/models/roberta/transforms.py | 2 +- 3 files changed, 2 insertions(+), 20 deletions(-) diff --git a/torchtext/models/roberta/bundler.py b/torchtext/models/roberta/bundler.py index 27042ef203..55e4b05281 100644 --- a/torchtext/models/roberta/bundler.py +++ b/torchtext/models/roberta/bundler.py @@ -22,17 +22,7 @@ @dataclass class RobertaModelBundle: - """[summary] - - [extended_summary] - - - Args: - _params: Encoder Parameters object - _path: URL or path to model state dict - _head: Module attached to the output of encoder - transform: Factory function to build model transform - + """ Example - Pretrained encoder >>> import torch, torchtext >>> xlmr_base = torchtext.models.XLMR_BASE_ENCODER diff --git a/torchtext/models/roberta/modules.py b/torchtext/models/roberta/modules.py index e9056980a2..5f0fc01d56 100644 --- a/torchtext/models/roberta/modules.py +++ b/torchtext/models/roberta/modules.py @@ -56,8 +56,6 @@ def __init__( def forward(self, input): bias = self.mlp(input) - # Using hasattr to make it backward compatible with models - # which were trained before attribute was added. if not hasattr(self, "add_residual"): self.add_residual = True if self.add_residual: @@ -211,8 +209,6 @@ def __init__( self.normalize_before = normalize_before def forward(self, input, key_padding_mask): - # Using hasattr to make it backward compatible with models - # which were trained before attribute was added. if not hasattr(self, "normalize_before"): self.normalize_before = False if self.normalize_before: @@ -227,7 +223,6 @@ def forward(self, input, key_padding_mask): attention = self.dropout(attention) biased_input = input + attention biased_input = self.attention_layer_norm(biased_input) - biased = self.residual_mlp(biased_input) return self.final_layer_norm(biased) @@ -277,8 +272,6 @@ def forward(self, tokens: torch.Tensor) -> List[torch.Tensor]: embedded = token_embeddings + embedded_positions - # Using hasattr to make it backward compatible with models - # which were trained before attribute was added. if not hasattr(self, "normalize_before"): self.normalize_before = False if not self.normalize_before: @@ -287,7 +280,6 @@ def forward(self, tokens: torch.Tensor) -> List[torch.Tensor]: padded_embedded = embedded * (1 - padding_mask.unsqueeze(-1).type_as(embedded)) - # B x T x C -> T x B x C encoded = padded_embedded.transpose(0, 1) states = [encoded] diff --git a/torchtext/models/roberta/transforms.py b/torchtext/models/roberta/transforms.py index 609195c548..e9be88067c 100644 --- a/torchtext/models/roberta/transforms.py +++ b/torchtext/models/roberta/transforms.py @@ -53,7 +53,7 @@ def load_state(self): self.eos_idx = self.vocab[self.eos_token] -def get_xlmr_transform(spm_model_url, vocab_url, **kwargs) -> XLMRobertaModelTransform: +def get_xlmr_transform(spm_model_url, vocab_url, *, **kwargs) -> XLMRobertaModelTransform: transform = XLMRobertaModelTransform(spm_model_url, vocab_url, **kwargs) transform.load_state() return transform From 3f142eae96b636ea5a0ebf3db9ba56980f5bd27f Mon Sep 17 00:00:00 2001 From: Parmeet Singh Bhatia Date: Mon, 11 Oct 2021 00:06:34 -0400 Subject: [PATCH 03/17] minor edits --- torchtext/models/roberta/transforms.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/torchtext/models/roberta/transforms.py b/torchtext/models/roberta/transforms.py index e9be88067c..1b4de7d267 100644 --- a/torchtext/models/roberta/transforms.py +++ b/torchtext/models/roberta/transforms.py @@ -14,7 +14,6 @@ def __init__( self, spm_model_url: str, vocab_url: str, - *, bos_token: str = "", cls_token: str = "", pad_token: str = "", @@ -53,7 +52,7 @@ def load_state(self): self.eos_idx = self.vocab[self.eos_token] -def get_xlmr_transform(spm_model_url, vocab_url, *, **kwargs) -> XLMRobertaModelTransform: +def get_xlmr_transform(spm_model_url, vocab_url, **kwargs) -> XLMRobertaModelTransform: transform = XLMRobertaModelTransform(spm_model_url, vocab_url, **kwargs) transform.load_state() return transform From 67372295fa8e0d5be213d50b71a39474fcff2479 Mon Sep 17 00:00:00 2001 From: Parmeet Singh Bhatia Date: Mon, 11 Oct 2021 12:30:34 -0400 Subject: [PATCH 04/17] minor edit --- torchtext/models/roberta/bundler.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/torchtext/models/roberta/bundler.py b/torchtext/models/roberta/bundler.py index 55e4b05281..7d56dac8cd 100644 --- a/torchtext/models/roberta/bundler.py +++ b/torchtext/models/roberta/bundler.py @@ -95,10 +95,6 @@ def get_model_from_path(self, path: str, head: Optional[Module] = None) -> Rober def params(self) -> RobertaEncoderParams: return self._params - @params.setter - def params(self, params: RobertaEncoderParams): - self._params = params - XLMR_BASE_ENCODER = RobertaModelBundle( _path=os.path.join(TEXT_BUCKET, "xlmr.base.encoder.pt"), From b3a3665e7b9cd66eab1226a902929ff2bd6f97e0 Mon Sep 17 00:00:00 2001 From: Parmeet Singh Bhatia Date: Mon, 11 Oct 2021 14:50:12 -0400 Subject: [PATCH 05/17] fix examples --- torchtext/models/roberta/bundler.py | 6 ++++++ torchtext/models/roberta/transforms.py | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/torchtext/models/roberta/bundler.py b/torchtext/models/roberta/bundler.py index 7d56dac8cd..10df0b317f 100644 --- a/torchtext/models/roberta/bundler.py +++ b/torchtext/models/roberta/bundler.py @@ -32,6 +32,12 @@ class RobertaModelBundle: >>> output = model(model_input) >>> output.shape torch.Size([1, 4, 768]) + >>> input_batch = ["Hello world", "How are you!"] + >>> from torch.nn.utils.rnn import pad_sequence + >>> model_input = pad_sequence([torch.tensor(transform(d)) for d in input_batch], batch_first = True, padding_value=transform.pad_idx) + >>> output = model(model_input) + >>> output.shape + torch.Size([2, 6, 768]) Example - Pretrained encoder attached to un-initialized classification head >>> import torch, torchtext diff --git a/torchtext/models/roberta/transforms.py b/torchtext/models/roberta/transforms.py index 1b4de7d267..2ec1afd483 100644 --- a/torchtext/models/roberta/transforms.py +++ b/torchtext/models/roberta/transforms.py @@ -4,7 +4,7 @@ class XLMRobertaModelTransform(Module): - padding_idx: int + pad_idx: int bos_idx: int eos_idx: int _vocab: Module @@ -47,7 +47,7 @@ def forward(self, input: str) -> List[int]: def load_state(self): self.sp_model = load_state_dict_from_url(self.spm_model_url) self.vocab = load_state_dict_from_url(self.vocab_url) - self.padding_idx = self.vocab[self.pad_token] + self.pad_idx = self.vocab[self.pad_token] self.bos_idx = self.vocab[self.bos_token] self.eos_idx = self.vocab[self.eos_token] From 6e919e628b7b30f0b2cf583c70d68b8c23baf122 Mon Sep 17 00:00:00 2001 From: Parmeet Singh Bhatia Date: Mon, 11 Oct 2021 17:23:19 -0400 Subject: [PATCH 06/17] fix flake --- torchtext/models/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/torchtext/models/__init__.py b/torchtext/models/__init__.py index 268ef3b71b..a7cbc0c88a 100644 --- a/torchtext/models/__init__.py +++ b/torchtext/models/__init__.py @@ -1 +1 @@ -from .roberta import * \ No newline at end of file +from .roberta import * # noqa: F401, F403 From 8573895fb5f798edc0144cc85b496d945de8291b Mon Sep 17 00:00:00 2001 From: Parmeet Singh Bhatia Date: Tue, 12 Oct 2021 00:12:52 -0400 Subject: [PATCH 07/17] minor fix --- torchtext/__init__.py | 2 +- torchtext/models/roberta/bundler.py | 20 +++++++++++--------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/torchtext/__init__.py b/torchtext/__init__.py index a6dda66847..2870e81dda 100644 --- a/torchtext/__init__.py +++ b/torchtext/__init__.py @@ -1,4 +1,4 @@ -TEXT_BUCKET = 'https://download.pytorch.org/models/text' +_TEXT_BUCKET = 'https://download.pytorch.org/models/text' from . import data from . import nn diff --git a/torchtext/models/roberta/bundler.py b/torchtext/models/roberta/bundler.py index 10df0b317f..149050fd82 100644 --- a/torchtext/models/roberta/bundler.py +++ b/torchtext/models/roberta/bundler.py @@ -9,6 +9,8 @@ from torch.nn import Module import logging +logger = logging.getLogger(__name__) + from .model import ( RobertaEncoderParams, RobertaModel, @@ -17,7 +19,7 @@ from .transforms import get_xlmr_transform -from torchtext import TEXT_BUCKET +from torchtext import _TEXT_BUCKET @dataclass @@ -65,7 +67,7 @@ def get_model(self, head: Optional[Module] = None, *, dl_kwargs=None) -> Roberta if head is not None: input_head = head if self._head is not None: - logging.log("A custom head module was provided, discarding the default head module.") + logger.log("A custom head module was provided, discarding the default head module.") else: input_head = self._head @@ -83,7 +85,7 @@ def get_model_from_path(self, path: str, head: Optional[Module] = None) -> Rober if head is not None: input_head = head if self._head is not None: - logging.log("A custom head module was provided, discarding the default head module.") + logger.log("A custom head module was provided, discarding the default head module.") else: input_head = self._head @@ -103,19 +105,19 @@ def params(self) -> RobertaEncoderParams: XLMR_BASE_ENCODER = RobertaModelBundle( - _path=os.path.join(TEXT_BUCKET, "xlmr.base.encoder.pt"), + _path=os.path.join(_TEXT_BUCKET, "xlmr.base.encoder.pt"), _params=RobertaEncoderParams(vocab_size=250002), transform=partial(get_xlmr_transform, - spm_model_url=os.path.join(TEXT_BUCKET, "xlmr.sentencepiece.bpe.model.pt"), - vocab_url=os.path.join(TEXT_BUCKET, "xlmr.vocab.pt"), + spm_model_url=os.path.join(_TEXT_BUCKET, "xlmr.sentencepiece.bpe.model.pt"), + vocab_url=os.path.join(_TEXT_BUCKET, "xlmr.vocab.pt"), ) ) XLMR_LARGE_ENCODER = RobertaModelBundle( - _path=os.path.join(TEXT_BUCKET, "xlmr.large.encoder.pt"), + _path=os.path.join(_TEXT_BUCKET, "xlmr.large.encoder.pt"), _params=RobertaEncoderParams(vocab_size=250002, embedding_dim=1024, ffn_dimension=4096, num_attention_heads=16, num_encoder_layers=24), transform=partial(get_xlmr_transform, - spm_model_url=os.path.join(TEXT_BUCKET, "xlmr.sentencepiece.bpe.model.pt"), - vocab_url=os.path.join(TEXT_BUCKET, "xlmr.vocab.pt"), + spm_model_url=os.path.join(_TEXT_BUCKET, "xlmr.sentencepiece.bpe.model.pt"), + vocab_url=os.path.join(_TEXT_BUCKET, "xlmr.vocab.pt"), ) ) From acd1b676d4ad67d4f08f4f6c20ee8155e81ca15d Mon Sep 17 00:00:00 2001 From: Parmeet Singh Bhatia Date: Sun, 17 Oct 2021 20:03:12 -0400 Subject: [PATCH 08/17] add transforms and tests --- test/asset/xlmr.base.output.pt | Bin 0 -> 25323 bytes test/models/test_models.py | 47 +++++++++++++++ test/test_functional.py | 58 ++++++++++++++++++ test/test_transforms.py | 33 +++++++++++ torchtext/__init__.py | 6 ++ torchtext/data/datasets_utils.py | 6 +- torchtext/functional.py | 39 ++++++++++++ torchtext/models/roberta/bundler.py | 16 ++--- torchtext/models/roberta/transforms.py | 61 +++++++++++-------- torchtext/transforms.py | 79 +++++++++++++++++++++++++ torchtext/utils.py | 4 +- 11 files changed, 311 insertions(+), 38 deletions(-) create mode 100644 test/asset/xlmr.base.output.pt create mode 100644 test/models/test_models.py create mode 100644 test/test_functional.py create mode 100644 test/test_transforms.py create mode 100644 torchtext/functional.py create mode 100644 torchtext/transforms.py diff --git a/test/asset/xlmr.base.output.pt b/test/asset/xlmr.base.output.pt new file mode 100644 index 0000000000000000000000000000000000000000..d32b3c7d8ef21c4e807873b52a588fcec5e6d910 GIT binary patch literal 25323 zcmZ^KdsIzd^lvY{BuOesDoG_3D%D+j-NZa{uUmv{u|9mOEL&Qy3(8g$osGF$Z zx-oFq4zVRd%E3FvEL$wJSa`AI5+Tu}JH!R=f^)I((H#=e8#iwE_usTJ*mrxtPU)pV z-kX9pdE5GK43^#@sq3;sDtL#q%Z{-I&Wj}^tq&R0WD0uy-!JF??}%lEH2=ql4UO!q z)&8&X{*S>6KIk#0VW-AFcon(@5_+?_s=>K1Iv7E8-z1O%6-CaW?*fD_eT@^Ii@_Yn zRj_#oMp#`3ODuX&VNmzDIh;P`OijV3KLtSzJ z)PFw*vcHtrc>gHkz1E#t@VCg7)lt~EK?tQ((@^gFQ%+v@E1nbf1j*kUIb;%`>Ek7s z>05=rG)k}}uNMk(G{NBLBWjxT0F0mghput4kbNr(5}z!E#BN!3U4bxfuvV4db0-U} z58VZ$eN}YmMLX(gYVtWU%~bZ-X84*e!HaxK#VP)m@YQ!&wq>^-hKUV9=DzdbF;<<( zC>^1nXC5V$uLsLcu3ic)4=8Ez_y|*;R}=3uPpP)#Dv(`KPtB{2;JrK+oBdNQ^l!Cs z8M*f54_1YvW=#;TX*!BMj2|X+FXxjc&W5BdyKr;B8H_9w;tQA`NUxb9+m}m9(`Bf{ zhx3^2Hv!62Zxi)`Y_9rs0~{H-g9*RiFvW~28Xk_u?)fL-)R;%W-FE|16ARw;sR_F! z?h)D+`r^oxlXQQwEPw9x7HG+Egx5jEFva{3Oeq>BatB&5V$586HI1;#A&v zStWcLEk~6d>WN4Zgx_;8eAG_XDZrM4A&C9@bt=~XyxPz1Qc#Yqu z-{tLAyuu9nj4SOZ}TvG&xMhi zx(ky3j3ZW=yRqAVfcRx8w&>U)?nvw;{IO>O8@Krc_;(qQjB4rfbut&AG3+bnUX_6l z>b{|o@LznDl16(R$Fa4!23Q)YjXQrGhV+a5czt*ow9I;e?VC;z@$;8KeaUrj-)V!N zSB|4iw3UG>K|WYy$6WH+Z~oE!f_)!AIZUfnt;)teuh#^zSyBpi|7Onjr&& z8lv#rI|7f(?SvYq8PK*z3DplhB*MY-@hF`#6;l*0 zFa!G@qv}W*9f%qw{Bn6RCR>vI@Od0$G?y>~%O&{qrJnq@J-RF@mxaHt-hh$MYfSSp zr8S@2p>*&K=i=fFZpxGKl_f`1#P#{WWyiU&r~a$}_({cB1=%D|p>i0S9)R z1&5mqTcoB48PAHiOV4)ktIQ%`Y+*dCZ_r}X#!-|Hd4m`2t)PGE8BDmi2{P43Q6p*s z=v>%}rrVNW{>)1l-L{iDpZ3Evyy=?s4AD*5A|W_%U%>aZ;B6NbD;D6UG4)T&FdbY%G;|RhxGXtET~)Y z9-m!ta7QuyvE(oAvk>H#-`D8AcT*r-!vjVJwfQY-2O;mrO03>76U~EE&`0z;4o_Uo zMSR>^?sCos+{E-yXJ9O`G#1C6C5;$nGnTeJcHjcY2C&kuW->1NmRkpYplppLwmXI( z$E{-dEv zO!=X;jgZjgh4;c=!=_$DOJ^B4dQpfNO*n*JRjV1bNr51uuog|fs-SUdo+}OL_+u)Fde}fl(gMs`9F8XnF5$(yufS$ZHh$D=C2ouqycPR|-X@^~EH%RNG7qls^2>15=BApTkuraU-HQ(nE@8%k8 zO8Z6E9d>}G>f4ZRmQ6JuSAmXiIp`K2g2PwO;M8CB;I`bEbQaB{QhWF2PB#WGsq0jINF%7Fki?%V4@?sNQvp9-hI&P6~YjWYgNhk6A zn+vFR!2ul11oytAg>LDpAY@GihQFW92K+X`HDB*zzHlCTBwVAZ7p9XpGtQ!3j{_Rj zmz9ewSLgNJcYx%^T~wrN5F>Qk%iCPP606$YG8$_~zV6oK+s)R1*cCusmuyhnQcCR~ zT0r;HL%dm`IkcKKfwo8-`uj!`o|9X?&-7A5n<4jRs|1{%DYyRw~Nf??dBb7T{?1fqG<1z>z)I(Wl!IBo&HL zEib7}3Gsf9(8Sna~f;usa z*Sb`JgDxqwYNHq&-+KVuPi%%XjS~>#^^06w;{$U8zEJxuLFj#&hqJ0DFmLH?FzWmQ zzKMg(mY1S0@HU0ukcf3MsRJHid2siqCs2KO9TEPy9D2SVU8oV!j#E?myV76tD zo7`szRk4RL?7KOA$`Hto?INK@#kl{51fR3_DwVc54lPAd#Krt93OT4y`H>AcU2QES zDawM~6+Qep(Sg5bPy)48 z7%t*%_3zSwoAKnZcQQU$VFPKlbLjLF8$tU@J}z^xW%vG_1fT!bz(~3q+a=(%ACr_kTnaie|G|q4UtsA)sa zuUygt=Y~ye#LC;W+fa{P-ZBA>&g~-`E`COT?kf09SjFlz7=uym53*iiKMV}Hg2K2+ z=zV9xu3Z0s7WaDMZ1O3vK^OnjS`9V|PsPgZ5f3h&<5ez8n zqVdum=5F#iZ1|o-jMnL5p8ge5{J5DUCd=aEWCd_Odjb5OD}i#16mj~T0@j@uXv#TV z&N}og*xt~jr!Bi~3}`FjJFXb9yPXCz-OoIXs*SKL}6${0HJcb74n?7C*;e z1=~#3@p4uSU z3c6x{V?ww#N)#zVsGB?W+7nBJv^BYr0%3Ua(wS)4rIQZ#b`W0E4+6Z-I@mUH$ur&) zA%hY|>BJVc?O`(wk5Pl^R)|kO{lOWcQmormQ}XLv6wZ|Og1`a-;TFfRNP0Uwo+Hm^ z7l`xbVO4Pbl?H$F&Kr2HCe8XCm4i9KqNu*2sQlhs5v(y?2^lR*!0uNmcqk2%o6Rf8 ztr_`Hx_B*bRg_b%nfZW5y2#_=L&LxYisNc?J=k|j1`3Wi;l=~2c|U?MnMtYrqJ zLW$P;8Q9n&gZ-{C?A|R}*qxEie2h?r<2MOA`oWAHwq8yo)}%vJ!a;BmNfu1)e% zY_Mgm8>*>}5XqcYq&mN;vuiv=yG7zaRvvhq^T6QW<m3pJN7L8#;-IQwfn zFLq!K+f?iiANv>ccaMs&IyV{A7+MF$Gp6%$t6q?mOCrI$Q;APdl3{;M$|85I4g&g6 zv^27Ul>WKYL(YAL|VCoons`H!lOx+I=6?N3$Ul6(L`Hqg*PXV7vG9+p1JJNB;5>k$z0H;Hb z=z*kH5O{8Y^p5MGXEYu_jKT&iuagB8UI-HE@0P0=XkgIyQ*bn(0zPc~4+bt=qRvLy zw5T^2|5&7e)|hOtGF@(=Yg&dLafJ7%+llUqCs@%%3-~965}^O%IvW0bLHOT0d4rig z=(~b~vEpNLcRRwn@o_k@gCUjKJmb=E9`fJX!qQQH$oQ#1U8`R*;~x})?Jfxnj#bPh!G8 z!5n_o#<)XI&{lpM9k?DypFSNWMnCLmtDY;k`fb4PHP2xrCcUlr^$%994Wo> zgvi>RAVIu2FSK}WnFleWlKyRs`&MDRW%>mMTCWq0I$O50!Vpu6)UlPn$crhJ(zzS< z;_#7!VE4lm)R*>I48Plt1wVG-IJ@J-utST_<4@D=-QP&6+#UE>{*@^!_akk+dbokT zg)5)t(j9X|So=l~91x$)4otH}|5iJ+I%b7(=cK^(^B@{%{{>_E59?L3(LPcNessq$ z?^2R5(Cq^sHlvFK?`@$Z`Y`o+bbyh0VFNW|)$m!mE^pi|1bStb?A`~){DckGd;yE5PGM~X^B9&^M?Oxw!;9^i$%^m33mrwJ*DzxCgT)9SkdgJIxo$sXBVJGZ3(?D8Jynr#;XHZ*N9rPOX!Ax-y)nEUE z)Mj3wLhG-TZt^~aHGLK2i~eTz-;9s&^0ccKS`n^0P_M={Am(>?~vl_a|3YpEN}7o8|yS7*iUi(N!Hmp_xZFe$Ry$6^h>T<)r>!u>eJ-I%E3l~l9~^D$gYLx) zNtk$>?nyn31~8eT|NQ_d(HW>2S@7v@#3 zXyLGG2A451jdwor0S4W^(1d;SQKGpEoE1}ukA^b7_;s4xSuhn&rb~g1EMc?i%uuFs zCJ}N5ko-G^ysF(z;?(Pzf!V`U^6x3$eKG@dOB*Nhz5&!y7o*5yY4}}`3;{ORAuoLj zaeBu>=HZ)g|70WIpRj_Y|96zgi)<&RQXAl~ptdHh*h8e3Ny5r|QRwE;h5M!7!{?dH zNrBM|?$xe6=(znmwO$vEyY36~K5i*oUe(D3OU2~ySiUgSPgw><+#Zna*h4&?4%69t zC-eJa`JEN9%9=40IjZZAzk;t)4F3TmIavKhO=xyR$f$TB-) zn3p5PJ69w^{qzhp>k((Nz8azZ)?;MBtDE?{dp@kpT@6J|;V2@lNo-n{!QEeuAoW%e zIz}hZ9kX5FBQ&DLxtVw=@+#zQnFZ}re9$z15>Xf1SFXOMpzKPqfG1SBj6MSkpgJa& z)Xx{A8{UMX)t=>y2t3E5|2$Zs!9Os3yND(`{vbkXW_-x-26pqs7-Do&lrjAhN9ELF zIY+k{(E4*K&0aGGqohAFJvU;24Z8x|EMYck$5NQ}=rn7l+(3kWek-HJ&XB3h6Z#>6 z(?53rmI@sw)q!Jx&N%@Z4Kq>bT_tnxjX3+IzMe2s#363xYTB#182#QvvIR0%sl%8n zwE49QFO<++y5r15^4sVH_pd7lr6x#_i1Gf6{>sUGTk#$$RJNaKyO58z<=NQ2PJw+Y z^^P=Yj)1ae028&>3V)n*#PBn6=-b`~FFPwR<7_w;ioaB5+-y&6{nKHJ_jumuY#;Y2 zDjOyL_E0s+i|mOx)3GVkllX2uOykb$V1}0*;FbZBthtvsk6huXx;KQ@yaDYsi=btC z3nrX=2;zphxLoWLNm^nLZ2N8qGd1A@9vZR*lkCyU(73#KgFGZ==Fm&)j#)kyO2_7D z-WacX3+?YDU|ScVq4o-_*2rH}Z`s0_Zn;i-?If^mr#?+OpHED`KZ1bo6*yt74vgxb zg{%M;=@|*i$(TQ>*r1F@v`g^&2<{xW-;qzq5`b->Hae#@^?qnji z@5NTx5L{Ejleh`{Y5bpl9A+l)dne2xABOCBSA|@7so=vqYi*{MeOb7C{%JJ&G{h{p zJ_lP=PGQ2MT~HaW#5Qk~W^eA1giw(z-f8Jp9Qv>j{Z@-(&K6Ik%HvV8$sDg$eT2?U zmuYtUWUybT#E1GtpnrZ4jF~?fM5{)?`_M}mb8!+0_EUzapC*jR`D>)}atS)#oP|!S zFGEUC7LjTDNl(o1Bw*V`)nbO{d93sH2AUuzAhxm3Bt$X9GB??Iam+naPwpY{BNFe~|dmTI^(JW45LZ{97i= z&kWfN^%>Xk+9MJEVTdc}kIcZzS*3K#f%6y{F@=4<+KeY6pD}CFPI4_lo0a}n21k1( zAlPCKE3@w^EgN|QZM*8JR%iuDm{VGw_g)lZ&u~y6sD*E()iL#x2%m8^0sWQMQ&r0{ zd}Nja)tmB2A?t{V26$-dwBPozhItszuL8wmXXF7gf zpdRTi)I0tsMv3$?&7!YJl1DhL&d>ud4LeL*B@W7QB5dAX4n)_^Vx42dx%%%hyxtdO zy!%g>cMN)j^Buy$M`R8hY`H;#U*go+mu&omx26-lYI34Dr)pdh;vwILc*S9 zW1c{h${BN-ZkZ$w*H6o^%b2I2HcpkzU+@-|Ut0&Mc^a%%?=LdPV=^1D_Ci_4&UOp% z?I1^=7n7U-Q6`|B2j#K0;HuV3r@YUC;7~VMcYX<*)AgI0Nsi+J+5>U%TSNY7%~ISE z9SgRD=5W5blAK@vl$5^PMWegElgtHUNm9WJCac~IcgLwfQw4+fl2ft4n1h_7lwZM%b8ZzzG|v3c~l*m>fgmV$P!r$}+dL!vir0N-9u$DmbFKN8N!nuy{KVgkN{f{%J1I-gOXn@_(cLHHip230r*l!w;g9zopZpj|*R z#dHM{tah^Ao86QPrPdf7z zR;eg6zJvRg@eU*}uOdRTBH`kfDXg#8B$U3uli49>K*VJ_Y2mMcjjud0>WpX1<8NTH zoj+*2XQ00>5E7kgNYvM2M!!ys_4nLJM~w@~@b^S$(*4dj_-65Ur)Kg6{g>(DrfF>P zDg%5{*#-|4cfhnY$3a5WguQ8(Rhd)n#1<#{fzIJS7%7`c3^M1T&c!R-b*`B7cIl#h zdY?s6&MDH-Q4hl?ScA8egTAFQjg^ui#a9Yx?e-f~<~rb#H5V(Z69D=EmJ8y@o=xUB8Yikb{*C6*|y*$7+aJx8y&n_SYZFe>I(gR)rxaQnsv-hNdb z++X?}yroXjM%hMG7k8qg%go7F!&0gC2rqIWkF9A&4V&V*OA$Zi}P zx_vEG%{x!aA|Ju0x6{G1>?xi8yoX5F+Cu*JDWr1vQ+ZQG0yqEqQB1nT(*xrYvG!>Y z4JRk)%##Pn-lIu$=^g{Hg^!G;)O2XH)WQM(e2bnr$)L9K9mq`gW-r&?qphEn=%JO# z{60<{9h-JThdxh2ZzO@UZ9V(8?KDKiE7Rc6MG!in3>WB`uqS>{VtafR2FZ-$71Bni z_*EXgMUL?wT~5$3S|=C0zU)NgLL^CsvNGf@izLqN#SpPBm>x>D;PnJKx9Z$|;2RUE z&X+Jo+h9KLKPwG(1^A8CWr{z?l)}yJs zz6m7?*-1>B)gL-&aTTbN8m<1a5PIz%5|@dK*yg^t@`Qd}(63Sg^;F&R-m@iSvi>g; za?^nIUU3qRZ@EJDx2wXJ1H!m;<5zOpT#;36m&b(5^MS=rWO(WVY_wBiOHJ3Kvb`r2 znk#4F5;O~)wN3a$_f#l3xCLU8bTC@dknX6P#MXM)!9y~iHT2sHbGL0Km3mw7j><7~ znDG{tRVqSs!7>z5ISvQ;VIn(A58nS60G~#r8AbXKl;*`4hpZunxQbsZt^i4?PNcEV z7mNx=x!~%{bfeC6FgT&WryMh6=Zi#;xzpolO871g*8c~^d6U3-(VD#`npYC%3J;jnW{=&jh+X-_ws> zuW98yPcXJ##S})2;jNwrGZ~KYRPDYuI_2-g(uakpsQVJUARC0vi_l~p15D6Zz!^mN z;s(zw7>&?Ixqf-p+fReEWdzXdTJ&`0>1Y?871sKUX1?8J-U~c0vv~UiDw%kR0pxP)&6rT)6(#Jui zcs=$k7{@j;^(21qC%KS+1D4)T#GKxFmeL=TNRVy{xe%twAKNpQ4~Y8@BD8$jQJE8H z6jw`S{~X84wF;zFSeHtM*x)$*)uj1uFgh*Jguvuf?&0JKpnheTdi&|{F#_$XzI`l9 zoK>iP#w5zk?;#_9=D`imm0<1|1@_x#5fhz0!gZzV(Yitv=5EcUk|%r_=MXIn{d0;-ivAD%&iaD7c{Lh*T8g_?ZGbARqmXg0 z4YHMjz`1MG;!}kt>-_#F_vXi0QYiZvPffZ(j2|ur9knt@j+aNLs)L~T{xHi;@nSuh zSQ@;hniP+-fPu=zR3^zf)C+ zpFD-6DxZav(vu+9kpwHAzlT||QgEnsGTZ#im=C#khHNku=#~!x;8d6}pY~}HX+C@e zHrOxX^@|Nzp}U_s>V~9iaRLfGpKnp(uL=?E<1OT5r-8Hh8ti|d%ipY(Vg0_W#x&h; z)W>Qqz1T}wpBo1JgC%FNcYGALL{b(8lV+o>LkT&OAPmmwZg~3l0BCRefXxOk=>GTL z85Ob9bQ9HsH+47AJ8=zK78~#?ciK^1O@yShwm`}s6HZ)Himq$j3yLDM*@GJA*_HjH z=y$|`m)Ut7Q!U1#bHEBlIj|0#yZhktX(b4poC6Cde1eZxUD+*H(qZEnP2M5e8^tvP zG0C8t>V43_ews==f4JksqtuUP6Uuy&_LO9w#}W38=b1f}0^~1e+~8aFgpD zQd!rDv!kAq6oD6V@qILY)?-9cE_uah*3G zLnpv1Wqn@mcqCN-gBu+^Xr{d;3+%VrfE%9sW{uYS@mHa;*S(7ytrPoa!C1EVwR zQPp3DmF#%J!M0NDBSqM`TAP2mY7sJ-YeBAlfH;ml1ihad=*`rxBrtkB=QuGlcO=Qj4S1!#{SbZiKWJKhh4I=`PqjnK1hE{<%(MST zw0E1r#=Z5V)4dlP*T1DjbQ`?BaE~ZHF+_P^UyNwl&zWjjpw*FsxM5En3UwFGxA-yx zq%T&2zUw{`WKu)s3AEBoO(PuI{F5r)&VpX2Hf%j8&U)RE;8(8gAo^EsLP2^eWW`P5 ztMV>G@V;*t++KrTTgACYqWeh3=vf;5Tae>igxIU~srcJAlB&^5aOjyHZ0(7`;w=Zz z>R>E+?CZtKJwAw+z8FK$YE`IvbCn%ORLKTwFMRA@#g+EGqF&GUmHVk2f~IdvK<$MM z)`~8}u0`ib@j5Yf?G|k`3o_$UtZgvF;33YCmH@4{e<3UB5PV!A%yOb)ywwp^X1-TC zG~AfOS7_+7whu+fuec^MpjAP{HA_h4lU|F~`RmyxYisEJq7TaJgwdh;5p^Ud>7jR% z*vjFk@`+~;f|!mX4*3|u*JBZ|rb>$(9k&FI2+if=v@UXOhtE-;&B`Dh(n%)22>}_D zU@MahaP9_Ma`f{QFluciLDx=!UAYvvy?;Zrw@Ts5OL8!`#ErKOCorg_2}#RhNc?^g zwh-pw{8$0+p7xwXJaU3j#jX4r-ExrF^@#j(Ghr<>F5{TG)3}2XhMsOYh=3O$wEAM@ z-9%AV{hhbP$j+1W$1Y`Xu`Z;^dw!9G71zoKCMjcsW&$-^JOg6O$Fq&&q}ax~qmVXS zLIW$M$g?&HxFMMa?J7&qvnfg-H{7Eg!7JFjx?<3A%Ot^vHA$Ff4{cn=q1q%fzVY`6 z4XIiu(7bEuTFaFXnP853y(|>8d6A-`5W2x(5qau2mP`wag!aW_pwst>#eBJF9Dk-6 z97q#$@sa}E8p>kz&%efCnaSv1C676gzvy7bTt39v1MDxXP5YP4| z+PWx)(VOJKyX;bjnkGf|;x#!~taTid1}izaB2{X?tDiYEtP2l%-(cGgBJh&;S@?OD z5NG8S&dq$3#+RMO&a|ye_>p&%U*?Hw61R!jA_wfq>w!N4zBf(T5nbG$Qm@7^*d#X= zBz5|z`}G8f%jL|LrJ5jRB`niM%Gsw%C9XX+w;E=GiN=T zY2(P&D@u@&Xgw;BkRWvPZJ4prfbG2ciW!y50c%4Q)>+ zpho6)9*6c@0meDQw~xzc|FjJB3i?Z@tThKumkRpB z%ago%UIjK+UO_l(1QE%y7IN-YoGsPi)Ov5=b_FrG*X_uPd5c4ceE>a?z7R;j2To6V z0eqOZmu+{?rZNJamrwEs+asUo92f5tsb79^MOW2zu z@Q}7Ww+K=@CD0RQ!}5=t*gP#Ys04-bJi**P2ww}~OGKbyFddS9mZJ3%4#f0Nf{<+_ zW4}e5wS5y!J-bVw@u&xN67?fGp{sd?yN*Oc<1{>%ljn7Q)S{S^0(E(C4N`V>;P=H6 zZ0Jrx@|$GA&E1wdMW}*$-UG^R`9mJK&A{I_gCsBHJe=>2h6`y|V7+uLiBha$Wmnt* zznx7eqY;FAf}fIx(N+>#X#u4xL?NZm2%7D`(rK0dk^Y%su;4}sXt`TLQO{f|CvyN7 z{?0_@h%}a4m`w~9o8z0WO1!k_d5E-o4RuPg*jBWNYyRmBLIG9W%EoQ{#6O!seqITP zj64v;NC{?)eMKWLO0mt-m%+>JBrOPhhg!ngXstX8GZpjTfUG~HzAHs5aGk# z8Pb&!wJ4|YoU^*0$z0dS!pEcg1%3$?G@j#1zcJE~qq`PU}yg$K}?KWHGqcXzJ=`SBraYuR}|EX;y>U-^cTYVzdBGZj{`_zU%X zoI))w8{&e=P0;%`hV!{4LHaK)AwA8@F{iqamfTcg)h=hzZn~c|d+0HZmJtx){0v@x zOF@)1hPX~OGO@lAt0h9wt$u`_Yt2Wsb+f?bmI42&FoQ-o( zO#1{6loRmW{<8$Q?B)RJiCoE_yA+KdG(FjVPYR;W-@rS|3e0ll=u)>G>@pL1=*wOY z3!DltW7AHqxFeSQcT|QQI>AyQMFS?L;Q>gx6_IjMMUoCmf@^;fMCM6AsKyXy-}03# z&`yD0Q(M6IwJiVeg(w_0)Z)wE{bg!Rj?n5@U|pRuxU3R6HqGfhI`-Xygg;ZsW%m;T z%$SFzw(9J_wovYC;{>=oO`wksyrH)HltA%sH14fC0O?bmpxWdRNZn2Zk)RRScVQWu zqP-gmzqAYV%(eW57a?4 z;U=t0j~Vuvz6Uj%7VHgpLeoyoVOf!vP@a5?e*SHR`ZYdud*)LT5_24f3r`cVq9*jd z@DiW?K22Kn)>cGxY|^4}e7s-{RC-&$P1(<(&WDq^%hwGd z`ISC~R*Yk#TR&3?r*Gi4cq1#|!2y}I%w)TG92{Q@Pc_;EKDw zfl)M?GlA>U{sV?ea(vGF=@@io9vpVM3qA*<;OP$u7&90MwcS^sSKysjFIHgEru>A@ z5xqo>8ztKD-|+6~*--AWiK@y^2Gz{@@ZjGs)M5f@K699NO;lsu@H-`%42o-YV#bp( zJllB)Ge2>_KJ_H8dzXROUsbHn>nC=7lhFKGIRqC?CB|hQzqg(6rppvQr;J6HK3jhF0UhjkX;0V9 zj0g2QN4TO)Rn(>tM0xZdtzP_tbk;7zgv?ap5--4Mx_J~LmVmL_3PLW8r=1(jxk~0P zr`D4L@jWZ?vHKseEmq?jE5-Onu{*&zC5sIDsgR-b3NS-96Bjh~wtvd(Ci5I!u{>A9r>CO#LmSmlhwUG9o=a!w7 zL~UOczU^ow({ttkPTi3OkH31cRtv|Jds(QLyZooZ`fT^(l?GM}PcTiV9{Aa-B6sg;*FTkyu5TP|4)KHdYgre6pebMFwP zo0D)}_+-2*c?P@V^DysaB6!VPLnWhhz`2cwCxv$G-16O6F!Knp9*~5nq+F`8x&WSx zNhOZ98^OKt3)RR@L&+l>;F?A=+2!|vID|%$+dk@SiDW#gAKTA(s71iY*t2xg-55T@ zN)uC(=fKjnn(XKk2NE=JmLwUCho1HGK)K;4TQ)7b(z;{!L?%XxbtM>#K zX7QUuxX&=3t&)L5df&;j1{O3wo3In_yaK28IWV~OJPzIzhs^FqQb1#2q47Z+wUz*} z4-!z>9uLN?&%r)N4ZQkPSwDe4Rmx>NAEx7mNdX@y<8lH@_b9+hi$yr$@==Tu?Eh-} zDn;)a>O$6U2{bUQq?^Ztksppqtn}tQYM;9e3m+ZF<%{FUoz`7YG~1O`Zz`r{wU=qA zuK~IS`jdo(L162p2Chfz(8szM@~5A}+L_7FHZ;t9d~CtD_#K00?hV=VwgS&Rs{@Cu zA7o_H9#EG~prNH(IeVp3Ah%T#XUWxKN`oowR9lHM2SymDm~?#G^P5&S{8lahCF^XTe!*BC$;^#m6_}hB(_+@iWF3|R!jTRR~Non{AXty$iv>a6`XLy%; zHC2hPOczTpQU+?T@|P$G@DgVwd5kYkCtz6iC}+RMs&r75_o*b2O@2QVeR1@n&z^x(cdIR9r3x=d5#8!lahx$ci3 zzFokHN18#NfRpZ(FP$O|ZJbPUV`uPf(Z(ylT5?R$>!wR19T22Joj(@Qg>XR!SW zQJ}rn8|qRMar>!z@Ge~%#hn*}aKc-Nvl@>I_sZE~$4GXxZUgTwaR-w$1U|Q8OW=fK z9e7W8gB{Ze^!uHHk5ed7H@t;czNqtAHARqXXb#aaU9=5mG9#uYL|vv7K6N#qllf$_ zLiQw-G#rCuL7s@vjHEsmvmj>fb~HGe!vBt=FnlV9mkC@>JvVP-8%r0%o9TC5HVl-x_8uk0r!59&1$t$Y*tL>^D0M2^pg(z6!x%_!^|eh_E@| z*J7%I1Skhrb8X@Ug4!s=zkenRW|Aq~GMx{gaOE|2PSIu-rJlAd`c^|b8V#}S>Op=? z)PKMpO~oEARuDI?yvJc1EX-&?@6bM`v&M-^y6E9U!M&$^tLHW){eX{irlsTfLmVp56;Z4=2-(<>QbID#SGqVc`WE1QAEpK$@tXP5Le!?!9@M3 z>;oM&_PSIgs(dHnHMS@lE!cBnFb$4;RA-k4Uc^v)OM$oe6CKvi1cQ$) z@P5TBVz!kb-0?}^CGKw_lc&UHHSGkabx%P;4fr2Xn9UdL`JV-WKL zqXRz?<%v9V_`4X}dHN9(@wK{qx6l-Rr1m!atqdhzrfSTeOA(l#Pw7g-H*`Ip2ttLY z%G3W{LWdQ@=#}uZT&6`AD&w}&-&G;xr%xa(+a}5Wufo0tDyFx6ccky4(uagpk|LD~ zX`cHjNs=T~NRfV#lH@CeNGg>OlB6$55|WTK&%K2xsf2_i35k*$>jS!|iEMRXzVdln!DUTGV7}Bn?^`hI`VLVe9)ToW9x* z=KY0%d~Fz=s1gI2&6Vt#Pmb_Nl^mDiFbVpr zEUw00AIgSj;-@zmBp!f>)-R_p)GGd zmfi>zQpr>7X6Jjb=rv{=pBZslpBsfKQ#0XksR9)r@*bPFyHT?3sA$PD5mac3L0_4W za>Wa|p7Rnccy}nuOZ;TPC+|Q@&lJ{_ycxsS_FMYzuY=qJ%1l=AE&9w`1^rLkNa5Sj z@W?Wfhy|X49I?kZVSxgE9DEnWjJBcJy5(5!y9L^y9O@3wVcy$E;cBhLbhogH^j^D2 zf|jmgjiiqh&6kG8qbAs*s0iJ@vG8lXKP=lKLA$D&py+!mQy3-%k%!-a<)Iq5CD(z; z^VTwrPl23IlM0R0?iJ}h(I;wi7jW6ff3ffnEjaDmHF)rJ7JNFNL%1YAG)ep|+-WcY z&(-9CU&It}cYBY;R^w2mvK37tR#DwWBdPh5c-HyA1zNbl5Kj2~|Kp>OJm|M5lIB3u zRzs}OEr1(lYe09`A(AJ~5mmop{+?k-g1c|9q^>#KTy0(Y>G)fyyBdVOS=YgJ=Q>zy zWXZK^PT~wA-;l$k1T5-*b0fc<#m9<9l!djBD`N*S{k`^V&4I^6%xoXqc=-iZ_fO_* z?26E*ArZBzrwVg+<&(&T#~|@^6qWiV;I^BcMd$a$Bq6ZUa{JLIU>0OWgFg$nQQf@u z!+}@Cq{LZN^XeUzeNf^mFI2EPQ%zFyJp!y0ScX{pv0ZR z<6ZmNy@ZimUCd>OaPp!|e-YQa(h1T(^1evgQ`+S&%f3kDqoMm2;(MGwj_1j2!h-~+ zUD-n($pk`ttP~pNTxashM}VmQLC5@b;VYMXB9WlZ4czkuPF9W!igAHa4F@60As^yq z#h_x26qUCRuoo;Ao>(?Ua!GJocGHO1;YBQpP16v2jAneA!(ip z6(4cBbctT4@Cu*nym!U}YL+gBQ4gPxwF*)LQ@EnkYpm0yh{SjJlH5h5!ryB@lSj4%%g78Z@7ece`&nw8G% z##WFWN{TEmpomyT#)6vZeR9TdFY1p80`6`eNGyLu>_a~@+vy3+a$YZ{T#co6pYvI= zt|Ik06$!;lcaf(3($L!(LlcL85Wbiq0XrY8!>?7nutBthlb?T)RVrw6WNs1Yoj%8Y zzZgO-Pq&fu^A_CgEAOCcPaGM3`!_Uwl%Rp03oyXo9V}wgNcWCyQ1A2urqT#9?MwxB zPLx2~7w%|z>LC#~jleJ2lew;UpIPen{g5*_14jo%;^vk*NXc7@jrUDCfvG>|w{8}F z>|{yR?Z(6U3rgJEAp;omsGq5{`U3gn3Bi*Rv2e%?YNdMwua+*M6_-+Bb-E09V%J)3 z^)N9If3CrO<(-wh3CnS7<1+5Rw0P+JGnh(NUxuzM4X%yHG<3IP6Fe&c`SZInflZ~a|o^z39-mXW-vc0gxFoUd_uf~cC>O>Pd?lS9? zl_i^f8(FZs5^d%&?3Fxz-C6axu+mhOs$32OX+3q)<2hCM*#15pmuJ8Q%%sfj`gjOF z#j#JD8%V*d;lh!IgQ;WG9=g`Sik9&b7e+eS&=LBcu+au0L(^cKtvni5t8JxDiQ2r+ zI!9Rf$`&%qTP!E5q>|ZBov5Q$4+{{Vg-;FnI^3y)_%^KDh?_JnQ3 zhp*!nPbnmySu~Gn^hN)1H?dD|6f`&Z@Z8@h`f-^x$Z$<0=g>JY8)$)@eD2cndLdhP zFbh2cTVa@Z7-Zx{vR}ILbl8+w)*`E6e zuAkz0xyR~h-zx*!*ek;YDm`UwRzuL5K#BTNBdFacjXOr}G>$fDE&snMvS29t}n?Gjc1s>=p?+a>W8I!`CMoIV!UqK@Qx@BmDCS(x+pB4B&bLvp8K2ZW0$(OYTfSmePLUN^!KUWRPo(yqHx z2fdM8$BZ;c-RVFJE}ybY=r^_sh%$hR@fSsH@4UIW9#{0zXbB#4R2hTqmNZ* zC9$GAO_aL(Duj`yECYfX^3m(qFVW9RX)Z*~l&BlHvf&S;xz=H=Y`%*oy+c<*w(584uw@zEtqkK- zs%tSfBomY;%0kr}ZI+&W0uRin#J7qjEMcfI)4=Vt%k%*hBy19vEK6o;)gn>d&KpeJ ziO06L$1)$CHdMd(Ty%$zeMcGlz|vu9#5uLX@{F}Sgx!gNzLr5i3sX4xygb&v`xG{h z?+|v$W(yUgPt!7*0b&Zz*#MvGQ#nVm!`gxKA6SobLh^`pLm-#^X9e?;E-0Dakq;`; zUHG+9nd|aZqJD3m@cj0(mQq)9iNZl|h!1!poF&ssC@6qz=L?eRUdX~7k5A8*@@BX4 z!yxb&j}cyzgpYWv?@3cbkZ%m9f`Mujzb-~C;)Z}&(;-lPtBQI@>uk0s5#8H@a?HXCW=_N`=og{i&#&8plE70(@D$_^5cHufB%5lK+EDV;6 z!-a=a*=Q|UZneT()Svc?d}`N$+>9gilCunVwbPhXxvj;UPG;cZCj%+l(nwt`z^AlC z;`PTHG9Qd(H<#>zJ&C^H`KE(R<+)$>6)#0s-klWbe4GS13GvXz4Z=~L18maK@!YMI zakO##0vtJR5H5n-XyMq4D@rs_AQuYx^C;=xX8~3s8HUc<&7}8d2(8y&V7FBF!};qQ zpsOp5glqe<(hd#uo<18x4D+GwkQFzv)ttJ^gwxB?2QhH29%q!B#ca~O@M#$bAw}N! zBYy)r75u|>ZfIpejKZDr3^?}n(2lSz)$3=F>$Wf@mhjy=~l@xG@7I`w2@ z($7QiW^5!#%OG97|1wKl{}7!{Wy1uI>u@+g5Be@YgZWE&fNM}7?#MHSCBfH3R?p+m zdP*ibW_ zqrKkLFa@}2oD;~e9|xDZwP;(YGrL4e@VMGlA~>cK!Q_f59 z6H8vn(*D*laCA@@Z0U8UwHx*eGZuw{U(azc+m=i`a$hoYnFOXkY7TU@-yk3TR)Eh& zIl56fka$Ul3A^SwL78D7T8@l|XjM(F`A04)g@?kAO%vEPiC18a;h^Mu0`xo7@#7(T zsJ$MDI$_1kV`Ty8%h@tg8BUxZSYt(5J-%@gaq1p{uqeEWOvsO-#{0 zD&_fWQwwVP8%7htOoc@J9ILI@46l&t<-1Ed;R~0WlQ3rjU5N?=~7Z{$5!yzv_;r@UD4jdRnU#^)B?Z-xQPXZtC z8YyNF>Tm`wdw*Xs8WwD87c_ zLmasaem$f);fScUPXoton#lI>^>0aF0fCYUXkwsx2VZ5da)8$qJ06H zEy`Glsw$D5q=kuAo_uXz2lP$mpsbVxwe}h(%3gL6TXJ|Yzl1b^PKIOpN;v5&`h$gV_16;DyH4h8A8m%w_^I=D1T zA0mQ+xbRoAV9^so^Rm<-YvfGMSSOh?x!=x86`L%JmkUG|#_^D=zXyz-#^TOmC8~VI zizXVpMr%18I^dRzce8p}Vayp883c7VXmXS&4%KQ?LULD=S2 zCR7zyM$LlPWOTC?jM09F?(a82i}4i@1CFz>ia!)Z$nn((Xr?I{yYsd@4=X*rRibeGwn?g)JvK2T#hedq{W zZ4?Ot?OV*hRRe7-%}@i1}0 zy>m^m>&13p3qHUnZ+(~}^5k(jgtpnVqxsVpn4ZwfM(Epc2i)fpQAHeD?#ThWXdk5V z*T`;{J6QR`6n+pps{U&up1VGs%I0Lky0geLz|E>?;*!YMX5>LY;^+7~B`aW#DUCxp>j6!RXGCgdy z158#R*);zViTbDo7S(dtYv_vVmo8%1n~aC6e9dVld)70N z8aKpa#*W=2OJ)f-zWp%n>U5=}+#kVj?~&93f3oeZ<1o*x6Hc!mM?AmF!);?(oSdNy zQ?t8BeOLelxyyijVLy8|bPLQiiYF#<3QXB%G2A+90q^W~WA`gbtO;2K_Phqf_I%!F z>+nZG(ah;V!)=&Bb2Ss*O?`X-xk2(`_|ZCi!5Tmju=KQB`!R!+tcnUWGh61arIYTpX(HnSdc;dH*0gI z>e<}tat3h*OJRa*CCKZ2!^Fe;n3#GdJTFXzR)GO*hGA&kKDcD_km20V-c6u$ri}%^ zqU^1C90(fEqKW%PbkAOjD;i_K$=;O)o_tBXa?g@`;l7;o(edaTG9BXb6sczRMf8Z2 z;eun&K>zI!yj}Aev|lTO%jg&w-!6{_+W^mqYsHx0P%HP`VAYE$P}U#Pn@0nR&luz}0x@ljAN zrUV>=Z=@4?IuAf@&KYR$PsC|&#$r&nlSm-0DC$lsXO-H2P<+}?rZ^^t-JZpLQfGy4|o*IR+nzxqg8%S4!5ABl(Nw2{Nri%^l*`4kU0Msnw8;{DNs zA^68-Hq(ZX%vBmC{TB+^1?Nk6>fkE+=zt1l-V~$HD;XKJ_676$qYJlYH<7d{8)1}c zA~Sq<63ocNx&cna^VnDb&ijtw`A4r|sz)VKxiPr(bRv1DeUjG% zxr{cuHK7{Q(A;hj?!DUy%T?0Aa#=Z<;NlF;TVqMV&Tb6uT?6+T_du4U0v@@405hRO zBuIY39?v?Di*CFl`^ADm^z%09->ZgW>cqL+yY(bm!wnTPh0tbci9_S&;SN_jB5=vF z%q`?KY*+AF5`_;$A(!GH;hccy65oKf2g8{CD|wdKCxRVC*^oT@2TAWlc&ph!yfWh0 znw`B^AU#JIG(WDStB2Rp91>$`wEroRBdc*nffljLO@=k^UyCM;h=8t_+u*%f9;_LN z!|tdiD6#Y;f+4RgpWMo$Ur#rZ(GO%njmI{(#xEmjO%mL9D@SfO&%4RH(}#ajB z1Qd0ea)!NMS(vR9)ER7NZMUR^W^za1e2*IYHgp+HdfUWn{(fPpOQdM=u42)`ms)sh z_dAGLH5z(P$AffzC!5;Db9d$_(;VSV7T~CiRjn@YHtnh~lPYn|Q$LDwbDAx++)tI9 z8(oiMv@AhSV=@F}MByrTGYIH?1}~HwvCNvs6Z0AlW+&EBo3LPJzuG`J?non6&zMQo zEPk>-!eadR`weQP`G`(@)90jZeDKFfZ_JlD#@5;n<_3rk=)R9*)>f}EGC+d-*dIea zB*~B*o|BcJtj@XUC_wPhy~u|pu*~xUDGfRcS;J0o=`Im??$b%Q^2>&p@BT_mM=ykB z{jrcR^f`3w(P!$*3ZP0p0+y-fVrHKqkyM$7^SYXOUdua}SW`_-y72go$DX1bA9*6_^qFmX&^vH=lb7nKe2le_WqEiJn5^biy- zQ^JPhM{(r`6Y4Z492@vL)vP}<*c>(#RHEDY9OZpW|EX4ZV4V=_np{yra}TDzP(Xnj zoo;Thg)QuTfval-V4=hpRb!(eYwLNKl>E$U{7XMrxHJ-L$}5<+{y#XRhsVXo4`#OC z>hXBpMeOsmg0tVhlcx(N!jNq#ctK?}Oj&Re^U^;MwZp#n?D7=m!QW5STBd-lSU;=j z7)I5Goq!m#bXdG8fxG#B1n2lf3|+1*V{5AJvJz_m*`hNn#9J5Mr-#F+szRZD!DObQ zbAxnN-6bhwGN5Vx3v4TI7j1oS3bxl3G1#OUJ0Is@X}J_SOx{J*^mRbOQUmsD{{wBe zvqX6Yhd_M64kW3|aesji+?^ba<#Go>+UX1g_QjDh&xv4H{RWClqeQ=Tl}XS8MNUQg z2Wv17#ruShJ&1@b^L+|aw>-xCa-+b@qsMaf-l?F!-wEv>9}$*p%_OniZt&IMDDLS| z<_djFFy=)O*pJ^QdepNC6H8=xe#2zaWcmWUp3Ws@b}sn%s28V0Zu_TD6_UN=d`~9hn#w`~{9|2BKLQ z4ZS~_u&2qIN^y2zuR2YnF1Hr~Uf%>6Ua!o)+yXZoD*)r*a@e>0EqJBX@Z3d2NpTf0 zlpT&fy0t)dZN^gnB2?Ki1F|1HA^SFo!<`cYK!0q2LgPYmcvlC;OWBYy#t+ESiPs=G z9`U*RP%fJ_;EcO@SYaCiHO`hyUdfSlzqEo!w#Cf)(^<5blf!B|??a;C7&<$x5w;pl z0RMCHU|>*>MqL+h*bT~KDiUY-RF`2;wJlWqZYTYZoS-zjSZF*Z8h4rx#*x3&IPc-d zVU7+5-&CuiOE&~{KLD)c)m`gr&jP{UCKAqbB_#j89?U=fj{mQ*{|j}z@X-JO literal 0 HcmV?d00001 diff --git a/test/models/test_models.py b/test/models/test_models.py new file mode 100644 index 0000000000..7e9ba7ef8f --- /dev/null +++ b/test/models/test_models.py @@ -0,0 +1,47 @@ +import torchtext +import torch + +from ..common.torchtext_test_case import TorchtextTestCase +from ..common.assets import get_asset_path + + +class TestModels(TorchtextTestCase): + def test_xlmr_base_output(self): + asset_name = "xlmr.base.output.pt" + asset_path = get_asset_path(asset_name) + xlmr_base = torchtext.models.XLMR_BASE_ENCODER + model = xlmr_base.get_model() + model = model.eval() + model_input = torch.tensor([[0, 43523, 52005, 3647, 13293, 113307, 40514, 2]]) + actual = model(model_input) + expected = torch.load(asset_path) + torch.testing.assert_close(actual, expected) + + def test_xlmr_base_jit_output(self): + asset_name = "xlmr.base.output.pt" + asset_path = get_asset_path(asset_name) + xlmr_base = torchtext.models.XLMR_BASE_ENCODER + model = xlmr_base.get_model() + model = model.eval() + model_jit = torch.jit.script(model) + model_input = torch.tensor([[0, 43523, 52005, 3647, 13293, 113307, 40514, 2]]) + actual = model_jit(model_input) + expected = torch.load(asset_path) + torch.testing.assert_close(actual, expected) + + def test_xlmr_transform(self): + xlmr_base = torchtext.models.XLMR_BASE_ENCODER + transform = xlmr_base.transform() + test_text = "XLMR base Model Comparison" + actual = transform([test_text]) + expected = [[0, 43523, 52005, 3647, 13293, 113307, 40514, 2]] + torch.testing.assert_close(actual, expected) + + def test_xlmr_transform_jit(self): + xlmr_base = torchtext.models.XLMR_BASE_ENCODER + transform = xlmr_base.transform() + transform_jit = torch.jit.script(transform) + test_text = "XLMR base Model Comparison" + actual = transform_jit([test_text]) + expected = [[0, 43523, 52005, 3647, 13293, 113307, 40514, 2]] + torch.testing.assert_close(actual, expected) diff --git a/test/test_functional.py b/test/test_functional.py new file mode 100644 index 0000000000..6d71dd71aa --- /dev/null +++ b/test/test_functional.py @@ -0,0 +1,58 @@ +import torch +from torchtext import functional +from .common.torchtext_test_case import TorchtextTestCase + + +class TestFunctional(TorchtextTestCase): + def test_to_tensor(self): + input = [[1, 2], [1, 2, 3]] + padding_value = 0 + actual = functional.to_tensor(input, padding_value=padding_value) + expected = torch.tensor([[1, 2, 0], [1, 2, 3]], dtype=torch.long) + torch.testing.assert_close(actual, expected) + + def test_to_tensor_jit(self): + input = [[1, 2], [1, 2, 3]] + padding_value = 0 + to_tensor_jit = torch.jit.script(functional.to_tensor) + actual = to_tensor_jit(input, padding_value=padding_value) + expected = torch.tensor([[1, 2, 0], [1, 2, 3]], dtype=torch.long) + torch.testing.assert_close(actual, expected) + + def test_truncate(self): + input = [[1, 2], [1, 2, 3]] + max_seq_len = 2 + actual = functional.truncate(input, max_seq_len=max_seq_len) + expected = [[1, 2], [1, 2]] + self.assertEqual(actual, expected) + + def test_truncate_jit(self): + input = [[1, 2], [1, 2, 3]] + max_seq_len = 2 + truncate_jit = torch.jit.script(functional.truncate) + actual = truncate_jit(input, max_seq_len=max_seq_len) + expected = [[1, 2], [1, 2]] + self.assertEqual(actual, expected) + + def test_add_token(self): + input = [[1, 2], [1, 2, 3]] + token_id = 0 + actual = functional.add_token(input, token_id=token_id) + expected = [[0, 1, 2], [0, 1, 2, 3]] + self.assertEqual(actual, expected) + + actual = functional.add_token(input, token_id=token_id, begin=False) + expected = [[1, 2, 0], [1, 2, 3, 0]] + self.assertEqual(actual, expected) + + def test_add_token_jit(self): + input = [[1, 2], [1, 2, 3]] + token_id = 0 + add_token_jit = torch.jit.script(functional.add_token) + actual = add_token_jit(input, token_id=token_id) + expected = [[0, 1, 2], [0, 1, 2, 3]] + self.assertEqual(actual, expected) + + actual = add_token_jit(input, token_id=token_id, begin=False) + expected = [[1, 2, 0], [1, 2, 3, 0]] + self.assertEqual(actual, expected) diff --git a/test/test_transforms.py b/test/test_transforms.py new file mode 100644 index 0000000000..ab941fb39c --- /dev/null +++ b/test/test_transforms.py @@ -0,0 +1,33 @@ +import torch +from torchtext import transforms +from torchtext.vocab import vocab +from collections import OrderedDict + +from .common.torchtext_test_case import TorchtextTestCase +from .common.assets import get_asset_path + + +class TestTransforms(TorchtextTestCase): + def test_spmtokenizer_transform(self): + asset_name = "spm_example.model" + asset_path = get_asset_path(asset_name) + transform = transforms.SpmTokenizerTransform(asset_path) + actual = transform(["Hello World!, how are you?"]) + expected = [['▁Hello', '▁World', '!', ',', '▁how', '▁are', '▁you', '?']] + self.assertEqual(actual, expected) + + def test_spmtokenizer_transform_jit(self): + asset_name = "spm_example.model" + asset_path = get_asset_path(asset_name) + transform = transforms.SpmTokenizerTransform(asset_path) + transform_jit = torch.jit.script(transform) + actual = transform_jit(["Hello World!, how are you?"]) + expected = [['▁Hello', '▁World', '!', ',', '▁how', '▁are', '▁you', '?']] + self.assertEqual(actual, expected) + + def test_vocab_transform(self): + vocab_obj = vocab(OrderedDict([('a', 1), ('b', 1), ('c', 1)])) + transform = transforms.VocabTransform(vocab_obj) + actual = transform([['a', 'b', 'c']]) + expected = [[0, 1, 2]] + self.assertEqual(actual, expected) diff --git a/torchtext/__init__.py b/torchtext/__init__.py index 2870e81dda..a5c2802614 100644 --- a/torchtext/__init__.py +++ b/torchtext/__init__.py @@ -1,10 +1,14 @@ +import os _TEXT_BUCKET = 'https://download.pytorch.org/models/text' +_CACHE_DIR = os.path.expanduser('~/.torchtext/cache') from . import data from . import nn from . import datasets from . import utils from . import vocab +from . import transforms +from . import functional from . import models from . import experimental from . import legacy @@ -21,6 +25,8 @@ 'datasets', 'utils', 'vocab', + 'transforms', + 'functional', 'models', 'experimental', 'legacy'] diff --git a/torchtext/data/datasets_utils.py b/torchtext/data/datasets_utils.py index 571b43c479..12297931f9 100644 --- a/torchtext/data/datasets_utils.py +++ b/torchtext/data/datasets_utils.py @@ -15,6 +15,8 @@ import defusedxml.ElementTree as ET except ImportError: import xml.etree.ElementTree as ET + +from torchtext import _CACHE_DIR """ These functions and classes are meant solely for use in torchtext.datasets and not for public consumption yet. @@ -213,7 +215,7 @@ def _wrap_split_argument_with_fn(fn, splits): raise ValueError("Internal Error: Given function {} did not adhere to standard signature.".format(fn)) @functools.wraps(fn) - def new_fn(root=os.path.expanduser('~/.torchtext/cache'), split=splits, **kwargs): + def new_fn(root=_CACHE_DIR, split=splits, **kwargs): result = [] for item in _check_default_set(split, splits, fn.__name__): result.append(fn(root, item, **kwargs)) @@ -250,7 +252,7 @@ def decorator(func): raise ValueError("Internal Error: Given function {} did not adhere to standard signature.".format(fn)) @functools.wraps(func) - def wrapper(root=os.path.expanduser('~/.torchtext/cache'), *args, **kwargs): + def wrapper(root=_CACHE_DIR, *args, **kwargs): new_root = os.path.join(root, dataset_name) if not os.path.exists(new_root): os.makedirs(new_root) diff --git a/torchtext/functional.py b/torchtext/functional.py new file mode 100644 index 0000000000..abaec09acb --- /dev/null +++ b/torchtext/functional.py @@ -0,0 +1,39 @@ +import torch +from torch import Tensor +from torch.nn.utils.rnn import pad_sequence +from typing import List, Optional + + +def to_tensor(input: List[List[int]], padding_value: Optional[int] = None) -> Tensor: + if padding_value is None: + output = torch.tensor(input, dtype=torch.long) + return output + else: + output = pad_sequence( + [torch.tensor(ids, dtype=torch.long) for ids in input], + batch_first=True, + padding_value=float(padding_value) + ) + return output + + +def truncate(input: List[List[int]], max_seq_len: int) -> List[List[int]]: + output: List[List[int]] = [] + + for ids in input: + output.append(ids[:max_seq_len]) + + return output + + +def add_token(input: List[List[int]], token_id: int, begin: bool = True) -> List[List[int]]: + output: List[List[int]] = [] + + if begin: + for ids in input: + output.append([token_id] + ids) + else: + for ids in input: + output.append(ids + [token_id]) + + return output diff --git a/torchtext/models/roberta/bundler.py b/torchtext/models/roberta/bundler.py index 149050fd82..22a2e3b402 100644 --- a/torchtext/models/roberta/bundler.py +++ b/torchtext/models/roberta/bundler.py @@ -30,13 +30,13 @@ class RobertaModelBundle: >>> xlmr_base = torchtext.models.XLMR_BASE_ENCODER >>> model = xlmr_base.get_model() >>> transform = xlmr_base.transform() - >>> model_input = torch.tensor(transform("Hello World")).unsqueeze(0) + >>> model_input = torch.tensor(transform(["Hello World"])) >>> output = model(model_input) >>> output.shape torch.Size([1, 4, 768]) >>> input_batch = ["Hello world", "How are you!"] - >>> from torch.nn.utils.rnn import pad_sequence - >>> model_input = pad_sequence([torch.tensor(transform(d)) for d in input_batch], batch_first = True, padding_value=transform.pad_idx) + >>> from torchtext.functional import to_tensor + >>> model_input = to_tensor(transform(input_batch), padding_value=transform.pad_idx) >>> output = model(model_input) >>> output.shape torch.Size([2, 6, 768]) @@ -47,7 +47,7 @@ class RobertaModelBundle: >>> classifier_head = torchtext.models.RobertaClassificationHead(num_classes=2, input_dim = xlmr_large.params.embedding_dim) >>> classification_model = xlmr_large.get_model(head=classifier_head) >>> transform = xlmr_large.transform() - >>> model_input = torch.tensor(transform("Hello World")).unsqueeze(0) + >>> model_input = torch.tensor(transform(["Hello World"])) >>> output = classification_model(model_input) >>> output.shape torch.Size([1, 2]) @@ -108,8 +108,8 @@ def params(self) -> RobertaEncoderParams: _path=os.path.join(_TEXT_BUCKET, "xlmr.base.encoder.pt"), _params=RobertaEncoderParams(vocab_size=250002), transform=partial(get_xlmr_transform, - spm_model_url=os.path.join(_TEXT_BUCKET, "xlmr.sentencepiece.bpe.model.pt"), - vocab_url=os.path.join(_TEXT_BUCKET, "xlmr.vocab.pt"), + vocab_path=os.path.join(_TEXT_BUCKET, "xlmr.vocab.pt"), + spm_model_path=os.path.join(_TEXT_BUCKET, "xlmr.sentencepiece.bpe.model"), ) ) @@ -117,7 +117,7 @@ def params(self) -> RobertaEncoderParams: _path=os.path.join(_TEXT_BUCKET, "xlmr.large.encoder.pt"), _params=RobertaEncoderParams(vocab_size=250002, embedding_dim=1024, ffn_dimension=4096, num_attention_heads=16, num_encoder_layers=24), transform=partial(get_xlmr_transform, - spm_model_url=os.path.join(_TEXT_BUCKET, "xlmr.sentencepiece.bpe.model.pt"), - vocab_url=os.path.join(_TEXT_BUCKET, "xlmr.vocab.pt"), + vocab_path=os.path.join(_TEXT_BUCKET, "xlmr.vocab.pt"), + spm_model_path=os.path.join(_TEXT_BUCKET, "xlmr.sentencepiece.bpe.model"), ) ) diff --git a/torchtext/models/roberta/transforms.py b/torchtext/models/roberta/transforms.py index 2ec1afd483..bff390a261 100644 --- a/torchtext/models/roberta/transforms.py +++ b/torchtext/models/roberta/transforms.py @@ -1,19 +1,18 @@ -from typing import List, Any +import os +import torch from torch.nn import Module from torch.hub import load_state_dict_from_url +from torchtext import transforms +from torchtext import functional +from typing import List -class XLMRobertaModelTransform(Module): - pad_idx: int - bos_idx: int - eos_idx: int - _vocab: Module - _sp_model: Any +class XLMRobertaModelTransform(Module): def __init__( self, - spm_model_url: str, - vocab_url: str, + vocab_path: str, + spm_model_path: str, bos_token: str = "", cls_token: str = "", pad_token: str = "", @@ -22,7 +21,6 @@ def __init__( unk_token: str = "", mask_token: str = "", max_seq_len: int = 514, - truncate: bool = True, ): super().__init__() self.bos_token = bos_token @@ -32,27 +30,38 @@ def __init__( self.mask_token = mask_token self.cls_token = cls_token self.sep_token = sep_token - self.truncate = truncate self.max_seq_len = max_seq_len - self.spm_model_url = spm_model_url - self.vocab_url = vocab_url - - def forward(self, input: str) -> List[int]: - tokens: List[int] = [self.bos_idx] + self.vocab(self.sp_model.EncodeAsPieces(input)) - if self.truncate: - tokens = tokens[: self.max_seq_len - 2] - tokens.append(self.eos_idx) - return tokens - def load_state(self): - self.sp_model = load_state_dict_from_url(self.spm_model_url) - self.vocab = load_state_dict_from_url(self.vocab_url) + self.token_transform = transforms.SpmTokenizerTransform(spm_model_path) + + if os.path.exists(vocab_path): + self.vocab = torch.load(vocab_path) + else: + self.vocab = load_state_dict_from_url(vocab_path) + + self.vocab_transform = transforms.VocabTransform(self.vocab) self.pad_idx = self.vocab[self.pad_token] self.bos_idx = self.vocab[self.bos_token] self.eos_idx = self.vocab[self.eos_token] + def forward(self, input: List[str], + add_bos: bool = True, + add_eos: bool = True, + truncate: bool = True) -> List[List[int]]: + tokens: List[List[int]] = self.vocab_transform(self.token_transform(input)) + + if truncate: + tokens = functional.truncate(tokens, self.max_seq_len - 2) + + if add_bos: + tokens = functional.add_token(tokens, self.bos_idx) + + if add_eos: + tokens = functional.add_token(tokens, self.eos_idx, begin=False) + + return tokens + -def get_xlmr_transform(spm_model_url, vocab_url, **kwargs) -> XLMRobertaModelTransform: - transform = XLMRobertaModelTransform(spm_model_url, vocab_url, **kwargs) - transform.load_state() +def get_xlmr_transform(vocab_path, spm_model_path, **kwargs) -> XLMRobertaModelTransform: + transform = XLMRobertaModelTransform(vocab_path, spm_model_path, **kwargs) return transform diff --git a/torchtext/transforms.py b/torchtext/transforms.py new file mode 100644 index 0000000000..43710ec4f3 --- /dev/null +++ b/torchtext/transforms.py @@ -0,0 +1,79 @@ +from torch.nn import Module +from torchtext.data.functional import load_sp_model +from torchtext.utils import download_from_url +import torchtext +from typing import List + +import os + +PRETRAINED_SP_MODEL = { + 'text_unigram_15000': 'https://pytorch.s3.amazonaws.com/models/text/pretrained_spm/text_unigram_15000.model', + 'text_unigram_25000': 'https://pytorch.s3.amazonaws.com/models/text/pretrained_spm/text_unigram_25000.model', + 'text_unigram_50000': 'https://pytorch.s3.amazonaws.com/models/text/pretrained_spm/text_unigram_50000.model', + 'text_bpe_15000': 'https://pytorch.s3.amazonaws.com/models/text/pretrained_spm/text_bpe_15000.model', + 'text_bpe_25000': 'https://pytorch.s3.amazonaws.com/models/text/pretrained_spm/text_bpe_25000.model', + 'text_bpe_50000': 'https://pytorch.s3.amazonaws.com/models/text/pretrained_spm/text_bpe_50000.model' +} + + +class SpmTokenizerTransform(Module): + """ + Transform for Sentence Piece tokenizer. + + Examples: + >>> from torchtext.transforms import PRETRAINED_SP_MODEL + >>> from torchtext.transforms import SpmTokenizerTransform + >>> transform = SpmTokenizerTransform(PRETRAINED_SP_MODEL["text_unigram_15000"]) + >>> transform(["hello world", "attention is all you need!"]) + """ + + def __init__(self, sp_model_path: str): + super().__init__() + if os.path.exists(sp_model_path): + local_path = sp_model_path + else: + local_path = download_from_url(sp_model_path) + self.sp_model = load_sp_model(local_path) + + def forward(self, input: List[str]) -> List[List[str]]: + tokens: List[List[str]] = [] + for text in input: + tokens.append(self.sp_model.EncodeAsPieces(text)) + return tokens + + +class VocabTransform(Module): + r"""Vocab transform + + Args: + vocab: an instance of torchtext.vocab.Vocab class. + + Example: + >>> import torch + >>> from torchtext.vocab import vocab_from_file_object + >>> f = open('vocab.txt', 'r') + >>> vocab_transform = VocabTransform(vocab_from_file_object(f)) + >>> jit_vocab_transform = torch.jit.script(vocab_transform) + """ + + def __init__(self, vocab): + super().__init__() + assert isinstance(vocab, torchtext.vocab.Vocab) + self.vocab = vocab + + def forward(self, input: List[List[str]]) -> List[List[int]]: + r""" + + Args: + input: list of tokens + + Example: + >>> vocab_transform(['here', 'is', 'an', 'example']) + + """ + + output: List[List[int]] = [] + for tokens in input: + output.append(self.vocab.lookup_indices(tokens)) + + return output diff --git a/torchtext/utils.py b/torchtext/utils.py index 9a7ad1dde7..a1776fa01f 100644 --- a/torchtext/utils.py +++ b/torchtext/utils.py @@ -8,7 +8,7 @@ import zipfile import gzip from ._download_hooks import _DATASET_DOWNLOAD_MANAGER - +from torchtext import _CACHE_DIR def reporthook(t): """ @@ -67,7 +67,7 @@ def _check_hash(path, hash_value, hash_type): raise RuntimeError("The hash of {} does not match. Delete the file manually and retry.".format(os.path.abspath(path))) -def download_from_url(url, path=None, root='.data', overwrite=False, hash_value=None, +def download_from_url(url, path=None, root=_CACHE_DIR, overwrite=False, hash_value=None, hash_type="sha256"): """Download file, with logic (from tensor2tensor) for Google Drive. Returns the path to the downloaded file. From 4c41ded22199b109e332a7ebaed06e5de8be6f3d Mon Sep 17 00:00:00 2001 From: Parmeet Singh Bhatia Date: Sun, 17 Oct 2021 20:12:40 -0400 Subject: [PATCH 09/17] add test for xlmr large --- test/asset/xlmr.large.output.pt | Bin 0 -> 33515 bytes test/models/test_models.py | 23 +++++++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 test/asset/xlmr.large.output.pt diff --git a/test/asset/xlmr.large.output.pt b/test/asset/xlmr.large.output.pt new file mode 100644 index 0000000000000000000000000000000000000000..5888eb54b1accd0f7a5be03990d96e5321546a3e GIT binary patch literal 33515 zcmZ^KeN;_P`0tSas59RWwuDT4-&UA*)rIDu=saP z;MOgY3j~@WTZSx}FPJ}QzVrftPi^EDZ8l z8?@HPZvC1NnJv=#E?b6$Y>{!HPm4vAjV0e~j4JWRA`F z|7*PeWANfXALdU$DG7Nv@W~$fG>kAnL!DT+ItU&24<-%GC9HFE73ol%%G!Ppg80K_ zq|q~;J=pDwmlKZ>?Qv?z-iMRn>PNt0z9GskyvE`$^|6HR=XmDVMl7t}0|&;Kg0fLA z`S3Xfb#6KlKa*(Dx+T6it}YH=UY~@HN#5u=IDko-d}fW>-&u}}8SHo`%dXYt@=Fg+ zFxNlUT!t zyhu%M(x*y_9w)=(p_$OTWEq;9+$5IP9qeSuJN*4Z9TjUv;nwM9m>HD{;oB=gcEu?; ztKP=WW?v+-cHPA4%MLJFnM?{#?}e4^_E3~H1hsiI`BXg@`nEn1ny+XfIwLnQpM=9? zlKKM-`n?f~b-ILS%Ys09lq2Svq_IbzFS8_V3*lLP7q~bh0S4SZidHr}6#ZS;NIvnw zd|=8MP&>U0UZgUxvCk$_5#gxq>4<5c@?n&N2X+DLeusEn^SyFw}>{aB1vH z2!+CFNlfL~VD@p=O>jzhkGc}J(77i~IHz_c+znm@iNOmoZ`pR{8MckFOexY{AV+4n ze86^hZ)oj`!pcAKVA5M6idy$TsCG$+Ti?p^@1MdUQg<=5yvxQkV+oMZkiqghIhfL8 zPXBuOKxcLkPVeX;fw%VI3FD!l;yYRxvor}$1-V1u({beI?=fid#gxRJl7hh4N|B-S zDXdIrwQ6)tCI8B-U{mdFaxTP%>78Q2)^>HFl$#{nQSutGRH5OafAgI(W z7HvwF$3zK5INF|%&GB1^io#~n^Kl|nYCB=XaXsvQ<4GP*YGfVXCS%IcW2`91gay2P z0moyn=OPgXDl)F{0;JM24ecFA$aLZ z1}JO@5QhDqMfbnkS+g?WsQ^(Dtz8Wbn8?xl%$D_E(`|Rew(O1xQ1ipkFZa# zW?|^hGEtu?2tP!c&|2HCplN$vq&mEw>tsB#00Plyox>hZ?HQJ`!P1hm^~f&7BwSmDzJAyEieDTx=%#o6Ww{*1zq$X2x-ZAh@{>gQxeQdL zU1if-mcyR6N+>PN1mnZu@W=QZ*4&xQlpbCpp{YlS@|+~-9Fj?1507E~!z$pc+(B|~ zc@!(ZxR+%8O(J$*&XU`?8O+fz0YX3T6$&m+s#tRM6M5<50(K8B2rGr_D;6adVOgdo zqJbR0eNU2CCFV0<6EpnennO~~4}+?~syP1UC!%1w24BV~KthnBb?D8zB(2z=R1|fS zE6qIuk+Q?p$AFSudBjKy9bMe-$2IyN` z3-I9_4bmEqmct*Dm`F+RIbT2mZF@+5=qgdxtR$uonO;$TI{~X~FJbP4M>z1i1CPmY zR`vZDxNi&wrMvY)tIm~ZFfa$Jiw(fLK?lgOPNMrujSRZ@2296pVK3dgneJQ#q&Jh` zuayyo?&uYsXpx5p!%T4Te=9JgL`3>dNa5I1is(N73E8`6fb~_Bh`hUB5tm0#+0^NY zOf7sLd~BV~9lM{R`5Z&IH#80VHnt0Y{#l4UjI*#ql4x*l7|MQ51-CQ>Z0rPRX!0gM zEn*or_W;}JF{o*sTjAC9K$MnqlSrq&5z6#PV8BpcRC<2|%A;+`p6V2|3}C{uJIf(R zQJoE(IxlKD7mu~6nfU5lJB#}00mT!pf!DYWSgtXYhGx5xu#LyC;KFfOl%kGmTO&#Q zu|O0|JUPQg&4wv7{1yrhUz#C3T9G#_Ok+Wdip6VK4Qd_y@j%viZk3$-Bl5|=e!J0obHQ#12={C)5_4@{uP?r9>LfJ zQ`oZi9w4yQ6$N#zfrzp}bmygWn9=uHblG7T#O@vkc9YUjQuQ@?n_33y{-eOz^&kda z5y0`WkD1vKM;N`}Jrl0nhzB0b#>(f0Lir331lo@p4EQ_aQ%GxqhOCJlNU2jRz2=r3^)ll9Jn_t{M}s(uA{<(QG$`(>nO zP%euKy~@s5|732bW8LaRW0n6jfC)H8!Vi7c#b49b-Q?3wg+D%!+zd(R1rx zMYF674BQwB6FR!-=d&DZQcCgl#b;38>PC?F` zV~}*HkmxKbCBq-nPjXGM%T@qVFN=td+Fde#pA3HTkp@X{B%Q;*3;Uy-u(8=1 z&o~^yYm%#QxAh6uusV)d?a3fVsup61S-dcO-ZUZ?=7ZD6$iTOOZWf`{$Qn9JA#857 zX!?o+r0vvqR(ryQZ(FE|Q zaEx8<2@;R(vCwN2glM)CV;LosxtED2dkY!%t4u;0FSa6oHYK<#hvA+=3&Y6g+L9=nnKrh}MuL;}SQkWl0gT&>tN#ez5 z=AmVQ9;54s=FGjqA!Mj0}0G8dm{!Ud?P0Yg)sMJI?y&&5&r6|02w>e@_WKUJ$spDISpd|{q%>c_%SV4S* zeu%bfak-in7W!9`2@ZFd@=Qq{`OX5vm*1H|E6l)BZvk&tF(wAG$Dn7^Rgm2@0Z%q4 z!t|tHOyQS_C}Oq*9XlnACA1fyS>{m)DL5&5l6Qt_eKAJs%rxQoix;tYU5>EEZ5-8~ z|BeNmn~V>eX5omjbJ?txMBHnW$}|^TtC+c43DjQChKqaNGEJ>bqTYYqM8S2aXwla= z)U=eeGObml-nvSVa9(z1ZNhzM%Rda?zCFasNg7rmre{gFrX~CI)*h~}^k%(Uon+hw zJ9K=c&YUItg|*L1FsV`t|ExSp`=a#mi|-AV1)A(pNC(l>T?EB*^MsZK3q%fg-BD0k zW;rgUg=OU)wL0pV1RkF>LE1EjrFxH|Y;FcxNi9Xcw1=W6{`b*i=~nFV)guG5MMTph znrT;Sg65tMp_SAmOcYjQ$w^0ywKc~_RT+45atn)k?1}+#s_;adh6syl(b8Zde7D-j zA{HEEb`DpB28L41dT^ZZ`O*E5>DdO`(v^6l!eXIe%s6aNGllZSrXqdP&MH3yTUBl= zzdFZC5n_HeGS`^b=(}JQIIp-sROmsJwNz!2hvKlc_+rI_5EY($xeazn{ziE>V>H=+ zR2cW+AX=P=K&hikS@jwPY;N_$!h`8}?9?SzAG`uje%gXP_9enWl~MeFlsB+tli&hQ z!(iPM_-bQ@C8hz;=Boht2Tr3^>;)n(I?B>Fn~=~6M}>}SN=bhAY3_XflW1R_5cw1@ zxb^)4X`QXbvX;!CYWJPs>b%#iZ01BzJhmG5Jvxn__C&E;1#-~yeI*nbHM6Yv?d(>^ zI+Tnb4XvkCMbamRlj0e5!o*n{$-qeqSkQG2>i0*Gs*#sSz4kGYpWQQI)%i*?>sclU zgyXQH^b9NC`9nCZdp|DE6vCoI0=o6}0k+Be5J~M^3gM^6L55Lu%c~^OxQnyh40=^VwPV*r{Qx!Fi6iTq;y#xy*}tmpfYnr5Oy@<{Mr zZ~_?EVWIaiY}u=Vh5h>Ii=dr)! zQ>`rezA(md5%*Y>=U}YJSb<$Pqe$B@BUrB5h91L}Ahs$2t_lugM$I1btnw=MU6LYK z{q})D)J?X%K@;Du>0;K|n=7umP2%z+)8MngDn9O#EyihO;ng4YO#56eJnlYBr>z)* zZ`D6xPH-!P>}1#z?nSI74grUCBlw-yp=e@MC0bY~MWquaLPPUk@-5v0pBnvucY%LV zU5q`Xt80a$i&7~0c!mcQ>7#~TJhbl)A@#d|6G!HTK|%*!_DvbJ^OWF?y&DZ3s|C;Y zRbiTc9S?1N4}rm&LhI^0t7q-cP${pKBo(hC=}vtRcC(o1ru*|K|Ls@a6hke#_GWi``$N#n8P}U*S9#zCA;4C;!1oQ=j1R*dkcp zdKW!&=3$iPXl^xS7Fju_0!kXEQIE7*_`D;PPkQhHrCvl~#Jcm$xMdg5X&(a4orh^i z`Z^*AwY&N}u!N;QOu*2OQP3|xg7?(zrEc%`LavkqouBZ4%_(Ss*@@P)S9u}S#|@`f zx|L|&fk37?q}3{Pp9NJ~Fp^s8Gz&*fli>zI%7AOuBLg0h9h3&om#7MQPb~@?)bn8XUY2Csw#XxNA#Az>ABbr1b?ACoGgvcJn=u zTf3F%^(Ejt=^iw2u;PN79Tm0-qPd z$fHGnLO>$Do;*L54<|n(ZqC=B+3(!=k{eUuhUF!8uceL^RNCSa-E@?6Z6iU=(p)Fj zjE5IbWLJ0RL;vz)AkqB@e9Dk~U)9LZzYAk}*ABy3CeIbt-^F?*#O&!gtpD~oVh|xg z8@KxMg3ZeOWvL>|iC)Ve*0rJhpIRJvu0wqbCsPj>S#J1YFh9Pr7ke^q(;=4D^sCxV z3_q*I16}v?!viW9m9~H^s+5JX%h$nkQ521S^$Y7c;&iK1kl2-mPpzM$cac6s21GNj zv}CCMejSv0w7`9MJdu5Wp0T9`a5;Jq&mSZrWkWUDshPU`#;hKky`ICV(~0cF*>OZ@ zsYD0fYLaEc)p*Vz0m__6p{>~{`qrzC9ftGRg)#fUbLnLuzb@hH{3Ebi;uWZ`T}^fV zGlhe?$uvu&RAjf~4NMNcj5%3bP@(%D)~$blTK0s`xDI^7`z8qddQ51t;0PTvQJy~@ zOmVu_W92)s3{XpYTrs&=4^1kM?PZ~PYV`b}=4y%ssJYX#uWTbIz}#w|M1aEg1h`Eq^t7F}2S9VfFReeij<@{rVAuPPF&Z1M~PvJpao#==$Ia({w zC~0?|Wmzh$j`|^L(=o?SmAd$@JB#l5u#f&)cbr#0Rfe`f!ziy=4S$xVf$YCDD2mvQ zZhM6obcjIH!cw%B+$kE;9$1lvD}ps~x@`p@O% zJaBoLuu$_K#!o&9i6!fS+3kWs^K0?Za8G(Kp$%IH(u6Go(|C{D1oYm10d<4F!^*9f zu{Ax^OK< zzcHjIqb=#*?R(&E?*~|?IsuAT%i|L*4QevWj@~);4Er*Kpg6w-w=2qAdl;3*tEbgN z&(|c8XW|V!XxNHfXAS6`U*%vrY&>>P|Bv@YF2L=Y_i(pgIQE~;ffX-iiliLH@~QLG-&Lg)WrRD(QEi&BHXk9$#OZz0~**TKZ6 zJE-~Mm(c3;y&~k*GPrSMD>h#?Aw547M0fkX;`+U|d_q$x%Q{mFbMKmB!}uC3te?Oq z+`5hnqmLu69YnnZfh5Q%l)elfPvjn*0)@5TghHb@=3o7f?I=2mb~mPx`zof$0+N~0 z&|!SC^bmecqK9WJt0ygUZ!oK6cJxK@VX!!Nkq$KPM$3J7@v&tH7A&sA`;D8?_Sb#P z`BqFN&dTu;OC20!u7_uEs^?5+R zk`z>K_z%mLo+stIMiZUrIbi6f$SadfvAs?f{crnWm+C&YyM71P^wh@{E2U`Z+9)8A zhe`6eQuz7Ml=`$x=gzk>M8$(Q3AF{;*pxX37uXM`D;K20$=3bQs<@dx8q>?Q^T+T* zUsJGD@-;3TI~*EZJ1|z`uIQJ^aVjRkX{*~JZZzngn2YRy`1_xsbekD|6D;8cb%bl? z4iZjyI*wIehuMV0+mCw$A@Oz6%SV6XM)YCdNoY@FE zkG08WFDpJFJRIV~SJG^YnbiDZ4!3i5MfIZ_c(j+L=E>-pg2f| zcFr42Rh~+5=aoWYsN({ILi_RluWp>^yP8X^SpYAMCsK|6`)IjgiRhZq2NnY2oY&if zTK*a(RHld7_M9+WL>#DW)MA`eSHPYeSK$WNBk@m7IxpVW2hPE%a4KsAU8yx0!Zt;6 z$!<^nz&x1ecUI$lofBl#u!m6DU_=6cm+;A-lu_#7LFC^UQ#tVrDvOthd`4U(S~F|V zugQ=XZ9Iy+(SSDSA12-YBY3)XBB0;!nVYs~!O>swpe(5j^$?Bp zCgF~JB!c0ev@*YZH3nj{ra*!|9D2*}7B!-ran8yo$?}Ek!dnk`u2F1}M zVc7N<79RJ6L|43Hx*FazrZJgH9XgEpA3KCI8j2wz)rh{2l&9?BW1m_8FF(%lOCk;yreU2|knPMo8)_)t98pAG#s#@wzp9Ho`6vEg53Xo~X|Flg21 zCoXn@GYqMalmOFW*z*( zY$BXcrDYC1_2wEIk)eURPQSt{f_T{eJdcizxJQ&OKL(wD5p-_T0n}|C3n%Y+@}voi zMPrs%VB*vgah%oSh1aiReryOToDk6U8e7=HMRRfc#W>b9^er=Ud_z5x&cc{DKkUnw zqSYt-VeZay47q=e|4}Pq@&k9^pvGhzoH`hLj%4umR%8C~$trF=X*!HFT*v3yPUolA zT||fV=fP+59d=>!NZh@q6=N*^v9u|p(56bAj})e{^81&BdhwCusMjQFqyLzd1s_F? zOiP;Z8{wP&Q5gLxh6%iXFuMUCC^VZ1x!D=;shZ|hQ8>HCnZ8Mz<(k}j}>x@3@d`GPY{AES0bBQ6;Mmn=_={|{DP+QfT1CAi_kFqocLM>YxB$q0^3dt)spi59R&uM=`PBuDEZD}3>nVmI7X3$izYQU{ z?k6VR{y`4a8}d=;X>H7a>EtWxIrAKnj|}Ho z&t7B6B`HkUF@-+ON#unwUO4Z29QFFWl19rY(xA`!T=(k%6a<=zG=;B0;z%|r)cwx# zkBK4h{t?%Fm&{;YhCf)p^;y!Mc?4@M{ovQ()hL&j2_dR0an!N`?wMrHle7yl(()>9 zy6=ZBBc}29{{E<#CCNPwFXu1Cn8K=6e`Z9uKX*AEOH5nWaNn2v;lu81mh>nQb{0wy zZRLN+I^uA8rUY*`b>~~Yo6qu(ReYIXx~M&}<2h zxqb;9o_qt#!Y3GI>_sBw)0u)K5{o;7X`0$vc9;?mR`oqoi^O>s1|RX^%2%}rhwhJ_o%l}n*TR{A1FOs%g5#AfnMc(Xl)ol&q`~t za2W+^`^SbRYsTTbzgG05>1ZlNec_2o0? z_jZ8&yT8XLMG<^?Ky9Kr9yPQjjul($q*p^29dLZ!1i)75>0cgG26e&|dw zZ$5}lYYjnfyExt(v516-j~q8n2HCedD9fCs44u7 ze#n|$8S^1uw(`r~m+;eI%GY)4@vxo8v39@oOpWKyXr}2EG<<#&S3hoI(OJhadFwXX zJFy#!`D`B8y%J*$RLK1`!)YLB!y&ydICAhZT5w;1|ClfXPn?OO>9Wdv<{}l&I;XLo z?IkSvt_Rdhy@I3TUf`$XdXk@4hnGd`>2Sl@bo!W9;15pYk5DCEFn%^&^lLb;{jr`l zm|If^h27Bl@0(SK(>vVQVZ`Ssb6R-YfPauPr-FI~%b!M8*pO2RHKyq}DpvuF?I+Tm zqf)5(VMVC@eTxP<9mLZ2GSpw;6M3=l3~j7@i33qRP9cwh>R~US4JXXdKjknF4ZVp?Yg1sYoi7jR{|L?*@obe` zCkFiSB}u`*!Rnj|y{u5lsvkwN<$l5Fm^}qct9q|boFxxzT9!yxb_(A+$%^MJFvXsy zc{mW0j>`|EqhGZ@I0gM?OB1fZrq6e=ErD`Z<>5T^%}UZGjOR*r0GfqM_~*O&l<#=R ziZX(D=I%>W=E?=wq;igRRGq?6Qg+x|Q%8zr6H)7~DlT8uf&)JSVcP0Uip^bkD1H=I zcrYK&72bJ3k=an_JrPKz z|6Whes%!wOOOILnRZaTAX)wLNNB~~e1xO~2rsI89L#yg49`Z6DTT*k;H2wyz9l4My z{+WhSe>L#GNK4*3QJ=FNs@zXx&TY!F(aa?gT8fh4ZN&koDU1yc9pLyPtOy#3N#}z0w>4*#rbjx zP*POR!dAS2(JzluuX%Yvf(Fh9)S{G!k5!meI>+GO@_61FK9#YuQU^PoGw^;!A~)lz&^4 z=`@SW{+`HJPI-jx{VUiv({Q@v+j*q^*;pSxRJ4-SLe|sERt;(+u+rF*Hnkh!Utt9K zJ8e4ros*7wJCk8ZmlDc1=X0-1xg<@oT^M%PiXI^*I8G%S%xpE$y|o(l-fqDZ?D2>dy`OM@n!HBKLsRy-hrIhk2uJ* z8!gVR;5|4W0vC)CH8V;2DK!DoPJe=l4F)JMVX!uQ8s9MJG3tq*LtwHJANgL9&gqfh zMw3*iK}{6~oZJ9KE&A|cVIFHU>q3``;oy5_2;4AwiJIqZsX)z)66qu;w<*D)MmzX` zeI8zu{e@rW4W~yfW>Zyfv5u3}S`lzonR)RN{O&lEi!x0)d;1R#G*w`2^IKxLNRBQS z>;mu47f`bDyvXZA9aiRVu6SAh1V_cm(f-Cw)V)-K3RH!3w5k^O`B{NvS0abM75J%B zh8iyPhs*B*@XD=nn)Gcl9lt*rn@>4_-<*61$VnI7ANB%Yh%sY$TI`Jc%bh~QMG`Cw zZ=t$_DmulyLVeBKpdjZ7wxZEE?v*~DuD=3%$FHM@8b@IB^(-vEzMeJBd(X}mt8xj` zG3YpDh^W720}F(w!fanvYBEpAXKr51ooiw-bhI-$ADM+468kWNNP}j^6#nh@0`#kw zK*!@Futg=1_9*Fa3*ke|(kmmiU(}#_N(_;F{*hQYeMC25E_OIM*zRJv&uE(ASsrXgL2(1VCVsH3pZf#r3Y}eV~ zmU0!$aN@!?cT=o@7kEu622ZIM;?jK=;i`ly4Lhg-PxZ~fX8L5XyJ17)Ms#D3#Z#6w zFrLHNA6`U9tq+LOBD9;P$4S~nJiNh_UjMim)hox)bgcrY@7^bzd+!9!Hay8s-+qmw z#J)oP)(enhrc1Bu7jf6y-8f~Y4BT0D75%QA6d4|E5jqbZCQ@11z#1Kzu~_mA{G&!sLwwfq76P&1Yavb;nG zq!~aFr;;f#m@0P;&hPxkJm!XznDY~9)_6~m*>@AzcX=pHT6r0kUA&4#JI)Z#uCer{ z&J2F|n=3!FELyC$hQKwM1kSpCiQ3*(!y8$1dMdr2}rr|M|?NAE^+-tEZXh z=3YFd69wU;U16#HNoqO71XhO)!CLph7!$mo_=vq~#bSS`POdz}Mju5}?IASaKsM&J z{fDOl-SKPJ8}O=-#j*WTyxZ--d^udV#Ffnd1jy7*0J7=@m%j-IU1$)fSX_eTKtiu{v)g)WumRvlc31me(Q5z^^;(r z>d3-t3aq;Prt!qMI(QIbPo>=-fv@v~Yj2)pQhM+(o?26i&gm=oKbu<^niLKSqLIYl zVi;e1J%L@dj_1|$I-%#z0jr_}5w|VW@gMpJcBtjuh71>$wt(*BHZJER><+zr4fG zpR#DkXtH9>eS5DXbh-ikZ0?{ImKuto%=# zR*4F+!eIjlVimdOo6%ID9>6--44j-Ie&5EcaxbsvV1Hr=*D27$6Xml|`tTK^vb7%M zB{|9_m*eS$ds$)40;W4=BRT|(p!R2vqx8P>kdk>1w)iZh`8u}HKRXxyERVu+uW5Xj za1t+;TFs?XvoRoPH~lff4g_CU!R?0Cth4^UmSN`4=mnXo%ONI+fk{lWeH9TOSp&)P!Lb}#C3VkR8Ocn3dT9_PiSRx})MV$t3h z$iE`S>4D#goH$qSi&tjJCL_Ss>NwpwL5TO<0YkK=!smh^bnD(zyz|jyGV8@bR33=t z29Jhe-w{(fGFcvy@5kcTdoNMx(oP~jW-FW>(*&iT3qWu(OeB}#&RvhkWAW*&^ti-B zRy|rsB^qBr$f5w2_~ILmey)NO1|(?z*NbR3Q3D#ri0ckUC8Pb^$!M+IGh?}l44-c%eV;-xX9xDml33}?;6;+4q}4MhtM(X1`JYr zj;^t()WP;2{t;{xTGujgAXbHX?4+J&aAh0q{D8!I=fI%%f=*cfajP zxk3U;e7TCTp^;u=xRarPudfyI zgNGMmhAv=ExhC}Gj^KLVwb0U{0Ip4p0L|j9CJ!2?_qkp!)Khx4GrZA7(90~fcL(~oBgm}J{~w3GCQ z+RGW>)<2R=O@6|%4(;R_ce8|TkM!YD*dg4St-?Re)#ALWhKBr{4P{RLSo z`ii?4rg)dcWv=F(8@_?diN|mYyJ6ddc~E7vhFU+n39o9l@$RyXI7;>=N{t@{D!We- z!B1bJZ<&khlFxy|+cv7%_m%Ap{)N3?4qKk^;O3*re9J693<|jmU5<`C zLoo}8;Ukj2_auaW0I+R*i!3^yq* z;CWV8QB&SXm=>zT*Y6C*lDUt;rjC;`qwVm>y@O1$KL#&Tms)>#sm<4mIi$K11w3ig zYht8%2*|V@uxQ7{UK3qfSn`#`W&jYUrRUG9$lxw(2P`q~@ z)vgbNv}IBx*j8LW<-800H~K)#;7R=B(ZgW>L!aeKyaCbtA+)i>m}kci=ddgd>VM9I z<@dgT=5hRIuf*APm()k9pt|}_oI@FE!4aSX3;up_@W*9 ze4hCMuzT}^sKk#UI;t5cU>>w)XDb>6%JRvT6laAG;|A)c`1ihmrn<@in^Ay1<)v^V zPnSQxyaOdq%22Q2SxnkSL`1T8Fm+ZFEUc))+rutneQpiBJlzekTS}mqFPMZAK$UN${rVC=P=759oSF(^7z*UEfXx=gy^=|24WvLpS zv27dsR5TywRatSp<@v&tl_SvM)g(Uu!A1CcL>HdEPvF)o&*H8>kr=0K$3tYZu>G0^ z3G~aX__T2s^u$ZU50z37)Tv+XzdDfz+?M1+;&XAUnHl#paDzqCx-@&rQRt6QX5{oz z?r%5+%5(L2mi`y`vE?WUFV(N8^xr|Mw~VA(wdwTM6Dib=Xn@vkQ!b}%%})k;@~_XQ z@Y0eP8uLpFe|8CY(!?r~vf2}89@gg;Wsdx`TN$W{vC*1QdNgn)2c>xp5G}CazTbz^ z{k?W*{pw#ukA@j4#yFy@%Q33g@`t3FY17^(6`!%V{q-G!KhxQh>C;4ur{m!LsSz` z`ObG%)qffa^JBrId=8H&P{*-K6QR#nhgKSlB2ITBG40@3^liU`D;jlqfp;90oo9wY zsWIeF{Vr~wqc3W@*Nq!)$x_)z#$cx$hIjvEq2K5%R=nR+G%aBoZ*mRAtXtyRkd9#b z#OV-q8~u=#cO;=nP$#M_IDs}-y&2sYjvXd-jBL9ky8k2{n+2E24YztUbf|z{-(vJg zt|npXl05pJF7^Et&V60$0Pju2--o7B&5L%}b5@L1PFtb#uc44?{279K=Tg6fQb@}! zXCcOhFn)m*-*9pcuNYJg6Q(6$*PAdLADV%okG=`JRU%N_iQ#SJ5=qvSEH zRar(7r&f|t%?dOo$c$doJ<4x2TkzD=xsa$D59M>;!b^1uU-o8;z5CnYdS(*6|KJH2 zJ<4JkYh7^TxLSmc&3s#X9BP@ip=;GEbU9#&ja6B=Q0&>V2|R-bf9GI$jfW_DnHXot zs!;zG4`B1jAawH?ixK}+p?zA5NX{h%U0a89(h|*TL&Y3M)^vFFWF-B&?lZiaqerhl zn2tp*E9vsda(u@01MqclJnm7cf$p=<;ecc`pZmz0CpJ4$pN4Ewb1WateRfI>-U1hs43f7ArdI z=~ND0^7PNHD%8!sLG)FPsa|;vp8F9A`?ekj!QtzW?9znk*b7cK?!l(NQD|VcjV*7i zgINy^>6E1^Tu@VMF*4hbMpsY7rR#Js>5Q0HYRD3Wo{lHd2IhRodJ*LP`3iaVDVO+g)0rcpSMzZSB_ z%@ciIWJjxPr05gND6nYl6wm(%E@RGV1I-l#0+=^4#Xy6PY1H@_k80_;%X zlPP44{{R~g#$nfcM#4Sftwt@phpG$2o*|hbT)CzO+2nrFl(!@K^oj{IG-Zq%~ zcOIiQhjh7IQWyH9da>_fzi|3OCq8>yBIfLlh9?6qSbf-lST&49BE|v6vu*&}<3_~R zI+}ZK7T*<-j{Y64pmQya-0jV!9#J~H@~4AU`~y#XttUa{+lxtWYb%kilOckffvW@J zTIcBRoU1SVh!1{@fS`;iP_rq8w}yCuhOvO!o9;%5Mc>KYQ}Kj-N`YV1nV_{qorY}f z$CNpBU>V&`q(H_?D*6-&M?>b*_5%5&tRwKkd=x|N& zkQQ^+cW$#dzdL;U8Fe1))PePD4&sB7uXw3{Hq}-ZVb6b>=-2ljIQgZK@$Zay&%8B+ zMf`zq7tWNTWSH3lb7nAWDBQ{x!HKZvaDT1|xGt~4px$)+wCpE@tDIr8wA*lre=wZc zBSn7*q*(qbHLBNcPVsCwO@Ez8(iY7k?DRu1er4>;$W~Mm^UC(7|IyoR?$mC7C1@U> z$yEgX?9QLn^wgPm?7qJ@s!VZ3vt7fv;dC8XbV*+vb7X1Tux)siDS!{G;b&*p!X_U3Uty-~bxB^pp^ zmLf@#Bo(UfUQdG*Q9?*ar4mBOl#nD1BuSE_iINaf>but}O6F9GM1~}p6N(Jo{k!*^ zd(S!l-rZ}rzx&mGYPW}HTA$B*J;!${)%d)o3{EO)GR|qr#5&ra2R}hP=xJP#w@j?g}=AZmQO#h3m%~&W<{h1TXQFbUgoQAeS_9JdNG*9DS$7Vb~d2lxXE}x|~0C zNHjB(At%|y{e@WN^O-AKE>CxMjOAA}oyPfNR?y!wCKTD%1eqtU^Cr`;P)N-_PRFVq z+kS4q&~eGwzP*P_IuJ`CePij0d;o)$W%%>5&=&jr5*g~1h%|2uBKModEPeYv&`nl@ zq#9FLCZ)@c$(S;^dlsZupv>lv>%vz~g`i=#g#0|Ec~76ayi8&MUO3#tGgnjLo_E0B zapuT$+BuU~670^JW90b9A2z=}hjICLdD%nea3`-HyXj&{NshImk0uHH^j9&^p$`0v zvBoUvaS5(WTa9@W_u;H?;oVQnV&62Y`S^z!a8J{ObA3GoK0la8TBixUS1N*5!aCkL z;2n3mD;u;=H}DEdhHUGVSkxW%vMj{n75YDYjQ98L#S4x9)?oHy76OnhQBDLr)|Qcb-EzcBf!fcPobH zeB)d{6u_{r@@xZt1@r6P0P`SJkXej1;vA5Cdmks2d*a`|6l~`fvAvOh(C*)Ky0v$Y z=){8I>|vZ6sU9E8!VP zyPKel5lYT4gdfI7P|v=E_Y%gw zOBAZaRiQ1M#~H?N^Q++VhBb@2rmlgq@)NMPRgTSBbdL8r-~xG9Wa%(}5N^jf(8$wH zwE1Nc1^nH{8>;s5hsJ5qx z@W*Y^;Iqm%h`W6pTy|e!A>V7o&&K*QKdmUP?wBQ8SM&=dAQw{V1pR^SbNDG6E%>Yl z!rSEwSm8@Mcy@6#8=NVD6vQjO|_3>=V4=#UKa`~8*<^__eIQKS(rOi7{?BFR-xpiWVH7Eac$4r#Y`&N zj?1_(1r^tZl0!&7xCf@wkO61G=-?E7v-MvLD!waffFs#SBv9&6Ovzi}FrvB%!dhB%^czFcm#7v^Lim~jKr6y(E zorU_E>74t;bZ%UGux(pvE`DEp0Bpw=(T0YDy!)sx;B#vrn5LOgeyIoRntBoc7(_DH z+Frg|yM_0JSib#p3ACQSh;4tj@NS+jp>5+~ZkD$Kj!_sw>t!y&lCm@K^-UZG4n9Un zvxPo;uNgH<8sG|tKfIy6<$rVJ;;x7fcv#hvwV!{+UCGvgrBfnN^VJ<@`qBhG+Kz(A z`yXIf#}oc{fB|{;imctnYO}4cdvNp;SBm|TfD;_$W~gpZAWzS9>*}zfukk zXqiSgst4momV@UTj`2TV`a+mW57=s6<<-ALv3GH)G3W9pU(d9kA zV8j=&T@?2%@DA2wt;M2Sc*T7VWfhM2%}> zaZ`pNotra|z42bayiFy^qv4mx<8L2&rKP(WrgO04MO4LW>S>GD;mDKVa8 z4`{%$docp5LWA9GmtsldMslCt>YlLuTR9#@#iK4lQ+9mNFo+IE^d4G_I$;9dTM>$jTUXe$`FfeI31RGXggYegK z;+l^u;r`d-toXABpVPYtLpW_Z@?aG2vpf@T_zHcP+)?4G%zlzjsH4CB>i`eGU2bes@9nyZx;HSS` zhQ5ereEPyTJh@5CCnOKyIwFg>)onGXQ7FsQ_j$viZ&TPr?>QLU3D7pm3sx4NbsBdHc)EHkpA`gNM+yvkBPEquq0hGRT8zoeg zat%90T*g!*^i)g3)9a?-RZQm&>&~RS>`HFI!yJ&8{l|AV8<6Y1fBb2u6KIfKj=okO zVfz|ow$4<tR&ZE`P(B6-EWE-%ny;u2dnX72r{CM7BauIast7D+}q^K>-m`}Hnr&nX$ zvDM`mZ@yfCX8LR;^QtJ=o8D|jG}h9>eN^{ac)dF*m@5^qDd_7?DdJzvzeT87T%4>5Rp z9rRx~nezT*06(D(mM;`EQo(B|_Sy-!J0l54>mxK}?GQOSEugP?wk%`&c-p)%mcKVz z5iIoFu))oV1&NxUhh0ALIZB@&qlPlq6ewuofa)1Z~|1BYd-N z0A!u7hVYegEObpH_SPH0FN3kft(wU-uQ#Msuz)EY5}ujNTijY}7r1}Hj%L4(fS{f( z@B|~!_q4-;a0if%7vk3|h>z_LG++2p_zm||zfT2k*rxb;kH z*NJB2yW%8FC_cfDI6sha`X%uFOf>Ipsl|P2(4~qUrku&e)%2wx;=HOxu|<16 zDtRQZ+!xvG=k%%cbX5)Cx$r!%`6-aqP6=i|r)jgRC8~U%tPC0d+l%?r`p{J(4(na( zq2lZ&7#24ZzYlxObylhYZ&ZLWt=X7xqYFx>C9=7hv%veUpE!CxN6|*Zn1be6bVzML zUhyUuaN#=72`r~Pw^!WSCCm9(VQfu1j)G-X7bLD8LFZFEbnihuk{3^UB1pu~++qLjv&Tvos^h&H{< zyDVSCLd(m+?Zzk2J0wY?X5HiOM9Pq@NmbcnZGYTwY!C(3$Dn1;RV>_^i3T@S`7IBB z^ZwT-v)s24OcZsUT2D#f^YK~uUPccZ$LvPq{58PokEO6-%G~~UC0OxX8BS+spy{e0 zE@*@gbn0&5rbZuUwwr&Hj!sKM-^mGlwbFC0s^}Ov>uZyRl%T7T`6YUno(&7UbMUT= zKE)e$@=1G!(9+{CP`hse{jth~{oqdB&yRASKYoWSiLtb%@+^3SUf^^7dU8AH2I`va z#$m%(V$AsCyn?F|OKft0lC`(_kKb;HEhhgIca|QfjyG@c?Lkiro?HU+B15FHvrnr4BlyX6KJ40L1Rq#z2l=>*4$kN8YO z241=(*-J}S=70I3E`sC-by z7Jlr{Hg8*tSJp__q?OqUbA@%_YF`A+`8}LndIhC+^g-AAli01}jM0C6xuU8X=sr+S zhsP$NcKAkW4JbhC;DPjO@dcK9$%I&rCY$@cKlE%|z${kf;J0!1s4>Wg)|;5Jx#3cb zpCqDHeQ|8}T^+VBFqbdhw4bJ37c{y?XEA!qMeG_o9z8C_a_2#+Pnl8#4N#cbK$rjr+J)g5OtbNH|S9auOqCufRQ{h4A)vG^BnnLsfkp zt|GI9+aP5`<93ySuWypV&AC}Dr>g<4RdQYwNlgO#8(;xOE}oXLI| zUB?|2zcJfUfx=|eV3075Dtb1SeGcG_UyXS2%4HVnoCLeCt1^e0^C;h> z!xoMEc5VL|j+NzCK(dO6yaFP)8KOrR)A^hVawI4wVIFBOzepWh_oD4jS?o1i09$*W zijuQN(c;t7n4Q`iT+xUyY0w?eOB_PogZ$am0bk(z+`X*0Ie@jiQ6NYEY_Om52CM9p zIHgiY(P*_Aq3>6V%VeymP;0V{x96U+>nHq4cKIuQx=er0%y1tSRS%>R6BQb@Pn`{S zY~d41M{)~Zc;VJ#Z@5>R#{8(_*29~ zNr-&;lV2$n$F~wSlnt~!3_E_rf%RoCERBCk{NH`l)_$JVUQi@g;FVcxYR=(7azOu-qx zS!z5=-g}S7jHNL{U|#&SRfj#ds)?^H7SH*m%NoOtneF}UqC540%=%=1I&YB3C&)+B z_a9pX9a^uoow*XL32OjT-4}2s#{fmg`oqnc$60jRFK$V7Cf@HDz{Y<4$UN@n;+C7g zFi?3f_wjfQ?wRUNBQU?M(jfqA|xhoD<8%(EGP9P(pOZ^3I0y zY=nFV7AkL{t*2LDz@dEX8gL$4pH70R)rZj5MV+QcMq_oXCeT@RrZI6MyY6-c#}+T7 z*#yA+QLy5z`ZhtOeB8*&+`;x*IHV~*Wy z-q}jJ+@=0H%c$Kb5~7DTG(&}{-M&O==a0hAM;lmtcRHBWn{Q}PSj{O|;%eA&kj{cVpSt3N<%-Dmz1yNe_H z)(P5;ku1((IMsN_^UC?Z;D)II?)&S+?x5f+njeR2O>gnHH>$9CL$xV5`U`A+CP#OS zk1;8w6ts`mg%bb%EY9KthAz+r%XzA#CGm=@6qpvicBNpj&jWvl%dzvitJvLRwitX< zgZ=Bj7#(-%vDP8QaC4#!eLHgq@5V2o&laP|aM>@e>z*e+BWcGK0Cx*v)0 zpT%*UDb`_~DjS%yq5@0(IDnG!g!!JndGPP+Dv0Qdz=k?^7HT}2g`ADS%m)Hfs=bkW zG;KJ^KGt5wGLnygxA+5> zcuENhgkuJN5V5zX2&y6k4|dcU2%L1BGUhmn9@R=}gpT19v=1rgqQ0EL?^S<6Bi)h`HwE(hzRsg=*=@|{R51$++dxi- zfui)RV8FG@ocFe?FuM96UN8I2x6C)DJsKP7cg{h|PqSpf+G(79k1W~Sy&;`Fhhh9h zJ*F1qhx4v*6!yvkpI9Y>7pH+o=Cy$B_)FyAl!p;@{`?3leg5UTUXk{JMXbW^Exx)O zLiI`Q(6wtO`03|Ee_;;g8^vPmhDMm)A^6Bzw_&iFH!HZ>2rG*aozsJO2lK^L*|QH9 zYaa$%gP<~PfxA(lIuA~sv7%|c%B*G^qh}vIXpa00{(ID9HtR!?D8fsTU0GMme>U@F zzlUV8KifmO-&%ih)S`#1ZOKox^GxR__l~5}zvt=Y!x(=0);4^1g)z}|M~oY$OmDpI zV}M}>6l!I|66r_qyhQMT=|6+8Ka#LIZVg!t-;A@<94W~BHW>Fer+>#KY1QXR>|ARO z{t7oF)@1}rYJc&relFoOc~(OCkZ5N{UMuIKYezO!tmV0RMdMh`pXH#kvLAasE(nqa zPb8UeJ-U3%j^>*GV@F@)V@B&f=!mOF?Umt>;qrmi{CEzI#SPreW%)QDv;%bH63I;` z0o6bCazAID#VSP+PTehd_NqTabczv`x7XwQ=K0Y3FqZXCRuK%L$NBvu{?m@v!l=Cm zaOuBUtfI!;`eX14-jF{6)9=k>wgDmRSLq0RKFOHo*zeJMAK|;4p;QVCrjUl19%xE%xG^+;by>B5o;SMAZZG_t!Bq(v!2+SyUgl^xP zyzJ8poc!-Xe)E{otV7V-CAtYU>Cpas_Uom%Bh(lteQ8Cb#QCJVOC)NISxtV6yTPR5 z8lITYfyLL~(0JE!_9Xrp;rLiE@xFw;w43YgHxYf`nlN?uYTj(#P+0L~6#3q~57k3b zFz6FUKK@Ef_ni;!>Plh5KMTyP!wV&oT7&7@2;y?adj(iv<~^rQ6rwzj+ohn$(q{Yy)$i9~!K!N*vS~M7 ze{qf&?1Ec*x48pHZCSp+8@RFi8z$5`;l(>c%4bK$(B+O5{EOHLtl3bFEQ2DUTX_tH zSyXZTPRfw^y){@z6R1nU7~Y*2L+ybhDQR;ne7szZ-);YbmC`ZDvGF9?bB#Fe-&%o( zu#Bx3Cj-WF?||zSJ2tuV0eY=n!fwiqL~eXB#+S=MQRYzUy4H;#7(3s{LDk70Q0BQ2Bpf#IB5N7iu)~9P>dgQc_m8d)x8WobLM){vcC)~;tu_yK zpBC6%+O%n10D1onLGvMhDbW2YHEjIN?!UXsp20+BbZ7+4NKv70>HD!mD~_6q57XFs zOO~?5ne?6Kb6&*<>53}dvJh? z1FOhwAfuf}DCM;snYAo}3uhF{gBmL6ddesC*|(BRiwaoaEp^9Taz^LgCe75iAhPZ~(%>^US<3$8&MLCxrFnh@k zW!sU|^+@!%_)U~7sgLt-o}zb~CY5V`n2LT5ljz1)6;%B)jOXCyp8fRWF3cn-ZJtza-Q`a+(uMhxd5^J6 z&_&$qD}>tCG2FXBv)J8KPYPL+i^DlPwzR(k)f`wwxBqFg)sb<0OF|vpGL2-v7R{%k zpJ{Y`E<)T(9Dd>hlUUp+){Z&A zywfw#cGp2tGL>gJdi~1naS_$Hg;ULw7M@;QA^V+;tSG1ywF}-;(a{BvUvz|RcDT;m z2VVi3(2Mk0e-6_*(TuS96Ybwq$`x-dLH~8T*}syxFnO2+Eq}n%%xg{p17IiHdUy%x zMeJe2C2z1XZ;Pn+UlTq)@PVx?v13I;Jm_^`0lzyrgJO5Lh}(E6dOJ{`)_jd;8JiE` z8jJ1BZ$~&~c)Y_oOWnw6wm)6VL2%eUpFv|cd2sWYrBAbX=(RIoZ1oyWJH;~HBfIID z^=>xXz7}UZTxX-O<1U$9cVWwGp7UR9OUbrv5$m@yndx@;U~1JhI9gxHr|2z1raOeH z7dY{1P9s=*TAjd7+0PuVmxEMb1w~%o$!up0BCR4l$~ILg*Ae>lr$<>a{eK53Y5FlX zTv?}l^$#nG-Li_hw7z1rf&>QZq@ae%2$nP37tf*uYpaYQce#7m=wHi5oIVKp+q>AY z06#Vj@3DlsyJ-DB4r>$uQg@`$ye>-$j9)^`$!<>^=?$gS^Te%T(xc z#uBzzn1ckH&-n88i)+$1rMdSYiT<*r6L5<*j;N)Eg*>@+`wKB56T+9CwJ8MLCmc zd4Nr3fIoe4vSf`9n`w1o9kveKz(P+oW1hlP@uJ~+EZ$%z1@6}pFPWG{k}g-^R;eWAJp9XC4ums1m0K*WNsIYfZ^xdh zW~}W4s2qFOP0UdauX^pbmFZI z8GK7yH7@Q>VuoWIz>7IEV^=xm=dy;aVIN2)Nv^!+x`Lgt`6XI+!klfLEf{L`J7d^8TvgPeA;Z-!$KoABhQ_(^flP5(uKAq z*D`3H3Gx4$6sRfRi9LIDfE{_TklqiM<9+>SQi8(^SUo(GOg;@F$^!Gr?lJ7@J80wFR{}X}hqLnh7p&sy2(BZ2 zBfbz`iJW0~VB9yL=lB6~3pHT0R~;5uSmE5w3uvF~EVk(EP+D;$1-?7(0Jk;v%)R6{ z_i=7H-I)BHe05#foBEq{t9}{@@r-i6WO;V(fxrXuUrqZq1e2|yE6r`c2f1PwvJ&)$ z+oC420T=dCi|JQpS<(+?^u(jN9><2%3izNTKid2EHnVIv#w-_lQ@)la)0(%MbuRYe z&K(YimEj^Ew~UST`jPB>dJv0F7z!?jbZm-he)CFt z`{}veAspH=gslp4q+LG;Ql!mw@Gbtr7CTyyym2iKdUX-D89ZRdwkKh~LlEn6?B&xp z>SFPFfA(OP1-s=w6#iXEgWlFF?9Y5bdpOOQZ8%tmcV_Qm8Jkp@ZhukP z-@Lh;nt=h8nC^k76;nxXwGG?8eJTt3^OG$;Bxy6}`FVx+R=4tD>&sR;yW#>QP1?o6r`{?Z{kIqgzH?@KZ)?~DpFc+)-K^X;Ba@G-7GmML z$}F$*CzX7+Wq%$$X9HzRzKBXKI^v)F*SiVraP;ktBv+&CK< zv9irsHRbZyw?EL>Y7q;$nu(JNrZYeFMXcPSp2YF9ZO*J$#F-9X&|*s!3|P60yN zH%l2tm-=bYUf&#ApyN&}9v{H*UDtT2kvy&H3T6AH%wYRK!Bg!1f@^9?qbIrMEO~tm z6sYZIy4{u(v2+L9pOQ{(9wz1O2ZZ<-r^FS#9L{^?#o@li$~FPI>6d7*T!U&Pzb064?q_2U?JSyPqsUxf}DUD79{Vdm8xQrcj zx(qwZ^eNzs75GNvF|WKZ=KbD+7DbN6a_udw;f^L-?Ji>a3F%N}S43~FIWr5X7nrpm znQt*nr_j>9(2zZV$vpmy-3kIz^X26E_z3+PBxqb)pYxG|9`x^%LaOvU z$kJFT{TL|33CDRrV5Y!Kb(N!CmlN5K+68QFeK-bP7csAOCrCmSM34LaQuT*Ftoe2< z%y7L(*^<>{(EAO>`${qI^ZB@kcW1s`0v}3$4m~+^gq`Jvl`r|Qj$ELHB?jDvjnx@U zspu3R*0hVhTAYM!`KQ?2>RBv$;VCSZ*-t6Cueq46;>B|!P)>vyo=gsx3}_I$ePX9XjJB-sk5Y&1?*FL(X8 zPq@y{`6$7=(J{M}4Zb+jrsSS6dJLOP@6U4VMWqEZ%X&>EVl6iO#z8a@9Y@z122`ia=>wX1yG90j&-+L!XcbM4S# zT}(O3VPy923NF%)Wr4v#;)a`LTx#?fvb-Hl@{POMt^K>1)VnHt<|>bQ=m!X5oO<$-I=+|CSey3%^64mCXCkmPF3VjVqfNh);%UweWufie$3l-8VcNh7-ri3Gv(|3`&Ap-A=VBwi*trX@rWC>0 zHv<1Fdk9!A;y6R0O;r761UpCNb1zbT*!k58Y-M#eR#sWidEGq}ulNnF%t>H2zRxjB zJDV9VoI*$6i*dAPHT5jJM%$-1vr|DP3}(nN2}L`$*P@x9*D{Dcbi}Z(m!$>=i9f@ruTR7(eE^F^c(Mz!!pBfrMG86$ zoyX}hS>`yn1*Ahg;DVwLnM_il!0Z~aQdK-Sxut_{NH|uU>cgDU*P`*>5Ag8lc-)@3 zh-qov;v%>1BPEd(OM6rVCOaH(Yvw5-uR$wHSmau&#;D=ae&11dizMbflEdRO1jhOPr5tVkFMnAPGoM;h^y32LT`FeDG7D{^k)$tM?uqpar~{ag}5N12o=)J(K{fP&w5=2ZT7=S z;=&O~3O1n~7GH7O-_h80~r3l&7+wCsrVlwA!-quqn!gT!j9#lY9*d@%WrLRDRcho_X6aLRX1BJ2sFk z4NS0Y%s^~;IF;_Tx#NSqnP8Y;#%Dd;2Kp~QK;oHOFny>Y{xQ6Zk8bB+(C0Xu-Oz_8 z4@#3s{AW_zkrV4o0qI1zh?)lO%MeSocd%qwC)#m{>0Y3Wt=# zY*|}o*W8cY=j-s#);$=JX9!Vi2T-)lU7Tnu!wx&W6JpG{u-_>VW#4tfpEWt$M)`dZ z;`R=WxEAO-Jq5Ccb@OF1XR&E`5-$IK19uuOL!;Z@U~~U7c!t?9Gf#mFc?Hnzu)wi3 z{|Hqr{mG!#jJQoZVQ6JCCM2l{@#Rdu>iuNaVJ`!5+CQ;&#>+A{3m?uqV-M3_rVkQ% zW}(_Xm1qv*x=omOMUWVT88^+xE zbl&n!8|+-L9m5Jl-1qTnu-_<(SDn(IU5b*%Cg;CgMo$xLk+Nim_T9jV`{uAycU73H zk}sPOFbE#MNnxoE(_!@eaG2iEgf3<~(J|f)-x==0$(zSgP}4DVaD2t-drjnICdRUi zjA_=v5RHoj{@Pxhxy+A^!|i6e?E00QJ2LQ4R?oGAbJ}p>fwky1M&N@EzKhjDek83K zl4S5$iMOk+#P`Oh#X7upYUmlwmTa6r($+>GJzVhhpIMHt?yE7!RyDTCQlA1REI{>$NcOXC zC7sJLV5b+#k)f{#U$dnhq!bUbmT|_+rr%Izr0Gh2og=Agp&@6VzmK-qD%J3P{UjE>_Bu+x4}(UVOPJJhQEcW|h90vP zVzW*Q7QeQEXGbrvXYYS=TK<2*r(FjonVPc$?Hn3Nyu;x5IAI=dHAaRCnn3MNp)L^W z^ms*qeezzk<-IMsO&b9POGCkQh#a+F@?e2`;y8zyN-X34C;X|giv2wABQ_k8%S?9< zWsdDFytm>A*rVr9pD!A+;XUKwvczqCYdnz6$<5+i&-GxS%~4$GJ{-gTs50&C+i;Su zl#ufynw!Org4FL7KpK8{iMzliJnqBi^{FuV(GMuN6b3=3D?#q{LEcWmiEINdJa-m1%jV*dj$a_31X z4)6o9&jHAg>j9GWhgyCl7D~y8{`{^N@}d&f{~8HlEpeDyQO{MUb%W=>WZu5P2Q*iD zV2|A$cq9=G@@{XQhB6t`Ag}Tcf(RkA)zW>F~sCUg2Ds5)d9=|AX z*!G4OM+@)M?RPLy-kCalYWPcoN3&t7#So$&hqW7D3VDX);NRd#PS%gd!!PUkQ@1Bk zh`crPSlGjBZQsS;?>LVhyJNY1zho&SeG|;sYD3X2uf=VqySdzITWm_2#c5^L!J!|M z1ka9i*x)SWf*snQ z8yTpJiv=xNO?MlB?QFr5yP1>}zH!&@6!7^8HZ0cl2~2kJgyy6{^j0<=LhcD{hQEt2 zQlf@W3m0-_jFDtBHxFhB596T!UKw_5ivg?7zm3lg>L6Cb5G%LV;ktEWS>dN6IL$_m zg^7o8Cw!ja&yG+Gt(^(U`##~v0~X*tRA1!kahTnoH;K6QO)zQE5GEZqACe;VxCL@= z_@iHR$;$sddfe#Y6cvZE*?+az+-)cLt7qOIi*Tc(jaT61?mRdVa}KB6S7g>21ZPaO zsVSvgtfyXq{6$aLa7G=C&+TA|z1{p4xQwZ#g1+(3H}2ZRer%q}1^BdV37K#Hg`?!6 zd0&m&Ap342*-ze#Sy63pHK~KpO-)KJKq zG-Nd->d$jkY0BK@qlf`96T=ys+Kr+&=jh2d&Z&@`;RufNldY4N<4r*`t_z z>ki1DZ_GNqr?YJf1b)8WELI&Lcx{G7V~|;pqi z#f&!`!THJqfbELo_*dziQHK|vbrbyYrOzPw+6~V0^k2xEcUa^iaUWhzcERBl6)@|l zHFU0u5_G~LqNXXsNaojkw!!fzD#UGo(!P^uVWA7n1O3?)&jT0+lh`8VMArU!JIg<_ znLZ@kgMd^oNR*VJYY9?JcKuvdA?q*74UGbyvbVC~DX?h_aKFhU zT5_=)H$Aao-ph2jGLQLe?s|9TpqtD|*x$8k-sI0p8-ZkYHe>ycEYKDhJ$jxJ@OJJ* zN?PWO?lZQ6w(~K3vh*51%3bhhRAysG;dMUj;W@tIk-jLUQj#@?mxF<5EAO-O5bMld zz@EI`jIL|tn9-XexVLE-oPMu{la+n&{J?sFvz5asoI4CFcderA6Rr3&{UQ|kRKl~k zDNv`@iSeQ4czD}E?AJSp?F~*u!<=vkv-*u^WXw_Gnk~71E9H+Z@u&TN+xglfAH@sH zZ=wYCa>^;spua>Eu8Wr@@4=JVq10fgxwR2}0$$@S{Xyt`dKWkcM{xqJf!@D+$Mp}* z@Q*ur;{?PdZsq@GxVLUgrP`p&QWPHsK0mB{t095c9ix657>6u=e>% zw$UXPWuc$=`>ye1nxQWqW_=XX&5iNUbs^u+6;-jk=L4Aiz?A*`eU>%!e&I{Jc4M?h z5jgyqDcsXW{^czr(LLq}!B6{(K9(HeVii_UdSr?C*II3I8={K`l+1aPn2{v&?I3JD zTPLy?JqN>w&7vH+ayV@+hFJ>M(EOtbLLRM!B^^?1>6n+;tGfq{e@z09oE)*jibmM^ zYdm>gx`OuA73df|kAAmoqO5XT79aHrDqbGL>AQNsbDSj09&CY~>GL^_@C<(SI!|Wg z+6K|{^x(lxBev$_6r5UH4llFiSfb7hG-Oij-?amfVBo<`J#diz2#jIR6e~wHl=U2r?qwC#c$e-CF*8TT^`&T03tGfqNaGWH06`kZv_e7x4=mf6M zNtL-|-p4czEBw^&GC$mHGws?wkhHbOz?hdp{dESg^q?U-F-DGsDCR;%i-LIT5pN+U zP61!>!i1~6U4&9k_F`;!5ysWk@hNv#L+d(E)Vygbc4hw{+_D8j;`8uvcqWf=`grvD za(;&5T-2<+Dsp~w7Mi{ffIpIj;5Ji6_IJlm1Bq}6iT_r&nf;_CBxGfUz5ioTl$a{v zvtfDQ%FX^0Hv4bbxN_ZEM|&Fy&;Ngmx%_`;P?x@9zLfCy{#)Haqx^xqHux8;AHk)Kb9k4f Date: Sun, 17 Oct 2021 20:13:03 -0400 Subject: [PATCH 10/17] fix flake --- torchtext/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/torchtext/utils.py b/torchtext/utils.py index a1776fa01f..d7506ee7e2 100644 --- a/torchtext/utils.py +++ b/torchtext/utils.py @@ -10,6 +10,7 @@ from ._download_hooks import _DATASET_DOWNLOAD_MANAGER from torchtext import _CACHE_DIR + def reporthook(t): """ https://github.com/tqdm/tqdm. From 62874976940fd6c2336b910b89ccf4cbea134a0e Mon Sep 17 00:00:00 2001 From: Parmeet Singh Bhatia Date: Sun, 17 Oct 2021 20:15:59 -0400 Subject: [PATCH 11/17] remove pretrained spm model --- torchtext/transforms.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/torchtext/transforms.py b/torchtext/transforms.py index 43710ec4f3..b813e81071 100644 --- a/torchtext/transforms.py +++ b/torchtext/transforms.py @@ -6,15 +6,6 @@ import os -PRETRAINED_SP_MODEL = { - 'text_unigram_15000': 'https://pytorch.s3.amazonaws.com/models/text/pretrained_spm/text_unigram_15000.model', - 'text_unigram_25000': 'https://pytorch.s3.amazonaws.com/models/text/pretrained_spm/text_unigram_25000.model', - 'text_unigram_50000': 'https://pytorch.s3.amazonaws.com/models/text/pretrained_spm/text_unigram_50000.model', - 'text_bpe_15000': 'https://pytorch.s3.amazonaws.com/models/text/pretrained_spm/text_bpe_15000.model', - 'text_bpe_25000': 'https://pytorch.s3.amazonaws.com/models/text/pretrained_spm/text_bpe_25000.model', - 'text_bpe_50000': 'https://pytorch.s3.amazonaws.com/models/text/pretrained_spm/text_bpe_50000.model' -} - class SpmTokenizerTransform(Module): """ From 0127f7e73ca8bdbb9caa41df0329472f1665fdd3 Mon Sep 17 00:00:00 2001 From: Parmeet Singh Bhatia Date: Sun, 17 Oct 2021 20:36:23 -0400 Subject: [PATCH 12/17] fix doc --- torchtext/transforms.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/torchtext/transforms.py b/torchtext/transforms.py index b813e81071..3dc4385d6a 100644 --- a/torchtext/transforms.py +++ b/torchtext/transforms.py @@ -41,9 +41,12 @@ class VocabTransform(Module): Example: >>> import torch - >>> from torchtext.vocab import vocab_from_file_object - >>> f = open('vocab.txt', 'r') - >>> vocab_transform = VocabTransform(vocab_from_file_object(f)) + >>> from torchtext.vocab import vocab + >>> from torchtext.transforms import VocabTransform + >>> from collections import OrderedDict + >>> vocab_obj = vocab(OrderedDict([('a', 1), ('b', 1), ('c', 1)])) + >>> vocab_transform = VocabTransform(vocab_obj) + >>> output = vocab_transform([['a','b'],['a','b','c']]) >>> jit_vocab_transform = torch.jit.script(vocab_transform) """ From 8e7a46715e4b405affe86e5e57d589c4092cf5de Mon Sep 17 00:00:00 2001 From: Parmeet Singh Bhatia Date: Sun, 17 Oct 2021 20:37:30 -0400 Subject: [PATCH 13/17] minor fix --- torchtext/transforms.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/torchtext/transforms.py b/torchtext/transforms.py index 3dc4385d6a..c97f17d1df 100644 --- a/torchtext/transforms.py +++ b/torchtext/transforms.py @@ -59,11 +59,7 @@ def forward(self, input: List[List[str]]) -> List[List[int]]: r""" Args: - input: list of tokens - - Example: - >>> vocab_transform(['here', 'is', 'an', 'example']) - + input: list of list tokens """ output: List[List[int]] = [] From 9a1757970579066c83767bf0525d4c1bc441424a Mon Sep 17 00:00:00 2001 From: Parmeet Singh Bhatia Date: Mon, 18 Oct 2021 10:35:03 -0400 Subject: [PATCH 14/17] reverting default path in download root --- torchtext/transforms.py | 5 +++-- torchtext/utils.py | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/torchtext/transforms.py b/torchtext/transforms.py index c97f17d1df..c28bc296c3 100644 --- a/torchtext/transforms.py +++ b/torchtext/transforms.py @@ -3,9 +3,10 @@ from torchtext.utils import download_from_url import torchtext from typing import List - import os +from torchtext import _CACHE_DIR + class SpmTokenizerTransform(Module): """ @@ -23,7 +24,7 @@ def __init__(self, sp_model_path: str): if os.path.exists(sp_model_path): local_path = sp_model_path else: - local_path = download_from_url(sp_model_path) + local_path = download_from_url(url=sp_model_path, root=_CACHE_DIR) self.sp_model = load_sp_model(local_path) def forward(self, input: List[str]) -> List[List[str]]: diff --git a/torchtext/utils.py b/torchtext/utils.py index d7506ee7e2..9a7ad1dde7 100644 --- a/torchtext/utils.py +++ b/torchtext/utils.py @@ -8,7 +8,6 @@ import zipfile import gzip from ._download_hooks import _DATASET_DOWNLOAD_MANAGER -from torchtext import _CACHE_DIR def reporthook(t): @@ -68,7 +67,7 @@ def _check_hash(path, hash_value, hash_type): raise RuntimeError("The hash of {} does not match. Delete the file manually and retry.".format(os.path.abspath(path))) -def download_from_url(url, path=None, root=_CACHE_DIR, overwrite=False, hash_value=None, +def download_from_url(url, path=None, root='.data', overwrite=False, hash_value=None, hash_type="sha256"): """Download file, with logic (from tensor2tensor) for Google Drive. Returns the path to the downloaded file. From 0915572be8659b04dc840ab5091d7da5e0971a56 Mon Sep 17 00:00:00 2001 From: Parmeet Singh Bhatia Date: Mon, 18 Oct 2021 18:10:55 -0400 Subject: [PATCH 15/17] removing get_from_local_path function --- torchtext/models/roberta/bundler.py | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/torchtext/models/roberta/bundler.py b/torchtext/models/roberta/bundler.py index 22a2e3b402..b7db892db9 100644 --- a/torchtext/models/roberta/bundler.py +++ b/torchtext/models/roberta/bundler.py @@ -4,7 +4,6 @@ from functools import partial from typing import Optional, Callable -import torch from torch.hub import load_state_dict_from_url from torch.nn import Module import logging @@ -51,11 +50,6 @@ class RobertaModelBundle: >>> output = classification_model(model_input) >>> output.shape torch.Size([1, 2]) - - Example - Working with locally saved model state dict - >>> import torch, torchtext - >>> xlmr_base = torchtext.models.XLMR_BASE_ENCODER - >>> model = xlmr_base.get_model_from_path(path="path/to/state_dict.pt") """ _params: RobertaEncoderParams _path: Optional[str] = None @@ -81,24 +75,6 @@ def get_model(self, head: Optional[Module] = None, *, dl_kwargs=None) -> Roberta model.load_state_dict(state_dict, strict=True) return model - def get_model_from_path(self, path: str, head: Optional[Module] = None) -> RobertaModel: - if head is not None: - input_head = head - if self._head is not None: - logger.log("A custom head module was provided, discarding the default head module.") - else: - input_head = self._head - - model = _get_model(self._params, input_head) - - state_dict = torch.load(path) - - if input_head is not None: - model.load_state_dict(state_dict, strict=False) - else: - model.load_state_dict(state_dict, strict=True) - return model - @property def params(self) -> RobertaEncoderParams: return self._params From bd7978509177cd63846b374803681f0a6403afda Mon Sep 17 00:00:00 2001 From: parmeet Date: Mon, 18 Oct 2021 18:24:17 -0400 Subject: [PATCH 16/17] Update torchtext/models/roberta/transforms.py Co-authored-by: Steven Liu --- torchtext/models/roberta/transforms.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/torchtext/models/roberta/transforms.py b/torchtext/models/roberta/transforms.py index bff390a261..febf6a858b 100644 --- a/torchtext/models/roberta/transforms.py +++ b/torchtext/models/roberta/transforms.py @@ -63,5 +63,4 @@ def forward(self, input: List[str], def get_xlmr_transform(vocab_path, spm_model_path, **kwargs) -> XLMRobertaModelTransform: - transform = XLMRobertaModelTransform(vocab_path, spm_model_path, **kwargs) - return transform + return XLMRobertaModelTransform(vocab_path, spm_model_path, **kwargs) From 96f0c2702396612c8a363e603522ee54e1fc78fb Mon Sep 17 00:00:00 2001 From: Parmeet Singh Bhatia Date: Mon, 18 Oct 2021 23:13:59 -0400 Subject: [PATCH 17/17] add __all__ in functionals and transforms --- torchtext/functional.py | 6 ++++++ torchtext/transforms.py | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/torchtext/functional.py b/torchtext/functional.py index abaec09acb..9231c9644e 100644 --- a/torchtext/functional.py +++ b/torchtext/functional.py @@ -3,6 +3,12 @@ from torch.nn.utils.rnn import pad_sequence from typing import List, Optional +__all__ = [ + 'to_tensor', + 'truncate', + 'add_token', +] + def to_tensor(input: List[List[int]], padding_value: Optional[int] = None) -> Tensor: if padding_value is None: diff --git a/torchtext/transforms.py b/torchtext/transforms.py index c28bc296c3..ece8ebbc80 100644 --- a/torchtext/transforms.py +++ b/torchtext/transforms.py @@ -7,6 +7,11 @@ from torchtext import _CACHE_DIR +__all__ = [ + 'SpmTokenizerTransform', + 'VocabTransform', +] + class SpmTokenizerTransform(Module): """