From 6ca2dbea758fd1f95fed0ab4b1a25fdc4687c889 Mon Sep 17 00:00:00 2001 From: nayef211 Date: Thu, 6 Jan 2022 21:32:27 -0800 Subject: [PATCH 1/5] Added new roberta encoders and tests --- test/asset/roberta.base.output.pt | Bin 0 -> 22251 bytes test/asset/roberta.large.output.pt | Bin 0 -> 29419 bytes test/integration_tests/__init__.py | 0 test/integration_tests/test_models.py | 67 ++++++++++++++++++++++++++ torchtext/models/roberta/__init__.py | 4 ++ torchtext/models/roberta/bundler.py | 59 +++++++++++++++++++++++ 6 files changed, 130 insertions(+) create mode 100644 test/asset/roberta.base.output.pt create mode 100644 test/asset/roberta.large.output.pt create mode 100644 test/integration_tests/__init__.py create mode 100644 test/integration_tests/test_models.py diff --git a/test/asset/roberta.base.output.pt b/test/asset/roberta.base.output.pt new file mode 100644 index 0000000000000000000000000000000000000000..d04c740b88838d2df102bf0c135ce9920454992d GIT binary patch literal 22251 zcmZ^Kc{o>J^sX{QgeVb7g$x-=5}&=cgi184D2b#Z63sKpOy-J^NF-x4@Y!nJp1hDth4uCd%f>^_fmUNF%c0-Ns<3&lNV7CaS!m= z;8)5~Ya$n^o98+Lj5Y+S$7bK4HT zfc3j)jPw`L(Qz6vRR4~llJ1*5{fGUZS4PhB@pIp~;{QAx=|9|Q*iac`e^DpVp{vLC zUH)Q=M8@v)7hh&CVn57B*VFn@`m^U!WTEX;pIxQB-ZD{_kVy|JPz!k;(tVVk6^ubJhPZ zd;f>=LpO&jquBYc72xNS%H~+U#PSspVD0b(ovA#M8y>_^W+#L$Sc`M#9cI^#?1hbW zmmr{`iTO`>0^c0u*si-Azg=x0hWA!M+bL7L*4~P#vOy458-!UOGr_+)m@UsgLB<8V zBTmr-n%xDM>Q*c$CW@84*JSXorvzLNmBN<5ZD9R-H>}Q^3OR6w{99a$<->NdO)pyo zDYfpv4vCN#&fm$s`F@}mwOt@KLX12uU(S-s#8IPU1WZB%teM zVD&Xs$j?Xuz)-yM{0O`cFvdv%B4`$UoBRtsZ(+Ve%rg7HF_zMOj!bp(h3sdan07}a z)v3{k|Awt5i><_9RpCj{J>mpiwHk0_gC2Cmydxgp4#C)YUf7n%EMi{E!qSvbXnt^? zBz;yRo8+S~s&W(lP7~m%=yaSHr9vG#qQK&14!fzM3auVhWZlDM0;!|=Y^re(dp-0A zzh4N33b|$MaFQ0h`5}g`r^cb$>(k`qELm9J@QoO#eI zd^Fg=9Ag%fE8TTOUOSe#=rAzau!`N9aRMd8#IYj#6`Wrw51sv5L^RSE9(RYbPS+_= zn^eW@+kE{O%w-^O~tg;(ZW*8b)fR%mjk=YwT9XB-;6*SGebiCWK#BCF<+@Sz4SZQRw{0 zroT=hWpRPvZm^gIuQvjt#|z->&uSu-Fb)SAywKIX94Fsc)fl-`~Gn@rkq#=Zby&9iBvUV zVp0~|H=a^y^>aGgryfs!CD^mV;-yUDpDZ2>oQ0t;R6+kHfRWW-p}J=qnKns;N7UIf zPal1>$^OkO3ueGr=)#Yu#z93-t99E$)oi4Vo!`D>s<{M9a-U?UL2-@K6bp zxi158MY8xNK?mDZHo~0W1G7`u+Qm+&{DG3ElAUaFay@=? zcEW+eE6g)N6b7cNGmRBzF|H^|Aew1}+K0UHMg1FL)W%c9)aNGo@BCrRA6X}`F+PQA zb-&pyeSp4>`$F-bwb*&#G@0aJgkm%Gh==YOruWSf2L>(^^?7SRM$!?sL}DN;-o?evd}ai=%P8 zPdBsJ=>ZLjyO~Xe7^*Cj!AL7bUf>}`XXq}6$hB+PSUXWr$jxEJ-a2sgKMk_)dxao; zSu(Me+z1Be2Z-Q&3R5l~0rO`pgmz_R{QgZ7N*WvK#y?^Fm*^fcV|y6vUGj&-sFp&c zO(HYP6+o-&9LU(!09q4*pu5|d9sQ>WeiLE@XGSQ)TJ?31t6fOK%gwOrN3JkS`KZ9a zQj$2T#?M}(@k^L^I8u;zz#OGcG_nuN)QP&*X83sJJ~MJXz;2yQV~K(tka412*fMh; zDc+?(c;-WRYBn3Yi$)69E4jjPx$+A6o*1?xV<~9Jo)*TdhLY^_H%VjCAi2104|A_f z0-3^pq|@Jrv`nvpw+Rd3U&#V^;1`6cOB=}c_jM2%<3-Lb=_A)4^pKT}(eV7hD2qVp zQ_Rjf8pDG_Sh?;gWJy0kZNe;K{zneW_Zjow_BYwlZRha$s7%3zK{1ePUxde7UlAt{ zUGn-2CCcCCk|$oq_)%;cnfYfc>18une~UIecqV|1O@kKy`KU6J#NlXNpb7PpCqmfi zQf408DVVfF5huP~0Ut6$NMyDJn^SZhr%YeVzF#v3iG&i|Qq{mVt#c(aOU&5m;d9`a z$#I;RWre>(6j{KurNr)043S!?jy(n=u`TixUjI54x9U7$W@5q6^W!_y{C5;buT*18 z#+QQP>~%ufb5gu$(mUaz?LJWFa|&dhEMOVK45~EtIYGg)r=|j6^ zS#~%SmKY7UxNu7ZB?}$!zVuj_u)CA&S-Fvp5mR9|&Mu~{pLJ1HVvnG5cRCw5KNdy8 zrLb`OIg4jo1n}ByE?f6=P;l7g9442(W>L#1dz3dH>bFt6tYazY%9Wv7N%GhrC?Ox5 zehap(=oJ*E0**K=hLGDR=r3G|@=uzGrc?uSN|MHn6Uxy_+!Cey7LzUamXUEDw?X2} z06D0+nG`n`vk0vs*!k5|Aer))$g3B_SBreuZIue%+98-9q=d?k)zMW}gXGzbpo1=N z*?QF`a1fV(Xx(u{#_A`_+@S}L?ifMFlNs2NrCaq&(h|+Y-GMtOLzC-hm{v0xrY#H; zzB_sfgKC0cMYS;=tmqa#?7t;+vbzE|TJAH)j*;-VvM31t5!@SxZb za#&6UYkc*=BW5Dm-m(P>7RG?XDFzgNSKXHwNp4anp~0&Y273xbi8szat!V8_t3Ku6Ovad z&x9qmnRw%30vYjA9}G-~vtkDsG?r{-PxrdP(b_xg>Q4zW_h=GH_<0o0TnvYvN=J0< z^%Qu0I6*9(Ct|zdX`yQCe3;<20tR^YZvrol{=q$+9 zodkZ~y99wWf%O!`3f}JW#3Dm|XgsD3hfXPAj0FSjf70x3k0O}fvO}eZmmz<-5N6os zK%d1tber>=I(oA&lG{MfZu*NmJQokuq*pRdygZGR8*{Jsxu1kXvmXCmW z`(4?s_VG-@x)$T>FT+vUtME7M8*`SwK{n5n!4YfIA#n94JU%e41t@SEF3O1A|pBiVWQ(HoFuCzv|by@ zc7)`R`8{u$j(#Zat6e8NCaN!NpR_~ZF;|QI>HEh<{uL74fE93inF%}itbl0V`oX?7 zEEUEXn8Ex6U05*D1*BDanYgO~@pv18Q*tk1e~mWLe=>sg$(qBbxdtFOkpMeF*0WmI zvk!^wJPE=Rl2Xi$I+4799=n*GHngmgx z;QL$%^%`uSO(DS7C-CrK0r|16kZjqy5pG&_qic0JnEpA3X$clIWX>%x+0)5gGjs zaEtU#Rpu{??0L=M3pmTOgM?iWr#+w2E34wRz~D+P`gGTD%im`!D}@%lpcxlNpDTaI zXOB(L`}1C!(yvNQ&Q5`t9zzUV;7^bG&LA-CKZPg z>C_hjtH)zdFls%%dpe#@xg$sQ&(z_oNw+ay-&6Wx%nbE={ob%rB(Fuv$;5Wi!Z3frqL%$MA@;fbaYnO4WC4uK>TAS?(wk4 zQaL?5edMC>>b72HHFqJLZ&<+BX4t{iUovPjwYB0+=`xb?QH76O(~2*Ar{kBuYaojj zRC)|Q!GAvq5G;xiA>RKYplm@TXq}nLw;Vq~Qop6}vCl&Be3~BqG7zB)RZc)u$#dMY z^bFs#LxqwnDRg6L!% zvYP57eq)YqHn2KC2q6J;xJRlSEfmoM%?>dhv&045x)y;;LO(7I^PvW-UJ-*v5&ULS z$^By=VTz7Dtcf~{Gw+9z3r8Ix(BKoI_IB9PD9WYBtO7+JF@Ak^2*`!4sH)Z|f|66- zc-+lgP}C8@)4qLU*9)w$Vna7s;V=dLD+X{UcLLLumcq-rsZ8%=2McRBjyLaphFL{# za9!?flxdbkzipYs%{dz@8<1rW)do_9CUQXMJFeMi^dG@>^0x}F%(u&T# zxY1=24(4Su+jF91`eQb+#x35J&4pgFtT z(NHpzO}1+W&(RIIcU2^J35>+XAq-V&C5^xKmE!w##bl4T8c~bdKzHRoU`M>=@aWZ# z>|^i*D!s6lT6&$qq754%7{7vQ<4xESG8GyP7~G8Tqm9@8!HFJ2x^!789Qt@0YBSme z?V}^e2Z1A5`#^-R8N$1PT*#8BGBr*&urzPX=dm|z@bcf8;F_ccf382o*EN@Ecf)c> zP?}FKmzfEUO&>2uZg=Lo=XuZg3>TJ-B zuLJb>^J)XS$k`8k)ZZ}S+Ng@CpJF)iv;z72_XtF&&4l=o)1k9n1LXEC1X*o+yxI7i z4Br>dW|f@;o7E*`@s{NfAvTo=E*bGnHCp)9auMrpJOn1{3NSH0flRP_hVM(lu}tqZ zw2aPXdsB4y`(Kh43b{AC@~`qjieANnb-Jiy55?jx#$!tyzlE@=P3+ zl?CQ2OsW5lakS{v2W%U@m{%r@;_3OaR3X_MjP|>k^S6)5qp~fqLi#X1Q)^(;9Y^B6 z%5pr}K>5}m$GG9=OeP<3i0x_EOrJFjGM)5kRJ~#Z?>$w^PdA*e9Q9S7Oy3y{TQ;o* zQhw)wM_@bk`HTCsU}N!D+aBN|dgQyaR&M{k*(9QP?~qlL=WqiLQ2pF@-ib{$?+U zyLJtJol)U~qTj)Eiy2&7G>iYYI%miS|KhPW=U}_O3tY5`fNJ$d$X|L_ur}*1OnRdP zP41L>ZQI1^ho=$$;ahRs9YyZ2;s;ExUPhB@BWcvhktAb-H%7j_M9yfqRjtTTrJIEY z7*eo_x$n7#M^B}*<~t9_qMTH8_MSpL%HjnlI<|r7z7SZH`p{y7fe5&zDB(HHU^uWn zh5qmU5`Ax<)1|J;Y zTPMAQ>!-!&rg!Jaf{Vj=s7DEAJvonOUH>wV(Q;s4J`B>kF0s35_t+noG58>QKdh4s zq!sTAnaYZ_uzK(>E16R(XrH7>A_p}ouC;-cMwZw*wVxk7ZbZwJCZfWfJPay5S1I-H zG)U`B<+@KpXkds7@2f=gpuu>{<}pnBZwdT;5k`fv@?7nN6t(=NOQ(bkVA@J?TtD^+ z)Gq44>wo$xW7ph(r%om~XLtjsE^(vnI@_4<&jdK|@G3q@8b${+GtkCv91K%4;P-M@ z34SMzRSMbRvjg?Kfc_{W;x3e47d-dq;HzM?rj69tgcLQZd>|&{^ z!?A4dcBqli!)wx_ygxJ?_7ekCk}U)u8yQ-(Z5O_{EKc1-&FC5ba5VA?qR(^6;E!%2 zyL9uLu&93&&zUfh7M@h)f=9PtTv`@3Ts#Hy8_js!rYtmYV|?8ECNjn*8DmUe;!;~A z7Jr9gY{hGse%gtqzyfH`CqdsMRz3m?i-U)l|u4A*1-!pCIkc?Dkh zC!O6@tpOfQx#`}?^k3(oMGvpCycT^3ZaKNK*7g&?hB&Qyi+7|qqJ(abkVEe_4@jQT zN7jnGK(p&Q!g2*oK2mmCWlhg6KIe}Z&3O`C9Wrh@eX&lN?!2`XCM)HU7t0hZ2PBWt zO9g85)lXO2xaU0nTNg?nO}vkhpDvIeB`0WSrwJ4qeTJ&R$B^{=6MQY#2c0NGx=Q1; zrTDAscx?S&*j|vtBRwv`hx%}OY^MTuuepaS6l}Swy(YDeNr2~0qWpHLG))TFgsFS` z;YHCPrd~LW@~Xzr9DN4l6H3qxbZ0293$ydKFBy!O+X?@FmG^zR_d|x|=3nG$* zyvCdje>RrK&%DFDj?br!M_P%Vbsot})4}`N?_h(zD!Jd)%BH_ppj&MW1zPU{`K*7F z;rAf2(>h;?c}E%<>2`x0mzJSZH4N~qOAO}pDpAc1Wu&~u5_<~nV3c|rQE%*ng3-ye zcM@YNQ+8w7m=8qrn5n?{aV^ZxQRBXCMr4j-K6Kp~Yq>!E2%TcA4TEFCA$wdI`B_qe z8#W)s6JeS(XxStDbHRpY6F*hZ7;L*a_%^YvBz-kPiKTVE4 z-8>iG(tW7!*U9Ft7lA`RRM0b8jF0@e4cea_!=0A}JhyL%XId*o1BNf-iyN0gXXQ!= z6spn7E-?^RAHui4KMdDQf5L(0zwkHcv&GM^fEMF|Efpe@sJzFpA?{9xw#39Rv;GON z$>0?@=f1?*&+n2=Ub=Mc^c)^{X)C5Bza!iu3>@|wfLc@>smqSDIJv7BKKC7?t~{RJ zT@b~GD<(l?s{x#hRQdSZDe|xNChm-h6l${sae+ ztm5>NKln+4Q zIenbGNrEq!5eo^O8JPZL62IH-%S&Eqla@niWL;w$Sv2Dh+=(a>wnbm$jbr4%XXRro z=@!6zlN`amT7tGJdQk5$NZjT=WNXZ?p?a(_zIYVJ@2%0t5x1)#xk`^ahrdV5UH?F2 z^IM3t%?54X2qsaY22G>Z@E)}%sQ0^${jBK1;oBwnrQ5b3t$zYs*B1)w#%K@4avpf* zxDTcj8&g>Kgm@TaVpP|4OwwP(&vmr0HuqUPyGEH$DL6^oLf;BM|9Z)~>Ikou35SC_ z0{P6kOGIWtD6w4Oj-CnrnEv!&mGPGsgh<`P*Ar53TdqEu?;6g_B?`eoYymtciFAht z0UeD}6u&VG4;yh5LVAfbL6q z>eO!sVY#Q!e5N=ztUdwH*9FlJHd_UE=N(6rm*V^?_E##19E9}TMf5_dGK^p0U>>)_ znD6dU=^v4QvXIeIHGLnYBlb@359olsMUU|O9 zER2%n4g9X&I~G_ukxlZSgCUU;+-lQv^pJKYZqsAIIeZ_J3X-HQubLoQ;UcMeZ^WNW z4~H&;J@CrVm}sRJk*CA%kuGOL>_}GNpGNEC$F&_B$PF9G8PLCfZ{K(pc%&XW99l{DYAmLXT_#xC*NTb~H?Uc+3Hz1x zG3?r08urE(ueW|79-`iS?f6bG`XNKV>!!n%as3PqKr__NKLm@zdQqPB(dPp?N|c&!0{@Lrwt*>8BDG&7 z2RGLv)tjNsH#7&(=Ot-4X^%GOJIk{_Bhq=oi6pvmhcaEj7lGW-<23GW7@KR&q1#G~ zZYc(Inx?_K*T#~MvJWv!c0JzAd(C8CJz}>#Z$aAS>%!0uEBeIxFeZO*CQseGctTW^ zAn8;U>JKiU4&g)N=c+XQ@i78^G>t}lHIx1gisc8kWU)fYLio7yBpnm$Nn1tCAzvpF z&ez#ONySUBR$T$Iy)w+sUxweG-OZdBvGBdOE}>|Al3}Z@Q$S=q`fl= zkL>)9*t=EBCOl( z&F7w1rQ-L?$sL`c_{cko$sf$-6C{^V3qdL*|4v8ogh^24;g51zTiC9U-;ci1Cp-& zW~h_`&-UJ;lT^-;wIa+vvr8 z0+yY37Q{V_Xmy|_Go0!Inv14F>0fE~;ewu}h)NdE5X;5kOZ`DM-jil-&P6q)S?saG zXlgkeELW8lL*AX1%57Jtp=g&nJUDb6s;kVn>C5$W=ic#j?6@)t`JW)!U5+m>xXXRF z9^;$pOmO(sNcgd-h-o7buUH=+m^GX^1q}mfYmB#^F(F%h1 zhWK|>89s}-PV&E>h1z5j{^hxlg>MN%wLD`kJ8v!;Ob2FznvIS)Jg1uW3TKS@pI`jgKm%M^B{fii{@(8RBUB+RymGC5Z6I|BZ@7glj)#dLa; ze23BX+Elk$k?Y;cf-B|{e3@bn&H5P20<;V8h4W9?Xmk)qSAJr*4_t#MZ8KT@Y&ouU z+LIc7zKvSv??S)G1~RblGrMxTjJzGR2E{d#QPxHc99w3yc?RDF8jdCy{C+OK^eGxY zeeTC*DfJ;fNQz$ctHXd%zwx+VIUHIiLTzWn^W5|s7LeEs=XXTX%LP%)rg%6%vg8)D zt8ay(urf4gKg$zV#=*n88;}Ovp}pY*lu}Q^krJ#da&M;5L; zMw8fRzW2Tz_rA3W6ZdcAY2~x4%KRI^#1U~__Gt{UmV}>M$8nv!-SpwqTX_Ap7PJg> z;YHD#xOa^YRh;iyDXrYjx0Y$s_^&MjnygICHaPJ{BRe$x?>K!I9*+f|Pr@|C%TQ{v z4rY3;g>}{QIn4ftwcQF;-z8pv2b;?`o@s%>;S1^Krw<@Ga3eL*ivsIjZNB1w2!^Tq z`~a~Q-|U6hVsvbXF=uV(0qDyHmEBHg1(dzIq%a1R<9?~ zllkV{TVM}GHS=+zgfx0QPD6ErI~d+qO;2n7hMXWjUUc~+xQXwDo;@YlXOa%9*G;4< z4{hj@)=o6KZ9xY=*yHHcX|Q8ppQS}(Gq=dwNVUp8qI94H9raC=1dHm@32)lbN9j6s zdh`cAT~ZP#$m(%EUJAtaRk4E}OX%e9iICSYr!vJVpLrb|OVdLR!YPwfaBg}G!#{n4 zBRkCSyRDm-mX~D4vaN^BsYXE;JfG1+ z*G~D2^G}q)#ORmgQtcpov=+lLe;>o15v%zh|M##hYNSwGf#8pET7syT*{sVUff*jX zP;r1-&?VhdxoD3p8rvx2Zuv-BSGtewHz#zXmo3#59fmn}o>XOTIIY_nB8VMR2??>I zY04NRO$}PY*yqYj^*{~Ctc*wLiCR3m#*5ZHH3sWd2j~R@0qSA^bi5B7;w1?EIczOo z_yj=JV;D`Ee2_jLHlC(VJAxN$qxqm|1Y31*g5b$MBN|Ys!F$v_pf02uk&l z4TMeVvJjy#rP|=J4<7k615eIWqJ!quFh$_ZeGO;x0~f?$zlAn^m>7uH8bZ+5Djh4n z05=RjinAB>z~L3qxbMYMKF}t?PS{%DjBTyV=&P zhsO7-(ZQz#O4fZKq8Iz%p0)^oa7c}OlnBQqYPIm%(2oD9I!oRU`L}{e3|EhP#vCs1 zfaUuf>GvVs$A#0|X>V60MjakO-RFqoVn1=NVKRcsOxP^EWHy?f^4JN}DrUi=akUV+ zbPkD5m`GF}W}^4V7}lda5pAR&Foho-$d1WD$?ln?KIO1Qe25g0emxo`WhY4aT;Ed^QA7L!`9F;t;fhsGV=#AfRikxI{FVB6EmwsJeZcm6q|ss4~h-jIdj z{h8$UyB4}&LLxL=+l({SL;33KnRxYU5mxQbq;Dis!NkIZzLTiI{M`p>yt*S6E)0RX zb0XB&{w5eCtRTxa=i{e@P{D&|E2&QKXpCFtB~aR2jB?lZ(7guZ=?yJ)tHSz)IQd5_ zsDGod#rM2J_U!Q@rltYViFSMDN5IQFW;>a>qu9 z8uN@m#ZR90I0oVV$#t0dpDet%*9A##qp74{78w_HV2ImpXTSZ6=*(XQ@OvPSk~mGA zb-)9RUl!oXnHqo2TXZThzq8-kfawWVEa9I zNXvVa1zb_3W~xf`dg^(y&ew=ndTH>4x|z8CL<#Ng^5b?hT_I@L05|kVLOIcCIC;lQ zjJ~&%=G!Ub*V|Tv|E*(#`#<5gS6ck&5nsr-QU{8|O!&0H@g%Hm5x=%y6(m2{GI5U# z{$=|#dO=x=9-OYsTR%Uc+A|B-)$WZ@di@*k^+sRe$0T*fDVUCtLN zcnjV9Uf{*mGUU^e82Iubjk{<^(^nap0_oUb(CK{!U2BH)VPUxtY2eH2hs{RAmMi?+ z+mEnc^6!u~dn7;PZA#nXcj21SjqroX(6=KHY>M4sf_l3!NAQz4H8$fnmv0y{Ib1mB zNIdu0X~OL6hQ{KusqFQNE|?Wq3RQZq@OH#Qrk9=vQ>WeoZNW*6aK~*iNd43U{ zE(K!V)o#!jevmB`v*fLkcZHju#Z+y6b`iw4CsRiUh7n68K{#w0KD-``pKqqp`r~m> zF4Piy!U#3nveV)9cfQ^S%~k z*011gYR|fGt=eOaE{8;Rsa7Nhu`zlx}>rqY9se*a) zI^f4^Z=vMQ0`O0nh~=+7F{NAjDDHUx9wtrVPT7}9dtesduKa^M>-S+(>fgZ}-%Bt@ z{44$y)PiwcePxl0DD}?!N3!<=o%*H>Uxgau^`C~VBA@^2cG!8 zhnff0P!188=5rI$jM=0~~D(p{mgT=~W zuxmja`M&=$)Nfb<#iiFFV38wDSd&lYr^UgIRsXS<^ej|=k)Z*?dYlnf1Ao7dfEKGV z+_5B_HW%2^m6tx?F^MIt(7=)qo#iZ>h_Z1x|AC{rDAzqVjed^p!OP08QTIzIsw){_ zUbP2V->gBs4%xttJ>k^E-@p z=>?}8h(A`tKRq%RUb39X9~}6O%X}rlqjEL;zUGOkJQHlT7L)$jzc8`#13A~Ymo&VM~tN1!9kF_Jc52v5DE>utw5|b2A&TZ zg2`HEmb&&SKbqiAHytYzxF^I9n3P{}y0IryuVz3;~OohlCNkSL0Yq z8Qx~5&D-lw!C;Q*fK* z{t$yX4uxp2Z8}>zI-I|=*QI05tl&?yH?xTGvUK(uQ^Fc!hx7*W!pgOcL%O3IP`%w8 z%fvll&(nU8)6^DxnLL%QnrX%MtFPj&od;pnxMlb-uMnU0j;G~Mqd?-|IJ~c13PAxa z@O8KjpLOdiIJdm@#qI0CckJ=jk7 zAw5)6Ib_QBF~trYBn@(0aP^D1Nkul8nysL_2SRD~_HZBptV62XnZLFZ~gFPmBtc~GT0`ZUs}%7LiFh7L!z`}`Yio3*V^Xh+OQ`->lsP7D=Hiqb<5=QrK)}WhhTEXt; zVpf`E5wT(=|(=A6GoL`6H$w9pb7q8CEeizL!kQVNqZ?CIu~VBGdVg(Ms( zq`f&I?DO$Q@VU5@M5go;;~nZ1Wk-_ujDgLpSEZID)GeX%rk#AwntQ}+mX<&oLAT|y1{-3xdy*%l`1F6N`$d)V$5 ztwf7YqkVHn(@r~&Do?+k%-am;qh-yYV0@psmD_Shjb>05Jq10&Pq6CD2~x0bh|f(p zPipHILipnZT%-C2?zXieC#4{fJq->z-ef|-CkXK17QHt$Xmf81OjnD5rE9;z(s3j4 zE6c#Mz6&wt+Dtk{BoD6FUxhzfWw=q}B3Ual2kj0#gUsAu61(|2>=MD0jKR<1&(_9a4i0tAmQ9^ z=DP;jX!CT)IpBd0?p;D@xd%uBCZP2eK+DhzxcB*NT#%>%O0{LMr{53N>sDjW6??FD z7J|p2+xUL@HTGVm5hJgT#v6C_`0Vk+!AUBIc)!j8PZJH=*zOF~j^?yEXpHdZj@6ia zCJ832y#(p0NLkYcXg)NrSSsH!y2mJq|AY z07|W47&EyA=HBojOUKuu*})WyHx@y|A)Hwt@{5cae9z>JO?lT!3%r{)q3XXy&A6|W z2ushrgu(KyAbQ*dz8ur!?)%M!cfVM{x(Rkw)+dUD(?(SY#tV1QkCpbcYHK{vy56DGKYg)acgh|3G@xx&YkLDvdYl$W zOU}dgy>T$dJ_M^`Zowk!S$tTXJ!uZfg$naucraC;CmtBmkUU%kc1tT^#p~;^a%G+1 zk^6b@ca4NS?T0Zh&5qu+E#U(Rn$T8u9M$Y@kge&I&e|l#TXy{gPsek_J4BK03yg#T zLpAEG6vTXT-$L6GOB`%ep|)=$>D>$WLD6jiPCq9``_4(x-D=T7)v^z;TSt<=lOGN* z2TUNp_yVoW{R6-2^S~l}9EsC=O-4RH!K~lvQke&fxXX{<#B81x&$u*+>=>g7ODj$Z zK0X(xyTw&#?B12+<+>2O)iw_c7rS8FQBx}K+X@Fh2kHd+PUEEG~j#PmAttqHwe48n_)v(e7>*?NGwIG}+N{%*uWzDsl zSSTj4JxjI{+16+hz03yZbgswEw4RWw^GTLRGHL9Q<7Ox4Ls+~v?zQ~MW*<1tFYnOd z4gL;%$)!;8v(JrqeDQ^*@6+haC)HH&w33}t=`|nmIEc*NeUF^CCVchacU(zoaJa@z zR9vseC%#T2#Z$h*4dXe)XWd$wJ^vOSc0WV1lqE^NVkaD&@(V=n=Roo47s9zsg?RV# zL7e)pkxY#oP4D(gQq}KAxNXf@n3SIgTTQbt#q0#$ku6~NN*D9ET4`?UV9X`re}K<| zGkDjroyPT@V+&)#3DsD{MOziQIE z;Dy-+RNmnRDbLLXzuk!M-v(8!&W)wj<*DescvRJzGrBn9Y$o=G9K_S@d31NvNSOb3 zj6g%xkxu`$q$2ZeG{m(uv2p)Wag50YYOFSzhMRbrQ$2MW<0*-DL-U7im=)D5TZXdc z&uHtaHkhb%4UQjvgKg_oVfB!1$4+f0@os(!g=yI|KdS(IqC;uS9V<#!>hTXVUgPGj zA>K{Y+|2feMX{DU zZbQ6#40fALr20FC)^jL~=aYKJ()NZNzE|cPZl}+9{rRo@6Gchh2h1* ziAA}fKB9+`X->mnrRCr<>UuK~=Q{47pB}kau6b)lm&+AV3%`0YrusF` zQ&)tQlZ4=S+=}*J9m>~Vo@H53*~60ICAnoi5yD?2!kH06cp~mT-I{idNmVIBPG>me zEk94M2D}B&?l*k-mZSKirI?RxSKf61e5?ot1vfyqOd4DM6oKNO>0I&T zeGsX=4!5`Fa+l%_)IClF{so<*G-EbZ`H;yUtT_pXhTVs*?3uJc`69-Z?4Y=7E&b5; z9YVLC;%T|DWZ!rPY;P4-i6s078YbEN=TQfC?1((X9af_c@ARRp;Ua;%e%; zMFy%yZDQh&x6zT&L;TG89A45=4QEXhK|B05+Z~VvHnuOpIPncEQ__GjzLAi=a-rq_ zD&$PSsp{H3j?9@N4I(K+i9+d#v+ivu6(SjWDin=Rvrrn1AsIr+7=6r)_@XH zDv1oytWla&e7oy?zxR9I_qx9CTj$zmoqg?fpKGmi*1gYJ>%RZL7h(%&`Nbyu?j1xt z?&YF*d^vu`Yrwuv;*M+hGSensB1_%9`P}rOha%?P2s4HY2l62&pY#k67vOBb4{ za5YCa!u2HwAQSK4&r3es*3b&f6qjSTJ+bWC4USBs!V#CFGq5OsFvT;q($KT*8>M0MN1SCEPA>Qta^sEeQ9--YkdU+AceOH8fj2&U-ISlkhA4ysx4Ao`=5 z>@gvz5wM24)w_$i6gQVy8`i{@`-p-_JPdzKpTHe}wUTU;+kw4DT;Dj#G=# zVA3D_0W$&v%^YXva3{jj*xUZiFfZT`I{L;iTfVlycm6lJzkWSq9y}LI8eZb%Pon(X z2ty;u8>D-76)Aox%pAM6nA!9?4>n1(2|W8IaI9D|d{lHLqJq8PczrHdZZAgt_bOb{ zhbuVbkPF#vPcZSTd{y)&8z%V22(B{aBO6+A82h8w<5z!qra}2QytGl|_Dh|>*VX;i5^oeUQGOM($w!mN3F1?w78IQ>~)vAs$Im-`1Yu0JKYZ|l5Z za9=j_JwJg-zaqs1ZXL<3Y@f*IZd$|eHY1{Ldz?rd@ns@k+ro-Fb@20#T$ueV0Ng_l zRykeD!NUfzD0*iET1LmBzt2gNt^0pbW>gGy92d#N7FNPH@1yWNU5B}LX)fb(U4&u8 zKf-1uGrDx+43x=F;moES#^ibp#vptmcXs9=*IH%CNu8>r8G2e=yi5RWkZeY=n+;ru zo;++R4rJO#{UXi&r>Q0%uTd!MU_VSaPxhZo;|!LcgEuP-xr6qn;m<--=F4V&e<;l# z=2nSv_Uvlrn#}^3l9)u4X|`bAgE^e~H4(1&fgH0w_9ble3E{Y|am?q%2{3D!1r!ch z!A6soD!t$g$Y~wJ9r0OCp6M**^(s|dMv@f!Vcja`uyhCU+CBvMw&v)dP zevrLx2z*xt;f(dSO&3cqLraN?a0qkQKE|EdyN~bBqw(DJfF|%<8qJu|L#Ri~$N_Z= zW?4xTY%b_TNuNrze@&Tcc`d4Z^clLA%Y$Z>F3hhR2`;U-vGBPhV`RJlgf?5VBl2cJ zmq-&?*tVSON$}!YjXbdUOfoL*4Z~40^~swN7b}10Jwv0qCEUZH6HH3pO8T{X5Vq@- zVR(ByPP}`I4VoZ>qJs_G)v~cTyw;c5JHC-qxOavl=}Yi&uN-sy62i~5k(}RRduCNc z0KBr_1=Kp6xz$!pYtk0sh0-3F^DUKWzG}{ev^<2{13!p%Yhd}tFWIyu#A#y=)7c+IIlKDVob?c|#khSFtscb@Yt;&L_i|&V-VVk4;sP#o zT`7}Z^8tQ6I1aim!oe#q90GScGk#GuRU>ucNq={_K&|))HYeVP{Ru0mPhU3eG!nxh zxwWAD>;@VC*$#B{6AqmT;FaZhj+^Ez;%#xscBuhI4Th`2F5rT^pojxT*b_Om@4R){DroJyn**+3FK}4~!J}|_&Qn@o~Q&1Z-S_Lx0p$>GhS+ zW8OwB+QxJ11GLPF(qG~)30}W1;{^}vZ4sS^X}ebzxcApW9h=T<(>#WaMw8i)nHN~6 zzp|Nif2_x@QY*$X;5_$7zzF8Dv?$Itl;KSKWNCViC6m-C%*`K^B#*ucS2Z>lq0$8% zM%nWwJ-5Y~c&f>R{f!}f$KTWL6nilG`KJLUZYHi5_rdm*ou-;~J-EZfj?3^j;mpq1 zkycP8ZqnHhp(zYPfserF^;j}OkO$({hJZBfiVf1C+Bq(!-{l^4OzZxj)At;P=fuV|gqf^mEK{J6`?STH;l znykV&rG`B2!~QU+$uZ$XHr^tBXVXA)bO?wF8o^@EA^5S-kRz=L5Mdi;Bb}hgvYRLV-mi!i{t8c7?Q2t(e#ALIjkKU%e=kX!sadPA@dvc z84HhUPA8}lcb2c>PHQO8{fiW_LPm*Mw_MBAY)1wmUCQ*}00(CeEW~$2t(zzds==pKC1s-`}=q~q5An{`(Y%9G2^{cY!jWac1^7ND-TZ*8%-ZW^Cd_j6O zrD-pUa<5O-!PCw}D%g<%V_t5isU!73)+wm6x;~#g&Bn^(rTK!B>ZdRw&K0NFEC!Pi zvB>|Ch2^Hs5IReXB&HRS!U-bW{A_Xb<~3Xf{mZ~@^GLY(r?5aZwg80tL)p!bD516E z=&z|=^r4yvxJXQfc#9jbO*a4~R-eYs@ETkn=tF{cc|yF)D@a{62YS@UqwLq=O5;oR zm@u^s^>5syP*P(D-%PaQx{vnmBbN zNAH=ltKSuYTa_E^$SS2}28RSrX)}m|??%Dn&kks9d!3zB8vy1-JAs_J#OlSrhFhxZ zXwQ6QT;aP83l~e1Ppibxy5|W*2Ops|yuWO2Y9et4@7RIJJjN#NCfHA&j0>9Ns7isC z;J_&vMkFtRE<7YnQ`35gw&Wag?{hbM-eooJG@T2pVsk*(u9dC}NhB*?ra-~|dqgQe zk$SfB8e)DjjTO^GiOqHFi^h5!j#Gg4(@Uv^m!I>fT|s?RAoV>^BBD^bDieUIAG4~p|2ZbF?ZHh+@)>`1xisg+C&yYt)^oO&(P?a zJQ*a)6>-Yf9due;Ek1nj3m*z+KnGF752+VP`>9_7u5Ke?&!mt$yCg6sJcYjTo{jHb zh+*2q&*aqpFhS%yWuA$_lJc>auq5>m-6k!EM5W;jS+enGuj7qQMh<#=|n6s#OmNUL9$5O43T^pWyR+QH9{ozhseOW^?kZYaGEERw;>e1pm{;D~6wPafo?%MIdU`Es5wx@C1UHvO;R9_O^43@e z&o5BJicU3J@0lX#t$0D)_AJ1wkLH>_ym=8Fsy*=@@6$7OIAc@7bA-Gvq_A-p4o@GA zoo@5dWA}LUF+ECNW~qQXjfRDs7NDXBzD-;UiL27kWA6euQT>Q~=ZeY5J_A@QIz;{J zH&dVcSMmAv8@TDS2OU|ZM&uT$;A#~)yu282+EFc%A=*qLW2Tcx-Rsn81D~%{R|dm% zNu(}a8y`G8FDN@!P5yXn2-Q2A`28VD@85|a@vR|Hx+s}=_5Y&Fb4!TclmoEwrUfRR zmVty1Wz59(eO&bDi)7~LN>bu}o4h^$5I*sG*!r%yAS~ifHtp>OiTWUDo@i3hnXFDu zjF85LHa)O($RPF$87S1hR=MR`FRQH@Sy_MI6swaQ-m$6jFgN zeMg|VSqnZ))}(`PPh$Gm;~Bt2{r8EI@r^UA&o-Xsi?ExXo0tg177KU)GK9p)szje@981PJ z0hc+A)M&+l+16KN?b2{y&19jaKa`ADKPvbZD4-YE%c!Ze5}ceoNX0;spyM%%K0-$L zw!aDY-!;UDyLvFgwFqBNP=?d)QKZLV75K_LqI(M3>B+GdiL8hlsWu-^X9*>N(iDBj zm}yD;N`h$C_-=ar${6T-9?GNEcH)v{C#l7QX2CcqKL|I~0I}K0^hbjP7cVS9HlLUZ zhL&A8s%SC(Ic*GirhkT(6ptnOT9J6DQOD$&<|T4vmoBX;R0fahd34-4S3%s^T-?R8 zKU%~5(5c6k6%Fbn-!}BpR+DiU$m?zjPOwyONP&}0A460Rz9!?gU4yPYybs&XZyt_TXUo^U&5kVN0F!*cCqkSaSG73}2DuW%G~|Ii30md&6xT^6t+dkti6a7MAk z4YcKZ4Y6Z3V6x&!8uPrC+VR@=2l`zVz)ABV znH;Q%W+NUFk4aJ>Vs-+|w#&d|{W%s+xteTRpeIZS%O(e$FR(Uy*PumxDl9j6id$5? zDmvyYfP__5?5*iWcv5~ewx$@sh>!B7<+^+63i}SaGcc1DHdCHkU{0;i>d;)h%jD>Y z3y^t53aZ<-(`g5MU_`$hNox8NvofdBReCAlVkpL{@o|!U&zB4OW~-6|OTNLd(OMeS zaoyC~z81v#P4TVp3h<800LO8Iv|z=0h`QZGZv>^2HFtGEdwDQj>V66)8amOm%zSoD zfh~CXM4%D-1ZK(v(ZOVKf#!D?o>B9Wr2DC&T5mMz-oFzz?VpbKB=&&#(;h*XP%wFO zRt`U=x58j`G5j&X735o$&{6#g$(yWS*v`?VT+W$$PA zSju7P#KWjfog2TJp^#7TJn(u*YdNbQ_M8ofd(ZR#c_;XHysnPB0V3+Jl;`YR&f{7Cuo-WH}4vT}_ zYDJVY9EWz26?E;CpY+L`$VKl{7&OoRWl z{d=$E|IOCH<$rAd`B^yFii_+0v(dy6`u}SF8wUHk{C9o%AE!di{9F7kyxniMtWd?wqoMHUytA)zuW(#?SBDulQ$dy literal 0 HcmV?d00001 diff --git a/test/asset/roberta.large.output.pt b/test/asset/roberta.large.output.pt new file mode 100644 index 0000000000000000000000000000000000000000..9f5287ca3fad00f9fea2608b7c102bdcaca0d204 GIT binary patch literal 29419 zcmZ^Kc~p;2{BN69EmR6^N-HTz)MwrkvZatBLXt|0lq|_NOM7Y2suFEPwosJxnfF9Q z5wa9nLQ-h4ZxOe9?q9!q&zoE)U2Q-m$M4iB~Zzi+4iuf+-?y8pvsGxNE#wEi!9 z|A+BIFC~kH;di%iX6`YD$*2D&JD;5ZLR3NJ?__c#Xaq*D9u4o;N3w%H>ZIN03Z7|b zW=0>+LPD(_re=J_pe8FeeU&c!GfgMmV=CCXJQ04j&xMK9{~$g;3rOd#aj@GfmOVQa zOUxZ}Aa7qIKHQ}XmVs_e#kQ8rnV*dQ%MGAP{Qzt;iGfcS4hY7J+wp6^-m!C%BVgLq zdxG7;6+|XZk9?21!`gM@VBj_ry!>X0&Ztq!6a7ZY6ul9UyAWc;akY zTl1k#6>DyMCQ>odFyGLPo!qK~t}B;QJW&>g$=@(E8}p{Mr|1UosG-To?WlqMIr_z;hB0Wd}?SV1rPvt zmRf@AgWK$5tS9t(OlCG7nZ&te7#{aLj2|vHk+=Ri%w-z55gjUcX|e zmpejcv>^mH_`zGxWB9pwAJnv65&XHne^U6+(214*QiIgbQ=#ooKDZp805_wmg&)t_5{>P`v_bAy%wrFpS6y}0aBFb2<8S4@zZ`+W)(RRFMhIvoo?^w>f{wNBh5Nf*fe$~5YmQpruKq+~Je9G!X~Dqmonrp& zf$04{fh`ae2rpfUg?m#AnS5{(mI>?ehxr+s5-~ZR>#<(&c$yUS%DBPl`e$g^cMWfa zmSLS`u^{WKxS&j96nmeb2!ElR9rJR5cIERVq3e=R;d&8xtji-^T^mq3fMdj@UKY8# zAC6U}LsoevJaXtG-f_Vwd2R%=pYj?e-O*)!Q}$xX^&CNdpDa<{R9;hgsDU+Xw<50N z>|j$$CYcwa3sZxtu;;l6OB^>5GB!M6M%xBqRPZpNT3RA=+A9tFmsJS=R+VDOTS-uv zpv?j!g*duO8$vaWF~{(;pyOY$;KuD;Bx%_=+)^tf=R_5RgBSj?)F4yryf~Fb8w=n~ z%PeL!{67p^a}Yu3GT5bMvI}ER5SP_f80EeIm$g(8m4M}J z*|`ST<*N#Xv(AvaN|)Hlj`J+#$9incd`9d}e<1BPhH&x68hE_719W?)!kzbT*jztJ zxOQa)vmD4LOJ3x%I-xy!jvdQPP60XD^NYNXSp=hoZvnYN2V&{%gzf1CxXWz<=>B@i zvcnIc;+v(={NW{R9Nr^H{7(wn4knY1eTUFQD8=AM82nW{%bE?=qinAqDtUVe-xV)H z#pY3XcGCzb&gINU>7LcNS$iS#D;*0Nxzn-mpdYO7ZN#Qm`8fTL23cJ>8{|6U$pqPcqLcAca8Y0m z8xn3Y+gDN8=a+#RhqQ2@x&n^>se&obCqSEPAu({2ffBO=M1P+-oK?OK+5UY@`MVPC z&u(THW}YY3)DPE=cM#eMQ<=)?d3f_b6KG#*2J<|w33EP(QsY(L1o=K3lLZsoP1 zpNcWLnfiha@9q~=uiijr=;}bm*A_NqdkD6Md&BeKw|L{jM1J_48`+$H3(noYOpM$K z#An!{?$*!nPxH#9+3JPDj_;EM_XEy=)MOFpJs&}qD&JtGFQw@X%|_h3{S;M0!Dcr*M08D%sKUMzh?>{slE#6RYk z>it6SC)|MLckIVCT~8r(zdw#S)&qABb}c>_>iL#UCB zZWvleMU-G_ff7DgrA!XT6Oi5I1q=RJbNujaE zG4D9p5~4v)clO}KAGg4B|7+HyH-Ytz-U(sr&)UpUdmz*s?v4BE`Z4z6cqps(hf0MX zBzW6ibTSDO;`RhAb3V&{Ee-+$I}V7-E z^JRkaF9bDSXTjS=mSx|if{jy#VM&S&=@|cxqC;djOf*5i$?E&x8+Nj=Enfy(16U1ilX1yN_Hnip1yNTA>VSB!-wpR-jW@ z$5uNRU|7a&6wT7&=8}H+RKb%S(rOa~2A7e0XUs@P$4E$46)=xj4fK(bgE1}7z*YGW zduSfTUWA+B&h9>@7*az5cV?mIzO69+bqmIOIkHJUlI*gs9r^J~9xW;=aLg17T<)d| za>LHR-}Vr6tdoM0u!(eb!97-jYe~w<88{>Lwcy6@iRhaZiIvyVQhvK++Wp*=Na_E1F08h@ay$>WQH5C{QtLiAZ9c}jmL7ylwoQ<@Qjcv7y8_dSj*_G# zEi67+CUl&p3bSQDvAxNjrb6$D+U z$6&7cUef)cMo_ok100{!F)_vM_= zwq|{j3&^CFitxkB9E4M*=n|g+4~QO?EgYC%Oe%|G;BP!(0a}rm{<(vIgFW9v*w)W77_dwZT=Y_4?#{)SQy#|dEgB8|(WfD@@vU{|-VLzv=``lz`~m$R z9>gV#!9RmNfLf;bx-}EVmRrH9tX^`dRupRdRSA(?0N0~AdwO{#nKy8iBt1GsvL>_$ zRToadJLzYTiSJ_v?1+GQy27|cXN0ReBx%(t2YeYG1L`01P;;RSHbhKCc377MX>b^$ z=PoR2Tga}i9M9Gs6rfA}Sk!p%k_@CjVO}kdK{ELPoQe*CEZJ{li*~GFn6xfBR4#_h zH_J$thCKX=oyf;ub){cd$>9kDMW%bD58LYWFzALfj_oVK4Mt}8R3nBhiAciJHn+(i zBUxfnuP?})APxMW6MC=+RMAWobd8UJslG$lB|IOluet^zi`AJSj^MSc_G0e6xftWy zOCD}+u2E2&3;{cm@QimZtG;!RO#c0z?O0sFq^9bESL9OMv^I@gbeAGHr3$9fk1VpU z3@tX6k<1bW{P^aKV3x`a)@PuAuF7x8fmmsdA`-&=3Yn}iArdi37m5QE@R;)`V(DcC zzDZwLqr(}CTDr@+s3cr4f z(p58~Af&U3t#sMR@h56WWN|ZmfC@*noOa*XcKN}ME*1L8ru3u;m7tod@lM0SIHFM-1)g^ z*{8;$Jw;&s%UYPFfI^Q?-=WiNEem>Pjj7!$!Tv!2CftuAejh@KfAT!CA#Wye{!tBu zdV3*##aWCzej3UjS71k772A7T3?e3e$L@*aVB~`BSP*jw#+)=EYYu+Fx;4p6yJ;Pn z>3E7&IvGODoozVnR5hu!JPd_X!(f(I2b1h76UZ;=C)fMlvEJ@aWTj3mvhsTJEl3it zPN@Xa^@|v;OK0!Y9H^S=3T_nS&xK*dbXkxLKezi7NR5idsdJmrY1SGz{?>$BiA{i- z^(To(@;SH@FGd$Mrtv1(^|;snA&l@7y=xO1n)>g z-_GJ_s3Z51l+b#P7=1Q10L9dDdDWv${NScetgN5L51St1o!0tX>+>vnae6A97&t%z zPi(^TpB`h%Mfutf?H%~#$3qq*D#u*~B6QJoWmcUp!w!uRmAGBf79MuQ3}!#J#{iw9U_JIH#2q^Xlje^mHeIdguW8TAA5Nvs8N;Z-$S_=% zUB#b1J`2~YF4LL2_tKX5!-7%2Uy+ZUzMSn|P0d8wFy)s#y(Bt~AKOi-qv%9#ZEzm9 zEY|>w3Pp%6E~bmBjri{4jga`PfL7Pq(5$K=DlN4e6fL#5P{)_MpFV*1t_^2GwG`?X z^M!0_(txQCO~GhO2G6!@aHh=+L4o~?tim!WaVpa->C~4t?E?y@Em$R zBOlM=OYU@YGA(%N!;f#ur&7gUxOuY$m6*`teVCApfS|MsXl@m}mx1leGAW!DO!O z$&k6Y(!hgnSjns;INPun3;xD&<0el!`rTeQ;XI888NGn^-TE9I|D%8o9<0Pu22s>@)(LdeoJ39YROqaQ6ZthoDHy21hU0dzgBor$&i6YCR8=T9 zNP`Dcgo1^`61lI%MwCsKL<<=y=ofrr*_I_ZHKmJ=*_(iepUbm7?d$2^@O#8+mN>t8 zDhg~$_V7H1XclU79cPwOGQ&22A90Nne*4#laoHNQ>ZceB%Zb55hLd=^W3W&rRfFDq ztxRpI$I)$asYq{Ckh`YuaH?M<%xpVk^GVZ~e)E}+V!5WYXGJOO*w~FSj@3*PSMXz| zh5T7`9^O`~fiG^(z$*Ny-t0y^`i0@u{9QTP7 z3JgNdGI2Xa%pV-$Ja!+3uvyCV;|q1x`ujU(+2_-1UfMWrVK>|AErQ)w(s9iLPckv8 zily+Bt-| zu91af%pfWFDh-_;f(80BXhnt=e`damOS`l{w#!E_j+f=Z>o?IZtuX2)V+_)T&6u@W zjMrw|#c6tbW*3|kgo!hglLZStU-yrshNULeyQ?$or7?uFA<`$ zF0jBehq!J)COqC4#?MV@qb_Tw^8-(3(@XY=#4dRcTdY5gM)f<<$lf|M3cg8O)?b1n zW0$f;OI2vAZy+SOHwj9@*Tc7qSGnF?5xV=~63Co9Mgq-9V3D z43yymAroloQ;NlIbNRTG9!NL7Bs}(J4&Sz2hhJ)42}WzTfJ=EIH+76-A``E(e&HE( zb+Lt$yWWGy^mCXHV@wCuWs@Q$q2TxZI*>BU#Ej9Qr1xhA_4?d}w@;7dViF%<{+uTE z^YK`yP`JxKI9AiKTid8Z-w3YpGz1sY#F}D-IC~TATr1=zT-SIH z*}h-lc)?Uqm$k69)QO|N00Fj^_6O;0xU(^k^T zl#b4zrrH1De|iVuVw@4li%Ao1e38UEhZjKVQy;t_jHNr~e1WVzZ@A0~H>`6U3IC;- zz%838^zRM@e9P-VFJc<4UfxE?=P8hSd?k(E;||(ZFbdgNS)9j!o{t&$~W? z{3khnX9&AwwrRtW{=@i0QUotd+DdKnm*Tr?nY4b|NN{YJwq=sLy` zsise?BE(7<=y;F@-+PQwuHJN>zX1(9O!$ham&sm<7ErE4XLuWPY%56t+Lx2dTa&oaQ)_ zdz@O2ckXS2opQgfEbLrBcHvU=P@IM*L(lRVsd6x5V+H1<{>E#LwHQ?!!nHq&(_oYN zkhyFme>1BaMk=4c?~|^wHSxW0{;3lmne9xoA2dPfH(9>R{Q)lPy1)|+I;q9h;e4!W z0$tWpjFZ+)=f?Lo(#GJWJic6k&Ng7MQ`!hJ6~5xZ&&kwQVsMDg7lKtr80vUI;PWo7N2-t{y*3n^#M1M|6nukCWYF=>3+Lx zm}tI@mo}!6yGKspp)Cnqt*{RsZLz?!P8=gY45J$lO{1Ms*SWgHAC{FC#G}4G#bA%C zHhWeMFoRFKXvN6w^uOu%ssD~F-n=;jYX-!5h`l7O-spk{;>>Wr%n&|{UO`u08e|ne z_p=8Ie=*76J$w17l5fx63LC4V=~Lr3?D36J{O+)cR4HDa%ZIvfhmvo^&EPgl%-=`f zK2xLBqpxDgvs4T@Bg*wFRH@WqJATD;ESH(%N1XQd2}<^?qYZwGXiffZYHjO=B41zP zT(i%@6CZ$Y`W?yToUdSzz8OCz;QZnpw*5s0kCKVU?<#GW-dO|Da<;tuSu_;} z)(e`=>RE!RC9k@x#kKpEQ+=%^RAJ&)5U+kgC(vZrf7yeNT2YCXDau?+y;P{QCyiz> z!pU4S@~1lzp17^z4KWe4{mOl4GWOx+9!C5{TM_aRM7+>o4}V$MnKG)^moMeta)z<=!e?;mLp1LkbfX_73Sop^8f(yu zrB0FR)X6-SD_bNZ3~7t=-#9?n&3vvMGMV3w66M>Q4fyeGhJ4BlByV-PZThYAXr*>L ziPUw(&pY*4)t5dDDmYK)h%e(0kM97d^OTSIyPS2$$8fteS1K;Gl1la7VKMi6(CfM4nW<&4W5=bhR;40fqt*&a-F}g zQBE<2o~V6>+Z+#5ukd&r6xImSoRXPyYc|yhImhfYNAQ7rF`#pM6YPvG7e*VXQkPBh zsPThhTr%()n*3gYxbt<)^C*T-8p(8$=^(y+phc_aU&MdahP>2t5)>RfMP~LC;jY)t z+~Y?fu3x_e%`}Tq-RmShTUSCGOLFl2e+NM3Sct%({3soCN#b^mv$_17^dUY#rpCnN z8PPc-3m0ymq=p+0!A)yvT06Xi&9m8U(~-HH`#s3QtJf|F4)|OkRz{Bmm!yR}Z1`kQ zZFk-ZvLh z7A=CPhJ(~1W)mg^^fLKPj~Hyr!+-0lNq3qZAJM!Mo-KNhzejptOxzmY;IGHWS8hPH z7ngC*$I0~3v>^KYiYE^+I?gm2O@watg|xrMghtiR;s3Hm^EMR)zFF)74)1aX(aH&Q z{tzY}*{wj5Y97;cZEYHumrGOkYM{i$BA(g)9c)9Cq3O^ljF?vo&Y63;n|-*=j;SrU zy!;(WNED+FW`AX`TTJN!Fd?G?XL9{$QLbxyn2a?)2Axy`Z zbw}oLdJb8)?Kv!2+=q833i#RCzlfK&8W%Ys4m#^E(%3TNJ`);<<< zG5k; z@uibf|D&!t>3r!_d7d*AryuVE-kw{;HP?6|sAS^c_4ioOkxVu1*3m~#Q}Nf8vAnEO zhqq|G=W`v$gQZ>#^rnra?SW}v)O(_)T}%lC4$?Srv@b7lIe@#HO7T;64Ii)am8tyP zPMv!)Vd*y?@b>P;8Yy3{YTC;q3yWa1lNR~bA;pXH?!vudS^j-RHu}X&^KS#ibZyf~ zq0hlnV3Hw4cLnXImbs=}Pd$cLt{n|GA9(QPO&nAV3m~UqKFe&*C5wv}Qek>KR(I^C zr6Hr}l*p_6oa0rVB$A7-M%7Z4=cd%f-w(~J3!?hR)3Sgl1OB z@UQ2*;EdK;@T*He$JI{k^6^M&uyPX@$+3Zd&N{ZIr;eosN)H%KMC(nf7Vyk($=o3` zj*nY*4>F%t;o=AL;7sQ@&`)&Xqdvs(nJvwHr@>k}y?p~0TcU=(ZmIB4|1SM7_aK|r zdzf$P{*OKpzesP|Z3bWesWkrATzI>26;;cOpk_Tc@LzK~%zfDjnlDW0W#7;6(<6)f zWO#$%TM4u(ZiaUwSJ4_DHD14|1mBPUD6l@g7H&wK<9IBAibYtURG=%rpl}k${I?QM znZ`kS%yZ})BLLNJIzriv#(bmNO;A29N!gZL{DP7ZUz?atzy75t<8hU4%E=}T&rk48 zrvvG#ma$x`G?FHM`-<--=kT(lC48CI4tl*;d5tt|$-I%YxyppCQtD zKYvOixhA^q42%fyN$K3{(pnFF!fBZRv3>%ZdElp~$!gMNgJrRQoq%P4A zuUT-;a~oZpJpqGU3TUu?K9qJv@UkHeDSVANO;WrKM?BVH;+&cI~hb4E~I-gEC9|xt8E}$Ry6#4CP-XohN@EqhY?D8o3g7CVbx0VWinF90woE)4*am`qcd%EL3x(*NPtFZR1D`Y7L-ILLqUO;ESO}WB7TME0Flnn~TSPV)cWrTzOQj4 zZB0z1YMLW?Rq0Kdy>cTSPoOPjc<^~))Zwb-4eG*RCZQwo6(kyez-cg=vi>gHUrCY0_Q(rpqvLh)AT6y7o; zhc}wJai6&fT(l$~r)j)pBK~=#_uX+8;d2UpMSOyd@9(11ja_tv%Q-mmAQg2oib&|- ze0oLI7e4na;9HH=X}jsknr)+$xkQ^W|9$iyx*cqX&ii|)d$rSVSmtUCV z5N9*!UG|c&_O@+?~Bu>)r1OH;wX20W^H7happ#ZHR^I`FlVyS0}=%B)0~ z^TwL~OBX_3#S&b(vJeh8E=JpKU#^2Q_^Y#_=MqmuTZT1mG9pxcTR#EIE1Ch##W4wEEi^4WQ_FeloLI;QLM6DoGLm;N52X13$7xbi34dm$H^ADh9) zb*7Yysc_{h>#%!=BBio&wz5x*@s7F=@3Biq=gs5jB9WiW_^Lj?TO!Jjy9+^P-Vuxm zD~DGQeQjr=de?WW67<_QWn0uF`@Z|Hlyk+50c1!6tgjNkcQd)8}(+mFdo+Mi6^DlFxlnM&};uf^(IIRPJtx;PaEs_*Y>+WPV!& z7OM~O*Awlzxwi|Sv34ZA?IB7eeGTc<+=Wm(x7a52?9VIfcS`a-ukWK??K0}1xrvuK z#q%egs$9LffJbso+OO+N9Bv0t(`b9lm^+Q%^8Cc=7d5bh3sQwvH&to;`b@sL2jONl zr7KU$z>ib~U@0a1ic=3=FiVY356h=vZr6zCZshlDKGQcIWjM`(!KAV7e9Gm+P+c7f z59U}sq&tS+wx(Q#vEa4&O_r;ZglO z{&)5>sIiNIyruSBuQ3pfwtuVeQhy98y=!z&~bkROVJ2y!8=*W5(0)X;(T}uD?L9I`801`{wY)FZAfjXOjHotvH_j zX+H6`jpg3U_F`JaFGxwLLkX8B^!*|U?i{|4mn#jY8q)XK=7@HT8ll9KYXkXWomDL9 zVkE7qIRVoCgHY4|9CSWB$BC=QVbd`~QpCdfz}uxZxt0@%M_(4#zZnfRPlNbF*EsrA zteAS&sPJ=zsh|<=#b=6j;E10B&Rl1Ki-{(UeLDjV?D6Jr*D3JvIxivo>UQ)Bo<%=x z-pl`@3lBGl!uvb_;FB(OFxxCe8?16!NLMI&Dz1Y|N<}zR!;_XJzoIVx${|SYFBs4> zpjK7LzZvHdI{5!ix#9*!a>DIXni#njqk7G z{3W7Bmw$M%5Sr<)Mf&auMm93Fu4kwz~;9xjKs%G-*&ntI^!+^?0t}6^j0M z%OFi4P78i#aS*G8lV6*uazhRk*)$Jz_I`)lqx%J#{m_B5b&>fJW63w6P-x zE0=#Eb)pei-!P2Z>5b&w=UxlDez&3Qc*ZAI+~6#HJg-rX<|o2t@O0J=q4pC|wQ?9= zF8>lNUTQEJ5Zp zoer1U2}Hqx&RjAa2Sj7B<&71MSQsz3)J6IPm#OI8b;Nhcw z;>KnXx+!25|KaihzkR-kS;vdX(-sB(EW!;_edFk^_;{+W;ZAe%6=`a=3=b@8plg$s z^Qcdk`FYjt^xzj6s@F6R)4el*Ph1Urf)H}o=F?}L%9LMuiCtMg$i+VqkP}{x8&^t0 zfrS~AOUCfLx(NJzRFTfGJ_;-Kns8ROIbSL_9cS!|6Ik8aL9c$^3?o#>@Z49E={$*_ z>}Nx(P*^!MmI6Jwe`O4B8)whdL)&qr)KSPhIE5yJzT_u&9>5!J!Q6321aJE!iCa=n zveZM1dE-i5UY%botU01ZGk>1IK#K&XkX1_4Ssi`#UKzdqxKl^9k$lWTFYcbc4>OMF zQOR6}qrz1A%c8Sj;c*S!El1N1yNmQ6~;P zw>iCn=l{*1b0-wRv0Z0i_^M9k_{pEA?X{veie5oQXC(iW;l__R{(S@HpZz(t0EKXbRTGHGRlKjiX zh45w4Z1zm$JnEiQhyUiu(l0()_@Q_W?@2$6@<&YRnILnrCPGvAKGcR63B`%5)NSy+ zbOoKx%w-}nk!P_Vc>P!WvF&oaZGI(8{v9ryr#%stSm|@;C0SUi zmW0_=YcXb?IejyE7GG7k2K7f8!Tq2L8t9+G8e1jks^N;fRyiGaTX$m5lLT&4>H=CO z?eL?doL@hm2|1@C>EnB{eD?R3fQM_*Yc&NkZHe0VrE4)@TRa-ccCgNlWUz=IMzhTq zqjl_lY<@qaH7z)V)o&yD@-4kEJ>P*IaG3{QevA3^lAq9hWB|QS<}pR}%eY#t7|!0> z48IiD(CQC!=og_QRj%?QJt;@H`D;<$EHQ&V&3*;j);rM>OLcw}4hpET6gAsBjQ!gs z3DUPEs8eX>{ePdY=b~L{t^>EF^QL`S>gD1`S9dOBvZa019!HJ z;FALPVo3G{H1gZaL0(XEaA~?E-gtZD|AC`1Gk}f=V{iep0~ut$q2I6}|XIv5kG3X2#cAJR`3UEAcr- zYr(GOz2MfS+xV<*sQ&p8d}7uJSYup|(laFR+8${-$8k6x93IVPcPjH&E3UJ9hURp< z%SxUfpv{k1-U9EA1X!@v0>NLpRp-h*{ zI-?^D`oO}`$AAum`dBlKS7-ODt_vxA71xy zrst%Jaqq@Q&=7MMww$iUm=;sM_US?P@P#iE+Wi;UdZ!uFy0raFcSM2+Z5Ye~N5mIz;?atq(I zX7Z!ghR$?+zXYE$dhyRG3p!s(j3>TP#G?J8gheUQ+}-23YStJkIS`8-mrJPY{&9G5 zS|b#JG^x1Oi<8U_z%}L#)lUm?o+CobH$(7N3?U-g_8^z(N(&5X@%W+NB-p2$*?M=w zZ!(iyQri#vMD@A4U_Q{vg{(Z{5Pn-cv=`!yGrUZYpeZGLZO+~=IY3*noT=e+8 z(5>n`zDzJ@yPqm>he->7n~#Qz0q$6oDasOh8_CA6DZ%;v=oXm%TQpzapaO zXALX(^}ZE1EI$EVX?_^-ax2)z2|-Hr3-0^Rgb1cr;!~edbku`GeCi4bcv=~Nt|2)X z!d5a92NxQ>HVpH8e_{BfR#@Ka$$q{*f;-lhV^fbgm+v{se*V&;6O-HM2X||b?eE34 z8D-Sqk2t-NwTc`V*iFwbpUi8Yi1IJyIm9{Gg2P}DbKEVkLZ(oMKJ&7X5JF1$S?58LptRgTp)K1cb_XG*8j#s#+s9Op?t`{ z%~GJ@6Gm`HBU5^^w2%FHY>R1i>8KizLq5AjKze*KYJ!zS4%@^pxU9g|vv z=Nv*Hb7U0ph_K))$BnUgPs*kK7(Dqw_T*-A2AD(uA z>wOrDv)*5)dTu~h=xk*l|C|KZkP~$ErpIi!%M&*IPYcetB*pD)w5aTv9kk#-FP3b7 z9g{4Vz`N0j5NJ7_W@=Al!zX2e$<{;U=owksrg;-SdnfRQXVEaBu?`j|hVystTG_a6 z#I0u0@Vmi^{ufq{JU|0t{+M!=+#C4D-jB^5JEZOS+=ENogt*w)2BcgRY}UR`CIRDm z@k!%;^w{(ev?T*DZCL<>4*Ih#e{R8w@ZIqA-Oiz{tFicFa11?oYbsaQjKT-wC&SL{ zGFCQO0dB{>hMA-97kSugU|LMPQMZ z8ICshz&&bbVQEA&4)hejsi8Hde^NBB$}FN6zwU!W{&KcOJ09cko;aprsYVCnuCqNA z{UEP<6-If!!tXB<;AeFM4D9lu@8b5uf4mb{D;y-Q=L$&g<3)lo3pU~muhm@1d6-~< zRHb0UTv_^bat0)9u;81v@fQa9LT?leFmOj6P?D|K3_%? zmy4{Ai-YWRA!M}{!?4B^_}5v2)!Uze#%wQ4U7^dZHur$WrD;5Hd?ZXZA4V1T1<7QrC~%*nPTN{X^ItKa$;Q!rn<@d?Oq4PN=?E(klf)4 znLdEsk8{Cl?GO$#&cOrD-RS%{AH$3MiEShBJyOf*hH)$4oY7+1aZa48%-%xN@c9Z3}=H2fh zUGpQ(d}WV{1GmWWDr4+&_d|!pvmouf7AVQZ^D&yQVBw!hj! zwC#n(`sZQWpei>hH=#x99L%-1kQaN*_@epiu`_BrYtH;jIv+-1X{!RCknkP2|(r{5?>2^I;b)b?Gs;X(*!j4DTv!;S&~V@#l-}v4)PLlKW-w zV^R#Qk5YvFPi%>Nzo+2wvrY^)-$U01RN}J(dCVg7HOg#$NKA9;FZBke(S){J$ZvnjM#av-9L;O&byO@^2Bo0(F*m+FLy{}J@}z4{?-HcHv7us5WI#># z6|s?a2fsb9@$=|Xv@>zwrQh~q?3A(m-FJD+(|p3jd$#gHLti{vzW|LUE<{s{<9OWQ zny`bM6jYxKC$o1b(GnvcDm^*_Hhs^=iU&*Sn2ulgHf0_g9Bs-Ejg+x1>D9Bf>He&U;iiUd z9$p=d(`4@9_|(z3baX7=xa_np*TX3!aa6Y2@Cf-rA z0K?Z8@%HU8bpDvrxcXHb%I|9x%>8-@y*8V1d^ZJ@R>lpTU(KN{XQ#kf^-@gV-aZnWN$0_H8#BCUk`dNeHm#%>u9tLMKkf`LGqL#NOph|!ildf=}w&$`12K#QYD>7zuL(gQIKTCyfTl5z9E*?hB zGme5t{dm-s{Rf+pO=wSI9f-TcLCTVAf}{a`I^cR+4T#n(Nc&#Dd18_aP09UP=K-DJfr;Vo`!??7TA?!Sg4ye3`fEC+e;9wn8lsv-7 zoj1_k#Eu@4@x{Fp{#RvZ9!}L8^?f2LM23>2lm=62km20x7AhqYp^_%eqNr%nFPTY( zB6E_8ib5gIUN?oLl8{nqP#IFBL_+oM-}Suj^FGgWUGE?7KG$*V%Y8fBKF-7$ ztUF{xMo9`MeFO32{OvTA9CyDy=pSW{T5uUSs&sjc*VyndKaVzgz z6#iPwZW;KY=-V#LTPVsR8w7Ovi2(f8c!@lXalxu%hXrQ;gp+#(B3O_lN~;1g$N@(s zR=8LKKWH|x2-8!b81w*E-AYHd@c9gMPD0~#85r%c62yKMvG>XqobJvU_}nc3MSaUr z&nt_2e?tyDS4^c3ZpK0XTW3_&=z;Ha5lak~X9i}eFnwDhGjs~WnAsJ~>H#0?ro_S< z9TRGQ<~vuqG#M9Nz6P_qFEAH{P`1amgA*T}L)SE(Wf7A9&?^>_V6CM?t{N9Baud=3O`q!E4)bJ-TSz`%geXfDf z-jAd!V;NWlMzIHWL!i9*G92(41SQ*jp!Jj2*&O!5GLuB%0=-1~;)o9O%b3hu8mDsi z->FcoEhcOqpDQ(qaEGl!qO`SH881a%L9ZNX7G~p1Mf28jmG5UjUGqrxc+qZpK{6X# znmWN{!#vXZ@c`|3bpV{Vr$f^C)2u0Q6>1!+qhW3Z(Ecb8#?|b_m+@_AesUOQ?;4IP z^hG%tGhY-RyvZC6x1r<7-E6~dNoq2#pBp*)Al%MZXWdhtqpM0dOTDy^iARYEL**ja zVbv)5SGEn)pQ=-TLm5tN&KWcu(QQ2V^dyalsv^b@%vh620JQ%V!4T{BP@bT{Qr@W2 zx~TbRS0|vEkNPmQ$dv6cwPh*lar99}ApLtm64pGLL)%^l!T3*kobSE&+_3{*ETnZ4 z(W);1so$RLjI$ea8Ky-e^ET6w>Sh>Jx{iHqmt-r|#$kMj9MoP`<6cz;V)qqo_ASno zo^E4w?bBdpanOWKx$Vy!x+Ay=y3XjIKa*{lw@kpD4q%_|NwaG*7r0qdHc>Q{M&}1@ z(4%k}eI^Wtbw9n>rL8^qeC!fZZ>z@k5B(;=?sIA1)G6#p?-h8yU=+)~yOlk*dxXo9 zx6(6LRl#@qWTWHfklA}}qoVQ)izC+EfmD?uXw3HqtBUP3Le-F-s8gpyc|XWwBYk>i z_X;vkX9~!Ue+GT8&S6i79i6fLDKJh(RUfPrhxO816+&qW$at`7<`WH0AX+rI<0(3@|`_c&el%Y`A~+QQcZ=* z^Hrqy(s8y|{}kP!s7rs^jiHUo4?#U9oeO(!4Gxa^xT+xyv}s-kH_L{>E6L#86(cw=e*q1*oMab^PSSWeA9my)0akm4LfXO`l$Ae%Ab&Ob zN984W`6x2gokifjHUuZeOr~Yu_R#vy0kAJS%AM1mj9;IsgxC=n zG>@d=7a5mZVg)K2#<2rmdyp5PFmBg4Dt}CYE@(9%O6_xSP4-=~&t)miHFDs}%>CH7 zS_5ivCLQHp%w$1m&3^50XTyHZV4`owP^)I?l9^V#j`#36Y9CY&Us}xRwf&P>#Cbcs zCpwE=OYnolg|!e{JDq*gh+tP*wP>>BW?UA=$HC`RnfXp7=Hh<>@*cUv$X_F=?x$6p z?{G1ir?`u~t<7W~wky$h<*od6Uzx}i^kakdQz}s+L67b7<0`hsfPCahCUw-37VCM@ z+PmiT;jxoo8fV3%<;w85gbUt0T1*w4yrBF*A4;FzL-mR~$qo7WFfnI37pBI$Bg(q) z{9eL}G#zp2H%VL?TLl*l=YpQ%7V4STjpLT)(J$+hu}%Cm;}UmKsRk!vHX?)R%NMZD zw|9w4YBH;TY|8xKi<#skC$a^Wli-A;A$<`Y&bC#x3j261gyy=@pi-wy{SBAng{FCx|Ce*Yy4=^6u1vMr_Z7F zqIs}&oCP~;@eyoIhEtt+iZsI39agS3z%y43VM5*=tgX>z*F9$9U-fWm(~t{CRv+V% zU0iWg{uk!-VLWR}Itt~<$z*A91USuH$9~8xX1fh+sq&AXP$XS|!yXl*lv)(?dJ@JW za%;KaYhK{xTfJ~XKacd>(SYPP&aAqGUjy&I#C#Xuhy9&FbX=x39+-X3vc9kr++w z*Q^}o?pTdq)=vQKwq4Y#V+XV44$=`DUcA0T$&1AxS=qsD?_3XU*BZEeVyP^*BOGsf?qVifA)8op7~6VdxMS~fP*0-+5>0s@ zyX9&l!|JiZP06*O@?sGiF;AWj{!AC1(aRu@gZEOGKYMTtY{Q)9i%==K06H!-Lt6D= zrr<7MBYsAc1s81Cm$w4i+?0Vc3^uUPky04FO^)_H_{BYvoeIYTSJTr;sZ?!6K9v5n zXX=&s|7t0E+AEY9VVW2bH$>ajI!n~)c zoTo)3MCP92EPg0ZZIx4SJZ2FzE{(*a-v)(ZO;hOVxMy5B|DK=WXHB;hB|~lIL7Jbf z#rA51un%%tRN?G!^!yw}8|LW}=}9W&?olIRNlwylS3kgY%{4Uj*;TMSKZAO>D0z{75>n3CC@> zqm#=-KzHFWMEln)#WEdQ9R7gS*OOGE(uTd*Ur?-`t%pSwqBPHU51Z3Bk#6wdYe)(^ znWn4)6K2n#&M!mg>T{JSm?=W%x`@%tqX~k}b1_)r^NiDuD<`ktmeZ5FmclXrIV^PP z1B)JGlD}rL=Ix64 z3f1ZNlC!+);G&UMksj;nS-|WI6KH-}BzJqodiHIXki>kaxJ+t3yS&AoHugR*c^fi? z_mU)Vuk3?xzT-5Zc3B`R9vRJ~a>Z%!gpDxII)(-Nb&~118MJS=276I>oon~(Ls#!h ztV!31u0jcJL`xKnvL1okcV$qkMd57aX(Q-RI>0V$KLoChH`t<2dolj!0&2WD7u3!a zp;y9QQsG!eZ3YkHr`$gB{Jtn&d%24`UQwfc8^+UNe`4W#R5mM*vZSY*La8)PVG7F* z(9aIp0$s;Ke9WSW-rKqC%6K=nO5r=oh71S!je|I~I#sy5?GUu;gfONXPpmw7-_Gh% zxZ|2eM~~=%h1O=&@z5)HbF>K-zYK#@YgOr+`IE8s?P{#)*eWO-mIq}CYK->9(~=X< zA!_=3ZqA3lu-#sgdR8rFNg`Y6O`Ee2em;}Usq03E`o}P;?;~hFq?{dZ_io<(9p~2Z zd9AZ4_@p?7i;YYqS=;)c;KUl*Y@h~b_m5z0o?5K#ojr(pYSR5Fhfz1x3c%?CES_-# zQlu2PQ?KNiu6P5eMQG4bdym3+XK6ZUk|zjSyq6BtET@Sl0OyU~NMHA6;vlC&idRLl zV4yaMXnBTn4b}UJ0pG@lJ3D}P4-MIaU6I$~A?G5Li38EgB zf%UaHOySvfYVzw8Tm5xDrft4XGWUz1QvY?febQKFKkg@AgU?5&a2*!ZVh*G2T&c^$ zQcV5U3st%fICOdrEo;n%+-w)_&g^z_-(jiYq)|6nW4kun@@qM#(>wu=&EClzUq<5M zEkfoMq5vOKTX5W#Y1HX0lE!#KUH85r8m754?e!{}(w_n+MR`qK(Ij}RT8;9Ro7nfA zkLdHL8Vn4*1WUqg*r6|R?2=41sM~GEs42(USBG(|-}oi!o|R__vMWHU`WN0dNM@do z&x7Ih0F0RF#IR%x9jHD`y>BFO9rv`MR{0vFdh}sOo+%r6=}i5M?Wxi_3YwFh@Zz#8 zUa$KA9Bz-M-O~(7k?eA2`Xid2yt0iNcHL*DIgjbbaqiq2jY_=zB#X$u89~*obRaWJ zhx*L>OUjPOqnU;T`BtgVQgpVnv6o&mz1F>;UY(7RGYZis&zf=t3n5t|n@)0HOV82jM!}hJ! z#!+=nY`(ZHbRBAeK=mc;+~~t}+s|y+f94$~2phcpu!48rC54fMT; z6zecg#a9!`@yZz=!G?R;#^F02gA|`ruw5+4TD&(iUGE6+3){jz2_@-H4?{YC*BWdx ziJ>xQd|B2n1_zi5KX!;`C&hqSqZq%g>jn2CZ@HYprQEUmB^bMZB~AHI%%rvbXp74b zdAnjE^$FO+mL)mRFW$G*}ld4|PH;%5L5_*NbZgpf|{8!-@&jEZj7(w=oYXr$-;(VO24}6F7>l}|vDBmQ- zes5;n6#Zy+Qt>m-JeYtj#Z#Gt_ylVGAcEfca-HV1X|u6=zrpY|-MF&h5|n!JzU~8~ zXv-;4B06_0`x+I928p4pqmI`PoO+M@m><>a-%rdQeg~7l-83TTGsX{Gh9K4_~`cqe8V9=>BYgT`?JKVg=&E zM{Xb|D`M_-*Kbm(ROL*Vc;o-O#11g2jNF?j4ERv$0T`s-9MA~u{Q-<)8w=9Vfx zTXKnglypL=Ya{6IXl)|tvO&j8wo)x!dOq=G!~$@i|sk3L9b7rK#PSXRH|bvJ*Shz z;sS<9|J=Fkfc6=-!`z3BYQ6%uQo}*%SDP?&xgShcodo+FKJoF|0=8bN9xF!=H|gB0 zNh_2;z^R4)C_97aAXcrRf9s#ZJncio!;)C+#znC4uM|6X^9?%geFL`q zxft(e&PsQ>vjlPj1IwSo+61EanJPb1KI~I<);yd8JDizMSE6Zj|yAeUN z0y}q4glR23K~2Ug62}dHp`kgP$py->Ql8WF&tL+Z#U61vE#;h{&Ulb`H-r{nHo)i{ zf`6r_^Ljy^b243xopyTy4*L=IW;S5ohQ$yweIiq|X~v`z;~}HW17>sSuxx@Z{r)(T zeNcC2Gsn)S6LUk^@uQw}a^g$6L!lU*mfiy|*Km9yD@6kXteCmMCHQAVDt z$bK~2(<$BQ_+r@{c3HyXdt7FFE-KQ)j(s>x@DiW2%hHD#!Pp}z zM=c%vnAo5Jvl{seZ(Rt$yT41geM6^EBTAELY`j5YZsc%*73#FRDZKdSQ)N7>;zo-s znqlte3_)7k31;f%2KAe}F>1LSec?DHP`%;%n17Gi1uQ z;la}V?0dv+_--W45<}hqO3kH$`q3sYo~>b@%_guge|xxRI2te3UB)H1vmobSieP5V zG7LRYh|XW_S;xEe?BIiRJe%AForBNVkvl)|d{+d#?*9T04oI-^KaPV!_d4PoF2QOX zzTo~{Utq->eIjpdOIueo;F$0tc0WCjt8lwd2ae8zdHo@{Nm7$;)mupuoe1V`k-!0Q zU$$Gg8&3;gLCU}nFr6}$nS04mPC1g9uXupY?#)=Bx)Dq4WT@HpKpdMF4SgPBeB8U5 zuNQdZj)oU7!(9|B7yp7!ZF8t&0bdXNSwOGG8=&jy*YKoToNaQBMmdFU&Sm)r(0Mc& zUf(jOtn|Aed_xmg)55bhqI|f35nZ%>TN<<`-4wpQBS|F>t)>-AX3^p55kzrY6isM& zL3^i#L7#pd>RAvrZ}&my?UEyU&#$26bxF=|V*^fJG?AV8x{bwpcX9hAP1(u~Hgu`q zVJi1bpFWNm1|PTY;bb=yQTGxRI^c1DUYK)+Jr2)c%|@NV<6niegV+2IJ2;vy?YRv% z$!lVkp2}t13d2Z=1yrZ@9{Xo!mvBR(7M*FgiWNm}AX$pabbOj67I#i2J>FHQuv?Yg z9KQ?pc=DR2h)Yzc7fv>$CqZoSRE$ks!ZLa0#tpKBmiRBE`!AnC_8^Qt_ccOoYRSge z&7r3cHnU%Qu8}DUskGlnNYr|C=)S`S=wNaVud17{Ey132VfGa?I2}e@#tagN(;YB( zN(?BQD4}_*F*RS-hp(RRWGmdVuyA0CpzVkaiwLmhrd*N6ROw*WG&>CbIMlIB%MV1M z=Q3^Wut&M8a%^ziNLCv721RYOnSf~1zNnK_CC8i1TA%_?(`tolV&YgB&xA_(aSlBN z*HG^B61pMnH7zqvz~2+xXxU96>+gvyZOmNuUBe0#eC_Dz z{+%%GZvn9m34~qOry|`vhkJHrA{9P)2dh@R$AqC;6zH~rw@?PcNBV)2Z!F#@)<@fG zH(?Z4Me+*|!GST4Q1?v~v(}UEEq$5Pji;Z$PGP4c7nES+R7 z$}O_9W}kiUVMyzJP#=FCU%J{;7yWda-94H{4cvy}yE3d*H5R!a;&c<^^N>d`vRsp~ z;9t2B_f)&EBN}RSN8A(6LrIQ)G!}A&2j{X;el>7)Y6X|KMvt3cNXwo_$szHCZ2VBG{V=Q?`--O zBlz=v4{fr#M(x9gv2~ZdvGC6}ChKAcoe^K4bkiZ~W}m?w*`vZDwgkPsJkoXpcIpXbPk=*BY}WZtB!+N<5g9#e2_| z^yMpXKD$`u6jvr(bBw*eHw6zXf8fNHenQbu9VW6e+Q4*aC>`N^g0_C-W4iZ2H01tA zuFB>x{Mcm8K0ftw_qa< z2Tv~*mg(cbuKVw%W2Oz17@v9#3j!Ciw`?Kv_i`Zvl?m+fy`AjC0YW8-C6jw1L(dF9 zk8a!#_(wL9`}IKM1~Q@_ATG{@Uy)#GTLXa9tx`3G9Mgf$ScIJwp$;xMTC!j}a}XWsyb;m9kJv zn`cCPUI)|Tcz;IVWXiAmP<+pN_ITbTdLm#2-MP~VGq=ec0_)@ihE?D&0kENWlyntlTa_2mPv9aYqb@dtG47ktNJJMhMZL zZRoj8;oO19^K7j{C0>r4#_APa*z}mE;6b9u)JSute&r19|9g&R3j(0h`5HSTHHO74 zH=|FtRfEQ#PcZ#+8uqSx1HZ!p1(zl7z`XVwO!3YsEaABdhe}q^=!jI1-gTdhyhLdO zUw;rp^`P4k8|YX&i>*r^3;O9*EFotp%X~AN^-4yepP(P+yZgY6FQ?IPnlw8et_fEb zMdBUVk#th(6lP<$7+59sPby1t?rY0LU@F|y03vs0Wvdw9;q$@uZ6MXRYxK<(3nJ{-{sL{kM~#`dW{}9AIR4! z4XFN{2(YvcMvEi*RGfd#@kgXg+VwUv6FXj?@VArmAF-6_*QJrN;0b(1HK=m9<*|i10irklj z)}KpIA2fz~_e;P(9!Y{?sz7gVw`Y4+7Sr{=4}*zKDC<$@*HUwQsZDhylfUCi3k)9+ zsaF?>;)<(WsH_E*YilFV(iJh$JqyDsgmiqwX7H*5u8L>ly#DOUe!jF}O=l!&U_l@F zoJ=9IzlK5SE2J)ENn~}(S$07ojkE>5fSX^Psm6s`*lh2?uFT8;kv4N?Vm6$;vFQNW zm}XFH3L^C)AI-|)-Y?QGox8SuPy9>wZXXlX?rZV@=MZP#nLIguSG zA;`vkpR@RUoC>S&;x&;mnk-K z7w>%_K~({GTlqC6Yu#d2Iena*Y(7qHvSwR8&SGJ^&$HoO;pC-982CPoq=nyAXsh3A zeDckm)bX&8}akb?bPYuMSLaf!GM7c)OY4i zA>BNSW@&lASDC|fZK@Rb@0cr4d-zi*Vh~bX92o%p8}5;Lgftky6wVPZsMrN#3pMCaI=yiur*WQ zc?w{4!B3*7b&*@Tay@@73MQq0)j;oDH0&r#M9G9}=o&VhXUGa+&f6M+M1>BHo-o+cJZg?wbTh z&W!?*{14>os%yfPUDx4legtHjOy|x{C=;yTkc^LRmXW%oml$q-9wRMBkc2If!uG@- zvNk}1wjX>1W^vuP_+5w4O1y!X{w_5x**^;Abc(V0z1KiUgCTpz31RTXFcR?26lB_4 z$(4Nx7+V{Io-cLqw(m#sgq+|8)J19Sp9?5c<^$t=>@Zb(g&;zDGc2-l6_z&)k?uSK zo@5qF(43>(AZ|7jJNw@o7(d4J0wHxtuN_K<2Z zMHD*b3Jgzu!sCmLxYi?8!uiv}iNdfr;y6wnNT@QjF{&T{lpZjTA z&bbFaAS#*pFLaEby){FfECKX44P)CTZ6xwZE|{*Y zf<94$LWP=#+_K4SM0rsm9($R>ZHh2MOIVOFoBQhUy6#5`4^v&zCIH;mAM)pz2-SkA5RxqKcR(H6tI%I9I7kt*4HBZW&~Vi+nF0^4RRBs(_;!3up%_!|9=r0ZJX zv*TG<)m{PHHt)r$^F{F6r_E$S{4CnomS8M1c|A1URfMXhWjOE5S)88M4%M@#p-tLY zJUF%Vu2cy^eR9Jv-jL2oG-Pm&=1vmZi4=Qk`(@rMEP3s4{61O=w~f`ZWd-1x1^ zAXAkq;HJsJb-TN`!{r S6c^s~92DO+$vdMqj*y@RSrk3b`G2*WpJ!1-lm-0Po( z!jIwckRl#~<&Q_PIeQl3;Anu|6W<8Rrl_FiCTH@cHw0c>=UKm3dtgDm3SIj$8?#d8 zqDRFQ?s$%|0!y%$OKJ~MBEltf=Al2$iWL1FsVHV z8g6K#G|z$l>N$_p)e`PQ?^iDFpW7H{aDdC1TMkV#+R4=?y~NV?5NOEFgiU?oM7^KD zq$V4ftCS^Jki}!FFT27!-x;vX{w~Idc?e2}RvI_u7@&#qOR$l+L>wP(=k~TM(HEid zXv2H8J9iCo_Mh5NWNs|6^RR=vT*sDx$Bdbsu7CQy|2MUB7EQpM5Q{$$r-No6Ty@1IQk+9yeg>-H&CZn3;N!=s?Sj~9Gk!228r~4LW9qWR} zZRy0APdu&;_vC$+V&u?xKfd3~3Bi2N5|Vb|HYcxKN&1(Da;4+-p}zSE*_9f^Eo|WZ zDTU!M+UqQ*d8m&Z-}r>wUM$c4L?Ed+W&%S7>rv;V5m#^a6A~gcgzowy(K$7U+%A!W z^GodEd-E;)^eUJ0N|PZ|WrENp@(y@)#&TM5!|07M-e7e=nd`G}gqn+E=+=xi#CDe` z?yWb+9Gw>O^LmeP&#LLf&f-%U{OIliWpl@)Zf!04H;WU^@m~0&)fWfW&d067AP8Ttg<<-+oc4}9ZtlBv zq@_ig$Ocz~&yNBQqAP)EouTZU-PXvfi#X>i|Tp7D}BnW#Xob6a|QaA!a~2S4uPk7G4>=2aMW zyfPsl4^1PZYQwR^JdKk-o{zU{=R>CIWoRg_f{tp!mZlnl+Mo%qx!3{|k6$NSAEt2z zNoB;gu$v6eP=<)g?}F%fo-t9L%gJqCAf$0GxmBC|@o4QaPUBAn{@puX;Jr$U9Q0A4 z_9IUi4bP6|GT%OfDSl_5LOz=d78OFm>XqD3Sv0m7u7G_7{QKd2rSNz+;DTqnxu#)C zJlpvk-jh=)u6ni>2Jh5xQ!P61Vs)-?xtSJ9Rt}+6Oc>|zA&eaVY6?v~W~e*AiYV`> zAV(*KaSMOfqTX-~rbhfxn;8n%$NeUHLu+B^rZyQ;Tuk%~M{t>)V|hNOF)h})1G*(X zLU)IewD@WS7ae&C^i9;6=F@g^?90#{_m2JCrqd^gbfzftzw3!heG9RnO9M|`(_obEu9D71ml-b~c zlb?ml!Vf^#j(sQ^vkC&dro*dD|6*SYRWj@KL@uk`1?Kc$5mq1Hh0blY z_`YE`mX`V=KDdncTi=s$BQ3ajC(UrpgJ1}SOPKN2hx=`oL$>aG0a<-5T)fO4aQe}N z&t!rHXO^aMoW)tZOFHFldJchMNoOY#D7 z*|B(fxM$8p3cT!F2r z8Y2EXKA^3;BGIf?g~cXKLe+wJY;c&`B{TQWvjRd!q6~XD&I&f=m0Q zi5^FJe^avo;Phth>-VX!{?S8nYw<5qR2YcE2V`**&l3pjQ=m)s<#O|HED{vgJ>+IO zRC6~I%Q+>lOz>59HFjL8g(pPT3HN0^;%;5ohx^J41!0O5(tE4$qFoN0u_KtSw*WYq z*_it9m9S4N3{27{a92%+giXq2DBkf}V0+^iX>i{Ud3sLV2g6O|O-v~nC{`z3FP5Uv zyF(BrwHJ8_nV3UN7Xd zK>}Xv`^v4h3c%`HQ@Bg}lF;nl6ZE?Eh|3RN1+}Nrxfgx%+^G18)N6bQ-ml3e@=f7* z{=s5S%y$E^ml6e^8<7y`UWslh9hma>0SYrmlc)DJIID+ZaA=(o4E5gxW&3NOpL&8A zNy!P8o?A*1GjzF{U=d=Kbee1DU#r4ZV;KyrrFNhAI#6>)v5v@Rd@_4ArakNB{(1YI zyH&mfW}gF`(D4$tpAi>ctW1He-IH3>-Ik2~4&L0%_Ft(P9YGk%L!y=R44#28(j7s7GVN}O?`fm}v zIeA>Xe5xR##0v)!9pH+j6?oqq4^k8K;8cGU9=7GMVsjCE844D9(0IX$oCWMB<#Uj; zErh{yb;$<7Iry?i6ict&hF@Fk$g4t6R548{4vmnCszVbHVUvW$JO_joPjnspG}-fi4! z(kc8ae;=)r6uEMx(P*3>L0Zcb-ndO>b z!EMpmaL}&-|M0HVm~m>@bvTo&olwGkH0UPY3$K%$C3SEm#f@{0aO1j#H6S**l-Ff< zLH4m2!HkGV*qt21sU+uvtB)`ANqoSYJh$SPmLIOsA+R9-8D~|j#AYT|kT?kw9MCm| z=YPKoYASwURzozIKQz-39Ayiie4K@T~>;`~SPUpsa}be|8u2-nMT) z-?eawnTg2%Y$*8uxj%2zzeiAeZimA#{s8{HuUszr@56ugbo{Tq59R*-!GGKSXNSlC zIs3oZ;Kzr7_zLihdqRYc_9CXy?c5107&dolj@@A&^1`@aAl*KLLX literal 0 HcmV?d00001 diff --git a/test/integration_tests/__init__.py b/test/integration_tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/integration_tests/test_models.py b/test/integration_tests/test_models.py new file mode 100644 index 0000000000..ee811beff1 --- /dev/null +++ b/test/integration_tests/test_models.py @@ -0,0 +1,67 @@ +import torch +import torchtext + +from ..common.assets import get_asset_path +from ..common.torchtext_test_case import TorchtextTestCase + + +class TestModels(TorchtextTestCase): + def test_roberta_base(self): + asset_path = get_asset_path("roberta.base.output.pt") + test_text = "Roberta base Model Comparison" + + roberta_base = torchtext.models.ROBERTA_BASE_ENCODER + transform = roberta_base.transform() + model = roberta_base.get_model() + model = model.eval() + + model_input = torch.tensor(transform([test_text])) + actual = model(model_input) + expected = torch.load(asset_path) + torch.testing.assert_close(actual, expected) + + def test_roberta_base_jit(self): + asset_path = get_asset_path("roberta.base.output.pt") + test_text = "Roberta base Model Comparison" + + roberta_base = torchtext.models.ROBERTA_BASE_ENCODER + transform = roberta_base.transform() + transform_jit = torch.jit.script(transform) + model = roberta_base.get_model() + model = model.eval() + model_jit = torch.jit.script(model) + + model_input = torch.tensor(transform_jit([test_text])) + actual = model_jit(model_input) + expected = torch.load(asset_path) + torch.testing.assert_close(actual, expected) + + def test_roberta_large(self): + asset_path = get_asset_path("roberta.large.output.pt") + test_text = "Roberta base Model Comparison" + + roberta_large = torchtext.models.ROBERTA_LARGE_ENCODER + transform = roberta_large.transform() + model = roberta_large.get_model() + model = model.eval() + + model_input = torch.tensor(transform([test_text])) + actual = model(model_input) + expected = torch.load(asset_path) + torch.testing.assert_close(actual, expected) + + def test_roberta_large_jit(self): + asset_path = get_asset_path("roberta.large.output.pt") + test_text = "Roberta base Model Comparison" + + roberta_large = torchtext.models.ROBERTA_LARGE_ENCODER + transform = roberta_large.transform() + transform_jit = torch.jit.script(transform) + model = roberta_large.get_model() + model = model.eval() + model_jit = torch.jit.script(model) + + model_input = torch.tensor(transform_jit([test_text])) + actual = model_jit(model_input) + expected = torch.load(asset_path) + torch.testing.assert_close(actual, expected) diff --git a/torchtext/models/roberta/__init__.py b/torchtext/models/roberta/__init__.py index 1057c6deb6..79830cb2c3 100644 --- a/torchtext/models/roberta/__init__.py +++ b/torchtext/models/roberta/__init__.py @@ -8,6 +8,8 @@ RobertaModelBundle, XLMR_BASE_ENCODER, XLMR_LARGE_ENCODER, + ROBERTA_BASE_ENCODER, + ROBERTA_LARGE_ENCODER, ) __all__ = [ @@ -17,4 +19,6 @@ "RobertaModelBundle", "XLMR_BASE_ENCODER", "XLMR_LARGE_ENCODER", + "ROBERTA_BASE_ENCODER", + "ROBERTA_LARGE_ENCODER", ] diff --git a/torchtext/models/roberta/bundler.py b/torchtext/models/roberta/bundler.py index 58774aacd5..1f81750fca 100644 --- a/torchtext/models/roberta/bundler.py +++ b/torchtext/models/roberta/bundler.py @@ -15,6 +15,7 @@ RobertaModel, ) +import torchtext.transforms as T from .transforms import get_xlmr_transform from torchtext import _TEXT_BUCKET @@ -187,3 +188,61 @@ def encoderConf(self) -> RobertaEncoderConf: Please refer to :func:`torchtext.models.RobertaModelBundle` for the usage. ''' ) + + +ROBERTA_BASE_ENCODER = RobertaModelBundle( + _path=urljoin(_TEXT_BUCKET, "roberta.base.encoder.pt"), + _encoder_conf=RobertaEncoderConf(vocab_size=50265), + transform=lambda: T.Sequential( + T.GPT2BPETokenizer( + encoder_json_path=urljoin(_TEXT_BUCKET, "gpt2_bpe_encoder.json"), + vocab_bpe_path=urljoin(_TEXT_BUCKET, "gpt2_bpe_vocab.bpe"), + ), + T.VocabTransform( + load_state_dict_from_url(urljoin(_TEXT_BUCKET, "roberta.vocab.pt")) + ), + T.Truncate(510), + T.AddToken(token=0, begin=True), + T.AddToken(token=2, begin=False), + ), +) + +ROBERTA_BASE_ENCODER.__doc__ = ( + ''' + Roberta Encoder with Base configuration + + Please refer to :func:`torchtext.models.RobertaModelBundle` for the usage. + ''' +) + + +ROBERTA_LARGE_ENCODER = RobertaModelBundle( + _path=urljoin(_TEXT_BUCKET, "roberta.large.encoder.pt"), + _encoder_conf=RobertaEncoderConf( + vocab_size=50265, + embedding_dim=1024, + ffn_dimension=4096, + num_attention_heads=16, + num_encoder_layers=24, + ), + transform=lambda: T.Sequential( + T.GPT2BPETokenizer( + encoder_json_path=urljoin(_TEXT_BUCKET, "gpt2_bpe_encoder.json"), + vocab_bpe_path=urljoin(_TEXT_BUCKET, "gpt2_bpe_vocab.bpe"), + ), + T.VocabTransform( + load_state_dict_from_url(urljoin(_TEXT_BUCKET, "roberta.vocab.pt")) + ), + T.Truncate(510), + T.AddToken(token=0, begin=True), + T.AddToken(token=2, begin=False), + ), +) + +ROBERTA_LARGE_ENCODER.__doc__ = ( + ''' + Roberta Encoder with Large configuration + + Please refer to :func:`torchtext.models.RobertaModelBundle` for the usage. + ''' +) From 7650b67821a69b18dc407f0a3a14ffad38e6b68d Mon Sep 17 00:00:00 2001 From: nayef211 Date: Fri, 7 Jan 2022 09:03:33 -0800 Subject: [PATCH 2/5] Added docs for roberta encoder --- docs/source/models.rst | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/docs/source/models.rst b/docs/source/models.rst index 500b2a6c7d..b5425c8da4 100644 --- a/docs/source/models.rst +++ b/docs/source/models.rst @@ -32,3 +32,19 @@ XLMR_LARGE_ENCODER .. autodata:: XLMR_LARGE_ENCODER :no-value: +ROBERTA_BASE_ENCODER +-------------------- + +.. container:: py attribute + + .. autodata:: ROBERTA_BASE_ENCODER + :no-value: + + +ROBERTA_LARGE_ENCODER +--------------------- + +.. container:: py attribute + + .. autodata:: ROBERTA_LARGE_ENCODER + :no-value: From 3303894e148a492591d435b7d128d9b3ca7b3dce Mon Sep 17 00:00:00 2001 From: nayef211 Date: Fri, 7 Jan 2022 12:45:45 -0800 Subject: [PATCH 3/5] Updated truncate length. Added info on how model was trained along with license info --- torchtext/models/roberta/bundler.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/torchtext/models/roberta/bundler.py b/torchtext/models/roberta/bundler.py index 183adf4eed..67f4cd7f55 100644 --- a/torchtext/models/roberta/bundler.py +++ b/torchtext/models/roberta/bundler.py @@ -205,7 +205,7 @@ def encoderConf(self) -> RobertaEncoderConf: T.VocabTransform( load_state_dict_from_url(urljoin(_TEXT_BUCKET, "roberta.vocab.pt")) ), - T.Truncate(510), + T.Truncate(254), T.AddToken(token=0, begin=True), T.AddToken(token=2, begin=False), ), @@ -215,6 +215,16 @@ def encoderConf(self) -> RobertaEncoderConf: ''' Roberta Encoder with Base configuration + RoBERTa iterates on BERT's pretraining procedure, including training the model longer, + with bigger batches over more data; removing the next sentence prediction objective; + training on longer sequences; and dynamically changing the masking pattern applied + to the training data. + + Originally published by the authors of RoBERTa under MIT License + and redistributed with the same license. + [`License `__, + `Source `__] + Please refer to :func:`torchtext.models.RobertaModelBundle` for the usage. ''' ) @@ -247,6 +257,16 @@ def encoderConf(self) -> RobertaEncoderConf: ''' Roberta Encoder with Large configuration + RoBERTa iterates on BERT's pretraining procedure, including training the model longer, + with bigger batches over more data; removing the next sentence prediction objective; + training on longer sequences; and dynamically changing the masking pattern applied + to the training data. + + Originally published by the authors of RoBERTa under MIT License + and redistributed with the same license. + [`License `__, + `Source `__] + Please refer to :func:`torchtext.models.RobertaModelBundle` for the usage. ''' ) From b64fbfa334dd68c73ee56918c2baa43070730ef8 Mon Sep 17 00:00:00 2001 From: nayef211 Date: Fri, 7 Jan 2022 13:52:18 -0800 Subject: [PATCH 4/5] Added datasets that roberta was trained on --- torchtext/models/roberta/bundler.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/torchtext/models/roberta/bundler.py b/torchtext/models/roberta/bundler.py index 67f4cd7f55..0cea5e2b42 100644 --- a/torchtext/models/roberta/bundler.py +++ b/torchtext/models/roberta/bundler.py @@ -220,6 +220,10 @@ def encoderConf(self) -> RobertaEncoderConf: training on longer sequences; and dynamically changing the masking pattern applied to the training data. + The RoBERTa model was pretrained on the reunion of five datasets: BookCorpus, + English Wikipedia, CC-News, OpenWebText, and STORIES. Together theses datasets + contain over a 160GB of text. + Originally published by the authors of RoBERTa under MIT License and redistributed with the same license. [`License `__, @@ -262,6 +266,11 @@ def encoderConf(self) -> RobertaEncoderConf: training on longer sequences; and dynamically changing the masking pattern applied to the training data. + The RoBERTa model was pretrained on the reunion of five datasets: BookCorpus, + English Wikipedia, CC-News, OpenWebText, and STORIES. Together theses datasets + contain over a 160GB of text. + + Originally published by the authors of RoBERTa under MIT License and redistributed with the same license. [`License `__, From 687e7f01169f72ce49d662948d281b1f59fff5f7 Mon Sep 17 00:00:00 2001 From: nayef211 Date: Fri, 7 Jan 2022 13:52:56 -0800 Subject: [PATCH 5/5] Removing unnecessary new line --- torchtext/models/roberta/bundler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/torchtext/models/roberta/bundler.py b/torchtext/models/roberta/bundler.py index 0cea5e2b42..fcfb82dbd6 100644 --- a/torchtext/models/roberta/bundler.py +++ b/torchtext/models/roberta/bundler.py @@ -270,7 +270,6 @@ def encoderConf(self) -> RobertaEncoderConf: English Wikipedia, CC-News, OpenWebText, and STORIES. Together theses datasets contain over a 160GB of text. - Originally published by the authors of RoBERTa under MIT License and redistributed with the same license. [`License `__,