From 693c5146e338648d2bc0b01d835a2cf2423d24cc Mon Sep 17 00:00:00 2001 From: Dominik Farhan Date: Mon, 11 Nov 2024 19:14:58 +0100 Subject: [PATCH 1/9] fix(tokens): :bug: correct bug due to incorrect use of numba that slowed down TokensCutter --- src/data_processors/tokens/tokens_cutter.py | 12 ++++++---- src/run_action_gin.py | 2 +- src/tokenization/pipeline/pipelines.py | 4 +++- src/tokenization/pipeline/savers/__init__.py | 4 ++-- .../tokens/test_tokens_cutter.py | 24 +++++++++---------- 5 files changed, 26 insertions(+), 20 deletions(-) diff --git a/src/data_processors/tokens/tokens_cutter.py b/src/data_processors/tokens/tokens_cutter.py index e7ad113..7e0e279 100644 --- a/src/data_processors/tokens/tokens_cutter.py +++ b/src/data_processors/tokens/tokens_cutter.py @@ -8,7 +8,10 @@ def fast_token_mention_span(all_tokens, label_token_id): - @nb.njit + # Premature optimalization is cause of all evil and here it is so so true + # The numba njit version crashes the code at OOM, maybe not working good with multiprocessing? + # TODO: We should investigate why is it broken. + # @nb.njit def _fast_token_mention_span(all_tokens, label_token_id): mention_start_idx, mention_end_idx = None, None for i, token in enumerate(all_tokens): @@ -157,6 +160,7 @@ def _more_on_left_cut(self): ) def _warn_about_padding(self): - _logger.warning( - "Padding tokens are present in the input text. This means that input text is shorter than expected." - ) + pass + # _logger.warning( + # "Padding tokens are present in the input text. This means that input text is shorter than expected." + # ) diff --git a/src/run_action_gin.py b/src/run_action_gin.py index 29ed99b..0362c01 100644 --- a/src/run_action_gin.py +++ b/src/run_action_gin.py @@ -2,6 +2,7 @@ logging.basicConfig(level=logging.INFO) +from fire import Fire import gin import wandb @@ -17,7 +18,6 @@ embed_links_for_generation, ) from finetunings.generate_epochs.generate import generate -from fire import Fire from multilingual_dataset.combine_embs import combine_embs_by_qid from multilingual_dataset.creator import create_multilingual_dataset, run_kb_creator from tokenization.runner import ( diff --git a/src/tokenization/pipeline/pipelines.py b/src/tokenization/pipeline/pipelines.py index 87204ea..85dda81 100644 --- a/src/tokenization/pipeline/pipelines.py +++ b/src/tokenization/pipeline/pipelines.py @@ -2,7 +2,7 @@ from .loaders import DaMuELDescriptionLoader, DaMuELLinkLoader, MewsliLoader from .loggers import LoggerStep, StatisticsLogger -from .savers import NPZSaver +from .savers import NPZSaver, NPZSaverIncremental from .tokenizers import CuttingTokenizer, SimpleTokenizer @@ -102,6 +102,7 @@ def __init__( remainder: int = None, mod: int = None, require_link_wiki_origin: bool = True, + save_every: int = 1000000, ): super().__init__() self.add( @@ -115,6 +116,7 @@ def __init__( ) self.add(CuttingTokenizer(tokenizer, expected_size, label_token)) self.add(NPZSaver(output_filename, compress)) + # self.add(NPZSaverIncremental(output_filename, compress, save_every)) class DamuelLinkMentionPipeline(Pipeline): diff --git a/src/tokenization/pipeline/savers/__init__.py b/src/tokenization/pipeline/savers/__init__.py index f379a04..4e8dae9 100644 --- a/src/tokenization/pipeline/savers/__init__.py +++ b/src/tokenization/pipeline/savers/__init__.py @@ -1,3 +1,3 @@ -from .npz import NPZSaver +from .npz import NPZSaver, NPZSaverIncremental -__all__ = ["NPZSaver"] +__all__ = ["NPZSaver", "NPZSaverIncremental"] diff --git a/tests/data_processors/tokens/test_tokens_cutter.py b/tests/data_processors/tokens/test_tokens_cutter.py index 9dd02c3..7a45f57 100644 --- a/tests/data_processors/tokens/test_tokens_cutter.py +++ b/tests/data_processors/tokens/test_tokens_cutter.py @@ -80,27 +80,26 @@ def test_cut_mention_with_context_short_text( assert list(result).count(tokenizer.vocab[label_token]) == 2 def test_cut_mention_with_context_small_text_large_expected_size( - self, tokenizer: BertTokenizerFast, label_token: str, caplog + self, + tokenizer: BertTokenizerFast, + label_token: str, ): small_text = "[M]John Smith[M]" expected_size = 20 - with caplog.at_level(logging.WARNING): - tokenizer_wrapper = TokenizerWrapper(tokenizer, expected_size=expected_size) - tokens_cutter = TokensCutter( - text=small_text, - tokenizer_wrapper=tokenizer_wrapper, - expected_size=expected_size, - label_token=label_token, - ) - result = tokens_cutter.cut_mention_with_context() + tokenizer_wrapper = TokenizerWrapper(tokenizer, expected_size=expected_size) + tokens_cutter = TokensCutter( + text=small_text, + tokenizer_wrapper=tokenizer_wrapper, + expected_size=expected_size, + label_token=label_token, + ) + result = tokens_cutter.cut_mention_with_context() assert len(result) == expected_size assert tokenizer.decode([tokenizer.vocab[label_token]]) == label_token assert list(result).count(tokenizer.vocab[label_token]) == 2 - assert len(caplog.text) > 0 - def test_cut_mention_with_context_large_mention( self, tokenizer: BertTokenizerFast, label_token: str ): @@ -130,6 +129,7 @@ def test_cut_mention_with_context_large_mention( ("Hi [M]John[M].", True), ], ) + @pytest.mark.skip(reason="Not implemented") def test_warn_about_length( self, tokenizer: BertTokenizerFast, From df9fa6067c51f5f82990f87216feef04097d1f9b Mon Sep 17 00:00:00 2001 From: Dominik Farhan Date: Mon, 11 Nov 2024 19:27:56 +0100 Subject: [PATCH 2/9] docs: add png version of the loop image --- README.md | 2 +- finetuning_loop.pdf | Bin 29241 -> 0 bytes finetuning_loop.png | Bin 0 -> 14621 bytes 3 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 finetuning_loop.pdf create mode 100644 finetuning_loop.png diff --git a/README.md b/README.md index 587740c..c4cd417 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ Entity linking is a cute problem in NLP where your are tasked with connecting a ## Overview of the finetuning pipeline -![Finetuning pipeline](finetuning_loop.pdf) +![Finetuning pipeline](finetuning_loop.png) ## History diff --git a/finetuning_loop.pdf b/finetuning_loop.pdf deleted file mode 100644 index 43974133ef34d504e689533891b96d846ff91b41..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 29241 zcmXs!19YW5u-k5Jn_F{hV{3bB+qSp7wQXZ-+qQM3XYEt?x(D&ow((%G zM)_*P(Gg%HER2Q}Aj$FTS9BIBr6e0zzf@fCA0z%D+Q2#5`=(}~+M?-w_V-;UOrY{d$!CQSRW@hW+wR_H*Pb%FFZL& zzF~le^+s`Q&PwnNc!!kC}Z}s1P?bfSWujnh8coCyye4DRx zx-V$f@_nxQ_73?cy8^iEKH7k&4TknDPj9jLukTyj9-S8$VfA>N=OZiF@VEUA z@Wc8xV1O7{8*UQ{(d7d84U6JWi z`RfHll2HO?^xG>*3X1U?n{O#Pxcw3=D&e>L^g_T$4Nb~zkD7nJopOkV>!Wh#rcuy{LuT+QLv+E;+7hTg9nDOma(LiZaH zBmTAN|5^R{l7Oe;Mf(ovE*;$};e-D;v7~XoUG+sBe4cx^J2bC`{1Y`OSq-5v!p5^; zcN%1r#ptMwe}l~ABCZX8gIpO!HXxh8?qEtEk{mM$f?1rm!idmq6^C$IcwBv=&zNCT zW&6fPEGWAGAzrDlfT30?xd2W{mQgZ9PXw!{vCxP>_lKvl&#BRnqQsC9Z!U)YxZkOA zSWJ-_icB%aNp|mKNh4BbF2F5nqFrIgrqP&Y*^mNQ3q>1hh?ioFx@^GSUW+GFi#4FI z$K(2GpI-oefD_KF$c#NPi*%3A@HZ`ZNh1h}sM*lAJ(eu50EuqlT_dJgBedyQ`tfh1 z;LMQ4h|#oVK)nBqC@+=~BG3``_v8+PPHX~t)ebTFg`*|BN|C( z+&kAwhR`R_>9??$3llpOMJMqY4R0+1?X)O9BgmD$Kg4pYp$>1NwQ**UhyD;vh95g| z$mxTguBSmzrj_rv?4T3a2xOp(wj)&NcJ$P?&q2`4`t1c`o3T%)+{D(q8t zu0{|ZkLd6AnY@9g?y1=XM?F8$Rgc(Ah6azdCOVc(t<-IPHX@fWnM8ld#Da z>fD_pc!~P6O(%`KtMXl1#zNuOuWt+*p}e(;Q9o2HCAIS87IH%Hi#1~HHev*9M%>7+ zp*y7~=atU&&Z&K<`!rkO-k*5a7!a~Lz4B7Ps5)_Ctg=Fuo)q37y`|FL;;U08l|wL6 z{qoL((8-vQk3gKqBOUv!UKCvkdK7##@^h5^5r|LBi5wqMoKlQ5WUP{8g!1YOydmtf zg%D)SXpwaUD|_Zo@LMFz_1PIGM7!#sAW*8)R1bo(gsSmeGf zPq^MMfB9`RJ4}DN50RD7Z_^{RZ!pLpRuGBt7`+pbrZ^)N{#fkT`Mzd-DnW^>3**;v zr>95f2@woK9|{K%)X8*;x68K+3GmIbT@Cg+f6ukV1iGe$OgOUpMdj|u-x)(c>|k~V ze)ou!92Bh_l>UwO<95_^pT&vd8MO_tq2MR;GNxCb+9|e0l^drpmWqM#yTlK#0J8B< z=n0+|Y*+F};6=XovEB>myYLkZq_8Xw#Ujj9zg51eJWe-(sy!n{SzA>CtKG@a~stZQ8 zr1>oFoexl+_r%N#{=D?(JbIu!6D&s`1qQrI2v)wX2AvM=1|@m0dGLF`cE95;-7cLF zdbE)uJxZaGY!Q*Es26d9B{c?w4`Je+q;K**Q5X3=*d^f-#cL+GYP)bc z0Qqw(9a{GAPZAl@IC0UzpjOOOcuXUh77%S^A9#`>Z|7<_`Dw7F#`m9rT!2cFTL<2Y zUcx8b7ZM}>l!0((N7RoPKTk$EqjD39a}Q*cT%xq-XP|GZu69IZqqgGT^8xX5Cvr!@ zt-h(&TpOHL1j5k~I**_%GP!*3A{s54^Zc4w?z`svJ@`17YtVO}Kil8aNcba&t|=yk zs@w`6z;$p!!8c+;xbOkBaKyWWSu|(j2fZ%93OQ6BhjN#14UO9tLPmouwiISN%x_v$ z`Eu9;sLVni0q3m<=Pu37Y8J#AG1`u-SXb{;JUX%ehUBmDuHiSri1#?&2;6s9Jici^ zZ$NK<|5ioW_0iZB@?=|*i^QY3Y<@Hxnl4=6rI|uH9!Sqh)G&7Cd%7}^w}Knm zoR&R1#*>}502>FHC|?>>hVFt!-fP5?qju1;Z2zY(kMjfj{@?ulKbHGQp785c`jUme zN!?X`WU}HxmKKp~u_KI1HRku)cV7JDU1ckEFPM?fuZ#A9YbI>|=I>BJTLvhIUZ`Gu z1)MnYOZF`q@)b^_y}p+(h)_W$OjlpkcatuQo%@&PEzCs<5(`X#41StFY+q%(9mt(0AN<6~3elSW~~t12)+khbkw` z8h|!DnR213ZtFH2IF(+RA^gTUGrn5yB8_~`2ffLAR|r#McCvLe5qe54wWmH21-%vv zbXXeSmM;0LCjJv@+i~Rf-PY{ctX@4A?+}3&J91w=XQF`uM2|7;KlIgjf=5(HAk@h>aJL$+5P85s8FdjR#8Mxu- zx~&F)gzp<^0o3A$a3q3p_~X;KH%vsa6MJ2zU=Op07zIv951Z);oL&>`a+wy?8fv>}qI2JjmX)gD{wHv4P8lVXGhihT{tj(!Io zqB7tQuE1^WFOqK^Gk8~Zm}V{~R}1ZDPt7mzO22VviGW7S$+?#^Gc5;4nma8?hX+Rk zyDbOL=?Ejy^xU@92uhuks==TFhm%>X21eB>6;&}+hS8R&->R70o)H-%CS2w1phnuw zZ8d+IaT$QR=@G(=yyy?ecu9G3BawhA9<=aY-8ghuAxJ!{k(`nV5i_O3zrDv1sC!?A z>WYai5gDmFG~_0F-ZV5cbS0)cg!&NWTq4gUm-?S-@Paa!lSQ&rMTpD>!$7EHon+!% zD9*Bk%tjF4Lpi7eN(l66YW^9F$VyR@u27UY%ny@a`u=DfA4g42{`f0RHJ-IVOGk%F z5U;H{p5u7E5xUUMC(6vSLY~Ea5_Pi1jb4v#Jfapwn8|5YiH~{FW;_xjR`!D_<$MuE z(0%||&7MtI@l&hmrXO=6l|gsDySDgbW7yN#T(%~1MV_nB?5*{*&RASkRV|ACC%%#v z8Xb{9B{d!m0(S7CfuCM)?!Q~F4My{rf$G&ttcEd**?%Td7}P>iz8=yQ8DY(ogBDMd ztTLKB8zqzlwwkzAuuXDuvbf`kirU%V1Ju>iuGiJR<{X z7v)~jRgs&qC9GF;iPWxC$vhf4-)@W^DE$|g5dZz27tLMDe8P*UorjZeKX%TfR~*WZ zd)r8KR&(|xg$ID}R51lCi=~o>A^2`Mtaem`0h&D}se<&e2kWWR16!-mIm6nfv$1kt z|8knAqle>qU`g=o@+7c!1TE)kKllh5&{>-B)x}6rN%UH=Uw-LUhmPr3PENEHs9IBQ zt(X*xgVS3^Mp_1qjitEBT>8hB+g^Gx1)}pJq#SH}ejAMt2U}`N*_qU;N^wpNzs`)4 zmU49A>+#xTwFbXn+HPy$7z#BKxH^>T2#q@0193YQR>vF5A* z7JuaMi^!Q3<=1G!hZ-_YQ>hRVwY-xY>2o)8<9?v&{i4c_aI-_Yd8ls*C%YIqpk3%&1ci#c6PB7&@!AH<#?+L6BE8Q z&|GA_nC_tszUyPOl#b&c2IHGe*(?9^)>^2yP2(j`rX}XpNGU<%PRxK1lv7#!c{Lww zJRc}zmSVv|H7F?@HL6pNP38KcDOXNVATLE-WnR*6W=ws@Z!c{nv8Kz%-XO23Mhw^vE<5)FS9zpYf2x1ok=KR@qr z?s2Xc+RMB<{`){cT~TOD_PBO}?Kv`l|xQK`A(&*0IG648xq87^kCdBmx3+`2;t zBSU%uIL;={)ReoJAa1r6S7l47pF@LNOlxcvobnt~>D!GWV2g6jnJ?7&iCR<~k5~zE z4H<4_zp{>$Qzlq6g-v1E;zHNte1Fe8zw?5AasI^ggi`n^jdsQ7OBrjRlu&Y$^jkz} zOOXI0GQsFCdCo3Y9d)_Ffoo|ipt$W`Bk8xsn7ktRRsZ4=JddTF$_5vu0Hc6_K)|xH z|4&0rUt`do8=3)MUo&h`%9D4;Nzob~uT;DD$jCddhg`!#)+UaEhTfxrUP54jH_ehJ_FWn7xkO9Gx?^o^nXows?%0sGSoa=x zw*K;mA&d6897z7#fqVz*_z^t$M=}Cwjqg*Pq0L1PtXt^)*rj5yO|Gz+GPK{@LW(JN z8@x~MaAIL&y{C~@e#+0nyf!n)M`f^e&=fJ+ndumESf+vesnExajNoBwIwf#(rQ%&^ z*?qmyWB`<@h?J$Z_SV8tQIdV4!cjLg_|lbERB9X}P@2G1*2gkw4}#2{y{9RawH9_# zu@cI5>?R5ea#D$DEev_d_y+XN$i&EK=Bpd$dOH4o@!LPG)Tn=CoIt%d1d>aoBtW z+u0?y7gh03_Okcg@l*INOI-ikGL%;V|H#R4PzYqB8XU!NmCDSBD}d$p!ip@iC?LbK zvo=rJL^E4$DwRcQqM{&F%j^*#u(GpOn1x9##!*egfY+=o1FS_eRMiDz)ID-}EuQ9- zWLqe$I(`-mFKciW(Vk)?BfDk_5*P_{MHXj_q^1<%AfgVZ#r=5unW2IC&vs5s{@|)- zsScwdXJtG(Y;Y1GGUih%Ueugkh~F4JZHg#oGG}H5lG_OjJqLrkLRD_ioHOL@%#K~? zs3bLd@!h;m`#l{@W>cJ>;PV+A7i{fpjRfy8)#=5Cf9UWmLCH4;SIgCNO@hqp^H*SiH zWe^rXsz)PLfVnFdDz28i+NJG-hRfXP8wrHY*a`TPwX_JG0o6gr|u9`AmR`-OJzU?A+$l!8i%5j7nn62 z4j*G1G8e(n0f8!-2i|7{(X#KMoFj@3EEB%-sbP*J#_VR=0IP$pUrxB|! zs{0&veBMtMhF2bmXC8@S1)<`ZU+?WjS>0h=zNoAxk+Tb3zE<>UMcyw3=s~V_UjZz( z2TIY*e9z5Sphg~>TjGCsy?vY;A4}pJA+VN8ESeoI06W|BD_=o4-(Lv#ot6A$#5N=* zPtClv;}=R%hoZQ9e#%8ZLOVS6tZr-iMWC$vTwwQ*ki^$_5v6y(;vRN=iM9dWcVRU! z!-a{{f5R&1!(w;~e^h@VqsVp&>AksiyPE&FTP`wUI;KK*Kzuq7EqsAsYx|GtdXn}V zP(+0vUJ*Fos^$#&5(6;G>!AXYPmzekTMWC8nRpoJQWy&;5Co!L!Uuj^s`mvppqBdI0%`4 zNV6t}a*WWok503Z-$Ts6>ZneT^FCnj zb6*C=3botJgMsJHoB_QU7BpVa!1vBM2R~)<_dccD!)U1l(`&R%+jh!LgiQ_S(39jjWFnfvtkO>bHt zs!o{PPH&L%Zz(D5tkV`2IZp$iZzwsaXt}B^J<&7q(LVchn0ZI3_ATD`3HSBst?YoL zvLv~SGiJ~I!DCZXh*I;T&v^;{OA1>7)R7sPZpXLzECOXOZy4KzG~F=I!Pl7eWo?## zzCSQWkvIrh3gl5WEfuwXPWFbRN7rS?AU<6p_ROPl$@l#&$>7`8j%$n~&+*ME4>WWx z84^XoHcr7_O-V-ZEQ()F+%kJKDfMg<_d)b4w-PhOG_q)wwQ|l$um3XX0RsW#Z|7JWhBu|+ntJXD_ z#5|{fm@7SKt*sIM7p#0}?csHn%sK8)nd>~~1Ali$XGFzF$yI$Q-io%&GY9;;dWw6L z{**`E_e(RHf_3&V>PO2Njgu@d1qxSF4{?GOXFh71;M4WQ11%GB^KQ47s$S1s8X-Q7-Aqx7#F!A7%P?&4ru(g{1j(L9x+UMu#Y-2L=?CF9LB{de8)#; zQhVjJQmXO|;T5OvFy<r9Ut(8@4?#G7iI?>*@qz z0(muJ;hWqYXw)n(cc5D(y`fXDube8ExB4)TXSe#2j~9`4X^GqlWq5-t>eP6HiH)*{ zx2D=Pyzy6V9KJKoZS}iq)O3YJxBm8q&%FA6&Y%-J(a1R$G||NA_`P~|t3SF`*c)B2 zKN*HrK>I$D~)#ClisseyQ8-do1K~Km!6@- zAoJBC!rt>%J>EaOvxQv&xqDFOh;QJWR`xroe4Xf=mi7klcFSA6x~r940UdRk>#^Qf z#%Ko){xqgSCLVeRPIw^nEa~+VU;TMN`$zue{GG z_+qMnksjc%=VEWTT|jt`1c1L#?gRKoge_3u4e=c)z!w5&^7X)5|3Xl)7Oz+I311C_ zyp6uA0|Q;F<>v#=9_>p_lfpT%2Utg=|Bl$%a?XGZpAWA8e9i!! zhmJQ=4G8v}yK{J8<=Xt5`)ni;=2#KcvtKk*C#5q(cB>k z@Yz2BWPO6Dv6$J}_E7bP8u56d(ObzGc0Bt+<{l#6c)5Fs=QKV6-+?Mz0YN~OCp3F2 zg>FfL}@a<-la_BQ@|(<#vhRB*A2Bsyt&zMRR-iwxJ0`|OEEg(L)a@h zcJt0CmRbPT@WeqD!9u)~YE42waQN_%u4pKc7_y#Lk0%F+n;V+^zBeD8e4!*9mCJlQiU)+^ zaxrLM)rA#q5`;~+rGvmes+%+jI^gdG{<0=t#8VdWH)&|rMjO?8vo2BWueKwqUoDzXQAyan0Hi;v$fc74}L znGR7x7_(4iKobKIa6zum$MrdW*rdDQf3;`MiEx*2O8n_%^4^$Ro%J!>%Q`QwzQqj> z{lIsf84$4Dtjfps)%g0@^{VM8Ad=zzE&g43J($d=Q>Q+uX62s?iYSMPxJWmwm#*?U zp*w91?WWGI%~F|G8S}~yJeGywINWaoJB`e5oF&nHSA@4Cd>?$F_UWGU%B&2mAzCGb zym!&uto0k)bSKYmapd44;z=ORLka1uYQ-1tSdFKpY~iyC>|pESY-Gg654+wQl*Fm` zKdnMQ*zY+<^)x&6ZJH8LL*KhDuxVB0k`6OI1jotjK;XN1!9^Tbh?Y3%JbnN+q^$g*7U- zey#|Cupy~15^^+2E?NlzBh%pQRIKJY=AVD@!soK2weX_h zI;M^z8OO_1;77JCr6;Z%$uSwFI||zonk1}ZnP8W)Oe~>#-6Wa{GiI*Gj8=(95ud;V zZDMFq)!A^CBnB?O0J%y7?SZQuxWgxphj78 z`jSOz_8@?i%}e3XXBf&)dIU|_RxEU*WD%)+P<(QIUwoqRJ$Xn(cd}YQw2_#msdTDl z>9dxCxY$`e4kL9@;a`G~WXyu~LVTeC$ok4eU3q|pCbL(RE*GG>-P7Pw_b0N=$+oyz zcXLRbc7e7Cd`r#NRjG`yF(=x~9$l<_UQ-Vl6>{|nP-xo!S!{l(ht1R4Q{u5vb7gw9 z@gO{4rel2=g%{@4o*pD`N#!y);9`=Qg^R8m8s~b_H+Ldi!UIq2UEs@H20c3a9$jxD zAJ8Jtq79|lBU=t@F1O%lT_}1h79b~}iH0ZzY?8waPb1$))ud>YB@s@D1Hk4wN}W4w z44P*P@E-__k<4hPw>_$wRU_WrmJeGk+V}9|za<;n7!g`|rbcU7)ko7)l_GtAf1jNI zKUk36S*R>%_AzlDCBhpr!kw4 z;hVAn+D!q0tR^g0a6Vf>4H4S(3`2DE6Oq~z-t;SoJp=Ez8{1ks0|~%gzfcUZvuB^E z_L;}t+VgG+0RQef>ltzN`@mh|CV8$L9L7{c^a9_I|TEbQpculipM7gW3>VUXP|kLsxn8J2-0#KD35N9soUF*|eNlrCph! z3N+fZdS9j0AYPI$62FNTuSk~yewpmCd0c8=U zXf^9!SL7$(*E=b6EJ4fS_Th6}jyaE1)4zTJ`pkuV}|@8rD*CMNPOZ(8F+jzwQlDI@Ezy`uOx+r;)frgyafq zfl{+m#oA<*+0T}Ysk1SY?#59yJ9cNuB#D?Q3$B$LdAnqy@S&uX=~YL&Lld=398Q0u z%g%wTYsJck(J_rPl%Sl-OkNd_v2Tg#l@`0dR@TrW^A|p0f;A>>nvoRXieoDqwZ4mq z=LaqXj{Uoa$dbyhv^V|58D`tG5!a*+B%I8M9(8IZ;rVNIcg+08`u8t29%G!V7`lb& zmoszh(2;MUBEU4sPHxY#gK{gBfMt(#uUvguU94u6(0w{pR@Uq}`73jawiWrH*K-n( z9?_?ArP14P6}|*dtHz-3roeWY;i*Ej!SsH5B40t#!erE0b{BOhZ-0b{c-`C}Q7QOx z5Xw57@IC-OUTbM;-kig^7bUaG*DcRW4^^D47$-sKL!>WG-cY58N3kTMY|DgN)Mp@+ z$HK%si@HIOG;*-mYV9=Rn47p>U|+|!rnPZ&R6SeMNZ+8l-%aiAeua5kk)!Twf8cZc zJp1Uj)t85LSFW61HL1zC1O&}b~|oh)@K453G8I%wrF1mireq%p)1Kcs#KJ(lb6`opyxJF+UaJdY4+a@CoRU@v>6yB2Fih}D1T*I zz$OfDC34KQWX{;Cf*+F5?CcxW`{S1t!#bIA7GMMxG_-O6(xipE!qP{3-QuLeCw9Ud zi5-%G1QbSbiwU@*BnWA1MHEu!6&^&HdcoG5N*W=9eT|t8=af?Cw|*Q|ToswF=^kyC z{hX15se-Pu&8S0|?s8V|N1gy9W;=KjIhc~`uU-4uEXtDI@){sh{7_u3rlNp#D3yF0|?6kKAzy6%naA#XX)a35$;xb(fzM44Pff|nSC z9WAz%T_*~X;LX50_z*7?F#cl(qaCWq0)byZC`;i*^I0yJd{2vC$NZEPjs$~32$ndF zk0LjCI)9R0>8i8L! zWqJRW^`X-`;8f#eU#)3>jjKL;&<|&s-8L|9+cxu7YuIsZ*sx~XJ^jXkSqX`~srg7~+sYEI2!>Gr;)cZQF4 z&1O~r&qO4j28$dbIIfj_*i;l`=}URtxHx)@0ap=CK7CI=oqQ%+t<|Q}G!56vvU5IO z)=Y=El`pQoOGF5JYUTRd+!XUTsuG+~KRNx6N=hs@zpo3~yQ%_01~CmgU&tbZ^}L`c z5Uk%X)QucV+d0Y;b=f3J-P<*N-bvDud+@F)vC)O2R1VXBL-HG7$;>%Suez>aY=39g zdLHp2mUb;wGl?j)y;hxDTwSe|znxkIh|vni-sMUB`LPr*v)t+bE>Z*C@=~4gwNAZj zbRE&xc05gx$H<1aQN8Y$JKnLYecqn2Kkq~oW#^FKp_R9&G6v#roX9X4P-T)$>&5U6 z2r#))r`f07reW!eBa7VzewbSuxI5(D2~YcN^KxHu2|m0nJR(Vmx-;QSm@)z5CYV$? z1F^KkTSw-UQlPZ#(?NW64%KyY^@}$#Z#l0J+-9m_GPM~P_-|jck5ouUlGf=uOse?o zX@=%W2TL?nrsPld!)+bqcav{LJ0=J4M@0RKJcjqimepl*WFJ>1EsaYPFOJVz8{~k) zGGjxy>5+xCf>rLIoGjIuS_%3?(E}<#gLiu67=+ksae=|AAyiI!@LOvIos;^4X%X)r|7!_^{L7Jv6Sh? ziBl7Ld@gH@>NF~2c7QUI=0JDppZW&K><_(w0F#YYWhT@LOmP)B8aW{S_-oHG3GfhdxBWb-CSUd`%0WETnZ>)*x6K#k_@4qC2qUImQDGa zMefZ1Vk3FV2_t6Ae(U`S5sOhTLZ9Ai14)i>ZIAxSHrBG!HApcIuy@Ows-taTsMXWE z)u_6j+2K-KXLPn;XyKx*KHp@|)e}bxeeY?+r|3nbBz>s#esr)zTVa6r5>`ZuXI)6&i0ath_PXelrCH`LlRQ-zr=XA_Rb1b(GEKY^P@Q;oMzJthwJ?n$yu*Q`v)Ijn zI-;(Xw$X;6BQ=NaNz_n``!I{2;vy0EmyaU^%`PcB*EfXPwms)$cJQa8wha@{eAh2% zuftXL%vg=?I+LI(0GaWaX_o(@QMZW6!x_qA>%@I0*ow0BaQ?0D;3uOwsi(O-pAP8= zM@whmekyEN;zW*da9Hf{F`W?W)+u~cbGvQwCJQ9bT4TvuXASt zHJu0sp#c16X>}(m3Ui0m-`vyD5>22z1oEIqsRSZ`B3WT>h8Wa%kmbACiH2K-ArX=k z6Sgq?6Wk=Z7m&)4TXjQ<9vE97(kD()yYXL>R~~WJ!ck(-$vW#+uWh0X7#ED$r4gYc z$m+-#Y%X;M-rOYqp3{^@sMJkBWml$_SK<)(t(s=$4oX%&omNOyHb7dxArBl+8)p8S z|40tBX>{Zai!Xv|Ojy@Uez+@gAKOoiy~c01mR!{Jb>QIUu{)ur9dFBCn8MTokazv4 zmJIkarJ;|)guhumE?HbBKh_Uf_%KGIhrnoZ-7juD_;2$cX#~}U<-LpdU_$Gd>{0TS z>ga$v>C6u8{%%Fw$kapv5V~A7%yEo9TPuXmW_^OPv2?mDd$hj{{e)vGI;V*PO-P8T zFs?637-|9T#1aAPqW-*&u-1};*`lIpsus2xp>lPeZYfM0AAqUNsKToa_Ai;C*UMy& zE-|`BmP5bN#5RmUzV4(nu9(#|$4PWdFZT(22=&MV(JGd}h}t=9XwrFMK0HO5W;h_r<`jyx!RCjQG{%Zut+`}A(Q z>U`_)<~QtQ@qqgSs$-Jk&$GrbzKMftRnI%OhD{nth~_^<&KXgMqz_TtG8L1!>y&zn z7BM)Wg0NiPDae={vLa)$F>v00kNI zvsJ~aae~!ij*untbAsk^HQhnl%=f{9mS@%U~}QWnMV5yffp`_L(%=o5cTAd;+oj; zBzCBmXy_oj>pbd9tk?*Q0@@2q_Y=65l0))G`W2#Z$5((au5#5$l<&%czbKnD3-2YMh2v4PTAsU*RyVL~zPr4q zNB`|8hhM*a@Hn+5kjWQSz?{k#{87agy^>NzSN`Sd0|V-~!Qy&Rz+x65i8VM${P9;| z7LaE&`o<^7py5rMhY>z>lIDZ?LYl=4MkEv^?MTaza~hqISs74y82tB6>in2PLMb`G zGlxJ-6)D_2*hBnH>x)4jkw6{GEQ?tjrekHpqW7bcwVSq+;on7~lt#?yj~6S?%KT-P zfq8n43AuK&0ZdTw{COzVfq&S~alrngYwno-u`W`f5*kGOczw)!rd*-P>_0M|wep3r zKx%RFLY0Il@gVvzFJV&YdifgjPd_DDH0=$N@QrM}Jad}V@yGj(;=dD{Hl$c8P=94{ zF<6U(?IJ3Wp~AK-Q5GMahz+$ZhPOMckGa! zCQ~>?R8mc9q8PH-7m^xJFpb?F9o_C+>D?fj*w07`n@gI9FUq&5qr<}0sTFLJ#666c zYhberYUWm8_|B5sa!2P4NQ4HOz}Y<*9%u)(crw3|q*Cx1AaM~Xy(XdDRCO|{CeD|e zzXwIlH3|nx_GKCFMQxDT3`OWC`VM2ji*CiHxJ9?eHqATnTxEvRA2}?yWh+VT+{0bi zl{6!M@yFR0RzX`QmAe61UzuFbwJG9<7nQT3V(tU4nD~^tMX+0+)u=!rEYj0xrU)mv zKx^yt8Xm;{qJ%i6xi!xFMCdJI4I#HKW`{@B`RN)WvhaXN1;&y-VuUQ{K|7l~P$BSusUl^vf)f9s&^I`V;T zeyrsOlMOavjRp>S}(#O|GZ$ zb}*^Ym0V&776!gzZFF%nFpHjPf+Lsk>v==u^hAOKN#uk1T$?7_-VB1?|Iv-y|M)f% z#IzeMw6U@iO%K1GLXbr^o6m7;hUzxQAB`5AYc`7U4ns@i%A$uBJvjMfj45NHB8LMI z?7g;>im{9KXz!1gYWx1FBL<>OBabqgrT+^fn7*eO?n15Sc2W-(=gZNtHfhLS6}z~+ zani>YwA}k(q*6iU9q>g~LFYZt`^oy~bQ#12pvMonYN~<|YEtEAnnDvT4 zexRX&nSj(=D2+D*s_&Q94oUVSV%#B>okJvZI*HL!>diualSW>%Keh${*VbYirysI*GHKvlgsMGjsVx%jVbc0nj*3 z3GU3edj^(cq0uCJzobbb9ZizGzU4eF1h*hf(qmPU3VuNZ5GQDMk3U3^r6b3E2-YFd{rQ4V0&Snbt0&?27S`4TH!Z(ZJd`5V&?jwYy_W57` z2ktRwLYZ5kyF}Rsg<=$KYwz2OEG{k+aW^45O}GUq*)QTkYK8+%D{@z(Vj6*UD>^*s z*E_NJ^x`e?qk;Q7v=pWft3jtp0xMJV#nF$I>)T$BvO}x-KD+6$i?uD)dD;OZ(`4gh@481c5X)%7 z0iR(t+lohyXYnj!6TW&zbk3i$254EzBb3}|)_?oG+{X<{c-bL-7rEEYt_#cx2#+(z zPtU(zMj4zjCSdLCSOXDhsans+2q!|~|2h!{$j_DY2ys5)f%E?6*`e|qHa2236uj1E zl{T9?PGR>oP+hTL2wofS417D+zFH83XjOL9x5w+`@)(#}qSeDGH!72i=bjZV(F+B< zg@Kbfr)#JEd}a_~lRAlNB8Y8jL|FRiSHNDMMNGj2Kg%SMVXQ&Tz!o++x%X?yy1rB% zL$(0WVJ*73Ba_HIo;H_^>EM`(P_N69c-i$aQ-k_w`q{bL+Z#{li(m6^z1h&1c#2ev zQ(gdt?ZfQd^AIuz@&oe)erh6FH3$B^W9caSwd>V6cSdlZ zFUenV>9@N^6bwFKextLj~)8-xqK;s4*d2r4vCqi_P^jn9*1M5wjTJ>u`$m6W7T|(8GY3 z*X=md{Qj+WOOOAqj7Y6xX=>9;4Lu%`I}Sq{TAleK%&#Oh^-$9qtZ=K2zEt&1R*f7gt_^&Px| z;5`})$G1BvNANSz5mz%GO(Z2zG^~n2DG-5Uc&l0xqFa}ceeM1#A{LC75oiFh&~Vj^ ze-68hn;d|(%KD?l9I7uTUF+fUArujKoI7%^DHAklevMK=et?qL;Fh>RsFM4{0azt1 zK3K(jXs1bZYRc|8SJ6d0u*a`b=j$`+4sCoQHZ%(oij$KvbV+H7v!yA@y5spef;!9u zA2c0h3mSq3ua9_dhmJS1|EIaHj*6;p+oh3Iq(cT#O1g&zB~%&-VL-ZuZjc6P0Rcs& zhLA217`nUVp&RLN=+3hTAD{Pm*LuJAJL|0To%62ikGbdi)xGbT-+f>eq3xsf;9lak;4b10;qK$g`<69yYLc+CJ)#k>wy!UlC<9l!k5BGQ zHg7dwb6qui^?ZfJUwpZEwP@GZ@Tl5t9KHt2*fOMas-t`IhF97!m?GY>?288lQU9GF zN_x)RIwH3vt${IenO<3Lt;lNWq0m}gS7z0D)*sm8^*VF-bC~so^z1(&<~&_{miNddqF0pt2)bNR$6t&rU%e@8-!rgxJq39&$5~iF`AA zy~0rcvLG(G)Xat|>MOa7QOQe0jM5brXv!RUTlDek^Cc=uJhub$$YlJ9nT=8MFGtuY_V^Vy-}W6_3H%MDGXg?sgC91)U`w#w&a z6NK>1_1yKtFDxS2LgW4NO{84bNnH;O&~r!k+pqMJMG|5|LZT}AeBImANNYHY;Qi1k z0ax2^h+y|)bGa!&Ld4aPoM`b`o7Sipw;hyiqSAdkHb`VM>kBzz@(!fl%wvCe~bW;e8|Aif(aAU-a_k2y3T|D%~r$sT!-R8?u zr+I8$->=gkol@7UZ7|X9N7y>GD)s{sD)&$iF@#ZD=FrdynBf;J6K{;%`>PJGskYzF z^2p(~cNY$lrBG5@`McVf@~n`4Iwd(=6>6frLv>Ck5PQo$Rxq6lx!+^ffX8^Bo^XP_7#1YC)AZ-l^+fWqicMHcYB>cQ|E~1V3q{TLJo^({E^KI8IdGv= z_LM@oaygt0;<^)4e|%0>H#}MvSt~MnKKe7LN3*sr+I<@MBZMJA=A8#tfnJ$=jp+VO zmlq2shqmjEtZmiB$>W)Z?2n8Ix&z-bax<-P-cyxQi22(3?kOn|u>+OYd28@Nq~U)1 z*AHDNV*wiB$AV!L3~bR-W25ZRlQA0wXxKblJ{5v$1UB(7N`>9ErAfBDMBT(?g%rkE zZ4;*0owe$Q>>}|9=*C8A@X?*T43?ENM))mu6GUaQGjsmh>)I4+b%8(bS%}OkxjqtE za~%!z?A&bj@qq1NU83B}2pOkz4%d^u+!J!e^zJXpu=+`R7BnN5Va@o2KH`4jNE(Ti znN6p;`oRNN%>xZF$PNkfg}^hkZ+P~CD029_KG;^M0}eze`OPqf^qkx~!VNLB0zbr) zXq7UsC>C1IPWluy-#;`Zeb4joPHbxqn(;%PU zN`I$~BPJ9byfs9%u=wWnv8UgDD1<+rx~Hln22w@$e(6IkaRfD!SPkd#aL4>GwK)%afB}$)>wiunT{rVzqF0Hk`rIxuH0$gM_Y#M)<2HLyX+p7g@FM9J$ zKysP>?{?&vgA`&3RdOXH86H-0D3O*T!lhG=K{|A1BHNfOup1s5B zT=c1w9e;Ay=2k|VCKLJNszDSyTkFxAi~6UhmZ)TLJ&9Nw$U4RUBlb>|N(Wl-qA+Uf^gpE`RV_L^+pK7Df1M?JQ>;`_h_ zd_C5z9c{X`T3%we6L;^F=8QUiH$fNI)YjzBJ1PMd^1e*C+U3z(3uP zNiSx=wuKJAeOojr8j%+8{maZ3X0=2PHNx|ZOm9DuBz(*Jq2Hg(`h>3wTVhK}>>N*w z*A^(3^P`@GfHVS8g^kNQENeZ2?AvBv#y|Zo8zeGV6gRaYWbz+qZyR$C=>9Em=O5yQ740du{TkE z;zB0LJcKSR?qwtuzfzIL8f5B0PtcuozuLUKJOhivhGVw8qU1BEn2a4>XVAO!m&`;l zO42+Jo-+)(D!`u^QvvyR4ll z#ylR%WQdP*>@VEQlxAw>s4dioi!xQ)7mO2Zog<*y5=W?UOaCh3{QsK~7XSYm-NP@aUNt32=~`Z?p=GbMKAB(R=(i*1LS*hab#zA&N})46b~2ogp3}Y zY&O(i_V{{UEFQF*0hca+5`vSJXb&VcU^lHeXoujf*Y>Z5s7o{i^{&9 zPQ@K7(F1m`HcQ)58hpY#;x3gRHiwibSyVty~k5(%rqLdFt z><-rvRhqWZt8nYh#j3*mZT-QIAkq5j#bY~pu>^_Uk8%)KA73xeEIF|h4I58UPumX2 zw1jKzlgoJ#uh-|7BG;%#`j@x176lLTw`c25XBNpDPF7$Im)q9|FKHVtmW!{}rz-4x zhG^rXx35mjR#^;h%lj6s9KQV6;Rc)YDPgPUkZ13SEoZCZkU!EFuRoWtYq(mqoAbGx zK|~^w1>si(<8vOH#cusR^~U9!Z_QXzY8Xqn;;Q4O% zKwz_Z9%9zM>lfZ$o<32#=clA|=VCvvclX6d$2=OYch44TmB+dg+T%WiG`#XiF4$05 zy&P*8$*x!(t)m@qSqGv(YyR_sDA{1PUUabr;wjov;U(IV4zk;6!*=QS-K7P2^mjzV zNb1=KqW1r=x(5E464M|w5V1IYt}LqTSQv7;3$jz*yn;Ox@RaqvG)m$vfoGrY4i4Cr zwCzUtTwy0ujahAaP^AhcgFy#AFl5lK&yIbs#~QAdcM;h{*E>MG#z3(8OVIiCCE~X4 zVA%Cx_5k_zZcMiF)yaBXiD-ps!uoK#m)^+ezV+n(;jV+3-t~2~ zgpW(Un-5bTY#m;78#3wQ0MRE`m*D(%nty3@wx3*i4qDisE^$q+l-oO`w3&i0ba~c) zgUN}!&nSaglJyMzblD-du{vt$czgU=X$|;+!s|6&amduJoT;)4KCw{dtMldfp}wm@ zmkG{t*qrFG^3rAH-YiwhCV6pkok#2Gr{nVH-PNg$)ndlsLH=ROcBb)|M#t4W^?=sm^?~x`I{U^T+#E3m zTn@`!1AdIqdT*+2O_hISY1EKJ!}(LI^-@jU!F#C(dUA#?wmQEa{c!+0^W0fwd)#|(Y7HaRB-|PJrH6;og7%nOWtw%IWM@PT%rG#Z$==%Op zirimH(m(fan3h<(^y$9$*#&gFek;D0+-^5!HLuR;Mf;`83&wbuv(`%86wj0kgz@KU zJ*V;Z^d|=iyWZ9Z3k?hPz2f32zQR!l*AuX)&f=?~-DQZ^8Z64qc22rzps;y?sOKod zSM*i(y657=oP&Rb+ZMXC-N58ggx5)BCue1)@2=dcxlr=$*ZE?Ew=XW>E1&BnVqsmM z`n9ev;79qT>|=7r2g2WY^^)0Tvm=#iqNepVsFD#I;|pUU(I>*L#??CK1iPvb4M=fd zH2NL6L9B?&_-&0csYNh~Y8$GFL~e09r*O5h1BsZr^&>)#4%E;`xQe$FE2Cz!z~T47 zNT|M`X(LHOqKqYv8O#7+A{!izd}RQWzED@SPnhRx%C!m47{|!QQ6fO{+`_3W$X zofFf;14?oGAA#$DCA6BfG0jPn__K1-=dA-uar&zj9Y_S5CNRv2lM1ukNxdoUy(sOY z)`~4&s5>+|P}@hj-u7rGE@p?Ce6$`MYP_|HFkW zA9E;nX5v)w-6v;S`s0mm!aI3lM}SoNMSf&Nl1YD(H$B1ke~~vm9sU{)kPH79uGraz zobz|jh?_b8PZruZKem=VJdvT{wi_Hkr+Ah5DP6p4*Gv&+GL(2KJvIOcV45JOBeaXUya;wU#$iGYA$yt_WmnxJD7m{LUf?`53 z)m~tgC*<&}SkdvDs#p=1V?ha!-EGoGr3wRw-%AxL4-4lM(zQ(^Elm}O6tp`Pt~4y5 zG9`($6#uef5;)UWBA)hX5BC?qbE?9I-+v^vQFD?n6PfHycU>8DRdpiI6^fYrJYs8M zXTc>{#so!$#uPVr?#C~liCo6NfEjkD(vX{a)~$*s6uiEEpucTdkz_2&Qg+WqD`SK` z+SLZC_2RsQLM4WYF`=SO0*qCg;rJr(YkFu@*uga*Y!z8PlMM@&nz3#y7Z)$Y0WK(QzB zCh@2odNS(5b&7z4&@x6;00(DEUe2pe04%f>1kYfHhHZvKNo+sx(?JKKXH#QZE>-9c z*kl!=lR$t0jMxFr0pV9+1;!}dSUrGqf4Yfut{nKd%0mI?kWOOrZzG+f1W+}zXF+{q zX#*A~;O-lH07Iq{S zj67ZRClc{D%AY+E~74Y#geuRvFP8!jx)na=J zJ=e4lnM`-3Z4sT!&pCYrHTU5k1tLILMqWnFw-7H^6OqtDOLJsT+X953iGap--X0w` z-S`8GP|&{8zlFfd5j}(&{EOJ>`qh3zF#T%(A05C|>EA?vZ);$Wo;ck&kA)?8Ls&ba z2U1h~BEEM0YQG^Ef3^ST4nTpT@cL0S;xV%OkxJsuW2gjl-J-+TL0v2s>%B z{j?I~OFAa?#sQ*^k+cuNJeftSU-sK3n1^6#mD~oeR`Bg46MD&`&E-Gg|QzIxNkJe$B?0Sh_; zHtL)L^lo8EIs$rDzk)zH5dQm7RAbBtag~GJ z!l$AbHc+{-aPgsBox2}cM+{U9-UUiU71LO9xnH7$`}|awaLEG+k0{sg$SGZS%AK|z zK7{Q*quSgIF^h`6_+d@5UZDuNR zi!}zpWBOyOceiFIjXk`M8$b;wmdfKLZnR}F{P7n*8*DqaPogV*J2s}_13seh>+pFP z!kuo&rE~XdmeslsAinG34+j&K6QVsDZi~s!dhp4yUx|_@t`>eTp1AT_^+zkekSM=g zN7TMT9Ai#;JQH5ZKSfZdWgzgVF5~5<&=J9K zzvK?Cd;&96vmX}LK&n9n>IN!Q6|iuiNLV71B`E;YhUGyH6;=TGmA{klzs@f56~ept;PrCv zRX31&BPQ#0g(u;ugTPq!@3s~ZlO8}pN{1JJj)IF&O|eJA$tPL=OmV+_HAH&I06+%t z@X!2z^!P<#(7WhySLCAe|ir77@vDQQ1r#T}0LiQlJKJ0&5)M zLRoVPQ{lkx!$(+9>KvO?_yl`CJM=tdWCgk@ zcO!s*f0mL3q<@6ad&85!h#5dV0PZ78gNnZ&9)<~>0Xp}(WFt1(<2zgdes^1!Fi=@; zy_Ax~Lwj{6jsl~X&JYWS_g0RSg6%u|z&F$nr|EuR2~|lo6WskpEG=Q-2tKX+Ma2DT zzafC!H%q{Bq^K8*Lc=-m;y5^Y*22SI!yR1TB7ZX_ECre#LmC4q7mpL z!=42gqY#53oBnjNA;8#V|GF-)8hj*80I?%N>d%NU9Y37_5+^@Ew*mqLDMW=-fenq+ zts~Nfz;*#Nr{>>9s$^jxi4%IcfGV&-kkWLN3Arzjb3y)Y0UHF#O))|fbC&7Vde6D% z7TwF8p8Xc50;=*Y`_x%%P;*V{0V^)iOv7$>lfW3A>Kw$ic#nkwQq&6MVCHb2haX8J zbr7%=8F4uVzxbh1cWOK_zxZ;%0Q?hAj-qn)Q_&6jUB3vSz{!Kkh}M0MyUJCccxXLh z7C)0aT;TDHeTg-e9WRaaFzk;KGA}!3`ta6+aReM?=-T@-qfUa(RKbXq%C#rzRLMqe z3dz$VE6l@qUX;k4=9xW#tiIlRd+`WL-o!tGN_oE#t z(h6Kt>i`e|+gNWnO3pdb6YYi;K&nTin;A2bH)2;4=KNUoyRNc~C%w7eJ}cZ-x8$AF@||W^Y8CHunSyQnvD{b2Rf@ z{=07UBrRW@c&`P#+~hli`@@xytzRDE*~meVIZmzyDa-#qNc3}e$Gv=S(f6l7YDRoG zzlnWmYVlEZI#NVir6F!=$i9V}sz{2LcX>??vf$mb-~`ah9xd>f`c+1X;Xl={coiUq zZ^Uj0^~ut2bqo-3ztu4ym;dh$xDmTIEBq;T0YVWehW}K*0#(1&aUk{h(ri9otfFywd)*W>BrZ1RZ(Vn6vp*-iJM16xIj&jeBK!FE^+z+=AgAOeYyVjNznw?js z9yG4Ysy3bfh+Xr;Q7%O}L?g1T#4uintcSu%l?YEOv#;5Tb3UOqC}z zI$ne}BD~ODq2IQ$Ln97{cnBE`=>LMoJ|JQst!?!?oHv6E7Wk*VPl6fp@9F=($A2;3 ze>2d0te)4%;!vpa?B}GnYoVyJ>TSsURjC!~WcjhEjF`2*{P!Yui?Kam_O#^0??oCG zV^{|yhYxX!p41$r=~;b$s}ttBC0!GrUJR;f8aP;xd%e3aycpvC3WlLdr!PO;(U4q! zjlS;P=Po*SnQhzeD{^_%1f1Hb?V!Elg3vl#X29SQrQ7R;haJ2+Dc5^vc?x}=RG0Ty4P2TW4mi|jpuGh|LP=};NMP?se9O)(s9bz7@3-wn3@15DS-#Yyl^pc zzM&26?X4}14V^9RY#;E*y4sp>K+PQu?JYQL+|6!!su(&N+Bnf6pFs1%^0g@)HK{%%Pr9Vp6Guq(Qq;1_k@(iP=YSs_g$o<2<72G2tapLQqBEcJ? zC<(>Ki9#EKTUVG?u}@uLG~yg6p;(`PJubFPrFik|BE_4@GFIR(D#SQ zUD(WphZU9Er`0#A9@G^!6={~Cmci(}&2A4{S9;8YchjpIqz0cBdA<8+g*rqR8#ubt zl0DnGTQkA;Lu+of-a{lUK~^m?Tt4dCl30X2R9eR1tdTp?g6fkBfl(15%JTe!Qx0XU zy}aE|(%>k;BCdK0(QIj!lIt!!7baeIM_vlHA^a$0OmI(vxQbdq$!Z9>RtSASB*gix}+`<#* z2P?(UxnuR(3T2a|V?z2h%o_Vl?my`i4Gy3Xs@?P~3T zbM$ZO495pB%_}MsHiKzt@jote5Gc(rn8&idw|X}pleOTyy!r)$>+3Q&w=sXEf<=8h zw9&j673L(lGojdY3|08UBB~r!D=sm^b}a~ef$D;OyZSrSPdF0po!&1&KP+UWvT7Sp zEra{asaq&v;~wP$^(Bdd%@-0mq?x2^A9Nh*mi^SabwTCMY5f^$zfZ&Y7w%Tw@A%i{mb`X(;p_@$Vh*+`B-qgupGmS`1Q14?xLCG|7p&p?mL?y@D6&_CV z9ydKrbb`jb`x8!NY9e2sH1Z3~3VVS9PbwH(EDB@KUhMUdC!cPL-g{>jO>mglrgUhr zyUbgjLu*p+bp#r#!w>&gg#`S!Lh>)KZ~0YSl#QS-oNaJ9mFT#6k(I&G$(areSic@x zM#s&~byG9`dLCIa*@W$)z>c0iceFN1jCt>ennJTc-;1EW&aG6Orr0_7BRaVUC zC>QOfBtObQ6(3}l5isXk@EIF!yYJT5NGs$@&-ZQL`n3Zy7Kkxaj4v@K5EdQo^C~Q4$K9kIayH5w3d`KUYIGp7^IGVk} z*ZVO!ABWZE%Cu`>-P?6idb4$xXvpYD&Ggl>+%_z^Cy+TPBE5H15l)~FrzSb$zXa7k z9>|`ZVX z&-n>WGo%FmNa=e&KF0_AAQ?F(y3d4h<4+0cn@GtYzo6%AFqP3R6gj5JOGzF!q$%y3K5`WCwU?dI%50ajMwmR9}bPXtu6t|Rmbb!A15?nHA;D%PhA zM{;uaek5?snrt*sv)eGM>T%?28TRIMa`WZg8oFk4-OQ8uqs3{n0z=v=y0k8XI8^rP zxF?6Dv~K}*BzJF;nl6F72-xgA6;J0DO%rdiNeH(1^3tKr;-xrzd0>D|j9z13FMsIUX~vgRWUVX3c#9&8Wkot9kd zKR&hpW5wBCCoJQD^wXQ~(I}o-{7Oej{*k_w@%-DNI;WLBt-;;1!lz5K{WHVevuFG> zCYxfWN1=Ej(t)tBz6Lf zlP=^s!2{CRjLdlcxVd97A@*9$inN`37LOxSC9R3z&zg!?v)=(vGNc9o1(}Yj|QWAqT)Bv66CtU!#w0*!Wjyp|#opg|;1OF`|Y&cEiF#c|M3O@GlO|$=T4++1(Lvj0Y?vB!tVrAfqh%zW~qgjwS#A diff --git a/finetuning_loop.png b/finetuning_loop.png new file mode 100644 index 0000000000000000000000000000000000000000..a1f036295c8ee2cd122033268f88b52081a11fcd GIT binary patch literal 14621 zcmaL81yoc~wD^684yB|NX^|EQkp^K%X^}4JF6kT*WJCm!0YoIE5fJGvr39rBX%La_ z7+{G18GY}4@4dC&w^&Txb5HHFyTLdj1m|(?P-1qsOw_{r94gHVIWFvMj`?0!qBW zo7Y4pM&453wVaH0m*nbwxOpw!gh3TasmO+hTy(oJ7_Dehhte-&yOBAhw)k3)-D?yc z!8L1aMy0)@+rJf;V&r?b;;Y;1=@jYXMvKzM@?>#gd*8d5-|qtcTxHG0qomZS^(m{z zg~Lb01tChxcn*355{YvoLWNddZ&h|^!Pg71!8^J8?gL&NjFJ*xi-t+G+&F*ApVHFRN*5T$kkFaK=(K+1Q1lErXiJA%3`q+z}r4Qe%rM0tqE?a49Ht z7>+cb1k5Ds#yS?h>@N>x7Is_fLdC$J*+9hlTVcwU5F1_cGooZiwvoeMsTrT-=PrNdWNNTN? zL!oH&Pk4H6UWSO1{vN-wh5d6KMp!G2-G+4aMnf!>L*Y;20kwVjZ`qrV7+``ESE|yO z5Py_c=`?bLohq*4D|+T>PGWR>_V&^Ft=YQwno(YFjK1Hs4w1?A{V->?#@}_6Kdm)g zD-AunN2-5Sx5ulgI~IufmW_%VKlJj* z`(c`h@=6;m)iK$I>#=1W7CjYp)#r@x!0C3k4Vbl|O4cg5+mai!f51sUO1nQufn94( zpSy^vz>;b_XwSr`xsMa`z2b+-i$t>h`%+>Dt+;THaJu|CBpZ}Jr=loDnTX_y0ei{l z{rnlS3xdO`%{AZE>G|b%g4ggQ?8dO&zx`hl+)ahuY@5s6w%uWs-y=dLWhVJx+Qgag zmW<-r(tX1`-HWefo~bsnu3i$Jsn;t+(@V3%U1uWn19SN&3-^cfB3${7O-c#2nM29n z!^f}aL{enH=y6Lrb*s#q*{(t{3CB_&AL!*L52!p#Vsp$F9X(0uZ+4AQt4%9=xg!x+ zwG+%p3tLoSj~EeH(KA^Cn-|njegwGgqB#$9Mfy8g;3)&P49T|pp@<*W-?fjzN^H!$URO``aP_@PydfB5KGb}KO}RRjgc30>*Z z!c}=>pO{I)rA+uZ1kXY6K#B6m5U7vvRj!5L(dJLsP8H=ItEyKI3@4v#g>(1})F+(XFkJo<**qDcrl50EaIxx0A%bS^4Z_3DZxP)jF}i9--o4rzcBxBz$<)P17RPhA(vxRJEo*~ z5l`jA)Oi|87jgay#|QaOUj8beC^ClU-RV;>B&Ru?gbv|ki%U^yl>-h*8e%|M<~mG~ zfZO)YXL`gRlcq25oLic0inQ5O@`gCCXdXQJpm$GYeUzrqam74tk>>lCIaXCzm9+Qi zq^N5J{|2(yDhAPK1w~VTg?1mNhd=22P?cJ;i4$K!d^Y?2Ryd67H5b@L1^(LNZ`7CS z`L0SJA75NTh2SGPuuWp*llsYg$mijZ&9_?a--k&3t)iNMR zkIw{=t-MMx8~5;>dv}{&ySk9i<(1v zvaxdvG9qGD>NOa~vaQB(L`0qL*>AtPvHr7}CbvU4mA}Su0W53^7nc+PwNVbL$G(mG zF_ZTF2OjC{vy*Mbzby>SnMFf8svWM^Bsr z62(8p&iL}uT1>9)f`i)B>{ZcQ)Kkb*rAO-#>-v+9Fr`#q;aIdtn#UujLoqR{cVdA# zOycP2I0iSSQ5QWpwd$!xThs2ZileUTmk{CsCxhpFhr(?^G;BX6n#}n^KT$gRc^$d; zQ8}b7;>s6IS4w;${$dnnxWoht2sI*Q)qj@tS`LD>h3q;I;>95gVkX}v4ZWAn5N_Bg z-FU{|@K(B0T{7Cn&d4Lx+@E%fIhgQqveZ_n%X9_r_O~l>R=Y9V>2GV#>u72hW3jXF^igCq)W%m zd>Ovk&F|tf-GtfixIETApLjPm<#5uUOs--nEO@cXP{M-2+EiptxAFAz<4s$I^U#$J z6%7HEe0#1N3Zz_qYi&6Y6O3SjXu%D#&cuJ82$AgD7VXdBpVc*-#pLxIEcU{3RwoZl z`yONVLaH;0(JLg7_)Sb^`#u9@mFBXn8OyXRbC@??S;((Je5x(xV#WE&jBJmIe$Ov= zJeP5v=lA}3WE@KTN8p@&^BaHbuS+AnZr0Jgz4KtYw>8BbBI|<#Pf02%Rnp8GIGM#BbSbNe zE48!`BJLB#X9}d^^0yW{7ZS=NZMbc#(q9`y{^@cxmFd+ZEPKkQq&A}ZCil^-cdx&c zktn%1x(|*@_ttL0FmEmfqE}uto>J{^4%2aJiT=_mV)m;+n`$}t2Ht9MxCb+zD%OZk zl$Gf}Gu6`*2~<+7sU=g;)AQC|6U9tVSVIGn)dDk;M`TolrmF|_5egMnIr2!gxTT_8 z)LtaT=s+xfT_%3^aiDcF#pgmoCOJ~3hfKXV6CbD1q;Xxiv7`OT&@cJlG~X{fj8}8<-)fKDc(|EhyrIU-xCFhK zrQEW;U(adwWGyNWf;2_a;5(?Vn;1K#KQk|qdzqBcIlLbAc5y2w!OV9j@Ftim4wQsg z;W7xH&h;pa=UL4XYVgQn=BccV$+L0AI;fJbb|&N`>l7{Up7h3Me?}DeaQQ0D%Qyy^ zc7dC}12s(Atkd}U*Xn|I6UTm~OS*a6kDCSW!v))7ej%}tp-p! zfi_pJw}$z?vFa{|=W&}ki7qrm0at0}vB05wFl&MD6RDY`^N55{#gU~4$%$u;*<&UBAB!WB zDn&Yo;FGj`$c3$ z68jCt$5(VF6*81guOKpTYYAIRBPq1B-c=onP>eZYnyT)6cPawr%TfR7cY1Mp8j@J1 zLIv|dDWLPsmlo->llR|0r|U zMMaqGCU5!+dSz-LJ?*<$@;5GDM-DygpR>TA7wV2V-snlufkyFncYJwgvk@xXy^$1q zuqh&z2SjD~tteO0B$%b(kR=h6yH~Z9l9!;--5(~hUYcRco=LXSDC$aIy%$pNv*r1r zc0D(}CPu~CQ+(mYctMHiouKQw!$Z??8^C@q_R`hj-q*JyNjI;#s=;acvn+)7NAX9G zu=r#F3&cp}rAth&BihZ)$}n@B$XMUNmvUIY0JtUbZ8?P^V1hb2DqwLjcbhFI&Oz(VuIi1c$f1WtQxE(X_DBb0MQIN%U-owkRrQdZ z46iq+nNJ+EkUdYOyQsb^;?D64X1addT11+J1N9w$y@+r~NE>_J6?!F_Qv2v5AvI5l z-b05m>I{lsaRW+n2pJmmi>pPN7oYr2D}PM99o66CJ6&;Jk8t_c(;La&`Fy0(`?|F5 zmYaKw(j_~KKZAps=w+LBjqKaJ0yU#D$G5CwMq-)7g{w9vOXUtn{hFTOF+*Z&#UE$e z&Haz}wwlgPS|(1?t1>we8LKrdux)X~I7I8S|13rTO`{n3v-&Yz=Mr7(iU;1vp#_GV z#W7RVsovF)pmV9FG5EsJ74=|tV6Nt@I;}yw(mTZwGppIiSh&E67p%@rfA&rgLs?I% z)3;N5b{WAh&p=s5?-&8(hjzIO5lWU|E%&67+1p6BV!{cW5OwEIcSP$w_En^l^c?J+ z?QQdxZM`a28txSF0)Nm7Ov!jEZ|M+7QR=Ao<94y8q^sUWz0Yy&&V<$!KCK<(_Zr!s zp@*E^j>nwW=%RknYhOid!Ng#e)hI7P$b~7yIf{@)3wt8MirYeFiKx#{{pX;O_1P_^ z61?8xkc*#&#aw~VnYH!2lhbGGqI6GzFpC}2VlE!|;E?T+$#^sZ*OnzvQSr=zYgKSH zEY)DeZ@TMAwSXxP4EWp8qKvpCzo)J>+@P!6`;vicDLC;4KiNQ}blkzd?{m4`ou80v z1lFt6&r9M@e4F$HY#h2An!4A@-2c`cY#UJe9V6?W=Hu>pqLQ{x+o96S# z55h<+>nZ)^9c?z7HYYm;lP@Pe+_V<$B7oWo(R*M5 zH)?-!K#^JWjwTgz@*0u%9in(hHHqf;(OPh2`?mO-?M5DXTT8P0l-ha|Dn))RX17{a z*%}Z2JSu9KU-Vx&doM_w+C|{+YJG^7BAPaS>Jm$-|v=vRfX?r&6wO1lI78@T! z)uB=tA|J9z9+}*N7M?PIPy1O4Fmb2gE2D5VI8$#I%~6^PS>AqTnuz?iPwC*~ga@Je z%)+fNE2r|4LN5$Z^=5Wv;tFVxg{QJA7_izd33uX1@THBiFMJy>G%R_a3 zxtGn-q~k3^LRx;Cp}1PNEF$yrlujR`$=-2%tUlpzPtH`9lQAwHB;c>5->V2+`+8is z_{YThqlO`2Y=JQG7SVD>gD=E&5LiqKTrD!w1g|^LbIO%3JKK$fFesUNk}D+j;Dnq9!1)Ocp3 zw?B?qKP4plJowDo!QqHvr)$O#>oxampW*F>0-cITn#dZ{R`5UZ%OaJ%F7_o`2a;xxFO-CZy5M*p$;l9XmxaiR*zIqs<2Xr({7Y~nlr zELFLVIVfkJxY)R1d=1a2KRg@Q(n~VbgAoIv^+b*<{ZS2Q`phMcPDM?Wm;r=Xh-+nY{eB;7-N_otoWBAlA<9hD#Iqi_?`@1lxT> z0Ts@OZX!k7sicUYpn0pDhe3G;3^j!0szPa+lb><}Y;F5V+jox06(trd4~^8c>Wer{ z*)u;e#}O+E>avyRdt@NhHuMQC(|R6wsM)ts4qkm1{(|ow9s5hUv^;ul9ecV>HKGw2 z5CK%ar!{T2WE9Kld&-`PT)=x(>;jQI6dWpTKBrZV%lH(CM(vG`&4%_zZ`7}@mCX1o z1_zyZ|E%HO-w+Ma9^3TMF{~`4cf0{Q9;%O%aXgvJ)_Pk!Q-f$c|CJD%F%xV6_daZy z4MB7C>6wtfa9c_@*)w1KvsXOh$3eWKDR$Q{R+u2;`p-CP_svg%{fP_fvpWU|(dc*r zAJxt3@?S#JzI7tC4|4U`wb~F2Jb){d@S-wu#hZ@g5KrMo->J&u(2`3>*}j=M-^U2 zNN|MgU%{Q@jf-;s&4om+D=k7JM&#SmWOKw&(*cGK`U&y#zCw0;$s;eZ!gI?Ve}154 zzubCovN*sRaa+xND$hQRf^z&>IRN|cd?^AOybL*JPPTFgASp_nqcVDKfXF{{DScR{ zzK;E9Jn4#Dc{xmlNXN>D8Hwyhkrw0Ui#8L$55E|D*cSv}0x;dB8(i-Kfmjs|uTH$g zVzFo~r~c(80FYR5jE?uB1|U>8F!wDFcn@B>^AKigW;~LI+bAmu>e}LJbKX)?gHuzY zqzORyX_GEpWwV<#=3nl$tjC`k%6LJ1Bg`-cCDC;H`~I)u&8J@HXXD)O&QB&oA$mkN zH5etV_RsFQzTZ7@_r`~DP$u*BMvan_S$w7^R zBt{}OQxbpq-;^Uk_>1h806ezUDhEuX9s6j*orp`(yA9S0OYA7)b5WP&Kpz6!KyK$n@M~{Y2006YUL47}%&&i}@`{&B*aeXs^-rTon zE{+;2|N6pwZQG|4eZLi`PY}!@I+f^nlKAluc3nZ=!ec2N!p;0{Ul-I|FDdbib0RRy z!)NO6lCTOgYe+k*LJ1iV=@41os+9AlZ0+Z$Bai>h%YZmy<>dWlWcUnBAGpB)V+Q`7 z@@rY~+BrM11}kV-OdglyNPA2Hq zL;eI=*FB8ic=Ishe)xTz_wR|ax!;8-aK)}DGn4GjY4K#~QQc?=YY>(E4X9h&7Y~n? z9q1np>?25){nZ5jj>SLmp0}rFe|St!K}k9OisCad%e`@EWc=^xp#OV1d{B%qSOH$E z4h3cW_vJ*V58ypGU`>TmI96Ag*ho^=jqYzU3(bAT*GIGKN@FmIQ6Ni#Jvr}1=fXqv z{aCZOCGj~U3M%W#!P)z$^|>Pz95^8BY;2P!_eyK6c?enl4zWZ8DEvbQzP18 zhDdW!q$QiAkw4~fG20{2iV))Mv^(2MPZ0$9X}*npL0~%MgbDvT&Sj)O_-EHw9qs_md~cXp9KZWBH_?aVd}0n*HHF&o{358%>raFo>cZtNkFlH;!!oDK zuRk`D2weE2jg?1{uYXBHJN08^mbQ-b_4z(mhDYuDm@^KATz&o4{QWu+5~-pnoqc_C zR_mi-L;+XSSQ(pEll21R@T50pYcGmy{V}t>+Wme!CZdhzg6jx>U>;CawKk%ndd=^r z2NP%~df&WmaA@Ty7(cU9!B{j{J6D&(kX>O&6#J0Cc>i*1skEWl^tS6nIiqCFNKEE&TWFBR_4s%$lJIyqhDw|+5serY5&X7BYiCA{`k&})1I;64|uFxR#jan;n;xO)hSw+b2(F<;IbIecdmm-_bzvwO=O})Zk+|#< zRLn50dK-!-`LRQ_?nEIU(JUb}PVm=`|1F64k96{X??OTC>MpaZ@cY@=Em4>r4%gro z=Ng(6WlCn8zg7w6*qAjREZ|p_-~D>#FARQ^|O`h6odi z9NBs=4e>;t%c&5v=AeX%>(2CD5EJ->U-WS=+$0HEBd*AWmoLF~6x881TDof3 z>|Gqp=hf;+gX?fzP=!;|1;L;%9lfN#=0(~jo1n=)~<`&~> ze+0Ts)sKxWXhK47p_lKdEJ6}T$5v#?nq#ZIFEyKAg(}GQlVPOy0@z9 z!$x*Mwq9EbO5FYp_X!Agyn0S*nS%1F9}UT#3piDHB*vQSZ9FYN!Vlci!v3ZEFS0X; z@-7s1gevdHUC_xfDa)=3@vs6%u%LZaM!Q?%$)WARWdtXEFk5oE$bNb#`o`0^-PbU# z5H=oiC0@;oAqhNwB55n1u!8Blbn7h&R?bL{dj8vTX08@R+K}sTV|IOiZGM~ z0-9lI6)q<}qrl1x0V{Uw18#Y^04q~n$`~77{fL*HJjzJaZc*KIWm7{U=x{m7`)@Y) z05ZD-$$qM*d*Z)P5*X}x;(SxB@`+B29#-vi&wVX$^#hq-%30pdRX1%yu?7=^=44h6 z-@CRgG76en5?>9#InaUw11bodMqnea{Br|=dpWLVRr*my<32i&4`fXV$W=)SHr_AH zRy(G;Ix53Atbth-g@{>sa#7du6<-0@&tw{(jU?`0Rv7YEb;Gx@&9fY^{fgzMotm|z zTg_%qf%`@;Ie+W-)ZH&lQ-B^%*SIqVNv0ds(Z|wrzSXSVR#~}%Mbf|t)Gl*@VS#mU z3to}m%L3sCl9w4;gNr2P+IB9nAw=0GOiZ3~(DYmPeHOxUYv8iAQ7*uh)Upg9p}aLw zSJOCUlKjkHb;-B=Rj9Cj+|aKStUIFi!andn)3XH+UhC>BEJE#SKjXV*soyO*bkt7& z^v_#2@va;WDR#huN`Qtz6H$}?2KO<40s8~8D?Ylw{Hu+dnBYR#xdrZv(E%p*XmsMc zq3(_pSSbPsWTL-1i?fxhW8ff0N zm1C|b$BnQrHDh27!`^o6>}=)FdSFO4WQ+H5*_CpQ1s?HA9{B0(-7(`+|!uLOmz zF0QGpr2w2B=5VnO@3EIBVuc;0o9kvWYY{t1xBckZZ)~+Ew52z|<RwCTpb(sfpmGW5Xlmmz2uamiu<`!?5xc^oezLJ6+31X*IVtJamSOMVxP z+e@-+z()sfl=bbRG4zu&6W74lqd@WROj462-Ek1*to@V6j}-^%X?^(Jcnxyo>x&t%+NAO}Ca zq=Uh32`Kx$M*r0o*oI9%4tMT-awc)2_1ao@ z$*5-j{%Q_rGf7vUlvQaB{W4xh2q8GocQF2xR`a=(`KFLs1jQ`}&OaC?*?##=@Z|&&&w_St2d^fh-`xgE#G5H0EXkZYE7PFP={_3{# z5<@X?(o37k>WdF-E-@GrT9%woe$rJ7U>ewU6Luw1rp`LEXF%BFK|BZsYrd6W)BqqECmi#m^SxD5e1rc!pvk8~Cy$A=; zXdtAT>J=5O|77PA8PwD%)Mh7geE2n?Yz73%HMQpb=ekQ?pX$q#DSq`sGf}j`(3~Sd zhJGES7v+j)b1k>~X?MFR#GIE>IfNq{P)p zoRl77{4dq@?ISogKkN6jD`W!FkDMySK`oCRX0^dQyQO5FNQF^Lx`miDbg#vtXRBF~ zGbXh}*U^rXuM6R?TM)AHxKg~vHuRFL3>CXx^PuqBCfRSsTAJwHKfD8!zNO2$iQe55}$-0iZi!yWt^W_eqW8&7l@my<7YQK>U9uGa^}wdTC8=BS7E*?)fd-tgko|D#P!FS*vYV1 zv-tFz8JJN59Qq$WV(wL_bB3j8fDDR=Xg$CjIFO1)ubh){eUK3Z%thm@Yk-}FBA7yo z`^HxXp|u2A#uRSFb48~#(X%&NAIr7fgI1OP9B%ovE1i2)9u-|X*0U_H?o`TjAdteSRjTY zR8G2p;Secey}Mj~qs1uR&gl8ZcPG37Cp&B9GB3Owds+)nNGt57aNjl5LM|6I;N|ivs8Y*T3nT@X_A@pGQO{%=1;&R`!Or7vGm=R4da4g0z8YglIs(y8@Jw! z#R>y9%*OWOxCs46D`>4Uz)lor!hvf*NFloLseVblCb`;i3E%7=WB1OT>)eax?cvOqZok=(@Pc}-^OCwgAacIN1wR&6Q7K;s8$M^6Gmi5=!&6V;231r7^L&W+%b zQ?wExJ5;{@VBZcBA8CKI^ZJXRlg`fu=7OrD@;Y3@}veEz*-w_)pP=EZ z;OuV&{dj!&o;?*Ot$Tu}UoyS!6JD(ID<4>XfYg&L245qgy8*X~tS|g8Y9+=$rSYuf$jF=S127^O;`tQicD2v-6_y zMA7HnRA04E;d->}FD;ey6O8hP0fEXJcC;JUReE$Ek~&wg5$@4!17U&dL*Qrl;{C;t zhdCC_^3yjWbvt$Y2n$2;-~YRN84f#k{%9tuZjT1GqdW( zmA^9LSkyrA6`;yfmhz9r51+Lk2gl@b zQ9%{16@E)eo=>CfWMWf*a3VtIzW;hH?L9p= zJ1>Z1xGk_ibp1ItlIA*%vC07>WV!;%VL2CsX27Jb66OQK>iZgo0%eAP#sJ8a3J}Xf zqiGn3v~z<6b+L@ZM&U0?P5MiH%2B{;`~U6b7K=0Mkz&<(K@R(uui zwHy?fwv8XSDmGMw7-T#CRuI1n&;HRE+Ev=q?6+Ab2;v!vyRv6d45cmG;pqIc-*0iT z`!^f1xAE3Fqu{7$^o%7)=rx6&3^}quF<&daap|-z*?9UT;`A&&3O46;*7ZY^>4JXt z*L?_F9Kt#Y_*qs&hMBt$9qIvkRkU_>t$34Q{2$HZ_`7(;pZ{`jMi8O>4=xTe7wPtz zaJVRYL^p-J9w`6)_jMjteXs?60G`C+4Ll}XY?(8+EhEAUM2`YW${;Sr694?U2nr{C zBM!FcU`^;NkP~njKbGueZCn54^?wn1{KJ1k$Ofqh@X94KXo#jZ)FFDzy5qr3cE`Ea z%pachc3K0Iw~XsAfsahis3`9I3#PC^S4XdHoJsjKsCfavpahUbV5V&B6QY-_Ci?*X z1$8unG8VllJc%v)&taYY>G}Ne7xP{`r}XGWd;ZuCzp;H*|~CC2yLDd zn}KZya!YJ}U>bj;5t1Uwdm`YwQj9>dbnN6a+qwS4hhUf_qRTC?%;tIe==>?d{4Y;H zihg?}<8EbrXoV;kthMfrsmLj2*3euJnezm<%ysD&NZZ?uBV)RgfzWr;erw(?WqunD zn$3YKg$&-JPhktB5aJCNVRX{1sA0-TS4qDBMp-Y3>v?PLg)b?+kuJ}`(JMyX;ll;B zDISxbgeyAG(h_vcE!CHyQv$Mn$GW5mF-Ed>xZ)$lL0nJ?Lu&Bg3X{n}%xHrKR?p1; zlEn9${B#La5kyK>xG92S2t%EH1f~uYp|SX!;smilFrM!IQ^$b0Q zK|Tzo2R{Azw|Fss5%f?(`)=a*aCSg?|3Y07tVQ`>6j}l(k-yx|Z#+TU0WE};FL~Ei z{IlXs`F7)r)5q*UmtK)CFtSZVs=)pO1_F7;tO9<94ePNX{kR^$fOi2=#)?WpcpSc&!*C3C^ zdHT(_B8-3)`-6Ejo{sr!>Zsojo($PJ=7zSNFo2!E(ch-Hb14ux78g7%a@r zQf5{_eR%@#w$*xs%jx4xi_rMa2HR1th>Zg!NTB*Gy4G590 zLnc*cMVXa#9ZQxgtd&8>;9Qh2Lf0ie^@mJx^Gja=Ts%m679e!!?aTTJz$(CxQ_6;h zqOxnZYg8I!hR=S2_4%oN{LEK5Lq>Rk;l{U#V}c^CDBP$67Co?-HwfbK25DpWLJf;t zpx3%y`wDQ%Hb0VE{2^j)jkeV3#$`kgNSVB3(@|bD*d^wY_CK=gnfCAHQhzH(3n+PG zy0G(@q-Y;6#?|S+;PyMI@d!tz$;*+Mc@@`u8E9lB&<|}Z z=KhHr_7Ta~_E#$t|2re6OGkn}Se2TnkA8wgehbm_)1Kzd!s-5m{k_=l41gr>2`KS^ z$n5sDfz>Y{%!Hwqd#P2piy|pFpx3bC1l=T)fjscTl)3zug0HPpg#oE9Gh)alV43u< z`umHRMwyDVVvU9iH$w6=$5K*E$7t}`fx(hUz06B|gu@b=q25Ol)2lR=AJTk&Qp6-3 z_-;+&F3-tuzm{m|#R~x$Kb%(b3yTaiTgi9M3-n+lG&h-NQd7}_^bNhpuC6@Z(Gm*F zKMerMRXm9Jgs@nRXnsUi9dX#^v%)OFiICv7O~ZFJJt3Sf{vQK8XZ{}peEBeJEq)^J z%f%0v+%XnGALjh~rH$Z!L(czx{`KDp^=a7Gj@%{Yy2*10f;hYt4ZUq1dD}`@d)a~~ zNa(hZ2>)$yenAmEK@kZdAqf#--rKh&Zr^sax%%pVG;npdakLBgzZ=ktzvlxDxc;jL zZ%0?#$KH=zJ^t@DT6~B-(B=xZjfah$guboUV@G#4NE@!iFL>*2lbGZN_!?4G)P#SO Hw+#P(6sSMX literal 0 HcmV?d00001 From 57d90c1cdc78689b98be265cb31ec61feba2b8e8 Mon Sep 17 00:00:00 2001 From: Dominik Farhan Date: Tue, 12 Nov 2024 07:34:33 +0100 Subject: [PATCH 3/9] feat(tokens): :sparkles: slightly stronger description creation with additional strip of the title --- src/tokenization/pipeline/loaders/damuel.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/tokenization/pipeline/loaders/damuel.py b/src/tokenization/pipeline/loaders/damuel.py index d448977..b01822b 100644 --- a/src/tokenization/pipeline/loaders/damuel.py +++ b/src/tokenization/pipeline/loaders/damuel.py @@ -130,14 +130,14 @@ def _process_with_context( title = self._extract_title(damuel_entry) if title is None: continue - original_titled = title + original_title = title title_wrapped = self._wrap_title(title, self.label_token) description = self._extract_description(damuel_entry) if description is None: description = "" text = self.construct_text_from_title_and_description( - title_wrapped, description, original_titled + title_wrapped, description, original_title ) qid = parse_qid(damuel_entry["qid"]) @@ -175,7 +175,9 @@ def _wrap_title(self, title: str, label_token: str) -> str: def construct_text_from_title_and_description( cls, title: str, description: str, original_title: str | None = None ) -> str: - if original_title is not None and description.startswith(original_title): + if original_title is not None and description.strip().startswith( + original_title.strip() + ): return f"{title}\n{description[len(original_title):]}" return f"{title}\n{description}" From c810bf2b685cd1afc7a8a86d8df162332cb4e23e Mon Sep 17 00:00:00 2001 From: Dominik Farhan Date: Tue, 12 Nov 2024 07:36:47 +0100 Subject: [PATCH 4/9] Revert "docs: add png version of the loop image" This reverts commit df9fa6067c51f5f82990f87216feef04097d1f9b. --- README.md | 2 +- finetuning_loop.pdf | Bin 0 -> 29241 bytes finetuning_loop.png | Bin 14621 -> 0 bytes 3 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 finetuning_loop.pdf delete mode 100644 finetuning_loop.png diff --git a/README.md b/README.md index c4cd417..587740c 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ Entity linking is a cute problem in NLP where your are tasked with connecting a ## Overview of the finetuning pipeline -![Finetuning pipeline](finetuning_loop.png) +![Finetuning pipeline](finetuning_loop.pdf) ## History diff --git a/finetuning_loop.pdf b/finetuning_loop.pdf new file mode 100644 index 0000000000000000000000000000000000000000..43974133ef34d504e689533891b96d846ff91b41 GIT binary patch literal 29241 zcmXs!19YW5u-k5Jn_F{hV{3bB+qSp7wQXZ-+qQM3XYEt?x(D&ow((%G zM)_*P(Gg%HER2Q}Aj$FTS9BIBr6e0zzf@fCA0z%D+Q2#5`=(}~+M?-w_V-;UOrY{d$!CQSRW@hW+wR_H*Pb%FFZL& zzF~le^+s`Q&PwnNc!!kC}Z}s1P?bfSWujnh8coCyye4DRx zx-V$f@_nxQ_73?cy8^iEKH7k&4TknDPj9jLukTyj9-S8$VfA>N=OZiF@VEUA z@Wc8xV1O7{8*UQ{(d7d84U6JWi z`RfHll2HO?^xG>*3X1U?n{O#Pxcw3=D&e>L^g_T$4Nb~zkD7nJopOkV>!Wh#rcuy{LuT+QLv+E;+7hTg9nDOma(LiZaH zBmTAN|5^R{l7Oe;Mf(ovE*;$};e-D;v7~XoUG+sBe4cx^J2bC`{1Y`OSq-5v!p5^; zcN%1r#ptMwe}l~ABCZX8gIpO!HXxh8?qEtEk{mM$f?1rm!idmq6^C$IcwBv=&zNCT zW&6fPEGWAGAzrDlfT30?xd2W{mQgZ9PXw!{vCxP>_lKvl&#BRnqQsC9Z!U)YxZkOA zSWJ-_icB%aNp|mKNh4BbF2F5nqFrIgrqP&Y*^mNQ3q>1hh?ioFx@^GSUW+GFi#4FI z$K(2GpI-oefD_KF$c#NPi*%3A@HZ`ZNh1h}sM*lAJ(eu50EuqlT_dJgBedyQ`tfh1 z;LMQ4h|#oVK)nBqC@+=~BG3``_v8+PPHX~t)ebTFg`*|BN|C( z+&kAwhR`R_>9??$3llpOMJMqY4R0+1?X)O9BgmD$Kg4pYp$>1NwQ**UhyD;vh95g| z$mxTguBSmzrj_rv?4T3a2xOp(wj)&NcJ$P?&q2`4`t1c`o3T%)+{D(q8t zu0{|ZkLd6AnY@9g?y1=XM?F8$Rgc(Ah6azdCOVc(t<-IPHX@fWnM8ld#Da z>fD_pc!~P6O(%`KtMXl1#zNuOuWt+*p}e(;Q9o2HCAIS87IH%Hi#1~HHev*9M%>7+ zp*y7~=atU&&Z&K<`!rkO-k*5a7!a~Lz4B7Ps5)_Ctg=Fuo)q37y`|FL;;U08l|wL6 z{qoL((8-vQk3gKqBOUv!UKCvkdK7##@^h5^5r|LBi5wqMoKlQ5WUP{8g!1YOydmtf zg%D)SXpwaUD|_Zo@LMFz_1PIGM7!#sAW*8)R1bo(gsSmeGf zPq^MMfB9`RJ4}DN50RD7Z_^{RZ!pLpRuGBt7`+pbrZ^)N{#fkT`Mzd-DnW^>3**;v zr>95f2@woK9|{K%)X8*;x68K+3GmIbT@Cg+f6ukV1iGe$OgOUpMdj|u-x)(c>|k~V ze)ou!92Bh_l>UwO<95_^pT&vd8MO_tq2MR;GNxCb+9|e0l^drpmWqM#yTlK#0J8B< z=n0+|Y*+F};6=XovEB>myYLkZq_8Xw#Ujj9zg51eJWe-(sy!n{SzA>CtKG@a~stZQ8 zr1>oFoexl+_r%N#{=D?(JbIu!6D&s`1qQrI2v)wX2AvM=1|@m0dGLF`cE95;-7cLF zdbE)uJxZaGY!Q*Es26d9B{c?w4`Je+q;K**Q5X3=*d^f-#cL+GYP)bc z0Qqw(9a{GAPZAl@IC0UzpjOOOcuXUh77%S^A9#`>Z|7<_`Dw7F#`m9rT!2cFTL<2Y zUcx8b7ZM}>l!0((N7RoPKTk$EqjD39a}Q*cT%xq-XP|GZu69IZqqgGT^8xX5Cvr!@ zt-h(&TpOHL1j5k~I**_%GP!*3A{s54^Zc4w?z`svJ@`17YtVO}Kil8aNcba&t|=yk zs@w`6z;$p!!8c+;xbOkBaKyWWSu|(j2fZ%93OQ6BhjN#14UO9tLPmouwiISN%x_v$ z`Eu9;sLVni0q3m<=Pu37Y8J#AG1`u-SXb{;JUX%ehUBmDuHiSri1#?&2;6s9Jici^ zZ$NK<|5ioW_0iZB@?=|*i^QY3Y<@Hxnl4=6rI|uH9!Sqh)G&7Cd%7}^w}Knm zoR&R1#*>}502>FHC|?>>hVFt!-fP5?qju1;Z2zY(kMjfj{@?ulKbHGQp785c`jUme zN!?X`WU}HxmKKp~u_KI1HRku)cV7JDU1ckEFPM?fuZ#A9YbI>|=I>BJTLvhIUZ`Gu z1)MnYOZF`q@)b^_y}p+(h)_W$OjlpkcatuQo%@&PEzCs<5(`X#41StFY+q%(9mt(0AN<6~3elSW~~t12)+khbkw` z8h|!DnR213ZtFH2IF(+RA^gTUGrn5yB8_~`2ffLAR|r#McCvLe5qe54wWmH21-%vv zbXXeSmM;0LCjJv@+i~Rf-PY{ctX@4A?+}3&J91w=XQF`uM2|7;KlIgjf=5(HAk@h>aJL$+5P85s8FdjR#8Mxu- zx~&F)gzp<^0o3A$a3q3p_~X;KH%vsa6MJ2zU=Op07zIv951Z);oL&>`a+wy?8fv>}qI2JjmX)gD{wHv4P8lVXGhihT{tj(!Io zqB7tQuE1^WFOqK^Gk8~Zm}V{~R}1ZDPt7mzO22VviGW7S$+?#^Gc5;4nma8?hX+Rk zyDbOL=?Ejy^xU@92uhuks==TFhm%>X21eB>6;&}+hS8R&->R70o)H-%CS2w1phnuw zZ8d+IaT$QR=@G(=yyy?ecu9G3BawhA9<=aY-8ghuAxJ!{k(`nV5i_O3zrDv1sC!?A z>WYai5gDmFG~_0F-ZV5cbS0)cg!&NWTq4gUm-?S-@Paa!lSQ&rMTpD>!$7EHon+!% zD9*Bk%tjF4Lpi7eN(l66YW^9F$VyR@u27UY%ny@a`u=DfA4g42{`f0RHJ-IVOGk%F z5U;H{p5u7E5xUUMC(6vSLY~Ea5_Pi1jb4v#Jfapwn8|5YiH~{FW;_xjR`!D_<$MuE z(0%||&7MtI@l&hmrXO=6l|gsDySDgbW7yN#T(%~1MV_nB?5*{*&RASkRV|ACC%%#v z8Xb{9B{d!m0(S7CfuCM)?!Q~F4My{rf$G&ttcEd**?%Td7}P>iz8=yQ8DY(ogBDMd ztTLKB8zqzlwwkzAuuXDuvbf`kirU%V1Ju>iuGiJR<{X z7v)~jRgs&qC9GF;iPWxC$vhf4-)@W^DE$|g5dZz27tLMDe8P*UorjZeKX%TfR~*WZ zd)r8KR&(|xg$ID}R51lCi=~o>A^2`Mtaem`0h&D}se<&e2kWWR16!-mIm6nfv$1kt z|8knAqle>qU`g=o@+7c!1TE)kKllh5&{>-B)x}6rN%UH=Uw-LUhmPr3PENEHs9IBQ zt(X*xgVS3^Mp_1qjitEBT>8hB+g^Gx1)}pJq#SH}ejAMt2U}`N*_qU;N^wpNzs`)4 zmU49A>+#xTwFbXn+HPy$7z#BKxH^>T2#q@0193YQR>vF5A* z7JuaMi^!Q3<=1G!hZ-_YQ>hRVwY-xY>2o)8<9?v&{i4c_aI-_Yd8ls*C%YIqpk3%&1ci#c6PB7&@!AH<#?+L6BE8Q z&|GA_nC_tszUyPOl#b&c2IHGe*(?9^)>^2yP2(j`rX}XpNGU<%PRxK1lv7#!c{Lww zJRc}zmSVv|H7F?@HL6pNP38KcDOXNVATLE-WnR*6W=ws@Z!c{nv8Kz%-XO23Mhw^vE<5)FS9zpYf2x1ok=KR@qr z?s2Xc+RMB<{`){cT~TOD_PBO}?Kv`l|xQK`A(&*0IG648xq87^kCdBmx3+`2;t zBSU%uIL;={)ReoJAa1r6S7l47pF@LNOlxcvobnt~>D!GWV2g6jnJ?7&iCR<~k5~zE z4H<4_zp{>$Qzlq6g-v1E;zHNte1Fe8zw?5AasI^ggi`n^jdsQ7OBrjRlu&Y$^jkz} zOOXI0GQsFCdCo3Y9d)_Ffoo|ipt$W`Bk8xsn7ktRRsZ4=JddTF$_5vu0Hc6_K)|xH z|4&0rUt`do8=3)MUo&h`%9D4;Nzob~uT;DD$jCddhg`!#)+UaEhTfxrUP54jH_ehJ_FWn7xkO9Gx?^o^nXows?%0sGSoa=x zw*K;mA&d6897z7#fqVz*_z^t$M=}Cwjqg*Pq0L1PtXt^)*rj5yO|Gz+GPK{@LW(JN z8@x~MaAIL&y{C~@e#+0nyf!n)M`f^e&=fJ+ndumESf+vesnExajNoBwIwf#(rQ%&^ z*?qmyWB`<@h?J$Z_SV8tQIdV4!cjLg_|lbERB9X}P@2G1*2gkw4}#2{y{9RawH9_# zu@cI5>?R5ea#D$DEev_d_y+XN$i&EK=Bpd$dOH4o@!LPG)Tn=CoIt%d1d>aoBtW z+u0?y7gh03_Okcg@l*INOI-ikGL%;V|H#R4PzYqB8XU!NmCDSBD}d$p!ip@iC?LbK zvo=rJL^E4$DwRcQqM{&F%j^*#u(GpOn1x9##!*egfY+=o1FS_eRMiDz)ID-}EuQ9- zWLqe$I(`-mFKciW(Vk)?BfDk_5*P_{MHXj_q^1<%AfgVZ#r=5unW2IC&vs5s{@|)- zsScwdXJtG(Y;Y1GGUih%Ueugkh~F4JZHg#oGG}H5lG_OjJqLrkLRD_ioHOL@%#K~? zs3bLd@!h;m`#l{@W>cJ>;PV+A7i{fpjRfy8)#=5Cf9UWmLCH4;SIgCNO@hqp^H*SiH zWe^rXsz)PLfVnFdDz28i+NJG-hRfXP8wrHY*a`TPwX_JG0o6gr|u9`AmR`-OJzU?A+$l!8i%5j7nn62 z4j*G1G8e(n0f8!-2i|7{(X#KMoFj@3EEB%-sbP*J#_VR=0IP$pUrxB|! zs{0&veBMtMhF2bmXC8@S1)<`ZU+?WjS>0h=zNoAxk+Tb3zE<>UMcyw3=s~V_UjZz( z2TIY*e9z5Sphg~>TjGCsy?vY;A4}pJA+VN8ESeoI06W|BD_=o4-(Lv#ot6A$#5N=* zPtClv;}=R%hoZQ9e#%8ZLOVS6tZr-iMWC$vTwwQ*ki^$_5v6y(;vRN=iM9dWcVRU! z!-a{{f5R&1!(w;~e^h@VqsVp&>AksiyPE&FTP`wUI;KK*Kzuq7EqsAsYx|GtdXn}V zP(+0vUJ*Fos^$#&5(6;G>!AXYPmzekTMWC8nRpoJQWy&;5Co!L!Uuj^s`mvppqBdI0%`4 zNV6t}a*WWok503Z-$Ts6>ZneT^FCnj zb6*C=3botJgMsJHoB_QU7BpVa!1vBM2R~)<_dccD!)U1l(`&R%+jh!LgiQ_S(39jjWFnfvtkO>bHt zs!o{PPH&L%Zz(D5tkV`2IZp$iZzwsaXt}B^J<&7q(LVchn0ZI3_ATD`3HSBst?YoL zvLv~SGiJ~I!DCZXh*I;T&v^;{OA1>7)R7sPZpXLzECOXOZy4KzG~F=I!Pl7eWo?## zzCSQWkvIrh3gl5WEfuwXPWFbRN7rS?AU<6p_ROPl$@l#&$>7`8j%$n~&+*ME4>WWx z84^XoHcr7_O-V-ZEQ()F+%kJKDfMg<_d)b4w-PhOG_q)wwQ|l$um3XX0RsW#Z|7JWhBu|+ntJXD_ z#5|{fm@7SKt*sIM7p#0}?csHn%sK8)nd>~~1Ali$XGFzF$yI$Q-io%&GY9;;dWw6L z{**`E_e(RHf_3&V>PO2Njgu@d1qxSF4{?GOXFh71;M4WQ11%GB^KQ47s$S1s8X-Q7-Aqx7#F!A7%P?&4ru(g{1j(L9x+UMu#Y-2L=?CF9LB{de8)#; zQhVjJQmXO|;T5OvFy<r9Ut(8@4?#G7iI?>*@qz z0(muJ;hWqYXw)n(cc5D(y`fXDube8ExB4)TXSe#2j~9`4X^GqlWq5-t>eP6HiH)*{ zx2D=Pyzy6V9KJKoZS}iq)O3YJxBm8q&%FA6&Y%-J(a1R$G||NA_`P~|t3SF`*c)B2 zKN*HrK>I$D~)#ClisseyQ8-do1K~Km!6@- zAoJBC!rt>%J>EaOvxQv&xqDFOh;QJWR`xroe4Xf=mi7klcFSA6x~r940UdRk>#^Qf z#%Ko){xqgSCLVeRPIw^nEa~+VU;TMN`$zue{GG z_+qMnksjc%=VEWTT|jt`1c1L#?gRKoge_3u4e=c)z!w5&^7X)5|3Xl)7Oz+I311C_ zyp6uA0|Q;F<>v#=9_>p_lfpT%2Utg=|Bl$%a?XGZpAWA8e9i!! zhmJQ=4G8v}yK{J8<=Xt5`)ni;=2#KcvtKk*C#5q(cB>k z@Yz2BWPO6Dv6$J}_E7bP8u56d(ObzGc0Bt+<{l#6c)5Fs=QKV6-+?Mz0YN~OCp3F2 zg>FfL}@a<-la_BQ@|(<#vhRB*A2Bsyt&zMRR-iwxJ0`|OEEg(L)a@h zcJt0CmRbPT@WeqD!9u)~YE42waQN_%u4pKc7_y#Lk0%F+n;V+^zBeD8e4!*9mCJlQiU)+^ zaxrLM)rA#q5`;~+rGvmes+%+jI^gdG{<0=t#8VdWH)&|rMjO?8vo2BWueKwqUoDzXQAyan0Hi;v$fc74}L znGR7x7_(4iKobKIa6zum$MrdW*rdDQf3;`MiEx*2O8n_%^4^$Ro%J!>%Q`QwzQqj> z{lIsf84$4Dtjfps)%g0@^{VM8Ad=zzE&g43J($d=Q>Q+uX62s?iYSMPxJWmwm#*?U zp*w91?WWGI%~F|G8S}~yJeGywINWaoJB`e5oF&nHSA@4Cd>?$F_UWGU%B&2mAzCGb zym!&uto0k)bSKYmapd44;z=ORLka1uYQ-1tSdFKpY~iyC>|pESY-Gg654+wQl*Fm` zKdnMQ*zY+<^)x&6ZJH8LL*KhDuxVB0k`6OI1jotjK;XN1!9^Tbh?Y3%JbnN+q^$g*7U- zey#|Cupy~15^^+2E?NlzBh%pQRIKJY=AVD@!soK2weX_h zI;M^z8OO_1;77JCr6;Z%$uSwFI||zonk1}ZnP8W)Oe~>#-6Wa{GiI*Gj8=(95ud;V zZDMFq)!A^CBnB?O0J%y7?SZQuxWgxphj78 z`jSOz_8@?i%}e3XXBf&)dIU|_RxEU*WD%)+P<(QIUwoqRJ$Xn(cd}YQw2_#msdTDl z>9dxCxY$`e4kL9@;a`G~WXyu~LVTeC$ok4eU3q|pCbL(RE*GG>-P7Pw_b0N=$+oyz zcXLRbc7e7Cd`r#NRjG`yF(=x~9$l<_UQ-Vl6>{|nP-xo!S!{l(ht1R4Q{u5vb7gw9 z@gO{4rel2=g%{@4o*pD`N#!y);9`=Qg^R8m8s~b_H+Ldi!UIq2UEs@H20c3a9$jxD zAJ8Jtq79|lBU=t@F1O%lT_}1h79b~}iH0ZzY?8waPb1$))ud>YB@s@D1Hk4wN}W4w z44P*P@E-__k<4hPw>_$wRU_WrmJeGk+V}9|za<;n7!g`|rbcU7)ko7)l_GtAf1jNI zKUk36S*R>%_AzlDCBhpr!kw4 z;hVAn+D!q0tR^g0a6Vf>4H4S(3`2DE6Oq~z-t;SoJp=Ez8{1ks0|~%gzfcUZvuB^E z_L;}t+VgG+0RQef>ltzN`@mh|CV8$L9L7{c^a9_I|TEbQpculipM7gW3>VUXP|kLsxn8J2-0#KD35N9soUF*|eNlrCph! z3N+fZdS9j0AYPI$62FNTuSk~yewpmCd0c8=U zXf^9!SL7$(*E=b6EJ4fS_Th6}jyaE1)4zTJ`pkuV}|@8rD*CMNPOZ(8F+jzwQlDI@Ezy`uOx+r;)frgyafq zfl{+m#oA<*+0T}Ysk1SY?#59yJ9cNuB#D?Q3$B$LdAnqy@S&uX=~YL&Lld=398Q0u z%g%wTYsJck(J_rPl%Sl-OkNd_v2Tg#l@`0dR@TrW^A|p0f;A>>nvoRXieoDqwZ4mq z=LaqXj{Uoa$dbyhv^V|58D`tG5!a*+B%I8M9(8IZ;rVNIcg+08`u8t29%G!V7`lb& zmoszh(2;MUBEU4sPHxY#gK{gBfMt(#uUvguU94u6(0w{pR@Uq}`73jawiWrH*K-n( z9?_?ArP14P6}|*dtHz-3roeWY;i*Ej!SsH5B40t#!erE0b{BOhZ-0b{c-`C}Q7QOx z5Xw57@IC-OUTbM;-kig^7bUaG*DcRW4^^D47$-sKL!>WG-cY58N3kTMY|DgN)Mp@+ z$HK%si@HIOG;*-mYV9=Rn47p>U|+|!rnPZ&R6SeMNZ+8l-%aiAeua5kk)!Twf8cZc zJp1Uj)t85LSFW61HL1zC1O&}b~|oh)@K453G8I%wrF1mireq%p)1Kcs#KJ(lb6`opyxJF+UaJdY4+a@CoRU@v>6yB2Fih}D1T*I zz$OfDC34KQWX{;Cf*+F5?CcxW`{S1t!#bIA7GMMxG_-O6(xipE!qP{3-QuLeCw9Ud zi5-%G1QbSbiwU@*BnWA1MHEu!6&^&HdcoG5N*W=9eT|t8=af?Cw|*Q|ToswF=^kyC z{hX15se-Pu&8S0|?s8V|N1gy9W;=KjIhc~`uU-4uEXtDI@){sh{7_u3rlNp#D3yF0|?6kKAzy6%naA#XX)a35$;xb(fzM44Pff|nSC z9WAz%T_*~X;LX50_z*7?F#cl(qaCWq0)byZC`;i*^I0yJd{2vC$NZEPjs$~32$ndF zk0LjCI)9R0>8i8L! zWqJRW^`X-`;8f#eU#)3>jjKL;&<|&s-8L|9+cxu7YuIsZ*sx~XJ^jXkSqX`~srg7~+sYEI2!>Gr;)cZQF4 z&1O~r&qO4j28$dbIIfj_*i;l`=}URtxHx)@0ap=CK7CI=oqQ%+t<|Q}G!56vvU5IO z)=Y=El`pQoOGF5JYUTRd+!XUTsuG+~KRNx6N=hs@zpo3~yQ%_01~CmgU&tbZ^}L`c z5Uk%X)QucV+d0Y;b=f3J-P<*N-bvDud+@F)vC)O2R1VXBL-HG7$;>%Suez>aY=39g zdLHp2mUb;wGl?j)y;hxDTwSe|znxkIh|vni-sMUB`LPr*v)t+bE>Z*C@=~4gwNAZj zbRE&xc05gx$H<1aQN8Y$JKnLYecqn2Kkq~oW#^FKp_R9&G6v#roX9X4P-T)$>&5U6 z2r#))r`f07reW!eBa7VzewbSuxI5(D2~YcN^KxHu2|m0nJR(Vmx-;QSm@)z5CYV$? z1F^KkTSw-UQlPZ#(?NW64%KyY^@}$#Z#l0J+-9m_GPM~P_-|jck5ouUlGf=uOse?o zX@=%W2TL?nrsPld!)+bqcav{LJ0=J4M@0RKJcjqimepl*WFJ>1EsaYPFOJVz8{~k) zGGjxy>5+xCf>rLIoGjIuS_%3?(E}<#gLiu67=+ksae=|AAyiI!@LOvIos;^4X%X)r|7!_^{L7Jv6Sh? ziBl7Ld@gH@>NF~2c7QUI=0JDppZW&K><_(w0F#YYWhT@LOmP)B8aW{S_-oHG3GfhdxBWb-CSUd`%0WETnZ>)*x6K#k_@4qC2qUImQDGa zMefZ1Vk3FV2_t6Ae(U`S5sOhTLZ9Ai14)i>ZIAxSHrBG!HApcIuy@Ows-taTsMXWE z)u_6j+2K-KXLPn;XyKx*KHp@|)e}bxeeY?+r|3nbBz>s#esr)zTVa6r5>`ZuXI)6&i0ath_PXelrCH`LlRQ-zr=XA_Rb1b(GEKY^P@Q;oMzJthwJ?n$yu*Q`v)Ijn zI-;(Xw$X;6BQ=NaNz_n``!I{2;vy0EmyaU^%`PcB*EfXPwms)$cJQa8wha@{eAh2% zuftXL%vg=?I+LI(0GaWaX_o(@QMZW6!x_qA>%@I0*ow0BaQ?0D;3uOwsi(O-pAP8= zM@whmekyEN;zW*da9Hf{F`W?W)+u~cbGvQwCJQ9bT4TvuXASt zHJu0sp#c16X>}(m3Ui0m-`vyD5>22z1oEIqsRSZ`B3WT>h8Wa%kmbACiH2K-ArX=k z6Sgq?6Wk=Z7m&)4TXjQ<9vE97(kD()yYXL>R~~WJ!ck(-$vW#+uWh0X7#ED$r4gYc z$m+-#Y%X;M-rOYqp3{^@sMJkBWml$_SK<)(t(s=$4oX%&omNOyHb7dxArBl+8)p8S z|40tBX>{Zai!Xv|Ojy@Uez+@gAKOoiy~c01mR!{Jb>QIUu{)ur9dFBCn8MTokazv4 zmJIkarJ;|)guhumE?HbBKh_Uf_%KGIhrnoZ-7juD_;2$cX#~}U<-LpdU_$Gd>{0TS z>ga$v>C6u8{%%Fw$kapv5V~A7%yEo9TPuXmW_^OPv2?mDd$hj{{e)vGI;V*PO-P8T zFs?637-|9T#1aAPqW-*&u-1};*`lIpsus2xp>lPeZYfM0AAqUNsKToa_Ai;C*UMy& zE-|`BmP5bN#5RmUzV4(nu9(#|$4PWdFZT(22=&MV(JGd}h}t=9XwrFMK0HO5W;h_r<`jyx!RCjQG{%Zut+`}A(Q z>U`_)<~QtQ@qqgSs$-Jk&$GrbzKMftRnI%OhD{nth~_^<&KXgMqz_TtG8L1!>y&zn z7BM)Wg0NiPDae={vLa)$F>v00kNI zvsJ~aae~!ij*untbAsk^HQhnl%=f{9mS@%U~}QWnMV5yffp`_L(%=o5cTAd;+oj; zBzCBmXy_oj>pbd9tk?*Q0@@2q_Y=65l0))G`W2#Z$5((au5#5$l<&%czbKnD3-2YMh2v4PTAsU*RyVL~zPr4q zNB`|8hhM*a@Hn+5kjWQSz?{k#{87agy^>NzSN`Sd0|V-~!Qy&Rz+x65i8VM${P9;| z7LaE&`o<^7py5rMhY>z>lIDZ?LYl=4MkEv^?MTaza~hqISs74y82tB6>in2PLMb`G zGlxJ-6)D_2*hBnH>x)4jkw6{GEQ?tjrekHpqW7bcwVSq+;on7~lt#?yj~6S?%KT-P zfq8n43AuK&0ZdTw{COzVfq&S~alrngYwno-u`W`f5*kGOczw)!rd*-P>_0M|wep3r zKx%RFLY0Il@gVvzFJV&YdifgjPd_DDH0=$N@QrM}Jad}V@yGj(;=dD{Hl$c8P=94{ zF<6U(?IJ3Wp~AK-Q5GMahz+$ZhPOMckGa! zCQ~>?R8mc9q8PH-7m^xJFpb?F9o_C+>D?fj*w07`n@gI9FUq&5qr<}0sTFLJ#666c zYhberYUWm8_|B5sa!2P4NQ4HOz}Y<*9%u)(crw3|q*Cx1AaM~Xy(XdDRCO|{CeD|e zzXwIlH3|nx_GKCFMQxDT3`OWC`VM2ji*CiHxJ9?eHqATnTxEvRA2}?yWh+VT+{0bi zl{6!M@yFR0RzX`QmAe61UzuFbwJG9<7nQT3V(tU4nD~^tMX+0+)u=!rEYj0xrU)mv zKx^yt8Xm;{qJ%i6xi!xFMCdJI4I#HKW`{@B`RN)WvhaXN1;&y-VuUQ{K|7l~P$BSusUl^vf)f9s&^I`V;T zeyrsOlMOavjRp>S}(#O|GZ$ zb}*^Ym0V&776!gzZFF%nFpHjPf+Lsk>v==u^hAOKN#uk1T$?7_-VB1?|Iv-y|M)f% z#IzeMw6U@iO%K1GLXbr^o6m7;hUzxQAB`5AYc`7U4ns@i%A$uBJvjMfj45NHB8LMI z?7g;>im{9KXz!1gYWx1FBL<>OBabqgrT+^fn7*eO?n15Sc2W-(=gZNtHfhLS6}z~+ zani>YwA}k(q*6iU9q>g~LFYZt`^oy~bQ#12pvMonYN~<|YEtEAnnDvT4 zexRX&nSj(=D2+D*s_&Q94oUVSV%#B>okJvZI*HL!>diualSW>%Keh${*VbYirysI*GHKvlgsMGjsVx%jVbc0nj*3 z3GU3edj^(cq0uCJzobbb9ZizGzU4eF1h*hf(qmPU3VuNZ5GQDMk3U3^r6b3E2-YFd{rQ4V0&Snbt0&?27S`4TH!Z(ZJd`5V&?jwYy_W57` z2ktRwLYZ5kyF}Rsg<=$KYwz2OEG{k+aW^45O}GUq*)QTkYK8+%D{@z(Vj6*UD>^*s z*E_NJ^x`e?qk;Q7v=pWft3jtp0xMJV#nF$I>)T$BvO}x-KD+6$i?uD)dD;OZ(`4gh@481c5X)%7 z0iR(t+lohyXYnj!6TW&zbk3i$254EzBb3}|)_?oG+{X<{c-bL-7rEEYt_#cx2#+(z zPtU(zMj4zjCSdLCSOXDhsans+2q!|~|2h!{$j_DY2ys5)f%E?6*`e|qHa2236uj1E zl{T9?PGR>oP+hTL2wofS417D+zFH83XjOL9x5w+`@)(#}qSeDGH!72i=bjZV(F+B< zg@Kbfr)#JEd}a_~lRAlNB8Y8jL|FRiSHNDMMNGj2Kg%SMVXQ&Tz!o++x%X?yy1rB% zL$(0WVJ*73Ba_HIo;H_^>EM`(P_N69c-i$aQ-k_w`q{bL+Z#{li(m6^z1h&1c#2ev zQ(gdt?ZfQd^AIuz@&oe)erh6FH3$B^W9caSwd>V6cSdlZ zFUenV>9@N^6bwFKextLj~)8-xqK;s4*d2r4vCqi_P^jn9*1M5wjTJ>u`$m6W7T|(8GY3 z*X=md{Qj+WOOOAqj7Y6xX=>9;4Lu%`I}Sq{TAleK%&#Oh^-$9qtZ=K2zEt&1R*f7gt_^&Px| z;5`})$G1BvNANSz5mz%GO(Z2zG^~n2DG-5Uc&l0xqFa}ceeM1#A{LC75oiFh&~Vj^ ze-68hn;d|(%KD?l9I7uTUF+fUArujKoI7%^DHAklevMK=et?qL;Fh>RsFM4{0azt1 zK3K(jXs1bZYRc|8SJ6d0u*a`b=j$`+4sCoQHZ%(oij$KvbV+H7v!yA@y5spef;!9u zA2c0h3mSq3ua9_dhmJS1|EIaHj*6;p+oh3Iq(cT#O1g&zB~%&-VL-ZuZjc6P0Rcs& zhLA217`nUVp&RLN=+3hTAD{Pm*LuJAJL|0To%62ikGbdi)xGbT-+f>eq3xsf;9lak;4b10;qK$g`<69yYLc+CJ)#k>wy!UlC<9l!k5BGQ zHg7dwb6qui^?ZfJUwpZEwP@GZ@Tl5t9KHt2*fOMas-t`IhF97!m?GY>?288lQU9GF zN_x)RIwH3vt${IenO<3Lt;lNWq0m}gS7z0D)*sm8^*VF-bC~so^z1(&<~&_{miNddqF0pt2)bNR$6t&rU%e@8-!rgxJq39&$5~iF`AA zy~0rcvLG(G)Xat|>MOa7QOQe0jM5brXv!RUTlDek^Cc=uJhub$$YlJ9nT=8MFGtuY_V^Vy-}W6_3H%MDGXg?sgC91)U`w#w&a z6NK>1_1yKtFDxS2LgW4NO{84bNnH;O&~r!k+pqMJMG|5|LZT}AeBImANNYHY;Qi1k z0ax2^h+y|)bGa!&Ld4aPoM`b`o7Sipw;hyiqSAdkHb`VM>kBzz@(!fl%wvCe~bW;e8|Aif(aAU-a_k2y3T|D%~r$sT!-R8?u zr+I8$->=gkol@7UZ7|X9N7y>GD)s{sD)&$iF@#ZD=FrdynBf;J6K{;%`>PJGskYzF z^2p(~cNY$lrBG5@`McVf@~n`4Iwd(=6>6frLv>Ck5PQo$Rxq6lx!+^ffX8^Bo^XP_7#1YC)AZ-l^+fWqicMHcYB>cQ|E~1V3q{TLJo^({E^KI8IdGv= z_LM@oaygt0;<^)4e|%0>H#}MvSt~MnKKe7LN3*sr+I<@MBZMJA=A8#tfnJ$=jp+VO zmlq2shqmjEtZmiB$>W)Z?2n8Ix&z-bax<-P-cyxQi22(3?kOn|u>+OYd28@Nq~U)1 z*AHDNV*wiB$AV!L3~bR-W25ZRlQA0wXxKblJ{5v$1UB(7N`>9ErAfBDMBT(?g%rkE zZ4;*0owe$Q>>}|9=*C8A@X?*T43?ENM))mu6GUaQGjsmh>)I4+b%8(bS%}OkxjqtE za~%!z?A&bj@qq1NU83B}2pOkz4%d^u+!J!e^zJXpu=+`R7BnN5Va@o2KH`4jNE(Ti znN6p;`oRNN%>xZF$PNkfg}^hkZ+P~CD029_KG;^M0}eze`OPqf^qkx~!VNLB0zbr) zXq7UsC>C1IPWluy-#;`Zeb4joPHbxqn(;%PU zN`I$~BPJ9byfs9%u=wWnv8UgDD1<+rx~Hln22w@$e(6IkaRfD!SPkd#aL4>GwK)%afB}$)>wiunT{rVzqF0Hk`rIxuH0$gM_Y#M)<2HLyX+p7g@FM9J$ zKysP>?{?&vgA`&3RdOXH86H-0D3O*T!lhG=K{|A1BHNfOup1s5B zT=c1w9e;Ay=2k|VCKLJNszDSyTkFxAi~6UhmZ)TLJ&9Nw$U4RUBlb>|N(Wl-qA+Uf^gpE`RV_L^+pK7Df1M?JQ>;`_h_ zd_C5z9c{X`T3%we6L;^F=8QUiH$fNI)YjzBJ1PMd^1e*C+U3z(3uP zNiSx=wuKJAeOojr8j%+8{maZ3X0=2PHNx|ZOm9DuBz(*Jq2Hg(`h>3wTVhK}>>N*w z*A^(3^P`@GfHVS8g^kNQENeZ2?AvBv#y|Zo8zeGV6gRaYWbz+qZyR$C=>9Em=O5yQ740du{TkE z;zB0LJcKSR?qwtuzfzIL8f5B0PtcuozuLUKJOhivhGVw8qU1BEn2a4>XVAO!m&`;l zO42+Jo-+)(D!`u^QvvyR4ll z#ylR%WQdP*>@VEQlxAw>s4dioi!xQ)7mO2Zog<*y5=W?UOaCh3{QsK~7XSYm-NP@aUNt32=~`Z?p=GbMKAB(R=(i*1LS*hab#zA&N})46b~2ogp3}Y zY&O(i_V{{UEFQF*0hca+5`vSJXb&VcU^lHeXoujf*Y>Z5s7o{i^{&9 zPQ@K7(F1m`HcQ)58hpY#;x3gRHiwibSyVty~k5(%rqLdFt z><-rvRhqWZt8nYh#j3*mZT-QIAkq5j#bY~pu>^_Uk8%)KA73xeEIF|h4I58UPumX2 zw1jKzlgoJ#uh-|7BG;%#`j@x176lLTw`c25XBNpDPF7$Im)q9|FKHVtmW!{}rz-4x zhG^rXx35mjR#^;h%lj6s9KQV6;Rc)YDPgPUkZ13SEoZCZkU!EFuRoWtYq(mqoAbGx zK|~^w1>si(<8vOH#cusR^~U9!Z_QXzY8Xqn;;Q4O% zKwz_Z9%9zM>lfZ$o<32#=clA|=VCvvclX6d$2=OYch44TmB+dg+T%WiG`#XiF4$05 zy&P*8$*x!(t)m@qSqGv(YyR_sDA{1PUUabr;wjov;U(IV4zk;6!*=QS-K7P2^mjzV zNb1=KqW1r=x(5E464M|w5V1IYt}LqTSQv7;3$jz*yn;Ox@RaqvG)m$vfoGrY4i4Cr zwCzUtTwy0ujahAaP^AhcgFy#AFl5lK&yIbs#~QAdcM;h{*E>MG#z3(8OVIiCCE~X4 zVA%Cx_5k_zZcMiF)yaBXiD-ps!uoK#m)^+ezV+n(;jV+3-t~2~ zgpW(Un-5bTY#m;78#3wQ0MRE`m*D(%nty3@wx3*i4qDisE^$q+l-oO`w3&i0ba~c) zgUN}!&nSaglJyMzblD-du{vt$czgU=X$|;+!s|6&amduJoT;)4KCw{dtMldfp}wm@ zmkG{t*qrFG^3rAH-YiwhCV6pkok#2Gr{nVH-PNg$)ndlsLH=ROcBb)|M#t4W^?=sm^?~x`I{U^T+#E3m zTn@`!1AdIqdT*+2O_hISY1EKJ!}(LI^-@jU!F#C(dUA#?wmQEa{c!+0^W0fwd)#|(Y7HaRB-|PJrH6;og7%nOWtw%IWM@PT%rG#Z$==%Op zirimH(m(fan3h<(^y$9$*#&gFek;D0+-^5!HLuR;Mf;`83&wbuv(`%86wj0kgz@KU zJ*V;Z^d|=iyWZ9Z3k?hPz2f32zQR!l*AuX)&f=?~-DQZ^8Z64qc22rzps;y?sOKod zSM*i(y657=oP&Rb+ZMXC-N58ggx5)BCue1)@2=dcxlr=$*ZE?Ew=XW>E1&BnVqsmM z`n9ev;79qT>|=7r2g2WY^^)0Tvm=#iqNepVsFD#I;|pUU(I>*L#??CK1iPvb4M=fd zH2NL6L9B?&_-&0csYNh~Y8$GFL~e09r*O5h1BsZr^&>)#4%E;`xQe$FE2Cz!z~T47 zNT|M`X(LHOqKqYv8O#7+A{!izd}RQWzED@SPnhRx%C!m47{|!QQ6fO{+`_3W$X zofFf;14?oGAA#$DCA6BfG0jPn__K1-=dA-uar&zj9Y_S5CNRv2lM1ukNxdoUy(sOY z)`~4&s5>+|P}@hj-u7rGE@p?Ce6$`MYP_|HFkW zA9E;nX5v)w-6v;S`s0mm!aI3lM}SoNMSf&Nl1YD(H$B1ke~~vm9sU{)kPH79uGraz zobz|jh?_b8PZruZKem=VJdvT{wi_Hkr+Ah5DP6p4*Gv&+GL(2KJvIOcV45JOBeaXUya;wU#$iGYA$yt_WmnxJD7m{LUf?`53 z)m~tgC*<&}SkdvDs#p=1V?ha!-EGoGr3wRw-%AxL4-4lM(zQ(^Elm}O6tp`Pt~4y5 zG9`($6#uef5;)UWBA)hX5BC?qbE?9I-+v^vQFD?n6PfHycU>8DRdpiI6^fYrJYs8M zXTc>{#so!$#uPVr?#C~liCo6NfEjkD(vX{a)~$*s6uiEEpucTdkz_2&Qg+WqD`SK` z+SLZC_2RsQLM4WYF`=SO0*qCg;rJr(YkFu@*uga*Y!z8PlMM@&nz3#y7Z)$Y0WK(QzB zCh@2odNS(5b&7z4&@x6;00(DEUe2pe04%f>1kYfHhHZvKNo+sx(?JKKXH#QZE>-9c z*kl!=lR$t0jMxFr0pV9+1;!}dSUrGqf4Yfut{nKd%0mI?kWOOrZzG+f1W+}zXF+{q zX#*A~;O-lH07Iq{S zj67ZRClc{D%AY+E~74Y#geuRvFP8!jx)na=J zJ=e4lnM`-3Z4sT!&pCYrHTU5k1tLILMqWnFw-7H^6OqtDOLJsT+X953iGap--X0w` z-S`8GP|&{8zlFfd5j}(&{EOJ>`qh3zF#T%(A05C|>EA?vZ);$Wo;ck&kA)?8Ls&ba z2U1h~BEEM0YQG^Ef3^ST4nTpT@cL0S;xV%OkxJsuW2gjl-J-+TL0v2s>%B z{j?I~OFAa?#sQ*^k+cuNJeftSU-sK3n1^6#mD~oeR`Bg46MD&`&E-Gg|QzIxNkJe$B?0Sh_; zHtL)L^lo8EIs$rDzk)zH5dQm7RAbBtag~GJ z!l$AbHc+{-aPgsBox2}cM+{U9-UUiU71LO9xnH7$`}|awaLEG+k0{sg$SGZS%AK|z zK7{Q*quSgIF^h`6_+d@5UZDuNR zi!}zpWBOyOceiFIjXk`M8$b;wmdfKLZnR}F{P7n*8*DqaPogV*J2s}_13seh>+pFP z!kuo&rE~XdmeslsAinG34+j&K6QVsDZi~s!dhp4yUx|_@t`>eTp1AT_^+zkekSM=g zN7TMT9Ai#;JQH5ZKSfZdWgzgVF5~5<&=J9K zzvK?Cd;&96vmX}LK&n9n>IN!Q6|iuiNLV71B`E;YhUGyH6;=TGmA{klzs@f56~ept;PrCv zRX31&BPQ#0g(u;ugTPq!@3s~ZlO8}pN{1JJj)IF&O|eJA$tPL=OmV+_HAH&I06+%t z@X!2z^!P<#(7WhySLCAe|ir77@vDQQ1r#T}0LiQlJKJ0&5)M zLRoVPQ{lkx!$(+9>KvO?_yl`CJM=tdWCgk@ zcO!s*f0mL3q<@6ad&85!h#5dV0PZ78gNnZ&9)<~>0Xp}(WFt1(<2zgdes^1!Fi=@; zy_Ax~Lwj{6jsl~X&JYWS_g0RSg6%u|z&F$nr|EuR2~|lo6WskpEG=Q-2tKX+Ma2DT zzafC!H%q{Bq^K8*Lc=-m;y5^Y*22SI!yR1TB7ZX_ECre#LmC4q7mpL z!=42gqY#53oBnjNA;8#V|GF-)8hj*80I?%N>d%NU9Y37_5+^@Ew*mqLDMW=-fenq+ zts~Nfz;*#Nr{>>9s$^jxi4%IcfGV&-kkWLN3Arzjb3y)Y0UHF#O))|fbC&7Vde6D% z7TwF8p8Xc50;=*Y`_x%%P;*V{0V^)iOv7$>lfW3A>Kw$ic#nkwQq&6MVCHb2haX8J zbr7%=8F4uVzxbh1cWOK_zxZ;%0Q?hAj-qn)Q_&6jUB3vSz{!Kkh}M0MyUJCccxXLh z7C)0aT;TDHeTg-e9WRaaFzk;KGA}!3`ta6+aReM?=-T@-qfUa(RKbXq%C#rzRLMqe z3dz$VE6l@qUX;k4=9xW#tiIlRd+`WL-o!tGN_oE#t z(h6Kt>i`e|+gNWnO3pdb6YYi;K&nTin;A2bH)2;4=KNUoyRNc~C%w7eJ}cZ-x8$AF@||W^Y8CHunSyQnvD{b2Rf@ z{=07UBrRW@c&`P#+~hli`@@xytzRDE*~meVIZmzyDa-#qNc3}e$Gv=S(f6l7YDRoG zzlnWmYVlEZI#NVir6F!=$i9V}sz{2LcX>??vf$mb-~`ah9xd>f`c+1X;Xl={coiUq zZ^Uj0^~ut2bqo-3ztu4ym;dh$xDmTIEBq;T0YVWehW}K*0#(1&aUk{h(ri9otfFywd)*W>BrZ1RZ(Vn6vp*-iJM16xIj&jeBK!FE^+z+=AgAOeYyVjNznw?js z9yG4Ysy3bfh+Xr;Q7%O}L?g1T#4uintcSu%l?YEOv#;5Tb3UOqC}z zI$ne}BD~ODq2IQ$Ln97{cnBE`=>LMoJ|JQst!?!?oHv6E7Wk*VPl6fp@9F=($A2;3 ze>2d0te)4%;!vpa?B}GnYoVyJ>TSsURjC!~WcjhEjF`2*{P!Yui?Kam_O#^0??oCG zV^{|yhYxX!p41$r=~;b$s}ttBC0!GrUJR;f8aP;xd%e3aycpvC3WlLdr!PO;(U4q! zjlS;P=Po*SnQhzeD{^_%1f1Hb?V!Elg3vl#X29SQrQ7R;haJ2+Dc5^vc?x}=RG0Ty4P2TW4mi|jpuGh|LP=};NMP?se9O)(s9bz7@3-wn3@15DS-#Yyl^pc zzM&26?X4}14V^9RY#;E*y4sp>K+PQu?JYQL+|6!!su(&N+Bnf6pFs1%^0g@)HK{%%Pr9Vp6Guq(Qq;1_k@(iP=YSs_g$o<2<72G2tapLQqBEcJ? zC<(>Ki9#EKTUVG?u}@uLG~yg6p;(`PJubFPrFik|BE_4@GFIR(D#SQ zUD(WphZU9Er`0#A9@G^!6={~Cmci(}&2A4{S9;8YchjpIqz0cBdA<8+g*rqR8#ubt zl0DnGTQkA;Lu+of-a{lUK~^m?Tt4dCl30X2R9eR1tdTp?g6fkBfl(15%JTe!Qx0XU zy}aE|(%>k;BCdK0(QIj!lIt!!7baeIM_vlHA^a$0OmI(vxQbdq$!Z9>RtSASB*gix}+`<#* z2P?(UxnuR(3T2a|V?z2h%o_Vl?my`i4Gy3Xs@?P~3T zbM$ZO495pB%_}MsHiKzt@jote5Gc(rn8&idw|X}pleOTyy!r)$>+3Q&w=sXEf<=8h zw9&j673L(lGojdY3|08UBB~r!D=sm^b}a~ef$D;OyZSrSPdF0po!&1&KP+UWvT7Sp zEra{asaq&v;~wP$^(Bdd%@-0mq?x2^A9Nh*mi^SabwTCMY5f^$zfZ&Y7w%Tw@A%i{mb`X(;p_@$Vh*+`B-qgupGmS`1Q14?xLCG|7p&p?mL?y@D6&_CV z9ydKrbb`jb`x8!NY9e2sH1Z3~3VVS9PbwH(EDB@KUhMUdC!cPL-g{>jO>mglrgUhr zyUbgjLu*p+bp#r#!w>&gg#`S!Lh>)KZ~0YSl#QS-oNaJ9mFT#6k(I&G$(areSic@x zM#s&~byG9`dLCIa*@W$)z>c0iceFN1jCt>ennJTc-;1EW&aG6Orr0_7BRaVUC zC>QOfBtObQ6(3}l5isXk@EIF!yYJT5NGs$@&-ZQL`n3Zy7Kkxaj4v@K5EdQo^C~Q4$K9kIayH5w3d`KUYIGp7^IGVk} z*ZVO!ABWZE%Cu`>-P?6idb4$xXvpYD&Ggl>+%_z^Cy+TPBE5H15l)~FrzSb$zXa7k z9>|`ZVX z&-n>WGo%FmNa=e&KF0_AAQ?F(y3d4h<4+0cn@GtYzo6%AFqP3R6gj5JOGzF!q$%y3K5`WCwU?dI%50ajMwmR9}bPXtu6t|Rmbb!A15?nHA;D%PhA zM{;uaek5?snrt*sv)eGM>T%?28TRIMa`WZg8oFk4-OQ8uqs3{n0z=v=y0k8XI8^rP zxF?6Dv~K}*BzJF;nl6F72-xgA6;J0DO%rdiNeH(1^3tKr;-xrzd0>D|j9z13FMsIUX~vgRWUVX3c#9&8Wkot9kd zKR&hpW5wBCCoJQD^wXQ~(I}o-{7Oej{*k_w@%-DNI;WLBt-;;1!lz5K{WHVevuFG> zCYxfWN1=Ej(t)tBz6Lf zlP=^s!2{CRjLdlcxVd97A@*9$inN`37LOxSC9R3z&zg!?v)=(vGNc9o1(}Yj|QWAqT)Bv66CtU!#w0*!Wjyp|#opg|;1OF`|Y&cEiF#c|M3O@GlO|$=T4++1(Lvj0Y?vB!tVrAfqh%zW~qgjwS#A literal 0 HcmV?d00001 diff --git a/finetuning_loop.png b/finetuning_loop.png deleted file mode 100644 index a1f036295c8ee2cd122033268f88b52081a11fcd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 14621 zcmaL81yoc~wD^684yB|NX^|EQkp^K%X^}4JF6kT*WJCm!0YoIE5fJGvr39rBX%La_ z7+{G18GY}4@4dC&w^&Txb5HHFyTLdj1m|(?P-1qsOw_{r94gHVIWFvMj`?0!qBW zo7Y4pM&453wVaH0m*nbwxOpw!gh3TasmO+hTy(oJ7_Dehhte-&yOBAhw)k3)-D?yc z!8L1aMy0)@+rJf;V&r?b;;Y;1=@jYXMvKzM@?>#gd*8d5-|qtcTxHG0qomZS^(m{z zg~Lb01tChxcn*355{YvoLWNddZ&h|^!Pg71!8^J8?gL&NjFJ*xi-t+G+&F*ApVHFRN*5T$kkFaK=(K+1Q1lErXiJA%3`q+z}r4Qe%rM0tqE?a49Ht z7>+cb1k5Ds#yS?h>@N>x7Is_fLdC$J*+9hlTVcwU5F1_cGooZiwvoeMsTrT-=PrNdWNNTN? zL!oH&Pk4H6UWSO1{vN-wh5d6KMp!G2-G+4aMnf!>L*Y;20kwVjZ`qrV7+``ESE|yO z5Py_c=`?bLohq*4D|+T>PGWR>_V&^Ft=YQwno(YFjK1Hs4w1?A{V->?#@}_6Kdm)g zD-AunN2-5Sx5ulgI~IufmW_%VKlJj* z`(c`h@=6;m)iK$I>#=1W7CjYp)#r@x!0C3k4Vbl|O4cg5+mai!f51sUO1nQufn94( zpSy^vz>;b_XwSr`xsMa`z2b+-i$t>h`%+>Dt+;THaJu|CBpZ}Jr=loDnTX_y0ei{l z{rnlS3xdO`%{AZE>G|b%g4ggQ?8dO&zx`hl+)ahuY@5s6w%uWs-y=dLWhVJx+Qgag zmW<-r(tX1`-HWefo~bsnu3i$Jsn;t+(@V3%U1uWn19SN&3-^cfB3${7O-c#2nM29n z!^f}aL{enH=y6Lrb*s#q*{(t{3CB_&AL!*L52!p#Vsp$F9X(0uZ+4AQt4%9=xg!x+ zwG+%p3tLoSj~EeH(KA^Cn-|njegwGgqB#$9Mfy8g;3)&P49T|pp@<*W-?fjzN^H!$URO``aP_@PydfB5KGb}KO}RRjgc30>*Z z!c}=>pO{I)rA+uZ1kXY6K#B6m5U7vvRj!5L(dJLsP8H=ItEyKI3@4v#g>(1})F+(XFkJo<**qDcrl50EaIxx0A%bS^4Z_3DZxP)jF}i9--o4rzcBxBz$<)P17RPhA(vxRJEo*~ z5l`jA)Oi|87jgay#|QaOUj8beC^ClU-RV;>B&Ru?gbv|ki%U^yl>-h*8e%|M<~mG~ zfZO)YXL`gRlcq25oLic0inQ5O@`gCCXdXQJpm$GYeUzrqam74tk>>lCIaXCzm9+Qi zq^N5J{|2(yDhAPK1w~VTg?1mNhd=22P?cJ;i4$K!d^Y?2Ryd67H5b@L1^(LNZ`7CS z`L0SJA75NTh2SGPuuWp*llsYg$mijZ&9_?a--k&3t)iNMR zkIw{=t-MMx8~5;>dv}{&ySk9i<(1v zvaxdvG9qGD>NOa~vaQB(L`0qL*>AtPvHr7}CbvU4mA}Su0W53^7nc+PwNVbL$G(mG zF_ZTF2OjC{vy*Mbzby>SnMFf8svWM^Bsr z62(8p&iL}uT1>9)f`i)B>{ZcQ)Kkb*rAO-#>-v+9Fr`#q;aIdtn#UujLoqR{cVdA# zOycP2I0iSSQ5QWpwd$!xThs2ZileUTmk{CsCxhpFhr(?^G;BX6n#}n^KT$gRc^$d; zQ8}b7;>s6IS4w;${$dnnxWoht2sI*Q)qj@tS`LD>h3q;I;>95gVkX}v4ZWAn5N_Bg z-FU{|@K(B0T{7Cn&d4Lx+@E%fIhgQqveZ_n%X9_r_O~l>R=Y9V>2GV#>u72hW3jXF^igCq)W%m zd>Ovk&F|tf-GtfixIETApLjPm<#5uUOs--nEO@cXP{M-2+EiptxAFAz<4s$I^U#$J z6%7HEe0#1N3Zz_qYi&6Y6O3SjXu%D#&cuJ82$AgD7VXdBpVc*-#pLxIEcU{3RwoZl z`yONVLaH;0(JLg7_)Sb^`#u9@mFBXn8OyXRbC@??S;((Je5x(xV#WE&jBJmIe$Ov= zJeP5v=lA}3WE@KTN8p@&^BaHbuS+AnZr0Jgz4KtYw>8BbBI|<#Pf02%Rnp8GIGM#BbSbNe zE48!`BJLB#X9}d^^0yW{7ZS=NZMbc#(q9`y{^@cxmFd+ZEPKkQq&A}ZCil^-cdx&c zktn%1x(|*@_ttL0FmEmfqE}uto>J{^4%2aJiT=_mV)m;+n`$}t2Ht9MxCb+zD%OZk zl$Gf}Gu6`*2~<+7sU=g;)AQC|6U9tVSVIGn)dDk;M`TolrmF|_5egMnIr2!gxTT_8 z)LtaT=s+xfT_%3^aiDcF#pgmoCOJ~3hfKXV6CbD1q;Xxiv7`OT&@cJlG~X{fj8}8<-)fKDc(|EhyrIU-xCFhK zrQEW;U(adwWGyNWf;2_a;5(?Vn;1K#KQk|qdzqBcIlLbAc5y2w!OV9j@Ftim4wQsg z;W7xH&h;pa=UL4XYVgQn=BccV$+L0AI;fJbb|&N`>l7{Up7h3Me?}DeaQQ0D%Qyy^ zc7dC}12s(Atkd}U*Xn|I6UTm~OS*a6kDCSW!v))7ej%}tp-p! zfi_pJw}$z?vFa{|=W&}ki7qrm0at0}vB05wFl&MD6RDY`^N55{#gU~4$%$u;*<&UBAB!WB zDn&Yo;FGj`$c3$ z68jCt$5(VF6*81guOKpTYYAIRBPq1B-c=onP>eZYnyT)6cPawr%TfR7cY1Mp8j@J1 zLIv|dDWLPsmlo->llR|0r|U zMMaqGCU5!+dSz-LJ?*<$@;5GDM-DygpR>TA7wV2V-snlufkyFncYJwgvk@xXy^$1q zuqh&z2SjD~tteO0B$%b(kR=h6yH~Z9l9!;--5(~hUYcRco=LXSDC$aIy%$pNv*r1r zc0D(}CPu~CQ+(mYctMHiouKQw!$Z??8^C@q_R`hj-q*JyNjI;#s=;acvn+)7NAX9G zu=r#F3&cp}rAth&BihZ)$}n@B$XMUNmvUIY0JtUbZ8?P^V1hb2DqwLjcbhFI&Oz(VuIi1c$f1WtQxE(X_DBb0MQIN%U-owkRrQdZ z46iq+nNJ+EkUdYOyQsb^;?D64X1addT11+J1N9w$y@+r~NE>_J6?!F_Qv2v5AvI5l z-b05m>I{lsaRW+n2pJmmi>pPN7oYr2D}PM99o66CJ6&;Jk8t_c(;La&`Fy0(`?|F5 zmYaKw(j_~KKZAps=w+LBjqKaJ0yU#D$G5CwMq-)7g{w9vOXUtn{hFTOF+*Z&#UE$e z&Haz}wwlgPS|(1?t1>we8LKrdux)X~I7I8S|13rTO`{n3v-&Yz=Mr7(iU;1vp#_GV z#W7RVsovF)pmV9FG5EsJ74=|tV6Nt@I;}yw(mTZwGppIiSh&E67p%@rfA&rgLs?I% z)3;N5b{WAh&p=s5?-&8(hjzIO5lWU|E%&67+1p6BV!{cW5OwEIcSP$w_En^l^c?J+ z?QQdxZM`a28txSF0)Nm7Ov!jEZ|M+7QR=Ao<94y8q^sUWz0Yy&&V<$!KCK<(_Zr!s zp@*E^j>nwW=%RknYhOid!Ng#e)hI7P$b~7yIf{@)3wt8MirYeFiKx#{{pX;O_1P_^ z61?8xkc*#&#aw~VnYH!2lhbGGqI6GzFpC}2VlE!|;E?T+$#^sZ*OnzvQSr=zYgKSH zEY)DeZ@TMAwSXxP4EWp8qKvpCzo)J>+@P!6`;vicDLC;4KiNQ}blkzd?{m4`ou80v z1lFt6&r9M@e4F$HY#h2An!4A@-2c`cY#UJe9V6?W=Hu>pqLQ{x+o96S# z55h<+>nZ)^9c?z7HYYm;lP@Pe+_V<$B7oWo(R*M5 zH)?-!K#^JWjwTgz@*0u%9in(hHHqf;(OPh2`?mO-?M5DXTT8P0l-ha|Dn))RX17{a z*%}Z2JSu9KU-Vx&doM_w+C|{+YJG^7BAPaS>Jm$-|v=vRfX?r&6wO1lI78@T! z)uB=tA|J9z9+}*N7M?PIPy1O4Fmb2gE2D5VI8$#I%~6^PS>AqTnuz?iPwC*~ga@Je z%)+fNE2r|4LN5$Z^=5Wv;tFVxg{QJA7_izd33uX1@THBiFMJy>G%R_a3 zxtGn-q~k3^LRx;Cp}1PNEF$yrlujR`$=-2%tUlpzPtH`9lQAwHB;c>5->V2+`+8is z_{YThqlO`2Y=JQG7SVD>gD=E&5LiqKTrD!w1g|^LbIO%3JKK$fFesUNk}D+j;Dnq9!1)Ocp3 zw?B?qKP4plJowDo!QqHvr)$O#>oxampW*F>0-cITn#dZ{R`5UZ%OaJ%F7_o`2a;xxFO-CZy5M*p$;l9XmxaiR*zIqs<2Xr({7Y~nlr zELFLVIVfkJxY)R1d=1a2KRg@Q(n~VbgAoIv^+b*<{ZS2Q`phMcPDM?Wm;r=Xh-+nY{eB;7-N_otoWBAlA<9hD#Iqi_?`@1lxT> z0Ts@OZX!k7sicUYpn0pDhe3G;3^j!0szPa+lb><}Y;F5V+jox06(trd4~^8c>Wer{ z*)u;e#}O+E>avyRdt@NhHuMQC(|R6wsM)ts4qkm1{(|ow9s5hUv^;ul9ecV>HKGw2 z5CK%ar!{T2WE9Kld&-`PT)=x(>;jQI6dWpTKBrZV%lH(CM(vG`&4%_zZ`7}@mCX1o z1_zyZ|E%HO-w+Ma9^3TMF{~`4cf0{Q9;%O%aXgvJ)_Pk!Q-f$c|CJD%F%xV6_daZy z4MB7C>6wtfa9c_@*)w1KvsXOh$3eWKDR$Q{R+u2;`p-CP_svg%{fP_fvpWU|(dc*r zAJxt3@?S#JzI7tC4|4U`wb~F2Jb){d@S-wu#hZ@g5KrMo->J&u(2`3>*}j=M-^U2 zNN|MgU%{Q@jf-;s&4om+D=k7JM&#SmWOKw&(*cGK`U&y#zCw0;$s;eZ!gI?Ve}154 zzubCovN*sRaa+xND$hQRf^z&>IRN|cd?^AOybL*JPPTFgASp_nqcVDKfXF{{DScR{ zzK;E9Jn4#Dc{xmlNXN>D8Hwyhkrw0Ui#8L$55E|D*cSv}0x;dB8(i-Kfmjs|uTH$g zVzFo~r~c(80FYR5jE?uB1|U>8F!wDFcn@B>^AKigW;~LI+bAmu>e}LJbKX)?gHuzY zqzORyX_GEpWwV<#=3nl$tjC`k%6LJ1Bg`-cCDC;H`~I)u&8J@HXXD)O&QB&oA$mkN zH5etV_RsFQzTZ7@_r`~DP$u*BMvan_S$w7^R zBt{}OQxbpq-;^Uk_>1h806ezUDhEuX9s6j*orp`(yA9S0OYA7)b5WP&Kpz6!KyK$n@M~{Y2006YUL47}%&&i}@`{&B*aeXs^-rTon zE{+;2|N6pwZQG|4eZLi`PY}!@I+f^nlKAluc3nZ=!ec2N!p;0{Ul-I|FDdbib0RRy z!)NO6lCTOgYe+k*LJ1iV=@41os+9AlZ0+Z$Bai>h%YZmy<>dWlWcUnBAGpB)V+Q`7 z@@rY~+BrM11}kV-OdglyNPA2Hq zL;eI=*FB8ic=Ishe)xTz_wR|ax!;8-aK)}DGn4GjY4K#~QQc?=YY>(E4X9h&7Y~n? z9q1np>?25){nZ5jj>SLmp0}rFe|St!K}k9OisCad%e`@EWc=^xp#OV1d{B%qSOH$E z4h3cW_vJ*V58ypGU`>TmI96Ag*ho^=jqYzU3(bAT*GIGKN@FmIQ6Ni#Jvr}1=fXqv z{aCZOCGj~U3M%W#!P)z$^|>Pz95^8BY;2P!_eyK6c?enl4zWZ8DEvbQzP18 zhDdW!q$QiAkw4~fG20{2iV))Mv^(2MPZ0$9X}*npL0~%MgbDvT&Sj)O_-EHw9qs_md~cXp9KZWBH_?aVd}0n*HHF&o{358%>raFo>cZtNkFlH;!!oDK zuRk`D2weE2jg?1{uYXBHJN08^mbQ-b_4z(mhDYuDm@^KATz&o4{QWu+5~-pnoqc_C zR_mi-L;+XSSQ(pEll21R@T50pYcGmy{V}t>+Wme!CZdhzg6jx>U>;CawKk%ndd=^r z2NP%~df&WmaA@Ty7(cU9!B{j{J6D&(kX>O&6#J0Cc>i*1skEWl^tS6nIiqCFNKEE&TWFBR_4s%$lJIyqhDw|+5serY5&X7BYiCA{`k&})1I;64|uFxR#jan;n;xO)hSw+b2(F<;IbIecdmm-_bzvwO=O})Zk+|#< zRLn50dK-!-`LRQ_?nEIU(JUb}PVm=`|1F64k96{X??OTC>MpaZ@cY@=Em4>r4%gro z=Ng(6WlCn8zg7w6*qAjREZ|p_-~D>#FARQ^|O`h6odi z9NBs=4e>;t%c&5v=AeX%>(2CD5EJ->U-WS=+$0HEBd*AWmoLF~6x881TDof3 z>|Gqp=hf;+gX?fzP=!;|1;L;%9lfN#=0(~jo1n=)~<`&~> ze+0Ts)sKxWXhK47p_lKdEJ6}T$5v#?nq#ZIFEyKAg(}GQlVPOy0@z9 z!$x*Mwq9EbO5FYp_X!Agyn0S*nS%1F9}UT#3piDHB*vQSZ9FYN!Vlci!v3ZEFS0X; z@-7s1gevdHUC_xfDa)=3@vs6%u%LZaM!Q?%$)WARWdtXEFk5oE$bNb#`o`0^-PbU# z5H=oiC0@;oAqhNwB55n1u!8Blbn7h&R?bL{dj8vTX08@R+K}sTV|IOiZGM~ z0-9lI6)q<}qrl1x0V{Uw18#Y^04q~n$`~77{fL*HJjzJaZc*KIWm7{U=x{m7`)@Y) z05ZD-$$qM*d*Z)P5*X}x;(SxB@`+B29#-vi&wVX$^#hq-%30pdRX1%yu?7=^=44h6 z-@CRgG76en5?>9#InaUw11bodMqnea{Br|=dpWLVRr*my<32i&4`fXV$W=)SHr_AH zRy(G;Ix53Atbth-g@{>sa#7du6<-0@&tw{(jU?`0Rv7YEb;Gx@&9fY^{fgzMotm|z zTg_%qf%`@;Ie+W-)ZH&lQ-B^%*SIqVNv0ds(Z|wrzSXSVR#~}%Mbf|t)Gl*@VS#mU z3to}m%L3sCl9w4;gNr2P+IB9nAw=0GOiZ3~(DYmPeHOxUYv8iAQ7*uh)Upg9p}aLw zSJOCUlKjkHb;-B=Rj9Cj+|aKStUIFi!andn)3XH+UhC>BEJE#SKjXV*soyO*bkt7& z^v_#2@va;WDR#huN`Qtz6H$}?2KO<40s8~8D?Ylw{Hu+dnBYR#xdrZv(E%p*XmsMc zq3(_pSSbPsWTL-1i?fxhW8ff0N zm1C|b$BnQrHDh27!`^o6>}=)FdSFO4WQ+H5*_CpQ1s?HA9{B0(-7(`+|!uLOmz zF0QGpr2w2B=5VnO@3EIBVuc;0o9kvWYY{t1xBckZZ)~+Ew52z|<RwCTpb(sfpmGW5Xlmmz2uamiu<`!?5xc^oezLJ6+31X*IVtJamSOMVxP z+e@-+z()sfl=bbRG4zu&6W74lqd@WROj462-Ek1*to@V6j}-^%X?^(Jcnxyo>x&t%+NAO}Ca zq=Uh32`Kx$M*r0o*oI9%4tMT-awc)2_1ao@ z$*5-j{%Q_rGf7vUlvQaB{W4xh2q8GocQF2xR`a=(`KFLs1jQ`}&OaC?*?##=@Z|&&&w_St2d^fh-`xgE#G5H0EXkZYE7PFP={_3{# z5<@X?(o37k>WdF-E-@GrT9%woe$rJ7U>ewU6Luw1rp`LEXF%BFK|BZsYrd6W)BqqECmi#m^SxD5e1rc!pvk8~Cy$A=; zXdtAT>J=5O|77PA8PwD%)Mh7geE2n?Yz73%HMQpb=ekQ?pX$q#DSq`sGf}j`(3~Sd zhJGES7v+j)b1k>~X?MFR#GIE>IfNq{P)p zoRl77{4dq@?ISogKkN6jD`W!FkDMySK`oCRX0^dQyQO5FNQF^Lx`miDbg#vtXRBF~ zGbXh}*U^rXuM6R?TM)AHxKg~vHuRFL3>CXx^PuqBCfRSsTAJwHKfD8!zNO2$iQe55}$-0iZi!yWt^W_eqW8&7l@my<7YQK>U9uGa^}wdTC8=BS7E*?)fd-tgko|D#P!FS*vYV1 zv-tFz8JJN59Qq$WV(wL_bB3j8fDDR=Xg$CjIFO1)ubh){eUK3Z%thm@Yk-}FBA7yo z`^HxXp|u2A#uRSFb48~#(X%&NAIr7fgI1OP9B%ovE1i2)9u-|X*0U_H?o`TjAdteSRjTY zR8G2p;Secey}Mj~qs1uR&gl8ZcPG37Cp&B9GB3Owds+)nNGt57aNjl5LM|6I;N|ivs8Y*T3nT@X_A@pGQO{%=1;&R`!Or7vGm=R4da4g0z8YglIs(y8@Jw! z#R>y9%*OWOxCs46D`>4Uz)lor!hvf*NFloLseVblCb`;i3E%7=WB1OT>)eax?cvOqZok=(@Pc}-^OCwgAacIN1wR&6Q7K;s8$M^6Gmi5=!&6V;231r7^L&W+%b zQ?wExJ5;{@VBZcBA8CKI^ZJXRlg`fu=7OrD@;Y3@}veEz*-w_)pP=EZ z;OuV&{dj!&o;?*Ot$Tu}UoyS!6JD(ID<4>XfYg&L245qgy8*X~tS|g8Y9+=$rSYuf$jF=S127^O;`tQicD2v-6_y zMA7HnRA04E;d->}FD;ey6O8hP0fEXJcC;JUReE$Ek~&wg5$@4!17U&dL*Qrl;{C;t zhdCC_^3yjWbvt$Y2n$2;-~YRN84f#k{%9tuZjT1GqdW( zmA^9LSkyrA6`;yfmhz9r51+Lk2gl@b zQ9%{16@E)eo=>CfWMWf*a3VtIzW;hH?L9p= zJ1>Z1xGk_ibp1ItlIA*%vC07>WV!;%VL2CsX27Jb66OQK>iZgo0%eAP#sJ8a3J}Xf zqiGn3v~z<6b+L@ZM&U0?P5MiH%2B{;`~U6b7K=0Mkz&<(K@R(uui zwHy?fwv8XSDmGMw7-T#CRuI1n&;HRE+Ev=q?6+Ab2;v!vyRv6d45cmG;pqIc-*0iT z`!^f1xAE3Fqu{7$^o%7)=rx6&3^}quF<&daap|-z*?9UT;`A&&3O46;*7ZY^>4JXt z*L?_F9Kt#Y_*qs&hMBt$9qIvkRkU_>t$34Q{2$HZ_`7(;pZ{`jMi8O>4=xTe7wPtz zaJVRYL^p-J9w`6)_jMjteXs?60G`C+4Ll}XY?(8+EhEAUM2`YW${;Sr694?U2nr{C zBM!FcU`^;NkP~njKbGueZCn54^?wn1{KJ1k$Ofqh@X94KXo#jZ)FFDzy5qr3cE`Ea z%pachc3K0Iw~XsAfsahis3`9I3#PC^S4XdHoJsjKsCfavpahUbV5V&B6QY-_Ci?*X z1$8unG8VllJc%v)&taYY>G}Ne7xP{`r}XGWd;ZuCzp;H*|~CC2yLDd zn}KZya!YJ}U>bj;5t1Uwdm`YwQj9>dbnN6a+qwS4hhUf_qRTC?%;tIe==>?d{4Y;H zihg?}<8EbrXoV;kthMfrsmLj2*3euJnezm<%ysD&NZZ?uBV)RgfzWr;erw(?WqunD zn$3YKg$&-JPhktB5aJCNVRX{1sA0-TS4qDBMp-Y3>v?PLg)b?+kuJ}`(JMyX;ll;B zDISxbgeyAG(h_vcE!CHyQv$Mn$GW5mF-Ed>xZ)$lL0nJ?Lu&Bg3X{n}%xHrKR?p1; zlEn9${B#La5kyK>xG92S2t%EH1f~uYp|SX!;smilFrM!IQ^$b0Q zK|Tzo2R{Azw|Fss5%f?(`)=a*aCSg?|3Y07tVQ`>6j}l(k-yx|Z#+TU0WE};FL~Ei z{IlXs`F7)r)5q*UmtK)CFtSZVs=)pO1_F7;tO9<94ePNX{kR^$fOi2=#)?WpcpSc&!*C3C^ zdHT(_B8-3)`-6Ejo{sr!>Zsojo($PJ=7zSNFo2!E(ch-Hb14ux78g7%a@r zQf5{_eR%@#w$*xs%jx4xi_rMa2HR1th>Zg!NTB*Gy4G590 zLnc*cMVXa#9ZQxgtd&8>;9Qh2Lf0ie^@mJx^Gja=Ts%m679e!!?aTTJz$(CxQ_6;h zqOxnZYg8I!hR=S2_4%oN{LEK5Lq>Rk;l{U#V}c^CDBP$67Co?-HwfbK25DpWLJf;t zpx3%y`wDQ%Hb0VE{2^j)jkeV3#$`kgNSVB3(@|bD*d^wY_CK=gnfCAHQhzH(3n+PG zy0G(@q-Y;6#?|S+;PyMI@d!tz$;*+Mc@@`u8E9lB&<|}Z z=KhHr_7Ta~_E#$t|2re6OGkn}Se2TnkA8wgehbm_)1Kzd!s-5m{k_=l41gr>2`KS^ z$n5sDfz>Y{%!Hwqd#P2piy|pFpx3bC1l=T)fjscTl)3zug0HPpg#oE9Gh)alV43u< z`umHRMwyDVVvU9iH$w6=$5K*E$7t}`fx(hUz06B|gu@b=q25Ol)2lR=AJTk&Qp6-3 z_-;+&F3-tuzm{m|#R~x$Kb%(b3yTaiTgi9M3-n+lG&h-NQd7}_^bNhpuC6@Z(Gm*F zKMerMRXm9Jgs@nRXnsUi9dX#^v%)OFiICv7O~ZFJJt3Sf{vQK8XZ{}peEBeJEq)^J z%f%0v+%XnGALjh~rH$Z!L(czx{`KDp^=a7Gj@%{Yy2*10f;hYt4ZUq1dD}`@d)a~~ zNa(hZ2>)$yenAmEK@kZdAqf#--rKh&Zr^sax%%pVG;npdakLBgzZ=ktzvlxDxc;jL zZ%0?#$KH=zJ^t@DT6~B-(B=xZjfah$guboUV@G#4NE@!iFL>*2lbGZN_!?4G)P#SO Hw+#P(6sSMX From a927f458180d6997e647b820db72038836cc7f2a Mon Sep 17 00:00:00 2001 From: Dominik Farhan Date: Tue, 19 Nov 2024 15:33:56 +0100 Subject: [PATCH 5/9] fix(multiling): :bug: do not copy multiple qids more than once per language if it is present multiple times It might happen that langage description contains multiple ocurrences of the same qid (in theory it shouldn't but due to remaping it happens) this makes sure that these multiple QIDs do not break the process. --- src/multilingual_dataset/creator.py | 31 ++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/src/multilingual_dataset/creator.py b/src/multilingual_dataset/creator.py index a798be2..16061fb 100644 --- a/src/multilingual_dataset/creator.py +++ b/src/multilingual_dataset/creator.py @@ -26,7 +26,7 @@ def __init__( self.dest_links_dir.mkdir(parents=True, exist_ok=True) self.single_mixer = Mixer(buffer_size=1) - self.standard_mixer = Mixer(buffer_size=200) + self.standard_mixer = Mixer(buffer_size=10) def run(self) -> None: """Gathers links from all languages and writes them to dest_dir. @@ -49,7 +49,7 @@ def run(self) -> None: out_file_paths.append(out_file_path) self.single_mixer.mix(out_file_paths, n_of_mixings=1, compress_output=False) - self.standard_mixer.mix(out_file_paths, n_of_mixings=5, compress_output=True) + self.standard_mixer.mix(out_file_paths, n_of_mixings=20, compress_output=True) def _copy_files( self, source_file_paths: Iterable[Path], dest_file_path: Path @@ -93,6 +93,7 @@ def __init__( def run(self) -> None: qid_lang_mapping = self._get_qid_lang_mapping() + assert all(len(v) <= self.langs_per_qid for v in qid_lang_mapping.values()) lang_qid_lists = self._group_qids_by_lang(qid_lang_mapping) self._copy_chosen_pages(lang_qid_lists) @@ -115,10 +116,20 @@ def _copy_chosen_pages(self, lang_qid_lists: dict[str, list[int]]) -> None: def _copy_chosen_pages_from_lang( self, wanted_qids: list[int], filepaths: list[Path], lang: str ) -> None: + # wanted_qids = list(wanted_qids) + processed_qids = set() for i, descs_file_path in enumerate(filepaths): tokens, qids = load_mentions(descs_file_path) + _, unique_qids_index = np.unique(qids, return_index=True) + qids = qids[unique_qids_index] + tokens = tokens[unique_qids_index] + + index = np.isin(qids, wanted_qids) + # This is likely better than the isin from numpy which needs linear scan. + for i, flag in enumerate(index): + if flag and qids[i] in processed_qids: + index[i] = False - index = np.isin(qids, list(wanted_qids)) chosen_tokens = tokens[index] chosen_qids = qids[index] @@ -130,6 +141,7 @@ def _copy_chosen_pages_from_lang( tokens=chosen_tokens, qids=chosen_qids, ) + processed_qids.update(chosen_qids) def _group_qids_by_lang( self, qid_lang_mapping: dict[int, str] @@ -194,6 +206,11 @@ def _get_mapping_from_counts_and_lang_sizes( desc="Mapping QIDs to languages", total=len(qid_lang_counts), ): + if qid in qid_lang_mapping: + _logger.warning( + f"QID {qid} already has a language mapping, not adding it again. This might happen when using qids remap dict. If that is the case, ignore this warning." + ) + continue items_by_importance = sorted( lang_counts.items(), key=lambda x: (-x[1], -lang_sizes[x[0]]) ) @@ -235,14 +252,14 @@ def __init__( ) def run(self) -> None: - _logger.info("Starting to create KB") - self._kb_creator.run() - _logger.info("Finished creating KB") - _logger.info("Starting to create links") self._links_creator.run() _logger.info("Finished creating links") + _logger.info("Starting to create KB") + self._kb_creator.run() + _logger.info("Finished creating KB") + def create_multilingual_dataset( source_dir: Union[str, Path], From 2bd912ae390e2b12e02d7f3deb24d8ae6015bcab Mon Sep 17 00:00:00 2001 From: Dominik Farhan Date: Tue, 3 Dec 2024 21:30:18 +0100 Subject: [PATCH 6/9] perf(models): :zap: Add DiscreteGuideTable to NegativeSampler Calling np.random.choice repeatedly is grossly inefficient. This speeds things up. Nevertheless, it can probably be optimized further. --- src/models/negative_sampler.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/models/negative_sampler.py b/src/models/negative_sampler.py index a90c17b..ec169aa 100644 --- a/src/models/negative_sampler.py +++ b/src/models/negative_sampler.py @@ -1,9 +1,9 @@ -from collections import Counter from enum import Enum import logging import numba as nb import numpy as np +from scipy.stats.sampling import DiscreteGuideTable from models.searchers.searcher import Searcher @@ -143,9 +143,13 @@ def __init__( self.sample_f = _get_sampler(sampling_type) self.qids_distribution = qids_distribution self.randomly_sampled_cnt = randomly_sampled_cnt + if self._should_sample_randomly(): + # self.urng = np.random.default_rng() + # DescreteAliasUrn seems slightly faster but there is a UNU.RAN error which I don't want to debug and + # DiscreteGuideTable looks okish. + self.rng = DiscreteGuideTable(self.qids_distribution) self._validate() - def sample( self, batch_embs: np.ndarray, batch_qids: np.ndarray, negative_cnts: int ) -> np.ndarray: @@ -178,13 +182,12 @@ def _sample_randomly(self, batch_qids): batch_qid = batch_qids[0] batch_qids = set(batch_qids) result = np.empty((batch_size, self.randomly_sampled_cnt), dtype=np.int32) + # TODO: Is sampling of size 1 really needed? These nested loops seem incredibly slow. for i in range(batch_size): for j in range(self.randomly_sampled_cnt): qid_to_add = batch_qid while qid_to_add in batch_qids: - qid_idx = np.random.choice( - self.returned_indices, size=1, p=self.qids_distribution - )[0] + qid_idx = self.returned_indices[self.rng.rvs(1)[0]] qid_to_add = self.qids[qid_idx] result[i][j] = qid_idx return result From 145b2009c5f4a992d557b7307a014341f24d07b4 Mon Sep 17 00:00:00 2001 From: Dominik Farhan Date: Tue, 3 Dec 2024 21:35:45 +0100 Subject: [PATCH 7/9] fix(models): :bug: correct rng init validation order in init of NegativeSampler --- src/models/negative_sampler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/models/negative_sampler.py b/src/models/negative_sampler.py index ec169aa..b27f6ad 100644 --- a/src/models/negative_sampler.py +++ b/src/models/negative_sampler.py @@ -143,12 +143,14 @@ def __init__( self.sample_f = _get_sampler(sampling_type) self.qids_distribution = qids_distribution self.randomly_sampled_cnt = randomly_sampled_cnt + + self._validate() + if self._should_sample_randomly(): # self.urng = np.random.default_rng() # DescreteAliasUrn seems slightly faster but there is a UNU.RAN error which I don't want to debug and # DiscreteGuideTable looks okish. self.rng = DiscreteGuideTable(self.qids_distribution) - self._validate() def sample( self, batch_embs: np.ndarray, batch_qids: np.ndarray, negative_cnts: int From fb6652e7e451079bccd4c9dd47439f474480f882 Mon Sep 17 00:00:00 2001 From: Dominik Farhan Date: Tue, 3 Dec 2024 21:36:26 +0100 Subject: [PATCH 8/9] feat(train): :sparkles: add qids distrubution calculation to generate; needed for randomnes in NegativeSampler --- src/finetunings/generate_epochs/generate.py | 5 ++++- src/utils/calculate_qids_distribution.py | 16 +++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/finetunings/generate_epochs/generate.py b/src/finetunings/generate_epochs/generate.py index 5c514b4..8d40787 100644 --- a/src/finetunings/generate_epochs/generate.py +++ b/src/finetunings/generate_epochs/generate.py @@ -98,8 +98,11 @@ def generate( negative_sampler_kwargs = {} if "distribution" in NEGATIVE_SAMPLING_TYPE: + # using sqrt seems like a sane default negative_sampler_kwargs["qids_distribution"] = ( - calculate_qids_distribution_from_links(LINKS_EMBS_DIR, index_qids) + calculate_qids_distribution_from_links( + LINKS_EMBS_DIR, index_qids, lambda x: np.sqrt(x) + ) ) negative_sampler_kwargs["randomly_sampled_cnt"] = 1 diff --git a/src/utils/calculate_qids_distribution.py b/src/utils/calculate_qids_distribution.py index 9b24386..ad00cc3 100644 --- a/src/utils/calculate_qids_distribution.py +++ b/src/utils/calculate_qids_distribution.py @@ -3,19 +3,29 @@ import numpy as np +from .loaders import load_qids + def calculate_qids_distribution_from_links( links_dir: Path, index_qids: np.ndarray, transform_fn: callable = lambda x: x ) -> np.ndarray: qid_to_cnt: dict[int, int] = defaultdict(int) + index_qids = set(index_qids) for file in links_dir.iterdir(): if not file.suffix == ".npz": continue - d = np.load(file) - qids = d["qids"] + qids = load_qids(file) for qid in qids: + if qid not in index_qids: + continue qid_to_cnt[qid] += 1 for key in qid_to_cnt: qid_to_cnt[key] = transform_fn(qid_to_cnt[key]) + print(len(list(qid_to_cnt.keys()))) + print(len(set(qid_to_cnt.keys()) & set(index_qids))) qids_observed_cnt = sum(qid_to_cnt.values()) - return np.array([qid_to_cnt[qid] / qids_observed_cnt for qid in index_qids]) + res = np.array( + [qid_to_cnt[qid] / qids_observed_cnt for qid in index_qids], dtype=np.float64 + ) + print(sum(res)) + return res From d69214a783cd5f507d89b6742600267c9fac4e5d Mon Sep 17 00:00:00 2001 From: Dominik Farhan Date: Tue, 3 Dec 2024 21:37:36 +0100 Subject: [PATCH 9/9] feat: :wrench: add configs and run scripts --- configs/tokenization_context.gin | 14 ++++++++------ configs/train_NO_distribution.gin | 20 ++++++++++++++++++++ src/scripts/multilingual/multilingual.sh | 5 +++++ src/scripts/train/evaluate_no_slurm.sh | 21 +++++++++++++++------ 4 files changed, 48 insertions(+), 12 deletions(-) create mode 100644 configs/train_NO_distribution.gin create mode 100644 src/scripts/multilingual/multilingual.sh diff --git a/configs/tokenization_context.gin b/configs/tokenization_context.gin index 258a620..e407cab 100644 --- a/configs/tokenization_context.gin +++ b/configs/tokenization_context.gin @@ -1,6 +1,6 @@ -expected_size=128 -output_base_dir="/lnet/work/home-students-external/farhan/troja/outputs/128_finetuning_damuel/" -output_base_dir_mewsli="/lnet/work/home-students-external/farhan/troja/outputs/128_finetuning_mewsli/" +expected_size=64 +output_base_dir="/lnet/work/home-students-external/farhan/troja/outputs/finetuning_damuel_2" +output_base_dir_mewsli="/lnet/work/home-students-external/farhan/troja/outputs/finetuning_mewsli_2/" languages = [ "af", "be", "ca", "da", "el", "es", "eu", "fi", "ga", "gl", "hi", "hu", "id", "ja", "la", "lv", "mt", "nn", "pt", "ru", "sk", "sr", "ta", "tr", "uk", "vi", @@ -8,11 +8,13 @@ languages = [ "it", "ko", "lt", "mr", "nl", "pl", "ro", "se", "sl", "sv", "te", "ug", "ur", "wo" ] +# languages = ["af"] damuel_base_path="/lnet/work/home-students-external/farhan/damuel/1.0-xz" +#damuel_base_path="/lnet/work/home-students-external/farhan/troja/damuel/1.0" label_token="[M]" -compress=True -remainder_mod=128 -num_processes=90 +compress=False +remainder_mod=90 +num_processes=60 run_damuel_description_context.expected_size=%expected_size run_damuel_description_context.output_base_dir=%output_base_dir diff --git a/configs/train_NO_distribution.gin b/configs/train_NO_distribution.gin new file mode 100644 index 0000000..3f688c2 --- /dev/null +++ b/configs/train_NO_distribution.gin @@ -0,0 +1,20 @@ +training_batch_size=2688 +#epochs=300 +epochs=100 +logit_mutliplier=20 +negative_sampling_type="top" +learning_rate=0.0001 +negatives_per_link=3 +context_size=64 +STEPS_PER_EPOCH=1000 + +generate.BATCH_SIZE=%training_batch_size +generate.EPOCHS=%epochs +generate.STEPS_PER_EPOCH=%STEPS_PER_EPOCH +generate.NEG=%negatives_per_link +generate.CONTEXT_SIZE=%context_size +generate.NEGATIVE_SAMPLING_TYPE=%negative_sampling_type + +train_ddp.EPOCHS=%epochs +train_ddp.LR=%learning_rate +train_ddp.LOGIT_MULTIPLIER=%logit_mutliplier diff --git a/src/scripts/multilingual/multilingual.sh b/src/scripts/multilingual/multilingual.sh new file mode 100644 index 0000000..c377689 --- /dev/null +++ b/src/scripts/multilingual/multilingual.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +# An example on how to build the multilingual dataset + +python run_action_gin.py ../configs/general.gin create_multilingual_dataset --source_dir=/lnet/work/home-students-external/farhan/troja/outputs/finetuning_damuel_2 --langs=[af,be,ca,da,el,es,eu,fi,ga,gl,hi,hu,id,ja,la,lv,mt,nn,pt,ru,sk,sr,ta,tr,uk,vi,zh,ar,bg,cs,de,en,et,fa,fr,gd,he,hr,hy,it,ko,lt,mr,nl,pl,ro,se,sl,sv,te,ug,ur,wo] --dest_dir=/lnet/work/home-students-external/farhan/troja/outputs/all2/ \ No newline at end of file diff --git a/src/scripts/train/evaluate_no_slurm.sh b/src/scripts/train/evaluate_no_slurm.sh index 2d226ab..cc50b4f 100755 --- a/src/scripts/train/evaluate_no_slurm.sh +++ b/src/scripts/train/evaluate_no_slurm.sh @@ -8,17 +8,26 @@ echo "Current directory: $(pwd)" MODEL_CONFIG_PATH="../configs/lealla_m.gin" TRAIN_CONFIG_PATH="../configs/train.gin" -# DAMUEL_FOR_INDEX_NEW_DIR="$OUTPUTS/workdirs/all/damuel_for_index_8" -DAMUEL_FOR_INDEX_NEW_DIR="$OUTPUTS/triplets/combined" -MEWSLI_TOKENS_RAW="$OUTPUTS/tokens_mewsli_finetuning" -WORKDIR="$OUTPUTS/workdirs/all" +DAMUEL_FOR_INDEX_NEW_DIR="$OUTPUTS/all2/index" +# DAMUEL_FOR_INDEX_NEW_DIR="$OUTPUTS/triplets/combined" +MEWSLI_TOKENS_RAW="$OUTPUTS/finetuning_mewsli_2" +DAMUEL_DESCS_TOKENS_RAW="$OUTPUTS/all2/descs_pages" +WORKDIR="$OUTPUTS/all2" ROUND_ID=7 -MODELS_DIR="$WORKDIR/models_$ROUND_ID" +MODELS_DIR="$OUTPUTS/workdirs/all/models_$ROUND_ID" ACTION_SCRIPT="run_action_gin.py $MODEL_CONFIG_PATH $TRAIN_CONFIG_PATH" LANGUAGES=("ar" "de" "en" "es" "ja" "fa" "sr" "ta" "tr") +if [ ! "$(ls -A $DAMUEL_FOR_INDEX_NEW_DIR)" ]; then + echo "Running embs generating for damuel" + ../venv/bin/python $ACTION_SCRIPT "embs_from_tokens_model_name_and_state_dict" \ + --source_path="$DAMUEL_DESCS_TOKENS_RAW" \ + --dest_path="$DAMUEL_FOR_INDEX_NEW_DIR" \ + --state_dict_path="$MODELS_DIR/final.pth" +fi + for LANG in "${LANGUAGES[@]}"; do echo "Processing language: $LANG" @@ -39,4 +48,4 @@ for LANG in "${LANGUAGES[@]}"; do echo "Completed processing for language: $LANG" echo "----------------------------------------" -done \ No newline at end of file +done