From 6250fa946892db18730239fc832fc4b828406ada Mon Sep 17 00:00:00 2001 From: Matt Watson <1389937+mattdangerw@users.noreply.github.com> Date: Wed, 10 Jan 2024 17:44:53 -0800 Subject: [PATCH 01/29] Fix test for recent keras 3 change (#1400) When dropout rate is zero, dropout will now create no variables (no rng seeds). We should set a non-zero dropout to test the presence of these seeds in tests. --- keras_nlp/layers/modeling/cached_multi_head_attention_test.py | 1 + keras_nlp/layers/modeling/f_net_encoder_test.py | 2 +- keras_nlp/layers/modeling/transformer_decoder_test.py | 2 ++ keras_nlp/layers/modeling/transformer_encoder_test.py | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/keras_nlp/layers/modeling/cached_multi_head_attention_test.py b/keras_nlp/layers/modeling/cached_multi_head_attention_test.py index 052ce66ec1..16b423e08c 100644 --- a/keras_nlp/layers/modeling/cached_multi_head_attention_test.py +++ b/keras_nlp/layers/modeling/cached_multi_head_attention_test.py @@ -28,6 +28,7 @@ def test_layer_behaviors(self): init_kwargs={ "num_heads": 2, "key_dim": 4, + "dropout": 0.1, }, input_data={ "query": random.uniform(shape=(2, 4, 6)), diff --git a/keras_nlp/layers/modeling/f_net_encoder_test.py b/keras_nlp/layers/modeling/f_net_encoder_test.py index e5d0b1ea77..47c189ef11 100644 --- a/keras_nlp/layers/modeling/f_net_encoder_test.py +++ b/keras_nlp/layers/modeling/f_net_encoder_test.py @@ -23,11 +23,11 @@ def test_layer_behaviors(self): cls=FNetEncoder, init_kwargs={ "intermediate_dim": 4, - "dropout": 0, "activation": "relu", "layer_norm_epsilon": 1e-5, "kernel_initializer": "HeNormal", "bias_initializer": "Zeros", + "dropout": 0.1, }, input_data=random.uniform(shape=(2, 4, 6)), expected_output_shape=(2, 4, 6), diff --git a/keras_nlp/layers/modeling/transformer_decoder_test.py b/keras_nlp/layers/modeling/transformer_decoder_test.py index 2b54324f02..fda2ea7e33 100644 --- a/keras_nlp/layers/modeling/transformer_decoder_test.py +++ b/keras_nlp/layers/modeling/transformer_decoder_test.py @@ -36,6 +36,7 @@ def test_layer_behaviors(self, normalize_first): "layer_norm_epsilon": 1e-05, "kernel_initializer": "HeNormal", "bias_initializer": "Zeros", + "dropout": 0.1, }, input_data=random.uniform(shape=(2, 4, 6)), expected_output_shape=(2, 4, 6), @@ -59,6 +60,7 @@ def test_layer_behaviors_with_cross_attention(self, normalize_first): "layer_norm_epsilon": 1e-05, "kernel_initializer": "HeNormal", "bias_initializer": "Zeros", + "dropout": 0.1, }, input_data={ "decoder_sequence": random.uniform(shape=(2, 4, 6)), diff --git a/keras_nlp/layers/modeling/transformer_encoder_test.py b/keras_nlp/layers/modeling/transformer_encoder_test.py index 844125c4b0..edcfdfc470 100644 --- a/keras_nlp/layers/modeling/transformer_encoder_test.py +++ b/keras_nlp/layers/modeling/transformer_encoder_test.py @@ -37,6 +37,7 @@ def test_layer_behaviors(self, normalize_first): "layer_norm_epsilon": 1e-05, "kernel_initializer": "HeNormal", "bias_initializer": "Zeros", + "dropout": 0.1, }, input_data=random.uniform(shape=(2, 4, 6)), expected_output_shape=(2, 4, 6), From c49bf9bf55b04cf10c6b448f7d6949cdb3551d32 Mon Sep 17 00:00:00 2001 From: Matt Watson <1389937+mattdangerw@users.noreply.github.com> Date: Wed, 10 Jan 2024 19:02:47 -0800 Subject: [PATCH 02/29] Pass less state to jax generate function (#1398) --- keras_nlp/models/generative_task.py | 33 ++++++----------------------- 1 file changed, 7 insertions(+), 26 deletions(-) diff --git a/keras_nlp/models/generative_task.py b/keras_nlp/models/generative_task.py index 9a461926e4..14d4c88f2f 100644 --- a/keras_nlp/models/generative_task.py +++ b/keras_nlp/models/generative_task.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import itertools - import tensorflow as tf import tree @@ -82,30 +80,18 @@ def wrapped_generate_function( @jax.jit def compiled_generate_function(inputs, end_token_id, state): - ( - sampler_variables, - trainable_variables, - non_trainable_variables, - ) = state - mapping = itertools.chain( - zip(self._sampler.variables, sampler_variables), - zip(self.trainable_variables, trainable_variables), - zip(self.non_trainable_variables, non_trainable_variables), - ) + # The only state we update during generation is sampler state, + # all weights are fixed and will not change. + mapping = zip(self._sampler.variables, state) with keras.StatelessScope(state_mapping=mapping) as scope: outputs = self.generate_step(inputs, end_token_id) # Get updated sampler variables from the stateless scope. - sampler_variables = [] + state = [] for v in self._sampler.variables: new_v = scope.get_current_value(v) - sampler_variables.append(new_v if new_v is not None else v) - state = ( - sampler_variables, - trainable_variables, - non_trainable_variables, - ) + state.append(new_v if new_v is not None else v) return outputs, state def wrapped_generate_function( @@ -113,20 +99,15 @@ def wrapped_generate_function( end_token_id=None, ): # Create an explicit tuple of all variable state. - state = ( - self._sampler.variables, - self.trainable_variables, - self.non_trainable_variables, - ) inputs = tree.map_structure(ops.convert_to_tensor, inputs) outputs, state = compiled_generate_function( inputs, end_token_id, - state, + self._sampler.variables, ) # Only assign the sampler variables (random seeds), as other # model variables should never be updated in generation. - for ref_v, v in zip(self._sampler.variables, state[0]): + for ref_v, v in zip(self._sampler.variables, state): ref_v.assign(v) return outputs From f89bf90cbd892698e1dbb2861408b4a16dba2046 Mon Sep 17 00:00:00 2001 From: Matt Watson <1389937+mattdangerw@users.noreply.github.com> Date: Sat, 13 Jan 2024 17:27:11 -0800 Subject: [PATCH 03/29] Add llama tokenizer (#1401) Mostly copy paste, this is to unblock showing a llama conversion script that uploads to kaggle --- keras_nlp/models/llama/llama_tokenizer.py | 81 ++++++++++++++++++ .../models/llama/llama_tokenizer_test.py | 46 ++++++++++ .../tests/test_data/llama_test_vocab.spm | Bin 0 -> 237763 bytes .../create_llama_test_proto.py | 32 +++++++ 4 files changed, 159 insertions(+) create mode 100644 keras_nlp/models/llama/llama_tokenizer.py create mode 100644 keras_nlp/models/llama/llama_tokenizer_test.py create mode 100644 keras_nlp/tests/test_data/llama_test_vocab.spm create mode 100644 tools/sentencepiece_testing/create_llama_test_proto.py diff --git a/keras_nlp/models/llama/llama_tokenizer.py b/keras_nlp/models/llama/llama_tokenizer.py new file mode 100644 index 0000000000..7acdf8687c --- /dev/null +++ b/keras_nlp/models/llama/llama_tokenizer.py @@ -0,0 +1,81 @@ +# Copyright 2023 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from keras_nlp.api_export import keras_nlp_export +from keras_nlp.tokenizers.sentence_piece_tokenizer import SentencePieceTokenizer + + +@keras_nlp_export("keras_nlp.models.LlamaTokenizer") +class LlamaTokenizer(SentencePieceTokenizer): + """Llama tokenizer layer based on SentencePiece. + + This tokenizer class will tokenize raw strings into integer sequences and + is based on `keras_nlp.tokenizers.SentencePieceTokenizer`. Unlike the + underlying tokenizer, it will check for all special tokens needed by + Llama models and provides a `from_preset()` method to automatically + download a matching vocabulary for a Llama preset. + + This tokenizer does not provide truncation or padding of inputs. It can be + combined with a `keras_nlp.models.LlamaPreprocessor` layer for input + packing. + + If input is a batch of strings (rank > 0), the layer will output a + `tf.RaggedTensor` where the last dimension of the output is ragged. + + If input is a scalar string (rank == 0), the layer will output a dense + `tf.Tensor` with static shape `[None]`. + + Args: + proto: Either a `string` path to a SentencePiece proto file, or a + `bytes` object with a serialized SentencePiece proto. See the + [SentencePiece repository](https://github.com/google/sentencepiece) + for more details on the format. + + Examples: + ```python + # Unbatched input. + tokenizer = keras_nlp.models.LlamaTokenizer.from_preset( + "llama_7b_en", + ) + tokenizer("The quick brown fox jumped.") + + # Batched input. + tokenizer(["The quick brown fox jumped.", "The fox slept."]) + + # Detokenization. + tokenizer.detokenize(tokenizer("The quick brown fox jumped.")) + ``` + """ + + def __init__(self, proto, **kwargs): + self.start_token = "" + self.end_token = "" + super().__init__(proto=proto, **kwargs) + + def set_proto(self, proto): + super().set_proto(proto) + if proto is not None: + for token in [self.start_token, self.end_token]: + if token not in self.get_vocabulary(): + raise ValueError( + f"Cannot find token `'{token}'` in the provided " + f"`vocabulary`. Please provide `'{token}'` in your " + "`vocabulary` or use a pretrained `vocabulary` name." + ) + self.start_token_id = self.token_to_id(self.start_token) + self.end_token_id = self.token_to_id(self.end_token) + self.pad_token_id = 0 + else: + self.start_token_id = None + self.end_token_id = None + self.pad_token_id = None diff --git a/keras_nlp/models/llama/llama_tokenizer_test.py b/keras_nlp/models/llama/llama_tokenizer_test.py new file mode 100644 index 0000000000..9a3c225456 --- /dev/null +++ b/keras_nlp/models/llama/llama_tokenizer_test.py @@ -0,0 +1,46 @@ +# Copyright 2023 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from keras_nlp.models.llama.llama_tokenizer import LlamaTokenizer +from keras_nlp.tests.test_case import TestCase + + +class LlamaTokenizerTest(TestCase): + def setUp(self): + self.init_kwargs = { + # Generated using create_llama_test_proto.py + "proto": os.path.join( + self.get_test_data_dir(), "llama_test_vocab.spm" + ) + } + self.input_data = ["the quick brown fox", "the earth is round"] + + def test_tokenizer_basics(self): + self.run_preprocessing_layer_test( + cls=LlamaTokenizer, + init_kwargs=self.init_kwargs, + input_data=self.input_data, + expected_output=[[3, 8, 4, 6], [3, 5, 7, 9]], + ) + + def test_errors_missing_special_tokens(self): + with self.assertRaises(ValueError): + LlamaTokenizer( + # Generated using create_no_special_token_proto.py + proto=os.path.join( + self.get_test_data_dir(), "no_special_token_vocab.spm" + ) + ) diff --git a/keras_nlp/tests/test_data/llama_test_vocab.spm b/keras_nlp/tests/test_data/llama_test_vocab.spm new file mode 100644 index 0000000000000000000000000000000000000000..d753476f535c762c5646937ae2e1c783a225afe2 GIT binary patch literal 237763 zcmZUc4P2Dhdf=a#;X~z1ui+A-h}N*i8f&bvY6XlnUbTiAuc5{o)>vcJ60hMZC0>Kf zz`#Jz5`u&n=_?j689A@|!W+v8HW1XpS4Qp7#>XcYx)lOW)8rQJy|6r2d-R1ZD z|DNZZ_q^x(IbZMl4in-hM5g8aDkmdR_8U?-};* z+L8G9uRHw{5+;np`PaYRoBylCcM~H&{=s9%wqNhx^~fV`g1X^v_P+VZ89N~o=k9&K zA38U0_x7CM_xSbRykEUG=JB&BkLzYk_*g5SYBir}wO?o-{`jYP!t`5A`?%FOb&|t?5s%QfXHJPM3RO? zem*ENv6J{a=Z~cS@yL|mS5kR zEO}2&l>IMEl>E;o$=^LTQ4W0eb9wXIiL#wEe)H|mmE%RM-1{{ChtdBS zS-wCgRp@~)M7F3n;nWkNUr*iWcMw+`ai)tDC+j3IQ7b{pRWyh2`*mVAM9DeA>O08; zTtqi;%|e@AMh|?gli;gb$s(>6+HDJR8|;K(6?QlAd>J7{lQeP@_a6N3B6U+m9|I52 z-PBRw!w3lyE-+3by|}I9v6Op4GFPs?HCu+u)@c7Fc8S)Ezo|+i9;6fWgBt3FG(OSF zzdiq&whq6^#2fx@r`9kjT8i#yotu`<(es^AtSisDDwf41LVno)YoVv zutX!l$24;Iq(%zy_mdZMqDD>~O_F~&{+OILCdnr+CdrvwN%D{SpGh_8RS@nR{pg>r z{+xS%E=#EM^ZhgApN~(G3xOx)voBKR;@3|~=LU^raIJYLT267_f9-!#20xuGUwr-t za{0U2vQXvicC?fbU*Nk4nT2~U;Y??v#7RD?xUU~M7yo^PS%$3OdZlXX`%w~))DJ|7 z;bhcUeFP&SrDy|X%!?Mi9sk}aDP0sTra94KTp?0L9L?C>O5!QI5iRAxXsN}o@?^9e zCarq(+w`U8>?rA=PnvPJf_X)htmB?7peN0Z$n9XyjglPXKaj`3cl3*=b#fZrjE#Sa zJe(=w;{Im(i3i!49wjE?Ev*r0NhVL_A}8@LYNc%kMC_Vq3F5AzZx$WX$T{K&#zxXd zBE^88sYD}XUX2_i&2!X4CwVidWBD~c1jNSMmCA{{rVKd#*+Zu1=aO)BwaX`}+b=0)_EkLWMlH%Rzu z{8V4Og5Ex!{*DZM&Go0WQpYtj3WcX?+O1g{SEz`@-NZJ zW0b{kO(%D_$8v0m`1A!FCx-P4oqQ z7InK>S#UP(YYm>}dvZbYTAlI{6%b^CX?zr9KVwG}1ZBe8HGv zU!jwm_?a1#bknhA^t;5LfPKLQK3l>5Q=;u%-c*s4I$)oWAa~sl-N#AK?{K4NmO8%D6 zF3hvD)HUiO_?DKw6eqK}{tEH8oJ|nZQjz~g-Iq5eNEH56lSCT1HiX}C+<%R(?@hq2 z6XY`aEr(xsm*M9##$3^@;Js3*>u)Q3ye59xnRnq9GsyRp&~q)ktYY$R-@ zAwdq8Cy27!h1|Cizvalzv)oI(TX3%f6YaDexu1M2<+?eNyxgZvT6B`5!r*s!E$w^< z`@;Pj!mYwj^@C#E2O(gipM5HFl`@%jiwq+TUTlzh`&Z@HtC0=dQ)PM7pUm8|33nO! zu%g#hiDcuxL0Dz`1L$Yr6r6|RRm4Rc2GCnHvXV9`%_ZMlGm^K8M3J{CM>%y}M%YU7 zzng1bI7}MNjM+_VHL{DmcaSIZX6A-g`Xm?>ltmBrT#fvSFcuZ>5$ch2)`6M&-j3V? zIbhgBc<7AO$Va5P48ID>v6uT=NUsjP8n9^zbdYasSr5fQojCFLz$pmyGfps1mopv& zrm4Qn`rtHeiroa#U>ExHS}D#HIY)TYAZ<5~v?-&3epEzSd8B!nYZqZy#Yvwmqdv+) zkMgM27y%>kyr>cLDxDaYGNuvkt_u4l?Y2fM`^f(->i!^cRiN9c169TcgfZY}zCc^! ze-Hv?5n{+AZZN8AF5(yF{gwiGt z{dnd%?hnwH(+I=5A$T15d4dGreB>8gLYQl`I}}3^T;g6MQmp}26MwmdIb@LfI~y&_ zac_e4D&1!_@*QFIL)6y|kzc6sa+JIWao<;U7%h7Vw-d@KW8f*)3+Vaq75P{9|97N5 zo3K0RHyZM=4V}KHUn{bhaLLqFJ7Lt?WIp;L*owai&af`}C21BVQ-24Tzo$eq7BZgE zpGauVwUyGFXv z122*v)_4ZmO@EfUM5^&$KT6vTXrzHKhnX+R7m55G@vAKN` z(e-L>VZ2DeZ*2SziM+*qMGyyNanvX6+)SUZCGNk+ZyEJzPSwZ?WXER43jCEFok7|Y zHS#Y0i;1rkzcRwlMxPJOw2eNF^ymYo1=QzS+HA_Bxp_I)Cvo3V!rM1c@1N6;ztc#| z3icRW`_Y~aw{`LXVc16K50kfV=r>L1hSS(Twr)c&LSN54#RX$6ZY_h=`t6$$(6#~#Pwr9Jr#R#m!OxY$UWjy>#W~W=5nq-#I5Gk@sux`v0EQU zdyTS>M>h|qi21;DG1GPlT(>Vuk>+jFrJ3|T;(jAx+mL4b|A=($#Ba={t!Ji4XZm#M zB+d7^ex33pa(^ELx@hl?X|fUjPRg?x*?@lv!Bd)p-seFjtv+h zqhvk$PS^^2p_q2i&*U2A*Mk9lIsTQZ9>|x0`SsVNX&5Cw>b(K~TIdG-Os>-}&9s{z z_Zc`30Tz$J=d{v2O($=1uZg|&75p#3Eb2d4ruq{5j-kjgyPBfv`Knf~bG?i@F=Ub# zc!_(%`27{KYH76mDdEaaM@tLtw^_5?<60g0H(iUCeDZ8y{?GS_A7j zLr{I8G!!L=xYtNutR(!~%=s0^SU((KzN=@PrS2*>vv$}MEf0x1pl6(UQY#&-cb+3o z6Zgy}|Aj~s@#v3GU$k+6z7_mBLZZ2L1b_3|XsJdf;5K`sB?&o|Z2TM9i$Q*5iI$rY zq#wyy5bl<3)K47tgZ?SQ+H}?{_T2=YNByQ_)7i8!YjI`&J=|vl6M7l8UGCDz+g^=S z&PkMh;+vt`|4r=o3(g7X4{FVIoxBu{a{m;O|HO3@{brU5_bhhtOoaR!*Bf>x${hR- zGrt5me+gj!ooh8RlWeH-2z3th;ju*dJC5mofuC`NazDlWPe;f<;^$!eZYf}Vq8x3z zHIhXcx>zr`=FmUJYb27m|1C=N{|+Z2HR4RyNI!9SM@4*t>;;IUN5lCS^g$YFtc227 z5of(lPL%7!O&GNwcpd+}kVqO|5#M!K!nN`eZ0)OP@i*#Z9O?ZxRURre_*E=@H&)75 zFxO@OGKeAyzXJN>y$4U4IdnH3W>;HJEIZU4@ zkC)o#nNz5T)*j(-2s^*cnu+l134RkGNoVPUIcrfJPkLzf3K2Xy#+28g2`*~*+j*q2F9wUz!XxbQbHp5R}+JA!jK+PRDI zM8(OxYa?v6CW7&Uv!-upKgz1|b(A^v5_z}`S72=a&bnKzQ`H%ogZouo>sjBaeOhoA zeds80OdQX7>dz(cay3&PvHrD%Jm@ztr^59a zW_C-`mDj! zGn!zR(C^7FYla5urer+l(Z~mc(ZMjpz+{*T2@nlV>MRMF0)Zavg|&y8P}P}>>i@Iw zS8SzO=qDE;bHv^!yyZ7t_oE#b!Ok9Zq?xPKk~%|G0~QQd!o z`nyBlB|eApSNVr6TwAHG@8LRis`^T?MI)0{{wc3YKM6esW)m*(DfRKCR!&gniz?lv zRFfvPTq}eiYtRZ6 z4pckUA&X%d>C9Is|I|l$3VcU7KfwmEy&m#qU|!cB!Tkw!gguvRAlzEDZlS+2w)}X$ zpx>P#23Svg>max!O#&~aiQz(;RM%%nU=3ka9Wu6WP7^MNoDmO}21N z&2eMn*i6zIST1#xO@EtuxWjpjMPwarV=HB4Ev?G75qH`B3@N#@T(+REhmzCFS(kBr zB7Cdn0os|q5bS53=d3?)Rn7O<%Pu6(V%1(f_$L0G`3E`s4}6XMl=W}saw#6A-FS9t zxS^Fz#Jv^tk?B%Wz`lU|E~Ng{Gr##vVFu=OJ#&PrzoHM8JO!;`&SUh zSpL~x6j3g9COCF|4tg&2i!H11K;2(WST&}rzEz1{3;Cow_KfDMbjrba^ELOx5x{XF=qc07eF zqwGZ&xOb2+K}bcX4wSu~<9avv;S5xF(*KF$Jo=cOCH@e89^unP3RQkpKOsIf{}mh1 zKTDToGqB$o%6=IuS=ZMRZV}~ZK$de3GOXew4mGcgm17Ujv{ahhS4Eod(ATPmmdU%C z2&p08M{AbJU9P`Jxzt!vO20VN$UP0BvZC% z;)p>G;trH)WfFSuY6g9sK9Ak}9sZ1^`qlUsgPw8*Kj->a8N$+y=L6_H8LZn_53oik zBAwFRyt}})66So>zcW<0uc_CxVXKpf>p0gtxUY-x=M&_|Nd2R7ISH?xO$6R&oOluc8ThL-V%4|-Z$bp++AMN< zEI-uSCmMMX{Ty6S_dmw{E>!o3S>9@>En zaMsRiEu>9;7>i1|Pwms<@hdexBmYEO>p2%F9gUXXs&LFjW!TI|=((WuHs!yD__2|{ zMDKvwk0a$h=9HSpiJ$hbewjTnX_k?vcfL!LQtIcoEsi;&Ne)^8(tUPIm@d3K4r`)l^B(;~$6 z6#nevoXCm;q+h_Z_5Bf2N?-GR#C{_Wzhs^Ra!=n2v^#r+p6}@+EyP7V{saE%*_27G z0heec2X`PsD;4CalI!`nE2of8#);pcA7SraMx5{S9Q)YQI;p|$17%AuXsHk4|Dpcg zSMmIi-(ICFf8y&$H;wRK!sj|^Wj-CoJxG~(?MQza*xu=nE}}jchz#!1%74c_NZdy$ z_gCmOGZ|;FhrgW2UhU^>jgmzF1yaS4j_&$GCtd8FRUCsaYvon^)V+f*5eMln#(i}^ z=Njz2`xB`f^j^JfIp{0zzvPdW0={k9re7cawvf1lrG6rE|AMEA~0C^l5N;L9Y_I9S@TJhOM-m{3* zdTIY>Id?m}SOPsunBSI2Lybn<^s9Qp9v|V%rBfsAZ*tB?x}U(YbjELF9ewA6@gn~P zeGsm$(W-ao#P*6-?9`(?F8c5Bk3jxA()6lMMvxBL%ZGi`Qt#uqcGaepSY+vU?164k zzAmjihTB1W?eEYgX@sY}CgN91npeMI95L$TJ>u|7tkmxjIlhhLxZX*Ba=xgQ8p2Og zaWWoWrCyx$do{m4h2M|YpGKbnPl8Ib^f2jTLn^Hr?EXD$?isFieNDX*r^@dV74~zj zbbU*nR9-&R%CkTC9pnCPu5msmY4|+{Z_^LEpT*v&&*yQk0VmgA_(5(%9;eRgVI%rq zfqI6%3%L(|0|sbVOMgaI!t3xJ)Wa1RfN$Vm;Ug$!Eo8W_krU{>@G+c$e+Db)yB_tM z&^YRseBHy3cckP9ZT$c}B=mLq!0^$DqV^xP<*XqdV;p&tF;*gE)1UHe9=5`E*a`L2 zRrv5OIo_y|9Nc^1cgNVHBlF=P6hd=3YZRmjTGz5hN7`Ch;~}e|7M!W{U1S58p_~1l zdnxBf3z$1`TcP(p>wrnD*QaoPf!nFVfd}**%spkyN9e`OQ^uDVD_@C_Q@D*h57!Sd zpUz`_!Z_rkZ=J*cJeaVh(&G{0WbVC)`x5vluWwqUxLFgp26b{7KM!+o-yNPUzI;(* zyvn&DYZM>Xyu{lNK03Yy+ii-|$rZvI>4?L~>tN{5^O855Z@fCOWp`(GUX(Fd33yDi}KUP>Gz+ zP5Y%3eUUFEGd>XGJ3cXuJCBY0tCYXKSW|ZOX-&oBhcs`0F9LeDfHb=VF=7_6iucQ$7)i*dpM$U%0Fdy{TkvdOO_PG%EGHA4D z<#(Kin^#23M=PSlGB8z|hNepM$tR?R=YFlB6lq(VB38{rv5ia?yM3~>YaSEF{Um8w z5GAe9mKi11RZ(I?+S#+WABhr23FrBG?EG<^EGM0nuo~9Ede{g(oa6WAMN3~nw0MyH zxV@Y!_;@zphmO`Lp3g!!P3JrbX~*4;bYRye?x}!EsD@f7^3n#Nhn{-Q)9v&x3-63Fw)W@IzY4Gw zq#t(&<3#5ut>lkF!DOwgz!sI5=PzyU)K|I5BUHdLg}t)Vq#3!rC}=&0}1fsIf&}7lW|Xl z6qp6Gp(UAldWBBfzzR07uf^VTv3D@eNS3*Tn-2?NF)V}Suo8N1Ft&||^o_D_yhHt0 zQUBm$?DRuNi06ZF0-VqVuG=E+LFO75j*XVk40whyd|b4wCeC%R9yY=z*aBN&JA~hl zmYv8PrI*pC%eAr>eINMrPe?w}jh_n+qI-7Gc2J0341U%aKHOf~t{=M7qr?p^=$grU zxQnnA=%n9vjE@pOec87mO1#K^Y}2y`n>Y|9y}41+gB;sG+ZhimoEZ>*B~(K#)PaL= z?bwi=`)pt}U=PRW^HovOf=nWe2^zqBI$6v}3$h8C(MtvUXMNQ+XS!IywsgAKp&i|k zI$fOTR>C>J3Ekixoh~l$pnC?VOCRqG_|Z>+e|L)bwxx&{*?)4nwA`64t z=^RawF2cD8??!eHsrL?2Qswiv_x2{0LwU@D}*Ea;&x_3nAp$2^P){kXljRUh-yzdJ09KX%3+ zq|?jzgYF`JcMW|noAF~HS!NT@T$m3Fp(&X)5ADAg-MU*VZQHc641GB`HnR@G&knX+ zt*pdvHLQd6un{)F7T5~EZ~xnI=fGYtv34(gB|?lZM94ne`CyEQl;Ih?Z;c*m!Df)* zqfbaN(gYPy3Dr;wb>M8&NdwXh&0qz00qYNNpnEp6{@B2t0lgdij1xZaLO)|o%Lc8q zQl>V_qsnS?Q8q7SL^|y3#ao!y)O~OY)Vt$nkb0zgUeSX+_d?%z+82HrpV9r$F~Bq4 zA# zx8z6bmzDUn++i$&wm8P7NX96zk7vwcJajM)t|rVn7_+~Lynlwi5jMdV=$XV=IE6ei zc6y+HgmnRD7(S%`2J4SZ#?mat)NIC9WEXDN0>)hE1~cz6Y$cxUuoH5isXSUNM|l4d z-OBrmZMgTL=YxZI+i}|stRIjE@pFb)Um%OYeT{btkrm*&L%zJ^i@L1DT@7RNHn!Pc zLme{yw#={N54RuLL42KYl%INaUZelf?k=Pox(Bd-OQcwd(*aKKKvQF+SX`0fN4K7hl(zek zQp&@A(==>qCbqE@8(Dy@T!@rY>b^T#IfFb0Zl1HBM_vS1i1jvexaWXQF5$imz6Cn* zr?d9QCcV&-iv2C3kEUaPNZUN(u44WN2VCKvVYm)A!N&SMjJyl?py!y5^%pj7A)P(M zn@7CZz7Ofg-El`Jo%eaS;Ur@KbagT|csU1fvHxt<$phke2)gCi0K`B7OonjF6XN5= z?9fr(-NQXRjrlf?{+mnxg{k~l+U_ESvO|S*RZ%!58cL}l;eLL&~ zCvBdCbRDC8(f2BUY`5KeFi`NdH>epM%)hT+fGtPzX)v3S}jUTny*@0`X{AJ}vy*TavGmuh4!)WL9CoHQUqFUN`*S;|?F zsh=}xbStzFPb=}XosAbO2dOp=NbM&{0~{mqV&u7ngK$ph1`qh*6r6#clL^ubec*w9 z@PZHgBMH*Mdk3AX;^hPf%1&fgcD%SKpBuWNWdrGNCVj$L>q(#T+0omP4sX1iBaQQL z5iY@HxB|m)9eT)rFZu7Y^De|H>X*9pB7LCN=N+d>KS=sWCuQqGx?H4B+}%iLWW3xY zo-o{nd*F^E3_L*hj3;hQygWqLJxAGd<`B59N9mB{S9G%NPiIfyTJShC%_3^8?ZlcLpRJP4(AQ6EJXVB5uDFy zWik3P7#lw(X~hG}ajyhl74bs9Mk`)ejo&&jvj<*}v@mCHL~eqn16tXF+zM98*#`DZ z$_Nhf*bXfNlz)h8^Vn~tGAE`pHzM0{JCNJCcPHe)Uf2f*p%99pr-bphobk6F`>LV+ zJ86IL9bo?f9rTCJYplOQtiSFv{@!5x9l|~b7=OLY{oi0)#54R@yi_26+*ekj7uh-6 z7v}X_ky4Gj7V4k@!mq}Q8QBb0D0!to97rb&zee21P#1YZhChrKKeC%~&iEZ;Hu@Pj z2TtakeB^n!2$#T(pNqNoGPek*)#a;%w5p$;=TGkjK{p+_GAC^y`q!r zge(1y{`NZGZ=r|bF5H6$F#LRkJVb^LvwycliEagBFT_9sG~s6W3n#yePtyNCMer7p(idGg$k{?quHiTz(- z{)H0*%)eagB8;n-^+zk~56Wrw^1dkXYy_i`bJBPC9w?6TCvpygpB32#rQ@hip7%Nm z$XhOXJV0JGQL>dVPCNY+xf9&2%mK)~;KILe4}Bm{Bl~dYgD;zLX%YQ&2mLgQ^#`^; zX8%Yl*uc&h&|Z&?)Ub!;oD^ygy#-~&H&+{XS#uzzr# z#{RK2mrE<|PTpOGI^t*mv*H@h4v-dPQ;7VaJHQETskqb0-#qe{O#TwdUjcD&%>#Zg zKKGXB6aQm*VLpWDKMRq=&+~31Qk~uE zA*`p*F6F%#{8mCV_O}`t>L`>mD*eyjl6ANj!+I#~p}%oHzY%>CY=Nz?9d<%^##@qu zbe0#$UgSQ=hlAigQo#P9Knl@4s}4yY6r-EKkM09+)*;p+hxk^dKq~OBgledTrpyAd zKpncZK>x4pSxmGBxBZbm7S}0be?RKHq7m-H#A#otBDot=gH+aAgr{E0q3?7o+ z5b1yi`ma&88nkX%6@1_$|Sr@VIRzy{axQ{}&j41@c$M(!f-f$IYO@+|!l9^ifmKKiG> zlYWYy7g~l2D1QOvFW~&8Ky1nMe+~VADg7UGE9ol`0|_v8j)xrE$4z4YI|=)rg8k3L z{-J+>d3lI&4CxQ@d~mIL#~J(IjQvAbCicGo`-g7ualT~EEf{+Sq2~Xl60J-n{uHoO zVVC9DC6w}9*A&Z~edLi{+VHc(EW#N-<-MaoB>Nkk%*O3Px^L@bF8X{}2z`6lAM9Y= zP{8`hKGfWXHb-i z4S#h=s`0Od@YO?7hjeOqR*5YAns4r&rVQxrMT{|#oF${HGts^f?`5Jpz|XwpyTJJe zarWb%gr5nV&<%~62>Eal@k|l<-RTHvVZLf*j%vHX{5Z;-bDOzmfH`Q0`H}gvgL$+w zE1L5)&O-NazMseWej?}l$(-*ad*<*|OL56=xk%iX zpmb86n83(G&5|R#_;z={RGj{`T;|#p`1_$lQn{c&4yWhJk@{SDcS?a&dw(OtTnoQ5 zUf!c_YB%tHUG@Q~8Gk@N$c>abZ=_u3`b`xELc9-g`~c_8{4Pgh3HFMgnU{8d$NS$F z%{ac><@=V_@zUfSFRhkDY2jIo71~A<#ojwfY@L(D!E>GV#);B7I!!vRC5!XM6gi=J zTwIzbrK@0y7Izl_6yJ~2g-O-$zv5taPXa@(_Kz&~kO&T$f0FxmJra}tL zf|f~<(wfW}!&1&6z&0;Z?9^ZT+DLJ%h?LocnG5q_AuNVvupCxGPd4MvZpNRJ*f;dw z$G%}~pWtHsF`jQs;Y5h>2fFeYe+n3XkllOm>x>X1&v1Qwn6R4o)`9y%gt#mbvL4+N zj9?Clkd5e@z~4*Uousjdv|tN<=8*{5irfxOH4&17+zXaa1mjl(;}>IBOPrJjVx@hA zbB$4+bL8@z1JwRM^lY@SwwB?Sd7l^CGz`&R13V8Vj(j)>g-{G8sDSWs#)D6>3-oGm zF5;Ox(&gfr60!l*GbZ0H_TA{7D&~S?jQJZF`^%XFkiN}~|F`3$bu>=eM&iV}hkaol z`@#bDzeoq}o!r|DR&an5y1@f}=ow-EJIWpv=~==48@y}Tf9JCQhK@w`zsc->k$g zoG#+_k_OU&dzd(`!%YaoUAPAi;352U{);_(pno@Y489@ic!2fa0Ohw(e&TXAQhsz7 zy1Sn8Q~&1Vc+st5j)i;JWDK$?o$^Bhx^)|Qqr8*Rlfbc?yy0ihBd-VIWh#CtFbihG zT$m3FVKI#DR}!h8WY&YYm%~ah@_vFEABG=Gkaf62SL4LUTwK~gUww>oR{S=?CfEX7 zVLOCBPLQ3*9M}u{ARi8bbB|66k?QQ-oyWUX=q9KDbq4Q&e)_+c{_nfN{2SsMLatRq zE!05+n4uZ0(2~pg{D4{q@O%(#$FLh{M|VI6wmx>gm&y6wD$e&-aK5mX^s%jO=$Xm+ z1oVLi`oRl6@PnQ;x_TE-eTSl+KZL0h2kAP&H<9NNpYmK0J@gsvi97{o;2fNXHu7eL zvGey=8MjWdE;!3Nfi;kQfZsDBjCmD(7kL?)7_Y7%hryD?dqT*Y5Qe*O4<5in2pc); zu|-JeGy2FcxaTpx-$lki0!#)c{VWMN6;fapxJQ})VJ^Dblk|b{YJ|+k?Qf*rz4TG& zhn7g@_vA-=OzXTydun?+^ZO#^_c`oQ$wv|6z2O>T6U-)#g-|*TyXv5RQ`!HXX8#*x z9!K`GF7P6Kxc$hE&Fp{S1aUc`YbpERRQA8nO}gg#C|OK=%V0UIgeEWZKdeT#a=i`K zp|1x=h`f!m1_2vv#Lr3oHX*lw`!?tK$nD@FzkNrT-&il~#GM1aJk}irtV0aUZ^#z% z(MrDB$d?st=yqs7K>72cWH0yZgM2s$g-{G8sDPdU%0EQ;$(M)x^k1X=w<$kTot1RN zQT|DkADsAijZl8Vxslzt&7GW~us<_kqv{>o)zo)WE9GzGnE~-ziNEb^lvJbpcCr5_ zkL?S{Bhq#{O6u@;-r-%NU60NO+}W%L@iT)fi?(0OxCG6(t>8@l`Q{Ar9Gr)Xa0xDhnK|_e(lV%(VdQmax}cSt z$S_z>W5ZzY#Wq~n2`hM0WA0C74X~8Ce-di|^cL*3 zm3+2=^%`l9l7~dj|B|VjIDVIIHTDJxFd33yDx|qa?B~(K#)Ip0SMOy7VV@Fzf=59mUaknEKiTo}BVa(7BR&an5y1@fI zaXkN@!t?)`JpYG&_BCp6OoJ&Vd_07o11;xOjg7E}~xoKhO4kCDX-QHC_5G{BFP;&fj-a56rL1hMUkY6YdIF z(0_cdVHmgZD*cLg9vngHE<_y;@csjB(0Yw`bZ&G0jI_~Kc4Rwl2h#T;zYjt`zs~(P zp>!g4zMtQAK)(w;1?v2XJsx=g4v+#O7*|F z=pO8=k3KpdeIfX9`vy59z~1_ii}70q%fWnuzKpcoXa9}98hstChqeLilkvco_-Kq6 zJO9J2-$uQI>VF#vw*`z6!Pt`x^)Bd*8$G*(UV|o{#wiYwSxHz z*~0bK&4eL-E7-se?cji^+>-*cU^YBDf9Lz9PS!WLdotO#W--raGtVRYaeMRF#}+Wp z=dy1-&Av6rJ{IX5V&95%;dUdt58!trTILhaLNIkE)BedgI&-HCtloQ!P*;{u| z{yl_4{=el{@BeJ#zAdm7w!==yfxXaEMgL|TP-M)&seF1j713S*deh08$wcibKMnxF){*MEx-v4nT z)mYSx9DDy@7wdn<8V_zioC4<&ot#0Q1EW4x&LiDk_5cRnOGfvwFX)3y=$FBd?gKCM zL(2;6pM12DFZHgBZ7umKVC_LU9ldJ(5hqu;Zy2tF@%31_i44PC=)s`(3NPur0W_TEXvGaKf@d{_vJVHqrko;;qv7x4TY>EZVT`cJdJ z4zj;S`bXLS6K5xT=@Z~AVgJj%*Tp{F4c*{m|GScSR>L}25AJz{fsN>%T+SBu#LFi1 zE#R*q?#(>^-oW#3_Q6~6GvnTl+zCzWk8_ZF!Ghj4hkgB`c-eI^~3GIJ`_Q$r=IeW)u z><>=dRC5$(GkLX`OT;#($lem!{;#Tka zG}Aw731^-XFLlTUr9(3`-RJ$^JFF+rt=N1UIMAKo*hbk7P{v%!itNTO{3UIN^usAQ z1Lxp8T!c$-8Lq&v;_G<1jtr@f(%3iuw10k%^*;V#xC=hD22}I!^Soz+`yuGo(WfB> z5+MAfRwg5pU@D}*ESL>*LCwGOkqcol4D(&mGGu58Yha`^i{Fw)u7-869^A>;KWs$z zjK}_$>SPo87WnD?zx%9}CSm^u>>rzHtH=I3v5f(2WC+_pI)aQ(LCy!pGd^)1zLop8 zLk{@(=4~&shqmp7zTLD9^i!u^@S*z|4?0*|bRNO}OR)bvv_IE=djF3wr8B6Xag4#l zl@AA@5Q;(Bxhj9?72XlRJxqO9AS4nyP1DL&Cyn{q1&Ooo_VS!N_KKjH+aAgr{D~ngY(cc!v6O@Yk?!|8B3TG zxzCIAf#1dc_cr@q!kidouY&Bt?YhAJ7rMcDhy5?>W`m1y3@#F<`vze`#KE_5o(04` zhj$e5zYP9F{Be6{k{0p`erD1s<($R@O?KW{$L~5=EI-Vvd8})2hoOBB<=Q|wk@g+T zzl$jUQo7D;y+Hqk7)XE~$~*S_pS1qJ_NRS+dj7A<3S-az zy_BE0yUD9_JZmH3NrI`60&WfE$A1>OCz0~QZ1lO{r;U8Mls}U4!+iYAnY=@WyoY@? ztzxf;z6>n*w^3HL23wAMC5)Zxq*KlnloJf>k<~Xo#xlN{WzSsNktQa-2lDY9&^p4e zhmEiaw!l`{4m%+ShQHyA1Q~jUy{e?iKJ+V{@R+=33&g&mHvP0{U|v_PH7Rgm%sV9nhin_;7+fzZ1I9UEt2<{ExAr zXAkFpoJID5hx5UH{MFh0*n5a`I8!7&2ROk4p;vVh{+91CpXd1{?o%+Fru-I5=zD(8 z2KPBQ4;P{IwdG>k#kc;?&gA`X)*1Ny_ zk6Vpt?V#Rqut7_5hP0+;NZY&&u_A5h8DdAa<8~ms$w$!$-}v)e0eYBC91CIWy`3TWF7i?@ZTnG@IpUq#LpbckWI)f(1gAfxg9KbDAy3>;}o7utQ~JsyHd~CPzTsJ^+q_6V*sw^B-FZgdk6R$MI~K@$C!dj;&GY1F<~*qmJuUAJ&XKCk zbL5>nPf3?$inxXz7ZY(C$FX0pN|a7b;@Go1r+3Pu^@knXwCJRQFlN?xze`S+Mm{Y4 zFgHy;I?cPfD{xQ#;|C1EA&ZYk_M!Xs!rH%8& zl=w$tobrcKQuT-O!L|&kqa8{=Wgh!dCw0UTdL=`Smt#YcA@A<~6RAG1SQ_vTKb|4) zWiFH2-sL>AT_!cze=4o@>C#f3F4o?3X|t!(Ch1~pOy>+eo#&hB;`=&6%-q`yR&an5 zy1@gzo71IdLpsl1(xq=lI?rCxSr?{@pE~j(I}Pd5LEKLKPn4%K_oho%Zn|{ib|cL* z)5K3ar{D~ngQj_DVuADM))i^YjcIZb{Sr8O(>S+G;~X=Mu_#S0<97v|1(X-5zI(im zyb0<%$T0FQcuq40fwz}2$i?|2ZK|Fl-s9Q>cnINF7>m_h$XSy5-ZFeRZ$bg2uVV3+Kl~g(8*MED|?(aNI{^EvF#6WO09vp*l@o>b;0(pd?sVIBPV{^5Fb5Bq8L{^3US zP2guw?ZeOOA}-j1pLrYa!6COp6Z%eM4p_*0TQ>O)@h(5^ebA2XU~g}`!}Doq>7;yK z!UYL8MEQ{F*+TmW^FRAIJ?kU`Yrvw@tm$53KgDkX+~PXg8$gL;t1=1+)fxfSIlMYcbh!``;gF6 zCy z*?&GCOBtAdE%bdaHqgj5@Ym26CsEI|uX>l!uJUW49q?;ewNzSjmr9uU?!rBI01rX8 zo<0jP;JZyf%A_w`pif+*uW-iWIZ3};#JX&Nen@?E$Fc6BpSjRaEMU!*&bo91{T5nS z8@EE6f%OI0K)qM=)BDe_$4P=p=hZlwj7);5kOH&dr~Q96?Vm-PWYRXY^VoMD$c{$V z|8~~@7V4*${eLU_|9bZSHJod(7YOl9W$1Ix0C*2@n6=St;+zZfVIdg#UB$)7Ww0Dp z!fFWf+lAq;czB2sX}P#@>Io zu>ZBQ|3%uo?2D1@xE)CSHGXICHop^y+(dlaVJGCkUg#NM|3Ae3AL+Tl{{KGvfAFFE zp<{^s{|bKNa20#~>;&mTx-!}SBfE*$nabXrc=F*O6oMN+*CM_@K=+go_Z0R&=oR2E zCvF32Ap5Hlq!K^#HulWOT4+MALpFdVmvUuOu9N()0o={demX%MLH7UT*N$wt!2W-b za7)=QgKZ9W0qxkZ`u^9-Jq~a}H;kPJuuoCn0DExv5T=)~|8f4t`8zh^13z?NXPqNI zoWC)KbPZwuuKzfH^ApD@aQ4z?KBjMA(`Rs>1N9B?kKg}#$Xnk``qmWs82R%bAp9ie zc*c%z20IMeGh@AOR*r5)AL>|6j0w2tCXD2)I*V7R-j9&QB)sYzy~% z2!E6y3z5s9Wy%lV|E^*!&f46@TxDl2ZO>zUU(fF!eUvD^x7e4AV|_7=@6fqtC9H;Z z&@-Zb6aJ zSrc@FnQ{6u@mzsnxDHJaN;DoLb?0*#dM|N|cGn;1!#Iqb$ z!fH@w^R7JJc|cd+l=Q)R^o`(OO5a;R8YPSaV427B8~nF`viq&b?O={%9fY)#KO2ml z`!NpW;J28%vIgmkuow3}Fn+>(EZ<5#`tR4@LEME<3_kY87061chFU1S%6I>~Hx%YO z`a0YVV1{O}f&-k;4IT)6%()P9_-Xz}74i(6gY$3^F2QBE0>jY4Iesg&ffa0Ehjws4 z2jfO3W5x-jlkuYq{m1u@x*1z~*yHyuV(p!-&XQPvXjp$RR;X`){fq@p=Ev)#a}&aF z7u+)$hw(SQm@FRbTYcN`0RM;J$F_a=d7;0P-x15CpF#LId$er6*ZGF}cr%%`PqOHK zNq>YGNPx+pp3Nm8hpEe{$P}0bvtcgGhlQ{h!mRU+w|M^l27Ltg@Qa*zB3Hs{SO@Bv z>3ZZw*aTZZeLG>ne=GWr-*0b6_q9xu|M&atmIZu&4sBosTQ<+7SMiKLljqdPdz85- z$Zz_Oq=Uc>YEIg}sTvIw^a|2+H)C_mD{ z^ZRD*v4R7f&<0$1v_TMINr0WAebR<*$Ym^^ovxo zQN|zQZ3o8y`}MWVHCe0&!OWQ8dq0Bn7}f=_k^4&d-8~bGeP77fZJWe8Ad>kdjxiaz zi7;DWD}=w~chrzOAxF8(_(Y`x(YB%2x8~zsf<}g-{G` z#&s8%&^@k4Ytst!N>F=JpMm)UslF9ojbAm?LLD@K8JfWgEgIhIhPFK3^JedDV=lHc zC%3nC+;)9k1)RW&ijaaHW=TG`jJhUwzE zi|cM=_W^z*Wi9(a=-bTsJoKY`;m6-6QuFUA(kXq4=Xp=F_C|Nt69=3__mp!l@t#M& z2>wy-g|Xj1IKp!Y{LE8W!yvCf(;)je*++}&d_j$Bl`xnQgnU-dYXPrUoBJE$~KBfOnI>Rv%r5KCx zn1DiibF%Myy7i`nE5F5#{DpBl^l6A=Zp+9Sn2FgatiK_CEM! zcBz{aZ>!s}*l$Y_$Hq4yd0bvQWo}T7u{+{dVine)_8|Y3_;vIR*o1OyLlxfNe%T?s z3wuzZ&aM1c*Z*mLO5sDu@$*-cN02~|@-9X8+KV7fRv16uM)v)yGKS3K1Ww{KVxQg_ z@*FPU5?bZ)*#5pk&rX;BCY$R)zlQvM`44f7Ko_oy>-nv9mB?Ou3)~{_;2s{J{{q)& z%)wx?1Vb?#BhmK}{WnYGA5>rEqmrlMx=*Ek8;jcg%7D5dp^^_Fp&V%FR`wiJ1|hXx z8H75vbv+u`+_9g$@j3SYJoZ0Q*ykDsN^2a(V*(~&3Z|j`hVlQm-y8qGQ=j}^ee(y6 zTPE|uM<=rXr|SRjr~e-um63c^P^#BiF%q{evn)eU1w(4lVJ@#`;>bd#79A|_&%F&y?Ybxc1#!fn!Hw+e=je^{da0#^t*a{ zgX3#FzlJ%U>jLxd^-ad{Lruad&!mw|I(C!e%CQYq*nwRb=eh48>ztz=4I`w3MtX9Y z_k-x4ciYlDok-n(Ol zVfaa5 z*QIC3);W9`^vch<-*1`2VE?2PO^) z-@9EL_TTtW`0n%%g?;Cr3g2lS7WNMKNOn;STC{7KQq~MWF$SnxfFyT@;d@ zP1D_?kh)o<{r%f;&v6gXf1$bngHeK^Xx})%{Eq?Ve+&p&vP++1H<=U8->^3M{-V%~ zW3%|G&|<8A2CeiqRF5hOxu5(t442kOl%jS~QK*?*6vomMOI#bq(Z?fo&>R~i(TEA+ z(){U@$Z2R*-jtCukm-0>_GA}C|M&e+^`qa09GfD~7U+^ky2-lB?thjxi2ky~YHbdF z$K+mZ5N(W9hxYwW{Xg={Yx2uXY0k!6%*R5sJGR5Ion$ttO_R~K>DI;}^Uixz8>aa@ z{{+&0ZxR1!d_bG!~bIL2w6AhKL5)7&~WyCNTBiN zef?thLlaV%;+Qx$YU>E)&DjUxkZ?6BzTqCxzW!b1-@D4ccfJ32z5jQ;|99E{?}q&C zclBrA*FL`=jvUy>;#VklE<`^z4HBp%ZC( z8}j?5z1ulnyzd>}4_R^5qwa@1c>=YQ?}wA*Y1B-2Ur)I&oDn{Urt|NHlxuA4=RQ&Q z_JdIGz8a7iBOXaKA%zS6a|u__i)*-!KHNh4PWSJ6I`_K&r2Egfwhq@=@BVSL=UsX9 zUH)hHdiP!9Q{2C6Xmt&3j!Q3k5bj9lr2KV{e1O)&j`83AUv~e`NE-v`gOPtq+Rr%$ zx+cC`7?)N(>VbVD9=@Ym=P7aAP}JObH#97JHzZK;HD$j3;-;6}*8=ys-hI+XI^04_=NudPZWho^u8~=8>W!c5ZCT0BXggZk3Tmc%%Hc* zj~RZlne?nWy0cp!6@4!9i+nfoS~uCHZm%CD4-RAhKl8AkDvs@Gly6e%uO@jp_ceXL z{H#=C6<&@P7MYlW;kl>wWBTZ~kU!`QfKl5^q1cezK4SU@3)@)4fgeIb-Q{$OLhr&&wE(6$9~zPOW5aQ*ymH( zQ_^i2!L~;0W%l`b^9S}@_e6UA7pYq?7`4sj4|JG6K#xB8xc<*j`f#NDFRne16d%_j z7%498x=P8hXytPlM~+AILlox!tuV%4coGWp0))FZ>!(<+|NlAnKiN5^yMNbL7?=N| zb0E(Do8mvyP=*mTtItQ5ZrDbKQL>ce-`NEZB0v$TKNA8~%q8pjmo_t4kT zD?g`P{Tlzkv)X^pc<0l-`!T)&#PvrvIA#+nisY9tB=o93`hI3zx(}&G#P@xRKSTR@ z8@=b>*puWA?7|-G!vRzu|3ElI){xcY5!BKXWD1GH@^{jAUn763ms9e7TtnkO_5ZP7 z)BfLvEb=&klQ@n4zy80%{Gao#4UgylXuB8sf1Z=yoZ}oW;RBwg{ZDW6KDKxtTWh?(r1rjX0Jo%h2lrz7>h|2HtUqY} z-vi-Zdk6RbU%dzZzky`$5Z{#Yb1=Q?^VS#mqOl6})>F=Lne9T)%KM#X^*PW-BH!)Z zH~R)Ud<)0zS1#RB@nbO#<1qn~Fa>pkO0+>s*q`ikG}4o3s%I18=tcJR4fge2w)AcG z^4=BPVo?d#^h^C6Vc6Z_e|yUh`x z&qV4z+xI3L_%2&`>yu%&xOm>mUs$KOOPTIk=8Bt-%vFAYjr;<0j9n03jND{?foJ#$ zX7LleXj}kWe5v^5Scz4reb)KMu>a`^*V53<{-1F#p($RMhX5y|EKRhkN?d7IaC_MF%qTd`@DVZ$ja~X z+y9&&!Q7m<_C(Kb%r_IiWvX@-;+*XX!m-SqL{7mplp*?FVi`Pxo|Vtz*nye!*~pKQ z|L|x&$6Rqee{HS{+51`FDY+O+u^cP03cdW2YshujfQnyRtKjFp#d^LxM4w)AukUET z?+BY5Q;u3?!8WoAaZbo<_w~7xTga`q)s^CRU>Eja9}b|hU4HtO_fyROKZ5^%HUGbQ zvr)a7B%6?8zaM>$|Nkxif9-=mnqTJ}?SAXfe(1btd;lJ;+n+ng2e8+F{C>#!s&NDf z#JPEy^U|kh$M~=Rr|E6TKdVd>*G+a!mZrFBb$*^afm-@W@-%98yY8J2ultPfITWtj zy)>?MA7tGy`ERS|h6Easd|mn9?78BC|6IZq^x_(>qYt;xuHD_i@7hUb`D41sZs8o6 zm&cDDR{kfI|24}04&}dh^}F@A?I$9Q*OYs4%;y%|6MlgHUy&zc`>*i>VIaNg33>mU z)&i%u$pgNp8Q(5_0^*pJobRz4ajtsq-}{G2;`@H1 ztwT;jSqy6f%phZ3Gn4E!{%3X!%NKKFSbzU~av>HY*8dgy|6={WRCqaJ{lAi2h5xAj zuTvJ(D`OfaD<@_t7ZxcO&_qvZzZdpL93;Pu@cm0?9X27ZEm`^W)7qb&|4q;Tw&%ao z^WW?FZ}t3<@Amu$MH@id-*M?4ehEYFW&m+8IdsEsZwx zRA@Vsy`Ru#jA76GL=3Yra=#cAPSQ`Kj}N)xx5gG2V{j&hzc4VIBQM|*uHeyk)JuE4-ZhMkGOt-`S<40C&p7O z#c+&7`ziD9&zpaL*8F>G?|0p`{{9W?@9($%K8`+P{{3_2-%BfPzI_Wl<6N!I)n+_F zdV6UomCjg8O2Z`j6y#4yTU@ueF6WshuKIXsC?jW}_U+Ozlbnqj zdP9x-N|uJX!t>EYPwjM{_3qPeb^7n?(QsG)KN{&tG)?#3yzBRm|E>6{{T-#T7*+f$ zm2X=A?yUCTMR}}8`x<{V{&$Z2zd-(9BL9$asKyZ_kU|=5$RdvuIE^!iZHAto>!a4^-}{Tw5a-=T-{w}tevxyI zc{Km;0{s%Mpcj1~_id5aqmElqO6LxF4-e4)e{&Aw+6R(@QG%f;jG-ZW^+TqW6C>%R z7>jXe6&Ke48c#3uKR#ofVfrNGwOMj_)c;sFnGLd4I|YAg{O^|gzZ?HO&HpR;(yOc! zR7S7(dQoVf<^7{`k@t@-dN*<#l@BP4|E)Lnpxd=(j6GoYWYFrjIR1BrbY@~U=3+h; zVln#ub#!6=L(f-yBf`DLo-Zd?VindPj{jXpZonp#VeBp> z$FRem<$GcK*0X&Z7O;O;uzxqREy=<>xn=4E|Jj8-*oQ*lWn6&~u>TP~g){zh4i|63`V}#{i7RQ7%Z;(#Xk!t<(CqA zTzg?CS;601v%r0AbYFA)M|m||eA6uVg}C-Y5?l1MZ#`w~+GYN#x2?VDn7D>dExnXp z>4jIlp?-cV+Q;f=c_%qa{Y<8Wd)^!#dOyZT@DciF+{?$T$?1RNFdn`CYV8lQ$NYjx z9{T!HD}0arVQ8X3N~@z2A-^?x4Td~0Mlbk};9zN6hQeKdTl`Fhy1dU*Ku+rz`& zvqQrhFMZe=l^+h@d3$Ksw&3UXZU1?wl2!s$J4c1e-5(3vr+&;Dm?OfDr=GS)pK*hJ zOJmpeVd0-(Deq43SUqr#qf9|_-n@gt$lZ&~bHW(~~sPlUY(2ZcB84%7FgKQUPx z@_svklQ@ktc=P6vaCmibI7hGUR}_A*-8Z|xDEx4DQ8+Tw!;2CuD8xL)lXL5xUS#rKjL?Fbo5bHNn;0gVGn*1`2EQPID~fB z(vfs+HSS*C#%9-i?6~_!OSAiT{#MVXt)9Igjb7{g9U*&uZQKQ! zLKFmHR>_MygH-mljN8ir@`XR(T zokOg{x{zZN#W5AtemjC%c|Ad*_a-kPngjhtEd8@RmpW zdB!=ZN$vHf=d>@#LVb5%d3~I(SX(6@(3!el4=+vqj+^`2hX@ci$~4KF+^RmS8A`qwnQ`VI*0K^mE!M zpPwGa(#K;0TE%5BiJmtZr9!uuU$E(m=&R-&^X`vhI+MhGZ=eckfOI*(S~Zz`=oSE;|$KBcF^H#72U+z0bNYX_G*W*e%o1G}&X`)~k<&~8nF4s`CfhQV%Y z7j#&|0J*c)Fu>6PzMo>>51Ah2`yn&Jtz?_y>JIvTlfID}-*2~Xf=miGk*Uqr4`|dT zK(%unK>{hH(S|zpc|97QRsSINDaW;C3;qAX8yu5G9w%@Tr*Q`7P_Tb@vw!i}{#DOE zs^1^4|9gY2d!PMFW-e$Wv^e5&LvzyFIqbs(`?Kb|?ufD3>d_kKOwa(G!xLc^XtzNz8zEIfz>Sg_E>e|#2_qo}9((9yASby-P_q8uJ zx_{vm?nvhz9-#l%TmuH91ViyZ-+xp7AHn{|v90bOEy$o1ZHRjZ4wueIlwvGe?>gqD zHGk;Y%kCfJ=@XD=>*vtDfc=k2;^H1kQ^;wEYbTeHGZ6Psieo1dm?=CPP5s;#V%sH& z!u~hvp~CvNeWuvwNk#Q~DF7k33@D-{kl0Uw&MjI+VuU`LhaVYr5CcB^M`Xj{WaRPWqi^goUI5Z8X#irBX>Nw|HH{(Ths**CID zUt*ggH$p!JcTSnbyA-{8TTTH zdlICQY;r7gJu}58#5F|wxx@PP!j*rco*$y@r^mGgGmA>Xe0p}hYr{hNV&vJsIdnUw zYX$pvt@}p<5@_6QYyp~(!czZPj+I!2HCTrY*o1b^sbeSmpUfU)|I@qUJ!RPc{JTeQ zx_=x)`Yrc=+}r}Rq7Bt|pA6;F*@i0YK$4_*~v{v z-8MGBF(+{vXAtM&pCd2e658dX4*94PS#+WMl>B*9{>0Ip?B5sF|8vy!WXlruKiMkW zM#k|gSESR6Yq*Zq7qy?Y3;O8Utnf|K>Xs?tR)im*h&U!WvXh~|a+ z#C}K12=+T#>1`;iFHp>W7nensxNbaJ-*=B|IpW>#BM;yZs$(1ysJm(W0(-uJou9a@ z-~ZJ6<}V(0&S%}H+?#VaITE#p^(&BL5yuDB*IR?(u=2_EHn{HOWpf~rx~Z*p-+g@Q=`hZ*<1qn~ zFa^_4hK|jAkcew^;ctk{H%MtZ{2V4kH(jrp`Sy= z%cFcB%Aft7&Fh}gJkJJsR8JfoRyyVaF5wDl7g_&*-ssRvk9*QL%rZ~>3HkkH@Aly6 z5a-Y*(fH!%aLqC4L8ak3*@ss8E%FXB^v(gs1r(Qtd%_Qp6V8k88m13l{L!n%2i!0| zfK1;vK7h=;V|>5};{(wCtnmetr88YRWY<&vH_8|QY2>BV|8KN|aL4yKm@L6i497^+ zowdIJ3gZB}|6qTCgQeD%8y!j=Hx}bC9<}0XFoB+M?+q`E4wL9pkP_b{oD|>a8NI*$ zo#yv4%s@qpXN>$W`i0rTm0wVHeujw`N*}%|2vI8+AIGfmeq~wqa>Q>skh7z zcidtu#d55~Dy+de6xN?VF8?EoF8tp5|K4{qjy=oQhnDHuhR^(N{AqSI{}eVzYZD%? z&wrVJ3US_ix%h3ULWOU!b))>hR^OFygSsZJF|~`n2Zen@w`)i2*N(tGan+7LKpsM^ z^s31tsF`dHvDM?kww?baRMx3SeP5OP|1wm)Jt0&)^_O8g{U67CIP7@#!y)1L6#i-V zN5igB9}VAp>ce5@Qy&fAUi;@^&(%Kl8(Q~mz&kA_2IO2a=NFAWFxj|^`;E$BnL5Gzw+Uf)^MvrN$>YPWc`YBJ3O&}`2O4S->kKvx!)_{ z*q~QJdd%j~((ms>X4v0{nbK{;SpUg=Qafje_VqXIuP8ho6EF!=a9DZzg9YsH%_02I z+LATHhJ@PNL&6bjQr3;J-sD?P@S%SrBnEwh&*~c?Icj}q+W2*Tn6HPrtJ)towLg$} zPWxlB_Q!PX4>Bb@P5Nb+fti?%xtNcIXuq%hu~QwrU0In_CwHiu5%(z0*JxiXu>ac< z`@fNo$KRjz4WqFB@W!IBSUO9w94pb%;azCjY+U0!?U1+3vs-Wc;Y->lXxONI za#6eBwswJgDy%rp?$OaAK?Cr-9Oni%KejZj6j|| zdQkgoz4n*m(qzj#ZJQdbZizzD4HhMIHJ5d|P|X z@w4WCSG_rY;;J)6;RJaSwNnR#)8rY{&>Kd1ueS$;bHW$Ubk=*l>^Hp$|?w zAaP1tolGhd8u4iTk4ye@1--b2>*&KR+(BXdKN~;#9t-P#9DL}5DA)<&niuGUMD#5j zL-aki`2O48VS|wA>1+t;JV5{dQyxO}J!CrA4D{%8h;#cY|AGJD`}(n`vhyeM-*0yw z40TNPMr%ruBT;+XTASopL?1--JtQzrcsvUG3T$?d$F+~JMZ0aQ_SZBM1pl9eo{D~93fcQp{ZeEZDt=TP(x=!H^o;r=&Hi-V)KXUYZ4f~Y+-~1x`^ab`QTIiWM?9;XEQ?zeo-=Y&)bfFtLlJKP_I#3TLTUl~ce^?C&v^#JLSg*tD*ve!SF_jn zdwQaW{c8;VI{F5r-tk{?$=lMxCUNQQ#{WnA|7q`(UM_AMGP~LSJB|N0*1t-42XYhn zCGcqce;qrzu>Qd;*Y~0{mq_~s=Rj&b8{B{PU>^?P5UOzm3A7*fe9&3!`3~@WM|r;e zJYN*%e;0ecr##<_p6_|jx7qVWYrW^|f7QpGM>=V=A&XjZHIB{G6aL@e|0n1t5w9uk z&6w=xTK#rf+!>t11r*vNlYLLh{Y%1E(2EM+Ob(Tw)(-hi2-n5+q3*ag_}TaTf7bAFXp{0x^_Rb%?tZ)(vPz9n-{SEm&j{m zVSTQf5A$9dnQivpdih9NxeE5jF!qOVTqCrLOlvy~_uEL6Vk}zs%AfM`IC{ek`Rlf^ ze)I`w5>Cxye~`(m#x{$q*5;T(PDAZkeF@|Y)I6o%fP8Jtd*7dLu&+oP?G0m{#m`0W zzuQ-cjN|_olS{E2E3pcdU)64T!d?LMxE9DdGPC(%|LO+%CX}O7TRV%cyS|UJ-Uag5 z=C^A3u!`J)+HU(mkb6**iQ|PIj_2Ped;m=&vKAmLQop>V zez~lEIj(*|e!6}4r1gILG%K9P30(8N)TuA((J+f28jXw8Z+J8gtuCql%c%dl)qiB8 zaFU(gq`i`2v!9gCX;ktN#y!Q((9hulE}?z3{`~>`1O501)XiP$?1oUqBomke|L=*b zUTdra+5ctlZ-h14$-$^u_B;E(M0hCT-i2`w!-D;PTl@bR?f=Q%vvzrcY<$jp-Tk}y zKg0cJBuX(B`QMWy9e+_!&Z(h+Kr*FGaei5F5nV5~an2&{s<4+coOYv>r-g2^4{)^*JR?@SZ zbQpVOXk^)kDvHWOoIlX}Z+{j_ zW89~U!dSBJ4P$eB3vrLMI96qxxbc_}fBU8Uz4O)3vG-MNoL57(=2bqeSCvn%DxY3e zKD`=_u6Q*xuYNTg+xTiouYEPNEP9o1{ngOwxVFrzp>Dyep?=w`p#cdrB8euXFv)q2 zxc4a$`fyA`8D^kPn)PTv0*$-NegEbB@8$gO*qO+p4cqj^G$asK)QDKf_M=qwzod z3e5-E|73a&`+tFNbBX$ijO)*&FK-S52`4q=ZLm6gZCT3$U+KY|9Nh;iBKp|8vLOYixRa^c_b@2|2a%EP7@_r+I3IobP(SHm68bR|8( zR%}3FTpO8EFUGxMDwUa4pHxn7x89E9YRD>b2WshY|DIj+J=li>c(i}dA$qo3+X2<| zBZz$hxvlC?vTLvU?^*tDG)&b#z~lXQm$(M@X~KU}NTUr|s;84&BEEzh6RYz4ipzi?|M9VP1&xzmjjaYKZTG9_Nk4xp&=o zeE*BwE8D!V|KDoQJ|q9v>;KUQbWK{<(T7{OgL`;@|8)G(Mc?0H-yhOwL0n_Jb+_*i z)$H;9Yvd^mMhR-!<~90&hSC!&*;XRF9HSceVRgmP>{6?P!5Wxk8tgSt8Lzw)gCiF)_c;oi_hPod)F;;_$g z2XF}0ID!On-zX04tL6Uz@~Cp)vHd5U6V8)IFY5oqF~_A>c>bF`f3g*A3#|WdELP87 zv-w-14{+ea@qdHZZ_2-vbEJ_)9w+esst-<*ePrL)9ceNpI`nKNDh9KMgHWOFSi)*-!KHNgx0`~DDHYi&)p|O5IL3XTuEzbjrv@_7A^x)X{t?$$ z$)0`BzVnQ8`@R3Y-v4&b$Gd;L{+sun5iY!c$EBZpGE9=r6ih=IT3>Mg?r{b^E3bA6 z&!o>rUN|QYcavSs#ubaJe#!iCav^F5JQ)^~OHt!K8-x??VY%>1G+p)Xg&S{cN0Psr zf3I!KpVfF!+nQ`bDx=-*KWnfK8?XuG*oG>!Yghiy)^|AiHv7MZ{m-6HqlKPH%HP`f zZP-$*ZTgh&9Xq7g_iK5R+=Co{PVYzLLG^jh7JHqDJA`Uf{fqkkvqd4>kA3ly^>xH0 zkRRkdi+kLExx#&jPooW4w2I5D=WC#!z)74&_3QEuS+n2#C;B;hg?`ZsWW#pl!wu!b z-I#}z5A@VkKU=pUFb)s)gA^Hhkf{smEmdSqi67(^FQPK z)3p!C9HKA3ZnyhG0}^O-FG)11$5YrOjdE;56@D;AojJK*_&(qE;irni4o_bo@t!_8P^Wpb#P@9GJLb+Z^KUd zYg7M6*oGtVT2Q(6??To7zY7(E{zurp`tQQ`_+JjFS5kgUqYXXZvcBOW{cBJBdHD9A z5@VM>7_#E~{_3OVhPf}ivH62x@8rRue&(1?4AD-GPnOCwDkK#sQCJ*aLMn5`Mtw@KcnOe_kTrPFRI%0?|*hwxK8iG zE!@F9JV5`y)n>wAlwc@^qj!t>$Yjs=jkzQHj2#?Hj>C9Nz$8q;G?XFk?>B?&<^P`< z!+f-}W4KZOk9I)xjmPWjds^SI`1#1Sm)ZkZ{(p!6ML5p!ue4XdQhKXrpV6LKPLFdA zJF$|!3i%G-=Ea9T$gbPIzh_5WJ4fKThfDPD0FGp&l{%`%N z$)`q##>wWIi%YLA4OQe0w5})(yU0Dri0{*i~l#xadPmXMcboHrlm(Vw6vR*Vh}D9r!+t-5oy`(Nh%*Smk5bWD2x z=x~}mgI49hIr0KB&7(u-Ead_&312~Ox^pN)x~6&u%8|MkM~C`2#scwNjhm&tU)$_; z=h~|M->6;af7fsweYk}?xQF(n_J=k_=W6F63;R!#IpI8c^py5Lj@A2CYP3JQ)eVQW zKhU<<_@S4c)_?JI=sf;($f66~$RUrTuX}ztHsrXvzb zXg~su7rj3;o$~&Umxc$fvHv=G1%pw7p%{*lD8*Pr8^3zqs4$LRvvE|Y-TQ$so<0GS z@OXayP4(}6_3v$Ej&FaO-+F$nej|Hd=~vi`V=I4G9FIRkTw(oz9~Om~^x2q;`B;d) zZ|ZxTrTveFCF*zOSmQ?Nqp4f^8Eud64+x7LyA;cj>naK>$yHc`_UAl*@3GT+&63fl z*L~jezwG(H?fJj0{eNEj|FS+vvPC%7&8?$6f3l}S|NqOSq1V`fo-bS9Uz!`R3FX*^ zD(t{6yc65~zqX&k7e|LZ!s)%nSda&B2-RrqaE@m8Lw~gXa$$c)zKr~C*XDj7t-m}a z{%4*(S@H;KW4|xihMM*K_B-|8>hH}8=g}mbQXe;xN%sGi*NqLtYx~vz>dzC7dDMS! zl71Rz@E^_pt9wEHJLh2?|3~W&toXBV&VMi960V>Z*Ki$uXkWzsN9RlGcXVx3zazKe zVf*6fTm1iT^Z%3Si`plrv`^6L*fvyOw&t32?%*CCp!O>J|F*T(z9Ih)a&3GE1L=eD zNAv&q*bCz;lpiI2ABs5tFZ%z6(<^_(?$`bpNiW4%jKg?LKwnEhj}W^aSg&9WL2H^ebn`9IfaO+?3C!*x9N zPu9za!tL`s|3#kvGWj1}BjkTPw*UWB{$DHqqYb@3j{SA|`v0EqzRP~o(yaKHwfx%I z^`DLNpG(3$aSzacz56ltXCOH^>ff+NCOH(t5yu~oBp;1GE~SsfIK+3DUE-aQ-P!{= zGLAp4l5Wjr^QEm}eTP2Yu@f)})#m0m?lm9$p!wir>RIFS#ZNx%BK|_Pf6C`SgXz*RbE&+1<_Tcd~96`}-MoDmyqqHcn@MlTE@YvS1&+ z*Zx`T|4XqP75ZXUlI^ak1D#X(7tlrTM()0T{@dmsEMb33=a}^3UV|;t&y3OkPH!8b z-?XMAtddUUCq0WVT0fIsE3U>o`gQa~k2&=FOTq^FCZw*Khj7>2`J3j@ljY*l@7Sx5 ztU~Kx4%+Hd>*aLL>SS@qN+*vKIEmWx z*57AGoTex48@qA4IGmxML+TlM1If$24QyE=|0)AtTfzRC$M$}cZEZYE>j-^ij*s(q zu8@WOxA^qB#CIcik^POj?biR=tB(%}zJW&leQ{iV(^_ZL02TT_`~RKx z3I-$hsR5yc9E#!S{hana+4D(r1ITK5YAiVp<1qoX%G6l)O`^xS&v6d)6#6uz&U=1H zI;Ii5zPp~l2k->@A6xFfSD$G2mpOh0W@9eqqwg0_hK1zfsDI5nU2UE(y)gfO$bhh% z-YPEom{!uWH}nl-6@3lzo=Z-7+szNqMHcEG_GMw;)o%4VJGHRC4w)J;FwFFyb=ZJS zD91KbVF%jT*&XWJ&SLfN0QK)E^)H!2elq{NI<^_d*yL%)wy+m7#=o_q4b{)Ff2Fer z`)~lY;^Mw`hv;$7p$1gbk07;|{a?@i=Lcvc-(Nl?;_pbK;!D1WerAyNj%QM*GLTHjt0|KjyLVXL4X@-n@M75EX-~Vig>#k3b zR<$vGlgOH+{S}PSn?j$4+$XKQ!pD$6{AQD*e8Xg_*gTgPj1O?k48%6)Ofr4g_!)98 zaxZ%3bL7kQ@;%wf_m*AF-|x4DSd68(Anz?FS7H^`U>)kzQ}qkp*FINwCFSX+X5|Av z;!$Nq^JV#)Oy7{dZ_D5J|1guUHe^vzVa_jk0w>X@Zy@*bXG4#A{G>DR^uL+EjAPPHqvfXg%V;&fz3qm! z<8FJJOQ%wKTJ9g6_-9(7RHj}&{|U# z#*-6}NtT7qm&(*{Wnq%=6y#i6-dX@%t6k@cvM^0t8D?N6dcSI&1+phh4|B=B-%bzn z$%R;qrC5%YScPiO?Xmx0t$E_&YR9<#XU!9*C){g;`>gnc=Z@5I*Y9~HdtCoJ&xbnq zT8{=Ku8I>Me`}I%3WfgviV!wQs~p=(xug&r~nG?>FM>FhSX&&3k{zc1L`4g>ZL!AGYl1>_Ji2DGwE?|F5 zHonhymenTjT&wNsw|pccZ9H1P=7hL7|L-Ju8nyD~8S)%z{I5YeiRpa!!k5r=Q@Mdg z$0bo$@BY!y;W#wXlW3aiU15uI@=^JF#s7PeYabAu2C0Ep~t=08;XrHrk5e^$s-k~753#B9vP`};TM3+G!DK6Uwm-?WgoV zq4Og9_pCO}U2PlW_G|y)=o0OZG1?zUPuKpCW(KYOv_FvEZjS@$?7%MUL92U=bM*Jo zvscyM`}yDLhmgNv`~te)F@E8;aSh^*Ab}LpXhRl#-zo}uvgfD#B4qC``AeQvUqpLG ze^-vbD(=&J+HZyOkNt6mUOQR)$MZc$k3NwGeFPWimk`%3jB6Ma{Qv8XH4v9pzW0*X z5Z9HwPWB-~FRTmX+q)%v2RY9skHS1uvQ8eUkNE@%G-iDN@@rvztR(-}FNcJC{`Ua= z|A#UkgHeK^XxI1FflmEySv+pv(_>%U(ev{EW%-{>zodTN_?~@0Z(C;Eyfj9l6xFx2 zrOBF!`rj~)J{}Vg#~>tzxjt=}YTtE}jf$Ko8s+$s70xcr~_gYs{I zy4m}9H2x~?vrzE`K87Lsnxt3p-65fz+=eRbz%J~;K1AQ<7W94GoMrNWxI?H$?ngty z_7?kveaPHtZQtJT(a^&WmGE2dm$e5zD^JnWs0>epDn5qv!QwDQxD8q4(aQeIWQxOy z715NT3mq_qU^8_n$u8!X4bh1N7hITSU9H>^kNd=Z`G9)*4@c9P&7N z{14XO6V9L&ZAiaU5(Y~rx20bwA&*GAwZ}2yhYDv0u;DS1J{);^4&D3~T`26o`-1oJ zH})~m7g_l$Iu8)?SfS#Y}?rD*ldEhkqZqiq=H#Kt{fRtc{`j-F5YPMdwF z*me0YtOs8l*7{o^~pBjIEFlq{fT3M8kH4E96CHC{PV^ShJ!DC zFuZy8iEyC#zlHCu|8HUc+0TUU_WakdZ}oo--&yfr_4Sp7Hz zHvh2SG;<@^tUofI=Bct!HKNS?!)amrlK&q5QM+l!Gm}EK{~f_UbxaPshD{0IoIEM) z+&W2r>XfkO;-HZG@raP{`*)g+7n)ZR-q<)e>|HiEd~ZNWIG{bA@>?2h;GPI?YA0ps zJzwyhl5uYT3GyUP;|$KB`Gw*7u!e_p{q=sw`u(C`rr$67wLEpbU*9*M3>W-f|CTOb8b`(B2IMr(YgE)EY(1BQmfH;qwnT=l7e;Rnb0CHj3V{BXsv+Pfo- zKQlgm2|xb=_T6gs-9~M%D9 zIW}w_N3RYD%{R=yBh&Zo|3=TSEnCllka|k|oHfLp%8nvO5Qcp~AzFqvzN1 zQ}82n({rvhk4M+tNzbAk9eA`)UCUYAe88~?ai znzdJpLd{!6VJST^%RRlsuS8#kRK`7vOQKP~QRP>SuXw^fb$&~4=ldt)9^!HD@D21$ zD93-)K4@H^99g2hu}rywI`wks{bExWZh@MyfzQF=3uA&mplID~2(K>~5zz!cf5PRRX@eouNE`u^QBCi6Iflc;`O z8S>o2HeKu+I4!Prr}+ywLr)BFFE~fPgcQ9AkNkg?ANTxzJStrATQ9ERI{G5!{oW#b zUba3V+53%w;U4(_{WtsGkyf@0CQC3B!_nHTjnSdci5|zK#6IX!`dH*|Yj>jiMRfqG z=h-h_dgJ^y9urWzR{ytt{z>%2-qE4q_0eGpeHv1e_3a|LeROEVmeudIXWe_>Z%2eO z$IQS?%tqypE`2^0qUTdjhs9*?PxK!r^?$$Z92fO}pOxm#(R}NpL(}xpAvM|B z^1lpWsbiO8C01b#)?owM&+7lbtpERp{{O4`|9kZRBZoYWny=qHZ*(|@G+J)z|3@p@ zP^Uk?ey8)hmqc9CfJ_QEbvS=!blBt^<=BQQ?7%MU!9KM6Pe)Qef6eHSjn7ouEy!vX0WLN$({Ra^!MdUlw!k)o%O_g->qNAt0c z4qbT7^RJ%eJuL7Z_~2{lxi{7Mj;*0L%##k}pHTTl^NJ=*%d>5aYZ#zT-m6!iH>}XU z-mG4KDYmbDqe!8GuQM;L6F7;}ID>PzfIrM{UFYRT{MfX2O|0@z`L=sI%p~QcNVmM;^XCzsQu^5l`o!bBMU?*Zb ztE=1lUoVesW@nK{ud@Gdu>bF}|8KJYFUp%}l|S1ueE%P_&V+P&KWUE(a%)X+;UP?+ zzeX1NKYncG$Vc+JzKACIm||;??7H1l0$cmx<=j7(=ssZ65spxMPU!Q z#eEmXa}KjkvTz~)e_1(8KZI%=LEn&~kceUFr^qzgkVRa>zeaqX9@ivna6KpJCy^S( z{zdW`{)gw-&f?;@!!zVLv~C_4E|8ay(cjR?{*UASt_b%c?j4j@*A~VB@-fs+=YQYo z9O}o!UTJnn8;{2SUGtyo=)*1CL1F!C?Vl|j)}HYUI=VfhjJn}Z<^Rj-U(f28}R|I(B7 zo4!~7wd(V)_$i;GV>8A8be`oGK%pG{X)Kr7TPyep*#6xct^cP#zy5|ketr1~G_s|W zXhI5$rLz>vu@bAW2J5f^?d;Hw9?u9_bY1mq-tmm|?dRF&N9P%TfMbZ`1X}L1Ssl}Q z!}tTTnvY_Wbjq;}Rj6$?|DSEUgPxE^19s8(AT`FdO?8da-9JCTK5=o~hXdpxwBFUG zBaa|++;>?o|0^F7!YLHieb_7iCw-?i^8Z!qGw|`mHfo%wSMaY*_x(@)|0uiv7%Q)O z(f@19AofzE9E!AuQf!bQmvXT|zDSUc4K_%y;SeN9kYIxZd$GX=2{sggGR)XRIf#ge zl!G{(VP^Ks>=`zD&yPKOZwV4?kRU;VgiCNThaeZp91e1jgD~IM+7!O6-h0pe{_)P| z^Q`?mYd`CGp0z&fvwq57i`i_~v>}HgO8HA3i|uVk9Jf!f{=IWM<;@(r&KO^E!T1ug zC||XVWB>hrP@HMd9oyTk?OaHhX&IC(E>_7$4B> z{_eWJOY-M6_eaKggt4Emp#O67+jkSO-|xI@(&R<52L)wOssB-5L@bx$T;w<>`Lf@x zq8B$%QD$ADG3pWYIQO{Lecqw>q2EU3Pm}+z@&9@MPnPBf?z9G)-v(kZhGIB+^ka=6 zyBn3eWH0+SnjDLg|9r{wyQcp(p7V8W9AA)EKd0U9yB`k|g-t@?M{GX%VElhMeFkQt z_bo>qp>AqU8x^T-9r4pPQEV!Z--u3Ovqlj;NXr6`_~2d=39Bes|JzrVB!3X>Q_xODusuA~(DBu7+iBkW=+u9H4X|y4Sy1nnGkBv@BOQW=w<~N*? zz7y&o&N+f(IF6Gzjk7q9c4c};%xB0U?x~uWA0PVP@%e0h-~V2w{9otYka3Jg8=0P< z4PQJxxQwgF&Q(VkWBmbo&U5Hw=Woz&p}1e%9sZ*=+HiFTVb$scePq8kJ-cDXDv|?H zT7O`=bPY02%<)h(vpK0t>bcHqy6L{pcrO>cr^DV83eS~=BD$`7PdM1*y=IK#AX{s^ z*S+5BP4AhW+3DT9qTK=y)-h_Pr_hKdZ0N970blq{?QyX$r}7{El8EgEh0n?(?x$`U z`@5R`T_=B$jjzgI8SVRd{dD9g=ZwZ!jK@Sw!W2wH``q{YAN3{V&^1IpA1VG}^8F*~ zAAFDx?)OvL59qc3um7=J95WEhjG1KSTE8%xoP+Fi^$~Iba!)Gb63VtmlyPKHIZz-M z`E3a*et9=h_{ComOXCp@z%J}TuexEcex>Tk+B6+k;{cMVy{VmVjOQ1RPkFZLG--MpVx6hk zanoVXnyeC*LlH;N`|jPuG4eQ0qUXDJ6Q{}UFZ?BOmOPJ(F&vkXzH~3~!S;`A9Et5}oJnC2r6w{>3|ar7uwsmS5eM=z6^`QFr%VqJGT1M8kx8i6q%L{a&Jp zY<8R?3(sm3bZu$;y|#cm^gi_4q|8J6#=b-cI(PRaa_D+X+!_DT+LtIc^(9&s-%GS! z>PsBF)|W_k_azR|GxY4azC;_UYx)uc#WNT~F&wqRYQ!;uo_xl=iFXuzG*aW-o3N(I z?se+D#8_eJEB6xP$%)AByqB0nPC-U^XY0L0uH#-}n&Wbm_JvUfbZx(v$fItg^h;}l zv?is!k!+glKaj!<*UZFh%)vY?z#=R``#I$gI*);q3Ld7sdx%s*3U|=MD3}&i5lUn=*hcx6Ad@-Cf3l`BBdVQ%+5ERxtnOj zhTHNt-h3&9&ENTCSnr&Q-^uI0Eeo6Im0w`LzhW#6eG9f>2X>*Lt-elN^@wBSO7;7l z#{cb=|H+hcV~_Lpp&AEJVXSPDOd*YS?f)H!a{zPb(x#tBfnG$5eBG)HICxZhUz7gF z4()-+(%WhvjvR_Of@3(2-alx+BfIsVohEy}rEWoPP;cI-4>yj9x#&23+Z;u*@|*g7 zzNRmoo?WbtgRAu1Eae7YrI&sKMP*6>dC#$HuKI?s?$7(qt@YXSuhtB4+=qS@zHbb~ zU<}1@-~DiM1V*9vU-aXXV^OV48&B4pvELdw3AOYow_-eh^q;01f9Se6Ze<2J6SFY~-CuU@zgYA1J7sLXeMp3r=5haC zy_+8A)@S&qi|9G_z4H-s{OC(jWLFEwC$t|3e>lHzCp*OkH?o&aN7XNox@q5g*Q~%w ztil?s#d>T)`)dC8I{!iDPN-j;VYk?`0$Ft2!uGa```}*wH(6T$j@^tlwhh%IwOo2Td)naFB<op@*a-NfFP zZYO>^|88Q>;=75T^uM0itt|L)&A%mfy?j0KqgQVxc22&T_+j^z#E#AXp7_E1KPI*> zzLIEd9T{3`MuzlTPlkgpm|K(hR5*0=$*|9VRpS8Ote403_bZJx`?-FH>i!=IaZkGa zd2sb6KJbpxGq7UyvhJ-Cdk=tcW#{WR_jWg&&e z?y}H?Zgc)B9@YQ!|y zxcA^pGDFTL=OCLg&XQb!+zZOcS;|JTYoW4{EI6)y@zKyG&10Rj2uo1A&KjP=m(r7O zJsKJ|J{p$OS0LrM*>MxuxLte8u)$&D$id;wXY{4>O;=jWqk7XKRu7xKSxh@L^#d2Kfbhw4|&PZdvG8*dM}54HN&8BJ)!0q3OMf09fgj{i@SZOC+M7syC=9*&DBu>Emf-ox?#L*AeN@QgU5x3vGl zeCb~-{n`vF>iF5NJBH&piPJcX^Jr)HJNWaRD6Q|9VgFaN|M7pwmdVnut>EAz?hA+5 z{>>cOF3=U((F&+~!2~#i)<(Prex(?n!-BHi^g6F!|bM<^1YkX%r zg%48y7{)dt`4Zc>P};Zf4t!NBZn@0c>J~Y zk4LnBoKOcps{P}Fx&fJW+(vf)mjC~lvHySK|37M-e{n9wa;(5gtU}>aWnnF;3LJXB zh+oSaj^A9S{fj*>wST>8UAMi)Y6!1FW~aKxTiRt%S*ITMd+%d0-$lCO-feLVWwr8h zfOFSi3$~$lq5gN};|_XK8QFkc^gT#zR5tHcMsHVEllz3lK7?xW0J8LCJP)xCp_5II z>++@@x1r#;=(sDR4nWpj;(uS$zCb4Lvj616_Jzf+bxjdRa16(B5~pz%57+;>%a6Uu z{$FGN+3&(j>_6EuhW($w{*&qH>_3@toFz-^|GfI)^?%NbvsPFQF4B`%*#B<+8NHxQ zHRZV3aTD3dKfWwH&U-A)FEB1Y>-}F9){Bg`na-Dd7h{aSb9@U0dU1m997TAH*E zAJsm5SX%%N(K9E+b6x&KdqVz3Cvy1f{oC${vk(2M)EBW!{u@XR#!%F~NZM}+_WNEzP5ak2nc*ov9eKgD#J_qwqFo&z+mwae?x|{wB^&kHH630uCWv7;t zEAU{vPCldEO1-{F#yt*J`mH{szg;`P8y~&jo;CS?{W~ZAB95T&`;p-|c@n)}_bmTld?-Ek zeV!%H<05(x$3VxuPcG9R%wMSdlINnoJ>~t!HX!a@XnaQcH|zi2uAdl5{n3s3rJK&^ z|30Ok_^A2q;<$=l+`ui|K_B{kU;1Cx|Bg=N(1kn-D56DQcdP$6xK&>_c}RbGhRix{ zBhybl9R`SJAO>S7vh$szk9s&gr;oZ*pZ5s*C=?wR7V7`jr`<)47FNCZ=`fZYkJ=&n zQ^`q)V;CC5lk9IC!Mo}^+Mk+7N|)nCvT2z9Z)vWV<_09+6MjxOn(3*-Pk(6rpDE(3 zw6;ancdV^KFUNnn{?Cj5hyI`2^1S<-A)c9-jXB6lS4R5g(Q|M4wl>-afW8Pt#|6i6 zj$hYy?YhF2qI$J`9LSm{pAIX?m59$`f%mkEo;<9N`#tZSjC%~kxy5mg@fyFa#d>T) z6)K*vwm-QIJFp9Tun*NpB89@Y@B1XB`7f`V>k`WD?|=6lzP8_cA2r5;>~1tRg6#R% zr$doEf@3(2lQ@mDIFH`1m@i27;Bu7j@}tSP{!cG?1GjJoeTaJs^?OU%hk+Q3tgy^{ z>jKhq&)l!~7JjB)?zlM4cOWc3Sv`2F{=cd6KN_~m|C{B1_Opp>M)Z+O>p$Hz))u2& zUujHtRayTqnm!io-fxF8zLU%?_D*KKZ~x`-B1-fBYuNuL`F}tAPcO~S^*?Rvj0+XV zL`=dIOhY+lp!fUcf2%)rf92^g({ayN>?J|Y!8|NL^^3|+atW4VIckN~DA!idlk#}O zEc>d_S0Ob{+JrStmR9L5?XSx=H1LI!ud)x?noIlZ(o@UKTXW4Cti^hKu>WV1PKji#DXsIrlf(hs3o7+mLl$-2ZL| zJ@phJXx}XTTiqiW_bTk7$Gw3H>NlnL ze_i@<5b2}Re^}a(MH|u`%5m|W#d%yrc8GJHWE1E)HoFs-=~ofg^e-UqoUSqYNnX%T zAp8buXMMC(7vCT+R(IPQahUydgvGSRJrA2v@mX`xjlL5 zM(tO$2cmPadlq)+J!Jv1=d|^qjy3%W82q8#J=)Hj+bC5 zR$wK1J+oEh8dP8C7uJ%Me~9xA>_1Mg-EI8ecHlm9?f+#BgzmR}n~vlDKPfVeHsqqN4p$_P;24ghH2-h6_7(br`G2SB=kaj; z(7nnk)b-~V@e3P9@;`_1i};*PXyzlQu-r9op5Wv2l{fI2&pN({N^={lTH^E8|BcSO zeB7J-d%kdfK0j?PKkX&{H=ni@5B<+4`E|$+;^&>y|IG*M5#ME8MK5Yk>HqfI4SH-7 zXuvJ{9i%R}H#8xx)z>F1?Y{bL@y#N;(RhAxFfzN}U;lrZvHXsQ<6-~*I{xEo{r~*T z`pN#|Iq65^d~v_%KM>d2AK{u&7>%(QkBOLsDJZ4?=3k|MyY>wf(ITC#m!#iwiEAbN zP5*z#hxb32CeGTU(l2f0^rUuyxc>hPdVzoOu>U{a>r{Qf!sD2q4Er;a9_N_NAuGS@ z{g?w%oT|RS&&ZGQzLBHHHA!#D>+_tq0E^rzqJUPmu0{Eh9wW~qGmZ@w z7RMfBQ5yfFoNKr#eS>1Zs5(-AbtA<2e<}ZwMh-*@Nf$t>E236qv3;#rF2Sb^Av5%>RFNzd(Oe|K7c zkiG^*cC~Pu{k_iql52%kuQrB(+=N;-wTj$=(*A#^Jl6}xGC1CW=Dg=x<2jQ}nb`he z{r{Wl=DwRG+334z@|`sME>h$MV>C9N@O>Orzy7-K>bu%0#Z%SjThTYwZahcFIq5gf zBaZdx+HP!zbQg9?|69`klJ9T6?{Bg1kGwzsR6EgiYa<}N+8iG7RpS7X$Y$h|W%3|B zxBY$_Qkvd|;$-(56TAZT*))W`8Ym_X2&Ug zRgGlRQNL}q?;q+B*CI{jjZsF^X8Qof-?VonaoV+KaUK`ZgUc|^IdtN|K7hpwo)O|4 zye_0W?EBXmLhF9dqDh(;OFJ^2&mmkDM=x&R7B)_1_Xlaqq`yfoG)@nF^nP27vHj-s zFpwOK%Flg1RQ*D^qP{=W@o(JYaB>8)!ZH{|&yAD+#^|S`k415$`$PU2_vg2s&;CE! z{rLVZ^6}^G^X#07n1m^)c-CC{pP8plFGueiQ^E{#X4IeH50GZ@J%B{In9yVMr57m!6AGRev6$S`jft4ujKQDY0J^9M>p<&tc zVGVsPQj4Ege|bJMz5IM=^xHqb{JF4i_2+@m%d&U_Hk9{ydm*;CnmRTRNEF8G~u~4fo ztUmE*sC!la*9rX?Xgc-S{r;~rLqhYp$Bbhe6E;m46ZVK>A1V(&8>)s(2o)p8h0Vjp zg?}73A#8Eq)qXpGe;WCjuuWRuKKGfhb>ws5hc8bKJ5EfF-|>bNe*E&(uxt0{!jJZU zF6>G zUGWZ1J+FPr{QkM-_v0YaIQ07So}u~uubSWQy|q7MeE@V$wLSp4##tX=nDqgMSRY`} z^I?O!>PE!zd_DfJps&8WWqPBgVQEA-q=d)s3$eJF|->}`+yC41dtxUdnZJ|X;?``a{XPj9`V=xf%esJMIpSVIMU?g>KCF&#TU`M& z{5BJH6RbaEyg_OGp(f>0#`uH0b|Nx$PWy(k;6eY!Z0FCxJXBdLrt+P#(5~$3K<8v- zp)&15;~(Es_i#_GIQW9H0*9t5Tag{24uVDE>iO*RVF}sY_#^56Rv?JRjDQ>#+${sOIZzA!}CK4~5);TKX<>50aOZ#b~;qeuW49|8@M#dbTc( zu}@BCm&UM5i`oDA?38-I0@vz*+`W9-?!LC6!!Cf(IU>)LHytT zeBEpO-W@rIFXm^ib-;~4oOJ@=lt+1VrX3gZh({y&?YpUVES zt#R-0YVlt7{T*{o?QU}d$djnKB3*gu+N%BFOX|blVq4hilK;O`I@htk?z`Ua4X!JV z|EEX0T$+E^#5WhuS)9j3^x!hCq8IIKdB@Ajc;wK9JRYp`)pAbyaZoyAS`WFOjOUU? z+g@QqjO`Q8E!;sLvXhOPhq@kCSnq%p&T<%XxHCN zwqN6KZsfA@)z0#zQVtur_k%3dxLr>y$v}OaRkS394B!az3;~L9|wi*4c32f-1C@qWXX$B z$7Q5%>Te=@aRaxIRp*QKk300-T;C?WkKXSG{_BB1_=0a$_(T79`uqOxM*i3`B#3VL#~d&4UF8l?KWH#9k?am>d*WS`dheLXgzLY=RG$}bp~W=-@h!nR=tc3}_p zp>C`Ezf=B4lKpB_9z+`v$B3rx>K}H_0VI(^F{rDNZOEa0o&0}T9zzZ#JNuqICjS-D zvYGvRQT~5PUL_ALmN&_)<2Ev`|5Fst5gfyDJQzQBk{-*HPImJ&{VYoRdhV8gxBD;h zys)_b&qcBawb~9YlUEV@z)J0iN98%kH_)ssNI7mK<2r+PgvI@LE6Vi6YyaEu9)JAq z{r3LCW9FCpz3}O>&~LkZfPv_J%v!JHP*nb2J;Q z<6nX{O?nl!U>n-m;Er=_R5$yNuI+5oUN(w-jeGsH%wqp=a4}oQP91uQZM)9?p$*lq z8D}7#UD$(tsC~uwgO`nSqet7_fCEuSXJse#82|AMJBda#O<@0p#dVI4x%PwY8^;}=#Ay^h!~gb- zI#Ajl@jdDHEb=JOi)ay7>vPgSS^6K5{wJj$S$Z4d{KpFGP*;B18tPwEzKgfE>Hhr3 zi}d78_jW~J0R1vj7u*|~5a(iC6_y^R4}iRZ>~!ypyn~GY>^!Glf?<1Jnt>RMp?Ev`x#S3pLi;uKUwOP!Ig;C` z{!7o(3wilk+0Y_yw&I}k((0dwlqqpXP9`>*1-f^y*-ma)5!So51+_=z!aYK(Rz5P^JKwpFIX zf!H@!8h>y~Uzy`o$hyBZqbIP-@@N#uM{%+Hv8lPlbxFTH9aSZLcxG-P(A%|Kk0Bo`2^0oO6n( z{E7bjKRp$yU-7KHnXvZ+o%+f3j`X)1m!@@dIa!A25eMM|PQ$pTBGTz!l>M z&{AXl`Bv-CBfZu7{X5OyziIxybK1x{{oVCwn4te&UwY#h{rB`{G9}I%?(r7xpb!0a zcn=-EqmT-TuSW%ubh)II0^(eQ%vP@0$Txiq^b zg-PO>f@vtn49vuA%t8A&{mh@{=fH~#vx^X2H7XI{UYrh z&lArAEW#3Gg=Mgmo*U%5$8!1#6zK)z(S?=5s=eP;AJ-UcqvzB&N_~O!Vx-%%k>B#y@6Ug$mhX=UKZfHtiP{eNr&XSzKPZ3B($6EV z2@ux?YI9K|872+DyYsj^z`~tuK>mjVCZ$cHe zU>kPe;r>4z{s)cq(vN1(AcbAd*@Jzk#sMUeLK^L!c?U}K?8&Zk_v=;#^{XP^?BV`M zZ?XS4G*|vZ7Hx?A4{hSfp@<`R(Eo5OhOcA8S8LOtpG0vdyMw&v)P>W+V*kTg@;qYy z!$qyW5uv`6qC;P9C->R;EFxK_u#Y1GqaU7%5=DdybwE@iX z+cLJC9{U{zh<6~e^pgMoFZbIPI^PmEe<0TTOMZak=(7)Y?obTJ2*kO&H5f%ty0->? z$!Pjmq#QRpZXz4qZ_n@8*hc>QJ8Ue*JEy|h029ecn1X32#|-p7rJt3Y9d-6Ojh{XVJAJqGQv23x-Y-`Ak5yQMwOEf$ zD73gAvhErFFB*`Xe&45Tn!v~8TgJKe3!JkB+pq(>um}54jduO~9enLhK6ma_{`G49 zwZ8QNS>$WC@NZl9^M8@XA!O?LzuWo0NYf99=d$l9Nv4o(at_k;+%mqk`)Z3iim&pm zh2>ZCwGs1u^)-G`tpCw#uf%!x@)2t24cZH0ANn!J$I(oW`xrc!=U?}-^oyrqo%6^> z@i*O+{=3d2D~&s=@*YpR_B77oJT9UKA6oyBU5$NC`J?h0O8I}Q^zW4Zz0xDSf3yFt z|F|rktLViIWLHc7C1W1wxw+Cm>*L`Ly${9R;@<8*UX*^}{eJ8{v|5LO9EjTIJ{|^> zLs8>dHw==lVfIIKJOa&U^eei@M&YpyrB2?jpCFHn(f%jDH7w^Qw^|e>CFYDMBvgL~WkAw1h`nvpoP98_LL;lam`%T_khW*d0 z|L(VzxHvOpT+43;J?DmGS=fANWti=}Ij9v@gL(Ai ztI~!__O&XuA1Q;H(X>ojg)eYUPcWbUtKYJg&ET-e@e(Y>a_q0~7k+lvT#&>4!p{e( zhiHTD<=gitlUDeBC03!LtSl6;h939)T1&3SCiMJPJ0scs6?GhP8+Kq9KG?so$MHT? zW83x-q4yp0KYp)l`mJ{;tmj{SlVlo|Pek9gEbP8CJp2TEu2^eu>cFt~u5}o%4-EU{ z$A6YDw~reYesE$$*fC~Q_@TP?PHO}HXxz}S>&fBvVjCLX?tUz69rIZDrwNaRZMTiJ z688c8iYrfq&HbMUm6x6fRo5R6ZGOw)&F7v9n}$6V-l;DOMaM^Q499U2 zKRs1uU8}P2BkOzYy!5f~W9-U&ES&b+S^RL}qhSa3Zx7+Tp|*QnsNeW{sN4VDkbLVqp&{`d-|V+TRi*va?}iw zF6SpP!|_ZspI{%vS6auRo874A|Fb0xdHz2?v2l#{zq#7~kiu-&%)vY?K*i^+;Y=>U zQnc6b|C{&-WG>GqAoGq3WXb=3j{lE?{LwTH@u45=Pt&%SUm=baSc&x1{$UlFQ76x; zm#m?$#d>t!7C*Z9;d#V4w8dro3cuBEWbf}9vrMmE#)f|NjZpc!H$v4rZ)n4kmY1c; zf5oxAaSe~S*Y`H(?!bSx{_n=k+61x7xo=+MgOK}BjRW{l`LCVrKVAP5Xubei~;H$2G;Z zz|Yfj(jV7mxk&Fp+>^Y3ymPuxy?0%x5`Gz1(TmzM>q1RtUARF{-d^V$S{H87?;tho zd!ZRk&S}I4_a9HUelPSnC$nt5{^#|f-%n!wTiI8}K3rKJa_G9fUj1;rKH&ACcw&7R z;GBUN?z+L`(5PbsMxpN2?}hrs-wO@%zZa65*N4W9>q8TokwVK0>q9FJB8@|<*Q-O` z|Nl0!eewFxF=2h^9OM6|`+s_Vr2il0|M@`Gch`r}{%0)4V+9>_or9bmMeL>#~<-%rQCT3#}=3xOAp|swTcBxoTS>kvpD)bcYp@pU(awkK*lY~UR(+>r*Ixb|S#WH=VQ6{X z_#Yf}Uiy+TFMPiYnMK*kUsJ z=G9LcL*%z=)L!^VI6x**b535nEzcq4IF06YY;d%_WYcQb|Eess3D2SMOLbrJ2#(=6 zKKT4jIzEjG-x~I#&F!gE!;sL`t6nn%r6)~c7MUTndIOY z_Nnz@Xbe+c3@1lm6h@=sTm8dWay%wt5_+FlAEuDgkiPzrP)^RkY|KISis#pD?GSqI zHNN31>X7ykto$`wIn}#gDE}!3VqbCh7uJUb&g=Qy_q0Ejg*VU1=k^5HP@`T%Ulf1) zg>Q%K{;hF5@d@rmZ8>C@@+Qhz*aV+~GdbC}o z{e51tuFPuV^VVs5Cv&s353p5v6zFk&Uiw9CqW__<<>3TB>_yLuUi%0;?7AXq=nehd_aNg393Mk7J#~|vA0{1sFU^me z?>r>mlD<~qdFiWn9@<=U94B!aXK@}E(S!C`{C~tTwK+VT|Dlc@%i0#t=;8C(>iKw< z8D!~go2Ak3m&J1xy|{r`H^@xYj!)0M!u~&LogaE1ii_EQqmsD7?~-!fnQf3f;MndS!^x~=|?EWNE; zz5J8>(8r7m71tz8!8DYk*L=PiWViex& zk1p}2A2G&8_!2C|a%3;K2dtpidB^oAt#8%gxzJ1PKjy`zhUrst&MK_ITCB$=RACF+ zpX9esu>Qc9`+j`ax%+;5;e@j6D8K#H{-Jd>KOX59`R$AO?bEG4Fqt1eu7Bt@C$jr1 z?ET~HeV6$gp7n`#SX@G@=6}=t(DUNC zh#p)K#z5D{F}GERthY%Y ziihhzDsOZ0e3$Y#e^(j5TmIiE|8JH47ua8{C6ijE7&i@OVxgjIVEQ^;wkrI(X4P&4Fx|NpQyE620ZOpkN_8c(qgD2;!c z&PSHtlk#xmZuuY0^pre2$2Ie?0E@5$OR*d)(B8!VU#9-?s`>}nwNCwmEI2NbE$_+y zI9M+)*2w>Pd2j!H|GsUf@+Gg_e!qNKDZW)G{Eq)ju17^9dqr0LL0R!=S*W6K!8Ytb zT(@f%xd;1DjW};O&LciR&%LC4K$4z9ag6d|>ihK<=}rr4i|57P(EHWF;RtyQ$I0W;~dks2faPnG}q1dY?>|K0D~_a=5Cul!@jo9NACZ2!LO9%J83X@Bg) z1Hv3}%tM^}xqxhE$2(A3XP4dYV)OH_vC-^(QJa5@^0XBPm91&E^`ZTHUHd;`AN3;f zEI}OiR{0rr{%hXL^pA$6!b;=ZR`YvaWcRSf{VW%r6BgUWtCxK=tZ=*%tFQ*K4ZH?x z>CyHzU_E^kqEG+9^@Ho?OaDUWI6tzz69p`M+CzBRkpu z4&Q|OKxzCBnK}{Y|LJG`hPAL_oL}!3PLpSG9v9KRTK-=rkC8d~yz892g#x{JnElW5 z|HXfBGyh*cJTzb4TrB^SZI0vkpC0j4d`#V)yov|oe|qT;*3Y^@zlGv%ac}n@WL!V1 zB}l*T{f^)I5Xb-Y+bd6@H2%joo*eX%FwpT}G@ntfT~N*qi|I69QP^;dKyTwm!zgkz z#$r4Qe|RcPBqw1S%F(@HK$t=HeBE zH1E~_yxqG0uGxZZ*nwSm^R_hWKizQG`q%aP|Is;1|2?`E>c4+c|NTq)@5z?y=KtR| z|NpSL{y5a3{~y^V{r@$_2Mp1_kAp~SqdbI+Hp}c{{rhuAh4$Bs4|v`9fEUD%uAS`5 zUiO7Fjy~*hKgYcHePlHbAc@L9Yj^mK{!MM3^~M!6oU%WC-v0DW_AU@-Gg3%PbISQ? zv>}JAbZ1^Pejy%bqzy;t$57m=|9_+Yf9G_$|Kq}{*F7ChlBZF7#(oOqdDJ94SLsjU zqT?PkAN7CldCuLQvvcdbgZho;xxhN5+BeL7dCNHER%?~wvTLrQ7dLPVchHA^|1AHp zu^nt}CmH9$bdh<-1+wV-ZrN|{OOrV;HNMXd{=d3q#`DX1esM0$0C9Y%ebez^3`OB_ z`^b?Mb^6lC(HM*In22riUV2u)Fo|B;f0nJDLeC1zU>ZF)PT9cbmeXgT_@w+hLHxdnsALM<|WI z_iW?3J8^%p6~dd{PppHK#u<_i*I#f>9H-y7+VijRZkoI!a>Es6E#ABdC$p7?*<2&}Me_zo4H-`V=drm*Y|3C&=w4pSvO?_~UxYnY0 zl#h)(`ya=-rT6v^>xFMZ72^26Y=7-r7y5@Sj!WzIvhCaGJJ7sQy#b|lG`4Fu6jseA z-b3y~?F-tR$pfg_-9Nm!{l6Pi`rkv+aSD|kp9ob~hTDJZ6Jhi2PlSKm|94>v(tc~h zKPh*&Eqpq>y?bcbI^^l_!`J>k?6~yzA?LT=KYS|u__{G0-rbLc?eyM%I?ftC(o^*N z5gbGJr$>Z$E~s~Wa&Xx9j5Qq9YxhnTHep2AV_&SF-X0ZpPaPG0(mE>aO8lKUY(v9N zeP%zJ`-!kazu6C;8y>c|4i7&Vw<2ttyCU=$KTuI-oc>eBqYU!CCVOAg?FHn&PU18Q zpR_L?c@e!|9uO+Os@>!3_Qi928CPRCD!w*2+=#mNuHK6G$N#mk`@c8-?;F~L{B{R@ z==U@A91KJu@D;wG@0UIl!!ZJ*FdAbq9@XkUKgWJ$@6Y6+<>EPuiGG`eDR^_YG;9y% zhOwXe@9Otj18%1|3}g7m)A`5z%4Vd{vW|btS3lUFFF!&5_Ef$rO6#~e|Ka)ohmDU% zR|nsHFCTuVetN!fdb;)h{bxC5U?#Gy;_&-ydhQDUf46@4@AC_M_eK7B0r_*{x^7

f3OP#)3xb*<@x9$Jv+BavhJx2#wH�#z1G^J zN5pXqHD`P`ND4dd_#~P${?FQijjn0R+y7Dhzy3x3$71#L`O1yg)yq+8e|U?(;hM8J zkBf+V*YuE=aTV>(=~$+Y@hbm={G0uU`5>)0h%^o%gDl$6D~=nug*)g&9JkYNKYMFk zgTgnwjtB1UPRAHTPI5E@4c%{kY2!p{fpy% zkloK(|MO$k!O+iN8V_+w`GF~}pN4YGz)Z}>9Q1x)o7l7Fj|@{^a=iJM(vcnAK9c*JMnz>hy7|oa|b(xv94Kxl~{!}Sc~=8 zg!Y%%=lSgOT=w}D_L)zgU&cPa%081V@3H?lc!95bhW~ql{a?+#uVdfI7*-{oE!c(~ z$li8NEXV0N_t%M1`~S1@e{BDMg#AtMf62J#X7zS{*bB-m)aH%3cHKUd*8iEsuAJsW5`PI_@I=dh&w*OxyucB6XFL?u{_W!y57q=YWLGv^Icbxl}?mqm_-?abF zRj)=fQs{F{zn`-&f3(&QIT%AR93$|bw*NmT|KZ?d`A>cQuiO7O$hRB$CU5dl;uyeD z;>?_pKlv`B>2dteSn|QV-7daB?0YPrh${D0`HjzqgX$&Yoi`CZA1e!!$nHOWG)y6T zgE}5rjv1JV*_eZQSb#-Xf}Wr8C&=zP{j+56znU98(zk^;KXrw$l~{$fD13Q(SWi~( zcTZpVe29CdZgO0O*cMUoE#ngCaco5#hu?HlzEdYjk#UcqO4n2!njW?}X9sp+5B8z= zi@sa3`?uPN$eu^lVaXKIXp8cu?~*Ly2#z7{Efm*yJx(vJ_cG4-HTr2hTz}w6-}0E) z{%d~uwlZJmgV$ zsXP>4C=V@7=BH=OPj59peXIHDyUkBW_AT?%QSG-l|K+l{uA&#U{h!x%^?bNNPrl~f zUU~k0{^w6UTz_DpdtLT?xaFMm_VRFt>_gUn_p4SfLZ+i!yOjG{R~`mB9*lx}iuYBz z&-w0er~BLM{_5SI|2(wc{UM7sv|p0GYtl#N?wUVO&tEWq{*3wap5ah&4#xZXN4ml5{`G1cYcN*va1>;bKEx}SO#|o^(-^~A0=Wjx3oep+l zm2=i$E!HE}|2L6U*n;+v>@Pd|zn%a0mi9j!94G(d5ZjbNmfogaAh!Q)6VDFp!X9K7 z@-MXQ?W5;jVn0z$KY-$0`3!kmIH|G??Z(R#|VtVXpF^p^nTj6L3V%s`7nv>`CKAw z7^f^6ro8mtrwNPm|Kb>mICh}i@eDke|2LEVVE*51`W*cKo&R@Cnx?p}RR1=XXC8e4 z7GVkgm-GL&vr#DG|7QN*Qt>Ru3amu!EBELBt)f4e|F?#|7JoheZ@sXzwu4P%6|&a{ zge~MYWcYQRevA9t>~Opb1wMZfT}RbZ$o5yXU!e0XZS3e`OY?|h>`L<&ho~QnR6k&Y z)6P4@=4Ozkx4q1M&SKx^f1v&p>lJKp^K0xZ7I=Oiv>$ui!#-5Ud-*osVJG`cFOC2E z5}%)5o6&!<+5gd#?xn#wZS)*ccg2k+G;VZ{!s7hDBjho}wu|HBNqlhr-)YBZ@i+7T z^8e@gf9G9u5k0t!nE$Vmy|{sg^Z)q#ADaKy%IC)a&HTSx;<VPaqH`x<3Z|hPGcXgg@uBg5?|q%;^2$HYCyl?J|1n1#vF%_Uxd4Ty z9u14gUhC`i{_K7K|BJ>4d|O@bC+5cZZ7G&x1>*RvmE(f9!4k-Nyebpi@x(YWe5erY}bwBcp`?>hePGI3`Hhvxj?kRp?= zO(Bgo9&(&5~pz%wS457Yt~w$KUja@BE1JGZ4=G>^atw?To(4h{C(ANFK!^tzmM~7ZqciL zukDVVxkK+mzgo}5_g?p^^uMKTR-0^cyS7>1Op|Y?`L?nE6+hEP@7#eHjG-8g5g3K` z|E&G*1?_+M-;O^R@{urFJYz8)6Oo<8{$dh6x0(IL6#6t2=>_C>Y9GYr&;Nahefo|VZeHCkvfDe@L-u@C8-A35&m6weRZ(S zXr`x7(P&(>^HNBo4LKBX1nsYWz&}(U?0STMIGukuS^hsK|Eo*2;-K@=%e23)Q#U7L z|6tpE{Rdw+m%#cAac+OCe;gBM9QU(Poh9~N9CsYs@K2Jbqi(OTQvKsM#?3e`tv|s2 zU!iO0~AZxS<*Y4~e`W^6}SByO)2VyXWBChS55k8!rYh_#aTa$)93Z?ynx3jS| zY;8UJ>A&iqVdL;{{yn|qr;?*xGZy195tA?l(@>7~OWGf_>35Pj?fYG1-f@8}YMUsn z|38`iW*6gJyF=1dntv~CZ3+HWz4iaqr#5EPtuRBJwfn9AFMK9FsXh_c|F8I&vG9&l zvF|{gqDlOX`X}ZHi~9PwJ+wl?)cnp zIzwrlp1ExLYiv83JLG@F@?)v%mSY80Vine4E!yjSe~4pDbLe_qyeQC%D9!&pBmW~^ z@B0&H+#5biZ`-f_aZ3BXcs8L5TTpwQ{k`2kY@^3DZyMy|9rRuJ(D<>0wsm3qP>lm9 zeYgCPBt3q|DKd@1r?k(JMI1rp?~E%@|2{@Pj+2P7B zN`5MQb7UCmy5Sgs*alD~bPe(RQ9xWHptSz?8TtPL`%fM^ zp=^E4^G6%Hzxw|8KLZ)!_`lKO9FNK;JfE+O3=`>-Fa^_4`14btob3Hq?L5Et{q8i* z$MH;?2iNZn*x9LK%S;~ea|SIs|P{d8#1hn>{7-MCY~w7&ml zdP+a{Sl2AUQY^;`ti&p;LHk?!|IxYI{C|BjUHbO(D6G@}zgqvl{_fTj`v1xF1^xeI z#&MQxb6)!9(_yW6)?*W@kR9n9Y@z2K5x0KyZS)-|IxaZQlU@3M;{LzYtxw0Iy8KO#k)Ds~i}YXV z_`k?LBaX?QCq4I?^uJ;(0{sZ$n8JtqAG~aT24U6Yaq=W;mpvU$lV?%m+S0lR;yUm6 zB1-!ooKny5zj6PA?yvlP=n>xYwf_<>lULD;8@Po#DA)t1kL*{kj`-Yv4Fk#EU)~9W z$sI5MxA4P}6T|j%|26zz)_)DV&gm~({dZyKApK>B|6ABIQh(YDp9s5WeIop1(D1O& zJ^k~L5n-=>yLVn!ci8`t@bkU;qjru8KYPm@_wQR1^JDJooA$>1i+=C_VT{Zh#)Q0e zFC0qzC8QJo;aUGjXifat*pYt^sqTLd&51vSrZN8(8n69ZNGAR@G^o?npE0)cg1M#E zf2@7)zo^gs7kx4x3vpfapI^6k+9hLu-2a=2@nI8&(?_84xrx@IpBO6M8Xq>l_}{`m zO3Rl1L$yD}`Gft#KMffkw&Cqo{f~=35q>yb|KkfILdCE8hl(&D{CKWDL;Ps&h_G|& z--n;B^Nq2`qr^QL8M(a9_W!@3{(slnFo3$^b* zr`?IZ1>4ZchUU@kehu0cF%W|>6vNSeLjV66{r|{a(*J)~|Nl-lZm+%)vSpU~KMpQr z|8a;O*M`eFukEJm`j>^=BW0o8`{;=8gR8G7-s3FuDA9budpjEYSG>nF<~F*=Q5cP}7>|jVgegd$GLM)nN8JBz2ARF?`@GGM zpg&lDWHx;c{(Ak9()dR<@ZtE!jCnWCTZEeYecKn;A1TehZ?tzE{e$a|{4e((ZdLzB z3UU3tCE{9&ec_{@gzMJ!x77f z9J-`GkJ9+`-Rgg<&71Ih8*(V3wslabS*9PCp4@rA@4xV_x|-v-cUm)=HuL}Sq4|Ht z`yY2shRpKAPtq&38CE^64#uC(@%Ouw&3S%!fj?d(>lX5VwH-7diAFS`87Z6=$61`m zMfBh@uA&$13I0DiksB%gLE@h*{^>D(-@kil!)Ah|Ig?DFLr*I>{nVf^iX4rx=;5c0 zC%d1pw>!CEnEE51_f2h8lN{HblK;=i|KsF;#5HA_&`eLEpv<4|I9c5bC4SH@b`+}wP z<@oFQf1Ce&)7Ud>(D`j8;@p!}g584zbqpdi?eLzxFH2FLb`5{6g33%CBYa zU%jqKwzL|**kSylwAi;@|DpRwR(jf`p-!KF{c7U`*Xau&Eja!(HqBJ>q>l8Nxot)hK+?J(DTK`3GsT>d(p$Z2@ic9Ev!Cv@{+gkK-gx zBd+Zd*El;%&++#=S6kPOei6mkHh{c3Ko`2d&HoqP({7H+FU%SEl`)@=uc9}Gf8O^= zcB{|cA}hW#Al!++`7Zj%evO{<>((nhVcZ{m!!y1wyr~?HYlAhu_kMd4UoVA$&KZn` zE55gGd%=ifsN=#HtYJu2enx)92>K}0O;xU=VUTj3ZEftYT&FjaDfYDG5q1FwpJXS7 zunW%3AnX6z-qXH*n_WTY3HJZ+`|a++OYQKjuBkE4-T#cmcuYjLNgS9&&s}pZrqHJ$ z))@-OUlNz_a$(i%)(mncYOf3kv&lKAp*PgKZ_IPN0L{DIH%j{q>>m&o30s1tSdJA~ ziB(vG2mNbn>G2y`Pi{itUrPCLL*?&{OXBx$5w;CGunT+8+fo+xk+J`_n(X;nSvWu@ zkwO}EyWh9}gS?Oa^8ZNhi!ALwF`51UK0D#O9Ev!CV>phJXg@6fpc6TCA&&xzD6RiL zM*e^1{qcYFIJWhH{W&d;vq*2{XOnT>??ti)Sz(vStH@nZ4s2vQ>6(N)98C#U$n)6SXXw<*YOH!hhh zN5(mIY+*eb7P>c-)+Jl*o?m1CUzMhB-VKF!`@#(8&%|uZ!8|NL`)%tVpw!MYOP*dV zZ@(aqqqvU$KaBk!$^Mh+_t^h)%2s6QZRq(V|6d&-?r+xf*hfD!esGcamtZMYU?t*O zLaWF%QU9{<|i*a**(Y7p&&S{!2jn0XCEj6QV z-Mvu1@t)uAg(MpJSxso}xaZy73(Li`1>3L#yRZlQP>uG?z0i@rr~RW(`$u2sLLLPa z(emEC(29c>?y3Lq;l!CicKf~1hV+no;lTe(*xg4}desTO-xNhD=}=0#$w*o%U0%se zWkN>0l9Aj{vAkVg%VoL5C9c8MxGb0D5|?tEXo|3F^(%x}U!kuM;7*!GFvgqM(igqPuu(Ral2$3TVa zEOTA%C0_0MyS@?E|1a$Gr??aU3;tL9OZ+SRZ}^@0$LgJ%{eOk7ufX@WsQ!14<78q@ z`=8AGUzD6`%nJWn8vi>ULeI=z_&>7R^w;+L{?yHR>S)BX)X_!kNOIUQqwD;GFpg+0 z-vqln$;T(NK6r(H`{eL1aNI08=eT(?C7dP~QX#y@dESQ);6wNaWywd#kKq&O+tEI_ z>fHUV0Yfvc0mJl>m~$T67d~abGdPDf-0(kmgnq++?@{_=SU90fa7@&Oo|k_f7xx67 z!qeFPjDMhz0~p4Y54$&V{q5Q;WuU6%n9G3 zZ@=MM_zt=L7uUkGWHkTpyX5!q1N;aH^~X=hpW^5EB_@5N(Hely)8{IDyZANz8Wz^o z4bF9A{lkp^eM4df6qYa|%Fhrj9 z{YBs4xoYi0cIa#ElWfxdsC`MepWgNc^|N&$W^J20;`++99gwCk9M(qaJP;nR-%-2~ zZ$huJiI~2Ox6lVSoZmigqrU?qTb_-$_^xM#_1EWwcZt99>iyx}3e{7uA8AlmM0uzY-%N+eOb{Sls5&JF>$!=$Zd|T~Lm^)Oh`q(%f`S%8C>@FK@U0?3VC<`bK+}vctFR zA6v@~-yxqxuek4$-@{;B8F0zHbzd(7M#!iP_V9m3*3TeY%jfE-A>*nd^@5szP2~yh4a=EoZIz|yq{FeJDTxs z-QcA_mk1sNM!#%KtGB%;!Wt$mVXQRHbis( z-a)<#G5SEgc4cE$c(?F-kg+S(3q#tL@gv5c_=5NI9Umb7VEw<2gW;fK-iHt1L-+{J zb$JFji)rKdGi>S1_SpI_yF&Q?y$m>GB-@^~^BlPCz zf5<-=enOAt1PloOlpfilBlW-Qe{iJEi=e2}Iu*zZQY+@Gs0JR|))`2ukaA)531ymWqzYuGpDo5w9kzs8yuN96rNWdNqL zQhVN2YwfD(}maDP-1%Ol5^9Yr^?|4H)7YkW&&8qc8R zHTyF*R>mJtzfv>@@EhWzIe@+DfN#;;-k=ZhbL?Q(7L@}-wemfNkLwSpkpI{Cf2kAx z+41+T1K~Z=_zs>$%WJeh$nW6?_z|Wr@}KK4&TZd0GV}l2r20C=H;o)sUykjle~>`N z|6ly1?Nemmh2O5f*UnByW?f%>0>kS0XsrDw&hb4{{#+5;}#smZP+%Bv;MW*a6A3V*9t>a zuirsm*FU@c{kt<`RXxhjuN3Bl?u*uRsn&k6{Z+UN_h9n0I=10JxQ{-kuATLKUrWCq z3k9AHQtI|-{(jW{_-yBI!UMLw5pP1K{o&cXg+93H-K_I3i2e`oBzt=0dfP0%p{?NA zK2?_+{>J@2nP=_Ct=#a0xjA1h&$A}vFG8m>^zrqb@WsuY@P)-)V{(6QtnMGe=XZ9) z=k~6K_N}XYVqY-+_3Q99=XwX)3Z4(=%byP|xxWsrJHHB#T>3@$Yh!EgvhCe?bp6)w z*@A=NGiARF7qg@nKNvpW{L=6q+un!AH}{7xHkg0ccuVNmxg|WYY(9pv<^#5U2+c*= z;Vho4)=#i2KWDvMe!e~QU%Wl^VWi_e<9YWQ1G+5?b>0?wVu!-h=++nUbLG#!Exd#8 z$sHl)e{o;!x-)!4njga_@F|?ZIke#sBsT5|;~0zot1!B8UwBm9V@PJbT3`GkgkdNE<2|R_TakKoGG1e?ReUklO!~U<+zn;q`D7-%QmZmRc z@qb)W_N^!b*OZ0igmAPrU(`Q8MGi=#<(t;WC!=-zC&^iCzuEWpZ>;?p`3La(yTXs@ zO7I=LoL|7Fi$eNW#*OXw3?@%Yb60zb9{pqKTjY1}EHeEx)ym^asxDdFiP|A3S+Y;sQge52k3K`9lLob z+(JKyg+}*n3^_&4FB|_qqTH)4@=d7+m4Uri4~5(8cROOX55$ezyN)}AqyL8um$~i= z_aXf_*=NkTpFZH)2FJuZCcf(amh889C>(O!tI!tBKm2f3xQl)d?!#*_U8DVdN&SjB z%p-MK`M#r#hf()3Chd%WfHL@||Cjnvp+mWe;iNXDLjo{3}6_M@A^17iPJKm7sx&isSp`u|R`KXGFn z@gaH~H`Woq=3b9F?u~d8-h#K`9e5X})#)>s#T+v0-;Al_uz*qZ)R=l|oJ{Pg|H-8A zK@CY8oV|W}-;3+(fUJT&M z2l+sd>-upL%kGUniCLs0-F+tU41KQOv*|t%zCr&M7S#7?aVgAuK7BL4^?xXIoye@q zSK;}WxlZ@^9ml-bzxY|<@8Wy-5vH{zXOLN6M!vq;zEmzPR4IRoJ^vEdNJeWaP0*7? zo_~So@0)Hr>$&`RU-*f%diZ_(l#Gq>@5ImPzr^zx*y10$!RI6L{UC=ivSJ-OZTMf? z-tu*M>UegzMvul4_9f*HM80+lhwKAme}{~ z-swH_(}idI%}>XCitQU?18X-fkbTYUe+*y{Lx^8w|F>TE|Npsb_$8jluW=3gCVgAD z1=C&Z|2X?U#{Os9|IYsh^+SvL7V#Kd`k%TJN5w_#T)S^yam5I|K#`V`%C&id<$c~ zg>f=*Qv0LIw^?KSkDL;o^~~n~QqK{S^r<}0G2e5< zK&R`rZ{LRdpbsDO-Y`_@J>qSy@f~;<-i`O*efR)AghXXt_z3wid;*`sWWDtV8uP*# z`kcCQ)|lZrdK(tBuhaS0+t~B!&qu`d6y$|R$;Z%JmKPo;pFpgF?~3|yaE~9)bJo2O z7maz1s2_*ak8!fEPW>NON5<@z5>Lj3hslv9zIyEF=AG;7Kc=nW+bfL$oL2|5nO8J@ zS^101f7X`eX=QK<8ULCN<*z(Gmgo7Ci6YORObSnuQ}$bb|I5PqlY9XFB%1&4yO;L5 zG5>M$%8&Ka4DAn-^jU1*Doj2T>F+iF-~S4}LH`z7{(OH(oZ1(@Lr;>Ec$WTMd=Il1 zwNG1JUzh76)56hOl0EU9@RaR8z>n|~^oon&r}Sv<{J^mT;pg;UVnldYI8F{#9th8i z`!%j%-<0x5Ih!eO%R@LIdCe~gsy zyYI^mud?4=xCi&)wYVP-U|K)d%*y}3|4pC3STXxwT8RqwKbb60t`@QXcSATTjdgz4 zZzQk$Dkr>&d<))&ci>%kH{OHy;RE;(K7#G%Jae{gX1rTFF}nAUiI3I}NM6zoq0g!N zXSESNMNhLIqB)T_>wk5BPtx(WW#<3AReq=UmKuM^()UjvJmvZ49|(`qAHztc=U?mj zBU-EBadC;1ekAfKOe)u(CVLUpL9_DtTn#%zco@;x&%%as;<)c%kN-@)?;n}@FXZq4 z-?rsv=GHrI60_0qcn06Vw=i9!?@v4Xzq|g%iuUiC@ozFw$Np{9PO$$ZeTux#ck>XEejPeu`fr{qhIH z^JLrC9t`JwcfY1z!@g;C2X4VZ{Lkk9-DaQLaR(0JRk#cH;6D6r`73{|dajG|7oxiU z59a^ zn;wk`y^s7rwEe9ie29$tDnCL-^PfIOegdCD+rN28JL18R=&A@m_gvm1?hMXha>f7K zlNF(jK6lms+jlC$BlJhHPgPmN5de75XUe^5as`2$BknWvq<9^d;`;aTBT;u7A;UF1EOJbX0VN4^$G z>juoKyXLNDh5LmcKxW-Z<;A?ZDuuq~iqMY%?{aXnLi^r(*{BG^TNPm>pM8JT`W#Ef z_7~a9#$coQcyE->`62D+`Tb#9eLPd5jxN^!wWs=67GZ_olpl__J5^8YdU9~1I* zvO@h-#{NCRPOYm5Z<5|y(A!iI-bSABJ@!~L=^ga=MR^`0>+0k!y7th!#Ap8F5obUD zqp>aFE5B05e{Fww5B+`k06v6|;A8j%K7}(lhiG0=8`-B!>30tUuK(u$PfDCiIP#Hs z#6FMWF+7eZ@Dy4`_lKv+>1Fr7>i%8d9Om2If0z63aQ_$FzqH2O>o^iQ?q4~U+;RVy z!uls43Rhmo_J8S7{|~pGgR(4I|GQV((Hdd{;H=zGX4-} zTW0LZTsEh3{t^Rh%fY3b@H{A7UqMXbvTM}^;r^l9~i@O*>( zNA|JH`%64u*E#4qqc&Ci*!4EnNW+2fCda%5Z^JwAF1#D>!TT_+KADl`EFxdic``~j zUF19k>Yu!*|3&+AL;F*GG(jeXC&?-M^|13lAe|2({bu>wx^%slwEtFf!bijn9+5XL z=Y)^ZKY*}Kj9JK7UjfaWX8Ym+qZ_t=}+J(JdHkMfBl{M z-!HT6l`)zBa$i#BV?>{JuYCqEjB!k27HK?#>;E7Bt^Xr`-+Aqu6w;mAIT*dD|NZ#& zG01=R|KrEx7wLQp3;SOdzC%umOIB$=)1x&FW>09B(7zY$Bb@G(ciQD4@(1F6grDH2 z*#0X20rHo49#>wj?Ln^p8~vkymKVC3A@MQ!`&Hy!=oNPl zc^{%V5Cg6~UhTbAD%*ske?0Y_X16u4KiTs)|9_Wc$Fb9+_5ZKA_SZW8emsDqIDwO> z#I$ksnL76OX?7&LaK3=uSj3Jb7mVkR>IWEGVb84TA0Q`$llAOKOgS#GdoWZAeuqRK`=knR#Z1Gxp9TwQ8Y2lP{^La93e~-P${$@MZ+b?5(ce^h(_+Zt+&?v5j z|HQEH$PW9Ij2}N3{^s~g!&7;=p$kuT{!#dHLs@ttzbtfi{%iBqO2Ze6OT**nXs!rf z$g2pSzwl6atl;&by`nsP4xc?;%m>!`cwv7y|COxJHixi6POF<{Qt}x2 z&yJrpr)@n(1Yd*g2N|NhMS zeTD8bKk{$5HOynzK0T|qhO6WrdX0}}&4;r&h&gURI)kZOL#}WhhO3loY^I^kTSFZA z;tEiRA{<6Bj$nH}E0mDypFS8$$t!Qq3T0$DDsT)5&*3d!6;S_!Fs`-E2IpH+D79;9~;bnC; zIkcV=>cl0gUlQuc21IssBiV$=&Yoq@&!Jhk6>0V90_KszhG$kK{sJzd9UbUI7y69d zXZi<{`UkeuxsHirm_D*$e7~Q4kMR@idrVZb?=eZA!gM45fa9*?nCqxz{{V8K zoc~{uxuDJyLqFo^t!96#hsWrHC)LeJ(q}NDZ8MBGhVbJ552tLugqAOUGJOflSivgR&{w1W?_i%}Fs6T2+dRIaoYFQQ!Mc4mu!$|~U>8@hhv{ndKW6oV z%q^<_*3^Gw8qr+$(WL$-X^uCm|1r^cy>EK5R{f7?{jIE&?-l8@uMD~5WR>ryY+uNu z&o#=M_4`6Ty#Ncotu$_|Us)(FT7SQYJdED_eW92Jzp>T|T94BxRXVvA+?(LlWX<^5=je4G6$PSgZRiPR+IEB-=^5K1+x+r_U{Uei)-Cu|Q8Fg?Ws7|B2Ju(NR0CO1rs9 zySdAMg2>2+WcDH-EA8OS|AAZiAuTF{w8!SPk5l4jq?y7cEaEbju#6RC{_CAG&Xgq| zk;B^VBNu(Uoxb4??ce&Muxh_GtYZVcwM8LzswiyI2QM1?#}<7DBlKa!F@#-liIqd) zD!GSA?ES{>jkTvx=9n1rZEM-DKPP(Mh}JhN5Z8Cu z_&omL{{&ga=P`;gj3a@G9An?P{11%nPnGdM$m4%d;5v$22RSdC+Tee%#s7fpX}>cR zI#1i(+V^kH3PtqZ3wIiix-%T6587t{#q=W>aoq5_e+caQmjFt{B}xv3QnCz@-$OZB zfyfVH);-SQnDB9=8??)tw9~s?2l|%X->Umt=YN2qO8y5J#t2S0<|Hang=*B`6i#D$ zP5xKEW&F!7O1D${A2<4UM@u~)j3a@GUHN}c`=6Y0T#t8ME1f#jqXE6k-v4S*XrvDs z_m9T@o9NBR{7=(2AK&o)(Oj!utybP>|C}=hb-_Lt(T)y8>u+_E=l@FH{n`G|Mejxo z{pf4e{_nW%(>EAXFQ$z3t!V!*t0xh+&lr-JK?;|!h|8F+*Z+uF%wb+%a7r1NCKr?& zqe<<5X^tCTOkko?|05N@x_`~x)*pCpc36_``b$GtCbvJA9ahLytVQwI z!1i133=z@#TbsgL*uhoYSbuAeeq;TumRHNWKUDv&$p5(U|IKN>9W(qcX0`L?Fz?x? z)|7>OEfy{pg;C$~7{>W=BrrkGeBY^}FqPCc?Dzd(V8-`@A$lCcjlLh8jmb}O`H9S0 z@SXpycECUG4>|N)w7SM!;Y>y~h5pSN}Hr^Y(vY^*|_go+BthDJC5sjh~m%=dzT6_9>@VU_m%t$bQdf zzmvzr^=OM6Cr=>iCpbw~BDSOLlGKlo$_Z7%)flccZjET|f;ggo=SF?Ung1KDXN4O3 z^q#iJ5}-~ykk+4FUO89xQ({ZO6f+ok^Ra4z||^Mk_AhBHA&n z4xhnnuJhzbw@CialK-(##{QSjZ;t=j?|16|Q(2)y8lC7ukM>+Q8Qav~qHd1S`w_?Q z_{WT~9TClsAG0lqw6Z($U!S3;a0!dZ_{$p`iR|6W!b@1jdGL=~!$HuhgbZhu(&<8JkL z|Gscaxb1`d0<0r=n%>*w-JzBq&1oE{+!yNT^@!G#8{W~zB!`Z<|5o<@MRqco`Trk1 z&h{TpvXj~D#~ssvMl_)rt+;@Tn2sCYx9_ZD=E!+t#;GPYI=R5UAMI!ViyyZ=K~C5| zTH8P4|4&B$|7({{2RhM($qM`A91Pv`IrjW)9e;RwKNc#Dzt^(=PdOJE<=yE2f0=P0 zdT*oax8E3I#@7dqxUVwf@7AtQiW@F*Uj@eB$#}8x_b&GzvtQhPo*u)v)_{c z4(KyD|9xgI)BB zk2M?&SLvDg1Jd22XI=9CUEeU`hhLjS{x903elPHzi`4JrRGv8r_PaTMa7&rNCd%{=ZfYN( z**RKq0f`FzG-R?w|APMic6!=4e*oJ;lQHxojxi)LgA^`d5tp%qZT|yE%j4@G_l*cg^Z!=J zRjgqhJv;lt2DynX?4Wm3{$Jl0cIkr+@;|Q9_i(fQrvGn9dDnMD{eyvu-}XQ2Pmjj- zha1@m=x|Kb$9AKCAnO@zI^RPMnTzSm>UTtS^c?2tDWuhz3mDC1*I>L*`v4R4B%(F` zr;tb<2zk3=`<+xoXj{AuxZ?w=g#IuI%xa}39E0w+<4DpX^7*Zpg& z%#wewesJ^u!2eObKjr#sq;bPv@Dx4rkvvV-q7Dn??zhzalF@j@jq!|P@1@@Mo;d%3 z#_QkU+2|Ym3%+0CBHzbG+oON~MZTSFZ+@7cpZNpAk#A?lf1v(AXtK}t_w~<^t+;@T zXh#P+(ef4FFuCyR#?Vb(`Jna@x&CqQZ(03G&(!~Cn?jsE$-a-qxW?$w*wd^!DoLNg zg1maO|J)cVlCdqkZzTqB=-uqmV*cL|HQj3q2%1v5MH_@=f$ z8T}(5-rd6a#){XTS2uEw96-M&oArz78 z+M7L>v%+C|F`WLtc_xfkp9zVoGhw3a zOh^`=QNErDQw3*2U;JU?&ku(I3~oKlrhYiY3my)``48*AeK?$OzLTg#721AbeUj-A zs_8YDw$DuA!(sOD!{+}ze7$_%(?;A?zANuX>mLqd7)Rpx!(rl>cXaaMFj?_%m?{(J zx=u;wG-^?Y$qv`pbSBi(=aTLX4fIAVTySr#?y=py#?FK$aXm$6LNnQl-sLml0(lX! za?f?gb49yw2Zk?su1lV?XA@uXe6%6Q^lgpv)k|Q4o?J42Z^!yJm}XPYU>0*t>aWx4 zuR8ZfF8EeErP+mUGNh^CF8kTOa9MZ>%UHoG(&k`Z`S_WzMqi(93hU$sHnD{r?BXi+ko9%n^w-XW95NSq zi01$1lLaV55pK-?JxrgI&t}#4=~pY`g>TOP-DT?|lYcK||F`^0KVrWU#N_dTCHYss zLaA`;5A|KsBfp`cG5Htk|ER5E+m*k#GgOe>2h!){@n~(ZI(j`83gvs-Q{?>Ntk58?5lv`D zD=y$7w!i*xXeZZyb0&0FCiUfn zF?!_tJ1`^Px6AkaY)f+3_M7#;`ehJ9N7N0K>WC`!-%0hRW214iBpE}`I{&iH{h_;X zfB4Fdeo4p7AgZJ17S&N}>MAmgg&p-@nfkv%{ZA(H)c*zQ_9FE^IhAYfTla@Uf-Rlt zf3v>eWVil$=Sd+srr%&gzk$B~OTvpt)1x(j=6Chu)Bon`C&N=4pR}H2V|eoLr^AtpO4r2XWR!vd*cVg=ZZcO&K<4`f4T90 zh0D@jLfgp?hVvZ_p{27nwBnIiL-^~NhOlhg3LdThX!vaNr^9E;>%&Dn=3JleZwjln zt>N+7#_&bY=L_qf3>{64*1q)bJN>omwryY&TX?cRgdMWSKeV3qKDx0h{BOr{L$7rz zWB$AL)lOqt?FDSJg3xcgudhBgj1;{*3~%PM(XNjJCd`8y-+5&iyQHs3f6?NF4y<7mmTtidkS(wKH2tH>_=_YLV9np@t?AsP(&ZB)rWE_ zCmg01W29Oh5*No%eNH$ct^_UnZw;kn8P-3{jwP@B?EelGzRH0x0 zc3-nTnz-|HO1E3OGx}&^&eLyAP4;)S{c3Our%{VKq<^Kq;e>C{GoEu@vt&wmo?LKk z=_cR4G)ASFkmfj=>O?k*&q7X_A4Lvhrwdyg7PSuYmo6TR6f)@PXij! zgctiaGz(9c8o$J>c1H9s#d)?tDyiL)VjqyBwc0_r>Hk!&EriK@Z4k75#dCjaUbrBw zi`f3^fzVENMEbY&3+UTe7q|X;K0@Xtbc*jnH)7~V9Epo^#8?RyomF4sNdx?9=t)>S*t#u#pP-59bhZr?ro zCE5>$tZ&F8n2a3^xnv%aapzul-N+X%KzfrczT(`wu7TbYyCoEgFT!CIqj$;ri8w+Z ztTP6T5_%~{Dy%_?c-5`?QvH7-*Ym|dk?SmQU58y4hB1ON$CRT2$8a1ca1xc6)`pu| zQOB_7=g4{CRHeEA3mBEg7{-geAMbJEh-d3Olg>A#f1*kn)u=&_^5zs7i{-H){O9lc zeCqVy*NPk1R2E@~&p_Ni0EaQM#-HW5=YCRo;`#UZ-s+;`kMY|Ia-nJ14TM$fFT%TDg6$Fbmg)ADl) z(fYb$;*yv_3hTxlFOku|e;3Kfrij*jjQRy4JNmM?B`hQIKZw?aT>k?9|JQ^6f3)_% zs%>jn#|AdBg&pkTDz^0tZ~vq9E8eSpwqN)3epS zja_ZX6TS^HY9nXt-zqjDqP1>^lIs7I`kx#TUXfOwGY2^j5H6QBIHM#6|w^74%~m zN$Gz_979te=$bO~|y%+O^>#8@}d1XcpIs3%H1Obf6Pm z=td0vh~vfe*O+k2S>N+nHr1{A?uPtpUVrqA_$y!XoXAU9L^K!eGP#6htY8&uSjPqu z^4unQ!@qxvzJp!NMmG9}{JE!2+LAxD%@=UhwmoEhQ~8TrGnxj62 zo^ES9NH2zd#L*i&6krv6IjsB-_c(rlDAhUp_%b<7&pv4Kr&VF$aIUUmPj zY1XyOVSdv!A#Ln`fiJ*lq5HtNxCAEblf)!_3O#Gr*D%P`6Gb6hm1eYtLCpQ_(FbiG zKr+fBm!tNBYeTelLG-SowX>qWr}V8^A=mavdLEgNsGN!VoaTznD|)l{#eXbKUnueJ zOML?O0o*kzjdO20mZe8hcF^PMgQ38*9Pt5gD{w7d#>%|P@(PG)Wh_a59fqx$JgK# zPUBpcZyskcqfD7bW}PrHg?TI>jj_Yp;3qtzVr_6FkMO}4H&vhwjy~7ezwF!?%u}uw zu#t1v##wA+)H+`s>d}bk-+@hJGg>ii|Cu7s`H1I?-~YcuX8rXl{#P}8t;h-QBUz^W zuTcJb{%z(Ip8sY}xFD_V`4BFW?dXWYpYeY0wFby94uwwPE3Z5hy2y1lL3b232AJ@y zV)TB*(V`Dva?v@~jQa}DExWcQVfqXf#(V>pz57e9mF!u~2@T>?xP(RY=BR&*4us3} zL3Qr{yM2kijOd?s!|e9Bx_5|N5%=Q$qgCNGtYZV2^}oNaUZ8Jb2fMh6J!Jg**!cq( z#E`Z_JVza`{V+mieM`N7T;w4i1t>%jrc1Oxmh_J;>L1hIo2TDg|Eo*;?*#wAD*l0$ ze6`g5H`gaF*8b2ANT_2EOQ#q|P=d&ZKiPgDl+x$O$cMj-UXF!5eH%z&-hLJ0dYbql zkjK$`iH%F1L~O;gyR7~uD}|%^e>eWCJY>K4jPZv}{WTc4%Kr~T^f+#;|9@5+t$9!T z=28e%(umePJO6QgPV^kl@4q|$v7N1i|JnNg+s2%0q;(3X5zULKCF@X+D?fZmXdt8h zuf`~BJgO-Q`(~TT$p5XCynx5~!CfS8_`kK&Z}`7;&^z(_{%>94x)DRqp1c_8Tju|< zRpazABr%{Yh{}i|L}f%&HjLoK{slAkOW_jI@4Pi!CfmNozJ4{I9{Ms?u!=RTBN}_? zm(K?8*`=cb<&saeznfd<#-+^*^1&-l3df1+e?xT!;0w>X%XZ#qI z^vEWS{!L#+ug1tJwrLX^6+>AELyfq|hCfA~#w5L#tV5DMYYzS#>V+GSrZ3>eI)QyP z?B6=)IL-cz(j?=;!{o@!%R-}Ln$V0^T);)NqXX0Cn9rDlFk9t3wbJE#kTMq`&G%s8 z6#s+E=B;CVRbK-p=t)f4FZ%yqr!=B@?33uCpVb!3tj}*;C{ZB2BI!vdjRnVa+b4!u z+valpKOk=X0^yAR&o_NL^j>i>jL`>Ax;G^0GZ>*qa}eT)<{yY)x5AO zoi(gu1Cu%GNp<5UeeRTgBlYqYeFqD*+PmUX4eVd+itFKnf0f)r?=k+w-&QsuR;gY+ zDc`DhbA|JenO9dLA1gQFXx?HUWBYH-!PNfAw@)G3UeB(5dv++IA4V~bAX@jkguG(j zd?{Ima#SGEuJ4{q&S?KPWrySR6F7-z&8s=wScfF7OpR>dzT@(LmFvXdY1h@{y2g|* zi2R$P@&8K4SD_j;IEB-wMP~h{I{E*!=bz{K7kd6k(-(3)|7CpyxcUE&2G76A^CzSE zSL>gYXJ2RTzxDqkKmR&u)}sN9XhJhC;3C?w{bO_g`37{*d%D;y#)dL}n3c+aC;4vG zYZD~*g-+Wu>;Jg7wl{m8-`*c?`v17^A!O{|n0kt>KS)MC?eT7PJbh&CKxmal7rGHc zKjIie5}EaX8`%F%>}ztqj@`J(zFuNqlcUlZllHi}F8V*vg!(k9TPNA$Q|jeJF?&-w zDO|!LCJ)=EO#6dASF8R#rTsx)#zK|)7pc?g-#YCNaXq%Ll56M{zfNu-R`0%~AFZpq zDZGW@eD{USIEVA}^|Svm;Q9u~#4ox3Quz-fID1)J6Vd$O9ml6XsZEMq`c?c-{ckJf zKa3+2vqILA`yca-96u0p=yM&uE#%Vk5dAwaUFsWa^{t`6 zu{{k3LcVPU=-o9&Oco(pLttQ8y@tcW#TfS8MeAB*{xfM;B92Gc*<2Zqn zs6-X2QG-)Rf7JTB;Xz{?#;O0@SlMS@VuNoH(KorR53KdiZGX8oh5fJm-MdSvs@mI~8naHHQVrSEY*>VK?||C{CiPWiuGp3ol1{Qpy)9YKr#h^LWq>?JJX zGM2E6=}PrKW)bzV&DVQ3NMiwiF#lg%5;y+;cQ)yHr93ZkReEis=H>tJ!LUZ(ey2XV zd(8`er*B&L%3p=BL2hCTJLswOEs_g=cUQPd?xD9){;yXKe@Ff=cWRh_qx#4Sr(qEczyVrve$>x&NsPo)R@E3utHBP9}RiVlaB%{tR4+%+f%mBV~#$% zc{CK-_Su?OhyIOReGo4VgE{)i^Yn|G+cxB%Z6n8wbERG$y4U$S*taLkd~15F%lx@V`xfH}N-%cmPr|6Nv&=YI)1QRoig7WN+E#{h zhl|5s9w`px^a`{s9tr20?+q=5M?xze$tnqdeYGSUv+V>P%_szlh70tI7`gDEw#kFWP96+HX!)jox2=lMZd(T?H!DIX*@Yy1HeM0tk`vt3@K@4=JzRAF424A z?jMWv!5#PS+qz6&!pLcTkE`x~+5N9S7?#B)j_eC7NoBYDi`s_6xbh3-!_Y%v z{S8O?Dm)Z&?VE>u6d+?CY12mYX$pn6`;LYp@^Ga8+0oEfV*TGbW3Q*LkHHS{myh$8 zA0|hP$tGP#w02VZhuNXnF>No`SNtVD7W5KKtDk40I$B*5`RdQ-sGIYy`>ZXf+eTN6 z!LAv5B@_0WXjHdjl0J1x{add7ZD#*>{&xM}8TLPhSK0q7FA1g2QHJBLqnxb3u_%la znAV1v(UzDso-l`bV-6|Y{Qp3l{jXnP9En!;f0ywG?UYIFm8p7h+5#t~Q;8~6W70Jx zU3(3EPTe}|T2IkWBU;liUF{w#-K(}nt+*cTkvg&-(Z2&5$VS8*JD}|m&FgOxZpLtt z_Q$Sv#$oM@0{1_|{>RP#e^n@(i@n2i2ToAs9cBDU=6{ef{2VKW_JaDvI&SP0_J57`hjMcq2}~fl$-iise-SeMhp~MjvmV~v*YicEw7U@f11$RY z*W6+6|HJCp=$!Ngedy8J-YN8~t82vfBaSgdwtMvNuOxj?eu@6~Geb{dq)z@37jH1u zhkn=G*UmR-$yyiPe3RDrC}Bi@&ak-jy?No1;}&rlOIXGVRuQeeI^F2r$yx2ROuyEu zJhrF~#+YX_io`L`;Dmc8C#u~$IVrp*jdg5b6I<9e4}XW;McNqK76072N?-rVN$&SIjFd5nqZjl%oR2(6gbh;%mm8JF~)Z;S=a>&k85W zN({F8#t^SoepLFlPWZ;i9NVf;jT)T7Y1E<)eYNaccJcrQ*|bAR>$;_^Q$~*Tv#;6I zW4r9@y=X2!`+AFg-NByjVqagdt{fYFrj>nuF|yBnpPpL^Y5D@1yVQrMcdn?7-9R>? z3C(DW#-_9vTj>{Y5$))R@ca{Xb|G*-Bu$^t5Iv6g~moT!!e!sweZ)U5L%i>nBiZ#4=d~IF0ML*02 zdHy-yK-eEP>08*rF8X{U{l1w2a8_Gj-p&^$3S$GC*uoBWaTR;Wde*mq z?H|7^^q$tYLeE32&OB|hC(f^jEX44-ypCwik2nU7dHyJ}?J)W})xQ|%S5IMR=6b*M zFnt8Y_IYvrb40iVrAU8V`zx;gPpTvJ8_bdO!YO_2X)Iv0oBfY*Brt(!Ufg6%{qMY0 z?kDO4h}Mr?q)%4pYnOI8qCSAg_Mb!BOWDWlpEP{|^YzYy$o8+W?=c+53G_A^zp(8j zeNfs_A3!C&3ei7xh7p%`wAOBB{(YT&PP_k$;#^a_)jsqQvf42P|4Z(loLF=Jac zAIr^4r?=zx=RbCc>+zmC$u9IBvz`Ij;=7MkxvwVob=>tRhx)}2*SIg?Au?`zUnl$b zlJ5e87(!hB9hR?0&}^*!EaHwmca@L8=ghzN-p7QKm|ir#zQo2R=hn0f=qY3T>0RxF zuHTN&8>1gb0uzY-{V|CtBu;1>NGF9$Sj42bBrenETBVI8`Z5;iX{0)w3oGJ!^7zb< zYv`q~lN*T9qksMkVpDhvnRSJPhnC$FqB?b5KSZ?7{uM<1<~#Pg^0y)El2@^Z?bn+> z`=7{PsWZvkNdHIvC+f0j>^90HEukpniO)y+8Z2Ow$HB=Oe=!a2^BPc;B<`(%a z@=KiIlQ_*EabbzyB2t&H&+$(b`F1ddQB2u)Qu{oK2}Emh4IrA^--nSV-!9@9LYZ@v zqXNfp94BxRmFU~DzQ3{gfl}k&$gGRsWsKR_^~l9Tp};;>s74J=;WTPdhv|Of-iMr1?gNwJ37#bE_5S?oAYlQwev8qEto>4 zowp+Yx4Qp}?w?F_xqmV#JV{PzizT+4UpjG&A&E(GNzBmaPPqSL?w5WE3&QCrOwLyx z3X9@;`VWQ6)k?vw00tPPF<3}Ogz3}XcAj@iH_ z((g46KQAmJ{RT58F>?F`Ca)XPam6@-5|m>7!8=14dF2h(av&>k4AFS}aqV&s2y>N+}4I~dA~N3_CaR;{h#NATJd$LM*|wsgl4qj z0=7rB>A%WH@cW0tMd2$i=hH)Wpc7q)`jorL82S-MX8wV?XN(?=Nz5WipTWYOIuy|u zdGxRNjQy>CiT-gn*sjb-Dm#{yA(xdUm-wPNCWT8_M5aGs%=^M+;b}g6GnmC3n|l5< z``Z{mnp`MS|6fpcVVrIL2mXKd=~2%uNoN_6AKD7pTY268ZP7y4 z^fmjeBU<}sgN)Yx*(A3xsr|7-?)pyWj=4s5f9kkv!~&uxr-GyA_As;vYf7GG=k7<{rqISuP`hTywe|7!%s=6K%o@){}{(sbyWp0;r z3Q>f^=q+^ro=Y)(FxRzJ7}KSfU?k7AA&w!GiYr4oD)3_a-(2ZBbv~R$&>1Tw9@A;NV_>ZT%cdX zf^b?mMb5Wphjwv2S$ufOPW0v)8z8$8qYo@9*OrtA!u=Q)9uXc={=~`5`rc9fE&sLz z_imr{ugW{*6=Sb6WD1wC{hAOK$;(*6GTMH~zu;He@$^-!VIAAwJ`gq{ebo0zM*sHR zB6qNhtJp)v;$F;t~`ry(k629_E?TsVKnj+WBUOFr;V}E?x`g0$& z4xsV<^}l^KdvZs9ziMp9_PJI0f7v(iV?OA@QJXZ46rz9C^zB$b4Fed&P$}CM!x%x* zH5NOr6lEw!1&-l3rZ@TbW0oyE*UB%R-JD_@r?HUqe}F3f=QaG#$%OqTu1XV=^r?0J z^No4ogmg}#5>@COV}H*a4Au0(PWE>@KLh$HL~G{{BaR{GI4v%r&!LvA!{jDk1F`{0 z`m8>SIeit!-AALiw7!W2eFpR5Q{C3BKWzLR1NejgAG+oy$26lA7Z8>4nf{*?Ul-x& za`%tf%kCf1oP-pjfBY?Aw8Z_FxqsK2`9DH|`!96=x$Yl5{nkH_&UHUr?^_>1Z`}Q( zQ~cnLYuhw`;cnv#uh1{H=Kh!6|AzY)-({b~mO6lpA@cp~C*z3L`OJ(<*ZH2KFw*pe z8t*=`LFs*b`1|?w4;+^NF@!jxH9ki$odn!Tz5CScKU*Skm)DvVAHbg zQw{QHll)JPZW#aGGX6~__W0h>lWY3P96M#-X#I-MF5VF?OM3~+=WS%1`Z ztkTyoq7Qudh-*aTL-16wH5@&QCpT{iT?N(~Jbt_JmRrJ?s&5aSmA8k->u(ERZ2sf$ zg{s>^$JTA(vE@U?xekTTCGQCB+JG(ncZSyDJ3|}JkNt62ca9C5tNzpQmz94SK6CuG za545T!lU@?%)gKU{wZtI_D9P9IQ(^7y4H;B@Etu~`?B!GwU>o0+qSLe|Jda}3ZHNK zqwu71?#d_4UlG6lCF=!}(fIKb%C0Xf57+mrkN+wsT>1V_v#;8=hphi14Q0>vXZQqs z6u;5$pzMj#X+Er-QEuDBR`xk%K{S6M*Z!II&%HSzkA7qRLOwlOlVTPH^g`Us|JtUJ zuX=~`<%loBVHD#CN>GYEHb(!3@qu-=28QUF^?H|$4`7u2J%({4Frki1BAUZLb;cU|}YnKh7 zQd||PQG-)Bjat+pdREPd%FBA;Xw0Yujr1n;Evo-loCDFneuws?U8HZcK)*?z`I+`< zMw_|U=U*v*(l6j5+Hv#$I~TP7G2ba)Ax&SvsC+i2tQ{v4oBH3B%Td`pNuSa$lc=%A zf^<63g_e2Sy6n@yju)Pr@&3lx^7MWzR7e{s%y;W66c^1O7$cMDT{Rv}rVuN2U%UDh z_smHVUPR`D`=Mg>Vu>~~8U1_nvSXI8j1|1N{kJMS-J$;P zQb%{I|1n>$ZHY7%FuJM!$M~+gUfnyfrv6W=|Hst-NPjy!tVw4bo7ln*cCr0qV_{^} zhqp&wxtD*$-}zpVSx@+tzsa6E>DNtIO#VM)$FqKsJzVnl*&|thpWT=L zPucyQ|CBwD^{ec`qJPMq$a*e2dHTO*r@#K2>=uk>{VaQ|YCC)UbmGT(27@0Y9=uE8lpw!!w-^nWQ=PYbuX z$Mc`w%dVyODq|xb$~yXBop+0RdIK_k)z!++O80h(|Ifv1+5H{YvIn}ZWe<`=W7o3d zscYH8{nxTbQ016LG@%)-xPXgj$FyfRQ}ROgEIC*4LiRkF5>Ar~OD|-PN@EP;&DXLM zm_YKvwd~2-7qX{{UdZl{#`>TACcBfo^2y(1cahzQp&xOKA&D7e>R;d0Ic(p$mz}7( zmYouR3F$XzpOKSI&e3=+`!ap5E~Uo`x4+$d0ePkUfM=afzL4*<0ieCOywx@+y+_+2hKE zlfGZ!tX0=2ydXSZB@dH*uC2e!^RMvyPk8>vJb!Yy-t$L}V{(y)d=#J%ML3M<6Y{&X zW^0^>oR?;*;d&b{>T4cV_Kz(of5^m|@`p?cM?T_HP5i%;d)dX(IfAGUwuJ1xyq6tA zDSc4=68YYi)6>SsM}#Ba+y9@myN~a(tQ-G-?6XZ6FqMP^1re1Lm6UWT=u}clN=m-T zrBsw$Mn=VDlp{$epTPzU7%*VKfB^#r8{62%?%7?GT*(!y$jB(Ua!pE7zLS!Sa`k(j z^XA+2z4?6~kKgZ)*W-QMoyU2e$9deokDIfSjLz`<=ksr({eS7gm5_TdiMen8-wqv=^v=EJP7{v^#z4b>Hp37jx_D z_gPE5bQs;hjY*-5TP13u@TI;L+4bb#6PwBIH{@IL5Dw!AI&logQJbF-PLS{IFFVQH zg;O}_Terv$t#jp%UiI@iW#P2(|5)wcbnOKC^xp;Azanic8J!ba;@Z7^S50JI9&A(@wf2S*GK(zj^`{Lw~ES|16 zl0pjke*apVoytBK@2x)=%G_g*<8X2*ee4qb;a#3}B>QNL!C36krj}qFbNK@AHqSVj zIStX7!TZl!V{q191O1iKeD9Ba=-dC-?EOo;f2H>)C%7gZlQ0=mFcs4<9Zj>9KWW<6 z&z8o0=lZd?bMMd=SD*C$;yEC`TC#8cqfGy(@SXXO=uD&;;+ctAn2m-p`cUJoEntr3 zJ`M^qmw6sK=F;WQerN8Zb%C`8+;(}_1!N}n2%kkpXAt+TKj0V5fwaq$x!m`U_Fleu zMY{J)^X`YW0X6#mGMq?B8@q**c+&kpnca`raafJ*WA{kM@mEjj}v z(gn1Nw=IgBIno8}QV-P#S06nK+7RggqOt!W=JLbp4jg7af-2@n2N0bx8tDK!xgEoC zq{-7g=I>50pF|hB=}k^W=9LNIGXpb`K53FNc@&h{0*ZL_) zK`I7gD28JsMq>=dVjRXJ4HJ-#N$5FB|M$8vy>oyjvv&`31_?P8(=Z*ir|bnGv$HP> zGs#(aZ~xyM=IHFnzWskY28Fro9i!#V3F?6H@+w(+#`w<|YM*gxIav{nt;xP~^2l@E zagllbtWy`M2rde<#W4>{HkpsJXSDD9zsbhgQ;oC9RB07J2FgX;%F%s{O|6S9PuU%}`E+XoKccA*b@`(fL zm0I;m-AUtrvf+^Nzi_qn_SlMN48~#{8ios_Z!w;^#d94*8uJ81|I}{h-^M@s*LFI$ z=nT9`x}RaRi+>hU4hDCn=mDyKbiA zC%e}tnFrCA!D*bqSwv&sbL4sSBH^Ux?{z*Ml9|iRJM7cfNMTMzm3e^uGqwNRD)^Op zhBD8x579n>|MLIcBJ1j1GZe!y68~KON3%EOdS~>_|La4vHfx*FK27_N>QmbPGur=N z?Y}-ypMK=9_WzLfU$|1=qRcnhmtvfbis8oP*w0*r>_>gCXXqYE^>?&EW8HHc#v?7d zzc9@>=$e+n@6_pS6I`Qj8})Ci)&Dh<^#3PEx(5A!vVN-mKiMeU1aYKe5~Bb1PA1>` zzs3~ishEbo|8uAhT8osu9A!}04l>%~JDvXwlqlCT$z8{tNlVVge&(u^@AN4u$$e*( z$79yayy;vK{-xv1KVl!sQIT&AAoiomy1lu=%tHniAQM^0K`xrw=;&u#|D+vmAzQV> zZDc!p2U)GJ>s%c5kNjz_v98$~z!Gb8imd@eyiHk3k$;EN=ZY^M?T=UwImCYct^U9M zjgjHFy0nmcQRLS;A{3J)C`0Xe^$1ymdNdFj6~ zrt6#i!eQ>+{q1)ldyab6tlX`I1X z?2)HRPB>4Bx%`m4b>8{N%)RJ4|Ia&DqC%JgZqd0USfHTpT-&Z zBS(euq0YQdb@sja{r#vy_25y~myHSs=uc~pj8eak3iXFZ$={>s|3-zT!=pko_k+i! zMVa*h&DIB?ow=jfHTLKacAudbj*)0MCXNz&(3xAJYtK1+&FII zk%kFKN7tjH!X&c0ZbX<&PQg_4gppwyIUO@F6T8x`2(!r9n1i|4V_v;vjCBdj<>uM< zWzZ9s8UwOdl~^0lZ0@|x9J+AvsnTw`{Dayf_Q(oTFI^j$Gntp{&`(N|2eb(-mA-ks zZ!hiJ$>=PG(n94Q`%;zvWZz_@Z$tL&KTVT2#gT(te#6Lm&XdNql zw4ap!kIVm4m49Il2vbYerOTUS!)WE-y^6(Af-*$^kgAOAg~~N^4Rbx3u}@hqS9UAT z3Df(|*r1BNjo%?0M%4d3LU!U9j-yXsD1SxkV@|N2L>HFUMdyDf_38gkv!B6P^t@^v zJb50y=ziN8_}{#f|C7lSq+&3JqIQG(lfhg*)4kF6|DQYQ?7QTxUU>`^D9tej%rpKa z%L}!C+Op_e-2H9Fi8a>eiDLmWk%b)OA`kg!(ndB{YX3^if2U~Qhil)_cm7YZ@&7UH z9}b`v|MUL=qcZ^u#8ZeO6r&-<_&32Ap1EbHxKYMjiH-%@)_L04WNj^KxOMGF3iV|7 z7xYI`rA3Y+7LyrCfPH?L|eL*-$cA;wIFZy6uH}8XC?ew9}S9~pT&5S=JZl85Q*p`+YiY8wcw$8aMY?(gX9G&wlW?T_A zO?I};qalnFfA<4-h4JLBnpoI*T)kA^FYL(meG_gE)hT!BTTToI^8BZ){zIWI;Tn4j zCx-T$2ZgdD9}T64J{rpHQ`^^d8~w#C>Zp%|s`Gb+{Rwx5;^B0cQM_|6^nbNzTsR=? zcD~}?oPl9a;Q-&=y8Qe62QB5b#Kp)S|A)k!p+mxw373YYotK8}9jPJR^~=Xx99E=W z5>^&n67o_5sm&OHITwBBmrYY|&(N34*Z&`z8uGa3qX322Gt)kT z8L6R&xqOPb;Y{Og<}y?jI{Pi(8E|C9wABCF{~u>9wJ`N$!x8%enEU$w{nf$jEr;m` z4$%*w9UVv$rX;!s&1l0R9L5opPBuTEW&eTe%Ta;KG4>z0M-^E;%YEkBe?Zqzi@G`1 z{-a@}{RczsKM<}tjeY_x$?ng+?X0>L9oRMN!qDkH+0P|}W8`t{$&pXEpI|P}HHKzC z$=rpiJo*Dvq5`M5oyHlQ#W|cuFA}`y}C2{zbEy7 z^$i;NN8`uw;z`2T>pN0K%>Fu@0(cYg*=2G#LVc!J(e^fA6VtGw>X*#y?>?mFVX*>>-{slf2Q{*JJ_qeZ_PRHkJ`iD-@Dg)?}mEsZ#>ZT zls>p|b=1%9zSB5c{F!JUk`%JY?21^(C6_)$KR+lr2BniN33my^gHUpgWow>7`Ob6WKgH(o3b% z`=?s_mm1p6rH1JLuQ~2>7)Q{FJ=P*eZNM?+Xz$v-vG&Nr+O3n^ zx^N1o@z47EGwf$^4t?iu(ifg*?nT0Hl?{}Zy!(H@HskLj+SC);R_tf4I;0<=ACw|o zDh6XHhGQf~I~4WDWrfH@rv?+#$!_N{%qDkl^J_SmLO+kzk?MwI^+GEBANgPYe_heK##o&>10->_nsHRB<5(1 z%M@~Imw6tlhU-@izAID;Q-KU_wZ(UZ1!N`~D*eNp%t3vPz5Y||OGhqy9@?2Z z(26#cPQNRZ&A!VTg}XvIS&=F3IquPH|2y(sQ-DGgp%^77LnWG~+y6e({&%uvu6Q%V zo9WC3vV*<)i2d(4aKip~)G^niL72vQclYI;U0rv}%*qhEHRHF9eouQd~-}(acHEqld_QltqyE7bOZn1a%V2!=x%tz2M!?R8EjKZ{{ zbei`^-}}!JFDmi=|G#!5gihBT!*QIzNp#^9nzV(@h}xJIyx;$4?m%_#KlT5$mvznF z0}afL`ku9?(!*)-oWWU~LqnH6^7ihZXO8~KdvHQ5^fD*B;`uY_{?N8S|8cN$CAjS> zP7f($D)yA5hr#4flvK()!j?~UrY8GH>=#$`51NYU%5i#3p;UGe)sOApOAOJPpV7iC zr88b~XOecA&Xc|CFZRBY|JU<>qVor5x_%aB zV-Dsb+WR+;{QrIa&pmgCZheXF{`T>IZ17IAy;{GVdx_s{=%KFRqMNybIS|7Z0(Ci?~`EiwMomi3)~)2vNluEc(A zP!$TqQGzm5q6YP7MjM*ung2)M|NTxH{~tE~r&DhyJ9>@(#Z{w?JAnUj{;l?Rn)Vmb z`8S8ea~Ma^iH5P-Uv1nm=9YQd-?{o9%qP%cEZ(kdZkwU~ou&W5Ejs_Ei#&xr!k#A2 zpk($teV20WJ9n;Sk<2^@SAk#`a?MIaV&hzg@!YTFeOZIt@OXBo*SK8mr zoQlC1is2ZE(dd3MDU2bbGx5ff<1ij+n1FOl!emTAPhC=&N=`$mG5`DPFUsg-l+8-) zXRboDet)`fGcXggFdK6)7fo}Of8&IM#*i&&Wo}DV_m5ZpQGG)B#{uDLXDk2Os(P{k zjoP5BwnQJ&+`GA)NCzIBO+Qb(?aE(+Hh2MZCR((;2NBKfwbs!6m#GKHEPi|D=XFh?B0(}#-mFTOWbgcNX z&wEE_a4jRF(~w%|F0hFJOBT*v^g)0 z(9Ya}YH3)51BlK_tQ-8!+~#}d|DP7$8JxvAoJTJbe&^kP?hGh08vmz|-MgKQFlcB< zWgd*7h~_?rlOr)2WANVpzs53`*LydNV;+z9&MmGi@veA({reQ-j2XribBrsnztA`X zr#wrV>n0!_lQ0=mFcnS4AkDqT8DxubO5gf_TJPj1xDqznmDR^XnMSeX#lOG43_i9szE<+-u1) z<9$jjO+0J z{}q@ku^-W1hxkcp^KSlY6aEmg?^M2>>v)8@X{7X*J_pB0|I^C<`KVvdpSc6oQ>6bi z>5p2}rFl2;Hjs_3+m)LdI>ocZ+<5f=k7LYx{F9`F`*G&-0{WjkX9O~zMAdovANxcq zbLfB2#jUnT*}q@8x+f_#Oiv7_xSdAd`62U+3#xRu-?~So;7|z6;g#8jG-8go(G*bMUKW8bZ3V!mh9_)kbUd-)CG(5wa2sX^6V4H zzV-L^_oXxMnWjFhxBij2T>ocZ2K@>16jY_DTM(^1i26WNxlO}#%)m^{!ff>D^R#~> z$^3vbHQBp|7$1`JkbworL>6)ot?$ew>-G7f|2yX~=c5488uFGL`J+hwNE2_m{6Ur_ zn}hamlydFjKJ8&;q=QEE@1JO|c$#aYwlUgs6pif**^5w&5;RZh7os_UzWsIdoqhZ3 z=sw@uUl;xRFZxGt^pD&F3EFjijJg_g-<9UT$zA#O{;xDYah>w3|5ql?J;Idmt7I-e z;<oxn+S;S^5e40=kF!&$QH*5q)G?DkK`^JFg)UNsKHu95z~H}0K2M3g=$-1Z#z z{Q4ZJ%;jf2|7l}9=IraV>4&82NzdQq`MLjZ_aFEc`^8&Dw!cpI=i0vgk5|f@%%d>| zW6_i;4Sd6cslEZ(DvjGxTsOh_1E|&?s>zV{!qt*>Gkk-|(%$!NBun#^k3xC0SouK3 zF?sW(JbL_{`KfX4F&=5?xY2%CG98mJ8BNN2^GxN_c)n$}@@Y)phIVwIZ+-#MvM$i%2acn{ukMzY_&4KKfk+x{%M;27v4Yr z@vQzA`u@Lu${PM(s!OzcS@f52d`UVJD}~jXTy8@ftvZXVV8f1=a7eR7)P+joOnse$k54LKGgg- zjxircRs9uVKPr>Wf8zwVljy=J{PXF_EPizWkdgI zeZhF^3(W2Bx2B+K^eFm~{}F!S|8)Ig_NHU>zbEK_>9||anos|Wc66Xx95v!RfLhe0 zSX+Si{;$^;zyI9yJTx*$im)`gN2`1O)H9AXKQPAs&hnjoHFRXyk!rpwennUzOm{2u zbMo|Yd-b)?(L1ev_u2;M21IB4SG;UZoU}Sv!tQ<7@$bm9e;;k!TCr=WIp6p0H#mgV zbH;={>C%6~6=9?BZjC`P9%*m~d&>)OB&#K5tam zwqSTDIzK#YO}IR)J2ft>9eh*BpF1|J!N&80!-lhiLqX1s_O6Yy7Jv?XJssNF4~4~f z*N5za>qAc0jbX{i4~L}-J{&6CcZ=s(KK-V!V(bU}_so8{;g^I}XNQE9CJYF*%$B3Z8_AWY z28F!c3E}7)vGAO!t{)6nz@6yYxfmlJbU*;&Se<$?Qql?Y=7j}gL5#9ivJBq050X{x(R#DC{yFtP)Ru5NY9B|_;Q`^`efozf zo}1f$ToB^F!5?uI`(Md1+kvBB=x7wnN98hu;wUcd}*1kl0oI)!m-qZc>*t{zY*lv=3)waBKcYJ8SXR4>pa_+$d=TrV!vQMI^?6VQS8y0uy*Av`iG_? zJLB;=#&;R!U*{!+#V1F{vgh;*Ip+SCU@5m{#rjm031PYU=oNYW!bXNNk%h+p!~K zNUXT|g0ORny*N4JV!KDH*T>OAa^G98eJ&mxE1NbX7X6>7akTenO^bEjnHKZ-@qLEw z6{x}w`27dE@G6e*zX(mm%7Sm%2u(-)=lpa{<0q4SQ*H;7)ltXghttlW>2>zZ5pC`Hvo1mgd~ZPHRl~Rgjg3 zlfzE#yONW`4)-XYU|qq;D#U;79d-^qFVn^@0lX*hy=ma$bv{2ac67uwvCfOHi5<I-k5f zW_mmRXX*2}JiZE@S6=Vh8)EyoeMEX4-R*gI2!H#BV*ioEe)~lEM|>Agh;=@CSL~?7 zY)>~Ac;?2~FZj2pQ~ODy=Jcy#KM{vtL&rVP7%%%GhI`KR4~B*jLD} zVIlW#lk-8<6~92bWVvsA7J~9`AJL+a6b5k5zF7Gm;=Hj}`Wp|&!m=Cvn`L)$SRS4V z7xG`Z=7R76@-lo7P3n^T=bjF0@BeOC_rQX16}O|-ux)rFB{ZqX)_gA|T+eMS`_aFs z!|S-yIes__H$u77FPIpa>AjouKkj1w7^dJ+l&# ztkj9IMF|sQi<2kDvWHKM<&2ydTQYcJZ0XR6v1MZ?#&X9^j4dBMF}7mN#MsIS@?dr> ztQtQtwmOZU`14O+6mp{b9rX^o3N7uJNAByB9{!;vJ9Pscc7KZq(k3 ztC+bYvNP|Np-ZMa*KVfs8@ZL1O^lUQPK-tUyh`?_H1qjU*awZ@>pjbjiQ(uF`aN#7 z)_2rd-*Ge;U!++7f$0C?O)AT#4ELp@XdQb|I3aAi!XL%4^wfpn=&Ryamp3&>ZN2(+ zw{jW3D%SaoGPhniPZ}HRd|jFOvT$@OO^37z$1e=au(8HJE`5*s=-Iqmb0pc@$V1pZ z=Z@G(vMX|r(&0JpEG-VRABo&fS?^08d)M4_+Oy*5(OY7tBDXQjuig@C8XdxE_A`;2 zG>yJ(l-7+?Q;p?PL%qf`((Oe5i#{t%&2VdD#?rx^Q@*X!X`5lrKW&h?;M8ys&1gcr zZ9vG<7g#hjDJ(uRK>v&W_M~-D#|MO^o$L?F|N0ioowvOrwSQPSxPQoVUiYf{Yh$ZV z4G3%I^w+oS@1MH;!@9ZZt80=&!N~q$Lyl`lBU}JF7mLZazMzRl5G7~ za#%MjIjlzk^M={UVdLQBuqiEi|KzZl+>)9cwhpxxPTRMQ+@9k7%ThvduXggBb~3^H zYa2_1+e7YkT`5_{zK<*)u3gPd@&3{I%>D)3n-Z!MQbJ9p^OZBS)2^?ZlVa_pc3SvG z*Ef;PU1MSgxwWA6*qB&bdP-k&0O+8zi>%{ej!$< zbFL(>!Ls|4f?;R;M(9$;Z-pUO{C7Ao$Xrh77hYq20}VP?js6{Uj`^=R$lS7(-X)vf zMW^r|+_w8y`mUo%AxS(bSR9hVg=CKTl#(lx!llg19!LsQeR11E)%CvlIQ=i3$jZ^? zaK`ovzvVVkyK*n8>2p3ueg~`Yg6m#KvTLt^Zbke)JdCfQ)w?Y^c|&aRsT*SPUghBY z4Y9uO?%QW~4KNOp@1^$&EJH3<<)nm6*RJeR4)RjM+U6T#>n2?rTT^&LEdTJpu)(<> z8#Atrtv{UV+|g@do7uOVxIVUtEL_kpta0t1v9)(#*x~(lqZFoj;?=MuEZ&4xd=Fdw zr@nUR!caOmB|OLeJdWb0SoF(h!_UcI;w8L{SMVxy_2X~gP5cFK!*C&4##w&Siw%XEL09J3s6xm>>4G&9^>yekeQhOxSnmnNU7D)0&@5 z^JbZ0|Io}(m6{o^7Y-OOi0{nTee19RhoXO-b+L-X`!)j7|G7P6IJ zTiUbw9M6W1asF$rEwAt#zap#1y15JO{jxqwTb6RQbSQo#);9jBa1rwhH3P8`7Z@mu^3f4~(thIoAX!jLt4VOZq8i!&A)|1JzU zMGM1{+=XFj*21tXXJN?ATNsv;E6ywoE6*(qdA$q6s?!U@YOE>rk9MprTj)RF3&Z;6 zg`of&m^We*zrtzHhRwsD4O_4^_1RE#?%A-d_t~(W+`(Qv^x3d;q_RNnW-poWY}hmQ z*{~O-t|^=SY}hyVS?eR7wLaomWB+Hve)cM|+I2N!o(%`aJsWDtx^(GJHq7`=Xgr<~ zntB(6=F=JB;Q5Tua$-SfWp3l%ek>z&oXAip=7+5G`C-wN`C;+o`5}AK{E#zsepqt; z$*?qYzVVN_I`-u==KBx&cf(5I@`}D2R@KmVjQnm`lm9QV{IT=H+N}9uUFZC;{>1!H zaC*LSFh6WOHY2vFcz!6H{khoYv-882sh^8&JvKiS&H8TGmh~^O?Z@XE|73>ZywAmU zrac_nl`=o<9z8#ljFbOzKNs6u__ z_&&ZkPX7=G@bCCBeum#7`NLit=_s5tAj~A^U@qn%7kOyLqPM>r7Qg!4aG1FhOTPJZ zSc>D!r|>4;#&tKT7jY|Y$33_oYwvj`JVHK+eEqoB$#AoBh-@rF`OlvTrLR5{%3gjZ zY-QhpQdDDS^)sP~d=7ub(R-%EI>VIM(YHTo-0`W{0A=C|Y{6bM;s^K%Uc`aYXF@H0 z&HNj51KSD*g&S@Q@mp{Q;+6K_QwS_FFj`Dzv*#SQ z_U5F%F}duFv-i#=hULA9VFkI8J&*e;{;SC~!spY)uf;m7_wSa1=@aERhGZ#tjW-mE@q3?g8?{D4f8S|s(%#Ze($0aM+ zE4lCIUqx06UsG>>)AhAv9ee$B^P4lxZ<0-O&5vf7AI&sBO184Mac}3}LAK6G3T+ul z)-)xB4zg)-QfQu<6b_Ot3zDoCO|o7zDKwCc!c>n;3N@)o;Xrbd^`uFmVmkfh%%rfN ztP-Y_EMwnCmWyY1zWyBcU@vx{7(1~GTc_y{VjH$&6AG~zTd;nx{vQAnjth`NC z-pFi8B3z15xEeR$CftHM5LaJh)zkl} zKNgeO>^aA+yFY2&J-O_RvEDiQzupA-KSBOCugZND|JCFg;q&Vg!dlm_BiFN=G7tBP z`(AyPZ)uC>}{ZV0QIt`DnQ$ND$Llc9Tue{VI3+xW9h-0e>~ z7ca#cF=NC<->{~ZTcKy_*7@ssbzbQCCi!#sUm3YsKf}L!gtNXPd8IQkBKd%|0A$ZA z#@3O{P7Eg^`AchRBdKHEMRq@LKg&zj?d?bk-LEEwd!zhPY7IaXW_MC}F?bGfosPol z-}i{W=g#O%Jm&kX3*deh=Wrf9k0gcONPa);;rNYhA2C{FPBhET@;TLT()?Lp{zS;ehYre|dtW9JtG$yUej)j{%XB21O^LeKG z*bDQtjceY0-m}`gYT?QD2kC6NM|(I5bLc9J@tTf-nG}Tk8tlHt~tW(0oP!&xt47w=(Vw^m4Eb1 z=a^rjS6;?n*lp%qwjblx>k9P&#qq;*`xBhDQ&d(YP?f8?^geIqB3sJ=KY>Myj%XISJjAj!d2eG{XX~i&e8X5-zv|Im*3+0 zR;8XpT#F1cyLZC?vilYJ`j~hO9t%$iAElu((epj;rY?!z@u?_12br_J7}p7JoYrl? z+C4&;aqonQo^g@1+?LMIa~0BWEq7ge^!{&S$2)Ou>-8RI-wCrJ`X%oa7sM&+Ugj@@Qfx^v;Xajbr%Vj5yY# zJG$SMcu=`ze=c%+ITi+8r<{v_Tc&%yz3wElkc|&Z^&17j_At|9i5{*QpZudT6y=r0+@f^v%RgzJSv*m@ zwK@v>O=+SI-6s8`ZxN;cKZa>*hr3_($Z+jt+I;34$eS>b`4;jv+<~5^Sh$#5;`%IkDK5t- zJdU1=pGvGHyIz`_*hqFivLLaA?7$E3L%jcg&lB2zWXnrA(HwO`Qdl}NIV>AP|CvVr zPnWiWUT-BmTi)#Cu!?!Lytbz7f7AX?H|ISwQU5nl|JS$4NYww82j(S)X#Zyswqg6! z#IU2@+JZLcO_IB;XWMwKhxa3C@Dsd%7x8QS25-n)-Q@4_ z8hWs3Yb^YUd<%cWKd|-OM?+EXM?<1-kT0K9ar+bw&ORPK6PdYxCi*^j7;oGk3!f*y zg#0zJ@L1$7+~efe@pP&X?fNh;{Va9~bN5649=nvh9HVeG%ITB-b)5bt`9suVmvLPS*|YoQ z*e zx_&*^Y>egi?-wp)R`lYRK{XKn5N^arpc;zbh5PVHJcxh6t%>H~(G`Eny{?FL5BXZG z>*7mdT{|v{srs#3yfD^tg)?{Rwf6KTaES-BnST@~%aP&`>R~pY0(?1{&tDuUEf7o+; zpIqzTp6mSQbG`Pvz@+eoUi+`fP3(oo`-RQphS0m9a{`G+nDYtCK})*ZhftUYNzM7lmcHuBqmT>Y74Sb^op!%D2iDr6y2P%kDI zAqUx5iX~_+QAW{%gLrRUdIxhmYEX?@968WVjy{C8o`M&WONTAz7R zc$4|9$bNKrV$YRJ6XW&6Le`OC=KltU#p4E=_qfo!hxPPW+B+;PJ86x3;bmd@>C3~4 zaU;XZ!MyMuO8=MY%v*K# z26gsEa+5mz{q;{77tsG%|CDCjmTufe?&x&R`El!?)UCV7-DmZGum^ildQ2U6{@s0C z#zz&#M3vazssD3W-Prcd8KVdEYoaqo>-2M?Ge#TOqccXEGWCDRgOk*;nbt?owqDRP zL}#dWOcVD|c})H&mp}KRQeLeZLoJbAT$5O3-PYnmidj56X8`&GkyM+5RW}yk+ z$42bMFOYRARRxf${cuJJa*&Os_VF(nH%!?|4V%Qdh1`syDXC#AxqV)$aYJgzx0hoL z|8?ZrTyv7-`odKIL`V%A*ncN~e8_#);b(XS7s{iz;C}p>J&+&3DBOmJ@I^d>2k{L& zi6vNz%_zew*oh`o;s^K@UPZra)yEi(>DYubky$=}43FdM__MILa=#Cs##qe29AslY zw&HpG1Qj@eZXCoF%F@L+%X|a*7cxdpz{l|*K8G*kDdb=!cB33kIEG*24ElMdAsB_R zn21T3fgh+7ewQSUWOK5(gOKIo+1P?^;SAs7LxsH#ci_{Qg(tBTn-Rou5w5^E z+=;LBr$2>mN&FxE)LALwyZw^zQF0vjn{g|y;dfda^rAVV-)VE6C;yvv>U!aNUb!Ts zkiV6+ekSY^j2GUtfBbFnrm}CuIAI3icf$P<|H*ADh6Og%8sAEoQ{1n| zc6=VgT>n?@uQFeXop=qG;V4SbgDdb8l;N)ku3I7AJmwi>BJSaKA0EI2T#mns`z>K^ zX1)l2U_Z{Dk4If|3%3v8zjOa3xAl08do}qLviqJ7(EnUQ=Yn6Ne<~daw+eju1IGHP z#<{qa`4I@;^K(3pD=!JJi^K1E;)BIE1ixhdIj&*u$M0*Tr4jK3?Y3~g^%Fh{S5m(cm(lFJS*sq zdm=Nxp8IeOhT-*#W$HUamc8nWdhZO2$!zwVq3L1C$n>z3T;~7jxoPRfzv*EGxspAP z`zrpc$u)ER`#mE)tj$ah>&W%&1>86A-$-r}zA!C4Y^=M#jMAHQ4Bj6x& z3tIWLp`Ez{@$=pvi>9Q9#mJtK9&)gRc`24H@cprzc?DMT%fl+>)mSsZ_eWp*KlK9t z*)c|%sr@H6vTw@J{%dzPlUvxgaxdb)joi+@qh9;pru`>(b!z{Q+dFmgf)MSY+RI*g z&i?3L?LS%0UctSR|9-NHy}I81CfE1vFQ00E({#Ggnb!N0jqFVs##ouwK$9)(t=!xA zx04<06~0|%iS(?Lo@6O|*=*@OS9+7Xee06RzQbgX>HMuK?NPZ2_MH}_%8DC zqV|niJ)R=$RY=4A_&oj%Sy+$1!ZNt{FkFj|VlqCBhw&)ppa9z*#tl;mhi zp(8}q0C5a7F68zZ{B97P6FEw~ztq_4@7ljA@^NFg9~t-kyRq2$aoX9N)57)SIlN+R z#&3+-wo|S=k!GLcHDkk8V}v4X!*=X7hu)1##dU)*;bLv-MdED1XI=XQzlS4ti7Q`6U7x5 zb~XQB^Zy6`Ev~7>i{kl*u(jg<3GOg%tu!!yn!H8apA}{W|IZ6oHk59_``+wc)nR0) zN%DWvFf7y#8XoFiy)4v+S3<+EQ=xIh??Y2~IW%9>6%Jnc+t3nT53RSJ32nFjclz<3 z&=Fn@uZh20{2ji_MV@E(gZl5z0oe11`TNj6ls;;G<(h$E-&co*@^4%kD$He8J~=q- zf93|Yi~Cb(jMBmTH!}Z9TKocgU3(k**S%W@ z|L=%rg!Flwd{Np@6W22Dc(>>3z~@~5Dc5}5wcp_0C`|m6@y{9KpK&*Z#a+ff$iWgU z?YhbM_ok3b@49?$wErn)|C4_rU={P~F|n{_p1D4(#X77XYz_b$uo0Wa)16~8wqWaE zb4%EU?bvbF`e*D!w3mPPV0#y_2YbDKbFA7GEh>aC~W8LPLSx7U)rDc4#a@*sOjnl&TX zi#;gD4(!5CRGy;`LlyQTT1!-lXf09nZ>+z`Yn$YmQS#A#`D3#@I8Yw>wmkJI`RIrI z|AZTbxq;gQWQ)8sP##)^pW!k716`lU|6$jBU0(c{xc-c5_bPHV`qtN_^vG^|Ro~Qo69_&TwG-Gt^%NP{Og{_!s{-fTvZ1YXY>P~s)xV&;w zo+0bl>(9wcz48><#NN#PApaJ!mA$Q={>SwlZ5M^lNRzMQY3aHJ-jmcMI3!JAQ$C1wQWjAMyV?|KEu-N0^_u$5q^)Kw=cP zIDdr4#o5obm*DrV`I_*b=l=)tMe=E3HhRvV@Y`5KSB?U#$2zP<{+Wr^w$hO!n!_$4 zw`TPVTkH$i%)bzuuynMwMZ>Kz8f>jm{WW26vbk<#MT11F?xL&bpT~bCxgx>ZBr=zM z8QIu7HZ&CYW;wncYMBopI(lDj|1^ zdl$Kry_noF%Nc^6VLvK8Qw7SI_o1x$8fTwalP>MMqwg#YZ<97JNV^w&m$z`ObgJU^ z?I0-9e+-Y~N4SIAi?~Z%|BCteh-bPB&*CHQ^=bSI z)?yp7Jl`hI^F2I|{8xsBm&ui$DW0bMr)&Sn#mZK;GL|z_`6rjMFFSHW$W;!PAG;x} z$WZ?0D*yaiUHb>D8Iu?umv(DMCx#oO!6#5KLHQr6{A1H_|Q_i4);<;((YVbQ42j^_O6%zAAld9cu4@#*%AqY(|L zK{aY|0F@)1iI2xU8lRA#KZD1x0>8#Z z^5l0gO1``eU&j-uMI$bfA78^w^5-Y8M&8=<2l?E!9qP*ekVltC`4r#6HhfrK{UolC zKR=95Vh_%u7sKV*u^1+H*(_>hNtG!!K zB;~IsumV+x{^gQ|=uFwg$ZnQqSi-y%%lPFU9}$*IzZK+4{f<1W!fLF^H%B;bgtJsz{j1mhwaNcv1$*E5V1@EOs+~Jsb4LC@uI|)ltwVJ7RRbE)bWB}Y zBL8PPL!(T&_08KxUmDtLqVsbS!VA*iMZAT6(&%zr8s!800Dr;_zT@X{w>v0LXKPX+%gL62IKjL>7eyKJJe`7A{ydeCA+)ktE7o9XGPDcL= zn903en|u}7#NLd9DcWY#opfIB80Q6!>lZ%GznZ<~u=9M)`S9B8b_bRUV=Ro6??H`G<{RT$HLAD_d5HWqN3RI;9hYrE-^RXc0u?w^O7=i^jKDC z{lO`7`<^?R<6qhP&K&z>=Om9!$gMuuwk!y)tmy#dIpT z4p(CwK7!$RSh(!17lzk`JBws)SKy0y3{M~j>rsf^a3)>+7)~sEFm{f-cCfJ_ZpFR$ zZ+IAA!XlKxezy2={0hIp@DI2Kx8p(l2TUcz{Zc%B7pCD~@gx>t2dMDlZ7_8cKaG2b z(5GQ4W?(kHhHv3nY(f!EeBr@Z3Aqnds6#W_aSYpy%eQqJ=O3CG+dAAlVzPO}k;4Ew5yn4+kKmK|48DL&?7%_%JAQ;0@Fp(w zT^t(|pNPBBVP8i30(&wt?8TU8kH#E(HKIKmv+Uhqj`nagOrs~5VQ+^$8?`vV9Gz)b z&97>Vy&$8F^T*m7GR_{6RC`5I-Z@jRlwP7wFEHMol5~243G@QO?sQF`o`8E?m@L;U zA{U>#C}h)NzEX6Y9VmVe|CGxOJ_|=|$4VmxywdA^ybSY#3`v!8Odu}2N**BA0 z_-`eP*te0}d+933V)mWnF8;g8681ghUe`zOP{zKGEazWARHu;H`%-dQraFLJ&c1?N z$v=-=#lD(c4L_Si(R*YlOm z@Q)7%&Gh}p+%s3?J9|*ww!bJfR0&gEW~_(<=ghIUssH%bvcpCu;Cxynr{A-DG8T5wq@gd<6=y9c8G+-_#>i zobds88gVSgYj^`A#W4YQ;eI@B0DmcnOKz`r`@VoWWHZb(6fg^ODT_B$@X~3M;23g}fTNg5z`sv+V=Sq%+VqZ&+a6-L!#W{UJI9 z=FM~Mf7VWKvWBAYEL{To_UQwy0~r{KW?NHHV{HX?Xy1#At@}A?ZAGUw7A4kNq#3K* zQ(ewpdTyX~umi(>`>3l>fl3^hs~<3RpmQzwmT5t4vC)#+u>DAs=f; z+y9UCD8Pm@+W#5ae{Fc-$#>8H$#{4Di*|h5VeS9C0q^ujN}PW%PWxZu-!$6t68GHW z+PxzOgi?KhGJS!4Wcdu|A50k#DlO>hJ7*_F``@MgU!eUTZm$69xYeU!g845rF*hUn zhj0s8nfuNMJg@(c@5=+v%R_zt4WIJ=@bxb6aaH%d@6WsVG*0YO3?fbVy0IXU;-&*$Ec^j-h8*IuvxT6^uiUV9JiB(zF#i0q)>u+_Xr zxk@8zXjjPafmIs#knareU4(w}5c+_89A+Nw2<>tedk-IDPW2FTtI$!#cMFh#eaz7u zpj{*VchJu9@8h@?>A`=qoBKc&_XT8=i~B^G)Bv>ySE>%G_w(KxRPLc%p&WY!lzu?J zoAbrU5-3{1zHFrZo>rQ18+{$~-c}AWcY=OB(#5d@>EwJSGK=GP@4cX7=-&t)n1}Ol zE$t-(9)~A^<%-NaXnV~`6kKnyYn^Ea>s{s9ia zBXAtH5SJU71ziw=+i-gT{sml61VMNars3b=5AZ&GjkfKC0$9s^;&0)v;CCGV9r9*y zKt9yLL3kSc&;u;dWbWsl@^$zCyO6&@F2FU|H^DIc0NqBQAF9?}V9nzKb3GU6?_E&Y z`U@(Bid7f*?%V~|`Y-U!iwmsvUr;mDK`rhL$a?VOuWii*=J75lu;PL|oNGOQfoI_b zd2tKwy1<^V3+ls9FLZ9XpbqHXctKsr@R17|f{|x0Xc!qfdVw{O3+x5BpaDp3y`Thn z8zr6;L^CcZviAae0WK&;yqU8fDGMCuKavx&-~ULi4?j}Qdmkwm@{WDP`2Qmnoc)OL z|3}QVd?fb{#(%rG?vV48!C5H3gXiDFJpYhY`*{8#Yo7oAc>Zy&89yz^)+FtK_ZYm# z_(qzW`ESPj{wBu%$aao9qKyBcE1&T{^wcr_hhFS`(EkDRUoe1u5QbKK#Pk0n4Ks%@ z0ujdfQHVht5}nI?j8nv6exv)7T$6tRRq!YjL)j&Nu3cPvuR$eznQLn$*T8OQp#DoB z38NgBac%uM_E+%p681N68-$OrkHHhf*9$+y?H@V*E@gE)alXSbpApRb5pH+kb}N+P zw#d0Z;D3m7{FGScyBz-xzK`3-kpb3}FW<~_4q6$$P4dhQBSRdAVRR$&udu-M^S>}x z;D9@E55ni*R@e+z!{34Z{WtDYj-%Wc_H&=`|3R+0KPcxg_l?I^=z9EYhnwL|;(QxU zkS6anW_tgOemT5_+ewa3L6W>2MXo_FUQx-+1#V}VmvP*H{2Qbn`B`Ki@+M>y`8nh% zp7-CR9Gbxc8R+~E5P(MhgW}?ow|7n zYTagh8&WRzH#vU}mf#&Yj$WlF7#~A9R6u2Cx%H>j*3Zy2 zJRk2EpnL}SJ`jpWqx5@Nv%uX6**r4~N&9KY&0ze$mht}w{VI6BUxn}W^ZswY-0!fa zF?aF%2lQ#mpq%s9lfTOR3aje;@|ucSTdAdwRCj7n^=AjwfW5Ko;`;~4mc5Msqm2K- zyK!)N|4Z5O9{s?c0c|I~jx^7I`Vw8p?hnZyWt0o0z~^T&`>S~VH=!T0fMfG*<^RF; zKg@+V66X0&{n+2b#7`VQ367ITD4*vkAEcwqXE`C0O^aDgJ}W2cvvS=&D;r#4pH)!h zvkEz%7xG#8bv~<@{(gy{_nZe_o!_b@6GSvyGLuxx2gFK63_Qe z=v~S81##JY-ql8Nk zR}!-S!0(N2;(1M(S;!CILny0cEuIf*w)~WL>K^5rRiFQuW?yc#UjDg@Z*RD)|B3(i zmw1lWzNY^|HvC_&=>jrz<5^vDp2tdMt2NO0d%ktZcWlgYcB_?pEo-H`(DUCpE?Uhv zYRJ;N;ApjUId0~+naX)5s)qOcqP#Ph%}YY>eu44TPc5zI{9l2;=rwIZx_KGn-K$vt zKSY=a>-Sxnd{?gsoEfX-;Jaa2J6Ch#`-F10tX9s()pBiJ%@|>|8V;?d zkH4C}+G_f`tJU&6-}5`l_xxC|ZG?(Ft5tr-YQB-Rn!OUtCmbM;`(giPu6PnfUj(|yj6Tdzizc!vsSC!#oiL6-@RIG5XU`+pJ)@m z#Q>wTdDgr9K12BbV6Ovy2Z?JSxLP64YW45xy?Anvb%-e8s3*SyS~@h2(W5<Y*B{jHr8aJ$73luCsN8TOUc!gQjR~R zTz5)&!7?j9ol?OySP6lVNiZdJV3VpX0>sS5wq zb1C-!m0Pusm0NYsms|DG6lK8o-#%piql$0F{Uz)F?=b&y1@j+#AI^K^WAgEF&bBQd zyEy;xn%bFX`TzZW1kU&K&x&%UM+h@5{-QHXu;n&Oba zo`h8NHH|{%-q)1{j)z{C6SA?pj=ZXz=U-JWGH>6j%0KX`3XZ(aIQVt;KfKQVhu7Kv z@H*q*SK0sgD*GQ_Rr&O*s({MbS5*bo*lVEngIC!z^D6JYzsflHRW;7NuBOAx|LkD? zXAb>O%rgIRhWU>(@|``Swuv+HA3MYQpJ&wm;Tih6XV{N;hTniY!yM=t_24In-(KiD zdWP@!o>2$})}3MecZTub8LsU&cz5_ft zH)`9!81{-c*~|JS?fp&K`k{RtEQxo8VTs z3l2ad;TsOW$^GC>eTU;-+3vD=%f-1{bDw>MRO=uQ2UbNSE%y=9E$>lo8# zG1kwQx|$g4GZ)Z-TRG`^ZH@4+*`Z6edfON&24<}R`3y@7SuAC zAP;JQ>84D63O3URpXT{+4EbxQ`oelu-p;qW@A?$qdH5@pUCTE=?`JIVb=GIu-`4c) z%hdSj$aJ3cK8B=@yF!O))Rww!EdN`zRf3?-Krojqk#MFzw zzLhcFt*N#`FH~Ct$iek74egKc`!X>NPYi41L`)IViJll%3>iNbQ(`uzB<`u{81tXx zVO5RQOuw$l6K8FskHRpE}S99b5PtMvW2 zO18#TT-9K?4~_6{+;J6tFrot7@{f(McRsG%RdMB{kIRM3c1P)B#^u-&S5^iuCGLuA z^qLW+K8&*`VMGaJ{DZh+R~%Oq8F@ac5#%uGgy-TKdMC~pFRlUd6>5sBzbdXi+$HkvHb7USZzq(-nhJX#N~NDuGT&ER%TtBvbN+| zj?OkY*L8@`on*R_ZOVD4Pq|m*S$Rz^E8o+m0{j(LwW(-h9{s*Ns~B1GPMbTBj{-HKC9B5NxMw^;;wXycorj{*jYCRZ` zC*8*W_gCaYw#~K4zoJ6{Wc!W|bzJj`I$a&=I+|^DztE>cdJe_AH_PMO&vu(@+wrTW^Hf65O zx3X5`TMnf2te@v!yIkx2x}9>!UF%m~hM&GsrwWivD`)blm`peB#jE@(S?Oo(%eebh zzRu74Z+_PQ+I0_ps%KBAW{01-4nO_gPSqnDCOX;wazgiVzPZz7wcyse*Ux@(zr4ib z%kZlW>0ei11s?LNy(!=7*y%_Ae5;G}x}%-U|GBIn`Rm=+%l@}~tN;0IE40VY?;rb_ z^YClvg?uY~BHtQDj*zd&!FJaF{Nk3B8K3hj0m&CSl|p_Ux6Eq-%Gwx^V`o6l9RX$U z59l9o&)FN$%lG73-^TtZ6m-7I{C_}2djfKMx>SrTsp?WG&$aJzzPv8Ssvw@qx`3)q z1yp?^pqjY={r&*oRmr#NI|Do$3arNS0W}>cu$pHBYB|)U)}uL=XKO&-=Y#UO3aqxR zxt4!zfW0ySwQmWiBUE5@dJ3#AWcTv{^-%A@WI(sl&N;)o(4d>UI@cwaDBL+m&;SYi7D#*{*i_k?nH0 za;>bR-3;hYs^myFdj;C%<{B+}zg>mT7Fq?Qm(Mkt_fES;VE90;74Gb2?S?)KX${~O zTGOq5V51a^P@a#xgNSZ+tuZ7r@xJUxMh~LDl5Ola!j}K{!5XSo!=_gjv~f^t=zkMnE%eS z^4+b>e|4*HOOaLN&a&L0Ru$u~#MP?O{Y6&Uo+7IpS<%$0$`fAt_N~l+^)PSSs@i8; zRky!K^~i>Ot-Sx+swUi<_qHSg6U#T70aiUe7 z89nMkcCYMVF7Q_hR`sa&P^Q&~TmPz7=093Bu&0&%&#f9lh94@ihS#;y|0%K}M_U!0 zD6(Rt7yqD#`R`0CN&Zrl@96tQR%SBO%F6P{5%tI!_E2ta%eB^R{Y>0 zFZ=(zDx%EY2fQk#?_7c`#c$b(UX>#&(!KP5UsKhQUfwNrST(rSc6wEp_Nso)tA-O^ z`oCWG|GBN^sF$%~vDLcHi~hxym-Kvfy=pt;u>8Bc3LNfa|5LHmvEFHQZuF`Pf8AGj z)#E9)f}vuo7uiR?`l;{GF)#a{digCmFZVw$&&pot+A1m$8pm z$)r~)@-^xw&sX%R-0xG_H6>PQm5*NF zhkbH$?G@oZi2K0G*A+tcQ;#D%`!tMucugPAdmnukpJGRxRy4m)ku9&YXVJ$zHg&ku zr^FGTe4O{r_A!R=sdXRM%zHisaP#y0ZbNp@`qXvG$J&`s9TPsT86WpfpMvPulk_P< z8;PcAAMXX(Qy)}udhb-q{K z(8IMH^b}dWt|F^1S!A`o&`aKWljt1sUT8ReL0;x@U{5`U6E) z!{H*U@la4r(7d)+El~PA_m?}&{UylQA*jloy{dxhb-mP8ukv0A%&R0s!o zMptrQE8ZGZ$u+cZ?ps+$u8_UI@8}iE{vctwRz|Iy4`NpCs;G4s|F!sk4%vugNS=8& z^N)o{rYkd3$X_xy`3mwckRPK@UW5E=B-4ADpGM|F)H=8o@SgBxyB@KyL4JPf7G zv&<3K6<0Agx`p-I>lpj|n0E`sZv$V;Jo{G0Q8yFx@0drvficWB_DSIW?d!R>X!`HyE=d_%`UKXf&l!eUc z>{agb+{civz1+8sv2SnZIQOyB^vStzd3bLDig0(oz#K%hPtD;zHM#rL$bF?@oztqH zqt6=ZQ*FA$s@dUFwX086tDIIP{wgw@R{8sV%==_orNKUxeAuVrV|{W@_c1Tj$2T|o zRNyMH@{iKqPn}WjH1`Ago~{pl%0@cZald%~jIxlKw24tR7^FHssifzVO1zM=;^9v! zmLIjESy3y(Jk7}A9t|h?O^5tXY6!oB_#Hro?zmF@LFVvgqgL-R_C>Mbt_Ru8M!c@| zJv=k6RL2^A8zGN+x+}}_ue_4IC08=n+9&TjPRsM5(`rR#u8LS$yVlCVoT>A}q?P@C zhg?^rteh=tmHQrj6c$DEp@4bR!V^iWC?jIIccrZ2&{~y1DRZx7p`=w#Jyx7u%kz(U zUlu#7_eZRnVA85Rn6m21l2-kph}D2>^jyL^@1<%Eb*m-+Qs(rS_kI3Sd6B-Vl+}jx zAG%b5rnSuVtyM>IE&boMe4n3r;Dg=jVIUq1rmS9n%Id3ItA2OX3ca&d1L?II{BSM( zKl=Z3Yc>1<^S~KND?+_QAvPDW;%vf7%rfu0GHIpOcWCs(l$E*bQf1LUb&wY)WN*Ag zuJZ{iXI+%vzZkXhIxkf|6s)~Og|i8(Xicx&{!7{W#y$w#OZP^tvT(vG-#cnmL=#r! z{-{;enXsz!nFkN9<@wj8y0W#ZKXi#2SY&G?T+`uEtC{1LthH*TEIduf-w+X0CJ8>PU`SomrQvD;=@A@!NynAhNg2&-&k`tp8ua9=L=xFvtBFhPWQW z?_8?k_b$=M)`S&V*{NuH)QX+DRB^^RiDxfW5>nHbXmmxw%3KlS_dgPrBkOWGA^RHM z<(M0#{)5c_T~51U-$&?j6>N-Ig=a^p|Agg!Vbm(#8M8`|rDd$|gfCNhR;MbWm#K0; z`#yq~QUB}%xg%lKKFj(?-DS)PUC#G^FK6G!<@EnAS2OXpggTj*;QKF43CmlRuza1D ztL<#u^0Uz`5WZaPEOd9|2h>@1Idh-9yRm1~>Z#-T0==P_)yIN(KWiJe{x-Y^X~@iv200tEh(+Nvq|(l-0N*ZZ&Z| z)bD4XCJXSjTY6N-Vth4wCTdQktV+*is)F*2xK*($X_c-*Z`y0|3zw+`ijMHRX!O+! z!t|35B(1z>SvTT3%{dgeav#1-*?W5A0{c5xbzFD*+nIN~jJDsSQAl*ME<;<1F&>GN zPJ}XzQjWv##H(=VYH3?W*6yy3ey6jQx3!GJP3s|1t$nUB(_s)`_m6 z+<16(P9&`W51#h*ll#$NN`X6C>*D~0_^ zXgGS6eu1nj+pOqs*)R3NR{cB2P1pR2{A}hsTzn^r=YwbcW_=vlp0QbNo~!gp>|Oa+ zu}_zEu;^9lOkc$wfX(U)UZoz~gD`;GSMb+={wl5m(%-jPBhPQv(ALeY^KDk_{j2Er zU!||%C%TqA?b)iIU{4&_svKnInk~wz+C=|glboJST>o3-f*kC*(M^p1x6uFHqH6pW zuGqpjY771UEh=_x;{DGpDm}PGWywt{$4|uvn^fuFq^e_^R6VhY`L9i?J+X=Fe-q#T z+oFbNx2TzTyzoCLFCWk4VeFnOHpzReX|iT0xF`n?9$@ zNWXtiCH?MI8YWn}x^#7~r|5wxhucqBq)Bms5 z1m9il+;_FQHeSu1lB?ANLFn~wV1Dsx{WWQL!W-nx-=Iy{+g4qzPa|)F_PP!Fdt@}a zL0>|~_g}5=A)kV;!8f3P&(+M;ZD4)=Y7KKd)$~iQR)Obg z6+#iXp?J#%#@AOXGkCSKc5IMi74Oup*r0zQ4p;uw%+GF6?z#<(uQxDPxk2@3uV#F8 zwHlwlPEA>_tNGA%%&%O>{M^;-$=RTC(x`Z71N&1q@cT3yRQ=cn*5Wp>;JJ(S?_F~2 z-7P0%KeSse$ibcqc}I84L7F-66DWoP@WLOsCQH}v=KYu5>bUC)rEq%@UV=Aa1^ryF zSi^gi?|gli_dK7J0?c{0|A4*-?}qz-`U$Q0SFR;kgWt7q1+0Sg&`!JWhAr5)!YA;z z7r74}f=f9*hB?W{G|;=gVs>)`RF^f?V$hj z&M5^Ro>KdPDRrDZrB34R+C%?m&zO4dSgGLADfa)f4qkN$-#_E|4+DHKcrbd2hB_}% z7=|C>`M>WH_Bvg{y7wi_e_o{^X(XdGt^Jl6TI~A4;I~$S)Z?qd&S-9AfYPQQiT7YV0*o`w;H{ zK>Zfp0buWd+56x09^b3rJ%AQyUBNp5-~}JFfgb|U4js@5UGS&h?_SB?|26b|5B^;J zq!ogJRlNTWL+j`(&!#oZ`v)WViM*d?&lm5%Lmd9!@4vVBcdHqis&=ap8VZ)Y|5SBwH+`Djs+?P7|ND8Bhq>nW?N;d>^E$zGGz({84o0}9uHt%&BQN2a z{1~i;-}2nM4)?_!j9Cu;URkSlbHVS{=Q%fu-~F81jGy>pyA^}zk=@Mi?bgV_c@2Bs z<(|G}C#W zw*s_h|Nh;~E$xgI@}A#N$W}E z58$Wp44lNR5_u1Pe~z>=xW6EOgZmkH9lGdG^umAR+`n@!PC4K9S?&QGUk9It#|VE8 zzhA_D6MPFY^VtUqjym=LKsI(4sf)FQ>i88+b z04W%SS~u@{(4OjPD-FoT^|Vi9Gsi8+(!=N3yTP~P?%*485A*G~5Z{ms^X)kHKhz-e z)~;cFWDVn^HM}FVMnxHX^8t#dkE>+I8g9hAUjYtqLN@1IkOR38IZyr|_5t~W1okAP z){#Hxe`r#n&g1m+&<{O_kl}auz6Ek*mixnz)9QSW@`3JMlj`AIkTU3{jQWtRgz-#p ze>ukeW#y#W2;+yqH02AKT%S*KT|B|H@MU-oegyvvKZ8Q}0p}}_zvbF#K_2DUhx`wY z`;c+?CHxESSAY|K%W)p^cgPFyAMgqKd!K^Ma6N1XH-4@}-VI-c|C2bsg)GPYHssHs z274nojU7thclc{Sc0)T1!t8n4G17zI?j6DW7W4{xAPh58+ zFCZU5J`T^p&*0BKOPa`0{0}0>k=MX+?9<2@I0=9LS@eUm*x!J+VG;gZO_khj5oa5KlB;oMgE9QMpLzfqQ1?`L166Pe9EN!JnD|MRqe z+GSqGZrd~09z3i1 z_h|pnc!Ks1_IK(}(f+s6{&&#+cd<^7Y~$E}fPM4i$LyQ$ILtRoj8|o}zu7rF|irnGb1!MreY1 zXnC+NqrQGtF>R>sfihma3QKoVk|+y@}??3AL&VK;ivj)#z87x%d!_W_Rk z_r0l*pLULpeb7T255B;C0G+zgtz!%O1(5;zR_*IA{;nd@%drpHx{iL=+Bfa?zm03* z33F{Ae}r_x=dtgB7VKX_wj&?-SLSKCW=64(!JXJAksEpL-;JMp;hXsRHvAU%KVzQt zZv0$%6Kw!K1)pZ@^tZ^(a5Y>H2MD(vc_-WrH^W!q5x5N=gm1#P;2ZdT6ut#tCGMZ$ z=7d7{F>WQuL&*C$_aHol{WX#@N;}O@<-r=T=*GOKn-+p-iLgSbi0v#9RCRWIqV_o|` z??i8oo!F<4nd^A|gX0?7KV*OSpK{Tba%fArw57Z&c#mW~&wpegZLjDE?f-e&KeB}5 z(p6(BgYrEesDgMZSJD2VdMD3+sKs6f^~BLY9F1#e|L@cOKjitpb&UQ$?H{~^^+6kU zKLiNd4jtG#p^J0fV83U>aqm&uKlEb{!2st5VF-H|hIg|68zR`F5aV1N64;ZFDjU;i z)fj!=^}LU~p7)X0vmfI{#w%y|w#$0fywlLMa$C}K2;OFe)eO=)L>orWc5oG^I>c7yjH$d}{+) z!g1;Q?5q8dJ-EmUjw^AmYWlsZ+4NuI`n_s{{7y-j-vLSTUK6sBb4{K6J`~sCF|NOF zB6GPGpFsWyerArL5Vu!3KZ1P>N;s~7N$juVUW5JD$T?Vm29E!nZzg;KE{9LSySS5e zzTw92XFv?d2EPH#?`J>?v_TtQ&H^W7g9~yX7xD}*XWFLmlXuL4SpbDlVX)6u*fCVt zXDe~7v|TH0*Gk*99oG&R)CO~Wj8tZcW46dJTZZX2*tTM9#kQ>kTZwHe#a3$DjNh`^ ze#~-hIOnJ{M!hy_BmcK*lQ!!rZPBN7H9wT~nT%`T24mQ^ZN|3U7&jVYhcWoT_GdC~ zHpVT+*ok4hZ8M>aSL4%oH2#b?{YAD)9B zz!CT%{0M#wN8u+hmz`;w4%>9vrpq>SY%|w3^KG-xHj8Yt*fvXSv(z@rY_lA5K4RjW zkC@ozBPQ&8w8@w;6LvmUY0S9Kn29XgG-1=OZrcnRvtY=WZgV#6HvZG4Zey15+h?Ci zH}DAhOuDhfn85(%LXF>;jd`|dQe0^4FlK9)G5bwyi&-ZA#r&u-8%*kpZ7E~=P2Lv$ zCf%i6<9{i4(3k@rV-A`UIG<~B_^<``8Mokci|ura?X*)FJ8{}+JMFZcP*E8eMll4LwUss| znZYD|Op?ST6-*+=BrZD|TOkwGHf_6BXu9nN{*5rFx7mfm7uvSjzPH&$vJJ0xacr{- zV%v6YvkPIHT?pH5)Hb^Sw%G-+ZHKnm%5S?#+iaD$-K=f4qT6oKHe0=IJGISLY8x3N zHcUd<>6`4?schHwYeji)zkz>qYI_F%wq={PbM{7U-y!;m+iw=Vr0qMkooH~hBQqwB zws#Yyozok2Z6_gi$! zwW8wfg1N;w+jYNFJFgX$Zx_=}<80Ue-|O%HewRMSPKjOGW&U?*w{F#~8+Gfg`hxao z&jIauJFn`nzNjxgtJ`$j?Yiyh>vS7#Pdu*Mb^Dif`!{s^Pjvf45>mQDU(%QMd|h|y z&M)fDyMCfO?>(qHkLoVnb+7JvKzBVZ9QSIk`G+rS?*Z+7KzqNgFY9jI{YBl4xli}# zo-ga3`)}7h-_t!Wj%c6ueNp>v)4toaj|lb=>%OmP-viqBp!Usl>MQ!nm-UtJ=_^m` ztGZYB-lco*-3Jfo-mmH2`|l6|_Umi<+H<;3_ua1h_US$>Kh*uY|BicfKXKl_PY3jX z9@xvj4Ti1zgL?2bJ@^eh_#J&+U*F6BUOl95=$rZnJ*Jbhf)gy=b-v-pVXs=`QHY({zQ*b zn2+hP2lUwYAJ;$WpUnRucv41S9@gV}oG8Dm@1omx59*K(nSc164n3tqKh&Wg>w9`a zPwdeX==TKPo_Ih{Jg6s3R;a%xFc0&;4YuovALt26@<~1UJw16?Pd=q5pMG3V{#Z{^ z{)dGxJRRPn!w=~2gF5^j9e!Mg4?U^F{BzozHy)nm@Hri(8Znd>N!2FrytNWdd3voGkf&Rm-GzB59*mm^vrX5R^Qk64?b$Jd>U)hm|tV<8tc$lr^dQ8)~&G~jRiHp+@YHUbjVT}!IEUK}X#^M@FXe_C*QH^J6JWJyajXO1-t#Oyeb2Of-@jQ*^YrH_? zg&Hr?xLf1J8ZXg!xyCCrUa9dajaO^DM&q>_uhV$F#v3%=sPQI^H*35_xgQVPcwFNN zjVHw>{t2fhvNhq-M2;qMHIb)@LQNED!mWv7O_XS&R1;;IsL({ECaN^S8exqlYBf=( ziF!>mXrfUQO`2%dM2jX`HQ~{OR}(%>v}wYxi4IM4YNAUM-J0mpL{JmGn&{I+za~PO z7|_I^CWbT-7H=p_jA$aFiKuuJVIr=Hgt$r3tbS3fUzF+>t@?#Wzv$L4`t*x_O}aH% ztjQ8hmTIz0ljWMM&}5}1t29}y$r?@8YO+p~^_pzZWTPgVHQA!cR!w>|>D8o9lL1Y( zYqCR=oto^@WVa@JG#S)nuO|C6NfJYv3~Mr?$*3k{nv82Qp~<8sQ<@yrRHmk~H098g zQ&ZWRa%n0@Q@NVT(^S5u3N%%ysUl6eHC3#s5>1tAs!UVmnyS!LrKYMhRjsKSP1S0u zPE+-oYS2`(rdl-R)l@)J?V9S)RHvr8G}W!C9!&)`)vKvKP4#Okq^SW-4Qgsc)1{g& z({#C}D>YrE={ij}Xu46;Et>Xd+NWv1rUPRC|8%FOyENUc>7dvKG~KW1kfsMTJ*ep+ zO@}od(R5VPalMqKmmGS@sh9HgQh{D7)JsKr$*q@4^-`H$D%VR@dZ|e-wdkd&UP?SA z204sy7$9L(XeLK9d78=BOo3*KG*hgZ63vuqrc5*Cn!!V*W~ww(t(h9l)M}9||Ri*>wF$D8y@re4X?D^9(Vt5=Hjid(M~ z>y}?WIjECEIvLi?zJIfV5>Jq1OK)(@ig*p4x03OvyrXF z@EF6Z*L_B|EgSd^7+bqBI`}uT^P=&3mTiCCfw1ky*2TX$KY%W8c=U!>Z}{{^o8Iv2 zjR5~|v>T(x7(rw78Y5(k0gPFfF>;KNYm7W&oDnenDjeL`W+^U4wH0;@zSX` zy7)J-U>ILUaMlRUcbTX$mOTWQJ&-~dDcGKSmp$9*b(ws2ne@8sld~>67{Ww?Fac*> zb_j%t-ejrEWT|V}6}N8wO}K6oTepd=+r-vwymlL}-NtLT368HG6E}u&Gs3veW^XWt zU1a#T)xxlSE+@~?vQHDr*oef|5yL(-DWa{3)d=Hb*^Uh@dQH;3CX`(ab|KhB(5E;0 z`Dgyba8`EC>_*;IiG$$)HIGW7*w~e9*)>XtIW3 zhhA2Sqh&{w9kl(M40C}D8BZAYp^1CQXtOMg50k`@iFC+zU9KVQ%SXgEWMUgK!NdHU z7%(mh+ttJnHZg=L@2Clf#RRg&2D=KlYBz8$d$s_}Y#v$6zUc8xx$$TRveOJNha|aS z{!QkGO=1}4gc0VHNfX0_8#YM~+cw*WIZyK(F`-6`GsZ>x#UtB?@jGH{5&q4IDE}ro z4Eu07vm7rU*_p#`!Wlux=&~JKjDHh0X8dCOQQ+4c86U=P4BKobZ5N~1HfGC=D6x%Y zd#SOTX*h&!lxsGNf1<|-8l%@XNMknZqG96AA_WX=4*rc#$80vGi@+zwvXjZ1IkG*P zAP!@5^8ZG=F-XQ~d^$}sPA*jpLSXz+81E*a(*)#vHupgcpE3N#Af9X!55`4df^uXB zFp+25*Ptzk$Q+rdOo|w`=Zl8zc{#J@#%Etwreb8yyG%-^5g5bxG=ew~%ct!GZM`v; z{h4bY4~RGVmMtc(PX#7gj6Vt!(&UNOYYZYRuwyW> z6OfW5WK_KOI~&(*O0%OnovG7KozB+j9G%Y9={%h-(rLF&7wdGHPS@(RN2l9#I-t{? zIvv#MKAjHf^q@}1bULonDV@pGnJk@g=!{D|9eFmM$t6J^k7x3Arch^!#8Z+h{7i-3 zYO8vyErF8&7f1c{@!Qg8JY0LOhChd4Kw3AteG zoEzj^7#TJaKO^;jeR9wz2YqtLe-7#85MK^)kx4bp>r6&!{j-F-w1vq_>15_f-Vu`P8t#N6d|1m zbr>P72z3}Cy$E#}A-MgJ)JN6|lu{!#RgqJI?qqv#(+|0w!L z(Laj*QS^_Ze-!O&=pRG>82ZQ1KZgD>^pBx`4EjG<=?J!9w@N6$EV#?dp5o^kYyqh}mFGmf5d^o*lt96jUc8As1Jdd8{eIQ1N-p5xSWoO+H^&vEpRqkkOzMeETyOQBl|-BRe5qV7`Ym!j@c=$JxBIw*4~^h{B2DRfPt>nJ*oqT?t!j-uZv`i-LB zD7uZJ*C@J-qRS|{jFSIR@;^%cN6G&v`5z^(qvUmzypEF3QSv!TK1a#tDES;EpQGe+ zlst}-pHcENN`6Mk&nWpBB|oF&Wi+9;GfSZYDq(>Ci{gK?Egyv6&J1(b2;1i9)Y~}( z=P~~_kNLNAh$g2V8lYEi=enU7O7wP~1F|6(@}UR_mq)m~HV8mB{};voW?Me!eLIgx z^Ng@<`B{29pD6Q*GM^~(TfqxKz-N9R^g{@SfXwGdAgZ?uGJ)^~gfDPG9uP-C4-me9 z@CAe~B+fz)5O1Lmx_~?u;;#^Ym8eyXpPE9w9Uz`Sj^6HWdYM$h9)31A3mlLK`A`6b zPy}u$h7u@+GAM^i!+q!YAUKlE4u7=8iZ0sHg-w67_5ShfLVm?Xw}q;jKiV35F4|bM z{-dqo;#m`WHPq-_7P-kHCt2hoiyUN;gDlQv)j~ZqLNm027yQr;ozM><7=$4hh7pKD z4C3dqP&g|ANu6`#opVqI4pLyljoEK=&Oyu$%F02^4(~Z<**Rz8oRb1G2i@nKRTs~m za}J(!*5kT-20z`{e9&e%=PWcn9OTCw;Fq#@kW)v%aL&<*BR2Bw=rEk4OdNf7V$=-B zB{jm&e3JCqeK;BBSi|pVN&h3(=a{=K*Y}F408}X=QLh9(_^q5OtRR>l2J8i zIOp{L$s-3B-GXG_j?biHoXovKl!%06SfRvXTdQ;O8lz8p&pZZu4P?dt(eK>vdu)q9mUBV#mOCo`;*B+J9I!N zn50RUY+tNT4vf1ANqJWi;j)baqpFOv$xhF51`(!wXlctao8mA^Q!*w-Y&2IVjnYX| zw9_6!(uSO-Moj(CY@Bg0sn9&iDcy41%WzBJWxV%SFHI#Ll7dG50gG)`+3Ip>>-4F zC;-x8{65c^e4a7(Jmc$m#?$kRpXV7f&of@0XS_Vm7KU zQ*jtonl3{+vs-B%plKeHY1%-VmYL>KOJ{chewfxtyIkOgN@xQ=;D@Q3G*dThe}o{6 zD4m1<9OBKvKYg`yP8C!`4Uk3-;dAiM)J&Qwm~;+da`T}G8lf3lfpdApmFI+9AbegE zkX|17&5J9Yp97?mKLEr-$0}XW06rLmA*BlmTZnrhc_}3BLeeiJ?!qV#PZ7Em5f9s> z(?z6Fg#JYlApRoscV|I1;LlAw?lPbp-1u{oUpM)2_XFvETJ4q`XB^^FC`zP_$@`JQtF}v|e^mvbM-}l`ljmx5ucpkaiKm*hs!5|7|233r4e`|wM=fdAp;I04)=^#! zltBY!*g%?%)LSFRjpVrz9h(c4ZYc)*wUWnH^4&_=dx+0Nd3eZ|mw3F?t(UZXwLn-O zY5RJCIDGi`QCDrG-$vSPluH|VZX-OmgmfEuYUA2!YhMVWMjOTJM-@M+`^l!CqW9y) zPxS3X-%e5;WVeGLow#=fl9Gk{Gq5KS}qOf!f~kD?|6 z#Ple|I~vsjgNTL9MlEE8wcyBtN-fa)S)d=Vz%y`xiRcAaHW*n1#n1{J5Ys{q$BgP0 zxFam&^ud4@X#ERJwl3rjYk}6kkVja`U?HyodSM7SmtP1K(4>WeDhO$TR=!YJ0=O5F zm%;?3v`|EzijrDz6OS9eZqj$hwNUH?@>ATYg%a{v;sWwgicY0oEtH{O8RyDIfOyL5 zf%GcyTS+NOcBBfMVz+!CtDdmrV4M ziC!|s^36h@8wk@!JhZxne(e22THx8g5Ng-L0E!NbYGIH98pQ7)#WzUW%&{*F5oQP_ zhwwi{{=?)wOtpo{XP9)u6c%&t3nQe%^3TEu?jz)dx$y<&vKN@sU0^VHb65(wU|`^-Ovvskkld% z)%Db=*ov)E#@?8k;#k2+)^!asV{P=FXnYX7Z6uo7=|GZqgrGZ zVlf|o`NWlvUuGhBGeC>XG%V5-7MWRCWJY1JkYk#_ViEE1U|Mv?v{;-CgfAxT;zIDl zfEG*0KM$Y9Qrx-p7t8QdM!vZG7t2CgEGNI^oGY)@Vg))@aIPYt#Y)0d;#S!LZ9u*& z(Y=y5D+hrxs3hN&lv5S?ts=cD&R3Imb-5O44vV$uQriT?QRe{CtV7>A%C4UBsYmAq z(&VAUyARL~xYH088?%7C^MG1xLf7 zA9-u*0?LL3Pjme*@}ybxqpP2M@wpY=u>jKWlXpLP36N&M4a6BJ0rC=v0AbsSm#5NV zJ85-LMxErT6Sq$Mb&}ss(&-|67iHQ_KD!CqP2JIM7JKp2hyOmxw2$=riH|4FVu<{Q z$mam%HGtb7a*%utQWk^gNjq4i@3}Zc*dg*fgw8|Mf0(iiQwCxD4HM5WVMnOf5%R&h z(;|J-Mb>H->1!^s_Oi&_?;`WNi_Ft5GIzR|Bwx&#E;2W|$Xw_mbDWDj5*Aa`3v+r) z%sDM(x}XTkVNgr-!Izl5S<32zen`QnmK^y|2tgRo66=;r*__L*1s`-k7jQn8cybe3 z$|HUz@|KvuTgt;9?P4jfMN7=MEivP^M5k|wS+*sv{iOopE+~U02mx^vB(+rNhk%xf za4#aAqG2t$$*a2pI4&l=V*HhmKRRejbkLSc8lfFXgO1r!Dd)>LUsj`~3ev8?t%CR~ z$YW)-ma5vcRO1H9gwE7bE%Db9Z!LP);a*n?#92q2T-!@^-CE-2zEqFB!KtHq2KVNr zVajosa^&8;G(vfg;LbYK65W|4)_Rs$&sbs&V~O>MCDtL9=&md=r@q8o`V#ZtORUW= zG538w(+3fq=l*-1*~0T#1CY>pM+uZcrOq?Ka-La$^NefHJCi!k8urfNL+A_j5Re=hb#J2oGx zRfWtQ>*dcMo&}Hp{7?TrKX!cn^2c}H{qwg!?sVI;^VjFXx4%9Y{dX?^?_ZxQn}2<7 ze*E>h^?fA690WB(m~-TOQ~@%ZoeYv1Sb zpT9o+@96X7yT`wN;?MiUzxU+de|`Fa{nPL7pXdHFeo+5B_s2PZ^!RhT@X!0{`_un! zKCk?Bxw`hu{`$QB>hag``v3m=y!F@2&j6qI{+Rdv^WKMl&MrOv7&CvKmK&e*J&%9< zAm!)_ottJy#4h4^7CK*Sl&~9{>z`E_lBR| z34VGf`04lLPw(44{XG;u{b>8?9owgOXrF$M{`BMR=YRaUdN=mze=DEfOMQ9|_353{ zr}sCX-rszB7xU>I%%^uS;=4*cua-J`qub#g>-#p(vKRiD@zdXM^ z6VKGM;92x6d6qpZo>kBP`(v$pHawf2Ezh=R$Fu9%^Xz*LJcphm&#}j!(|@0O&OGOy z3(uwJ%Hz-Vzi&LZo;%OIXXcrE9y~h!`@cO;o@dXC=hgG(dG~1aDQ5il|9bxA`M2jk zo_{?5=lS#3@PD82r2k3(ll~|DPx}4;`gqds|KG=x{wMuU`k(Ya>Gw0*c+&r*|4ILo z{wMuU`u+MaoG%;x?>?UNKk0W8Z9M6J(*LCYN&l06zt@c?{XSuhC;dKU4WF{clYTz~ zjwk(3`k(Ya>3`Dyq~E8o@udGrzfWW1Nx$DS$CG}4o0svV|4F}Jr^b{1C;fgu9sU+L z<4OOM{wMv<`k(bb>wnh&tp8d6v;JrOeg+-S`k(bb>wnholizsO|E&L6|FeF-PmgE) z&-$PBKkI+i|E&L6|FiyQ{m=USx;dWpKkI+i@7K`rtp8d6v;JrO&-$PBKkN7F>v-1h z*Vyr_|5^XD{%8Hq`k(bb>wnh&tp8d6v;JrO&-$PBKkN6oV?67B*6(ME;b-gN6YTKo z_VDZWc+vl&|3&|ce!q?nzmAU={bp6etZMjKeE3;>`2K6W=zr1wqTkOa!>{+_MgNO_ zKdX!v{V)1o^uOqT(f^|VMgNQb7yU2#U-ZA|_p{A-(eKxa@uJ_)Im6F6<3<0A{uli( z`hD*VMU5Gc;cHzv_S0|Em90|EvC2{jd68_4|2m zyy}0||Ek~jrNif;@v8q-|EvC2{jd68^}p(W)&HvBXQuJ0|4qNoPQ%ZQ!?zgYP5+zz zH~nw=-}JxfH>Vrsbi3`GzrvFX9pG(J^ z{x|(^`rq`w>Gv*kyy^Ed>+o~Jc+>x;-|rRUO~21|!{@r;+m-RA|4qMdS%x#m!_O4s zP5+zzH~nw=-}Jxf_p`?EIdHt`f75S=HGDHO-t@oef7Ac2|6TvP{&)TF`h9L3@A}{M zzw3Y3|E~XC|GWNo{qOqU^}p+X*Z;2nUH`lOcm2M%9`E|!^}p+X*Z;2nUH`lOcm41B z{fs`|^}p+X*Z;2nUH`lOcm41B-}S%if7k!6|6TvP{&)TF`rq~YwrTjeYWTf%yz770 z|E~XC|GR$QM2&a-e(oCY`h6}K@A^OVf9U_v|Dpdw|A+n${XRF05B(qdKlJ-~ZG7ne z(Ep+TL;r{V5B(qdKlFd-|Iq)T|3m+W{tx{=gAdE*yN4>(QlJC{OmLS zN54(pu*n-XdBY}e*yIhHyy1Ppu*n;~9UL}!!zORo`@NMI;$s67|44b^+v*57F8#Z~vCU5v$ zIBfEUP2RA{8#Z~vCU4l}4V%32zxr+RhE3kE$s0C#!zORo`u*nt!zORo`u*n-XdBY}e*yIhHyz#&KZSsaq-mu9V zHhIJMH^U}x*yIhHyy2VEVUss}Z94o+Ic)NVP2RA{8#Z~vSE<7$Z}>WO*yIhHyy550 z;X9yVlQ+C?88&&tCU1D}GQ4*gHhJS;`fc)tP2RA{8{WkXo4jF@H*E5TP2RA{8#Z~v zCU1C8Gi>sPP2TV`?XbxkHhJS;`v0Zh_f6wp`v0Zh&%47ronfmtZ1u*!^n1TE{0uyN z-8*dchOOSP)f@lP?=#ZyE^OH94ex-4t={mJ@UYbzwtB-?!^2i@*y;^m5f59v;l0_g z)f>L!8h)-GwtB->Z`kS$TfN~O(y-MVwtBJ3}H;j8Ikt2cbFHf;5Vt=_QJ z8@77GR&UtqjeqO6)f?Vr4O_iot2b=*#=rI3>J3}HVXHTM#XW5GhOOSP)f={Y!&YzD z>J3}H;p^{Vt2b=*hOOSP)f+xLjDPF@xBh?Y_c>zt95Mc_--d5^X*T|&--d7a+I;wY zF>LsT4d1Zg8~@R7!#8}LK5Y1g4d3wjWBB|rZ1{#3Zo`Ie_$)GfXE<#5hR-D9Kl*L? zhArQ)-}sMyo4(=m@UZC{ zHhsgUZ`kw=o4#SwH*ETbP2aHT8#aBzrf>LOXV~-&pP`3M->~T$J|7L6zG2fheCIQ4 z`i4#4u<08%eZyy`VbeEk`o=%{|Iz=Ce%rp`Gu5!|8@7GJwr~8S{~!HcD-7Qq4WGA$ z*9*hOZ`k;afArh~@` zHh;tBZ+KNPZ2pGV6~pFl*!&IO$BuvW+x(4x^!x5=c$G11|HeQ1?f=F<`fUJ*@3e;R zw8lUBZ2`wW`v1}YkA5#nhh5;X3mkTV!!B_6?rV5)I_v_6UEr__9KH)1UXu*3Nrumr z!{^Fj7dY$!hh5;X3mpHa-!5?Y&TQBP4zE*&UEr__9KJ&vzQY(kiw?WM;k%6CGwJw$ zpGoZkhh5;X3mkTV!!B^x1rEEwVHY^Ojv01=!!B^x1rD!fh8McSE^zohWY`4`yTD-= zIP3z4??;C3M}}?Sunioxfx|X%cmX`@1BdVO#{cQ}GI;#a|3|;g;PAR<*bI(8`hC|u z>;{M3;P|88c5wXBZ$CJ^5*mN>|Iu$tIBW@rE#dg1|BwDZ`t1sb*G9vxaM%?NyTb8D z{~!H-^#9TSN58G%ur(aEhU1TZd&6OGIP49Fz2UGo9QKC8-f;Nteb^k1Kl=aZ|D*qp z{y+Nd5XT?=c8J5Ps$q*bY!Qbo;_%vP`0jH2(QiL6{^<7|{`m9ful$#5j=s~3m*V-a zzdheP-#tG(KRv%ZzdaMr)U)7O^elOnJu9A7&zfi5v*FqFYR}6=iGDQx%6Clu01!NThE>6-ZS&eJrABo&wqQKJkOpN&#ULn^X~cZ{Ez2< zJ^%9j+w&jKKc4^d=>MYsi~cYAzvy?xCSUaXPB&llf6@O%zwg*$Y8Pw1e9>>wmoNIQ z`tn7;WnaGNx9-aq{T6=tqTk9dU-Vo0<%@o6zkJbe@s}_9eQ%sE`Yr$RMgJH57J&Jp z-wH5a^!sivU-Wy&k}vxG?RfG_Q5{#8#yqb)qV7!aT7yZ_PF?Pro z{k|WJ_cQSwb$myiull{l%vb$i^?%j>RsUE0U-f_0?|ban6~=ej`Kte`{;&Fd&zP_J zeb<<;`oHS;onyZ0w@JxY{a^Ka7Zh8>eAWL||5yEXiTSGktNyS0zv}<0-`p^Eh54%g ztNyS04F+Q{7~{Zv)&EuhSN&i0o7m;6e&4UhZZ5`gF^-Ga!n7qa0E#_^p zzlwQV%-iy}extVht^aTRMr|=_%isFF){Xa9F=LDUS`63nxBkEN`(8DF>o-!1U0N); zVttms^;?z2Tr7X<_gXmKf5pHgR$VdOiosTVCmZvsm`=rO<5)(;h9{O$v4V;fR18vL z`xG0e*gnOeDF#h3bC>s=`&p`-yN>`KF(%3Re}cv%^*8n|`h;TvfQLa8==|!d1n0%HgW=O+Q-| zwkqHBf7Aa>zgvA_t>Sy<@K)ii!dvB=e&#CP4d$DE<|^Oxf7Aa>KYx{P`oHNn%n64T z4l5j1IIP$^<(vL*`gyE;)BjEXH~nl@zUlv_|C|1A`oHP_rvICM?_px-6GNYT*Z*Dr zcl}=7$9u|r*UxpuYy9}GJm2+y*YA~nd~Y7F_48f-cm3b>f7kzAzp+rh>;JBw2MZ4t z9xUJWf7kzA|9Ac0^?%oI8W2t_oLD%q7#hXUD4bY~jbb(sW25+;JC*yZ-O`4VUsmzwuIj=;zDAm*t24 zANo17{LueH{}25?^cy+Fj3Gbtn=-^eJw{J4P!Ee1d^n1S$!}S=h$9v!W z(9f&ohyEY>xwZVz|3kl7M1JW1q5p^eANqgj=h|XG6$7fUZTX?!d?Jin7`K>FgmH`a z7%{NQ5B)|~`Jw-Z{vY~(=>MVrhyEY>f9U_I|EK<+`hV*GssE>b??S@Kg_Db+R(|UL zso(pN7~aRb>v&HR@2}%sNq*|*=<-wlPyIjjGj;Ki?i?<#r+)7k^Hcv%{Xg}ao5VZE*bT<$EZ#llm;PV+4bWnW5}+Wz^#9WDy<~pr z|E2$ze(&xB8U!@RFa5vt|I+_UzwuhU!^|)Jzw~>TnP2)1+VV@k_nP^o|Cjz>`hV$% z74l0zun;@O*aXD*Ep`F13&=11zx12M?H@2p}w5Zi(L((nCM+y;&D zTx<#QOaCwZzx3M`;J9aG$+6H|JMIo|8M=j_1i4uw|={Y{MK*2li&J(>;J9)w|;wu{MP?l|8M=j_5arY zTmNtUzxDst|64zS5$}WJeQHG$|3v>p|3v>pzjx4?=%47H=(oqn zME^wpME^wpM8A<@Ci=av&P4x2|3v>p|3tq5V7W6OZU(mmxe?h;kLKgHd=wHykpnpOC zf_{6REa+d*zo36X|APJn{R{dR^e^bQcg})-n};muH)_p-{ssNUty$1-7MBJ6hOSxE zZ|s^y{fqjIUSsr{Mg5EV7xf#!#{2Fp>R;5qsDDxaqJHzb7{|uGBK8$Al8w1t7WFUc zU(|1Z9mCly>NlQ^yTn=4zo>sv|Dygy{k9od)W4{IQU9WTv%MJGW>Np5{zd(Z`WN*t z>Nmp8qW(qw#<*G3Z}*f%{fqh+_50hI#||pye_7OTteZvs2D@3(zog%AH>QEHmCBO- zCH*#2S<-Jel_mYQQ(4l#q<=~OlKv(AOZu1eFX>;>@8)ur^e^dO(!ZpCN&k}mCH+hK zm-H{`H$}{n{w4iO`j_0i=s`;sO7OZu1fFY8~{Z?qhf$e2WCS^u(rBj(u2#7-v5`fX*h ztbbYmvi@cL%len~FY8~{Z)=}r{mc57_1oQNS^u(rJDV))w>^tdbe8om>tEJyARSZ8 z7)fVYzdcTt^)KsR)^9kSW&O+gm-XA~WLdwxPL}l>Q)gNKvi@cL%len~+i7H3|FZsN z{mc57^)KsR(Z8a9MgNNa75yvv?c%bc-!MD&aaqyN4rE2Yp>|gEujpUVzoLId|BC(< z{Wc+4(Z8a9MgNNa75yvv%}it9ofZ8n`d9R?=(oSiivAV-EBaUTujpUVzoLId|BC(< z{VV!c^sne&(Z8a9MgNNa75yvvSM;ywHzbcCc~R;8rs()4gs{U2|tNK^R;8rs()3#tzuU7uj*gb zzp8&#zX@+v^{?t*)xWBLRsX7f)84G=U(>&)e@*|I{x$t}k6F{drhiSp4P@5zujx0| z&6@r-{cHNy^snh()4!&FP5+wyHT`S)Z6~v)-;6kG`q%WY={M%jn*KHYYx)iQv!;Jd z|C;_a{cHNy^snh()4!&FP5+wyHT`S)*YvOHU(>&)e@*|I{x$t;`q%U`I$6`t>10hm zK#(>4Yx>vquj^mezpmfRIqUk@^{?w+*T1gc{xj?P*Y&UKU)R5`e_j8&{&oH9`q%ZZ z>tENuu76$sy8d-ud-v#x(#|GNHl{ptEMzo0oO{_IX*?zpfvU$h!V@{pfhAAsee=drv6R+FiOnhv#EbmzYTCU^>6Cm)X!aIQ~##^P5lg3HuZ1n-_&m} zoK5|k`Zx7&>fhAAso&&2oBB8PZ|dLFzo~yy|EB&;{hRtX_1ooTQ~##^P5qntH}!Ao z-_*aU-)=8k`nU9N=?7@CrGHDmS%0?lZ|UFCZ{L?K{agCC^l$0k(!ZtOt~p!!xAbr6 z-_nl+WJ^C5kS+aN`nU9hIoZ;`rGHERmi{gMTl(>VZ0X<9zomam|CatO{agCC^xGI_ zOaGRB)F4~>xAcQP+0wtIANYwKVz%_72-(uVrGHERmi{gMTl%;4Z|etxvaNqxKemu< z{oDGt^>6Fn*1xTPTmQEHZT;K&xAkx9-`2mae_Q{y{%!r+`nUCO>)+PDt$$npw*GDX z+xoZlZ|k=S46Fn*1xTPTmQEHZT;K&xAkx9-`2mae_Q{y{%!r+`nUCO z>)+PDt$$npw*GDX+xoZlZ|mRDzoUOg|Bn70{X6fhDBtAAHNoRVGrwyfFJZ_k=t{k!_@oCA_&SO2d5UH!ZIclGb; z-_^gXe^>vmew*m*>fhDBtAAJjuKr#9yZU$a@9N*xZ#x}8E%wvd)o(+cUH!ZIclGb; z-_^gXe^>vmeuyT!`dNbP>bJSguKr#9yZY^K!x&^&|E~T${d@ZN^zZ54)4!+RMmKx< zZML(ge^39O{yqJB`uFtj>EF{2>|{^>p8h@kd;0hE@9E#uzo&ms|DJxZCwuz$^zZ54 z)4!*GPye3&J^g$7_w?`S-_vh1o<040`uFtj>1QOer+-iXp8h@kd-|cF?CG~Hj%{&l zi(^}yJ^g$7_w?`SXI#TrWMBWj{(b%X`uFwk>j#gruYX_vzW#mv`}+6w@9W>!zpsB^ z|Gxfx{rmd&_3!K7*T1iSU;n=Tef|6T_x11V-`Bsde_#K;e!J(`J!fD4zW#mv%t!2? zv#)<&|Gxfx{rmb^knHP+qq47mU;n;-J|z44ZKt!ZADYU({(b%X`uFwk>)+SEuOFxi zPm%-u2l@~6ALu{Of1n?>3T%}F{RjFF^dIQA-_L>m1N{g35A+}CKhS@mAEFGPl>_|; z`VaKOS~<{vpr2m}zmfy}09Ow5+j-|e|AGDk{kFvaM1N{g35A+}C2Q+h_ z-#$DC`VaIU=s(bZp#MPsf&K&i2l@~6ALu{Of1v+R|Dpav{fGJw^&jd#)PJbo<~)b` z5A`4FKh%GyA2!RO{zLsX={eL7pXE^hp?(|n9O^&Rf2jXZ|Dpav{fGMP*K?@p#|itp8a5vHoNI$NCwC z9P2;Uf2{vlKeG_#CCpZi^&jg$)_<)3SpTv9WBteakM$qxKh}S&|5*RA{$u^e`niWd z$~o45tp8a5vHoNI$NG=;AL~EXf2{vl|A~H1A}9J!^q=TwCUTOa-bkLFbWss2;_ zr}|IzpXxu=4~pkh|Ec~{{R~P@^|MVm)qkp=amuOwQ~jsOa+gs{d5~ss2;_r}|IzpXxu=f2#jf|Ec~{{ipg*^`GhIUUR1ZO#hjF1~zB< z&-9<^KhuAv|4ctim^1y1Y|iwb=|9tdrvFU;nf^2VXZp|dpXoo-f2RLT|C#rWLbN%P~&-I_{KiAJr=3M`|{&W53`p@;B z>p$0juK!&Bx&Cwg=lVIdoa;Z=f3BZ5%DMh?{pb4GqnzvK*uo&?T>rWLbN%P~&-I_{ zKi7Y*|6KpM{&W53`kAFTE|7Em4h-a6|GEBi{pb1}90(tmbN!55xX&<7IoE%#|6Kot z{tNvM6XZhwh5ifujuhlV|Aqbw{TKQ#^k3+|(0`%-LO*wx3;h@RFZ46hao8Xi`Y-ff z=)chK&_OQrJ9dx@{TKQ#^k3+|(C+|3F7#jMztDf7-$8`%SmCk4tL8%gh5ifu7y2*s zU+BNkf1&?E|Aqbw{TKQ#^k3+|(9gQ&LO<`C3;h@RFZ5sP=LW-h=TiTr{!9Ir`Y-ig z>c7-~ssB>{rT$C(m-;XD^RjWFA(#3u^}E!NOZ^OOF7;pPztn%J|5E>@ezrE3`Y-ig z>c7c7-~ssB>{rT$C(jAXdpTA%v?&*e)0m41#cSNgB?U+KTnf2IFQ|CRnL{a5-q`&{Y2(to9&y~~yU zEB#mcuk^Ecxzf)c=Su&T{ww`g`mgj~>A%u{rTyuk~N+zt+z<=UV@@{%if$`mgn0>%Z2|nC4plwf<}U*ZQyZU+cft zf35#o|F!;W{nz@h^yuk~N+zt(@P|62dG{%if$`mgn0>*o@4 zt^Zp8wf<}U*ZQyZU+cftf35#o|F!;W{TySi^)s=#(eJiFZuH;iztMlA|3?3f{u})_ z`fv2#=)ci_qyI+#js6?`H~QU1$c_FR{Wtn=^xx>e(SM`=Mn4~(8~r!>Z}i{jztMlA z|3?3f{u})_`dv54js6?`H~Me%-{`;5?|xEl^xx>e(SM`=M*ofe8~r!>Z}i{jztMlA z|3<(2+quzyqyI)ftDal^xB74O-|D~Bf2;pi|E>O8{kQsW_225h)qku1R{yR3Tm85C zZ}s2mztw-M|5pF4{#*UG`fv5$>c7>0tN&L2t^Qm6xB74O-|D~B@1jd?_225h)z6&f zR{yR3Tm85CZ}s2mztw-M|5iV5pIiO6`fv5$>c7>`;OAEVt^Qm6xB74O-|4^8f2aRW z|DFCj{dfBB^xx^f(|@P`PXC?$JNA%x|r~gj> zo&G!hclz)2-|4^8f2aRW|DFCj{dfBB^xx^f(|@P`PXC?$JNA%x|r~gj>o&G!hclz)2-|N5Ef3N>u|GoZu{rCFs_228i*MG17UjM!R zd;RzN@Acp7zt?}S|6c#S{(JrR`tSAM>%Z53um4{Ez5aXs_xkVk-|N5Ef3N>u|GoZu z{rCFs_228i*MG17UjM!Rd;RzN@Acp7zt?}S|6c#S{(JrR`tS8K?Qv@%_xkVk-|N5E z&$;Jb|4cvco|*oce)mi=(?8Qc)9<2CX8PU#&P=}>;F;-n2Rt+VGyOCD40LAtXZmOQ zXZmOQXZqbX%S``F|4jc(|4jc(zZ(#7J0&yyGyOCDEOlo3XZl?;%1r-E|4jc(|4jc( zKX09x{+WL6I&PC?rhle?rk`=nOh3z?nSM9R!)C|b^0<(cnf|%{x&FESx&FESx&FES zx&FESx&FC+C-~z8f9Cq<`se!R`seyt>df`K0h77@x&FC+Hv}@*&x2>Kf3AP7f3AP7 zf3AP7-`$wZ_0RRs^}E`Xx&FESx&FESx&FESx&FC+zC3gNbNzGubNzGubNzGu?$2bd z-wm3~_0RRs_0RRs_0RRYZ;%K55BeYUKj?QUDi8V}^grl-(C-F99`rxxf6)J+-|db( z=zq}vp#MR?zdb=7^grl#Z7L7?AM`)yf6)J+-;KLG=yw+<5BeYUKj?qZ|DgXt|AYPq z{SW#d^grl-(Ep(SLBD%CdC>o$|3UwQe)o0qp#MStgZ>Bo5BeYUySo#2ck-bBLH~pP z2mNl3VMSl(pDbzKk9eqCy)9c^*`!=)c>geQU9a5C0$jKm33A|M36e|NT96J^K0o@c-fe!~ci>5C0$jKm33A z|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW z@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K z|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<# z;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e z|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe z!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0` z|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+` zhyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=> z{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci> z5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q% z{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@% zAO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk z{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$j zKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8 z{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5 zfB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG z`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A z|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW z@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K z|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<# z;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e z|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe z!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0` z|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+` zhyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=> z{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci> z5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q% z{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@% zAO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk z{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$j zKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8 z{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5 zfB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG z`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A z|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW z@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K z|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<# z;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e z|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe z!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0` z|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36i|I7cE|1bYv{=fWx`Tz3&<^Rk7 zm;W#SU;e-RfBFCN|Kh;FaKZuzx;ps|9;ia|Cj$S|6l&U{D1lX z^8e-k%m0`EFaKZuzx;ps|MLIk|I7cE|1bYv{=fWx`Tz3&<^Rk7_iz3DfBFCN|KR;5qsDDxa zqW(qw{D1lXF6v*@zo>svKmT9;zx;ps|MLIk|I7cE|1bYv{=fWx`Tz3&<^Rk7m;W#S zU;e-RfBFCN|KI^`g z0jM(obq1i$0Mr?PIs;H=0O|}todKvb0CfhS&H&UIfI0(EX8`I9K%D`oGXQl4pw0l) z8Gt$iP-g(@3_zU$s51a{2B6LW)ER&}15jrG>I^`g0jM(obq1i$0Mr?PIs;H=0O|}t zodKvb0CfhS&H&UIfI0(EX8`I9K%D`oGXQl4pw0l)8Gt$iP-g(@3_zU$s51a{2B6LW z)ER&}15jrG>I^`g0jM(obq1i$0Mr?PIs;H=0O|}todKvb0CfhS&H&UIfI0(EX8`I9 zK%D`oGXQl4pw0l)8Gt$iP-g(@3_zU$s51a{2B6LW)ER&}15jrG>I^`g0jM(obq1i$ z0Mr?PIs;H=0O|}todKvb0CfhS&H&UIfI0(EX8`I9K%D`oGXQl4pw0l)8Gt$iP-g(@ z3_zU$s51a{2B6LW)ER&}15jrG>I^`g0jM(obq1i$0Mr?PIs;H=0O|}todKvb0CfhS z&H&UIfI0(EX8`I9K%D`oGXQl4pw0l)8Gt$iP-g(@3_zU$s51a{2B6LW)ER&}15jrG z>I^`g0jM(obq1i$0Mr?PIs;H=0O|}todKvb0CfhS&H&UIfI0(EX8`I9K%D`oGXQl4 zpw0l)8Gt$iP-g(@3_zU$s51a{2B6LW)ER&}15jrG>I^`g0jM(obq1i$0Mr?PIs;H= z0O|}todKvb0CfhS&H&UIfI0(EX8`I9K%D`oGXQl4pw0l)8Gt$iP-g(@3_zU$s51a{ z2B6LW)ER&}15jrG>I^`g0jM(obq1i$0Mr?PIs;H=0O|}todKvb0CfhS&H&UIfI0(E zX8`I9K%D`oGXQl4pw0l)8Gt$iP-g(@3_zU$s51a{2B6LW)ER&}15jrG>I^`g0jM(o zbq1i$0Mr?PIs;H=0O|}todKvb0CfhS&H&UIfI0(EX8`I9K%D`oGXQl4pw0l)8Gt$i zP-g(@3_zU$s51a{2B6LW)ER&}15jrG>I^`g0jM(obq1i$0Mr?PIs;H=0O|}todKvb z0CfhS&H&UIfI0(EX8`I9K%D`oGXQl4pw0l)8Gt$iP-g(@3_zU$s51a{2B6LW)ER&} z15jrG>I^`g0jM(obq1i$0Mr?PIs;H=0O|}todKvb0CfhS&H&UIfI0(EX8`I9K%D`o zGXQl4pw0l)8Gt$iP-g(@3_zU$s51a{2B6LW)ER&}15jrG>I^`g0jM(obq1i$0Mr?P zIs;H=0O|}todKvb0CfhS&H&UIfI0(EX8`I9K%D`oGXQl4pw0l)8Gt$iP-g(@3_zU$ zs51a{2B6LW)ER&}15jrG>I^`g0qC9nJNo&G!hclz)2-|4^8 zf2aRW|DFCj{dfBB^xx^f(|@P`PQNn%bq1i$0Mr?PIs;H=0O|}todKvb0CfhS&H&UI zfI0(EX8`I9K%D`oGXQl4pw0l)8Gt$iP-g(@3_zU$s51a{2B6LW)ER&}15jrG>I^`g z0jM(obq1i$0Mr?PIs;H=0O|}todKvb0CfhS&H&UIfI0(EX8`I9K%D`oGXQl4pw0l) z8Gt$iP-g(@3_zU$s51a{2B6LW)ER&}15jrG>I^`g0jM(obq1i$0Mr?PIs;H=0O|}t zodM{*{(JrR`tS8S15jrG>I^`g0qDK{d;RzN@Acp7zt?}S|6c#S{(JrR`tSAM>%Z53 zum4{EO#e*(O#e*(O#e*(O#e*(O#e*(O#e*(O#e*(O#e*(O#e*(O#e*(O#e*(O#e*( zO#e*(O#e*(O#e*(O#e*(O#e*(O#e*(O#e*(O#e*(O#e*3GXQl4pfmk5{WJYD{muZ? z8Gt$iP-g(@3_zU$s51a{2B6LW)ER&}15jrG>I^`g0jM(obq1hw{d4_u{d4_u{d4_u z{d4_u{d4`!0Mr?P&h^jr&-KssI|EQ>0O|}todKvb0CfhS&H&UIfI0(EX8`I9K%D`o zGXQl4pw0l)8Gz39&-Kss&-Kss&-Kss&-Kss&-Kss&-Kss&-Kss&-Kss&-Kss&-Kss z|9@57%Whm*0fteY9YhbBa|VDA3_v*XEeUNd%?N~eAoPFIegu|*hGvkRD$6SSv+Sx< z`?B8jzv+L||EB*<|C|0d{crl;^uOtU(~kid24EO~VE~2!7zSV%fMEcJ0T>2g7=U2_ zh5;A`U>Ja50EPh=24EO~VE~2!7zSV%fMEcJ0T>2g7=U2_h5;A`U>Ja50EPh=24EO~ zVE~2!7zSV%fMEcJ0T>2g7=U2_h5;A`U>Ja50EPh=24EO~VE~2!7zSV%fMEcJ0T>2g z7=U2_h5;A`U>Ja50EPh=24EO~VE~2!7zSV%fMEcJ0T>2g7=U2_h5;A`U>Ja50EPh= z24EO~VE~2!7zSV%fMEcJ0T>2g7=U2_h5;A`U>Ja50EPh=24EO~VE~2!7zSV%fMEcJ z0T>2g7=U2_h5;A`U>Ja50EPh=24EO~VE~2!7zSV%fMEcJ0T>2g7=U2_h5;A`U>Ja5 z0EPh=24EO~VE~2!7zSV%fMEcJ0T>2g7=U2_h5;A`U>Ja50EPh=24EO~VE~2!7zSV% zfMEcJ0T>2g7=U2_h5;A`U>Ja50EPh=24EO~VE~2!7zSV%fMEcJ0T>2g7=U2_h5;A` zU>Ja50EPh=24EO~VE~2!7zSV%fMEcJ0r2l1PFs%#FdD#U0HXnn1~3}HXaJ)Dj0P|o zz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQt zFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)D zj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaM_GKMi0sfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(E#@T`$+>B4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifp zG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C z4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU( z0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy z07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=F zfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IYe0MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1 zAR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ( z8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2 zKs1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4Immo zG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4 zfM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCF zXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks118 z0MP)V0Yn3c1`rJ(8bCCFXaN0CKMf!nKs1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$ zhz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<8tY`e^{s0HOgz1BeC?4Immo zG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4 zfM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCF zXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks118 z0MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT z(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G z0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLaw zq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V z0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?W zL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz z1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$ zhz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c z1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh z5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC? z4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1 zAR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ( z8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2 zKs1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4Immo zG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4 zfM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCF zXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<8t+{WO4R0MP)V z0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?W zL<5Kh5DlQ8=%)ch1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ( z8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2 zKs1180MP)V0d(oV^k4cf{WO4R0MP)V0d(oV^k4cf{g?hr|E2%Zf9b#UU-~com;Out zrT@}@>A&<}`Y-*L{!9O*|I&Zyzw}@FFa4MPOaG<+(tqi{^k4cf{g?hr|E2%Zf9b#U zU-~com;OutrT@}@>%aBi`fvTW{#*a8|JHx&zxChxZ~eFaTmP;9)_?22_22q${kQ&G z|E>Slf9t>X-}-O;xBgrIt^d|f1BeC?4ImmoG=OLT(Ey?WL<5Kh(5?U0PXmYs5Dg$2 zK)3!||E>Slf9t>X-}-O;xBgrIt^d}4>%aBi`fvR-fM@{G0HOgz1BeC?4ImmoG=OLT z(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G z0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLaw zq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V z0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?W zL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz z1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$ zhz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c z1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh z5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC? z4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1 zAR0hF(@z741`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT z(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G z0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLaw zq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP*Y zxqcczG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1 zAR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ( z8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2 zKs1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4Immo zG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4 zfM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCF zXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks118 z0MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT z(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G z0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLaw zq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V z0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?W zL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz z1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$ zhz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c z1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh z5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC? z4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1 zAR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ( z8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2 zKs1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4Immo zG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4 zfM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCF zXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks118 z0MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT z(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G z0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLaw zq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V z0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT=cg75 z4ZnULerK@GWB7x??2F+txQ;jc?S1&W!Ara0{bSC2p5OfVm;OD!-}u%4-EhZa{pX|p z_n7~?^T+Ss`P29BeCy6vfAOjB5BcvK?z}(JfA73M(pP`|?olz8)`P3b^WbIaJUD@W9vlxo4-UDV2d6>K2k!&U2Ty+IgTMUq!D++Yc_`88q9*7kDh+cM*!g=z?cb=F4=gCg|^JHK6d9uO7;EO+7UO&&)Dh+=&SQ#|D-!jL3o=ps#XRrL{ z*~`Uw_B?!^om)N64uqZO-}@TR_N$*~o378Z9V+L=vdVd}MsQwC)}I$Mt>?wm-g)tU z>%4eFab8?Oo)@r$!MFDEjmLX=|JGh?(>^cu)Sefciq4B&Byni z&#M!;=hZ&+^J;_Fd9~~4d~&&G@B{ngiRgTC$l`pm-S&L8c6dHp*g2oAc%09sP0nZU mh0kXfr04TDe$T#GIzC^lv7Rp$QO*~$=jV&*v-8EZ*gpZgev@zj literal 0 HcmV?d00001 diff --git a/tools/sentencepiece_testing/create_llama_test_proto.py b/tools/sentencepiece_testing/create_llama_test_proto.py new file mode 100644 index 0000000000..c57a0074e2 --- /dev/null +++ b/tools/sentencepiece_testing/create_llama_test_proto.py @@ -0,0 +1,32 @@ +# Copyright 2023 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tools.sentencepiece_testing.utils import train_sentencepiece + + +def main(): + train_sentencepiece( + ["the quick brown fox", "the earth is round"], + "llama_test_vocab.spm", + vocab_size=10, + model_type="WORD", + pad_id=-1, + unk_id=0, + bos_id=1, + eos_id=2, + ) + + +if __name__ == "__main__": + main() From c41e844703230d948d5eadcd8ac45555e511fd26 Mon Sep 17 00:00:00 2001 From: Mohamed Abu El-Nasr <64566340+abuelnasr0@users.noreply.github.com> Date: Mon, 15 Jan 2024 06:33:13 +0200 Subject: [PATCH 04/29] Add Bloom Model (#1382) * Add Bloom Model * Add Backbone test and some fixes * Add BloomBackbone to keras_nlp.models * Fix a typo in layer naming * Remove self.built = True * Revert "Remove self.built = True" This reverts commit 889f2040bb56ca7dda6e64c110ff6168b41eb494. * Add built=True to MLP layer * Add Checkpoint conversion script * Change LayerNorm name * Fix typo * Fix getting HF model output * Add and to allclose function in checkpoint conversion script * Remove allclose check * Add doc for bloom * Write batch size instead of _ * Rename out_dense to output_dense * Rename out_dense to output_dense * Format to 80 chars and remove unnecessery check * Remove exporting BloomDecoder * Add intermediate_dim Arg * Format the code * Remove unnecessery comment * Use keras gelu * Remove MLP layer and implement it inside BloomDecoder * Split q k v heads * Remove shapes comments * Revert "Split q k v heads" This reverts commit 2d03d2c0e872b5e26f9e96b1cc5d610e5a306289. * Revert "Revert "Split q k v heads"" This reverts commit 531b1ff64f9e4d30515dfaa19345db4677c87e15. * Revert "Remove shapes comments" This reverts commit 2eeb5f48a7e6d30b66a1fa61df449d2d39f33950. * Add bias axes * Add bias axes to the correct axes * Update conversion script for splitting q,k,v * format the code * Rename _dropout -> _dropout_layer * use clone initializer instead of paasing str name * Serialize kernal & bais initializers * Format the code * Add alibi_bias_max to _build_alibi_tensor function * Format the code * Lowercase vairiable names --- keras_nlp/models/__init__.py | 1 + keras_nlp/models/bloom/__init__.py | 13 ++ keras_nlp/models/bloom/bloom_attention.py | 212 ++++++++++++++++++ keras_nlp/models/bloom/bloom_backbone.py | 153 +++++++++++++ keras_nlp/models/bloom/bloom_backbone_test.py | 51 +++++ keras_nlp/models/bloom/bloom_decoder.py | 204 +++++++++++++++++ .../convert_bloom_checkpoints.py | 195 ++++++++++++++++ 7 files changed, 829 insertions(+) create mode 100644 keras_nlp/models/bloom/__init__.py create mode 100644 keras_nlp/models/bloom/bloom_attention.py create mode 100644 keras_nlp/models/bloom/bloom_backbone.py create mode 100644 keras_nlp/models/bloom/bloom_backbone_test.py create mode 100644 keras_nlp/models/bloom/bloom_decoder.py create mode 100644 tools/checkpoint_conversion/convert_bloom_checkpoints.py diff --git a/keras_nlp/models/__init__.py b/keras_nlp/models/__init__.py index ab04d8eae0..30736594d0 100644 --- a/keras_nlp/models/__init__.py +++ b/keras_nlp/models/__init__.py @@ -35,6 +35,7 @@ ) from keras_nlp.models.bert.bert_preprocessor import BertPreprocessor from keras_nlp.models.bert.bert_tokenizer import BertTokenizer +from keras_nlp.models.bloom.bloom_backbone import BloomBackbone from keras_nlp.models.deberta_v3.deberta_v3_backbone import DebertaV3Backbone from keras_nlp.models.deberta_v3.deberta_v3_classifier import ( DebertaV3Classifier, diff --git a/keras_nlp/models/bloom/__init__.py b/keras_nlp/models/bloom/__init__.py new file mode 100644 index 0000000000..ba0c2545e4 --- /dev/null +++ b/keras_nlp/models/bloom/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2023 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/keras_nlp/models/bloom/bloom_attention.py b/keras_nlp/models/bloom/bloom_attention.py new file mode 100644 index 0000000000..7af2e7a34d --- /dev/null +++ b/keras_nlp/models/bloom/bloom_attention.py @@ -0,0 +1,212 @@ +# Copyright 2023 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import math + +from keras_nlp.backend import keras +from keras_nlp.backend import ops +from keras_nlp.utils.keras_utils import clone_initializer + + +class BloomAttention(keras.layers.Layer): + def __init__( + self, + num_heads, + dropout=0.0, + kernel_initializer="glorot_uniform", + bias_initializer="zeros", + **kwargs, + ): + super().__init__(**kwargs) + self.num_heads = num_heads + self.dropout = dropout + self.kernel_initializer = keras.initializers.get(kernel_initializer) + self.bias_initializer = keras.initializers.get(bias_initializer) + + def build(self, inputs_shape): + batch_size, seq_length, hidden_dim = inputs_shape + + self.head_dim = hidden_dim // self.num_heads + + # Layer-wise attention scaling + self.inv_norm_factor = 1.0 / math.sqrt(self.head_dim) + + self._query_dense = keras.layers.EinsumDense( + equation="btm,mnh->btnh", + output_shape=(None, self.num_heads, self.head_dim), + bias_axes="nh", + kernel_initializer=clone_initializer(self.kernel_initializer), + bias_initializer=clone_initializer(self.bias_initializer), + dtype=self.dtype_policy, + name="query_dense", + ) + self._query_dense.build(inputs_shape) + + self._key_dense = keras.layers.EinsumDense( + equation="bsm,mnh->bsnh", + output_shape=(None, self.num_heads, self.head_dim), + bias_axes="nh", + kernel_initializer=clone_initializer(self.kernel_initializer), + bias_initializer=clone_initializer(self.bias_initializer), + dtype=self.dtype_policy, + name="key_dense", + ) + self._key_dense.build(inputs_shape) + + self._value_dense = keras.layers.EinsumDense( + equation="bsm,mnh->bsnh", + output_shape=(None, self.num_heads, self.head_dim), + bias_axes="nh", + kernel_initializer=clone_initializer(self.kernel_initializer), + bias_initializer=clone_initializer(self.bias_initializer), + dtype=self.dtype_policy, + name="value_dense", + ) + self._value_dense.build(inputs_shape) + + self._output_dense = keras.layers.Dense( + hidden_dim, + kernel_initializer=clone_initializer(self.kernel_initializer), + bias_initializer=clone_initializer(self.bias_initializer), + dtype=self.dtype_policy, + name="output_dense", + ) + self._output_dense.build(inputs_shape) + + self._dropout_layer = keras.layers.Dropout( + rate=self.dropout, dtype=self.dtype_policy, name="dropout" + ) + self._softmax = keras.layers.Softmax( + dtype=self.dtype_policy, name="softmax" + ) + + self.built = True + + @staticmethod + def _build_alibi_tensor(num_heads, seq_length, alibi_bias_max=8): + # this function is adopted from fairseq + # https://github.com/ofirpress/attention_with_linear_biases/blob/a35aaca144e0eb6b789dfcb46784c4b8e31b7983/fairseq/models/transformer.py#L742 + def get_slopes(n): + def get_slopes_power_of_2(n): + start = 2 ** ( + -(2 ** -(math.log2(n) - math.log2(alibi_bias_max))) + ) + ratio = start + return [start * ratio**i for i in range(n)] + + if math.log2(n).is_integer(): + return get_slopes_power_of_2(n) + else: + closest_power_of_2 = 2 ** math.floor(math.log2(n)) + return ( + get_slopes_power_of_2(closest_power_of_2) + + get_slopes(2 * closest_power_of_2)[0::2][ + : n - closest_power_of_2 + ] + ) + + slopes = ops.convert_to_tensor(get_slopes(num_heads), dtype=float) + slopes = ops.expand_dims(slopes, 1) + + alibi = slopes * ops.expand_dims(ops.arange(seq_length, dtype=float), 0) + alibi = ops.expand_dims(alibi, 1) + alibi = ops.expand_dims(alibi, 0) + + return alibi + + def call( + self, + hidden_states, + attention_mask=None, + cache=None, + cache_update_index=None, + ): + batch_size, seq_length, hidden_dim = ops.shape(hidden_states) + + query = self._query_dense(hidden_states) + key = self._key_dense(hidden_states) + value = self._value_dense(hidden_states) + + if cache is not None: + key_cache = cache[:, 0, ...] + value_cache = cache[:, 1, ...] + if cache_update_index is None: + key = key_cache + value = value_cache + else: + start = [0, cache_update_index, 0, 0] + key = ops.slice_update(key_cache, start, key) + value = ops.slice_update(value_cache, start, value) + cache = ops.stack((key, value), axis=1) + else: + if cache_update_index is not None: + raise ValueError( + "`cache_update_index` should not be set if `cache` is " + f"`None`. Received: cache={cache}, " + f"cache_update_index={cache_update_index}" + ) + + # query (batch_size, num_heads, query_length, head_dim) + query = ops.transpose(query, [0, 2, 1, 3]) + # value (batch_size, num_heads, kv_length, head_dim) + value = ops.transpose(value, [0, 2, 1, 3]) + # key (batch_size, num_heads, head_dim, kv_length) + key = ops.transpose(key, [0, 2, 3, 1]) + + alibi = self._build_alibi_tensor( + num_heads=self.num_heads, seq_length=seq_length + ) + + scores = ( + ops.matmul(query, key) * self.inv_norm_factor + alibi + ) # [batch_size, num_heads, query_length, kv_length] + + scores = self._softmax(scores, ops.expand_dims(attention_mask, 1)) + + scores = self._dropout_layer(scores) + + attention_output = ops.matmul( + scores, value + ) # [batch_size, num_heads, query_length, head_dim] + + attention_output = ops.transpose( + attention_output, [0, 2, 1, 3] + ) # [batch_size, query_length, num_heads, head_dim] + attention_output = ops.reshape( + attention_output, + [batch_size, seq_length, self.num_heads * self.head_dim], + ) # [batch_size, query_length, hidden_dim] + + attention_output = self._output_dense(attention_output) + attention_output = self._dropout_layer(attention_output) + + if cache is not None: + return attention_output, cache + + return attention_output + + def get_config(self): + config = super().get_config() + config.update( + { + "num_heads": self.num_heads, + "dropout": self.dropout, + "kernel_initializer": keras.initializers.serialize( + self.kernel_initializer + ), + "bias_initializer": keras.initializers.serialize( + self.bias_initializer + ), + } + ) + return config diff --git a/keras_nlp/models/bloom/bloom_backbone.py b/keras_nlp/models/bloom/bloom_backbone.py new file mode 100644 index 0000000000..e3d66998bc --- /dev/null +++ b/keras_nlp/models/bloom/bloom_backbone.py @@ -0,0 +1,153 @@ +# Copyright 2023 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from keras_nlp.api_export import keras_nlp_export +from keras_nlp.backend import keras +from keras_nlp.layers.modeling.reversible_embedding import ReversibleEmbedding +from keras_nlp.models.backbone import Backbone +from keras_nlp.models.bloom.bloom_decoder import BloomDecoder + + +def _bloom_kernel_initializer(stddev=0.02): + return keras.initializers.RandomNormal(stddev=stddev) + + +@keras_nlp_export("keras_nlp.models.BloomBackbone") +class BloomBackbone(Backbone): + """A Bloom decoder network. + + This network implements a Transformer-based decoder network, BigScience + Language Open-science Open-access Multilingual (BLOOM), as descriped in + ["BLOOM: A 176B-Parameter Open-Access Multilingual Language Model"](https://arxiv.org/pdf/2211.05100.pdf). + + The default constructor gives a fully customizable, randomly initialized + Bloom model with any number of layers, heads, and embedding dimensions. To + load preset architectures and weights, use the `from_preset()` constructor. + + Disclaimer: Pre-trained models are provided on an "as is" basis, without + warranties or conditions of any kind. + + Args: + vocabulary_size: int. The size of the token vocabulary. + num_layers: int. The number of transformer layers. + num_heads: int. The number of attention heads for each transformer. + The hidden size must be divisible by the number of attention heads. + hidden_dim: int. The dimensionality of the embeddings and hidden states. + intermediate_dim: int. The output dimension of the first Dense layer in + the MLP network of each transformer. + dropout: float. Dropout probability for the Transformer decoder. + layer_norm_epsilon: float. Epsilon for the layer normalization layers in + the transformer decoder. + max_sequence_length: int. The maximum sequence length that this decoder + can consume. + + Examples: + ```python + input_data = { + "token_ids": np.ones(shape=(1, 12), dtype="int32"), + "padding_mask": np.array([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0]]), + } + + # Randomly initialized BLOOM decoder with a custom config. + model = keras_nlp.models.BloomBackbone( + vocabulary_size=10, + num_layers=2, + num_heads=2, + hidden_dim=32, + intermediate_dim=32*4, + dropout=0.0, + layer_norm_epsilon=1e-5, + max_sequence_length=128, + ) + model(input_data) + ``` + + """ + + def __init__( + self, + vocabulary_size, + num_layers, + num_heads, + hidden_dim, + intermediate_dim, + dropout=0.0, + layer_norm_epsilon=1e-5, + max_sequence_length=512, + **kwargs, + ): + token_ids = keras.Input(shape=(None,), dtype="int32", name="token_ids") + padding_mask = keras.Input( + shape=(None,), dtype="int32", name="padding_mask" + ) + + # Embed tokens + token_embedding_layer = ReversibleEmbedding( + input_dim=vocabulary_size, + output_dim=hidden_dim, + embeddings_initializer=_bloom_kernel_initializer(stddev=0.02), + tie_weights=False, + name="token_embedding", + ) + token_embedding = token_embedding_layer(token_ids) + + x = keras.layers.LayerNormalization( + epsilon=layer_norm_epsilon, name="token_embedding_layernorm" + )(token_embedding) + + for i in range(num_layers): + x = BloomDecoder( + num_heads=num_heads, + intermediate_dim=intermediate_dim, + dropout=dropout, + layer_norm_epsilon=layer_norm_epsilon, + name=f"transformer_layer_{i}", + )(x, decoder_padding_mask=padding_mask) + + sequence_output = keras.layers.LayerNormalization( + epsilon=layer_norm_epsilon, name="final_layernorm" + )(x) + + super().__init__( + inputs={ + "token_ids": token_ids, + "padding_mask": padding_mask, + }, + outputs=sequence_output, + **kwargs, + ) + self.vocabulary_size = vocabulary_size + self.num_layers = num_layers + self.num_heads = num_heads + self.hidden_dim = hidden_dim + self.intermediate_dim = intermediate_dim + self.dropout = dropout + self.layer_norm_epsilon = layer_norm_epsilon + self.max_sequence_length = max_sequence_length + self.token_embedding = token_embedding_layer + + def get_config(self): + config = super().get_config() + config.update( + { + "vocabulary_size": self.vocabulary_size, + "num_layers": self.num_layers, + "num_heads": self.num_heads, + "hidden_dim": self.hidden_dim, + "intermediate_dim": self.intermediate_dim, + "dropout": self.dropout, + "layer_norm_epsilon": self.layer_norm_epsilon, + "max_sequence_length": self.max_sequence_length, + } + ) + return config diff --git a/keras_nlp/models/bloom/bloom_backbone_test.py b/keras_nlp/models/bloom/bloom_backbone_test.py new file mode 100644 index 0000000000..99cb9d357e --- /dev/null +++ b/keras_nlp/models/bloom/bloom_backbone_test.py @@ -0,0 +1,51 @@ +# Copyright 2023 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from keras_nlp.backend import ops +from keras_nlp.models.bloom.bloom_backbone import BloomBackbone +from keras_nlp.tests.test_case import TestCase + + +class BloomTest(TestCase): + def setUp(self): + self.init_kwargs = { + "vocabulary_size": 10, + "num_layers": 2, + "num_heads": 4, + "hidden_dim": 8, + "intermediate_dim": 32, + "max_sequence_length": 10, + } + self.input_data = { + "token_ids": ops.ones((2, 5), dtype="int32"), + "padding_mask": ops.ones((2, 5), dtype="int32"), + } + + def test_backbone_basics(self): + self.run_backbone_test( + cls=BloomBackbone, + init_kwargs=self.init_kwargs, + input_data=self.input_data, + expected_output_shape=(2, 5, 8), + ) + + @pytest.mark.large + def test_saved_model(self): + self.run_model_saving_test( + cls=BloomBackbone, + init_kwargs=self.init_kwargs, + input_data=self.input_data, + ) diff --git a/keras_nlp/models/bloom/bloom_decoder.py b/keras_nlp/models/bloom/bloom_decoder.py new file mode 100644 index 0000000000..b3f8b80da7 --- /dev/null +++ b/keras_nlp/models/bloom/bloom_decoder.py @@ -0,0 +1,204 @@ +# Copyright 2023 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# from keras_nlp.backend import keras +# from keras_nlp.backend import ops +from keras_nlp.backend import keras +from keras_nlp.backend import ops +from keras_nlp.layers.modeling.transformer_layer_utils import ( + compute_causal_mask, +) +from keras_nlp.layers.modeling.transformer_layer_utils import ( + merge_padding_and_attention_mask, +) +from keras_nlp.models.bloom.bloom_attention import BloomAttention +from keras_nlp.utils.keras_utils import clone_initializer + + +class BloomDecoder(keras.layers.Layer): + def __init__( + self, + num_heads, + intermediate_dim, + dropout=0.0, + layer_norm_epsilon=1e-5, + kernel_initializer="glorot_uniform", + bias_initializer="zeros", + **kwargs, + ): + super().__init__(**kwargs) + + self.num_heads = num_heads + self.intermediate_dim = intermediate_dim + self.dropout = dropout + self.layer_norm_epsilon = layer_norm_epsilon + self.kernel_initializer = keras.initializers.get(kernel_initializer) + self.bias_initializer = keras.initializers.get(bias_initializer) + + def build(self, decoder_sequence_shape): + hidden_dim = decoder_sequence_shape[-1] + head_dim = int(hidden_dim // self.num_heads) + + if head_dim * self.num_heads != hidden_dim: + raise ValueError( + f"`hidden_dim` must be divisible by num_heads (got `hidden_dim`" + f": {hidden_dim} and `num_heads`: {self.num_heads})." + ) + + self._pre_attention_layernorm = keras.layers.LayerNormalization( + epsilon=self.layer_norm_epsilon, + dtype=self.dtype_policy, + name="pre_attention_layernorm", + ) + self._pre_attention_layernorm.build(decoder_sequence_shape) + + self._self_attention_layer = BloomAttention( + num_heads=self.num_heads, + dropout=self.dropout, + kernel_initializer=clone_initializer(self.kernel_initializer), + bias_initializer=clone_initializer(self.bias_initializer), + dtype=self.dtype_policy, + name="self_attention", + ) + self._self_attention_layer.build(decoder_sequence_shape) + + self._post_attention_layernorm = keras.layers.LayerNormalization( + epsilon=self.layer_norm_epsilon, + dtype=self.dtype_policy, + name="post_attention_layernorm", + ) + self._post_attention_layernorm.build(decoder_sequence_shape) + + self._mlp_intermediate_dense = keras.layers.Dense( + self.intermediate_dim, + kernel_initializer=clone_initializer(self.kernel_initializer), + bias_initializer=clone_initializer(self.bias_initializer), + dtype=self.dtype_policy, + name="mlp_intermediate_dense", + ) + self._mlp_intermediate_dense.build(decoder_sequence_shape) + + self._mlp_output_dense = keras.layers.Dense( + hidden_dim, + kernel_initializer=clone_initializer(self.kernel_initializer), + bias_initializer=clone_initializer(self.bias_initializer), + dtype=self.dtype_policy, + name="mlp_output_dense", + ) + intermediate_shape = list(decoder_sequence_shape) + intermediate_shape[-1] = self.intermediate_dim + self._mlp_output_dense.build(tuple(intermediate_shape)) + + self._dropout_layer = keras.layers.Dropout( + rate=self.dropout, dtype=self.dtype_policy, name="dropout" + ) + + self.built = True + + def call( + self, + decoder_sequence, + decoder_padding_mask=None, + decoder_attention_mask=None, + attention_cache=None, + attention_cache_update_index=None, + use_causal_mask=True, + ): + self_attention_mask = self._compute_attention_mask( + decoder_sequence=decoder_sequence, + decoder_padding_mask=decoder_padding_mask, + decoder_attention_mask=decoder_attention_mask, + use_causal_mask=use_causal_mask, + attention_cache=attention_cache, + attention_cache_update_index=attention_cache_update_index, + ) + + residual = decoder_sequence + x = self._pre_attention_layernorm(decoder_sequence) + + attention_output = self._self_attention_layer( + hidden_states=x, + attention_mask=self_attention_mask, + cache=attention_cache, + cache_update_index=attention_cache_update_index, + ) + + if attention_cache is None: + x = attention_output + else: + x, attention_cache = attention_output + + x = x + residual + residual = x + x = self._post_attention_layernorm(x) + x = self._mlp_intermediate_dense(x) + x = keras.activations.gelu(x, approximate=True) + x = self._mlp_output_dense(x) + x = self._dropout_layer(x) + x = x + residual + + if attention_cache is not None: + return x, attention_cache + else: + return x + + def _compute_attention_mask( + self, + decoder_sequence, + decoder_padding_mask, + decoder_attention_mask, + use_causal_mask, + attention_cache, + attention_cache_update_index, + ): + decoder_mask = merge_padding_and_attention_mask( + decoder_sequence, decoder_padding_mask, decoder_attention_mask + ) + if use_causal_mask: + batch_size = ops.shape(decoder_sequence)[0] + input_length = output_length = ops.shape(decoder_sequence)[1] + if attention_cache is not None: + input_length = ops.shape(attention_cache)[2] + + causal_mask = compute_causal_mask( + batch_size, + input_length, + output_length, + 0 + if attention_cache_update_index is None + else attention_cache_update_index, + ) + return ( + ops.minimum(decoder_mask, causal_mask) + if decoder_mask is not None + else causal_mask + ) + return decoder_mask + + def get_config(self): + config = super().get_config() + config.update( + { + "num_heads": self.num_heads, + "intermediate_dim": self.intermediate_dim, + "dropout": self.dropout, + "layer_norm_epsilon": self.layer_norm_epsilon, + "kernel_initializer": keras.initializers.serialize( + self.kernel_initializer + ), + "bias_initializer": keras.initializers.serialize( + self.bias_initializer + ), + } + ) + return config diff --git a/tools/checkpoint_conversion/convert_bloom_checkpoints.py b/tools/checkpoint_conversion/convert_bloom_checkpoints.py new file mode 100644 index 0000000000..1fc895c54c --- /dev/null +++ b/tools/checkpoint_conversion/convert_bloom_checkpoints.py @@ -0,0 +1,195 @@ +# Copyright 2023 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import torch +import transformers +from absl import app +from absl import flags +from checkpoint_conversion_utils import get_md5_checksum + +from keras_nlp.models.bloom.bloom_backbone import BloomBackbone + +FLAGS = flags.FLAGS + +PRESET_MAP = { + "bloom_tiny": "bigscience/bloom-560m", + "bloom_extra_small": "bigscience/bloom-1b1", + "bloom_small": "bigscience/bloom-1b7", + "bloom_meduim": "bigscience/bloom-3b", + "bloom_large": "bigscience/bloom-7b1", + "bloom_extra_large": "bigscience/bloom", +} + +flags.DEFINE_string( + "preset", None, f'Must be one of {",".join(PRESET_MAP.keys())}' +) +flags.mark_flag_as_required("preset") + + +def convert_checkpoints(hf_model): + # get huggingface model configuration. + hf_config = hf_model.config.to_dict() + + cfg = {} + cfg["vocabulary_size"] = hf_config["vocab_size"] + cfg["num_layers"] = hf_config["n_layer"] + cfg["num_heads"] = hf_config["n_head"] + cfg["hidden_dim"] = hf_config["hidden_size"] + cfg["intermediate_dim"] = hf_config["hidden_size"] * 4 + cfg["dropout"] = hf_config["hidden_dropout"] + cfg["layer_norm_epsilon"] = hf_config["layer_norm_epsilon"] + + hidden_dim = cfg["hidden_dim"] + num_heads = cfg["num_heads"] + head_dim = hidden_dim // num_heads + + # Intialize Bloom model with the weights. + keras_model = BloomBackbone(**cfg) + + # get huggingface model weights. + hf_wts = hf_model.state_dict() + + # assign huggingface weights to the keras model. + # Embedding layer. + keras_model.get_layer("token_embedding").embeddings.assign( + hf_wts["word_embeddings.weight"] + ) + # LayerNorm. + keras_model.get_layer("token_embedding_layernorm").gamma.assign( + hf_wts["word_embeddings_layernorm.weight"] + ) + keras_model.get_layer("token_embedding_layernorm").beta.assign( + hf_wts["word_embeddings_layernorm.bias"] + ) + + keras_model.get_layer("final_layernorm").gamma.assign(hf_wts["ln_f.weight"]) + keras_model.get_layer("final_layernorm").beta.assign(hf_wts["ln_f.bias"]) + + # Decoder layers. + for i in range(cfg["num_layers"]): + decoder_layer = keras_model.get_layer(f"transformer_layer_{i}") + # LayrNorm. + decoder_layer._pre_attention_layernorm.gamma.assign( + hf_wts[f"h.{i}.input_layernorm.weight"] + ) + decoder_layer._pre_attention_layernorm.beta.assign( + hf_wts[f"h.{i}.input_layernorm.bias"] + ) + decoder_layer._post_attention_layernorm.gamma.assign( + hf_wts[f"h.{i}.post_attention_layernorm.weight"] + ) + decoder_layer._post_attention_layernorm.beta.assign( + hf_wts[f"h.{i}.post_attention_layernorm.bias"] + ) + + # Attention layer. + attention_layer = decoder_layer._self_attention_layer + + fused_qkv_kernal = hf_wts[ + f"h.{i}.self_attention.query_key_value.weight" + ].T + fused_qkv_kernal = fused_qkv_kernal.view( + hidden_dim, num_heads, 3, head_dim + ) + query_kernal = fused_qkv_kernal[..., 0, :] + key_kernal = fused_qkv_kernal[..., 1, :] + value_kernl = fused_qkv_kernal[..., 2, :] + + fused_qkv_bais = hf_wts[f"h.{i}.self_attention.query_key_value.bias"] + fused_qkv_bais = fused_qkv_bais.view(num_heads, 3, head_dim) + query_bais = fused_qkv_bais[:, 0, :] + key_bais = fused_qkv_bais[:, 1, :] + value_bais = fused_qkv_bais[:, 2, :] + + attention_layer._query_dense.kernel.assign(query_kernal) + attention_layer._query_dense.bias.assign(query_bais) + attention_layer._key_dense.kernel.assign(key_kernal) + attention_layer._key_dense.bias.assign(key_bais) + attention_layer._value_dense.kernel.assign(value_kernl) + attention_layer._value_dense.bias.assign(value_bais) + + attention_layer._output_dense.kernel.assign( + hf_wts[f"h.{i}.self_attention.dense.weight"].T + ) + attention_layer._output_dense.bias.assign( + hf_wts[f"h.{i}.self_attention.dense.bias"] + ) + + # mlp. + decoder_layer._mlp_intermediate_dense.kernel.assign( + hf_wts[f"h.{i}.mlp.dense_h_to_4h.weight"].T + ) + decoder_layer._mlp_intermediate_dense.bias.assign( + hf_wts[f"h.{i}.mlp.dense_h_to_4h.bias"] + ) + decoder_layer._mlp_output_dense.kernel.assign( + hf_wts[f"h.{i}.mlp.dense_4h_to_h.weight"].T + ) + decoder_layer._mlp_output_dense.bias.assign( + hf_wts[f"h.{i}.mlp.dense_4h_to_h.bias"] + ) + + # Save the model. + print(f"\n-> Saving KerasNLP model weights to `{FLAGS.preset}.weights.h5`.") + keras_model.save_weights(f"{FLAGS.preset}.weights.h5") + + return keras_model + + +def check_output(keras_model, hf_model): + hf_model_input = { + "input_ids": torch.tensor([[59414, 15, 2670, 35433, 632, 207595]]), + "attention_mask": torch.tensor([[1, 1, 1, 1, 1, 1]]), + } + + hf_model_outputs = hf_model(**hf_model_input) + hf_model_outputs = hf_model_outputs.last_hidden_state + hf_model_outputs = hf_model_outputs.detach().numpy() + + keras_model_input = { + "token_ids": torch.tensor([[59414, 15, 2670, 35433, 632, 207595]]), + "padding_mask": torch.tensor([[1, 1, 1, 1, 1, 1]]), + } + + keras_model_outputs = keras_model.predict(keras_model_input) + + # Comparing the outputs. + print("KerasNLP output:", keras_model_outputs[0, 0, :10]) + print("HF output:", hf_model_outputs[0, 0, :10]) + print("Difference:", np.mean(keras_model_outputs - hf_model_outputs)) + + # Show the MD5 checksum of the model weights. + print("Model md5sum: ", get_md5_checksum(f"./{FLAGS.preset}.weights.h5")) + + +def main(_): + assert ( + FLAGS.preset in PRESET_MAP.keys() + ), f'Invalid preset {FLAGS.preset}. Must be one of {",".join(PRESET_MAP.keys())}' + + hf_model_name = PRESET_MAP[FLAGS.preset] + + print("\n-> Loading HF model.") + hf_model = transformers.AutoModel.from_pretrained(hf_model_name) + + print("\n-> Converting model checkpoint.") + keras_model = convert_checkpoints(hf_model) + + print("\n-> Checking keras model output.") + check_output(keras_model, hf_model) + + +if __name__ == "__main__": + app.run(main) From 0ce32ca4a403a2596c52ed61e17c0e54a68f4285 Mon Sep 17 00:00:00 2001 From: Matt Watson <1389937+mattdangerw@users.noreply.github.com> Date: Wed, 24 Jan 2024 22:09:27 -0800 Subject: [PATCH 05/29] Try fixing tests (#1411) --- keras_nlp/tests/test_case.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/keras_nlp/tests/test_case.py b/keras_nlp/tests/test_case.py index 455a8569b7..1010262853 100644 --- a/keras_nlp/tests/test_case.py +++ b/keras_nlp/tests/test_case.py @@ -142,6 +142,12 @@ def call(self, x): else: return self.layer(x) + input_data = tree.map_structure( + lambda x: ops.convert_to_numpy(x), input_data + ) + output_data = tree.map_structure( + lambda x: ops.convert_to_numpy(x), output_data + ) model = TestModel(layer) # Temporarily disable jit compilation on torch backend. jit_compile = config.backend() != "torch" From f897134c2b3f78092f5f0bd286042c6c89942a77 Mon Sep 17 00:00:00 2001 From: Matt Watson <1389937+mattdangerw@users.noreply.github.com> Date: Thu, 25 Jan 2024 00:43:53 -0800 Subject: [PATCH 06/29] Revert "Pass less state to jax generate function (#1398)" (#1412) This reverts commit c49bf9bf55b04cf10c6b448f7d6949cdb3551d32. --- keras_nlp/models/generative_task.py | 33 +++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/keras_nlp/models/generative_task.py b/keras_nlp/models/generative_task.py index 14d4c88f2f..9a461926e4 100644 --- a/keras_nlp/models/generative_task.py +++ b/keras_nlp/models/generative_task.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import itertools + import tensorflow as tf import tree @@ -80,18 +82,30 @@ def wrapped_generate_function( @jax.jit def compiled_generate_function(inputs, end_token_id, state): - # The only state we update during generation is sampler state, - # all weights are fixed and will not change. - mapping = zip(self._sampler.variables, state) + ( + sampler_variables, + trainable_variables, + non_trainable_variables, + ) = state + mapping = itertools.chain( + zip(self._sampler.variables, sampler_variables), + zip(self.trainable_variables, trainable_variables), + zip(self.non_trainable_variables, non_trainable_variables), + ) with keras.StatelessScope(state_mapping=mapping) as scope: outputs = self.generate_step(inputs, end_token_id) # Get updated sampler variables from the stateless scope. - state = [] + sampler_variables = [] for v in self._sampler.variables: new_v = scope.get_current_value(v) - state.append(new_v if new_v is not None else v) + sampler_variables.append(new_v if new_v is not None else v) + state = ( + sampler_variables, + trainable_variables, + non_trainable_variables, + ) return outputs, state def wrapped_generate_function( @@ -99,15 +113,20 @@ def wrapped_generate_function( end_token_id=None, ): # Create an explicit tuple of all variable state. + state = ( + self._sampler.variables, + self.trainable_variables, + self.non_trainable_variables, + ) inputs = tree.map_structure(ops.convert_to_tensor, inputs) outputs, state = compiled_generate_function( inputs, end_token_id, - self._sampler.variables, + state, ) # Only assign the sampler variables (random seeds), as other # model variables should never be updated in generation. - for ref_v, v in zip(self._sampler.variables, state): + for ref_v, v in zip(self._sampler.variables, state[0]): ref_v.assign(v) return outputs From b76c149fac6e0fd321e25a115cc499df1634f8f0 Mon Sep 17 00:00:00 2001 From: Mohamed Abu El-Nasr <64566340+abuelnasr0@users.noreply.github.com> Date: Fri, 26 Jan 2024 03:22:33 +0200 Subject: [PATCH 07/29] Bloom tokenizer (#1403) * Add Tokenizer and new conversion script * Add bloom tokenizer to init * Convert weights * check backbone output from bloom tokenizer * Convert inputs into tensors * Fix backbone input * convert backbone input to torch tensor * Change presets naming * Change presets naming * Comment unused code for now * Format the code * change tokenizer example * Add a preset and initial for tests * Add backbone preset test * Add tokenizer test * Format the code * Fix tokenizer test * Rename BloomBackboneTest * Small Fixes * Fix tokenizer example * Add example for backbone * Update preset to Keras --------- Co-authored-by: Matt Watson --- keras_nlp/models/__init__.py | 1 + keras_nlp/models/bloom/bloom_backbone.py | 15 +- keras_nlp/models/bloom/bloom_backbone_test.py | 27 ++- keras_nlp/models/bloom/bloom_presets.py | 30 +++ keras_nlp/models/bloom/bloom_tokenizer.py | 123 ++++++++++++ .../models/bloom/bloom_tokenizer_test.py | 63 +++++++ .../convert_bloom_checkpoints.py | 177 ++++++++++++------ 7 files changed, 372 insertions(+), 64 deletions(-) create mode 100644 keras_nlp/models/bloom/bloom_presets.py create mode 100644 keras_nlp/models/bloom/bloom_tokenizer.py create mode 100644 keras_nlp/models/bloom/bloom_tokenizer_test.py diff --git a/keras_nlp/models/__init__.py b/keras_nlp/models/__init__.py index 30736594d0..7b898138fd 100644 --- a/keras_nlp/models/__init__.py +++ b/keras_nlp/models/__init__.py @@ -36,6 +36,7 @@ from keras_nlp.models.bert.bert_preprocessor import BertPreprocessor from keras_nlp.models.bert.bert_tokenizer import BertTokenizer from keras_nlp.models.bloom.bloom_backbone import BloomBackbone +from keras_nlp.models.bloom.bloom_tokenizer import BloomTokenizer from keras_nlp.models.deberta_v3.deberta_v3_backbone import DebertaV3Backbone from keras_nlp.models.deberta_v3.deberta_v3_classifier import ( DebertaV3Classifier, diff --git a/keras_nlp/models/bloom/bloom_backbone.py b/keras_nlp/models/bloom/bloom_backbone.py index e3d66998bc..2fd18ed760 100644 --- a/keras_nlp/models/bloom/bloom_backbone.py +++ b/keras_nlp/models/bloom/bloom_backbone.py @@ -11,11 +11,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import copy + from keras_nlp.api_export import keras_nlp_export from keras_nlp.backend import keras from keras_nlp.layers.modeling.reversible_embedding import ReversibleEmbedding from keras_nlp.models.backbone import Backbone from keras_nlp.models.bloom.bloom_decoder import BloomDecoder +from keras_nlp.models.bloom.bloom_presets import backbone_presets +from keras_nlp.utils.python_utils import classproperty def _bloom_kernel_initializer(stddev=0.02): @@ -35,7 +39,8 @@ class BloomBackbone(Backbone): load preset architectures and weights, use the `from_preset()` constructor. Disclaimer: Pre-trained models are provided on an "as is" basis, without - warranties or conditions of any kind. + warranties or conditions of any kind. The underlying model is provided by a + third party and subject to a separate license, available [here](https://huggingface.co/spaces/bigscience/license). Args: vocabulary_size: int. The size of the token vocabulary. @@ -58,6 +63,10 @@ class BloomBackbone(Backbone): "padding_mask": np.array([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0]]), } + # Pretrained BLOOM decoder. + model = keras_nlp.models.BloomBackbone.from_preset("bloom_560m_multi") + model(input_data) + # Randomly initialized BLOOM decoder with a custom config. model = keras_nlp.models.BloomBackbone( vocabulary_size=10, @@ -151,3 +160,7 @@ def get_config(self): } ) return config + + @classproperty + def presets(cls): + return copy.deepcopy(backbone_presets) diff --git a/keras_nlp/models/bloom/bloom_backbone_test.py b/keras_nlp/models/bloom/bloom_backbone_test.py index 99cb9d357e..83732e4945 100644 --- a/keras_nlp/models/bloom/bloom_backbone_test.py +++ b/keras_nlp/models/bloom/bloom_backbone_test.py @@ -19,7 +19,7 @@ from keras_nlp.tests.test_case import TestCase -class BloomTest(TestCase): +class BloomBackboneTest(TestCase): def setUp(self): self.init_kwargs = { "vocabulary_size": 10, @@ -49,3 +49,28 @@ def test_saved_model(self): init_kwargs=self.init_kwargs, input_data=self.input_data, ) + + @pytest.mark.large + def test_smallest_preset(self): + self.run_preset_test( + cls=BloomBackbone, + preset="bloom_560m_multi", + input_data={ + "token_ids": ops.array([[101, 1996, 4248, 102]], dtype="int32"), + "padding_mask": ops.ones((1, 4), dtype="int32"), + }, + expected_output_shape=(1, 4, 1024), + # The forward pass from a preset should be stable! + expected_partial_output=ops.array( + [2.4394186, 1.4131186, -2.7810357, -6.330823, -1.0599766] + ), + ) + + @pytest.mark.extra_large + def test_all_presets(self): + for preset in BloomBackbone.presets: + self.run_preset_test( + cls=BloomBackbone, + preset=preset, + input_data=self.input_data, + ) diff --git a/keras_nlp/models/bloom/bloom_presets.py b/keras_nlp/models/bloom/bloom_presets.py new file mode 100644 index 0000000000..d3e9c780c0 --- /dev/null +++ b/keras_nlp/models/bloom/bloom_presets.py @@ -0,0 +1,30 @@ +# Copyright 2023 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""BLOOM model preset configurations.""" + +backbone_presets = { + "bloom_560m_multi": { + "metadata": { + "description": ( + "24-layer Bloom model. trained on 45 natural languages and " + "12 programming languages." + ), + "params": 816115712, + "official_name": "BLOOM", + "path": "bloom", + "model_card": "https://huggingface.co/bigscience/bloom", + }, + "kaggle_handle": "kaggle://keras/bloom/keras/bloom_560m_multi/1", + }, +} diff --git a/keras_nlp/models/bloom/bloom_tokenizer.py b/keras_nlp/models/bloom/bloom_tokenizer.py new file mode 100644 index 0000000000..cc3fcc2fc3 --- /dev/null +++ b/keras_nlp/models/bloom/bloom_tokenizer.py @@ -0,0 +1,123 @@ +# Copyright 2023 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy + +from keras_nlp.api_export import keras_nlp_export +from keras_nlp.models.bloom.bloom_presets import backbone_presets +from keras_nlp.tokenizers.byte_pair_tokenizer import BytePairTokenizer +from keras_nlp.utils.python_utils import classproperty + + +@keras_nlp_export("keras_nlp.models.BloomTokenizer") +class BloomTokenizer(BytePairTokenizer): + """A BLOOM tokenizer using Byte-Pair Encoding subword segmentation. + + This tokenizer class will tokenize raw strings into integer sequences and + is based on `keras_nlp.tokenizers.BytePairTokenizer`. Unlike the + underlying tokenizer, it will check for all special tokens needed by BLOOM + models and provides a `from_preset()` method to automatically download + a matching vocabulary for a BLOOM preset. + + This tokenizer does not provide truncation or padding of inputs. + + If input is a batch of strings (rank > 0), the layer will output a + `tf.RaggedTensor` where the last dimension of the output is ragged. + + If input is a scalar string (rank == 0), the layer will output a dense + `tf.Tensor` with static shape `[None]`. + + Args: + vocabulary: string or dict, maps token to integer ids. If it is a + string, it should be the file path to a json file. + merges: string or list, contains the merge rule. If it is a string, + it should be the file path to merge rules. The merge rule file + should have one merge rule per line. Every merge rule contains + merge entities separated by a space. + + Examples: + + ```python + # Unbatched input. + tokenizer = keras_nlp.models.BloomTokenizer.from_preset("bloom_560m_multi") + tokenizer("The quick brown fox jumped.") + + # Batched input. + tokenizer(["The quick brown fox jumped.", "The fox slept."]) + + # Detokenization. + tokenizer.detokenize(tokenizer("The quick brown fox jumped.")) + + # Custom vocabulary. + vocab = {"": 0, "": 1, "": 2, "a": 3, "Ġquick": 4, "Ġfox": 5} + merges = ["Ġ q", "u i", "c k", "ui ck", "Ġq uick"] + merges += ["Ġ f", "o x", "Ġf ox"] + tokenizer = keras_nlp.models.BloomTokenizer(vocabulary=vocab, merges=merges) + tokenizer("a quick fox.") + ``` + """ + + def __init__( + self, + vocabulary=None, + merges=None, + **kwargs, + ): + self.bos_token = "" + self.eos_token = "" + self.pad_token = "" + + super().__init__( + vocabulary=vocabulary, + merges=merges, + unsplittable_tokens=[ + self.bos_token, + self.eos_token, + self.pad_token, + ], + **kwargs, + ) + + def set_vocabulary_and_merges(self, vocabulary, merges): + super().set_vocabulary_and_merges(vocabulary, merges) + + if vocabulary is not None: + # Check for necessary special tokens. + for token in [self.bos_token, self.eos_token, self.pad_token]: + if token not in self.get_vocabulary(): + raise ValueError( + f"Cannot find token `'{token}'` in the provided " + f"`vocabulary`. Please provide `'{token}'` in " + "your `vocabulary` or use a pretrained `vocabulary` name." + ) + + self.bos_token_id = self.token_to_id(self.bos_token) + self.eos_token_id = self.token_to_id(self.eos_token) + self.pad_token_id = self.token_to_id(self.pad_token) + else: + self.bos_token_id = None + self.eos_token_id = None + self.pad_token_id = None + + @classproperty + def presets(cls): + return copy.deepcopy(backbone_presets) + + def get_config(self): + config = super().get_config() + # In the constructor, we pass the list of special tokens to the + # `unsplittable_tokens` arg of the superclass' constructor. Hence, we + # delete it from the config here. + del config["unsplittable_tokens"] + return config diff --git a/keras_nlp/models/bloom/bloom_tokenizer_test.py b/keras_nlp/models/bloom/bloom_tokenizer_test.py new file mode 100644 index 0000000000..9ae9c0cc00 --- /dev/null +++ b/keras_nlp/models/bloom/bloom_tokenizer_test.py @@ -0,0 +1,63 @@ +# Copyright 2023 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from keras_nlp.models.bloom.bloom_tokenizer import BloomTokenizer +from keras_nlp.tests.test_case import TestCase + + +class BloomTokenizerTest(TestCase): + def setUp(self): + self.vocab = ["!", "air", "Ġair", "plane", "Ġat", "port"] + self.vocab += ["", "", ""] + self.vocab = dict([(token, i) for i, token in enumerate(self.vocab)]) + self.merges = ["Ġ a", "Ġ t", "Ġ i", "Ġ b", "a i", "p l", "n e"] + self.merges += ["Ġa t", "p o", "r t", "Ġt h", "ai r", "pl a", "po rt"] + self.merges += ["Ġai r", "Ġa i", "pla ne"] + self.init_kwargs = {"vocabulary": self.vocab, "merges": self.merges} + self.input_data = [ + "airplane at airport", + " airplane airport", + ] + + def test_tokenizer_basics(self): + self.run_preprocessing_layer_test( + cls=BloomTokenizer, + init_kwargs=self.init_kwargs, + input_data=self.input_data, + expected_output=[[6, 1, 3, 4, 2, 5, 8], [6, 2, 3, 2, 5, 8]], + ) + + def test_errors_missing_special_tokens(self): + with self.assertRaises(ValueError): + BloomTokenizer(vocabulary=["a", "b", "c"], merges=[]) + + @pytest.mark.large + def test_smallest_preset(self): + self.run_preset_test( + cls=BloomTokenizer, + preset="bloom_560m_multi", + input_data=["The quick brown fox."], + expected_output=[[2175, 23714, 73173, 144252, 17]], + ) + + @pytest.mark.extra_large + def test_all_presets(self): + for preset in BloomTokenizer.presets: + self.run_preset_test( + cls=BloomTokenizer, + preset=preset, + input_data=self.input_data, + ) diff --git a/tools/checkpoint_conversion/convert_bloom_checkpoints.py b/tools/checkpoint_conversion/convert_bloom_checkpoints.py index 1fc895c54c..38acd099cf 100644 --- a/tools/checkpoint_conversion/convert_bloom_checkpoints.py +++ b/tools/checkpoint_conversion/convert_bloom_checkpoints.py @@ -12,51 +12,86 @@ # See the License for the specific language governing permissions and # limitations under the License. -import numpy as np -import torch -import transformers -from absl import app -from absl import flags -from checkpoint_conversion_utils import get_md5_checksum +import json +import os -from keras_nlp.models.bloom.bloom_backbone import BloomBackbone +os.environ["KERAS_BACKEND"] = "torch" +os.environ["CUDA_VISIBLE_DEVICES"] = "-1" + +import huggingface_hub # noqa: E402 +import numpy as np # noqa: E402 +import torch # noqa: E402 +import transformers # noqa: E402 +from absl import app # noqa: E402 +from absl import flags # noqa: E402 + +import keras_nlp # noqa: E402 +from keras_nlp.models import BloomBackbone # noqa: E402 +from keras_nlp.models import BloomTokenizer # noqa: E402 FLAGS = flags.FLAGS PRESET_MAP = { - "bloom_tiny": "bigscience/bloom-560m", - "bloom_extra_small": "bigscience/bloom-1b1", - "bloom_small": "bigscience/bloom-1b7", - "bloom_meduim": "bigscience/bloom-3b", - "bloom_large": "bigscience/bloom-7b1", - "bloom_extra_large": "bigscience/bloom", + "bloom_560m_multi": "bigscience/bloom-560m", + "bloom_1.1b_multi": "bigscience/bloom-1b1", + "bloom_1.7b_multi": "bigscience/bloom-1b7", + "bloom_3b_multi": "bigscience/bloom-3b", + "bloom_7b_multi": "bigscience/bloom-7b1", + "bloom_176b_multi": "bigscience/bloom", } +EXTRACT_DIR = "./model" + + flags.DEFINE_string( "preset", None, f'Must be one of {",".join(PRESET_MAP.keys())}' ) flags.mark_flag_as_required("preset") -def convert_checkpoints(hf_model): +def download_hf_model(hf_model_name): + hf_model_dir = huggingface_hub.snapshot_download( + repo_id=hf_model_name, + allow_patterns=["*.json", "*.bin"], + ignore_patterns=["onnx/*"], + local_dir=EXTRACT_DIR, + ) + + return hf_model_dir + + +def convert_model(hf_model): # get huggingface model configuration. hf_config = hf_model.config.to_dict() - cfg = {} - cfg["vocabulary_size"] = hf_config["vocab_size"] - cfg["num_layers"] = hf_config["n_layer"] - cfg["num_heads"] = hf_config["n_head"] - cfg["hidden_dim"] = hf_config["hidden_size"] - cfg["intermediate_dim"] = hf_config["hidden_size"] * 4 - cfg["dropout"] = hf_config["hidden_dropout"] - cfg["layer_norm_epsilon"] = hf_config["layer_norm_epsilon"] - - hidden_dim = cfg["hidden_dim"] - num_heads = cfg["num_heads"] - head_dim = hidden_dim // num_heads + kwargs = {} + kwargs["vocabulary_size"] = hf_config["vocab_size"] + kwargs["num_layers"] = hf_config["n_layer"] + kwargs["num_heads"] = hf_config["n_head"] + kwargs["hidden_dim"] = hf_config["hidden_size"] + kwargs["intermediate_dim"] = hf_config["hidden_size"] * 4 + kwargs["dropout"] = hf_config["hidden_dropout"] + kwargs["layer_norm_epsilon"] = hf_config["layer_norm_epsilon"] + + return BloomBackbone(**kwargs) + + +def convert_tokenizer(hf_model_dir): + tokenizer_file_path = os.path.join(hf_model_dir, "tokenizer.json") + with open(tokenizer_file_path) as tokenizer_file: + hf_tokenizer = json.load(tokenizer_file) - # Intialize Bloom model with the weights. - keras_model = BloomBackbone(**cfg) + vocab = hf_tokenizer["model"]["vocab"] + merges = hf_tokenizer["model"]["merges"] + + return BloomTokenizer(vocabulary=vocab, merges=merges) + + +def convert_weights(keras_model, hf_model): + hidden_dim = keras_model.hidden_dim + num_heads = keras_model.num_heads + head_dim = hidden_dim // num_heads + num_layers = keras_model.num_layers # get huggingface model weights. hf_wts = hf_model.state_dict() @@ -78,7 +113,7 @@ def convert_checkpoints(hf_model): keras_model.get_layer("final_layernorm").beta.assign(hf_wts["ln_f.bias"]) # Decoder layers. - for i in range(cfg["num_layers"]): + for i in range(num_layers): decoder_layer = keras_model.get_layer(f"transformer_layer_{i}") # LayrNorm. decoder_layer._pre_attention_layernorm.gamma.assign( @@ -141,54 +176,72 @@ def convert_checkpoints(hf_model): hf_wts[f"h.{i}.mlp.dense_4h_to_h.bias"] ) - # Save the model. - print(f"\n-> Saving KerasNLP model weights to `{FLAGS.preset}.weights.h5`.") - keras_model.save_weights(f"{FLAGS.preset}.weights.h5") - return keras_model - - -def check_output(keras_model, hf_model): - hf_model_input = { - "input_ids": torch.tensor([[59414, 15, 2670, 35433, 632, 207595]]), - "attention_mask": torch.tensor([[1, 1, 1, 1, 1, 1]]), - } - - hf_model_outputs = hf_model(**hf_model_input) - hf_model_outputs = hf_model_outputs.last_hidden_state - hf_model_outputs = hf_model_outputs.detach().numpy() +def validate_output( + hf_model, + keras_model, + hf_tokenizer, + keras_tokenizer, +): + input_str = ["the quick brown fox ran, galloped and jumped."] + # KerasNLP + token_ids = torch.tensor(keras_tokenizer(input_str)) + padding_mask = token_ids != 3 keras_model_input = { - "token_ids": torch.tensor([[59414, 15, 2670, 35433, 632, 207595]]), - "padding_mask": torch.tensor([[1, 1, 1, 1, 1, 1]]), + "token_ids": token_ids, + "padding_mask": padding_mask, } - keras_model_outputs = keras_model.predict(keras_model_input) - # Comparing the outputs. - print("KerasNLP output:", keras_model_outputs[0, 0, :10]) - print("HF output:", hf_model_outputs[0, 0, :10]) - print("Difference:", np.mean(keras_model_outputs - hf_model_outputs)) + hf_model_input = hf_tokenizer(input_str, return_tensors="pt") + + hf_model_outputs = hf_model(**hf_model_input).last_hidden_state + hf_model_outputs = hf_model_outputs.detach().numpy() - # Show the MD5 checksum of the model weights. - print("Model md5sum: ", get_md5_checksum(f"./{FLAGS.preset}.weights.h5")) + # Comparing the outputs. + print("🔶 KerasNLP output:", keras_model_outputs[0, 0, :10]) + print("🔶 HF output:", hf_model_outputs[0, 0, :10]) + print("🔶 Difference:", np.mean(keras_model_outputs - hf_model_outputs)) def main(_): + preset = FLAGS.preset + assert ( - FLAGS.preset in PRESET_MAP.keys() - ), f'Invalid preset {FLAGS.preset}. Must be one of {",".join(PRESET_MAP.keys())}' + preset in PRESET_MAP.keys() + ), f'Invalid preset {preset}. Must be one of {",".join(PRESET_MAP.keys())}' + + print(f"✅ Coverting {preset}") - hf_model_name = PRESET_MAP[FLAGS.preset] + hf_model_name = PRESET_MAP[preset] + hf_model_dir = download_hf_model(hf_model_name) + print("✅ Huggingface model downloaded from hub") - print("\n-> Loading HF model.") - hf_model = transformers.AutoModel.from_pretrained(hf_model_name) + hf_model = transformers.BloomModel.from_pretrained(hf_model_dir) + hf_tokenizer = transformers.BloomTokenizerFast.from_pretrained(hf_model_dir) + print("✅ Huggingface model loaded") - print("\n-> Converting model checkpoint.") - keras_model = convert_checkpoints(hf_model) + keras_model = convert_model(hf_model) + keras_tokenizer = convert_tokenizer(hf_model_dir) + print("✅ Keras model loaded") - print("\n-> Checking keras model output.") - check_output(keras_model, hf_model) + convert_weights(keras_model, hf_model) + print("✅ Weights converted") + + validate_output( + hf_model, + keras_model, + hf_tokenizer, + keras_tokenizer, + ) + print("✅ Numerics validated") + + keras_nlp.src.utils.preset_utils.save_to_preset(keras_model, preset) + keras_nlp.src.utils.preset_utils.save_to_preset( + keras_tokenizer, preset, config_filename="tokenizer.json" + ) + print("✅ Preset saved") if __name__ == "__main__": From 5038e5893297b769b7ab7a7e985d5ada4accf323 Mon Sep 17 00:00:00 2001 From: Matt Watson <1389937+mattdangerw@users.noreply.github.com> Date: Mon, 29 Jan 2024 10:21:21 -0800 Subject: [PATCH 08/29] Update black formatting (#1415) --- keras_nlp/layers/modeling/transformer_decoder.py | 8 +++++--- keras_nlp/models/bloom/bloom_decoder.py | 8 +++++--- keras_nlp/models/gpt_neo_x/gpt_neo_x_decoder.py | 8 +++++--- keras_nlp/models/llama/llama_decoder.py | 8 +++++--- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/keras_nlp/layers/modeling/transformer_decoder.py b/keras_nlp/layers/modeling/transformer_decoder.py index 3a3cda3f21..cb7c779332 100644 --- a/keras_nlp/layers/modeling/transformer_decoder.py +++ b/keras_nlp/layers/modeling/transformer_decoder.py @@ -469,9 +469,11 @@ def _compute_self_attention_mask( batch_size, input_length, output_length, - 0 - if self_attention_cache_update_index is None - else self_attention_cache_update_index, + ( + 0 + if self_attention_cache_update_index is None + else self_attention_cache_update_index + ), ) return ( ops.minimum(decoder_mask, causal_mask) diff --git a/keras_nlp/models/bloom/bloom_decoder.py b/keras_nlp/models/bloom/bloom_decoder.py index b3f8b80da7..c5dbb5135e 100644 --- a/keras_nlp/models/bloom/bloom_decoder.py +++ b/keras_nlp/models/bloom/bloom_decoder.py @@ -174,9 +174,11 @@ def _compute_attention_mask( batch_size, input_length, output_length, - 0 - if attention_cache_update_index is None - else attention_cache_update_index, + ( + 0 + if attention_cache_update_index is None + else attention_cache_update_index + ), ) return ( ops.minimum(decoder_mask, causal_mask) diff --git a/keras_nlp/models/gpt_neo_x/gpt_neo_x_decoder.py b/keras_nlp/models/gpt_neo_x/gpt_neo_x_decoder.py index ae646fb2b6..f80cafd528 100644 --- a/keras_nlp/models/gpt_neo_x/gpt_neo_x_decoder.py +++ b/keras_nlp/models/gpt_neo_x/gpt_neo_x_decoder.py @@ -211,9 +211,11 @@ def _compute_self_attention_mask( batch_size, input_length, output_length, - 0 - if self_attention_cache_update_index is None - else self_attention_cache_update_index, + ( + 0 + if self_attention_cache_update_index is None + else self_attention_cache_update_index + ), ) return ( ops.minimum(decoder_mask, causal_mask) diff --git a/keras_nlp/models/llama/llama_decoder.py b/keras_nlp/models/llama/llama_decoder.py index 47bac478cc..2137831be1 100644 --- a/keras_nlp/models/llama/llama_decoder.py +++ b/keras_nlp/models/llama/llama_decoder.py @@ -172,9 +172,11 @@ def _compute_self_attention_mask( batch_size, input_length, output_length, - 0 - if self_attention_cache_update_index is None - else self_attention_cache_update_index, + ( + 0 + if self_attention_cache_update_index is None + else self_attention_cache_update_index + ), ) return ( ops.minimum(decoder_mask, causal_mask) From 71b756ea610fde98cc26393da92e1e3c936c84a5 Mon Sep 17 00:00:00 2001 From: Mohamed Abu El-Nasr <64566340+abuelnasr0@users.noreply.github.com> Date: Mon, 29 Jan 2024 21:54:50 +0200 Subject: [PATCH 09/29] Add Alibi bias layer (#1404) * Add AlibiBias layer * Add example * Convert layer to recieve attn_scores and add the alibi bias to it. * Change layer logic * Add layer test * Format the code * Fix seq_range creation to be int range * Change bloom model to use alibi bias * Format the code * Remove print function * Change logic to only compute alibi bias once * Change bloom model API calls to much new alibi layer API * Format the code * Add dtype kwarg for the layer weight * Revert "Add dtype kwarg for the layer weight" This reverts commit 91c0e04aba806f9dafe074fcaa986aab6bc5b97e. * Cast after adding alibi bias * Return to compute ALibi bias at each call * Add compute output shape method for bloom decoder * Force shape to be (batch_size, num_heads, query_length, key_length) * Format the code * Fix documentation * Fix the example * Fix tensorflow2 test fail * Fix tensorflow2 test fail --- keras_nlp/layers/modeling/alibi_bias.py | 137 +++++++++++++ keras_nlp/layers/modeling/alibi_bias_test.py | 202 +++++++++++++++++++ keras_nlp/models/bloom/bloom_attention.py | 54 +---- keras_nlp/models/bloom/bloom_decoder.py | 3 + 4 files changed, 353 insertions(+), 43 deletions(-) create mode 100644 keras_nlp/layers/modeling/alibi_bias.py create mode 100644 keras_nlp/layers/modeling/alibi_bias_test.py diff --git a/keras_nlp/layers/modeling/alibi_bias.py b/keras_nlp/layers/modeling/alibi_bias.py new file mode 100644 index 0000000000..8a66ad05af --- /dev/null +++ b/keras_nlp/layers/modeling/alibi_bias.py @@ -0,0 +1,137 @@ +# Copyright 2023 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import math + +from keras_nlp.api_export import keras_nlp_export +from keras_nlp.backend import keras +from keras_nlp.backend import ops + + +@keras_nlp_export("keras_nlp.layers.AlibiBias") +class AlibiBias(keras.layers.Layer): + """A layer that adds the alibi bias to attention scores. + + This layer adds the alibi bias to the attention scores. Alibi bias is a + linear, non-learned bias. Defined and formalized in + [Train Short, Test Long: Attention with Linear Biases Enables Input Length Extrapolation](https://arxiv.org/abs/2108.12409). + + This layer takes as input the attention scores. and returns the attention + scores after adding the alibi bias to it. The output will have the same + shape as the input. + + Args: + alibi_bias_max: int. This value will be used to compute the slope of + each head. The heads' slopes are a geometric sequence that starts at + `2**(-alibi_bias_max/num_heads)` and uses that same value as its + ratio. Defaults to 8. + Call arguments: + attention_scores: The result of multipying the query and the key of the + multi-head attention layer of the transformer to add alibi bias to + it. With shape `(batch_size, num_heads, query_length, key_length)`. + + Examples: + ```python + query_length = 10 + key_length = 10 + num_heads = 4 + batch_size = 2 + hidden_dim = 8 + + # Create new alibi layer. + alibi_layer = keras_nlp.layers.AlibiBias() + + query = np.zeros((batch_size, num_heads, query_length, hidden_dim)) + key = np.zeros((batch_size, num_heads, hidden_dim, key_length)) + + attention_scores = keras.ops.matmul(query, key) + + # Add alibi bias to attention scores. + attention_scores = alibi_layer(attention_scores) + ``` + + References: + - [Press et al., 2021](https://arxiv.org/abs/2108.12409) + """ + + def __init__( + self, + alibi_bias_max=8, + **kwargs, + ): + super().__init__(**kwargs) + self.alibi_bias_max = alibi_bias_max + + def call(self, attention_scores): + shape = ops.shape(attention_scores) + if len(shape) != 4: + raise ValueError( + "Expected `attention_scores` shape to be " + "`(batch_size, num_heads, query_length, key_Length)`." + f" Recived shape={shape}" + ) + + key_length = shape[-1] + num_heads = shape[-3] + + alibi_bias = self._get_alibi_bias(num_heads, key_length) + + return ops.add(attention_scores, alibi_bias) + + def _get_alibi_bias(self, num_heads, key_length): + slopes = ops.convert_to_tensor( + self._get_slopes(num_heads), dtype=self.compute_dtype + ) + slopes = ops.expand_dims(slopes, 1) + + seq_range = ops.expand_dims(ops.arange(1 - key_length, 1), 0) + seq_range = ops.cast(seq_range, dtype=self.compute_dtype) + + alibi_bias = ops.multiply(slopes, seq_range) + alibi_bias = ops.expand_dims(alibi_bias, 1) + + # return shape is `(1, num_heads, 1, key_length)` + return ops.expand_dims(alibi_bias, 0) + + def _get_slopes(self, num_heads): + # this function is adopted from Alibi original implementation. + # https://github.com/ofirpress/attention_with_linear_biases/blob/a35aaca144e0eb6b789dfcb46784c4b8e31b7983/fairseq/models/transformer.py#L742 + def get_slopes_power_of_2(n): + start = 2 ** ( + -(2 ** -(math.log2(n) - math.log2(self.alibi_bias_max))) + ) + ratio = start + return [start * ratio**i for i in range(n)] + + if math.log2(num_heads).is_integer(): + return get_slopes_power_of_2(num_heads) + else: + closest_power_of_2 = 2 ** math.floor(math.log2(num_heads)) + return ( + get_slopes_power_of_2(closest_power_of_2) + + self._get_slopes(2 * closest_power_of_2)[0::2][ + : num_heads - closest_power_of_2 + ] + ) + + def compute_output_shape(self, input_shape): + return input_shape + + def get_config(self): + config = super().get_config() + config.update( + { + "alibi_bias_max": self.alibi_bias_max, + } + ) + return config diff --git a/keras_nlp/layers/modeling/alibi_bias_test.py b/keras_nlp/layers/modeling/alibi_bias_test.py new file mode 100644 index 0000000000..120c7622b1 --- /dev/null +++ b/keras_nlp/layers/modeling/alibi_bias_test.py @@ -0,0 +1,202 @@ +# Copyright 2023 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from keras_nlp.backend import keras +from keras_nlp.backend import ops +from keras_nlp.backend import random +from keras_nlp.layers.modeling.alibi_bias import AlibiBias +from keras_nlp.tests.test_case import TestCase + + +class AlibiBiasTest(TestCase): + def test_layer_behaviors(self): + alibi_bias_max = 8 + batch_size = 4 + num_heads = 8 + query_length = 10 + key_length = 10 + self.run_layer_test( + cls=AlibiBias, + init_kwargs={ + "alibi_bias_max": alibi_bias_max, + }, + input_data=random.uniform( + shape=(batch_size, num_heads, query_length, key_length) + ), + expected_output_shape=( + batch_size, + num_heads, + query_length, + key_length, + ), + ) + + def test_float16_dtype(self): + # Create a 4-dimensional input (the first dimension is implicit). + alibi_bias_max = 8 + num_heads = 8 + query_length = 5 + key_length = 10 + test_layer = AlibiBias(alibi_bias_max=alibi_bias_max, dtype="float16") + input_tensor = keras.Input(shape=(num_heads, query_length, key_length)) + output_tensor = test_layer(input_tensor) + + # the output is expected to be the same as the input shape in all + # dimensions. here, the first dimension is implicit and is for batch + expected_output_shape = (None, num_heads, query_length, key_length) + self.assertEqual(expected_output_shape, output_tensor.shape) + # The default output dtype for this layer should be "float32". + self.assertEqual("float16", output_tensor.dtype) + + def test_dynamic_layer_output_shape(self): + query_length = 10 + key_length = 10 + num_heads = 4 + + test_layer = AlibiBias() + # Create a 4-dimensional input (the first dimension is implicit). + input_tensor = keras.Input(shape=(num_heads, query_length, key_length)) + output_tensor = test_layer(input_tensor) + + # the output is expected to be the same as the input shape in all + # dimensions. + expected_output_shape = ( + None, + num_heads, + query_length, + key_length, + ) + self.assertEqual(expected_output_shape, output_tensor.shape) + + def test_value_error_when_inputs_shape_is_not_4(self): + with self.assertRaises(ValueError): + AlibiBias()(random.uniform(shape=(12, 12))) + + def test_num_heads_is_not_power_of_two(self): + inputs_shape = (1, 12, 12, 12) + inputs = random.uniform(shape=inputs_shape) + layer = AlibiBias() + outputs = layer(inputs) + self.assertEqual(inputs_shape, outputs.shape) + + def test_correct_output(self): + batch_size = 1 + num_heads = 8 + query_length = 1 + key_length = 3 + input_shape = (batch_size, num_heads, query_length, key_length) + input_tensor = ops.zeros(input_shape) + layer = AlibiBias() + output_tensor = layer(input_tensor) + print(output_tensor) + self.assertAllClose( + output_tensor, + ops.convert_to_tensor( + [ + [ + [[-1.0, -0.5, 0.0]], + [[-0.5, -0.25, 0.0]], + [[-0.25, -0.125, 0.0]], + [[-0.125, -0.0625, 0.0]], + [[-0.0625, -0.03125, 0.0]], + [[-0.03125, -0.015625, 0.0]], + [[-0.015625, -0.0078125, 0.0]], + [[-0.0078125, -0.00390625, 0.0]], + ] + ] + ), + ) + + def test_correct_output_num_heads_not_power_of_two(self): + batch_size = 1 + num_heads = 14 + query_length = 1 + key_length = 3 + input_shape = (batch_size, num_heads, query_length, key_length) + input_tensor = ops.zeros(input_shape) + layer = AlibiBias() + output_tensor = layer(input_tensor) + print(output_tensor) + self.assertAllClose( + output_tensor, + ops.convert_to_tensor( + [ + [ + [[-1.0, -0.5, 0.0]], + [[-0.5, -0.25, 0.0]], + [[-0.25, -0.125, 0.0]], + [[-0.125, -0.0625, 0.0]], + [[-0.0625, -0.03125, 0.0]], + [[-0.03125, -0.015625, 0.0]], + [[-0.015625, -0.0078125, 0.0]], + [[-0.0078125, -0.00390625, 0.0]], + [[-1.4142135, -0.70710677, 0.0]], + [[-0.70710677, -0.35355338, 0.0]], + [[-0.35355338, -0.17677669, 0.0]], + [[-0.17677669, -0.08838835, 0.0]], + [[-0.08838835, -0.04419417, 0.0]], + [[-0.04419417, -0.02209709, 0.0]], + ] + ] + ), + ) + + def test_correct_output_alibi_bias_max(self): + alibi_bias_max = 12 + batch_size = 1 + num_heads = 2 + query_length = 1 + key_length = 3 + input_shape = (batch_size, num_heads, query_length, key_length) + input_tensor = ops.zeros(input_shape) + layer = AlibiBias(alibi_bias_max=alibi_bias_max) + output_tensor = layer(input_tensor) + print(output_tensor) + self.assertAllClose( + output_tensor, + ops.convert_to_tensor( + [ + [ + [[-0.03125, -0.015625, 0.0]], + [[-0.00048828, -0.00024414, 0.0]], + ] + ] + ), + ) + + def test_correct_output_alibi_bias_max_num_heads_not_power_of_two( + self, + ): + alibi_bias_max = 6 + batch_size = 1 + num_heads = 3 + query_length = 1 + key_length = 3 + input_shape = (batch_size, num_heads, query_length, key_length) + input_tensor = ops.zeros(input_shape) + layer = AlibiBias(alibi_bias_max=alibi_bias_max) + output_tensor = layer(input_tensor) + print(output_tensor) + self.assertAllClose( + output_tensor, + ops.convert_to_tensor( + [ + [ + [[-0.25, -0.125, 0.0]], + [[-0.03125, -0.015625, 0.0]], + [[-0.70710677, -0.35355338, 0.0]], + ] + ] + ), + ) diff --git a/keras_nlp/models/bloom/bloom_attention.py b/keras_nlp/models/bloom/bloom_attention.py index 7af2e7a34d..366169d1fb 100644 --- a/keras_nlp/models/bloom/bloom_attention.py +++ b/keras_nlp/models/bloom/bloom_attention.py @@ -15,6 +15,7 @@ from keras_nlp.backend import keras from keras_nlp.backend import ops +from keras_nlp.layers.modeling.alibi_bias import AlibiBias from keras_nlp.utils.keras_utils import clone_initializer @@ -74,6 +75,8 @@ def build(self, inputs_shape): ) self._value_dense.build(inputs_shape) + self._alibi_layer = AlibiBias() + self._output_dense = keras.layers.Dense( hidden_dim, kernel_initializer=clone_initializer(self.kernel_initializer), @@ -92,38 +95,6 @@ def build(self, inputs_shape): self.built = True - @staticmethod - def _build_alibi_tensor(num_heads, seq_length, alibi_bias_max=8): - # this function is adopted from fairseq - # https://github.com/ofirpress/attention_with_linear_biases/blob/a35aaca144e0eb6b789dfcb46784c4b8e31b7983/fairseq/models/transformer.py#L742 - def get_slopes(n): - def get_slopes_power_of_2(n): - start = 2 ** ( - -(2 ** -(math.log2(n) - math.log2(alibi_bias_max))) - ) - ratio = start - return [start * ratio**i for i in range(n)] - - if math.log2(n).is_integer(): - return get_slopes_power_of_2(n) - else: - closest_power_of_2 = 2 ** math.floor(math.log2(n)) - return ( - get_slopes_power_of_2(closest_power_of_2) - + get_slopes(2 * closest_power_of_2)[0::2][ - : n - closest_power_of_2 - ] - ) - - slopes = ops.convert_to_tensor(get_slopes(num_heads), dtype=float) - slopes = ops.expand_dims(slopes, 1) - - alibi = slopes * ops.expand_dims(ops.arange(seq_length, dtype=float), 0) - alibi = ops.expand_dims(alibi, 1) - alibi = ops.expand_dims(alibi, 0) - - return alibi - def call( self, hidden_states, @@ -163,20 +134,17 @@ def call( # key (batch_size, num_heads, head_dim, kv_length) key = ops.transpose(key, [0, 2, 3, 1]) - alibi = self._build_alibi_tensor( - num_heads=self.num_heads, seq_length=seq_length - ) - - scores = ( - ops.matmul(query, key) * self.inv_norm_factor + alibi + attention_scores = ( + ops.matmul(query, key) * self.inv_norm_factor ) # [batch_size, num_heads, query_length, kv_length] - - scores = self._softmax(scores, ops.expand_dims(attention_mask, 1)) - - scores = self._dropout_layer(scores) + attention_scores = self._alibi_layer(attention_scores) + attention_scores = self._softmax( + attention_scores, ops.expand_dims(attention_mask, 1) + ) + attention_scores = self._dropout_layer(attention_scores) attention_output = ops.matmul( - scores, value + attention_scores, value ) # [batch_size, num_heads, query_length, head_dim] attention_output = ops.transpose( diff --git a/keras_nlp/models/bloom/bloom_decoder.py b/keras_nlp/models/bloom/bloom_decoder.py index c5dbb5135e..a0c62a2541 100644 --- a/keras_nlp/models/bloom/bloom_decoder.py +++ b/keras_nlp/models/bloom/bloom_decoder.py @@ -204,3 +204,6 @@ def get_config(self): } ) return config + + def compute_output_shape(self, decoder_sequence_shape): + return decoder_sequence_shape From 6350cea942003909dfce01091289c2c8b362e062 Mon Sep 17 00:00:00 2001 From: Ramesh Sampath <1437573+sampathweb@users.noreply.github.com> Date: Wed, 31 Jan 2024 12:41:04 -0600 Subject: [PATCH 10/29] Pin to (#1420) --- requirements-common.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/requirements-common.txt b/requirements-common.txt index 5c2a8a3d90..dedd3b3874 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -17,3 +17,6 @@ namex rouge-score sentencepiece tensorflow-datasets +# TODO #1419: CI fails with `tensorflow-hub==0.16.1` installed from +# `tensorflow-text-nightly` dependency. Remove once GH issue 1419 is closed. +tensorflow-hub==0.16.0 \ No newline at end of file From a4088f2a50007820ea0bca9cbc37129fad086f56 Mon Sep 17 00:00:00 2001 From: Ramesh Sampath <1437573+sampathweb@users.noreply.github.com> Date: Fri, 2 Feb 2024 17:14:20 -0600 Subject: [PATCH 11/29] Update TF Text and remove TF Hub deps (#1423) * Update TF Text and remove TF Hub deps * Update TF Text and remove TF Hub deps --- requirements-common.txt | 3 --- requirements-jax-cuda.txt | 4 ++-- requirements-tensorflow-cuda.txt | 4 ++-- requirements-torch-cuda.txt | 4 ++-- requirements.txt | 4 ++-- 5 files changed, 8 insertions(+), 11 deletions(-) diff --git a/requirements-common.txt b/requirements-common.txt index dedd3b3874..5c2a8a3d90 100644 --- a/requirements-common.txt +++ b/requirements-common.txt @@ -17,6 +17,3 @@ namex rouge-score sentencepiece tensorflow-datasets -# TODO #1419: CI fails with `tensorflow-hub==0.16.1` installed from -# `tensorflow-text-nightly` dependency. Remove once GH issue 1419 is closed. -tensorflow-hub==0.16.0 \ No newline at end of file diff --git a/requirements-jax-cuda.txt b/requirements-jax-cuda.txt index c09b306264..1d378d6bfe 100644 --- a/requirements-jax-cuda.txt +++ b/requirements-jax-cuda.txt @@ -1,6 +1,6 @@ # Tensorflow cpu-only version. -tf-nightly-cpu==2.16.0.dev20231221 # Pin a working nightly until rc0. -tensorflow-text-nightly==2.16.0.dev20231221 # Pin a working nightly until rc0. +tf-nightly-cpu==2.16.0.dev20240201 # Pin a working nightly until rc0. +tensorflow-text-nightly==2.16.0.dev20240201 # Pin a working nightly until rc0. # Torch cpu-only version. --extra-index-url https://download.pytorch.org/whl/cpu diff --git a/requirements-tensorflow-cuda.txt b/requirements-tensorflow-cuda.txt index 21a8ed2463..be95915996 100644 --- a/requirements-tensorflow-cuda.txt +++ b/requirements-tensorflow-cuda.txt @@ -1,6 +1,6 @@ # Tensorflow with cuda support. -tf-nightly[and-cuda]==2.16.0.dev20231221 # Pin a working nightly until rc0. -tensorflow-text-nightly==2.16.0.dev20231221 # Pin a working nightly until rc0. +tf-nightly[and-cuda]==2.16.0.dev20240201 # Pin a working nightly until rc0. +tensorflow-text-nightly==2.16.0.dev20240201 # Pin a working nightly until rc0. # Torch cpu-only version. --extra-index-url https://download.pytorch.org/whl/cpu diff --git a/requirements-torch-cuda.txt b/requirements-torch-cuda.txt index c71c51e478..7ea2981478 100644 --- a/requirements-torch-cuda.txt +++ b/requirements-torch-cuda.txt @@ -1,6 +1,6 @@ # Tensorflow cpu-only version. -tf-nightly-cpu==2.16.0.dev20231221 # Pin a working nightly until rc0. -tensorflow-text-nightly==2.16.0.dev20231221 # Pin a working nightly until rc0. +tf-nightly-cpu==2.16.0.dev20240201 # Pin a working nightly until rc0. +tensorflow-text-nightly==2.16.0.dev20240201 # Pin a working nightly until rc0. # Torch with cuda support. --extra-index-url https://download.pytorch.org/whl/cu121 diff --git a/requirements.txt b/requirements.txt index fa1dc91943..b226229d15 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ # Tensorflow. -tf-nightly-cpu==2.16.0.dev20231221 # Pin a working nightly until rc0. -tensorflow-text-nightly==2.16.0.dev20231221 # Pin a working nightly until rc0. +tf-nightly-cpu==2.16.0.dev20240201 # Pin a working nightly until rc0. +tensorflow-text-nightly==2.16.0.dev20240201 # Pin a working nightly until rc0. # Torch. --extra-index-url https://download.pytorch.org/whl/cpu From d3c66142b2d38b5974fc559481ae0e1ad4e09bc1 Mon Sep 17 00:00:00 2001 From: Ramesh Sampath <1437573+sampathweb@users.noreply.github.com> Date: Fri, 9 Feb 2024 11:05:56 -0600 Subject: [PATCH 12/29] Pin Jax Version in GPU CI (#1430) --- requirements-jax-cuda.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements-jax-cuda.txt b/requirements-jax-cuda.txt index 1d378d6bfe..903a603352 100644 --- a/requirements-jax-cuda.txt +++ b/requirements-jax-cuda.txt @@ -8,7 +8,8 @@ torch>=2.1.0 torchvision>=0.16.0 # Jax with cuda support. +# TODO: 0.4.24 has an updated Cuda version breaks Jax CI. --find-links https://storage.googleapis.com/jax-releases/jax_cuda_releases.html -jax[cuda12_pip] +jax[cuda12_pip]==0.4.23 -r requirements-common.txt From 2cfd900f62e071a9abb382e4e35b01a58466101b Mon Sep 17 00:00:00 2001 From: Mohamed Abu El-Nasr <64566340+abuelnasr0@users.noreply.github.com> Date: Sat, 10 Feb 2024 05:34:00 +0200 Subject: [PATCH 13/29] Add Bloom preprocessor (#1424) * Add bloompreprocesoor * Add bloom preporcessor docstring * Format docstring * Add bloom causal lm preprocessor * Fix bloom max_sequence_length * class Naming changes * Delete mutisegment comment from the docs --- keras_nlp/models/bloom/bloom_backbone.py | 4 +- .../bloom/bloom_causal_lm_preprocessor.py | 182 ++++++++++++++++++ .../bloom_causal_lm_preprocessor_test.py | 94 +++++++++ keras_nlp/models/bloom/bloom_preprocessor.py | 182 ++++++++++++++++++ .../models/bloom/bloom_preprocessor_test.py | 80 ++++++++ 5 files changed, 540 insertions(+), 2 deletions(-) create mode 100644 keras_nlp/models/bloom/bloom_causal_lm_preprocessor.py create mode 100644 keras_nlp/models/bloom/bloom_causal_lm_preprocessor_test.py create mode 100644 keras_nlp/models/bloom/bloom_preprocessor.py create mode 100644 keras_nlp/models/bloom/bloom_preprocessor_test.py diff --git a/keras_nlp/models/bloom/bloom_backbone.py b/keras_nlp/models/bloom/bloom_backbone.py index 2fd18ed760..86a2e2b4c3 100644 --- a/keras_nlp/models/bloom/bloom_backbone.py +++ b/keras_nlp/models/bloom/bloom_backbone.py @@ -28,7 +28,7 @@ def _bloom_kernel_initializer(stddev=0.02): @keras_nlp_export("keras_nlp.models.BloomBackbone") class BloomBackbone(Backbone): - """A Bloom decoder network. + """A BLOOM decoder network. This network implements a Transformer-based decoder network, BigScience Language Open-science Open-access Multilingual (BLOOM), as descriped in @@ -92,7 +92,7 @@ def __init__( intermediate_dim, dropout=0.0, layer_norm_epsilon=1e-5, - max_sequence_length=512, + max_sequence_length=2048, **kwargs, ): token_ids = keras.Input(shape=(None,), dtype="int32", name="token_ids") diff --git a/keras_nlp/models/bloom/bloom_causal_lm_preprocessor.py b/keras_nlp/models/bloom/bloom_causal_lm_preprocessor.py new file mode 100644 index 0000000000..01f3c88d30 --- /dev/null +++ b/keras_nlp/models/bloom/bloom_causal_lm_preprocessor.py @@ -0,0 +1,182 @@ +# Copyright 2023 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import tensorflow as tf +from absl import logging + +from keras_nlp.api_export import keras_nlp_export +from keras_nlp.backend import ops +from keras_nlp.models.bloom.bloom_preprocessor import BloomPreprocessor +from keras_nlp.utils.keras_utils import ( + convert_inputs_to_list_of_tensor_segments, +) +from keras_nlp.utils.keras_utils import pack_x_y_sample_weight + + +@keras_nlp_export("keras_nlp.models.BloomCausalLMPreprocessor") +class BloomCausalLMPreprocessor(BloomPreprocessor): + """BLOOM Causal LM preprocessor. + + This preprocessing layer is meant for use with + `keras_nlp.models.BloomCausalLM`. By default, it will take in batches of + strings, and return outputs in a `(x, y, sample_weight)` format, where the + `y` label is the next token id in the `x` sequence. + + For use with generation, the layer also exposes two methods + `generate_preprocess()` and `generate_postprocess()`. When this preprocessor + is attached to a `keras_nlp.models.BloomCausalLM` instance, these methods + will be called implicitly in `generate()`. They can also be called + standalone (e.g. to precompute preprocessing inputs for generation in a + separate process). + + Args: + tokenizer: A `keras_nlp.models.BloomTokenizer` instance. + sequence_length: The length of the packed inputs. + add_start_token: If `True`, the preprocessor will prepend the tokenizer + start token to each input sequence. + add_end_token: If `True`, the preprocessor will append the tokenizer + end token to each input sequence. + + Call arguments: + x: A string, `tf.Tensor` or list of python strings. + y: Label data. Should always be `None` as the layer generates labels. + sample_weight: Label weights. Should always be `None` as the layer + generates label weights. + sequence_length: Pass to override the configured `sequence_length` of + the layer. + + Examples: + ```python + # Load the preprocessor from a preset. + preprocessor = keras_nlp.models.BloomCausalLMPreprocessor.from_preset( + "bloom_560m_multi" + ) + + # Tokenize and pack a single sentence. + sentence = tf.constant("League of legends") + preprocessor(sentence) + # Same output. + preprocessor("League of legends") + + # Tokenize a batch of sentences. + sentences = tf.constant(["Taco tuesday", "Fish taco please!"]) + preprocessor(sentences) + # Same output. + preprocessor(["Taco tuesday", "Fish taco please!"]) + + # Map a dataset to preprocess a single sentence. + features = tf.constant( + [ + "Avatar 2 is amazing!", + "Well, I am not sure.", + ] + ) + labels = tf.constant([1, 0]) + ds = tf.data.Dataset.from_tensor_slices((features, labels)) + ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE) + + # Map a dataset to preprocess unlabled sentences. + ds = tf.data.Dataset.from_tensor_slices(features) + ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE) + ``` + """ + + def call( + self, + x, + y=None, + sample_weight=None, + sequence_length=None, + ): + if y is not None or sample_weight is not None: + logging.warning( + "`BloomCausalLMPreprocessor` generates `y` and `sample_weight` " + "based on your input data, but your data already contains `y` " + "or `sample_weight`. Your `y` and `sample_weight` will be " + "ignored." + ) + sequence_length = sequence_length or self.sequence_length + + x = convert_inputs_to_list_of_tensor_segments(x)[0] + x = self.tokenizer(x) + # Pad with one extra token to account for the truncation below. + token_ids, padding_mask = self.packer( + x, + sequence_length=sequence_length + 1, + add_start_value=self.add_start_token, + add_end_value=self.add_end_token, + ) + # The last token does not have a next token, so we truncate it out. + x = { + "token_ids": token_ids[..., :-1], + "padding_mask": padding_mask[..., :-1], + } + # Target `y` will be the next token. + y, sample_weight = token_ids[..., 1:], padding_mask[..., 1:] + return pack_x_y_sample_weight(x, y, sample_weight) + + def generate_preprocess( + self, + x, + sequence_length=None, + ): + """Covert strings to integer token input for generation. + + Similar to calling the layer for training, this method takes in strings + or tensor strings, tokenizes and packs the input, and computes a padding + mask masking all inputs not filled in with a padded value. + + Unlike calling the layer for training, this method does not compute + labels and will never append a `tokenizer.end_token_id` to the end of + the sequence (as generation is expected to continue at the end of the + inputted prompt). + """ + if not self.built: + self.build(None) + + x = convert_inputs_to_list_of_tensor_segments(x)[0] + x = self.tokenizer(x) + token_ids, padding_mask = self.packer( + x, sequence_length=sequence_length, add_end_value=False + ) + return { + "token_ids": token_ids, + "padding_mask": padding_mask, + } + + def generate_postprocess( + self, + x, + ): + """Covert integer token output to strings for generation. + + This method reverses `generate_preprocess()`, by first removing all + padding and start/end tokens, and then converting the integer sequence + back to a string. + """ + if not self.built: + self.build(None) + + token_ids, padding_mask = x["token_ids"], x["padding_mask"] + token_ids = ops.convert_to_numpy(token_ids) + padding_mask = ops.convert_to_numpy(padding_mask) + # Strip any special tokens during detokenization (e.g. the start and + # end markers). In the future we could make this configurable. + padding_mask = ( + padding_mask + & (token_ids != self.tokenizer.eos_token_id) + & (token_ids != self.tokenizer.bos_token_id) + ) + token_ids = tf.ragged.boolean_mask(token_ids, padding_mask) + return self.tokenizer.detokenize(token_ids) diff --git a/keras_nlp/models/bloom/bloom_causal_lm_preprocessor_test.py b/keras_nlp/models/bloom/bloom_causal_lm_preprocessor_test.py new file mode 100644 index 0000000000..6caf8fddcf --- /dev/null +++ b/keras_nlp/models/bloom/bloom_causal_lm_preprocessor_test.py @@ -0,0 +1,94 @@ +# Copyright 2023 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from keras_nlp.models.bloom.bloom_causal_lm_preprocessor import ( + BloomCausalLMPreprocessor, +) +from keras_nlp.models.bloom.bloom_tokenizer import BloomTokenizer +from keras_nlp.tests.test_case import TestCase + + +class BloomCausalLMPreprocessorTest(TestCase): + def setUp(self): + self.vocab = ["", "", ""] + self.vocab += ["!", "air", "Ġair", "plane", "Ġat", "port"] + self.vocab = dict([(token, i) for i, token in enumerate(self.vocab)]) + self.merges = ["Ġ a", "Ġ t", "Ġ i", "Ġ b", "a i", "p l", "n e"] + self.merges += ["Ġa t", "p o", "r t", "Ġt h", "ai r", "pl a", "po rt"] + self.merges += ["Ġai r", "Ġa i", "pla ne"] + self.tokenizer = BloomTokenizer( + vocabulary=self.vocab, + merges=self.merges, + ) + self.init_kwargs = { + "tokenizer": self.tokenizer, + "sequence_length": 8, + } + self.input_data = ["airplane at airport"] + + def test_preprocessor_basics(self): + self.run_preprocessing_layer_test( + cls=BloomCausalLMPreprocessor, + init_kwargs=self.init_kwargs, + input_data=self.input_data, + expected_output=( + { + "token_ids": [[1, 4, 6, 7, 5, 8, 2, 0]], + "padding_mask": [[1, 1, 1, 1, 1, 1, 1, 0]], + }, + [[4, 6, 7, 5, 8, 2, 0, 0]], # Pass through labels. + [[1, 1, 1, 1, 1, 1, 0, 0]], # Pass through sample_weights. + ), + ) + + def test_no_start_end_token(self): + input_data = ["airplane at airport"] * 4 + + preprocessor = BloomCausalLMPreprocessor( + **self.init_kwargs, + add_start_token=False, + add_end_token=False, + ) + x, y, sw = preprocessor(input_data) + self.assertAllEqual(x["token_ids"], [[4, 6, 7, 5, 8, 0, 0, 0]] * 4) + self.assertAllEqual(x["padding_mask"], [[1, 1, 1, 1, 1, 0, 0, 0]] * 4) + self.assertAllEqual(y, [[6, 7, 5, 8, 0, 0, 0, 0]] * 4) + self.assertAllEqual(sw, [[1, 1, 1, 1, 0, 0, 0, 0]] * 4) + + def test_generate_preprocess(self): + input_data = "airplane at airport" + preprocessor = BloomCausalLMPreprocessor(**self.init_kwargs) + x = preprocessor.generate_preprocess(input_data) + self.assertAllEqual(x["token_ids"], [1, 4, 6, 7, 5, 8, 0, 0]) + self.assertAllEqual(x["padding_mask"], [1, 1, 1, 1, 1, 1, 0, 0]) + + def test_generate_postprocess(self): + input_data = { + "token_ids": [1, 4, 6, 7, 5, 8, 0, 0], + "padding_mask": [1, 1, 1, 1, 1, 1, 0, 0], + } + preprocessor = BloomCausalLMPreprocessor(**self.init_kwargs) + x = preprocessor.generate_postprocess(input_data) + self.assertAllEqual(x, "airplane at airport") + + @pytest.mark.extra_large + def test_all_presets(self): + for preset in BloomCausalLMPreprocessor.presets: + self.run_preset_test( + cls=BloomCausalLMPreprocessor, + preset=preset, + input_data=self.input_data, + ) diff --git a/keras_nlp/models/bloom/bloom_preprocessor.py b/keras_nlp/models/bloom/bloom_preprocessor.py new file mode 100644 index 0000000000..d57ccd2414 --- /dev/null +++ b/keras_nlp/models/bloom/bloom_preprocessor.py @@ -0,0 +1,182 @@ +# Copyright 2022 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy + +from keras_nlp.api_export import keras_nlp_export +from keras_nlp.layers.preprocessing.start_end_packer import StartEndPacker +from keras_nlp.models.bloom.bloom_presets import backbone_presets +from keras_nlp.models.bloom.bloom_tokenizer import BloomTokenizer +from keras_nlp.models.preprocessor import Preprocessor +from keras_nlp.utils.keras_utils import ( + convert_inputs_to_list_of_tensor_segments, +) +from keras_nlp.utils.keras_utils import pack_x_y_sample_weight +from keras_nlp.utils.python_utils import classproperty + + +@keras_nlp_export("keras_nlp.models.BloomPreprocessor") +class BloomPreprocessor(Preprocessor): + """BLOOM preprocessing layer which tokenizes and packs inputs. + + This preprocessing layer will do 2 things: + + - Tokenize the inputs using the `tokenizer`. + - Construct a dictionary with keys `"token_ids"`, `"padding_mask"`, that can + be passed directly to a `keras_nlp.models.BloomBackbone`. + + This layer can be used directly with `tf.data.Dataset.map` to preprocess + string data in the `(x, y, sample_weight)` format used by + `keras.Model.fit`. + + The call method of this layer accepts three arguments, `x`, `y`, and + `sample_weight`. `x` can be a python string or tensor representing a single + segment, a list of python strings representing a batch of single segments, + or a list of tensors representing multiple segments to be packed together. + `y` and `sample_weight` are both optional, can have any format, and will be + passed through unaltered. + + Args: + tokenizer: A `keras_nlp.models.BloomTokenizer` instance. + sequence_length: The length of the packed inputs. + add_start_token: If `True`, the preprocessor will prepend the tokenizer + start token to each input sequence. + add_end_token: If `True`, the preprocessor will append the tokenizer + end token to each input sequence. + + Call arguments: + x: A string, `tf.Tensor` or list of python strings. + y: Any label data. Will be passed through unaltered. + sample_weight: Any label weight data. Will be passed through unaltered. + sequence_length: Pass to override the configured `sequence_length` of + the layer. + + Examples: + + Directly calling the layer on data. + ```python + preprocessor = keras_nlp.models.BloomPreprocessor.from_preset( + "bloom_560m_multi" + ) + # Tokenize and pack a single sentence. + preprocessor("The quick brown fox jumped.") + + # Tokenize a batch of single sentences. + preprocessor(["The quick brown fox jumped.", "Call me Ishmael."]) + + # Custom vocabulary. + features = ["a quick fox.", "a fox quick."] + vocab = {"": 0, "":1, "":2, "a": 3, "Ġquick": 4, "Ġfox": 5} + merges = ["Ġ q", "u i", "c k", "ui ck", "Ġq uick"] + merges += ["Ġ f", "o x", "Ġf ox"] + tokenizer = keras_nlp.models.BloomTokenizer( + vocabulary=vocab, + merges=merges, + ) + preprocessor = keras_nlp.models.BloomPreprocessor(tokenizer=tokenizer) + preprocessor("The quick brown fox jumped.") + ``` + + Mapping with `tf.data.Dataset`. + ```python + preprocessor = keras_nlp.models.BloomPreprocessor.from_preset( + "bloom_560m_multi" + ) + + text = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."]) + label = tf.constant([1, 1]) + + # Map labeled single sentences. + ds = tf.data.Dataset.from_tensor_slices((text, label)) + ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE) + + # Map unlabeled single sentences. + ds = tf.data.Dataset.from_tensor_slices(text) + ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE) + ``` + """ + + def __init__( + self, + tokenizer, + sequence_length=2048, + add_start_token=True, + add_end_token=True, + **kwargs, + ): + super().__init__(**kwargs) + + self.tokenizer = tokenizer + self.sequence_length = sequence_length + self.add_start_token = add_start_token + self.add_end_token = add_end_token + + def build(self, input_shape): + # Defer packer creation to `build()` so that we can be sure tokenizer + # assets have loaded when restoring a saved model. + self.packer = StartEndPacker( + start_value=self.tokenizer.bos_token_id, + end_value=self.tokenizer.eos_token_id, + pad_value=self.tokenizer.pad_token_id, + sequence_length=self.sequence_length, + return_padding_mask=True, + ) + self.built = True + + def call( + self, + x, + y=None, + sample_weight=None, + sequence_length=None, + ): + x = convert_inputs_to_list_of_tensor_segments(x) + if len(x) != 1: + raise ValueError( + "BLOOM requires each input feature to contain only " + f"one segment, but received {len(x)}. If you are using BLOOM " + "for a multi-segment classification task, please refer to " + "classification models like BERT or RoBERTa." + ) + sequence_length = sequence_length or self.sequence_length + token_ids, padding_mask = self.packer( + self.tokenizer(x[0]), + sequence_length=sequence_length, + add_start_value=self.add_start_token, + add_end_value=self.add_end_token, + ) + x = { + "token_ids": token_ids, + "padding_mask": padding_mask, + } + return pack_x_y_sample_weight(x, y, sample_weight) + + def get_config(self): + config = super().get_config() + config.update( + { + "sequence_length": self.sequence_length, + "add_start_token": self.add_start_token, + "add_end_token": self.add_end_token, + } + ) + return config + + @classproperty + def presets(cls): + return copy.deepcopy(backbone_presets) + + @classproperty + def tokenizer_cls(cls): + return BloomTokenizer diff --git a/keras_nlp/models/bloom/bloom_preprocessor_test.py b/keras_nlp/models/bloom/bloom_preprocessor_test.py new file mode 100644 index 0000000000..bb80006396 --- /dev/null +++ b/keras_nlp/models/bloom/bloom_preprocessor_test.py @@ -0,0 +1,80 @@ +# Copyright 2023 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from keras_nlp.models.bloom.bloom_preprocessor import BloomPreprocessor +from keras_nlp.models.bloom.bloom_tokenizer import BloomTokenizer +from keras_nlp.tests.test_case import TestCase + + +class BloomPreprocessorTest(TestCase): + def setUp(self): + self.vocab = ["", "", ""] + self.vocab += ["!", "air", "Ġair", "plane", "Ġat", "port"] + self.vocab = dict([(token, i) for i, token in enumerate(self.vocab)]) + self.merges = ["Ġ a", "Ġ t", "Ġ i", "Ġ b", "a i", "p l", "n e"] + self.merges += ["Ġa t", "p o", "r t", "Ġt h", "ai r", "pl a", "po rt"] + self.merges += ["Ġai r", "Ġa i", "pla ne"] + self.tokenizer = BloomTokenizer( + vocabulary=self.vocab, + merges=self.merges, + ) + self.init_kwargs = { + "tokenizer": self.tokenizer, + "sequence_length": 8, + } + self.input_data = ["airplane at airport"] + + def test_preprocessor_basics(self): + self.run_preprocessing_layer_test( + cls=BloomPreprocessor, + init_kwargs=self.init_kwargs, + input_data=self.input_data, + expected_output={ + "token_ids": [[1, 4, 6, 7, 5, 8, 2, 0]], + "padding_mask": [[1, 1, 1, 1, 1, 1, 1, 0]], + }, + ) + + def test_no_start_end_token(self): + input_data = ["airplane at airport"] * 4 + + preprocessor = BloomPreprocessor( + tokenizer=BloomTokenizer( + vocabulary=self.vocab, + merges=self.merges, + ), + sequence_length=8, + add_start_token=False, + add_end_token=False, + ) + x = preprocessor(input_data) + self.assertAllEqual(x["token_ids"], [[4, 6, 7, 5, 8, 0, 0, 0]] * 4) + self.assertAllEqual(x["padding_mask"], [[1, 1, 1, 1, 1, 0, 0, 0]] * 4) + + def test_sequence_length_override(self): + input_data = "airplane at airport" + preprocessor = BloomPreprocessor(**self.init_kwargs) + x = preprocessor(input_data, sequence_length=4) + self.assertAllEqual(x["token_ids"], [1, 4, 6, 2]) + + @pytest.mark.extra_large + def test_all_presets(self): + for preset in BloomPreprocessor.presets: + self.run_preset_test( + cls=BloomPreprocessor, + preset=preset, + input_data=self.input_data, + ) From c3c268a0c8768411db000014e9dce306c448fd84 Mon Sep 17 00:00:00 2001 From: Matt Watson <1389937+mattdangerw@users.noreply.github.com> Date: Mon, 12 Feb 2024 09:35:58 -0800 Subject: [PATCH 14/29] Add layer attributes for all functional models (#1421) * Add layer attributes for all functional models This proposes a new style for backbone and tasks: ```python self.token_embedding = ... self.transformer_layers = [] for i in range(num_layers): self.transformer_layers.append(...) inputs = keras.Input(...) x = self.token_embedding(inputs) for layer in self.transformer_layers: x = layer(x) outputs = x super.__init__(inputs, outputs) self.num_layers = num_layers ``` The main goal is to allow more readable manipulations of the model, e.g. a custom LoRA routine: ```python backbone = GPT2Backbone(...) for layer in self.transformer_layers: # Use different rank for different matrices. layer.self_attention.query_projection.enable_lora(16) layer.self_attention.key_projection.enable_lora(4) ``` * Small consistency changes --- keras_nlp/models/albert/albert_backbone.py | 126 +++++++------- keras_nlp/models/albert/albert_classifier.py | 29 ++-- keras_nlp/models/albert/albert_masked_lm.py | 31 ++-- keras_nlp/models/backbone.py | 34 ++-- keras_nlp/models/bart/bart_backbone.py | 129 +++++++------- keras_nlp/models/bart/bart_seq_2_seq_lm.py | 66 +++----- keras_nlp/models/bert/bert_backbone.py | 87 +++++----- keras_nlp/models/bert/bert_classifier.py | 29 ++-- keras_nlp/models/bert/bert_masked_lm.py | 28 +-- keras_nlp/models/bloom/bloom_backbone.py | 52 +++--- .../models/deberta_v3/deberta_v3_backbone.py | 61 ++++--- .../deberta_v3/deberta_v3_classifier.py | 38 +++-- .../models/deberta_v3/deberta_v3_masked_lm.py | 29 ++-- .../distil_bert/distil_bert_backbone.py | 52 +++--- .../distil_bert/distil_bert_classifier.py | 33 ++-- .../distil_bert/distil_bert_masked_lm.py | 29 ++-- keras_nlp/models/electra/electra_backbone.py | 95 ++++++----- keras_nlp/models/f_net/f_net_backbone.py | 84 ++++----- keras_nlp/models/f_net/f_net_classifier.py | 28 +-- keras_nlp/models/f_net/f_net_masked_lm.py | 30 ++-- keras_nlp/models/gpt2/gpt2_backbone.py | 84 ++++----- keras_nlp/models/gpt2/gpt2_causal_lm.py | 34 ++-- .../models/gpt_neo_x/gpt_neo_x_backbone.py | 51 +++--- .../models/gpt_neo_x/gpt_neo_x_causal_lm.py | 24 ++- keras_nlp/models/llama/llama_backbone.py | 50 +++--- keras_nlp/models/mistral/mistral_backbone.py | 47 +++-- keras_nlp/models/opt/opt_backbone.py | 47 ++--- keras_nlp/models/opt/opt_causal_lm.py | 24 ++- keras_nlp/models/roberta/roberta_backbone.py | 53 +++--- .../models/roberta/roberta_classifier.py | 44 +++-- keras_nlp/models/roberta/roberta_masked_lm.py | 29 ++-- keras_nlp/models/t5/t5_backbone.py | 141 ++++++++------- keras_nlp/models/task.py | 35 +++- keras_nlp/models/task_test.py | 4 +- keras_nlp/models/whisper/whisper_backbone.py | 160 +++++++++--------- .../xlm_roberta/xlm_roberta_classifier.py | 43 +++-- .../xlm_roberta/xlm_roberta_masked_lm.py | 29 ++-- keras_nlp/models/xlnet/xlnet_backbone.py | 74 ++++---- keras_nlp/utils/pipeline_model.py | 73 +++----- keras_nlp/utils/pipeline_model_test.py | 155 ++--------------- 40 files changed, 1119 insertions(+), 1172 deletions(-) diff --git a/keras_nlp/models/albert/albert_backbone.py b/keras_nlp/models/albert/albert_backbone.py index 414bb97e87..99a7cc97a3 100644 --- a/keras_nlp/models/albert/albert_backbone.py +++ b/keras_nlp/models/albert/albert_backbone.py @@ -118,67 +118,47 @@ def __init__( f"`num_layers={num_layers}` and `num_groups={num_groups}`." ) - # Index of classification token in the vocabulary - cls_token_index = 0 - # Inputs - token_id_input = keras.Input( - shape=(None,), dtype="int32", name="token_ids" - ) - segment_id_input = keras.Input( - shape=(None,), dtype="int32", name="segment_ids" - ) - padding_mask = keras.Input( - shape=(None,), dtype="int32", name="padding_mask" - ) - - # Embed tokens, positions, and segment ids. - token_embedding_layer = ReversibleEmbedding( + # === Layers === + self.token_embedding = ReversibleEmbedding( input_dim=vocabulary_size, output_dim=embedding_dim, embeddings_initializer=albert_kernel_initializer(), name="token_embedding", ) - token_embedding = token_embedding_layer(token_id_input) - position_embedding = PositionEmbedding( + self.position_embedding = PositionEmbedding( initializer=albert_kernel_initializer(), sequence_length=max_sequence_length, name="position_embedding", - )(token_embedding) - segment_embedding = keras.layers.Embedding( + ) + self.segment_embedding = keras.layers.Embedding( input_dim=num_segments, output_dim=embedding_dim, embeddings_initializer=albert_kernel_initializer(), name="segment_embedding", - )(segment_id_input) - - # Sum, normalize and apply dropout to embeddings. - x = keras.layers.Add()( - (token_embedding, position_embedding, segment_embedding) ) - x = keras.layers.LayerNormalization( + self.embeddings_add = keras.layers.Add( + name="embeddings_add", + ) + self.embeddings_layer_norm = keras.layers.LayerNormalization( name="embeddings_layer_norm", axis=-1, epsilon=1e-12, dtype="float32", - )(x) - x = keras.layers.Dropout( + ) + self.embeddings_dropout = keras.layers.Dropout( dropout, name="embeddings_dropout", - )(x) - - # Project the embedding to `hidden_dim`. - x = keras.layers.Dense( + ) + self.embeddings_projection = keras.layers.Dense( hidden_dim, kernel_initializer=albert_kernel_initializer(), name="embedding_projection", - )(x) - - def get_group_layer(group_idx): - """Defines a group `num_inner_repetitions` transformer layers and - returns the callable. - """ - transformer_layers = [ - TransformerEncoder( + ) + self.transformer_layers = [] + for group_idx in range(num_groups): + inner_layers = [] + for inner_idx in range(num_inner_repetitions): + layer = TransformerEncoder( num_heads=num_heads, intermediate_dim=intermediate_dim, activation=gelu_approximate, @@ -187,43 +167,51 @@ def get_group_layer(group_idx): kernel_initializer=albert_kernel_initializer(), name=f"group_{group_idx}_inner_layer_{inner_idx}", ) - for inner_idx in range(num_inner_repetitions) - ] - - def call(x, padding_mask): - for transformer_layer in transformer_layers: - x = transformer_layer(x, padding_mask=padding_mask) - return x - - return call - - num_calls_per_group = num_layers // num_groups - for group_idx in range(num_groups): - # Define the group. A group in ALBERT terminology is any number of - # repeated attention and FFN blocks. - group_layer = get_group_layer(group_idx) - - # Assume num_layers = 8, num_groups = 4. Then, the order of group - # calls will be 0, 0, 1, 1, 2, 2, 3, 3. - for call in range(num_calls_per_group): - x = group_layer(x, padding_mask=padding_mask) - - # Construct the two ALBERT outputs. The pooled output is a dense layer on - # top of the [CLS] token. - sequence_output = x - pooled_output = keras.layers.Dense( + inner_layers.append(layer) + self.transformer_layers.append(inner_layers) + self.pooled_dense = keras.layers.Dense( hidden_dim, kernel_initializer=albert_kernel_initializer(), activation="tanh", name="pooled_dense", - )(x[:, cls_token_index, :]) + ) - # Instantiate using Functional API Model constructor + # === Functional Model === + # Inputs + token_id_input = keras.Input( + shape=(None,), dtype="int32", name="token_ids" + ) + segment_id_input = keras.Input( + shape=(None,), dtype="int32", name="segment_ids" + ) + padding_mask_input = keras.Input( + shape=(None,), dtype="int32", name="padding_mask" + ) + # Embed tokens, positions, and segment ids. + tokens = self.token_embedding(token_id_input) + positions = self.position_embedding(tokens) + segments = self.segment_embedding(segment_id_input) + # Sum, normalize and apply dropout to embeddings. + x = self.embeddings_add((tokens, positions, segments)) + x = self.embeddings_layer_norm(x) + x = self.embeddings_dropout(x) + x = self.embeddings_projection(x) + # Call transformer layers with repeated groups. + num_calls_per_group = num_layers // num_groups + for group in self.transformer_layers: + for _ in range(num_calls_per_group): + for transformer_layer in group: + x = transformer_layer(x, padding_mask=padding_mask_input) + # Construct the two ALBERT outputs. The pooled output is a dense layer + # on top of the [CLS] token. + sequence_output = x + cls_token_index = 0 + pooled_output = self.pooled_dense(x[:, cls_token_index, :]) super().__init__( inputs={ "token_ids": token_id_input, "segment_ids": segment_id_input, - "padding_mask": padding_mask, + "padding_mask": padding_mask_input, }, outputs={ "sequence_output": sequence_output, @@ -231,7 +219,8 @@ def call(x, padding_mask): }, **kwargs, ) - # All references to `self` below this line + + # === Config === self.vocabulary_size = vocabulary_size self.num_layers = num_layers self.num_heads = num_heads @@ -244,7 +233,6 @@ def call(x, padding_mask): self.max_sequence_length = max_sequence_length self.num_segments = num_segments self.cls_token_index = cls_token_index - self.token_embedding = token_embedding_layer def get_config(self): config = super().get_config() diff --git a/keras_nlp/models/albert/albert_classifier.py b/keras_nlp/models/albert/albert_classifier.py index b0ed7bca7c..e8621cf6b4 100644 --- a/keras_nlp/models/albert/albert_classifier.py +++ b/keras_nlp/models/albert/albert_classifier.py @@ -155,30 +155,37 @@ def __init__( dropout=0.1, **kwargs, ): - inputs = backbone.input - pooled = backbone(inputs)["pooled_output"] - pooled = keras.layers.Dropout(dropout)(pooled) - outputs = keras.layers.Dense( + # === Layers === + self.backbone = backbone + self.preprocessor = preprocessor + self.output_dense = keras.layers.Dense( num_classes, kernel_initializer=albert_kernel_initializer(), activation=activation, name="logits", - )(pooled) - # Instantiate using Functional API Model constructor + ) + self.output_dropout = keras.layers.Dropout( + dropout, + name="output_dropout", + ) + + # === Functional Model === + inputs = backbone.input + pooled = backbone(inputs)["pooled_output"] + pooled = self.output_dropout(pooled) + outputs = self.output_dense(pooled) super().__init__( inputs=inputs, outputs=outputs, - include_preprocessing=preprocessor is not None, **kwargs, ) - # All references to `self` below this line - self._backbone = backbone - self._preprocessor = preprocessor + + # === Config === self.num_classes = num_classes self.activation = keras.activations.get(activation) self.dropout = dropout - # Default compilation + # === Default compilation === logit_output = self.activation == keras.activations.linear self.compile( loss=keras.losses.SparseCategoricalCrossentropy( diff --git a/keras_nlp/models/albert/albert_masked_lm.py b/keras_nlp/models/albert/albert_masked_lm.py index e95af7c207..139be0a762 100644 --- a/keras_nlp/models/albert/albert_masked_lm.py +++ b/keras_nlp/models/albert/albert_masked_lm.py @@ -97,32 +97,35 @@ class AlbertMaskedLM(Task): """ def __init__(self, backbone, preprocessor=None, **kwargs): + # === Layers === + self.backbone = backbone + self.preprocessor = preprocessor + self.masked_lm_head = MaskedLMHead( + vocabulary_size=backbone.vocabulary_size, + token_embedding=backbone.token_embedding, + intermediate_activation=gelu_approximate, + kernel_initializer=albert_kernel_initializer(), + name="mlm_head", + ) + + # === Functional Model === inputs = { **backbone.input, "mask_positions": keras.Input( shape=(None,), dtype="int32", name="mask_positions" ), } - backbone_outputs = backbone(backbone.input) - outputs = MaskedLMHead( - vocabulary_size=backbone.vocabulary_size, - token_embedding=backbone.token_embedding, - intermediate_activation=gelu_approximate, - kernel_initializer=albert_kernel_initializer(), - name="mlm_head", - )(backbone_outputs["sequence_output"], inputs["mask_positions"]) - + outputs = self.masked_lm_head( + backbone_outputs["sequence_output"], inputs["mask_positions"] + ) super().__init__( inputs=inputs, outputs=outputs, - include_preprocessing=preprocessor is not None, - **kwargs + **kwargs, ) - self.backbone = backbone - self.preprocessor = preprocessor - + # === Default compilation === self.compile( loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True), optimizer=keras.optimizers.Adam(5e-5), diff --git a/keras_nlp/models/backbone.py b/keras_nlp/models/backbone.py index 69da56593b..9e22be3f44 100644 --- a/keras_nlp/models/backbone.py +++ b/keras_nlp/models/backbone.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from keras_nlp.backend import config from keras_nlp.backend import keras from keras_nlp.utils.preset_utils import check_preset_class from keras_nlp.utils.preset_utils import load_from_preset @@ -23,24 +24,40 @@ class Backbone(keras.Model): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self._token_embedding = None self._functional_layer_ids = set( id(layer) for layer in self._flatten_layers() ) + self._initialized = True def __dir__(self): - # Temporary fixes for weight saving. This mimics the following PR for + if config.keras_3(): + return super().__dir__() + + # Temporary fixes for Keras 2 saving. This mimics the following PR for # older version of Keras: https://github.com/keras-team/keras/pull/18982 def filter_fn(attr): - if attr == "_layer_checkpoint_dependencies": + if attr in [ + "_layer_checkpoint_dependencies", + "transformer_layers", + "encoder_transformer_layers", + "decoder_transformer_layers", + ]: return False return id(getattr(self, attr)) not in self._functional_layer_ids return filter(filter_fn, super().__dir__()) def __setattr__(self, name, value): - # Work around torch setattr for properties. - if name in ["token_embedding"]: + # Work around setattr issues for Keras 2 and Keras 3 torch backend. + # Since all our state is covered by functional model we can route + # around custom setattr calls. + is_property = isinstance(getattr(type(self), name, None), property) + is_unitialized = not hasattr(self, "_initialized") + is_torch = config.backend() == "torch" + is_keras_2 = not config.keras_3() + if is_torch and (is_property or is_unitialized): + return object.__setattr__(self, name, value) + if is_keras_2 and is_unitialized: return object.__setattr__(self, name, value) return super().__setattr__(name, value) @@ -48,18 +65,13 @@ def __setattr__(self, name, value): def token_embedding(self): """A `keras.layers.Embedding` instance for embedding token ids. - This layer integer token ids to the hidden dim of the model. + This layer embeds integer token ids to the hidden dim of the model. """ return self._token_embedding @token_embedding.setter def token_embedding(self, value): - # Workaround tf.keras h5 checkpoint loading, which is sensitive to layer - # count mismatches and does not deduplicate layers. This could go away - # if we update our checkpoints to the newer `.weights.h5` format. - self._setattr_tracking = False self._token_embedding = value - self._setattr_tracking = True def get_config(self): # Don't chain to super here. The default `get_config()` for functional diff --git a/keras_nlp/models/bart/bart_backbone.py b/keras_nlp/models/bart/bart_backbone.py index 2679b84a9f..fdb8e5df5b 100644 --- a/keras_nlp/models/bart/bart_backbone.py +++ b/keras_nlp/models/bart/bart_backbone.py @@ -102,59 +102,34 @@ def __init__( max_sequence_length=1024, **kwargs, ): - # Encoder inputs - encoder_token_id_input = keras.Input( - shape=(None,), dtype="int32", name="encoder_token_ids" - ) - encoder_padding_mask = keras.Input( - shape=(None,), dtype="int32", name="encoder_padding_mask" - ) - - # Decoder inputs. - decoder_token_id_input = keras.Input( - shape=(None,), dtype="int32", name="decoder_token_ids" - ) - decoder_padding_mask = keras.Input( - shape=(None,), dtype="int32", name="decoder_padding_mask" - ) - - # Token embedding layer. This layer is shared by encoder and decoder. - token_embedding_layer = ReversibleEmbedding( + # === Layers === + self.token_embedding = ReversibleEmbedding( input_dim=vocabulary_size, output_dim=hidden_dim, embeddings_initializer=bart_kernel_initializer(), name="token_embedding", ) - - # ===== Encoder ===== - - # Embed tokens and positions. - token_embedding = token_embedding_layer(encoder_token_id_input) - # Position embedding parameters are not shared by encode and decoder. - position_embedding = PositionEmbedding( + self.encoder_position_embedding = PositionEmbedding( initializer=bart_kernel_initializer(), sequence_length=max_sequence_length, name="encoder_position_embedding", - )(token_embedding) - - # Sum, normalize and apply dropout to embeddings. - x = keras.layers.Add(name="encoder_embeddings_add")( - (token_embedding, position_embedding) ) - x = keras.layers.LayerNormalization( + self.encoder_embeddings_add = keras.layers.Add( + name="encoder_embeddings_add", + ) + self.encoder_embeddings_layer_norm = keras.layers.LayerNormalization( name="encoder_embeddings_layer_norm", axis=-1, epsilon=1e-5, dtype="float32", - )(x) - x = keras.layers.Dropout( + ) + self.encoder_embeddings_dropout = keras.layers.Dropout( dropout, name="encoder_embeddings_dropout", - )(x) - - # Apply successive transformer encoder blocks. + ) + self.encoder_transformer_layers = [] for i in range(num_layers): - x = TransformerEncoder( + layer = TransformerEncoder( num_heads=num_heads, intermediate_dim=intermediate_dim, activation=keras.activations.gelu, @@ -162,39 +137,29 @@ def __init__( layer_norm_epsilon=1e-5, kernel_initializer=bart_kernel_initializer(), name=f"transformer_encoder_layer_{i}", - )(x, padding_mask=encoder_padding_mask) - - encoder_output = x - - # ===== Decoder ===== - - # Embed tokens and positions. - token_embedding = token_embedding_layer(decoder_token_id_input) - # Position embedding parameters are not shared by encode and decoder. - position_embedding = PositionEmbedding( + ) + self.encoder_transformer_layers.append(layer) + self.decoder_position_embedding = PositionEmbedding( initializer=bart_kernel_initializer(), sequence_length=max_sequence_length, name="decoder_position_embedding", - )(token_embedding) - - # Sum, normalize and apply dropout to embeddings. - x = keras.layers.Add(name="decoder_embeddings_add")( - (token_embedding, position_embedding) ) - x = keras.layers.LayerNormalization( + self.decoder_embeddings_add = keras.layers.Add( + name="decoder_embeddings_add", + ) + self.decoder_embeddings_layer_norm = keras.layers.LayerNormalization( name="decoder_embeddings_layer_norm", axis=-1, epsilon=1e-5, dtype="float32", - )(x) - x = keras.layers.Dropout( + ) + self.decoder_embeddings_dropout = keras.layers.Dropout( dropout, name="decoder_embeddings_dropout", - )(x) - - # Apply successive transformer decoder blocks. + ) + self.decoder_transformer_layers = [] for i in range(num_layers): - transformer_decoder_layer = TransformerDecoder( + layer = TransformerDecoder( intermediate_dim=intermediate_dim, num_heads=num_heads, dropout=dropout, @@ -203,22 +168,51 @@ def __init__( kernel_initializer=bart_kernel_initializer(), name=f"transformer_decoder_layer_{i}", ) - x = transformer_decoder_layer( + self.decoder_transformer_layers.append(layer) + + # === Functional Model === + encoder_token_id_input = keras.Input( + shape=(None,), dtype="int32", name="encoder_token_ids" + ) + encoder_padding_mask_input = keras.Input( + shape=(None,), dtype="int32", name="encoder_padding_mask" + ) + decoder_token_id_input = keras.Input( + shape=(None,), dtype="int32", name="decoder_token_ids" + ) + decoder_padding_mask_input = keras.Input( + shape=(None,), dtype="int32", name="decoder_padding_mask" + ) + # Encoder. + tokens = self.token_embedding(encoder_token_id_input) + positions = self.encoder_position_embedding(tokens) + x = self.encoder_embeddings_add((tokens, positions)) + x = self.encoder_embeddings_layer_norm(x) + x = self.encoder_embeddings_dropout(x) + for transformer_layer in self.encoder_transformer_layers: + x = transformer_layer(x, padding_mask=encoder_padding_mask_input) + encoder_output = x + # Decoder. + tokens = self.token_embedding(decoder_token_id_input) + positions = self.decoder_position_embedding(tokens) + x = self.decoder_embeddings_add((tokens, positions)) + x = self.decoder_embeddings_layer_norm(x) + x = self.decoder_embeddings_dropout(x) + for transformer_layer in self.decoder_transformer_layers: + x = transformer_layer( decoder_sequence=x, encoder_sequence=encoder_output, - decoder_padding_mask=decoder_padding_mask, - encoder_padding_mask=encoder_padding_mask, + decoder_padding_mask=decoder_padding_mask_input, + encoder_padding_mask=encoder_padding_mask_input, ) - decoder_output = x - # Instantiate using Functional API Model constructor super().__init__( inputs={ "encoder_token_ids": encoder_token_id_input, - "encoder_padding_mask": encoder_padding_mask, + "encoder_padding_mask": encoder_padding_mask_input, "decoder_token_ids": decoder_token_id_input, - "decoder_padding_mask": decoder_padding_mask, + "decoder_padding_mask": decoder_padding_mask_input, }, outputs={ "encoder_sequence_output": encoder_output, @@ -227,7 +221,7 @@ def __init__( **kwargs, ) - # All references to `self` below this line + # === Config === self.vocabulary_size = vocabulary_size self.num_layers = num_layers self.num_heads = num_heads @@ -235,7 +229,6 @@ def __init__( self.intermediate_dim = intermediate_dim self.dropout = dropout self.max_sequence_length = max_sequence_length - self.token_embedding = token_embedding_layer def get_config(self): config = super().get_config() diff --git a/keras_nlp/models/bart/bart_seq_2_seq_lm.py b/keras_nlp/models/bart/bart_seq_2_seq_lm.py index 2131519ce3..c17eafdb02 100644 --- a/keras_nlp/models/bart/bart_seq_2_seq_lm.py +++ b/keras_nlp/models/bart/bart_seq_2_seq_lm.py @@ -185,24 +185,21 @@ def __init__( preprocessor=None, **kwargs, ): + # === Layers === + self.backbone = backbone + self.preprocessor = preprocessor + + # === Functional Model === inputs = backbone.input hidden_states = backbone(inputs)["decoder_sequence_output"] outputs = backbone.token_embedding(hidden_states, reverse=True) - - # Instantiate using Functional API Model constructor. super().__init__( inputs=inputs, outputs=outputs, - include_preprocessing=preprocessor is not None, **kwargs, ) - self.backbone = backbone - self.preprocessor = preprocessor - self.generate_function = None - self._sampler = None - - # Default compilation + # === Default compilation === self.compile( loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True), optimizer=keras.optimizers.Adam(2e-5), @@ -280,33 +277,28 @@ def call_decoder_with_cache( cross-attention layer. """ # Embedding layers. - token_embedding = self.backbone.get_layer("token_embedding")( - decoder_token_ids + tokens = self.backbone.token_embedding(decoder_token_ids) + positions = self.backbone.decoder_position_embedding( + tokens, + start_index=self_attention_cache_update_index, ) - position_embedding = self.backbone.get_layer( - "decoder_position_embedding" - )(token_embedding, start_index=self_attention_cache_update_index) - # Sum, normalize and apply dropout to embeddings. - x = self.backbone.get_layer("decoder_embeddings_add")( - (token_embedding, position_embedding) - ) - x = self.backbone.get_layer("decoder_embeddings_layer_norm")(x) - x = self.backbone.get_layer("decoder_embeddings_dropout")(x) + x = self.backbone.decoder_embeddings_add((tokens, positions)) + x = self.backbone.decoder_embeddings_layer_norm(x) + x = self.backbone.decoder_embeddings_dropout(x) # Every decoder layer has a separate cache for the self-attention layer # and the cross-attention layer. We update all of them separately. self_attention_caches = [] cross_attention_caches = [] - for i in range(self.backbone.num_layers): + for i, layer in enumerate(self.backbone.decoder_transformer_layers): current_self_attention_cache = self_attention_cache[:, i, ...] current_cross_attention_cache = cross_attention_cache[:, i, ...] - ( x, next_self_attention_cache, next_cross_attention_cache, - ) = self.backbone.get_layer(f"transformer_decoder_layer_{i}")( + ) = layer( decoder_sequence=x, encoder_sequence=encoder_hidden_states, encoder_padding_mask=encoder_padding_mask, @@ -315,7 +307,6 @@ def call_decoder_with_cache( cross_attention_cache=current_cross_attention_cache, cross_attention_cache_update_index=cross_attention_cache_update_index, ) - if self_attention_cache_update_index is not None: self_attention_caches.append(next_self_attention_cache) if cross_attention_cache_update_index is not None: @@ -337,26 +328,13 @@ def call_decoder_with_cache( def call_encoder(self, token_ids, padding_mask): """Does a forward pass on the encoder and returns the encoder output.""" - - # Embedding layers. - token_embedding = self.backbone.get_layer("token_embedding")(token_ids) - position_embedding = self.backbone.get_layer( - "encoder_position_embedding" - )(token_embedding) - - # Sum, normalize and apply dropout to embeddings. - x = self.backbone.get_layer("encoder_embeddings_add")( - (token_embedding, position_embedding) - ) - x = self.backbone.get_layer("encoder_embeddings_layer_norm")(x) - x = self.backbone.get_layer("encoder_embeddings_dropout")(x) - - # Transformer encoder layers. - for i in range(self.backbone.num_layers): - x = self.backbone.get_layer(f"transformer_encoder_layer_{i}")( - x, padding_mask=padding_mask - ) - + tokens = self.backbone.token_embedding(token_ids) + positions = self.backbone.encoder_position_embedding(tokens) + x = self.backbone.decoder_embeddings_add((tokens, positions)) + x = self.backbone.encoder_embeddings_layer_norm(x) + x = self.backbone.encoder_embeddings_dropout(x) + for transformer_layer in self.backbone.encoder_transformer_layers: + x = transformer_layer(x, padding_mask=padding_mask) return x def _initialize_cache(self, encoder_token_ids, decoder_token_ids): diff --git a/keras_nlp/models/bert/bert_backbone.py b/keras_nlp/models/bert/bert_backbone.py index 174b0f0e42..f511de3687 100644 --- a/keras_nlp/models/bert/bert_backbone.py +++ b/keras_nlp/models/bert/bert_backbone.py @@ -99,57 +99,40 @@ def __init__( num_segments=2, **kwargs, ): - # Index of classification token in the vocabulary - cls_token_index = 0 - # Inputs - token_id_input = keras.Input( - shape=(None,), dtype="int32", name="token_ids" - ) - segment_id_input = keras.Input( - shape=(None,), dtype="int32", name="segment_ids" - ) - padding_mask = keras.Input( - shape=(None,), dtype="int32", name="padding_mask" - ) - - # Embed tokens, positions, and segment ids. - token_embedding_layer = ReversibleEmbedding( + # === Layers === + self.token_embedding = ReversibleEmbedding( input_dim=vocabulary_size, output_dim=hidden_dim, embeddings_initializer=bert_kernel_initializer(), name="token_embedding", ) - token_embedding = token_embedding_layer(token_id_input) - position_embedding = PositionEmbedding( + self.position_embedding = PositionEmbedding( initializer=bert_kernel_initializer(), sequence_length=max_sequence_length, name="position_embedding", - )(token_embedding) - segment_embedding = keras.layers.Embedding( + ) + self.segment_embedding = keras.layers.Embedding( input_dim=num_segments, output_dim=hidden_dim, embeddings_initializer=bert_kernel_initializer(), name="segment_embedding", - )(segment_id_input) - - # Sum, normalize and apply dropout to embeddings. - x = keras.layers.Add()( - (token_embedding, position_embedding, segment_embedding) ) - x = keras.layers.LayerNormalization( + self.embeddings_add = keras.layers.Add( + name="embeddings_add", + ) + self.embeddings_layer_norm = keras.layers.LayerNormalization( name="embeddings_layer_norm", axis=-1, epsilon=1e-12, dtype="float32", - )(x) - x = keras.layers.Dropout( + ) + self.embeddings_dropout = keras.layers.Dropout( dropout, name="embeddings_dropout", - )(x) - - # Apply successive transformer encoder blocks. + ) + self.transformer_layers = [] for i in range(num_layers): - x = TransformerEncoder( + layer = TransformerEncoder( num_heads=num_heads, intermediate_dim=intermediate_dim, activation=gelu_approximate, @@ -157,24 +140,45 @@ def __init__( layer_norm_epsilon=1e-12, kernel_initializer=bert_kernel_initializer(), name=f"transformer_layer_{i}", - )(x, padding_mask=padding_mask) - - # Construct the two BERT outputs. The pooled output is a dense layer on - # top of the [CLS] token. - sequence_output = x - pooled_output = keras.layers.Dense( + ) + self.transformer_layers.append(layer) + self.pooled_dense = keras.layers.Dense( hidden_dim, kernel_initializer=bert_kernel_initializer(), activation="tanh", name="pooled_dense", - )(x[:, cls_token_index, :]) + ) - # Instantiate using Functional API Model constructor + # === Functional Model === + token_id_input = keras.Input( + shape=(None,), dtype="int32", name="token_ids" + ) + segment_id_input = keras.Input( + shape=(None,), dtype="int32", name="segment_ids" + ) + padding_mask_input = keras.Input( + shape=(None,), dtype="int32", name="padding_mask" + ) + # Embed tokens, positions, and segment ids. + tokens = self.token_embedding(token_id_input) + positions = self.position_embedding(tokens) + segments = self.segment_embedding(segment_id_input) + # Sum, normalize and apply dropout to embeddings. + x = self.embeddings_add((tokens, positions, segments)) + x = self.embeddings_layer_norm(x) + x = self.embeddings_dropout(x) + for transformer_layer in self.transformer_layers: + x = transformer_layer(x, padding_mask=padding_mask_input) + # Construct the two BERT outputs. The pooled output is a dense layer on + # top of the [CLS] token. + sequence_output = x + cls_token_index = 0 + pooled_output = self.pooled_dense(x[:, cls_token_index, :]) super().__init__( inputs={ "token_ids": token_id_input, "segment_ids": segment_id_input, - "padding_mask": padding_mask, + "padding_mask": padding_mask_input, }, outputs={ "sequence_output": sequence_output, @@ -183,7 +187,7 @@ def __init__( **kwargs, ) - # All references to `self` below this line + # === Config === self.vocabulary_size = vocabulary_size self.num_layers = num_layers self.num_heads = num_heads @@ -193,7 +197,6 @@ def __init__( self.max_sequence_length = max_sequence_length self.num_segments = num_segments self.cls_token_index = cls_token_index - self.token_embedding = token_embedding_layer def get_config(self): config = super().get_config() diff --git a/keras_nlp/models/bert/bert_classifier.py b/keras_nlp/models/bert/bert_classifier.py index 2a9aa548bf..3ddb90a9d2 100644 --- a/keras_nlp/models/bert/bert_classifier.py +++ b/keras_nlp/models/bert/bert_classifier.py @@ -140,30 +140,37 @@ def __init__( dropout=0.1, **kwargs, ): - inputs = backbone.input - pooled = backbone(inputs)["pooled_output"] - pooled = keras.layers.Dropout(dropout)(pooled) - outputs = keras.layers.Dense( + # === Layers === + self.backbone = backbone + self.preprocessor = preprocessor + self.output_dropout = keras.layers.Dropout( + dropout, + name="classifier_dropout", + ) + self.output_dense = keras.layers.Dense( num_classes, kernel_initializer=bert_kernel_initializer(), activation=activation, name="logits", - )(pooled) - # Instantiate using Functional API Model constructor + ) + + # === Functional Model === + inputs = backbone.input + pooled = backbone(inputs)["pooled_output"] + pooled = self.output_dropout(pooled) + outputs = self.output_dense(pooled) super().__init__( inputs=inputs, outputs=outputs, - include_preprocessing=preprocessor is not None, **kwargs, ) - # All references to `self` below this line - self.backbone = backbone - self.preprocessor = preprocessor + + # === Config === self.num_classes = num_classes self.activation = keras.activations.get(activation) self.dropout = dropout - # Default compilation + # === Default compilation === logit_output = self.activation == keras.activations.linear self.compile( loss=keras.losses.SparseCategoricalCrossentropy( diff --git a/keras_nlp/models/bert/bert_masked_lm.py b/keras_nlp/models/bert/bert_masked_lm.py index d4c12d1091..555c562f1f 100644 --- a/keras_nlp/models/bert/bert_masked_lm.py +++ b/keras_nlp/models/bert/bert_masked_lm.py @@ -101,6 +101,18 @@ def __init__( preprocessor=None, **kwargs, ): + # === Layers === + self.backbone = backbone + self.preprocessor = preprocessor + self.masked_lm_head = MaskedLMHead( + vocabulary_size=backbone.vocabulary_size, + token_embedding=backbone.token_embedding, + intermediate_activation="gelu", + kernel_initializer=bert_kernel_initializer(), + name="mlm_head", + ) + + # === Functional Model === inputs = { **backbone.input, "mask_positions": keras.Input( @@ -108,22 +120,16 @@ def __init__( ), } backbone_outputs = backbone(backbone.input) - outputs = MaskedLMHead( - vocabulary_size=backbone.vocabulary_size, - token_embedding=backbone.token_embedding, - intermediate_activation="gelu", - kernel_initializer=bert_kernel_initializer(), - name="mlm_head", - )(backbone_outputs["sequence_output"], inputs["mask_positions"]) - - # Instantiate using Functional API Model constructor + outputs = self.masked_lm_head( + backbone_outputs["sequence_output"], inputs["mask_positions"] + ) super().__init__( inputs=inputs, outputs=outputs, - include_preprocessing=preprocessor is not None, **kwargs, ) - # All references to `self` below this line + + # === Default compilation === self.backbone = backbone self.preprocessor = preprocessor self.compile( diff --git a/keras_nlp/models/bloom/bloom_backbone.py b/keras_nlp/models/bloom/bloom_backbone.py index 86a2e2b4c3..4f12bacd19 100644 --- a/keras_nlp/models/bloom/bloom_backbone.py +++ b/keras_nlp/models/bloom/bloom_backbone.py @@ -95,46 +95,55 @@ def __init__( max_sequence_length=2048, **kwargs, ): - token_ids = keras.Input(shape=(None,), dtype="int32", name="token_ids") - padding_mask = keras.Input( - shape=(None,), dtype="int32", name="padding_mask" - ) - - # Embed tokens - token_embedding_layer = ReversibleEmbedding( + # === Layers === + self.token_embedding = ReversibleEmbedding( input_dim=vocabulary_size, output_dim=hidden_dim, embeddings_initializer=_bloom_kernel_initializer(stddev=0.02), tie_weights=False, name="token_embedding", ) - token_embedding = token_embedding_layer(token_ids) - - x = keras.layers.LayerNormalization( - epsilon=layer_norm_epsilon, name="token_embedding_layernorm" - )(token_embedding) - + self.embeddings_layer_norm = keras.layers.LayerNormalization( + epsilon=layer_norm_epsilon, + name="token_embedding_layernorm", + ) + self.transformer_layers = [] for i in range(num_layers): - x = BloomDecoder( + layer = BloomDecoder( num_heads=num_heads, intermediate_dim=intermediate_dim, dropout=dropout, layer_norm_epsilon=layer_norm_epsilon, name=f"transformer_layer_{i}", - )(x, decoder_padding_mask=padding_mask) - - sequence_output = keras.layers.LayerNormalization( - epsilon=layer_norm_epsilon, name="final_layernorm" - )(x) + ) + self.transformer_layers.append(layer) + self.layer_norm = keras.layers.LayerNormalization( + epsilon=layer_norm_epsilon, + name="final_layernorm", + ) + # === Functional Model === + token_id_input = keras.Input( + shape=(None,), dtype="int32", name="token_ids" + ) + padding_mask_input = keras.Input( + shape=(None,), dtype="int32", name="padding_mask" + ) + x = self.token_embedding(token_id_input) + x = self.embeddings_layer_norm(x) + for transformer_layer in self.transformer_layers: + x = transformer_layer(x, decoder_padding_mask=padding_mask_input) + sequence_output = self.layer_norm(x) super().__init__( inputs={ - "token_ids": token_ids, - "padding_mask": padding_mask, + "token_ids": token_id_input, + "padding_mask": padding_mask_input, }, outputs=sequence_output, **kwargs, ) + + # === Config === self.vocabulary_size = vocabulary_size self.num_layers = num_layers self.num_heads = num_heads @@ -143,7 +152,6 @@ def __init__( self.dropout = dropout self.layer_norm_epsilon = layer_norm_epsilon self.max_sequence_length = max_sequence_length - self.token_embedding = token_embedding_layer def get_config(self): config = super().get_config() diff --git a/keras_nlp/models/deberta_v3/deberta_v3_backbone.py b/keras_nlp/models/deberta_v3/deberta_v3_backbone.py index aa5077ec67..87531aefcb 100644 --- a/keras_nlp/models/deberta_v3/deberta_v3_backbone.py +++ b/keras_nlp/models/deberta_v3/deberta_v3_backbone.py @@ -108,46 +108,32 @@ def __init__( bucket_size=256, **kwargs, ): - # Inputs - token_id_input = keras.Input( - shape=(None,), dtype="int32", name="token_ids" - ) - padding_mask = keras.Input( - shape=(None,), dtype="int32", name="padding_mask" - ) - - # Embed tokens. - token_embedding_layer = ReversibleEmbedding( + # === Layers === + self.token_embedding = ReversibleEmbedding( input_dim=vocabulary_size, output_dim=hidden_dim, embeddings_initializer=deberta_kernel_initializer(), name="token_embedding", ) - x = token_embedding_layer(token_id_input) - - # Normalize and apply dropout to embeddings. - x = keras.layers.LayerNormalization( + self.embeddings_layer_norm = keras.layers.LayerNormalization( epsilon=1e-7, dtype="float32", name="embeddings_layer_norm", - )(x) - x = keras.layers.Dropout( + ) + self.embeddings_dropout = keras.layers.Dropout( dropout, name="embeddings_dropout", - )(x) - - # Relative embedding layer. - rel_embeddings = RelativeEmbedding( + ) + self.relative_embeddings = RelativeEmbedding( hidden_dim=hidden_dim, bucket_size=bucket_size, layer_norm_epsilon=1e-7, kernel_initializer=deberta_kernel_initializer(), name="rel_embedding", - )(x) - - # Apply successive DeBERTa encoder blocks. + ) + self.transformer_layers = [] for i in range(num_layers): - x = DisentangledAttentionEncoder( + layer = DisentangledAttentionEncoder( num_heads=num_heads, intermediate_dim=intermediate_dim, max_position_embeddings=max_sequence_length, @@ -157,22 +143,36 @@ def __init__( layer_norm_epsilon=1e-7, kernel_initializer=deberta_kernel_initializer(), name=f"disentangled_attention_encoder_layer_{i}", - )( + ) + self.transformer_layers.append(layer) + + # === Functional Model === + token_id_input = keras.Input( + shape=(None,), dtype="int32", name="token_ids" + ) + padding_mask_input = keras.Input( + shape=(None,), dtype="int32", name="padding_mask" + ) + x = self.token_embedding(token_id_input) + x = self.embeddings_layer_norm(x) + x = self.embeddings_dropout(x) + rel_embeddings = self.relative_embeddings(x) + for transformer_layer in self.transformer_layers: + x = transformer_layer( x, rel_embeddings=rel_embeddings, - padding_mask=padding_mask, + padding_mask=padding_mask_input, ) - - # Instantiate using Functional API Model constructor super().__init__( inputs={ "token_ids": token_id_input, - "padding_mask": padding_mask, + "padding_mask": padding_mask_input, }, outputs=x, **kwargs, ) - # All references to `self` below this line + + # === Config === self.vocabulary_size = vocabulary_size self.num_layers = num_layers self.num_heads = num_heads @@ -182,7 +182,6 @@ def __init__( self.max_sequence_length = max_sequence_length self.bucket_size = bucket_size self.start_token_index = 0 - self.token_embedding = token_embedding_layer def get_config(self): config = super().get_config() diff --git a/keras_nlp/models/deberta_v3/deberta_v3_classifier.py b/keras_nlp/models/deberta_v3/deberta_v3_classifier.py index b03122064d..f5249cb34b 100644 --- a/keras_nlp/models/deberta_v3/deberta_v3_classifier.py +++ b/keras_nlp/models/deberta_v3/deberta_v3_classifier.py @@ -163,32 +163,44 @@ def __init__( dropout=0.0, **kwargs, ): - inputs = backbone.input + # === Layers === + self.backbone = backbone + self.preprocessor = preprocessor + self.pooled_dropout = keras.layers.Dropout( + dropout, + name="pooled_dropout", + ) hidden_dim = hidden_dim or backbone.hidden_dim - - x = backbone(inputs)[:, backbone.start_token_index, :] - x = keras.layers.Dropout(dropout, name="pooled_dropout")(x) - x = keras.layers.Dense( + self.pooled_dense = keras.layers.Dense( hidden_dim, activation=keras.activations.gelu, name="pooled_dense", - )(x) - x = keras.layers.Dropout(backbone.dropout, name="classifier_dropout")(x) - outputs = keras.layers.Dense( + ) + self.output_dropout = keras.layers.Dropout( + backbone.dropout, + name="classifier_dropout", + ) + self.output_dense = keras.layers.Dense( num_classes, kernel_initializer=deberta_kernel_initializer(), activation=activation, name="logits", - )(x) + ) - # Instantiate using Functional API Model constructor + # === Functional Model === + inputs = backbone.input + x = backbone(inputs)[:, backbone.start_token_index, :] + x = self.pooled_dropout(x) + x = self.pooled_dense(x) + x = self.output_dropout(x) + outputs = self.output_dense(x) super().__init__( inputs=inputs, outputs=outputs, - include_preprocessing=preprocessor is not None, **kwargs, ) - # All references to `self` below this line + + # === Config === self.backbone = backbone self.preprocessor = preprocessor self.num_classes = num_classes @@ -196,7 +208,7 @@ def __init__( self.hidden_dim = hidden_dim self.dropout = dropout - # Default compilation + # === Default compilation === logit_output = self.activation == keras.activations.linear self.compile( loss=keras.losses.SparseCategoricalCrossentropy( diff --git a/keras_nlp/models/deberta_v3/deberta_v3_masked_lm.py b/keras_nlp/models/deberta_v3/deberta_v3_masked_lm.py index bf6a850a54..fadb4c0e24 100644 --- a/keras_nlp/models/deberta_v3/deberta_v3_masked_lm.py +++ b/keras_nlp/models/deberta_v3/deberta_v3_masked_lm.py @@ -104,32 +104,33 @@ def __init__( preprocessor=None, **kwargs, ): - inputs = { - **backbone.input, - "mask_positions": keras.Input( - shape=(None,), dtype="int32", name="mask_positions" - ), - } - backbone_outputs = backbone(backbone.input) - outputs = MaskedLMHead( + # === Layers === + self.backbone = backbone + self.preprocessor = preprocessor + self.masked_lm_head = MaskedLMHead( vocabulary_size=backbone.vocabulary_size, token_embedding=backbone.token_embedding, intermediate_activation=keras.activations.gelu, kernel_initializer=deberta_kernel_initializer(), name="mlm_head", - )(backbone_outputs, inputs["mask_positions"]) + ) - # Instantiate using Functional API Model constructor + # === Functional Model === + inputs = { + **backbone.input, + "mask_positions": keras.Input( + shape=(None,), dtype="int32", name="mask_positions" + ), + } + x = backbone(backbone.input) + outputs = self.masked_lm_head(x, inputs["mask_positions"]) super().__init__( inputs=inputs, outputs=outputs, - include_preprocessing=preprocessor is not None, **kwargs, ) - # All references to `self` below this line - self.backbone = backbone - self.preprocessor = preprocessor + # === Default compilation === self.compile( loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True), optimizer=keras.optimizers.Adam(5e-5), diff --git a/keras_nlp/models/distil_bert/distil_bert_backbone.py b/keras_nlp/models/distil_bert/distil_bert_backbone.py index a3634215fa..f97e24f55d 100644 --- a/keras_nlp/models/distil_bert/distil_bert_backbone.py +++ b/keras_nlp/models/distil_bert/distil_bert_backbone.py @@ -100,39 +100,29 @@ def __init__( max_sequence_length=512, **kwargs, ): - # Inputs - token_id_input = keras.Input( - shape=(None,), dtype="int32", name="token_ids" - ) - padding_mask = keras.Input( - shape=(None,), dtype="int32", name="padding_mask" - ) - - # Embed tokens and positions. - embedding_layer = TokenAndPositionEmbedding( + # === Layers === + self.embeddings = TokenAndPositionEmbedding( vocabulary_size=vocabulary_size, sequence_length=max_sequence_length, embedding_dim=hidden_dim, embeddings_initializer=distilbert_kernel_initializer(), name="token_and_position_embedding", ) - x = embedding_layer(token_id_input) - - # Normalize and apply dropout to embeddings. - x = keras.layers.LayerNormalization( + # Keep the token_embedding property for consistency across models. + self.token_embedding = self.embeddings.token_embedding + self.embeddings_layer_norm = keras.layers.LayerNormalization( axis=-1, epsilon=1e-12, dtype="float32", name="embeddings_layer_norm", - )(x) - x = keras.layers.Dropout( + ) + self.embeddings_dropout = keras.layers.Dropout( dropout, name="embeddings_dropout", - )(x) - - # Apply successive transformer encoder blocks. + ) + self.transformer_layers = [] for i in range(num_layers): - x = TransformerEncoder( + layer = TransformerEncoder( num_heads=num_heads, intermediate_dim=intermediate_dim, activation="gelu", @@ -140,18 +130,31 @@ def __init__( layer_norm_epsilon=1e-12, kernel_initializer=distilbert_kernel_initializer(), name=f"transformer_layer_{i}", - )(x, padding_mask=padding_mask) + ) + self.transformer_layers.append(layer) - # Instantiate using Functional API Model constructor + # === Functional Model === + token_id_input = keras.Input( + shape=(None,), dtype="int32", name="token_ids" + ) + padding_mask_input = keras.Input( + shape=(None,), dtype="int32", name="padding_mask" + ) + x = self.embeddings(token_id_input) + x = self.embeddings_layer_norm(x) + x = self.embeddings_dropout(x) + for transformer_layer in self.transformer_layers: + x = transformer_layer(x, padding_mask=padding_mask_input) super().__init__( inputs={ "token_ids": token_id_input, - "padding_mask": padding_mask, + "padding_mask": padding_mask_input, }, outputs=x, **kwargs, ) - # All references to `self` below this line + + # === Config === self.vocabulary_size = vocabulary_size self.num_layers = num_layers self.num_heads = num_heads @@ -160,7 +163,6 @@ def __init__( self.dropout = dropout self.max_sequence_length = max_sequence_length self.cls_token_index = 0 - self.token_embedding = embedding_layer.token_embedding def get_config(self): config = super().get_config() diff --git a/keras_nlp/models/distil_bert/distil_bert_classifier.py b/keras_nlp/models/distil_bert/distil_bert_classifier.py index 42de1cee83..cf0db9786a 100644 --- a/keras_nlp/models/distil_bert/distil_bert_classifier.py +++ b/keras_nlp/models/distil_bert/distil_bert_classifier.py @@ -150,39 +150,46 @@ def __init__( dropout=0.2, **kwargs, ): - inputs = backbone.input + # === Layers === + self.backbone = backbone + self.preprocessor = preprocessor hidden_dim = hidden_dim or backbone.hidden_dim - - x = backbone(inputs)[:, backbone.cls_token_index, :] - x = keras.layers.Dense( + self.pooled_dense = keras.layers.Dense( hidden_dim, activation="relu", kernel_initializer=distilbert_kernel_initializer(), name="pooled_dense", - )(x) - x = keras.layers.Dropout(dropout, name="classifier_dropout")(x) - outputs = keras.layers.Dense( + ) + self.output_dropout = keras.layers.Dropout( + dropout, + name="output_dropout", + ) + self.output_dense = keras.layers.Dense( num_classes, kernel_initializer=distilbert_kernel_initializer(), activation=activation, name="logits", - )(x) + ) - # Instantiate using Functional API Model constructor + # === Functional Model === + inputs = backbone.input + x = backbone(inputs)[:, backbone.cls_token_index, :] + x = self.pooled_dense(x) + x = self.output_dropout(x) + outputs = self.output_dense(x) super().__init__( inputs=inputs, outputs=outputs, - include_preprocessing=preprocessor is not None, **kwargs, ) - # All references to `self` below this line - self.backbone = backbone - self.preprocessor = preprocessor + + # === Config === self.num_classes = num_classes self.activation = keras.activations.get(activation) self.hidden_dim = hidden_dim self.dropout = dropout + # === Default compilation === logit_output = self.activation == keras.activations.linear self.compile( loss=keras.losses.SparseCategoricalCrossentropy( diff --git a/keras_nlp/models/distil_bert/distil_bert_masked_lm.py b/keras_nlp/models/distil_bert/distil_bert_masked_lm.py index 71cb117d5b..9be43f8aa1 100644 --- a/keras_nlp/models/distil_bert/distil_bert_masked_lm.py +++ b/keras_nlp/models/distil_bert/distil_bert_masked_lm.py @@ -104,6 +104,18 @@ def __init__( preprocessor=None, **kwargs, ): + # === Layers === + self.backbone = backbone + self.preprocessor = preprocessor + self.masked_lm_head = MaskedLMHead( + vocabulary_size=backbone.vocabulary_size, + token_embedding=backbone.token_embedding, + intermediate_activation="gelu", + kernel_initializer=distilbert_kernel_initializer(), + name="mlm_head", + ) + + # === Functional Model === inputs = { **backbone.input, "mask_positions": keras.Input( @@ -111,25 +123,16 @@ def __init__( ), } backbone_outputs = backbone(backbone.input) - outputs = MaskedLMHead( - vocabulary_size=backbone.vocabulary_size, - token_embedding=backbone.token_embedding, - intermediate_activation="gelu", - kernel_initializer=distilbert_kernel_initializer(), - name="mlm_head", - )(backbone_outputs, inputs["mask_positions"]) - - # Instantiate using Functional API Model constructor + outputs = self.masked_lm_head( + backbone_outputs, inputs["mask_positions"] + ) super().__init__( inputs=inputs, outputs=outputs, - include_preprocessing=preprocessor is not None, **kwargs, ) - # All references to `self` below this line - self.backbone = backbone - self.preprocessor = preprocessor + # === Default compilation === self.compile( loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True), optimizer=keras.optimizers.Adam(5e-5), diff --git a/keras_nlp/models/electra/electra_backbone.py b/keras_nlp/models/electra/electra_backbone.py index 66d1db8ccc..2e2a0197af 100644 --- a/keras_nlp/models/electra/electra_backbone.py +++ b/keras_nlp/models/electra/electra_backbone.py @@ -94,65 +94,44 @@ def __init__( num_segments=2, **kwargs, ): - # Index of classification token in the vocabulary - cls_token_index = 0 - # Inputs - token_id_input = keras.Input( - shape=(None,), dtype="int32", name="token_ids" - ) - segment_id_input = keras.Input( - shape=(None,), dtype="int32", name="segment_ids" - ) - padding_mask = keras.Input( - shape=(None,), dtype="int32", name="padding_mask" - ) - - # Embed tokens, positions, and segment ids. - token_embedding_layer = ReversibleEmbedding( + # === Layers === + self.token_embedding = ReversibleEmbedding( input_dim=vocab_size, output_dim=embedding_dim, embeddings_initializer=electra_kernel_initializer(), name="token_embedding", ) - token_embedding = token_embedding_layer(token_id_input) - position_embedding = PositionEmbedding( + self.position_embedding = PositionEmbedding( initializer=electra_kernel_initializer(), sequence_length=max_sequence_length, name="position_embedding", - )(token_embedding) - segment_embedding = keras.layers.Embedding( + ) + self.segment_embedding = keras.layers.Embedding( input_dim=num_segments, output_dim=embedding_dim, embeddings_initializer=electra_kernel_initializer(), name="segment_embedding", - )(segment_id_input) - - # Add all embeddings together. - x = keras.layers.Add()( - (token_embedding, position_embedding, segment_embedding), ) - # Layer normalization - x = keras.layers.LayerNormalization( + self.embeddings_add = keras.layers.Add() + self.embeddings_layer_norm = keras.layers.LayerNormalization( name="embeddings_layer_norm", axis=-1, epsilon=1e-12, dtype="float32", - )(x) - # Dropout - x = keras.layers.Dropout( + ) + self.embeddings_dropout = keras.layers.Dropout( dropout, name="embeddings_dropout", - )(x) + ) if hidden_dim != embedding_dim: - x = keras.layers.Dense( + self.embeddings_projection = keras.layers.Dense( hidden_dim, kernel_initializer=electra_kernel_initializer(), name="embeddings_projection", - )(x) - - # Apply successive transformer encoder blocks. + ) + self.transformer_layers = [] for i in range(num_layers): - x = TransformerEncoder( + layer = TransformerEncoder( num_heads=num_heads, intermediate_dim=intermediate_dim, activation=gelu_approximate, @@ -160,24 +139,49 @@ def __init__( layer_norm_epsilon=1e-12, kernel_initializer=electra_kernel_initializer(), name=f"transformer_layer_{i}", - )(x, padding_mask=padding_mask) - - sequence_output = x - # Construct the two ELECTRA outputs. The pooled output is a dense layer on - # top of the [CLS] token. - pooled_output = keras.layers.Dense( + ) + self.transformer_layers.append(layer) + self.pooled_dense = keras.layers.Dense( hidden_dim, kernel_initializer=electra_kernel_initializer(), activation="tanh", name="pooled_dense", - )(x[:, cls_token_index, :]) + ) - # Instantiate using Functional API Model constructor + # === Functional Model === + token_id_input = keras.Input( + shape=(None,), dtype="int32", name="token_ids" + ) + segment_id_input = keras.Input( + shape=(None,), dtype="int32", name="segment_ids" + ) + padding_mask_input = keras.Input( + shape=(None,), dtype="int32", name="padding_mask" + ) + # Embed tokens, positions, and segment ids. + tokens = self.token_embedding(token_id_input) + positions = self.position_embedding(tokens) + segments = self.segment_embedding(segment_id_input) + # Add all embeddings together. + x = self.embeddings_add((tokens, positions, segments)) + x = self.embeddings_layer_norm(x) + x = self.embeddings_dropout(x) + if hidden_dim != embedding_dim: + x = self.embeddings_projection(x) + # Apply successive transformer encoder blocks. + for transformer_layer in self.transformer_layers: + x = transformer_layer(x, padding_mask=padding_mask_input) + # Index of classification token in the vocabulary + cls_token_index = 0 + sequence_output = x + # Construct the two ELECTRA outputs. The pooled output is a dense layer on + # top of the [CLS] token. + pooled_output = self.pooled_dense(x[:, cls_token_index, :]) super().__init__( inputs={ "token_ids": token_id_input, "segment_ids": segment_id_input, - "padding_mask": padding_mask, + "padding_mask": padding_mask_input, }, outputs={ "sequence_output": sequence_output, @@ -186,7 +190,7 @@ def __init__( **kwargs, ) - # All references to self below this line + # === Config === self.vocab_size = vocab_size self.num_layers = num_layers self.num_heads = num_heads @@ -197,7 +201,6 @@ def __init__( self.max_sequence_length = max_sequence_length self.num_segments = num_segments self.cls_token_index = cls_token_index - self.token_embedding = token_embedding_layer def get_config(self): config = super().get_config() diff --git a/keras_nlp/models/f_net/f_net_backbone.py b/keras_nlp/models/f_net/f_net_backbone.py index ac4d290b02..9103a10d48 100644 --- a/keras_nlp/models/f_net/f_net_backbone.py +++ b/keras_nlp/models/f_net/f_net_backbone.py @@ -101,61 +101,44 @@ def __init__( num_segments=4, **kwargs, ): - # Index of classification token in the vocabulary - cls_token_index = 0 - # Inputs - token_id_input = keras.Input( - shape=(None,), dtype="int32", name="token_ids" - ) - segment_id_input = keras.Input( - shape=(None,), dtype="int32", name="segment_ids" - ) - - # Embed tokens, positions, and segment ids. - token_embedding_layer = ReversibleEmbedding( + # === Layers === + self.token_embedding = ReversibleEmbedding( input_dim=vocabulary_size, output_dim=hidden_dim, embeddings_initializer=f_net_kernel_initializer(), name="token_embedding", ) - token_embedding = token_embedding_layer(token_id_input) - position_embedding = PositionEmbedding( + self.position_embedding = PositionEmbedding( initializer=f_net_kernel_initializer(), sequence_length=max_sequence_length, name="position_embedding", - )(token_embedding) - segment_embedding = keras.layers.Embedding( + ) + self.segment_embedding = keras.layers.Embedding( input_dim=num_segments, output_dim=hidden_dim, embeddings_initializer=f_net_kernel_initializer(), name="segment_embedding", - )(segment_id_input) - - # Sum, normalize and apply dropout to embeddings. - x = keras.layers.Add()( - (token_embedding, position_embedding, segment_embedding) ) - x = keras.layers.LayerNormalization( + self.embeddings_add = keras.layers.Add() + self.embeddings_layer_norm = keras.layers.LayerNormalization( name="embeddings_layer_norm", axis=-1, epsilon=1e-12, dtype="float32", - )(x) - - x = keras.layers.Dense( + ) + self.embedding_projection = keras.layers.Dense( hidden_dim, kernel_initializer=f_net_kernel_initializer(), bias_initializer=f_net_bias_initializer(), name="embedding_projection", - )(x) - x = keras.layers.Dropout( + ) + self.embeddings_dropout = keras.layers.Dropout( dropout, name="embeddings_dropout", - )(x) - - # Apply successive FNet encoder blocks. + ) + self.transformer_layers = [] for i in range(num_layers): - x = FNetEncoder( + layer = FNetEncoder( intermediate_dim=intermediate_dim, activation=gelu_approximate, dropout=dropout, @@ -163,19 +146,41 @@ def __init__( kernel_initializer=f_net_kernel_initializer(), bias_initializer=f_net_bias_initializer(), name=f"f_net_layer_{i}", - )(x) - - # Construct the two FNet outputs. The pooled output is a dense layer on - # top of the [CLS] token. - sequence_output = x - pooled_output = keras.layers.Dense( + ) + self.transformer_layers.append(layer) + self.pooled_dense = keras.layers.Dense( hidden_dim, kernel_initializer=f_net_kernel_initializer(), bias_initializer=f_net_bias_initializer(), activation="tanh", name="pooled_dense", - )(x[:, cls_token_index, :]) + ) + # === Functional Model === + token_id_input = keras.Input( + shape=(None,), dtype="int32", name="token_ids" + ) + segment_id_input = keras.Input( + shape=(None,), dtype="int32", name="segment_ids" + ) + # Embed tokens, positions, and segment ids. + tokens = self.token_embedding(token_id_input) + positions = self.position_embedding(tokens) + segments = self.segment_embedding(segment_id_input) + # Sum, normalize and apply dropout to embeddings. + x = self.embeddings_add((tokens, positions, segments)) + x = self.embeddings_layer_norm(x) + x = self.embedding_projection(x) + x = self.embeddings_dropout(x) + # Apply successive FNet encoder blocks. + for transformer_layer in self.transformer_layers: + x = transformer_layer(x) + # Index of classification token in the vocabulary + cls_token_index = 0 + # Construct the two FNet outputs. The pooled output is a dense layer on + # top of the [CLS] token. + sequence_output = x + pooled_output = self.pooled_dense(x[:, cls_token_index, :]) # Instantiate using Functional API Model constructor super().__init__( inputs={ @@ -189,7 +194,7 @@ def __init__( **kwargs, ) - # All references to `self` below this line + # === Config === self.vocabulary_size = vocabulary_size self.num_layers = num_layers self.hidden_dim = hidden_dim @@ -198,7 +203,6 @@ def __init__( self.max_sequence_length = max_sequence_length self.num_segments = num_segments self.cls_token_index = cls_token_index - self.token_embedding = token_embedding_layer def get_config(self): config = super().get_config() diff --git a/keras_nlp/models/f_net/f_net_classifier.py b/keras_nlp/models/f_net/f_net_classifier.py index f6485485e1..f4ee31d1e8 100644 --- a/keras_nlp/models/f_net/f_net_classifier.py +++ b/keras_nlp/models/f_net/f_net_classifier.py @@ -109,29 +109,37 @@ def __init__( dropout=0.1, **kwargs, ): - inputs = backbone.input - pooled = backbone(inputs)["pooled_output"] - pooled = keras.layers.Dropout(dropout)(pooled) - outputs = keras.layers.Dense( + # === Layers === + self.backbone = backbone + self.preprocessor = preprocessor + self.output_dropout = keras.layers.Dropout( + dropout, + name="output_dropout", + ) + self.output_dense = keras.layers.Dense( num_classes, kernel_initializer=f_net_kernel_initializer(), activation=activation, name="logits", - )(pooled) - # Instantiate using Functional API Model constructor + ) + + # === Functional Model === + inputs = backbone.input + pooled = backbone(inputs)["pooled_output"] + pooled = self.output_dropout(pooled) + outputs = self.output_dense(pooled) super().__init__( inputs=inputs, outputs=outputs, - include_preprocessing=preprocessor is not None, **kwargs, ) - # All references to `self` below this line - self.backbone = backbone - self.preprocessor = preprocessor + + # === Config === self.num_classes = num_classes self.activation = keras.activations.get(activation) self.dropout = dropout + # === Default compilation === logit_output = self.activation == keras.activations.linear self.compile( loss=keras.losses.SparseCategoricalCrossentropy( diff --git a/keras_nlp/models/f_net/f_net_masked_lm.py b/keras_nlp/models/f_net/f_net_masked_lm.py index d7048cd525..c0eb231d78 100644 --- a/keras_nlp/models/f_net/f_net_masked_lm.py +++ b/keras_nlp/models/f_net/f_net_masked_lm.py @@ -101,6 +101,18 @@ def __init__( preprocessor=None, **kwargs, ): + # === Layers === + self.backbone = backbone + self.preprocessor = preprocessor + self.masked_lm_head = MaskedLMHead( + vocabulary_size=backbone.vocabulary_size, + token_embedding=backbone.token_embedding, + intermediate_activation="gelu", + kernel_initializer=f_net_kernel_initializer(), + name="mlm_head", + ) + + # === Functional Model === inputs = { **backbone.input, "mask_positions": keras.Input( @@ -108,24 +120,16 @@ def __init__( ), } backbone_outputs = backbone(backbone.input) - outputs = MaskedLMHead( - vocabulary_size=backbone.vocabulary_size, - token_embedding=backbone.token_embedding, - intermediate_activation="gelu", - kernel_initializer=f_net_kernel_initializer(), - name="mlm_head", - )(backbone_outputs["sequence_output"], inputs["mask_positions"]) - - # Instantiate using Functional API Model constructor + outputs = self.masked_lm_head( + backbone_outputs["sequence_output"], inputs["mask_positions"] + ) super().__init__( inputs=inputs, outputs=outputs, - include_preprocessing=preprocessor is not None, **kwargs, ) - # All references to `self` below this line - self.backbone = backbone - self.preprocessor = preprocessor + + # === Default compilation === self.compile( loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True), optimizer=keras.optimizers.Adam(5e-5), diff --git a/keras_nlp/models/gpt2/gpt2_backbone.py b/keras_nlp/models/gpt2/gpt2_backbone.py index 89c23f71de..d3f4a41541 100644 --- a/keras_nlp/models/gpt2/gpt2_backbone.py +++ b/keras_nlp/models/gpt2/gpt2_backbone.py @@ -97,68 +97,73 @@ def __init__( max_sequence_length=1024, **kwargs, ): - # Inputs - token_ids = keras.Input(shape=(None,), dtype="int32", name="token_ids") - padding_mask = keras.Input( - shape=(None,), dtype="int32", name="padding_mask" - ) - - # Embed tokens, positions. - token_embedding_layer = ReversibleEmbedding( + # === Layers === + self.token_embedding = ReversibleEmbedding( input_dim=vocabulary_size, output_dim=hidden_dim, embeddings_initializer=_gpt_2_kernel_initializer(stddev=0.01), name="token_embedding", ) - token_embedding = token_embedding_layer(token_ids) - - # Can't use `TokenAndPositionEmbedding` layer here because of different - # initializers. - position_embedding = PositionEmbedding( + self.position_embedding = PositionEmbedding( initializer=_gpt_2_kernel_initializer(stddev=0.02), sequence_length=max_sequence_length, name="position_embedding", - )(token_embedding) - - # Sum and apply dropout to embeddings. - x = keras.layers.Add(name="embeddings_add")( - (token_embedding, position_embedding) ) - x = keras.layers.Dropout( + self.embeddings_add = keras.layers.Add( + name="embeddings_add", + ) + self.embeddings_dropout = keras.layers.Dropout( dropout, name="embeddings_dropout", - )(x) - - # Apply successive transformer decoder blocks. + ) + self.transformer_layers = [] for i in range(num_layers): - x = TransformerDecoder( - intermediate_dim=intermediate_dim, - num_heads=num_heads, - dropout=dropout, - layer_norm_epsilon=1e-05, - activation=gelu_approximate, - kernel_initializer=_gpt_2_kernel_initializer(stddev=0.02), - normalize_first=True, - name=f"transformer_layer_{i}", - )(x, decoder_padding_mask=padding_mask) - - sequence_output = keras.layers.LayerNormalization( + self.transformer_layers.append( + TransformerDecoder( + intermediate_dim=intermediate_dim, + num_heads=num_heads, + dropout=dropout, + layer_norm_epsilon=1e-05, + activation=gelu_approximate, + kernel_initializer=_gpt_2_kernel_initializer(stddev=0.02), + normalize_first=True, + name=f"transformer_layer_{i}", + ) + ) + self.layer_norm = keras.layers.LayerNormalization( name="layer_norm", axis=-1, epsilon=1e-05, dtype="float32", - )(x) + ) - # Instantiate using Functional API Model constructor + # === Functional Model === + token_id_input = keras.Input( + shape=(None,), dtype="int32", name="token_ids" + ) + padding_mask_input = keras.Input( + shape=(None,), dtype="int32", name="padding_mask" + ) + # Embed inputs. + tokens = self.token_embedding(token_id_input) + positions = self.position_embedding(tokens) + x = self.embeddings_add((tokens, positions)) + x = self.embeddings_dropout(x) + # Apply transformer layers. + for transformer_layer in self.transformer_layers: + x = transformer_layer(x, decoder_padding_mask=padding_mask_input) + sequence_output = self.layer_norm(x) + # Instantiate using the Functional constructor. super().__init__( inputs={ - "token_ids": token_ids, - "padding_mask": padding_mask, + "token_ids": token_id_input, + "padding_mask": padding_mask_input, }, outputs=sequence_output, **kwargs, ) - # All references to `self` below this line + + # === Config === self.vocabulary_size = vocabulary_size self.num_layers = num_layers self.num_heads = num_heads @@ -166,7 +171,6 @@ def __init__( self.intermediate_dim = intermediate_dim self.dropout = dropout self.max_sequence_length = max_sequence_length - self.token_embedding = token_embedding_layer def get_config(self): config = super().get_config() diff --git a/keras_nlp/models/gpt2/gpt2_causal_lm.py b/keras_nlp/models/gpt2/gpt2_causal_lm.py index 44eebd0a20..e154c88bb1 100644 --- a/keras_nlp/models/gpt2/gpt2_causal_lm.py +++ b/keras_nlp/models/gpt2/gpt2_causal_lm.py @@ -155,23 +155,21 @@ def __init__( preprocessor=None, **kwargs, ): + # === Layers === + self.backbone = backbone + self.preprocessor = preprocessor + + # === Functional Model === inputs = backbone.input hidden_states = backbone(inputs) outputs = backbone.token_embedding(hidden_states, reverse=True) - - # Instantiate using Functional API Model constructor. super().__init__( inputs=inputs, outputs=outputs, - include_preprocessing=preprocessor is not None, **kwargs, ) - self.backbone = backbone - self.preprocessor = preprocessor - self.generate_function = None - self._sampler = None - # Default compilation + # === Default compilation === self.compile( loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True), optimizer=keras.optimizers.Adam(2e-5), @@ -216,27 +214,25 @@ def call_with_cache( the final hidden representation of the input tokens, and `cache` is the decoding cache. """ - token_embedding = self.backbone.get_layer("token_embedding")(token_ids) - position_embedding = self.backbone.get_layer("position_embedding")( - token_embedding, start_index=cache_update_index - ) - x = self.backbone.get_layer("embeddings_add")( - (token_embedding, position_embedding) + tokens = self.backbone.token_embedding(token_ids) + positions = self.backbone.position_embedding( + tokens, start_index=cache_update_index ) - x = self.backbone.get_layer("embeddings_dropout")(x) + x = self.backbone.embeddings_add((tokens, positions)) + x = self.backbone.embeddings_dropout(x) # Each decoder layer has a cache; we update them separately. caches = [] - for i in range(self.backbone.num_layers): + for i, transformer_layer in enumerate(self.backbone.transformer_layers): current_cache = cache[:, i, ...] - x, next_cache = self.backbone.get_layer(f"transformer_layer_{i}")( + x, next_cache = transformer_layer( x, self_attention_cache=current_cache, self_attention_cache_update_index=cache_update_index, ) caches.append(next_cache) cache = ops.stack(caches, axis=1) - hidden_states = x = self.backbone.get_layer("layer_norm")(x) - logits = self.backbone.get_layer("token_embedding")(x, reverse=True) + hidden_states = x = self.backbone.layer_norm(x) + logits = self.backbone.token_embedding(x, reverse=True) return logits, hidden_states, cache def _build_cache(self, token_ids): diff --git a/keras_nlp/models/gpt_neo_x/gpt_neo_x_backbone.py b/keras_nlp/models/gpt_neo_x/gpt_neo_x_backbone.py index 6804331aed..5bbc11af70 100644 --- a/keras_nlp/models/gpt_neo_x/gpt_neo_x_backbone.py +++ b/keras_nlp/models/gpt_neo_x/gpt_neo_x_backbone.py @@ -77,29 +77,20 @@ def __init__( max_sequence_length=512, **kwargs, ): - # Inputs - token_ids = keras.Input(shape=(None,), dtype="int32", name="token_ids") - padding_mask = keras.Input( - shape=(None,), dtype="int32", name="padding_mask" - ) - - # Embed tokens - token_embedding_layer = ReversibleEmbedding( + # === Layers === + self.token_embedding = ReversibleEmbedding( input_dim=vocabulary_size, output_dim=hidden_dim, embeddings_initializer=_gpt_neo_x_kernel_initializer(stddev=0.01), name="token_embedding", ) - token_embedding = token_embedding_layer(token_ids) - - x = keras.layers.Dropout( + self.embeddings_dropout = keras.layers.Dropout( dropout, name="embeddings_dropout", - )(token_embedding) - - # Apply successive transformer decoder blocks. + ) + self.transformer_layers = [] for i in range(num_layers): - x = GPTNeoXDecoder( + layer = GPTNeoXDecoder( intermediate_dim=intermediate_dim, num_heads=num_heads, dropout=dropout, @@ -110,25 +101,38 @@ def __init__( activation=gelu_approximate, kernel_initializer=_gpt_neo_x_kernel_initializer(stddev=0.02), name=f"transformer_layer_{i}", - )(x, decoder_padding_mask=padding_mask) - - sequence_output = keras.layers.LayerNormalization( + ) + self.transformer_layers.append(layer) + self.layer_norm = keras.layers.LayerNormalization( name="layer_norm", axis=-1, epsilon=layer_norm_epsilon, dtype="float32", - )(x) + ) - # Instantiate using Functional API Model constructor + # === Functional Model === + token_id_input = keras.Input( + shape=(None,), dtype="int32", name="token_ids" + ) + padding_mask_input = keras.Input( + shape=(None,), dtype="int32", name="padding_mask" + ) + # Embed tokens. + x = self.token_embedding(token_id_input) + x = self.embeddings_dropout(x) + for transformer_layer in self.transformer_layers: + x = transformer_layer(x, decoder_padding_mask=padding_mask_input) + sequence_output = self.layer_norm(x) super().__init__( inputs={ - "token_ids": token_ids, - "padding_mask": padding_mask, + "token_ids": token_id_input, + "padding_mask": padding_mask_input, }, outputs=sequence_output, **kwargs, ) - # All references to `self` below this line + + # === Config === self.vocabulary_size = vocabulary_size self.num_layers = num_layers self.num_heads = num_heads @@ -139,7 +143,6 @@ def __init__( self.rotary_max_wavelength = rotary_max_wavelength self.max_sequence_length = max_sequence_length self.layer_norm_epsilon = layer_norm_epsilon - self.token_embedding = token_embedding_layer def get_config(self): config = super().get_config() diff --git a/keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm.py b/keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm.py index 0f813470aa..bef32017ea 100644 --- a/keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm.py +++ b/keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm.py @@ -52,23 +52,21 @@ def __init__( preprocessor=None, **kwargs, ): + # === Layers === + self.backbone = backbone + self.preprocessor = preprocessor + + # === Functional Model === inputs = backbone.input hidden_states = backbone(inputs) outputs = backbone.token_embedding(hidden_states, reverse=True) - - # Instantiate using Functional API Model constructor. super().__init__( inputs=inputs, outputs=outputs, - include_preprocessing=preprocessor is not None, **kwargs, ) - self.backbone = backbone - self.preprocessor = preprocessor - self.generate_function = None - self._sampler = None - # Default compilation + # === Default compilation === self.compile( loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True), optimizer=keras.optimizers.Adam(2e-5), @@ -109,20 +107,20 @@ def call_with_cache( the final hidden representation of the input tokens, and `cache` is the decoding cache. """ - token_embedding = self.backbone.get_layer("token_embedding")(token_ids) - x = self.backbone.get_layer("embeddings_dropout")(token_embedding) + token_embedding = self.backbone.token_embedding(token_ids) + x = self.backbone.embeddings_dropout(token_embedding) # Each decoder layer has a cache; we update them separately. caches = [] - for i in range(self.backbone.num_layers): + for i, transformer_layer in enumerate(self.backbone.transformer_layers): current_cache = cache[:, i, ...] - x, next_cache = self.backbone.get_layer(f"transformer_layer_{i}")( + x, next_cache = transformer_layer( x, self_attention_cache=current_cache, self_attention_cache_update_index=cache_update_index, ) caches.append(next_cache) cache = ops.stack(caches, axis=1) - x = self.backbone.get_layer("layer_norm")(x) + x = self.backbone.layer_norm(x) hidden_states = x logits = self.backbone.token_embedding(hidden_states, reverse=True) return logits, hidden_states, cache diff --git a/keras_nlp/models/llama/llama_backbone.py b/keras_nlp/models/llama/llama_backbone.py index 63438544cc..46cfdc37f2 100644 --- a/keras_nlp/models/llama/llama_backbone.py +++ b/keras_nlp/models/llama/llama_backbone.py @@ -75,26 +75,17 @@ def __init__( max_sequence_length=4096, **kwargs, ): - # Inputs - token_ids = keras.Input(shape=(None,), dtype="int32", name="token_ids") - padding_mask = keras.Input( - shape=(None,), dtype="int32", name="padding_mask" - ) - - # Embed tokens - token_embedding = ReversibleEmbedding( + # === Layers === + self.token_embedding = ReversibleEmbedding( input_dim=vocabulary_size, output_dim=hidden_dim, embeddings_initializer=_llama_kernel_initializer(stddev=0.01), tie_weights=False, name="token_embedding", - )(token_ids) - - x = token_embedding - - # Apply successive transformer decoder blocks. + ) + self.transformer_layers = [] for i in range(num_layers): - x = LlamaDecoder( + layer = LlamaDecoder( intermediate_dim=intermediate_dim, num_query_heads=num_query_heads, num_key_value_heads=num_key_value_heads, @@ -105,23 +96,34 @@ def __init__( activation=ops.silu, kernel_initializer=_llama_kernel_initializer(stddev=0.02), name=f"transformer_layer_{i}", - )(x, decoder_padding_mask=padding_mask) - - sequence_output = LlamaLayerNorm( + ) + self.transformer_layers.append(layer) + self.layer_norm = LlamaLayerNorm( name="layer_norm", epsilon=layer_norm_epsilon, - )(x) + ) - # Instantiate using Functional API Model constructor + # === Functional Model === + token_id_input = keras.Input( + shape=(None,), dtype="int32", name="token_ids" + ) + padding_mask_input = keras.Input( + shape=(None,), dtype="int32", name="padding_mask" + ) + x = self.token_embedding(token_id_input) + for transformer_layer in self.transformer_layers: + x = transformer_layer(x, decoder_padding_mask=padding_mask_input) + sequence_output = self.layer_norm(x) super().__init__( inputs={ - "token_ids": token_ids, - "padding_mask": padding_mask, + "token_ids": token_id_input, + "padding_mask": padding_mask_input, }, outputs=sequence_output, **kwargs, ) - # All references to `self` below this line + + # === Config === self.vocabulary_size = vocabulary_size self.num_layers = num_layers self.num_query_heads = num_query_heads @@ -150,7 +152,3 @@ def get_config(self): } ) return config - - @property - def token_embedding(self): - return self.get_layer("token_embedding") diff --git a/keras_nlp/models/mistral/mistral_backbone.py b/keras_nlp/models/mistral/mistral_backbone.py index 42cec8b218..107e5699cb 100644 --- a/keras_nlp/models/mistral/mistral_backbone.py +++ b/keras_nlp/models/mistral/mistral_backbone.py @@ -109,17 +109,9 @@ def __init__( dropout=0, **kwargs, ): - # Get the dtype + # === Layers === dtype = kwargs.pop("dtype", keras.backend.floatx()) - - # Inputs - token_ids = keras.Input(shape=(None,), dtype="int32", name="token_ids") - padding_mask = keras.Input( - shape=(None,), dtype="int32", name="padding_mask" - ) - - # Embed Tokens - token_embedding_layer = ReversibleEmbedding( + self.token_embedding = ReversibleEmbedding( input_dim=vocabulary_size, output_dim=hidden_dim, tie_weights=False, @@ -127,11 +119,9 @@ def __init__( dtype=dtype, name="token_embedding", ) - x = token_embedding_layer(token_ids) - - # Apply successive transformer decoder blocks + self.transformer_layers = [] for i in range(num_layers): - x = MistralTransformerDecoder( + layer = MistralTransformerDecoder( intermediate_dim=intermediate_dim, num_query_heads=num_query_heads, num_key_value_heads=num_key_value_heads, @@ -144,25 +134,35 @@ def __init__( dropout=dropout, dtype=dtype, name=f"transformer_layer_{i}", - )(x, decoder_padding_mask=padding_mask) - - sequence_output = MistralLayerNormalization( - name="sequence_output_layernorm", + ) + self.transformer_layers.append(layer) + self.layer_norm = MistralLayerNormalization( epsilon=layer_norm_epsilon, dtype=dtype, - )(x) + name="sequence_output_layernorm", + ) - # Instantiate using Functional API Model constructor + # === Functional Model === + token_id_input = keras.Input( + shape=(None,), dtype="int32", name="token_ids" + ) + padding_mask_input = keras.Input( + shape=(None,), dtype="int32", name="padding_mask" + ) + x = self.token_embedding(token_id_input) + for transformer_layer in self.transformer_layers: + x = transformer_layer(x, decoder_padding_mask=padding_mask_input) + sequence_output = self.layer_norm(x) super().__init__( inputs={ - "token_ids": token_ids, - "padding_mask": padding_mask, + "token_ids": token_id_input, + "padding_mask": padding_mask_input, }, outputs=sequence_output, **kwargs, ) - # All references to `self` below this line + # === Config === self.vocabulary_size = vocabulary_size self.num_layers = num_layers self.num_query_heads = num_query_heads @@ -174,7 +174,6 @@ def __init__( self.sliding_window = sliding_window self.layer_norm_epsilon = layer_norm_epsilon self.dropout = dropout - self.token_embedding = token_embedding_layer def get_config(self): config = super().get_config() diff --git a/keras_nlp/models/opt/opt_backbone.py b/keras_nlp/models/opt/opt_backbone.py index ff1495ba9f..d04f4b571d 100644 --- a/keras_nlp/models/opt/opt_backbone.py +++ b/keras_nlp/models/opt/opt_backbone.py @@ -93,25 +93,18 @@ def __init__( max_sequence_length=2048, **kwargs, ): - # Decoder inputs. - token_ids = keras.Input(shape=(None,), dtype="int32", name="token_ids") - padding_mask = keras.Input( - shape=(None,), dtype="int32", name="padding_mask" - ) - - # Embed tokens and positions. - embedding_layer = TokenAndPositionEmbedding( + # === Layers === + self.embeddings = TokenAndPositionEmbedding( vocabulary_size=vocabulary_size, sequence_length=max_sequence_length, embedding_dim=hidden_dim, embeddings_initializer=opt_kernel_initializer(), name="embeddings", ) - x = embedding_layer(token_ids) - - # Apply successive transformer decoder blocks. + self.token_embedding = self.embeddings.token_embedding + self.transformer_layers = [] for i in range(num_layers): - x = TransformerDecoder( + layer = TransformerDecoder( intermediate_dim=intermediate_dim, num_heads=num_heads, dropout=dropout, @@ -120,27 +113,36 @@ def __init__( normalize_first=True, kernel_initializer=opt_kernel_initializer(), name=f"transformer_layer_{i}", - )(x, decoder_padding_mask=padding_mask) - - # Add a final layer norm. - x = keras.layers.LayerNormalization( - name="layer_norm", + ) + self.transformer_layers.append(layer) + self.layer_norm = keras.layers.LayerNormalization( axis=-1, epsilon=1e-5, dtype="float32", - )(x) + name="layer_norm", + ) - # Instantiate using Functional API Model constructor + # === Functional Model === + token_id_input = keras.Input( + shape=(None,), dtype="int32", name="token_ids" + ) + padding_mask_input = keras.Input( + shape=(None,), dtype="int32", name="padding_mask" + ) + x = self.embeddings(token_id_input) + for transformer_layer in self.transformer_layers: + x = transformer_layer(x, decoder_padding_mask=padding_mask_input) + x = self.layer_norm(x) super().__init__( inputs={ - "token_ids": token_ids, - "padding_mask": padding_mask, + "token_ids": token_id_input, + "padding_mask": padding_mask_input, }, outputs=x, **kwargs, ) - # All references to `self` below this line + # === Config === self.vocabulary_size = vocabulary_size self.num_layers = num_layers self.num_heads = num_heads @@ -148,7 +150,6 @@ def __init__( self.intermediate_dim = intermediate_dim self.dropout = dropout self.max_sequence_length = max_sequence_length - self.token_embedding = embedding_layer.token_embedding def get_config(self): return { diff --git a/keras_nlp/models/opt/opt_causal_lm.py b/keras_nlp/models/opt/opt_causal_lm.py index 6197a87ffd..9715bc6b75 100644 --- a/keras_nlp/models/opt/opt_causal_lm.py +++ b/keras_nlp/models/opt/opt_causal_lm.py @@ -155,23 +155,21 @@ def __init__( preprocessor=None, **kwargs, ): + # === Layers === + self.backbone = backbone + self.preprocessor = preprocessor + + # === Functional Model === inputs = backbone.input hidden_states = backbone(inputs) outputs = backbone.token_embedding(hidden_states, reverse=True) - - # Instantiate using Functional API Model constructor. super().__init__( inputs=inputs, outputs=outputs, - include_preprocessing=preprocessor is not None, **kwargs, ) - self.backbone = backbone - self.preprocessor = preprocessor - self.generate_function = None - self._sampler = None - # Default compilation + # === Default compilation === self.compile( loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True), optimizer=keras.optimizers.Adam(2e-5), @@ -216,21 +214,19 @@ def call_with_cache( the final hidden representation of the input tokens, and `cache` is the decoding cache. """ - x = self.backbone.get_layer("embeddings")( - token_ids, start_index=cache_update_index - ) + x = self.backbone.embeddings(token_ids, start_index=cache_update_index) # Each decoder layer has a cache; we update them separately. caches = [] - for i in range(self.backbone.num_layers): + for i, transformer_layer in enumerate(self.backbone.transformer_layers): current_cache = cache[:, i, ...] - x, next_cache = self.backbone.get_layer(f"transformer_layer_{i}")( + x, next_cache = transformer_layer( x, self_attention_cache=current_cache, self_attention_cache_update_index=cache_update_index, ) caches.append(next_cache) cache = ops.stack(caches, axis=1) - x = self.backbone.get_layer("layer_norm")(x) + x = self.backbone.layer_norm(x) hidden_states = x logits = self.backbone.token_embedding(hidden_states, reverse=True) return logits, hidden_states, cache diff --git a/keras_nlp/models/roberta/roberta_backbone.py b/keras_nlp/models/roberta/roberta_backbone.py index 8495b5cb69..614104d8d7 100644 --- a/keras_nlp/models/roberta/roberta_backbone.py +++ b/keras_nlp/models/roberta/roberta_backbone.py @@ -98,39 +98,28 @@ def __init__( max_sequence_length=512, **kwargs, ): - # Inputs - token_id_input = keras.Input( - shape=(None,), dtype="int32", name="token_ids" - ) - padding_mask = keras.Input( - shape=(None,), dtype="int32", name="padding_mask" - ) - - # Embed tokens and positions. - embedding_layer = TokenAndPositionEmbedding( + # === Layers === + self.embeddings = TokenAndPositionEmbedding( vocabulary_size=vocabulary_size, sequence_length=max_sequence_length, embedding_dim=hidden_dim, embeddings_initializer=roberta_kernel_initializer(), name="embeddings", ) - embedding = embedding_layer(token_id_input) - - # Sum, normalize and apply dropout to embeddings. - x = keras.layers.LayerNormalization( - name="embeddings_layer_norm", + self.token_embedding = self.embeddings.token_embedding + self.embeddings_layer_norm = keras.layers.LayerNormalization( axis=-1, epsilon=1e-5, # Original paper uses this epsilon value dtype="float32", - )(embedding) - x = keras.layers.Dropout( + name="embeddings_layer_norm", + ) + self.embeddings_dropout = keras.layers.Dropout( dropout, name="embeddings_dropout", - )(x) - - # Apply successive transformer encoder blocks. + ) + self.transformer_layers = [] for i in range(num_layers): - x = TransformerEncoder( + layer = TransformerEncoder( num_heads=num_heads, intermediate_dim=intermediate_dim, activation="gelu", @@ -138,18 +127,31 @@ def __init__( layer_norm_epsilon=1e-5, kernel_initializer=roberta_kernel_initializer(), name=f"transformer_layer_{i}", - )(x, padding_mask=padding_mask) + ) + self.transformer_layers.append(layer) - # Instantiate using Functional API Model constructor + # === Functional Model === + token_id_input = keras.Input( + shape=(None,), dtype="int32", name="token_ids" + ) + padding_mask_input = keras.Input( + shape=(None,), dtype="int32", name="padding_mask" + ) + x = self.embeddings(token_id_input) + x = self.embeddings_layer_norm(x) + x = self.embeddings_dropout(x) + for transformer_layer in self.transformer_layers: + x = transformer_layer(x, padding_mask=padding_mask_input) super().__init__( inputs={ "token_ids": token_id_input, - "padding_mask": padding_mask, + "padding_mask": padding_mask_input, }, outputs=x, **kwargs, ) - # All references to `self` below this line + + # === Config === self.vocabulary_size = vocabulary_size self.num_layers = num_layers self.num_heads = num_heads @@ -158,7 +160,6 @@ def __init__( self.dropout = dropout self.max_sequence_length = max_sequence_length self.start_token_index = 0 - self.token_embedding = embedding_layer.token_embedding def get_config(self): config = super().get_config() diff --git a/keras_nlp/models/roberta/roberta_classifier.py b/keras_nlp/models/roberta/roberta_classifier.py index 9098d95429..e3d7666f5b 100644 --- a/keras_nlp/models/roberta/roberta_classifier.py +++ b/keras_nlp/models/roberta/roberta_classifier.py @@ -144,38 +144,50 @@ def __init__( dropout=0.0, **kwargs, ): - inputs = backbone.input + # === Layers === + self.backbone = backbone + self.preprocessor = preprocessor + self.pooled_dropout = keras.layers.Dropout( + dropout, + name="pooled_dropout", + ) hidden_dim = hidden_dim or backbone.hidden_dim - - x = backbone(inputs)[:, backbone.start_token_index, :] - x = keras.layers.Dropout(dropout, name="pooled_dropout")(x) - x = keras.layers.Dense( - hidden_dim, activation="tanh", name="pooled_dense" - )(x) - x = keras.layers.Dropout(dropout, name="classifier_dropout")(x) - outputs = keras.layers.Dense( + self.pooled_dense = keras.layers.Dense( + hidden_dim, + activation="tanh", + name="pooled_dense", + ) + self.output_dropout = keras.layers.Dropout( + dropout, + name="output_dropout", + ) + self.output_dense = keras.layers.Dense( num_classes, kernel_initializer=roberta_kernel_initializer(), activation=activation, name="logits", - )(x) + ) - # Instantiate using Functional API Model constructor + # === Functional Model === + inputs = backbone.input + x = backbone(inputs)[:, backbone.start_token_index, :] + x = self.pooled_dropout(x) + x = self.pooled_dense(x) + x = self.output_dropout(x) + outputs = self.output_dense(x) super().__init__( inputs=inputs, outputs=outputs, - include_preprocessing=preprocessor is not None, **kwargs, ) - # All references to `self` below this line - self.backbone = backbone - self.preprocessor = preprocessor + + # === Config === self.num_classes = num_classes self.activation = keras.activations.get(activation) self.hidden_dim = hidden_dim self.dropout = dropout - # Default compilation + # === Default compilation === logit_output = self.activation == keras.activations.linear self.compile( loss=keras.losses.SparseCategoricalCrossentropy( diff --git a/keras_nlp/models/roberta/roberta_masked_lm.py b/keras_nlp/models/roberta/roberta_masked_lm.py index 1517f25914..0e62f4cff6 100644 --- a/keras_nlp/models/roberta/roberta_masked_lm.py +++ b/keras_nlp/models/roberta/roberta_masked_lm.py @@ -103,6 +103,18 @@ def __init__( preprocessor=None, **kwargs, ): + # === Layers === + self.backbone = backbone + self.preprocessor = preprocessor + self.masked_lm_head = MaskedLMHead( + vocabulary_size=backbone.vocabulary_size, + token_embedding=backbone.token_embedding, + intermediate_activation="gelu", + kernel_initializer=roberta_kernel_initializer(), + name="mlm_head", + ) + + # === Functional Model === inputs = { **backbone.input, "mask_positions": keras.Input( @@ -110,25 +122,16 @@ def __init__( ), } backbone_outputs = backbone(backbone.input) - outputs = MaskedLMHead( - vocabulary_size=backbone.vocabulary_size, - token_embedding=backbone.token_embedding, - intermediate_activation="gelu", - kernel_initializer=roberta_kernel_initializer(), - name="mlm_head", - )(backbone_outputs, inputs["mask_positions"]) - - # Instantiate using Functional API Model constructor + outputs = self.masked_lm_head( + backbone_outputs, inputs["mask_positions"] + ) super().__init__( inputs=inputs, outputs=outputs, - include_preprocessing=preprocessor is not None, **kwargs, ) - # All references to `self` below this line - self.backbone = backbone - self.preprocessor = preprocessor + # === Default compilation === self.compile( loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True), optimizer=keras.optimizers.Adam(5e-5), diff --git a/keras_nlp/models/t5/t5_backbone.py b/keras_nlp/models/t5/t5_backbone.py index 6e76094d71..3fdf69ff30 100644 --- a/keras_nlp/models/t5/t5_backbone.py +++ b/keras_nlp/models/t5/t5_backbone.py @@ -85,45 +85,21 @@ def __init__( tie_embedding_weights=True, **kwargs, ): - # Encoder inputs - encoder_token_ids = keras.Input( - shape=(None,), dtype="int32", name="encoder_token_ids" - ) - encoder_padding_mask = keras.Input( - shape=(None,), dtype="int32", name="encoder_padding_mask" - ) - - # Decoder inputs. - decoder_token_ids = keras.Input( - shape=(None,), dtype="int32", name="decoder_token_ids" - ) - decoder_padding_mask = keras.Input( - shape=(None,), dtype="int32", name="decoder_padding_mask" - ) - # Token embedding layer. This layer is shared by encoder and decoder. - token_embedding_layer = ReversibleEmbedding( + self.token_embedding = ReversibleEmbedding( input_dim=vocabulary_size, output_dim=hidden_dim, tie_weights=tie_embedding_weights, embeddings_initializer=keras.initializers.TruncatedNormal(1.0), name="token_embedding", ) - - # ===== Encoder ===== - - # Embed tokens. - token_embedding = token_embedding_layer(encoder_token_ids) - x = keras.layers.Dropout( + self.encoder_embedding_dropout = keras.layers.Dropout( dropout, name="encoder_embedding_dropout", - )(token_embedding) - - encoder_attention_mask = encoder_padding_mask[:, None, :] - - position_bias = None + ) + self.encoder_transformer_layers = [] for i in range(num_layers): - output = T5TransformerLayer( + layer = T5TransformerLayer( is_decoder=False, hidden_dim=hidden_dim, intermediate_dim=intermediate_dim, @@ -135,39 +111,23 @@ def __init__( use_gated_activation=use_gated_activation, use_relative_attention_bias=bool(i == 0), name=f"transformer_encoder_layer_{i}", - )( - x, - attention_mask=encoder_attention_mask, - position_bias=position_bias, - use_causal_mask=False, ) - if isinstance(output, tuple): - x, position_bias = output - - x = T5LayerNorm( + self.encoder_transformer_layers.append(layer) + self.encoder_layer_norm = T5LayerNorm( epsilon=layer_norm_epsilon, name="encoder_output_layer_norm", - )(x) - x = keras.layers.Dropout( + ) + self.encoder_dropout = keras.layers.Dropout( dropout, name="encoder_output_dropout", - )(x) - encoder_output = x - - # ===== Decoder ===== - - # Embed tokens. - token_embedding = token_embedding_layer(decoder_token_ids) - x = keras.layers.Dropout( + ) + self.decoder_embedding_dropout = keras.layers.Dropout( dropout, name="decoder_embedding_dropout", - )(token_embedding) - - decoder_attention_mask = decoder_padding_mask[:, None, :] - - position_bias = None + ) + self.decoder_transformer_layers = [] for i in range(num_layers): - output = T5TransformerLayer( + layer = T5TransformerLayer( is_decoder=True, hidden_dim=hidden_dim, intermediate_dim=intermediate_dim, @@ -179,7 +139,54 @@ def __init__( use_gated_activation=use_gated_activation, use_relative_attention_bias=bool(i == 0), name=f"transformer_decoder_layer_{i}", - )( + ) + self.decoder_transformer_layers.append(layer) + self.decoder_layer_norm = T5LayerNorm( + epsilon=layer_norm_epsilon, + name="decoder_output_layer_norm", + ) + self.decoder_dropout = keras.layers.Dropout( + dropout, + name="decoder_output_dropout", + ) + + # === Functional Model === + encoder_token_id_input = keras.Input( + shape=(None,), dtype="int32", name="encoder_token_ids" + ) + encoder_padding_mask_input = keras.Input( + shape=(None,), dtype="int32", name="encoder_padding_mask" + ) + decoder_token_id_input = keras.Input( + shape=(None,), dtype="int32", name="decoder_token_ids" + ) + decoder_padding_mask_input = keras.Input( + shape=(None,), dtype="int32", name="decoder_padding_mask" + ) + # Encoder. + x = self.token_embedding(encoder_token_id_input) + x = self.encoder_embedding_dropout(x) + encoder_attention_mask = encoder_padding_mask_input[:, None, :] + position_bias = None + for transformer_layer in self.encoder_transformer_layers: + output = transformer_layer( + x, + attention_mask=encoder_attention_mask, + position_bias=position_bias, + use_causal_mask=False, + ) + if isinstance(output, tuple): + x, position_bias = output + x = self.encoder_layer_norm(x) + x = self.encoder_dropout(x) + encoder_output = x + # Decoder. + x = self.token_embedding(decoder_token_id_input) + x = self.decoder_embedding_dropout(x) + decoder_attention_mask = decoder_padding_mask_input[:, None, :] + position_bias = None + for transformer_layer in self.decoder_transformer_layers: + output = transformer_layer( x, attention_mask=decoder_attention_mask, position_bias=position_bias, @@ -189,23 +196,15 @@ def __init__( ) if isinstance(output, tuple): x, position_bias = output - - x = T5LayerNorm( - epsilon=layer_norm_epsilon, - name="decoder_output_layer_norm", - )(x) - x = keras.layers.Dropout( - dropout, - name="decoder_output_dropout", - )(x) + x = self.decoder_layer_norm(x) + x = self.decoder_dropout(x) decoder_output = x - super().__init__( { - "encoder_token_ids": encoder_token_ids, - "encoder_padding_mask": encoder_padding_mask, - "decoder_token_ids": decoder_token_ids, - "decoder_padding_mask": decoder_padding_mask, + "encoder_token_ids": encoder_token_id_input, + "encoder_padding_mask": encoder_padding_mask_input, + "decoder_token_ids": decoder_token_id_input, + "decoder_padding_mask": decoder_padding_mask_input, }, outputs={ "encoder_sequence_output": encoder_output, @@ -213,7 +212,8 @@ def __init__( }, **kwargs, ) - # All references to `self` below this line + + # === Config === self.vocabulary_size = vocabulary_size self.hidden_dim = hidden_dim self.intermediate_dim = intermediate_dim @@ -225,7 +225,6 @@ def __init__( self.use_gated_activation = use_gated_activation self.layer_norm_epsilon = layer_norm_epsilon self.tie_embedding_weights = tie_embedding_weights - self.token_embedding = token_embedding_layer def get_config(self): config = super().get_config() diff --git a/keras_nlp/models/task.py b/keras_nlp/models/task.py index ee28e3a984..1fe8d0b789 100644 --- a/keras_nlp/models/task.py +++ b/keras_nlp/models/task.py @@ -31,18 +31,25 @@ class Task(PipelineModel): """Base class for Task models.""" def __init__(self, *args, **kwargs): - self._backbone = None - self._preprocessor = None super().__init__(*args, **kwargs) self._functional_layer_ids = set( id(layer) for layer in self._flatten_layers() ) + self._initialized = True def __dir__(self): - # Temporary fixes for weight saving. This mimics the following PR for + if config.keras_3(): + return super().__dir__() + + # Temporary fixes for Keras 2 saving. This mimics the following PR for # older version of Keras: https://github.com/keras-team/keras/pull/18982 def filter_fn(attr): - if attr == "_layer_checkpoint_dependencies": + if attr in [ + "_layer_checkpoint_dependencies", + "transformer_layers", + "encoder_transformer_layers", + "decoder_transformer_layers", + ]: return False return id(getattr(self, attr)) not in self._functional_layer_ids @@ -99,17 +106,28 @@ def compile(self, optimizer="rmsprop", loss=None, **kwargs): super().compile(optimizer=optimizer, loss=loss, **kwargs) def preprocess_samples(self, x, y=None, sample_weight=None): - return self.preprocessor(x, y=y, sample_weight=sample_weight) + if self.preprocessor is not None: + return self.preprocessor(x, y=y, sample_weight=sample_weight) + else: + return super().preprocess_samples(x, y, sample_weight) def __setattr__(self, name, value): - # Work around torch setattr for properties. - if name in ["backbone", "preprocessor"]: + # Work around setattr issues for Keras 2 and Keras 3 torch backend. + # Since all our state is covered by functional model we can route + # around custom setattr calls. + is_property = isinstance(getattr(type(self), name, None), property) + is_unitialized = not hasattr(self, "_initialized") + is_torch = config.backend() == "torch" + is_keras_2 = not config.keras_3() + if is_torch and (is_property or is_unitialized): + return object.__setattr__(self, name, value) + if is_keras_2 and is_unitialized: return object.__setattr__(self, name, value) return super().__setattr__(name, value) @property def backbone(self): - """A `keras.Model` instance providing the backbone submodel.""" + """A `keras.Model` instance providing the backbone sub-model.""" return self._backbone @backbone.setter @@ -123,7 +141,6 @@ def preprocessor(self): @preprocessor.setter def preprocessor(self, value): - self.include_preprocessing = value is not None self._preprocessor = value def get_config(self): diff --git a/keras_nlp/models/task_test.py b/keras_nlp/models/task_test.py index 09fe1b0086..bf82e4fa68 100644 --- a/keras_nlp/models/task_test.py +++ b/keras_nlp/models/task_test.py @@ -32,11 +32,11 @@ def __init__(self, **kwargs): class SimpleTask(Task): def __init__(self, preprocessor=None, activation=None, **kwargs): + self.preprocessor = preprocessor + self.activation = keras.activations.get(activation) inputs = keras.Input(shape=(5,)) outputs = keras.layers.Dense(5)(inputs) super().__init__(inputs, outputs, **kwargs) - self.preprocessor = preprocessor - self.activation = keras.activations.get(activation) class TestTask(TestCase): diff --git a/keras_nlp/models/whisper/whisper_backbone.py b/keras_nlp/models/whisper/whisper_backbone.py index 32cfab215b..2e84219091 100644 --- a/keras_nlp/models/whisper/whisper_backbone.py +++ b/keras_nlp/models/whisper/whisper_backbone.py @@ -116,75 +116,40 @@ def __init__( ): assert_tf_backend(self.__class__.__name__) - # Encoder inputs. Note that the encoder does not have a padding mask: - # https://github.com/openai/whisper/blob/v20230124/whisper/model.py#L132. - encoder_feature_input = keras.Input( - shape=(None, num_mels), dtype="float32", name="encoder_features" - ) - - # Decoder inputs. - decoder_token_id_input = keras.Input( - shape=(None,), dtype="int32", name="decoder_token_ids" - ) - decoder_padding_mask = keras.Input( - shape=(None,), dtype="int32", name="decoder_padding_mask" - ) - - # ====== Encoder ====== - - # Embed the input features. This consists of two 1D convolutional - # layers. - # For the first layer, we use `padding="same"` since that corresponds to - # a padding size of 1. - encoder_conv_layer_1 = keras.layers.Conv1D( + # === Layers === + self.encoder_conv_layer_1 = keras.layers.Conv1D( filters=hidden_dim, kernel_size=3, strides=1, padding="same", name="encoder_token_embedding_conv_layer_1", ) - embedded_features = keras.activations.gelu( - encoder_conv_layer_1(encoder_feature_input), - approximate=False, - ) - - # For the second conv. layer, we cannot use `padding="same"` since - # that corresponds to a padding size of 1.5 (since stride is 2). Hence, - # we will manually pad the input. - embedded_features = Padder()(embedded_features) - encoder_conv_layer_2 = keras.layers.Conv1D( + self.encoder_conv_layer_2 = keras.layers.Conv1D( filters=hidden_dim, kernel_size=3, strides=2, padding="valid", name="encoder_token_embedding_conv_layer_2", ) - embedded_features = keras.activations.gelu( - encoder_conv_layer_2(embedded_features), - approximate=False, + self.encoder_padder = Padder( + name="encoder_padder", ) - - # The position embedding layer for the encoder is a sinusoidal embedding - # layer: https://github.com/openai/whisper/blob/v20230124/whisper/model.py#L137. - # Hence, we set it to be non-trainable. - # TODO: We can use `keras_nlp.layers.SinePositionEncoding` layer. - position_embedding = PositionEmbedding( + self.encoder_position_embedding = PositionEmbedding( initializer=whisper_kernel_initializer(), sequence_length=max_encoder_sequence_length // 2, name="encoder_position_embedding", trainable=False, - )(embedded_features) - - # Sum and apply dropout to embeddings. - x = keras.layers.Add()((embedded_features, position_embedding)) - x = keras.layers.Dropout( + ) + self.encoder_embeddings_add = keras.layers.Add( + name="encoder_embeddings_add", + ) + self.encoder_embeddings_dropout = keras.layers.Dropout( dropout, name="encoder_embeddings_dropout", - )(x) - - # Apply successive transformer encoder blocks. + ) + self.encoder_transformer_layers = [] for i in range(num_layers): - x = WhisperEncoder( + layer = WhisperEncoder( num_heads=num_heads, intermediate_dim=intermediate_dim, activation=keras.activations.gelu, @@ -193,37 +158,29 @@ def __init__( kernel_initializer=whisper_kernel_initializer(), normalize_first=True, name=f"transformer_encoder_layer_{i}", - )(x) - - x = keras.layers.LayerNormalization( + ) + self.encoder_transformer_layers.append(layer) + self.encoder_layer_norm = keras.layers.LayerNormalization( name="encoder_layer_norm", axis=-1, epsilon=1e-5, dtype="float32", - )(x) - encoder_output = x - - # ====== Decoder ====== - - # Embed tokens and positions. - embedding_layer = TokenAndPositionEmbedding( + ) + self.decoder_embeddings = TokenAndPositionEmbedding( vocabulary_size=vocabulary_size, sequence_length=max_decoder_sequence_length, embedding_dim=hidden_dim, embeddings_initializer=whisper_kernel_initializer(), name="decoder_token_and_position_embedding", ) - x = embedding_layer(decoder_token_id_input) - - # Apply dropout to embeddings. - x = keras.layers.Dropout( + self.token_embedding = self.decoder_embeddings.token_embedding + self.decoder_embeddings_dropout = keras.layers.Dropout( dropout, name="decoder_embeddings_dropout", - )(x) - - # Apply successive transformer decoder blocks. + ) + self.decoder_transformer_layers = [] for i in range(num_layers): - transformer_decoder_layer = WhisperDecoder( + layer = WhisperDecoder( intermediate_dim=intermediate_dim, num_heads=num_heads, dropout=dropout, @@ -233,26 +190,70 @@ def __init__( normalize_first=True, name=f"transformer_decoder_layer_{i}", ) - x = transformer_decoder_layer( - decoder_sequence=x, - encoder_sequence=encoder_output, - decoder_padding_mask=decoder_padding_mask, - ) - - x = keras.layers.LayerNormalization( + self.decoder_transformer_layers.append(layer) + self.decoder_layer_norm = keras.layers.LayerNormalization( name="decoder_layer_norm", axis=-1, epsilon=1e-5, dtype="float32", - )(x) - decoder_output = x + ) - # Instantiate using Functional API Model constructor + # === Functional Model === + # Note that the encoder does not have a padding mask: + # https://github.com/openai/whisper/blob/v20230124/whisper/model.py#L132. + encoder_feature_input = keras.Input( + shape=(None, num_mels), dtype="float32", name="encoder_features" + ) + decoder_token_id_input = keras.Input( + shape=(None,), dtype="int32", name="decoder_token_ids" + ) + decoder_padding_mask_input = keras.Input( + shape=(None,), dtype="int32", name="decoder_padding_mask" + ) + # Encoder. + # Embed the input features. This consists of two 1D convolutional + # layers. + # For the first layer, we use `padding="same"` since that corresponds to + # a padding size of 1. + embedded_features = keras.activations.gelu( + self.encoder_conv_layer_1(encoder_feature_input), + approximate=False, + ) + # For the second conv. layer, we cannot use `padding="same"` since + # that corresponds to a padding size of 1.5 (since stride is 2). Hence, + # we will manually pad the input. + embedded_features = Padder()(embedded_features) + embedded_features = keras.activations.gelu( + self.encoder_conv_layer_2(embedded_features), + approximate=False, + ) + # The position embedding layer for the encoder is a sinusoidal embedding + # layer: https://github.com/openai/whisper/blob/v20230124/whisper/model.py#L137. + # Hence, we set it to be non-trainable. + # TODO: We can use `keras_nlp.layers.SinePositionEncoding` layer. + positions = self.encoder_position_embedding(embedded_features) + x = self.encoder_embeddings_add((embedded_features, positions)) + x = self.encoder_embeddings_dropout(x) + for transformer_layer in self.encoder_transformer_layers: + x = transformer_layer(x) + x = self.encoder_layer_norm(x) + encoder_output = x + # Decoder. + x = self.decoder_embeddings(decoder_token_id_input) + x = self.decoder_embeddings_dropout(x) + for transformer_layer in self.decoder_transformer_layers: + x = transformer_layer( + decoder_sequence=x, + encoder_sequence=encoder_output, + decoder_padding_mask=decoder_padding_mask_input, + ) + x = self.decoder_layer_norm(x) + decoder_output = x super().__init__( inputs={ "encoder_features": encoder_feature_input, "decoder_token_ids": decoder_token_id_input, - "decoder_padding_mask": decoder_padding_mask, + "decoder_padding_mask": decoder_padding_mask_input, }, outputs={ "encoder_sequence_output": encoder_output, @@ -261,7 +262,7 @@ def __init__( **kwargs, ) - # All references to `self` below this line + # === Config === self.vocabulary_size = vocabulary_size self.num_layers = num_layers self.num_heads = num_heads @@ -271,7 +272,6 @@ def __init__( self.dropout = dropout self.max_encoder_sequence_length = max_encoder_sequence_length self.max_decoder_sequence_length = max_decoder_sequence_length - self.token_embedding = embedding_layer def get_config(self): config = super().get_config() diff --git a/keras_nlp/models/xlm_roberta/xlm_roberta_classifier.py b/keras_nlp/models/xlm_roberta/xlm_roberta_classifier.py index 67a9dd5bef..45d79eb304 100644 --- a/keras_nlp/models/xlm_roberta/xlm_roberta_classifier.py +++ b/keras_nlp/models/xlm_roberta/xlm_roberta_classifier.py @@ -157,30 +157,45 @@ def __init__( dropout=0.0, **kwargs, ): - inputs = backbone.input + # === Layers === + self.backbone = backbone + self.preprocessor = preprocessor + self.pooled_dropout = keras.layers.Dropout( + dropout, + name="pooled_dropout", + ) hidden_dim = hidden_dim or backbone.hidden_dim - - x = backbone(inputs)[:, backbone.start_token_index, :] - x = keras.layers.Dropout(dropout, name="pooled_dropout")(x) - x = keras.layers.Dense( - hidden_dim, activation="tanh", name="pooled_dense" - )(x) - x = keras.layers.Dropout(dropout, name="classifier_dropout")(x) - outputs = keras.layers.Dense( + self.pooled_dense = keras.layers.Dense( + hidden_dim, + activation="tanh", + name="pooled_dense", + ) + self.output_dropout = keras.layers.Dropout( + dropout, + name="output_dropout", + ) + self.output_dense = keras.layers.Dense( num_classes, kernel_initializer=roberta_kernel_initializer(), activation=activation, name="logits", - )(x) + ) + # === Functional Model === + inputs = backbone.input + x = backbone(inputs)[:, backbone.start_token_index, :] + x = self.pooled_dropout(x) + x = self.pooled_dense(x) + x = self.output_dropout(x) + outputs = self.output_dense(x) # Instantiate using Functional API Model constructor super().__init__( inputs=inputs, outputs=outputs, - include_preprocessing=preprocessor is not None, **kwargs, ) - # All references to `self` below this line + + # === Config === self.backbone = backbone self.preprocessor = preprocessor self.num_classes = num_classes @@ -188,6 +203,7 @@ def __init__( self.hidden_dim = hidden_dim self.dropout = dropout + # === Default compilation === logit_output = self.activation == keras.activations.linear self.compile( loss=keras.losses.SparseCategoricalCrossentropy( @@ -198,9 +214,6 @@ def __init__( jit_compile=True, ) - def preprocess_samples(self, x, y=None, sample_weight=None): - return self.preprocessor(x, y=y, sample_weight=sample_weight) - def get_config(self): config = super().get_config() config.update( diff --git a/keras_nlp/models/xlm_roberta/xlm_roberta_masked_lm.py b/keras_nlp/models/xlm_roberta/xlm_roberta_masked_lm.py index f0dfc85e84..b29aa30dd9 100644 --- a/keras_nlp/models/xlm_roberta/xlm_roberta_masked_lm.py +++ b/keras_nlp/models/xlm_roberta/xlm_roberta_masked_lm.py @@ -106,6 +106,18 @@ def __init__( preprocessor=None, **kwargs, ): + # === Layers === + self.backbone = backbone + self.preprocessor = preprocessor + self.masked_lm_head = MaskedLMHead( + vocabulary_size=backbone.vocabulary_size, + token_embedding=backbone.token_embedding, + intermediate_activation="gelu", + kernel_initializer=roberta_kernel_initializer(), + name="mlm_head", + ) + + # === Functional Model === inputs = { **backbone.input, "mask_positions": keras.Input( @@ -113,25 +125,16 @@ def __init__( ), } backbone_outputs = backbone(backbone.input) - outputs = MaskedLMHead( - vocabulary_size=backbone.vocabulary_size, - token_embedding=backbone.token_embedding, - intermediate_activation="gelu", - kernel_initializer=roberta_kernel_initializer(), - name="mlm_head", - )(backbone_outputs, inputs["mask_positions"]) - - # Instantiate using Functional API Model constructor. + outputs = self.masked_lm_head( + backbone_outputs, inputs["mask_positions"] + ) super().__init__( inputs=inputs, outputs=outputs, - include_preprocessing=preprocessor is not None, **kwargs, ) - # All references to `self` below this line - self.backbone = backbone - self.preprocessor = preprocessor + # === Default compilation === self.compile( loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True), optimizer=keras.optimizers.Adam(5e-5), diff --git a/keras_nlp/models/xlnet/xlnet_backbone.py b/keras_nlp/models/xlnet/xlnet_backbone.py index 1d1b4d2343..45a4b8f407 100644 --- a/keras_nlp/models/xlnet/xlnet_backbone.py +++ b/keras_nlp/models/xlnet/xlnet_backbone.py @@ -103,42 +103,25 @@ def __init__( bias_initializer="zeros", **kwargs, ): - # Inputs - token_id_input = keras.Input( - shape=(None,), dtype="int32", name="token_ids" - ) - padding_mask = keras.Input( - shape=(None,), dtype="int32", name="padding_mask" - ) - segment_ids = keras.Input( - shape=(None,), dtype="int32", name="segment_ids" - ) - - # Content and Query Embedding - word_emb, pos_emb = ContentAndQueryEmbedding( + # === Layers === + self.content_query_embedding = ContentAndQueryEmbedding( vocabulary_size=vocabulary_size, hidden_dim=hidden_dim, dropout=dropout, name="content_query_embedding", - )(token_id_input=token_id_input) - - # Apply XLNetAttentionMaskLayer and XLNetSegmentMatrixLayer Layers - # to get the processed attention masks and segment matrix. - attn_mask_content, attn_mask_query = XLNetAttentionMaskLayer( + ) + self.attn_mask_layer = XLNetAttentionMaskLayer( hidden_dim=hidden_dim, kernel_initializer_range=kernel_initializer_range, name="encoder_block_attn_mask_layer", - )(padding_mask) - seg_mat = XLNetSegmentMatrixLayer(name="encoder_block_seg_mat_layer")( - segment_ids ) - - output_content = word_emb - - # Encoders + self.seg_mat_layer = XLNetSegmentMatrixLayer( + name="encoder_block_seg_mat_layer", + ) head_dim = hidden_dim // num_heads + self.transformer_layers = [] for i in range(num_layers): - output_content, output_query = XLNetEncoder( + layer = XLNetEncoder( num_heads=num_heads, hidden_dim=hidden_dim, head_dim=head_dim, @@ -149,27 +132,52 @@ def __init__( kernel_initializer_range=kernel_initializer_range, bias_initializer=bias_initializer, name=f"xlnet_encoder_{i}", - )( + ) + self.transformer_layers.append(layer) + self.dropout = keras.layers.Dropout( + dropout, + name="dropout", + ) + + # === Functional Model === + token_id_input = keras.Input( + shape=(None,), dtype="int32", name="token_ids" + ) + padding_mask_input = keras.Input( + shape=(None,), dtype="int32", name="padding_mask" + ) + segment_id_input = keras.Input( + shape=(None,), dtype="int32", name="segment_ids" + ) + # Content and Query Embedding + word_emb, pos_emb = self.content_query_embedding(token_id_input) + # Apply XLNetAttentionMaskLayer and XLNetSegmentMatrixLayer Layers + # to get the processed attention masks and segment matrix. + attn_mask_content, attn_mask_query = self.attn_mask_layer( + padding_mask_input + ) + seg_mat = self.seg_mat_layer(segment_id_input) + output_content = word_emb + for transformer_layer in self.transformer_layers: + output_content, output_query = transformer_layer( output_content=output_content, attn_mask_content=attn_mask_content, attn_mask_query=attn_mask_query, pos_emb=pos_emb, seg_mat=seg_mat, ) - - output = keras.layers.Dropout(dropout)(output_content) - + output = self.dropout(output_content) super().__init__( inputs={ "token_ids": token_id_input, - "padding_mask": padding_mask, - "segment_ids": segment_ids, + "padding_mask": padding_mask_input, + "segment_ids": segment_id_input, }, outputs=output, **kwargs, ) - # All references to `self` below this line + # === Config === self.vocabulary_size = vocabulary_size self.num_layers = num_layers self.num_heads = num_heads diff --git a/keras_nlp/utils/pipeline_model.py b/keras_nlp/utils/pipeline_model.py index fa08aaf929..89a2f81822 100644 --- a/keras_nlp/utils/pipeline_model.py +++ b/keras_nlp/utils/pipeline_model.py @@ -142,27 +142,15 @@ def _split(t, start, end): class PipelineModel(keras.Model): """A model which allows automatically applying preprocessing.""" - def __init__(self, *args, include_preprocessing=True, **kwargs): + def __init__(self, *args, **kwargs): # Workaround for https://github.com/keras-team/keras/issues/17270 # Reset any attempt to overwrite this classes base class to this class # can continue to be used for functional and non-functional models. PipelineModel.__bases__ = (keras.Model,) super().__init__(*args, **kwargs) - self.include_preprocessing = include_preprocessing - - def preprocess_features(self, x): - """An overridable function which preprocesses features.""" - return x - - def preprocess_labels(self, y): - """An overridable function which preprocesses labels.""" - return y def preprocess_samples(self, x, y=None, sample_weight=None): """An overridable function which preprocesses entire samples.""" - x = self.preprocess_features(x) - if y is not None: - y = self.preprocess_labels(y) return pack_x_y_sample_weight(x, y, sample_weight) # ======================================================================== @@ -184,10 +172,9 @@ def fit( ) x = _convert_inputs_to_dataset(x, y, sample_weight, batch_size) - if self.include_preprocessing: - x = x.map( - self.preprocess_samples, num_parallel_calls=tf.data.AUTOTUNE - ).prefetch(tf.data.AUTOTUNE) + x = x.map( + self.preprocess_samples, num_parallel_calls=tf.data.AUTOTUNE + ).prefetch(tf.data.AUTOTUNE) if validation_data is not None: if not isinstance(validation_data, tf.data.Dataset): @@ -221,10 +208,9 @@ def evaluate( # needs preprocessing. kwargs.pop("_use_cached_eval_dataset", None) x = _convert_inputs_to_dataset(x, y, sample_weight, batch_size) - if self.include_preprocessing: - x = x.map( - self.preprocess_samples, num_parallel_calls=tf.data.AUTOTUNE - ).prefetch(tf.data.AUTOTUNE) + x = x.map( + self.preprocess_samples, num_parallel_calls=tf.data.AUTOTUNE + ).prefetch(tf.data.AUTOTUNE) return super().evaluate( x=x, y=None, @@ -239,11 +225,9 @@ def predict( **kwargs, ): x = _convert_inputs_to_dataset(x, None, None, batch_size) - if self.include_preprocessing: - x = x.map( - self.preprocess_samples, num_parallel_calls=tf.data.AUTOTUNE - ).prefetch(tf.data.AUTOTUNE) - + x = x.map( + self.preprocess_samples, num_parallel_calls=tf.data.AUTOTUNE + ).prefetch(tf.data.AUTOTUNE) return super().predict( x=x, batch_size=None, @@ -257,14 +241,13 @@ def train_on_batch( sample_weight=None, **kwargs, ): - if self.include_preprocessing: - data = self.preprocess_samples(x, y, sample_weight) - x, y, sample_weight = keras.utils.unpack_x_y_sample_weight(data) - x = ops.convert_to_tensor(x) - if y is not None: - y = ops.convert_to_tensor(y) - if sample_weight is not None: - sample_weight = ops.convert_to_tensor(sample_weight) + data = self.preprocess_samples(x, y, sample_weight) + x, y, sample_weight = keras.utils.unpack_x_y_sample_weight(data) + x = ops.convert_to_tensor(x) + if y is not None: + y = ops.convert_to_tensor(y) + if sample_weight is not None: + sample_weight = ops.convert_to_tensor(sample_weight) return super().train_on_batch( x=x, y=y, @@ -279,14 +262,13 @@ def test_on_batch( sample_weight=None, **kwargs, ): - if self.include_preprocessing: - data = self.preprocess_samples(x, y, sample_weight) - x, y, sample_weight = keras.utils.unpack_x_y_sample_weight(data) - x = ops.convert_to_tensor(x) - if y is not None: - y = ops.convert_to_tensor(y) - if sample_weight is not None: - sample_weight = ops.convert_to_tensor(sample_weight) + data = self.preprocess_samples(x, y, sample_weight) + x, y, sample_weight = keras.utils.unpack_x_y_sample_weight(data) + x = ops.convert_to_tensor(x) + if y is not None: + y = ops.convert_to_tensor(y) + if sample_weight is not None: + sample_weight = ops.convert_to_tensor(sample_weight) return super().test_on_batch( x=x, y=y, @@ -299,10 +281,9 @@ def predict_on_batch( x, **kwargs, ): - if self.include_preprocessing: - data = self.preprocess_samples(x) - x, _, _ = keras.utils.unpack_x_y_sample_weight(data) - x = ops.convert_to_tensor(x) + data = self.preprocess_samples(x) + x, _, _ = keras.utils.unpack_x_y_sample_weight(data) + x = ops.convert_to_tensor(x) return super().predict_on_batch( x=x, **kwargs, diff --git a/keras_nlp/utils/pipeline_model_test.py b/keras_nlp/utils/pipeline_model_test.py index 4c7c7f1964..ae71f9f570 100644 --- a/keras_nlp/utils/pipeline_model_test.py +++ b/keras_nlp/utils/pipeline_model_test.py @@ -36,8 +36,9 @@ def __init__(self, **kwargs): super().__init__(**kwargs) self.dense = keras.layers.Dense(1) - def preprocess_features(self, x): - return tf.strings.to_number(x) + def preprocess_samples(self, x, y=None, sample_weight=None): + x = tf.strings.to_number(x) + return keras.utils.pack_x_y_sample_weight(x, y, sample_weight) def call(self, inputs): return self.dense(inputs) @@ -48,8 +49,10 @@ def __init__(self, **kwargs): super().__init__(**kwargs) self.dense = keras.layers.Dense(1) - def preprocess_labels(self, y): - return tf.strings.to_number(y) + def preprocess_samples(self, x, y=None, sample_weight=None): + if y is not None: + y = tf.strings.to_number(y) + return keras.utils.pack_x_y_sample_weight(x, y, sample_weight) def call(self, inputs): return self.dense(inputs) @@ -63,8 +66,7 @@ def __init__(self, **kwargs): self.dense = keras.layers.Dense(1) def preprocess_samples(self, x, y=None, sample_weight=None): - x = tf.strings.to_number(x) - y = x + y = x = tf.strings.to_number(x) return keras.utils.pack_x_y_sample_weight(x, y, sample_weight) def call(self, inputs): @@ -77,8 +79,9 @@ def __init__(self, **kwargs): outputs = keras.layers.Dense(1)(inputs) super().__init__(inputs, outputs, **kwargs) - def preprocess_features(self, x): - return tf.strings.to_number(x) + def preprocess_samples(self, x, y=None, sample_weight=None): + x = tf.strings.to_number(x) + return keras.utils.pack_x_y_sample_weight(x, y, sample_weight) def get_config(self): return {} @@ -167,19 +170,6 @@ def test_fit_with_preprocessing(self): model.fit(x=x, y=y, batch_size=8) model.fit(tf.data.Dataset.from_tensor_slices((x, y)).batch(8)) - def test_fit_no_preprocessing(self): - x = np.random.uniform(size=(100, 5)) - y = np.random.uniform(size=(100, 1)) - sw = np.random.uniform(size=(100, 1)) - model = FeaturePipeline(include_preprocessing=False) - model.compile(loss="mse") - # With sample weight. - model.fit(x=x, y=y, sample_weight=sw, batch_size=8) - model.fit(tf.data.Dataset.from_tensor_slices((x, y, sw)).batch(8)) - # Without sample weight. - model.fit(x=x, y=y, batch_size=8) - model.fit(tf.data.Dataset.from_tensor_slices((x, y)).batch(8)) - def test_evaluate_with_preprocessing(self): x = tf.strings.as_string(np.random.uniform(size=(100, 5))) y = np.random.uniform(size=(100, 1)) @@ -193,19 +183,6 @@ def test_evaluate_with_preprocessing(self): model.evaluate(x=x, y=y, batch_size=8) model.evaluate(tf.data.Dataset.from_tensor_slices((x, y)).batch(8)) - def test_evaluate_no_preprocessing(self): - x = np.random.uniform(size=(100, 5)) - y = np.random.uniform(size=(100, 1)) - sw = np.random.uniform(size=(100, 1)) - model = FeaturePipeline(include_preprocessing=False) - model.compile(loss="mse") - # With sample weight. - model.evaluate(x=x, y=y, sample_weight=sw, batch_size=8) - model.evaluate(tf.data.Dataset.from_tensor_slices((x, y, sw)).batch(8)) - # Without sample weight. - model.evaluate(x=x, y=y, batch_size=8) - model.evaluate(tf.data.Dataset.from_tensor_slices((x, y)).batch(8)) - def test_predict_with_preprocessing(self): x = tf.strings.as_string(np.random.uniform(size=(100, 5))) model = FeaturePipeline() @@ -213,13 +190,6 @@ def test_predict_with_preprocessing(self): model.predict(x=x, batch_size=8) model.predict(tf.data.Dataset.from_tensor_slices(x).batch(8)) - def test_predict_no_preprocessing(self): - x = np.random.uniform(size=(100, 5)) - model = FeaturePipeline(include_preprocessing=False) - model.compile(loss="mse") - model.predict(x=x, batch_size=8) - model.predict(tf.data.Dataset.from_tensor_slices(x).batch(8)) - def test_on_batch(self): x = tf.strings.as_string(np.random.uniform(size=(8, 5))) y = np.random.uniform(size=(8, 1)) @@ -234,19 +204,6 @@ def test_on_batch(self): model.test_on_batch(x=x, y=y) model.predict_on_batch(x=x) - def test_on_batch_no_preprocessing(self): - x = np.random.uniform(size=(8, 5)) - y = np.random.uniform(size=(8, 1)) - sw = np.random.uniform(size=(8, 1)) - model = FeaturePipeline(include_preprocessing=False) - model.compile(loss="mse") - # With sample weight. - model.train_on_batch(x=x, y=y, sample_weight=sw) - model.test_on_batch(x=x, y=y, sample_weight=sw) - # Without sample weight. - model.train_on_batch(x=x, y=y) - model.test_on_batch(x=x, y=y) - def test_saved_model(self): model = FeaturePipeline() x = tf.strings.as_string(np.random.uniform(size=(8, 5))) @@ -278,19 +235,6 @@ def test_fit_with_preprocessing(self): model.fit(x=x, y=y, batch_size=8) model.fit(tf.data.Dataset.from_tensor_slices((x, y)).batch(8)) - def test_fit_no_preprocessing(self): - x = np.random.uniform(size=(100, 5)) - y = np.random.uniform(size=(100, 1)) - sw = np.random.uniform(size=(100, 1)) - model = LabelPipeline(include_preprocessing=False) - model.compile(loss="mse") - # With sample weight. - model.fit(x=x, y=y, sample_weight=sw, batch_size=8) - model.fit(tf.data.Dataset.from_tensor_slices((x, y, sw)).batch(8)) - # Without sample weight. - model.fit(x=x, y=y, batch_size=8) - model.fit(tf.data.Dataset.from_tensor_slices((x, y)).batch(8)) - def test_evaluate_with_preprocessing(self): x = np.random.uniform(size=(100, 5)) y = tf.strings.as_string(np.random.uniform(size=(100, 1))) @@ -304,19 +248,6 @@ def test_evaluate_with_preprocessing(self): model.evaluate(x=x, y=y, batch_size=8) model.evaluate(tf.data.Dataset.from_tensor_slices((x, y)).batch(8)) - def test_evaluate_no_preprocessing(self): - x = np.random.uniform(size=(100, 5)) - y = np.random.uniform(size=(100, 1)) - sw = np.random.uniform(size=(100, 1)) - model = LabelPipeline(include_preprocessing=False) - model.compile(loss="mse") - # With sample weight. - model.evaluate(x=x, y=y, sample_weight=sw, batch_size=8) - model.evaluate(tf.data.Dataset.from_tensor_slices((x, y, sw)).batch(8)) - # Without sample weight. - model.evaluate(x=x, y=y, batch_size=8) - model.evaluate(tf.data.Dataset.from_tensor_slices((x, y)).batch(8)) - def test_predict_with_preprocessing(self): x = np.random.uniform(size=(100, 5)) model = LabelPipeline() @@ -338,20 +269,6 @@ def test_on_batch(self): model.test_on_batch(x=x, y=y) model.predict_on_batch(x=x) - def test_on_batch_no_preprocessing(self): - x = np.random.uniform(size=(8, 5)) - y = np.random.uniform(size=(8, 1)) - sw = np.random.uniform(size=(8, 1)) - model = LabelPipeline(include_preprocessing=False) - model.compile(loss="mse") - # With sample weight. - model.train_on_batch(x=x, y=y, sample_weight=sw) - model.test_on_batch(x=x, y=y, sample_weight=sw) - # Without sample weight. - model.train_on_batch(x=x, y=y) - model.test_on_batch(x=x, y=y) - model.predict_on_batch(x=x) - def test_saved_model(self): model = LabelPipeline() x = np.random.uniform(size=(8, 5)) @@ -377,14 +294,6 @@ def test_fit_with_preprocessing(self): model.fit(x=data, batch_size=8) model.fit(tf.data.Dataset.from_tensor_slices(data).batch(8)) - def test_fit_no_preprocessing(self): - x = np.random.uniform(size=(100, 1)) - y = np.random.uniform(size=(100, 1)) - model = DataPipeline(include_preprocessing=False) - model.compile(loss="mse") - model.fit(x=x, y=y, batch_size=8) - model.fit(tf.data.Dataset.from_tensor_slices((x, y)).batch(8)) - def test_evaluate_with_preprocessing(self): data = tf.strings.as_string(np.random.uniform(size=(100, 1))) model = DataPipeline() @@ -392,14 +301,6 @@ def test_evaluate_with_preprocessing(self): model.evaluate(x=data, batch_size=8) model.evaluate(tf.data.Dataset.from_tensor_slices(data).batch(8)) - def test_evaluate_no_preprocessing(self): - x = np.random.uniform(size=(100, 1)) - y = np.random.uniform(size=(100, 1)) - model = DataPipeline(include_preprocessing=False) - model.compile(loss="mse") - model.evaluate(x=x, y=y, batch_size=8) - model.evaluate(tf.data.Dataset.from_tensor_slices((x, y)).batch(8)) - def test_predict_with_preprocessing(self): x = tf.strings.as_string(np.random.uniform(size=(100, 1))) model = DataPipeline() @@ -407,13 +308,6 @@ def test_predict_with_preprocessing(self): model.predict(x=x, batch_size=8) model.predict(tf.data.Dataset.from_tensor_slices(x).batch(8)) - def test_predict_no_preprocessing(self): - x = np.random.uniform(size=(100, 1)) - model = DataPipeline(include_preprocessing=False) - model.compile(loss="mse") - model.predict(x=x, batch_size=8) - model.predict(tf.data.Dataset.from_tensor_slices(x).batch(8)) - def test_on_batch(self): data = tf.strings.as_string(np.random.uniform(size=(8, 1))) model = DataPipeline() @@ -426,20 +320,6 @@ def test_on_batch(self): model.test_on_batch(x=data) model.predict_on_batch(x=data) - def test_on_batch_no_preprocessing(self): - x = np.random.uniform(size=(8, 1)) - y = np.random.uniform(size=(8, 1)) - sw = np.random.uniform(size=(8, 1)) - model = DataPipeline(include_preprocessing=False) - model.compile(loss="mse") - # With sample weight. - model.train_on_batch(x=x, y=y, sample_weight=sw) - model.test_on_batch(x=x, y=y, sample_weight=sw) - # Without sample weight. - model.train_on_batch(x=x, y=y) - model.test_on_batch(x=x, y=y) - model.predict_on_batch(x=x) - def test_saved_model(self): model = DataPipeline() data = tf.strings.as_string(np.random.uniform(size=(8, 1))) @@ -472,19 +352,6 @@ def test_fit(self): model.fit(x=x, y=y, batch_size=8) model.fit(tf.data.Dataset.from_tensor_slices((x, y)).batch(8)) - def test_fit_no_preprocessing(self): - x = np.random.uniform(size=(100, 5)) - y = np.random.uniform(size=(100, 1)) - sw = np.random.uniform(size=(100, 1)) - model = FunctionalPipeline(include_preprocessing=False) - model.compile(loss="mse") - # With sample weight. - model.fit(x=x, y=y, sample_weight=sw, batch_size=8) - model.fit(tf.data.Dataset.from_tensor_slices((x, y, sw)).batch(8)) - # Without sample weight. - model.fit(x=x, y=y, batch_size=8) - model.fit(tf.data.Dataset.from_tensor_slices((x, y)).batch(8)) - def test_saved_model(self): model = FunctionalPipeline() x = tf.strings.as_string(np.random.uniform(size=(8, 5))) From 22c1e30d412e847a8aefad6fd238e3eff0918b97 Mon Sep 17 00:00:00 2001 From: Matt Watson <1389937+mattdangerw@users.noreply.github.com> Date: Mon, 12 Feb 2024 22:06:20 -0800 Subject: [PATCH 15/29] Allow setting dtype per model (#1431) * Allow setting dtype per modeling This gives the following new syntax for setting dtype on a model. model = keras_nlp.models.BertBackbone.from_preset( "bert_base_en_uncased", dtype="bfloat16", ) model = keras_nlp.models.BertBackbone( vocabulary_size=30552, num_layers=4, num_heads=4, hidden_dim=256, intermediate_dim=512, max_sequence_length=128, dtype="bfloat16", ) classifier = keras_nlp.models.BertClassifier.from_preset( "bert_base_en_uncased", num_classes=4, dtype="bfloat16", ) * xlnet fix * whisper fix * torch gpu fix --- .../cached_multi_head_attention_test.py | 2 +- .../layers/modeling/masked_lm_head_test.py | 1 + .../modeling/sine_position_encoding_test.py | 10 ---- .../modeling/transformer_decoder_test.py | 1 - keras_nlp/models/albert/albert_backbone.py | 17 +++++- keras_nlp/models/albert/albert_classifier.py | 2 + keras_nlp/models/albert/albert_masked_lm.py | 1 + keras_nlp/models/backbone.py | 6 +- keras_nlp/models/bart/bart_backbone.py | 22 +++++-- keras_nlp/models/bert/bert_backbone.py | 16 ++++- keras_nlp/models/bert/bert_classifier.py | 2 + keras_nlp/models/bert/bert_masked_lm.py | 1 + keras_nlp/models/bloom/bloom_attention.py | 11 +++- keras_nlp/models/bloom/bloom_backbone.py | 9 +++ .../models/deberta_v3/deberta_v3_backbone.py | 11 +++- .../deberta_v3/deberta_v3_classifier.py | 4 ++ .../models/deberta_v3/deberta_v3_masked_lm.py | 1 + .../disentangled_attention_encoder.py | 7 +++ .../deberta_v3/disentangled_self_attention.py | 19 +++++- .../models/deberta_v3/relative_embedding.py | 4 +- .../distil_bert/distil_bert_backbone.py | 10 +++- .../distil_bert/distil_bert_classifier.py | 3 + .../distil_bert/distil_bert_masked_lm.py | 1 + keras_nlp/models/electra/electra_backbone.py | 21 ++++++- keras_nlp/models/f_net/f_net_backbone.py | 21 ++++++- keras_nlp/models/f_net/f_net_classifier.py | 2 + keras_nlp/models/f_net/f_net_masked_lm.py | 1 + keras_nlp/models/gpt2/gpt2_backbone.py | 14 ++++- .../models/gpt_neo_x/gpt_neo_x_attention.py | 15 ++++- .../models/gpt_neo_x/gpt_neo_x_backbone.py | 12 +++- .../models/gpt_neo_x/gpt_neo_x_decoder.py | 7 +++ keras_nlp/models/llama/llama_attention.py | 13 +++- keras_nlp/models/llama/llama_backbone.py | 11 +++- keras_nlp/models/llama/llama_decoder.py | 6 ++ keras_nlp/models/mistral/mistral_attention.py | 21 ++++--- keras_nlp/models/mistral/mistral_backbone.py | 7 ++- .../models/mistral/mistral_layer_norm.py | 1 - .../mistral/mistral_transformer_decoder.py | 13 ++-- keras_nlp/models/opt/opt_backbone.py | 9 ++- keras_nlp/models/roberta/roberta_backbone.py | 10 +++- .../models/roberta/roberta_classifier.py | 4 ++ keras_nlp/models/roberta/roberta_masked_lm.py | 1 + keras_nlp/models/t5/t5_backbone.py | 16 ++++- .../models/t5/t5_multi_head_attention.py | 19 ++++-- keras_nlp/models/t5/t5_transformer_layer.py | 41 ++++++++++--- keras_nlp/models/task.py | 5 ++ keras_nlp/models/whisper/whisper_backbone.py | 25 ++++++-- .../xlm_roberta/xlm_roberta_backbone.py | 4 ++ .../xlm_roberta/xlm_roberta_classifier.py | 4 ++ .../xlm_roberta/xlm_roberta_masked_lm.py | 1 + keras_nlp/models/xlnet/relative_attention.py | 5 ++ keras_nlp/models/xlnet/xlnet_backbone.py | 10 ++++ .../xlnet_content_and_query_embedding.py | 16 +++-- keras_nlp/models/xlnet/xlnet_encoder.py | 21 +++++-- keras_nlp/tests/test_case.py | 59 +++++++++++++------ 55 files changed, 459 insertions(+), 117 deletions(-) diff --git a/keras_nlp/layers/modeling/cached_multi_head_attention_test.py b/keras_nlp/layers/modeling/cached_multi_head_attention_test.py index 16b423e08c..6bf4311423 100644 --- a/keras_nlp/layers/modeling/cached_multi_head_attention_test.py +++ b/keras_nlp/layers/modeling/cached_multi_head_attention_test.py @@ -39,7 +39,7 @@ def test_layer_behaviors(self): expected_num_non_trainable_variables=1, # Keras 2 does not handle mixed precision correctly when not set # globally. - run_mixed_precision_check=config.keras_3(), + run_precision_checks=config.keras_3(), ) def test_cache_call_is_correct(self): diff --git a/keras_nlp/layers/modeling/masked_lm_head_test.py b/keras_nlp/layers/modeling/masked_lm_head_test.py index 8d22ea0343..69a6288911 100644 --- a/keras_nlp/layers/modeling/masked_lm_head_test.py +++ b/keras_nlp/layers/modeling/masked_lm_head_test.py @@ -58,6 +58,7 @@ def test_layer_behaviors_with_embedding(self): }, expected_output_shape=(4, 5, 100), expected_num_trainable_weights=6, + run_precision_checks=False, ) def test_value_error_when_neither_embedding_or_vocab_size_set(self): diff --git a/keras_nlp/layers/modeling/sine_position_encoding_test.py b/keras_nlp/layers/modeling/sine_position_encoding_test.py index 80dad26cbc..e7ea4b1b05 100644 --- a/keras_nlp/layers/modeling/sine_position_encoding_test.py +++ b/keras_nlp/layers/modeling/sine_position_encoding_test.py @@ -107,13 +107,3 @@ def test_start_index(self): sequential_output, (0, i, 0), parial_output ) self.assertAllClose(full_output, sequential_output) - - def test_float16_dtype(self): - pos_encoding = SinePositionEncoding(dtype="float16") - seq_length = 100 - hidden_size = 32 - inputs = keras.Input(shape=(seq_length, hidden_size)) - outputs = pos_encoding(inputs) - - # output dtype for this layer should be tf.float16. - self.assertEqual(outputs.dtype, "float16") diff --git a/keras_nlp/layers/modeling/transformer_decoder_test.py b/keras_nlp/layers/modeling/transformer_decoder_test.py index fda2ea7e33..4757a0a45c 100644 --- a/keras_nlp/layers/modeling/transformer_decoder_test.py +++ b/keras_nlp/layers/modeling/transformer_decoder_test.py @@ -49,7 +49,6 @@ def test_layer_behaviors(self, normalize_first): ("with_norm_first", True), ) def test_layer_behaviors_with_cross_attention(self, normalize_first): - pass self.run_layer_test( cls=TransformerDecoder, init_kwargs={ diff --git a/keras_nlp/models/albert/albert_backbone.py b/keras_nlp/models/albert/albert_backbone.py index 99a7cc97a3..1e342e791c 100644 --- a/keras_nlp/models/albert/albert_backbone.py +++ b/keras_nlp/models/albert/albert_backbone.py @@ -72,6 +72,10 @@ class AlbertBackbone(Backbone): embeddings. num_segments: int. The number of types that the 'segment_ids' input can take. + dtype: string or `keras.mixed_precision.DTypePolicy`. The dtype to use + for model computations and weights. Note that some computations, + such as softmax and layer normalization, will always be done at + float32 precision regardless of dtype. Examples: ```python @@ -110,6 +114,7 @@ def __init__( dropout=0.0, max_sequence_length=512, num_segments=2, + dtype=None, **kwargs, ): if num_layers % num_groups != 0: @@ -123,35 +128,41 @@ def __init__( input_dim=vocabulary_size, output_dim=embedding_dim, embeddings_initializer=albert_kernel_initializer(), + dtype=dtype, name="token_embedding", ) self.position_embedding = PositionEmbedding( initializer=albert_kernel_initializer(), sequence_length=max_sequence_length, + dtype=dtype, name="position_embedding", ) self.segment_embedding = keras.layers.Embedding( input_dim=num_segments, output_dim=embedding_dim, embeddings_initializer=albert_kernel_initializer(), + dtype=dtype, name="segment_embedding", ) self.embeddings_add = keras.layers.Add( + dtype=dtype, name="embeddings_add", ) self.embeddings_layer_norm = keras.layers.LayerNormalization( - name="embeddings_layer_norm", axis=-1, epsilon=1e-12, - dtype="float32", + dtype=dtype, + name="embeddings_layer_norm", ) self.embeddings_dropout = keras.layers.Dropout( dropout, + dtype=dtype, name="embeddings_dropout", ) self.embeddings_projection = keras.layers.Dense( hidden_dim, kernel_initializer=albert_kernel_initializer(), + dtype=dtype, name="embedding_projection", ) self.transformer_layers = [] @@ -165,6 +176,7 @@ def __init__( dropout=dropout, layer_norm_epsilon=1e-12, kernel_initializer=albert_kernel_initializer(), + dtype=dtype, name=f"group_{group_idx}_inner_layer_{inner_idx}", ) inner_layers.append(layer) @@ -173,6 +185,7 @@ def __init__( hidden_dim, kernel_initializer=albert_kernel_initializer(), activation="tanh", + dtype=dtype, name="pooled_dense", ) diff --git a/keras_nlp/models/albert/albert_classifier.py b/keras_nlp/models/albert/albert_classifier.py index e8621cf6b4..32a4e0847d 100644 --- a/keras_nlp/models/albert/albert_classifier.py +++ b/keras_nlp/models/albert/albert_classifier.py @@ -162,10 +162,12 @@ def __init__( num_classes, kernel_initializer=albert_kernel_initializer(), activation=activation, + dtype=backbone.dtype_policy, name="logits", ) self.output_dropout = keras.layers.Dropout( dropout, + dtype=backbone.dtype_policy, name="output_dropout", ) diff --git a/keras_nlp/models/albert/albert_masked_lm.py b/keras_nlp/models/albert/albert_masked_lm.py index 139be0a762..1958713b9f 100644 --- a/keras_nlp/models/albert/albert_masked_lm.py +++ b/keras_nlp/models/albert/albert_masked_lm.py @@ -105,6 +105,7 @@ def __init__(self, backbone, preprocessor=None, **kwargs): token_embedding=backbone.token_embedding, intermediate_activation=gelu_approximate, kernel_initializer=albert_kernel_initializer(), + dtype=backbone.dtype_policy, name="mlm_head", ) diff --git a/keras_nlp/models/backbone.py b/keras_nlp/models/backbone.py index 9e22be3f44..6fccf6013a 100644 --- a/keras_nlp/models/backbone.py +++ b/keras_nlp/models/backbone.py @@ -22,7 +22,7 @@ @keras.saving.register_keras_serializable(package="keras_nlp") class Backbone(keras.Model): - def __init__(self, *args, **kwargs): + def __init__(self, *args, dtype=None, **kwargs): super().__init__(*args, **kwargs) self._functional_layer_ids = set( id(layer) for layer in self._flatten_layers() @@ -74,8 +74,8 @@ def token_embedding(self, value): self._token_embedding = value def get_config(self): - # Don't chain to super here. The default `get_config()` for functional - # models is nested and cannot be passed to our Backbone constructors. + # Don't chain to super here. `get_config()` for functional models is + # a nested layer config and cannot be passed to Backbone constructors. return { "name": self.name, "trainable": self.trainable, diff --git a/keras_nlp/models/bart/bart_backbone.py b/keras_nlp/models/bart/bart_backbone.py index fdb8e5df5b..803d5a2a9f 100644 --- a/keras_nlp/models/bart/bart_backbone.py +++ b/keras_nlp/models/bart/bart_backbone.py @@ -60,6 +60,10 @@ class BartBackbone(Backbone): can consume. If None, `max_sequence_length` uses the value from sequence length. This determines the variable shape for positional embeddings. + dtype: string or `keras.mixed_precision.DTypePolicy`. The dtype to use + for model computations and weights. Note that some computations, + such as softmax and layer normalization, will always be done at + float32 precision regardless of dtype. Examples: ```python @@ -100,6 +104,7 @@ def __init__( intermediate_dim, dropout=0.1, max_sequence_length=1024, + dtype=None, **kwargs, ): # === Layers === @@ -107,24 +112,28 @@ def __init__( input_dim=vocabulary_size, output_dim=hidden_dim, embeddings_initializer=bart_kernel_initializer(), + dtype=dtype, name="token_embedding", ) self.encoder_position_embedding = PositionEmbedding( initializer=bart_kernel_initializer(), sequence_length=max_sequence_length, + dtype=dtype, name="encoder_position_embedding", ) self.encoder_embeddings_add = keras.layers.Add( + dtype=dtype, name="encoder_embeddings_add", ) self.encoder_embeddings_layer_norm = keras.layers.LayerNormalization( - name="encoder_embeddings_layer_norm", axis=-1, epsilon=1e-5, - dtype="float32", + dtype=dtype, + name="encoder_embeddings_layer_norm", ) self.encoder_embeddings_dropout = keras.layers.Dropout( dropout, + dtype=dtype, name="encoder_embeddings_dropout", ) self.encoder_transformer_layers = [] @@ -136,25 +145,29 @@ def __init__( dropout=dropout, layer_norm_epsilon=1e-5, kernel_initializer=bart_kernel_initializer(), + dtype=dtype, name=f"transformer_encoder_layer_{i}", ) self.encoder_transformer_layers.append(layer) self.decoder_position_embedding = PositionEmbedding( initializer=bart_kernel_initializer(), sequence_length=max_sequence_length, + dtype=dtype, name="decoder_position_embedding", ) self.decoder_embeddings_add = keras.layers.Add( + dtype=dtype, name="decoder_embeddings_add", ) self.decoder_embeddings_layer_norm = keras.layers.LayerNormalization( - name="decoder_embeddings_layer_norm", axis=-1, epsilon=1e-5, - dtype="float32", + dtype=dtype, + name="decoder_embeddings_layer_norm", ) self.decoder_embeddings_dropout = keras.layers.Dropout( dropout, + dtype=dtype, name="decoder_embeddings_dropout", ) self.decoder_transformer_layers = [] @@ -166,6 +179,7 @@ def __init__( activation=keras.activations.gelu, layer_norm_epsilon=1e-5, kernel_initializer=bart_kernel_initializer(), + dtype=dtype, name=f"transformer_decoder_layer_{i}", ) self.decoder_transformer_layers.append(layer) diff --git a/keras_nlp/models/bert/bert_backbone.py b/keras_nlp/models/bert/bert_backbone.py index f511de3687..2248260da7 100644 --- a/keras_nlp/models/bert/bert_backbone.py +++ b/keras_nlp/models/bert/bert_backbone.py @@ -61,6 +61,10 @@ class BertBackbone(Backbone): embeddings. num_segments: int. The number of types that the 'segment_ids' input can take. + dtype: string or `keras.mixed_precision.DTypePolicy`. The dtype to use + for model computations and weights. Note that some computations, + such as softmax and layer normalization, will always be done at + float32 precision regardless of dtype. Examples: ```python @@ -97,6 +101,7 @@ def __init__( dropout=0.1, max_sequence_length=512, num_segments=2, + dtype=None, **kwargs, ): # === Layers === @@ -104,30 +109,35 @@ def __init__( input_dim=vocabulary_size, output_dim=hidden_dim, embeddings_initializer=bert_kernel_initializer(), + dtype=dtype, name="token_embedding", ) self.position_embedding = PositionEmbedding( initializer=bert_kernel_initializer(), sequence_length=max_sequence_length, + dtype=dtype, name="position_embedding", ) self.segment_embedding = keras.layers.Embedding( input_dim=num_segments, output_dim=hidden_dim, embeddings_initializer=bert_kernel_initializer(), + dtype=dtype, name="segment_embedding", ) self.embeddings_add = keras.layers.Add( + dtype=dtype, name="embeddings_add", ) self.embeddings_layer_norm = keras.layers.LayerNormalization( - name="embeddings_layer_norm", axis=-1, epsilon=1e-12, - dtype="float32", + dtype=dtype, + name="embeddings_layer_norm", ) self.embeddings_dropout = keras.layers.Dropout( dropout, + dtype=dtype, name="embeddings_dropout", ) self.transformer_layers = [] @@ -139,6 +149,7 @@ def __init__( dropout=dropout, layer_norm_epsilon=1e-12, kernel_initializer=bert_kernel_initializer(), + dtype=dtype, name=f"transformer_layer_{i}", ) self.transformer_layers.append(layer) @@ -146,6 +157,7 @@ def __init__( hidden_dim, kernel_initializer=bert_kernel_initializer(), activation="tanh", + dtype=dtype, name="pooled_dense", ) diff --git a/keras_nlp/models/bert/bert_classifier.py b/keras_nlp/models/bert/bert_classifier.py index 3ddb90a9d2..09d2b8810c 100644 --- a/keras_nlp/models/bert/bert_classifier.py +++ b/keras_nlp/models/bert/bert_classifier.py @@ -145,12 +145,14 @@ def __init__( self.preprocessor = preprocessor self.output_dropout = keras.layers.Dropout( dropout, + dtype=backbone.dtype_policy, name="classifier_dropout", ) self.output_dense = keras.layers.Dense( num_classes, kernel_initializer=bert_kernel_initializer(), activation=activation, + dtype=backbone.dtype_policy, name="logits", ) diff --git a/keras_nlp/models/bert/bert_masked_lm.py b/keras_nlp/models/bert/bert_masked_lm.py index 555c562f1f..17b9669619 100644 --- a/keras_nlp/models/bert/bert_masked_lm.py +++ b/keras_nlp/models/bert/bert_masked_lm.py @@ -109,6 +109,7 @@ def __init__( token_embedding=backbone.token_embedding, intermediate_activation="gelu", kernel_initializer=bert_kernel_initializer(), + dtype=backbone.dtype_policy, name="mlm_head", ) diff --git a/keras_nlp/models/bloom/bloom_attention.py b/keras_nlp/models/bloom/bloom_attention.py index 366169d1fb..e36c6fac62 100644 --- a/keras_nlp/models/bloom/bloom_attention.py +++ b/keras_nlp/models/bloom/bloom_attention.py @@ -75,7 +75,9 @@ def build(self, inputs_shape): ) self._value_dense.build(inputs_shape) - self._alibi_layer = AlibiBias() + self._alibi_layer = AlibiBias( + dtype=self.dtype_policy, + ) self._output_dense = keras.layers.Dense( hidden_dim, @@ -87,10 +89,13 @@ def build(self, inputs_shape): self._output_dense.build(inputs_shape) self._dropout_layer = keras.layers.Dropout( - rate=self.dropout, dtype=self.dtype_policy, name="dropout" + rate=self.dropout, + dtype=self.dtype_policy, + name="dropout", ) self._softmax = keras.layers.Softmax( - dtype=self.dtype_policy, name="softmax" + dtype="float32", + name="softmax", ) self.built = True diff --git a/keras_nlp/models/bloom/bloom_backbone.py b/keras_nlp/models/bloom/bloom_backbone.py index 4f12bacd19..5737dcc889 100644 --- a/keras_nlp/models/bloom/bloom_backbone.py +++ b/keras_nlp/models/bloom/bloom_backbone.py @@ -55,6 +55,10 @@ class BloomBackbone(Backbone): the transformer decoder. max_sequence_length: int. The maximum sequence length that this decoder can consume. + dtype: string or `keras.mixed_precision.DTypePolicy`. The dtype to use + for model computations and weights. Note that some computations, + such as softmax and layer normalization, will always be done at + float32 precision regardless of dtype. Examples: ```python @@ -93,6 +97,7 @@ def __init__( dropout=0.0, layer_norm_epsilon=1e-5, max_sequence_length=2048, + dtype=None, **kwargs, ): # === Layers === @@ -101,10 +106,12 @@ def __init__( output_dim=hidden_dim, embeddings_initializer=_bloom_kernel_initializer(stddev=0.02), tie_weights=False, + dtype=dtype, name="token_embedding", ) self.embeddings_layer_norm = keras.layers.LayerNormalization( epsilon=layer_norm_epsilon, + dtype=dtype, name="token_embedding_layernorm", ) self.transformer_layers = [] @@ -114,11 +121,13 @@ def __init__( intermediate_dim=intermediate_dim, dropout=dropout, layer_norm_epsilon=layer_norm_epsilon, + dtype=dtype, name=f"transformer_layer_{i}", ) self.transformer_layers.append(layer) self.layer_norm = keras.layers.LayerNormalization( epsilon=layer_norm_epsilon, + dtype=dtype, name="final_layernorm", ) diff --git a/keras_nlp/models/deberta_v3/deberta_v3_backbone.py b/keras_nlp/models/deberta_v3/deberta_v3_backbone.py index 87531aefcb..e7bd8ca20a 100644 --- a/keras_nlp/models/deberta_v3/deberta_v3_backbone.py +++ b/keras_nlp/models/deberta_v3/deberta_v3_backbone.py @@ -67,6 +67,10 @@ class DebertaV3Backbone(Backbone): `max_sequence_length`. bucket_size: int. The size of the relative position buckets. Generally equal to `max_sequence_length // 2`. + dtype: string or `keras.mixed_precision.DTypePolicy`. The dtype to use + for model computations and weights. Note that some computations, + such as softmax and layer normalization, will always be done at + float32 precision regardless of dtype. Example: ```python @@ -106,6 +110,7 @@ def __init__( dropout=0.1, max_sequence_length=512, bucket_size=256, + dtype=None, **kwargs, ): # === Layers === @@ -113,15 +118,17 @@ def __init__( input_dim=vocabulary_size, output_dim=hidden_dim, embeddings_initializer=deberta_kernel_initializer(), + dtype=dtype, name="token_embedding", ) self.embeddings_layer_norm = keras.layers.LayerNormalization( epsilon=1e-7, - dtype="float32", + dtype=dtype, name="embeddings_layer_norm", ) self.embeddings_dropout = keras.layers.Dropout( dropout, + dtype=dtype, name="embeddings_dropout", ) self.relative_embeddings = RelativeEmbedding( @@ -129,6 +136,7 @@ def __init__( bucket_size=bucket_size, layer_norm_epsilon=1e-7, kernel_initializer=deberta_kernel_initializer(), + dtype=dtype, name="rel_embedding", ) self.transformer_layers = [] @@ -142,6 +150,7 @@ def __init__( activation=keras.activations.gelu, layer_norm_epsilon=1e-7, kernel_initializer=deberta_kernel_initializer(), + dtype=dtype, name=f"disentangled_attention_encoder_layer_{i}", ) self.transformer_layers.append(layer) diff --git a/keras_nlp/models/deberta_v3/deberta_v3_classifier.py b/keras_nlp/models/deberta_v3/deberta_v3_classifier.py index f5249cb34b..d6eea63601 100644 --- a/keras_nlp/models/deberta_v3/deberta_v3_classifier.py +++ b/keras_nlp/models/deberta_v3/deberta_v3_classifier.py @@ -168,22 +168,26 @@ def __init__( self.preprocessor = preprocessor self.pooled_dropout = keras.layers.Dropout( dropout, + dtype=backbone.dtype_policy, name="pooled_dropout", ) hidden_dim = hidden_dim or backbone.hidden_dim self.pooled_dense = keras.layers.Dense( hidden_dim, activation=keras.activations.gelu, + dtype=backbone.dtype_policy, name="pooled_dense", ) self.output_dropout = keras.layers.Dropout( backbone.dropout, + dtype=backbone.dtype_policy, name="classifier_dropout", ) self.output_dense = keras.layers.Dense( num_classes, kernel_initializer=deberta_kernel_initializer(), activation=activation, + dtype=backbone.dtype_policy, name="logits", ) diff --git a/keras_nlp/models/deberta_v3/deberta_v3_masked_lm.py b/keras_nlp/models/deberta_v3/deberta_v3_masked_lm.py index fadb4c0e24..d050dde6c0 100644 --- a/keras_nlp/models/deberta_v3/deberta_v3_masked_lm.py +++ b/keras_nlp/models/deberta_v3/deberta_v3_masked_lm.py @@ -112,6 +112,7 @@ def __init__( token_embedding=backbone.token_embedding, intermediate_activation=keras.activations.gelu, kernel_initializer=deberta_kernel_initializer(), + dtype=backbone.dtype_policy, name="mlm_head", ) diff --git a/keras_nlp/models/deberta_v3/disentangled_attention_encoder.py b/keras_nlp/models/deberta_v3/disentangled_attention_encoder.py index 081c345c79..be79bc98ff 100644 --- a/keras_nlp/models/deberta_v3/disentangled_attention_encoder.py +++ b/keras_nlp/models/deberta_v3/disentangled_attention_encoder.py @@ -99,22 +99,26 @@ def build(self, inputs_shape): dropout=self.dropout, kernel_initializer=clone_initializer(self.kernel_initializer), bias_initializer=clone_initializer(self.bias_initializer), + dtype=self.dtype_policy, name="self_attention_layer", ) self._self_attention_layer.build(inputs_shape) self._self_attention_layer_norm = keras.layers.LayerNormalization( epsilon=self.layer_norm_epsilon, + dtype=self.dtype_policy, name="self_attention_layer_norm", ) self._self_attention_layer_norm.build(inputs_shape) self._self_attention_dropout = keras.layers.Dropout( rate=self.dropout, + dtype=self.dtype_policy, name="self_attention_dropout", ) # Feedforward layers. self._feedforward_layer_norm = keras.layers.LayerNormalization( epsilon=self.layer_norm_epsilon, + dtype=self.dtype_policy, name="feedforward_layer_norm", ) self._feedforward_layer_norm.build(inputs_shape) @@ -123,6 +127,7 @@ def build(self, inputs_shape): activation=self.activation, kernel_initializer=clone_initializer(self.kernel_initializer), bias_initializer=clone_initializer(self.bias_initializer), + dtype=self.dtype_policy, name="feedforward_intermediate_dense", ) self._feedforward_intermediate_dense.build(inputs_shape) @@ -130,6 +135,7 @@ def build(self, inputs_shape): hidden_dim, kernel_initializer=clone_initializer(self.kernel_initializer), bias_initializer=clone_initializer(self.bias_initializer), + dtype=self.dtype_policy, name="feedforward_output_dense", ) intermediate_shape = list(inputs_shape) @@ -137,6 +143,7 @@ def build(self, inputs_shape): self._feedforward_output_dense.build(tuple(intermediate_shape)) self._feedforward_dropout = keras.layers.Dropout( rate=self.dropout, + dtype=self.dtype_policy, name="feedforward_dropout", ) self.built = True diff --git a/keras_nlp/models/deberta_v3/disentangled_self_attention.py b/keras_nlp/models/deberta_v3/disentangled_self_attention.py index 1c9ae569c7..48730f2ad6 100644 --- a/keras_nlp/models/deberta_v3/disentangled_self_attention.py +++ b/keras_nlp/models/deberta_v3/disentangled_self_attention.py @@ -86,6 +86,7 @@ def build(self, inputs_shape, rel_embeddings_shape=None): output_shape=(None, self.num_heads, self.attn_head_size), bias_axes="de", **self._get_common_kwargs_for_sublayer(use_bias=True), + dtype=self.dtype_policy, name="query", ) self._query_dense.build(inputs_shape) @@ -94,6 +95,7 @@ def build(self, inputs_shape, rel_embeddings_shape=None): output_shape=(None, self.num_heads, self.attn_head_size), bias_axes="de", **self._get_common_kwargs_for_sublayer(use_bias=True), + dtype=self.dtype_policy, name="key", ) self._key_dense.build(inputs_shape) @@ -102,17 +104,27 @@ def build(self, inputs_shape, rel_embeddings_shape=None): output_shape=(None, self.num_heads, self.attn_head_size), bias_axes="de", **self._get_common_kwargs_for_sublayer(use_bias=True), + dtype=self.dtype_policy, name="value", ) self._value_dense.build(inputs_shape) # Relative attention. - self._position_dropout_layer = keras.layers.Dropout(self.dropout) + self._position_dropout_layer = keras.layers.Dropout( + self.dropout, + dtype=self.dtype_policy, + ) self._attn_dropout_layer = keras.layers.Dropout( - self.dropout, name="attention_dropout" + self.dropout, + dtype=self.dtype_policy, + name="attention_dropout", + ) + self._softmax = keras.layers.Softmax( + axis=-1, + dtype="float32", + name="attention_softmax", ) - self._softmax = keras.layers.Softmax(axis=-1, name="attention_softmax") # Output. self._output_dense = keras.layers.EinsumDense( @@ -120,6 +132,7 @@ def build(self, inputs_shape, rel_embeddings_shape=None): output_shape=(None, self.hidden_dim), bias_axes="d", **self._get_common_kwargs_for_sublayer(use_bias=True), + dtype=self.dtype_policy, name="attention_output", ) self._output_dense.build(inputs_shape) diff --git a/keras_nlp/models/deberta_v3/relative_embedding.py b/keras_nlp/models/deberta_v3/relative_embedding.py index 6ae29a5fd7..f727ce0568 100644 --- a/keras_nlp/models/deberta_v3/relative_embedding.py +++ b/keras_nlp/models/deberta_v3/relative_embedding.py @@ -57,7 +57,9 @@ def __init__( name="rel_embedding", ) self.layer_norm = keras.layers.LayerNormalization( - epsilon=layer_norm_epsilon, name="rel_embeddings_layer_norm" + epsilon=layer_norm_epsilon, + dtype=self.dtype_policy, + name="rel_embeddings_layer_norm", ) def call(self, inputs): diff --git a/keras_nlp/models/distil_bert/distil_bert_backbone.py b/keras_nlp/models/distil_bert/distil_bert_backbone.py index f97e24f55d..1ae0840ea8 100644 --- a/keras_nlp/models/distil_bert/distil_bert_backbone.py +++ b/keras_nlp/models/distil_bert/distil_bert_backbone.py @@ -62,6 +62,10 @@ class DistilBertBackbone(Backbone): can consume. If None, `max_sequence_length` uses the value from sequence length. This determines the variable shape for positional embeddings. + dtype: string or `keras.mixed_precision.DTypePolicy`. The dtype to use + for model computations and weights. Note that some computations, + such as softmax and layer normalization, will always be done at + float32 precision regardless of dtype. Examples: ```python @@ -98,6 +102,7 @@ def __init__( intermediate_dim, dropout=0.1, max_sequence_length=512, + dtype=None, **kwargs, ): # === Layers === @@ -106,6 +111,7 @@ def __init__( sequence_length=max_sequence_length, embedding_dim=hidden_dim, embeddings_initializer=distilbert_kernel_initializer(), + dtype=dtype, name="token_and_position_embedding", ) # Keep the token_embedding property for consistency across models. @@ -113,11 +119,12 @@ def __init__( self.embeddings_layer_norm = keras.layers.LayerNormalization( axis=-1, epsilon=1e-12, - dtype="float32", + dtype=dtype, name="embeddings_layer_norm", ) self.embeddings_dropout = keras.layers.Dropout( dropout, + dtype=dtype, name="embeddings_dropout", ) self.transformer_layers = [] @@ -129,6 +136,7 @@ def __init__( dropout=dropout, layer_norm_epsilon=1e-12, kernel_initializer=distilbert_kernel_initializer(), + dtype=dtype, name=f"transformer_layer_{i}", ) self.transformer_layers.append(layer) diff --git a/keras_nlp/models/distil_bert/distil_bert_classifier.py b/keras_nlp/models/distil_bert/distil_bert_classifier.py index cf0db9786a..e82aaf2781 100644 --- a/keras_nlp/models/distil_bert/distil_bert_classifier.py +++ b/keras_nlp/models/distil_bert/distil_bert_classifier.py @@ -158,16 +158,19 @@ def __init__( hidden_dim, activation="relu", kernel_initializer=distilbert_kernel_initializer(), + dtype=backbone.dtype_policy, name="pooled_dense", ) self.output_dropout = keras.layers.Dropout( dropout, + dtype=backbone.dtype_policy, name="output_dropout", ) self.output_dense = keras.layers.Dense( num_classes, kernel_initializer=distilbert_kernel_initializer(), activation=activation, + dtype=backbone.dtype_policy, name="logits", ) diff --git a/keras_nlp/models/distil_bert/distil_bert_masked_lm.py b/keras_nlp/models/distil_bert/distil_bert_masked_lm.py index 9be43f8aa1..fcf54e014d 100644 --- a/keras_nlp/models/distil_bert/distil_bert_masked_lm.py +++ b/keras_nlp/models/distil_bert/distil_bert_masked_lm.py @@ -112,6 +112,7 @@ def __init__( token_embedding=backbone.token_embedding, intermediate_activation="gelu", kernel_initializer=distilbert_kernel_initializer(), + dtype=backbone.dtype_policy, name="mlm_head", ) diff --git a/keras_nlp/models/electra/electra_backbone.py b/keras_nlp/models/electra/electra_backbone.py index 2e2a0197af..13be2d8eb8 100644 --- a/keras_nlp/models/electra/electra_backbone.py +++ b/keras_nlp/models/electra/electra_backbone.py @@ -58,6 +58,10 @@ class ElectraBackbone(Backbone): can consume. If None, `max_sequence_length` uses the value from sequence length. This determines the variable shape for positional embeddings. + dtype: string or `keras.mixed_precision.DTypePolicy`. The dtype to use + for model computations and weights. Note that some computations, + such as softmax and layer normalization, will always be done at + float32 precision regardless of dtype. Examples: ```python @@ -92,6 +96,7 @@ def __init__( dropout=0.1, max_sequence_length=512, num_segments=2, + dtype=None, **kwargs, ): # === Layers === @@ -99,34 +104,42 @@ def __init__( input_dim=vocab_size, output_dim=embedding_dim, embeddings_initializer=electra_kernel_initializer(), + dtype=dtype, name="token_embedding", ) self.position_embedding = PositionEmbedding( initializer=electra_kernel_initializer(), sequence_length=max_sequence_length, + dtype=dtype, name="position_embedding", ) self.segment_embedding = keras.layers.Embedding( input_dim=num_segments, output_dim=embedding_dim, embeddings_initializer=electra_kernel_initializer(), + dtype=dtype, name="segment_embedding", ) - self.embeddings_add = keras.layers.Add() + self.embeddings_add = keras.layers.Add( + dtype=dtype, + name="embeddings_add", + ) self.embeddings_layer_norm = keras.layers.LayerNormalization( - name="embeddings_layer_norm", axis=-1, epsilon=1e-12, - dtype="float32", + dtype=dtype, + name="embeddings_layer_norm", ) self.embeddings_dropout = keras.layers.Dropout( dropout, + dtype=dtype, name="embeddings_dropout", ) if hidden_dim != embedding_dim: self.embeddings_projection = keras.layers.Dense( hidden_dim, kernel_initializer=electra_kernel_initializer(), + dtype=dtype, name="embeddings_projection", ) self.transformer_layers = [] @@ -138,6 +151,7 @@ def __init__( dropout=dropout, layer_norm_epsilon=1e-12, kernel_initializer=electra_kernel_initializer(), + dtype=dtype, name=f"transformer_layer_{i}", ) self.transformer_layers.append(layer) @@ -145,6 +159,7 @@ def __init__( hidden_dim, kernel_initializer=electra_kernel_initializer(), activation="tanh", + dtype=dtype, name="pooled_dense", ) diff --git a/keras_nlp/models/f_net/f_net_backbone.py b/keras_nlp/models/f_net/f_net_backbone.py index 9103a10d48..309f312a17 100644 --- a/keras_nlp/models/f_net/f_net_backbone.py +++ b/keras_nlp/models/f_net/f_net_backbone.py @@ -66,6 +66,10 @@ class FNetBackbone(Backbone): embeddings. num_segments: int. The number of types that the 'segment_ids' input can take. + dtype: string or `keras.mixed_precision.DTypePolicy`. The dtype to use + for model computations and weights. Note that some computations, + such as softmax and layer normalization, will always be done at + float32 precision regardless of dtype. Examples: ```python @@ -99,6 +103,7 @@ def __init__( dropout=0.1, max_sequence_length=512, num_segments=4, + dtype=None, **kwargs, ): # === Layers === @@ -106,34 +111,42 @@ def __init__( input_dim=vocabulary_size, output_dim=hidden_dim, embeddings_initializer=f_net_kernel_initializer(), + dtype=dtype, name="token_embedding", ) self.position_embedding = PositionEmbedding( initializer=f_net_kernel_initializer(), sequence_length=max_sequence_length, + dtype=dtype, name="position_embedding", ) self.segment_embedding = keras.layers.Embedding( input_dim=num_segments, output_dim=hidden_dim, embeddings_initializer=f_net_kernel_initializer(), + dtype=dtype, name="segment_embedding", ) - self.embeddings_add = keras.layers.Add() + self.embeddings_add = keras.layers.Add( + dtype=dtype, + name="embeddings_add", + ) self.embeddings_layer_norm = keras.layers.LayerNormalization( - name="embeddings_layer_norm", axis=-1, epsilon=1e-12, - dtype="float32", + dtype=dtype, + name="embeddings_layer_norm", ) self.embedding_projection = keras.layers.Dense( hidden_dim, kernel_initializer=f_net_kernel_initializer(), bias_initializer=f_net_bias_initializer(), + dtype=dtype, name="embedding_projection", ) self.embeddings_dropout = keras.layers.Dropout( dropout, + dtype=dtype, name="embeddings_dropout", ) self.transformer_layers = [] @@ -145,6 +158,7 @@ def __init__( layer_norm_epsilon=1e-12, kernel_initializer=f_net_kernel_initializer(), bias_initializer=f_net_bias_initializer(), + dtype=dtype, name=f"f_net_layer_{i}", ) self.transformer_layers.append(layer) @@ -153,6 +167,7 @@ def __init__( kernel_initializer=f_net_kernel_initializer(), bias_initializer=f_net_bias_initializer(), activation="tanh", + dtype=dtype, name="pooled_dense", ) diff --git a/keras_nlp/models/f_net/f_net_classifier.py b/keras_nlp/models/f_net/f_net_classifier.py index f4ee31d1e8..512182d2cd 100644 --- a/keras_nlp/models/f_net/f_net_classifier.py +++ b/keras_nlp/models/f_net/f_net_classifier.py @@ -114,12 +114,14 @@ def __init__( self.preprocessor = preprocessor self.output_dropout = keras.layers.Dropout( dropout, + dtype=backbone.dtype_policy, name="output_dropout", ) self.output_dense = keras.layers.Dense( num_classes, kernel_initializer=f_net_kernel_initializer(), activation=activation, + dtype=backbone.dtype_policy, name="logits", ) diff --git a/keras_nlp/models/f_net/f_net_masked_lm.py b/keras_nlp/models/f_net/f_net_masked_lm.py index c0eb231d78..c715a70843 100644 --- a/keras_nlp/models/f_net/f_net_masked_lm.py +++ b/keras_nlp/models/f_net/f_net_masked_lm.py @@ -109,6 +109,7 @@ def __init__( token_embedding=backbone.token_embedding, intermediate_activation="gelu", kernel_initializer=f_net_kernel_initializer(), + dtype=backbone.dtype_policy, name="mlm_head", ) diff --git a/keras_nlp/models/gpt2/gpt2_backbone.py b/keras_nlp/models/gpt2/gpt2_backbone.py index d3f4a41541..d93b2199b0 100644 --- a/keras_nlp/models/gpt2/gpt2_backbone.py +++ b/keras_nlp/models/gpt2/gpt2_backbone.py @@ -61,6 +61,10 @@ class GPT2Backbone(Backbone): can consume. If `None`, `max_sequence_length` uses the value from sequence length. This determines the variable shape for positional embeddings. + dtype: string or `keras.mixed_precision.DTypePolicy`. The dtype to use + for the models computations and weights. Note that some + computations, such as softmax and layer normalization will always + be done a float32 precision regardless of dtype. Example: ```python @@ -95,6 +99,7 @@ def __init__( intermediate_dim, dropout=0.1, max_sequence_length=1024, + dtype=None, **kwargs, ): # === Layers === @@ -102,18 +107,22 @@ def __init__( input_dim=vocabulary_size, output_dim=hidden_dim, embeddings_initializer=_gpt_2_kernel_initializer(stddev=0.01), + dtype=dtype, name="token_embedding", ) self.position_embedding = PositionEmbedding( initializer=_gpt_2_kernel_initializer(stddev=0.02), sequence_length=max_sequence_length, + dtype=dtype, name="position_embedding", ) self.embeddings_add = keras.layers.Add( + dtype=dtype, name="embeddings_add", ) self.embeddings_dropout = keras.layers.Dropout( dropout, + dtype=dtype, name="embeddings_dropout", ) self.transformer_layers = [] @@ -127,14 +136,15 @@ def __init__( activation=gelu_approximate, kernel_initializer=_gpt_2_kernel_initializer(stddev=0.02), normalize_first=True, + dtype=dtype, name=f"transformer_layer_{i}", ) ) self.layer_norm = keras.layers.LayerNormalization( - name="layer_norm", axis=-1, epsilon=1e-05, - dtype="float32", + dtype=dtype, + name="layer_norm", ) # === Functional Model === diff --git a/keras_nlp/models/gpt_neo_x/gpt_neo_x_attention.py b/keras_nlp/models/gpt_neo_x/gpt_neo_x_attention.py index 40cdc0d5a9..dee8addf14 100644 --- a/keras_nlp/models/gpt_neo_x/gpt_neo_x_attention.py +++ b/keras_nlp/models/gpt_neo_x/gpt_neo_x_attention.py @@ -64,7 +64,8 @@ def __init__( self.rotary_max_wavelength = rotary_max_wavelength self.rotary_dim = int(self.attn_head_size * rotary_percentage) self.rotary_embedding_layer = RotaryEmbedding( - max_wavelength=rotary_max_wavelength + max_wavelength=rotary_max_wavelength, + dtype=self.dtype_policy, ) self.kernel_initializer = keras.initializers.get(kernel_initializer) self.bias_initializer = keras.initializers.get(bias_initializer) @@ -76,15 +77,22 @@ def build(self, input_shape): output_shape=(None, self.num_heads, 3 * self.attn_head_size), bias_axes="de", **self._get_common_kwargs_for_sublayer(use_bias=True), + dtype=self.dtype_policy, name="query_key_value", ) self._qkv_dense.build(input_shape) self._attn_dropout_layer = keras.layers.Dropout( - self.dropout, name="attention_dropout" + self.dropout, + dtype=self.dtype_policy, + name="attention_dropout", ) - self._softmax = keras.layers.Softmax(axis=-1, name="attention_softmax") + self._softmax = keras.layers.Softmax( + axis=-1, + dtype="float32", + name="attention_softmax", + ) # Output. self._output_dense = keras.layers.EinsumDense( @@ -92,6 +100,7 @@ def build(self, input_shape): output_shape=(None, self.hidden_dim), bias_axes="d", **self._get_common_kwargs_for_sublayer(use_bias=True), + dtype=self.dtype_policy, name="attention_output", ) diff --git a/keras_nlp/models/gpt_neo_x/gpt_neo_x_backbone.py b/keras_nlp/models/gpt_neo_x/gpt_neo_x_backbone.py index 5bbc11af70..1955ed5801 100644 --- a/keras_nlp/models/gpt_neo_x/gpt_neo_x_backbone.py +++ b/keras_nlp/models/gpt_neo_x/gpt_neo_x_backbone.py @@ -61,6 +61,10 @@ class GPTNeoXBackbone(Backbone): can consume. If `None`, `max_sequence_length` uses the value from sequence length. This determines the variable shape for positional embeddings. + dtype: string or `keras.mixed_precision.DTypePolicy`. The dtype to use + for model computations and weights. Note that some computations, + such as softmax and layer normalization, will always be done at + float32 precision regardless of dtype. """ def __init__( @@ -75,6 +79,7 @@ def __init__( rotary_max_wavelength=10000, layer_norm_epsilon=1e-5, max_sequence_length=512, + dtype=None, **kwargs, ): # === Layers === @@ -82,10 +87,12 @@ def __init__( input_dim=vocabulary_size, output_dim=hidden_dim, embeddings_initializer=_gpt_neo_x_kernel_initializer(stddev=0.01), + dtype=dtype, name="token_embedding", ) self.embeddings_dropout = keras.layers.Dropout( dropout, + dtype=dtype, name="embeddings_dropout", ) self.transformer_layers = [] @@ -100,14 +107,15 @@ def __init__( layer_norm_epsilon=layer_norm_epsilon, activation=gelu_approximate, kernel_initializer=_gpt_neo_x_kernel_initializer(stddev=0.02), + dtype=dtype, name=f"transformer_layer_{i}", ) self.transformer_layers.append(layer) self.layer_norm = keras.layers.LayerNormalization( - name="layer_norm", axis=-1, epsilon=layer_norm_epsilon, - dtype="float32", + dtype=dtype, + name="layer_norm", ) # === Functional Model === diff --git a/keras_nlp/models/gpt_neo_x/gpt_neo_x_decoder.py b/keras_nlp/models/gpt_neo_x/gpt_neo_x_decoder.py index f80cafd528..0a7bad7cd9 100644 --- a/keras_nlp/models/gpt_neo_x/gpt_neo_x_decoder.py +++ b/keras_nlp/models/gpt_neo_x/gpt_neo_x_decoder.py @@ -99,18 +99,21 @@ def build(self, decoder_sequence_shape): max_sequence_length=self.max_sequence_length, kernel_initializer=clone_initializer(self.kernel_initializer), bias_initializer=clone_initializer(self.bias_initializer), + dtype=self.dtype_policy, name="self_attention", ) self._self_attention_layer.build(decoder_sequence_shape) self._self_attention_layer_norm = keras.layers.LayerNormalization( epsilon=self.layer_norm_epsilon, + dtype=self.dtype_policy, name="self_attention_layer_norm", ) self._self_attention_layer_norm.build(decoder_sequence_shape) self._self_attention_dropout = keras.layers.Dropout( rate=self.dropout, + dtype=self.dtype_policy, name="self_attention_dropout", ) @@ -120,6 +123,7 @@ def build(self, decoder_sequence_shape): activation=self.activation, kernel_initializer=clone_initializer(self.kernel_initializer), bias_initializer=clone_initializer(self.bias_initializer), + dtype=self.dtype_policy, name="feedforward_intermediate_dense", ) self._feedforward_intermediate_dense.build(decoder_sequence_shape) @@ -128,6 +132,7 @@ def build(self, decoder_sequence_shape): hidden_dim, kernel_initializer=clone_initializer(self.kernel_initializer), bias_initializer=clone_initializer(self.bias_initializer), + dtype=self.dtype_policy, name="feedforward_output_dense", ) @@ -137,12 +142,14 @@ def build(self, decoder_sequence_shape): self._feedforward_layer_norm = keras.layers.LayerNormalization( epsilon=self.layer_norm_epsilon, + dtype=self.dtype_policy, name="feedforward_layer_norm", ) self._feedforward_layer_norm.build(decoder_sequence_shape) self._feedforward_dropout = keras.layers.Dropout( rate=self.dropout, + dtype=self.dtype_policy, name="feedforward_dropout", ) self.built = True diff --git a/keras_nlp/models/llama/llama_attention.py b/keras_nlp/models/llama/llama_attention.py index a2604e5351..529e73b009 100644 --- a/keras_nlp/models/llama/llama_attention.py +++ b/keras_nlp/models/llama/llama_attention.py @@ -58,6 +58,7 @@ def build(self, inputs_shape): equation="bqm,muh->bquh", output_shape=(None, self.num_query_heads, self.attn_head_size), kernel_initializer=clone_initializer(self.kernel_initializer), + dtype=self.dtype_policy, name="query", ) self._query_dense.build(inputs_shape) @@ -65,6 +66,7 @@ def build(self, inputs_shape): equation="bkm,mvh->bkvh", output_shape=(None, self.num_key_value_heads, self.attn_head_size), kernel_initializer=clone_initializer(self.kernel_initializer), + dtype=self.dtype_policy, name="key", ) self._key_dense.build(inputs_shape) @@ -73,16 +75,22 @@ def build(self, inputs_shape): equation="bkm,mvh->bkvh", output_shape=(None, self.num_key_value_heads, self.attn_head_size), kernel_initializer=clone_initializer(self.kernel_initializer), + dtype=self.dtype_policy, name="value", ) self._value_dense.build(inputs_shape) - self._softmax = keras.layers.Softmax(axis=-1, name="attention_softmax") + self._softmax = keras.layers.Softmax( + axis=-1, + dtype="float32", + name="attention_softmax", + ) self._output_dense = keras.layers.EinsumDense( equation="bqm,mh->bqh", output_shape=(None, self.hidden_dim), kernel_initializer=clone_initializer(self.kernel_initializer), + dtype=self.dtype_policy, name="attention_output", ) self._output_dense.build(inputs_shape) @@ -90,6 +98,7 @@ def build(self, inputs_shape): self._rotary_embedding_layer = RotaryEmbedding( max_wavelength=self.rope_max_wavelength, scaling_factor=self.rope_scaling_factor, + dtype=self.dtype_policy, ) self._rotary_embedding_layer.build(inputs_shape) @@ -173,10 +182,10 @@ def _compute_attention(self, query, key, value, attention_mask=None): ) attention_scores /= norm_factor - attention_scores = self._masked_softmax( attention_scores, attention_mask ) + attention_scores = ops.cast(attention_scores, self.compute_dtype) attention_output = ops.einsum( "acbe,aecd->abcd", attention_scores, value ) diff --git a/keras_nlp/models/llama/llama_backbone.py b/keras_nlp/models/llama/llama_backbone.py index 46cfdc37f2..cc628ad7a5 100644 --- a/keras_nlp/models/llama/llama_backbone.py +++ b/keras_nlp/models/llama/llama_backbone.py @@ -58,7 +58,10 @@ class LlamaBackbone(Backbone): can consume. If `None`, `max_sequence_length` uses the value from sequence length. This determines the variable shape for positional embeddings. - + dtype: string or `keras.mixed_precision.DTypePolicy`. The dtype to use + for model computations and weights. Note that some computations, + such as softmax and layer normalization, will always be done at + float32 precision regardless of dtype. """ def __init__( @@ -73,6 +76,7 @@ def __init__( rope_max_wavelength=10000, layer_norm_epsilon=1e-5, max_sequence_length=4096, + dtype=None, **kwargs, ): # === Layers === @@ -81,6 +85,7 @@ def __init__( output_dim=hidden_dim, embeddings_initializer=_llama_kernel_initializer(stddev=0.01), tie_weights=False, + dtype=dtype, name="token_embedding", ) self.transformer_layers = [] @@ -95,12 +100,14 @@ def __init__( layer_norm_epsilon=layer_norm_epsilon, activation=ops.silu, kernel_initializer=_llama_kernel_initializer(stddev=0.02), + dtype=dtype, name=f"transformer_layer_{i}", ) self.transformer_layers.append(layer) self.layer_norm = LlamaLayerNorm( - name="layer_norm", + dtype=dtype, epsilon=layer_norm_epsilon, + name="layer_norm", ) # === Functional Model === diff --git a/keras_nlp/models/llama/llama_decoder.py b/keras_nlp/models/llama/llama_decoder.py index 2137831be1..3b9d6906b8 100644 --- a/keras_nlp/models/llama/llama_decoder.py +++ b/keras_nlp/models/llama/llama_decoder.py @@ -64,11 +64,13 @@ def build(self, decoder_sequence_shape): max_sequence_length=self.max_sequence_length, rope_scaling_factor=self.rope_scaling_factor, kernel_initializer=clone_initializer(self.kernel_initializer), + dtype=self.dtype_policy, ) self._self_attention_layer.build(decoder_sequence_shape) self._self_attention_layernorm = LlamaLayerNorm( epsilon=self.layer_norm_epsilon, + dtype=self.dtype_policy, ) self._self_attention_layernorm.build(decoder_sequence_shape) @@ -76,6 +78,7 @@ def build(self, decoder_sequence_shape): self._feedforward_intermediate_dense = keras.layers.Dense( self.intermediate_dim, kernel_initializer=clone_initializer(self.kernel_initializer), + dtype=self.dtype_policy, ) self._feedforward_intermediate_dense.build(decoder_sequence_shape) @@ -83,12 +86,14 @@ def build(self, decoder_sequence_shape): self.intermediate_dim, activation=self.activation, kernel_initializer=clone_initializer(self.kernel_initializer), + dtype=self.dtype_policy, ) self._feedforward_gate_dense.build(decoder_sequence_shape) self._feedforward_output_dense = keras.layers.Dense( self.hidden_dim, kernel_initializer=clone_initializer(self.kernel_initializer), + dtype=self.dtype_policy, ) intermediate_shape = list(decoder_sequence_shape) @@ -97,6 +102,7 @@ def build(self, decoder_sequence_shape): self._feedforward_layernorm = LlamaLayerNorm( epsilon=self.layer_norm_epsilon, + dtype=self.dtype_policy, ) self._feedforward_layernorm.build(decoder_sequence_shape) diff --git a/keras_nlp/models/mistral/mistral_attention.py b/keras_nlp/models/mistral/mistral_attention.py index 680f1f6d1b..1c26943d0a 100644 --- a/keras_nlp/models/mistral/mistral_attention.py +++ b/keras_nlp/models/mistral/mistral_attention.py @@ -69,7 +69,7 @@ def build(self, inputs_shape): equation="bqm,muh->bquh", output_shape=(None, self._num_query_heads, self._head_dim), kernel_initializer=self._kernel_initializer, - dtype=self.compute_dtype, + dtype=self.dtype_policy, name="query", ) self._query_dense.build(inputs_shape) @@ -82,7 +82,7 @@ def build(self, inputs_shape): self._head_dim, ), kernel_initializer=self._kernel_initializer, - dtype=self.compute_dtype, + dtype=self.dtype_policy, name="key", ) self._key_dense.build(inputs_shape) @@ -95,22 +95,27 @@ def build(self, inputs_shape): self._head_dim, ), kernel_initializer=self._kernel_initializer, - dtype=self.compute_dtype, + dtype=self.dtype_policy, name="value", ) self._value_dense.build(inputs_shape) - self._softmax = keras.layers.Softmax(axis=-1, name="attention_softmax") + self._softmax = keras.layers.Softmax( + axis=-1, + dtype="float32", + name="attention_softmax", + ) self._dropout_layer = keras.layers.Dropout( - rate=self._dropout, dtype=self.compute_dtype + rate=self._dropout, + dtype=self.dtype_policy, ) self._output_dense = keras.layers.EinsumDense( equation="bquh,uhm->bqm", output_shape=(None, self._hidden_dim), kernel_initializer=self._kernel_initializer, - dtype=self.compute_dtype, + dtype=self.dtype_policy, name="attention_output", ) self._output_dense.build( @@ -120,7 +125,7 @@ def build(self, inputs_shape): self.rotary_embedding_layer = RotaryEmbedding( max_wavelength=self._rope_max_wavelength, scaling_factor=self._rope_scaling_factor, - dtype=self.compute_dtype, + dtype=self.dtype_policy, ) self._dot_product_equation = "bquh,bkuh->buqk" @@ -265,10 +270,10 @@ def _compute_attention(self, query, key, value, attention_mask=None): norm_factor = ops.sqrt(ops.cast(self._head_dim, self.compute_dtype)) attention_scores = attention_scores / norm_factor - attention_scores = self._masked_softmax( attention_scores, attention_mask ) + attention_scores = ops.cast(attention_scores, self.compute_dtype) attention_output = ops.einsum( self._combine_equation, attention_scores, value ) diff --git a/keras_nlp/models/mistral/mistral_backbone.py b/keras_nlp/models/mistral/mistral_backbone.py index 107e5699cb..375d3c54b1 100644 --- a/keras_nlp/models/mistral/mistral_backbone.py +++ b/keras_nlp/models/mistral/mistral_backbone.py @@ -64,7 +64,10 @@ class MistralBackbone(Backbone): layers in each transformer decoder. Only `sliding_window` number of tokens are saved in the cache and used to generate the next token. Defaults to `512`. - dtype (str, optional): The dtype policy for the mistral model. + dtype: string or `keras.mixed_precision.DTypePolicy`. The dtype to use + for model computations and weights. Note that some computations, + such as softmax and layer normalization, will always be done at + float32 precision regardless of dtype. Examples: @@ -107,10 +110,10 @@ def __init__( layer_norm_epsilon=1e-6, sliding_window=512, dropout=0, + dtype=None, **kwargs, ): # === Layers === - dtype = kwargs.pop("dtype", keras.backend.floatx()) self.token_embedding = ReversibleEmbedding( input_dim=vocabulary_size, output_dim=hidden_dim, diff --git a/keras_nlp/models/mistral/mistral_layer_norm.py b/keras_nlp/models/mistral/mistral_layer_norm.py index 9f9ddf26b5..e714a8540d 100644 --- a/keras_nlp/models/mistral/mistral_layer_norm.py +++ b/keras_nlp/models/mistral/mistral_layer_norm.py @@ -32,7 +32,6 @@ def build(self, input_shape): trainable=True, shape=(self._dim,), initializer="ones", - dtype=self.compute_dtype, ) self.built = True diff --git a/keras_nlp/models/mistral/mistral_transformer_decoder.py b/keras_nlp/models/mistral/mistral_transformer_decoder.py index 9b6f7fdbf8..d6505af0d7 100644 --- a/keras_nlp/models/mistral/mistral_transformer_decoder.py +++ b/keras_nlp/models/mistral/mistral_transformer_decoder.py @@ -73,20 +73,20 @@ def build(self, decoder_sequence_shape): sliding_window=self.sliding_window, kernel_initializer=clone_initializer(self.kernel_initializer), dropout=self.dropout, - dtype=self.compute_dtype, + dtype=self.dtype_policy, name="self_attention", ) self._self_attention_layer.build(decoder_sequence_shape) self._self_attention_layernorm = MistralLayerNormalization( epsilon=self.layer_norm_epsilon, + dtype=self.dtype_policy, name="self_attention_layernorm", - dtype=self.compute_dtype, ) self._self_attention_layernorm.build(decoder_sequence_shape) self._self_attention_dropout = keras.layers.Dropout( rate=self.dropout, - dtype=self.compute_dtype, + dtype=self.dtype_policy, name="self_attention_dropout", ) @@ -95,7 +95,7 @@ def build(self, decoder_sequence_shape): self.intermediate_dim, kernel_initializer=clone_initializer(self.kernel_initializer), use_bias=False, - dtype=self.compute_dtype, + dtype=self.dtype_policy, name="feedforward_intermediate_dense", ) self._feedforward_intermediate_dense.build(decoder_sequence_shape) @@ -105,6 +105,7 @@ def build(self, decoder_sequence_shape): activation=self.activation, kernel_initializer=clone_initializer(self.kernel_initializer), use_bias=False, + dtype=self.dtype_policy, name="feedforward_gate_dense", ) self._feedforward_gate_dense.build(decoder_sequence_shape) @@ -113,7 +114,7 @@ def build(self, decoder_sequence_shape): self.hidden_dim, kernel_initializer=clone_initializer(self.kernel_initializer), use_bias=False, - dtype=self.compute_dtype, + dtype=self.dtype_policy, name="feedforward_output_dense", ) @@ -125,8 +126,8 @@ def build(self, decoder_sequence_shape): self._feedforward_layernorm = MistralLayerNormalization( epsilon=self.layer_norm_epsilon, + dtype=self.dtype_policy, name="feedforward_layernorm", - dtype=self.compute_dtype, ) self._feedforward_layernorm.build(decoder_sequence_shape) diff --git a/keras_nlp/models/opt/opt_backbone.py b/keras_nlp/models/opt/opt_backbone.py index d04f4b571d..0b98a6c64e 100644 --- a/keras_nlp/models/opt/opt_backbone.py +++ b/keras_nlp/models/opt/opt_backbone.py @@ -57,6 +57,10 @@ class OPTBackbone(Backbone): can consume. If `None`, `max_sequence_length` uses the value from sequence length. This determines the variable shape for positional embeddings. + dtype: string or `keras.mixed_precision.DTypePolicy`. The dtype to use + for model computations and weights. Note that some computations, + such as softmax and layer normalization, will always be done at + float32 precision regardless of dtype. Examples: ```python @@ -91,6 +95,7 @@ def __init__( intermediate_dim, dropout=0.1, max_sequence_length=2048, + dtype=None, **kwargs, ): # === Layers === @@ -99,6 +104,7 @@ def __init__( sequence_length=max_sequence_length, embedding_dim=hidden_dim, embeddings_initializer=opt_kernel_initializer(), + dtype=dtype, name="embeddings", ) self.token_embedding = self.embeddings.token_embedding @@ -112,13 +118,14 @@ def __init__( layer_norm_epsilon=1e-5, normalize_first=True, kernel_initializer=opt_kernel_initializer(), + dtype=dtype, name=f"transformer_layer_{i}", ) self.transformer_layers.append(layer) self.layer_norm = keras.layers.LayerNormalization( axis=-1, epsilon=1e-5, - dtype="float32", + dtype=dtype, name="layer_norm", ) diff --git a/keras_nlp/models/roberta/roberta_backbone.py b/keras_nlp/models/roberta/roberta_backbone.py index 614104d8d7..1ab61eeeb7 100644 --- a/keras_nlp/models/roberta/roberta_backbone.py +++ b/keras_nlp/models/roberta/roberta_backbone.py @@ -61,6 +61,10 @@ class RobertaBackbone(Backbone): consume. The sequence length of the input must be less than `max_sequence_length` default value. This determines the variable shape for positional embeddings. + dtype: string or `keras.mixed_precision.DTypePolicy`. The dtype to use + for model computations and weights. Note that some computations, + such as softmax and layer normalization, will always be done at + float32 precision regardless of dtype. Examples: ```python @@ -96,6 +100,7 @@ def __init__( intermediate_dim, dropout=0.1, max_sequence_length=512, + dtype=None, **kwargs, ): # === Layers === @@ -104,17 +109,19 @@ def __init__( sequence_length=max_sequence_length, embedding_dim=hidden_dim, embeddings_initializer=roberta_kernel_initializer(), + dtype=dtype, name="embeddings", ) self.token_embedding = self.embeddings.token_embedding self.embeddings_layer_norm = keras.layers.LayerNormalization( axis=-1, epsilon=1e-5, # Original paper uses this epsilon value - dtype="float32", + dtype=dtype, name="embeddings_layer_norm", ) self.embeddings_dropout = keras.layers.Dropout( dropout, + dtype=dtype, name="embeddings_dropout", ) self.transformer_layers = [] @@ -126,6 +133,7 @@ def __init__( dropout=dropout, layer_norm_epsilon=1e-5, kernel_initializer=roberta_kernel_initializer(), + dtype=dtype, name=f"transformer_layer_{i}", ) self.transformer_layers.append(layer) diff --git a/keras_nlp/models/roberta/roberta_classifier.py b/keras_nlp/models/roberta/roberta_classifier.py index e3d7666f5b..887bc657d4 100644 --- a/keras_nlp/models/roberta/roberta_classifier.py +++ b/keras_nlp/models/roberta/roberta_classifier.py @@ -149,22 +149,26 @@ def __init__( self.preprocessor = preprocessor self.pooled_dropout = keras.layers.Dropout( dropout, + dtype=backbone.dtype_policy, name="pooled_dropout", ) hidden_dim = hidden_dim or backbone.hidden_dim self.pooled_dense = keras.layers.Dense( hidden_dim, activation="tanh", + dtype=backbone.dtype_policy, name="pooled_dense", ) self.output_dropout = keras.layers.Dropout( dropout, + dtype=backbone.dtype_policy, name="output_dropout", ) self.output_dense = keras.layers.Dense( num_classes, kernel_initializer=roberta_kernel_initializer(), activation=activation, + dtype=backbone.dtype_policy, name="logits", ) diff --git a/keras_nlp/models/roberta/roberta_masked_lm.py b/keras_nlp/models/roberta/roberta_masked_lm.py index 0e62f4cff6..bf96189860 100644 --- a/keras_nlp/models/roberta/roberta_masked_lm.py +++ b/keras_nlp/models/roberta/roberta_masked_lm.py @@ -111,6 +111,7 @@ def __init__( token_embedding=backbone.token_embedding, intermediate_activation="gelu", kernel_initializer=roberta_kernel_initializer(), + dtype=backbone.dtype_policy, name="mlm_head", ) diff --git a/keras_nlp/models/t5/t5_backbone.py b/keras_nlp/models/t5/t5_backbone.py index 3fdf69ff30..cf747c503c 100644 --- a/keras_nlp/models/t5/t5_backbone.py +++ b/keras_nlp/models/t5/t5_backbone.py @@ -67,7 +67,11 @@ class T5Backbone(Backbone): layer normalization layers in the Transformer layers. tie_embedding_weights: boolean. If `True`, the weights of the token embedding and the weights projecting language model outputs from - `hidden_dim` + `hidden_dim`. + dtype: string or `keras.mixed_precision.DTypePolicy`. The dtype to use + for model computations and weights. Note that some computations, + such as softmax and layer normalization, will always be done at + float32 precision regardless of dtype. """ def __init__( @@ -83,6 +87,7 @@ def __init__( use_gated_activation=True, layer_norm_epsilon=1e-06, tie_embedding_weights=True, + dtype=None, **kwargs, ): # Token embedding layer. This layer is shared by encoder and decoder. @@ -91,10 +96,12 @@ def __init__( output_dim=hidden_dim, tie_weights=tie_embedding_weights, embeddings_initializer=keras.initializers.TruncatedNormal(1.0), + dtype=dtype, name="token_embedding", ) self.encoder_embedding_dropout = keras.layers.Dropout( dropout, + dtype=dtype, name="encoder_embedding_dropout", ) self.encoder_transformer_layers = [] @@ -110,19 +117,23 @@ def __init__( num_heads=num_heads, use_gated_activation=use_gated_activation, use_relative_attention_bias=bool(i == 0), + dtype=dtype, name=f"transformer_encoder_layer_{i}", ) self.encoder_transformer_layers.append(layer) self.encoder_layer_norm = T5LayerNorm( epsilon=layer_norm_epsilon, + dtype=dtype, name="encoder_output_layer_norm", ) self.encoder_dropout = keras.layers.Dropout( dropout, + dtype=dtype, name="encoder_output_dropout", ) self.decoder_embedding_dropout = keras.layers.Dropout( dropout, + dtype=dtype, name="decoder_embedding_dropout", ) self.decoder_transformer_layers = [] @@ -138,15 +149,18 @@ def __init__( num_heads=num_heads, use_gated_activation=use_gated_activation, use_relative_attention_bias=bool(i == 0), + dtype=dtype, name=f"transformer_decoder_layer_{i}", ) self.decoder_transformer_layers.append(layer) self.decoder_layer_norm = T5LayerNorm( epsilon=layer_norm_epsilon, + dtype=dtype, name="decoder_output_layer_norm", ) self.decoder_dropout = keras.layers.Dropout( dropout, + dtype=dtype, name="decoder_output_dropout", ) diff --git a/keras_nlp/models/t5/t5_multi_head_attention.py b/keras_nlp/models/t5/t5_multi_head_attention.py index 77e7109efe..2e3647b1d5 100644 --- a/keras_nlp/models/t5/t5_multi_head_attention.py +++ b/keras_nlp/models/t5/t5_multi_head_attention.py @@ -45,36 +45,43 @@ def __init__( self.query_projector = keras.layers.Dense( self.inner_dim, use_bias=False, - name="query_projector", kernel_initializer=keras.initializers.RandomNormal( mean=0, stddev=(self.inner_dim * self.key_value_dim) ** -0.5 ), + dtype=self.dtype_policy, + name="query_projector", ) self.key_projector = keras.layers.Dense( self.inner_dim, use_bias=False, - name="key_projector", kernel_initializer=keras.initializers.RandomNormal( mean=0, stddev=self.inner_dim**-0.5 ), + dtype=self.dtype_policy, + name="key_projector", ) self.value_projector = keras.layers.Dense( self.inner_dim, use_bias=False, - name="value_projector", kernel_initializer=keras.initializers.RandomNormal( mean=0, stddev=self.inner_dim**-0.5 ), + dtype=self.dtype_policy, + name="value_projector", ) self.output_projector = keras.layers.Dense( self.hidden_dim, use_bias=False, - name="output_projector", kernel_initializer=keras.initializers.RandomNormal( mean=0, stddev=self.inner_dim**-0.5 ), + dtype=self.dtype_policy, + name="output_projector", + ) + self.dropout_layer = keras.layers.Dropout( + dropout, + dtype=self.dtype_policy, ) - self.dropout_layer = keras.layers.Dropout(dropout) if self.use_relative_attention_bias: self.relative_attention_bias = self.add_weight( @@ -298,7 +305,7 @@ def project( mask = (1.0 - ops.cast(mask, position_bias.dtype)) * -1e9 position_bias = position_bias + mask - scores += position_bias + scores += ops.cast(position_bias, scores.dtype) weights = ops.nn.softmax( scores, axis=-1 ) # (batch_size, num_heads, query_length, key_length) diff --git a/keras_nlp/models/t5/t5_transformer_layer.py b/keras_nlp/models/t5/t5_transformer_layer.py index 27b4c9892c..697af20899 100644 --- a/keras_nlp/models/t5/t5_transformer_layer.py +++ b/keras_nlp/models/t5/t5_transformer_layer.py @@ -47,10 +47,17 @@ def __init__( num_heads=num_heads, dropout=dropout, use_relative_attention_bias=use_relative_attention_bias, + dtype=self.dtype_policy, name="self_attention", ) - self.self_attention_layer_norm = T5LayerNorm(layer_norm_epsilon) - self.self_attention_dropout = keras.layers.Dropout(dropout) + self.self_attention_layer_norm = T5LayerNorm( + layer_norm_epsilon, + dtype=self.dtype_policy, + ) + self.self_attention_dropout = keras.layers.Dropout( + dropout, + dtype=self.dtype_policy, + ) if self.is_decoder: self.cross_attention = T5MultiHeadAttention( @@ -60,39 +67,55 @@ def __init__( num_heads=num_heads, dropout=dropout, use_relative_attention_bias=False, + dtype=self.dtype_policy, name="cross_attention", ) - self.cross_attention_layer_norm = T5LayerNorm(layer_norm_epsilon) - self.cross_attention_dropout = keras.layers.Dropout(dropout) + self.cross_attention_layer_norm = T5LayerNorm( + layer_norm_epsilon, + dtype=self.dtype_policy, + ) + self.cross_attention_dropout = keras.layers.Dropout( + dropout, + dtype=self.dtype_policy, + ) self.input_projector = keras.layers.Dense( intermediate_dim, use_bias=False, - name="input_projector", activation=keras.activations.get(activation), kernel_initializer=keras.initializers.RandomNormal( mean=0, stddev=hidden_dim**-0.5 ), + dtype=self.dtype_policy, + name="input_projector", ) if self.use_gated_activation: self.gate_projector = keras.layers.Dense( intermediate_dim, use_bias=False, - name="gate_projector", kernel_initializer=keras.initializers.RandomNormal( mean=0, stddev=hidden_dim**-0.5 ), + dtype=self.dtype_policy, + name="gate_projector", ) self.output_projector = keras.layers.Dense( hidden_dim, use_bias=False, - name="output_projector", kernel_initializer=keras.initializers.RandomNormal( mean=0, stddev=intermediate_dim**-0.5 ), + dtype=self.dtype_policy, + name="output_projector", + ) + self.layer_norm = T5LayerNorm( + epsilon=layer_norm_epsilon, + dtype=self.dtype_policy, + ) + self.dropout_layer = keras.layers.Dropout( + dropout, + dtype=self.dtype_policy, ) - self.layer_norm = T5LayerNorm(epsilon=layer_norm_epsilon) - self.dropout_layer = keras.layers.Dropout(dropout) def call( self, diff --git a/keras_nlp/models/task.py b/keras_nlp/models/task.py index 1fe8d0b789..783cc0b41b 100644 --- a/keras_nlp/models/task.py +++ b/keras_nlp/models/task.py @@ -220,9 +220,14 @@ def from_preset( # Backbone case. if preset_cls == cls.backbone_cls: + # Forward dtype to the backbone. + config_overrides = {} + if "dtype" in kwargs: + config_overrides["dtype"] = kwargs.pop("dtype") backbone = load_from_preset( preset, load_weights=load_weights, + config_overrides=config_overrides, ) if "preprocessor" in kwargs: preprocessor = kwargs.pop("preprocessor") diff --git a/keras_nlp/models/whisper/whisper_backbone.py b/keras_nlp/models/whisper/whisper_backbone.py index 2e84219091..c66a61d4e5 100644 --- a/keras_nlp/models/whisper/whisper_backbone.py +++ b/keras_nlp/models/whisper/whisper_backbone.py @@ -75,6 +75,10 @@ class WhisperBackbone(Backbone): positional embedding layer. max_decoder_sequence_length: int. The maximum sequence length that the text decoder can consume. + dtype: string or `keras.mixed_precision.DTypePolicy`. The dtype to use + for model computations and weights. Note that some computations, + such as softmax and layer normalization, will always be done at + float32 precision regardless of dtype. Examples: @@ -112,6 +116,7 @@ def __init__( dropout=0.0, max_encoder_sequence_length=3000, max_decoder_sequence_length=448, + dtype=None, **kwargs, ): assert_tf_backend(self.__class__.__name__) @@ -122,6 +127,7 @@ def __init__( kernel_size=3, strides=1, padding="same", + dtype=dtype, name="encoder_token_embedding_conv_layer_1", ) self.encoder_conv_layer_2 = keras.layers.Conv1D( @@ -129,22 +135,27 @@ def __init__( kernel_size=3, strides=2, padding="valid", + dtype=dtype, name="encoder_token_embedding_conv_layer_2", ) self.encoder_padder = Padder( + dtype=dtype, name="encoder_padder", ) self.encoder_position_embedding = PositionEmbedding( initializer=whisper_kernel_initializer(), sequence_length=max_encoder_sequence_length // 2, + dtype=dtype, name="encoder_position_embedding", trainable=False, ) self.encoder_embeddings_add = keras.layers.Add( + dtype=dtype, name="encoder_embeddings_add", ) self.encoder_embeddings_dropout = keras.layers.Dropout( dropout, + dtype=dtype, name="encoder_embeddings_dropout", ) self.encoder_transformer_layers = [] @@ -157,25 +168,28 @@ def __init__( dropout=dropout, kernel_initializer=whisper_kernel_initializer(), normalize_first=True, + dtype=dtype, name=f"transformer_encoder_layer_{i}", ) self.encoder_transformer_layers.append(layer) self.encoder_layer_norm = keras.layers.LayerNormalization( - name="encoder_layer_norm", axis=-1, epsilon=1e-5, - dtype="float32", + dtype=dtype, + name="encoder_layer_norm", ) self.decoder_embeddings = TokenAndPositionEmbedding( vocabulary_size=vocabulary_size, sequence_length=max_decoder_sequence_length, embedding_dim=hidden_dim, embeddings_initializer=whisper_kernel_initializer(), + dtype=dtype, name="decoder_token_and_position_embedding", ) self.token_embedding = self.decoder_embeddings.token_embedding self.decoder_embeddings_dropout = keras.layers.Dropout( dropout, + dtype=dtype, name="decoder_embeddings_dropout", ) self.decoder_transformer_layers = [] @@ -188,14 +202,15 @@ def __init__( layer_norm_epsilon=1e-5, kernel_initializer=whisper_kernel_initializer(), normalize_first=True, + dtype=dtype, name=f"transformer_decoder_layer_{i}", ) self.decoder_transformer_layers.append(layer) self.decoder_layer_norm = keras.layers.LayerNormalization( - name="decoder_layer_norm", axis=-1, epsilon=1e-5, - dtype="float32", + dtype=dtype, + name="decoder_layer_norm", ) # === Functional Model === @@ -222,7 +237,7 @@ def __init__( # For the second conv. layer, we cannot use `padding="same"` since # that corresponds to a padding size of 1.5 (since stride is 2). Hence, # we will manually pad the input. - embedded_features = Padder()(embedded_features) + embedded_features = self.encoder_padder(embedded_features) embedded_features = keras.activations.gelu( self.encoder_conv_layer_2(embedded_features), approximate=False, diff --git a/keras_nlp/models/xlm_roberta/xlm_roberta_backbone.py b/keras_nlp/models/xlm_roberta/xlm_roberta_backbone.py index b83b4596b2..c74a0fd6fc 100644 --- a/keras_nlp/models/xlm_roberta/xlm_roberta_backbone.py +++ b/keras_nlp/models/xlm_roberta/xlm_roberta_backbone.py @@ -52,6 +52,10 @@ class XLMRobertaBackbone(roberta_backbone.RobertaBackbone): consume. The sequence length of the input must be less than `max_sequence_length` default value. This determines the variable shape for positional embeddings. + dtype: string or `keras.mixed_precision.DTypePolicy`. The dtype to use + for model computations and weights. Note that some computations, + such as softmax and layer normalization, will always be done at + float32 precision regardless of dtype. Examples: ```python diff --git a/keras_nlp/models/xlm_roberta/xlm_roberta_classifier.py b/keras_nlp/models/xlm_roberta/xlm_roberta_classifier.py index 45d79eb304..fcd8bfe9b8 100644 --- a/keras_nlp/models/xlm_roberta/xlm_roberta_classifier.py +++ b/keras_nlp/models/xlm_roberta/xlm_roberta_classifier.py @@ -162,22 +162,26 @@ def __init__( self.preprocessor = preprocessor self.pooled_dropout = keras.layers.Dropout( dropout, + dtype=backbone.dtype_policy, name="pooled_dropout", ) hidden_dim = hidden_dim or backbone.hidden_dim self.pooled_dense = keras.layers.Dense( hidden_dim, activation="tanh", + dtype=backbone.dtype_policy, name="pooled_dense", ) self.output_dropout = keras.layers.Dropout( dropout, + dtype=backbone.dtype_policy, name="output_dropout", ) self.output_dense = keras.layers.Dense( num_classes, kernel_initializer=roberta_kernel_initializer(), activation=activation, + dtype=backbone.dtype_policy, name="logits", ) diff --git a/keras_nlp/models/xlm_roberta/xlm_roberta_masked_lm.py b/keras_nlp/models/xlm_roberta/xlm_roberta_masked_lm.py index b29aa30dd9..e231f3dc7a 100644 --- a/keras_nlp/models/xlm_roberta/xlm_roberta_masked_lm.py +++ b/keras_nlp/models/xlm_roberta/xlm_roberta_masked_lm.py @@ -114,6 +114,7 @@ def __init__( token_embedding=backbone.token_embedding, intermediate_activation="gelu", kernel_initializer=roberta_kernel_initializer(), + dtype=backbone.dtype_policy, name="mlm_head", ) diff --git a/keras_nlp/models/xlnet/relative_attention.py b/keras_nlp/models/xlnet/relative_attention.py index a11ae3fd9d..6ae56d1450 100644 --- a/keras_nlp/models/xlnet/relative_attention.py +++ b/keras_nlp/models/xlnet/relative_attention.py @@ -154,6 +154,7 @@ def build(self, content_stream_shape): output_rank - 1, [self._num_heads, self._key_dim] ), bias_axes=bias_axes if self._use_bias else None, + dtype=self.dtype_policy, name="query", **self._get_common_kwargs_for_sublayer(), ) @@ -168,6 +169,7 @@ def build(self, content_stream_shape): output_rank - 1, [self._num_heads, self._key_dim] ), bias_axes=bias_axes if self._use_bias else None, + dtype=self.dtype_policy, name="key", **self._get_common_kwargs_for_sublayer(), ) @@ -182,6 +184,7 @@ def build(self, content_stream_shape): output_rank - 1, [self._num_heads, self._value_dim] ), bias_axes=bias_axes if self._use_bias else None, + dtype=self.dtype_policy, name="value", **self._get_common_kwargs_for_sublayer(), ) @@ -197,6 +200,7 @@ def build(self, content_stream_shape): output_rank - 1, [self._query_shape[-1]] ), bias_axes=None, + dtype=self.dtype_policy, name="attention_output", **self._get_common_kwargs_for_sublayer(), ) @@ -213,6 +217,7 @@ def build(self, content_stream_shape): output_rank - 1, [self._num_heads, self._key_dim] ), bias_axes=None, + dtype=self.dtype_policy, name="encoding", **self._get_common_kwargs_for_sublayer(), ) diff --git a/keras_nlp/models/xlnet/xlnet_backbone.py b/keras_nlp/models/xlnet/xlnet_backbone.py index 45a4b8f407..0d660bead9 100644 --- a/keras_nlp/models/xlnet/xlnet_backbone.py +++ b/keras_nlp/models/xlnet/xlnet_backbone.py @@ -52,6 +52,10 @@ class XLNetBackbone(Backbone): bias_initializer: string or `keras.initializers` initializer, defaults to "zeros". The bias initializer for the dense and multiheaded relative attention layers. + dtype: string or `keras.mixed_precision.DTypePolicy`. The dtype to use + for model computations and weights. Note that some computations, + such as softmax and layer normalization, will always be done at + float32 precision regardless of dtype. Call arguments: token_ids: Indices of input sequence tokens in the vocabulary of shape @@ -101,6 +105,7 @@ def __init__( activation="gelu", kernel_initializer_range=0.02, bias_initializer="zeros", + dtype=None, **kwargs, ): # === Layers === @@ -108,14 +113,17 @@ def __init__( vocabulary_size=vocabulary_size, hidden_dim=hidden_dim, dropout=dropout, + dtype=dtype, name="content_query_embedding", ) self.attn_mask_layer = XLNetAttentionMaskLayer( hidden_dim=hidden_dim, kernel_initializer_range=kernel_initializer_range, + dtype=dtype, name="encoder_block_attn_mask_layer", ) self.seg_mat_layer = XLNetSegmentMatrixLayer( + dtype=dtype, name="encoder_block_seg_mat_layer", ) head_dim = hidden_dim // num_heads @@ -131,11 +139,13 @@ def __init__( layer_norm_epsilon=1e-12, kernel_initializer_range=kernel_initializer_range, bias_initializer=bias_initializer, + dtype=dtype, name=f"xlnet_encoder_{i}", ) self.transformer_layers.append(layer) self.dropout = keras.layers.Dropout( dropout, + dtype=dtype, name="dropout", ) diff --git a/keras_nlp/models/xlnet/xlnet_content_and_query_embedding.py b/keras_nlp/models/xlnet/xlnet_content_and_query_embedding.py index a2bc2d0d07..2de2c31780 100644 --- a/keras_nlp/models/xlnet/xlnet_content_and_query_embedding.py +++ b/keras_nlp/models/xlnet/xlnet_content_and_query_embedding.py @@ -58,7 +58,8 @@ def positional_embedding(self, pos_seq, inv_freq, bsz=None): ops.shape(pos_emb)[0], ops.shape(pos_emb)[1] * bsz, ops.shape(pos_emb)[2], - ] + ], + dtype=self.compute_dtype, ) * pos_emb ) @@ -67,12 +68,14 @@ def positional_embedding(self, pos_seq, inv_freq, bsz=None): def relative_positional_encoding(self, qlen, klen, bsz=None, clamp_len=-1): """create relative positional encoding.""" - freq_seq = ops.arange(0, self.hidden_dim, 2.0, dtype=self.compute_dtype) + freq_seq = ops.arange(0, self.hidden_dim, 2.0, dtype="float32") + freq_seq = ops.cast(freq_seq, self.compute_dtype) inv_freq = 1 / (10000 ** (freq_seq / self.hidden_dim)) beg, end = klen, -qlen - fwd_pos_seq = ops.arange(beg, end, -1.0, dtype=self.compute_dtype) + fwd_pos_seq = ops.arange(beg, end, -1.0, dtype="float32") + fwd_pos_seq = ops.cast(fwd_pos_seq, self.compute_dtype) if clamp_len > 0: fwd_pos_seq = ops.clip( fwd_pos_seq, x_min=-clamp_len, x_max=clamp_len @@ -85,11 +88,14 @@ def build(self, input_shape): self.word_embed = keras.layers.Embedding( input_dim=self.vocabulary_size, output_dim=self.hidden_dim, + dtype=self.dtype_policy, name="word_embedding", ) self.word_embed.build(input_shape) - self.dropout_layer = keras.layers.Dropout(self.dropout) - + self.dropout_layer = keras.layers.Dropout( + self.dropout, + dtype=self.dtype_policy, + ) super().build(input_shape) def call( diff --git a/keras_nlp/models/xlnet/xlnet_encoder.py b/keras_nlp/models/xlnet/xlnet_encoder.py index bb8e56e4cc..6aa44b150e 100644 --- a/keras_nlp/models/xlnet/xlnet_encoder.py +++ b/keras_nlp/models/xlnet/xlnet_encoder.py @@ -94,25 +94,34 @@ def build(self, input_shape): key_dim=self.head_dim, kernel_initializer=self.kernel_initializer, bias_initializer=self.bias_initializer, + dtype=self.dtype_policy, name="rel_attn", ) self.layer_norm = keras.layers.LayerNormalization( - epsilon=self.layer_norm_epsilon, name="layer_norm_rel_attn" + epsilon=self.layer_norm_epsilon, + dtype=self.dtype_policy, + name="layer_norm_rel_attn", ) self.layer_norm.build(input_shape) - self.dropout_attn = keras.layers.Dropout(self.dropout) + self.dropout_attn = keras.layers.Dropout( + self.dropout, + dtype=self.dtype_policy, + ) # Feed-Forward Part self.layer_norm_ff = keras.layers.LayerNormalization( - epsilon=self.layer_norm_epsilon, name="layer_norm_ff" + epsilon=self.layer_norm_epsilon, + dtype=self.dtype_policy, + name="layer_norm_ff", ) self.layer_norm_ff.build(input_shape) self.feedforward_intermediate_dense = keras.layers.Dense( self.intermediate_dim, kernel_initializer=self.kernel_initializer, + dtype=self.dtype_policy, name="feedforward_intermediate_dense", ) self.feedforward_intermediate_dense.build(input_shape) @@ -120,6 +129,7 @@ def build(self, input_shape): self.feedforward_output_dense = keras.layers.Dense( self.hidden_dim, kernel_initializer=self.kernel_initializer, + dtype=self.dtype_policy, name="feedforward_output_dense", ) self.feedforward_output_dense.build( @@ -128,7 +138,10 @@ def build(self, input_shape): ) ) - self.dropout_ff = keras.layers.Dropout(self.dropout) + self.dropout_ff = keras.layers.Dropout( + self.dropout, + dtype=self.dtype_policy, + ) self.activation_function_ff = keras.activations.get(self.activation) diff --git a/keras_nlp/tests/test_case.py b/keras_nlp/tests/test_case.py index 1010262853..50e7733513 100644 --- a/keras_nlp/tests/test_case.py +++ b/keras_nlp/tests/test_case.py @@ -87,7 +87,7 @@ def run_layer_test( expected_num_non_trainable_weights=0, expected_num_non_trainable_variables=0, run_training_check=True, - run_mixed_precision_check=True, + run_precision_checks=True, ): """Run basic tests for a modeling layer.""" # Serialization test. @@ -187,24 +187,8 @@ def call(self, x): if run_training_check: run_training_step(layer, input_data, output_data) - # Never test mixed precision on torch CPU. Torch lacks support. - if run_mixed_precision_check and config.backend() == "torch": - import torch - - run_mixed_precision_check = torch.cuda.is_available() - - if run_mixed_precision_check: - layer = cls(**{**init_kwargs, "dtype": "mixed_float16"}) - if isinstance(input_data, dict): - output_data = layer(**input_data) - else: - output_data = layer(input_data) - for tensor in tree.flatten(output_data): - if is_float_dtype(tensor.dtype): - self.assertDTypeEqual(tensor, "float16") - for weight in layer.weights: - if is_float_dtype(weight.dtype): - self.assertDTypeEqual(weight, "float32") + if run_precision_checks: + self.run_precision_test(cls, init_kwargs, input_data) def run_preprocessing_layer_test( self, @@ -283,6 +267,40 @@ def run_serialization_test(self, instance): lst.remove("__annotations__") self.assertEqual(set(ref_dir), set(new_dir)) + def run_precision_test(self, cls, init_kwargs, input_data): + # Keras 2 has some errors as non-float32 precision. + if not config.keras_3(): + return + # Never test mixed precision on torch CPU. Torch lacks support. + if config.backend() == "torch": + import torch + + if not torch.cuda.is_available(): + return + + for policy in ["mixed_float16", "mixed_bfloat16", "bfloat16"]: + policy = keras.mixed_precision.Policy(policy) + layer = cls(**{**init_kwargs, "dtype": policy}) + if isinstance(layer, keras.Model): + output_data = layer(input_data) + elif isinstance(input_data, dict): + output_data = layer(**input_data) + else: + output_data = layer(input_data) + for tensor in tree.flatten(output_data): + if is_float_dtype(tensor.dtype): + self.assertDTypeEqual(tensor, policy.compute_dtype) + for weight in layer.weights: + if is_float_dtype(weight.dtype): + self.assertDTypeEqual(weight, policy.variable_dtype) + for sublayer in layer._flatten_layers(include_self=False): + if isinstance( + sublayer, (keras.layers.Softmax, keras.layers.InputLayer) + ): + continue + self.assertEqual(policy.compute_dtype, sublayer.compute_dtype) + self.assertEqual(policy.variable_dtype, sublayer.variable_dtype) + def run_model_saving_test( self, cls, @@ -310,6 +328,7 @@ def run_backbone_test( input_data, expected_output_shape, variable_length_data=None, + run_mixed_precision_check=True, ): """Run basic tests for a backbone, including compilation.""" backbone = cls(**init_kwargs) @@ -351,6 +370,8 @@ def run_backbone_test( name = re.sub("([a-z])([A-Z])", r"\1_\2", name).lower() self.assertRegexpMatches(backbone.name, name) + self.run_precision_test(cls, init_kwargs, input_data) + def run_task_test( self, cls, From 1951b5c9789f4b8fb446b82c261ca8b958957348 Mon Sep 17 00:00:00 2001 From: Tirth Patel Date: Tue, 13 Feb 2024 13:18:04 -0700 Subject: [PATCH 16/29] Add a Causal LM model for Mistral (#1429) * Add Mistral Causal LM Preprocessor * Add the Causal LM for Mistral * Remove sliding window attention from Mistral's attention layer JAX complains about dynamic slicing when compiled with XLA. This is unavoidable since, at runtime, the slice of the current key/value array to use for that iteration is determined by `cache_update_index` which is itself a JAX `TracedArray`. Any workaround would lead to using dynamic shapes at some point. Hence, I had to remove this and instead use vanilla caching for now. For some reason, TensorFlow doesn't complain with XLA. I think this might be because TensorFlow is as stringent about statis shapes as JAX. In any case, adding sliding window attention that is XLA compatible is a story for the future. * Enable JIT compile in the Mistral LM model * Fix Mistral transformer decoder * Port the causal LM to the new infra * Fix a minor bug in sliding window attention caching * Fix a small bug in mistral transformer decoder * Remove the RoPE shenanigan in mistral attention layer * Address review comments and add mistral to the public API --- keras_nlp/models/__init__.py | 6 + keras_nlp/models/mistral/mistral_attention.py | 96 ++------ keras_nlp/models/mistral/mistral_causal_lm.py | 213 ++++++++++++++++++ .../mistral/mistral_causal_lm_preprocessor.py | 171 ++++++++++++++ .../mistral_causal_lm_preprocessor_test.py | 81 +++++++ .../models/mistral/mistral_causal_lm_test.py | 130 +++++++++++ .../mistral/mistral_transformer_decoder.py | 23 +- 7 files changed, 640 insertions(+), 80 deletions(-) create mode 100644 keras_nlp/models/mistral/mistral_causal_lm.py create mode 100644 keras_nlp/models/mistral/mistral_causal_lm_preprocessor.py create mode 100644 keras_nlp/models/mistral/mistral_causal_lm_preprocessor_test.py create mode 100644 keras_nlp/models/mistral/mistral_causal_lm_test.py diff --git a/keras_nlp/models/__init__.py b/keras_nlp/models/__init__.py index 7b898138fd..8fd6a70ac0 100644 --- a/keras_nlp/models/__init__.py +++ b/keras_nlp/models/__init__.py @@ -93,6 +93,12 @@ from keras_nlp.models.gpt_neo_x.gpt_neo_x_tokenizer import GPTNeoXTokenizer from keras_nlp.models.llama.llama_backbone import LlamaBackbone from keras_nlp.models.mistral.mistral_backbone import MistralBackbone +from keras_nlp.models.mistral.mistral_causal_lm import MistralCausalLM +from keras_nlp.models.mistral.mistral_causal_lm_preprocessor import ( + MistralCausalLMPreprocessor, +) +from keras_nlp.models.mistral.mistral_preprocessor import MistralPreprocessor +from keras_nlp.models.mistral.mistral_tokenizer import MistralTokenizer from keras_nlp.models.opt.opt_backbone import OPTBackbone from keras_nlp.models.opt.opt_causal_lm import OPTCausalLM from keras_nlp.models.opt.opt_causal_lm_preprocessor import ( diff --git a/keras_nlp/models/mistral/mistral_attention.py b/keras_nlp/models/mistral/mistral_attention.py index 1c26943d0a..6511a22446 100644 --- a/keras_nlp/models/mistral/mistral_attention.py +++ b/keras_nlp/models/mistral/mistral_attention.py @@ -141,7 +141,6 @@ def call( cache_update_index=None, training=None, ): - seq_len = ops.shape(hidden_states)[1] start_index = ( cache_update_index if cache_update_index is not None else 0 ) @@ -153,89 +152,34 @@ def call( query = self._query_dense(hidden_states) - # Note that the original PyTorch implementation uses - # view_as_complex/view_as_real while we use split/concatenate to - # convert to/from complex numbers. The transformations below make - # the rope computation numerically equivalent to the original - # implementation. - def _mistral_rope(x): - x = ops.concatenate([x[..., ::2], x[..., 1::2]], axis=-1) - x = self.rotary_embedding_layer(x, start_index=start_index) - x = ops.reshape( - ops.stack(ops.split(x, 2, axis=-1), axis=-1), ops.shape(x) - ) - return x - # Compute RoPE for queries - query = _mistral_rope(query) + query = self.rotary_embedding_layer(query, start_index=start_index) def _compute_key_value(x): key, value = self._key_dense(x), self._value_dense(x) - key = _mistral_rope(key) + # Compute RoPE for keys + key = self.rotary_embedding_layer(key, start_index=start_index) return key, value if cache is not None: - cache_k = cache[:, 0, ...] - cache_v = cache[:, 1, ...] - + key_cache = cache[:, 0, ...] + value_cache = cache[:, 1, ...] + if cache_update_index is None: + key = key_cache + value = value_cache + else: + key_update, value_update = _compute_key_value(hidden_states) + start = [0, cache_update_index, 0, 0] + key = ops.slice_update(key_cache, start, key_update) + value = ops.slice_update(value_cache, start, value_update) + cache = ops.stack((key, value), axis=1) + else: if cache_update_index is not None: - # Compute the new keys and values - key, value = _compute_key_value(hidden_states) - - # Cache is a rotating buffer, we want to warp around if - # the sequence length exceeds the sliding window. - update_end_index = ( - cache_update_index + seq_len - 1 - ) % self._sliding_window + 1 - update_end_index = ops.cast(update_end_index, "int32") - cache_update_index = cache_update_index % self._sliding_window - update_start_index = ops.cond( - update_end_index > cache_update_index, - lambda: ops.cast(cache_update_index, "int32"), - lambda: ops.cast(0, "int32"), - ) - # Also note that the update step below assumes that the - # sequence length is always one when `cache_update_index != 0`. - # This is necessary to support XLA compilation. Ideally, we - # would want to use - # `key[:, -(update_end_index - update_start_index):, ...]` - # as the update but updating using a dynamic slice gives an - # XLA compilation error in TensorFlow. - # Passing a sequence of length > 1 with cache update might give - # incorrect results (since there is no way to determine how - # many most recent tokens are to be saved if the tokens exceed - # the sliding window length). - cache_k = ops.slice_update( - cache_k, - [0, update_start_index, 0, 0], - # We slice the keys and values since if the user has passed - # a sequence of length > `self._sliding_window`. We want to - # prefill the cache using just the most recent values in the - # sliding window. - ops.cast( - key[:, -self._sliding_window :, ...], cache_k.dtype - ), + raise ValueError( + "`cache_update_index` should not be set if `cache` is " + f"`None`. Received: cache={cache}, " + f"cache_update_index={cache_update_index}" ) - cache_v = ops.slice_update( - cache_v, - [0, update_start_index, 0, 0], - ops.cast( - value[:, -self._sliding_window :, ...], cache_v.dtype - ), - ) - cache = ops.stack([cache_k, cache_v], axis=1) - - # Get the required keys and values from the cache. - # Since we expect the user to pass a fixed-size cache, we just - # pick the first few slices up-to and including the newly computed - # keys and values. - cache_k = cache_k[:, :update_end_index, ...] - cache_v = cache_v[:, :update_end_index, ...] - - key = ops.cast(cache_k, dtype=self.compute_dtype) - value = ops.cast(cache_v, dtype=self.compute_dtype) - else: - # Compute keys and values key, value = _compute_key_value(hidden_states) # [batch_shape, seq_len, num_key_value_heads, head_dim] @@ -265,7 +209,7 @@ def _masked_softmax(self, attention_scores, attention_mask=None): return self._softmax(attention_scores) def _compute_attention(self, query, key, value, attention_mask=None): - attention_scores = ops.einsum(self._dot_product_equation, key, query) + attention_scores = ops.einsum(self._dot_product_equation, query, key) norm_factor = ops.sqrt(ops.cast(self._head_dim, self.compute_dtype)) diff --git a/keras_nlp/models/mistral/mistral_causal_lm.py b/keras_nlp/models/mistral/mistral_causal_lm.py new file mode 100644 index 0000000000..22defbc456 --- /dev/null +++ b/keras_nlp/models/mistral/mistral_causal_lm.py @@ -0,0 +1,213 @@ +# Copyright 2023 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from keras_nlp.api_export import keras_nlp_export +from keras_nlp.backend import keras +from keras_nlp.backend import ops +from keras_nlp.models.generative_task import GenerativeTask +from keras_nlp.models.mistral.mistral_backbone import MistralBackbone +from keras_nlp.models.mistral.mistral_causal_lm_preprocessor import ( + MistralCausalLMPreprocessor, +) +from keras_nlp.utils.python_utils import classproperty + + +@keras_nlp_export("keras_nlp.models.MistralCausalLM") +class MistralCausalLM(GenerativeTask): + """An end-to-end Mistral model for causal language modeling. + + A causal language model (LM) predicts the next token based on previous + tokens. This task setup can be used to train the model unsupervised on + plain text input, or to autoregressively generate plain text similar to + the data used for training. This task can be used for pre-training or + fine-tuning a GPT-NeoX model, simply by calling `fit()`. + + This model has a `generate()` method, which generates text based on a + prompt. The generation strategy used is controlled by an additional + `sampler` argument on `compile()`. You can recompile the model with + different `keras_nlp.samplers` objects to control the generation. By + default, `"top_k"` sampling will be used. + + Args: + backbone: A `keras_nlp.models.MistralBackbone` instance. + preprocessor: A `keras_nlp.models.MistralCausalLMPreprocessor` or `None`. + If `None`, this model will not apply preprocessing, and inputs + should be preprocessed before calling the model. + """ + + def __init__(self, backbone, preprocessor=None, **kwargs): + # === Layers === + self.backbone = backbone + self.preprocessor = preprocessor + + # === Functional Model === + inputs = backbone.inputs + hidden_states = backbone(inputs) + outputs = backbone.token_embedding(hidden_states, reverse=True) + super().__init__( + inputs=inputs, + outputs=outputs, + **kwargs, + ) + + # === Default compilation === + self.compile( + loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True), + optimizer=keras.optimizers.Adam(2e-5), + metrics=[keras.metrics.SparseCategoricalAccuracy()], + jit_compile=True, + ) + + @classproperty + def backbone_cls(cls): + return MistralBackbone + + @classproperty + def preprocessor_cls(cls): + return MistralCausalLMPreprocessor + + def call_with_cache( + self, + token_ids, + cache, + cache_update_index, + ): + """Forward pass of `MistralCausalLM` with cache. + + `call_with_cache` adds an additional forward pass for the model for + autoregressive inference. Unlike calling the model directly, this method + allows caching previous key/value Tensors in multi-head attention layer, + and avoids recomputing the outputs of seen tokens. + + Args: + token_ids: a dense int Tensor with shape `(batch_size, max_length)`. + cache: a dense float Tensor, the cache of key and value. + cache_update_index: int, or int Tensor. The index of current inputs + in the whole sequence. + + Returns: + A (logits, hidden_states, cache) tuple. Where `logits` is the + language model logits for the input token_ids, `hidden_states` is + the final hidden representation of the input tokens, and `cache` is + the decoding cache. + """ + x = self.backbone.token_embedding(token_ids) + # Each decoder layer has a cache; we update them separately. + updated_cache = [] + for i in range(self.backbone.num_layers): + current_cache = cache[:, i, ...] + x, next_cache = self.backbone.transformer_layers[i]( + x, + self_attention_cache=current_cache, + self_attention_cache_update_index=cache_update_index, + ) + updated_cache.append(next_cache) + cache = ops.stack(updated_cache, axis=1) + hidden_states = x = self.backbone.layer_norm(x) + logits = self.backbone.token_embedding(x, reverse=True) + return logits, hidden_states, cache + + def _build_cache(self, token_ids): + """Build an empty cache for use with `call_with_cache()`.""" + batch_size = ops.shape(token_ids)[0] + max_length = ops.shape(token_ids)[1] + num_layers = self.backbone.num_layers + num_key_value_heads = self.backbone.num_key_value_heads + head_dim = self.backbone.hidden_dim // self.backbone.num_query_heads + shape = [ + batch_size, + num_layers, + 2, + max_length, + num_key_value_heads, + head_dim, + ] + cache = ops.zeros(shape, dtype=self.compute_dtype) + # Seed the cache. + _, hidden_states, cache = self.call_with_cache(token_ids, cache, 0) + return hidden_states, cache + + def generate_step( + self, + inputs, + end_token_id=None, + ): + """A compilable generation function for a single batch of inputs. + + This function represents the inner, XLA-compilable, generation function + for a single batch of inputs. Inputs should have the same structure as + model inputs, a dictionary with keys `"token_ids"` and `"padding_mask"`. + + Args: + inputs: A dictionary with two keys `"token_ids"` and + `"padding_mask"` and batched tensor values. + end_token_id: The id of the end token to stop on. If all + sequences have produced a new `end_token_id`, generation + will stop. + """ + token_ids, padding_mask = inputs["token_ids"], inputs["padding_mask"] + # Create and seed cache with a single forward pass. + hidden_states, cache = self._build_cache(token_ids) + # Compute the lengths of all user inputted tokens ids. + row_lengths = ops.sum(ops.cast(padding_mask, "int32"), axis=-1) + # Start at the first index that has no user inputted id. + index = ops.min(row_lengths) + + def next(prompt, cache, index): + # The cache index is the index of our previous token. + cache_update_index = index - 1 + batch_size = ops.shape(prompt)[0] + prompt = ops.slice(prompt, [0, cache_update_index], [batch_size, 1]) + logits, hidden_states, cache = self.call_with_cache( + prompt, + cache, + cache_update_index, + ) + return ( + ops.squeeze(logits, axis=1), + ops.squeeze(hidden_states, axis=1), + cache, + ) + + token_ids = self._sampler( + next=next, + prompt=token_ids, + cache=cache, + index=index, + mask=padding_mask, + end_token_id=end_token_id, + hidden_states=hidden_states, + ) + + # Compute an output padding mask with the token ids we updated. + if end_token_id is not None: + # Build a mask of `end_token_id` locations not in the original + # prompt (not in locations where `padding_mask` is True). + end_locations = ops.logical_and( + ops.equal(token_ids, end_token_id), + ops.logical_not(padding_mask), + ) + end_locations = ops.cast(end_locations, "int32") + # Use cumsum to get ones in all locations after end_locations. + cumsum = ops.cast(ops.cumsum(end_locations, axis=-1), "int32") + overflow = cumsum - end_locations + # Our padding mask is the inverse of these overflow locations. + padding_mask = ops.logical_not(ops.cast(overflow, "bool")) + else: + # Without early stopping, all locations will have been updated. + padding_mask = ops.ones_like(token_ids, dtype="bool") + return { + "token_ids": token_ids, + "padding_mask": padding_mask, + } diff --git a/keras_nlp/models/mistral/mistral_causal_lm_preprocessor.py b/keras_nlp/models/mistral/mistral_causal_lm_preprocessor.py new file mode 100644 index 0000000000..c8a0821733 --- /dev/null +++ b/keras_nlp/models/mistral/mistral_causal_lm_preprocessor.py @@ -0,0 +1,171 @@ +# Copyright 2023 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import tensorflow as tf +from absl import logging + +from keras_nlp.api_export import keras_nlp_export +from keras_nlp.backend import ops +from keras_nlp.models.mistral.mistral_preprocessor import MistralPreprocessor +from keras_nlp.utils.keras_utils import ( + convert_inputs_to_list_of_tensor_segments, +) +from keras_nlp.utils.keras_utils import pack_x_y_sample_weight + + +@keras_nlp_export("keras_nlp.models.MistralCausalLMPreprocessor") +class MistralCausalLMPreprocessor(MistralPreprocessor): + """Mistral Causal LM preprocessor. + + This preprocessing layer is meant for use with + `keras_nlp.models.MistralCausalLM`. By default, it will take in batches of + strings, and return outputs in a `(x, y, sample_weight)` format, where the + `y` label is the next token id in the `x` sequence. + + For use with generation, the layer also exposes two methods + `generate_preprocess()` and `generate_postprocess()`. When this preprocessor + is attached to a `keras_nlp.models.MistralCausalLM` instance, these methods + will be called implicitly in `generate()`. They can also be called + standalone (e.g. to precompute preprocessing inputs for generation in a + separate process). + + Args: + tokenizer: A `keras_nlp.models.MistralTokenizer` instance. + sequence_length: The length of the packed inputs. + add_start_token: If `True`, the preprocessor will prepend the tokenizer + start token to each input sequence. Default is `True`. + add_end_token: If `True`, the preprocessor will append the tokenizer + end token to each input sequence. Default is `False`. + + Call arguments: + x: A string, `tf.Tensor` or list of python strings. + y: Label data. Should always be `None` as the layer generates labels. + sample_weight: Label weights. Should always be `None` as the layer + generates label weights. + sequence_length: Pass to override the configured `sequence_length` of + the layer. + + Examples: + ```python + # Load the preprocessor from a preset. + preprocessor = keras_nlp.models.MistralCausalLMPreprocessor.from_preset( + "mistral_base_en" + ) + + # Tokenize and pack a single sentence. + sentence = tf.constant("League of legends") + preprocessor(sentence) + # Same output. + preprocessor("League of legends") + + # Tokenize a batch of sentences. + sentences = tf.constant(["Taco tuesday", "Fish taco please!"]) + preprocessor(sentences) + # Same output. + preprocessor(["Taco tuesday", "Fish taco please!"]) + + # Map a dataset to preprocess a single sentence. + features = tf.constant( + [ + "Avatar 2 is amazing!", + "Well, I am not sure.", + ] + ) + labels = tf.constant([1, 0]) + ds = tf.data.Dataset.from_tensor_slices((features, labels)) + ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE) + + # Map a dataset to preprocess unlabled sentences. + ds = tf.data.Dataset.from_tensor_slices(features) + ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE) + ``` + """ + + def call( + self, + x, + y=None, + sample_weight=None, + sequence_length=None, + ): + if y is not None or sample_weight is not None: + logging.warning( + "`MistralCausalLMPreprocessor` generates `y` and " + "`sample_weight` based on your input data, but your data " + "already contains `y` or `sample_weight`. Your `y` and " + "`sample_weight` will be ignored." + ) + sequence_length = sequence_length or self.sequence_length + + x = convert_inputs_to_list_of_tensor_segments(x)[0] + x = self.tokenizer(x) + # Pad with one extra token to account for the truncation below. + token_ids, padding_mask = self.packer( + x, + sequence_length=sequence_length + 1, + add_start_value=self.add_start_token, + add_end_value=self.add_end_token, + ) + # The last token does not have a next token, so we truncate it out. + x = { + "token_ids": token_ids[..., :-1], + "padding_mask": padding_mask[..., :-1], + } + # Target `y` will be the next token. + y, sample_weight = token_ids[..., 1:], padding_mask[..., 1:] + return pack_x_y_sample_weight(x, y, sample_weight) + + def generate_preprocess( + self, + x, + sequence_length=None, + ): + x = convert_inputs_to_list_of_tensor_segments(x)[0] + x = self.tokenizer(x) + token_ids, padding_mask = self.packer( + x, sequence_length=sequence_length, add_end_value=False + ) + return { + "token_ids": token_ids, + "padding_mask": padding_mask, + } + + def generate_postprocess( + self, + x, + ): + """Covert integer token output to strings for generation. + + This method reverses `generate_preprocess()`, by first removing all + padding and start/end tokens, and then converting the integer sequence + back to a string. + """ + token_ids, padding_mask = x["token_ids"], x["padding_mask"] + # Convert the inputs to numpy arrays if they aren't a tensor already. + if not isinstance(token_ids, tf.Tensor): + token_ids = ops.convert_to_numpy(token_ids) + # Make sure the numpy array has type `int32` since + # `SentencePieceProcessor.detokenize` only accepts `int32` arrays. + token_ids = token_ids.astype("int32") + if not isinstance(padding_mask, tf.Tensor): + padding_mask = ops.convert_to_numpy(padding_mask) + padding_mask = padding_mask.astype("bool") + # Strip any special tokens during detokenization (e.g. the start and + # end markers). In the future we could make this configurable. + padding_mask = padding_mask & (token_ids != self.tokenizer.end_token_id) + padding_mask = padding_mask & ( + token_ids != self.tokenizer.start_token_id + ) + token_ids = tf.ragged.boolean_mask(token_ids, padding_mask) + return self.tokenizer.detokenize(token_ids) diff --git a/keras_nlp/models/mistral/mistral_causal_lm_preprocessor_test.py b/keras_nlp/models/mistral/mistral_causal_lm_preprocessor_test.py new file mode 100644 index 0000000000..420995016b --- /dev/null +++ b/keras_nlp/models/mistral/mistral_causal_lm_preprocessor_test.py @@ -0,0 +1,81 @@ +# Copyright 2024 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from keras_nlp.models.mistral.mistral_causal_lm_preprocessor import ( + MistralCausalLMPreprocessor, +) +from keras_nlp.models.mistral.mistral_tokenizer import MistralTokenizer +from keras_nlp.tests.test_case import TestCase + + +class MistralCausalLMPreprocessorTest(TestCase): + def setUp(self): + self.tokenizer = MistralTokenizer( + # Generated using create_mistral_test_proto.py + proto=os.path.join( + self.get_test_data_dir(), "mistral_test_vocab.spm" + ) + ) + self.init_kwargs = { + "tokenizer": self.tokenizer, + "sequence_length": 8, + } + self.input_data = (["the quick brown fox"],) + + def test_preprocessor_basics(self): + self.run_preprocessing_layer_test( + cls=MistralCausalLMPreprocessor, + init_kwargs=self.init_kwargs, + input_data=self.input_data, + expected_output=( + { + "token_ids": [[1, 3, 8, 4, 6, 0, 0, 0]], + "padding_mask": [[1, 1, 1, 1, 1, 0, 0, 0]], + }, + [[3, 8, 4, 6, 0, 0, 0, 0]], # Pass through labels. + [[1, 1, 1, 1, 0, 0, 0, 0]], # Pass through sample_weights. + ), + ) + + def test_no_start_end_token(self): + input_data = ["the quick brown fox"] * 4 + + preprocessor = MistralCausalLMPreprocessor( + **self.init_kwargs, + add_start_token=False, + add_end_token=False, + ) + x, y, sw = preprocessor(input_data) + self.assertAllEqual(x["token_ids"], [[3, 8, 4, 6, 0, 0, 0, 0]] * 4) + self.assertAllEqual(x["padding_mask"], [[1, 1, 1, 1, 0, 0, 0, 0]] * 4) + self.assertAllEqual(y, [[8, 4, 6, 0, 0, 0, 0, 0]] * 4) + self.assertAllEqual(sw, [[1, 1, 1, 0, 0, 0, 0, 0]] * 4) + + def test_generate_preprocess(self): + input_data = "the quick brown fox" + preprocessor = MistralCausalLMPreprocessor(**self.init_kwargs) + x = preprocessor.generate_preprocess(input_data) + self.assertAllEqual(x["token_ids"], [1, 3, 8, 4, 6, 0, 0, 0]) + self.assertAllEqual(x["padding_mask"], [1, 1, 1, 1, 1, 0, 0, 0]) + + def test_generate_postprocess(self): + input_data = { + "token_ids": [1, 3, 8, 4, 6, 0, 0, 0], + "padding_mask": [1, 1, 1, 1, 1, 0, 0, 0], + } + preprocessor = MistralCausalLMPreprocessor(**self.init_kwargs) + x = preprocessor.generate_postprocess(input_data) + self.assertAllEqual(x, "the quick brown fox") diff --git a/keras_nlp/models/mistral/mistral_causal_lm_test.py b/keras_nlp/models/mistral/mistral_causal_lm_test.py new file mode 100644 index 0000000000..3f9d7fab36 --- /dev/null +++ b/keras_nlp/models/mistral/mistral_causal_lm_test.py @@ -0,0 +1,130 @@ +# Copyright 2023 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from unittest.mock import patch + +import pytest + +from keras_nlp.backend import ops +from keras_nlp.models.mistral.mistral_backbone import MistralBackbone +from keras_nlp.models.mistral.mistral_causal_lm import MistralCausalLM +from keras_nlp.models.mistral.mistral_causal_lm_preprocessor import ( + MistralCausalLMPreprocessor, +) +from keras_nlp.models.mistral.mistral_tokenizer import MistralTokenizer +from keras_nlp.tests.test_case import TestCase + + +class MistralCausalLMTest(TestCase): + def setUp(self): + self.preprocessor = MistralCausalLMPreprocessor( + MistralTokenizer( + # Generated using create_mistral_test_proto.py + proto=os.path.join( + self.get_test_data_dir(), "mistral_test_vocab.spm" + ) + ), + sequence_length=8, + ) + self.backbone = MistralBackbone( + vocabulary_size=self.preprocessor.tokenizer.vocabulary_size(), + num_layers=2, + num_query_heads=4, + num_key_value_heads=2, + hidden_dim=8, + intermediate_dim=16, + ) + self.init_kwargs = { + "preprocessor": self.preprocessor, + "backbone": self.backbone, + } + self.train_data = (["the quick brown fox", "the earth is round"],) + self.input_data = self.preprocessor(*self.train_data)[0] + + def test_causal_lm_basics(self): + self.run_task_test( + cls=MistralCausalLM, + init_kwargs=self.init_kwargs, + train_data=self.train_data, + expected_output_shape=(2, 8, 10), + ) + + def test_generate(self): + causal_lm = MistralCausalLM(**self.init_kwargs) + # String input. + prompt = "the quick brown fox" + output = causal_lm.generate(prompt) + self.assertTrue(prompt in output) + # Int tensor input. + prompt_ids = self.preprocessor.generate_preprocess([prompt]) + causal_lm.preprocessor = None + outputs = causal_lm.generate(prompt_ids) + # Assert prompt is in output in token id space. + self.assertAllEqual( + outputs["token_ids"][:, :5], + prompt_ids["token_ids"][:, :5], + ) + self.assertAllEqual( + outputs["padding_mask"][:, :5], + prompt_ids["padding_mask"][:, :5], + ) + + def test_early_stopping(self): + causal_lm = MistralCausalLM(**self.init_kwargs) + call_with_cache = causal_lm.call_with_cache + + def wrapper(*args, **kwargs): + """Modify output logits to always favor end_token_id""" + logits, hidden_states, cache = call_with_cache(*args, **kwargs) + index = self.preprocessor.tokenizer.end_token_id + update = ops.ones_like(logits)[:, :, index] * 1.0e9 + update = ops.expand_dims(update, axis=-1) + logits = ops.slice_update(logits, (0, 0, index), update) + return logits, hidden_states, cache + + with patch.object(causal_lm, "call_with_cache", wraps=wrapper): + prompt = ["the quick brown fox", "the earth"] + output = causal_lm.generate(prompt) + # We should immediately abort and output the prompt. + self.assertEqual(prompt, output) + + def test_generate_compilation(self): + causal_lm = MistralCausalLM(**self.init_kwargs) + # Assert we do not recompile with successive calls. + causal_lm.generate("the quick brown fox") + first_fn = causal_lm.generate_function + causal_lm.generate("the quick brown fox") + second_fn = causal_lm.generate_function + self.assertEqual(first_fn, second_fn) + # Assert we do recompile after compile is called. + causal_lm.compile(sampler="greedy") + self.assertIsNone(causal_lm.generate_function) + + @pytest.mark.large + def test_saved_model(self): + self.run_model_saving_test( + cls=MistralCausalLM, + init_kwargs=self.init_kwargs, + input_data=self.input_data, + ) + + @pytest.mark.extra_large + def test_all_presets(self): + for preset in MistralCausalLM.presets: + self.run_preset_test( + cls=MistralCausalLM, + preset=preset, + input_data=self.input_data, + ) diff --git a/keras_nlp/models/mistral/mistral_transformer_decoder.py b/keras_nlp/models/mistral/mistral_transformer_decoder.py index d6505af0d7..7c90ab91b9 100644 --- a/keras_nlp/models/mistral/mistral_transformer_decoder.py +++ b/keras_nlp/models/mistral/mistral_transformer_decoder.py @@ -36,7 +36,7 @@ def __init__( num_key_value_heads, rope_max_wavelength=10000, rope_scaling_factor=1.0, - activation="relu", + activation="silu", layer_norm_epsilon=1e-5, kernel_initializer="glorot_uniform", sliding_window=512, @@ -146,6 +146,8 @@ def call( decoder_sequence=decoder_sequence, decoder_padding_mask=decoder_padding_mask, decoder_attention_mask=decoder_attention_mask, + self_attention_cache=self_attention_cache, + self_attention_cache_update_index=self_attention_cache_update_index, ) residual = decoder_sequence @@ -185,23 +187,36 @@ def _compute_self_attention_mask( decoder_sequence, decoder_padding_mask, decoder_attention_mask, + self_attention_cache, + self_attention_cache_update_index, ): decoder_mask = merge_padding_and_attention_mask( decoder_sequence, decoder_padding_mask, decoder_attention_mask ) batch_size = ops.shape(decoder_sequence)[0] input_length = output_length = ops.shape(decoder_sequence)[1] + # We need to handle a rectangular causal mask when doing cached + # decoding. For generative inference, `decoder_sequence` will + # generally be length 1, and `cache` will be the full generation length. + if self_attention_cache is not None: + input_length = ops.shape(self_attention_cache)[2] + + cache_update_index = ( + 0 + if self_attention_cache_update_index is None + else self_attention_cache_update_index + ) # Mistral uses a banded attention mask causal_mask_lower = compute_causal_mask( - batch_size, input_length, output_length, 0 + batch_size, input_length, output_length, cache_update_index ) # Below is a workaround for `ops.triu` for Keras 2. # TODO(tirthasheshpatel): Use `ops.triu` once Keras 2 support is removed. # causal_mask = ops.triu(causal_mask_lower, k=-self.sliding_window) - i = ops.arange(output_length)[:, None] + i = ops.arange(output_length)[:, None] + cache_update_index j = ops.arange(input_length)[None, :] - causal_mask_upper = ops.cast(i <= j + self.sliding_window, "int32") + causal_mask_upper = ops.cast(i < j + self.sliding_window, "int32") causal_mask = ops.minimum(causal_mask_lower, causal_mask_upper) return ( From 0de100b7abafbe2c625f36a33e38fd413f25156a Mon Sep 17 00:00:00 2001 From: Matt Watson <1389937+mattdangerw@users.noreply.github.com> Date: Tue, 13 Feb 2024 14:44:22 -0800 Subject: [PATCH 17/29] Fix bart (#1434) * Fix transformer decoder bug * Fix bart and error with TransformerDecoder with cached cross attention --- .../layers/modeling/transformer_decoder.py | 4 +- .../modeling/transformer_decoder_test.py | 58 +++++++++++++++++-- 2 files changed, 56 insertions(+), 6 deletions(-) diff --git a/keras_nlp/layers/modeling/transformer_decoder.py b/keras_nlp/layers/modeling/transformer_decoder.py index cb7c779332..15c245768c 100644 --- a/keras_nlp/layers/modeling/transformer_decoder.py +++ b/keras_nlp/layers/modeling/transformer_decoder.py @@ -416,10 +416,10 @@ def call( cache=cross_attention_cache, cache_update_index=cross_attention_cache_update_index, ) - if self_attention_cache is None: + if cross_attention_cache is None: x = attention_output else: - x, self_attention_cache = attention_output + x, cross_attention_cache = attention_output x = self._cross_attention_dropout(x) x = x + residual if not self.normalize_first: diff --git a/keras_nlp/layers/modeling/transformer_decoder_test.py b/keras_nlp/layers/modeling/transformer_decoder_test.py index 4757a0a45c..aa16d9ae5a 100644 --- a/keras_nlp/layers/modeling/transformer_decoder_test.py +++ b/keras_nlp/layers/modeling/transformer_decoder_test.py @@ -127,10 +127,7 @@ def test_mask_propagation_without_cross_attention(self): self.assertAllEqual(outputs._keras_mask, mask) def test_cache_call_is_correct(self): - batch_size = 2 - seq_len = 5 - num_heads = 2 - key_dim = 4 + batch_size, seq_len, num_heads, key_dim = 2, 5, 2, 4 hidden_dim = num_heads * key_dim input_shape = (batch_size, seq_len, hidden_dim) @@ -171,6 +168,59 @@ def call(outputs, cache): self.assertAllClose(output, no_loop_outputs) self.assertAllClose(output_cache, no_loop_cache) + def test_cache_call_is_correct_with_cross_attention(self): + batch_size, seq_len, num_heads, key_dim = 2, 5, 2, 4 + hidden_dim = num_heads * key_dim + + input_shape = (batch_size, seq_len, hidden_dim) + cache_shape = (batch_size, 2, seq_len, num_heads, key_dim) + decoder_sequence = random.uniform(shape=input_shape) + encoder_sequence = random.uniform(shape=input_shape) + empty_cache = ops.zeros(cache_shape) + outputs = ops.zeros_like(decoder_sequence) + + layer = TransformerDecoder( + intermediate_dim=4, + num_heads=num_heads, + ) + no_loop_outputs, no_loop_self_cache, no_loop_cross_cache = layer( + decoder_sequence, + encoder_sequence, + self_attention_cache=empty_cache, + self_attention_cache_update_index=0, + cross_attention_cache=empty_cache, + cross_attention_cache_update_index=0, + ) + + def loop_body(i, outputs, self_cache, cross_cache): + # Compute the rest tokens. + start, size = (0, i, 0), (batch_size, 1, hidden_dim) + next_input = ops.slice(decoder_sequence, start, size) + next_output, self_cache, cross_cache = layer( + decoder_sequence=next_input, + encoder_sequence=encoder_sequence, + self_attention_cache=self_cache, + self_attention_cache_update_index=i, + cross_attention_cache=cross_cache, + ) + outputs = ops.slice_update(outputs, start, next_output) + return i + 1, outputs, self_cache, cross_cache + + def call(outputs, self_cache, cross_cache): + _, outputs, self_cache, cross_cache = ops.while_loop( + cond=lambda i, outputs, self_cache, cross_cache: i < seq_len, + body=loop_body, + loop_vars=[0, outputs, self_cache, cross_cache], + ) + return outputs, self_cache, cross_cache + + output, self_cache, cross_cache = call( + outputs, empty_cache, no_loop_cross_cache + ) + self.assertAllClose(output, no_loop_outputs) + self.assertAllClose(self_cache, no_loop_self_cache) + self.assertAllClose(cross_cache, no_loop_cross_cache) + def test_different_feature_dimension_for_encoder_and_decoder_sequence(self): decoder = TransformerDecoder( intermediate_dim=4, From bc3852f9fc3e9edaab7bc6c5d7ee40e6d3618ac2 Mon Sep 17 00:00:00 2001 From: Matt Watson <1389937+mattdangerw@users.noreply.github.com> Date: Wed, 14 Feb 2024 13:08:41 -0800 Subject: [PATCH 18/29] Add a settable property for sequence_length (#1437) For all preprocessors, we add a settable sequence length property. This removes a very annoying gotcha with preprocessing, where if you set the preprocessing length after build, it would not be reflected in the actual output length. --- .../albert_masked_lm_preprocessor_test.py | 2 +- .../models/albert/albert_preprocessor.py | 13 +++- .../models/albert/albert_preprocessor_test.py | 2 +- keras_nlp/models/bart/bart_preprocessor.py | 59 ++++++++++++++-- .../models/bart/bart_preprocessor_test.py | 3 +- .../bart/bart_seq_2_seq_lm_preprocessor.py | 70 +++++++++---------- .../bart_seq_2_seq_lm_preprocessor_test.py | 3 +- .../bert/bert_masked_lm_preprocessor_test.py | 2 +- keras_nlp/models/bert/bert_preprocessor.py | 13 +++- .../models/bert/bert_preprocessor_test.py | 2 +- .../bloom_causal_lm_preprocessor_test.py | 2 +- keras_nlp/models/bloom/bloom_preprocessor.py | 13 +++- .../models/bloom/bloom_preprocessor_test.py | 2 +- .../deberta_v3_masked_lm_preprocessor_test.py | 2 +- .../deberta_v3/deberta_v3_preprocessor.py | 13 +++- .../deberta_v3_preprocessor_test.py | 2 +- ...distil_bert_masked_lm_preprocessor_test.py | 2 +- .../distil_bert/distil_bert_preprocessor.py | 12 ++++ .../distil_bert_preprocessor_test.py | 2 +- .../f_net_masked_lm_preprocessor_test.py | 2 +- keras_nlp/models/f_net/f_net_preprocessor.py | 13 +++- .../models/f_net/f_net_preprocessor_test.py | 2 +- .../gpt2/gpt2_causal_lm_preprocessor_test.py | 2 +- keras_nlp/models/gpt2/gpt2_preprocessor.py | 13 +++- .../models/gpt2/gpt2_preprocessor_test.py | 2 +- .../gpt_neo_x_causal_lm_preprocessor_test.py | 2 +- .../gpt_neo_x/gpt_neo_x_preprocessor.py | 14 +++- .../gpt_neo_x/gpt_neo_x_preprocessor_test.py | 2 +- .../mistral_causal_lm_preprocessor_test.py | 2 +- .../models/mistral/mistral_preprocessor.py | 17 ++++- .../mistral/mistral_preprocessor_test.py | 2 +- .../opt/opt_causal_lm_preprocessor_test.py | 2 +- keras_nlp/models/opt/opt_preprocessor.py | 13 +++- keras_nlp/models/opt/opt_preprocessor_test.py | 2 +- .../roberta_masked_lm_preprocessor_test.py | 2 +- .../models/roberta/roberta_preprocessor.py | 13 +++- .../roberta/roberta_preprocessor_test.py | 2 +- .../models/whisper/whisper_preprocessor.py | 22 +++++- .../whisper/whisper_preprocessor_test.py | 3 +- ...xlm_roberta_masked_lm_preprocessor_test.py | 2 +- .../xlm_roberta/xlm_roberta_preprocessor.py | 13 +++- .../xlm_roberta_preprocessor_test.py | 2 +- keras_nlp/tests/test_case.py | 36 ++++++++++ 43 files changed, 320 insertions(+), 84 deletions(-) diff --git a/keras_nlp/models/albert/albert_masked_lm_preprocessor_test.py b/keras_nlp/models/albert/albert_masked_lm_preprocessor_test.py index 79d3a36bbb..b9bf693c17 100644 --- a/keras_nlp/models/albert/albert_masked_lm_preprocessor_test.py +++ b/keras_nlp/models/albert/albert_masked_lm_preprocessor_test.py @@ -43,7 +43,7 @@ def setUp(self): self.input_data = ["the quick brown fox"] def test_preprocessor_basics(self): - self.run_preprocessing_layer_test( + self.run_preprocessor_test( cls=AlbertMaskedLMPreprocessor, init_kwargs=self.init_kwargs, input_data=self.input_data, diff --git a/keras_nlp/models/albert/albert_preprocessor.py b/keras_nlp/models/albert/albert_preprocessor.py index 5d5628a729..19f4bd9a7b 100644 --- a/keras_nlp/models/albert/albert_preprocessor.py +++ b/keras_nlp/models/albert/albert_preprocessor.py @@ -158,9 +158,9 @@ def __init__( ): super().__init__(**kwargs) self.tokenizer = tokenizer + self.packer = None self.truncate = truncate self.sequence_length = sequence_length - self.packer = None def build(self, input_shape): # Defer packer creation to `build()` so that we can be sure tokenizer @@ -195,6 +195,17 @@ def call(self, x, y=None, sample_weight=None): } return pack_x_y_sample_weight(x, y, sample_weight) + @property + def sequence_length(self): + """The padded length of model input sequences.""" + return self._sequence_length + + @sequence_length.setter + def sequence_length(self, value): + self._sequence_length = value + if self.packer is not None: + self.packer.sequence_length = value + @classproperty def tokenizer_cls(cls): return AlbertTokenizer diff --git a/keras_nlp/models/albert/albert_preprocessor_test.py b/keras_nlp/models/albert/albert_preprocessor_test.py index 7d6fb4cfd4..ad5da8a47b 100644 --- a/keras_nlp/models/albert/albert_preprocessor_test.py +++ b/keras_nlp/models/albert/albert_preprocessor_test.py @@ -40,7 +40,7 @@ def setUp(self): ) def test_preprocessor_basics(self): - self.run_preprocessing_layer_test( + self.run_preprocessor_test( cls=AlbertPreprocessor, init_kwargs=self.init_kwargs, input_data=self.input_data, diff --git a/keras_nlp/models/bart/bart_preprocessor.py b/keras_nlp/models/bart/bart_preprocessor.py index ffe2148839..3310b1e532 100644 --- a/keras_nlp/models/bart/bart_preprocessor.py +++ b/keras_nlp/models/bart/bart_preprocessor.py @@ -140,10 +140,10 @@ def __init__( ): super().__init__(**kwargs) self.tokenizer = tokenizer - self.encoder_sequence_length = encoder_sequence_length - self.decoder_sequence_length = decoder_sequence_length self.encoder_packer = None self.decoder_packer = None + self.encoder_sequence_length = encoder_sequence_length + self.decoder_sequence_length = decoder_sequence_length def build(self, input_shape): # Defer packer creation to `build()` so that we can be sure tokenizer @@ -174,7 +174,17 @@ def build(self, input_shape): ) self.built = True - def call(self, x, y=None, sample_weight=None): + def call( + self, + x, + y=None, + sample_weight=None, + *, + encoder_sequence_length=None, + decoder_sequence_length=None, + # `sequence_length` is an alias for `decoder_sequence_length` + sequence_length=None, + ): if not ( isinstance(x, dict) and all(k in x for k in ("encoder_text", "decoder_text")) @@ -184,6 +194,12 @@ def call(self, x, y=None, sample_weight=None): f' and `"decoder_text"`. Received x={x}.' ) + if encoder_sequence_length is None: + encoder_sequence_length = self.encoder_sequence_length + decoder_sequence_length = decoder_sequence_length or sequence_length + if decoder_sequence_length is None: + decoder_sequence_length = self.decoder_sequence_length + encoder_text = x["encoder_text"] decoder_text = x["decoder_text"] @@ -199,12 +215,14 @@ def call(self, x, y=None, sample_weight=None): encoder_inputs = self.tokenizer(encoder_text[0]) encoder_token_ids, encoder_padding_mask = self.encoder_packer( - encoder_inputs + encoder_inputs, + sequence_length=encoder_sequence_length, ) decoder_inputs = self.tokenizer(decoder_text[0]) decoder_token_ids, decoder_padding_mask = self.decoder_packer( - decoder_inputs + decoder_inputs, + sequence_length=decoder_sequence_length, ) x = { @@ -226,6 +244,37 @@ def get_config(self): ) return config + @property + def encoder_sequence_length(self): + """The padded length of encoder input sequences.""" + return self._encoder_sequence_length + + @encoder_sequence_length.setter + def encoder_sequence_length(self, value): + self._encoder_sequence_length = value + if self.encoder_packer is not None: + self.encoder_packer.sequence_length = value + + @property + def decoder_sequence_length(self): + """The padded length of decoder input sequences.""" + return self._decoder_sequence_length + + @decoder_sequence_length.setter + def decoder_sequence_length(self, value): + self._decoder_sequence_length = value + if self.decoder_packer is not None: + self.decoder_packer.sequence_length = value + + @property + def sequence_length(self): + """Alias for `decoder_sequence_length`.""" + return self.decoder_sequence_length + + @sequence_length.setter + def sequence_length(self, value): + self.decoder_sequence_length = value + @classproperty def tokenizer_cls(cls): return BartTokenizer diff --git a/keras_nlp/models/bart/bart_preprocessor_test.py b/keras_nlp/models/bart/bart_preprocessor_test.py index 23cb7cae79..7872e35efa 100644 --- a/keras_nlp/models/bart/bart_preprocessor_test.py +++ b/keras_nlp/models/bart/bart_preprocessor_test.py @@ -46,7 +46,7 @@ def setUp(self): ) def test_preprocessor_basics(self): - self.run_preprocessing_layer_test( + self.run_preprocessor_test( cls=BartPreprocessor, init_kwargs=self.init_kwargs, input_data=self.input_data, @@ -60,6 +60,7 @@ def test_preprocessor_basics(self): [1], # Pass through labels. [1.0], # Pass through sample_weights. ), + token_id_key="decoder_token_ids", ) def test_error_multi_segment_input(self): diff --git a/keras_nlp/models/bart/bart_seq_2_seq_lm_preprocessor.py b/keras_nlp/models/bart/bart_seq_2_seq_lm_preprocessor.py index 3d398d29d1..1c72e6e935 100644 --- a/keras_nlp/models/bart/bart_seq_2_seq_lm_preprocessor.py +++ b/keras_nlp/models/bart/bart_seq_2_seq_lm_preprocessor.py @@ -124,28 +124,17 @@ class BartSeq2SeqLMPreprocessor(BartPreprocessor): ``` """ - def __init__( + def call( self, - tokenizer, - encoder_sequence_length=1024, - decoder_sequence_length=1024, - **kwargs + x, + y=None, + sample_weight=None, + *, + encoder_sequence_length=None, + decoder_sequence_length=None, + # `sequence_length` is an alias for `decoder_sequence_length` + sequence_length=None, ): - # Since we truncate the last token from `decoder_token_ids`, we need to - # forcefully set the `decoder_sequence_length` to one greater than the - # value passed. - super().__init__( - tokenizer=tokenizer, - encoder_sequence_length=encoder_sequence_length, - decoder_sequence_length=decoder_sequence_length + 1, - **kwargs - ) - - # Maintain a private copy of the sequence lengths for config purposes. - self._encoder_sequence_length = encoder_sequence_length - self._decoder_sequence_length = decoder_sequence_length - - def call(self, x, y=None, sample_weight=None): if y is not None or sample_weight is not None: logging.warning( "`BartSeq2SeqLMPreprocessor` infers `y` and `sample_weight` " @@ -154,7 +143,17 @@ def call(self, x, y=None, sample_weight=None): "These values will be ignored." ) - x = super().call(x) + if encoder_sequence_length is None: + encoder_sequence_length = self.encoder_sequence_length + decoder_sequence_length = decoder_sequence_length or sequence_length + if decoder_sequence_length is None: + decoder_sequence_length = self.decoder_sequence_length + + x = super().call( + x, + encoder_sequence_length=encoder_sequence_length, + decoder_sequence_length=decoder_sequence_length + 1, + ) decoder_token_ids = x.pop("decoder_token_ids") decoder_padding_mask = x.pop("decoder_padding_mask") @@ -173,6 +172,10 @@ def call(self, x, y=None, sample_weight=None): def generate_preprocess( self, x, + *, + encoder_sequence_length=None, + # `sequence_length` is an alias for `decoder_sequence_length` + decoder_sequence_length=None, sequence_length=None, ): """Convert encoder and decoder input strings to integer token inputs for generation. @@ -190,10 +193,6 @@ def generate_preprocess( if not self.built: self.build(None) - # If `sequence_length` is not provided, we use the default value. - if sequence_length is None: - sequence_length = self._decoder_sequence_length - if isinstance(x, dict): encoder_text = x["encoder_text"] decoder_text = x["decoder_text"] @@ -202,6 +201,12 @@ def generate_preprocess( # Initialize empty prompt for the decoder. decoder_text = tf.fill((tf.shape(encoder_text)[0],), "") + if encoder_sequence_length is None: + encoder_sequence_length = self.encoder_sequence_length + decoder_sequence_length = decoder_sequence_length or sequence_length + if decoder_sequence_length is None: + decoder_sequence_length = self.decoder_sequence_length + # Tokenize and pack the encoder inputs. # TODO: Remove `[0]` once we have shifted to `MultiSegmentPacker`. encoder_text = convert_inputs_to_list_of_tensor_segments(encoder_text)[ @@ -209,7 +214,8 @@ def generate_preprocess( ] encoder_token_ids = self.tokenizer(encoder_text) encoder_token_ids, encoder_padding_mask = self.encoder_packer( - encoder_token_ids + encoder_token_ids, + sequence_length=encoder_sequence_length, ) # Tokenize and pack the decoder inputs. @@ -219,7 +225,7 @@ def generate_preprocess( decoder_token_ids = self.tokenizer(decoder_text) decoder_token_ids, decoder_padding_mask = self.decoder_packer( decoder_token_ids, - sequence_length=sequence_length, + sequence_length=decoder_sequence_length, add_end_value=False, ) @@ -261,16 +267,6 @@ def generate_postprocess( ) return self.tokenizer.detokenize(decoder_token_ids) - def get_config(self): - config = super().get_config() - config.update( - { - "encoder_sequence_length": self._encoder_sequence_length, - "decoder_sequence_length": self._decoder_sequence_length, - } - ) - return config - @classproperty def presets(cls): return copy.deepcopy(backbone_presets) diff --git a/keras_nlp/models/bart/bart_seq_2_seq_lm_preprocessor_test.py b/keras_nlp/models/bart/bart_seq_2_seq_lm_preprocessor_test.py index 33fbd5fc3a..2f40e69722 100644 --- a/keras_nlp/models/bart/bart_seq_2_seq_lm_preprocessor_test.py +++ b/keras_nlp/models/bart/bart_seq_2_seq_lm_preprocessor_test.py @@ -45,7 +45,7 @@ def setUp(self): ) def test_preprocessor_basics(self): - self.run_preprocessing_layer_test( + self.run_preprocessor_test( cls=BartSeq2SeqLMPreprocessor, init_kwargs=self.init_kwargs, input_data=self.input_data, @@ -59,6 +59,7 @@ def test_preprocessor_basics(self): [[0, 4, 5, 4, 7, 2, 1, 1]], [[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0]], ), + token_id_key="decoder_token_ids", ) def test_generate_preprocess(self): diff --git a/keras_nlp/models/bert/bert_masked_lm_preprocessor_test.py b/keras_nlp/models/bert/bert_masked_lm_preprocessor_test.py index ff58962215..479d9e879b 100644 --- a/keras_nlp/models/bert/bert_masked_lm_preprocessor_test.py +++ b/keras_nlp/models/bert/bert_masked_lm_preprocessor_test.py @@ -39,7 +39,7 @@ def setUp(self): self.input_data = ["the quick brown fox"] def test_preprocessor_basics(self): - self.run_preprocessing_layer_test( + self.run_preprocessor_test( cls=BertMaskedLMPreprocessor, init_kwargs=self.init_kwargs, input_data=self.input_data, diff --git a/keras_nlp/models/bert/bert_preprocessor.py b/keras_nlp/models/bert/bert_preprocessor.py index bad38f22a5..02f5a45985 100644 --- a/keras_nlp/models/bert/bert_preprocessor.py +++ b/keras_nlp/models/bert/bert_preprocessor.py @@ -139,9 +139,9 @@ def __init__( ): super().__init__(**kwargs) self.tokenizer = tokenizer + self.packer = None self.sequence_length = sequence_length self.truncate = truncate - self.packer = None def build(self, input_shape): # Defer packer creation to `build()` so that we can be sure tokenizer @@ -176,6 +176,17 @@ def get_config(self): ) return config + @property + def sequence_length(self): + """The padded length of model input sequences.""" + return self._sequence_length + + @sequence_length.setter + def sequence_length(self, value): + self._sequence_length = value + if self.packer is not None: + self.packer.sequence_length = value + @classproperty def tokenizer_cls(cls): return BertTokenizer diff --git a/keras_nlp/models/bert/bert_preprocessor_test.py b/keras_nlp/models/bert/bert_preprocessor_test.py index 6d1e5fee57..c109d1006d 100644 --- a/keras_nlp/models/bert/bert_preprocessor_test.py +++ b/keras_nlp/models/bert/bert_preprocessor_test.py @@ -36,7 +36,7 @@ def setUp(self): ) def test_preprocessor_basics(self): - self.run_preprocessing_layer_test( + self.run_preprocessor_test( cls=BertPreprocessor, init_kwargs=self.init_kwargs, input_data=self.input_data, diff --git a/keras_nlp/models/bloom/bloom_causal_lm_preprocessor_test.py b/keras_nlp/models/bloom/bloom_causal_lm_preprocessor_test.py index 6caf8fddcf..a281519340 100644 --- a/keras_nlp/models/bloom/bloom_causal_lm_preprocessor_test.py +++ b/keras_nlp/models/bloom/bloom_causal_lm_preprocessor_test.py @@ -40,7 +40,7 @@ def setUp(self): self.input_data = ["airplane at airport"] def test_preprocessor_basics(self): - self.run_preprocessing_layer_test( + self.run_preprocessor_test( cls=BloomCausalLMPreprocessor, init_kwargs=self.init_kwargs, input_data=self.input_data, diff --git a/keras_nlp/models/bloom/bloom_preprocessor.py b/keras_nlp/models/bloom/bloom_preprocessor.py index d57ccd2414..734c9f4bf8 100644 --- a/keras_nlp/models/bloom/bloom_preprocessor.py +++ b/keras_nlp/models/bloom/bloom_preprocessor.py @@ -116,8 +116,8 @@ def __init__( **kwargs, ): super().__init__(**kwargs) - self.tokenizer = tokenizer + self.packer = None self.sequence_length = sequence_length self.add_start_token = add_start_token self.add_end_token = add_end_token @@ -173,6 +173,17 @@ def get_config(self): ) return config + @property + def sequence_length(self): + """The padded length of model input sequences.""" + return self._sequence_length + + @sequence_length.setter + def sequence_length(self, value): + self._sequence_length = value + if self.packer is not None: + self.packer.sequence_length = value + @classproperty def presets(cls): return copy.deepcopy(backbone_presets) diff --git a/keras_nlp/models/bloom/bloom_preprocessor_test.py b/keras_nlp/models/bloom/bloom_preprocessor_test.py index bb80006396..938113ef4b 100644 --- a/keras_nlp/models/bloom/bloom_preprocessor_test.py +++ b/keras_nlp/models/bloom/bloom_preprocessor_test.py @@ -38,7 +38,7 @@ def setUp(self): self.input_data = ["airplane at airport"] def test_preprocessor_basics(self): - self.run_preprocessing_layer_test( + self.run_preprocessor_test( cls=BloomPreprocessor, init_kwargs=self.init_kwargs, input_data=self.input_data, diff --git a/keras_nlp/models/deberta_v3/deberta_v3_masked_lm_preprocessor_test.py b/keras_nlp/models/deberta_v3/deberta_v3_masked_lm_preprocessor_test.py index 217980ea59..f041a6f7ff 100644 --- a/keras_nlp/models/deberta_v3/deberta_v3_masked_lm_preprocessor_test.py +++ b/keras_nlp/models/deberta_v3/deberta_v3_masked_lm_preprocessor_test.py @@ -43,7 +43,7 @@ def setUp(self): self.input_data = ["the quick brown fox"] def test_preprocessor_basics(self): - self.run_preprocessing_layer_test( + self.run_preprocessor_test( cls=DebertaV3MaskedLMPreprocessor, init_kwargs=self.init_kwargs, input_data=self.input_data, diff --git a/keras_nlp/models/deberta_v3/deberta_v3_preprocessor.py b/keras_nlp/models/deberta_v3/deberta_v3_preprocessor.py index 93f4fbbd22..88fa08fd70 100644 --- a/keras_nlp/models/deberta_v3/deberta_v3_preprocessor.py +++ b/keras_nlp/models/deberta_v3/deberta_v3_preprocessor.py @@ -156,9 +156,9 @@ def __init__( ): super().__init__(**kwargs) self.tokenizer = tokenizer + self.packer = None self.truncate = truncate self.sequence_length = sequence_length - self.packer = None def build(self, input_shape): # Defer packer creation to `build()` so that we can be sure tokenizer @@ -192,6 +192,17 @@ def call(self, x, y=None, sample_weight=None): } return pack_x_y_sample_weight(x, y, sample_weight) + @property + def sequence_length(self): + """The padded length of model input sequences.""" + return self._sequence_length + + @sequence_length.setter + def sequence_length(self, value): + self._sequence_length = value + if self.packer is not None: + self.packer.sequence_length = value + @classproperty def tokenizer_cls(cls): return DebertaV3Tokenizer diff --git a/keras_nlp/models/deberta_v3/deberta_v3_preprocessor_test.py b/keras_nlp/models/deberta_v3/deberta_v3_preprocessor_test.py index a50022f3c7..a9e2a59c29 100644 --- a/keras_nlp/models/deberta_v3/deberta_v3_preprocessor_test.py +++ b/keras_nlp/models/deberta_v3/deberta_v3_preprocessor_test.py @@ -42,7 +42,7 @@ def setUp(self): ) def test_preprocessor_basics(self): - self.run_preprocessing_layer_test( + self.run_preprocessor_test( cls=DebertaV3Preprocessor, init_kwargs=self.init_kwargs, input_data=self.input_data, diff --git a/keras_nlp/models/distil_bert/distil_bert_masked_lm_preprocessor_test.py b/keras_nlp/models/distil_bert/distil_bert_masked_lm_preprocessor_test.py index b01b1da8ac..85ee5bdd43 100644 --- a/keras_nlp/models/distil_bert/distil_bert_masked_lm_preprocessor_test.py +++ b/keras_nlp/models/distil_bert/distil_bert_masked_lm_preprocessor_test.py @@ -41,7 +41,7 @@ def setUp(self): self.input_data = ["the quick brown fox"] def test_preprocessor_basics(self): - self.run_preprocessing_layer_test( + self.run_preprocessor_test( cls=DistilBertMaskedLMPreprocessor, init_kwargs=self.init_kwargs, input_data=self.input_data, diff --git a/keras_nlp/models/distil_bert/distil_bert_preprocessor.py b/keras_nlp/models/distil_bert/distil_bert_preprocessor.py index 107275f80a..63f4e3637b 100644 --- a/keras_nlp/models/distil_bert/distil_bert_preprocessor.py +++ b/keras_nlp/models/distil_bert/distil_bert_preprocessor.py @@ -127,6 +127,7 @@ def __init__( ): super().__init__(**kwargs) self.tokenizer = tokenizer + self.packer = None self.sequence_length = sequence_length self.truncate = truncate @@ -162,6 +163,17 @@ def get_config(self): ) return config + @property + def sequence_length(self): + """The padded length of model input sequences.""" + return self._sequence_length + + @sequence_length.setter + def sequence_length(self, value): + self._sequence_length = value + if self.packer is not None: + self.packer.sequence_length = value + @classproperty def tokenizer_cls(cls): return DistilBertTokenizer diff --git a/keras_nlp/models/distil_bert/distil_bert_preprocessor_test.py b/keras_nlp/models/distil_bert/distil_bert_preprocessor_test.py index 22d69c88dc..f58b42cd39 100644 --- a/keras_nlp/models/distil_bert/distil_bert_preprocessor_test.py +++ b/keras_nlp/models/distil_bert/distil_bert_preprocessor_test.py @@ -40,7 +40,7 @@ def setUp(self): ) def test_preprocessor_basics(self): - self.run_preprocessing_layer_test( + self.run_preprocessor_test( cls=DistilBertPreprocessor, init_kwargs=self.init_kwargs, input_data=self.input_data, diff --git a/keras_nlp/models/f_net/f_net_masked_lm_preprocessor_test.py b/keras_nlp/models/f_net/f_net_masked_lm_preprocessor_test.py index 5f72081a0d..7d2ecc0f17 100644 --- a/keras_nlp/models/f_net/f_net_masked_lm_preprocessor_test.py +++ b/keras_nlp/models/f_net/f_net_masked_lm_preprocessor_test.py @@ -41,7 +41,7 @@ def setUp(self): self.input_data = ["the quick brown fox"] def test_preprocessor_basics(self): - self.run_preprocessing_layer_test( + self.run_preprocessor_test( cls=FNetMaskedLMPreprocessor, init_kwargs=self.init_kwargs, input_data=self.input_data, diff --git a/keras_nlp/models/f_net/f_net_preprocessor.py b/keras_nlp/models/f_net/f_net_preprocessor.py index 296493c930..b4cb5836bb 100644 --- a/keras_nlp/models/f_net/f_net_preprocessor.py +++ b/keras_nlp/models/f_net/f_net_preprocessor.py @@ -129,9 +129,9 @@ def __init__( ): super().__init__(**kwargs) self.tokenizer = tokenizer + self.packer = None self.truncate = truncate self.sequence_length = sequence_length - self.packer = None def build(self, input_shape): # Defer packer creation to `build()` so that we can be sure tokenizer @@ -165,6 +165,17 @@ def call(self, x, y=None, sample_weight=None): } return pack_x_y_sample_weight(x, y, sample_weight) + @property + def sequence_length(self): + """The padded length of model input sequences.""" + return self._sequence_length + + @sequence_length.setter + def sequence_length(self, value): + self._sequence_length = value + if self.packer is not None: + self.packer.sequence_length = value + @classproperty def tokenizer_cls(cls): return FNetTokenizer diff --git a/keras_nlp/models/f_net/f_net_preprocessor_test.py b/keras_nlp/models/f_net/f_net_preprocessor_test.py index f67737c828..c9096ac59f 100644 --- a/keras_nlp/models/f_net/f_net_preprocessor_test.py +++ b/keras_nlp/models/f_net/f_net_preprocessor_test.py @@ -38,7 +38,7 @@ def setUp(self): ) def test_preprocessor_basics(self): - self.run_preprocessing_layer_test( + self.run_preprocessor_test( cls=FNetPreprocessor, init_kwargs=self.init_kwargs, input_data=self.input_data, diff --git a/keras_nlp/models/gpt2/gpt2_causal_lm_preprocessor_test.py b/keras_nlp/models/gpt2/gpt2_causal_lm_preprocessor_test.py index 400273b792..0623d983a9 100644 --- a/keras_nlp/models/gpt2/gpt2_causal_lm_preprocessor_test.py +++ b/keras_nlp/models/gpt2/gpt2_causal_lm_preprocessor_test.py @@ -40,7 +40,7 @@ def setUp(self): self.input_data = ["airplane at airport"] def test_preprocessor_basics(self): - self.run_preprocessing_layer_test( + self.run_preprocessor_test( cls=GPT2CausalLMPreprocessor, init_kwargs=self.init_kwargs, input_data=self.input_data, diff --git a/keras_nlp/models/gpt2/gpt2_preprocessor.py b/keras_nlp/models/gpt2/gpt2_preprocessor.py index 29182f77b6..82be34776f 100644 --- a/keras_nlp/models/gpt2/gpt2_preprocessor.py +++ b/keras_nlp/models/gpt2/gpt2_preprocessor.py @@ -118,8 +118,8 @@ def __init__( **kwargs, ): super().__init__(**kwargs) - self.tokenizer = tokenizer + self.packer = None self.sequence_length = sequence_length self.add_start_token = add_start_token self.add_end_token = add_end_token @@ -175,6 +175,17 @@ def get_config(self): ) return config + @property + def sequence_length(self): + """The padded length of model input sequences.""" + return self._sequence_length + + @sequence_length.setter + def sequence_length(self, value): + self._sequence_length = value + if self.packer is not None: + self.packer.sequence_length = value + @classproperty def presets(cls): return copy.deepcopy(backbone_presets) diff --git a/keras_nlp/models/gpt2/gpt2_preprocessor_test.py b/keras_nlp/models/gpt2/gpt2_preprocessor_test.py index d7dcd261ed..35129c200d 100644 --- a/keras_nlp/models/gpt2/gpt2_preprocessor_test.py +++ b/keras_nlp/models/gpt2/gpt2_preprocessor_test.py @@ -38,7 +38,7 @@ def setUp(self): self.input_data = ["airplane at airport"] def test_preprocessor_basics(self): - self.run_preprocessing_layer_test( + self.run_preprocessor_test( cls=GPT2Preprocessor, init_kwargs=self.init_kwargs, input_data=self.input_data, diff --git a/keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor_test.py b/keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor_test.py index f5a7c57421..e873c38c79 100644 --- a/keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor_test.py +++ b/keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor_test.py @@ -40,7 +40,7 @@ def setUp(self): self.input_data = ["airplane at airport"] def test_preprocessor_basics(self): - self.run_preprocessing_layer_test( + self.run_preprocessor_test( cls=GPTNeoXCausalLMPreprocessor, init_kwargs=self.init_kwargs, input_data=self.input_data, diff --git a/keras_nlp/models/gpt_neo_x/gpt_neo_x_preprocessor.py b/keras_nlp/models/gpt_neo_x/gpt_neo_x_preprocessor.py index 1db4fe4c9b..8dc374332b 100644 --- a/keras_nlp/models/gpt_neo_x/gpt_neo_x_preprocessor.py +++ b/keras_nlp/models/gpt_neo_x/gpt_neo_x_preprocessor.py @@ -74,12 +74,11 @@ def __init__( **kwargs, ): super().__init__(**kwargs) - self.tokenizer = tokenizer + self.packer = None self.sequence_length = sequence_length self.add_start_token = add_start_token self.add_end_token = add_end_token - self.packer = None def build(self, input_shape): # Defer packer creation to `build()` so that we can be sure tokenizer @@ -132,6 +131,17 @@ def get_config(self): ) return config + @property + def sequence_length(self): + """The padded length of model input sequences.""" + return self._sequence_length + + @sequence_length.setter + def sequence_length(self, value): + self._sequence_length = value + if self.packer is not None: + self.packer.sequence_length = value + @classproperty def tokenizer_cls(cls): return GPTNeoXTokenizer diff --git a/keras_nlp/models/gpt_neo_x/gpt_neo_x_preprocessor_test.py b/keras_nlp/models/gpt_neo_x/gpt_neo_x_preprocessor_test.py index c87329af4a..92ea191596 100644 --- a/keras_nlp/models/gpt_neo_x/gpt_neo_x_preprocessor_test.py +++ b/keras_nlp/models/gpt_neo_x/gpt_neo_x_preprocessor_test.py @@ -38,7 +38,7 @@ def setUp(self): self.input_data = ["airplane at airport"] def test_preprocessor_basics(self): - self.run_preprocessing_layer_test( + self.run_preprocessor_test( cls=GPTNeoXPreprocessor, init_kwargs=self.init_kwargs, input_data=self.input_data, diff --git a/keras_nlp/models/mistral/mistral_causal_lm_preprocessor_test.py b/keras_nlp/models/mistral/mistral_causal_lm_preprocessor_test.py index 420995016b..80b55c02e5 100644 --- a/keras_nlp/models/mistral/mistral_causal_lm_preprocessor_test.py +++ b/keras_nlp/models/mistral/mistral_causal_lm_preprocessor_test.py @@ -36,7 +36,7 @@ def setUp(self): self.input_data = (["the quick brown fox"],) def test_preprocessor_basics(self): - self.run_preprocessing_layer_test( + self.run_preprocessor_test( cls=MistralCausalLMPreprocessor, init_kwargs=self.init_kwargs, input_data=self.input_data, diff --git a/keras_nlp/models/mistral/mistral_preprocessor.py b/keras_nlp/models/mistral/mistral_preprocessor.py index d5d838303e..b96afb5ed8 100644 --- a/keras_nlp/models/mistral/mistral_preprocessor.py +++ b/keras_nlp/models/mistral/mistral_preprocessor.py @@ -121,15 +121,15 @@ def __init__( ): super().__init__(**kwargs) self.tokenizer = tokenizer - self.add_start_token = add_start_token - self.add_end_token = add_end_token - self.sequence_length = sequence_length self.packer = StartEndPacker( start_value=self.tokenizer.start_token_id, end_value=self.tokenizer.end_token_id, sequence_length=sequence_length, return_padding_mask=True, ) + self.add_start_token = add_start_token + self.add_end_token = add_end_token + self.sequence_length = sequence_length def get_config(self): config = super().get_config() @@ -170,6 +170,17 @@ def call( } return pack_x_y_sample_weight(x, y, sample_weight) + @property + def sequence_length(self): + """The padded length of model input sequences.""" + return self._sequence_length + + @sequence_length.setter + def sequence_length(self, value): + self._sequence_length = value + if self.packer is not None: + self.packer.sequence_length = value + @classproperty def tokenizer_cls(cls): return MistralTokenizer diff --git a/keras_nlp/models/mistral/mistral_preprocessor_test.py b/keras_nlp/models/mistral/mistral_preprocessor_test.py index 40528fd4e8..47c0bc542c 100644 --- a/keras_nlp/models/mistral/mistral_preprocessor_test.py +++ b/keras_nlp/models/mistral/mistral_preprocessor_test.py @@ -38,7 +38,7 @@ def setUp(self): ) def test_preprocessor_basics(self): - self.run_preprocessing_layer_test( + self.run_preprocessor_test( cls=MistralPreprocessor, init_kwargs=self.init_kwargs, input_data=self.input_data, diff --git a/keras_nlp/models/opt/opt_causal_lm_preprocessor_test.py b/keras_nlp/models/opt/opt_causal_lm_preprocessor_test.py index 9ba6851d4b..e04436f092 100644 --- a/keras_nlp/models/opt/opt_causal_lm_preprocessor_test.py +++ b/keras_nlp/models/opt/opt_causal_lm_preprocessor_test.py @@ -39,7 +39,7 @@ def setUp(self): self.input_data = ["airplane at airport"] def test_preprocessor_basics(self): - self.run_preprocessing_layer_test( + self.run_preprocessor_test( cls=OPTCausalLMPreprocessor, init_kwargs=self.init_kwargs, input_data=self.input_data, diff --git a/keras_nlp/models/opt/opt_preprocessor.py b/keras_nlp/models/opt/opt_preprocessor.py index cdca904870..8f52bb67e6 100644 --- a/keras_nlp/models/opt/opt_preprocessor.py +++ b/keras_nlp/models/opt/opt_preprocessor.py @@ -120,10 +120,10 @@ def __init__( super().__init__(**kwargs) self.tokenizer = tokenizer + self.packer = None self.sequence_length = sequence_length self.add_start_token = add_start_token self.add_end_token = add_end_token - self.packer = None def build(self, input_shape): # Defer packer creation to `build()` so that we can be sure tokenizer @@ -176,6 +176,17 @@ def call( } return pack_x_y_sample_weight(x, y, sample_weight) + @property + def sequence_length(self): + """The padded length of model input sequences.""" + return self._sequence_length + + @sequence_length.setter + def sequence_length(self, value): + self._sequence_length = value + if self.packer is not None: + self.packer.sequence_length = value + @classproperty def presets(cls): return copy.deepcopy(backbone_presets) diff --git a/keras_nlp/models/opt/opt_preprocessor_test.py b/keras_nlp/models/opt/opt_preprocessor_test.py index b80c409b92..901efc7bee 100644 --- a/keras_nlp/models/opt/opt_preprocessor_test.py +++ b/keras_nlp/models/opt/opt_preprocessor_test.py @@ -37,7 +37,7 @@ def setUp(self): self.input_data = ["airplane at airport"] def test_preprocessor_basics(self): - self.run_preprocessing_layer_test( + self.run_preprocessor_test( cls=OPTPreprocessor, init_kwargs=self.init_kwargs, input_data=self.input_data, diff --git a/keras_nlp/models/roberta/roberta_masked_lm_preprocessor_test.py b/keras_nlp/models/roberta/roberta_masked_lm_preprocessor_test.py index ae762079e2..a842e99f5d 100644 --- a/keras_nlp/models/roberta/roberta_masked_lm_preprocessor_test.py +++ b/keras_nlp/models/roberta/roberta_masked_lm_preprocessor_test.py @@ -44,7 +44,7 @@ def setUp(self): self.input_data = [" airplane airport"] def test_preprocessor_basics(self): - self.run_preprocessing_layer_test( + self.run_preprocessor_test( cls=RobertaMaskedLMPreprocessor, init_kwargs=self.init_kwargs, input_data=self.input_data, diff --git a/keras_nlp/models/roberta/roberta_preprocessor.py b/keras_nlp/models/roberta/roberta_preprocessor.py index 556561d17c..57a421590f 100644 --- a/keras_nlp/models/roberta/roberta_preprocessor.py +++ b/keras_nlp/models/roberta/roberta_preprocessor.py @@ -143,9 +143,9 @@ def __init__( super().__init__(**kwargs) self.tokenizer = tokenizer + self.packer = None self.truncate = truncate self.sequence_length = sequence_length - self.packer = None def build(self, input_shape): # Defer packer creation to `build()` so that we can be sure tokenizer @@ -180,6 +180,17 @@ def get_config(self): ) return config + @property + def sequence_length(self): + """The padded length of model input sequences.""" + return self._sequence_length + + @sequence_length.setter + def sequence_length(self, value): + self._sequence_length = value + if self.packer is not None: + self.packer.sequence_length = value + @classproperty def tokenizer_cls(cls): return RobertaTokenizer diff --git a/keras_nlp/models/roberta/roberta_preprocessor_test.py b/keras_nlp/models/roberta/roberta_preprocessor_test.py index 5e7ad77514..699742ea08 100644 --- a/keras_nlp/models/roberta/roberta_preprocessor_test.py +++ b/keras_nlp/models/roberta/roberta_preprocessor_test.py @@ -41,7 +41,7 @@ def setUp(self): ) def test_preprocessor_basics(self): - self.run_preprocessing_layer_test( + self.run_preprocessor_test( cls=RobertaPreprocessor, init_kwargs=self.init_kwargs, input_data=self.input_data, diff --git a/keras_nlp/models/whisper/whisper_preprocessor.py b/keras_nlp/models/whisper/whisper_preprocessor.py index abcff0d770..c21705a481 100644 --- a/keras_nlp/models/whisper/whisper_preprocessor.py +++ b/keras_nlp/models/whisper/whisper_preprocessor.py @@ -169,11 +169,11 @@ def __init__( audio_feature_extractor = WhisperAudioFeatureExtractor() self.audio_feature_extractor = audio_feature_extractor self.tokenizer = tokenizer + self.decoder_packer = None self.decoder_sequence_length = decoder_sequence_length self.language = language self.task = task self.no_timestamps = no_timestamps - self.decoder_packer = None def build(self, input_shape): # Defer packer creation to `build()` so that we can be sure tokenizer @@ -307,6 +307,26 @@ def from_config(cls, config): return cls(**config) + @property + def decoder_sequence_length(self): + """The padded length of decoder input sequences.""" + return self._decoder_sequence_length + + @decoder_sequence_length.setter + def decoder_sequence_length(self, value): + self._decoder_sequence_length = value + if self.decoder_packer is not None: + self.decoder_packer.sequence_length = value + + @property + def sequence_length(self): + """Alias for `decoder_sequence_length`.""" + return self.decoder_sequence_length + + @sequence_length.setter + def sequence_length(self, value): + self.decoder_sequence_length = value + @classproperty def audio_feature_extractor_cls(cls): return WhisperAudioFeatureExtractor diff --git a/keras_nlp/models/whisper/whisper_preprocessor_test.py b/keras_nlp/models/whisper/whisper_preprocessor_test.py index 6837dc8bfa..8517a6c102 100644 --- a/keras_nlp/models/whisper/whisper_preprocessor_test.py +++ b/keras_nlp/models/whisper/whisper_preprocessor_test.py @@ -66,10 +66,11 @@ def setUp(self): } def test_feature_extractor_basics(self): - self.run_preprocessing_layer_test( + self.run_preprocessor_test( cls=WhisperPreprocessor, init_kwargs=self.init_kwargs, input_data=self.input_data, + token_id_key="decoder_token_ids", ) def test_sequence_length_override(self): diff --git a/keras_nlp/models/xlm_roberta/xlm_roberta_masked_lm_preprocessor_test.py b/keras_nlp/models/xlm_roberta/xlm_roberta_masked_lm_preprocessor_test.py index c1bfc7242a..6d77e71319 100644 --- a/keras_nlp/models/xlm_roberta/xlm_roberta_masked_lm_preprocessor_test.py +++ b/keras_nlp/models/xlm_roberta/xlm_roberta_masked_lm_preprocessor_test.py @@ -45,7 +45,7 @@ def setUp(self): self.input_data = ["the quick brown fox"] def test_preprocessor_basics(self): - self.run_preprocessing_layer_test( + self.run_preprocessor_test( cls=XLMRobertaMaskedLMPreprocessor, init_kwargs=self.init_kwargs, input_data=self.input_data, diff --git a/keras_nlp/models/xlm_roberta/xlm_roberta_preprocessor.py b/keras_nlp/models/xlm_roberta/xlm_roberta_preprocessor.py index 23b48073f7..c94f5f2421 100644 --- a/keras_nlp/models/xlm_roberta/xlm_roberta_preprocessor.py +++ b/keras_nlp/models/xlm_roberta/xlm_roberta_preprocessor.py @@ -156,9 +156,9 @@ def __init__( super().__init__(**kwargs) self.tokenizer = tokenizer + self.packer = None self.truncate = truncate self.sequence_length = sequence_length - self.packer = None def build(self, input_shape): # Defer packer creation to `build()` so that we can be sure tokenizer @@ -193,6 +193,17 @@ def call(self, x, y=None, sample_weight=None): } return pack_x_y_sample_weight(x, y, sample_weight) + @property + def sequence_length(self): + """The padded length of model input sequences.""" + return self._sequence_length + + @sequence_length.setter + def sequence_length(self, value): + self._sequence_length = value + if self.packer is not None: + self.packer.sequence_length = value + @classproperty def tokenizer_cls(cls): return XLMRobertaTokenizer diff --git a/keras_nlp/models/xlm_roberta/xlm_roberta_preprocessor_test.py b/keras_nlp/models/xlm_roberta/xlm_roberta_preprocessor_test.py index 3c3bbf2612..85c76fa282 100644 --- a/keras_nlp/models/xlm_roberta/xlm_roberta_preprocessor_test.py +++ b/keras_nlp/models/xlm_roberta/xlm_roberta_preprocessor_test.py @@ -44,7 +44,7 @@ def setUp(self): ) def test_preprocessor_basics(self): - self.run_preprocessing_layer_test( + self.run_preprocessor_test( cls=XLMRobertaPreprocessor, init_kwargs=self.init_kwargs, input_data=self.input_data, diff --git a/keras_nlp/tests/test_case.py b/keras_nlp/tests/test_case.py index 50e7733513..0541ae6451 100644 --- a/keras_nlp/tests/test_case.py +++ b/keras_nlp/tests/test_case.py @@ -230,6 +230,42 @@ def run_preprocessing_layer_test( if expected_output: self.assertAllClose(output, expected_output) + def run_preprocessor_test( + self, + cls, + init_kwargs, + input_data, + expected_output=None, + expected_detokenize_output=None, + token_id_key="token_ids", + ): + """Run basic tests for a Model Preprocessor layer.""" + self.run_preprocessing_layer_test( + cls, + init_kwargs, + input_data, + expected_output=expected_output, + expected_detokenize_output=expected_detokenize_output, + ) + + layer = cls(**self.init_kwargs) + if isinstance(input_data, tuple): + output = layer(*input_data) + else: + output = layer(input_data) + output, _, _ = keras.utils.unpack_x_y_sample_weight(output) + shape = ops.shape(output[token_id_key]) + self.assertEqual(shape[-1], layer.sequence_length) + # Update the sequence length. + layer.sequence_length = 17 + if isinstance(input_data, tuple): + output = layer(*input_data) + else: + output = layer(input_data) + output, _, _ = keras.utils.unpack_x_y_sample_weight(output) + shape = ops.shape(output[token_id_key]) + self.assertEqual(shape[-1], 17) + def run_serialization_test(self, instance): """Check idempotency of serialize/deserialize. From 1322aaa6a379853f866739b7b69eb5cd8e7c62c9 Mon Sep 17 00:00:00 2001 From: Pedro Kaj Kjellerup Nacht Date: Wed, 14 Feb 2024 21:28:19 -0300 Subject: [PATCH 19/29] Add dependabot to update GH Actions and Python dependencies (#1380) * Add dependabot to update GHA and Python dependencies Signed-off-by: Pedro Kaj Kjellerup Nacht * Bump GitHub Actions to latest versions commit 1be8ca54d78e778b5fc6b45483315e780e2d1ff2 Author: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed Dec 27 18:43:45 2023 +0000 Bump the github-actions group with 3 updates Bumps the github-actions group with 3 updates: [actions/checkout](https://github.com/actions/checkout), [actions/setup-python](https://github.com/actions/setup-python) and [actions/cache](https://github.com/actions/cache). Updates `actions/checkout` from 3 to 4 - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) Updates `actions/setup-python` from 1 to 5 - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v1...v5) Updates `actions/cache` from 2 to 3 - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/v2...v3) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions - dependency-name: actions/cache dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions ... Signed-off-by: dependabot[bot] --------- Signed-off-by: Pedro Kaj Kjellerup Nacht --- .github/dependabot.yml | 23 +++++++++++++++++++++++ .github/workflows/actions.yml | 18 +++++++++--------- .github/workflows/nightly.yml | 6 +++--- .github/workflows/publish-to-pypi.yml | 6 +++--- 4 files changed, 38 insertions(+), 15 deletions(-) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..0df37b1230 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,23 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" + groups: + github-actions: + patterns: + - "*" + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "monthly" + groups: + python: + patterns: + - "*" diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 64a41ca16e..1d77f5b9ae 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -15,9 +15,9 @@ jobs: name: Test the code with Keras 2 runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python 3.9 - uses: actions/setup-python@v1 + uses: actions/setup-python@v5 with: python-version: 3.9 - name: Get pip cache dir @@ -26,7 +26,7 @@ jobs: python -m pip install --upgrade pip setuptools echo "::set-output name=dir::$(pip cache dir)" - name: pip cache - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ${{ steps.pip-cache.outputs.dir }} key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }} @@ -51,9 +51,9 @@ jobs: backend: [tensorflow, jax, torch] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python 3.9 - uses: actions/setup-python@v1 + uses: actions/setup-python@v5 with: python-version: 3.9 - name: Get pip cache dir @@ -62,7 +62,7 @@ jobs: python -m pip install --upgrade pip setuptools echo "::set-output name=dir::$(pip cache dir)" - name: pip cache - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ${{ steps.pip-cache.outputs.dir }} key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }} @@ -81,9 +81,9 @@ jobs: name: Check the code format runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python 3.9 - uses: actions/setup-python@v1 + uses: actions/setup-python@v5 with: python-version: 3.9 - name: Get pip cache dir @@ -92,7 +92,7 @@ jobs: python -m pip install --upgrade pip setuptools echo "::set-output name=dir::$(pip cache dir)" - name: pip cache - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ${{ steps.pip-cache.outputs.dir }} key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }} diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 677a641658..8c1e5e2f6e 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -16,9 +16,9 @@ jobs: needs: [run-test-for-nightly] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.9 - name: Get pip cache dir @@ -27,7 +27,7 @@ jobs: python -m pip install --upgrade pip setuptools echo "::set-output name=dir::$(pip cache dir)" - name: pip cache - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ${{ steps.pip-cache.outputs.dir }} key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }} diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml index c3f6767350..24cec9d0f9 100644 --- a/.github/workflows/publish-to-pypi.yml +++ b/.github/workflows/publish-to-pypi.yml @@ -10,9 +10,9 @@ jobs: name: Build and publish to PyPI runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.9 - name: Get pip cache dir @@ -21,7 +21,7 @@ jobs: python -m pip install --upgrade pip setuptools echo "::set-output name=dir::$(pip cache dir)" - name: pip cache - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: ${{ steps.pip-cache.outputs.dir }} key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }} From b8045f9376f6b47aeea3f32d6afc2f92db3a220a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 14 Feb 2024 16:48:45 -0800 Subject: [PATCH 20/29] Bump the github-actions group with 1 update (#1438) Bumps the github-actions group with 1 update: [actions/cache](https://github.com/actions/cache). Updates `actions/cache` from 3 to 4 - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/cache dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/actions.yml | 6 +++--- .github/workflows/nightly.yml | 2 +- .github/workflows/publish-to-pypi.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/actions.yml b/.github/workflows/actions.yml index 1d77f5b9ae..16916c8fbd 100644 --- a/.github/workflows/actions.yml +++ b/.github/workflows/actions.yml @@ -26,7 +26,7 @@ jobs: python -m pip install --upgrade pip setuptools echo "::set-output name=dir::$(pip cache dir)" - name: pip cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ steps.pip-cache.outputs.dir }} key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }} @@ -62,7 +62,7 @@ jobs: python -m pip install --upgrade pip setuptools echo "::set-output name=dir::$(pip cache dir)" - name: pip cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ steps.pip-cache.outputs.dir }} key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }} @@ -92,7 +92,7 @@ jobs: python -m pip install --upgrade pip setuptools echo "::set-output name=dir::$(pip cache dir)" - name: pip cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ steps.pip-cache.outputs.dir }} key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }} diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 8c1e5e2f6e..686421fe00 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -27,7 +27,7 @@ jobs: python -m pip install --upgrade pip setuptools echo "::set-output name=dir::$(pip cache dir)" - name: pip cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ steps.pip-cache.outputs.dir }} key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }} diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml index 24cec9d0f9..186b7668da 100644 --- a/.github/workflows/publish-to-pypi.yml +++ b/.github/workflows/publish-to-pypi.yml @@ -21,7 +21,7 @@ jobs: python -m pip install --upgrade pip setuptools echo "::set-output name=dir::$(pip cache dir)" - name: pip cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ steps.pip-cache.outputs.dir }} key: ${{ runner.os }}-pip-${{ hashFiles('setup.py') }} From 70c57ccd0dfba8c6b6ef917ddd07e7254da68243 Mon Sep 17 00:00:00 2001 From: Tirth Patel Date: Thu, 15 Feb 2024 01:26:28 +0000 Subject: [PATCH 21/29] Add 7B presets for Mistral (#1436) * Refactor the checkpoints script * Add the 7B preset for Mistral * Upate the preset version [skip ci] * Fix the bug in Mistral preprocessor * Fix merge artifacts * Fix the tokenizer test [skip ci] * Mark smallest preset test as extra_large for now [skip ci] --- keras_nlp/models/mistral/mistral_backbone.py | 8 + .../models/mistral/mistral_backbone_test.py | 26 + keras_nlp/models/mistral/mistral_causal_lm.py | 6 + .../mistral/mistral_causal_lm_preprocessor.py | 14 + .../mistral_causal_lm_preprocessor_test.py | 11 + .../models/mistral/mistral_preprocessor.py | 20 +- .../mistral/mistral_preprocessor_test.py | 11 + keras_nlp/models/mistral/mistral_presets.py | 38 ++ keras_nlp/models/mistral/mistral_tokenizer.py | 8 + .../models/mistral/mistral_tokenizer_test.py | 20 + .../convert_mistral_checkpoints.py | 639 ++++++++---------- 11 files changed, 432 insertions(+), 369 deletions(-) create mode 100644 keras_nlp/models/mistral/mistral_presets.py diff --git a/keras_nlp/models/mistral/mistral_backbone.py b/keras_nlp/models/mistral/mistral_backbone.py index 375d3c54b1..3e2cfae148 100644 --- a/keras_nlp/models/mistral/mistral_backbone.py +++ b/keras_nlp/models/mistral/mistral_backbone.py @@ -11,6 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import copy + from keras_nlp.api_export import keras_nlp_export from keras_nlp.backend import keras from keras_nlp.backend import ops @@ -19,9 +21,11 @@ from keras_nlp.models.mistral.mistral_layer_norm import ( MistralLayerNormalization, ) +from keras_nlp.models.mistral.mistral_presets import backbone_presets from keras_nlp.models.mistral.mistral_transformer_decoder import ( MistralTransformerDecoder, ) +from keras_nlp.utils.python_utils import classproperty def _mistral_kernel_initializer(stddev=0.02): @@ -196,3 +200,7 @@ def get_config(self): } ) return config + + @classproperty + def presets(cls): + return copy.deepcopy(backbone_presets) diff --git a/keras_nlp/models/mistral/mistral_backbone_test.py b/keras_nlp/models/mistral/mistral_backbone_test.py index fc2b0a592b..fbfcb91124 100644 --- a/keras_nlp/models/mistral/mistral_backbone_test.py +++ b/keras_nlp/models/mistral/mistral_backbone_test.py @@ -54,3 +54,29 @@ def test_num_parameters(self): model = MistralBackbone(**self.init_kwargs) # Reference value calculated using the PyTorch model self.assertEqual(model.count_params(), 2704) + + @pytest.mark.extra_large + def test_smallest_preset(self): + self.run_preset_test( + cls=MistralBackbone, + preset="mistral_7b_en", + input_data={ + "token_ids": ops.array([[1, 1824, 349, 524, 11234, 28804]]), + "padding_mask": ops.ones((1, 6), dtype="int32"), + }, + expected_output_shape=(1, 6, 4096), + # The forward pass from a preset should be stable! + # Reference values computed using PyTorch HF model. + expected_partial_output=ops.array( + [-1.6875, 0.5117, -1.7188, 2.3125, -0.0996] + ), + ) + + @pytest.mark.extra_large + def test_all_presets(self): + for preset in MistralBackbone.presets: + self.run_preset_test( + cls=MistralBackbone, + preset=preset, + input_data=self.input_data, + ) diff --git a/keras_nlp/models/mistral/mistral_causal_lm.py b/keras_nlp/models/mistral/mistral_causal_lm.py index 22defbc456..3296bb9495 100644 --- a/keras_nlp/models/mistral/mistral_causal_lm.py +++ b/keras_nlp/models/mistral/mistral_causal_lm.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import copy from keras_nlp.api_export import keras_nlp_export from keras_nlp.backend import keras @@ -20,6 +21,7 @@ from keras_nlp.models.mistral.mistral_causal_lm_preprocessor import ( MistralCausalLMPreprocessor, ) +from keras_nlp.models.mistral.mistral_presets import backbone_presets from keras_nlp.utils.python_utils import classproperty @@ -211,3 +213,7 @@ def next(prompt, cache, index): "token_ids": token_ids, "padding_mask": padding_mask, } + + @classproperty + def presets(cls): + return copy.deepcopy(backbone_presets) diff --git a/keras_nlp/models/mistral/mistral_causal_lm_preprocessor.py b/keras_nlp/models/mistral/mistral_causal_lm_preprocessor.py index c8a0821733..893036cd58 100644 --- a/keras_nlp/models/mistral/mistral_causal_lm_preprocessor.py +++ b/keras_nlp/models/mistral/mistral_causal_lm_preprocessor.py @@ -131,6 +131,20 @@ def generate_preprocess( x, sequence_length=None, ): + """Covert strings to integer token input for generation. + + Similar to calling the layer for training, this method takes in strings + or tensor strings, tokenizes and packs the input, and computes a padding + mask masking all inputs not filled in with a padded value. + + Unlike calling the layer for training, this method does not compute + labels and will never append a `tokenizer.end_token_id` to the end of + the sequence (as generation is expected to continue at the end of the + inputted prompt). + """ + if not self.built: + self.build(None) + x = convert_inputs_to_list_of_tensor_segments(x)[0] x = self.tokenizer(x) token_ids, padding_mask = self.packer( diff --git a/keras_nlp/models/mistral/mistral_causal_lm_preprocessor_test.py b/keras_nlp/models/mistral/mistral_causal_lm_preprocessor_test.py index 80b55c02e5..dbacce37ce 100644 --- a/keras_nlp/models/mistral/mistral_causal_lm_preprocessor_test.py +++ b/keras_nlp/models/mistral/mistral_causal_lm_preprocessor_test.py @@ -14,6 +14,8 @@ import os +import pytest + from keras_nlp.models.mistral.mistral_causal_lm_preprocessor import ( MistralCausalLMPreprocessor, ) @@ -79,3 +81,12 @@ def test_generate_postprocess(self): preprocessor = MistralCausalLMPreprocessor(**self.init_kwargs) x = preprocessor.generate_postprocess(input_data) self.assertAllEqual(x, "the quick brown fox") + + @pytest.mark.extra_large + def test_all_presets(self): + for preset in MistralCausalLMPreprocessor.presets: + self.run_preset_test( + cls=MistralCausalLMPreprocessor, + preset=preset, + input_data=self.input_data, + ) diff --git a/keras_nlp/models/mistral/mistral_preprocessor.py b/keras_nlp/models/mistral/mistral_preprocessor.py index b96afb5ed8..38dc6da5b6 100644 --- a/keras_nlp/models/mistral/mistral_preprocessor.py +++ b/keras_nlp/models/mistral/mistral_preprocessor.py @@ -11,9 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import copy from keras_nlp.api_export import keras_nlp_export from keras_nlp.layers.preprocessing.start_end_packer import StartEndPacker +from keras_nlp.models.mistral.mistral_presets import backbone_presets from keras_nlp.models.mistral.mistral_tokenizer import MistralTokenizer from keras_nlp.models.preprocessor import Preprocessor from keras_nlp.utils.keras_utils import ( @@ -121,15 +123,21 @@ def __init__( ): super().__init__(**kwargs) self.tokenizer = tokenizer + self.packer = None + self.add_start_token = add_start_token + self.add_end_token = add_end_token + self.sequence_length = sequence_length + + def build(self, input_shape): + # Defer packer creation to `build()` so that we can be sure tokenizer + # assets have loaded when restoring a saved model. self.packer = StartEndPacker( start_value=self.tokenizer.start_token_id, end_value=self.tokenizer.end_token_id, - sequence_length=sequence_length, + sequence_length=self.sequence_length, return_padding_mask=True, ) - self.add_start_token = add_start_token - self.add_end_token = add_end_token - self.sequence_length = sequence_length + self.built = True def get_config(self): config = super().get_config() @@ -184,3 +192,7 @@ def sequence_length(self, value): @classproperty def tokenizer_cls(cls): return MistralTokenizer + + @classproperty + def presets(cls): + return copy.deepcopy(backbone_presets) diff --git a/keras_nlp/models/mistral/mistral_preprocessor_test.py b/keras_nlp/models/mistral/mistral_preprocessor_test.py index 47c0bc542c..e3ddd38f6f 100644 --- a/keras_nlp/models/mistral/mistral_preprocessor_test.py +++ b/keras_nlp/models/mistral/mistral_preprocessor_test.py @@ -14,6 +14,8 @@ import os +import pytest + from keras_nlp.models.mistral.mistral_preprocessor import MistralPreprocessor from keras_nlp.models.mistral.mistral_tokenizer import MistralTokenizer from keras_nlp.tests.test_case import TestCase @@ -57,3 +59,12 @@ def test_errors_for_2d_list_input(self): ambiguous_input = [["one", "two"], ["three", "four"]] with self.assertRaises(ValueError): preprocessor(ambiguous_input) + + @pytest.mark.extra_large + def test_all_presets(self): + for preset in MistralPreprocessor.presets: + self.run_preset_test( + cls=MistralPreprocessor, + preset=preset, + input_data=self.input_data, + ) diff --git a/keras_nlp/models/mistral/mistral_presets.py b/keras_nlp/models/mistral/mistral_presets.py new file mode 100644 index 0000000000..82a2ec44f6 --- /dev/null +++ b/keras_nlp/models/mistral/mistral_presets.py @@ -0,0 +1,38 @@ +# Copyright 2023 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Mistral model preset configurations.""" + +# Metadata for loading pretrained model weights. +backbone_presets = { + "mistral_7b_en": { + "metadata": { + "description": "Mistral 7B base model", + "params": 7241732096, + "official_name": "Mistral", + "path": "mistral", + "model_card": "https://github.com/mistralai/mistral-src/blob/main/README.md", + }, + "kaggle_handle": "kaggle://keras/mistral/keras/mistral_7b_en/3", + }, + "mistral_instruct_7b_en": { + "metadata": { + "description": "Mistral 7B instruct model", + "params": 7241732096, + "official_name": "Mistral", + "path": "mistral", + "model_card": "https://github.com/mistralai/mistral-src/blob/main/README.md", + }, + "kaggle_handle": "kaggle://keras/mistral/keras/mistral_instruct_7b_en/3", + }, +} diff --git a/keras_nlp/models/mistral/mistral_tokenizer.py b/keras_nlp/models/mistral/mistral_tokenizer.py index 12636f69f1..59a00d302f 100644 --- a/keras_nlp/models/mistral/mistral_tokenizer.py +++ b/keras_nlp/models/mistral/mistral_tokenizer.py @@ -11,8 +11,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import copy + from keras_nlp.api_export import keras_nlp_export +from keras_nlp.models.mistral.mistral_presets import backbone_presets from keras_nlp.tokenizers.sentence_piece_tokenizer import SentencePieceTokenizer +from keras_nlp.utils.python_utils import classproperty @keras_nlp_export("keras_nlp.models.MistralTokenizer") @@ -77,3 +81,7 @@ def set_proto(self, proto): else: self.start_token_id = None self.end_token_id = None + + @classproperty + def presets(cls): + return copy.deepcopy(backbone_presets) diff --git a/keras_nlp/models/mistral/mistral_tokenizer_test.py b/keras_nlp/models/mistral/mistral_tokenizer_test.py index ea9e04f67d..6b700bf711 100644 --- a/keras_nlp/models/mistral/mistral_tokenizer_test.py +++ b/keras_nlp/models/mistral/mistral_tokenizer_test.py @@ -14,6 +14,8 @@ import os +import pytest + from keras_nlp.models.mistral.mistral_tokenizer import MistralTokenizer from keras_nlp.tests.test_case import TestCase @@ -44,3 +46,21 @@ def test_errors_missing_special_tokens(self): self.get_test_data_dir(), "no_special_token_vocab.spm" ) ) + + @pytest.mark.large + def test_smallest_preset(self): + self.run_preset_test( + cls=MistralTokenizer, + preset="mistral_7b_en", + input_data=["The quick brown fox."], + expected_output=[[415, 2936, 9060, 285, 1142, 28723]], + ) + + @pytest.mark.extra_large + def test_all_presets(self): + for preset in MistralTokenizer.presets: + self.run_preset_test( + cls=MistralTokenizer, + preset=preset, + input_data=self.input_data, + ) diff --git a/tools/checkpoint_conversion/convert_mistral_checkpoints.py b/tools/checkpoint_conversion/convert_mistral_checkpoints.py index 3bc443d910..8e10089efd 100644 --- a/tools/checkpoint_conversion/convert_mistral_checkpoints.py +++ b/tools/checkpoint_conversion/convert_mistral_checkpoints.py @@ -11,433 +11,342 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import datetime +import gc import json +import os import pathlib -from dataclasses import dataclass -from pathlib import Path -from typing import Optional -from typing import Tuple - -import torch -from torch import nn - +import traceback + +import keras +import numpy as np +import requests +from absl import app +from absl import flags +from keras import ops +from transformers import AutoTokenizer +from transformers import MistralForCausalLM + +import keras_nlp from keras_nlp.models import MistralBackbone +from keras_nlp.models import MistralCausalLMPreprocessor +from keras_nlp.models import MistralTokenizer -MODEL_PATH = pathlib.Path("mistral-7B-v0.1") - -# Torch model taken from: -# https://github.com/mistralai/mistral-src/blob/147c4e68279b90eb61b19bdea44e16f5539d5a5d/one_file_ref.py - - -@dataclass -class ModelArgs: - dim: int - n_layers: int - head_dim: int - hidden_dim: int - n_heads: int - n_kv_heads: int - sliding_window: int - norm_eps: float - vocab_size: int - - max_batch_size: int = 0 - - -def repeat_kv(keys: torch.Tensor, values: torch.Tensor, repeats: int): - keys = torch.repeat_interleave(keys, repeats=repeats, dim=2) - values = torch.repeat_interleave(values, repeats=repeats, dim=2) - return keys, values - - -def _reshape_for_broadcast( - freqs_cis: torch.Tensor, x: torch.Tensor -) -> torch.Tensor: - """ - freqs_cis: complex - (seq_len, head_dim / 2) - x: complex - (bsz, seq_len, head_dim / 2) - """ - ndim = x.ndim - assert 1 < ndim - assert freqs_cis.shape == (x.shape[1], x.shape[-1]), ( - freqs_cis.shape, - (x.shape[1], x.shape[-1]), - ) - shape = [d if i == 1 or i == ndim - 1 else 1 for i, d in enumerate(x.shape)] - return freqs_cis.view(*shape) - - -def apply_rotary_emb( - xq: torch.Tensor, - xk: torch.Tensor, - freqs_cis: torch.Tensor, -) -> Tuple[torch.Tensor, torch.Tensor]: - xq_ = torch.view_as_complex(xq.float().reshape(*xq.shape[:-1], -1, 2)) - xk_ = torch.view_as_complex(xk.float().reshape(*xk.shape[:-1], -1, 2)) - freqs_cis = _reshape_for_broadcast(freqs_cis, xq_) - xq_out = torch.view_as_real(xq_ * freqs_cis).flatten(3) - xk_out = torch.view_as_real(xk_ * freqs_cis).flatten(3) - return xq_out.type_as(xq), xk_out.type_as(xk) - - -class Attention(nn.Module): - def __init__(self, args: ModelArgs): - super().__init__() - self.args = args - - self.n_heads: int = args.n_heads - self.n_kv_heads: int = args.n_kv_heads - - self.repeats = self.n_heads // self.n_kv_heads - self.sliding_window = self.args.sliding_window - - self.scale = self.args.head_dim**-0.5 - - self.wq = nn.Linear(args.dim, args.n_heads * args.head_dim, bias=False) - self.wk = nn.Linear( - args.dim, args.n_kv_heads * args.head_dim, bias=False - ) - self.wv = nn.Linear( - args.dim, args.n_kv_heads * args.head_dim, bias=False - ) - self.wo = nn.Linear(args.n_heads * args.head_dim, args.dim, bias=False) - self.cache_k = torch.empty( - ( - args.max_batch_size, - args.sliding_window, - self.n_kv_heads, - self.args.head_dim, - ), - dtype=torch.float16, - ) - self.cache_v = torch.empty( - ( - args.max_batch_size, - args.sliding_window, - self.n_kv_heads, - self.args.head_dim, - ), - dtype=torch.float16, - ) - - def forward( - self, - x: torch.Tensor, - freqs_cis: torch.Tensor, - positions: torch.Tensor, - mask: Optional[torch.Tensor], - ) -> torch.Tensor: - bsz, seqlen, _ = x.shape - - xq, xk, xv = self.wq(x), self.wk(x), self.wv(x) - xq = xq.view(bsz, seqlen, self.n_heads, self.args.head_dim) - xk = xk.view(bsz, seqlen, self.n_kv_heads, self.args.head_dim) - xv = xv.view(bsz, seqlen, self.n_kv_heads, self.args.head_dim) - xq, xk = apply_rotary_emb(xq, xk, freqs_cis=freqs_cis) - - # The cache is a rotating buffer - scatter_pos = (positions[-self.sliding_window :] % self.sliding_window)[ - None, :, None, None - ] - scatter_pos = scatter_pos.repeat( - bsz, 1, self.n_kv_heads, self.args.head_dim - ) - self.cache_k[:bsz].scatter_( - dim=1, - index=scatter_pos, - src=xk[:, -self.sliding_window :].to(self.cache_k.dtype), - ) - self.cache_v[:bsz].scatter_( - dim=1, - index=scatter_pos, - src=xv[:, -self.sliding_window :].to(self.cache_v.dtype), - ) - - if positions.shape[0] > 1: - # prefill - key, value = repeat_kv(xk, xv, self.repeats) - else: - cur_pos = positions[-1].item() + 1 - key, value = repeat_kv( - self.cache_k[:bsz, :cur_pos, ...].to(xk.dtype), - self.cache_v[:bsz, :cur_pos, ...].to(xv.dtype), - self.repeats, - ) - - query = xq.transpose(1, 2) - key = key.transpose(1, 2) - value = value.transpose(1, 2) - # scores : [bsz, n_heads, seqlen | 1, seqlen] - scores = torch.matmul(query, key.transpose(2, 3)) * self.scale - - if mask is not None: - scores += mask[None, None, ...] - - scores = scores.float() - scores = nn.functional.softmax(scores, dim=-1).type_as(query) - output = torch.matmul( - scores, value - ) # (bs, n_local_heads, slen, head_dim) - output = output.transpose(1, 2).contiguous().view(bsz, seqlen, -1) - return self.wo(output) - - -class FeedForward(nn.Module): - def __init__(self, args: ModelArgs): - super().__init__() - - self.w1 = nn.Linear(args.dim, args.hidden_dim, bias=False) - self.w2 = nn.Linear(args.hidden_dim, args.dim, bias=False) - self.w3 = nn.Linear(args.dim, args.hidden_dim, bias=False) - - def forward(self, x) -> torch.Tensor: - return self.w2(nn.functional.silu(self.w1(x)) * self.w3(x)) - - -class RMSNorm(torch.nn.Module): - def __init__(self, dim: int, eps: float = 1e-6): - super().__init__() - self.eps = eps - self.weight = nn.Parameter(torch.ones(dim)) - - def _norm(self, x): - return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) - - def forward(self, x): - output = self._norm(x.float()).type_as(x) - return output * self.weight - - -class TransformerBlock(nn.Module): - def __init__(self, args: ModelArgs): - super().__init__() - self.n_heads = args.n_heads - self.dim = args.dim - self.attention = Attention(args) - self.feed_forward = FeedForward(args=args) - self.attention_norm = RMSNorm(args.dim, eps=args.norm_eps) - self.ffn_norm = RMSNorm(args.dim, eps=args.norm_eps) - self.args = args - - def forward( - self, - x: torch.Tensor, - freqs_cis: torch.Tensor, - positions: torch.Tensor, - mask: Optional[torch.Tensor], - ) -> torch.Tensor: - r = self.attention.forward( - self.attention_norm(x), freqs_cis, positions, mask - ) - h = x + r - r = self.feed_forward.forward(self.ffn_norm(h)) - out = h + r - return out - - -def precompute_freqs_cis( - dim: int, end: int, theta: float = 10000.0 -) -> torch.Tensor: - freqs = 1.0 / ( - theta ** (torch.arange(0, dim, 2)[: (dim // 2)].float() / dim) - ) - t = torch.arange(end, device=freqs.device) # type: ignore - freqs = torch.outer(t, freqs).float() # type: ignore - return torch.polar(torch.ones_like(freqs), freqs) # complex64 +PRESET_MAP = { + "mistral_7b_en": "mistralai/Mistral-7B-v0.1", + "mistral_instruct_7b_en": "mistralai/Mistral-7B-Instruct-v0.1", +} +FLAGS = flags.FLAGS +flags.DEFINE_string( + "preset", None, f'Must be one of {",".join(PRESET_MAP.keys())}' +) -class TorchTransformer(nn.Module): - def __init__(self, args: ModelArgs): - super().__init__() - self.args = args - self.vocab_size = args.vocab_size - self.n_layers = args.n_layers - assert self.vocab_size > 0 - self.tok_embeddings = nn.Embedding(args.vocab_size, args.dim) +def convert_checkpoints(keras_nlp_model, hf_model): + config = hf_model.config - self.layers = torch.nn.ModuleList( - [TransformerBlock(args=args) for _ in range(args.n_layers)] - ) - - self.norm = RMSNorm(args.dim, eps=args.norm_eps) - - self.output = nn.Linear(args.dim, args.vocab_size, bias=False) - - self.freqs_cis = precompute_freqs_cis(self.args.head_dim, 128_000) - - def forward( - self, - input_ids: torch.Tensor, - positions: torch.Tensor, - ): - h = self.tok_embeddings(input_ids) - freqs_cis = self.freqs_cis[positions] - - mask: Optional[torch.Tensor] = None - if input_ids.shape[1] > 1: - seqlen = input_ids.shape[1] - tensor = torch.full( - (seqlen, seqlen), - dtype=h.dtype, - fill_value=1, - device=h.device, - ) - mask = torch.tril(tensor, diagonal=0).to(h.dtype) - # make the mask banded to account for sliding window - mask = torch.triu(mask, diagonal=-self.args.sliding_window) - mask = torch.log(mask) - - for layer in self.layers: - h = layer(h, freqs_cis, positions, mask) - - return self.output(self.norm(h)).float() - - @staticmethod - def from_folder( - folder: Path, max_batch_size: int = 1, device="cpu", dtype=torch.float16 - ): - with open(folder / "params.json", "r") as f: - model_args = ModelArgs(**json.loads(f.read())) - model_args.max_batch_size = max_batch_size - model = TorchTransformer(model_args).to(device=device, dtype=dtype) - loaded = torch.load(folder / "consolidated.00.pth") - model.load_state_dict(loaded) - return model - - -def port_weights( - model_k3: MistralBackbone, model_torch: TorchTransformer, params: ModelArgs -): - model_k3.get_layer("token_embedding").embeddings.assign( - model_torch.tok_embeddings.weight.detach().cpu().numpy() + keras_nlp_model.token_embedding.embeddings.assign( + hf_model.model.embed_tokens.weight.detach().cpu().numpy() ) - for i in range(model_k3.num_layers): - model_k3.get_layer( - f"transformer_layer_{i}" - )._self_attention_layer._key_dense.set_weights( + for i in range(keras_nlp_model.num_layers): + keras_nlp_model.transformer_layers[ + i + ]._self_attention_layer._key_dense.set_weights( [ - model_torch.layers[i] - .attention.wk.weight.T.reshape( - params.dim, params.n_kv_heads, params.head_dim + hf_model.model.layers[i] + .self_attn.k_proj.weight.T.reshape( + config.hidden_size, + config.num_key_value_heads, + config.hidden_size // config.num_attention_heads, ) .detach() .cpu() .numpy() ] ) - model_k3.get_layer( - f"transformer_layer_{i}" - )._self_attention_layer._query_dense.set_weights( + keras_nlp_model.transformer_layers[ + i + ]._self_attention_layer._query_dense.set_weights( [ - model_torch.layers[i] - .attention.wq.weight.T.reshape( - params.dim, params.n_heads, params.head_dim + hf_model.model.layers[i] + .self_attn.q_proj.weight.T.reshape( + config.hidden_size, + config.num_attention_heads, + config.hidden_size // config.num_attention_heads, ) .detach() .cpu() .numpy() ] ) - model_k3.get_layer( - f"transformer_layer_{i}" - )._self_attention_layer._value_dense.set_weights( + keras_nlp_model.transformer_layers[ + i + ]._self_attention_layer._value_dense.set_weights( [ - model_torch.layers[i] - .attention.wv.weight.T.reshape( - params.dim, params.n_kv_heads, params.head_dim + hf_model.model.layers[i] + .self_attn.v_proj.weight.T.reshape( + config.hidden_size, + config.num_key_value_heads, + config.hidden_size // config.num_attention_heads, ) .detach() .cpu() .numpy() ] ) - model_k3.get_layer( - f"transformer_layer_{i}" - )._self_attention_layer._output_dense.set_weights( + keras_nlp_model.transformer_layers[ + i + ]._self_attention_layer._output_dense.set_weights( [ - model_torch.layers[i] - .attention.wo.weight.T.reshape( - params.n_heads, params.head_dim, params.dim + hf_model.model.layers[i] + .self_attn.o_proj.weight.T.reshape( + config.num_attention_heads, + config.hidden_size // config.num_attention_heads, + config.hidden_size, ) .detach() .cpu() .numpy() ] ) - model_k3.get_layer( - f"transformer_layer_{i}" - )._self_attention_layernorm.set_weights( - [model_torch.layers[i].attention_norm.weight.detach().cpu().numpy()] + keras_nlp_model.transformer_layers[ + i + ]._self_attention_layernorm.set_weights( + [ + hf_model.model.layers[i] + .input_layernorm.weight.detach() + .cpu() + .numpy() + ] ) - model_k3.get_layer( - f"transformer_layer_{i}" - )._feedforward_intermediate_dense.set_weights( + keras_nlp_model.transformer_layers[ + i + ]._feedforward_intermediate_dense.set_weights( [ - model_torch.layers[i] - .feed_forward.w3.weight.T.detach() + hf_model.model.layers[i] + .mlp.up_proj.weight.T.detach() .cpu() .numpy() ] ) - model_k3.get_layer( - f"transformer_layer_{i}" - )._feedforward_output_dense.set_weights( + keras_nlp_model.transformer_layers[ + i + ]._feedforward_output_dense.set_weights( [ - model_torch.layers[i] - .feed_forward.w2.weight.T.detach() + hf_model.model.layers[i] + .mlp.down_proj.weight.T.detach() .cpu() .numpy() ] ) - model_k3.get_layer( - f"transformer_layer_{i}" - )._feedforward_gate_dense.set_weights( + keras_nlp_model.transformer_layers[ + i + ]._feedforward_gate_dense.set_weights( [ - model_torch.layers[i] - .feed_forward.w1.weight.T.detach() + hf_model.model.layers[i] + .mlp.gate_proj.weight.T.detach() .cpu() .numpy() ] ) - model_k3.get_layer( - f"transformer_layer_{i}" - )._feedforward_layernorm.set_weights( - [model_torch.layers[i].ffn_norm.weight.detach().cpu().numpy()] + keras_nlp_model.transformer_layers[ + i + ]._feedforward_layernorm.set_weights( + [ + hf_model.model.layers[i] + .post_attention_layernorm.weight.detach() + .cpu() + .numpy() + ] ) - model_k3.get_layer("sequence_output_layernorm").set_weights( - [model_torch.norm.weight.detach().cpu().numpy()] + keras_nlp_model.layer_norm.set_weights( + [hf_model.model.norm.weight.detach().cpu().numpy()] ) - model_k3.get_layer("token_embedding").reverse_embeddings.assign( - model_torch.output.weight.T.detach().cpu().numpy() + keras_nlp_model.token_embedding.reverse_embeddings.assign( + hf_model.lm_head.weight.T.detach().cpu().numpy() ) -if __name__ == "__main__": - with open(MODEL_PATH / "params.json", "r") as params_file: - params = ModelArgs(**json.load(params_file)) +def test_model( + keras_nlp_model, keras_nlp_tokenizer, hf_model, hf_model_tokenizer +): + # First, test that the number of parameters match + keras_nlp_params = keras_nlp_model.count_params() + hf_params = hf_model.num_parameters() + assert keras_nlp_params == hf_params + + # Test the outputs of both the models + hf_outputs = hf_model( + **hf_model_tokenizer(["What is Keras?"], return_tensors="pt") + ) + hf_output_logits = hf_outputs.logits.detach().cpu().numpy() - model_torch = TorchTransformer.from_folder( - MODEL_PATH, device="cpu", dtype=torch.float16 + keras_nlp_preprocessor = MistralCausalLMPreprocessor(keras_nlp_tokenizer) + keras_nlp_output = keras_nlp_model( + keras_nlp_preprocessor(["What is Keras?"], sequence_length=6)[0] ) - print("Torch model loaded") - model_k3 = MistralBackbone( - vocabulary_size=32000, - hidden_dim=4096, - num_layers=32, - num_query_heads=32, - num_key_value_heads=8, - intermediate_dim=14336, - sliding_window=4096, - layer_norm_epsilon=1e-6, - dtype="float16", + keras_nlp_logits = keras_nlp_model.token_embedding( + keras_nlp_output, reverse=True ) - print("Keras 3 model loaded.") + keras_nlp_logits = ops.convert_to_numpy(keras_nlp_logits) + + # High tolerence since bfloat16 is used as the default dtype for Mistral + try: + np.testing.assert_allclose( + keras_nlp_logits, hf_output_logits, atol=1e-4 + ) + except AssertionError as err: + print("\n") + print(traceback.format_exc()) + print(err.args[0]) + print("\n") + + +def test_tokenizer(keras_nlp_tokenizer, hf_tokenizer): + hf_output = hf_tokenizer(["What is Keras?"], return_tensors="pt") + hf_output = hf_output["input_ids"].detach().cpu().numpy() + keras_nlp_preprocessor = MistralCausalLMPreprocessor(keras_nlp_tokenizer) + keras_nlp_output = keras_nlp_preprocessor( + ["What is Keras?"], sequence_length=6 + ) + keras_nlp_output = ops.convert_to_numpy(keras_nlp_output[0]["token_ids"]) + + np.testing.assert_equal(keras_nlp_output, hf_output) - port_weights(model_k3, model_torch, params) - print("Weight transfer done.") - model_k3.save_weights("mistral_7b.weights.h5") - print("Weights saved.") +def main(_): + # === Get the preset name === + if FLAGS.preset not in PRESET_MAP.keys(): + raise ValueError( + f"Invalid preset {FLAGS.preset}. Must be one " + f"of {','.join(PRESET_MAP.keys())}" + ) + preset = FLAGS.preset + hf_preset = PRESET_MAP[preset] + + # === Create the save directories === + model_dir = pathlib.Path(__file__).parent / f"{preset}" + tokenizer_dir = model_dir / "assets" / "tokenizer" + if not model_dir.exists(): + os.makedirs(model_dir) + if not tokenizer_dir.exists(): + os.makedirs(tokenizer_dir) + + # === Load the Huggingface model === + hf_model = MistralForCausalLM.from_pretrained(hf_preset) + hf_tokenizer = AutoTokenizer.from_pretrained(hf_preset) + hf_model.eval() + print("\n-> Huggingface model and tokenizer loaded") + + # === Load the KerasNLP model === + keras_nlp_config = dict( + vocabulary_size=hf_model.config.vocab_size, + hidden_dim=hf_model.config.hidden_size, + num_layers=hf_model.config.num_hidden_layers, + num_query_heads=hf_model.config.num_attention_heads, + num_key_value_heads=hf_model.config.num_key_value_heads, + intermediate_dim=hf_model.config.intermediate_size, + sliding_window=hf_model.config.sliding_window, + layer_norm_epsilon=hf_model.config.rms_norm_eps, + rope_max_wavelength=hf_model.config.rope_theta, + dtype="float32", + ) + keras_nlp_model = MistralBackbone(**keras_nlp_config) + + # === Download the tokenizer from Huggingface model card === + spm_path = ( + f"https://huggingface.co/{hf_preset}/resolve/main/tokenizer.model" + ) + response = requests.get(spm_path) + if not response.ok: + raise ValueError(f"Couldn't fetch {preset}'s tokenizer.") + tokenizer_path = tokenizer_dir / "vocabulary.spm" + with open(tokenizer_path, "wb") as tokenizer_file: + tokenizer_file.write(response.content) + keras_nlp_tokenizer = MistralTokenizer(str(tokenizer_path.absolute())) + print("\n-> Keras 3 model and tokenizer loaded.") + + # === Port the weights === + convert_checkpoints(keras_nlp_model, hf_model) + print("\n-> Weight transfer done.") + + # === Check that the models and tokenizers outputs match === + test_tokenizer(keras_nlp_tokenizer, hf_tokenizer) + test_model(keras_nlp_model, keras_nlp_tokenizer, hf_model, hf_tokenizer) + print("\n-> Tests passed!") + + # === Save the model weights in float32 format === + keras_nlp_model.save_weights( + str((model_dir / "model.weights.h5").absolute()) + ) + print("\n-> Saved the model weights in float16") + + del keras_nlp_model, hf_model + gc.collect() + + keras_nlp_config["dtype"] = "float16" + + # === Save the weights again in float16 === + keras_nlp_model = MistralBackbone(**keras_nlp_config) + keras_nlp_model.load_weights( + str((model_dir / "model.weights.h5").absolute()) + ) + keras_nlp_model.save_weights( + str((model_dir / "model.weights.h5").absolute()) + ) + print("-> Saved the model weights in float16") + + # === Save the model config === + keras_nlp_config["dtype"] = "bfloat16" + model_config = { + "module": "keras_nlp.src.models.mistral.mistral_backbone", + "class_name": "MistralBackbone", + "config": {**keras_nlp_config}, + "registered_name": "keras_nlp>MistralBackbone", + "assets": [], + "weights": "model.weights.h5", + } + model_config_json = json.dumps(model_config) + with open(model_dir / "config.json", "w") as model_config_file: + model_config_file.write(model_config_json) + print("\n-> Saved model config") + + # === Save the tokenizer config === + tokenizer_config = { + "module": "keras_nlp.src.models.mistral.Mistral_tokenizer", + "class_name": "MistralTokenizer", + "config": { + "name": "mistral_tokenizer", + "trainable": True, + "dtype": "int32", + "proto": None, + "sequence_length": None, + }, + "registered_name": "keras_nlp>MistralTokenizer", + "assets": ["assets/tokenizer/vocabulary.spm"], + "weights": None, + } + tokenizer_config_json = json.dumps(tokenizer_config) + with open(model_dir / "tokenizer.json", "w") as tokenizer_config_file: + tokenizer_config_file.write(tokenizer_config_json) + print("\n-> Saved tokenizer config") + + # === Save metadata === + metadata_config = { + "keras_version": keras.__version__, + "keras_nlp_version": keras_nlp.__version__, + "parameter_count": keras_nlp_model.count_params(), + "date_saved": datetime.datetime.utcnow().strftime("%Y-%m-%d@%H:%M:%S"), + } + metadata_config_json = json.dumps(metadata_config) + with open(model_dir / "metadata.json", "w") as metadata_config_file: + metadata_config_file.write(metadata_config_json) + print("\n-> Saved metadata") + + +if __name__ == "__main__": + flags.mark_flag_as_required("preset") + app.run(main) From 8097631f6441a6635b4705bcc0addcecb993189f Mon Sep 17 00:00:00 2001 From: Divyashree Sreepathihalli Date: Fri, 16 Feb 2024 09:52:29 -0800 Subject: [PATCH 22/29] Update byte_pair_tokenizer.py to close merges file properly (#1440) --- keras_nlp/tokenizers/byte_pair_tokenizer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/keras_nlp/tokenizers/byte_pair_tokenizer.py b/keras_nlp/tokenizers/byte_pair_tokenizer.py index 55992a16d7..902af812e9 100644 --- a/keras_nlp/tokenizers/byte_pair_tokenizer.py +++ b/keras_nlp/tokenizers/byte_pair_tokenizer.py @@ -350,7 +350,8 @@ def set_vocabulary_and_merges(self, vocabulary, merges): f"`type(vocabulary)={type(vocabulary)}`." ) if isinstance(merges, str): - self.merges = [bp.rstrip() for bp in open(merges, encoding="utf-8")] + with open(merges, encoding="utf-8") as f: + self.merges = [bp.rstrip() for bp in f] elif isinstance(merges, Iterable): self.merges = list(merges) else: From abdccb73b77b8d569f884b8c7ce0f3b124ec6a16 Mon Sep 17 00:00:00 2001 From: Matt Watson <1389937+mattdangerw@users.noreply.github.com> Date: Fri, 16 Feb 2024 13:44:23 -0800 Subject: [PATCH 23/29] bump version to 0.8 (#1441) --- keras_nlp/version_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keras_nlp/version_utils.py b/keras_nlp/version_utils.py index 15fede3a08..4d6a8186d4 100644 --- a/keras_nlp/version_utils.py +++ b/keras_nlp/version_utils.py @@ -15,7 +15,7 @@ from keras_nlp.api_export import keras_nlp_export # Unique source of truth for the version number. -__version__ = "0.7.0" +__version__ = "0.8.0" @keras_nlp_export("keras_nlp.version") From 092cbde2085d8e5c892d9e518d49b021c175e1e0 Mon Sep 17 00:00:00 2001 From: Matt Watson <1389937+mattdangerw@users.noreply.github.com> Date: Tue, 20 Feb 2024 15:56:13 -0800 Subject: [PATCH 24/29] Update our sampler documentation to reflect usage (#1444) We will update our samplers in the near future to push the backend specific compilation details out: https://github.com/keras-team/keras-nlp/pull/1425 Also in general, we want our documentation to reflect the main usage of our classes, which is using them with Seq2SeqLM and CausalLM classes. So with that in mind, this updates our sampler docs to show the practical usage of the sampling classes with our modeling classes. For the base class, we show the main use case of overriding the `get_next_token()` function. --- keras_nlp/samplers/beam_sampler.py | 59 +++--------------- keras_nlp/samplers/contrastive_sampler.py | 35 +++-------- keras_nlp/samplers/greedy_sampler.py | 34 +++------- keras_nlp/samplers/random_sampler.py | 27 +++----- keras_nlp/samplers/sampler.py | 76 +++++++---------------- keras_nlp/samplers/top_k_sampler.py | 27 +++----- keras_nlp/samplers/top_p_sampler.py | 27 +++----- 7 files changed, 77 insertions(+), 208 deletions(-) diff --git a/keras_nlp/samplers/beam_sampler.py b/keras_nlp/samplers/beam_sampler.py index 9562f95d14..87948439a8 100644 --- a/keras_nlp/samplers/beam_sampler.py +++ b/keras_nlp/samplers/beam_sampler.py @@ -18,11 +18,8 @@ from keras_nlp.api_export import keras_nlp_export from keras_nlp.backend import ops from keras_nlp.samplers.sampler import Sampler -from keras_nlp.samplers.sampler import call_args_docstring -from keras_nlp.utils.python_utils import format_docstring -@format_docstring(call_args=call_args_docstring) @keras_nlp_export("keras_nlp.samplers.BeamSampler") class BeamSampler(Sampler): """Beam Sampler class. @@ -42,55 +39,17 @@ class BeamSampler(Sampler): {{call_args}} Examples: - Return only the beam with the highest accumulated probability. ```python - # Use a simple alphabet of lowercase characters with ids in range [0, 25]. - int_lookup = {i: chr(i + ord('a')) for i in range(26)} - char_lookup = {v: k for k, v in int_lookup.items()} - batch_size, length, vocab_size = 1, 12, len(int_lookup) - - def next(prompt, cache, index): - prompt_batch_size = tf.shape(prompt)[0] - hidden_states = np.ones((prompt_batch_size, 10)) - # A uniform distribution over our alphabet. - logits = np.ones((prompt_batch_size, vocab_size)) - return logits, hidden_states, cache - - output = keras_nlp.samplers.BeamSampler()( - next=next, - prompt=np.full((batch_size, length), char_lookup["z"], dtype="int32"), - index=5, - ) - print(["".join([int_lookup[i] for i in s]) for s in output.numpy()]) - # >>> ['zzzzzeeeeeee'] - ``` + causal_lm = keras_nlp.models.GPT2CausalLM.from_preset("gpt2_base_en") - Return all beams and their probabilities. - ```python - # Use a simple alphabet of lowercase characters with ids in range [0, 25]. - int_lookup = {i: chr(i + ord('a')) for i in range(26)} - char_lookup = {v: k for k, v in int_lookup.items()} - batch_size, length, vocab_size = 1, 8, len(int_lookup) - - def next(prompt, cache, index): - prompt_batch_size = tf.shape(prompt)[0] - hidden_states = np.ones((prompt_batch_size, 10)) - # A uniform distribution over our alphabet. - logits = np.ones((batch_size, vocab_size)) - return logits, hidden_states, cache - - beams, probs = keras_nlp.samplers.BeamSampler(return_all_beams=True)( - next=next, - prompt=np.full((batch_size, length,), char_lookup['z'], dtype="int32"), - index=5, - ) - - print(beams.shape) - # >>> (1, 5, 8) - print(probs.shape) - # >>> (1, 5) - print(["".join([int_lookup[i] for i in s]) for s in beams[0].numpy()]) - # >>> ['zzzzzeee', 'zzzzzeed', 'zzzzzeec', 'zzzzzeea', 'zzzzzeeb'] + # Pass by name to compile. + causal_lm.compile(sampler="beam") + causal_lm.generate(["Keras is a"]) + + # Pass by object to compile. + sampler = keras_nlp.samplers.BeamSampler(num_beams=5) + causal_lm.compile(sampler=sampler) + causal_lm.generate(["Keras is a"]) ``` """ diff --git a/keras_nlp/samplers/contrastive_sampler.py b/keras_nlp/samplers/contrastive_sampler.py index bac65bcfbe..8b3d52d9a5 100644 --- a/keras_nlp/samplers/contrastive_sampler.py +++ b/keras_nlp/samplers/contrastive_sampler.py @@ -17,11 +17,8 @@ from keras_nlp.api_export import keras_nlp_export from keras_nlp.backend import ops from keras_nlp.samplers.sampler import Sampler -from keras_nlp.samplers.sampler import call_args_docstring -from keras_nlp.utils.python_utils import format_docstring -@format_docstring(call_args=call_args_docstring) @keras_nlp_export("keras_nlp.samplers.ContrastiveSampler") class ContrastiveSampler(Sampler): """Contrastive Sampler class. @@ -44,28 +41,16 @@ class ContrastiveSampler(Sampler): Examples: ```python - # Use a simple alphabet of lowercase characters to [0, 26). - int_lookup = {i: chr(i + ord("a")) for i in range(26)} - char_lookup = {v: k for k, v in int_lookup.items()} - batch_size, length, vocab_size = 1, 12, len(int_lookup) - hidden_size = 5 - index = 5 - - def next(prompt, cache, index): - prompt_batch_size = tf.shape(prompt)[0] - hidden_states = np.ones((prompt_batch_size, hidden_size)) - # A uniform distribution over our alphabet. - logits = np.ones((prompt_batch_size, vocab_size)) - return logits, hidden_states, cache - - output = keras_nlp.samplers.ContrastiveSampler()( - next=next, - prompt=np.full((batch_size, length), char_lookup["z"], dtype="int32"), - index=index, - hidden_states=np.ones([batch_size, index, hidden_size]), - ) - print(["".join([int_lookup[i] for i in s]) for s in output.numpy()]) - # >>> "zzzzzeeeeeee" + causal_lm = keras_nlp.models.GPT2CausalLM.from_preset("gpt2_base_en") + + # Pass by name to compile. + causal_lm.compile(sampler="contrastive") + causal_lm.generate(["Keras is a"]) + + # Pass by object to compile. + sampler = keras_nlp.samplers.ContrastiveSampler(k=5) + causal_lm.compile(sampler=sampler) + causal_lm.generate(["Keras is a"]) ``` """ diff --git a/keras_nlp/samplers/greedy_sampler.py b/keras_nlp/samplers/greedy_sampler.py index 8e178b7468..ee8a6ecc2d 100644 --- a/keras_nlp/samplers/greedy_sampler.py +++ b/keras_nlp/samplers/greedy_sampler.py @@ -15,11 +15,8 @@ from keras_nlp.api_export import keras_nlp_export from keras_nlp.backend import ops from keras_nlp.samplers.sampler import Sampler -from keras_nlp.samplers.sampler import call_args_docstring -from keras_nlp.utils.python_utils import format_docstring -@format_docstring(call_args=call_args_docstring) @keras_nlp_export("keras_nlp.samplers.GreedySampler") class GreedySampler(Sampler): """Greedy sampler class. @@ -27,29 +24,18 @@ class GreedySampler(Sampler): This sampler is implemented on greedy search, i.e., always picking up the token of the largest probability as the next token. - Call arguments: - {{call_args}} - Examples: ```python - # Use a simple alphabet of lowercase characters with ids in range [0, 25]. - int_lookup = {i: chr(i + ord('a')) for i in range(26)} - char_lookup = {v: k for k, v in int_lookup.items()} - batch_size, length, vocab_size = 1, 12, len(int_lookup) - - def next(prompt, cache, index): - hidden_states = np.ones((batch_size, 10)) - # A uniform distribution over our alphabet. - logits = np.ones((batch_size, vocab_size)) - return logits, hidden_states, cache - - output = keras_nlp.samplers.GreedySampler()( - next=next, - prompt=np.full((batch_size, length,), char_lookup['z'], dtype="int32"), - index=5, - ) - print(["".join([int_lookup[i] for i in s]) for s in output.numpy()]) - # >>> ['zzzzzaaaaaaa'] + causal_lm = keras_nlp.models.GPT2CausalLM.from_preset("gpt2_base_en") + + # Pass by name to compile. + causal_lm.compile(sampler="greedy") + causal_lm.generate(["Keras is a"]) + + # Pass by object to compile. + sampler = keras_nlp.samplers.GreedySampler() + causal_lm.compile(sampler=sampler) + causal_lm.generate(["Keras is a"]) ``` """ diff --git a/keras_nlp/samplers/random_sampler.py b/keras_nlp/samplers/random_sampler.py index b922d29b2a..1ff39c9f9b 100644 --- a/keras_nlp/samplers/random_sampler.py +++ b/keras_nlp/samplers/random_sampler.py @@ -16,11 +16,8 @@ from keras_nlp.backend import ops from keras_nlp.backend import random from keras_nlp.samplers.sampler import Sampler -from keras_nlp.samplers.sampler import call_args_docstring -from keras_nlp.utils.python_utils import format_docstring -@format_docstring(call_args=call_args_docstring) @keras_nlp_export("keras_nlp.samplers.RandomSampler") class RandomSampler(Sampler): """Random Sampler class. @@ -37,24 +34,16 @@ class RandomSampler(Sampler): Examples: ```python - # Use a simple alphabet of lowercase characters with ids in range [0, 25]. - int_lookup = {i: chr(i + ord('a')) for i in range(26)} - char_lookup = {v: k for k, v in int_lookup.items()} - batch_size, length, vocab_size = 1, 12, len(int_lookup) + causal_lm = keras_nlp.models.GPT2CausalLM.from_preset("gpt2_base_en") - def next(prompt, state, index): - hidden_states = np.ones((batch_size, 10)) - # A uniform distribution over our alphabet. - logits = np.ones((batch_size, vocab_size)) - return logits, hidden_states, state + # Pass by name to compile. + causal_lm.compile(sampler="random") + causal_lm.generate(["Keras is a"]) - output = keras_nlp.samplers.RandomSampler()( - next=next, - prompt=np.full((batch_size, length,), char_lookup['z'], dtype="int32"), - index=5, - ) - print(["".join([int_lookup[i] for i in s]) for s in output.numpy()]) - # >>> ['zzzzzcpnjqij'] + # Pass by object to compile. + sampler = keras_nlp.samplers.RandomSampler(temperature=0.7) + causal_lm.compile(sampler=sampler) + causal_lm.generate(["Keras is a"]) ``` """ diff --git a/keras_nlp/samplers/sampler.py b/keras_nlp/samplers/sampler.py index e28fbe9d6e..2101c9277d 100644 --- a/keras_nlp/samplers/sampler.py +++ b/keras_nlp/samplers/sampler.py @@ -17,33 +17,8 @@ from keras_nlp.backend import keras from keras_nlp.backend import ops from keras_nlp.backend import random -from keras_nlp.utils.python_utils import format_docstring - -call_args_docstring = """next: A function which takes in the - `prompt, cache, index` of the current generation loop, and outputs - a tuple `(logits, hidden_states, cache)` with `logits` being the - logits of next token, `hidden_states` being the representation of - the next token, and `cache` for next iteration. - prompt: A 2D integer tensor with shape `(batch_size, max_length)`. This - tensor will be iteratively updated column by column with new sampled - values, starting at `index`. - cache: Optional. A tensor or nested structure of tensors that will be - updated by each call to `next`. This can be used to cache - computations from early iterations of the generative loop. - index: Optional. The first index of `prompt` to start sampling at. - Usually this is set as the length of the shortest non-padded - sequence in `prompt`. - mask: Optional. A 2D integer tensor with the same shape as `prompt`. - Locations which are `True` in the mask are never updated during - sampling. Usually used to mark all locations in the dense prompt - tensor which were present in a user input. - end_token_id: Optional. The token marking the end of the sequence. If - specified, sampling will stop as soon as all sequences in the prompt - produce a `end_token_id` in a location where `mask` is `False`. -""" - - -@format_docstring(call_args=call_args_docstring) + + @keras_nlp_export("keras_nlp.samplers.Sampler") class Sampler: """Base sampler class. @@ -57,35 +32,32 @@ class Sampler: {{call_args}} This base class can be extended to implement different auto-regressive - sampling methods. Subclasses can either: - - - Override the `get_next_token()` method, which computes the next token - based on a probability distribution over all possible vocab entries. - - Override `__call__`, if the sampling method needs additional information - beyond the next tokens probability distribution to sample a sequence. - - Please check available subclass samplers for examples. + sampling methods. To do so, override the `get_next_token()` method, which + computes the next token based on a probability distribution over all + possible vocab entries. Examples: ```python - # Use a simple alphabet of lowercase characters with ids in range [0, 25]. - int_lookup = {i: chr(i + ord('a')) for i in range(26)} - char_lookup = {v: k for k, v in int_lookup.items()} - batch_size, length, vocab_size = 1, 12, len(int_lookup) - - def next(prompt, cache, index): - # return a uniform distribution over our alphabet. - logits = ops.ones((batch_size, vocab_size)) - return logits, None, cache - - output = keras_nlp.samplers.GreedySampler()( - next=next, - prompt=ops.fill((batch_size, length,), char_lookup['z']), - index=5, - ) - print(["".join([int_lookup[i] for i in s]) for s in output.numpy()]) - # >>> ['zzzzzaaaaaaa'] + causal_lm = keras_nlp.models.GPT2CausalLM.from_preset("gpt2_base_en") + + # Greedy search with some tokens forbidden. + class CustomSampler(keras_nlp.samplers.Sampler): + def __init__(self, forbidden_tokens, **kwargs): + super().__init__(**kwargs) + self.forbidden_tokens = forbidden_tokens + + def get_next_token(self, probs): + batch_size, vocab_size = keras.ops.shape(probs) + for id in self.forbidden_tokens: + update = keras.ops.zeros((batch_size, 1)) + probs = keras.ops.slice_update(probs, (0, id), update) + return keras.ops.argmax(probs, axis=-1) + + # 257 = "a" with a leading space, 262 = "the" with a leading space. + causal_lm.compile(sampler=CustomSampler(forbidden_tokens=[257, 262])) + causal_lm.summary() + causal_lm.generate(["That's strange"]) ``` """ diff --git a/keras_nlp/samplers/top_k_sampler.py b/keras_nlp/samplers/top_k_sampler.py index 3456694848..513dd738c7 100644 --- a/keras_nlp/samplers/top_k_sampler.py +++ b/keras_nlp/samplers/top_k_sampler.py @@ -16,11 +16,8 @@ from keras_nlp.backend import ops from keras_nlp.backend import random from keras_nlp.samplers.sampler import Sampler -from keras_nlp.samplers.sampler import call_args_docstring -from keras_nlp.utils.python_utils import format_docstring -@format_docstring(call_args=call_args_docstring) @keras_nlp_export("keras_nlp.samplers.TopKSampler") class TopKSampler(Sampler): """Top-K Sampler class. @@ -38,24 +35,16 @@ class TopKSampler(Sampler): Examples: ```python - # Use a simple alphabet of lowercase characters with ids in range [0, 25]. - int_lookup = {i: chr(i + ord('a')) for i in range(26)} - char_lookup = {v: k for k, v in int_lookup.items()} - batch_size, length, vocab_size = 1, 12, len(int_lookup) + causal_lm = keras_nlp.models.GPT2CausalLM.from_preset("gpt2_base_en") - def next(prompt, cache, index): - hidden_states = np.ones((batch_size, 10)) - # A uniform distribution over our alphabet. - logits = np.ones((batch_size, vocab_size)) - return logits, hidden_states, cache + # Pass by name to compile. + causal_lm.compile(sampler="top_k") + causal_lm.generate(["Keras is a"]) - output = keras_nlp.samplers.TopKSampler(k=3)( - next=next, - prompt=np.full((batch_size, length,), char_lookup['z'], dtypes="int32"), - index=5, - ) - print(["".join([int_lookup[i] for i in s]) for s in output.numpy()]) - # >>> ['zzzzzacbbcaa'] + # Pass by object to compile. + sampler = keras_nlp.samplers.TopKSampler(k=5, temperature=0.7) + causal_lm.compile(sampler=sampler) + causal_lm.generate(["Keras is a"]) ``` """ diff --git a/keras_nlp/samplers/top_p_sampler.py b/keras_nlp/samplers/top_p_sampler.py index a04b39aa2b..326f5797a6 100644 --- a/keras_nlp/samplers/top_p_sampler.py +++ b/keras_nlp/samplers/top_p_sampler.py @@ -16,11 +16,8 @@ from keras_nlp.backend import ops from keras_nlp.backend import random from keras_nlp.samplers.sampler import Sampler -from keras_nlp.samplers.sampler import call_args_docstring -from keras_nlp.utils.python_utils import format_docstring -@format_docstring(call_args=call_args_docstring) @keras_nlp_export("keras_nlp.samplers.TopPSampler") class TopPSampler(Sampler): """Top-P Sampler class. @@ -46,24 +43,16 @@ class TopPSampler(Sampler): Examples: ```python - # Use a simple alphabet of lowercase characters with ids in range [0, 25]. - int_lookup = {i: chr(i + ord('a')) for i in range(26)} - char_lookup = {v: k for k, v in int_lookup.items()} - batch_size, length, vocab_size = 1, 12, len(int_lookup) + causal_lm = keras_nlp.models.GPT2CausalLM.from_preset("gpt2_base_en") - def next(prompt, cache, index): - hidden_states = np.ones((batch_size, 10)) - # A uniform distribution over our alphabet. - logits = np.ones((batch_size, vocab_size)) - return logits, hidden_states, cache + # Pass by name to compile. + causal_lm.compile(sampler="top_p") + causal_lm.generate(["Keras is a"]) - output = keras_nlp.samplers.TopPSampler(p=0.1)( - next=next, - prompt=np.full((batch_size, length,), char_lookup['z'], dtype="int32"), - index=5, - ) - print(["".join([int_lookup[i] for i in s]) for s in output.numpy()]) - # >>> ['zzzzzbabcccb'] + # Pass by object to compile. + sampler = keras_nlp.samplers.TopPSampler(p=0.1, k=1_000) + causal_lm.compile(sampler=sampler) + causal_lm.generate(["Keras is a"]) ``` """ From dd3ceb7957ab993fc7cc53890a72239e7a566218 Mon Sep 17 00:00:00 2001 From: Matt Watson <1389937+mattdangerw@users.noreply.github.com> Date: Tue, 20 Feb 2024 20:04:32 -0800 Subject: [PATCH 25/29] Add Gemma model (#1448) The Keras implementation of the Gemma model was the effort of a number of contributors: - Initial architecture: Gabriel Rasskin, Francois Chollet, Matt Watson - Model parallelism: Qianli Scott Zhu - Model export for inference: Neel Kovelamudi - Lora implementation: Francois Chollet, Samaneh Saadat - Benchmarking: Haifeng Jin - Intepretability extensions: Ryan Mullins - Testing infrastructure: Ramesh Sampath Many more helped with documentaiton and Kaggle integration. Co-authored-by: Francois Chollet Co-authored-by: Gabriel Rasskin <43894452+grasskin@users.noreply.github.com> Co-authored-by: Qianli Scott Zhu Co-authored-by: Neel Kovelamudi <60985914+nkovela1@users.noreply.github.com> Co-authored-by: Samaneh Saadat Co-authored-by: Haifeng Jin <5476582+haifeng-jin@users.noreply.github.com> Co-authored-by: Ramesh Sampath <1437573+sampathweb@users.noreply.github.com> Co-authored-by: Ryan Mullins --- .../modeling/transformer_layer_utils.py | 9 +- keras_nlp/models/__init__.py | 7 + keras_nlp/models/backbone.py | 77 +++ keras_nlp/models/bart/bart_seq_2_seq_lm.py | 1 + keras_nlp/models/gemma/__init__.py | 13 + keras_nlp/models/gemma/gemma_attention.py | 197 ++++++++ keras_nlp/models/gemma/gemma_backbone.py | 267 +++++++++++ keras_nlp/models/gemma/gemma_backbone_test.py | 128 +++++ keras_nlp/models/gemma/gemma_causal_lm.py | 441 ++++++++++++++++++ .../gemma/gemma_causal_lm_preprocessor.py | 173 +++++++ .../gemma_causal_lm_preprocessor_test.py | 92 ++++ .../models/gemma/gemma_causal_lm_test.py | 245 ++++++++++ keras_nlp/models/gemma/gemma_decoder_block.py | 189 ++++++++ keras_nlp/models/gemma/gemma_lora_test.py | 102 ++++ keras_nlp/models/gemma/gemma_preprocessor.py | 199 ++++++++ .../models/gemma/gemma_preprocessor_test.py | 74 +++ keras_nlp/models/gemma/gemma_presets.py | 66 +++ keras_nlp/models/gemma/gemma_tokenizer.py | 108 +++++ .../models/gemma/gemma_tokenizer_test.py | 67 +++ keras_nlp/models/gemma/rms_normalization.py | 40 ++ keras_nlp/models/generative_task.py | 17 +- keras_nlp/models/gpt2/gpt2_causal_lm.py | 1 + .../models/gpt_neo_x/gpt_neo_x_causal_lm.py | 1 + keras_nlp/models/opt/opt_causal_lm.py | 1 + keras_nlp/models/t5/t5_transformer_layer.py | 3 +- keras_nlp/samplers/beam_sampler.py | 2 + keras_nlp/samplers/contrastive_sampler.py | 2 + keras_nlp/samplers/sampler.py | 58 ++- .../tests/test_data/gemma_test_vocab.spm | Bin 0 -> 237805 bytes .../tokenizers/sentence_piece_tokenizer.py | 2 + keras_nlp/utils/preset_utils.py | 8 + tools/gemma/export_gemma_to_hf.py | 328 +++++++++++++ tools/gemma/export_gemma_to_torch_xla.py | 322 +++++++++++++ tools/gemma/run_gemma_xla.py | 287 ++++++++++++ .../create_gemma_test_proto.py | 36 ++ 35 files changed, 3538 insertions(+), 25 deletions(-) create mode 100644 keras_nlp/models/gemma/__init__.py create mode 100644 keras_nlp/models/gemma/gemma_attention.py create mode 100644 keras_nlp/models/gemma/gemma_backbone.py create mode 100644 keras_nlp/models/gemma/gemma_backbone_test.py create mode 100644 keras_nlp/models/gemma/gemma_causal_lm.py create mode 100644 keras_nlp/models/gemma/gemma_causal_lm_preprocessor.py create mode 100644 keras_nlp/models/gemma/gemma_causal_lm_preprocessor_test.py create mode 100644 keras_nlp/models/gemma/gemma_causal_lm_test.py create mode 100644 keras_nlp/models/gemma/gemma_decoder_block.py create mode 100644 keras_nlp/models/gemma/gemma_lora_test.py create mode 100644 keras_nlp/models/gemma/gemma_preprocessor.py create mode 100644 keras_nlp/models/gemma/gemma_preprocessor_test.py create mode 100644 keras_nlp/models/gemma/gemma_presets.py create mode 100644 keras_nlp/models/gemma/gemma_tokenizer.py create mode 100644 keras_nlp/models/gemma/gemma_tokenizer_test.py create mode 100644 keras_nlp/models/gemma/rms_normalization.py create mode 100644 keras_nlp/tests/test_data/gemma_test_vocab.spm create mode 100644 tools/gemma/export_gemma_to_hf.py create mode 100644 tools/gemma/export_gemma_to_torch_xla.py create mode 100644 tools/gemma/run_gemma_xla.py create mode 100644 tools/sentencepiece_testing/create_gemma_test_proto.py diff --git a/keras_nlp/layers/modeling/transformer_layer_utils.py b/keras_nlp/layers/modeling/transformer_layer_utils.py index 863da59a36..f375bf1b9d 100644 --- a/keras_nlp/layers/modeling/transformer_layer_utils.py +++ b/keras_nlp/layers/modeling/transformer_layer_utils.py @@ -55,9 +55,12 @@ def compute_causal_mask(batch_size, input_length, output_length, cache_index=0): `(batch_size, output_length, input_length)` that can be passed to a attention layer. """ - i = ops.expand_dims(ops.arange(output_length), axis=1) + cache_index - j = ops.arange(input_length) - mask = ops.expand_dims(ops.cast(i >= j, dtype="int32"), axis=0) + i = ops.arange(output_length, dtype="float32") + i = i + ops.cast(cache_index, "float32") + i = ops.expand_dims(i, axis=1) + j = ops.arange(input_length, dtype="float32") + mask = ops.expand_dims(i >= j, axis=0) + return ops.broadcast_to(mask, (batch_size, output_length, input_length)) diff --git a/keras_nlp/models/__init__.py b/keras_nlp/models/__init__.py index 8fd6a70ac0..cdd50670f3 100644 --- a/keras_nlp/models/__init__.py +++ b/keras_nlp/models/__init__.py @@ -75,6 +75,13 @@ ) from keras_nlp.models.f_net.f_net_preprocessor import FNetPreprocessor from keras_nlp.models.f_net.f_net_tokenizer import FNetTokenizer +from keras_nlp.models.gemma.gemma_backbone import GemmaBackbone +from keras_nlp.models.gemma.gemma_causal_lm import GemmaCausalLM +from keras_nlp.models.gemma.gemma_causal_lm_preprocessor import ( + GemmaCausalLMPreprocessor, +) +from keras_nlp.models.gemma.gemma_preprocessor import GemmaPreprocessor +from keras_nlp.models.gemma.gemma_tokenizer import GemmaTokenizer from keras_nlp.models.gpt2.gpt2_backbone import GPT2Backbone from keras_nlp.models.gpt2.gpt2_causal_lm import GPT2CausalLM from keras_nlp.models.gpt2.gpt2_causal_lm_preprocessor import ( diff --git a/keras_nlp/models/backbone.py b/keras_nlp/models/backbone.py index 6fccf6013a..9c8cdaa60e 100644 --- a/keras_nlp/models/backbone.py +++ b/keras_nlp/models/backbone.py @@ -152,3 +152,80 @@ def from_preset(calling_cls, *args, **kwargs): example_preset_name=next(iter(cls.presets), ""), preset_names='", "'.join(cls.presets), )(cls.from_preset.__func__) + + def enable_lora(self, rank): + """Enable Lora on the backbone. + + Calling this method will freeze all weights on the backbone, + while enabling Lora on the query & value `EinsumDense` layers + of the attention layers. + """ + target_names = ["query_dense", "value_dense", "query", "value"] + self.trainable = True + self._lora_enabled_layers = [] + self._lora_rank = rank + for layer in self._flatten_layers(include_self=False): + layer.trainable = False + all_layers = self._flatten_layers(include_self=False) + all_layers = [lyr for lyr in all_layers if lyr.weights] + for i, layer in enumerate(all_layers): + for name in target_names: + if layer.name == name: + if hasattr(layer, "enable_lora"): + layer.trainable = True + layer.enable_lora(rank) + self._lora_enabled_layers.append(i) + + def save_lora_weights(self, filepath): + if not getattr(self, "_lora_enabled_layers", []): + raise ValueError( + "There are no lora-enabled layers in this model. " + "Make sure to call `.enable_lora(rank)` first." + ) + if not str(filepath).endswith(".lora.h5"): + raise ValueError( + "The filename must end in `.lora.h5`. " + f"Received: filepath={filepath}" + ) + + store = keras.src.saving.saving_lib.H5IOStore(filepath, mode="w") + lora_store = store.make("lora") + lora_store["rank"] = self._lora_rank + # We cannot identify layers by name since names are non-unique, + # so we identify them by index in the topologically sorted list + # of layers that have weights. + all_layers = self._flatten_layers(include_self=False) + all_layers = [lyr for lyr in all_layers if lyr.weights] + for layer_index in self._lora_enabled_layers: + # We only lora the einsumdense layers, + # so the factored weights are always named `kernel` + layer = all_layers[layer_index] + inner_store = store.make(f"lora/{layer_index}") + inner_store["lora_kernel_a"] = layer.lora_kernel_a + inner_store["lora_kernel_b"] = layer.lora_kernel_b + store.close() + + def load_lora_weights(self, filepath): + store = keras.src.saving.saving_lib.H5IOStore(filepath, mode="r") + lora_store = store.get("lora") + rank = int(lora_store["rank"][()]) + + if not getattr(self, "_lora_enabled_layers", []): + self.enable_lora(rank) + else: + if self._lora_rank != rank: + raise ValueError( + f"The Lora rank expected by file '{filepath}' " + f"is rank={rank}, but the model was called with " + f"`.enable_lora(rank={self._lora_rank})`. " + "Both ranks must match." + ) + all_layers = self._flatten_layers(include_self=False) + all_layers = [lyr for lyr in all_layers if lyr.weights] + for layer_index in self._lora_enabled_layers: + layer = all_layers[layer_index] + lora_kernel_a = store.get(f"lora/{layer_index}")["lora_kernel_a"] + lora_kernel_b = store.get(f"lora/{layer_index}")["lora_kernel_b"] + layer.lora_kernel_a.assign(lora_kernel_a) + layer.lora_kernel_b.assign(lora_kernel_b) + store.close() diff --git a/keras_nlp/models/bart/bart_seq_2_seq_lm.py b/keras_nlp/models/bart/bart_seq_2_seq_lm.py index c17eafdb02..c530555b3d 100644 --- a/keras_nlp/models/bart/bart_seq_2_seq_lm.py +++ b/keras_nlp/models/bart/bart_seq_2_seq_lm.py @@ -479,6 +479,7 @@ def repeat_tensor(x): mask=decoder_padding_mask, end_token_id=end_token_id, hidden_states=hidden_states, + model=self, ) # Compute an output padding mask with the token ids we updated. diff --git a/keras_nlp/models/gemma/__init__.py b/keras_nlp/models/gemma/__init__.py new file mode 100644 index 0000000000..ba0c2545e4 --- /dev/null +++ b/keras_nlp/models/gemma/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2023 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/keras_nlp/models/gemma/gemma_attention.py b/keras_nlp/models/gemma/gemma_attention.py new file mode 100644 index 0000000000..80c2ac6a63 --- /dev/null +++ b/keras_nlp/models/gemma/gemma_attention.py @@ -0,0 +1,197 @@ +# Copyright 2024 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import numpy as np + +from keras_nlp.backend import keras +from keras_nlp.backend import ops +from keras_nlp.utils.keras_utils import clone_initializer + + +class CachedGemmaAttention(keras.layers.Layer): + """A cached grouped query attention layer.""" + + def __init__( + self, + head_dim, + num_query_heads, + num_key_value_heads, + kernel_initializer="glorot_uniform", + dropout=0, + **kwargs, + ): + super().__init__(**kwargs) + self.num_query_heads = num_query_heads + self.num_key_value_heads = num_key_value_heads + self.head_dim = head_dim + self.dropout = dropout + + self._kernel_initializer = keras.initializers.get( + clone_initializer(kernel_initializer) + ) + self.num_key_value_groups = num_query_heads // num_key_value_heads + + def build(self, inputs_shape): + self.hidden_dim = inputs_shape[-1] + + self.query_dense = keras.layers.EinsumDense( + "btd,ndh->btnh", + output_shape=(None, self.num_query_heads, self.head_dim), + kernel_initializer=self._kernel_initializer, + dtype=self.dtype_policy, + name="query", + ) + self.query_dense.build(inputs_shape) + + self.key_dense = keras.layers.EinsumDense( + "bsd,kdh->bskh", + output_shape=(None, self.num_key_value_heads, self.head_dim), + kernel_initializer=self._kernel_initializer, + dtype=self.dtype_policy, + name="key", + ) + self.key_dense.build(inputs_shape) + + self.value_dense = keras.layers.EinsumDense( + "bsd,kdh->bskh", + output_shape=(None, self.num_key_value_heads, self.head_dim), + kernel_initializer=self._kernel_initializer, + dtype=self.dtype_policy, + name="value", + ) + self.value_dense.build(inputs_shape) + + self.dropout_layer = keras.layers.Dropout( + rate=self.dropout, + dtype=self.dtype_policy, + ) + + self.output_dense = keras.layers.EinsumDense( + equation="btnh,nhd->btd", + output_shape=(None, self.hidden_dim), + kernel_initializer=self._kernel_initializer, + dtype=self.dtype_policy, + name="attention_output", + ) + self.output_dense.build( + (None, None, self.num_query_heads, self.head_dim) + ) + self.softmax = keras.layers.Softmax(dtype="float32") + self.built = True + + def _apply_rope(self, x, positions): + """Rope rotate q or k.""" + # TODO: refactor to use RotaryEmbedding layer? + max_wavelength = 10000 + x_shape = ops.shape(x) + freq_exponents = (2.0 / x_shape[-1]) * ops.cast( + ops.arange(x_shape[-1] // 2, dtype="float32"), self.compute_dtype + ) + timescale = max_wavelength**freq_exponents + radians = positions[..., None] / timescale[None, None, :] + radians = radians[..., None, :] + sin, cos = ops.sin(radians), ops.cos(radians) + x1, x2 = ops.split(x, 2, axis=-1) + # Avoid `ops.concatenate` for now, to avoid a obscure bug with XLA + # compilation on jax. We should be able to remove this once the + # following PR is in all jax releases we care about: + # https://github.com/openxla/xla/pull/7875 + output = ops.stack([x1 * cos - x2 * sin, x2 * cos + x1 * sin], axis=-1) + return ops.reshape(output, x_shape) + + def _compute_attention( + self, + q, + k, + v, + attention_mask, + training=False, + ): + query_normalization = 1 / np.sqrt(self.head_dim) + + q *= ops.cast(query_normalization, dtype=q.dtype) + q_shape = ops.shape(q) + q = ops.reshape( + q, + ( + *q_shape[:-2], + self.num_key_value_heads, + self.num_query_heads // self.num_key_value_heads, + q_shape[-1], + ), + ) + b, q_len, _, _, h = ops.shape(q) + + attention_logits = ops.einsum("btkgh,bskh->bkgts", q, k) + attention_mask = attention_mask[:, None, None, :, :] + orig_dtype = attention_logits.dtype + attention_softmax = self.softmax(attention_logits, mask=attention_mask) + attention_softmax = ops.cast(attention_softmax, orig_dtype) + + if self.dropout: + attention_softmax = self.dropout_layer( + attention_softmax, training=training + ) + + results = ops.einsum("bkgts,bskh->btkgh", attention_softmax, v) + return ops.reshape(results, (b, q_len, self.num_query_heads, h)) + + def call( + self, + x, + attention_mask=None, + cache=None, + cache_update_index=0, + training=False, + ): + seq_len = ops.shape(x)[1] + start_index = cache_update_index + positions = ops.cast( + ops.arange(seq_len, dtype="float32"), self.compute_dtype + ) + positions = positions + ops.cast(start_index, self.compute_dtype) + query = self.query_dense(x) + query = self._apply_rope(query, positions) + + if cache is not None: + key_cache = cache[:, 0, ...] + value_cache = cache[:, 1, ...] + key_update = self.key_dense(x) + key_update = self._apply_rope(key_update, positions) + value_update = self.value_dense(x) + start = [0, cache_update_index, 0, 0] + key = ops.slice_update(key_cache, start, key_update) + value = ops.slice_update(value_cache, start, value_update) + cache = ops.stack((key, value), axis=1) + else: + key = self.key_dense(x) + key = self._apply_rope(key, positions) + value = self.value_dense(x) + + attention_vec = self._compute_attention( + query, key, value, attention_mask, training=training + ) + + # Wipe attn vec if there are no attended tokens. + no_attended_tokens = ops.all( + ops.equal(attention_mask, 0), axis=-1, keepdims=True + )[..., None] + attention_vec = ops.where( + no_attended_tokens, ops.zeros_like(attention_vec), attention_vec + ) + + attention_output = self.output_dense(attention_vec) + + if cache is not None: + return attention_output, cache + return attention_output diff --git a/keras_nlp/models/gemma/gemma_backbone.py b/keras_nlp/models/gemma/gemma_backbone.py new file mode 100644 index 0000000000..e5814940aa --- /dev/null +++ b/keras_nlp/models/gemma/gemma_backbone.py @@ -0,0 +1,267 @@ +# Copyright 2024 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy + +from keras_nlp.api_export import keras_nlp_export +from keras_nlp.backend import config +from keras_nlp.backend import keras +from keras_nlp.backend import ops +from keras_nlp.layers.modeling.reversible_embedding import ReversibleEmbedding +from keras_nlp.models.backbone import Backbone +from keras_nlp.models.gemma.gemma_decoder_block import GemmaDecoderBlock +from keras_nlp.models.gemma.gemma_presets import backbone_presets +from keras_nlp.models.gemma.rms_normalization import RMSNormalization +from keras_nlp.utils.python_utils import classproperty + + +@keras_nlp_export("keras_nlp.models.GemmaBackbone") +class GemmaBackbone(Backbone): + """Gemma core network with hyperparameters. + + This backbone implements the base Transformer network for the Gemma model. + It includes the embedding lookups and transformer layers. This backbone + will output the final hidden states for each token, not generative + predictions over the vocabulary space. For a higher-level object for text + generation, see `keras_nlp.models.GemmaCausalLM`. + + The default constructor gives a fully customizable, randomly initialized + Gemma model with any number of layers, heads, and embedding dimensions. To + load preset architectures and weights, use the `from_preset` constructor. + + Args: + vocabulary_size: int. The size of the token vocabulary. + num_layers: int. The number of transformer layers. + num_query_heads: int. The number of heads for the query projections in + the attention layer. + num_key_value_heads: int. The number of heads for the key and value + projections in the attention layer. + hidden_dim: int. The size of the transformer hidden state at the end + of each transformer layer. + intermediate_dim: int. The output dimension of the first Dense layer in + a two-layer feedforward network for each transformer. + head_dim: int. The size of each attention head. + layer_norm_epsilon: float. The epsilon value user for every layer norm + in the transformer model. + dropout: float. Dropout probability for the Transformer encoder. + dtype: string or `keras.mixed_precision.DTypePolicy`. The dtype to use + for the models computations and weights. Note that some + computations, such as softmax and layer normalization will always + be done a float32 precision regardless of dtype. + + Example usage: + ```python + input_data = { + "token_ids": np.ones(shape=(1, 12), dtype="int32"), + "padding_mask": np.array([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0]]), + } + + # Pretrained Gemma decoder. + model = keras_nlp.models.GemmaBackbone.from_preset("gemma_2b_en") + model(input_data) + + # Randomly initialized Gemma decoder with custom config. + model = keras_nlp.models.GemmaBackbone( + vocabulary_size=50257, + num_layers=12, + num_query_heads=12, + num_key_value_heads=1, + hidden_dim=768, + intermediate_dim=3072, + head_dim=64, + ) + model(input_data) + ``` + """ + + def __init__( + self, + vocabulary_size, + num_layers, + num_query_heads, + num_key_value_heads, + hidden_dim, + intermediate_dim, + head_dim, + layer_norm_epsilon=1e-6, + dropout=0, + dtype=None, + **kwargs, + ): + if not config.keras_3(): + raise ValueError( + "`GemmaBackbone` requires Keras 3. Run `pip install -U keras` " + "upgrade your Keras version, or see https://keras.io/getting_started/ " + "for more info on Keras versions and installation." + ) + + # === Layers === + self.token_embedding = ReversibleEmbedding( + input_dim=vocabulary_size, + output_dim=hidden_dim, + tie_weights=True, + embeddings_initializer=keras.initializers.VarianceScaling( + scale=1.0, + mode="fan_in", + distribution="untruncated_normal", + seed=None, + ), + dtype=dtype, + name="token_embedding", + ) + self.transformer_layers = [] + for i in range(num_layers): + layer = GemmaDecoderBlock( + intermediate_dim=intermediate_dim, + hidden_dim=hidden_dim, + num_query_heads=num_query_heads, + head_dim=head_dim, + num_key_value_heads=num_key_value_heads, + dropout=dropout, + dtype=dtype, + name=f"decoder_block_{i}", + ) + self.transformer_layers.append(layer) + self.layer_norm = RMSNormalization( + epsilon=layer_norm_epsilon, + dtype=dtype, + name="final_normalization", + ) + + # === Functional Model === + token_id_input = keras.Input( + shape=(None,), dtype="float32", name="token_ids" + ) + padding_mask_input = keras.Input( + shape=(None,), dtype="float32", name="padding_mask" + ) + x = self.token_embedding(token_id_input) + x = x * ops.cast(ops.sqrt(hidden_dim), x.dtype) + for transformer_layer in self.transformer_layers: + x = transformer_layer(x, padding_mask=padding_mask_input) + sequence_output = self.layer_norm(x) + super().__init__( + inputs={ + "token_ids": token_id_input, + "padding_mask": padding_mask_input, + }, + outputs=sequence_output, + **kwargs, + ) + + # === Config === + self.vocabulary_size = vocabulary_size + self.num_layers = num_layers + self.num_query_heads = num_query_heads + self.num_key_value_heads = num_key_value_heads + self.hidden_dim = hidden_dim + self.intermediate_dim = intermediate_dim + self.head_dim = head_dim + self.layer_norm_epsilon = layer_norm_epsilon + self.dropout = dropout + + def get_config(self): + config = super().get_config() + config.update( + { + "vocabulary_size": self.vocabulary_size, + "num_layers": self.num_layers, + "num_query_heads": self.num_query_heads, + "num_key_value_heads": self.num_key_value_heads, + "hidden_dim": self.hidden_dim, + "intermediate_dim": self.intermediate_dim, + "head_dim": self.head_dim, + "layer_norm_epsilon": self.layer_norm_epsilon, + "dropout": self.dropout, + } + ) + return config + + @classproperty + def presets(cls): + return copy.deepcopy(backbone_presets) + + @staticmethod + def get_layout_map(device_mesh, model_parallel_dim_name="model"): + """Get a `keras.distribution.LayoutMap` for model parallel distribution. + + The returned `LayoutMap` contains the sharding spec for the gemma + backbone weights, so that you can use it to distribute weights across + the accelerators. + + Sample usage: + ``` + # Feel free to change the mesh shape to balance data and model parallel + mesh = keras.distribution.DeviceMesh( + shape=(1, 8), axis_names=('batch', 'model'), + devices=keras.distribution.list_devices()) + layout_map = GemmaBackbone.get_layout_map( + mesh, model_parallel_dim_name="model") + + distribution = keras.distribution.ModelParallel( + mesh, layout_map, batch_dim_name='batch') + with distribution.scope(): + gemma_model = keras_nlp.models.GemmaCausalLM.from_preset() + ``` + + Args: + device_mesh: The `keras.distribution.DeviceMesh` instance for + distribution. + model_parallel_dim_name: The axis name of the device mesh, where + the weights should be partition on. + Return: + `keras.distribution.LayoutMap` that contains the sharding spec + of all the model weights. + """ + # The weight path and shape of the Gemma backbone is like below (for 2G) + # token_embedding/embeddings, (256128, 2048), 524550144 + # repeat block for decoder + # ... + # decoder_block_17/pre_attention_norm/scale, (2048,), 2048 + # decoder_block_17/attention/query/kernel, (8, 2048, 256), 4194304 + # decoder_block_17/attention/key/kernel, (8, 2048, 256), 4194304 + # decoder_block_17/attention/value/kernel, (8, 2048, 256), 4194304 + # decoder_block_17/attention/attention_output/kernel, (8, 256, 2048), 4194304 + # decoder_block_17/pre_ffw_norm/scale, (2048,), 2048 + # decoder_block_17/ffw_gating/kernel, (2048, 16384), 33554432 + # decoder_block_17/ffw_gating_2/kernel, (2048, 16384), 33554432 + # decoder_block_17/ffw_linear/kernel, (16384, 2048), 33554432 + if not isinstance(device_mesh, keras.distribution.DeviceMesh): + raise ValueError( + "Invalid device_mesh type. Expected `keras.distribution.Device`," + f" got {type(device_mesh)}" + ) + if model_parallel_dim_name not in device_mesh.axis_names: + raise ValueError( + f"{model_parallel_dim_name} is not found in the " + f"device_mesh.axis_names. {device_mesh.axis_name=}" + ) + model_dim = model_parallel_dim_name + # The sharding is partition for the hidden_dim of the model. + layout_map = keras.distribution.LayoutMap(device_mesh) + layout_map["token_embedding/embeddings"] = (None, model_dim) + layout_map["decoder_block.*attention.*(query|key|value).*kernel"] = ( + None, + model_dim, + None, + ) + layout_map["decoder_block.*attention_output.*kernel"] = ( + None, + None, + model_dim, + ) + layout_map["decoder_block.*ffw_gating.*kernel"] = (model_dim, None) + layout_map["decoder_block.*ffw_linear.*kernel"] = (None, model_dim) + + return layout_map diff --git a/keras_nlp/models/gemma/gemma_backbone_test.py b/keras_nlp/models/gemma/gemma_backbone_test.py new file mode 100644 index 0000000000..c66d318fd5 --- /dev/null +++ b/keras_nlp/models/gemma/gemma_backbone_test.py @@ -0,0 +1,128 @@ +# Copyright 2024 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pytest + +from keras_nlp.backend import keras +from keras_nlp.backend import ops +from keras_nlp.models.gemma.gemma_backbone import GemmaBackbone +from keras_nlp.tests.test_case import TestCase + + +@pytest.mark.keras_3_only +class GemmaBackboneTest(TestCase): + def setUp(self): + self.init_kwargs = { + "vocabulary_size": 256128, + "num_layers": 2, + "num_query_heads": 4, + "num_key_value_heads": 4, + "hidden_dim": 128, + "intermediate_dim": 256, + "head_dim": 128, + "layer_norm_epsilon": 1e-6, + } + self.input_data = { + "token_ids": ops.ones((2, 5), dtype="int32"), + "padding_mask": ops.ones((2, 5), dtype="int32"), + } + + def test_backbone_basics(self): + self.run_backbone_test( + cls=GemmaBackbone, + init_kwargs=self.init_kwargs, + input_data=self.input_data, + expected_output_shape=(2, 5, 128), + ) + + @pytest.mark.large + def test_saved_model(self): + self.run_model_saving_test( + cls=GemmaBackbone, + init_kwargs=self.init_kwargs, + input_data=self.input_data, + ) + + @pytest.mark.large + def test_smallest_preset(self): + self.run_preset_test( + cls=GemmaBackbone, + preset="gemma_2b_en", + input_data={ + "token_ids": ops.array([[651, 4320, 8426, 25341, 235265]]), + "padding_mask": ops.ones((1, 5), dtype="int32"), + }, + expected_output_shape=(1, 5, 2048), + # The forward pass from a preset should be stable! + expected_partial_output=ops.array( + [1.073359, 0.262374, 0.170238, 0.605402, 2.336161] + ), + ) + + @pytest.mark.extra_large + def test_all_presets(self): + for preset in GemmaBackbone.presets: + self.run_preset_test( + cls=GemmaBackbone, + preset=preset, + input_data=self.input_data, + ) + + def test_architecture_characteristics(self): + model = GemmaBackbone(**self.init_kwargs) + self.assertEqual(model.count_params(), 33407616) + self.assertEqual(len(model.layers), 6) + + def test_distribution(self): + if keras.backend.backend() != "jax": + return + devices = keras.distribution.list_devices("CPU") + if len(devices) == 1: + # Need more than 1 device for distribution testing. + return + device_mesh = keras.distribution.DeviceMesh( + shape=(1, len(devices)), + axis_names=("batch", "model"), + devices=devices, + ) + + layout_map = GemmaBackbone.get_layout_map(device_mesh) + distribution = keras.distribution.ModelParallel(device_mesh, layout_map) + with distribution.scope(): + model = GemmaBackbone(**self.init_kwargs) + + for w in model.weights: + if "token_embedding/embeddings" in w.path: + self.assertEqual(tuple(w.value.sharding.spec), (None, "model")) + if "attention/query/kernel" in w.path: + self.assertEqual( + tuple(w.value.sharding.spec), (None, "model", None) + ) + if "attention/key/kernel" in w.path: + self.assertEqual( + tuple(w.value.sharding.spec), (None, "model", None) + ) + if "attention/value/kernel" in w.path: + self.assertEqual( + tuple(w.value.sharding.spec), (None, "model", None) + ) + if "attention/attention_output/kernel" in w.path: + self.assertEqual( + tuple(w.value.sharding.spec), (None, None, "model") + ) + if "ffw_gating/kernel" in w.path: + self.assertEqual(tuple(w.value.sharding.spec), ("model", None)) + if "ffw_gating_2/kernel" in w.path: + self.assertEqual(tuple(w.value.sharding.spec), ("model", None)) + if "ffw_linearl" in w.path: + self.assertEqual(tuple(w.value.sharding.spec), (None, "model")) diff --git a/keras_nlp/models/gemma/gemma_causal_lm.py b/keras_nlp/models/gemma/gemma_causal_lm.py new file mode 100644 index 0000000000..45c7c6abe0 --- /dev/null +++ b/keras_nlp/models/gemma/gemma_causal_lm.py @@ -0,0 +1,441 @@ +# Copyright 2024 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy + +from keras_nlp.api_export import keras_nlp_export +from keras_nlp.backend import keras +from keras_nlp.backend import ops +from keras_nlp.models.gemma.gemma_backbone import GemmaBackbone +from keras_nlp.models.gemma.gemma_causal_lm_preprocessor import ( + GemmaCausalLMPreprocessor, +) +from keras_nlp.models.gemma.gemma_presets import backbone_presets +from keras_nlp.models.generative_task import GenerativeTask +from keras_nlp.utils.python_utils import classproperty + + +@keras_nlp_export("keras_nlp.models.GemmaCausalLM") +class GemmaCausalLM(GenerativeTask): + """An end-to-end Gemma model for causal language modeling. + + A causal language model (LM) predicts the next token based on previous + tokens. This task setup can be used to train the model unsupervised on + plain text input, or to autoregressively generate plain text similar to + the data used for training. This task can be used for pre-training or + fine-tuning a Gemma model, simply by calling `fit()`. + + This model has a `generate()` method, which generates text based on a + prompt. The generation strategy used is controlled by an additional + `sampler` argument on `compile()`. You can recompile the model with + different `keras_nlp.samplers` objects to control the generation. By + default, `"greedy"` sampling will be used. + + This model can optionally be configured with a `preprocessor` layer, in + which case it will automatically apply preprocessing to string inputs during + `fit()`, `predict()`, `evaluate()` and `generate()`. This is done by default + when creating the model with `from_preset()`. + + Args: + backbone: A `keras_nlp.models.GemmaBackbone` instance. + preprocessor: A `keras_nlp.models.GemmaCausalLMPreprocessor` or `None`. + If `None`, this model will not apply preprocessing, and inputs + should be preprocessed before calling the model. + + Examples: + + Use `generate()` to do text generation. + ```python + gemma_lm = keras_nlp.models.GemmaCausalLM.from_preset("gemma_2b_en") + gemma_lm.generate("I want to say", max_length=30) + + # Generate with batched prompts. + gemma_lm.generate(["This is a", "Where are you"], max_length=30) + ``` + + Compile the `generate()` function with a custom sampler. + ```python + gemma_lm = keras_nlp.models.GemmaCausalLM.from_preset("gemma_2b_en") + gemma_lm.compile(sampler="top_k") + gemma_lm.generate("I want to say", max_length=30) + + gemma_lm.compile(sampler=keras_nlp.samplers.BeamSampler(num_beams=2)) + gemma_lm.generate("I want to say", max_length=30) + ``` + + Use `generate()` without preprocessing. + ```python + prompt = { + # Token ids for " Keras is". + "token_ids": np.array([[2, 214064, 603, 0, 0, 0, 0]] * 2), + # Use `"padding_mask"` to indicate values that should not be overridden. + "padding_mask": np.array([[1, 1, 1, 0, 0, 0, 0]] * 2), + } + + gemma_lm = keras_nlp.models.GemmaCausalLM.from_preset( + "gemma_2b_en", + preprocessor=None, + ) + gemma_lm.generate(prompt) + ``` + + Call `fit()` on a single batch. + ```python + features = ["The quick brown fox jumped.", "I forgot my homework."] + gemma_lm = keras_nlp.models.GemmaCausalLM.from_preset("gemma_2b_en") + gemma_lm.fit(x=features, batch_size=2) + ``` + + Call `fit()` without preprocessing. + ```python + x = { + # Token ids for " Keras is deep learning library" + "token_ids": np.array([[2, 214064, 603, 5271, 6044, 9581, 1, 0]] * 2), + "padding_mask": np.array([[1, 1, 1, 1, 1, 1, 1, 0]] * 2), + } + y = np.array([[214064, 603, 5271, 6044, 9581, 3, 0, 0]] * 2) + sw = np.array([[1, 1, 1, 1, 1, 1, 0, 0]] * 2) + + gemma_lm = keras_nlp.models.GemmaCausalLM.from_preset( + "gemma_2b_en", + preprocessor=None, + ) + gemma_lm.fit(x=x, y=y, sample_weight=sw, batch_size=2) + ``` + + Custom backbone and vocabulary. + ```python + tokenizer = keras_nlp.models.GemmaTokenizer( + proto="proto.spm", + ) + preprocessor = keras_nlp.models.GemmaCausalLMPreprocessor( + tokenizer=tokenizer, + sequence_length=128, + ) + backbone = keras_nlp.models.GemmaBackbone( + vocabulary_size=30552, + num_layers=4, + num_heads=4, + hidden_dim=256, + intermediate_dim=512, + max_sequence_length=128, + ) + gemma_lm = keras_nlp.models.GemmaCausalLM( + backbone=backbone, + preprocessor=preprocessor, + ) + gemma_lm.fit(x=features, batch_size=2) + ``` + """ + + def __init__( + self, + backbone, + preprocessor=None, + **kwargs, + ): + # === Layers === + self.backbone = backbone + self.preprocessor = preprocessor + + # === Functional Model === + inputs = backbone.input + hidden_states = backbone(inputs) + outputs = backbone.token_embedding(hidden_states, reverse=True) + super().__init__( + inputs=inputs, + outputs=outputs, + **kwargs, + ) + + # === Default compilation === + self.compile( + loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True), + optimizer=keras.optimizers.Adam(2e-5), + metrics=[keras.metrics.SparseCategoricalAccuracy()], + sampler="greedy", + jit_compile=True, + ) + + @classproperty + def presets(cls): + return copy.deepcopy(backbone_presets) + + @classproperty + def backbone_cls(cls): + return GemmaBackbone + + @classproperty + def preprocessor_cls(cls): + return GemmaCausalLMPreprocessor + + def call_with_cache( + self, + token_ids, + cache, + cache_update_index, + ): + """Forward pass of `GemmaCausalLM` with cache. + + `call_with_cache` adds an additional forward pass for the model for + autoregressive inference. Unlike calling the model directly, this method + allows caching previous key/value Tensors in multi-head attention layer, + and avoids recomputing the outputs of seen tokens. + + Args: + token_ids: a dense int Tensor with shape `(batch_size, max_length)`. + cache: a dense float Tensor, the cache of key and value. + cache_update_index: int, or int Tensor. The index of current inputs in the + whole sequence. + + Returns: + A (logits, hidden_states, cache) tuple. Where `logits` is the + language model logits for the input token_ids, `hidden_states` is + the final hidden representation of the input tokens, and `cache` is + the decoding cache. + """ + x = self.backbone.token_embedding(token_ids) + x = x * ops.cast(ops.sqrt(self.backbone.hidden_dim), x.dtype) + # Each decoder layer has a cache; we update them separately. + caches = [] + for i, transformer_layer in enumerate(self.backbone.transformer_layers): + current_cache = cache[:, i, ...] + x, next_cache = transformer_layer( + x, + cache=current_cache, + cache_update_index=cache_update_index, + ) + caches.append(next_cache) + cache = ops.stack(caches, axis=1) + hidden_states = x = self.backbone.layer_norm(x) + logits = self.backbone.token_embedding(x, reverse=True) + return logits, hidden_states, cache + + def _build_cache(self, token_ids): + """Build an empty cache for use with `call_with_cache()`.""" + batch_size = ops.shape(token_ids)[0] + max_length = ops.shape(token_ids)[1] + num_layers = self.backbone.num_layers + num_heads = self.backbone.num_key_value_heads + head_dim = self.backbone.head_dim + shape = [batch_size, num_layers, 2, max_length, num_heads, head_dim] + cache = ops.zeros(shape, dtype=self.compute_dtype) + # Seed the cache. + _, hidden_states, cache = self.call_with_cache(token_ids, cache, 0) + return hidden_states, cache + + def generate_step( + self, + inputs, + end_token_id=None, + ): + """A compilable generation function for a single batch of inputs. + + This function represents the inner, XLA-compilable, generation function + for a single batch of inputs. Inputs should have the same structure as + model inputs, a dictionary with keys `"token_ids"` and `"padding_mask"`. + + Args: + inputs: A dictionary with two keys `"token_ids"` and + `"padding_mask"` and batched tensor values. + end_token_id: The id of the end token to stop on. If all + sequences have produced a new `end_token_id`, generation + will stop. + """ + token_ids, padding_mask = inputs["token_ids"], inputs["padding_mask"] + # Create and seed cache with a single forward pass. + hidden_states, cache = self._build_cache(token_ids) + # Compute the lengths of all user inputted tokens ids. + row_lengths = ops.sum(ops.cast(padding_mask, "int32"), axis=-1) + # Start at the first index that has no user inputted id. + index = ops.min(row_lengths) + + def next(prompt, cache, index): + # The cache index is the index of our previous token. + cache_update_index = index - 1 + batch_size = ops.shape(prompt)[0] + prompt = ops.slice(prompt, [0, cache_update_index], [batch_size, 1]) + logits, hidden_states, cache = self.call_with_cache( + prompt, + cache, + cache_update_index, + ) + return ( + ops.squeeze(logits, axis=1), + ops.squeeze(hidden_states, axis=1), + cache, + ) + + token_ids = self._sampler( + next=next, + prompt=token_ids, + cache=cache, + index=index, + mask=padding_mask, + end_token_id=end_token_id, + hidden_states=hidden_states, + model=self, + ) + + # Compute an output padding mask with the token ids we updated. + if end_token_id is not None: + # Build a mask of `end_token_id` locations not in the original + # prompt (not in locations where `padding_mask` is True). + end_locations = ops.logical_and( + ops.equal(token_ids, end_token_id), + ops.logical_not(padding_mask), + ) + end_locations = ops.cast(end_locations, "int32") + # Use cumsum to get ones in all locations after end_locations. + cumsum = ops.cast(ops.cumsum(end_locations, axis=-1), "int32") + overflow = cumsum - end_locations + # Our padding mask is the inverse of these overflow locations. + padding_mask = ops.logical_not(ops.cast(overflow, "bool")) + else: + # Without early stopping, all locations will have been updated. + padding_mask = ops.ones_like(token_ids, dtype="bool") + return { + "token_ids": token_ids, + "padding_mask": padding_mask, + } + + def score( + self, + token_ids, + padding_mask=None, + scoring_mode="logits", + layer_intercept_fn=None, + target_ids=None, + ): + """Score a generation represented by the provided token ids. + + Args: + token_ids: A [batch_size, num_tokens] tensor containing tokens + to score. Typically, this tensor captures the output from a call + to `GemmaCausalLM.generate()`, i.e., tokens for both the input + text and the model-generated text. + padding_mask: A [batch_size, num_tokens] tensor indicating the + tokens that should be preserved during generation. This is an + artifact required by the GemmaBackbone and isn't influential on + the computation of this function. If omitted, this function uses + `keras.ops.ones()` to create a tensor of the appropriate shape. + scoring_mode: The type of scores to return, either "logits" or + "loss", both will be per input token. + layer_intercept_fn: An optional function for augmenting activations + with additional computation, for example, as part of + interpretability research. This function will be passed the + activations as its first parameter and a numeric index + associated with that backbone layer. _This index _is not_ an + index into `self.backbone.layers`_. The index -1 accompanies the + embeddings returned by calling `self.backbone.token_embedding()` + on `token_ids` in the forward direction. All subsequent indexes + will be 0-based indices for the activations returned by each of + the Transformers layers in the backbone. This function must + return a [batch_size, num_tokens, hidden_dims] tensor + that can be passed as an input to the next layer in the model. + target_ids: An [batch_size, num_tokens] tensor containing the + predicted tokens against which the loss should be computed. If a + span of tokens is provided (sequential truthy values along + axis=1 in the tensor), the loss will be computed as the + aggregate across those tokens. + + Raises: + ValueError: If an unsupported scoring_mode is provided, or if the + target_ids are not provided when using ScoringMode.LOSS. + + Returns: + The per-token scores as a tensor of size + [batch_size, num_tokens, vocab_size] in "logits" mode, or + [batch_size, num_tokens] in "loss" mode. + + Examples: + + Compute gradients between embeddings and loss scores with TensorFlow: + ```python + gemma_lm = keras_nlp.models.GemmaCausalLM.from_preset( + "gemma_2b_en" + ) + generations = gemma_lm.generate( + ["This is a", "Where are you"], + max_length=30 + ) + preprocessed = gemma_lm.preprocessor.generate_preprocess(generations) + generation_ids = preprocessed["token_ids"] + padding_mask = preprocessed["padding_mask"] + target_ids = keras.ops.roll(generation_ids, shift=-1, axis=1) + + embeddings = None + with tf.GradientTape(watch_accessed_variables=True) as tape: + def layer_intercept_fn(x, i): + if i == -1: + nonlocal embeddings, tape + embeddings = x + tape.watch(embeddings) + return x + + losses = gemma_lm.score( + token_ids=generation_ids, + padding_mask=padding_mask, + scoring_mode="loss", + layer_intercept_fn=layer_intercept_fn, + target_ids=target_ids, + ) + + grads = tape.gradient(losses, embeddings) + ``` + """ + if scoring_mode not in ("logits", "loss"): + raise ValueError( + "Unsupported scoring_mode. Must be one of 'logits' or 'loss'." + ) + + if scoring_mode == "loss" and target_ids is None: + raise ValueError( + "Cannot compute loss without targets. Please provide target " + "token ids via the target_ids parameter." + ) + + batch_shape = ops.shape(token_ids)[:2] + assert len(batch_shape) == 2 + + if padding_mask is None: + padding_mask = ops.ones(shape=batch_shape) + + if layer_intercept_fn is None: + + def default_layer_intercept_fn(x, unused_i): + return x + + layer_intercept_fn = default_layer_intercept_fn + + token_embeddings = self.backbone.token_embedding(token_ids) + x = layer_intercept_fn(token_embeddings, -1) + + x = token_embeddings * ops.cast( + ops.sqrt(self.backbone.hidden_dim), dtype=self.compute_dtype + ) + for i, transformer_layer in enumerate(self.backbone.transformer_layers): + x = transformer_layer(x, padding_mask=padding_mask) + x = layer_intercept_fn(x, i) + x = self.backbone.layer_norm(x) + logits = self.backbone.token_embedding(x, reverse=True) + + if scoring_mode == "logits": + return logits + + per_token_loss_fn = keras.losses.SparseCategoricalCrossentropy( + from_logits=True, reduction="none" + ) + per_token_loss = per_token_loss_fn(target_ids, logits) + return per_token_loss diff --git a/keras_nlp/models/gemma/gemma_causal_lm_preprocessor.py b/keras_nlp/models/gemma/gemma_causal_lm_preprocessor.py new file mode 100644 index 0000000000..20c66edff3 --- /dev/null +++ b/keras_nlp/models/gemma/gemma_causal_lm_preprocessor.py @@ -0,0 +1,173 @@ +# Copyright 2024 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import tensorflow as tf +from absl import logging + +from keras_nlp.api_export import keras_nlp_export +from keras_nlp.backend import ops +from keras_nlp.models.gemma.gemma_preprocessor import GemmaPreprocessor +from keras_nlp.utils.keras_utils import ( + convert_inputs_to_list_of_tensor_segments, +) +from keras_nlp.utils.keras_utils import pack_x_y_sample_weight + + +@keras_nlp_export("keras_nlp.models.GemmaCausalLMPreprocessor") +class GemmaCausalLMPreprocessor(GemmaPreprocessor): + """Gemma Causal LM preprocessor. + + This preprocessing layer is meant for use with + `keras_nlp.models.GemmaCausalLM`. By default, it will take in batches of + strings, and return outputs in a `(x, y, sample_weight)` format, where the + `y` label is the next token id in the `x` sequence. + + For use with generation, the layer also exposes two methods + `generate_preprocess()` and `generate_postprocess()`. When this preprocessor + is attached to a `keras_nlp.models.GemmaCausalLM` instance, these methods + will be called implicitly in `generate()`. They can also be called + standalone (e.g. to precompute preprocessing inputs for generation in a + separate process). + + Args: + tokenizer: A `keras_nlp.models.GemmaTokenizer` instance. + sequence_length: The length of the packed inputs. + add_start_token: If `True`, the preprocessor will prepend the tokenizer + start token to each input sequence. + add_end_token: If `True`, the preprocessor will append the tokenizer + end token to each input sequence. + + Call arguments: + x: A string, `tf.Tensor` or list of python strings. + y: Label data. Should always be `None` as the layer generates labels. + sample_weight: Label weights. Should always be `None` as the layer + generates label weights. + sequence_length: Pass to override the configured `sequence_length` of + the layer. + + Examples: + ```python + # Load the preprocessor from a preset. + preprocessor = keras_nlp.models.GemmaCausalLMPreprocessor.from_preset( + "gemma_2b_en" + ) + + # Tokenize and pack a single sentence. + preprocessor("The quick brown fox jumped.") + + # Tokenize a batch of sentences. + preprocessor(["The quick brown fox jumped.", "Call me Ishmael."]) + + # Apply tokenization to a `tf.data.Dataset`. + features = tf.constant(["The quick brown fox.", "Call me Ishmael."]) + ds = tf.data.Dataset.from_tensor_slices(features) + ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE) + + # Prepare tokens for generation (no end token). + preprocessor.generate_preprocess(["The quick brown fox jumped."]) + + # Map generation outputs back to strings. + preprocessor.generate_postprocess({ + 'token_ids': np.array([[2, 714, 4320, 8426, 25341, 32292, 235265, 0]]), + 'padding_mask': np.array([[ 1, 1, 1, 1, 1, 1, 1, 0]]), + }) + ``` + """ + + def call( + self, + x, + y=None, + sample_weight=None, + sequence_length=None, + ): + if y is not None or sample_weight is not None: + logging.warning( + "`GemmaCausalLMPreprocessor` generates `y` and `sample_weight` " + "based on your input data, but your data already contains `y` " + "or `sample_weight`. Your `y` and `sample_weight` will be " + "ignored." + ) + sequence_length = sequence_length or self.sequence_length + + x = convert_inputs_to_list_of_tensor_segments(x)[0] + x = self.tokenizer(x) + # Pad with one extra token to account for the truncation below. + token_ids, padding_mask = self.packer( + x, + sequence_length=sequence_length + 1, + add_start_value=self.add_start_token, + add_end_value=self.add_end_token, + ) + # The last token does not have a next token, so we truncate it out. + x = { + "token_ids": token_ids[..., :-1], + "padding_mask": padding_mask[..., :-1], + } + # Target `y` will be the next token. + y, sample_weight = token_ids[..., 1:], padding_mask[..., 1:] + return pack_x_y_sample_weight(x, y, sample_weight) + + def generate_preprocess( + self, + x, + sequence_length=None, + ): + """Covert strings to integer token input for generation. + + Similar to calling the layer for training, this method takes in strings + or tensor strings, tokenizes and packs the input, and computes a padding + mask masking all inputs not filled in with a padded value. + + Unlike calling the layer for training, this method does not compute + labels and will never append a `tokenizer.end_token_id` to the end of + the sequence (as generation is expected to continue at the end of the + inputted prompt). + """ + if not self.built: + self.build(None) + + x = convert_inputs_to_list_of_tensor_segments(x)[0] + x = self.tokenizer(x) + token_ids, padding_mask = self.packer( + x, sequence_length=sequence_length, add_end_value=False + ) + return { + "token_ids": token_ids, + "padding_mask": padding_mask, + } + + def generate_postprocess( + self, + x, + ): + """Covert integer token output to strings for generation. + + This method reverses `generate_preprocess()`, by first removing all + padding and start/end tokens, and then converting the integer sequence + back to a string. + """ + if not self.built: + self.build(None) + + token_ids, padding_mask = x["token_ids"], x["padding_mask"] + token_ids = ops.convert_to_numpy(token_ids) + mask = ops.convert_to_numpy(padding_mask) + # Also strip any special tokens during detokenization (e.g. the start + # and end markers). In the future we could make this configurable. + mask = mask & (token_ids != self.tokenizer.start_token_id) + mask = mask & (token_ids != self.tokenizer.pad_token_id) + mask = mask & (token_ids != self.tokenizer.end_token_id) + token_ids = tf.ragged.boolean_mask(token_ids, mask) + return self.tokenizer.detokenize(token_ids) diff --git a/keras_nlp/models/gemma/gemma_causal_lm_preprocessor_test.py b/keras_nlp/models/gemma/gemma_causal_lm_preprocessor_test.py new file mode 100644 index 0000000000..121621da85 --- /dev/null +++ b/keras_nlp/models/gemma/gemma_causal_lm_preprocessor_test.py @@ -0,0 +1,92 @@ +# Copyright 2024 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pytest + +from keras_nlp.models.gemma.gemma_causal_lm_preprocessor import ( + GemmaCausalLMPreprocessor, +) +from keras_nlp.models.gemma.gemma_tokenizer import GemmaTokenizer +from keras_nlp.tests.test_case import TestCase + + +@pytest.mark.keras_3_only +class GemmaCausalLMPreprocessorTest(TestCase): + def setUp(self): + self.tokenizer = GemmaTokenizer( + proto=os.path.join( + self.get_test_data_dir(), "gemma_test_vocab.spm" + ), + ) + self.init_kwargs = { + "tokenizer": self.tokenizer, + "sequence_length": 8, + } + self.input_data = ["the quick brown fox"] + + def test_preprocessor_basics(self): + self.run_preprocessing_layer_test( + cls=GemmaCausalLMPreprocessor, + init_kwargs=self.init_kwargs, + input_data=self.input_data, + expected_output=( + { + "token_ids": [[1, 4, 9, 5, 7, 2, 0, 0]], + "padding_mask": [[1, 1, 1, 1, 1, 1, 0, 0]], + }, + [[4, 9, 5, 7, 2, 0, 0, 0]], # Labels shifted. + [[1, 1, 1, 1, 1, 0, 0, 0]], # Zero out unlabeled examples. + ), + ) + + def test_no_start_end_token(self): + input_data = ["the quick brown fox"] * 4 + + preprocessor = GemmaCausalLMPreprocessor( + **self.init_kwargs, + add_start_token=False, + add_end_token=False, + ) + x, y, sw = preprocessor(input_data) + self.assertAllEqual(x["token_ids"], [[4, 9, 5, 7, 0, 0, 0, 0]] * 4) + self.assertAllEqual(x["padding_mask"], [[1, 1, 1, 1, 0, 0, 0, 0]] * 4) + self.assertAllEqual(y, [[9, 5, 7, 0, 0, 0, 0, 0]] * 4) + self.assertAllEqual(sw, [[1, 1, 1, 0, 0, 0, 0, 0]] * 4) + + def test_generate_preprocess(self): + input_data = "the quick brown fox" + preprocessor = GemmaCausalLMPreprocessor(**self.init_kwargs) + x = preprocessor.generate_preprocess(input_data) + self.assertAllEqual(x["token_ids"], [1, 4, 9, 5, 7, 0, 0, 0]) + self.assertAllEqual(x["padding_mask"], [1, 1, 1, 1, 1, 0, 0, 0]) + + def test_generate_postprocess(self): + input_data = { + "token_ids": [1, 4, 9, 5, 7, 2, 0, 0], + "padding_mask": [1, 1, 1, 1, 1, 1, 0, 0], + } + preprocessor = GemmaCausalLMPreprocessor(**self.init_kwargs) + x = preprocessor.generate_postprocess(input_data) + self.assertAllEqual(x, "the quick brown fox") + + @pytest.mark.extra_large + def test_all_presets(self): + for preset in GemmaCausalLMPreprocessor.presets: + self.run_preset_test( + cls=GemmaCausalLMPreprocessor, + preset=preset, + input_data=self.input_data, + ) diff --git a/keras_nlp/models/gemma/gemma_causal_lm_test.py b/keras_nlp/models/gemma/gemma_causal_lm_test.py new file mode 100644 index 0000000000..0e1d7a14f8 --- /dev/null +++ b/keras_nlp/models/gemma/gemma_causal_lm_test.py @@ -0,0 +1,245 @@ +# Copyright 2024 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from unittest.mock import patch + +import keras +import pytest + +from keras_nlp.backend import ops +from keras_nlp.models.gemma.gemma_backbone import GemmaBackbone +from keras_nlp.models.gemma.gemma_causal_lm import GemmaCausalLM +from keras_nlp.models.gemma.gemma_causal_lm_preprocessor import ( + GemmaCausalLMPreprocessor, +) +from keras_nlp.models.gemma.gemma_tokenizer import GemmaTokenizer +from keras_nlp.tests.test_case import TestCase + + +@pytest.mark.keras_3_only +class GemmaCausalLMTest(TestCase): + def setUp(self): + self.tokenizer = GemmaTokenizer( + proto=os.path.join( + self.get_test_data_dir(), "gemma_test_vocab.spm" + ), + ) + self.preprocessor = GemmaCausalLMPreprocessor( + self.tokenizer, + sequence_length=8, + ) + self.backbone = GemmaBackbone( + vocabulary_size=self.preprocessor.tokenizer.vocabulary_size(), + num_layers=2, + num_query_heads=2, + num_key_value_heads=1, + hidden_dim=4, + intermediate_dim=8, + head_dim=2, + ) + self.init_kwargs = { + "preprocessor": self.preprocessor, + "backbone": self.backbone, + } + self.train_data = (["the quick brown fox", "the quick brown fox"],) + self.input_data = self.preprocessor(*self.train_data)[0] + + def test_causal_lm_basics(self): + self.run_task_test( + cls=GemmaCausalLM, + init_kwargs=self.init_kwargs, + train_data=self.train_data, + expected_output_shape=(2, 8, 11), + ) + + def test_generate(self): + causal_lm = GemmaCausalLM(**self.init_kwargs) + # String input. + prompt = "the quick brown fox" + output = causal_lm.generate("the quick brown fox") + self.assertTrue(prompt in output) + # Int tensor input. + prompt_ids = self.preprocessor.generate_preprocess([prompt]) + causal_lm.preprocessor = None + outputs = causal_lm.generate(prompt_ids) + # Assert prompt is in output in token id space. + self.assertAllEqual( + outputs["token_ids"][:, :4], + prompt_ids["token_ids"][:, :4], + ) + self.assertAllEqual( + outputs["padding_mask"][:, :4], + prompt_ids["padding_mask"][:, :4], + ) + + def test_generate_with_bfloat16(self): + original_floatx = keras.config.floatx() + keras.config.set_floatx("float16") + try: + causal_lm = GemmaCausalLM(**self.init_kwargs) + # String input. + prompt = "the quick brown fox" + output = causal_lm.generate("the quick brown fox") + self.assertTrue(prompt in output) + # Int tensor input. + prompt_ids = self.preprocessor.generate_preprocess([prompt]) + causal_lm.preprocessor = None + outputs = causal_lm.generate(prompt_ids) + # Assert prompt is in output in token id space. + self.assertAllEqual( + outputs["token_ids"][:, :4], + prompt_ids["token_ids"][:, :4], + ) + self.assertAllEqual( + outputs["padding_mask"][:, :4], + prompt_ids["padding_mask"][:, :4], + ) + finally: + # Restore floatx to the original value to prevent impact on other + # tests even if there is an exception. + keras.config.set_floatx(original_floatx) + + def test_early_stopping(self): + causal_lm = GemmaCausalLM(**self.init_kwargs) + call_with_cache = causal_lm.call_with_cache + + def wrapper(*args, **kwargs): + """Modify output logits to always favor end_token_id""" + logits, hidden_states, cache = call_with_cache(*args, **kwargs) + index = self.preprocessor.tokenizer.end_token_id + update = ops.ones_like(logits)[:, :, index] * 1.0e9 + update = ops.expand_dims(update, axis=-1) + logits = ops.slice_update(logits, (0, 0, index), update) + return logits, hidden_states, cache + + with patch.object(causal_lm, "call_with_cache", wraps=wrapper): + prompt = ["the quick brown fox", "the quick"] + output = causal_lm.generate(prompt) + # We should immediately abort and output the prompt. + self.assertEqual(prompt, output) + + def test_generate_compilation(self): + causal_lm = GemmaCausalLM(**self.init_kwargs) + # Assert we do not recompile with successive calls. + causal_lm.generate("the quick brown fox") + first_fn = causal_lm.generate_function + causal_lm.generate("the quick brown fox") + second_fn = causal_lm.generate_function + self.assertEqual(first_fn, second_fn) + # Assert we do recompile after compile is called. + causal_lm.compile(sampler="greedy") + self.assertIsNone(causal_lm.generate_function) + + @pytest.mark.large + def test_saved_model(self): + self.run_model_saving_test( + cls=GemmaCausalLM, + init_kwargs=self.init_kwargs, + input_data=self.input_data, + ) + + @pytest.mark.extra_large + def test_all_presets(self): + for preset in GemmaCausalLM.presets: + self.run_preset_test( + cls=GemmaCausalLM, + preset=preset, + input_data=self.input_data, + ) + + def test_score_logits(self): + # Setup prompts, models, and associated expected shapes. + prompts = ["the quick brown fox", "the quick brown fox"] + causal_lm = GemmaCausalLM(**self.init_kwargs) + expected_score_shape = (2, 8, 11) + + # Preprocess prompts to get tokenized representations and padding masks. + preprocessed_prompts = causal_lm.preprocessor.generate_preprocess( + prompts + ) + token_ids = preprocessed_prompts["token_ids"] + padding_mask = preprocessed_prompts["padding_mask"] + + # Get the scores and assert their shape. + scores = causal_lm.score( + token_ids=token_ids, + padding_mask=padding_mask, + scoring_mode="logits", + ) + + self.assertEqual(ops.shape(scores), expected_score_shape) + + def test_score_loss(self): + # Setup prompts, models, and associated expected shapes. + prompts = ["the quick brown fox", "the quick brown fox"] + causal_lm = GemmaCausalLM(**self.init_kwargs) + expected_score_shape = (2, 8) + + # Preprocess prompts to get tokenized representations and padding masks. + preprocessed_prompts = causal_lm.preprocessor.generate_preprocess( + prompts + ) + token_ids = preprocessed_prompts["token_ids"] + padding_mask = preprocessed_prompts["padding_mask"] + target_ids = keras.ops.roll(token_ids, shift=-1, axis=1) + + # Get the scores and assert their shape. + scores = causal_lm.score( + token_ids=token_ids, + padding_mask=padding_mask, + scoring_mode="loss", + target_ids=target_ids, + ) + + self.assertEqual(ops.shape(scores), expected_score_shape) + + def test_score_layer_intercept_fn_exfiltration(self): + # Setup prompts, models, and associated expected shapes. + prompts = ["the quick brown fox", "the quick brown fox"] + causal_lm = GemmaCausalLM(**self.init_kwargs) + expected_embedded_shape = (2, 8, 4) + expected_score_shape = (2, 8, 11) + + # Preprocess prompts to get tokenized representations and padding masks. + preprocessed_prompts = causal_lm.preprocessor.generate_preprocess( + prompts + ) + token_ids = preprocessed_prompts["token_ids"] + padding_mask = preprocessed_prompts["padding_mask"] + + # Setup a custom intercept function that extracts the embeddings to a + # a variable from the embeddings layer and otherwise asserts on shapes. + embedded_prompts = None + + def layer_intercept_fn_for_testing(x, i): + if i == -1: + nonlocal embedded_prompts + embedded_prompts = x + else: + nonlocal expected_embedded_shape + self.assertEqual(ops.shape(x), expected_embedded_shape) + return x + + # Get the scores. + scores = causal_lm.score( + token_ids=token_ids, + padding_mask=padding_mask, + scoring_mode="logits", + layer_intercept_fn=layer_intercept_fn_for_testing, + ) + + # Assert shapes for info exfiltrated into the parent context. + self.assertEqual(ops.shape(embedded_prompts), expected_embedded_shape) + self.assertEqual(ops.shape(scores), expected_score_shape) diff --git a/keras_nlp/models/gemma/gemma_decoder_block.py b/keras_nlp/models/gemma/gemma_decoder_block.py new file mode 100644 index 0000000000..0a91655fc4 --- /dev/null +++ b/keras_nlp/models/gemma/gemma_decoder_block.py @@ -0,0 +1,189 @@ +# Copyright 2024 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from keras_nlp.backend import keras +from keras_nlp.backend import ops +from keras_nlp.layers.modeling.transformer_layer_utils import ( + compute_causal_mask, +) +from keras_nlp.layers.modeling.transformer_layer_utils import ( + merge_padding_and_attention_mask, +) +from keras_nlp.models.gemma.gemma_attention import CachedGemmaAttention +from keras_nlp.models.gemma.rms_normalization import RMSNormalization + + +class GemmaDecoderBlock(keras.layers.Layer): + def __init__( + self, + hidden_dim, + intermediate_dim, + head_dim, + num_query_heads, + num_key_value_heads, + layer_norm_epsilon=1e-6, + dropout=0, + **kwargs, + ): + super().__init__(**kwargs) + + self.intermediate_dim = intermediate_dim + self.hidden_dim = hidden_dim + self.num_query_heads = num_query_heads + self.num_key_value_heads = num_key_value_heads + self.head_dim = head_dim + self.layer_norm_epsilon = layer_norm_epsilon + self.dropout = dropout + + self.pre_attention_norm = RMSNormalization( + epsilon=self.layer_norm_epsilon, + dtype=self.dtype_policy, + name="pre_attention_norm", + ) + + self.attention = CachedGemmaAttention( + head_dim=head_dim, + num_query_heads=num_query_heads, + num_key_value_heads=num_key_value_heads, + dropout=dropout, + dtype=self.dtype_policy, + name="attention", + ) + + if self.dropout > 0: + self.attention_dropout = keras.layers.Dropout(rate=dropout) + self.feedforward_dropout = keras.layers.Dropout(rate=dropout) + + self.pre_ffw_norm = RMSNormalization( + epsilon=self.layer_norm_epsilon, + dtype=self.dtype_policy, + name="pre_ffw_norm", + ) + + self.gating_ffw = keras.layers.EinsumDense( + equation="btd,df->btf", + output_shape=(None, self.intermediate_dim // 2), + dtype=self.dtype_policy, + name="ffw_gating", + ) + + self.gating_ffw_2 = keras.layers.EinsumDense( + equation="btd,df->btf", + output_shape=(None, self.intermediate_dim // 2), + dtype=self.dtype_policy, + name="ffw_gating_2", + ) + + self.ffw_linear = keras.layers.EinsumDense( + equation="btf,fd->btd", + output_shape=(None, self.hidden_dim), + dtype=self.dtype_policy, + name="ffw_linear", + ) + + def build(self, input_shape): + self.pre_attention_norm.build(input_shape) + self.attention.build(input_shape) + + shape = input_shape + self.pre_ffw_norm.build(shape) + self.gating_ffw.build(shape) + self.gating_ffw_2.build(shape) + + shape = self.gating_ffw.compute_output_shape(shape) + self.ffw_linear.build(shape) + self.built = True + + def compute_output_shape(self, input_shape): + # Isometric + return input_shape + + def _compute_attention_mask( + self, x, padding_mask, cache, cache_update_index + ): + decoder_mask = merge_padding_and_attention_mask( + inputs=x, padding_mask=padding_mask, attention_mask=None + ) + batch_size = ops.shape(x)[0] + input_length = output_length = ops.shape(x)[1] + if cache is not None: + input_length = ops.shape(cache)[2] + + causal_mask = compute_causal_mask( + batch_size=batch_size, + input_length=input_length, + output_length=output_length, + cache_index=cache_update_index, + ) + + return ( + ops.minimum(decoder_mask, causal_mask) + if decoder_mask is not None + else causal_mask + ) + + def call( + self, + x, + padding_mask=None, + cache=None, + cache_update_index=0, + ): + normalized_x = self.pre_attention_norm(x) + attention_mask = self._compute_attention_mask( + normalized_x, padding_mask, cache, cache_update_index + ) + if cache is not None: + attention, new_cache = self.attention( + normalized_x, + attention_mask=attention_mask, + cache=cache, + cache_update_index=cache_update_index, + ) + else: + attention = self.attention( + normalized_x, + attention_mask=attention_mask, + ) + + if self.dropout: + attention = self.attention_dropout(attention) + + attention_x = x + attention + normalized_x = self.pre_ffw_norm(attention_x) + + x1 = self.gating_ffw(normalized_x) + x2 = self.gating_ffw_2(normalized_x) + x = keras.activations.gelu(x1, approximate=True) * x2 + x = self.ffw_linear(x) + + x = x + attention_x + + if cache is not None: + return x, new_cache + return x + + def get_config(self): + config = super().get_config() + config.update( + { + "hidden_dim": self.hidden_dim, + "intermediate_dim": self.intermediate_dim, + "head_dim": self.head_dim, + "num_query_heads": self.num_query_heads, + "num_key_value_heads": self.num_key_value_heads, + "layer_norm_epsilon": self.layer_norm_epsilon, + "dropout": self.dropout, + } + ) + return config diff --git a/keras_nlp/models/gemma/gemma_lora_test.py b/keras_nlp/models/gemma/gemma_lora_test.py new file mode 100644 index 0000000000..1cbbdfa67f --- /dev/null +++ b/keras_nlp/models/gemma/gemma_lora_test.py @@ -0,0 +1,102 @@ +# Copyright 2024 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import numpy as np +import pytest + +from keras_nlp.models.gemma.gemma_backbone import GemmaBackbone +from keras_nlp.tests.test_case import TestCase + + +@pytest.mark.keras_3_only +class GemmaLoraTest(TestCase): + def setUp(self): + self._init_kwargs = { + "vocabulary_size": 50, + "num_layers": 2, + "num_query_heads": 2, + "num_key_value_heads": 2, + "hidden_dim": 32, + "intermediate_dim": 16, + "head_dim": 16, + "layer_norm_epsilon": 1e-6, + } + + def test_lora_fine_tuning(self): + # Set up backbone and preprocessor. + backbone = GemmaBackbone(**self._init_kwargs) + backbone.enable_lora(4) + # 4 layers, 2 weights per layer + self.assertLen(backbone.trainable_weights, 4 * 2) + self.assertLen(backbone.non_trainable_weights, 20) + input_data = { + "token_ids": np.ones((2, 5), dtype="int32"), + "padding_mask": np.ones((2, 5), dtype="int32"), + } + targets = np.random.normal(size=(2, 5, self._init_kwargs["hidden_dim"])) + + # Test fine-tuning + backbone.compile(optimizer="sgd", loss="mse") + backbone.fit(input_data, targets, epochs=1) + + # Test saving and reloading. + temp_filepath = os.path.join( + self.get_temp_dir(), "lora_model.weights.h5" + ) + backbone.save_weights(temp_filepath) + new_backbone = GemmaBackbone(**self._init_kwargs) + new_backbone.load_weights(temp_filepath) + ref_out = backbone(input_data) + new_out = new_backbone(input_data) + self.assertAllClose(ref_out, new_out) + + def test_lora_saving_and_reloading(self): + backbone = GemmaBackbone(**self._init_kwargs) + initial_model_filepath = os.path.join( + self.get_temp_dir(), "base.weights.h5" + ) + backbone.save_weights(initial_model_filepath) + + backbone.enable_lora(4) + input_data = { + "token_ids": np.ones((2, 5), dtype="int32"), + "padding_mask": np.ones((2, 5), dtype="int32"), + } + targets = np.random.normal(size=(2, 5, self._init_kwargs["hidden_dim"])) + backbone.compile(optimizer="sgd", loss="mse") + backbone.fit(input_data, targets, epochs=1) + + lora_filepath = os.path.join(self.get_temp_dir(), "lora_model.lora.h5") + backbone.save_lora_weights(lora_filepath) + + # New backbone with same initial weights + new_backbone = GemmaBackbone(**self._init_kwargs) + new_backbone.load_weights(initial_model_filepath) + new_backbone.enable_lora(4) + new_backbone.load_lora_weights(lora_filepath) + + ref_out = backbone(input_data) + new_out = new_backbone(input_data) + self.assertAllClose(ref_out, new_out) + + # Test exceptions + backbone = GemmaBackbone(**self._init_kwargs) + with self.assertRaisesRegex(ValueError, "no lora-enabled layers"): + backbone.save_lora_weights(lora_filepath) + backbone.enable_lora(5) + with self.assertRaisesRegex(ValueError, "ranks must match"): + backbone.load_lora_weights(lora_filepath) + with self.assertRaisesRegex(ValueError, "filename must end in"): + backbone.save_lora_weights("bad_filepath") diff --git a/keras_nlp/models/gemma/gemma_preprocessor.py b/keras_nlp/models/gemma/gemma_preprocessor.py new file mode 100644 index 0000000000..8fc3beb48c --- /dev/null +++ b/keras_nlp/models/gemma/gemma_preprocessor.py @@ -0,0 +1,199 @@ +# Copyright 2024 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy + +from keras_nlp.api_export import keras_nlp_export +from keras_nlp.layers.preprocessing.start_end_packer import StartEndPacker +from keras_nlp.models.gemma.gemma_presets import backbone_presets +from keras_nlp.models.gemma.gemma_tokenizer import GemmaTokenizer +from keras_nlp.models.preprocessor import Preprocessor +from keras_nlp.utils.keras_utils import ( + convert_inputs_to_list_of_tensor_segments, +) +from keras_nlp.utils.keras_utils import pack_x_y_sample_weight +from keras_nlp.utils.python_utils import classproperty + + +@keras_nlp_export("keras_nlp.models.GemmaPreprocessor") +class GemmaPreprocessor(Preprocessor): + """Gemma preprocessing layer which tokenizes and packs inputs. + + This preprocessing layer will do 2 things: + + - Tokenize the inputs using the `tokenizer`. + - Construct a dictionary with keys `"token_ids"`, `"padding_mask"`, that can + be passed directly to a `keras_nlp.models.GemmaBackbone`. + + This layer can be used directly with `tf.data.Dataset.map` to preprocess + string data in the `(x, y, sample_weight)` format used by + `keras.Model.fit`. + + The call method of this layer accepts three arguments, `x`, `y`, and + `sample_weight`. `x` can be a python string or tensor representing a single + segment, a list of python strings representing a batch of single segments, + or a list of tensors representing multiple segments to be packed together. + `y` and `sample_weight` are both optional, can have any format, and will be + passed through unaltered. + + `GemmaPreprocessor` expects the input to have only one segment, as Gemma is + mainly used for generation tasks. For tasks having multi-segment inputs + please combine inputs into a single string input before passing to the + preprocessor layer. + + Args: + tokenizer: A `keras_nlp.models.GemmaTokenizer` instance. + sequence_length: The length of the packed inputs. + add_start_token: If `True`, the preprocessor will prepend the tokenizer + start token to each input sequence. + add_end_token: If `True`, the preprocessor will append the tokenizer + end token to each input sequence. + + Call arguments: + x: A string, `tf.Tensor` or list of python strings. + y: Any label data. Will be passed through unaltered. + sample_weight: Any label weight data. Will be passed through unaltered. + sequence_length: Pass to override the configured `sequence_length` of + the layer. + + Examples: + + Directly calling the layer on data. + ```python + preprocessor = keras_nlp.models.GemmaPreprocessor.from_preset( + "gemma_2b_en" + ) + + # Tokenize and pack a single sentence. + preprocessor("The quick brown fox jumped.") + + # Tokenize a batch of sentences. + preprocessor(["The quick brown fox jumped.", "Call me Ishmael."]) + + # Custom vocabulary. + bytes_io = io.BytesIO() + ds = tf.data.Dataset.from_tensor_slices(["The quick brown fox jumped."]) + sentencepiece.SentencePieceTrainer.train( + sentence_iterator=ds.as_numpy_iterator(), + model_writer=bytes_io, + vocab_size=8, + model_type="WORD", + pad_id=0, + bos_id=1, + eos_id=2, + unk_id=3, + pad_piece="", + bos_piece="", + eos_piece="", + unk_piece="", + ) + tokenizer = keras_nlp.models.GemmaTokenizer( + proto=bytes_io.getvalue(), + ) + preprocessor = keras_nlp.models.GemmaPreprocessor(tokenizer=tokenizer) + preprocessor("The quick brown fox jumped.") + ``` + + Apply preprocessing to a `tf.data.Dataset`. + ```python + preprocessor = keras_nlp.models.GemmaPreprocessor.from_preset( + "gemma_2b_en" + ) + + text = tf.constant(["The quick brown fox jumped.", "Call me Ishmael."]) + label = tf.constant([1, 1]) + + # Map labeled single sentences. + ds = tf.data.Dataset.from_tensor_slices((text, label)) + ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE) + + # Map unlabeled single sentences. + ds = tf.data.Dataset.from_tensor_slices(text) + ds = ds.map(preprocessor, num_parallel_calls=tf.data.AUTOTUNE) + ``` + """ + + def __init__( + self, + tokenizer, + sequence_length=8192, + add_start_token=True, + add_end_token=True, + **kwargs, + ): + super().__init__(**kwargs) + + self.tokenizer = tokenizer + self.sequence_length = sequence_length + self.add_start_token = add_start_token + self.add_end_token = add_end_token + + def build(self, input_shape): + # Defer packer creation to `build()` so that we can be sure tokenizer + # assets have loaded when restoring a saved model. + self.packer = StartEndPacker( + start_value=self.tokenizer.start_token_id, + end_value=self.tokenizer.end_token_id, + pad_value=self.tokenizer.pad_token_id, + sequence_length=self.sequence_length, + return_padding_mask=True, + ) + self.built = True + + def call( + self, + x, + y=None, + sample_weight=None, + sequence_length=None, + ): + x = convert_inputs_to_list_of_tensor_segments(x) + if len(x) != 1: + raise ValueError( + "GemmaPreprocessor requires each input to contain only " + f"one segment, but received {len(x)}. If you are using Gemma " + "for a multi-segment classification task, please combine your " + "input into a single string." + ) + sequence_length = sequence_length or self.sequence_length + token_ids, padding_mask = self.packer( + self.tokenizer(x[0]), + sequence_length=sequence_length, + add_start_value=self.add_start_token, + add_end_value=self.add_end_token, + ) + x = { + "token_ids": token_ids, + "padding_mask": padding_mask, + } + return pack_x_y_sample_weight(x, y, sample_weight) + + def get_config(self): + config = super().get_config() + config.update( + { + "sequence_length": self.sequence_length, + "add_start_token": self.add_start_token, + "add_end_token": self.add_end_token, + } + ) + return config + + @classproperty + def presets(cls): + return copy.deepcopy(backbone_presets) + + @classproperty + def tokenizer_cls(cls): + return GemmaTokenizer diff --git a/keras_nlp/models/gemma/gemma_preprocessor_test.py b/keras_nlp/models/gemma/gemma_preprocessor_test.py new file mode 100644 index 0000000000..f54a509979 --- /dev/null +++ b/keras_nlp/models/gemma/gemma_preprocessor_test.py @@ -0,0 +1,74 @@ +# Copyright 2024 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pytest + +from keras_nlp.models.gemma.gemma_preprocessor import GemmaPreprocessor +from keras_nlp.models.gemma.gemma_tokenizer import GemmaTokenizer +from keras_nlp.tests.test_case import TestCase + + +@pytest.mark.keras_3_only +class GemmaPreprocessorTest(TestCase): + def setUp(self): + self.tokenizer = GemmaTokenizer( + proto=os.path.join( + self.get_test_data_dir(), "gemma_test_vocab.spm" + ), + ) + self.init_kwargs = { + "tokenizer": self.tokenizer, + "sequence_length": 8, + } + self.input_data = ["the quick brown fox"] + + def test_preprocessor_basics(self): + self.run_preprocessing_layer_test( + cls=GemmaPreprocessor, + init_kwargs=self.init_kwargs, + input_data=self.input_data, + expected_output={ + "token_ids": [[1, 4, 9, 5, 7, 2, 0, 0]], + "padding_mask": [[1, 1, 1, 1, 1, 1, 0, 0]], + }, + ) + + def test_no_start_end_token(self): + input_data = ["the quick brown fox"] * 4 + preprocessor = GemmaPreprocessor( + tokenizer=self.tokenizer, + sequence_length=8, + add_start_token=False, + add_end_token=False, + ) + x = preprocessor(input_data) + self.assertAllEqual(x["token_ids"], [[4, 9, 5, 7, 0, 0, 0, 0]] * 4) + self.assertAllEqual(x["padding_mask"], [[1, 1, 1, 1, 0, 0, 0, 0]] * 4) + + def test_sequence_length_override(self): + input_data = "the quick brown fox" + preprocessor = GemmaPreprocessor(**self.init_kwargs) + x = preprocessor(input_data, sequence_length=4) + self.assertAllEqual(x["token_ids"], [1, 4, 9, 2]) + + @pytest.mark.extra_large + def test_all_presets(self): + for preset in GemmaPreprocessor.presets: + self.run_preset_test( + cls=GemmaPreprocessor, + preset=preset, + input_data=self.input_data, + ) diff --git a/keras_nlp/models/gemma/gemma_presets.py b/keras_nlp/models/gemma/gemma_presets.py new file mode 100644 index 0000000000..f63fef17fa --- /dev/null +++ b/keras_nlp/models/gemma/gemma_presets.py @@ -0,0 +1,66 @@ +# Copyright 2024 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Gemma model preset configurations.""" + +# Metadata for loading pretrained model weights. +backbone_presets = { + "gemma_2b_en": { + "metadata": { + "description": ( + "18-layer Gemma model (Gemma with 2B parameters). " + ), + "params": 2506172416, + "official_name": "Gemma", + "path": "gemma", + "model_card": "https://www.kaggle.com/models/google/gemma", + }, + "kaggle_handle": "kaggle://keras/gemma/keras/gemma_2b_en/1", + }, + "gemma_instruct_2b_en": { + "metadata": { + "description": ( + "18-layer Gemma model (Gemma with 2B parameters). " + ), + "params": 2506172416, + "official_name": "Gemma", + "path": "gemma", + "model_card": "https://www.kaggle.com/models/google/gemma", + }, + "kaggle_handle": "kaggle://keras/gemma/keras/gemma_instruct_2b_en/1", + }, + "gemma_7b_en": { + "metadata": { + "description": ( + "28-layer Gemma model (Gemma with 7B parameters). " + ), + "params": 8537680896, + "official_name": "Gemma", + "path": "gemma", + "model_card": "https://www.kaggle.com/models/google/gemma", + }, + "kaggle_handle": "kaggle://keras/gemma/keras/gemma_7b_en/1", + }, + "gemma_instruct_7b_en": { + "metadata": { + "description": ( + "28-layer Gemma model (Gemma with 7B parameters). " + ), + "params": 8537680896, + "official_name": "Gemma", + "path": "gemma", + "model_card": "https://www.kaggle.com/models/google/gemma", + }, + "kaggle_handle": "kaggle://keras/gemma/keras/gemma_instruct_7b_en/1", + }, +} diff --git a/keras_nlp/models/gemma/gemma_tokenizer.py b/keras_nlp/models/gemma/gemma_tokenizer.py new file mode 100644 index 0000000000..6a4bb76ea0 --- /dev/null +++ b/keras_nlp/models/gemma/gemma_tokenizer.py @@ -0,0 +1,108 @@ +# Copyright 2024 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import copy + +from keras_nlp.api_export import keras_nlp_export +from keras_nlp.models.gemma.gemma_presets import backbone_presets +from keras_nlp.tokenizers.sentence_piece_tokenizer import SentencePieceTokenizer +from keras_nlp.utils.python_utils import classproperty + + +@keras_nlp_export("keras_nlp.models.GemmaTokenizer") +class GemmaTokenizer(SentencePieceTokenizer): + """Gemma tokenizer layer based on SentencePiece. + + This tokenizer class will tokenize raw strings into integer sequences and + is based on `keras_nlp.tokenizers.SentencePieceTokenizer`. Unlike the + underlying tokenizer, it will check for all special tokens needed by + Gemma models and provides a `from_preset()` method to automatically + download a matching vocabulary for a Gemma preset. + + If input is a batch of strings (rank > 0), the layer will output a + `tf.RaggedTensor` where the last dimension of the output is ragged. + + If input is a scalar string (rank == 0), the layer will output a dense + `tf.Tensor` with static shape `[None]`. + + Args: + proto: Either a `string` path to a SentencePiece proto file, or a + `bytes` object with a serialized SentencePiece proto. See the + [SentencePiece repository](https://github.com/google/sentencepiece) + for more details on the format. + + Examples: + + ```python + # Unbatched input. + tokenizer = keras_nlp.models.GemmaTokenizer.from_preset("gemma_2b_en") + tokenizer("The quick brown fox jumped.") + + # Batched input. + tokenizer(["The quick brown fox jumped.", "The fox slept."]) + + # Detokenization. + tokenizer.detokenize(tokenizer("The quick brown fox jumped.")) + + # Custom vocabulary. + bytes_io = io.BytesIO() + ds = tf.data.Dataset.from_tensor_slices(["The quick brown fox jumped."]) + sentencepiece.SentencePieceTrainer.train( + sentence_iterator=ds.as_numpy_iterator(), + model_writer=bytes_io, + vocab_size=8, + model_type="WORD", + pad_id=0, + bos_id=1, + eos_id=2, + unk_id=3, + pad_piece="", + bos_piece="", + eos_piece="", + unk_piece="", + ) + tokenizer = keras_nlp.models.GemmaTokenizer( + proto=bytes_io.getvalue(), + ) + tokenizer("The quick brown fox jumped.") + ``` + """ + + def __init__(self, proto, **kwargs): + self.start_token = "" + self.end_token = "" + self.pad_token = "" + + super().__init__(proto=proto, **kwargs) + + def set_proto(self, proto): + super().set_proto(proto) + if proto is not None: + for token in [self.end_token, self.pad_token]: + if token not in self.get_vocabulary(): + raise ValueError( + f"Cannot find token `'{token}'` in the provided " + f"`vocabulary`. Please provide `'{token}'` in your " + "`vocabulary` or use a pretrained `vocabulary` name." + ) + self.start_token_id = self.token_to_id(self.start_token) + self.end_token_id = self.token_to_id(self.end_token) + self.pad_token_id = self.token_to_id(self.pad_token) + else: + self.start_token_id = None + self.end_token_id = None + self.pad_token_id = None + + @classproperty + def presets(cls): + return copy.deepcopy(backbone_presets) diff --git a/keras_nlp/models/gemma/gemma_tokenizer_test.py b/keras_nlp/models/gemma/gemma_tokenizer_test.py new file mode 100644 index 0000000000..1c617dd937 --- /dev/null +++ b/keras_nlp/models/gemma/gemma_tokenizer_test.py @@ -0,0 +1,67 @@ +# Copyright 2024 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import pytest + +from keras_nlp.models.gemma.gemma_tokenizer import GemmaTokenizer +from keras_nlp.tests.test_case import TestCase + + +@pytest.mark.keras_3_only +class GemmaTokenizerTest(TestCase): + def setUp(self): + self.init_kwargs = { + # Generated using create_gemma_test_proto.py + "proto": os.path.join( + self.get_test_data_dir(), "gemma_test_vocab.spm" + ) + } + self.input_data = ["the quick brown fox", "the earth is round"] + + def test_tokenizer_basics(self): + self.run_preprocessing_layer_test( + cls=GemmaTokenizer, + init_kwargs=self.init_kwargs, + input_data=self.input_data, + expected_output=[[4, 9, 5, 7], [4, 6, 8, 10]], + ) + + def test_errors_missing_special_tokens(self): + with self.assertRaises(ValueError): + GemmaTokenizer( + # Generated using create_no_special_token_proto.py + proto=os.path.join( + self.get_test_data_dir(), "no_special_token_vocab.spm" + ) + ) + + @pytest.mark.large + def test_smallest_preset(self): + self.run_preset_test( + cls=GemmaTokenizer, + preset="gemma_2b_en", + input_data=["The quick brown fox."], + expected_output=[[651, 4320, 8426, 25341, 235265]], + ) + + @pytest.mark.extra_large + def test_all_presets(self): + for preset in GemmaTokenizer.presets: + self.run_preset_test( + cls=GemmaTokenizer, + preset=preset, + input_data=self.input_data, + ) diff --git a/keras_nlp/models/gemma/rms_normalization.py b/keras_nlp/models/gemma/rms_normalization.py new file mode 100644 index 0000000000..ce9bdaf880 --- /dev/null +++ b/keras_nlp/models/gemma/rms_normalization.py @@ -0,0 +1,40 @@ +# Copyright 2024 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from keras_nlp.backend import keras +from keras_nlp.backend import ops + + +class RMSNormalization(keras.layers.Layer): + def __init__(self, epsilon=1e-6, **kwargs): + super().__init__(**kwargs) + self.epsilon = epsilon + + def build(self, input_shape): + self.scale = self.add_weight( + name="scale", + trainable=True, + shape=(input_shape[-1],), + initializer="zeros", + ) + self.built = True + + def call(self, x): + # Always compute normalization in float32. + x = ops.cast(x, "float32") + scale = ops.cast(self.scale, "float32") + var = ops.mean(ops.square(x), axis=-1, keepdims=True) + normed_inputs = x * ops.reciprocal(ops.sqrt(var + 1e-06)) + normed_inputs = normed_inputs * (1 + scale) + return ops.cast(normed_inputs, self.compute_dtype) diff --git a/keras_nlp/models/generative_task.py b/keras_nlp/models/generative_task.py index 9a461926e4..598217d964 100644 --- a/keras_nlp/models/generative_task.py +++ b/keras_nlp/models/generative_task.py @@ -101,12 +101,7 @@ def compiled_generate_function(inputs, end_token_id, state): for v in self._sampler.variables: new_v = scope.get_current_value(v) sampler_variables.append(new_v if new_v is not None else v) - state = ( - sampler_variables, - trainable_variables, - non_trainable_variables, - ) - return outputs, state + return outputs, sampler_variables def wrapped_generate_function( inputs, @@ -115,18 +110,20 @@ def wrapped_generate_function( # Create an explicit tuple of all variable state. state = ( self._sampler.variables, - self.trainable_variables, - self.non_trainable_variables, + # Use the explicit variable.value to preserve the + # sharding spec of distribution. + [v.value for v in self.trainable_variables], + [v.value for v in self.non_trainable_variables], ) inputs = tree.map_structure(ops.convert_to_tensor, inputs) - outputs, state = compiled_generate_function( + outputs, sampler_variables = compiled_generate_function( inputs, end_token_id, state, ) # Only assign the sampler variables (random seeds), as other # model variables should never be updated in generation. - for ref_v, v in zip(self._sampler.variables, state[0]): + for ref_v, v in zip(self._sampler.variables, sampler_variables): ref_v.assign(v) return outputs diff --git a/keras_nlp/models/gpt2/gpt2_causal_lm.py b/keras_nlp/models/gpt2/gpt2_causal_lm.py index e154c88bb1..b0bd529da4 100644 --- a/keras_nlp/models/gpt2/gpt2_causal_lm.py +++ b/keras_nlp/models/gpt2/gpt2_causal_lm.py @@ -298,6 +298,7 @@ def next(prompt, cache, index): mask=padding_mask, end_token_id=end_token_id, hidden_states=hidden_states, + model=self, ) # Compute an output padding mask with the token ids we updated. diff --git a/keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm.py b/keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm.py index bef32017ea..b1df4a6706 100644 --- a/keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm.py +++ b/keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm.py @@ -188,6 +188,7 @@ def next(prompt, cache, index): mask=padding_mask, end_token_id=end_token_id, hidden_states=hidden_states, + model=self, ) # Compute an output padding mask with the token ids we updated. diff --git a/keras_nlp/models/opt/opt_causal_lm.py b/keras_nlp/models/opt/opt_causal_lm.py index 9715bc6b75..2ca8ee07b4 100644 --- a/keras_nlp/models/opt/opt_causal_lm.py +++ b/keras_nlp/models/opt/opt_causal_lm.py @@ -294,6 +294,7 @@ def next(prompt, cache, index): mask=padding_mask, end_token_id=end_token_id, hidden_states=hidden_states, + model=self, ) # Compute an output padding mask with the token ids we updated. diff --git a/keras_nlp/models/t5/t5_transformer_layer.py b/keras_nlp/models/t5/t5_transformer_layer.py index 697af20899..ddff7a164a 100644 --- a/keras_nlp/models/t5/t5_transformer_layer.py +++ b/keras_nlp/models/t5/t5_transformer_layer.py @@ -131,8 +131,7 @@ def call( shape = ops.shape(hidden_states) batch_size, length = shape[0], shape[1] causal_mask = compute_causal_mask(batch_size, length, length) - attention_mask = ops.cast(attention_mask, "int32") - attention_mask = causal_mask & attention_mask + attention_mask = causal_mask & ops.cast(attention_mask, "bool") x = hidden_states # Intermediate result. diff --git a/keras_nlp/samplers/beam_sampler.py b/keras_nlp/samplers/beam_sampler.py index 87948439a8..297ec203de 100644 --- a/keras_nlp/samplers/beam_sampler.py +++ b/keras_nlp/samplers/beam_sampler.py @@ -72,6 +72,7 @@ def __call__( mask=None, end_token_id=None, hidden_states=None, + model=None, ): batch_size, max_length = ops.shape(prompt)[0], ops.shape(prompt)[1] index = ops.cast(index, "int32") @@ -167,6 +168,7 @@ def gather_beams(x): body=body, loop_vars=(prompt, cache, index, log_probs), maximum_iterations=(max_length - index), + model=model, ) all_prompts = unflatten_beams(prompt) diff --git a/keras_nlp/samplers/contrastive_sampler.py b/keras_nlp/samplers/contrastive_sampler.py index 8b3d52d9a5..4259167c8c 100644 --- a/keras_nlp/samplers/contrastive_sampler.py +++ b/keras_nlp/samplers/contrastive_sampler.py @@ -73,6 +73,7 @@ def __call__( mask=None, end_token_id=None, hidden_states=None, + model=None, ): if hidden_states is None: raise ValueError( @@ -209,6 +210,7 @@ def gather_best_token(beams): body=body, loop_vars=(prompt, cache, index, logits, hidden_states), maximum_iterations=(max_length - index), + model=model, ) return prompt diff --git a/keras_nlp/samplers/sampler.py b/keras_nlp/samplers/sampler.py index 2101c9277d..3ecf16ac28 100644 --- a/keras_nlp/samplers/sampler.py +++ b/keras_nlp/samplers/sampler.py @@ -92,6 +92,7 @@ def __call__( mask=None, end_token_id=None, hidden_states=None, + model=None, ): max_length = ops.shape(prompt)[-1] # Make sure `max_length` and `index` are the same dtype. @@ -133,6 +134,7 @@ def body(prompt, cache, index): body, loop_vars=(prompt, cache, index), maximum_iterations=(max_length - index), + model=model, ) return prompt @@ -147,32 +149,68 @@ def compute_probabilities(self, logits): probs = keras.activations.softmax(logits / self.temperature) return ops.cast(probs, logits_dtype) - def run_loop(self, cond, body, loop_vars=None, maximum_iterations=None): + def run_loop( + self, cond, body, model=None, loop_vars=None, maximum_iterations=None + ): """Run ops.while_loops with a `StatelessScope` if necessary.""" if config.backend() == "jax": + import itertools + + if model: + model_trainable_variables = model.trainable_variables + model_non_trainable_variables = model.non_trainable_variables + else: + model_trainable_variables = [] + model_non_trainable_variables = [] - def stateless_cond(variables, *loop_vars): + def stateless_cond(state, *loop_vars): return cond(*loop_vars) - def stateless_body(variables, *loop_vars): - mapping = zip(self.variables, variables) + def stateless_body(state, *loop_vars): + ( + sampler_variables, + trainable_variables, + non_trainable_variables, + ) = state + mapping = itertools.chain( + zip(self.variables, sampler_variables), + zip(model_trainable_variables, trainable_variables), + zip(model_non_trainable_variables, non_trainable_variables), + ) with keras.StatelessScope(state_mapping=mapping) as scope: loop_vars = body(*loop_vars) - variables = [] + sampler_variables = [] for v in self.variables: new_v = scope.get_current_value(v) - variables.append(new_v if new_v is not None else v) - return variables, *loop_vars + sampler_variables.append(new_v if new_v is not None else v) + state = ( + sampler_variables, + trainable_variables, + non_trainable_variables, + ) + return state, *loop_vars variables = [ops.convert_to_tensor(v) for v in self.variables] - variables, *loop_vars = ops.while_loop( + trainable_variables = [ + ops.convert_to_tensor(v) for v in model_trainable_variables + ] + non_trainable_variables = [ + ops.convert_to_tensor(v) for v in model_non_trainable_variables + ] + state = ( + variables, + trainable_variables, + non_trainable_variables, + ) + state, *loop_vars = ops.while_loop( cond=stateless_cond, body=stateless_body, - loop_vars=(variables, *loop_vars), + loop_vars=(state, *loop_vars), maximum_iterations=maximum_iterations, ) - [ref_v.assign(v) for ref_v, v in zip(self.variables, variables)] + for ref_v, v in zip(self.variables, state[0]): + ref_v.assign(v) else: loop_vars = ops.while_loop( cond=cond, diff --git a/keras_nlp/tests/test_data/gemma_test_vocab.spm b/keras_nlp/tests/test_data/gemma_test_vocab.spm new file mode 100644 index 0000000000000000000000000000000000000000..a049c032c296607c2c96febc6e7d2934884b85c5 GIT binary patch literal 237805 zcmZU+3s{ubdg%Yo3>OVI-G)t!B3d`cSYwSfR;_@s#;w*+<91VH4aZnx)e<*hDhM9>q)>vn1JceU9hNDwrja57G7>@B6{+{2$B;BWf zp6B8HvK*6S~Kg`>%8N{Xg!V|H+;ItDNtzYafey zEb3q0HSFKLGx3REcljSncx(dBzy5V@!LJhENsRpHdyk#lf4zVABacTn{7vo~kDRfO zMRLQQeZL<%FMrRDoZt8Ob#DHzULE)N$um#rX2yJ^)qSGXe5%!cu9bh&My9C@{V$}- z`2Rtw3>WJj73@jVNAZtM_|@M1+kW*oIXixG-ybjCS`nivujn-p_)aUmks?R_0R2n! zbPej1v0EpaQnH}Yi0+7pwn8MbRwSZYB)UN)>X^vHHjxQUA~AN6$1EaoeIl`5kpzfG zpL|+m(n*miL6M)F6-gWvdHjM%(y+)+heW1cqcuY!({77Ac|#<5RAl-ck(4o!8TUoD zFP$hk#y?_a#DYX6Q$s&K1HbL~1-_Rg(nKT!XkCHuFDPFA=!(y%I!L(N=Wm!5gETdg^YUN4t z_?9L@Dh72@ep@G16Gg1t`!xQC(f=4(u}~+~=z-5gwyHSc)RUs$K;7tf5?35?ri+v$ z>m)EqD?!RtJeTnMbz(L|$vML6yT}7vL^p8FLYrPj4}7JQ;450mBCb~2Z7Xs+?1B*$ zb`SA<5h2BsHF6X8Ui|MOb<;#20}s&M)KTDr2niA{FhL`IxUJ-|jC(>dPp-Z>M@Gum zYX2p6sn(3Ysahi*q!aW*8tR5LKE}eHe^pzL-xT5vf4fU-m>ey|cQo?#6J^?pEb8@| zMy7Gi6w;6p+;NoofXD&zWI*a`H4<2=k>KMRIebzhMfm&4i#bsvr;aAcKOBEtP8*Zt z;}?_U%&jE(NBvKvhV&{4caDDaPgj4+y+4(u)cN^=nexxar^>C-{t5DMrihFCTj(bqWLJ8Wn25KmR-`qVJXMIC#J{+W zwiy($YoaBHyPCdPd`u(fh$9#qNgs(61AeAbjg)&ea*#C7Q4d|@^)P+odGhq{&MK?V8RM;xl}AI7gZPUIqCs@92g-k|=tc9*!# zbLls!#FwR!O8i%U*nn+ z+3Lbx)qSMvK?c4f{73XR^mEF;R40#97Q;22+~FS64W0b6s!NMbE^>VgKRfQrD(o?A zc~~chz102B$lFX%_WTUC%UEinFX*$V+btsZxUYor9i|`L*Ox=X_&8(t}*5d#uWQXo!rFF%$TH`fi0uoC7u@Cdg{TDEYgKuK3}As zGPKmjG({2j@*xYE?Vm6CrF@`zGI7)GTPFVl|X+_kXYKVSQ9HHxb?K1fw7~7 zd{xoLhi{A2qK{C9O7c~{H$f`6_b1%bqwY%;c~P|k_4Sv?X_R*g%!Apm2x34#momXa z?wLX!jR%<9kcLe9P7~t~{^l|Ax0H5ao}I0(Q6IrKwe+PpnZxy$iNE!1f|!NKJejvJH?+|w!Jwcl zda&nd=(4#TCMCO z|F@|7gTz&dZl?}Z86ObFfS>sSZH@my2$V;NA)mOxsIIw)UzqoQsTCdl_bvR4*vZ?- z7~E4J@M*jRo~Aq!DZ!T`B}`iX3j*)OGtY52W7zu;2BT%+Bg z1d8Dj_ZpFE4X}pzD=f?*L)72dXjy@KGi*@lKC6*$39BEbzHW&8T#c7wX~U)}#dk@{@H?xf#n$isGY`ksEB$P&UOQ&$~?QEQV0=!;<+ z{w6rXy66|AS)5G$9bo>R8qHY9ct(FR&87aTqh*x*y|stFG!!M}$cjSJn7Et^k$;b$yl&q02QOSuo+q6|CIHom->1;TK)xp1MMGpB3Xhn zwW9V-|BhdIqv_dR$3VYlE{}z zLyf0_S4j8ODEWrCQs5F{tLJLSJog4uxSVOVn|GpM$#~3 zr^sPLikMcWP=4BNH*-T~iWn>@@)T*6v~iuVhcC>KdALok6y_$b9|P*C#EZKWy&^^K z5uaLT{gyIUaQz`}HK$Ibe9?^E`Z(HajD0-1c_>B92WE(wwoBl;eQ}DkY@Z=5r1v5B z8wuNvG~@qAq-!UBV;*fiD@D4}XGj-mzQ^_JlqZq<`ytRxdv{KkP55_Fo-N2m{8K1@ z5-fl@pzovnP&|uzWX)!zUY6irR)HfQ_3QZ?6y zqojJ2@Dd@xTZ{`kDGzn^7u-`oxI*GvtL}e-x?F}$;#UNLmx(J?WFhUT=3lkuH!su4 zv$&T)3t?Ke5HDv0<=B#WqfQ+7y+T|id*h^~L8RG^Ke8*1aLkXNbFIaNKLk{oj30)< zX!%E_bN#;|*HHa`gS1zOyoS^V2}3&uo>%kx3^9&sdC+9 zTC^09X9M%M9?}U{e4q0V&I8mMSkD=P>I-F|C^^KvM*3nE;ooA;uRO;3;Q;ep1LG`p zSG9$;!`^6lNZbKEN&m8h!gftP4{s{F&8wcoH!LK4Dnrlb!H?ND9 z8e{@)vo~6jkkiPqLZrx7(#9=?^pD?V=V7+4BL*V(;Z#p)e zO&hZoSN7k_eKs(mmt)%%E{(k9)kxLcL>VBynX3KYz=RhAEOO(IknC|EJ z8AmDiQ{4Y_g#06Z4#w}+LdGY`(Y{9`S(Krh^@3|I{bQm=B8mIoqD24ia3WG8&UB3o z5Qle6#5crVfH-;udD2o+wHt6I;g-+arQTu_{@Xv)r()g12uESEURg_|D zUq*|+NhcFX?|-QBP^rN$W9hrGQn8Y`CX0O!ec~-Bb+N7oQ!QtJ2AxEZ_h@(w-g}yJ zTiT|~6FIswPTmo%9H*TdEE;*gf^iz&JEoCi$iPhI zUA6yz4*4>YN(*up8yMgmO(mpZPa2mXxy0AcJ*AN%1?bzrS4!WQ%UM8bg5=;Xo|Pb_ zb49vIrosxo5A=s%9E_FH8}V|K#xV`YiwD1BJV!W%JO@3nigf+R z#{HayA*-Jt;Unyo4Dqc0>Yis#p&r_Lg~K82{5ESQ!fzn>O=Rho1SzTD zOoB!=td3`26E9)>E$n?tw^J8iKQ6Oba`b)7nH@$QWY0LhpE^k&!QcOO+zEk_O;BNZRQR0|1k@M7_O5mk<_WAUS$Kx31AESS8j>X0( za5V{VFO8AlHxtJD*H-eN-^iQ_*JqNix9LB$RTw?+Iq}?LO|JX}f6@%{Tq($zLhzTw zznipQV|~f>SFyFD#~52aS}uC*-7rL7Ag{q!3HJu!-o(#NUxw11v|$7KM*QGth;fOv zz8V{9PP32a{+q;EKI>6`ysOe@4W^#a1iOWPPkvc5G*UOE6FHAYJ|K(^Mj!^Jz%)pJ zXmCjs2;*PFt$}YgX5jMK<7PRXX*%SzXWK`rb$B*Q}=939D>t z1=s2bH*SB#+xY$c>+x^-{{Bts{u|Wa9r`ZuIh4Q3KWydNDs_D?*Qrz0S4u1znWFMf zc~$yJ=rJ&daDh*#k1w=xf-+xJ=`JH*a2b6BF2NP$PZ?BsuA|RW@t?*$NSmmA8tWiE zx&gmZ?kzdXI7a*~RgcPF<&AU~;isd$OzM98RG*H)U2}}`_c5ocHr+|sbnJ;RHQAgI zD?h>oU#Fk%kCdaDG^yiS5d>L-R;qBI+NmB{0?SEffkOGGKFU+zTgv${Hi+%@k}m`E zy8a06kEtWq8+dvQ6oMTfh`%U{2RF zN2vNM4kD?eVl~Im&-7b>xn8w@C2@@BpZ!HK+#aZfDqGe@cOjbi#r$+Z=dK-eTmfhF8CmpmRsu2OFDy$1Qm z_~|L9VI^h7oy&C-G6$(o#eeP!sj22#p~%yOJ&OGYKBp}?_t{67nk@3B`YHMI(55B$ zsW49B7$5hW9_3y2V}zK(E}RXpDO&y zsgIS&LQws;9;wE+CS+L%8>al3>KgfIQ4$+1r_IXe(ythQHYq=(%BLN@bUu5M56I7J z0cA>9Z>VoPGo?yc3x{CZSuodRK|U3pB9f_neiF#T-4(~;IE!dnbaC^saA4u2O_jmNuH{>UVyu5D*0rb_zn6I_U`4x z`5w=)k3FrETKwKuw)BFQ`XK)A>+d}k&-eN5Q@Zjez5#U8DDNeFrjs`2(-GW5l$qC# z^p}C{o&M-z>T{vU&~B~#PuxSqeUx&4iC#O4aRz(%%Sr6je#+J;N%Ws1RUGN)uFrMS z&E8qXG4zsFUcpb@JM>H9ApIq{ukPnugT41aB6Wkjs@KX9{0$$_?zgn^;U}D1zsz;Q zH>&VPtvde@Gx~?$YUS!ToZAE_`=>mE#LszEC(Yc`LLS~>o%y%;`A&-DA?uKDAd8?3 zl>Gz-R)~)Udo}KXH)6$$-v3Rkczzixy~y{JKlJp*%11~S51dYXz;ntEVx^DW|V2)=5vgPTnD(F66On_WGHOLD<;) zdpH9?9*4$Kjr^9qo$0t%e0Gs{Eh2SZ+W%S3-3~92K<`rKx24ips}VQ-s)4Y_M>%up z(n!Y}oU@Vc$8ap2@f%rB-+6zc$bUm0f~#w_>K!_{yY35kpGS}y`qy* zq=WYIVIOtW`vk6CwP__5S@tb^pj(u$TPu&_b`W33+q6j<;c2f)_|=i-)z2A6j5>Lj zIQ$YT4SPk7ZznmfchR4mFKVTh@Y7YCjE7gL7bpE*&96`4_rvw4(PzTXK&4rBnDntB zl~yfw|1LK74A;89qF#wp<#(wH`-+3)?1aeoijIG>X={GNli=!ZSe zVsF&v^SIZ7lj|>hFSjF)Q)dmZ3H`4?JwxA(+y}n_12nFqKO?K)HFy^q;0g@F*YL0K zA(XHdGThh53G_br2+qJigBA4MkNQn$0`*J2?%~HfQgVd0et;el`Z|4J~j=aGbD-p8!PkA;E+h7Omf(Gg;e0aAUZ_-E(?p*lYG4|-l0yqdo&{Dw~1!;n| zb*#~mwl>yy$Qq~vXDWRc*$8InVZY~I#`)1g=1$yJ=)2E4U^46VshnTncB*jT0evTP zPdW1udI|HC@t2I1FGt8J+(w><>xY?7=d(Ux9P-h(&f$L^OxRM{@d$A;_g=((34D~- zH$76^tO;C0I=PIWhdH?a4$l@}x~MT;<=l`pijQkv;vE1V9p8fOHpl7Y3gL}(#1Z6m zFms=cdE5>igmFy8{oypN{fj5H?WAi><^C6@if`?ca+B~rPJp|wKB;wmmaKI@GeaAu z&BFMHKH$7_>66+K#;ktfJoMG+THiO*wSJkTt&X^;dHc)%t-1O<&sQg8XzvpK9z1}D z;ImB^9oz9}h=Bx{0!c6p44r$aM9$}?|3ZpCFObri?~CzmpO_|`$435DDqdTnDZl!( zrt*nHnzufmqB(NaqB;EaUumkoyQ%s6m#Q?KgGth**)1oI_1NgK3hMu~NGl-Q7V_Us);qQp_kdA=Sye?liK zNM{wSf%UKfHbF1v_r1mVi4sO12`mfeZisJyGHzUpsK`0$&^P!T@-|Ox!v6=fXZHfaWdK z0saTkts2TUKT3+wOTckES~|cEHfSY{w)q-q$BwM&oF^gexI2&z?ApXVl~4sWPzS|c z+5q&>+rW9co&IIvol(ZtfqeQ`A-00_-s z#)B553+V(3whl_9tE=z%V7nqg5Dd9ZKERnW9%F6Q2*7`Klm6s{m>cW`5>GCCv=1BwupO(xdujJqa`#G zo?#515G`wnb3JT;O|TiZ!Zz3e;rF6t7cxib<@D(at>mKb1E2m$DL}gMbHPD$&raG7 ziqK2I&ly?E$Ta_00GurkSV}AN~r-kvy&iI3LdKrJvUBvIMrSD}kehend9O9V=3t$m6C$r|E z{gpu)A^HN5n!_Hp3m3d9w2d6;2JAMYK zN2=!)z1VXf^iQOH;m7eA-4C6EJmVeaTo~!xL;W)zyTFa!vz_k`df~Ga8CeRk)8rg+ zpNETZ8Lq$xT!)+BOPeTRWY{=O6y_$a+{In?RfL$>oZUljE~Ne70lHO{jWRw&*Ks`J z7^8229c(Mf6Z7OHu0?Y#1`@zs!FY5eN~WND7$^F1C!tRR|0v@TZZG4=05S!?*)Rv@ z!2(zWOCW4amgUG0>%tYtk?$gS8I5{NezbmBga#w^A|2jk!x z!mNjJ`Wr3&0^De_S;@JVaAP1T&qQ!EA_b<_{yua9vdmnlMIEc3cx81<{0eKKVXNdI$vIN}M zc&89q39dWj%S*nf%PQP8Fg|Z%n*+7fA>)5*GUZQwv_`QaZIs(iIXcL%gS^&pPdzk( z8CsyYk2QH0=kCbf+w{Lu`ri@yA9A1p`$zh4`;nc**A++ksaNMU`XBA?Lb{=65c{`8 zij_DW-~)A+YzaJ@OJnT13$EIds8_Tefh1kl4NI9kMyQ7se$aCQ4 zIs19!MR0{!Z!?E`4(Q|(?#tj?s1tuWYkzFg3$3Zx-(vb`I`)UO%_r_^=6`U&748{< z>u?intlz`PyKoPBkLg%{VdECk*-O0n#Eb3wkbc~qcXZNqpLZKhG6q0*7h{8$a{w3n z&o-SrAdZKiTY(Kg3?#r52)8~dK3>cY9p&9U+#}POZ{z5{dGueHhM##|Rzo~{hj=leSAwH7UOJA&(?;=PLymvvG?DtirmMIf zeqy53AnTwWM$+P>5gB?ZR?NsU&WcO}oJpfwp_O>rh^PH*yjVF%wQ)ddKS>(k7>ySr z&n+B;b3zYzzz?V34D_B%kUr=K4-9}8eBd8VkWSt^=vo~wCpb`cBD=HW#YOqt&;zX- zNq-CJ6VBQ|`jpR(-hp&@B7NfSK{_MjG%(3ivb_%T^49$0~U75J)&7X~z1@xmJX)`OWn z@CKxXIeQawGc+I2%2wnyuu{%;uxC<6aFE9iXdR^d!(5xsek+wZF`c;)*@4@E+`+xO zAO~_`9~^`tD1qKm#@`CY-v;cfmiF(W{lRyD{Rec?AG)rw{tB`Fy3hD~gYkD5`y6Ea z^)mN=jcpOn$m8)+iTq(-S%qF~=WJh?*Kb8i4emOqheim$5-(zK2w~X27XW$$-nR5z|=iwq;0ylmx=HAQbo+jon=GQCeBj9Jx z;IlJ#!GMeR>i04p^M2b8{m=KZPOcNK>|6TVYka?j9)`Pc4<5kC^AYk889L1V-4Z3b zm5jX*0}0TKpM|)mpj(*>)qW)jeHu8(OUHcXNU)K|6#Sf=Gt5TL0r$FSnTK2eF5buJ z2M;X5y##!VSr;th{9rZfgmm`*#L>na*^acbp0PC$_YuZgyo7@z0K^8osf<7+1Ne}VZIP7E^ta;=*%u0GZuZLB{ir`gNMIS77MWIL2ipgwut>nJ2|dF1f`dDTS8Ho`dV^i$+6aJMlBAalWmfB#X^y+tw&8J@(q zCCJ$@2j;;72+@BQAxEC)-AJT5yVXNjPoG`JdolQ}f@th-4KmbOBxh9mpS>yTaW8=l zP}WO-<9vP-`exV)+h7Omg7C~YB?sxOD3n~}J}7{L;675w{-IEc&^@aUNk5dJo4}9m z18>$L)*^@aR;5rX@vnj!sDtLrLa{(Sy0!F>v_m7h864;xV6QzSHc;b_bK#*M#~=5d zA~Et@qJ^-oS)?5-63>OAN8{*c>Rhgfak+?fLJ{kPB5A!=ByFQb(mqxsR;2BAk=PfL zM*1OfAgwA*a6%7wzz?V34D=2ilD-h>fCmPyQMMbD@z^2pLuck8zRNx&C%_5aqld(G z|B$!|(*tJmbB=h}XR{H0KA$@G5>{eKz#A9Sne zD-Z(-Fn*4Q9N))HX8$`G`=5&a&%*v;V32uvm~jm05Au9)oqER^``?27Lw6?jzYzO} z9`JF#WX>xbe+HrE|K?JyOe6jjuvBB071$+|@m$vw%bb1WkzLyGv%_q{89(8@qd+A4 z8=cI-?LxY5>tr7K0$2q7d)XiCWZh86_=NOZ4 zvR;?^rZ?sN`)^9|_SdAObdOvl?n_WMIbTd*QL-PDj%u=^C3Zo!64Zd1I=a&^#e7&CjH}aH<%22I+(G?MKa7(PNT=*YzR`Y_ z@5j+~&tn&Zw97Ei)|W)X|F+^ z2NOpD9E2h$0TWb0_&DRiC)fpg4LBF`%pK`+@k|NX27RLYEand#xC+(wgV%^KWFrR&4A^TsX1NScOZ2>DdzzIFz0YCJPvi}`pkBaoH zWd9A`b?m?M*ndN3BKzND_PA*cg9M|C{gyAmSg9q>semwuho;@(I zhdKt|Fm*i0`frf(TPQzqIh!ayx(nUiK>4YEb8@`sRx`)KJ!~=t*_=-KApzaGoxD-r zDd!hzf&N(Z7n_x3+g>A3{!XG8bE@Tem!agW~gW%k&lOm)#dw1vaZWX!- zDnXsWdtiY6@1_6yZZQ9b_=b>cHBblj&p1luv}20G9k(21>& zpYLUIzPFn5y_K9VtRsDFs|R{#aXtb4;DG_~f)D(lXN|7j1ytXmsOJx1>cm01PVi0Q zdBi6?S40neN_!$t!5KIQ=b@dvSz-MA{T0ToldKEQvQA(PWFO@Bj0j_1P2WXchGxdA zE65SBWbvL5@+O4gF5H6$@DRdA&U$PS68e-r@^kKaobPv$F^~XLz)3$#LQaDem<{eR z=6{%nuJ$DTV7wY33vm0JXm>At6b7I*lKDOP(H_$}|Iwb>p3eNfnE8D!dsOmK%y@6O z#@GaNh+`3yO~((5+l=hxOC`Rc|LLnxX5q+ z5#~463%hXVfG?kQMav6N2;@u&N#|Hneu}Z|L#%BPdGQS2e-M4GZgk`25eNlW4ng> zZf>LeO*}Ioek<{}pN*0lbl-0F|Kzb_A$dgFPDe>S{?0qRYqa~(`G7l{^&oy`aAnc< z>ll}y1-BJ^>Gb1SwEZ&rz+&RV&k07xNS}VX^q_}0=k_2+gfdL{Bk`l30yW>9L7s#2 za1k!SWiT_RUO`%hv@(Lc4$T*|auXQ_>uGEl?0wjV3maiQt|NVNt~V~ZMNx-!{= zfRnw+iTTX^sjLB(G51eq4S?Q?y|$6hcCcO}?J@F@$oXF~brZ+$(yhVXAOWU85=?^> zm<_$Gv->7e&xH4I-@roZ6~_1fqm2K|2|w=tX`AlV*k&gCe`F7BQT9PH?|1MFmbk*a zGdT~r02V3t$5CQ@ZMuZ`K4A&&kxtH|kZOOhLb;!(PfcZ?g1!dUL+P!jnOjMN^}q() zp3~%QjBg;&H-rByc{@oSkpqM?4<$=9{##)i?11LP8DbetmR;!9NS;meJU$0K7aS9L z{;lB|HPS{n_0F#W)V^>Z*9+hvn0QuK_9pWIdWmvFB~(ET)ImM8T2iFV&NFtTm1piY zq#bt$(visT5)j4=Eno!)IH3nT&>P3||EWCxpT+Zk7+_zc_BK8aYWycTf6e1J0^kHV zSM&UTCC~pedH$cy^M8I%z{xuXe&RU=XW$&T@pHj>bdQVo7vLiLCGhiX-&ZYZffE}#%9&US`fQj3%C1!Zn}_bfzWV8-3(yyVAGdFaGXm^w0J#Le<*)+GH|Wbq z%YF9W=xfl|!v<&{#6B4hY>AJ?i1G73-1_a*JE;D*iEvxNC=sk}Sd*ideH0<47uk=a z?}8l2g?&%}t#<6+i~YM){M3IU^`A`rBbEKD=N|_NR|F+sf=Z}@8t5f#-+XK=75fi9 zvMKK{?S_5%v9r!X>OYJ6&!+y7-I>&XI`zMp`scpT6Emcact)1aka}bzn4tx%-~cE1 zUWt$%q#>Vq0>W2Q#`km{{QNM!{&yFLklL z!QGq5zBP+^KAU+SIe^=n&px)0c|MPQ>uL6_LH4mo=P>(Lqzktj*>eED8_}|Wcou>2 zBiikA+9pI8{FmT2UQP|?@8#GX=Q)1L?ZfRws(M*Y7_*(+`THFyr<;#kzepbNfQir)FQoaA zz3c%DyqApbVPDV>m(VYRAKeFD7=YH5*gyGbCtvDa8QVJYRmj?dayt6d`Xf%RaNh`A z2jgq8auXSbyU>e0_hHxlpx*Txz}~%y)c<7aA3D!+{s|`->zvSy?gBUAdh)UVH`x2X z$3BeT7YgzF0{4jX0XziVpJP)H0|_t%Zc+X)-(-Zp!ZvWLxmS&W=2G6-M_MYhl7jzi zXm+tLd5-rEK8_VDGOWa#{{=Jds-|U08;b+FZ1Gx*D*&pX1bHRe% zK9_y{;&|DIy8t>!&v79^Y^jtpk>}rD%71}yV4chOzl`x8I?x^P`}^mE++PGG;M8ig!ad_)H!?S7VHmB+*Y=R{f%LN$RFOnsU!~d%uc=YTZQgi7cVu) zvZahSe2?f~NL=K@JBzrH9^zK-`n1qL>Ii3^8ZY(8Mx{dwG~eg_-#e@)(5={fJ2=pt z;Mh*t4p7EC%8Km4FZ=~Ze@y%N={d$6S>ce^nu^S{`WTfU&5RiW3Phj#_hVm{ug?{d58Tk z>t=(CaSSdJr~3wBLd3zhah`?5J(qVB@xKiIMEr4kXOR~23VvqNDdU{R1kHBdS;y}> zSS;VqtNE;JaEGB|FXh@uIg$3A%)g5%|1!cEC_mW14jtei-n-m$4<5in(7izag&0VH zUdlWE{GYV`f9+5E{`mY~l@-RH|9dGvarcl{=S0><#FGTmAO+kS%8&nSbWbAXhdJo; zz)u_b@+f~K<%b3MnKOBZ40#XxYF^D=5q&vW@NcKAY7Mpm_bM1a*GZ?GD=8-!*dwcN ze2nFMGs~X2tTRnad=KQ~JD~N1-vFCnGi-%zumg5M4vc)w83{7<40}~clYQs~a1e^1 z1WKPx=iO=64-oF;%ol?eXCRdwwT$qr1SJ&G{c=L+@VB|2T{62M_0i1Nf`6`|=W?bWBEV?NLGOWdbmBu)7(kft6`{Dh<%jj2N z1o~NLcwhj$-~&Hg=h{sO!(F%s58xq$zh(ciJ554qyd#rIJe2j8n1vH-Ul(>g%C<6wi<B0mY-gC~ zX?*i#KRsDGhInSUY?8F@oFHwQSZN>PdC$Z+d28VxNoCU_sUq$g`1?Ne$$uh;lmA$b zWd1SFo&SrJw9TXcFh;x*A!SVq#FY3)Vx0PiQd<3o^8WS=siz&vK4BjFLMQda5qdd8 zj#pqqk|FQx`4g!*utXa14?mG1?`AHSy1o@Wvt2H=*?%f+4e8QakuKK0bZNJz(Ein;Ld@LT0#m;C3U;v(m&*Jg49coP*~1X<~u%=+>2K%#CSs5&aT4`qDVJ zOye9gjjj=Tx#JIFBdE_hBe27$MaG04UFByFmmBi`fM19%AG zml=!IT*z6H`ra~pIOEa$|6PXY{zBP*hQuJ%`Co!^&t%;{s*@?`ChDo|wFpT&bSrzDc1S^=4USayV2lrT#s^!vPU=WE8vi*k4;H{8SOUwTmAR%(%_B#cM@pGD znD6Y!AHEOl9Akc8$^IIg&<(D2?628-_dxF~_UF)_%Km&A`}4W%&lB07C$m2vmn}Lil2Eq@4+E=Kr{L-WDZ!!dwVwd z4)HEO?tRdK?qF|kyTkKoXzikWUcve)iaDDyx2I6dnm18czI)2!)UWk1Dl zDw>3EREVnxdM~j48)E$zV*LjL)vW)Rw|up%|08Q{)5x5x&{>ApLL(&cHc14;SGQTn6>-yZQz-%y;p{ z2l!tE_>F+^6V|u%@0;i@q#GWs|Ji>&A4?gSe=YQVFE-G`HSpKc7bjECw6A)X(XR4q zp&jsRUA;`&@|H=M`0m0zcmNMUw}CzjG2pvRKgy&pT%b=}qpxtr<2gycTg~rn*8g_a{}$?}kNtle`~L>^|FxWJuonpNO=aja&H#81 zaD=td9O9e@3t$l#`CY{&$mOsCR>2ww^V@~tFL{4vomST4-T<3mD>UP0fo& zKrZwSvi~1u|Bv+CVE=!g{Xh87{m?ng{(mLEak!ejes+R%BVC#7|B*e!>r7>DPCNy0 z5Q@N!pKCGSAE0|miF+#hAM{G_R}i;>G>`+;2~vfhc{_V%WF0i4*CQLjl1I6+Dc4E< z*8uJo=s2Aqjv)Jg@@q%7USR(}M7U+_m%%m{yMPXCSbhI%DL%N5tf7gGUzxj#d6gd0nGau16u<0|n&w=^| z_=oR*J>;!_7JX|feT@A14-kGbb39|ndBR@DE-oT3LGumv|BOYK(XH6J`mN9_=p*1* zNZGbf#*LJ9yGE|#cN4;J7u31iJ>&>``3Fe`Ikyy*` z#Slilub7K8ALDngkOk0DNIzOhUGJpMX|wix`V@W#xmE-vV1i1hg7BC8<_oe8>S1J3 zyfh+1{2rkhsh|J-c}pT??BtzC6(9bl$N5H!`QL%A%H|?HC%WhRy6I6m_?x(ozUv*P zuAr5t)$;UZjuUgq>Z=s(N23j;%p!{7@t zE}!Q7oi#w$0oDiWnCqdNu&z|bW!3~eU}l`YOgvX$1g=AKE#KcUUf)EwdO1UP@!S9C zcfrB@-NCxS-oP9V_waiF4?(w){t4>0zF78&9%67Oz!Wfkz`XMz`z7@8?-Q<4{?n9y zkn)4|8s#6O{K`f^*~&D+roe2N1M^@3EP^G_i#_*Yxs*}N;CcOKByHzobB0eutrm(lkYl13@x09fYp z{09H6pzMAdatD|rSqCBQIruH1uB<`&BIM%U2gZ+?kL4RFK>z*vJBYgo zO2Eh7xDr_fHBbj-SNZOr_lClJM_-S-5zNp6R&an5dcXsrk2n`Xjy%o(s6w8Bb8sFm z!X>y2S6~EMImd5+4VeP7VGhiL1+WN~K$vyD z@fOejU#E}Y9(j>7Pvk0C1M5LOGu?pP1e;+isBb4M_-{l1;rs0!=)TtJ@}IxoZe7Ut z=g*v@a^N<3BV8Z( zp);BCU!(l^JCWUlb0OWhdyvjye&c|6&cS)O2tWLOg7GEBlv%`0Ieky@?3QbO+`eU` zv5>ToKlDFyE$<@G|17QXw~puH?-&To;Q1M8*{OpIk}^a^#OD;*LSf8J8_@+eU$nA81p;0E12I6?8A_K z(7%`U*-q9Rg{(c0KHPq!dbXlJ#ygj!rQT6>B749CerS!0l(s_ZFOzq~nwXz0%+IV* z#-C3xS9g}OkE>>1*T%k%HB2|xU0in~dk*j$DeKq=LjM-d=V1Wd3qSlmk(z%`kxtoD zJkNWYwKuxEfjHnCx~GDBiT6DEMevVtFO2{G!4aNI;Afu78U}dZL z5hEhTigdbl_vzC|JAL}s={_e)rmT`F>sn=%DCz3BG9_+Di4rBoeY}q$lOOxtKkgsT zdcEH7^EvO&&-ahl>-`H4(Em&76bwcQ+V%f-ppz}0MHjsrxy$l^@AGKA_Aib}GcC;) z&pLzFMcTi}eMTq%G{tDV|T=_#44;o z?Lq!6@$2XtunFbZhAOv1_LM0zULOIaTt?W6d3_@zXG6;2S>v}Y>xnnDWr#|_;`sNQBw@l`Rk4|L&Pu2h5Pyat!?y~LC$}iA%UjO?>=SRcN z67@8@{2;r0Ye{ILr#ebPnRCp*Ow7hy%*R43M*D95hy8p6hxs4SHIHv#2_L~M{s$a= z+ujOn{bQH;8psymOf&xj+7Rd8FO|-6ti&p`y3Wk7C&LwC^@qISmdiN&W z?U*j|HF>Qt|6X2-`|s4g=y&z@2FKTUehqUx*9GR^>zj<@hnj>_o=GE_bnGU_m17&K zumig=&U4>G);UK#8b(M5jr8O)?+4L8@z}rMn@ZniUwHo+@Ac96l0WI6x+&iyjt|`D z90za+)i{FKzn>tZJs;Qfi{k@Q!fCW2i#$#seX}^6Bv0cE&Y^W{ame%(hYR%VJne>A z9|)J|S5Rm-2**CKF0y)Jap)Bv{Rh{{KGeQY9BRfChg zM>)`}j5tQ79oO=b_p`+NS-}3AV@+`BOu`gQLm6hE?=x(QA3Ygj-+#{+ORO(xjL@4; zhM9hQE!LZx<$KH)o{QX%)xR&AuS?I6t#kM?=#`&yzuz*4$#;=eMs(eGePpi3^_@3X zeD4$Cp9ee<4qh$}Z%!W)4on;pzIVGg?7#7$@ZISj3j5AK6~5CvEbJZdk?_XVVPQ{Z zMELf^kA>Y+KNh}K{B+p0?CEgLb5FCMm&S9e8yAwx(W+ivNv=Y6uX+^S3-}0-M-FTJ zwhn*Pzpzz3nbD_QXYF0TZ$kAm1421jL)JcLd>Xw9JFp9Tunz}t2-P@(1X4(&x_&_T z!P%m4_@a4^E9_r9W>nZVc~ofgTNagj^~GKOi2kh+VLSe@cv#rc{E?9N+X?*BTTg{u z=ZA!EzWt%FbJSDe+slf>p7q7np|sAc-@o(LAafCm!W%n>=-(O?zW0LhaH#y32eA*l zus80Ojn-ftQxslPUL1Z){o#Mr{eByMaFIPf?0)!R=6*P0(azd_zmfNU6YBQf)jqz> zM|(RYn{V=!|H`-dD?Zwrp}FTqINI-*%GO_mWA}gQ+j@tO;hk_!nip^hSI~=VxQ;&D z!X4D@EDH5|i$VhuHASJZyC@_*o2I)(A$7Az`}?=yp5q>%|3Y;E2BQQ+(Y|qj`5yz! z{}>RmWS2h2ZZapFzhQ0i{Y9Y}$7bO{_p#t>PNo~IW|R} zEzl*8bdz)~+AlewI4(a*H8WXnFNe8WAWef_)2 zzju{??|T35djIcw|L?N@-wpZO@9NLKuYG<$977r{3*PmOzN`FuSNZp@IpMx_=~XtW zAK8!zdh6QzA+yo>>DdMMLnqSoHstq9d$)7Ec;7p`AF|@AN8Jy3@&sxp-w!9r)2Ny5 zzMgVlI3s)xP3PYYDc9K8&wZlq?FXUWeKjC4Mm&;eLJAlB=Mt`<7uRqdeYl18o$lZD zbnbQkN%x;|Z5^($-u>ff&%5&IyZq1Y_3pdIr?`LD(CQl69G717Al#A8N%`v@`2ekl z9pk_KzwG{>kv0a>2P6NKw4ZYhbWMD>FfOfn)C2oQJbXvB&Qs#Jp{Tj>ZfIEcZb+cw zYs!56#Z51{uLbUNz5Aq(bX@Oe-VLQ>&nx#I&i{MDzR|+tFdh?7@d^F^pC}5G=zU*! zH%uX?A+Ft1M&>>*AAfE@m_cuqA2a-9GwE4%bZ56dD*9aH7x`}FwQjOY-CjRR9vsI0 zf97F7RUF&XDBq;iUrq9I?rZvf{cj-_V=0znC03#1u5a+Byf;T4B)eabuC(LYBRP8W zAbC(ePs@kL`nj%)^59uxgM?%MeDueykxuWg%wHil;QjW)CgE~y!w&4i9`t=m+lj1x zhb=)qn*Vo*Ua(L0vt8(M&SHb|;7HU_@V~jQ!hPp8CwNvK*9UmCB=r6w`d=CZx-)}wp8|>@b>UQ;h zmh2Mlp7*eBkNvVom$1*ru+OKmr=;65f^Ch~%k1;><`3+*?uqpJFH*N)Flw94ALuZD zfF6DFas8j6^x;VPUtD`2DL$@6Fj8FFb(NB1(aPsAjvSBZhbYYdTVag9@FW!G1qgR- z)=#ls|NnFDf3kB-cmJ-hFfRW^=RlnQH^qOZp$s!H6SFZF^U?m!pSb_sd<1*>xp(qU z6uW=ddd#(^$(H->pKKLwBjeb`h0i=WErv1MSS>$m7Cvh78fBk=j`9J4f8y?U9(RMHN|2!wZ zImbC%!WH!58m^-c?Ve)?{-}Mey`8_R{jZ(hyhQt-OiQn2vb;G}`=8$CeQfbQw$^xm zN$q{(0B%Y14(`SD)$O@YS%1*{zX!s-_73jT{rvM84a*Z}tsz_!f@auUxvN;>Thf#$y5|VG8O7m1u*Ous_-7 zXrw36RL>^F(TnWs8|>@5Z0XzV>%Hu2wtFkuG9{s%UEi_H_lGRH(2X4Os9sY1yZNu9 z%u#pF+Skp0=R+u?C-$>_cbg+XpNZ6cw(m_g@Ljg>)+fVkaq+yBzpzemmonY8%oR5u znXCK)8~Fw17`q_67`e&(0?+Uh%;G0_(YOG%_)_u9u@b9L`>gYiVgJ(;uBD-y{ZHS3 zRL^_ockV|1jZNapu?$asKyZ#`oqtA-z|&Tu3nvN-7NY!m~SS2%T(mWJu9Ebu>&*dvymSq|KZVmj=AD`{@Pp@viGyTQ*tquVmVe~6?*w4*O2S5 z0TsWtR>9AGi}ieYh(5jKUfij%; z0=4v$lAO|G9)K=*2Z$ zM;~sXUAw!3-?fv>^2c09m;?2>UZmJ+fPIquPOK9n9nV^ zC;R~Yzame@_Fv-%!a#b}6Y~Bytp!eRlMhPBp-9hCwv!`Kim_-Fm%%uC_F3QGGrnE= z1jI2bIp1S9;#~FIzxNN5#P|J1TZf#6vKZC|m_f$6W+vHd{LkzdmM`YUu>St}$Nd#dN=x#m&3)BGF1A41J`Z67lB$sZvT zXj!aM_#}sT*0I7sFxn+L|1;!c!VE69j*)a;TG=T9v+}hA94Kv^Y6{0PmqoM z%)cLG{{2+*?{UpB{g)_nF&HHnis2ZE_EYBHpEv*htoirW-tW3={rwx(-`{WjeH?wp z{QKw3zn50peESx9#<^OZtIc?V^!CzFDxI+yhw*5A-7%Ozk7;)Hl!i(4DafCawzzI_ zUCuL2T=ntNP)5!`?c1ecCOI25^oAPul`IW&h3BJ*p4#a?>)of{>h#~&qv5Xpe>BpQ zXqxW5dDrhB|6B1@`#VZwF{=1iD&MsJ-C6Cwi}F~H_BH-!{O=t3e}VkJME)lW^S^Iu z|DiDcS0DUR=`F`ftU~Kc#t1Ca=S$D-miFt~^7IWTjQmxW54#Cq~jsF&5*{DlV=8G@f4Qe|*L|!}LkWYqR9=sQeVnn*UewrB_)esEl6m^`g)|%lk*?BJUqv^ls!fDj!f7|66bDLAPto z7<<6($)MG5as2NL>CD7z%*A{x#A5XQ>*&J#hn}zaMudBfJzq|)#44;o9RItH+<;9e z$D{GT+vths+#BCY6@3R%!`NL&j$wyA%lE?et!Mi-EMWhxVE=ArTatx&a?8{S{<8~v zun&dyt9D~t|Mq}z!Tx0fciw%^{%7}gAAFzv+r$3FF{JOaf7!elwr^Y~zwM&9%f_xt zry56)K>RIpoBd7CJ|k^ZYKK+v5$5?Na$}r>>~igC$3!1OTRaxEhn1n^3Doc-JX%-a zr0{7pt&r|&_qo=6k_G#Jy5nAV|7es}VgDm~3TOQ194_DzuAmp!a2@SC+5hf2_HATq zTvNAeTkHL!F#q0jX@1-N$9qM~Dffm}$F*g|^+8Q% z8H{|f`$sprFj!pmi+>hQ$}c7Kxc0(OvVy<2W`Xwu_T6!tH(hIM8L;d_#w2#%#@=kJ;`k71#_q;hg^nQ$w;3M?U zxR;MvlhgmkVLW>O)!H9qkNE|Y$SLUil;eK>OqfQGYjG#%J@oaZR`?$K$&Z74vx}T> zfi}f2i$j^?W?&|AzbOv0$+?)1!af4p3%O-{erUhJAJ49MwBA_U3pI0|UFiPW$fNA# zfM9J331unz}t2)iy{4{`mU z+|NovkNxVa#r1CSzO-|;YP-CKBjWb2E)Cy(t27)qH8Ol};F8w`PRsA=&toH zeMh@r`e^u8^YyT2_3-fRw}*$lXNQJ2Uiz>#DnA^)^Y+lNZNbm&+y3)VC9MRic8&^_ zyFV7TPyLuRFh_(PPd#mqKH~=cmd38_!@@u9*Y`E`L*biyM}>Mp9p&o4hnDF9j5O~e`2yYaE@NxuPFRryKi=XQTXBR zqHtuyU)h7m*q=dv73!Y-D|ypAtbXQQ_4{;a9QE0ddiK-$Xg;lM`HcRr&&Wrg49D(& zhJV+by0f1Q7o>R!SI~=VxQ^Jyj$@8qyT~UhT=x|J2ERswF$0MO{2P1)$+i3&>-jfU zmxNo6xrc}Qk63ex-v6ueA_k*fKYqtNV+4>z*9v0;R_iw>^P`M^7{xzwoPPvqw7kVX z@;3hn*|yP`i1Q_(L^?w;93#xE=`BUNUvXGYX2?}!g)*nr{KVX+iozPl%TRnS08ok!}J3{vS+PDicg*4hw@p1jCWF9AQ5`Dk4F9mr9>D$&ABn$ly-z*Bb zuM~yY2byuem&9i`ySDY#W~28aKh?W&zugNSmPOT*2ZWX4FW?%kqjtABrO)s$&=cLp z-es%>M8AX7UHz}O^}p`b|B5ZjnXPz@ot@rkEpx}jv2$?`p$GK-%j6Yw&S9&+$Yx){ zc7K5lk30tZtps=EtD)p@j6^BMqR>83M>f2q9z&!0tk6D*^*A5G(W~!`KP2Nm04?wE z9iVlQ@rV2q?d<#xbRvs5_MqD}#IXmlf1>(0ABgje#{^75E&oH!S?mAM6N8-F`KQs# zkW%M1A&EwH^!wL8o8kA|ciHTiNuQ0%U_Skijo+uw$3iT|QY=TE`nrC#bGV;B<$qY| zm{nMVb=ZJSD91Lmvne{*7KQa^PO*R4w7<9hSu^{igZ=S3`vWc7B^k8(KHGesX?^rn z(%FGs*n?K}ZwCA5kG`J+^h1bwI)_+?bs@(lieoCO{dNSk@_K?yp@!ZtQP?+^7Owc2 zc7yy7*VS(vAs-_551);G;VqB$^Ne#+liKS|&uL$fh5GKk^7=Smv9?M+rc2i11ikXB zp6{2e0YpEIGw9{7J4g2XkpG74`-;3kUWvcq8m^-cx6u1d{aR$tXN$r;@&Wq)@4j19 ze4KxsEWuC=N8if>!$`6e>F2ajK0iH-rH{u1w2I4M5Cg-B>X?+FyW#-ch_g^$UETpHs*TrOJ zg!enhToC$ltVCx&_6fSsjU3|mq`K|ThkA3`8<2SGdFA8tAxSnJe?Fw{KOa{4?;5Pb z25dq(wxJ5`^UBnJW$M2&^rutQqoQ2T;8`Q#qd%yV6{To?8UA3zg5g+?S%=>MAM`#^ekSvcgFjQQoQFP4RB zdgbTX4f?h^g|kas-yGLB-Ss`|`o@%nx;gHDf%71-Lb{u!`?~w@a2_0yMgl3M(S|JY zIDz(M@*g^pMHjk}Lmo%pcK_$yKjK;kE!HB))VqJrq7Bua_etrT#u=PL?V#tazxaH( zKugc&KBBVRQ;%YBl)UwS_D{Itv(yPoA|tsUW* zn7@{jE3pbOf36{8|7YJBnQtx_=k+W1@uu-l)($Rr%r;bE z2XN zrrDTzdRAMa6W8biksrY(a$N=c-*v`(Uwu`%`GPhnYMrCcako%&TfKVGeW9@b)yw+T z)U~N4?sK#Iq}NHKu>Rmn?`vOdbpOIB+>y>bJV5`ixdseI35Mc-zW=8DKZ5;_V_V%n zT9832+7R~+94?)aD8*Q`-gV4PYyQx)m)$?c(p$}m*FIQGF2!=J#47w={%7NM z9h5ia%N+8V|2+S1SWq?u_}K42Za;%6TS8^})SX7@^er}QUEA9=*Q zzsc|0zy4AvC%2*YzJ3T)(GwSy0cZ6U#QE=C^8RLJ8{!<}#^dHQI%XI4U?2L9`hI?F zOb`7KdSBt&c!O`F-8`J1nr|)c2ogvkjW$$#UcLV1;!yC9O?E%i^+$-$;{*!(|9z!6 zoTi__Ib6UcTtP3c;X3+oD@yi$MPsq~KfXiZUhRW>!B7lGTw6G< zF*lN4=zknzP5^x@^85!mbn6r9dQScSoblz8^>I&Udys|k|J(H|BGvI^IO$yD{BJxa zAg=we6|rw&l5qPX{rf2Nvu|XRzQi^~ZiIez9IY|_zux$NX{DQu|6j|tMO>f14K@1K zbN`^MmsTx%c?wyE>ZCq@GVVnX_asOq+2mO0dS;4Gh---UbBFcog)9F?JwHU*PmgO0 zW)_u%`Sk31*M^1k#mKXNbLe(V*9!LUTKA6zB+$6q*a9>mg{A(p94oO3Yp@O*unFy+ zQ^!vBKbbwq{-<}xd&;o?`FD@rbpJSp^jq%#xVZ&rMH{N`J{iiTvkg_)f!e2xKN#_3 z*hP=;qyc;A`;Z#o+K_ZiT*I*Ny$$m|M)Ak`{Sc~A@rE`CDnDmmn4cSK5dVub+K@#a zb+5C3cgz2KwbMJa%g<|Ux@~NLV@~2U&LGakKSy4`CA7;&9r95pvgkthDf#oJ z{E4GG*}pHU|L3Ud$(AMRf3j7$jf~@0u1KdB*Ki%JFKRz)7xdAyTcthA9=G&6$P4H8 zJI8M4BJYW-US!T8+5i9W?E2|1B?qHM+0anzzTWB|N`!|Z?)jGzZX}c123z=nw%*V$ zF2D4(Tfe|DJwJ{%oIZ_zv#zml?w3VjEZO%&2;&OB^Sh9}Uo)>Gu%G_MIAq~;x3&zK zNwV*2wE5}NP=?MIv=7lGZ+6e|Zpi#y{(ryC#B9vPd@RIb^!?0ryj&D|{>^tN-21e3 zF3FWxg*B+2$Ol4Je8MM^c^V7`#Gf_=(v6$vhE%AKjI#* z2{h7^h-1v+TD`;l=e4&VjunVw2C9U2q4Hx7#|N~Z<^R5@j#7_h(Y0IMbWk0&mmmIh zdzV9B_a>Tpe zM;^c-RL3|ZP~A!40#DPD<1j|66)J4!o6s^>wIWL5_SGt&qi;UW&D9} zqLD41B%6FADe{{CynlSab>TkTLheibLi<+z-}=2fx2tdVv;SXLws$MzGx|!jNsmrv z|MwXGgER{L-|y)EW?!};J3#)rXng#q;`~Q>wpjiW_h@`i+%xB%b3H(X-}|rhe6Gr$ z%7%gT!n&C+m`_75K@&YSOa7WCe_^P&xHso;awKXG>sKJhB90HLueS!nVdazSZE)Sm z%jQ5LbyHjIzWeyp(_x%r$72E}VG5?93>};KAQ9Kz?nGRBy9;@G-0Q4)uV>(yq~~}B z^vn{^0Byppm?4dsn2ouZkA+x_rFg&nv|M-<)}ZP)p4ls-vcT* z6Uwm-RoH=D*n@pIfVd9gA+j1rkce^r;=ai=+G1FmSu)NK%#(5bzY{U6Tt6AZU;DF% z>#xT8_*wUO-@4!8AB`_LLqCU#mq+$6Y~4Z-tEEBAp>POJ}-t$gZdSZ<#sJ8ORd6vhE`|H1wO2TQFjH#(F$ZY;)OJZi<&U;;hi z-Wy&R9VXGIASJ#@I4Qo-GkSmhJI(K9n1PBG&lve%^b50vE5D%Z{0tkh%Ua;V^O0+j z|92XHv{(K|EUO#UM@cl%Q*W6c?zqKRise{|Rak>{D6Bt!T>eKEUHHBA|Gn>K9DA0p z4=vNR4WIel_|xoa{wZvb)+Rh&pZ_xd6ym)3a`D?xg$mzd>qhy1t-dSa26atbV`>+D z4+{H+Zr6_3uN{GX;;J2gfINg+=~a_QP&3&YVynl6Z9D%Mz4~`ah2OaM3Kz4p(;o~wTz(thiEge#stUnKXM~n{Nb8QEdVOhWDaRNOb|8RKoruzNk9}S1bl!kvkUK$SW9~s_! zZlrzT^rLkT5BsMM58rJb9`?OHG<>IiXxNK4*8gSLGj)RbZxh1q<_Y0jlgEc$$-fA_ ze_tB@?-2O!tjRBropg>ZH?=E}s~Z?j3!lL`#P{F#snT$PehEE4l+S*`|M!ah6}pV& z7uSnxsCbhNOy*wpAMzINpzlNce?_CiJ^BOme@Py|V3c4ehGQhQOxIo?qPVIaZ>zyGS2RQCLOK z>W}QyAGwCU4*6l)9#geJCTfdpEeadNRZlMpo5*t1-Yp8-$STy(8_pMLcNB#k!n@G4 z*|^4e+97Y5XSd$?!pn%^lY=aeT&T1i#qcA`L_0&<7dtPu6lF)#8qdC!U^&uYNrker^z#@p*M{3UT+Tw z=Y%hy>8$s9*?ZRSkc{uXLm!-WK;o3PI+;`^G~&_vAD8^+3VLx3*U^VtxP!v@e>Q&f zJr>sgIQY;9QLq!lH80QyiRfE6hUj~2@%^{G!v-PK)7cQxd4T@^r#yt{d&qRK8R*gH z5a;$){saHP_w{2>W#>=izu)dW80whnjnPT)&Cvpf8^--8ulsszxhS>=?m;rw9qqi z*r#jRr)b~GzC|ap=t4Jg$YY?i=VKujW1r`^lw6LLsJp0t84au1*DKhp?D-_wgwz7| z?{;(OpYsfcg~IsPRsK^eu4b?C_w+;$`_~x!b@UBLz2m>)lDDOWP2$qqjsK7K|I^+n zyT=5Z?or%)_Tv^|EiBWk95*# zLl(8-Y8;!VC;Y#`|4-0QB3@J6n=#qXwfgO}xHCA13n;WlCi|Y0`C!unh{ALhL_GTZFG_41Llauw{4VeAj#xJGCfnbvj~ z?zfRB#aOiNl|SX>arA~8^4D!+{pb_WB%GSZ{veZAjcpcJt<5oooQB%7`Vz<)sCi1i z0r}dP_r5>hU|*3o+8f3?i=T_$f48p?8OQ%GCYNG4R$>(@zpCBxguMXhaV?N_WM=cj z{?!fiO(;jFwssa>cYPmcy$j^A&2QE6VHLRpwcYlCAorjq6UPfb9M8W`_yC$lXcwY! zqIM#ANL(B@P)+uJIX@|U`?v3HU{$NOuDddzPeVLmPQ1DOJ zs~^;zkN2lPZ2g>f)Gz2frG9x!{c>6Ta$Nm_{B-;7N$dUgX;wIo6S(GksZ(FnqhS_5 zG#VGF-|%Q0T3u58mr?(9tN+MG;Uqh~NqZ&5W z2m0|3sGGag%du|GJ*$5oN4N4X;Fvl#y@G#1{hL|FzkoPCAU&CFE}dRn!*vww|7VoD z^hfspE&3hAHbu@e>TZ5$|KAf=z1CO#S?05EmiSSUwy$j&!Knw-*`fJvy>qHcTI{u=bk-@KwfPTzK;{31L9Gcg-;F&_&N z$Db@Fm*U&Lz2#)9{1?ZctfXf*%irtme@$P5{8agSvi!Y3{zmm={wwhtunFa;-OYdh zj5&w&gnMkrMB7vTK9)MRR~|?1NA?gEzX$v9{_^jD@F7&A@?YFfP*%`WNTUr| z;g=C2Ppqq~pHyN~ql8x?MxV4d2IA-$#${;~n;) zvLl6&j_+Y7R1}qmIDeq`-~KF=#<))xg|TGc8^-4N7UCXhajeQXapN%|{`O1xd*`d6 zWACflIIo6m&8vJ`uPUEjRX)9{e0nt;UGZvYUj1q~w(-@FUi)fjS@bI3`m3SUac!Ab zL*0T`L;bQ>Ljw|ML=sI%VUqJ4aqm+i^x>F>GR#1oH0#lT1R8gj`~J)M-^=;m%fn2^ z%*I^I$3iT|QY=UN_Hyn2a_#?e{`YeJ_j3NPa{jOKkY7^n`!DBzFXw+R=YKEfe=p~M zFXw+R=YJP>w>+$r&MK_IIrP@LW)MPL8dk*UxhtJ+WE5=biMto9=qQH`UkjRMltH z3I4kW`)~k

tVPe}0*kx%bF^WyEgnUo!4xaY9`6{>|Yec^b9#o5LCM9BSwd!?f*3+5cAf z5}M9y+h5ks7@)0BUJ=)eYq*Zb<1?qKi-m6?u0eEUH`KV!Zr4R7h2z}hSkK13 z^C^y-hBC~+Ow7hyv=CyRT| znmy84h{afnN8^W<(_@>x6RYTJ@Q3sF-@pI0GAOKbOy4idLtmHgm)`Rs<-)(=-(O`< zl!r|*?u)O6a4%E`) z{yn?sd$11&@M!;@L-cI7wgal^M-cl2a$D7(WY=Ex-?RMRXqc*ffXDmqE^!U)(}e$| zkVYG_$m0Y~qWvB9KRWNK|ItP7MsA7vU;X~LebbMvJ4*Y;y=ByqYt-m2lwy* z|LORni@v|ZzCWbVg1E+b>u%p4s@dcH*T_>Cj1ttc&1>`l4W%bm$Q$zSaQaB3xOu`gQ!(;ywA9b1V49rIEv-)bu%D}Gvi1h*KaXkJ)GBcfhFxlET z^rgsu^MmpiTLE3#Esy&*w0oBOz4sga!b&pw&sLFZunrrr3FX*^D(pa9%X}BP2X%Ag zf8|>P67}w>!@Z%2o{$wlK7Fhq^SgfAEX7jg1AK<`;au7uRqd zeYl0X1?=NRY*4mpLcexnlI^*d4XWQdb({ULn|+I8NaL}6+swX2ySR>n>|_6lc6gV5 z`EEQ~pD#V=$#BPc?%@IYe_bA$%tju-w?NOX_g@UAmmt4@?*ZK_9*(siDz18teG18u zs9pPHC?&_D<^|V1(RHKZhuVs0ddhX9ag6&wU5)#5PYq{lJ9bE~@7MAsxd%D^oZgSfgX;61E%rJQ zcL>#}`WN;6XNy9%AN%4Z>+6V1AV0`^7WcUSa)tX4pGF(9Xcd=P&(}adfs;6m>euBR zvSz>ePxN#23jLxN$cF98ha1X=yD<+bALyy8#ufYR3UZ$x5GuYnAoS9&;X3+o3wLl2 z5Ago+GyT^pvp!~i&sUzb|0k4VgT<9#D2Ai3KJ(WGgpu@8jKw(gz4Bz}tMmP=_bm!f zz$8pT>qOt!uzq10Jv++xLocJxK)#1I@Zm0TL{nl_$ z{d86RbYC0ctojKl9Bt-5=-@w?#g<0PBKEcMf31l7>ptH9XR7)ch5dh?Rd0?^KO?UH zztnk_VJkvP;Gp-%H>)^() zL)yEr2XPOdnu&iF_R-^;=C4U=sy!f}F zTH3iSPlta}*6v#IWcX&s--ey^*QWlDunkAzwV-nC--W9Ee-|nS{g1GH_1}f>@xL5U zucZ8zMjLv*Wqrd%`q!TL^YHCKCB`m&Fl5E|{nba!4Rc?3WAg{Y-pPYQ{mzeuy6qng ziGCkaR(~`k*M2xOWRd^((P%t@Ta8JxofH23?YI`}WmP5Ce3*gO9vWN!aOXzBNfQ1SIq;ga7A^LvN+ zen!a`?*EFoUR1T~-~a5WaGluFSQ4-{QnOB zi*TIdUumy^rSw+MKBGOeoF3;Kc48%c74jXv&5I9xkX^TZf6tBz*WAxSzpcSK)b1Y@ zYIctb8|Vr10UNN1UXIj8{ondklTVEfjg!qa7nfdL8mh=0XkAenc9DCK5#OnQHH&@1 z2atP4zr8;Fu8I2bQFmDXJQ|QdBa&!B3Wxls8b^>o3Td<pEFmw=IB!1Aqd!4UtQZ{{ zP?-PsTXpAZ_rJ{juXq1A>6rBX(cv_C2Cd40bL0hNnn#DuS;_@m625}mbmvfpbWQaR zlp}R7jt=#4j0NJk8aGROzqZ-y&b3whzfrr;|E}RW`fv+(a1ZTC?GJ5=&ehIC7WSVe zbHaJ@=qc@g9IN-O)M$Tps~Zk$f1quz@k1{?t^eZb(0TmnkVO}|kwYFwU-$fQY|PUk zZ60XLsHc^$Pb*)a<{x-E)GhP=(0~LQFM5AyI_3QxFAWb|WB+yX3I?MDLopm9QHrsM zHh%TIQDGdtX5*+(yY~ZOJbeNt;qmfhVS9N+#lzxDiD{YLh_(yy==$5#HX zI39n7xWf7aKP(C}>9a8x^RWs6Uwm-RoH=Dcqg{|e{Da7FOCj-gwuPCu^Q%OuFd^ET7P*;{Legnvg8re#(rP24K?fe?RVOb)t@IE^QiyeB>gnb;6IxGSNDSYch18){*TrlSn+4!oc~_HC0s!- zuHicR(7uTMkIt9W@95g7en)P_!}i6|xA_0x=Km+t7qw4LX`i6gv2CcnY|S<4+`&CO zKDtgfWrL0o&5g?`FHnfmn)x>Xp(1B$L0U;8)xmfDVT;b%)m^{M*BwJA3ERW|9?xH z?y|NWnL`}weN-9JjAOIpe?0a-(c5C&F#E?yXFe8UFBQtnAA0^{I&+QoB8v}4X9Z!U-k1HVv}$=nl8#$Xhah6gWifbzG0i+;~IoJ z$f`Q+`>r8j7kv-*;Q$Vy8b^>o3Td<pvUkKbM4i;vS&?diP`O&p>i;)W2bkOmZlOBaS~F zNj@5XTuL8{aft6QyTm&qyR`>$WE_87CEc3M=1W_{`VM`(V<%t|s?E)B+-pAgLG!`M z)U(Fti=T!D;lxh;iC3-DCp-f)F&nMoGPj=$bLrW`?00?N^XUtbuVKHlv%8zw?_}LD z_V+XFRCaKJY@E*iCYyv)WWhdsul=*w|CeGpD)hyyB->q62Rf(nFQALwjof|x{I|_N zSi=66&N1o5y#`yPpBbb7o!&M=ziCZLSS6jxPkI(#w0GrxzDWo+#|dXkLCq-?RTB1a~<`r zt+4;XRQK=NlMCFx>yG0i4*1U@RO1K|NFj|jwBPpq$={vGq6=|vm$=4b9*_55807xZ z(xd)IE85gg)yd+Jl};Wfa1yoWt-sHXI89I7H+JK8aX3RihtxCj29lS38`!c${#6FP zwu1dNkL~>?+uC@T))D&393SWJTp#zZvP>yY=!Va{vvpdwcoyF?k0qWmT>R&R4{AB)jb!;<^ zvB}epZDB8FjDKrI8>*jU|4L^M_Td0(#l?N?4$ftZ?j_4_& zTk3!9(hM2*h-xD%zyH|~*Il0=t!iWXCXqEs`zsiuH-$b8xldYqg^wYD_{}Cq`G(0< zv3V{p7$4x68HjDpnPmE~@iXLH;P% z|6wwRxc_k6KYN4oY(hDi42Q%D+44yO0{;-k)*}WaAj^mzTUB6vk)z{QwT3 z8gU+R#mjt`Z}baCgnRV?CCC&izwEjHwSCFyZOEdc!kl071Wuw)-$3r=&xRiL{AuCd zuUU_nJckRog!EZshRI%B!*#^Iow)vYA3fXc+rTaQ9pu+b6OZdhx^81a0F zbIltkJ|B|vo)1k(%`xxZf8Ia7=brEb^#6O`Jlf4`@3{PY=p?hV%6!vh{QKtD)AKKs z>3=hS8ONlZM$1j}m(gl|d)p0d$KCccmrkYfwCYF3T+(aLK5zZ+=fhBX;=XIcaQaB3 zCc8GoH3}M0DlUDnEQ}?`p|z$gj3*}`lPn9JFO{j^%EBb!Dag6DytM$jR=ds>Wnr4Q zGR(kC^nTSk3uI529_EsLznvcDlMAsJOR*d)u?p3m+hhO1TJyxk)sAuf&zdJrPq^0x z_gV1?&mF1buHW-Y_PG9co)2~IwH^&fToori{?;Vj6bk+S6(MYrRynqz3Ole1?VCOS zex5(F=t8&k0&)|dH_q{S<*(D-d$(<5-FEqFul$8Xjc-~TGI`!N zepdd*9_QGH1IT~k|LJ##ti}-}(7r(aS|neumcLfWUz_D~GAEoTk7n4v(mb}6{fm~h z@+Vr+hB*H(C7m?d5cdIUUBLdBYZEUQi4xmMfNZ}~_@+IX~n%?WXF{@+RRG-~C| zGvqnc_+Nu`64Uwcg)gD$rg8&~j!UAh-uiKt`Hc40EcVv|_SX{SFZqY-KlXjZ9&geq%zyalfN+Oi z@n!W3+4tk;!vnI1?7z{uKW&WyWynB!+_!x&*}C}yp@bZY%zFK_WO@ZVfGkD953$U3 zU3OjPUFSvDsoWpyw{fU8zWbVUjHge)B-9$$U863TLXUf~HxwIZOfN&+lc!0268CCv zBxi`5iP@No_xEqi7tXz`PnRt8f9ua#OkaxSSc$q0{hH(S|Jg zzEu?RWY16eMabS?@|QfTzKHgW{;nK_zpkLP=i9(^JW`Uo!2 zFCng77}qc;`2W`%YalMIeD5W%A+9TVo$Nz~URW2%w|7hU4sxDN9))?RWSu-xAM*(k zXw3Nj<=4XcSV{h`Uk(ZP{O& zeo6hj@jd&1-nPuRd1;J9DXMR4OOrJd^}k^peLN;0jzLHabA8$z$>#U!fAZ0|tHOMA z$5sA{-TYN$#D|82X~G+P+hybo%*1TW#e6Kpe^&l2cFa;N$4aci8mz+xw6B-{H+#-x zR$0*{kHtBNxl{80arr;<2j$-ab+h;JX#7>&XQAQ?d<;YMHA%1HyF)@bxeZm=fnC^x zeTcrzE$I8WIm_e$afeWi+>eHY?Jf2T`;fWQ+P=NvqoIc%D&e=@FKZ8cR-U4#Q5l{H zReTKTgT-Nra2vA7qm})a$rOhZG48Il3vb%vjea`*K7wtnU%A^cT{t7I+W7u+zA>Akw7CJ?{7!H?mvCFg*&*12k5`a zw}^IY*>%h_&L3HHtu?*?IplHl_#dplC!9em+K_&yBn*~LZcD#VLLQNJYmZ~Z4;9W1 zV8df1eK_*;9J=`}x=`4E_XY3cZ|q~BFS7Di#@GLwz8}Yq#W+-aT^}tu0X;v{-%s{_ z!E-08Un&XH$TG~pOw=ype-J;Lp4jf$U@m<=QuMesLmY$Ah=t96g`(oi_VUvFq|*SP#B9tn=Fj#QF1^$a2K-@rC*G>XU84aSVAJ z`xD0iH7YBTICOYO_~(ru3gy{DZ!CL0>{ zlaw`ic1n(ZNZd(fS@acjDvMsbZ2n=tY34?-S$||a%~NHeYDAg&httCLCI3DAqjuAd zXC{Se|2u+z>X;mM4Vx0aIeAjpxpk8M)G1-l#X%wW;}Id@_wO_tFEpd`*t=|S z_}+k$a6o%J<+n82z&#P()K1FMd%oa1CF9)w6XZ#p#u=PL^9#fEVGR%I`s@9U_4`G? zOut|DYkBH=zrJri87}y}{w;lI{hsDy)qlD>_Pq=Zjn?=~T^t^o1`G{{ZyKZExav~_ z!w-)0OZ59#_~D9y;fQ&TwRcAve`b9C5`O*#?7P+MyN&$%XriZxt$ar8dv z+(LyphIh!?yV@T&2ZVd{gzIYLxqPUN6)Y2r{G8ErsrI19*?fOlb%I8I`C+px|YqZd%gSE=ss|C znEMjfw8V3Y&+SFmL~p1u=Yp*3{$Qw2elU!5&QgrUIE=>xOu`ft_E*sEX?RZiXO{NQ z9PJU9!k7FTmqep}qsp%u zU-5)}>im}8&i7BoJ;dYQ;T!0iP>%noebBf-IkH51W0`UVb?WzeafR`C^Rz!2t)uJz zRoH>J{{1d;5B8x;K8|@j=IyxW*B|-=TXt)2;L&)aqx5DRLmCI9aR}8of&}8afhn?A zosjz*{hstT^!>YMOy+R{CsF;nGUU03ZMxVua9UjLPV*OVhMpMUUT}_n2`PFL9{K+& zKkoVccvQIJw_aSsb@WBd`@Kc>ylj0!viBPU!#(l=`fv8VBdu&1OqO6MhNHDv8>2&? z6FrVeiG9$e^s&g_*6u|2i|PPW&$C~=^v3yZJSL!at^RNQ{FCU3y`w|J>!ZUI`ZS~_ z>)SY!Evw&a&${=%-;M}nj+ud(n2pLG$+zC&T>5-0M9-(54vWd&pXfhM>i>S* zIWFq|J}b?eqxsfHhoftiuMhpVj|=S^xhH{r^|>|M%$s zM-F)$HDAAZ-so@)X|&wb|BqI*p-z8({Z8k1FNwIO0httT>Tv$d=&;E-%CQYq*nwTx zgMDcCpN^z{{+iJt8=tARU3|8M^9lG$E=U+X`dsyH-@WI#8b8o8i9a}?hm?s^`KcVuA z<`qqrmS@`-*DyexyjQP2Z&;yyy;;5fQfy!QMv+1VUuRxgCvXy{aR%pb0e_g^y3Wgw z{Bb!l|F-u3S?ymOLmGeRe^1{~KS}2ba$AZ*FWEYg{onI}aE+cF#{S23dLN?CE{AS; zs0+8mRkPRckoQo#-daFp|5x3A&F{*Qfx?6Fs9&M+1^0=%i|&7n{#PW>h$NbjLW%zj z#c;&-&q%TqV=*4>JGKAi!A`_>R#&(8zg`~O%+4Z@UShyF#*rJa3FU}&Xh!@tdbVEOa+r-w-+{b3 zB!})Ab&a~Er)6N+CBFCXi^3jqi~BB&=Nx98WZ^>o|FUwHehAe#g1#X|ArZsUPmyV~ zA&a<%e~tJ&J+4XE;CfEbPa-vn{fp!?{14BuoyEm*hiAxhXx%(8Tp%wYqrah({U68u zT@mg@+&d_*t}ToMN7xrKu4&V@~ zk^A(J(EgVE&&GN@{`;oqio*Es{mOseMDrZ^pG>P8TF8uWT%)h;zWi@~{}E~Rs_PSE z3Td<*cc;O!oZNTK42n497^6Vl2jCJSL#;t3$#hatf;F6o+YK8D?N6YB%eDe!)Hu^uqpo zTiL(#`AEIa{uNG=ajm{s|D`AEH+`@EYt`pp@l!rW$7YNH=se3WfI>O?(^xLEw^r~I zu>HF?TK`Xfe*F!7{QB|}Xk<$#(S#HhOJ^yTV6Bv|s!-c({y*Dx2R$K;2JE8mL28U^ zo9Y^;yMKOwed6M}4+qFYXuYdVM;<}uxbL!F{#QOEgi|Q2`>%T+ z^#9s2h`kgkha&Bv6dNSSrCcP)7YWj_!3GI79D)Q15^RuQFE-d9!GvdJ;P@2`LSp3EkS|}5+q2Fa0yQ45ac46!$A&mkk0qDHid7i_ug~Af4uYg zJZnGC+Ru8PXRXirte-XV|7GhlXyb`>)HqMC)V?;u_diAcTEb?#rVTk1QOaNPSZr@Q z>bQM^_3xeADR1V`b=LTji^i9bMfs{_9Q*J0L*h)6hn=4p#y=Qo{d@9+I4a)OW=~dc zQcosp-jx4awb|1vKUt-ny5F{eBl*lO`{b zJt!!PO8t-eB4W7|=OV{B$yfY#4ZXODiZbgGjZu%F$GOL~?(;6a5B)YNf13P%jsMU4 zf3h?`aF;dE{5BASF%-kmqaSMo+1;qzC41Sw(d1Z^{O8M_-*x@B@tm(~rlsD)w5R(Y+{l>+StwZTxWwWv)jk` z_iSJ53)jG- zYF}jkXB!tlPafbKFZyWMN#Bjsj`#cj>-oz14L;cazt`{kQH{8NL;(lsNtF5@-qLgK3_nI+|gKVwwUiW#gx4dV1W|w#KvUUqRSjVWDo;1wUaxStn z)JMpL$UUiyODNkOQO1!)Kbh;^oB$4y5(YqCmM4n-VA?>qMr$H@~og`V%;OPnFQzwnpD zIr0K7#c*6f`to~;54L|?6P6X0x%FP6m!5m6FVT7ay~Is=#lLtbFZU%1!t!hS5?!zL zCF<_Im#81}UZP>bdx<33IODxU6WQ!IMHZgbCg|GI_4s`D6 zOXSe?rnocyqqQ$lZ0bw2EO{@{dbuxg=z3ow-QAZsOwZ7>=lc?EsIKWt3>42`48?HN z3ab&v2zv4v_a@#^^wCI-b8o_$rnuK>?3fM9;j8J% zd-oCzx9%m@($^uS9^TB(H=VtgXvD@l@;BaiF@!DO`DEDOoQmJc>%T1vo9UHbV86d& zEDe1twqqxDqoA$6PF(efW8_Np`(4KW?UVn>lyYOQ^Y)_}2T@_HY?4eNjdtz-9f)%P zbLi5hpGSdSM2mdgsth=EOnYCG{>KjOfymO^Y9Njria3hnIDy_jXul)7^`D&~d%mS^ zL2guU-lPvVj)}SCIDN+)MY8gn`hC8pFP)xUqK<=W^xSOa24AI@eiKDyN&$J#v1^|C zhOq9>`_8TP+4HZ~3~}6teigoN48&jz#c<#KaB>7jq4!_($W8-EI2)pTfQ5t`^ zi*0+JjT8RQFWgD&uep~b}YG?Xl)%CT53jy^qWtHL(iLAllW9PeCx@u-+xu( zAl|5##}4!>jWzqZeuwJ*9|>_!x&wK05Ra+b$8-5{KWEo#thxBk#X;fV=|@7XeyRGz zmN`_e90C|C!jb z?SCfz@wIr zKN7p|^bbE;^dE^`_^I}by+_oeTzeG9aRP6vBb_4an&KLO`WQ~?f4Znofo!^`kKvYn zm66sBbj~?kz$NtH3a+6S?Q8VYtk?hah<+P%xh9{`|AgWU{WP!Yr@gR{sCIftB!Y{ocCav^fhDsQm)VjVh<@Q=}jJPIhHu1WttU$}uUoa7rfl1=>N=;Nla!L^%Fg{|1A z+%1()d41Q8+fVBMKg-WYjxXQE=g;rb|G$HukCxf|ZyZ|0|Hff@23hB|-5MOKUok&b zJaKKjz2tt>>U*ydUQJJK(>8!o{o`Be&h%z9p%Dk2lXm|}GKDz)KTWnF)2&?~Bi(s8 zE~3Ep$9Z`V$Nvv`fBwTW;*j3b{tFAFe~I*KGpMNJXS?n=PT&;I;2bWXo!#%?&v&A< zzGH^{U&H>#{~=qZNWZp%Lyx#G9A^9D-Vj-|on`;)2Zc-G>A@9TLuvef&7jarPsX+Z z;{$HeZzJ^#yTNun9REMa`_R@K?crU&_n}{<_w@tc@ayu*AIri($Ad8x!!ZJ*FdC(K z8Sb&P4&@l{WrB8*efCKtGmbm+-mB-_UgJH}3lqH8VcxTMoF6kdj1|XtOvGeN#dMTo zCQ9o%cn5XIJm-s^>psub^KGp0o$V4nNd03N+lb_gY~vznLo+?~g8mQd63-IP9Lz;U zds&!IF2rKA-_!o_yyuS`y71xg*V;cG(f)B#9sHQ~kBjOCWY%#T+5KDo|6|7f|B3(q zsCE9uxeP0?605Kpg-?}*b)+hA==~yoEpIq}W4ZP(_Po^o^@?@f_8F@oyb76J>K<=u zmqBHndf4y1k0pE;>56-|#W9rC%F6-HU5l;Qj@m`~-<6L$=}Bc|19sE*BDG1`yhjvs>y@M(v$H##6E;hHa)J(n|9oWg5#p&u8cYWS$CQLeO>zknY_pTlMmY$ zmblh6MI6O(oWLoZ!8tr!|K}b*_7?kpo&9IO3oo+&WXl-#e**hYrf0DKWX5rpEUo|Z z%7@qgxggG3VKul!PhMsJyZLAIf;QEZ<7UTAWF!Chitsq^u{6KHxcsd5e@$2~GTLT3 zU-Vs!G5*f+Z4~Im3BHp_z7w=GX&*kOefWsB034=gPKxJ-{E7C2{EbfJ@Ynmd-4$ma z`cM^kQ|JmsC&`#)qc_-?a3*gul(P1S^pMNG5??KABH<`1V&*rYWwpa<%hBK z0Fc*h~i2gLD?y?^>>m?L~H=A&Q^ zSH&;+(DZaS{TJ##{Q0Ghmm$ketsqz8!FZi~M!S`IeUXfN9IWzNcl58xp0D}NzpX!= z{aovK9X4Pys<0K?u@k$|`(6Ei@<6@3&>&4Qt&OLp4b6z-pZ7X%KdNyMNu-cQ8`@v? ze$e@d_k)M?|LDd3-f!mp@qc7oEAa4Z-Y>HBwwJX3|I)Yhj=F!lc7WGEdcQqu%KiFx zPW(k2Md9}&!wK>fdcW>j{=xWAdhGi=M_#}s^dOFbj(eY6p+A_vQ28a#MSpwB`;Tov z+`G{DjP!5O|Gh&$F_QYD8}&;!oz?$+T0ikI^V`L74ZXOD+qjE9^!vW_zoh>ioyegJ zc@$7Yi@xqw|8Z!WzHaic{_+f&b=*d#pL{wD5YIpi#!zGzI7c7#aC%N3b*Db>5%f_g zIxZ~I|E*8EiySSiddbsaEIA&vL-ePTlM%--G>9kJ-#CJI)OWN$HII}o$BksuF#X@s zTrbTHNWLrlyl^zrQ%9cu(E2}9#aU@>i>mKfTZLYZ|8)JI7yb|ZKX>GL_cv2KvoHs9 zk(I8D^v$Q|-t=v4vJU`#F^Y~0j^iA^t{vKSg)Kw%8v8hqHBUYrR+6g_pT$D&X*E50 zL?8FN-a8rh7>IL=;~e9)ep`nP*o-PvJYj8rayxcnH}+ybs*ywrg>T>YNlNoyUNhGv zl-=L|?mK*KzxO_Bj0M@nxE@`+SVHvDvpVmjH#H8a?C{U_s#!Sf9n3q(_xn5p0C(Tf}D%_ zScvKul%M2MEW--a3ae4Bt)wUA@rK# z_SdDSmYcWcnzdMm4ftUH&t}K%m)~C>_zqi*uB+@i;uwJ%6%C-A;P$8FdKfl*a#kSGhmN`P2NzB>zS3c22eb+Dq<7 z96L}=9z<#XJHN#-^hw7lG_RMgm!y-7edu-9%|B32X%Jr$jXQ*+87ZV)(}o<1IEv#q zfm3MTBK_OkBN_K9?4rlLfePw3rT2eB`f&*9W72;_+K@#X(jCfi@tng2Ttaq;bDm@q z=s7mK6IbZh5ZClCAn%;6G5SfK*H0k)CTeGYv{V=0C@)rb+Z%D1{d9!Iw8lLTn^Eyu zbJ66R_^umJ__g)#T^rYt=p!@d&42wz`3YI}uX3aD^qip>_N1GL1ImqOJ~C zB#+`aPM|dZZ;$pB`h)p@XXqF3aQ)DI$|}_L=NIt{8%FX!hw+Q}oK0xvBd4&!HE*2c z?F4^Va{3&U<{^Tl{;zaDD+lZ5}`EMgBLRwiOTk&nNkH$PVJ? zo!9@(2kQ~v6wsLw>yFgt3enMIFt%;u7n-}QdX0Vz&XU*Kot$9Uh!(c_w=x8?Qu&RdAZs3_AG z^rkgG=nvMvTShN_dRkaPHf)g}ulr{2_;&GN{kxv8YTNz=-<7Re>AY1~jkQ>Z4cLq- zY(?*<}aoIPB z96FIl7m6sLm91-0zNE*<^T>>2gN4Ph2U(QH|0w4gZb{#u*e|M%)L-2QasFS*f25H^ z5l3+xCvXa-^?zn~rq?~&%bxK)&zdYaE|M)1)c>aN@0^n+4?F(g`ac_m$1#6#|I=Q3 z4OF%mbN@T-p5pG+Hc;CCPusvb$8kQy1v2_Om*TPZg1A3Z5B&X>{^KH_lb&mJ zZ}r|c{Vq!D|2WPcaIa*Wus$@rpp0Q38p);?*$OhnZuEOgzQRCMpSBhP8P}$(WiyA; zhvS3ugGV?Xh0%!p?_ z7Up0s=3^liqkTf`w>HMXc!u2jV;b_tR}@g(XZ)r1`_?7;v7MJ5qu+Xlerqy|wqe4g zWvO_UVFgwqwqeBm|5nj+d)VJy)*qy=MUh=C++lxju)pLwVbyDlVIVi7mQAf9x1zNF z-)YbFqOlB)ccMA(xz>2jWK$-#e^~$jmb$s`CP_B>Zkl{2&Ay8ixzQMnO(%UH$JDRC z?z{S~c1rP7HTqWcO|=`((Q#h-jq`|OJ-T)n+acYBUDE%i^uOr)Tj2X!;`<}-&p*{p zbi>*RNUt%6M|{;dh$ONZ`DD2~NYCxK--eW?x1l&izEV!)H)%s6bHd^tltr?_xR=@+ z)&!>?N8E$5K|5FStZ_b$PoddyN?%nY*>ucr+wA*?dc?IzQ+Z>Q(X_=rfblo&T}hmA z?Kxb)CG_A5jB^g1c(4y(@uFvhI0vr_=??q;wT94oz_VzQ<|WdOjOTM0*Tm6_o4Acl zQ`r4M+A`^H&F8ro9O?d$f5!d!t>?4vHgE;`#?^9bE|rIkH_AixBhQ8H ziBE+A!dGGyO8d_XUrkTG{9I^Q{#;l~Ux(C^=hR=G3r#OQ7aINc&o6y0>|gV_@b;0< zhJBf7;iolItXX63*K3~%dtRLse*DaYuzSm-@S|6r4ZD^<8-DoOr^C*g3E>CxJ{5M* zTL(QJTKYd8(&i-{TJX5Gktf38%oAaQ|JsZyY{ho$#LrKE-Z-s9_}Q9i`oBIO4!moe zvGCXj^K*H=W@MT5Apb|a4LKC?wzBFdc^oHj3TLp} zw|0)afJ>;`@tpP<@8J4#A=&Mjqv@)5aQZpzQ|9;2Gru2)kjCNHp7RXN?|;Soe($aQ z8S4X}bDH%5&^6Bb0K=>gFvR)*gPsc;)m1kkj_2#~e+7N@-7Pc1HG0n%tXuFo_MLtc zy?-hXx5>Mx{0TqUeD1WgReY>0^a<ax=K=n!Czbp$iM`nakjz^=GK9(GhDkr=B+C)^5-Os!FYys|ggfdAnCZ7!sGDH@A>$27>kl<4momm5@Fl@&NVP1%a<5OokN7FW+_ zp9@RL?#AcBGP3t|?VjXHtU}y>d^Oqqx_n6X{^7Z>j@*FFs6sVgXDeB=#(pT|PSnzO zlY5c8tSm;;MfEE@=>MzdU0TBaFJPzC0~We=KdNys#)A~nXhX^W z-^50u8-2VYVj z{ubN9R+s$$UDCOp{dM2;I&ViUq&PQon$)AiXp)bZ#EW-*^@IPa}!Aklntj1bo z@A!`!{BnAZf8L1=^vx)~%e>z1WWic{Pc@ zULT?7m&X4*X8k^E^~8C48()3#HmKk#1< z{K4matHK}pzcb$Ve>d@exAA|s@P9KO4+mX0z;y#L7$0o^8|t`yAOF9J|69ZV=kIs% z{qsxs|Fil3&-4FJ@_*0rfBCbe{qMw?$@BloHrG{?F%J(H*9eS4?I8Z|BOec=>B%Ad zUwUc&i!oTK*WBBy?lHmt6+YHE<1rEG%-}GY%(#cFdzk9pr#haFPGkLZ=+d|U@c!Im z-OGGXG>9{aIH#Zq%}Al#wKFjbb1)b4u@H;V?m2aAlYTPiS#;6!(pQkK`|W?ye?$6_ zKF0q)BK^ps4b^w_`HE*5R$wJ+he-dE#!}FeQ`{R?)7K)^-@T#9IgMjJ{vrFc&hHzr z85Qb$1yp{)xHM~`Zxyy3JFy#ku^)BY?d#?LBk~w>DB0O}M?5CaIlhTzWkJeu zBN^8jyelm3zgtnJFJAlK#&`MS_wKj%7alXe-0y`?mxX>iUMv>jW9~?%Ly^kBCK#s>mOvY49M>%F<7Up0s=3^liBewr-Df>HP^n4?Z z;~D{_^<&t{SiUS1mY$@YpRD+`Iep}6WXGvv@V(a3zcLNSa9i@GV#M`uq z4I@*+(h+^~ZDfYbkws)L`}W=cQF<8uh%ZM+`A{^IH9^@-v7Ja|9S6U`%a^8pb5>sq1ab6z%>Ig z7(?+^^mEA(7=`xh>c8@Mr*b5>N&T0erx)__wX&f_-fYDo=cUy@4=YpR-hf%ODQn`| z)1$>R7UMAywXb>qzKhB9xb;K;eC z_>?hlWYsr~M|$h2u#mnOOR)?quoAIvt~CDOw7xROtC4koYsq!U9g)Uc(l*4iJ1=cy z;mJhU;J3}F!d7g@PV7dnKK;F9H$P!N*<&5QYVshGNTJleqm3u#=d|O(SDy+MU$wTs zw%cA~g1fcxbpOTs|2+T9^*QGhQTY@7`G0yURKM(5>t{VG>^M#!&Yg^9-zj?X5zlXi z@{@iJDc?feJ237sAJ;&u+c7fKTkpT&z{rq9( zy}tCuG5YW6&16cPH{IiH+(jSy?erYvg#qM1q%V&OgUO*7juFUS_a7KV&rOlGXN)tT zk45oSY16Ns_n%$<U8 zBhK}Dxca;{oea)ePZNJRQy1j*t7oc)7m6{ zr`=On8@Az(3d@;92_wv^*V# z`fWHyU=&7UEGjC>Lfl_)JUyd*F*_^~CekNkDjtp>yCv=SJOi?LU7IUgQ=eyJGVI^G zYzo`egl2l`h%v^lEyqmE!W_)Sd@RJn{dWhkyV~r#kVgSUw7kjw;m|95p5@x#*K2<# zvxxie?)EIIrUT?){Uh(N91&W%ygK+%Naz8l0>A zz8Y~2>b2xLRDOZq|Md_y&^MzBTd^HG@o@j24*!G3dg({AXOO~f=j_FPRO29$NFj}O z&%6VrdG=)2`TKRNg8EgFZ}xD1q&L}r9G)ltA&WM|{)aa4Jm`No9>dqO;cK*M z&`+Vbi`_xqbLzqwVX^<=9C-n;|KSqZgHr#4_~W|56^*Vle(}M+v5ikkC+aS$|6gY# z*yF#MUpu2;xF()n+(h9UkA~aiUG$;fF6md-??C5y_5aiA$8S~F zKNRcw^5S7K<2a7dX>;DD1=;{+`)xT}PLKT#1H?NJS$fI;|Cjr13!QI@n?Dfi{UtxZ zarD^-J9j9CV+7*d-5QLdC*4~EzhpFhEK-h}9XFAU?ziW6Y-}U{{cScD%3RQJlEy{3QYF!!OL?o~t=J+E$dNO-!*Jcc3i5wi5QedaM_%wwR(y`kb75q10I z<6XX6dZ~TuRqq$8{Ksmn#X4-jW)xc74_WsN{}&BN&baSWHcjAT@-5?B`-RTgitX5m z-Pnu$s7AYf{tmu&C!ag_3jcZy|61RAfh_X1TllxF2l&58<1jMy{NEk?U!>^=#dF2? zlq6HgHaQ1rdTu%2+I_V}9mQAp*23~@_}Ym1zWO@9DAxbzwO8Z3d-(`8^akw(u@C*Y z;}dA6$9)VQ%=52%N&3apu-RlgAc9$$gajd zr~EN_4W<0QP5O69|32xF-oM#@*MD3Q&o%VoCbDa!|FSU;^xQn@pZ)Q0m)?is9&zvR zA1_G1@P0q`9$Kx#Kn_Ihq>qQenF$~ zW3>Ot?~P=W<7P4?pN(?OXpF^pOvGdqzN3wuY~LpT^Tj*yaDDAH?8|!gg>1Ph|KpIn zp1vXfpO?px?U4U7@_v)|mSO+%>c0o9B`(ek8Q1cgNss#n%pyCV5kI<+AE$g7q6|Rw zb#woJQ5LowUKQpzZ!T(u)nGn7`HHlml6|d;?MKR>W;89AR^bbs(-X|6|LV7_WivP| zcDxkJumT6_`-Pv~GZ*AYzwq-x>LJ>od-?V~%A}QkUxn4EC@TvEtfj|2zt)i(uo*qS z)y_zEe?=XK+>V{tjSu!O>~*{!)!4pcMCg6n{Ey!&n||vZ3hVh--z1qveQ?0#~% zz1W6^x4It-+r~T={%OKvVf!6pt;Bs0|2XaOuyxntq2lTjVN3rfLgnQrLe-7OLz~}n zcw^F2Ve_!3!rS#_q3HN1j^hMQ;isp|tZP*keq?=*U6(%=evI9jkA*XSJBJ@GdNl0B zfgK@Sa9o{uJ^Xx+{=1IX!#g`(4+r;rH`I2o5A~Z~3v~y+8v_-5J+x}M?xqktk>l)mOE=8FlikddXV)I&47a9r2@!AD%~?Lt9+Vukc&#CiecGG0XJo z?8`toNdwp+r?oRw?>;G=rqD>IHo%_ahJ_xxV z)i{U`mH*n={?qk;)%iYH|4WJ^g*4jG{TcObe3;Y5-C;f44mP_;=#68Ii$UCPC)%(_mD&bdf4ZWy6 zyFS!p)`y$)-R#Rb27^}=zrc2`u!x`O187Z_pzag~Z5YjljW`jE9{r_(x+m~zz9TPT$&N2RfhX1GMNBaL^{+|z2eQ!e; z?SIB%JSL*{neSP@|9fFFJz3+wF_k_YsZG+qL;5q{(--u;P%dmHW?>HIVm=mPF-q$# zX_t!il%$8Ki)?J9*#eIQa$|vF0quOa zjxEN(Y}0p2cJ1Tekp;)*8-|wGjQ_zQ=cO+j^TPMbkXf{y73TZdES@TC#dc)(_~d zC7af`{#Rw8O?VE4U#k0(M{yh{@WJPI%JCUg7>|5`yo5N1r-!Wkl6HZwSl5SsEr#PJ zZsRWc&~J}CV1B^>vil3x%_IlMuup9WLt~ioVmLVhqc9p3-|8R6lH)NElhOOchA@?! zj`WR>gmQ8w=3p+eS3SROYlqNtuksCFR)@5YVCApb%4y#HBKc1_5c`U|zpx=JbY9Qr zzNh`MEWB}EKDQ^p#v1h^`r`Q8FMK;>_iv5kAy=Tc;(K8w*`Ulx`gT^)SEIDP%Rcox z_0PDz%UWU83#{o(Zb0pe-Zxo=nitF+@?2)ow_-bXVmJ0;KkD}IE6{MA9l6JDT;^BM zOY8q!WZ&4J)+Y9ijAPjk)1&Pw?eFuVb!FBVpSNDyJDHoUeSodXqd<@I^U^P96ZJm_ zkwgmF7sY`zy|n(#^ZM^U#y)C%F1E&N*+0L>^>5x-A`d6{VJ~=I^x8+*Vb>K=LvQHs zz6TjU;P^P2>8V@n{4nY8due{$0_P$5ru4N6&r4su^U&s+6F7x4IEM?kgdVid=Kmv( zsm-^CB zP+Y?PBfm)cUo}SHr{1sl29N_$`W$uvLU@E!GkWa(|)>gAv0hdyRpsJJF$DyE|xz2@`HB)jE@S!9pB{0e_I zMu0vS^AX4Lbn+_-D$)xb7yVZOd31?C{fIF(!k1zhRv>%PJzyoh&O5G0X??2>&xKxU z|1mE%HB6tHb5>(5)?ouSqY7Km{v^MBg7pW+-1pas5NLIg#C8VecPj@4L+3@T@o1hp=7TZ%orKY0ZG0@&CSWeFf_T z?4|EV@2>}hUULVl$0>guA4C!<)ZV*aA4-ne+ zTlg4!)VQWxssF!&uaU97oNJ0WisLweQ#gZjXwU2aHom@-@1G;P`2Kma;J8S(oaO&s z+D-1k^|PTe#5|KsTsQFL5DUff-nEUemdm`YAZExnwaiJBqr z`~OF@Svj7AW_q0a*La$JKxzEj3_h~_o|K0h_sIWfrl;iLxvrUyg;E4E`N;<{bC$-UT*YQ%ZNaUStOdhSK#1CsO z{J+uUSd7O+#5GeVlT$Gr<)~dG|Ibr)(&L&24VXorgVY%Lf13QyCup1@|L=L5q8Os$N8}xwee-?N3-XW zLX~T_Vmo$XH}+ybs?mOj|BtxGUv3MVypL@r3n*sPkFH8T4ml@{!}QEW_Wxb>A8}6P zLGdJ!LK@k44&S_uo^$V=$kB_DC&leJk1oVHe$@xme_ryA5XTN2bzajH>`yf# zHS+&@d5p}-=UwOJEfna*BkX^k|1bVSTloL-;o$}H<`VgzY;zpP|MZBb;$!OW#1 zIV(K#it>HAvHA4MPmYw|M~2Qj>;oH8>K{6<3~yy0(EGCx<~naa7Gg1$Vj1FoCl%j% zDs=x^|Ci%9|F6e5)|K?`d0)#6-DZZR%vEx8W0SH1ggV-x5}_tdZ^5jNASkeb1_ zMAHH`0Cm^(|LZSrn5O?3jpOu7qj{hH=N;Dlcg0V8+W8x|LMki*1xXT|Bueu z`tQ-TNdNr{`tM)Ve^0jDF#rFK`Ts}E^~d23{r|`|>Hn`WK46IceH=ns8|7hSv{`1C z=-;0=Dzv|9e86kQ2Rtu+bnRka_OUOdar9xY`#J8t?g)xKfw%bUh2 zw_2+dS6p)qy|{_nxQjmY`)B!&jqPA-JIOc~ri;uwE|5jvcgq2DUz*H;squYw@c-2< zGoD}8^NVv~28iQB?VFAVV<-xb+eeP9sMD85j>cGw$3$$G_tLZbg~{~N{-Wl`Bg@}7-G^oQTuWNI3ycEq> z-_O6d)Kln%ihg04@D*5zRalL+Scea`A8c^E8Cy~DS>xo%^aT40kyXYWRDRhyH}tHq z4EEA<(YJdo5%$xoQOwJ`$al!g^Z4)I9Tg4=@A-}Ii%g;0xXW}5`=&CE%%OULP92HKGr*H=6Z~>RlgDdF$;?v<8+5JC^1&HBKJsob6chQG_`^{H+Sw8fxt1qjg zdjB=#0O1vnmxWq-9J@HsadMM#KzY;js(imk9dM0u0Cmr>f9y!Z7ja495txvsE4V8~eA7{oBI+iSt9_|JlF{ve((S%YHv;e%@L0 z^G<6ExNdv^-#l|q`@vQ91GH<8?--+gF+uwQSz5QBEI2Nbquj&jct7rOEIA$%F&R@) zH%$82%m#5M#oI_WiL=>%raJs*e`6e+SB{yOg_8d*d=5SCsnf}3&ZW;saS{7F?|uJ! zx%m;os-IVfA(tZh-^<7fK0;~yy=NQO-HH2)trXtueqtS@G|rHGxc-83;yC@rHJ*Qs zchlq@ksGfnYw^ZC`T0}Y13xp^{#M31h-)?4kIDZh<$p4FQU0et9N)1|{rjT!zcKs| z-*fsI{s%J1q79{SZR&$-#kCH_V|;An+5b4sExoUQ*dTl}su0KjW&3O2y4XK#bzEAv zmu=rp--+f;>J2EZqp?G~p|EN`@m_L2YM zI^6zSp9ov_d?Nhgfxin|k@i~~{z`Z}{^3*M z$2W}G@a}#jY?t@`({a}Dk)ER8kK#DGKRqJ6eNnyRlY_(lXRP6zpLM84Y*zAFpS|J&)^^PE1Qu*%X{D z|2_KQzt1o5-52@i1?10*>xO*+oKyYUAa+1qX`^|`eD?Xm7NX`f|JS+6ChPb+UW#UM zrO=p>4)<}W{}laFpA9Xqd`3E^gk{cIfy0SUYo{3(R?=5tHP&JsHefTV@beeq8d~;= zpdWZK5o)#j987#JykkCM&3gT6iO+}R3o}C9>}jF?@-$`cq>x%aDKsuImo4!b^)KxY z?m5nJNTTsc_l4#`?sI~9^5Wfz-Pnu$sK!Ag(LPf9i|5?w+#K08!?Tg+V;@iPr1pme z-p>;0BunpSif8M*2k$4nX;4UsCygA6$nLTKgS{CGUnvVYdMCZ|`(>f3OP#)3xb*<@ zckKV@+BasiJ;w%FH�#z1G^JN5yd*HD`S{ND4dQ_!OEm{?FQijjn0R+y7Dhzy1aO z#}f7Q1;hJ-}fJ=yb*YuEAa1HIw=~%9g@e2Qg{G0uU`5>)0gftE# zgDl$6D~_AEjl1YW9JkZ&0DEg)gTgnwjtB1 zUPRAHTPI5E@4cf0q7{~&)ar6-Shwpd1AffT(NP0IJ^>(vcnAK9c*JMjYc zhy7|oa|b(xv94K(RalL+SceVRjP@7V=LPKZJofoz_L)zgU(P%M7*-{ot=Nv8$lh^IEXV0N_t%M1`~S1@e{BDMg#AtMf62J#X7vt! z*z?LP)aH%3cHMrI*8iE!uAgNn z{!?H7>-PVR^6e(R$s2r>I0kT(I5TJEPrl1&dK~{VmV7X8w~H?j`yLA@qRM?$e&h4u zkb22@=S@V<$I8NFvipx84O7Y9ppHkDV=n;lmnwnbEY%eVx399t2`;Wyoq@6<_B zWZYw@(lu3wXN2v}*@@lQi~Z>RqVJaM{;f75vgc8CSTcn)+M>MWyCjP^isOiT3&k~F zPtZ&2y^J${jeZ6X*B^M&w>&1c|C(PuEmghnXvxE&wSwd4)c7)(|pVG6<4)*COU62reZp3 zM@qkQ%IV2zuATH;m`R_7RDai^X^8YYFRrZ_>zs4^7RMOOCFi3w|L-y5PUHN)U>vHj zrC5d)Scz5moB4n0{7opW)4@)xcFtO?!v@6q|7NlZThTs}{bfi0xAXtr)c%J<?f+}2T{BypCOMfB!#7rMjN^x^`6L{ zI`5g>q#nL;lDhOil$Y}HQDKEYmxW4g^;KWC=IEpRfM2Vt$7{|1J4K$sIb6UcR6MKB zL0-W%#QA@{1T&=F=BqF_z+gIsb158-*hNZ|46k6VD2)#46Oje1HDmYWjou ze{1RM@YnPIHV8{=JJ?KCA$wy$*h+3khF{m|x46H}PRF}Z;PV&Jbxb{lY=2q%1v=l< z#*QwwG>_>IHmv8eOcCpX&()hnG@%ibs8T}Vq{2x8(UK*U!M$aL2Puyri<0khgEYANs zN*+gSyEs9f!UyO7opF2)e>49t|9_tUcfmE6(1R<8`TrW(i<@{j|BuiAq4|HUd~W>T z%>TPBp1bHnzZ&U(?f(3~0rUs+ZwAr_at*?I2 z{ggR|j(h$xAdDr)qw;tBZ)@xQXXF1SI(IUrVmit(6SFV}9~%Gn?g!fU>BS@buZ;39 zul)0T()jE7A9KYK+YaWF3sHFL(Xg28wZ2~O&))a{zi52Gx7GE2Vs4DzmSF`}B97l$ zMXttLtiyxxe;eoz#{X@mZ^hq^{~HvB3$Ma<>_ln(cYfM#`h)e~OYML9e^SHvrfA~V zHDaSR_BUB)?~Mc2|F8LY*z20i0sf`A_WSA8IEc;{j1Mp#xr@wOC!jzUjjOKXm)7%7 z8(!l7uIK+Q7k6fGXwDA~DKhEW6w+uz4n-WrakLxf+_A>^1LF>IWS4OV`61#*5iKVN zYhN79|8RfgVaFNgWnZ=aKdP_VPfeN-fl+Ax&)WZ<*Zzn9?f8Qs9|@zyGZy195!u=7FDBD-Ti9Pr zrB6qZUO;}A_Caj<{NIN-zO3ADGf}IrZ5BBPHS~rf{CdoFJRi;U6dtUz9qTL$g+HiQ zE~YQVGOR>g|8Et!8f&o*8?YHw=>3Cz63J5k!#`-#r|*p6=GE;cyS;I;Kb#pTI54J7PfADp439Qc$=k~|?$8mASaX*{XSz_PC3CFPw z{}g#9>h=mN)jxh?+>GPW`UC9$C3;#}(?e#~8$Z29otl0Pz37~$d`1_4p)@{)EMDj1 z`0X4TTBQ@Ead9Kr7&av}$l5f1yN$c3USK^0vPPS5?XLcz-$DO*)z~v~AO>S7;@Z9$ z;lt^JxGO|B9a(3-34;`wrA8n#A9ze`2n%xUW!Y{r@Mt*Npv;ge^oSFYUAW zXA9K79WO;;n&*zL7rY0u&VALZGnCfpna8HT%C@7qL;goBKbE;}1y*7eR%0#Jp}pSs zhd9PGhpyMeivqof(){1E@;}n`zCUrsz2UR;wgc)Pr?u~kXEUm>6}30m-#h)oc6wa% zra?a5N#BhRjUP*BTNkz;)i{XKcgr71(&KlWBGV{*O8XpH#8FiK&bR{g@8k3nIE6U> zzUPzn+oE^>DiO|+y`M3ToxFq|T){Q;;wEDI+imhL`p~b=_t$a1{cQj}H;->JTiXSF zFyff+xbIE=dHw@gH{pNN{>Wyh-%q}r}T4=b)c;Rr9B0Wk=cR8w9oC6w12&@y*^$n{R(kFcaqCClPTz^5 zM2wH~Qb`4KGO7i|^0B|B19aPC0H|Af4o1 zaqdUOr^G`ZL=tVjrQKv*#`D=Co^9eG8$FvQ&!zdEc-(u7@YY?@hjfkf<=y)S-f!vq znCJUIT#MG_zxNdJ|N|BLK1;+W!j(sQp$|I5ZA(2pXHDSWv9!AtgM5LQi| zAWxxo`P1PHc@8zMEv<_nt_zMYp|ttBn9^pM-`!C@Nc@4d|iQBk~ zf<18h$bR+eh?D+n7)bX1@@^PR?tJOLg&&Tb77R#;2>bNgz5SZH!-0>4pYPKjwQE%P*_-CLf8Uyz zA9G*dv^VBo^n3pgV`N@8Cgjcc!r{bULOSsup7no(*2JHU9r^c=>i+l8ocL2{8uM?V z@%q1oWa3{#gF0RPSz|jdnpAM79gX~^)f9dEVje_Zm3@WUDUADnnM@zD4@ue*0GoM9qiv4_J0eTx{qyT z$D6g$r=H}uyLLDBVn3>J5J{x)q4@><#XnN~{=0x8TJ%Y@;?QgQ8gQ8Y!TAZ*{wpn> zHsnx5?Rx(AFg`Us+2r1EoPGjvPk?5}ag1amP6<1MbGU$q_JQrYb4pGvq$~^y!wC4mp9b^5!aZA>rcn}f1mUEHE37FKn%uE3`hG({r_k6|08!< z|NlMx|GU_@efmnsmf7n6IJAiU$6s>-9%U;~&|;hvOeJ=G{1NF>3PnZC_k} zq%{A&(cX3R53WD*zubSgRsA0+#P#=ds~_Nx+Fy1?{TtTKAopTF zs*&z6_lnHatN+X6NqQ=VBbE_4bV+|6rSa)|)c@9)H{tg-!47xTt6;7x$A!4 zf8iZj6a>@?{_Ji^Zf7vf4oT6 zE#m)bJ7_=>jc7tMQaB@ybGU#@=)o0SLoeDB{C{*JH&XnA#6Ly+Gh+O{fA`YL-#^6H zPru0jU%>xg;{NDu&-4GsnL{9++qjE9)IP)i7v3)^{rvp~?HL2;0};m##I*sN$he2x zU|~Zs93#;CsX<{BIT~Zp!%rJec0XZncXHz}^+!JM8``QSJFYt||DTut$I1VQYsxgC znVv#HnLpimmFmNB&F^yhOw7U@{4eMKZIb_C?{e*L@;?q;S3WBr4$Jo$GKU(4OUdR>uhX*GVa!}vvMv2VNnL-&uY^t4Gsoj(8iHO2|9 z*B3y>IsHwijh{pcoBd}Mw&F^?d_?ZVZtTUzd)E7V#QS(Mg#C`IQTU>JCR2#>57K1S zpOqon0@~;~6mb-3X*^Dzz$u(TT-znCadwWL}ks*>;euw$xaMm7o3|x*8jJ?t9|_ryMoS>?EjJX+ueni+TmMWQ)8aH z{~3$%n22nXI53%>yY5;{rB6q!GZc`&EH2^Y!m8P=ndB_gUL6qTkaJN(Z>V?QnD2NY zn)kSGl=c}oFd!@zwiL^-0xPi!tFaaj`q$Rc<2SN_+>FA%l=9=o%HJE8#P8oKY&&*h zH};~pr7Y|xWB+Y6+4HrsaF9$Qg*58+yl?*pc_01d|B>DoS=xVM3j6G?FxBw5$O{V*Y+)< ztA>wHPW64Koio{QQ<2?aTrydXjC1PP!g@3;a&IWDOSZ;6zsmlm1ckA7(U;9~JF#WJkKD#W#fR+DR^{$=0CTlc~`di9=E6%PM^z@Hs=!?F&HPs7zdHCF60Yn2u}{)< zPOe-F|1Z7m-SRnR<{=Y)kSK#|wRR6ojaWb){{ZD58FG@}|W`%z(jemoO&@;0a{w*0>-V41ed!gmq z+2L)%X+AhnAK+k)Z^ySBCx?Yc$jIj}^1bTGWzW{|KQGJNZ;|VNaV>m@jOPD+m;4@nfFB{D{`d*`Q~Vsi z#H4RDS_AMo`do!?7r&-o!@|0{!MSd%f0*(AuVmw5aEVR3!nR#wEZo}=k1HZHWpEg0p{?1ZIBeVV=+kOoHqxJuKRu62$!aa!A0F0sKDc>Ah zd;n4Z@vCebQD+a&XoF}w43TGjf6+I1u3Gz$9eSUAl1etZj2g zTwl4i1Jd+`!`eul2f_pPJBrui4d^vC5!09PCi>up^V{bw^tWMT%d-&|-}S7p{@R@I z4)IrBxj(#7t>_u&J07$3$*@o}8NIc)!Ge`q87uIm5Db&h<`zeJkl(k^xm`Uv@` zeICQ(cmg-Z$s+sVN#W@Y_UD#&OwR4e|71!yO)gZ+|DNNRag=c+FoEd5%+Wsvr_i^o z{!<@Cd2>*nit=Wh9L7kQeD+@bbI#L?0SsdtlbFSHvHV%$`I2+;>pYoRpG%%w!01W& z`>;G9K4JR=nY8~TIpti5Rz5gim#3r^%>n#Ml(x8JM|SucJ@a3$3(Aof8?S##nwzdk zS&<^=v%-Dk`n?Cj{bV#Y z650O`(2wHvcmsO0<=;fU1<~BUw~_Baj6P7WUD=ow-YNVpWb8`y!jQIQ{D|=M@whAo{L->+tWpJ0C@^PfF7_vjA)^Q-*N z*~*C-^8@;?`}R+Eng8F!{?`Z4zsCO0W&dNSko}*-{%6OJkRNuwkK*G<^UXXHoge2v z=9}P$HC^ERMb3Yi{f~KlS~usvun|Yu_+zeboXwwL`%lo5W$gb__CMOBwQViLN6E+V zIG(_hcpA^37Xui^IIi^Ng-LS#lX)R=iSIo##hkt>iroTE8`ESUn!ac_%-p-9Kc?6z&Ge^uhWP4X?Cz{ zi^_qaTKOKs$MpwP$p35nztjo;?D%`vf$%PAd<)+}%d53N$nW6?_z|Wr@}KK4&TZd0 zGV}l2r20C=H;o)sUykjle~>`N|6ly1?Nemmh2O5f*UnByW?f%>0>kS0XsrDw&hb4{{#+5 z;}#smZP+%Bv;O7Wa6A3VmkUEwuirsm*FU@c{kt<`RXxhjFBay6?u*uRsn&k6{S~+i z_h9n0I=10Jcolt4T|4Xf-bcS53k9AHQtI|-{(jW{_*Ca_!UMLw9&bRV{o&cXi9WdL z-K_I3i2e`o6nlE*dfP0%p{?NAK3$g^{>J@2m1ph8t=#aWxjA1b&$A}vFG8m>^ojMI z@VU*M@Y%&(V{(6QtnMGeXLfeOr}wUg_N}XYVxKqu_3Q8!=Xx933Z4t+%byD^xxWsr zJHHB#UiwA&Yh!Egubx!V#14gL(5)}x=gObIExd#8$sHl)e{o;xx-&d1%@5kgkWbk6Nj!~baI^fFG1e?R zeUklO!~U<+zn;q`D7-%QmZmRc@qb)W_N^!b*OZ0igmAPrU(`Q8MGi=#cNyO&r-Y}I`~y-Te*XfQC@Koi zInS?g4f_)A$NovjAE3`&cI@V%a0~q)78>2RG2|3Ezij;fh;pyG$Ty`PR0j55Jrr)U z-|dLmJ`gug=fj{YAuT;{qf+=ukzWS=qTe)@oG8ypkwnE0yuTe9Ecp>W7?uRvQg z|L}uZ;V$|;copu$bdC1+CG{)jFptz_<@=5{9!A~En6xwg0m|T;{$J{cDdB+<&$X;5 z+%K&MFf2UM@p9jx|1cC6>EAI%;~x!2ZF@a>^NYe8$TuNoQc9xgP*?#yJ@SEU3Hw8veILd74{4wMW`B5`{sf-H(|88G7{D+h-}P~F z60=blU%}U~{rx+`H^|6`?_1=RmuH3Vkl)4k@B_>>_?~OEIpmig2`?yH(nw)mzKHVN zPsINeKgTaI*(AR#vO7MMqy5YNaWB!_oI!HPJw=AAo6yEeVv;`9?Vm$;^L2O)zo(DL-!Jo> zOY-;Bn_k~vo--UbdD)iRqkKNc8?xRO@ zDkJ}&`{@thcjg})*Z+5l{fQgvh!4@@xUr7-HTQbdaj(Z4@Fu(kZ^Jtaxgx@SW0m7v6*S;fDX<2k1BI&xh$B#)A4Z zT3a_o&X@jn{lB<=)8oFw8u>qVAbiw*A4g35z-e}MopO_Hb>3Lyu<(d*W}ZEH#(vwM zFb|zqKT9%B{w}_UA7NTsat4|8W#sFd z?Mvm-LY4BT*z+%OjbyZ@(gZzOsA!O|Mcy@TqcX^lX_uy4%ou~-+k@w>P9L4Jq z`Os#@_mod>5KfO~hc}VuU#ooj{;lC{^mpK$co*J-?Y}q>-ba1_>1WN&{p^A8Fnygb z(uc{9;^R1jM5VrCvJH>oF+{$7$;PbkIQ>Td+Y|IBv9PCX*i}a4cvpB@+%xD!%j^83 zmK?@7w&nFna{ZOQA@Yj3Ty3BBzJ7CS_=@n?aF*@YtPDQK{_9!O<`)-}ey{Uf;C;Qt_V3{bNaIKN34V&cF7yAR{(*A+1Et2i*`=BJ|9tgF z_^yu@vhOi&TLKfyY~NM($`bpY-8;Q!e!B2%zxnByPqBStY+&ui1+uT1{f_|*VhHhz z?Elv5{{KIB4Zp;5_%*I!-=uE~w_v)9{U2xl$JqaD```KhpnizaW%fVDk-$Wf{hwn0 zlT(gMu&)nF=QiAqJCIp_Q2ZhKoVM;PUO~SL3yrS5-aW9%=aKP0IeGBE`=6XR81AuO z`lb92#Sc_^ZfJR>z8vwxWu9Aw=XlI>L|?i77YtNi-)~6TaSXS6KWJ0moNo!?e#bt5 zqj){ufH&cH=b!wZeSb;+hi_rbw=hm7PHKNt`8I2e|B+L|v!2>Z9JQK zdaA*5%kqpcTIxAsl0KE^Ip%wg80d7}_U+qnAN1j4-W!H0y+^#oHNFk+z&r6Sya(^Y z2au@D3lEbY#z*mSOx9a}pfN9;q0gx+XN?)2qqku}`#PO}y^TGu{(MwiPeERIjC>ru zWqIKV@=3%h_^zlQ2lx2#JZs$xanYF9i289z{TL_v>eT;nb!5zbDe+`nc$ge%;;YAw zZr-`R{$tu2zP-{Izm5@knykSQ2xr}V|kuGnJDu7$)xZk zIc2}~_r4^oKg9>&Ponw%zI$n}8}lD0ul!g)&CvcZNuR~`t-|D2BK_Uw|NCFT*XZ9s z%b)KLiBtQ+x9CZ765pYJ7vIC|MeWm8*VpCx$h2^@mSj&nCp>NY5AY-W1ij*7_$fV_ zJ3nyjK=?WRmlzQq7LJocl?TFe;(m>5*f*s-QqE?|+wu?&2;YK($oRkgnLZx+?YIMn z(5DXWPwJmY{no!ASKodq`yV4E{O^!#f*e?)6FJRvTT(vL(wjY;MDGh{EKI%rltpQ~YK2oEC~ z`&rmfP8|0g?D3ze_x&R?|AqYh|J%0w%-nj%O=31W9$&%N@C{7Y==;;o{vWQtv7-IE zX8fB>)UkgXwG-?=NuMHL<-7TobiRYAE&E+ER?43R-=qHkKf*wf{8`}o^5oB?Ho+MC zTst6Rf8?tF3Pbpb{eFsHBK^__!*gWYmmdu0eRsd6U&Fp>bq8+2LHw`g|J`Pv+i?dD z;T5BP@$zY#r3RK zgeS-+(c4}To+h6`%y9$K8BA4#Uf}@@R~=RU9t}hGi=$7t-(3EIQ%A!fh7iXvMiAxy zVaJVQ60=C-EBG3wk3XdU#XNs=`RD4)>o?y&Wxju!To4|;^iUYXI1-pZ5|fz1*%{?K zqG$IF>8+nV8oonbxw|5Kmu%C=dwxH^-4|Dc?+O0^Kf+J&QzZHy3g04sj$h(AOm;hm z_+QiKjvo!PxJKVMZ%m_mMcO(2E5C` z%?j;%?`5MR3~yD0k$m?3RqJys8QWiEFB^l6=HtCyI_HP9pXc|7Y4!0;i8{Ji|JSDe zudC{2M7}`_7*(c>HOl|T$09QG6U{a1PPD zqBgQmnbPkb23-Hm|DTjNmvH1G^Qe6u!{c}YPvU8`jP4K5kkiZVf7SiFzB$adyZFk!z{V|cSpnaH+@I+uOQ($XXZb;|LETXUlaEYBKy`~*KmX8k;M*Z?x?1IN`p&FUcbZ8R1>+QGi;VqcO8+a}a!(Y&{4 z9^91hwD_4$b)bIwKUn`?+>r5yINLH~Pv){Yo%5F%U|S9@<%H+xG4l+1@oV}u?3-1F zqeomqJLDGnLEMJhF{#c>9zPK7pwE>%xB4{w-VhE6XV%YEpQZ}g--p$|7tCEyW;f%k zzPnf0Co})A^+n+>`ruVx|k|@nhH9SR)Mw!W$g(CcFi2 z!#nU!ybJHawEAR5nzM*}P3Ord-E@)j6sUjlqW%}{&kgNQ_0a^G6rLof?AOE2f1h+d zfb<*XZ|lM_=#4OVI3aDf zIq~ZK^)H&=|3!WLZ+%f{{p_vbfcWh<{%N=++WyLY;UIbCLwAJR$lGxTw)@mc&+6Cu zqV*te&GQ}a3x~wV;tp&4Czj?eS{wty0-09R1^|?=-uuf&Iyzzxn^W zBs-3s9&9&d>`1|nyj^YGPq7u`_)o1G1->2D;?85m1c4HAcl3XyJKdK*KY=u3u zrhkB(5Kh*!BQfQ;#O}dRC7o*2;1njy?8BZsO`pqWf3wAF>2+9Oo2G?R#?9x+jQu_K z0{ffoTyMXO{oU=p*x-Xz2ScN{7XA~%!XrEEQ!;-1VECKkFAh)V<%TXi)%i!^3k_xA z$^5d=+4--{S1S#lD=rOBprg4Wd^WEleCEPK;qijkhW3i`@M(PNbTJ=T>*Iy};rth~ zLX-0}qZQlFW`zsnIv>}IWK_<#N0Ki~M->0!ywKg96*}oX%D!i`iDOIq!!7;Zvpehfh}A7cQ3E7alLZKYV8S#i8ZTl^yDzCkp;3e9rU!Y>n~1D&v2?uPnUp+6{ibynn~JV{Pn^!=vvPw7J# zsItyrtvM<7jhnkNTxXrZ{JhZM9OsmGhDtK_*{bjw9`jwN_5UZjZZ*bY%vandwlEpH zRr~kWutT5oPrTV3|K7OU{J%f5eqW*c%#ZvVZVmI;wNKCLt>G%UhhF2OS@Yp+4q}cQ zkj`N0){rZlhv6#a8k=dT^VSeYzPJJuq6mjkj3d~d&k7~v`X>&CQu4}MvqBkJjtU$@ z!gDxIp1?^|V$$=B#u%#Une{(TX&)G8Ij=9_clu8bTUV({-}j0+e2aYPP~({1Cgp!) zPB=v$ylVbmcTPA>uf>RZVR%`cO%AQ+ggSAF>KBE2vH_7@-AFbeva@H|^K)nxZbe#s zx`274u;H0iiNAo0Xh#P+(S<%^_nH2Ir2c^|b*^LL7^aVG7~k(_-(&m)`yLb3?0Zbo zr!d{fKj66QIOaNPoR^+D$v=QxDChrIWG<-l#L$m8daK#r>ftf^;7N5elJpsjXxj`U zjv>78|HCQUFQMi0>R)!(BE2mbul;IPxJ+NdGFGsPHT2b}|2x>{7>wzk)i#f>D5tc| zN3d?64QyfyJJ`il>|wfE{f}AwAajfAzcuwAnMO3%eKe{6Nt)x$>VHf$UhkWptX2Ob zT7N4m<$Fc??8`$gIa%fVDccwF=yQ$oX8pd9PcOiNZ!3)(>sJJs5n3}03EV90)Pl#8oCdcQI4a3~z3AIAxt#94KDvwJ(|ep=Y^ZKIy& zXR|}4ZB?j74Nl=Su6%G`s3q5x#dYNNpX>|uWCI$}gl4qj0@i<_{(X(Qm%V)DWe37V zaqZ|pCvJ`(6o$|x+>Ms`qOkpW>+=|!i`L)L>c68KEtI}?6(r)hZpCB^wA(_3%$4Wam^MBx0en^YTAnmbv?c)=xJvF}(ztn6+V_N{@$=bkb> zlsP7beA`;~>(7baH=^~;3dHptHa?F(_&-6G@p+754C6>(BFETwF8>2#`%`875AygQ z6u6Ee*Fnw;r#AQ>Z1F!Jd)n^|h0fD4WweKr#IYMjSW1 z?jHiX{w07Cafy;cp_D8`P2}-O*Cd2jfU!Vpsm()BY!? z9M|Jr*Gi`j^=LrvviHAQ6dLJ+#{HwQ|0a4fGXK-`&Br&qe>B&sSF4pb+CS%vL0z!V zMYN*>(fV7R-9fk7IT=_7o1WCrpX26#%NOeUz+2_7ZaGM)c=S{Okw@)#@~nTyzXCfxAh0! zogJ2>yZ+)3mdWi;XNMJX6>CvEHn9EXJ3~aY{??}O7IttIH`d?Uqu*G6tL2sQ?hnV2B>aaHH=BXJhhHTz(?67JTR5)(-f`{UL{*i&odTi#&Qh3Q&k597bQg`UwLV z#E>#7S~oYdjxL#5-+zPszoi@_C-#(Eex3)%1a?052*xE^hhGIP-s_^{h~1pWf5f|7m(rI7J_9X8+qKTK^|Kq)rx&<`Bl)wMEF& z_DS@!!=6-5+q|a$@&*o=`E&2`?X5{TLd7PZj$_~5YBLASP z_CuBaKY4gS8+j1FKmL)nPS_FqmY@`6h}J5O z^3+*OdxkT}Jb&+C-gCeC{OP0e`hOq)@cexbN#8@J{pGmz_iKOrg8d(@|MM%~x%4Y= z3@30Bm8e2B(#HO4$n6hmf84G9?%x+q3AeqUUx0N4Pt$vwygSs=qdAQOmHR>+y&ln; za>F~?nB>qg_utC?zsOD|Gyngi$JzeFNp>=u{kUTq(1<29qZJo$5z}$w`}Uo6%p5sy z%sAD=Mkg28_oMynfAQnCC&&r=M{E0M{Qt@5|9|b$=|CsCFj--roP(j8KF6M)t>X_* z@5e%=@%LKx|0(Apqr4ma|1UERMDJ~M{q`F}%=r4i5%*PQ{N391NpZs^?yJD~I~gxF z{@&&OWA=;Nj~sHHas39vYxZ06-vNCF=ilocJ2r(&Sj1&4VY<;a%r41)i{Ag5_fKZV zf8@PU>5W}<|E=!zg!f%o!HSna&~i^giB|Nd== zTk(z0c@DFlLC@)fVN=`|cCd?H@v(-3;VL~de?Yo>^sGzXzv~-D9FZ?aj<{TGfBPjN zkIY8_3Q>f^NWVWj93e}v{&uz~dF8zaLm3&>{mnS19nkVgdHNUX7wbKg+g5>NIF1uI ziAq$V8Z|hD)2Kxq>d}BkG@*}gP(KFr5e{OA9>=iqA)22wstzAR=KrEi>h}Wgxk&v^ zPUV@CV85I52e*_NY@$s6;HLHgnw_H+7m%pXPeUe4^e^cDZ>Oh?^LLW7Gx{emA6Iv` zstfk?qwg9YIU$WI_m7rf{=R3Bewks$tzi$3tHyJ}e;uu2`Gf3eQ z7I7I%*!Dkwv^>845#NY#H2-gfT*VsJ(X+EJY>=DS!VY>j<^T14VV6GGAphekeGfO= zZ~Fg+ly`kc)IS)g_-+5Q{`6>Uf4GsIfDXq*eQY=S2eQ7RP3L>aA#*W(S^bWvj-JCj zJ%zM7a{;5d>>7+0Y9C;Nov}pZGtj_orNcjWllf3!b7!K9Z-&TGU~o z-2IliUosl6xG|nl?7h_6-V^6P(0KhDJR5z3f5G=lT;%)MXnXYUzsR?6!Zf zY*UESC)xMW7}pp*8he^mMd)~F%2(0Yel*4(`4&g(Vvne^hHK=r z_cw)<<1S$lm$8IptYBtG9^cgVC!>D^Wc=KP7fw2lb46osqeqOrWnCYGOD-!T3iTb5 zldHxxkElO7qB#kkq3^smr9C3uHN=w2oh{{#wAY0`8TlxXAZn;fV6QTzkc~?e8B5S^r*}1-ZyWKDJ+N z4(z>W!n(2k0^w+Fw8BV!B!nVzU3;_Va#lD@FGlN!8p9E?_Y&VT@g?*@a$qqll+w!( ztpPGTru?_hkUFbxt;st$69%%*gh31;j$w>o!}XOrt^&t!9KSRFujEXaSw9nIF^74i zkj4T=kDUo)C(nfO>N6oxbtX)doe9a}Gs@R9VXELv=!-vM{P~eEfWfUt*wl}Nc)=rK zIR6p-w~vGq&UX@(s6yK>tWPo>LN&bx)ApGud?d^se#HF0N3NIed)kP*%6H}cX#FE$ z4C6=~e3JjRx;3M)o6)G4yF`^kcxY7~JrG zLC+>m4r64+x$N^o|Kb_p6fPnCcKya@)nz~17cL7gVHqn}McN$9D<3%%*68cgO<|qf zz$Uh^gI!$39-Sg&KUirkCa9nu(Y*siyo_GaQJRYqL zR!6VLLZN(bdy1StoD~|xHKGa4XvGCw#P(Mo3GL+iZ_b1c^2#qB@z9i04cZ&Ng-&r@ z=tj>~>4q1FXwGlHa2&mz%A~%WFh-Aje+Opd`*!)hpKVDF+kUhDSHBEm=!m+ZQXNsH z{yVAObZj(kmLy~7S?6EYxj%Fl?hjww(J$$k8ANsT+@d;aOUNU)_d{cqM6ob1+L?>s3a$MhR)=r_>Ue@S=|X?nB<(EP4` zeEQ#9{aAQ<<73v7Yz$8w{zUk~&L_f?r<#o4)rZfoekgoy{X^l2#*c)Kx{rj<;xqAD z|BQQoXm5Of_;k_3;oRZ6@Ru9^SGX+gCA6JmUK5L!BGLn|JQHH5#OX$Z@Cfx z4~0)Pe#T14;de)-41-Z!7bGg2sz3h-D+*6Pf^2xTp zVn1rD7Sel*jsKM8gd+N2tv-}fIpHw97$ep4khnO8>T|*oaV2Qke`_cu%dq}Ib}V`2 zXa9GoAdlfVw)+l*6XZ!$q6+=`xBHs)(ZrpnQ@Y*KozX`VbDn-{YO=qp?N@_SIE`A= zA^j`;4JUkqp7EUPnk7@h^W=hSOE>xUr7sk)c7T4wKJlBDbBMEQc3NW6#IZ2trdDvX9x116k~2 z-1JXpU*7cpFJ|AeV-w`WVYWLaF@@)601 zCYPnLdN6F!qkjg@?i~y}^z^;zJjbV_H0(2PpUCd)*<$}kc4(#cTgSn$Yu~Ggb-C^t z*WJpFx31cGHpXzX>&B36ar^GsFVTK5WPME@!DQ@U$R+cTj63(b>qfqC0n(dn@fGLZ zbq(~M*e#(@d=U<#7`;o@Ps9=WV4X2wl+a5tQeh2B#H()Am+Jo$xt=cuid<)b>pJYZ zFpLqDIi?&HIELdmfs?4jv^Lz#iaLfpKS$0Brz+J2Siq<>#xP#&{dkWPM?72SnRLD> z{S#Hvs74KXlsBiySS*hX;Xi-h=ToQuzE<48rm_e_dgk#Fe}a0rL_J(Ae^tm|?PX{%ep%Bbv~RR$RbE zv}5`-`@T*aft+i)KIbnboEBbK)BcOG0pj`x$b|ON1U=cMp6Sp(P$MpPU+9odC%Vv$ z$vk-!F?wcwU3PLmJ&py}o0gwbh}PE~6PLscQdl?cc!`Yu{kup;Hbu1NW7IDY+0mE9 zEnyjv|3S1aMc>5o%U-543v;E4q4t4!E zm2VAf3fr^(gZrT8khzG~o}R7tZR~18p73puQ5!j9|5mXP5v_GQlvMwx)c@p&@QSqZ z9G{N@6ru=+QH<$Y^)+sc^UoKmmyym>PiL|JSJeO71mk4FeiMzx=Mk;XGj+=NyKP6L zQ-V^Ip|^tligJ21Cob}Tub>~pNJ{@Z;uy+0;KTL(X%C+}t^Hi-ynpBY)=MA9?cer8 z`Cfn2Y>jqAxpu=5*T*j*UF7;!-G7Jvk1ppR|GE1=A&rx$L=~!0gHt$->3aDOvzWup z^cQ@o;^ZKJ_#9#Tm z=R{t@BBHrym&qk8V+E^N!#XyQkmokZ8~*)U^d0PCHnPz-&jo`A`kf}K%X+9A2-VfdK|+T!RRsh`=tDRTwbn{zZ<=OOcu)Dm_DL>z-+nmW1gN_ zpUAezmnSpsldtSLe0|-*7+L7NMd&@UFB~S1V6a>NgucIcKihvs|3ruW3A7kPNWZZt zoPV$J|DT!1s}HNh{!uwlN^U=u9m>dZRNxqn;{;Bk5>=>14Nl=SYOy^Lwcm@v`X_uJ z!qGo~>d6K)q6rD@@n*6W7jO}gZ)b9suNi%=+qb!@KKSv=L#OaUoE}pSjQLjC1AWHC z`wNX99x;B1%>4dhL3%OtBaYtKp%6=QLMrUeO!9FaBd``a+3!U+Nn`f&B^*`Cg>i`bG5ZKE4lRF&6GFW#iu&j?mX% zdnlBUr6@x=DsT)vOZ?Etn6~8!@+5l2RgzVR{;d=J3n;E#JcJSXDf-7wzcy$eAB4dy z+jDIvhYD@irXHrZd@v_eJH7^|a2n^jeDgSq8D+{eGV6qqDa>O5X^b7#20!5$6>Ebd zd4vzXxTykdaP+yx{$=OJV4iZdfQ_8PHqK%rqt^NAP>)7L{|;;-o6(AC`_B}4&PP0F z{QmzPGV8Bb@xQ9!Yei0YAIUQ1e}(ej^KUb!@ch?v!Ubt<&xdf4Y)3~F{-pPNuQfn^ zaVT^OUwQeV&_%AZ3A&@OF~Edp6{Ghfjuw3YlZ(!=X53eJZrQah3Dak=Fy2- ztz^$?PG}IH!X+%CH%I+jbRb-&52||y*zHU7Wkmn98)mo1)xAUHintf{AFT?nVI3RD ztpEL0^#XkhJJ`il>>=ad$Ic(XAcnLZ;yLPg?S~OE>l^9?DxdG^Y*I{*VDugfjo}hOKe>7Bw{O`-DUMRSt%UN|GV*DaW4TRsMe%qQ`M#{r|JtXw7@tHEm?0UPLIWA~e>Fy7<55jf*f-luM*eTDpPUkaCye*3NA zGTHWJ_Vp|I^w5{Ff>o?x9nsiBzkEJ`K@7?Japgc}{pIqg{>=*;_S?i3cCd@9*u!** z{4c-V9RHQCGV8y1myw@GX8q+#`Cr~o9FhN}nJkw7F;&2}=X03#A3ayTggIm`qB$6O zWIkFx_>xc%#raQIAz6gO*wz*(CU34MUt;X%n{0@8Yfp=h+8-s+e#Qz)$;|%`_zslQ zD{u_Q(Zlv!bRT8(6F7<9Jmbfxq(?Ss^l$nqdNoE)u}z!Us2IvR7;3~tHvB2_G$!e_ zWF3<9S#$8`P%qqoG<^X#)(PyZVgJ@S$7%L&lqMM$9wtX-UJ@D|(}ZTU;sP$B9UYi9 z$9%>dgxM8xQL8<@;dPpTU?>2s&_8>yGK=sQ@b)!r4CYGD6j zS6mMt{Hx?1dXMoZ{-&}Cu}byoN%>a2n=71$%)Gi1`B=FTNAni@7~9`A2UGhe-#&$C zdo8>6t=XZ7ei+3#f@t0E67q_9^QB}N%29ztyS{reIivmGlpT)KPv9h?HLvDyV;z#T zGBvV+`;N>1Rjw0*r(IW*>l#zOAo6dD#{VlFUxjMa;1o`y7Mb;*>g4~^o`0U_U+DQG zO<&0I{Fn6+;O7578a)3d&!3FuU#)*ko_&qE|JMJH{QT>rS&s%Zq6y8ofQx9y_K(f| z=Nr&L@9AQ<7#qs?VOA;wp5nVzuT7BL7dmavtpDTQ+TQ4Sesh1g>Hp)thmf&No3alZD9X5v9HPbI(FkC`+A9eO^!-uOxok> zy6FEv6YA5bZk=S0PpOv^#q3S#q;Ls~m^^HsGVKrgT&?=|l=cUG84FeFU!+c}f9teA z#P!&|O0J<-{5rXTSiSp_ezdOcrtlVq^W7IR;~dV@*U$dPfa@C^6TjsCOXWX|;Ou2> zO+@p9cO0Mom^LYP=~wZ;^uMi?|1gedt(f2Uzm5J$xF?;qyVdL8&k9*f?tjcTa{NHZ zq0e>rwvbEDL-g;!bg6Hw)whNM$M!TF2>G@Zpm*08Fcy!|EfMk4vTeW8JDL=&3PiVJ9Y@WF7AY)1z= zam9FK7a5IzCo26rj2=Tj;+SlV`hc|k>52g4eD`|bMZ?lmv??Y?Q@D}NQj2DynX?4YO8w@5Dh-Cf}-xrg3H z`M+K{{4M#v+`X0V3pw;$Wd8r>8;KXVSA4qUm7zW7?(lft-QhEcVTnC)-~gE*6!B$B&nV&lHx0wr7nO>yv$=x;%XD)N8|McV81a=udV( z7`|}4BAj$=C7PSmcX;a9gCX)6>)U)N^zS{SjrEXc@K6{kdWik=5c}sL_RmA1$}!cb z!6}?ZE$UE@Y5UC-SpNWX#nwNl@U8B7=DSDvKO7CCP1Zj+X8i*sFoEPr>mS%}s_dcA z)BIp)kWM3-(2U;J2Scpm!O%({qz}wI7%tE+V&uYu+9nSgJ9#h+q2=rP-L@)1yKNnq z+^h(lWEYb3*?2{mOICz#;TY0g6=C79wnMBUq&lqskZ1jYBI^$vw*Ekc^#_WrKTu-* z0dl0e;&`g3^P$$(`6Iyl~sgsByuXkgmWdk z+5U*u-|OjrFr=iFnct^exkT@cyMHXw2Y1}RZ|gFB2_vWVJ+8X{W%s}SU|1HHII=IS zkgJG%pVr8AME@|GU2$D&eDs7jk#1BjV1Chcl3U_-(6b>Qld&?sdgLB@=~@3tTMC2u z^0E7iCzajqFKQbOW$jJ=+|J_b9)Up~%XewZ9FCYy8}(b`GrA7+PQ$F#jvU-9SpSkOx_t$v<~ z>S%RM4ZQx0(Il`P=n>XW0K3 zUS6GaU@#V|6Rr( zv{NRvSElO4X$zc`P9>^PjY-#(bnP|tId$u-Ydu9jjc857bhUe|bg$YPwc>iTN9xFW zME?$KAR7^L?0~jMG_Su&xEaGm+8?{x8Hcqq3f%t;`yV&||5c%EF7|Fq)YD|EV_w+5 zazXea+L8WHR+w()A9PXwN2~siPW>Mj*bC|t>$tI3*#9-!AIi;fBrt*GCjX*k{zb_2 zAIA2D%zAitU(Xkv((Xd^53uOpUvr1O{|~BXqjS<1^r1&U5)fCugAM? zJp3JU7inW`SNwD9Dt-ME`xz(d>s3^l?BU#{@am-!Y9z%o)u1#l^AUGjUisG{HXM8o$!s3Ikr`y8Z|hD)2Kxq`fAy? z?BoFqvT28s)^$r+r;HrwXJ50a$9CD*d(m8e_VpI~x`RF4#lF5^T{$-VOe_2RVq~BD zK0UV-()0y1cc~9i?_5zEyMb&(6PnQ$jZJAUw$d-)BHGar;rS;z+3@r(bR&kz8rOh+ z`dp=RH@Rl|7#2#|@+a8z$FBE5^i;FW#m^vxOX%&=|A9sNU_0A9bue6}FJWYd{eFS{ z-pp1fm&L7M6>E6m_}aQ~i+-35^8B;Dfv`Vp(zmdKUG(`z`h7D4UFb#({fOp0#K|!vF@yCX

$|ub(aTjaJC-MeLbhvC}_zFkFiE zxz~3TtshX%M!1!)|Ff>=-8td=`ao~|zx1;G<~*aQ4X{LC#zKQ~38_WT5-U+!(p@F5 zyp=CZ6vhTNv4tJ%;wtu#^&Q^=wtxJR(0f|n3Ox_8I`g#2o;bfAvJk`T@;ah5KjIiT z=J}(@w!`S_RR3b2Up_WuU*>Zi248`+kXyiFJ>RJf70{? z%-1^)BHO>hzQ=GJC(zqy{KB@A^g(GyeE^m8Dn$R(8Ae>%(OSEi`S*49Iqm*0igQiz zR{PLL$ZE&b;1o`y7Immc1E%dWW8YcF-TePSTpJlas_$>TFauA!H{PHrGZkN){Hh)v-wWY!fD9$I!!i0agJ{SeVQ z`&SV4oA21~%C|$^6g*@qnNVqr1p6d z6NuL28bCC+zYil#zFovIgfizSM+J`II8NXsD$%!PeSc%~1Et2lky#hN%NVn<>ye9x zLVmX#!fDi^4%7X{zcFh)|2h9XpRYHrj&y@@_S!pr|Hi+UjDIg0|Bf5~o-vk< zN&1v={sg^VIt^$<%(I>>vJaYs=W?Z8eke54qxJh2nvAU@v;Kd+v2$&Ro;~xNk9!Z5 z{Qs(LS1+_iY0(GlJ9zTY_3>}z`LMJ`Fl4+xv;KdL@&7veowlEShse0_FgYUK3(~oW zc66W|y$X{Eu0AeJ;wAGx8?V^2Gv1H>87^3m`zF7ZWkObVB< zh)jRNnD>Rt!qa^CW-yC6Hud~z_O~&BG`Ucu{=cB?!Z_Rf5B&e^)1#hSlFl+BKeQFH zxAMCG-zxpa`a7BW{}aB+)C>Io=xg>_N3{0O1{tmWvq^4YQu||v-1VK#9dnKB{?u{T zhy_Gz0QEI0J8@&4V5a`}EN;vb+;#j_>>+DKy9>F;chYH$jt zk$&*jP)pXK9@CfQf6T7RpX$^3tMVt(o$@C}kIVljf9s!Dqy8683P--_Q}#<7*Y_`- zMl_)rlPA^xXr<3xkalx+xIn*%1>v-CikxrH4(;N4viR_lo#@RqHb8bGMju#It}Q7K zg!?fpJR&@#{E3s9^}VC|TmEec?%h7?Uy^soE5=@D$P_MN`_&;Vl9#cBWwiZ}f5ES` zwVy#{T%G_2=Gi9YEvz>)-xP_T-NIe%08H?Q^U0|FUo3$9&L*qc&+8 zDMbIO>D#e>8U`?kp;ERjhB1PqYb-mo$T*+eg^bYh}O;@ zMjS)VaavqLpF=HKhsjO824n-0^jUotbNVWdyN^b3X?+t5`V8j9r@F0Mf7tjt2Ji>} zKXlDaj%h|KE+8u7GyOj)zAnPkqTbN{Y4^M8Z__h0D# zbKO6B`mKK;o$G$M-nTx2-njcmr})7g*S2Z?!rjIfUZ!7c&HXRC{|)yqzRNy|Ep-4H zL*)C}PsS0g^O+f!uJb)dVWjB`HQs$>gVOu>@b~lUA2=-kV+e6XYkZDi#xW^e!Xlo@ z*Jq9|y6$dv?eqovAk$CS!KP)~ryAtZCi$Nn-7x;WW&E2=?D4&$C)f0oId;mv(fSpi zTD&7%mi7{s(c7*6Q2Yu#v;L^-Sf#IFL?8I@5!Z;whv4aAYdCrqPi@{3x(ci}c>H$b zEw_ZvSKl5wD{l`^)ZZ39*ZjxfvsJf+j;-6mm0e#@9_f(|lMvqujQMt?YBkf@uCiuKhFZpL=sc z9{tArg?xInCdDiY=!LkM|FumcU-b^>%Mo9M!zjiPl%N!SY>fU5;{)q#4GhsU>-8=h zAHXR4dko`9U_u?2L^Owg>XP+;$;|oyjoJsutiM;R?#)*R=c)fs~!csE{^NnD5qC zC@z{mFh(ZPyJ|d|Od(e4zIOF1?wOM!yok(yUmEoXDJSA&-&OTLGV^oP_d~_%#S(2~ zGWz%CWydUG87p{U`)^fvx|*=J#=^*`4{wjWaxedgzw^B!v!3wFf0I3T(z%fo?i4;^(t_SD(-R+pK@gJ}j=gz#6GVY>NtI zO#VM)$FqKsJzVnl*&|thpWT=LPucyQ|CBwD^{ec`qJPMq$a*$AdHTO)r@#7}>=uk> z{VaQ|YCC)UbmGT(27 z@0Y9=uE8lpw!!w7^?xZ>PYbuX$Mc`q%dVyODq|xb$~yXBop+0RdIK_k)z!++O80h( z|Ifv1+5H{YvIn}ZWe<`=W7o3dscYH8{nxTbQ016LG@%)-xPXgj$FyfRQ}TTFEIC*4 zeD*w<5>Ar~OV4MIN@EP;&DXLMm_YKvwd~2-=d-7Zp3m-(#`>TACcBfo^0D7!cahzQ zp&xOKA&D7e>R;d0Ic(p$mz}7(mYouR3F+5qpOKSI&e3=+`!ap5E~Uo`x5n&yKG=pFM<4 zafzL4*<0ieCOywx@+y+_+2hKElfGZ!tX0=2ydXSZB@dH*uC2e!^RMvyPk8>vJb!Yy z-t$L}V{(y)d=#J%ML3M<6Y{&XW^0^>oR?;*;d&b{>T4cV_Kz(of5^m|@`p?cM?T_H zP5i%;d)dX(IfAGUwuJ1xyq6tA>HkmK-N$!b*Ngvu?Xyi7FqMP^1re1Lm6UWT=u}cl zN=m-TQ7TG~kx_AsawO^GGuVIu0|pEjFkrx7V;kGpHM@$FoaDqRGBQd|u1QJCcT$p4 z?)1CA=f$`4z4(1^x8LuN$L;yLdcWT9_v`(-e7-K;%;nmbXzlF;=63VrRqWB)+e$Jz z!}A}{zlrw$r3+U=?!hGHzWskgjKRn1hmTResAE3RFDPe|g_(k>n1<@Zj+te@Wn8r-{Y0uHDemF?B%v2|kZS3u2 z$9d~%)GM{A%8+}_LDY86CFdd919(swX;FSM*cTw$``0m3KPN*!hdGm57IKh_Jaid9 zpi5{Dg3Ve1Cu0N#-t`!a?7Z~i^q`2U=?Nt=3r ztetKCJ=6RM5biI)jQpgYb*V^n<_Q806{lQS?9(x>zlS}Dim*@}g z@~k7-M`H}eVvjbp1ml>?7kIaM#>vcSh|Ub&f8H8{v-TS3uaxF{f9ylw{=a7LU*i2M zy+1j@HR+gy$(Vwvn1<`bUND z&VNK_BFzxbOw7V;G>p-Q8gFd@b2RsHP?)*Q^UyJuE`RpBb04h>tTo`a%eyWhGqFeb zEHXNSxNrRdzi1AmU7pP4zJIj$^35yKy=R(tKdcR?(f23!AsYKvkX7fDbL6=u9|b5x z5sFcQGDKtlrbEis31y6o#{R9`qOpJbS!Gi>i*x|d+56ERy=Z@WqyuO~qyvagu?G)} zl+8#N(EgIMx5)-niofr_jr3{J86c4^pjEtWQQXXtE?}2>s7AQ@=vmN)NCyy&{SPsh zA69qZF!K>qF-JOp=#0@w2hhpw7>*-Np6)S!cY^sOy3kE;aw;;fObDmRGdPR$=tX?4 z_Blg)lu3VJ9pjQW=mVZg4GF)|PeBS&F&INJ93wFrV=xxuFdk``fOJek&r$ln*Ny3& z12mbvdzdpw$f=ly>8L$rF9?~PeNmW6&cgfq|K>19XHWL+|JyMr%w_KwEpJXx2aK0j z$%x#GF)bF40u8yOE{NVq4 z)4V@EINx9%{Ra-Y<}i++6aT1RJI3C0Ox!q#7WZxKGX6(9I#4~>`{Mv=QFqpJp#hDk z^=v1^)3^S&$$Av#$S=~7bTPLa7dK8ZpGHTTxY2gly>N!xt|H@0@*MWmJNJ+5MTxYI zbSKdohJ=^oN9>>X?t4YLlV}aY2jhQz-SRwTzfifu{#mbsYCm{N#c02zz{?Wg-)44@w;7uZ@Ao}<1RB{?hrmNEycrPruUmNRPqBF&+ z)GZZcCIA21|G%b-X9i|s7NUOv#tV%D`^rVM_DKDY5BC3@RJQc3J~;o;`_;60|19r{ z5B|TX-xi(!I9r@^Fc&?)P~NqN^O!RtyYYYf>-7Jx`+o&{_XzqZG8zwMMRsQ}M&}Rb zFy|r<`FQ{Q;R5FO&mS&iE<)5MwIB10L!}?0f4)xUUW^iyVUM+XCD#8{GWVVTf7Tj4 z=6Y0Vi=s93mBLgYK27~ML;W|4E<&H7ISM!L-Tn7%%zgX+9#Vgw(Ld1kwjw&Wr~R1m zf2VVOggJ~O=)^G`N6$S;;RM-rGaWzKy*|l2h`tO?;|$Is8vC9j&!ZO!Cp~|!^XZVx zTyEZBpT0&4b1JIL1MHuv{pVJ}uhcV?d6s>M_6hu#|L+!ASMQpk7><$n$MQd#y(!l_ zqi_CSAELEc+l=;U+J97^(*B>({`YGC^?~~IBZsyBhqV8~mHHNCzRA86<8)LEH!jD1 z<|<@A>U%vy_fV?8qYWDCp5rhcY0>?KX~sdL=TQ}zGJM&TxiBOQ|v{kL~A`TqYkrZ7*%H1z$SLw(R%r0nG=gTi)@(H`IF{AZv< zxt>YxI_^wbayIreSDk#fPfxSff*H4Itue z%2JB_JDfgOeEDd9#Cpgf_VaJ`|MhQ;49C@_h1`oGzt$0-Bo_IsWnyMC4s zPLh4=FWwvxqW}MOvF}p1ogz=;49;SYJXLbSc~Z>fhvcpE&PQhMMc?^<-nkML!W3|e z&Lzom#(f+K|Dl~g3Q|#;GRpZh&cGizDwGd(=6$NO@6GS;M-{3EkFvgOR5(C?T6<)a z`h8TWKQv1I9!38*Dl{D)6`HvpJT5KDtPg0mJ^<~^9mTG(M}M&U48?GaM8h$0l-PsL z+!9@T&iVVyW6|M0?Gwb2E-vB5aT|{`Oh7ui9vu}Xk==D8!env^rlKc|4AaQzn1Pwt zm3Bp#Mb5??%*7t_>Lp{WOJFWH&%Q5%p19N)kiDwJ+JI(r=WXWDg^N#>cGKk_)E==% zR+xI}+Q6L2ykv)dQj$ENO=zj~&Fg)8Y2QvpXEBr(D*xD*s{AMWCL?_tvTy%sn!G8F z9ONPo`6xgkiqJGh`5!0#3F1fVSn;F%r2Kze{-3J+3v)o2TCy%(-Xt4FEC23QERGVC zA^L|@Wn?c@u9<6?>(PvT%6hr7TX9a9-gn0aRqSp24&gAO{_hd86UT5IefmQAD_S3O zg8d}Au(U2Z|2wHq|96`G49=qGRqNo%^XNtQJJ!Je_TBuSOr{_egE17fv&`+2BQY9d z@c#avvCPq!{~*ROk4HzBxAkoA!&7!VKn_=ouuRlCv=fb0fDWudx2j{=cXVo}?~QXU^mP{{Ei~=JJ{D zjlTc?+(~EOC2#f0W2iuBjxk`K@h@3ksQuHHMd#w~Z!=D;u|7{63y_H{v?E<$p976V^#*E)mCf?9@TSHIC zt!I<-|MU8V?Ec!;VBEISng;HBa`hjw`h`yB@&bJW9AiF?D!POHXZ0U)^dE47TkYu! z!b!3V4a}#=)2L^DU;lT8{Ve_${h#r88T$4qrc3`Z(tn)vpCbJy>D##GJbIDvvbq^5 zNX20E>HnrE|1;!I?M~|g`4jE(ao_*{4*$RC|AZSVj^P-IUDlS3CQHaY7{k1J^dA$) zlKWES-(>lBr2I?nXRjhRnp4{_+uYjZ!C`&Thr_yg9}a7$4|TrcYl&-S{4sI+tP8@n zwB%4U`LeKe&ShcC^x@{{oM$oPim++2vuz#?VVwB8AGj-wCwJAv!p`IBrTTthN3QRi zaC@juxl7-2VmOfJKV|hF33UnA*jqR;wBI}^lpXnaC_VJ?P;Q^vzOLKoFK$sseJoU+ zzboudxHA+Fr@M^eoqM7Gt3~6&0b#fE75C;040{R(`0m!_-{(JQDX%3iM)vqWCguzs z5|&K3G%W4BG-U5c4e735KIY=EBK4B6vgneKmpUS>8a^tl))ta5@&hjWL8byF@6 z>oYG81xyDaz#t6*0=sUk` zntFSNzFfZk|Jc-!$2}hfD8!zb_7TiT4Moi5Q_Kx#8gDa~p{mf?Z~4xEBP*t*{@4Eh zIBThesV5tb*cZUu*Z=RY4rXsTOh0glegN(0K$BE>z{wAD|KyIK}NW&fqN0;XHbg(51ZMAJ-*FD3>SVRMq>=dVjP-s)N^_7?f)s#|7CAu zZznt2-s}Hs^A4QSKhf^hozee2ssF2Q(8xa;KaLkq8YUne4LR~pfiX36wB{~4`+hR> z6!iW7ZY&)fqCG(EN0Y)XWo#pW*#8y+7H( zUhREr&Ut^-9`^p;z218_)O&y9fv%_Y!HuhtlmoQk5cbJ`Q64O9Rt{?9yC{LtWXD6Pq7>Ds_n4$kM&Dv?^mSS!ARPzAL>F6la|KQd}w$?lMhudWI zypbHHkW(=Y(=h|_bp4TNd^s|Nne1K9+e=7xJC9*Dxl0^#$UgnwJ7%rqaxUV!)Y0?`@t z(SI;X=joqfAIkB+{Qsun&QR#OA{3(pWvE0An&y~a!@(5#d9;pHHzcbUQtAK5|MLGE z;XXK@kGWC3P%n;Vv|*RA#UZj}yzvJPGao@GqJINK{|Kni?}*Ore*b*lo?-gGL#45@ z9_iwb33~!d-o8`4X>9>>7f#_cy1&&g43+=7zHaX=`~OY8J zEXH9x(l7x%_9~>4S()B@UI>$zqctv5$f=RPcBB0vXD_Tt2-DcRogp}#oPn8G@+;}{ zrakjT>EXEdn#^q$W@FFsbo#&aFo(JPyuI0H)5Bcmd8it$Up4ryP$^6WGPueTPq8l@x$JppXYN2N+E6e&l%NciXqs;S`%L@a$(Fg|%@A*!oDQ<5GAlS5HbDenkdKGm6;>?5&XT+u&h zDyA#P=`n>;*+o=8w*Me8L~DLV3%8Wcc*&hf+GRRV_O7?>eI>^u4HJ-#Ntlc&h|d3+ zO1^*o&ot)gn1TPV=l?|K56*P`EX>9n%tf^KZyx#o`~07K?hf7h65ajns_Icz3=>=Inr>JdVnnA_Wt=l#mpa^|MPs3^C^;yi;Vxz z>UT``4NzKQ{HZPLJO8Fxo5Ea){o0@^6o{h)WvE0A>d}lgG|e;rkG}u=oizSGZ2V8B z-cEM(8vl!{MjLkk|Kt2y?e8@0FQW5r4vFV5j-V3_W3|89xMR#M^R&Nn^*@+TpuX zDF2=3-;0D(>fx8{^CFkT>F=(zzneJ~gE17tF%qNE{bW)YLq=!fjU~rnJkl@$>6nDc zn1Y_Vq%f77hEikx57u9l(a9*AmDtZ*g=qc$bm3-TCT3wa=3p+G<|_Zj2?vcKThPkf zma6U_ul%F>gz}FA!qv`J{_RP`OrH9Hke%+O)Kgn&$+w>{XevPwq1^UC4`ok4uR2M{N zg5?U6hkO*E5Jl*g*2QF(das0x&IT$YD^Y`bG@}iNPM`z_!Fh^^P_s>vIsiWeZ(kMm#kTxe>--Y&P z=nU$txgRko^qv2&9_u^*|FpC@FOAU7+<|IoSc3zI&PuEs{O;W5`{)0k7T+11#W|cu zFA{$5-GAW>C^8!Vr;y#dosBSPXh>xqjG>6;K8KSdF&bm={{O$mGMCqTH;iK*kN3|l zt}OAc_+b6}6yuB;#uamnE3m)NI0L6VOPcE@ARUu18B;J7O~xS2y~Y`2i*ZWd`hVl) zcJ!_PKW$us1H#mvH7+@BTyoMlg>3BPcE}n2QU9%8{e@}bjP?f98{@PaleLT!H?qH) z7^d^<7|wmL`R_6AHP#*hZoAxT$ui~C85Pmm`I+2ip`^sveUA6SZ1y?WKizv73subZ zp2qmKxz1eUJ`WjKfQBsh=bp*jQsDl1)?G8_pyRyzdyh8vY(*}&JmjMQ-Gfp?A=x#I z{)FtQ7!-=h5|p75HK<23N)wFh@WKBTm@Ba#(O!r6Non(5{%aHd5VG%7zMboMgt=*? z^p`#d$4LLv%K!POU(cVp1JzTc|1{~3TGXX^H}N)*jjr33n;JUBv&7tZ^#6}z%zONk zq=fr%=JEpipFC#-GM_}%dHNswL@IOWf6&FPwn*8(U%9#`DKtz^45zrAM&J1%^Nb6u z131He7VYDW3(z`E8ANryHA02f1{JG=PrT3kY-jHv z&x!LqdXex4-{_g)A(>1;hP+!kTKQJ)qBF2Q&|jW4en1scg&T~a7>=F?oi{~}#u#*G zhcK4x>wl1a>-W?Ji}ba}v+wfk6Ue^x_xAUtGw+$EKCHL?k-1#|XI}>W3G)vhYooR?+H(|* z?F-q9P>d2ZPwE$Pb$yJw8gt*3=D^8a z`S$*=G(T~j@~i(>CeA&=l<=!$E(Qrwn8$B{=R9Wafm>~%`;vzc^(zjO zN6`2GAY=XLT*6NFW9a)Y;bi?3?rn4c$GM%rNp#^9PU8%EN|VD`vg_95aE|QuPsj6Q zFA`og4#cjJ{=YZw-9AK=J}KPx9QXYC9I4FZXFdOEV?5^U>$K^Ir0Yq~-{twa|8Msn z_!j%cTSd0NPWR{9zWt9^%A3riF$QDNlqn5-!-J{50of{z+frON!TAHI)*q_LkoLmW zl65nDgUQm~_iZFg^OcW6d9+yhK*ce6^Q1g_{N4Gfaqcl5Y3R7oepoUclQ0=g%6s!n z<NGqy3}rtIJTnGnH?$k$s9drlM9kokrFxJK4+B^~?>* z$Kj~XxNd~}rH#$dw$`|Zwz-3BZ?^t#PAtq2b|z+FHuj8E{)RhqkhyRDa=JCW%=1vC zE{V#1B^j+>F4YD`XOQn}^ZsN-rT14Z_A4J%Q8~GBM96U60%Rf!xmfyj>jOrN3{A>Z zbDsVe*`jQ_n*J9)IREji{ulcGzkbRZ{$Hs}w0l|fmvQ8YJ0At;dCs{G zWEb5)5!rpaJp$GT6i06IQVCgxzV&|6Ci|Wd`goq7TMg>bG21(!?YL*d`}={K`L$t} ze~9Ohhj182u*aNuNy*62$y`3v{5OscBNGp&}tkUqXGCbwN1loY14j49)y@iSsb|JbsKsab%TzEY`t1 zRDRlAU_Nd7)v?xm^<7Q$z14TxlqPj*GrxoEd&NFtu8;-_fVZHqW|H-^|i2VAfJ&eKH2-l{f>Fj`TF}YJZT|(xL9T+}LR=G#( z&D#3s)B{ml!tJPi98HG@goF3#AEtP2ZvTEki2oLU!d2{lCC88h`Weq+7|!D+RB@k3 z){~!(qJDeQ^l=gDWd&mgb!Y+oT; zQm=~rlKJS6kHsSTmpZNhBFj*KC(;^qs&&MEfd0Bs zO!iH=9ZXh79hVBTPaH@=bm zfU@`fk#(gCAOY#Zd`a4*iJSeQ1 zr=FfYC_KZyV)B)-$2@;-+D);qk>9{V?%yTngQ_cjfpp1o-}o#9<=;M{MIk8+;GTW4 z@;}6RW3Ti#9*%`&H~Kfr?&PpMJQXhFzjDn5;X~wQ_%NE(CHc=i9oF9e{jlzV1>q`g zN3CJo@J32#Qjx9sK}xuu+gkRcZ>z)WxYId)GzvFDxzjJ07@6t4oAf{KV*UiC;8B#W z>>tWrc{)7KT>eZ_=nLoidF)5E+Zkld`lmyaK1dTi%F%c9TRi)M$X}l%lZ@_h@KyiF z%uWie;hE6(=EBhac4p{!=Q|3`#8_79#Mq*QiLu4W6Jy!KC&qF{PK+%XJTbO(=)~Bv zu@hst<0i(IkDeG?F=k?H<;KKtbO`+(w_58v>a6cL8jLSetp7mt|L`W2WmAUx(owXIy(pXz zwq4!f^Ce@vF<5nxnQ}{kmJZj9(S&d`6jDubd~1jdi}R%zRZix|OCw+Jxg5 zhGp1T;~$s4M}72c-mN*3>}}*BY@c&S>?GM0xku^noOhNMhuM!rZl|pGC6B#lZaVE* zarEddu~U)T80J@Ri8YN5;WYc1$W59?-!@9?#;K{sa;c$S;~D98qW?vo6{cpmwJ~Gq z;La)E*6Fm(Fz25($Xsx0IEZF6A>KA1Wa$em8k!Uq9~q$kMSpwJx~Ss=!qQIm2jzc# zi{;MSUXj{AtQ_1wEMwnCmJipi=B9Z6=zM1X0`5%- z)d?w~Ce!)K8QN*r*Ud?>c2YYne531|$mXswv4h-N(0Xi4tSvnyv@b}pk2NJMzBxG* zKh-b%*}I(7w(e#wd7xjoBtgFrE7UnxlGk9_{Yk;FGkzmwBn7uEDRUm(ASRd~U5uOr#DS3tKS zejgsjH_+ebJ*T&W#PId0+HL=a?TTWab+e8*F=oi+w_Al7l zJ233(ENSP~X*LMwiNt^QMAJ9J?v9h?%LV}Bk;@iQ#?)wAIj0WY51t>&&O8(L9eO5|kIuB_C)2!HX4pS8GgPH!hU$z=`7cxc%d~&<*-$t0*-$_A z+0ek;xZqi3_}S1r{aJO+v!R7-<=2+>tUkxHp<|r?nrq7|JjbueDza|wLVLfg&(fBq zTrC}nABnY%e=1zW{K9zSgo*Oa_4>Q`DSnGT;LrFQ5^vD&#W%1BYq1jt@UQqCevd!m ziW@^bK7C=xn!PYAa^J-n3ypslhMb~>VM*@7urzC7SeCOenmOP2$6Ouz@)`5}2mSkDrEqyg-w&&5=sQM! zKdj0Br&#{j`C)C={IITbepr8EekeFSUpbf`HXfT1+f+P16wdxaZ1dUqVawDn#I_!r zABtvuKWxkTr`Y!6^NoKpLvh|0Vms3wj_pdBA9jzPA4nyH>nqKD{jX7bekN>X-+@w8V`udW5YihVV?rl?=YuQdKG=@-_yW<6Oj-G4A<8vl}; z=0@Y+1moX?u!Y>pUc`MH|Lx=sW9#C2I<~gxEQ*A%o4w??@$bomu$L@7V~*mS_PN*k zSh9k>lKX!CRb;jBHTCqXuCFEQ*z2d$ug-jTe!FjeJcE8VlkSu3JO6-xJO2)H`^TgQ z{*1TqH~by_ZVB;$7==4@nU zW|$w%G(SqVvbS+>=ifoL&PfVw8A;YOC4~;MX>w9%o|+U6k}V68tQSqPUNk8*kd4Ar zk4y?RsY&5La+39=Nugpo{pHN0u%E0Frj#sW-$#~X7O;z5=b?ht0D39ZmM{;$#@|vu?rYf)G3ifR7IsBK9 zOWEU-m3P-KA{VnSlm`}L1y*AnI`BjM7{5g~{*HdP%U8GwpTlE#98X{iwxbF!;J4^O zzX{r7T!Kq+Ij+SGxCNiX=kYL}z<2R9R%0DDVmo$WFUnDc4!nrp;175WJ@_;JhQA|G zU*RHLicz>4H{d4Rf;$jbUu4zO|EWI~liBP!$E~|RY27`!?2NJAIr_ie1o=Nf{x`46 zeHH)Jl4CS*RLbjvzsyx_lo;oeU|TNi{|5oYqTX#83P-OH*;U~GiyXP>5N%l zl;tdiMXy{C7B5Q**}u3vv?rv==nDJ3-@0c zxmiEMzk7tUz9M<0Gch9hfVBW*&nw2(k<3mECnEVPYic8@W8Fn|KW{(FOV;h}NDAGr zCWd>X{8DNSKon+oQg|_V4so51!s_4mh`;B~=uAB3`>YG#eir9&9zBmFh2BVhNqq5l zNsxBkn}mNdDI_zeAQgi#0zHokC+)6eZVTFf+{SzjzB@L=FC3#?M;%^9_il0BN8j_2 z5Ra#PCbr~$?a>)&C0*K|k#>W%StI3%FX`jl!#vhE|CTo6(O6hv4rbc~e#3?5Z<`~|I(Fd~Z8Fwf&rQDB{gi9I z&fKg`WG*x&t;&vtn>=R}XW#RAru*0n^R$g?-h1A&+PrGv$@T~7Y`I5!I0|#B=JVl*?_hY4eVw|OY@AZE|ibB+A_vhd8?5`;?;NAxVhbl)i6=$W~H-TU^u!9DrC z=T5yKe4E>%CtY75+M+@w;x(#hzPS?cd^8!tWvOZT&4 zvxZV0DI9AK>r2Yyn{@AyyY#4SQeLjJhU_VAwD)SfQ#kKd|4yPZZqMfZo;|!*{-syd zhKY?_kHf zac=AN9%tVTvmyE>(qmf`F1qhe{BK4a>(L$E?@BzV+_FCxxxE|_?-1p`nlK_%%g2Rd?QwcwMA$afeP^40Q}1<4^QXxV+JiUbt2;;Z<%f>WQJ!a1 zSojbAQ5lNz%3^L&y7%RuG|w!asNGr}h5fcPQHO4m{?WIH(*N(nw6(+CuX<#-_A+fg z^9|%p7|47Jc^mFPPg5-1P2L-s(QBTk10b7c(T9#Ux6gc# zUz;$k!gffLcIHc5`zdjKp1c&7V-y}o&&5wA){O-*bhyB}GQ*g|&Thxidb_`l}~ z?LV^RrJQJvIw2`69hn@KjiLWcqyMK%TS2e4lAbMZc5+z7yjoscQ}(}U|EHVto|&ls zo2dWm+hio_|H=dN5<|5Avk2R;eQILZQEzQQoAV~gUDmVhKIE)w>|x&9YkewN*7@!k zR^=zGSvg|uN}4q+>Ga=ZwK!^~TR%6``Z=g=}SSOOxMS zAFadtu{8K8Ucig^4StI^q6A3TgV?vI5pl3zjonpk)&au@D#@|*Y~^H=Z~zKJLBU3?EuV<8q} zDOO-L*5P2@@vxEHg6()hT;Ihm=0(cOUa}nhxDCWbc;j|`n3sMYyM(#>p?`~AN?wjp zxEkg3N&h@ff0O(XYO%|>u7&K`{c>y<**#=wtZU7mV_hqYV%=-rBFotuV%_2MiQVy? zv8&zlj5vDgK9ktfS{v)RW<_j1dw*TOo@+M7^85D-7cwh)@ynnZh<^k(;$u(^#qYv> z_%t5GKjGFybMWYjKjmIm#JY!kBi42CC9$p@7sXWl)-7Hb>$&mn*e9bf58e^$x%G}% z_nlY8bj9L5*Nuwx4Ej*)zcJt7`EJ4^%wNVW%(r1a^CB$6DyU-OoAC_3in;ib=eR@I zyP-Oa`~2~HnLmYA=I>(@{uyiWFW8M=;7@oR@oCPh!y+t3_B?AcGR?nbTo9Jdy&x=` z?msyCC(A31XNu_`kcU-JMaDnsIsTPg>))R1{O5DM_PfBO@P=OdugOj9g~$7a&Etm9 z!}beX*^9Kv+w!#aV~2#oF?1Bzg3TzxR$S{HwvqXVE(mMRUJ%wDzaXqVX+K1|K0Y?` z+kjmCnPpgk<;cTIti~#2AyQB;CKn+G*;tAtXf9Dk(Sn0`e_eV9b31BKjanQ)^lywh zG@=1z`iQ0KpmK7beq-faIuiD(k#r;|X5OJ*+r@vUd5)5N{Y-9q$vuVEsz|%vZIxN*w@tw z^Nb0@0Cx{pL*#{Mjni*pgbUrV2A?^25fbp7J)#I8p!OYDB>rbPe$O6(f)r^K#%UP$b| z=a2YdVzV%((DT%5i9KKXQ=)0y_-m2F} zXx_tmdMxc77M7i~#=Y>eu>ADpVa2$SVdde=L*CiT!>Y`S!|JZf!W!=Rlbv@y^`fwj zTyNgCfZV{ok=&H&|9s?T_ATPsI^&{Hgl$tU3fr-x`JzyaokuPTyRf@{U?^#GHUqi$ zxbykXmPfLheev;Q9rbIkvu$zqRY92{={KU&XP z3yF4g;IGo67eO(-5?x91JNpZ`ko{JC24BZQY`{tR{4cmj*bm9$_w_e7h`X8Zz9z(< zLK%L7j*nQEgP-8%_$AV=Q(s~$cHl5NaTyXOe06hOZSTNLCnNA z@CGVlzp#}dQ`>jPsE79)E+{U4TKDV7bU|HE>uz{+{_ ze^`apSTmUZ4{P<8)(xfqOLgY0I(vgUdn36?9sa@kr;H2e|EzyXGj2;aZX7UF7bw`ajr%y(m4VjywO}J}%><3S**5?C;e7IjnAMd-sgd1Nt@58KZUjInf!T z4eZewqfMFmKjgtl>ex){qi0($=oz9jR6C}Ld#F4nf0WCg`%o#bR-mfPK8G6TUSyc3 zn>8@(Iz2G#oGNd24Gg=_%bOFTH46PhvyO68eyX#r%&U`a&F0x%cbl|rCu>fmhU#Of z*1xBQ17tn_I_{0^4dh+IeHOFOgnz|G?8PsUbt+W_kgEM~MhS9|jivVSFBvyX*-8zY z#JPpsjG`&2VJo?PUaE0JYRI>jV-5dx()Jgfo#@ zK7S044+1Nk-?BPZaKco1K} zSMd~buoAmbjwT$#Z*T_vJkt=2!dOhiB+S4M)d{~(5=XK*SzK~~``|&ya`9|zLAP** z@A09+UWYsISxqm*ZR|+LboLT_kQZE6!G1DN%%N9j{D8H71!`P ztqpq7oYC*KInR^-O*?hHa6PYF5>m+D$yz@bb_vD{Z`wcpj(AhqH)5PHgYbLd{)GSF zwiQE#%O$6ie;2-t+<1*|CCn-A*JC@rh+(e(EB9BKFU3y0hRbjiCFsEw_$kWpR|MCs z5N{sy3^Eb-aJvr=U;-}3e~SApVQyx=2!CWh&Yq7)U2_Y!58=Oa{}s3Oc#L~B`8Bfp zo)6LgTter9U!i|09SFAyeDy=d`l-gbxRv=42;cJyJdZ0c39pO8?|I^b#Ww`MV*UlL zVeZH88>FQX@df0q>s}<#Z!Yr_XFh3copwl9D1^`2{+Wg~s42#KZ_MD;V zVadq!u#{Zp|LM7D>Bhh5VFkI8J&*e;{;SC~bN%~0BR#CmOb_eG_3Q=QH}Ky`ZW6vQ zEj?^@{T6a7dlC0-{I`=k&ZLLpbLnAcZ+h58?q)CHzK8!_vQ+r8H2cV1UrtuAS90Ia zzlyA$>;LW<{_mdY|L$ZRdp-At!Ret9P2U(XmE80AuOe5ougP#mfwKhGlIz&lb1xWvci50} zci4zc6YdU$*vz~ITlp1X8}oMTIDdC2#!lv4*v+p5dzkm4bi&=C4EvbNQNgbg`o;=hgD&c35w z``@PhCwFyf|Bu@{b@GA`?V;MsUV6^{=w9tVS%tdyQ)DSO#$={;9^le>NElF7cu zRNsT#!Csu~d->M8$krn1St31M-$6F9H|I$2Jn2o=o%B7pH}G#HtJ!M~`(B;C7rAoQ z#E^$oSdC?qCWc%r#|mW6niz7h1WOT5m>9BPy=GX9tut>AMW^JaGx8I;iM??0?fTod z>l55=J<9E&VD{}{!`$1$Mskg7^05}{@I!UbZ(uo7{LfgRej1~`e1^=%!^}I#pWp|0 zPTlsBw&`hY(N91H9lr^m#AEnA^6{efjaogPBJ5R2!~OUo{smcBkH5k)xcD$!i;rV6 zK8uI(DCVF5+akMX{L+PVMEH#4XiA|YMAZOs3^Xp}_Bs515S&bI?#oCPD8nbPuTzMkRKF4duhONd3Mc9Vz*lP~G8<&df z24lj-+SH50*@Dlz_6dFuNAAXlJ?^vFeKrd7TmHNFy}-R#yS$Wr9s9N78Rpr3?0!A$ zLp;NWguj;m4dS~^Tp!o({|P3FD=zG6{=eb>cm7*kQ;QeH^LJru#s5>>Vcc42VE#0D zi?}~8%nJTr6s~M2-GKMK*}ba6$WW8y|D<79s2wys)V+FHs1L7%hGC~dzUs2Z2LS@W|TJG?+)+x1g;kM zXV4g>gZFP_{Zlkx9OA(!5D`P^v#Q_TJ+|3ttl=G9|jVa+^q zeOQZiSU=bt05)JFHjSq{$7XE7*1_hMunpU>RtBi3w?`Rr@Rb!8V?(_yW~Icqj%8&8niPz9-DHP-CVM>UcS^j9@)SE|#kZ$EFZC3{n@wLIiO_L4MfMz9xqP>dbeg`KE8M<0eN>_@bgs1(s!qUhgPf0NfX z$upzmqy6&7W_fU+Jn~(6>NE1ukNE!?HwtqDw+F};d1;_Lv!uKA|C z_z7|S1=q-Db*@<-<(IpTEUh_zS;U*Z<9RZ^*Alg?|-Kaevx1Kf?FiZ;Na1 zc8^_nP1vX1>k@IT#$Pi46*WV(Z%Z2@#aP{uBG74k7 zoOSlPu*m-1#mHVD|6>XBQY@Qc{Ey|wulK(@y>Hp(o08R?^2~90<)l1A*0I;0lb3qsDYA*ZnfpQhEo3WuTRr`c>pR*m z3ZIiE-^A0>bql^LJ^zeE-{x*ilwPZ)-B-B%2D_!%Tk4H3bDu5^Gq774zaZUzBc7$c z%eBJ&KKicYWxmVDr2kzQCCtP4qHvA4O}v-8=V#pKZSnq4_|J;BP@H!Q*W){WiF*Y; z>G~h@|4;tE7iW$zKXs3*xIcl!C~k577>|pypKC9{A6)Yd;lIfLkK~Kw)52`@oImBa zv52l51z3-DSd08K6RmBfBS$ocT|{op>KC@y7qFRsAvR&@XlsjxTVpiXTBG`F!s29e z-N=dtiCEo5SI<9>|4MR2g0)FxF8eaFv3G1}DDcg4d^^-KA3$_wU$maPBj2}D-rLOS zwnpWi{a~4IN|zm-eYIB{d&E^j?iTkhawmH+xnq_y1U63)ZzdtM>-Q9k9)>%qKvr|<<17)hZ6hGch49_ zm#Ix3J1P`U8Wnc1?;>|j85N40bH5eai>>!6Gbf)jDr~|QY(@dM_53y_8&i?_W%}^u z#@98E(uW^m>|jk==j~y2RtT%+n2(xi&Du0;)h1iBHq~5qr8#Y7WvMc?M4gwFYO)54 z4_ga@rJlJQ9rz7CB|m=-k6{IVgNx+J?_rdDc^kfoCs2znVZ8d?9SKzVb7TnVQ+!_`{C`~Csn1%6=FU2S>n|rMOS8>?Ojg!rssDsq zb@-03+O=!aeE)RoKgo5SzQ6BKaMIi`xsiR-Ip4q6`El-bAuh*txEZ(O6ZjPTnmztc z_#z&|xA7D*u?Wkt8im+~Jy`79l#@S3I2t8*G<^0Zoch;@LA?1W#;IytkC*{Q|9(PcQnVpviIFN{#ESJ9RHf43+(@~HbB^Q zmGoaV(yLy2iDzRo{Z|hCR~G#jwlHrMw#YTx3h2KmN_NckJag#37SMlXdfr){cedx{ zzL$IHbni2b{tM;I6;tTHrqX{+rvI8m|CLVvHG%%?i1IJL)hX}w@?HbkNH(2T{?EKS zR%($CTf3D1Q~FfgqVvJBj1l9;fs3(-IY--&o#OvD#?baPH1}W6zDkMq}VYacmOT7Gvhk#-K%`je(7uw{zddz4(MN z@JVCeW6`<)#=dhtAKN|m^Rd0!;OH#zvR-50^UmgB-^U)EO%(k@qe@)SS)w&#KOd_e z_xadEzS%vPhA&{DeEbJx;I@nDRB#=x#yETo!|||i*;_9RuM2k;$=t5Mm+=^$Kn~WU z5WC?_y7)1iSoUD-9C__vV?*4Ed-31!FusCCD1-fM@#FY4ev9EBat&_BgZOutN{IWV zc>FF*!$0FmEWi#>;m6xx>Lz{~_YR>?!&J<`Yom?kG&8n!xOv26^N1szVK>;kV$GniF>jE)Z1!}N4AKu7WRK6F@JrYI363I+ ze*_=Hr|~&_37Ob|gZMZ67%$*WTbro+fVq{~=}Wyr;HtiVd-VU_T!J^LCm z-}P(BbtCCg$O85a97dCb#h4N*1wiBe(a`RglH(JIP)AcatUTd&s@6kKUn- zeIHrQzk;k}-%nO$N&~Wn{Qz0ZzmBYDZy+1JZ|Scml}zy@r@CKO^bwqPrY zunpU>a8*N2oaC1MoEBSdQ2521bfw0`9{7cvzXZM0~?=Gxnka z&*3lPy2v%Fah0$q-Fvd|Y5eBk2&QuX2KN#);Zo)DYTSf%>Iu_>@fYzD61nxq6T&@- zHY~E1VKL%o28Jwh$&9416xrjF%mXEb<=WI0mB#8Od2{C_nfFOD?~@c(PE87VHFO2X z=?rGu2bf7`pl#l;z`DC>1H<}5bPCLy=i2|Qo!(>(Md4Yx1orLI2U-U*Fci(UrlQ8$ z3hdCn7Z+RibJE(1PHQYmthGopR=20RoW1niKku;J{p1Ehm>I;QL)$PR>2|^SSpUeb;~Owb$#v)?RzB*WQEO{yyLb^#9>S zu7N48A^Ug3_tH*6s}zUG4*CsS&3lxqG_r7k29c6sC02$cF9L)jRHPU|v?HvC;j$4r){5QL~4^(kqKsLF!Pn1aw zPY#c*@4Z3g9?BKUu~$Ip2lTr+UyLk)q804RM%wRbr5U%;*D>#H=+`4% z96OLs&SxUCIDYru3p$4Wjo^WKI1ks-UNYcucoJBy$jpNVXob7sZFrb-`)Qw@Fa|Hd zYw%yNf_Od#x$pwSAcHV}16$x9-~c=V$6*U`xsh4W1tGW%w+G-~zy(DRgco5N{vG}R z@59$<+fFEewcIEE7XAu;$MN4GZw3eCLmeE1r@;?Bz!FX7e(ot>hYzp|`5WW{T!VcR z48sr5Z3OzEYTX6aJT5TTbAkTe1(mJ8pi-z;ak)fj(SQELxUVsZ4faKN-N|3iv;z>a?`TvjSALp9!(}HYG(hhi!!F!Bvq`8^@X3XzzV*HP6=eQ%v_#e9R8UI619piuK z#oh<~A29y~1K0;)XvIf7|3A_&a|k04VVoa@7{npbxxB|XMI7cgxl@DR<{%9I~?;F!OS1wb{B59LMd*Gocja*hd9SiiDkaa@$caKxP2TMU`_e*%{=Fz zmEqeY&)hIF#Bmr#H!}YU3rs)%3v&eyxD)pvd=74f&2TmR9oXN0<38m$%6(x!_X+|9{Z0g7^DX_+CHn|MtuM4r>~77r%c%pQa4TIe$I*tIV&ks?IO3shG8uTKY(J zrv_Djc2Eu28_O=fe}HV+%lJRa_#eC*2bcH1lr8Vk59}GxcH-+u^ZchT(S_{(ko-|b zxljsxekQZOisye5`XLKAHs4nMA6);#T!vd1yw$)kn?#VpOs(dvx@2Om-u-Piu3La#{4IE9z*FY z{h+fvucvulALDrqRTI3&PJf|+enRbgJkOySzb&VD?>)UzP5Ae2;XU|`E7f`p@5Mu4 z=kj-t+II2Y{0_c*w8ng!n(rX-eD8$bm3&_ix1Loi6@-E3>5m-c`-IQZAAw=w8zH`- z$5tv#e6e+W_h>!eL4qjpqzF4oxCC(}A^Q*f-smQt*OZxs`~W_LvP#zC`JiUYPkE>A zQNCIA`HyM#hRgb&_sI1C77u zTZeqd#vEt2TDjM})K|DEHa)r_NtEWHbkR!f)TW`3KgoOhyXc+W4&JA>K0B=qhV z7+?L=(t6JS75Ix@(wmoeVGiuM0Pgo&`uKg50h4aVxkSB*}6yZA2M{m74P)YbTp z-NEramafH~+#^*JrjK|03fHYx!TQy#$E{{PZZ&(__@3S>zNg1`^@_lmv04tk8JaU+&cDGZ=vsBHL5%)%oG!s zKB1E28Wm?HRK|~GmxdDTmuRxe%bKjJU_zBpb83`1;8E30jH(tILM>K3<=51iqFlJH zc*mjnDErG-jH;D2RbN#|URP57tfboNl4?f=Iz#HrPpSj=Zrr-C2j5Go$DLH)`6T-p zk_v4cRew`bgL|jZGpVpAsi9;NJ(C*V8&dSpH17b06rbRGug5}4GS*2j&KTXuScrUN zewboDw%Kw#mXdQKrR=hlTumwE_*2Str<4~gv+~m^6sK1e zKgn-5oYe5_YZ`&b`PUSM*axpE4hif@NJU@MC}i$^U0L9G=yf?E8@ubstIB!)Rplb{ z_Pwh71Fx#!$m@)QUuXZr>+FAco&685GY)>0{g1D*|M69oPrs@PsGNOORZxw+25LWe zl|3`B^6vYqjDufQkd5;Lx(R+LH|hTk`xSqqwhfG7uXvNatZ&lZ-=w|2sY2`p@Dca7{w;hD z`|Pik11`u0=j^ZLIQ45~a38q|ZiTzx05lT5;qaT>58l*wIDQ&x4!p@;&^H;oy{XE* zZ!&g!Q{{)=)K58=4?WOZ##p|NF?|+e{d}pbiLpL&0UfxNb3X9wD(1wwHrB3^@3C6T zyM}M(K`X?QyfciSPzx)9GL30lBpaz(3%H*eDGkx%Bo)5>6zlN$WtXJjje5?De zPw}0Hzf#$?eDm{u#sXhweU|-gP2avujgMZgmb@#~{N!5IJ+n@=Kj3$8nIx*$U&>mZ zg3SC&iv(fW$hk(^>jhpJdMZEo6*7(+Io-?_9$3guRi!fxo6p z^96ViegNJm<883Nv*c$D6E|*mnQd7s^m$}vR)qP_m>gFOv!67^{AWzAjl;};#+17& zro2@#=09WX*@$s%#CZQFCin9(=09R8c`T+G6w_ham*jH%^?m|EAy6Vvd-utrYA z6d|4HiDAW%@nbP1W@Ad?o|=v^|4ANJ)mY8+>zX`q)@B>5hLt0#569WFU1Qa{;;PwL zV^yc)e77^D%4A#>?zqa4Wzo1w-;b+gYh1-u4VL@R2=B%nSK$XED!?uO*a&;);*9a)8X#Yxrnvg6;_Ab__iS9j4@bB+#MKR5(#$p6BCg-BWL6*0m{XOP=NEY?E_chxpt{rYqT|oOk+^dqtj=*W|MDJ#8w$ zUtv|7iZvj+0v%gg8_NcZR~%4MLuNPT$}tWIut;*@90p+HLs}C)uFDV z*;e-pZR$DNrr?e?^%76tiZ=D{Z)4uC&>C3P#`;&DHAH&h_w%e_{EbX^D6%fwiju$B z`aZ?ec~)Yd%St}m#ynt~M(=1-=GuHKYel~0KswL*dH%J_wcfAWDTmy(e&uEO=^J&b z0Lip+CZCGQbmLyU%CC}@e%8K>yIXZ^2T_u!{`_JnG7_?heQ)9>w6J+fh< zll?CzbT8+dJ6%=_ZmoO$>?iljOFX^|zuJ)gbp=-7A-~$2@~w`Ye)P|`x=61(+R6N% z%LvF6L;;F0)sOnTe)h7a~nG4YG5Aa=;e5<}Qz_X#iYCIoM(}4o3 zc{ZSyLtSb;nqzsk2IPG{D4(mqYTKG?`PT;6D-%%rmVi1!1y-l0!0JMFKOayJ^&U(H z)a&n3UnoHP4=D6r7w^^uH26+HLmvbbelMV5(iuqy6!8QU4F;J14=5fEu>ZA9$*KTr zPOoYdSrctnb+}zs?sn=99o7|E<#RpsY1&nq&b3;$wySw(xA^=|W@C^#oNHJ8)A%kImftWrrVY6YNsFBE{7}E$~xN3fc~UPj&!qEpj~dR(W3X;RrqY7RX}?A zT%&pKv}**059C_m&TiIj=);iK0B)f*-Reh@PMxP-M@ip!G0W!L?ojL7x z{TO?Ic>VbODRw^ZlIcB0UXXdvm*1}5$4)Xo%Cna1p}VtPUH*3Z+vtZ|W?8GU@=Gko zbSv+_6j|B%t#a)sVjS4Yy{m`$?@TM--OBt|w+gouSw-$F%N=S}G5$(itt#DLWR>kH zvdWPaO|7at;iYfi%KTRk^R}(3eYRC~`+HQ6Y}nVz`@gMf!o7KKEAwBitpBvibF5X~ z*;e^ZwX*-yEC2hg3atB;+L0Y6TGg4+qb_9k${ywdf2Ckmk9rSfT79_nuWDufqg4ZY zTG{{Hsv%_fp%QC&T`T>cA}eyVRndteD@Jt0rYu#23GWVEAc^;qgw|Z2tBdEfS9`1i0xmN^Lj4WC2QRyKM^UxlZAN8o> zh)0!cJ$(Pzqv};2_H21no8?m-vi|%j)>b`gT;WmET#)Y*c^C%=)w=4GJpQ1(4`o@t zIS+fqJn~=RQGj^bH+tCr?NR47#a0*TbO+s5&ov(OcU!&dJnHKVs()ve6*}x;exr;1 z5N>PeaHbXB@8SJtk47Hz@ceUI(K)vjL&kS`lz7&sgT*_IN_!L>t+9++iH$_87mfBt?Rt#Uu=0v&sW#0wnGlf zzssw@;ZF8H6X+`M}n$*w#ceKP-Hb6F0vXA1=R%2YkSoKrO$JJxx?IFf{Y!4s@&PD zDyUx9OI`IU?}dbwzcXqT?7u>VaFAzoCHJ-BtwEJsL;L2wm38C_+57vBUZLy{5|(Ra z)XMoFX63GmT8Hspi~r}4jYx*%nRhe)Scqi0GBbtzC3BOnApZjSG5X{+$iGH1y_flE zWX`u)GtXr1_#QU6;Wh6kCIbi>;wUeG0?y+8*BD?BV+jJ&M9P{2pf< zcOB*MMffs&4IFz;%NgutfvQJY$ehkzbX;tE{BExBwzu(8a zPo`BG>{H2yeJVcIC--z8^Fn=mbF)tct`aN%DDC~!8RbrMKcMgF`p~Crq;nnji}%kc z3zd#0c7Zo zE7c!l4sSMU^&Vqi6dUe(klk#=>ssH#Gvi8itl_s2@~Ee~vMm3~E7@CeC3CHP^1kD= zJRdr(R%GU?h?TW#tsKmmIzLQW+3$DAbw$d`*|Ju-@6ktLQ8XV4m`5!v9o%A#HtA8navZAky2OBHBZ%Us`DbtKo)|6R-X`I!ek*sUH0;=y3b z>h-6rzPh#QcSo(zJ8LzNUaP?m*V6x^|39}@!yhmYoRPF5)Jqg%a}g`fCalCP^S&#S zR%(5RMn6nhnY%7k7X4EPd2vGa#!KWnpRjV)Mfv@UQ7f?lqgF*UVO8#rT2-A1t2&=~@ZegWe_g68TdVp*m#Be7wnoA= z9UirsIc~{Xt5(Xw(}ez2mooo#soL@{k^iACzJFroI!CRJmmHkr5b+k5{+z4Sdo>Til#@c*r`huXPlFG_EIGw zHGPRjS0t>=6)}GQBVjqRE|(Lsui;&exl!sr$o${sv>Wz)gf3UX#+X%jc9i;0Snd}_ zt>T?As{~nE#`;e9GL>g_sv>%sD)+PRBX}A0&pwbl5?1ZAtbf#9#+=aQeE;`y_I+GV z|Nn9|6K_kXlX(fg|I(DOyj2Oy*Lk_x&c-c28{Goo%hk?8cSn9eon@CZ_sP2(dq%CE zI-W1k8;V(dEQt5B))Lw=Y7K0q|8p#14ZU-@!aEDB;fF5Q2t-bp{iNuBK4!(!2`j;d zx+J97pEA0NdRURPTHZ@pjVt0-6W2rie)egy0AIVMM|CX5SF>lL=0wV>^jxMYD9?yn z6}ysF=_>T5y%xW4nM$DO2)~O)U%enqKlwn?%6pb|Bd*h&Lvbtj;mef0r$;WZzjIZ` zb+^BrdB@9W`#la9^gLu4XC(=^}F+odqbP`6OR7{>Y;GeRr(d~`Hx+tV)tg|Jg!m=e%-j2 zG4 zccOScc-C*$$C2$Bo7LvIN}t5um46lcbXfcKq-1Gs$!fBomL z;yNJxeVaA%{ALYp-OM`QX2ssWihlo9`Wk+sYsu4|t@;V}#DT5KL1wPmqO7V-^ba=4 z>Dk2fzeO&{!JZr4#Q1*;{ogIB#&6+@EsUeK(Es0}V%H|#|J7q8Y|lZGd} zLEiig+JwDr)z$hm@+N4n+n~QkMxz__C1ia6)%qUtDfk+E1N!$|&0O6E*5|L*Fvml# z4g7A}1`Ve-Xynue<&|AczvOBac&=6<6oDIxw`^d1eYG-!S1W7B202#oPVI^f`WNDG z;~no+rapG19O!dRDbqr##dLX@%ii2l=Zrr4_(Ln%5}`oUCo}H4Js#%iib9^ zKXn7YPqRVQk8NNrZUYORyIB9;CCA>~azge)yXAr$?75J4bhjL&nFBw8VkiJF{DEt- zbp3ALf7z{$yRJ|Qw-@0hcoSC8&-IEmyhr)Y*LQi(^GPYdoOk;V=!@`fxc{f0(29TM zT7otBT?<#hDp(KgwEJ$@f_*D|0)Kmv``{tCl;eZQ$KZDQntk{^ioFWT;IBE~g#24% zC(;i=NWu#+i2uyPw<_!Ct#Wv7rTz2$4{!yil*9KLa{ctt`JO{QeWQY^Dds;|_hwz% zjV%6PiuTXC_rvsm9%EnHLDs>KusGQ3zJhgd#`=xOrmQsM zf7Ze2qqMG~|3et>W1p9gzEj%{`akcSQsCh!wI7&L$JtZrB<`*~^ndn@sppQB3Lc$e z|3B;CRhRJnGoJr2zz2f|qnBu?^Ad$&_#vME`z~Rx(7F{$uD*9txjF|MV|;=M4R!1WJ$m zlCd-Tqf5mh_WmE`9RR4tUIVoc@eTmgZ{ZyP_70f6|4r}ly$ap~Xo1!hyaNDU@If2+ zApq^r0iDnVfBOCImF)drL*Mt{&(%*_AsAT2`|mKcj=u72TEo15FoK`R`)T%k@%}r+ z;s5>qdy9X!nxUy`w;G}0z{{%7+O4|$-Hi8l(}(#_RR?#|r`fH_xkdKBpI3R9YkuEu zmEJM06I@5La2DoZglp<5uD3Yy60XUQ!D{#|&%Nt#U);f%<>2p?wQ4sP{BC`obEEj( z&$-R`i9fbmF^C@7&HUbOjU1fUu;*Rw>AN-b>~0Mn+O2_uyA?wA`~QPA{N3u!e^fIJ^O`z^~xf@Gkr(tY!@Qmv9w)o%`!8$S)H9 zM&xJU3vd_w1Kf%GqsXtry>Ohgo<#ltehSaPN!%)t_u%*ENGpT;3-UL(pMlq*i~d9} z{5Q`1JLlq*^KGBy9>DQ+@M(CA@aORRMeH}hw;(g0eW2i|V-EmiV|PIgYi7BScXnL) zZuXVBSUYQCpDEJKaWS%F#iUAEM=OJJsDMhSf@-LN+V@{(EIF%s;Ee-^R5T&sh+mdfNWe(`$RT#+=47Ue4f1< zd^_$Az9IK8-;N9M4Y@Gij${8r4Ki=-8rDbFFg{wtJ3?zzl)*P2pm_SYN_MQ_M$G#a z-~cCNbIt`hkPDIXz057|l>&jk0EW87a>PO6PCeh5rczL3fF`83zX6I=^lhUef% z@XzowD1;wyz5@AMuALU-QI37c|KPX}8HZoOzu_(nL z{uy&W{n)QWehR|aV{kL}JK^`lbvN-5Mrj{OJN zH&1@dzWI*Be6!>T`|8P0H?oK0Anv`K??d+EKeU7WpM)Po4sjelLHmFah|n&g5PScu z;@l$=w2$N|+Sggy7qXf8kQQi!Ca8x7sD(PHu45hpDvz;;16kh0bKb>sA6ZsKKj19= zfHeIAr27>8gcs;1gx^%&3Hl9?Geam==Wr0{G5IW`G5o@A=b%# z03y#$DT*9+qX+GH2pM*9p9^vy;JAO^n+o}9=jhl6J+$%Q3)~0LsT~#1__-IpiJx!7Z*l)K=2`E?&y_dP2H;chX~s@}i`)!X!}V~0aNCi0!rgE) zd=(yn+u%X?CVUINf!{~rTkuul{uypgD1;y5R)Rc)ypMAa!c*9vgKuK*T(UJaK%2x#z!t$8q}pFsuGKo{qI$mdA6 z8`;P4kFcM^9->Zuj~v1+1^~&C291XOCH!An>lv|;x{fUxb*fxQ#DIM)sKdo~>R9;N+5KlTs|aBdKW zu!muIC;Pu4f;|c`&cz{tJqf9@F^yJ@(dS*y`^f8gA9+3dFn-b*w|>3ySFTsVn)NE&%6iMg>*dMlLe?5KQ^$I{cw1Nk`-~+Q+KMNe}J99#Io#cWX z?75H^mGYtBju%wumWp`c&)vkgHjpJ8m%h)w+7H=-i>%aHU|23}Pt2W5* zl!W;mkR{yOe8 z*nf?jg9T{d_|N%f!YANz_!PX0J6Y!&Zv1`*#DHw@8_@iI2Bbh6wBh9}a6&e?AO~_G z&+u}lZ5ls$#~hdiPzV(U`)q|BLxp{|64y%GwbFL2v|ZbA?SMgTFvrJ8WtKQ*iwv`6 zm~Mk@E5=rA+e)yN*tSw^rMAuZEt~DfEZ2r}jyhx1Yoj*uf2%fWv#!z>eOg!ZLs_56 zxCU-8hHcwsY}<`-qcL_EgAZ(fCgWyf++vKK7{=Q+6UulsK8;7?&v-MwjHf;DMYs)a zhdba)a3|aad*RD)H{1jJ;4AP|xEJ=r*Wf<59}d6+@F0909)fScLHG_l0*}ID@K5kK zd>0PE_uvV55)Q*t@H9LF&%*cNIrsq_fgi$;;Ky(jegbpZnYQV$O{ZI|^XPYL)g~kqJwssk_ z-^8|A-cGWq#!yGi%A{U)FP=IAkIRkkr}+y1TfqUD_i+tO^x>fd}O)W7wc9R6F>3ZFu8?Mz2*Xf40v$D7AhDAqI8Opd;8P_S} z2L6qpB#haZhiQAjhdG~NpUALJWY{ORDPtQaw%I4P*(bKyC$=kNJ14f=C$`&8H!9;s zoNlz8ZnT|tD1!y#s5Xq&D4wwtug zR(ada+GZ=d?G|mb)!VjH+iaz_kuhS!B$S=L$)26cc5S~_l=t=<_&2AvXYg-ZwrM+O zZ`Af3qMx|^X36>)ZwW6wRwQn-c zc5UCRo39nseY2e@yC`lpC+&K_MYmilD&8)bTa2?^_dB)oT2c9SG3_+YcK!do{{HWG z>2vIq*ri?Of0uUaR^7T$x8ABRXpiaGWL*W~vW z`-4iR9#zNW7|r~7o@ z?YeKD?!)p!-LL!axJUOB=l%P1Ko97Fz5Lr?*t$Qc2XE7Z-_V2K(bx6$z5MUhL;8lk zsejPJ`bT|B-_}7LyzcOII(Ypf@SG05l+t(forB-kBYNcer{RbmIryX=;qXyCa+v>Z zuw9Sp(J$)JgL?EyJ$jh`ZGh`f^eBb-m>zpTkA44f{geL5{2ziRWd!D7J+8-z^1J#j zx_$Sc4(X8jhwtgoQ#$lR9s04prziBp9zB76PvGr|2lT{)dctId`g;QNF#p?NyPo)g zo}eV3)RW)SlZW->Q+o30$Mxio^(5tgSop%z;XOM1fDS*X!{5>2$94G7lRC^lr_Fie z;b{(^(_u>PqPa~^>8U+>>Pvd+0X_Agp8B4iI;^Lj)6;tT0X?H~N6&mo&v5*p zo_R#iJf~;%eSQDnqXs*!i*lQu({ta~b3f1zbVNVY4-fHgb4WkZkM-k2{2$U${Y3w) zpXzzN=+%opz3A7A0lnCx7yEC6FA1H-GBuW^F^9&S8q3z0OJg}2%hgz(#_}~*ps_-Y z6=}?^v0{ytXsk?Qheu{MqQHP)`N z4vlqctV?6v8tc(mP-DFs>(f}j#zGn!(Ac2HhBOw|*s#W;8jEQxuCav1k{TP;c&5g) zH15#2Q{&khcWFFF~SUaRps zjn`|uLF0`YZ_;?P##=Prs&S9Ty&CsvyiMbNjR!Q|uJI0ycWS&#4N)xOR)@Y(u6Lp%X*F=LR8a2_RiDpf-XrfgU9!+>P;nPH$Cj6S{ z&_t&ux-`+Pi5^V^HPNeyK27v%BBY4{O$=&cNE2c4hQh>%CL)@MiZ>A^;+jZ^n-tCJ z7sdKTseaL_UwHJ3ZvCQ9zv$PbTa(3_EYW1CCd)KguE`2bR%)_JlhvB6(PXVA>oi%f z$p%d}YO-0AEt+iAq(_rpP5Lw$&}6$NJ2cs;$u3QHYqCd^K~45*vQLvFF{H_`CL@}R zYBHwDxF!>tOlmTv$x%&ZYAQ=p4ox{Vm8~h4rgAiutEoIq%QG*zyt3QbjNs!CJUnyS%Mt)}WURj;WAO*Lz(MN?i)1vJ&JsSZtbYN|_9-J0sr zR8UjBn(EV3zotT(8qn0BrbaYfs_8OKmutFG(^ZQSSOIdo!p_iO`DPJ!Y=%qrvRHT>O zdZ|<|mFcB&y;P-_n)FhOUW)3a#8YCB!w81~5=Mn)ax{~tnS9L@Xr@Rr#hNM6OsQtd zG*hk_JXC6?N;B1(snJZWX6iIkubBqTG-{?vGtHW5(M+pmJeu)orcE<`%>*>lu9*&T z)0^qiOt)rwG!xWJuV(r*)32G3W(G7fsF@+ngf%m)nGwxIG!xZKOfzxKB>3M0UxEkV zL3kXVgv0QhW|EpoX=YT%Gj%*m#~nKE)bVT`cjUfckyLG%+ z#~XFLNv~w;l`OsD)GN7qrAV*1^-8f`snaV_y^_?40-Y$*iBg>?(}@b5sL_c=ooLdD zX7NVOi4mR5(n*I-mg;1gPL}Irg-%xMWQ|VN>tusYHtJ-PPB!agi%xoV(x;PcI_cNR zfKImSWS35M>tv5k26eJmC;N1=UnfI4IiQn+Iyt11VV#WVWK^g8uQ_yT=#<|W-NpzS zV_2tB7_XP||JsN#BF0EyyxyqS%lJ13Wv`9u^>Y5rK?VOtR`PFT)oW2>V5>Gptug9e z8!`sAdSf*3Z%Y$TBmd^0Nv}5>*=h`rF}!-+XJp&5f!~0!wHu>@ewKfb#M3^T%^!Z+8Q5bC>5UHlO^O{RMGO;Hhe;S? z*-p3)lYWOuzr&>8VWQ|TNp~18oqD5-e-jIa@nr;Ojo^Hji5g?sLvYyxDRhy7?YVc^ zvz=a-$!C{Iugg9;>#~C(Oe6>saMopqK$z%Fmby%qx|UsW>*n8t>o&1_d|x+L~C6 zFg}*;*wCWaB;9L5*~MTNf?WiCdZV9z=1)xSLvIYu=FMhfu^rA7nJmeuG|0b+1H;6E zFwqY#+wGhTnshLh-R;N+O`L-!YZ!LuWu-V;c0}1h+rP;$7s!zDgkc|=xQC23%fk3D zNer1thiup78p6JOL~KJQwjmQd%)f~NL)b>SX0!MwdW@hkdToO=X0t9DCf+Pk zz`*9<-}rRQW>dNdd}1s+nY@`J+oK8MFg7RuZ?qeOWSqvQ(q_H|_{ zM&`WBq+}X_F^o?mhy$^F+D_2c8)Mm@x%TmZc%yIGV&Ym3Wm>ecFMBb~ZCPnFU$&E1 z+m@r*9ODF2* zRABN{V4}tNqc9;&o@l+sAi@GW1`|60DM>;`#e2WAam}VQJF3%}I_=cyY@N>0>0F)8 z)9E6ecI$MpPM7I)txkJ%x=p78I^C($L7nc?>5xtj>U2z}<2s$vnM|F@(iw-&xWvCAq@SROqd?s<+w_i0Wlgvn(2yg=blKmqjPux|gke%OZr! zklqS--U<*$z`HE^Y|&n*x7v&JR!8+)9h~f_Sr(1UqI+2kEQ^ROdOUCS;IqeTi(sDK z3KB<d+{bZV7LSptoY(Z^i1C zg-372-EYNkOb{lS`Bo~7aJ-coK@=^EvSm@dEE<+Y=&dAyk_ePXq{vFD)DE61x4onq zmxX6p1eQhjvKaVNn1SV>gMV@uG!Ao_0nIrZ#jH1Efdib74KB!mT*!lbD1bsJ0yh*x z36w$^ltU#{K|Pqbo1ht5!2@3KK|6FpH}pUddZ7;nU|4f5^5r6Z7wNfbpcd+YxLm~L zA}$wkx!S-F=;I<@R|gQcs|$$VMf@(}cM(6Yr_8zfAq1r38iXMTYc2=9bI>~neRI$! z2YqtTCx`s!kX{b)v7J5;5+i5g|_z(uq)q5z>lKhY`|?P=^uHj8KOW>M(-N5z>#4=LmX7 zsK*FtMo2S4*+Lfy)M5vPp`HN5|5$Yt;0n|x^ zGK^3s5y~+_IYy|H2z3&nJR_6~ZD%e*c}6JD2;~{!I*5!w1fmcFbdDr|GK!>NRC7`E zkD`AR{iEm~MgJ)JN6|lu{!#RgqJI?qqv#(+|0w!L(Laj*QS^_Ze-!O&=pRG>82ZQ1KZgD> z^pBx`4E?#G&Bf3&hMqC>jG<=?J!9w@L(dp`#?Ui{o-y={p=S&|W9S(}&lq~f&@+af z+z#hr=ov%L7<$IgGlrfq^o*fr3_WA$8AHz)ddAQ*hMsZsjH729J>%#ZN6$EV#?dp5 zo^kYyqh}mFGmf5d^o*ltoO+H^&vEKG zPCdt|=Q#BoNB=nb$I(BI{&Dn=qkkOzNP>V zCaBj0^_oQAB>E=NH;KMU^i85~5E-MFNuCh^h=^&68)0s zmqb6F#&bN1=aT4@M4u$OB+OH<@MMcz~7Jw@JAanOA1|5=#oO06uP9)C50|2bV*TXDfCHEXDM__QD-UiN^wo5&@F{-DRfJr zTMFG$=$1ma6uPC*Ero6=bW5RI3f)rZmO{4_x~0%9g>EV8E`@$6>Mn(jDRiWRGM7Tn z6!n%u*A%*rqT?t!j-ulz`i-LBDEf_}+bDXCqRS|{jH1gZ`5z_!qvU^-{Ew3VQSv%U zUPsC6DES;EpQGe+lzfho&r$L@N;b0a_XT0di8d$8;YSsZ|6B68*(8Z zihyu=gv)D#0Ce+zQT%VV<%8b0^N2Lh2-}vQrML5mGM^~(i88+xybuI@=J!EAgkT8B ze0~I?db=PK2wy<>0vF@~aTN3b;R^^~K=?x9Ec5{J7W$wI$YUY?3h`HoTGjZeDb(8m z;tAyF?e3t(Pt+{&2UVz4$>p1T-WCuT?RW!R53PEqyQWy)owct|D+;2f=er0%=vnac-#IN+cBq8!Cy65^PM(8kSyDX#yO=0zLYEEg6P>j)XY zY{_vtHVSg3IJiS{2XYN|iYD6^HFQ$-&V0Lc zjS9|!bIxK!Art{6WFHrwyIADT5?o5`qBL=r8f=BA-);_Q!^;=&|H|&g3)*<6Z8<9p z%bqK-R-WTB=SnpBF%fWV>TB6XDLZMWb`$vHb!IY5lknN88bzw?GsbJR;T$=oNFD9V zvDDy3Gj`URcpam5C=+`vr$Z2ZC)DO+Z83&UJ&7+*sEyukaw+f1i zJ*DRwxN9_UzL|#7O6(pq_HgXW1vGDK1mfwyAFq?2>!u_aU!Uu#)43pJ(Mx6ab4i70 zC`0H!%w@;;g9K$aG2zN}mGl~C0L9>88x1H@O$j|$hPG~bY``2bgdB$1ujGg8g-$1$? zE5ng_h7|L3rRVAL&eOe}r#m}OH*!7|hf$^JGNd!RmF5AO<}sP34WwzAX)d*Nb{F7> zX`Qso1#YN>Ht+*}n7TeXS4-zqK{eC>Y2*+-2megXq?v+A z=MW}0ABvz6nxPdqmq%QAPRIqq=QRQ8<&odKxYGGKKsxyYKsfitUlal2FG7EJ7Gwkd+{EK91IodTKR5Yx zlOK0Kke-`(`OqJRRDTa5q~v#u15E2%DkF*s!6Mw zG^+7mL%G%vUk!28l4cz`)e&zU<<&qLG*E^Oq}fQlHFDfYo*U7zxlrkrV!&T3d2A)$ zt(3io_&k({hkSX7$4lLMNy}FYg!PfOuNR2JhkqY+)kgYlq}@ijw2|jF!gEVVw~?nd zuC2EAg&=CQQM`Us@uRw*Z2BpBKVJMq-%j-HB-KH7I|$N=duKrDE~!(QjiL)QKe%wN>z;TF6 zC`9#z2os_R1}F+9p3_4Vb(lEAq!&izFz3SwrH4^@IHvRn`5f^8af}dtg#3&UeuQcq zA)nk8(%b>k%si!;SxQF-7xIXT$)z+CNogjL(o7zunY>9eF_UIOCe1`lnhBIN6DVmW zNzzP`q?r&&Gm(*IvLYQPS|%gXOgf~QSV%KrkY+HRW)PlcfSqQ5on|1NW)PiD5+8Sn zGy~o=1KBhK*faytG=tAHgUIwKYBE4fk5as&Q7tftSjcSDLRMG{jx4Cu0==IF`T+|( z0~eTxUT|fDkws7pt_sBVEf!a`0T3}}JYzrbYcLhi5@X#ER$gry7?@*1EQ zhJbVVg-`)aS}3T3kQQj=3xy?sdm(u#Oh8HtMdYa{sRcLjxbf>IeRo_7#ZDkU#hqFx zA)h5KATOopRO;128TyrRu51K|r@S6WuL8f7O{M6^JQTd1yqIv}p< zexR&sh`WZgXmJZQKJaUyws>KHY}At7T29pl&3${J4lQVh3-xGTUk2o+p5ulbAhrf$ zC#6P8hb^ZIjrgaHE;Ld?jg)#L+BH$KO+?#7&YCFECX#6;2hI3v83and6?+J;Sz}`UuU7YJic9VDy#nOWwJ!B$? z{~)T;f)|2RXK(~4hF%ivr5bz5L@$}>C1Wh#EcCg7Fnz>Bt6S*D-an)Tp8X4rlgQU$I`@#@mhEQ?{|3l1EACW1Euw8%`uB28hDnT17W6c!6P zrU@(-5f2ZhMR!b##o0jkV$v=y1V0RDv4s5d@L4RyolAeQ3_oS$i_3qpETqM9@>|Zi z@>(rcpkoE+Dgs)pBupi4l`YT)M5UkbZ#I`9!k9X0PTP~4Pmh{3&=YUsKqAqZNeW9sl{ge zwBWym__^R0J(RD9vh)@~nHGKc^I`Xqx3(^zY*_F#*Z(3%?Ct`Ryc~F2Z+FrrqSTo3P!~ z9qneZ7e9UY@1sonNWY)>c;YOE$bX1@4p3eLxD6r)$=4udF^HbDgGKtDi$jDRBF{tU zJVgD6Da$Zr5XRpy@eC7ognAtzAFMkq(l=dXt!9zF<|1n^i_HBlGQYdXJnbTLr;ADQ z#hmFPbEAvQg)TD3xyU17F-5&Fr?1rx!hXtK?igJ=W~fCH=(6G;%6dni3z-=Jp9ovmhxJ(#Ejb#Gj2jNx+{R=V$v(dUkUl6gSJEmZKmlDh z2b2JL@-+i~eUycdaBZaLC!C*r`6**&3YP+uPk_9(lec!lcMx9}adlDVUF4H%ajBbg z-T3Jy@7=`JP1qjd=^;-+%AcOoQZHrHKcb}&akAXL6siLJ4p6=WZLIG3Uh66Z(bUv9ET}K?#)Xhl=lektV1o)ompb7XNmQUCDt&O zSdUm@9b$>@$`W(xOU$J&F%Q1P+WZo8-{&)Z5Yc(=zvr1PJfAfH37vP8Kp9l(JQFPE znFTn{xc0m=sq?IXpU-a4|IN{TB}R6pN4w?FRmHNI@ZNiyNC1MDx|^!;;ENcEGwaj?Z8I_|Ch3{`SY6 zZhLnA`ds+-*XN@D&gK98>vLuEug}eozdpDA%QN-dd;H^_1JA=>pS%7!|9kjkD?fMr zv3C7&_Wbwk`E%d%$KCVq-Sf}y`D5+>?K$)K*Yvf_j&y1uTTFw`aJpW@voow^FHzKJ^A-vpMGHf^!xkg zx&MqG)IZPtan2t-{@gD7^M3mN^uL?WD}P$3K4Z*X3#ZuTQ^sf4=zZ|K^YP?yu{|zyA94Ui#Df z(@#I%etLiT`7eJg?zi`s`PaQ)XfeEe(7KmHo` zYkv`Po-dxSp1(ccJl{P(JU>0ZJik2?&(yQvS@bM30%sJn4VZ|D^v( z|C4^d*NrFrK4Fa~{XS(4pR&f2em?__C;d0zgr2k33Ph;arzuz;* zlYW1jm+_?kNxxsG#*_Xh{eC|k{uVgnN&l1nC;iX*pY=cMf7bu3|5^XD{%8Gu1|84( zpY=cMf7b7l-+0#ltp8d6vwpu%k7xbQ`k(bb>wnh&tp8d6v;JrO&-(qkIiB@D>wnho z*U<5-|5^XD{%8Hq`k(bb>-X#Hc-HUN*zv6YS^u;CXZ_FmpY=cMf7bu3|5^XD{%8Hq z`k(bb>-V{1JnMhf?`MkPXY1h;?C|UM@ay(?(f^|VMgNO_zm5;Tj*l1pW>v$iYWP`v z_*s1T{%gGGf6@P<-_Iw*ulM6c|BHSVMV$s{d90tNvI0ulis0`+0A?>VMV$s^9me!{?##s{d90tNvI0ulis0zv_S0 z|Ek|-rtzx(O~21h!_SSww;1D1|C|0d{crl;^uOshryJ&U!<=rs>3`Gzrr)<7<4ymY z{x|(^`rq{XPISEKf7Ab_|4qN2OUIl3H~nw=-}Jxf_bzk1>Gw11@N>d=)BmR5?-k=s zzt45U=eptBmGP$kO}}qhhBL>*&lKZL|C|0d{crl;^uOu%v&Qf_aJ=b%({G10d^0oN z^uOtU)BmpjUH`lOcm41BeQq4@`rq}x>wnk(uK!*CyZ(3m@A}{Mzw3Y3|E~XC|GWNo z{l2#z@A}{Mzw3Y3|E~XC|GWNo{qOqyj6UA=zw3Y3|E~XC|GWNo{qOqU^}p+X*Z;2n zUH`lOcm41B-}U>pY52Kn_`P+!>wnk(uK!*CyMEt9jd%Ti?i%mO3F zq5nhwhyD-!J~xaH{U7>2^!s^jeCYqs|Dpdw|A+n${U7>2^nd98(Ep+TL;r{V5B)xa z58s3hGt6OzIm|G}hyD-!z8xFB9UC9|KlJ;SZ1|RJeCYqs|DoUK_wk|sL;r{V5B+|2 z9Da5jzD*k+`akshoHIW3f9U_v|DoUK{^941@uB}izi-}#Z{Eg-e&5><-`fwLjfU^; zhfUtF$s0C#!zORo@)sHzfIn-$s0C#!zORo`u*n-XdBgWN!zORoAI{ZvIZ1RRp z-mu9VHhIHWslz63_&RmiJ48J4_m$Az1gtU8@}Tjey$$2dc#(4*y;^iz2P0wu+J3}H;XAQmt2cbrJbX4Az8@RjHw{0R4?mX=TfJecH~f4)d<{K(4Ly8*8@77G zR&Utq4O_k8tLb5@H+-))Z1sk%-mujhwtB->Z`kUMf9to^8{TCNTfJecH*EFBzxCVd z4O_iot2cbbJ#6)ct=_QJ8@77GR&Utq4O_k8>+fN!H*EEWt=_QJ8$LUXf9wCZ{(tND zIb!%6G5)RJhHrRjHvXgEhHv=VeE57ZZ1{!^->~5u|Iu&5H+-EwZ1{!^-|+ck`1~X>@zTxxmu<08%eZ!`2*z^sXzG2fhZ2E>x->~T$HhsgUZ}?tk z*z^sbp@&W1u<08<9}SznVbeE!=QC{jhE3nF=^HkE!)K>q(>HAT#y|T1(f^Nr+rHs5 z)v)awwtd64Z~UYGAN^h{4Bs6MpSOnB3&X~5*!Ycq^xOE2fArh=jeqpp`i+0|+xv}w z^xOOmFDi#u62oWwVe>a^{)Wxpu=yJ{f5YZ)cvUfM{)X2T!{%?;{0-m7j(_yq{EdI~ z`|fIZl`(Ap#y|S)|HeQ1Z2*Vww1)4r#y|RP0mnc3|Iz=CelJOfUEr__9Cm@jE^zqn zYj|-w>;i{f;IIoEz6%>(lMJs(hR>D5=gMIhIP3z4UEr__9RH`^E^zqHY}f@3uTzFy z;IIoEzC#G*%2N$moMUEr__9Cm@jE^ycd4!gi%7dX6*8Fqog zE^ycd4zFg07rMhPaQHrC*aZ%|z+o3S>;i}HM~3f5hHc=m4IH+C!!~eu0X*yjhwt*n z|LONKc>K}-N59SB@VaN%430ngeb+tg28Z3?_@m!;aQxA4KRCP+8h`Zv(QivQYzc=g z;rOHfkN!XU?Fxt2M#HXf*cA@D!tqD{AN_yy|IzNXvJwH4@J-_+;ic%^jvwaJvW|P&&+e@nS1U% z51vQQe|w%h&z=|0tLM%0?)mWikLQ0q|ML9X^B>PYp8xac|Dykk{xAB!=y$~?U-bJ< zH(&IB(f>ui@7Q8$7i+$J(Qnb0FZ!+e@?5moNH#Z=5gsE&uXG{}=rhfcc`|3NT;v`))5^^n1sWFZ%uMc=AQR@0;WM=2!^E zcYm=GjFn)#nvA7jyo<>f{nmmpcE}g~z8{SDGw~gDd`F$H`n|@?SN&i0f7Sn0|5yEA z^?%jxd+gX1#&_BIs{gD0uljw@n6LVM*O;&Rzv}m$W4`LQNy%6JU-f$z6kEi6)&Euh zSN(R0`Kte`{;&GK>i??W+%R^9`Kte`{;&EC24gT7=7R zF2->&j*IHitmAAzm&iAb5-H0!d1oliEvfHD&O>f)BjDsTYX`z;(O=tR^hF}TjiU6<|^I| z=9_-zD&O>f)BjCBf0b|gzv(y335OL9D;!ohtk^r{oBnV5d8~ZX|4sil{cKjg>Hntx zoBnV5zv=&`|C@gAVPfbLL!W%t|6Tug{a)S2d&+#*&vnIX{P?as-}Qgj@0EUhZyvAp z^IiXU{onO}*Z*C=u~5G2|E`}03lA0^EZ_Bi*Z*Drcm3b>f7fps5Kb(dSU9m58pY5k zoLG#FVm1(Cqxhaa21nt{AwTq+GQ>bVMo%$N z4~rJ#rx-uQ_$fd1d%qCF^%$e(3+9|A+n``hV!> z+G0Qz1FEoX`Jvx@B8*!Yx0q3caf|mDF|f)H{YF;#q5p^eANqgj|Dpef{vY~(=>Mtz zr~aS%f9n6K|EGTMLc+;~lZ&BNe(L|J-}{jm-p9M^cux}Vuj5@we(LAw@>Bm${Xg|H zb@8q*TwQ+Z|Eb^DD~w%CKEm0>=qo?<|J3iDV1DZVssE?`pZdKc%uoH^6XvJ>pZdKo z%uoG4_5ak*>cxB{-XZ3vej~D&lH{lUpZb65|Ed3{e(xCbQ~yu>KlPiN#5>2>4aVp! z-aY1*{$KhH&|-=bpdi2W|I+WhWPa)YrT>?H@9qN{1T@Gm{lE18(*H}p@mjpY%rE`F z^m~_?U-}K&@=L$>n)#*wm;PV+f9Z!6@=HIk5Ie@$1jP6)b^);q$S?iB^qa-xm;PV+ zf9e0F|CfI6tYSM5+kyPj@BLNW295DtYzgv9|1bT&^xGBWw|>$gzxDstPdwzeej~fs z9OSot`XRsdo8RQOe&f6R*8f|-O+tR_|E=FNC%^Uo*8f}oZ~edZ+bra_e!GSI)^EO( z-}-;+|E>SGetU-e*8f}oZ~edZ|JMIo|8M=j_5arYTR(vj?}OugaDMA2Fyg&%e(NVO zLSn=_yBG(?yW)6PoZtF?>-Qcn{`U0wt^c=v`-zYmndqPBCpJQCWTJnfpWKLnVJ7;$ zZ_Y&jME^wpME^v;chH&WpXi_Hx5vmt|3v>p|3v>pzmZ}l`n|8tME^wpME^wpM85%J zCi*A(C;BJ)C;BJ)C;BJ)?L0z|gdmB%M<)8c`_4qa_unyo%tZf0|3v>p|3v>p|3v>p zzjx%B=%47H>YwVL>YwVL>YwVL>NldyRKJN;ruwJ)r~0S*r~0S*r~0S*r~0S*O|UZ6 zKh^JjX{P$8`ltG*`ltG*`i(L()la~LfC&K;0>+g}kACm(Gu1!UKh;mhgp3Ip6EY@K z{Zsu@{Zsu@{Zsu@{f3^I>YwVL>Nom~(PyUmZDTUkZz7hd{ssMZ!dcM2px?XQ7>C9< zG$v$O(7&L6LH~mO1^o;97xXXaU(mmxe?k9({ssL;rCHE#Sega>3;GRAv!H)Lzg-&VqiM zhb-tfYR!WF1^vdYSsv|Dt}gy%^hOQU9X;Mg5EV7xgddH^R-L{zd)9xLMS1_moBbi~1M!``ekv4l3q< zS=4W=n??NwyIIn|q~CBirh&1Q%98#i{Weos(r-7FCH=NjS<=6xe@Xw6{w4iO`j_-C z>0i?C=5m(wFX>;>zodUj|C0VC{Y(0n^e^c*Ma+`^CH+hKm-O3bWl8^%{w4jUidoXX zq<=}j{YsYfFX>;>zog%uD@*#9^e^c*Le7%@CH+hKm-H{`U(#>;k|q61`j_=D>tEJy zv>cPjm_%k-|FV7~=Ge)^PA1FxZDq2oe_8*s{$>5k`j_=D>tEJyYoBHP%len~+udhb z|FV8Nn=I?MJ&RFvmh~^|U)FCR9aGF0NoQHVJx-SOFY8~{Z#bQ0{mc57_1o%XS--tb zmh~G`XIcNU{$>5k`j_?FX=GXdvi@cL%len~FY8~?zoLId|BC(<{VV$I;R;8rs()4gs{U2|_JCQ{ZwHlC{j2&{^{?t*)xWBLRsX7f zv)QcbU)8^=e^vjg{#E^}`d9U@>bEbR;8rs()3#32#>Quj*gbzp8&#|Ehk|-mL0h)4!&F zP5+wyHT`yvS<}C!e@(v)WY+Yr={MHRn*KHYYx>vpujyaYzovgp|C;_a{cHMdC$pyC zj5urh*YvOHH|Ecp{x$t;`VITDrhiTUn*KHYYx>vpujyaYzovgp|C;_a{cHNy^snh( z)4!&FP5+wyHT`S)*YqtENuuHVc#>-yLAuj^mezpmf@ zGwb@-^{?w+*T1fRUH`iNb^Yu5*Y&UKU)R5`e_j8&{&oH9`fW(Fu76$sy8d-yLA zuj^mezpmfTH0%1;^{?w+*T1fRUH`g%o4l;+U)OJ&mv#O2d0E%Lt{;%dy8d-yLA zuj{wp%eww`{p6Cm)W4~JQ~##^P5qntH}!Ao-_*aUe^dXa{!RTbO3dT4see6Cm)Ne1GP5qntH}!Ao-_*aU-{d}<`Zx7&>fhAAsee=d zrv6R+oBB8P+vR0b|EB&;{hRtX^>6Cm)W50UZZBKEF_C z-EF`7rGHERmi{gMTl%;3+Zbj`|CWB#AY1yk^n*Ux(!Zr2_=z22w)CS2+0wtI ze@p+C{w@7m`nUCO>j#6ft$$lTwvcW8+xoZlZ|mRIzpZ~;|F-^Z{oDGt^>6Fn*1xTP zTmQEHZT;K&xAkx9-`2mae_Q{y{%!r+`nUCO>$eFHB9U$V+xoZlZ|mRIzpZ~;|F-^Z z{oDGt^>6Fn*1xTPTmQEHZT;K&xAkx9-`2mae_Q{y{%!r+`nUCO>)+A8qkl*Lj{Y6} zJNj*Uv!j1U|Bn70{X6=1^zZ23(Z8eLjy*g2?SI2`WJmvw{vG{0`gip2=-<)5qu(wx zcJJBIZ~GnwBs=4)xWFX20y#{clGb;-_^gXe^)=8l3o3_tl8CX z&zfERyZY^%1CnJ||E~UB{k!^i_3!H6)xWEMSO2blo9OK7-_^gXe^>vm{$2gM`gis3 z>fhCGI~_nR_S4zbZ$q73{k!^i_3!H6)xWEMSO2blh$g%GS%U29x4F%({$2gM`t5JS z7-U!fuKqp!d;0hE@9E#uzo*|uH+%YRwzH>yPye3&J^g$7_w?`S-_sB5WKaK|{yqJB z`uFtj>EF}8r+-iXo_??=d;0hE@9E#uzo&ms|DOIm{d@ZN^zZ54({D4LJ^g$7_w?`S zXC$(xe^39O{yqJB`k|rh>9;M8ZE)+SEuYX_vzW#mv`}+6w@9W>!zpsB^|Gxfx{rmd&_3!K7 z*T1iSU;n;-yXV+FXJ7xm{(b$-N9>@puYX_vzW#mv`}$ds?CXc4vaf$%|Gs`cB>Vbp zr?amgn##WZef|6T_x11V-`BsdAE*jXk^}t*`VaIU=s(bZpdYphY?TB32l@~6ALzH= z&w>5}{RjFF^dIOy(0`yGq70yw1N{g35A?%YInaNgpI-^Tk^}t!R}S>sdFMd?f&K&i zw#s1ya-jb}|AGDk{RjFF^dIO4G;^TeK0F8d5A+}CKhS@m|3LqN{sa97`VaIU=s(bZ zp#M<+q5ebthx!loAL>8Uf2iN)Jcs%Z^&jd#)PJZSHp`*@L;W`CIn)oIOa(fsQ*y^q5ebthx+Z;bEyAN|Dk?c_8jUz)PJb|Q2(L+L;W`HIn>Wei{zLtC z+&R>LsQ*y^q5ebthx!loAL>8U&sXG7KM?2df8x=9sQ*y^k$zS^NBWQSAL)nY!eit} z|B?P9{YUy4qa5i!(to7?NdJ-kBmGDEkMtkuKhl4sAH2(v{v-WI`j7M<=|9qcr2k0& zk^Uq7NBWQSAL&2Rf25yh%8~vf{YUzb^dIRz(to7?NdJ-kBmGDEkMtkuKhl4s|49Fl z{v-WtNsja%=|9qcr2k0&vHoNI$NG=;AL~EXf2{vl|FQmK{m1$_gBp#|itp8X)vk>Mb%vO%|AL~EXf2{vl|FQmK z{m1%`^&jg$)_<)3SpTv9WBteaxrac?Io5xy|5*RA{$u^e`j7P=>p#|itp8a5iGEHZ zC;CtHpXg^Ma-#o4|B3z+{U`c4ik#?&pmU=CME{BY6a6RpPxPPYKhb}pAGQp4krVwV z`cL$q=s(eaqMz}}iT)G)C;CtHpXfi)f1>|H|A~HXBPaS#^q=TI(SM@feTAIp2eJcX z=S2UB{uBKt`cL$q=s(eaqM!Z9iT)G)C;CtHGe$Ypf2#jf|Ec~{{ipg*^`GiL)qkr0 zR6mcDQ~hjRPW7MaKh@8V=2ZWw{!{&@`cL(r>Oa*Fisw}Sss2;_3`$P*vrReGf2yBx z%BlWS{ipg*^`GiL)qkr0RR5{|Q~jsOa+gs{d5~ zss2;_r}|IzpXujbbEf}H|CxRUHfQ?J^q=WJ(|@M_Og~GQGyRNg&h($@KhuAv|4jdx z{xkh&`p@*A=|9tdrvFU;nf^2VXZp|dpXoo-f2RLT|C#rWLbN%P~&-I_{Ki7Y*|6KpM{&W53`Z=|n>p$0juAeu`x&Cwg z=la>Boa^VrWLbN%P~nWZ=`kaPVG4CGw@x&Cwg z=lUHS2p^Yo{fu0=&oE9o*MF}6T>pjs3;hlg@{!9Ir`Y-kKvT>mym-;XDyVQ_N z{S0j`^{rT$C(m-;XDU+TZq zf2sdc|E2y*{g?V(hsdS=OZ}JnFZEyQztn%J|5E>@{!9IgWVqg3>1TU$rTH(ztYdoA%u{ zrTH(ztVrD z|4RRr{ww`g`mgjm&YUa#+;Xn;U+cft&-dk8|F!;W{nz@h^y zuk~N+zt+!~=34)?{%if$`mgn0>%Z22t^Zp8wf<}U*ZQyZU+cftf32TU&9(k({nz@h z^yuk~N+=Mr%Z22t^Zp8wf<}U9AmEa zGqJhR@3uj1^xx>e(SM`=M*ofe8~r!>Z}i{jztMlA|3?3f{u})_`rSszjs6?`H~Me% z-{`;5f203KKOdeO{Wtn=^xx>e(SM`=M*ofe8~r!>T{p>%{u})_`fv2#=)ckLeo}7q z-{`;5f203K|Be0|{Wtn=^xx>e(SM`=M!);pxzT^4|3*Kno?HF5`fv5$>c7>0tN&L2 zt^Qm6xB74O-|D~Bf2;pi|E>O8{kQsW_225h)qku1R{yR3Tm85CZ}s2mztw-M|5pF4 z{#*UG`fv5$>c7?RqDyY|-|D~B&z$B~|E>O8{kQsW_225h)qku1RzGi_Tm85CZ}s2m zztzv+=T`r%{#*UG`fv5m^w0Fq^w0Fq^w0Fq^w0Fq^w0Fq^w0Fq^w0Fq^w0Fq^w0Fq z^w0Fq^w0Fq^w0Fq^w0Fq^w0Fq^w0Fq^w0Fq^w0Fq^w0Fq^w0Fq^w0Fq^w0Fq^w0Fq z^w0Fq^w0Fq^w0Fq^w0Fq^w0Fq^w0Fq^w0Fq^w0Fq^w0Fq^w0F)>A%x|r~gj>o&G!h zclz)2-|4^8f2aRW|DFCj{dfBB^xx^f(|@P`PXC?$JNA%x|r~gj>o&G!hclz)2-|4^8f2aRW|DFCj{dfBB^xx^f(|@P`PXC?$JNA%y@x#v#*TtDxgx&FC+_e?U^Ki5Ck@1juV`rZG|T)!LOnd^54 zJaheX{d4^cbmsc!`se!R`se!R`rS6mT>o7ET>o7ET>o6Z8xV0jC3F3A{d4^+b>{l# z`du^1T>o7ET>o7ET>o4@Z=JdRxqj|CZj)uMf3AP7pK;DyKg*xFemBd*X2;#~xR8{& z{(JrR`tSAM>%Z53um4{Ez5aXs_xkVkJHa0(_;auSUjM!Rd;RzNS?b*DcLOH(`tSAM z>vuyS_xgG8-0Q#Bf3N>u|GoZu{rCFajmf?Kd;RzNU2V#}{(JrR`tSAM>%Z53um4^@ zU!HsY_xkVk-|N5Ef3N>uzxy+}*Y5^R?)Bg6zt?}S|6c#Se)kRXp#MStgZ>BoE=A=* z|AYPq{SW%xK*)ps2mKHFAN0H3kq7+``XBT^==ZlL$brLH~pP2mKHFAN0F% zmk0gs;^aa9gZ>Bo5BeYUKj?qZ|DgXt|AYPq{SW#d^grl#PbUxhAM`)yf6(u~P9F3> z=zq}vp#MStgMN2+;_gl!^grl-(Ep&{?U6j{f7I_XRvz^~>UWPMkNO|=Kk9$f?`BUP z^*`!=)c>geQU9aCY|D*m#{g3(|^*`!=)c>ge zQU9a5C0$jKm5PH zr>;jo{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+` zhyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=> z{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci> z5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q% z{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@% zAO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk z{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$j zKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8 z{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5 zfB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG z`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A z|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW z@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K z|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<# z;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e z|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe z!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0` z|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+` zhyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=> z{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci> z5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q% z{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@% zAO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk z{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$j zKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8 z{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5 zfB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG z`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A z|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW z@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K z|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<# z;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e z|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe z!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0` z|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+` zhyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=> z{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci> z5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q% z{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk{y+SG`2X<#;s3+`hyM@% zAO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$jKm33A|M36e|HJ=>{}2Bk z{y+SG`2X<#;s3+`hyM@%AO1i5fB66K|Kb0`|A+q%{~!K8{D1iW@c-fe!~ci>5C0$j zzx;ps|MLIk|I7cE|1bYv{=fWx`Tz3&<^Rk7m;W#SU;e-RfBFCN|Kp|3p9kU;e)n z{S*BY{S*BY{S*BY{S*BY{S*BY{S*BY{S*BY{S*BY{S*ECfBFCN|KR;5qsGt8Y|KCOZi~1M!FY4$2%m0`EFaKZuzx;ps z|MLIk|I7cE|1bYv{=fWx`Tz3&<^Rk7m;W#SU;e-RfBFCN|KW|6l&U{D1lX^8e-k%m0`EFaKZuzx;ps|MLIk z|I7cE|1bYv{=fWx`Tz3&<^Rk7m;W#SU;e-RfBFCN|2@}#uAl!e|6l&U{D1lX^8e-k z%m0`EFaKZuzx;ps|MLIk|I7cE|1bYv{=fWx`Tz3&<^Rk7m;W#SU;e-RfBFCN|KI^`g0jM(obq1i$0Mr?P zIs;H=0O|}todKvb0CfhS&H&UIfI0(EX8`I9K%D`oGXQl4pw0l)8Gt$iP-g(@3_zU$ zs51a{2B6LW)ER&}15jrG>I^`g0jM(obq1i$0Mr?PIs;H=0O|}todKvb0CfhS&H&UI zfI0(EX8`I9K%D`oGXQl4pw0l)8Gt$iP-g(@3_zU$s51a{2B6LW)ER&}15jrG>I^`g z0jM(obq1i$0Mr?PIs;H=0O|}todKvb0CfhS&H&UIfI0(EX8`I9K%D`oGXQl4pw0l) z8Gt$iP-g(@3_zU$s51a{2B6LW)ER&}15jrG>I^`g0jM(obq1i$0Mr?PIs;H=0O|}t zodKvb0CfhS&H&UIfI0(EX8`I9K%D`oGXQl4pw0l)8Gt$iP-g(@3_zU$s51a{2B6LW z)ER&}15jrG>I^`g0jM(obq1i$0Mr?PIs;H=0O|}todKvb0CfhS&H&UIfI0(EX8`I9 zK%D`oGXQl4pw0l)8Gt$iP-g(@3_zU$s51a{2B6LW)ER&}15jrG>I^`g0jM(obq1i$ z0Mr?PIs;H=0O|}todKvb0CfhS&H&UIfI0(EX8`I9K%D`oGXQl4pw0l)8Gt$iP-g(@ z3_zU$s51a{2B6LW)ER&}15jrG>I^`g0jM(obq1i$0Mr?PIs;H=0O|}todKvb0CfhS z&H&UIfI0(EX8`I9K%D`oGXQl4pw0l)8Gt$iP-g(@3_zU$s51a{2B6LW)ER&}15jrG z>I^`g0jM(obq1i$0Mr?PIs;H=0O|}todKvb0CfhS&H&UIfI0(EX8`I9K%D`oGXQl4 zpw0l)8Gt$iP-g(@3_zU$s51a{2B6LW)ER&}15jrG>I^`g0jM(obq1i$0Mr?PIs;H= z0O|}todKvb0CfhS&H&UIfI0(EX8`I9K%D`oGXQl4pw0l)8Gt$iP-g(@3_zU$s51a{ z2B6LW)ER&}15jrG>I^`g0jM(obq1i$0Mr?PIs;H=0O|}todKvb0CfhS&H&UIfI0(E zX8`I9K%D`oGXQl4pw0l)8Gt$iP-g(@3_zU$s51a{2B6LW)ER&}15jrG>I^`g0jM(o zbq1i$0Mr?PIs;H=0O|}todKvb0CfhS&H&UIfI0(EX8`I9K%D`oGXQl4pw0l)8Gt$i zP-g(@3_zU$s51a{2B6LW)ER&}15jrG>I^`g0jM(obq1i$0Mr?PIs;H=0O|}todKvb z0CfhS&H&UIfI0(EX8`I9K%D`oGXQl4pw0l)8Gt$iP-g(@3_zU$s51ba>7VJJ>7VI$ z2B6LWbf$l%f2Mz?f2Mz?f2Mz?f2Mz?f2Mz?f2Mz?f2Mz?f2Mz?-x+{915jrG>I^`g z0jM(obq1i$0Mr?PIs;H=0O|}todKvb0CfhS&H&UIfI0(EX8`I9K%D`oGXQl4pw0l) z8Gt$iP-g(@3_zU$s51a{2B6LW)ER&}15jrG>I^`g0jM(obq1i$0Mr?PIs;H=0O|}t zodKvb0CfhS&H&UIfI0(EX8`I9K%D`oGXQl4pw0l)8Gt$iP-g(@3_zU$s51a{2B6LW z)ER&}15jrG>I^`g0jM(obq1i$0Q64(o&G!hclwA%x|r~gj>o&G!hclz)2-|4^8f2V)0f3AP7f3AP7f3AP7f3AP7f3AP7f3AP7f3AP7 zf3AP7f3AP7f3AP7f3AP7f3AP7f3AP7f3AP7f3AP7f3AP7f3AP7f3AP7f3AP7f3AP7 zf3DvdfI0)vx&FESx&FC+X8`I9K%D`oGXQl4pw0l)8Gt$iP-g(@3_zU$s51a{2B6LW z)ER&}1JHZ@_xkVk-|N5Ef3N>u|GoZu{rCEv0jM(oz1M%Q|6c#SerEvc3_zU$s51a{ z2B6LW)ER&}15jrG>I^`g0jM(obq1i$0Mr?PIs?#q{rCFs_228i*MG17UjM!Rd;RzN z@Acp7zt?}S|6c$7|5wGm?8cQ9U>N1uLG++GX8;Jn0E83YlF;VTj6jG7LjNc2M_?If zXa?D-vaGT{%dR@LFH8TM{x|(^`rq`w>3`GzrvFX2g7=U2_h5;A`U>Ja50EPh=24EO~VE~2!7zSV%fMEcJ0T>2g7=U2_h5;A` zU>Ja50EPh=24EO~VE~2!7zSV%fMEcJ0T>2g7=U2_h5;A`U>Ja50EPh=24EO~VE~2! z7zSV%fMEcJ0T>2g7=U2_h5;A`U>Ja50EPh=24EO~VE~2!7zSV%fMEcJ0T>2g7=U2_ zh5;A`U>Ja50EPh=24EO~VE~2!7zSV%fMEcJ0T>2g7=U2_h5;A`U>Ja50EPh=24EO~ zVE~2!7zSV%fMEcJ0T>2g7=U2_h5;A`U>Ja50EPh=24EO~VE~2!7zSV%fMEcJ0T>2g z7=U2_h5;A`U>Ja50EPh=24EO~VE~2!7zSV%fMEcJ0T>2g7=U2_h5;A`U>Ja50EPh= z24EO~VE~2!7zSV%fMEcJ0T>2g7=U2_h5;A`U>Ja50EPh=24EO~VE~2!7zSV%fMEcJ z0T>2g7=U2_h5;A`U>Ja50EPh=24EO~VE~2!7zSV%fMEcJ0T>2g7=U2_h5;A`z`uVu zZ9N*mXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y z0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U z0HXnn1~3}HXaJ)Dj0P|oz-R!Y0gMJP8o+1(qXCQtFdD#U0HXnn1~3}HXaJ)Dj0P|o zz-R!Y0qk4-G=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfP zU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR z7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|n zMgtfPU^IZy07e5C4PZ2Y(EvsR7!6=FfYAU(0~ifpG=R|nMgtfPU^IZy07e5C4PZ2Y z(EvsR7!6=FfYAU(1K9iTCkA&<}`Y-*L{!9O*|I&Zyzw}@FFa4MPOaG<+(tqi{^k4cf z{g?hr|E2%Zf9b#UU-~com;OutrT@}@>A&<}`Y-*L{!9O@|JHx&zxChxZ~eFaTmP;9 z)_?22_22q${kQ&G|E>Slf9t>X-}-O;xBgrIt^d}4>%aBi`fvTW{#!o{AR0h4fM@{G z0HOgz1BeC?4ImmoxBgo{4ImmoG=OLT-TH6+xBgrIt^d}4>%aBi`fvTW{#*a8|JHx& zzxC4qq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks118 z0MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT z(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G z0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLaw zq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V z0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?W zL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz z1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$ zhz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c z1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh z5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaN08KMf!nKs1180MP)V0Yn3c1`rJ(8bCCF zXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks118 z0MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT z(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G z0HOgz1BeC?4ImmoG=OLT(Ey?WL<8vO`e^{s0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh z5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC? z4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1 zAR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ( z8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2 zKs1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4Immo zG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4 zfM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCF zXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks118 z0MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT z(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G z0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLaw zq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V z0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?W zL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz z1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$ zhz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c z1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh z5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC? z4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1 zAR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ( z8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2 zKs1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4Immo zG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4 zfM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCF zXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks118 z0MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G0HOgz1BeC?4ImmoG=OLT z(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLawq5(t$hz1Z1AR0h4fM@{G z0HOgz1BeC?4ImmoG=OLT(Ey?WL<5Kh5Dg$2Ks1180MP)V0Yn3c1`rJ(8bCCFXaLaw zq5(t$hz1Z1AR0h4fM@{G0HOh$pIRt1{Q7^pBG5A2gCqVvfi zi}T5L+w Loading KerasNLP Gemma model with preset `{preset}`...") + keras_nlp_model = keras_nlp.models.GemmaCausalLM.from_preset(preset) + else: + hf_id, keras_preset = SIZE_MAP[size.lower()] + print(f"\n-> Loading Keras weights from file `{weights_file}`...") + keras_nlp_model = keras_nlp.models.GemmaCausalLM.from_preset( + keras_preset + ) + keras_nlp_model.load_weights(weights_file) + + print(f"\n-> Loading HuggingFace Gemma `{size.upper()}` model...") + hf_model = transformers.GemmaForCausalLM(CONFIG_MAPPING[size.lower()]) + + print("\n✅ Model loading complete.") + print("\n-> Converting weights from KerasNLP Gemma to HuggingFace Gemma...") + + # Token embedding (with vocab size difference handling) + keras_embedding = keras_nlp_model.backbone.token_embedding.weights[0] + hf_vocab_size = hf_model.model.embed_tokens.weight.shape[0] + keras_nlp_vocab_size = keras_embedding.value.shape[0] + if hf_vocab_size < keras_nlp_vocab_size: + diff = keras_nlp_vocab_size - hf_vocab_size + update_state_dict( + hf_model.model.embed_tokens, + "weight", + keras_embedding.value[:-diff, :], + ) + else: + update_state_dict( + hf_model.model.embed_tokens, + "weight", + keras_embedding.value, + ) + + # Decoder blocks + for i in range(keras_nlp_model.backbone.num_layers): + decoder_block = keras_nlp_model.backbone.get_layer(f"decoder_block_{i}") + + # Pre-attention norm + update_state_dict( + hf_model.model.layers[i].input_layernorm, + "weight", + decoder_block.pre_attention_norm.weights[0].value, + ) + + # Attention + query_target_shape = hf_model.model.layers[ + i + ].self_attn.q_proj.weight.shape + query_tensor = decoder_block.attention.query_dense.weights[0].value + query_tensor = query_tensor.transpose(1, 2).reshape(query_target_shape) + update_state_dict( + hf_model.model.layers[i].self_attn.q_proj, "weight", query_tensor + ) + + key_target_shape = hf_model.model.layers[ + i + ].self_attn.k_proj.weight.shape + key_tensor = decoder_block.attention.key_dense.weights[0].value + key_tensor = key_tensor.transpose(1, 2).reshape(key_target_shape) + update_state_dict( + hf_model.model.layers[i].self_attn.k_proj, "weight", key_tensor + ) + + value_target_shape = hf_model.model.layers[ + i + ].self_attn.v_proj.weight.shape + value_tensor = decoder_block.attention.value_dense.weights[0].value + value_tensor = value_tensor.transpose(1, 2).reshape(value_target_shape) + update_state_dict( + hf_model.model.layers[i].self_attn.v_proj, "weight", value_tensor + ) + + out_target_shape = hf_model.model.layers[ + i + ].self_attn.o_proj.weight.shape + keras_out_tensor = decoder_block.attention.output_dense.weights[0].value + out_tensor = keras_out_tensor.reshape( + (out_target_shape[1], out_target_shape[0]) # Transpose target size + ).transpose(0, 1) + + update_state_dict( + hf_model.model.layers[i].self_attn.o_proj, "weight", out_tensor + ) + + # Post-attention norm + update_state_dict( + hf_model.model.layers[i].post_attention_layernorm, + "weight", + decoder_block.pre_ffw_norm.weights[0].value, + ) + + # MLP (Feed-forward) + update_state_dict( + hf_model.model.layers[i].mlp.gate_proj, + "weight", + decoder_block.gating_ffw.weights[0].value.transpose(0, 1), + ) + update_state_dict( + hf_model.model.layers[i].mlp.up_proj, + "weight", + decoder_block.gating_ffw_2.weights[0].value.transpose(0, 1), + ) + update_state_dict( + hf_model.model.layers[i].mlp.down_proj, + "weight", + decoder_block.ffw_linear.weights[0].value.transpose(0, 1), + ) + + # Final norm + update_state_dict( + hf_model.model.norm, + "weight", + keras_nlp_model.backbone.layers[-1].weights[0].value, + ) + + print("\n✅ Weights converted successfully.") + print(f"\n-> Saving HuggingFace model to `{output_dir}`...") + + # Save model to HF Transformers format + os.makedirs(output_dir, exist_ok=True) + hf_model.save_pretrained(output_dir) + + print(f"\n✅ Saving complete. Model saved at `{output_dir}`.") + + # Tokenizer + + if not vocab_path: + tokenizer_preset = preset or SIZE_MAP[size.lower()] + print( + "\n-> Loading KerasNLP Gemma tokenizer with " + f"preset `{tokenizer_preset}`..." + ) + keras_nlp_tokenizer = keras_nlp.models.GemmaTokenizer.from_preset( + tokenizer_preset + ) + # Save tokenizer state + keras_nlp_tokenizer.save_assets(output_dir) + vocab_path = os.path.join(output_dir, "vocabulary.spm") + print("\n✅ Tokenizer loading complete.") + + hf_tokenizer = transformers.GemmaTokenizer(vocab_path) + + print(f"\n-> Saving HuggingFace Gemma tokenizer to `{output_dir}`...") + # Save tokenizer to HF Transformers format + hf_tokenizer.save_pretrained(output_dir) + + print(f"\n✅ Saving complete. Tokenizer saved at `{output_dir}`.") + + +def update_state_dict(layer, weight_name: str, tensor: torch.Tensor) -> None: + """Updates the state dict for a weight given a tensor.""" + assert ( + tensor.shape == layer.state_dict()[weight_name].shape + ), f"{tensor.shape} vs {layer.state_dict()[weight_name].shape}" + layer.state_dict()[weight_name].copy_(tensor) + + +def flag_error_handler(): + if not FLAGS.preset and not FLAGS.weights_file: + raise ValueError( + "Please pass either a valid Keras preset to `--preset`" + " or supply a Keras weights file (`.weights.h5`) and model size" + " (`2b` or `7b`) to `--weights_file` and `--size`, respectively." + ) + if FLAGS.weights_file: + if FLAGS.preset: + raise ValueError( + "Both `--preset` and `--weights_file` flags cannot be supplied " + "at the same time. Either supply a valid Keras preset to " + "`--preset`or supply a Keras `.weights.h5` file and " + "model size (`2b` or `7b`) to `--weights_file` and `--size`, " + "respectively." + ) + if not str(FLAGS.weights_file).endswith(".weights.h5"): + raise ValueError( + "Please pass a valid Keras weights file ending in `.weights.h5`." + ) + if not FLAGS.size: + raise ValueError( + "The `size` flag must be passed if a weights file is passed. " + "Please pass the appropriate size (`2b` or `7b`) for your " + "model to the `--size` flag." + ) + if FLAGS.size.lower() not in ["2b", "7b"]: + raise ValueError( + "Invalid `size`. Please pass the appropriate size (`2b` or `7b`) " + "for your model to the `--size` flag." + ) + + +def main(_): + flag_error_handler() + convert_checkpoints( + FLAGS.preset, + FLAGS.weights_file, + FLAGS.size, + FLAGS.output_dir, + FLAGS.vocab_path, + ) + + +if __name__ == "__main__": + flags.mark_flag_as_required("size") + app.run(main) diff --git a/tools/gemma/export_gemma_to_torch_xla.py b/tools/gemma/export_gemma_to_torch_xla.py new file mode 100644 index 0000000000..005eac272d --- /dev/null +++ b/tools/gemma/export_gemma_to_torch_xla.py @@ -0,0 +1,322 @@ +# Copyright 2024 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import contextlib +import os + +import gemma +import torch +import torch_xla.core.xla_model as xm +from absl import app +from absl import flags +from gemma import model_xla as gemma_model + +import keras_nlp + +os.environ["KERAS_BACKEND"] = "torch" + +""" +Sample usage: + +For converting a Keras model to PyTorch format using a custom or fine-tuned +checkpoint from Keras, make sure to pass the path for the Keras weights file +(ending in `.weights.h5`) and the model size (`2b` or `7b`) to `--weights_file` +and `--size`, respectively. + +Optionally, you can specify the output path for the converted model at +`--output_file`. (This defaults to `gemma.ckpt`) +``` +python tools/gemma/export_gemma_to_torch_xla.py \ + --weights_file fine_tuned_imdb.weights.h5 \ + --size 2b \ + --output_file fine_tuned_imdb.ckpt +``` + +For converting a Keras model to PyTorch format from a preset, +simply pass the Keras preset name to `--preset`. +``` +python tools/gemma/export_gemma_to_torch_xla.py \ + --preset gemma_2b_en \ + --output_file path/to/keras_torch_model.ckpt +``` +""" + + +PRESET_MAP = { + "gemma_2b_en": gemma.config.get_config_for_2b(), + "gemma_instruct_2b_en": gemma.config.get_config_for_2b(), + "gemma_7b_en": gemma.config.get_config_for_7b(), + "gemma_instruct_7b_en": gemma.config.get_config_for_7b(), +} + +SIZE_MAP = { + "2b": (gemma.config.get_config_for_2b(), "gemma_2b_en"), + "7b": (gemma.config.get_config_for_7b(), "gemma_7b_en"), +} + +FLAGS = flags.FLAGS +flags.DEFINE_string( + "preset", + None, + f'Must be one of {",".join(PRESET_MAP.keys())}' + " Alternatively, a Keras weights file (`.weights.h5`) can be passed" + " to --weights_file flag.", +) +flags.DEFINE_string( + "weights_file", + None, + "A Keras weights file (`.weights.h5`)." + " Alternatively, a model preset can be passed to --preset flag.", +) +flags.DEFINE_string( + "size", + None, + "Size of model. Must be passed if `weights_file` is passed. " + "This should be either `2b` or `7b`.", +) +flags.DEFINE_string( + "output_file", + "gemma.ckpt", + "An output file for the converted PyTorch checkpoint. Default: `gemma.ckpt`", +) +flags.DEFINE_string( + "vocab_dir", + "gemma_tokenizer", + "A directory in which the vocabulary for the tokenizer will be stored.", +) +flags.DEFINE_string( + "dtype", + "float32", + "Set the precision of the converted checkpoint. Must be a valid PyTorch dtype.", +) + + +@contextlib.contextmanager +def _set_default_tensor_type(dtype: torch.dtype): + """Sets the default torch dtype to the given dtype.""" + torch.set_default_dtype(dtype) + yield + torch.set_default_dtype(torch.float) + + +def _reconcile_attention_dims(qkv, target_shape): + return torch.cat(qkv).reshape(tuple(target_shape)) + + +def convert_checkpoints(preset, weights_file, size, output_file, vocab_dir): + device = xm.xla_device() + + if preset is not None: + print( + f"\n-> Loading PyTorch Gemma model config for preset `{preset}`..." + ) + model = gemma_model.GemmaForCausalLM( + PRESET_MAP[preset], world_size=1, rank=0, device=device + ) + print(f"\n-> Loading KerasNLP Gemma model with preset `{preset}`...") + keras_nlp_model = keras_nlp.models.GemmaCausalLM.from_preset(preset) + else: + print(f"\n-> Loading PyTorch Gemma model config for `{size}` model...") + config, size_preset = SIZE_MAP[size.lower()] + model = gemma_model.GemmaForCausalLM( + config, world_size=1, rank=0, device=device + ) + print(f"\n-> Loading Keras weights from file `{weights_file}`...") + keras_nlp_model = keras_nlp.models.GemmaCausalLM.from_preset( + size_preset + ) + keras_nlp_model.load_weights(weights_file) + + print("\n✅ Model loading complete.") + print("\n-> Converting weights from KerasNLP Gemma to PyTorch Gemma...") + + # Token embedding (with vocab size difference handling) + keras_embedding = keras_nlp_model.backbone.token_embedding.weights[0] + torch_vocab_size = model.embedder.weight.shape[0] + keras_nlp_vocab_size = keras_embedding.value.shape[0] + if torch_vocab_size < keras_nlp_vocab_size: + diff = keras_nlp_vocab_size - torch_vocab_size + update_state_dict( + model.embedder, + "weight", + keras_embedding.value[:-diff, :], + ) + else: + update_state_dict( + model.embedder, + "weight", + keras_embedding.value, + ) + + # Decoder blocks + for i in range(keras_nlp_model.backbone.num_layers): + decoder_block = keras_nlp_model.backbone.get_layer(f"decoder_block_{i}") + # Pre-attention norm + update_state_dict( + model.model.layers[i].input_layernorm, + "weight", + decoder_block.pre_attention_norm.weights[0].value, + ) + + # Attention + qkv = ( + decoder_block.attention.query_dense.weights[0].value.transpose( + 1, 2 + ), + decoder_block.attention.key_dense.weights[0].value.transpose(1, 2), + decoder_block.attention.value_dense.weights[0].value.transpose( + 1, 2 + ), + ) + qkv_target_shape = model.model.layers[i].self_attn.qkv_proj.weight.shape + combined_tensor = _reconcile_attention_dims(qkv, qkv_target_shape) + + update_state_dict( + model.model.layers[i].self_attn.qkv_proj, "weight", combined_tensor + ) + + out_target_shape = model.model.layers[i].self_attn.o_proj.weight.shape + keras_out_tensor = decoder_block.attention.output_dense.weights[0].value + out_tensor = keras_out_tensor.reshape( + (out_target_shape[1], out_target_shape[0]) # Transpose target size + ).transpose(0, 1) + + update_state_dict( + model.model.layers[i].self_attn.o_proj, "weight", out_tensor + ) + + # Post-attention norm + update_state_dict( + model.model.layers[i].post_attention_layernorm, + "weight", + decoder_block.pre_ffw_norm.weights[0].value, + ) + + # MLP (Feed-forward) + update_state_dict( + model.model.layers[i].mlp.gate_proj, + "weight", + decoder_block.gating_ffw.weights[0].value.transpose(0, 1), + ) + update_state_dict( + model.model.layers[i].mlp.up_proj, + "weight", + decoder_block.gating_ffw_2.weights[0].value.transpose(0, 1), + ) + update_state_dict( + model.model.layers[i].mlp.down_proj, + "weight", + decoder_block.ffw_linear.weights[0].value.transpose(0, 1), + ) + + # Final norm + update_state_dict( + model.model.norm, + "weight", + keras_nlp_model.backbone.layers[-1].weights[0].value, + ) + + print("\n✅ Weights converted successfully.") + print(f"\n-> Saving PyTorch model checkpoint to `{output_file}`...") + + # Save model checkpoint + torch.save({"model_state_dict": model.state_dict()}, output_file) + + print( + f"\n✅ Saving complete. Model checkpoint available at `{output_file}`." + ) + + if preset is not None: + # Tokenizer + print( + f"\n-> Loading KerasNLP Gemma tokenizer with preset `{preset}`..." + ) + keras_nlp_tokenizer = keras_nlp.models.GemmaTokenizer.from_preset( + preset + ) + print("\n✅ Model loading complete.") + print(f"\n-> Saving tokenizer state to directory `{vocab_dir}`...") + + # Save tokenizer state + os.makedirs(vocab_dir, exist_ok=True) + keras_nlp_tokenizer.save_assets(vocab_dir) + + print( + "\n✅ Saving complete. Tokenizer state " + f"available at `{vocab_dir}/vocabulary.spm`." + ) + + +def update_state_dict(layer, weight_name: str, tensor: torch.Tensor) -> None: + """Updates the state dict for a weight given a tensor.""" + assert ( + tensor.shape == layer.state_dict()[weight_name].shape + ), f"{tensor.shape} vs {layer.state_dict()[weight_name].shape}" + layer.state_dict()[weight_name].copy_(tensor) + + +def flag_error_handler(): + if not FLAGS.preset and not FLAGS.weights_file: + raise ValueError( + "Please pass either a valid Keras preset to `--preset`" + " or supply a Keras weights file (`.weights.h5`) and model size" + " (`2b` or `7b`) to `--weights_file` and `--size`, respectively." + ) + if FLAGS.weights_file: + if FLAGS.preset: + raise ValueError( + "Both `--preset` and `--weights_file` flags cannot be supplied " + "at the same time. Either supply a valid Keras preset to " + "`--preset`or supply a Keras `.weights.h5` file and " + "model size (`2b` or `7b`) to `--weights_file` and `--size`, " + "respectively." + ) + if not str(FLAGS.weights_file).endswith(".weights.h5"): + raise ValueError( + "Please pass a valid Keras weights file ending in `.weights.h5`." + ) + if not FLAGS.size: + raise ValueError( + "The `size` flag must be passed if a weights file is passed. " + "Please pass the appropriate size (`2b` or `7b`) for your " + "model to the `--size` flag." + ) + if FLAGS.size.lower() not in ["2b", "7b"]: + raise ValueError( + "Invalid `size`. Please pass the appropriate size (`2b` or `7b`) " + "for your model to the `--size` flag." + ) + if FLAGS.dtype: + dtype = getattr(torch, FLAGS.dtype) + if not isinstance(dtype, torch.dtype): + raise ValueError( + "Invalid `dtype`. Please pass a valid PyTorch data type (e.g. " + "`float32', 'float16`, etc.) to the `--dtype` flag." + ) + + +def main(_): + flag_error_handler() + with _set_default_tensor_type(getattr(torch, FLAGS.dtype)): + convert_checkpoints( + FLAGS.preset, + FLAGS.weights_file, + FLAGS.size, + FLAGS.output_file, + FLAGS.vocab_dir, + ) + + +if __name__ == "__main__": + app.run(main) diff --git a/tools/gemma/run_gemma_xla.py b/tools/gemma/run_gemma_xla.py new file mode 100644 index 0000000000..9fa50cbd2b --- /dev/null +++ b/tools/gemma/run_gemma_xla.py @@ -0,0 +1,287 @@ +# Copyright 2024 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import contextlib +import os +import random +import sys +from typing import List + +import gemma.xla_model_parallel as xla_model_parallel +import numpy as np +import torch +import torch.multiprocessing +import torch_xla.core.xla_model as xm +import torch_xla.distributed.xla_multiprocessing as xmp +from absl import app +from absl import flags +from gemma.config import GemmaConfig +from gemma.config import get_config_for_2b +from gemma.config import get_config_for_7b +from gemma.model_xla import GemmaForCausalLM +from gemma.tokenizer import Tokenizer + +PAD_TOKEN_ID = -1 + +FILE_PATH = "gemma.ckpt" +TOKENIZER_DIR = "gemma_tokenizer" + +PRESET_MAP = { + "gemma_2b_en": get_config_for_2b(), + "gemma_instruct_2b_en": get_config_for_2b(), + "gemma_7b_en": get_config_for_7b(), + "gemma_instruct_7b_en": get_config_for_7b(), +} + +SIZE_MAP = { + "2b": get_config_for_2b(), + "7b": get_config_for_7b(), +} + +FLAGS = flags.FLAGS +flags.DEFINE_string( + "preset", None, f'Must be one of {",".join(PRESET_MAP.keys())}' +) +flags.DEFINE_string( + "size", + None, + "Size of model. Must be passed if `preset` is not passed. " + "This should be either `2b` or `7b`.", +) +flags.DEFINE_string( + "checkpoint_file", + "gemma.ckpt", + "A PyTorch checkpoint file containing the converted weights.", +) +flags.DEFINE_string( + "vocab_file", + "gemma_tokenizer/vocabulary.spm", + "The file containing the vocabulary for the tokenizer.", +) +flags.DEFINE_string( + "prompt", + "The capital of France is", + "A test prompt for verifying functionality of the PyTorch Gemma model.", +) + +# This is a modified version of `run_xla.py` script in the Hex-LLM Gemma repo +# to ensure proper functionality after porting checkpoints from Keras. + + +@contextlib.contextmanager +def _set_default_tensor_type(dtype: torch.dtype): + """Sets the default torch dtype to the given dtype.""" + torch.set_default_dtype(dtype) + yield + torch.set_default_dtype(torch.float) + + +def generate( + i: int, + model_config: GemmaConfig, + checkpoint_file: str, + vocab_file: str, + prompts: List[str], + output_lens: List[int], + temperatures: List[float], + top_ps: List[float], + top_ks: List[int], +): + # Set seed from config + seed = model_config.seed + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + + device = xm.xla_device() + xm.set_rng_state(seed, device) + + rank = xla_model_parallel.get_model_parallel_rank() + world_size = xla_model_parallel.get_model_parallel_world_size() + if rank > 0: + sys.stdout = open(os.devnull, "w") + + # Load model with ported weights and place on device + with _set_default_tensor_type(model_config.get_dtype()): + model = GemmaForCausalLM(model_config, world_size, rank, device) + model.load_weights(checkpoint_file) + model = model.to(device).eval() + + # Create tokenizer with saved Keras tokenizer state + tokenizer = Tokenizer(vocab_file) + + prompt_tokens = [tokenizer.encode(prompt) for prompt in prompts] + min_prompt_len = min(len(p) for p in prompt_tokens) + + batch_size = len(prompts) + assert batch_size == len(temperatures) + assert batch_size == len(top_ps) + assert batch_size == len(top_ks) + max_seq_len = max([len(p) + o for p, o in zip(prompt_tokens, output_lens)]) + assert max_seq_len <= model_config.max_position_embeddings + if model_config.num_key_value_heads < world_size: + assert world_size % model_config.num_key_value_heads == 0 + n_local_heads = 1 + else: + assert model_config.num_key_value_heads % world_size == 0 + n_local_heads = model_config.num_key_value_heads // world_size + + # build KV caches + kv_caches = [] + for _ in range(model_config.num_hidden_layers): + k_cache = torch.zeros( + size=( + batch_size, + max_seq_len, + n_local_heads, + model_config.head_dim, + ), + dtype=model_config.get_dtype(), + device=device, + ) + v_cache = torch.zeros( + size=( + batch_size, + max_seq_len, + n_local_heads, + model_config.head_dim, + ), + dtype=model_config.get_dtype(), + device=device, + ) + kv_caches.append((k_cache, v_cache)) + + # prepare inputs + token_ids_tensor = torch.full( + (batch_size, max_seq_len), PAD_TOKEN_ID, dtype=torch.int64 + ) + input_token_ids_tensor = torch.full( + (batch_size, min_prompt_len), PAD_TOKEN_ID, dtype=torch.int64 + ) + for i, p in enumerate(prompt_tokens): + token_ids_tensor[i, : len(p)] = torch.tensor(p) + input_token_ids_tensor[i, :min_prompt_len] = torch.tensor( + p[:min_prompt_len] + ) + token_ids_tensor = token_ids_tensor.to(device) + prompt_mask_tensor = token_ids_tensor != PAD_TOKEN_ID + input_token_ids_tensor = input_token_ids_tensor.to(device) + input_positions_tensor = torch.arange( + 0, min_prompt_len, dtype=torch.int64 + ).to(device) + mask_tensor = torch.full( + (1, 1, max_seq_len, max_seq_len), -2.3819763e38 + ).to(torch.float) + mask_tensor = torch.triu(mask_tensor, diagonal=1).to(device) + curr_mask_tensor = mask_tensor.index_select(2, input_positions_tensor) + output_positions_tensor = torch.LongTensor([min_prompt_len - 1]).to(device) + temperatures_tensor = torch.FloatTensor(temperatures).to(device) + top_ps_tensor = torch.FloatTensor(top_ps).to(device) + top_ks_tensor = torch.LongTensor(top_ks).to(device) + output_index = torch.tensor(min_prompt_len, dtype=torch.int64).to(device) + xm.mark_step() + + # Prefill up to min_prompt_len tokens, then treat other prefill as decode and ignore output. + for i in range(max_seq_len - min_prompt_len): + next_token_ids = model( + input_token_ids=input_token_ids_tensor, + input_positions=input_positions_tensor, + kv_write_indices=None, + kv_caches=kv_caches, + mask=curr_mask_tensor, + output_positions=output_positions_tensor, + temperatures=temperatures_tensor, + top_ps=top_ps_tensor, + top_ks=top_ks_tensor, + ) + curr_prompt_mask = prompt_mask_tensor.index_select( + 1, output_index + ).squeeze(dim=1) + curr_token_ids = token_ids_tensor.index_select(1, output_index).squeeze( + dim=1 + ) + output_token_ids = torch.where( + curr_prompt_mask, curr_token_ids, next_token_ids + ).unsqueeze(dim=1) + token_ids_tensor.index_copy_(1, output_index, output_token_ids) + + input_token_ids_tensor = output_token_ids + input_positions_tensor = output_index + curr_mask_tensor = mask_tensor.index_select(2, input_positions_tensor) + output_positions_tensor = torch.tensor(0, dtype=torch.int64).to(device) + output_index = output_index + 1 + xm.mark_step() + + # Detokenization. + token_ids = token_ids_tensor.tolist() + results = [] + for i, tokens in enumerate(token_ids): + trimmed_output = tokens[ + len(prompt_tokens[i]) : len(prompt_tokens[i]) + output_lens[i] + ] + if tokenizer.eos_id in trimmed_output: + eos_index = trimmed_output.index(tokenizer.eos_id) + trimmed_output = trimmed_output[:eos_index] + results.append(tokenizer.decode(trimmed_output)) + + for prompt, result in zip(prompts, results): + print("======================================") + print(f"PROMPT: {prompt}") + print(f"RESULT: {result}") + print("======================================") + + +def flag_error_handler(): + if not FLAGS.preset and not FLAGS.size: + raise ValueError( + "Please pass either a valid Keras preset to `--preset`" + " or supply a model size (`2b` or `7b`) to `--size`." + ) + if FLAGS.size and FLAGS.size.lower() not in ["2b", "7b"]: + raise ValueError( + "Invalid `size`. Please pass the appropriate size (`2b` or `7b`) " + "for your model to the `--size` flag." + ) + + +def main(_): + flag_error_handler() + if FLAGS.preset: + model_config = PRESET_MAP[FLAGS.preset] + else: + model_config = SIZE_MAP[FLAGS.size.lower()] + prompts = [ + FLAGS.prompt, + ] + n = len(prompts) + output_lengths = [10] * n + temperatures = [0.95] * n + top_ps = [1.0] * n + top_ks = [100] * n + xmp.spawn( + generate, + args=( + model_config, + FLAGS.checkpoint_file, + FLAGS.vocab_file, + prompts, + output_lengths, + temperatures, + top_ps, + top_ks, + ), + ) + + +if __name__ == "__main__": + app.run(main) diff --git a/tools/sentencepiece_testing/create_gemma_test_proto.py b/tools/sentencepiece_testing/create_gemma_test_proto.py new file mode 100644 index 0000000000..c3ce418a4b --- /dev/null +++ b/tools/sentencepiece_testing/create_gemma_test_proto.py @@ -0,0 +1,36 @@ +# Copyright 2023 The KerasNLP Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tools.sentencepiece_testing.utils import train_sentencepiece + + +def main(): + train_sentencepiece( + ["the quick brown fox", "the earth is round"], + "gemma_test_vocab.spm", + vocab_size=11, + model_type="WORD", + pad_id=0, + bos_id=1, + eos_id=2, + unk_id=3, + pad_piece="", + bos_piece="", + eos_piece="", + unk_piece="", + ) + + +if __name__ == "__main__": + main() From b38c10f7e039c95341027d025c69e1ccdbc3abea Mon Sep 17 00:00:00 2001 From: Matt Watson <1389937+mattdangerw@users.noreply.github.com> Date: Fri, 16 Feb 2024 14:18:42 -0800 Subject: [PATCH 26/29] Version bump for dev release (#1443) --- keras_nlp/version_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keras_nlp/version_utils.py b/keras_nlp/version_utils.py index 4d6a8186d4..2be29e26fb 100644 --- a/keras_nlp/version_utils.py +++ b/keras_nlp/version_utils.py @@ -15,7 +15,7 @@ from keras_nlp.api_export import keras_nlp_export # Unique source of truth for the version number. -__version__ = "0.8.0" +__version__ = "0.8.0.dev0" @keras_nlp_export("keras_nlp.version") From 381a1dd36dd87923cbb393b055e8e8510edb37f6 Mon Sep 17 00:00:00 2001 From: Matt Watson <1389937+mattdangerw@users.noreply.github.com> Date: Fri, 16 Feb 2024 14:54:43 -0800 Subject: [PATCH 27/29] Unexport models from the 0.8 release (#1360) (#1442) This unexports our "not yet ready from prime time models". - electra - gpt-neox - t5 - whisper - xlnet These are all still in flight to some degree. --- keras_nlp/models/bloom/bloom_backbone.py | 3 +-- keras_nlp/models/bloom/bloom_causal_lm_preprocessor.py | 4 ++-- keras_nlp/models/bloom/bloom_preprocessor.py | 4 ++-- keras_nlp/models/bloom/bloom_tokenizer.py | 4 ++-- keras_nlp/models/electra/electra_backbone.py | 3 +-- keras_nlp/models/electra/electra_tokenizer.py | 4 ++-- keras_nlp/models/gpt_neo_x/gpt_neo_x_backbone.py | 3 +-- keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm.py | 3 +-- .../models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor.py | 4 ++-- keras_nlp/models/gpt_neo_x/gpt_neo_x_preprocessor.py | 4 ++-- keras_nlp/models/gpt_neo_x/gpt_neo_x_tokenizer.py | 4 ++-- keras_nlp/models/llama/llama_backbone.py | 3 +-- keras_nlp/models/t5/t5_backbone.py | 3 +-- keras_nlp/models/t5/t5_tokenizer.py | 4 ++-- keras_nlp/models/whisper/whisper_audio_feature_extractor.py | 4 ++-- keras_nlp/models/whisper/whisper_backbone.py | 3 +-- keras_nlp/models/whisper/whisper_preprocessor.py | 3 +-- keras_nlp/models/whisper/whisper_tokenizer.py | 4 ++-- keras_nlp/models/xlnet/xlnet_backbone.py | 3 +-- 19 files changed, 29 insertions(+), 38 deletions(-) diff --git a/keras_nlp/models/bloom/bloom_backbone.py b/keras_nlp/models/bloom/bloom_backbone.py index 5737dcc889..5e2251a693 100644 --- a/keras_nlp/models/bloom/bloom_backbone.py +++ b/keras_nlp/models/bloom/bloom_backbone.py @@ -13,7 +13,6 @@ # limitations under the License. import copy -from keras_nlp.api_export import keras_nlp_export from keras_nlp.backend import keras from keras_nlp.layers.modeling.reversible_embedding import ReversibleEmbedding from keras_nlp.models.backbone import Backbone @@ -26,7 +25,7 @@ def _bloom_kernel_initializer(stddev=0.02): return keras.initializers.RandomNormal(stddev=stddev) -@keras_nlp_export("keras_nlp.models.BloomBackbone") +@keras.saving.register_keras_serializable(package="keras_nlp") class BloomBackbone(Backbone): """A BLOOM decoder network. diff --git a/keras_nlp/models/bloom/bloom_causal_lm_preprocessor.py b/keras_nlp/models/bloom/bloom_causal_lm_preprocessor.py index 01f3c88d30..ceec2b67fb 100644 --- a/keras_nlp/models/bloom/bloom_causal_lm_preprocessor.py +++ b/keras_nlp/models/bloom/bloom_causal_lm_preprocessor.py @@ -15,7 +15,7 @@ import tensorflow as tf from absl import logging -from keras_nlp.api_export import keras_nlp_export +from keras_nlp.backend import keras from keras_nlp.backend import ops from keras_nlp.models.bloom.bloom_preprocessor import BloomPreprocessor from keras_nlp.utils.keras_utils import ( @@ -24,7 +24,7 @@ from keras_nlp.utils.keras_utils import pack_x_y_sample_weight -@keras_nlp_export("keras_nlp.models.BloomCausalLMPreprocessor") +@keras.saving.register_keras_serializable(package="keras_nlp") class BloomCausalLMPreprocessor(BloomPreprocessor): """BLOOM Causal LM preprocessor. diff --git a/keras_nlp/models/bloom/bloom_preprocessor.py b/keras_nlp/models/bloom/bloom_preprocessor.py index 734c9f4bf8..45003916ba 100644 --- a/keras_nlp/models/bloom/bloom_preprocessor.py +++ b/keras_nlp/models/bloom/bloom_preprocessor.py @@ -14,7 +14,7 @@ import copy -from keras_nlp.api_export import keras_nlp_export +from keras_nlp.backend import keras from keras_nlp.layers.preprocessing.start_end_packer import StartEndPacker from keras_nlp.models.bloom.bloom_presets import backbone_presets from keras_nlp.models.bloom.bloom_tokenizer import BloomTokenizer @@ -26,7 +26,7 @@ from keras_nlp.utils.python_utils import classproperty -@keras_nlp_export("keras_nlp.models.BloomPreprocessor") +@keras.saving.register_keras_serializable(package="keras_nlp") class BloomPreprocessor(Preprocessor): """BLOOM preprocessing layer which tokenizes and packs inputs. diff --git a/keras_nlp/models/bloom/bloom_tokenizer.py b/keras_nlp/models/bloom/bloom_tokenizer.py index cc3fcc2fc3..6ab26c6353 100644 --- a/keras_nlp/models/bloom/bloom_tokenizer.py +++ b/keras_nlp/models/bloom/bloom_tokenizer.py @@ -14,13 +14,13 @@ import copy -from keras_nlp.api_export import keras_nlp_export +from keras_nlp.backend import keras from keras_nlp.models.bloom.bloom_presets import backbone_presets from keras_nlp.tokenizers.byte_pair_tokenizer import BytePairTokenizer from keras_nlp.utils.python_utils import classproperty -@keras_nlp_export("keras_nlp.models.BloomTokenizer") +@keras.saving.register_keras_serializable(package="keras_nlp") class BloomTokenizer(BytePairTokenizer): """A BLOOM tokenizer using Byte-Pair Encoding subword segmentation. diff --git a/keras_nlp/models/electra/electra_backbone.py b/keras_nlp/models/electra/electra_backbone.py index 13be2d8eb8..0cc0358e82 100644 --- a/keras_nlp/models/electra/electra_backbone.py +++ b/keras_nlp/models/electra/electra_backbone.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from keras_nlp.api_export import keras_nlp_export from keras_nlp.backend import keras from keras_nlp.layers.modeling.position_embedding import PositionEmbedding from keras_nlp.layers.modeling.reversible_embedding import ReversibleEmbedding @@ -25,7 +24,7 @@ def electra_kernel_initializer(stddev=0.02): return keras.initializers.TruncatedNormal(stddev=stddev) -@keras_nlp_export("keras_nlp.models.ElectraBackbone") +@keras.saving.register_keras_serializable(package="keras_nlp") class ElectraBackbone(Backbone): """A Electra encoder network. diff --git a/keras_nlp/models/electra/electra_tokenizer.py b/keras_nlp/models/electra/electra_tokenizer.py index acd665c2a3..4fb7829424 100644 --- a/keras_nlp/models/electra/electra_tokenizer.py +++ b/keras_nlp/models/electra/electra_tokenizer.py @@ -12,11 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from keras_nlp.api_export import keras_nlp_export +from keras_nlp.backend import keras from keras_nlp.tokenizers import WordPieceTokenizer -@keras_nlp_export("keras_nlp.models.ElectraTokenizer") +@keras.saving.register_keras_serializable(package="keras_nlp") class ElectraTokenizer(WordPieceTokenizer): """A ELECTRA tokenizer using WordPiece subword segmentation. diff --git a/keras_nlp/models/gpt_neo_x/gpt_neo_x_backbone.py b/keras_nlp/models/gpt_neo_x/gpt_neo_x_backbone.py index 1955ed5801..0ef65a42f1 100644 --- a/keras_nlp/models/gpt_neo_x/gpt_neo_x_backbone.py +++ b/keras_nlp/models/gpt_neo_x/gpt_neo_x_backbone.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from keras_nlp.api_export import keras_nlp_export from keras_nlp.backend import keras from keras_nlp.layers.modeling.reversible_embedding import ReversibleEmbedding from keras_nlp.models.backbone import Backbone @@ -24,7 +23,7 @@ def _gpt_neo_x_kernel_initializer(stddev=0.02): return keras.initializers.RandomNormal(stddev=stddev) -@keras_nlp_export("keras_nlp.models.GPTNeoXBackbone") +@keras.saving.register_keras_serializable(package="keras_nlp") class GPTNeoXBackbone(Backbone): """GPT-NeoX core network with hyperparameters. diff --git a/keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm.py b/keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm.py index b1df4a6706..7725a9f6d8 100644 --- a/keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm.py +++ b/keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from keras_nlp.api_export import keras_nlp_export from keras_nlp.backend import keras from keras_nlp.backend import ops from keras_nlp.models.generative_task import GenerativeTask @@ -23,7 +22,7 @@ from keras_nlp.utils.python_utils import classproperty -@keras_nlp_export("keras_nlp.models.GPTNeoXCausalLM") +@keras.saving.register_keras_serializable(package="keras_nlp") class GPTNeoXCausalLM(GenerativeTask): """An end-to-end GPTNeoX model for causal language modeling. diff --git a/keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor.py b/keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor.py index 92ff9bbb03..665622540e 100644 --- a/keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor.py +++ b/keras_nlp/models/gpt_neo_x/gpt_neo_x_causal_lm_preprocessor.py @@ -15,7 +15,7 @@ import tensorflow as tf from absl import logging -from keras_nlp.api_export import keras_nlp_export +from keras_nlp.backend import keras from keras_nlp.backend import ops from keras_nlp.models.gpt_neo_x.gpt_neo_x_preprocessor import ( GPTNeoXPreprocessor, @@ -26,7 +26,7 @@ from keras_nlp.utils.keras_utils import pack_x_y_sample_weight -@keras_nlp_export("keras_nlp.models.GPTNeoXCausalLMPreprocessor") +@keras.saving.register_keras_serializable(package="keras_nlp") class GPTNeoXCausalLMPreprocessor(GPTNeoXPreprocessor): """GPT-NeoX Causal LM preprocessor. diff --git a/keras_nlp/models/gpt_neo_x/gpt_neo_x_preprocessor.py b/keras_nlp/models/gpt_neo_x/gpt_neo_x_preprocessor.py index 8dc374332b..4e675c9c98 100644 --- a/keras_nlp/models/gpt_neo_x/gpt_neo_x_preprocessor.py +++ b/keras_nlp/models/gpt_neo_x/gpt_neo_x_preprocessor.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from keras_nlp.api_export import keras_nlp_export +from keras_nlp.backend import keras from keras_nlp.layers.preprocessing.start_end_packer import StartEndPacker from keras_nlp.models.gpt_neo_x.gpt_neo_x_tokenizer import GPTNeoXTokenizer from keras_nlp.models.preprocessor import Preprocessor @@ -23,7 +23,7 @@ from keras_nlp.utils.python_utils import classproperty -@keras_nlp_export("keras_nlp.models.GPTNeoXPreprocessor") +@keras.saving.register_keras_serializable(package="keras_nlp") class GPTNeoXPreprocessor(Preprocessor): """GPTNeoX preprocessing layer which tokenizes and packs inputs. diff --git a/keras_nlp/models/gpt_neo_x/gpt_neo_x_tokenizer.py b/keras_nlp/models/gpt_neo_x/gpt_neo_x_tokenizer.py index d109c5849d..cc63e99af6 100644 --- a/keras_nlp/models/gpt_neo_x/gpt_neo_x_tokenizer.py +++ b/keras_nlp/models/gpt_neo_x/gpt_neo_x_tokenizer.py @@ -12,11 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from keras_nlp.api_export import keras_nlp_export +from keras_nlp.backend import keras from keras_nlp.tokenizers.byte_pair_tokenizer import BytePairTokenizer -@keras_nlp_export("keras_nlp.models.GPTNeoXTokenizer") +@keras.saving.register_keras_serializable(package="keras_nlp") class GPTNeoXTokenizer(BytePairTokenizer): """A GPTNeoX tokenizer using Byte-Pair Encoding subword segmentation. diff --git a/keras_nlp/models/llama/llama_backbone.py b/keras_nlp/models/llama/llama_backbone.py index cc628ad7a5..6534fcc0ec 100644 --- a/keras_nlp/models/llama/llama_backbone.py +++ b/keras_nlp/models/llama/llama_backbone.py @@ -11,7 +11,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from keras_nlp.api_export import keras_nlp_export from keras_nlp.backend import keras from keras_nlp.backend import ops from keras_nlp.layers.modeling.reversible_embedding import ReversibleEmbedding @@ -24,7 +23,7 @@ def _llama_kernel_initializer(stddev=0.02): return keras.initializers.RandomNormal(stddev=stddev) -@keras_nlp_export("keras_nlp.models.LlamaBackbone") +@keras.saving.register_keras_serializable(package="keras_nlp") class LlamaBackbone(Backbone): """ LLaMA core network with hyperparameters. diff --git a/keras_nlp/models/t5/t5_backbone.py b/keras_nlp/models/t5/t5_backbone.py index cf747c503c..5fb383458f 100644 --- a/keras_nlp/models/t5/t5_backbone.py +++ b/keras_nlp/models/t5/t5_backbone.py @@ -13,7 +13,6 @@ # limitations under the License. import copy -from keras_nlp.api_export import keras_nlp_export from keras_nlp.backend import keras from keras_nlp.layers.modeling.reversible_embedding import ReversibleEmbedding from keras_nlp.models.backbone import Backbone @@ -23,7 +22,7 @@ from keras_nlp.utils.python_utils import classproperty -@keras_nlp_export("keras_nlp.models.T5Backbone") +@keras.saving.register_keras_serializable(package="keras_nlp") class T5Backbone(Backbone): """T5 encoder-decoder backbone model. diff --git a/keras_nlp/models/t5/t5_tokenizer.py b/keras_nlp/models/t5/t5_tokenizer.py index b5dee49b85..5feb2d9ab8 100644 --- a/keras_nlp/models/t5/t5_tokenizer.py +++ b/keras_nlp/models/t5/t5_tokenizer.py @@ -13,13 +13,13 @@ # limitations under the License. import copy -from keras_nlp.api_export import keras_nlp_export +from keras_nlp.backend import keras from keras_nlp.models.t5.t5_presets import backbone_presets from keras_nlp.tokenizers.sentence_piece_tokenizer import SentencePieceTokenizer from keras_nlp.utils.python_utils import classproperty -@keras_nlp_export("keras_nlp.models.T5Tokenizer") +@keras.saving.register_keras_serializable(package="keras_nlp") class T5Tokenizer(SentencePieceTokenizer): """T5 tokenizer layer based on SentencePiece. diff --git a/keras_nlp/models/whisper/whisper_audio_feature_extractor.py b/keras_nlp/models/whisper/whisper_audio_feature_extractor.py index e41519bbc9..5fade1d63b 100644 --- a/keras_nlp/models/whisper/whisper_audio_feature_extractor.py +++ b/keras_nlp/models/whisper/whisper_audio_feature_extractor.py @@ -17,7 +17,7 @@ import numpy as np import tensorflow as tf -from keras_nlp.api_export import keras_nlp_export +from keras_nlp.backend import keras from keras_nlp.layers.preprocessing.preprocessing_layer import ( PreprocessingLayer, ) @@ -26,7 +26,7 @@ from keras_nlp.utils.python_utils import format_docstring -@keras_nlp_export("keras_nlp.models.WhisperAudioFeatureExtractor") +@keras.saving.register_keras_serializable(package="keras_nlp") class WhisperAudioFeatureExtractor(PreprocessingLayer): """ Whisper audio feature extractor layer. diff --git a/keras_nlp/models/whisper/whisper_backbone.py b/keras_nlp/models/whisper/whisper_backbone.py index c66a61d4e5..6da83c4b0d 100644 --- a/keras_nlp/models/whisper/whisper_backbone.py +++ b/keras_nlp/models/whisper/whisper_backbone.py @@ -14,7 +14,6 @@ import copy -from keras_nlp.api_export import keras_nlp_export from keras_nlp.backend import keras from keras_nlp.backend import ops from keras_nlp.layers.modeling.position_embedding import PositionEmbedding @@ -38,7 +37,7 @@ def call(self, x): return ops.pad(x, [[0, 0], [1, 1], [0, 0]]) -@keras_nlp_export("keras_nlp.models.WhisperBackbone") +@keras.saving.register_keras_serializable(package="keras_nlp") class WhisperBackbone(Backbone): """A Whisper encoder-decoder network for speech. diff --git a/keras_nlp/models/whisper/whisper_preprocessor.py b/keras_nlp/models/whisper/whisper_preprocessor.py index c21705a481..5ddec8732b 100644 --- a/keras_nlp/models/whisper/whisper_preprocessor.py +++ b/keras_nlp/models/whisper/whisper_preprocessor.py @@ -16,7 +16,6 @@ from absl import logging -from keras_nlp.api_export import keras_nlp_export from keras_nlp.backend import keras from keras_nlp.layers.preprocessing.start_end_packer import StartEndPacker from keras_nlp.models.preprocessor import Preprocessor @@ -32,7 +31,7 @@ from keras_nlp.utils.python_utils import classproperty -@keras_nlp_export("keras_nlp.models.WhisperPreprocessor") +@keras.saving.register_keras_serializable(package="keras_nlp") class WhisperPreprocessor(Preprocessor): """A Whisper preprocessing layer which handles audio and text input. diff --git a/keras_nlp/models/whisper/whisper_tokenizer.py b/keras_nlp/models/whisper/whisper_tokenizer.py index 7b68dfd790..4446193738 100644 --- a/keras_nlp/models/whisper/whisper_tokenizer.py +++ b/keras_nlp/models/whisper/whisper_tokenizer.py @@ -15,7 +15,7 @@ import copy import json -from keras_nlp.api_export import keras_nlp_export +from keras_nlp.backend import keras from keras_nlp.models.whisper.whisper_presets import backbone_presets from keras_nlp.tokenizers.byte_pair_tokenizer import BytePairTokenizer from keras_nlp.utils.python_utils import classproperty @@ -28,7 +28,7 @@ def _load_dict(dict_or_path): return dict_or_path -@keras_nlp_export("keras_nlp.models.WhisperTokenizer") +@keras.saving.register_keras_serializable(package="keras_nlp") class WhisperTokenizer(BytePairTokenizer): """Whisper text tokenizer using Byte-Pair Encoding subword segmentation. diff --git a/keras_nlp/models/xlnet/xlnet_backbone.py b/keras_nlp/models/xlnet/xlnet_backbone.py index 0d660bead9..1fe6086436 100644 --- a/keras_nlp/models/xlnet/xlnet_backbone.py +++ b/keras_nlp/models/xlnet/xlnet_backbone.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from keras_nlp.api_export import keras_nlp_export from keras_nlp.backend import keras from keras_nlp.models.backbone import Backbone from keras_nlp.models.xlnet.xlnet_content_and_query_embedding import ( @@ -23,7 +22,7 @@ from keras_nlp.models.xlnet.xlnet_encoder import XLNetSegmentMatrixLayer -@keras_nlp_export("keras_nlp.models.XLNetBackbone") +@keras.saving.register_keras_serializable(package="keras_nlp") class XLNetBackbone(Backbone): """XLNet encoder network. From 0d2fe7b9542ac3849fce259fb9ac135e80698c92 Mon Sep 17 00:00:00 2001 From: Matt Watson <1389937+mattdangerw@users.noreply.github.com> Date: Tue, 20 Feb 2024 20:10:06 -0800 Subject: [PATCH 28/29] Version bump for dev release (#1449) --- keras_nlp/version_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keras_nlp/version_utils.py b/keras_nlp/version_utils.py index 2be29e26fb..b725a00111 100644 --- a/keras_nlp/version_utils.py +++ b/keras_nlp/version_utils.py @@ -15,7 +15,7 @@ from keras_nlp.api_export import keras_nlp_export # Unique source of truth for the version number. -__version__ = "0.8.0.dev0" +__version__ = "0.8.0.dev1" @keras_nlp_export("keras_nlp.version") From cca2050cbb0d6121c2f87c6adf23798d307f31ac Mon Sep 17 00:00:00 2001 From: Matt Watson <1389937+mattdangerw@users.noreply.github.com> Date: Tue, 20 Feb 2024 20:24:23 -0800 Subject: [PATCH 29/29] Version bump to 0.8.0 (#1450) --- keras_nlp/version_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/keras_nlp/version_utils.py b/keras_nlp/version_utils.py index b725a00111..4d6a8186d4 100644 --- a/keras_nlp/version_utils.py +++ b/keras_nlp/version_utils.py @@ -15,7 +15,7 @@ from keras_nlp.api_export import keras_nlp_export # Unique source of truth for the version number. -__version__ = "0.8.0.dev1" +__version__ = "0.8.0" @keras_nlp_export("keras_nlp.version")