From adfb671d8ebf9174feba6461bd04937358fbecbd Mon Sep 17 00:00:00 2001 From: ChengZi Date: Tue, 21 Jun 2022 11:27:15 +0800 Subject: [PATCH] init --- README.md | 121 ++++++++++++++++++++++++++++++++- __init__.py | 20 ++++++ mdmmt.py | 137 ++++++++++++++++++++++++++++++++++++++ requirements.txt | 4 ++ vect_simplified_text.png | Bin 0 -> 7443 bytes vect_simplified_video.png | Bin 0 -> 7748 bytes 6 files changed, 281 insertions(+), 1 deletion(-) create mode 100644 __init__.py create mode 100644 mdmmt.py create mode 100644 requirements.txt create mode 100644 vect_simplified_text.png create mode 100644 vect_simplified_video.png diff --git a/README.md b/README.md index 1a92208..dc22870 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,121 @@ -# mdmmt +# Video-Text Retrieval Embdding with MDMMT + +*author: Chen Zhang* + + +
+ + + +## Description + +This operator extracts features for video or text with [MDMMT: Multidomain Multimodal Transformer for Video Retrieval](https://arxiv.org/pdf/2103.10699.pdf), which can generate embeddings for text and video by jointly training a video encoder and text encoder to maximize the cosine similarity. + + +
+ + +## Code Example + +Load a video embeddings extracted from different upstream expert networks, such as video, RGB, audio. + +Read the text to generate a text embedding. + + *Write the pipeline code*: + +```python +import towhee +import torch + +torch.manual_seed(42) + +# features are embeddings extracted from the upstream models. +features = { + "VIDEO": torch.rand(30, 2048), + "CLIP": torch.rand(30, 512), + "tf_vggish": torch.rand(30, 128), +} + +# features_t is the time series of the features, usually uniformly sampled. +features_t = { + "VIDEO": torch.linspace(1, 30, steps=30), + "CLIP": torch.linspace(1, 30, steps=30), + "tf_vggish": torch.linspace(1, 30, steps=30), +} + +# features_ind is the mask of the features. +features_ind = { + "VIDEO": torch.as_tensor([1] * 25 + [0] * 5), + "CLIP": torch.as_tensor([1] * 25 + [0] * 5), + "tf_vggish": torch.as_tensor([1] * 25 + [0] * 5), +} + +video_input_dict = {"features": features, "features_t": features_t, "features_ind": features_ind} + +towhee.dc([video_input_dict]).video_text_embedding.mdmmt(modality='video', device='cpu').show() + +towhee.dc(['Hello world.']).video_text_embedding.mdmmt(modality='text', device='cpu').show() +``` +![](vect_simplified_video.png) +![](vect_simplified_text.png) + +*Write a same pipeline with explicit inputs/outputs name specifications:* + +
+ + + +## Factory Constructor + +Create the operator via the following factory method + +***mdmmt(modality: str)*** + +**Parameters:** + +​ ***modality:*** *str* + +​ Which modality(*video* or *text*) is used to generate the embedding. + +​ ***weight_path:*** *Optional[str]* + +​ pretrained model weights path. + +​ ***device:*** *Optional[str]* + +​ cpu or cuda. + +​ ***mmtvid_params:*** *Optional[dict]* + +​ mmtvid model params for custom model. + +​ ***mmttxt_params:*** *Optional[dict]* + +​ mmttxt model params for custom model. + + +
+ + + +## Interface + +When video modality, load a video embeddings extracted from different upstream expert networks, such as video, RGB, audio. +When text modality, read the text to generate a text embedding. + + +**Parameters:** + +​ ***data:*** *dict* or *str* + +​ The embedding dict extracted from different upstream expert networks or text, based on specified modality). + + + +**Returns:** *numpy.ndarray* + +​ The data embedding extracted by model. + + + diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..eeededc --- /dev/null +++ b/__init__.py @@ -0,0 +1,20 @@ +# Copyright 2021 Zilliz. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .mdmmt import MDMMT + + +def mdmmt(modality: str, **kwargs): + return MDMMT(modality, **kwargs) + diff --git a/mdmmt.py b/mdmmt.py new file mode 100644 index 0000000..ac03b06 --- /dev/null +++ b/mdmmt.py @@ -0,0 +1,137 @@ +# Copyright 2021 Zilliz. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch + +from typing import Dict, Union +from towhee.models.mdmmt.mmt import MMTVID, MMTTXT +from towhee.operator.base import NNOperator +from towhee import register +from pathlib import Path +from transformers.models.bert.modeling_bert import BertModel as TxtBertModel +from transformers import AutoTokenizer + +import warnings +warnings.filterwarnings('ignore') + + +@register(output_schema=['vec']) +class MDMMT(NNOperator): + """ + MDMMT multi-modal embedding operator + """ + + def __init__(self, modality: str, weight_path: str = None, device: str = None, mmtvid_params: Dict = None, + mmttxt_params: Dict = None): + super().__init__() + self.modality = modality + if weight_path is None: + weight_path = str(Path(__file__).parent / 'mdmmt_3mod.pth') + # print('weight_path is None, use default path: {}'.format(weight_path)) + if device is None: + self.device = "cuda" if torch.cuda.is_available() else "cpu" + else: + self.device = device + self.mmtvid_model = None + self.mmttxt_model = None + state = torch.load(weight_path, map_location='cpu') + if self.modality == 'video': + if mmtvid_params is None: + expert_dims = { + "VIDEO": {"dim": 2048, "idx": 1, "max_tok": 30}, + "CLIP": {"dim": 512, "idx": 2, "max_tok": 30}, + "tf_vggish": {"dim": 128, "idx": 3, "max_tok": 30}, + } + vid_bert_params = { + "vocab_size_or_config_json_file": 10, + "hidden_size": 512, + "num_hidden_layers": 9, + "intermediate_size": 3072, + "hidden_act": "gelu", + "hidden_dropout_prob": 0.2, + "attention_probs_dropout_prob": 0.2, + "max_position_embeddings": 32, + "type_vocab_size": 19, + "initializer_range": 0.02, + "layer_norm_eps": 1e-12, + "num_attention_heads": 8, + } + + class Struct: + def __init__(self, **entries): + self.__dict__.update(entries) + + config = Struct(**vid_bert_params) + self.mmtvid_model = MMTVID( + expert_dims=expert_dims, + same_dim=512, + hidden_size=512, + vid_bert_config=config + ) + else: + self.mmtvid_model = MMTVID(**mmtvid_params) + self.mmtvid_model.load_state_dict(state['vid_state_dict']) + self.mmtvid_model.to(device) + self.mmtvid_model.eval() + elif self.modality == 'text': + if mmttxt_params is None: + txt_bert_params = { + 'hidden_dropout_prob': 0.2, + 'attention_probs_dropout_prob': 0.2, + } + self.mmttxt_model = MMTTXT( + txt_bert=TxtBertModel.from_pretrained('bert-base-cased', **txt_bert_params), + tokenizer=AutoTokenizer.from_pretrained('bert-base-cased'), + max_length=30, + modalities=["CLIP", "tf_vggish", "VIDEO"], + add_special_tokens=True, + add_dot=True, + same_dim=512, + dout_prob=0.2, + ) + else: + self.mmttxt_model = MMTTXT(**mmttxt_params) + self.mmttxt_model.load_state_dict(state['txt_state_dict']) + self.mmttxt_model.to(device) + self.mmttxt_model.eval() + + def __call__(self, data: Union[Dict, str]): + if self.modality == 'video': + vec = self._inference_from_video(**data) # {"features"=..., "features_t"=..., "features_ind"=...} + elif self.modality == 'text': + vec = self._inference_from_text(data) # str + else: + raise ValueError("modality[{}] not implemented.".format(self._modality)) + return vec + + def _inference_from_text(self, text: str): + self.mmttxt_model.eval() + output = self.mmttxt_model([text]) + # self.assertTrue(output.shape == (batch_size, 1024)) + return output.detach().flatten().cpu().numpy() + + def _inference_from_video(self, features, features_t, features_ind): + self.mmtvid_model.eval() + output = self.mmtvid_model( + features=self._preprocess_video_input(features), + features_t=self._preprocess_video_input(features_t), + features_ind=self._preprocess_video_input(features_ind), + features_maxp=None, + ) + return output.detach().flatten().cpu().numpy() + + def _preprocess_video_input(self, data: Dict): + for k, v in data.items(): + data[k] = v.unsqueeze(0).to(self.device) + return data diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..1bf57e6 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +transformers +torch +towhee.models +towhee \ No newline at end of file diff --git a/vect_simplified_text.png b/vect_simplified_text.png new file mode 100644 index 0000000000000000000000000000000000000000..91004b6eff9031d4dfd1214abd02b1472faccba3 GIT binary patch literal 7443 zcmd6M_dlEc7q{NjsI6*`s=Y<*8ntVcnx!QnBt}(LZK9|h^rna+sg-C^wSrnzgql?< zwhBd!ruODZ`@Nq(;d#BDUvgc!a-GkW>wV67zt88KB&$2dj8}NBP*6}Xn%p+9rl9y| zfPAh=OHKZKGP5B?LBTg=VxVgq;<8yVTOMi1-E&6Mve_ps-?I9NUYA8L@s@~zK?$u1 zgW(&AHzi*c1`iH0=4_+~ddmi^0XNKm-D7Owb_1uAd9Sjk@}OQfwQsgJuV})c<4ytl z+AFK)aPZ;T(45bpi_iD~a6WW7*D-t$0l;%X5CFUVKEXdn@|W{riH_4+^w~qy|9%S< z{ra_;W8m*?&`qzO&h*f~H`eag?Y_HQ``6wv+$oF@Gx>Lg3N6`IpN@xrjmJ^hbM;`A zzqUS>W+W#C-aod;dTeR1EHd?v?ZDLN|MTUgIR~(4`0>UwN~Vby=et;)ADz&s!%jhH z!)(y>#`y^eeYOs|=mS||#M`l`Fz=w1g5V+e>7wB9`T0TnN>z&H$wuwY;vW+ZQh8VV z#TjW*pIJ+(`A7EsVyw5iv>mJtZ~ z9)09@_A9uZS5tet9CYR#bh6nnTou?CYV-a0uz(bGaZ)>fx~y8u35oy+DkjaHVl>lx zb3h^>0XPbO5SyET-mUD7ekWH$c70{_V&p&Vi1wlZifcKi5P@ zY=1u8bH`=OTOGl|9+o$f+D@RV^AS5Os8o-&Jwt(r-yKIy`W~OO?gr@`kEDxT4B1YX z&J#l|WZO4m!aqlzCFq%q{kwv%6z=!?^p)4;4@_|jUVF}RUL*SGDvF{D^^=90ip_T zX#^vFV*)}v8|K1%v?G}GR*D7BJhzVLwzC@s&JTNZmTgpYJ%}OOo_9Z>k^8g3^AV;F zNB)CH6Zvz;;|w?e$QB7{!wF<()g~KVG^eoV*Ke^%<>J*V^tC<{ip) zLhGFUwDi+>m+fgRo2EugJo3*0!9&lz@BDn_{S%Y-XzKAJ%V`%;1`%eR)87^Zecto^ zhV}wxL3i#W#;o?$>Pl8>P*`^R=@0oGf*sxk6t7V}f3hU5liQAptu1vI~!ce*GP6`;`-Tig4 zXO5$m{xyD7GaZqAQo#1XWojIBzMXTrC*l+qJWaK8GI!8+bXz!0Mf#G1(MSwoxQr5XQrQUkx~)D*3*X6`pLxvGvdB-Gc&fGK_b!LkjnRNm zm>{5`gA+w=@pVU-C9Mzcz68moSZgdJAu9Ph6OeS@zMH{U#L2-KAMh;NUamGDe!hcV z85Qg7)uhKnMD_?qMQB%=`oXn^DYeN#@24h5u$Xce1A!!e1S!;OM=b-QVZ*iv?dQt>#rpe~Pul0ERgfJiJ?^|6wQ}mAiVSRO=VLaOgCL8d(%=RQg{W11 z!KvTsM{jO1Y5xA$M4FsvYOJbuS}cPU}w1?_?mz>wNJGit9Kv4N@cR z4XftA>fJuEsBOxmTJpc#%Y*AnCj4Vi7}$NyL8hgge5DFj6*}m{F>hE!Km2_CYwKdD zEBGVu6tCJ=twu9EY2WceLg#tKJ2Wh)S13WIjd$I5<8#$P=*;6z#uUd{eM6eBTwdS7ZJaLwhL{d>BRpwxmk@i}!pp^w z;qR)tvK(BR7G`9S=&19fIIBgFGc)n|8*oAJ7hM`(wW^+=1-sFe>a$Vm#gCb94rUXf zqcX^bbk&vx#vMiVi_;xW^d2tt%~_b=^x>voRP$>1VVV`g6urt(e!!4pzN%OqOEa5^Oww)q;gW;)|wt@!up5Zh5#roPii1qxFMEJpb{h)1A`@pU0$QeuQ`YrI@yK zH~OujVY5*@pyW0CoWkWSb)sYs*JxpRbWY1BQ8KYa3cQN`2+8k{mUR6Q`m;1gPCLCI zKwxHUwQ^XQmSjcy`Nhf&O!i0>>yExe@*J*Js*LwoOQV?quz3vcm1bx?+Itz5?i+WH z6!4aA8MWT8@Omj^st6iY15Q&(Ow+om^@)~t1ky}=t(Kso%RthLAgPei%Hd+VI&I@O zjhw1Fz@^g5HGqz>f~fU7mSI zLo&*)6npxt@s2ix70(4f9!1M-yI#6a7n-|R|5!dE-WW$!#Q3kiRGcJ2G|hT(C&ja5 zE{wWJwH>w6@w8bx(^y^~!R=womRzZjXzmyJZi_5_0dUPd$&?@>7{dFaA#F_YSd_(x z=k?O%8?9tM=$iZonreDmt>tLYZwTc3XQgdYb>&B3V*6FtH}9CHmE#)^(A4#W2LXdc zzn~p5!*a1>Y}FP|qKbNgKHu*}+)7g8wF%F@X_5{$n(?1sTaqanOK($3OW7Mlh+2PU9Rd zr|9m5J1Wr1=9dcdx#g0TkQ~Uetn1bm3OQUK+Gv4}d@1*0UxCF85N})!0_hC6R)qe! zp@lO8Dou`vBsM!VAspH3XU(q-z|fKVM7U{E7&q+em~j_NTRux<6RO{6oNZ6+iU!@? zhi}(xB>|8e*r(6fy5CMe$*d6GP}cAH(cxA7AVYmB3f_wK^|Vjuf>OscMjSVwYLaTp zS}8wg&&{%S@L_-WTsGfGOFH6)xWM}%8UpY3-OFV^wK_C z2}!L{sL;pzA;8%*mAKyLUN1b$!uy$-tF>($@m*&Rf@cFIu@_l}l>kppmW-AVbHX~1 zM+V>&I>P61f9IZM^hm|FOq?D?u`@qV>0~4mMB1^!>L;WFlD9@gP&C8Z;g%3hKTVe; ze);WNK8}02u=21z62hZ*HRrWVpk>!y)~eTZc}(2bQ+qFC-a@8=8iD@-Tje`g;iRxw zu%x#3el3Iy@as0K9y^@KRLp9<^H_(yv-`3M!TBasJog52@z%>r97Ot&=K*o;kfWrT z1^|~U%kXs_)K%EO@7s_03RCryxGk+C;EDQ$`z%%I^WgY2-GGSfrmE4`2?ij%`V4Q- z9dvD3VEK3VyYLMIr7o-K)lC9W-#50>ZcqK-R@3cfi7iLTr7G&kyrI-}Fy;%HWXR7Q z^Y0I@fW%{T!gp+neNy}C&e&|E^uOzl`F*)pBb4I*O8xF#VlNK=0t`Rh>QViXvLcd# z4Q@Y~O$i9=PyhJ}DgGnrFnLs4L^B)mU4vL8EsIYkVlI~n$cGav}s&q7pOsoE)Pt;I!-vx#k+KPn)VHC%3 z4$&x>Pl}etWoH6Q|A1fFX^zKz2T9~)hhdEPR9SV5`W>LOM@P+->1#BP7&3x|ZK#0% z*dAnVN~#!bs@$12&dZuN`KSksG)BfHQZ9d$V_R#W&1Jc(U}IU!=#IHXh?&g--k3UT z{OH7@_$he8B=jL`=59YEGhgy<2=163Fs4Elfpjn4Y6Q_-_g7Wp+{SyaLKSCv7$3Rn zn6}>%b>e2Y%QoO|Fd6nyBmPYnN9)m>q8yyU)mKhS^E^*cAxAHDm^H%%=yYBSts$+8 zD7`ESI$wa(=K2DLwSx6~hLnsecCQSPWZON>w?i9dp7a+HA;07GI9WvQE_GM7xy=CS zI-jdXhj)&?RTSDX$F@ueU&U(p5coP-52I30r(x&>qCgfcz^dmNS1O8=d%E$m5VhDN z%AO|wstq+a5E)i_fk~aGSM-f^^kL=eJi<%eNV1->6D-W3sa#M%m*Nhu)P=|7xZM?~ zZ)7%11%G9EY8Sp+Bs&@;RKd<{;l=}3l{<`;jLDr8n7tY(G~DId{6(;O_AUHdkm1*~ zTL5Z4zz{{Nl1~rLgU!i*D{y0RTjm)!x*~04VAP5stgiVAsjvAHgJ_(Lk-sT&m_^?g z!hLp(#1$$Fo4(9N0h%9sTIurGq(47yDtuqVBilNSOO_w$11DyH(~5tMqc9rK0sVCeIUWro*TnyK?+=>oF;Z*=O2nZ@;{<@J$RKEFs}~dtx+=UHSXIgs7c5 zrUy?xi>5?)yK%--?Qo3=dG-}GoW>_^g_{dQk*3K76r(5pW_+xW;q3qx0H$I0J|qdy z^K@KPfkFd%`_|r{Y+nKhe{*}f7~}%!yi&JksR`rWa|Be#UUGo+t{M|7`u7MRbo*S_ z7;aX7=ITvxUu@Y#d-eisPrHYqo3Hb4WL8d$_exO#ZxU`_UntOj(OnW|!oN(acA06! z%J@>_yqj6*j%4#`Q>S=d%zm=%Kpf=%PSWJB>M}QcAR?#hG;M#U7UZa~(^4EFU77(F zm?Al?)EuV6GIt)9R!?V+e<||)%H`w>VbDJdOR}C$6jJQU%t_3n>WYh&MLMFb1;{&S z{w#=etdMC;E|ZDs^JqC%!W#Sw^i?QqAs&Ko(Nz|9vyjlsGcSj1H0Qpw3B9~ zJXkD>9Z;mI<>0?vwwpcAwKUmpQ|r%@5?;79M#Wcv|FRbir+58gh7dRE%}x|vbb3&; z`X~OnR4K=Ut~af{E$8TQA9hMzr5n#{*$wuKG%{EZfBxAP!xb5qWi{;m;|2z|>GJ#g zd*yqQ0D39_sx8m0@KsEl*5 z4J1+`=!kFP5s#I`Z2Hr;-Aac10IiNUf;h~1>8McJSfG^e0D5H4KT0%&x-UTDIo6Op zw!VRWLD@$u_O>6~22Ily86^5aaio(W#5y!ymO1&k8X*DshYoEnY&cGms*QBQNJN!5 zWipC(F{%#;)=p|&wO<$pkye==%c{2LtvKacUaDG`HsNM&&I}Kw02Z1nTDX+x`|SOp z!@p;T*gFuZ?kA&P8Tf!6Bf=HYF6rR$WB$H+C9th|q=`bNH&guKSEkl4w`&_KgN3Zn zZ&IG$qG*r%D3Di&5s`bvw;1PJpE~!a1nRQlz}d^PY7Ibw#lC_gJ->j8OM`>W6-K$g z3DgMi%x^vRLG|3Wg!^9*GJ2D%GG0=+_B}v%J6JIlvO0QQI3)V(RmC?#FMA!5H{ddG z?D!?<-*(H$Xb118V)cgTi}t-rKiMJ4Q0t^j+u2lkmkIC4?5&Nxf9eqM=V|u{cY$O! z@U6NsRrL;AN4zPw&;3F59u%(sV;*wftU^zL(zh(TF7Gm7a6Y+m%%x^*()v{4Ci`a} z0Z2pmLx+62sk8=Dxjf03BLwiAliKz^iKVl6$rUYygdh#PV63PVl(U5b!R1|{uewQ? zXutz^gu*~q++52;N{rmG&16jo2wflaj;tlcSP;pV_m$zT$2stF@%T_r-BlLZ)@Ddd z?x`&cf+_@fklPA9tlR|tMdLR!Z07lb#sgYEI3aOrMhkTL zoQ!RnN*~WtkNM;sJ^z{(@tb<1p^}>MG4a2JT4K zU`a_vTJkO{uOk|zSJ zFGz;AyI5OgYeQ(rieXcN`v1CSz-lePpaD*w$4>KV@{()y0r2pmLpDCv+jV(cbsXHpDs@Mu2yuCdV~zM@!$3%SDrfI0+Yc|D|Bo575YSS(v}_2E=xSx^N}Mm-nb?spf~{QcdNz*2Yd-IOXO&GIqw7fx-|( zxSTE9wYIv~QXFD71OK8RqH{PJRm%o(UK^a-Z7v%^% z^;0gES1L8;#!Z+`r)QVYpkY^z`#l1Xoa z>8j1@>a-2pE*??rTXFGW1an(eE4+9hNOXP3=f`ZWB^1A!kI$wwCl{Vt_a$66%svH9 ze$V>uNIt8!WOc*fxiZ*hI9=X(%sTwhs01*`T@QrENBQ%OCX`fD!)~tcJTgqVc`d#1 zii8742wNX;Nmj74X2+-685el7Z~&?6+ViSxa|diFt2puuV7Z;L#@-25tU zE&msP^ISGXsFZo-;+6swD`fk6_jq%YqRQphdV`cp(5swGFm;d-F)XPZE}D4~xTblK znm9yHg(bfSdHU2Ml@Z4T@ZF35GZm{12b*dh?3HRYg6_N*i#`cydalkwnHG$$+`_-p z4HbR8@>11lmF#c_x&MvG5R;q#-;LVR*#0*W#8*iFHygw&z5j30hl&1gEC^tf`R@(9 beuIJ*HDm#0>uuy=CxwaO9fNmz;F$jd8yD3U literal 0 HcmV?d00001 diff --git a/vect_simplified_video.png b/vect_simplified_video.png new file mode 100644 index 0000000000000000000000000000000000000000..056d876b9f4872e2d6e217e3f811a00f9d1f78fc GIT binary patch literal 7748 zcmdT}_g@ozvsFPsI;enDrAU{qbOb_^8hWoHNQZzBl7JNHMU-BGfE1}BEp$1wNzlQ59nxN(DAT@9djM3+fFBX}&oRj;vb($J zpsTa3SN~pwlUEV{e!Kpklft+Ef5sSfx*0l4pP|jU*oW{PP33fmj|Uemxo2A_#g+D#D0#FzcLLf5`=ANlvV~^Y?Zd1uL|OlOjUZjVTHr@ z+b)kP$LBYQqV@TU4Kpw_PUdOft&MS3+0$NEH^ZXzJZ^AP zKj(DxbPjZm$H(N%<6PpIxaw0K?Q|%V*8#Ji72xR_g+}7)hP{>^Q-XU^Imo>N7t?KChO(a9$kp7;!PZv71cI8C>#B`5_mA7IV6(e>yLp^z zP>nD@%N?ZCSN^CdcC$1Ja|#It?VElHytNm0GlO!vk&zDpfZ?8l8;mgq)zezPOhZ25z9 zqzwMX10I2(aa=vzP4gtW|M3nuEV@Mt_#2G}>mu<06tHwJCrxnr)=dG(`A|Y&5~auI z;J}k#tbrUs@CVt|v>JU811rldVg6ADFqh8b`oV;+4sAsrn`U?1iFoE$Q>|MYB5*;0 zxIXKOt+3-VIvWMztt=kiI#oxEEhWU{VCJ|PC5t;GKN%$zY~JgUYiLuJm3F#Bpu(5J{71H#5~ zFF014LA~M7{Z2X$W~8q+IEql!sTmE*IddJ%jX&`9d5dl5lG~3ehkiZa=Z3H~D09l^ zW#J}Ks1roDtos&2@dPL)(eY}mGSHU~eV&alzW>J%;x265I*(5d)mBJF+W}HOXp6y> zUwR2j-pZpi$_^iS2}Sl?!(GyHRJ?XPi41(Zm1QKgRaT90Fc(rR0YO4C>v-1GFC5Lk z=%<*PwbnPU3aG{()1WoJESdAQw%N56Wn8al7t42u;bWPHCZ6?)6R=J`t3=B3_BQI< zYgs()mQ^kDm)LsMLnC*H)z#^^hb4i}9xJX#)jd14(xv6#p%i^%{M*9bVilI}%|rcW zYM$~aN#x|IjFdV9UwA}tWS1y>KX#xBC5r}|j~^AUsE$_nl9OxtQ~>4+Pw;$JFPS<@ zLkFx9zxjOwYvdFal0}Vu@Q`yxRg1wp8Q}vY!5laF2&A zGtN&~5D?Qd|AJRve2T17rEoyz8*IBpI|qs-K`$5Z464S=%Oj{Td$|`OZ_%csDXu&B zC1qi)p`<#LTvR|kTZZiiu{Coa`%{g+q7B8C!wsFF0DU9Zr{72GjbK5YfeEC5gCm&E z9oiYyMP^_&z+4sTy{+XDFtTZN+&KR4r;|O8s=%$Q$~Ol3Rl%U@%^t1sH!9-ySw@V! z+*n{N=~jwKS)gid)1UwSdWVG1t+kWcgsF$9L~l8AyfpePNJ1A*C+bC)J1+%=_8`~B*w)4$KNxsRG5H4B>rHQ-5HAc z#@AsbqO{*cl|Ll5kZuyxkIu}`U}Z}|dJ=-I7~%nOj`BHH z{8R3oa8dTvUtWBIbKPX*q4A{$?M1Oj4RW;C=wf%*&X^ z^Zv;ZYVufzdPlA(W#M&eQGDCkEP60ve0=?c&l=ZTZl8=f`msjUwlW?)C3sYBIB#7w zzZ`P=c+s}^{!!w*Wy^u2*4^ELI=Ty=z~?vj%a%#pZ!#0W=4gIzu#8Z$0n}JQpu@;rd|9po=4%=RNX`l~lt$O9?o0ahL>fU-LdU zfYfg>g!HKiN@=!!EsZ8>yk!o*7E!-ci&wfoB+I#w#|5(?hSClH!H%;SvXtgaCKk@0 zL4B>FNrEarOogTtf$n1usP!yI?nyw6v+vC_`+i4CqOv@(ViDi}mC)4l^>=%cijGOo zTleF(22=H7y=w{mn4-3EFMKJgsfv>-yC)rUS_WfeK$5(zwVP*rScpY2vHRc9A>8M= zDtVe`zle>k-Z9^Pj$YNUJ{>OKT8DN6zZ2)biQ)kGKPJQ7{7gp@) zT0c_w7uoF)`-ywt*w4>Ci0eO45y$XQ&j3~oZ2NQF+dZoWZGp@8$IkC__GjPpnsi8Y zt9T>8Fv9f0N+!0Ri(&n@@V2J&>MxBiXV7&e)+y+qZ$LxT6V;Q!fV=(=&0|w2M>Ai= zKR}6yX%O@&vPdzO&(dD|t@1YgZ5T;jB*UPCuxXgG5p>Y6FGdWJv&km?y}-$xA*48B zEkI;SSd~+rp|1g*-w?y`_EGT^=+)t5_ZtQNSZawEyNT=>y0xetC!EP%VROL2n1dsL z|8YFvkTzw}#8*4ZD6b*9|H_`8l`-fSCDXpUM#ukv+c z^oP8AnT0Tas@DtXV{N8#&L?6DdT(sCJ&|O!nBQHU=dW)OGtA zOAnLSbiRqH$}c&Vyd&Xn{grO6l0+w9_|so7_Lc<*uc)o5B(bmm=5ZpHg}huGo&oX) zV$?;(&OI$87Un)Qher#4hf*9TA%F(k6kJ(2aiSWgka)1Lp-d(sx*qwT@1 zzb!5AKf7gBFPWok`!GCGf;PWiKkxCUI2x{zsT`)_CVzKLS%s<^C>cQ~^r6bnKM&M_ zc{jJI>FRk_mEW*cYR5;d6KJ~_8+4a9v?-FVPw2VEzYO#eb`Xr-yY_Y<3UYJ>ws~c; z8`D{=6Zk9=Wt61IUxnga%?t?`)_t@(c3~Gf&Jg$6p6Y$V+VVZ+sQF+S9vut%v_1g} z+y7!~I%C5cNi#5Je2Q1REbr;p+MeAM%YD|D7^r`#lFCQjNn>2i-p5gfonTAo7R`KH z%moWk3W>Ycb7$Nx;*T-A{eJp#5r_*e{z$6g20^watgF~Ug>=_0y?ViY$C7zR<&r3u zD@f&=AQ3K&B&759RCBMlPFkc@-&cL`cqCbFG{IA@@?G#&B-)b!KN(udQRkN-W#@ZTzxtoP%}M` z6FG|MgDcBz@J2Nf+vzm`u{UgBxIb+0Drgj*no>#h>;YO`R@HM`stdlFv8h1FFYT-> zV*e$@@~vHH_hANi*OFZpUhwc`5?=7EV+UU|svy)P_Q)W(pIHm9M<1ymzB!(Y$BEf- zOczMSopD=R2%U}=$%(Vh=^mBiX8U>0E=w+0=u${_5d>5rJS3vf7@ruqgpf>ayKuGd z@qK?Mq9Pmtz4}xBfxHAa?^54>-u|98NuBwrmVM-wp;;Zp)P=^eJ)gX#=tn!rG59L6 zcZ_6v3*~go2&4Vapl(M-0><+k)XSADltL4g^!FJ^7~=I3UYGslWcu)x5od_B%HG6M zam3(n>gN7djD0%*Y69rSWPL`wXQ1TyZXjlp6q$Ll%;ZG@6uV_b!Nwr=MC?WxaeR2u z;+!*%3QShLhUOFNjvCE)LY4d2jwQ9Xy>0(}lu{lbhDVy5%Px|oJf2#3{(-q-{|(^N~=I&kSJMWILvi4ioUi`wjWmoO>B zqGJkU64RX@PJ?BcIIxI{^~9NqEc<=p>J@IU#&tg#d!MKIjrJ$;#5!8 zLb(5xQM7!?+EgtI5y*7;NXZ?)L>D(pOJz`Tpd-eGpEKlvI?dpLqIo*k#~T zSCLRBK~SHlKIFyb-QBO&?gqN=ppRNzpp@tC+CYSzj*N>`Y(CFZl4SD5N>tjtnX?+% znp**)t@XfaQ5+2+k6=P)OxZMGah7bQ$AK#l{=JT!c_|OIxCmqYU;i& zAP2;0#JwDRwv@Zooz3nj+R@#|W|{$SHX&D{42n`%tM@S?yC z-7{fhyfcQOxRz3EH0QU3*$}~ZWYNYoPf0FyX__u|$=D zCxLG&YOf2$8sy;PZvLVUFt*Q)hzc*A3;4=~fV6p&kcbq^?4uLKR?pTVNe|4;F%D~l z$HW?L34XfM_qsaUoe2u(9LuNaTZ}CL7gOV{&sgCO2JH0$g2EZ5F)0i2l#JcOY!sC% zkK6j6;vTy%!9M}Mm{Dl2HxCh#Rf~kB8)vF6-8nSdFOzg`>6G})Qc3H!a)vAbE_qTi zj*0@1er!i|jY|z`+fEua+&>{Lw&6Z_uI58fbNFE1+E6EZI9t^fuh60D*)+{MxJB~` z3?YB#^Y2DxZJ@1iOBL*5(T0%0;~b4dgk82&Vg3hp6a77HRY(k**DHnMKQ%m2WlB?` z)UONGYd(he@L`{IwjRVdHFl5QF9)^@MA|Hu=IMO&SAHj8TLRtBUcXSB=bo%r{11FU z>X{I3dG;5CN*J-czl|ZZ0`1K86~E9zvq?jgg~~QtQ@}4=2=jaoMTI+eyCv*?UKP<~ zmo+4=Nsbz6#Z5%Rh<9BfGL%!)o7JjW-UUO38?|5GnjAC0Q0b@iTY)_J3=MAJD10?o zL-@QWcLnUCNSP)!tY(P$C8=5^D}K&ySL4c|a9_NoOLmZTgsPBM*Qu$RHv0Q|Q9j?Q z(hK1-@{?{;aDp zO$>*$8j79QaN@ndbu!vho}|mvXW*IvZiYo9SOY>oi(kV5g8(JTzxQVJk&5hr4B657 zw6d%bEutiOe!)BsG~Y04RS^_Dd;RKSIBTI|0xrd*pWP-=!qr|rGy+pWn3Cq6@y{+2 zs)&0}V`%czYuE4YN+3lk^}EL!=`m^?r5{;a0G6MEQV>KVnmv`thxQEgNphhsl`U^U zIddUe%C>45d|#~?9NfvPy1_uRL~|ZirIX$wy7H)luQewckk44F33_D%9eI^-|8%ns z$v9s7_Y&Qe;vVw5B_iJuc^NruK#9n6rhK=B8bBwU56QBU8)XXsU#xmLTLm45%--RPZ8VBcRhqfO(O zp?vOyo1YN{r8Z`n@zt}-#9m(hrTo|CwQiO3-|B6pNKkDehgD~hGnek+@p$a?#Eq_Y zC9cl*l*#8DW=<<_!ix0zcpL*r+Yrxo?SACZt#$UH7tgz!5W7R{ z76s^)$Npiz40}Ef%o%-7@z|X)Ih#kxqLGfTLRq}cpKQMbT+uvHH8;tuV6Y92{_0qn zv%lqXKnn9oKQ+Ex%4M9`@@0)@kDT@XdJg{?P*hW)K>KmoY>(Cz`{K48mc@ z%?daaINh`CD=L=EbBK@jC_lN3zPL_$q*Wn?ojccl@-l_kUANKU)YC+>9JexukzsX^Fd57J1Jq#e7^=I35h8Z++6)}KaE6U9&lB%gMrE>{fFG*l#G ziCg1v1aykGgT}52=iTEPR}3l9<10|iy25racDIxC0V?$S3qW__ zn!5w$CDy2yw8ai7UFlu1LdX92WkbA~Qn$u7IR2|6<*BJv0YV}XuuMB2Tq1kCo-C+x zilUK!An4zg&kY1G&6oF>v|4Y$`bMaDv1IzKu5|dWHJ=Wryp#(r!-8w3^7*Q2Eb%VF z>a(F7oK>HattlzNPa9Q}x0xR8@je<|IpHx@kKK0Qm6tk{O7ROa7~;7`OmnHB^E6g299cEak4WagE;)i*g^tp8ME;6KNzJMf$-1K^Gk>zJ5k5C? z6?_}3mX$NjBIO1?=2!8zn@i%G=R-deaey#n4>ETE|Ce#nXl4#y7AIK%Il^s01h@BX zUlG-Ev>|qe)us$R30nFlTw?BaGehh{=9pv;UBfuoeD}E=P-%Ru6kH4iG=da-M`NK0 z3y)50KX0q~n3=KrS|YB)btotkFNvSTk8#M~F0RuIBi96HBxGQ)TJi1y2-3EmalDU5btZ09|!7Vv;KsH@w^P>21;K@ucn7RSn*~!A%xWTM4hN3@3uGjwn`p=@mK!NRo}83A`)_kAOyMwMk*mCe<)&BUkM=*{ssKUt@Wvks`gGeP zGRAa((aE1xw|rGDZ2QirB$Cfh0s#ueMRP1a1SsAhp@Q&>Q9%ur5-gP4|Jj6SJWQP< zy#X~!fR!;rY3!sk-}1zA#a#E4SGvUk;8Y9!+ljG3UxZn*1%rAphEP5O=#}2tY-jy$ zX!HWDrs+!hmgBhU#Xfh9qs(&*a>!;o_6-{*73irkvFMlfQPBXb?vp`5@bzgG6M6@66RQwZepcwzDn1+wLU(Cnmj|6v!kh!)He-%yr&7BZbCnRXQ^ z@MeqOez(WzG0hkSa!TI<2> z4cJ*VJ*zh}9xDu62zI2!hM&>1&G{YvOw+&lGi31(;?ZE55~u{mH)QS;j0M#%$nl%m z435@Ii%tpfUQ%ex1T{ThbqE8-L513xKfNUrdVQba7TY6`(qY#O@W;N3hpD~xgQBVW zHRWL1nTXr0U%-D_T9(jcPQCUk_;HJQkjKuc9^Oyp__Pn)Io!@+9O=0QxSmwbG}v;> zZ-zE64=|Ij@mHT@qG{@BWCs7>ike;N_u6iW3QXwad=_%ydu#j8zuo?sV)kbz0^?lU p|NWsE{C^{s;rP$YRapFzRJcq+?0qwg96uetp{}eAs8F;C`5!IjjZ**s literal 0 HcmV?d00001