From e1066ae460ad6684ac929cc64f585a8e95b2213d Mon Sep 17 00:00:00 2001 From: konstantopoulos-tetractys Date: Wed, 24 Jul 2024 12:42:21 +0100 Subject: [PATCH 01/28] Update .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index e4d6b4df..809f3dee 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,5 @@ obj/table.c.o /regr_smlp/data/smlp_s2_tx.csv /regr_smlp/master/Test130_*.* /regr_smlp/master/Test131_*.* +/result +/data From 68bb40f88d9a86741a8f3b93864c898507831bc1 Mon Sep 17 00:00:00 2001 From: konstantopoulos-tetractys Date: Wed, 24 Jul 2024 12:44:18 +0100 Subject: [PATCH 02/28] Update .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 809f3dee..4dcb2653 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,5 @@ obj/table.c.o /regr_smlp/master/Test131_*.* /result /data +.idea +src/logs.log From 8c3fc2c06d10751a40e74414fcff5d12f829db7c Mon Sep 17 00:00:00 2001 From: konstantopoulos-tetractys Date: Wed, 24 Jul 2024 13:18:03 +0100 Subject: [PATCH 03/28] pysmt files --- src/smlp_py/NN_verifiers/compare_models.py | 75 +++ src/smlp_py/NN_verifiers/fingerprint.pb | 1 + src/smlp_py/NN_verifiers/model.pb | Bin 0 -> 5201 bytes .../NN_verifiers/reconstructed_model.h5 | Bin 0 -> 18024 bytes src/smlp_py/NN_verifiers/saved_model.pb | Bin 0 -> 93992 bytes .../NN_verifiers/saved_model/fingerprint.pb | 1 + .../saved_model/keras_metadata.pb | 7 + .../NN_verifiers/saved_model/saved_model.pb | Bin 0 -> 93990 bytes .../variables/variables.data-00000-of-00001 | Bin 0 -> 6806 bytes .../saved_model/variables/variables.index | Bin 0 -> 1779 bytes src/smlp_py/NN_verifiers/smlp_toy.onnx | Bin 0 -> 1777 bytes src/smlp_py/NN_verifiers/test.onnx | Bin 0 -> 1777 bytes src/smlp_py/NN_verifiers/test_marabou.py | 134 +++++ .../variables/variables.data-00000-of-00001 | Bin 0 -> 6806 bytes .../NN_verifiers/variables/variables.index | Bin 0 -> 1779 bytes src/smlp_py/NN_verifiers/verifiers.py | 502 ++++++++++++++++++ src/smlp_py/marabou/fake_marabou.py | 36 ++ src/smlp_py/marabou/marabou.py | 141 +++++ src/smlp_py/marabou/query.txt | 51 ++ src/smlp_py/marabou/res.log | 167 ++++++ src/smlp_py/smtlib/__init__.py | 0 src/smlp_py/smtlib/parser.py | 96 ++++ src/smlp_py/smtlib/smt_to_pysmt.py | 190 +++++++ src/smlp_py/smtlib/text_to_sympy.py | 491 +++++++++++++++++ src/smlp_py/vnnlib/__init__.py | 0 src/smlp_py/vnnlib/vnnlib_parser.py | 356 +++++++++++++ 26 files changed, 2248 insertions(+) create mode 100755 src/smlp_py/NN_verifiers/compare_models.py create mode 100755 src/smlp_py/NN_verifiers/fingerprint.pb create mode 100755 src/smlp_py/NN_verifiers/model.pb create mode 100755 src/smlp_py/NN_verifiers/reconstructed_model.h5 create mode 100755 src/smlp_py/NN_verifiers/saved_model.pb create mode 100755 src/smlp_py/NN_verifiers/saved_model/fingerprint.pb create mode 100755 src/smlp_py/NN_verifiers/saved_model/keras_metadata.pb create mode 100755 src/smlp_py/NN_verifiers/saved_model/saved_model.pb create mode 100755 src/smlp_py/NN_verifiers/saved_model/variables/variables.data-00000-of-00001 create mode 100755 src/smlp_py/NN_verifiers/saved_model/variables/variables.index create mode 100755 src/smlp_py/NN_verifiers/smlp_toy.onnx create mode 100755 src/smlp_py/NN_verifiers/test.onnx create mode 100755 src/smlp_py/NN_verifiers/test_marabou.py create mode 100755 src/smlp_py/NN_verifiers/variables/variables.data-00000-of-00001 create mode 100755 src/smlp_py/NN_verifiers/variables/variables.index create mode 100755 src/smlp_py/NN_verifiers/verifiers.py create mode 100755 src/smlp_py/marabou/fake_marabou.py create mode 100755 src/smlp_py/marabou/marabou.py create mode 100755 src/smlp_py/marabou/query.txt create mode 100755 src/smlp_py/marabou/res.log create mode 100755 src/smlp_py/smtlib/__init__.py create mode 100755 src/smlp_py/smtlib/parser.py create mode 100755 src/smlp_py/smtlib/smt_to_pysmt.py create mode 100755 src/smlp_py/smtlib/text_to_sympy.py create mode 100755 src/smlp_py/vnnlib/__init__.py create mode 100755 src/smlp_py/vnnlib/vnnlib_parser.py diff --git a/src/smlp_py/NN_verifiers/compare_models.py b/src/smlp_py/NN_verifiers/compare_models.py new file mode 100755 index 00000000..b9187c82 --- /dev/null +++ b/src/smlp_py/NN_verifiers/compare_models.py @@ -0,0 +1,75 @@ +import tensorflow as tf +import numpy as np +import h5py + + +def read_h5_weights(h5_file_path): + with h5py.File(h5_file_path, 'r') as f: + weights = {} + for layer_name in f.keys(): + layer = f[layer_name] + for weight_name in layer.keys(): + weights[layer_name + '/' + weight_name] = np.array(layer[weight_name]) + return weights + + +def verify_pb_file(pb_file_path): + # Verify the protobuf file is valid + try: + with tf.io.gfile.GFile(pb_file_path, "rb") as f: + graph_def = tf.compat.v1.GraphDef() + graph_def.ParseFromString(f.read()) + return True + except tf.errors.InvalidArgumentError as e: + print(f"Error verifying the PB file: {e}") + return False + + +def read_pb_weights(pb_file_path): + # Verify the file first + if not verify_pb_file(pb_file_path): + raise ValueError(f"The file at {pb_file_path} is not a valid TensorFlow protobuf file.") + + # Load the protobuf graph + graph_def = tf.compat.v1.GraphDef() + with tf.io.gfile.GFile(pb_file_path, "rb") as f: + graph_def.ParseFromString(f.read()) + + # Import the graph and get the weights + with tf.compat.v1.Graph().as_default() as graph: + tf.import_graph_def(graph_def, name="") + + weights = {} + with tf.compat.v1.Session(graph=graph) as sess: + for op in graph.get_operations(): + if op.type == "Const": + weights[op.name] = sess.run(op.outputs[0]) + return weights + + +def compare_weights(h5_weights, pb_weights): + for key in h5_weights: + if key in pb_weights: + if np.allclose(h5_weights[key], pb_weights[key]): + print(f"Weights for {key} match.") + else: + print(f"Weights for {key} do not match.") + else: + print(f"Weight {key} not found in PB model.") + + for key in pb_weights: + if key not in h5_weights: + print(f"Weight {key} not found in H5 model.") + + +def check_weights(h5_file_path, pb_file_path): + h5_weights = read_h5_weights(h5_file_path) + pb_weights = read_pb_weights(pb_file_path) + compare_weights(h5_weights, pb_weights) + + +# Example usage: +# check_weights('path_to_model.h5', 'path_to_model.pb') + +# Example usage: +check_weights("/home/ntinouldinho/Desktop/smlp/result/abc_smlp_toy_basic_nn_keras_model_complete.h5", "/home/ntinouldinho/Desktop/smlp/src/smlp_py/NN_verifiers/saved_model.pb") diff --git a/src/smlp_py/NN_verifiers/fingerprint.pb b/src/smlp_py/NN_verifiers/fingerprint.pb new file mode 100755 index 00000000..b6744820 --- /dev/null +++ b/src/smlp_py/NN_verifiers/fingerprint.pb @@ -0,0 +1 @@ +Д=ߩ\ߕn ̫㝚H(2 \ No newline at end of file diff --git a/src/smlp_py/NN_verifiers/model.pb b/src/smlp_py/NN_verifiers/model.pb new file mode 100755 index 0000000000000000000000000000000000000000..948b921d188fed21e03d1dd396fd1676a8ebfde8 GIT binary patch literal 5201 zcmeHLeQXnD9Nyb?TYbySc4KOe%`6kPdC1yz8?d>)$2u6y2wR43hTzJ(_1LP%UFlsl z^CK+Lm_>~kiGP3yVW>d-NQ}WqvR+Xa4S(oJM(_icNMJF+ZzYS2_;DS*yLR^Ey>;;~ zoAmy;YoGV|J-^@cymtrJL1%yyB+efcLJ^sAcJR258xZ*b7iLOejx>Nn997gA98Ijfs(iRt zLdQ-GD-E|^RHpJ?M2GLmjcqP;v;B=s?2BKD*!;(qu+clKn&>CT(1p1QI`5iDH|%l0It!1lZ7)8W>y?kn67t=(0EmQ zQ{e=N)pdN2I?rT{`p&TE!mlTlJ%0?L-UQdL+8Q@4py+SgTJBD2V%<9 z<6GFOt4}wb-}bdqed#RfWbS9*8(NMkcNMTZ?>MfsFO`(yqxrGB4|!2na6mcm(VuAe zseaVky$=oVb)wr!r@XM$;O|Mxf0?FIiH}aGWG^omoA|-I#D2&dSLFR483eZ&IJFv1 zgZ5XquR3WUY~^L$(tZVvKZh!x{RvH99&FsU;j$9SuVlw(+M?f_{0gOtj125L4ZBps zUhJb1v~1;$qem`zpZVmPH}(JP;j-3<(C6ALW0~6?;X80x4$47M-~ug}=UubTaH)gg zXj84CUK*s6J;UTdaNn#6TF8KEl7KYR7$746A~Pluw=)uFfO*%fXt9<+=7y5L!I{?B6bicVL;yrR3V6H555jyvjveB#{6|gv835O0iee^Ps9G!CxOmfmm5ZaOuiwh zMoJ2Sd1#?)PP%KlNuE+ zXb`MS*GO`>{t zY>yx5H9clO`p5li`UcS~IYQdoblOJ@pgO{YM6(r)jA@uPg|o!i7JZD|frD&s3%hqXIabA)a`)Q8t zzW3ZUD^VkXkOv4=nvnJZLmLug8lWLGv`LLqpn~on8mZF=ut}S?3N*gThpMg(7@_hv#8R~hj_(y%N}ae z_z+e+SZ1)Cu9FoHTh3st75kf!zp_kr4C2Jk?(6SooRauciPLfn8gdKS1%o*dQVkwqa&pIC%nv<-saur~6ro*1pjM{Hc=15o7U zY7%@T9wW)bm=Oj3>-fz;rgCy6SMWjP2U|4HuUYW3P0dIciD(2_R^s;pEDrhA^J+PJ z;Dh_}>j_z~O7`IR$0VVG^lnjG2srIQ2$w%EHdY>sQ9WVlyeZ0vAIku^p8J;6itpE3bo2 z+&8c_#P+~8EOB8QXgz!;a3Z@%a?Wqyo&3tkqUiUoJNMo#BmjfJFIqpr7-;tg+kK3Y znU#S8V>ZV%X^zE{kH~o>zpx*?>!^1mu345qz}uyG_v?q!dcroexOa=2}C|Dco6oc(dXqt zYxed4Ot(^+X#wGw7PoZXVQSH+KDwZT#FP3XxO*ojw{XD=*VhMn=8&jhE{qA|sSU^B zj<%VG8{!J0DZ3fX(5hRII#8wqmS59nr6iYNRUtYq9}U%>ZNXYg*+uk1h$4f5h*!}Jh>`c;dyba?g$(A9~vD}j6jiM#Epcm32W<`73r+Vlr0gn za&U5zpWEdn;%(u7<4Ftbzjs{M5)tcAS_50Gn`Tlx%5mK`jS=Bti(KqIe8xRu{D7&Y zQbnGY{Mxwmc%@v}m~clWZaq195X_J~pmBlna-i)5vFZGlx2VaKZH$B6#&ZXle1`|V zV^nkcBodT{;#`l#;l};SFj`4*%92~-_E5Q)km}mInGJI9XRPJ%ZLl#?AN#YI(UyZZbeNk7h+8EX9t&v z_jrlNpj@0vaadu$KSxLND9*(Yk(GCSQjFMd3PRbKI@hYF!T|5uFFs}>GCZC5bmv3I2 zpLl%ukRb-BlwVaoA)hJfh5LkS3jt?-mEyJ#=G_SmiaW_(4A4N8rL;%&IeT;(17G5Q z5%aFCC>aDxCH^q7UkoZ>r&X-2DbJX;&~nS7%*N}CmA-N z#aZS#xlxi#Ate<@sB%q8saJJ5yLfWT;$jdkG4JAZwKm#4_cQhU%de{)moBJp4!)v( z?y2voJ#XF|{@Qc@$kxwnn!WU$s5;+xn|ikI{p?SkwAANc&15^*zLPz-;<)aRD3n;yL>JoBlx?)8(W)VEJO*Yi;5SaziO>)jXnR?XCJTNB=M>V@nN zv=6eznrGClzj$7K{CBIv-%JJ6jUQdnv+S#HoOtM!PW2DJY*%;BeSYTq>({98EbjZ^ zjp~`Uz1=NmJ+r~VKdSp)V<&dK7FOT>+sW+X|C~}^s(Us2bknXdyZBD$)NjqKX51Q{ z{np>r4foGx&mDar-1hz#yWbf4O}72r*VTQS?+yQG>Na)LvF7lhTVBfU-DqXkOgGKm z{%ntWpApMG^V5H+nJ+}u;R8>pnI~7OtsBnuoI3lbo}22Q?EzVS{O8T;0qYxC^~_h) zO<(?tdf~l^uAzbVvZDoZh=Q1^PEgTwY@U literal 0 HcmV?d00001 diff --git a/src/smlp_py/NN_verifiers/saved_model.pb b/src/smlp_py/NN_verifiers/saved_model.pb new file mode 100755 index 0000000000000000000000000000000000000000..88b09eca418034586178669a018beb74b183ab73 GIT binary patch literal 93992 zcmeHwYj7ONc^GD~zz!N9NH$-~;TwD$xjGWNgT-qtpN@cdcf3131o9~9By+L^mb^Xg z1KM4HN7;@aij|Zqu8M3)v6YGwGw~yerjV!%19rDlRAG zDkrIw^YzU1Oi%aMGqVd2N6I{vxPzJQ{@!1I{q@T<=#$?aA%A<4{?RUSm+ZOR?baW( z?o>PV>b*wo#yXuKW~bI|Z**2``TcZ~jBHjL8?~vCwcg|P+SD%MJH+t(E zJ@{j4ioi#=<;o#)z}{zMhO;B1=GgeeuF2hd_R@U^4^3Sp;{dMe+;4Z9)gB$~_B!>} z18bCKufOrh<;ob5%)ZR8Bu`|*00 zab>x^{`f{~rB+>wa)*(4iI`W{YOP+q_n47bZ+WCd=^6{@A{%7%TD5m=qd`ZBQ3aye z?>)we36fjywAVIPT~-_-r~{!J>kQV07zvInY1S+8T-bvdk-Em zDfyy7=(SqsLG6yc{9bMK;d;B?>UHTMvTrr`(^>D-?$%4S0zSC_Imrq zI=M!&H>;};LBU2{1!KxkFs?`WHbIQ*bQmx_ZxoHGkc(5hNw)eyz3YCnGWj90OvZ0E zs;jm40Iga_6}F_e`^ku>uHKK8tz_~Bd6i5uA?u96iZ=$P4Od8ZrPkP>yNG$eR_$$o zB{9+NA>-B6UVYPLCVMk=G!)ievfJL}Y5LgYUQqFI(k5do!2Nay#06S4>InwKvWHA~ z%IdVLO;=F+$sU&yr`lR`x{dlOdu#0_Ij>yT9(4`3JC!H?6D5s$w|Ab|?*+zJhIfOE z-l}d62#xj&?PCB9lxVdFd>UJAx7O-R$?}Am0#knWTVs5SIEPUD)_c{?T5at$ z&@jddW|;S?UH2LIH0H{;vz8wQ1tYDECi706wQhbG%*a2d`~38sK48-(yU0-4W;m=#b}S^JEI7wO;M@YMmBui@Z(9E!T~{32p&|GV0a? zMPe2i*3C_k&>+DPjdg45)lL;q<1JZ04*8lv#=(Z(0F7il3gdX#qp)f~Y_D2x)gslV z4wG>vq}rOR+hU87XPyE(gp$Au?`P^X+3orrsJd3$WF|l-TpjoNh6`4PFyAvZ(yeti zYlTt!QmK5YXdl`19xirPOBe0Z#RuiJ>gxTO`!jP3wFQIz?ElP?9+?E^4j$N^W*ye_ zCKV|w93+#D$5*fS17Alnm7&5o?7fn!94`6s5J&6a~~&@R7@b#BlT32R7w>{&>^0Y zp{F8OX{m5rA$Wp_$V7+^BwN}x$!4{Z4SP_p5>ROI@S4Qs_n)y7ED%7^g)hrx~s_`U^Kp=H;QiPa_$Vdw@5;da| z{lIrf+bBFr_rA>?@V?E^Mmf|z2<81m zGe)ABs6Sq?Q7sZsA6 zC)q^rI|!MQWK()yL9&4#?Mt#@tr*EBdf!0^mgJh?eVdZ^O-H2Yed8pW;C%-nPm*UU z@7q+oZ#pug-Z#lcC1UTAk=6D_t2cnt1uyP}yet?#80|cB)N7V}k{ov1Pi2H&om2rg zWVJ?m?OqiIaKb1gRaNGh;}Cb7Wr({E=@55|rh-oHk(|pBr?gen^HH9UQQk253_0S% zNQFMHmzn^V+^0c+-!LPUNHu1+S!NUNLn;D1RMH{-7s%+n8l-#etzr&&Q!woyy)1+#=@WZc>eyG3)e^AjI&JREy5`M&G zq&zr5_z^%(B{x5E=Gevrl3unbGWZK5=>_WpkSYa|S=}(+@gRhf^oIw1NYbkjBS~p! z5J-}T*#nR#Nz#kOb-6_jCzK?A(~l&1oft_XLxKh&VR~c$LM1txg(FdyGC_s*NsSy2 zp`{{Ds2w9gDLU@)EYo&cqJFI7+hLPun!LYLk>`(fyc#i*LlQ85ksM(ua~2J))$Ug} z8a?=B>2P;`fQnOYg_o4<>jAQ_vX_pVf5!haSY9}i{U{ve0XJ5Ze05kqJENPnvEop+ zBFwM^$@BC%S8>0j?;H46?#lRAFGWFfC7pUx$YwJM-qyg7N*vP@%wPsm`vPCZ`-|?S8FOYpuehJ}eG! z=P{i}o$C5JU_4)5AQRpsE=;krsoY<==+CiJZ>oK2*R(ecQvq0&Vw#RSSNPMH`E>mM zczNj$Up8}eXzBM}Ccl;;dDEc#8`Z}E`%%63p7W?yfAC(fd$CBy;Y-f_0)OPq3|g!N z#WK8D35pG)#cU`x%!}D%7ffJ`ZThPsy5x{KLifRZJj{o~dH^TPVIZ__A!8ckCk-gID@x9!j-1_k za`r^Y*&88;rY47&`{=0WjfiRPhxG$6vlb$HAd2Wg9ioTyh~}e+9*!Vt^?~RS0nwu} zqQ|0$9@imyLXYUlD59q#h)(r^=xG7bGXkQQ$Ov%TD8kQvYLwvT2sLc@IZBNg_&H6D zGW?vO#;hxWe#>VHL*(Z&{>prvki6`HN;rs>c>jYfkjWr7Oh&d%jToNhyu+4x4^|RKlTy+M+q3p zzP`*~u&)>SAMC5kasw>ql0T9n=CnVSErI(l2IJZ>vqZg-HrZ?1Qs%+RF}-N!Z^Hv< zMl=5oJb-RA^LO9@w4<5-Bs_qAH1nT=2hfma{?qUPwKntb!UNRW%zp+Rpw?#ov+w}5 zHuK*B4^V3}e-|F0)@I&uNrsh1<_z5}g%dyz1^p9L)Gds?V|4#8Q?+uOs?7?jHYcju ze3GgysH(PQpE+R;>LkyZG_^ME1{GQ4DpCYCu@@Eb#b@aqGI{6n%GJwDZ(nilTz>nV zD|`ymbHvu1XJ*k3mQM*?x_tZg%GITJZeO`I9c{&x)~D!-(z+N~mC?4{^%My$Gd%$H zV(tr+s}$&l{4UUqZ;R87j|#f+?V@hHlB63SQ*~p@-PD9RXuovMwEEJGS4CA|x^Y>2 zahHBdkzO$66{8(IP@YbZ;k)#<(!QVn@J7uRrXT(gJfUr-AK+}5et@e9^+SFa=*Lo= zek=?6@o`Z2FSp=3^oGLUoIlfLm4iX&v>!H47pL8i zNg9AQ>rWTr&3ZGq2_j=|`V$^mD9?vo8rF2B0jk2 zEMtkrL;g}u9ym3v6*Ttoy4C4RH*Vg(dhP1FS5}-`Z{N5*9a=;)Nm841U4iZAw%ov~ zX~A-MHD%(hiQ59)@1mOuTzS17PIVYB3{Le26fztr$NfCrG)^~rwXeoF z-HAG$2swuCK#6g>gU?TgZeNXYx)XIgk#6WpSyHv*ZdYt4PCn7QaMv1=}X)iTicak|AvJ`hlg$5P}kcPI+?{Ft6h7=Q%}A*vE3-6b|}t zE;_N3OPbixY0t^nc^oz+)aU%oSvG%ja^*NVD$d`qjO;4xi*aWS78gt8t1-H1ztGJv zj*vB{$#u)u9R4@g{->auFe~UPa=PBeWgz4zKK@XGMM(bg#{iv1S zUzvbmzq=ZN?Qejcu4DcurwAWS_~=e}!=z~Np(NDuY6`V%p;lI`H6zq=*RhDO%0jIS z+X+_(jM--4Jll8q#L#vIUmj%YTV#jDSz!hA6~o6;_-KcZGvQlB{J4W#d&rs ze^&#kuwPkw0UgNHu~6ISUcN~^zzTZ218lY#_DI5(bGFGF{^->{=uudu`tS-{ov;F{ zzw6Cf0j9&f1qOVVBlMEU%8trV_&?k3f8|EywmC#^l9b%ZlCxyLvjG!IPItYwTEAcC zJ4bg>(GDu7!PHNXW17Mvba-xI1`1`83e7^H;iN+IP$-*JXb}pHBotaKL7~y4LNnwy z3Hf8#0xYVx=Mzh!!;IJM`Gp0x%iQ1eYtSM&4o53AHyZ9r6TaA?y&fz#5j>vfCohr{ zEF`P0v9%18pRXifOlGg^xo;d$a*8f$7)uvsQ;_SfWsv#QB&tZPCS8)@FEG2S2vM+T#&c z@sFkeZ+09P)fQ;I>7Sb5o}FOr@!JnQRsYqE(ilhU8CobvZ zy1E`&{cHU-IDJKCj!y&ik4dt`SkAhnDJ_Ku1NTLy^%aApRTqU>eOqx%*Bo7LZv8!j zUMa#-dF_zO=qfuf(5i-1xq;fkPQp7?@xw#x*v){ht@y?axbIY2KGV+K-$ z`VS3a$!rQeZ)gY#F9S9SrXYwt;3e+E2R4Q*#MFw116%}iz);ea!5jgEQ3*0?a>LR9^ux&wo7Q&chE40K=Z4L1 ziQI7ZSz zpJol_tdIo)g-LJ4Nsf%z!?2S*y0B#XcCnv{UHt@Sgj*RS)9(tXt8B9W<10(pY-mJI z@jc$$%>19*nf`RsMxJXJWQM=5iJ&ybo;bNb?@pvAOzzJw&X6DcZu0*oQ!G@12_SZU zcyJ^>ohHYs?j;JsthX0rdCUBT$#O3s`Up~?!w4CsBlwcbr*v4Bx3ARy!ZIoae2v+3-&8jblWFmy6V1s&`YR5OiW3gG!jJK*Tj*CNm%cm+C-X4BadZjWxLo8by7jvn zm#{7CeB+V!umluZxMfIR4}-?H`{-+7w1pdW^z|@ke3Om7R{CAgOa1I}Rtm@MVqWTF zoAS~C`;V6f+F6U&O4?3|gF)Tx~ z8>~%?m)s-=YZ1dTbbQ)2+IIgrF-TRiG4 zzj0;xt(!NlUcc=g19W5Q%`3~do!3_`-+aURX;@ehcD?Qy=)#R+TkU?|;%ECUExj7_ zy+ca_{m=UQ8S!?s@S-$J9G6=sLTbc7BAiM-9Tt$u4GPI88U7_h*=W%r>+J5ho%}x> z)yS`Mb9aUr?tf$6PxRlM^%-=seP2L)#lI)6Qk01eR$b>6q9H-Y2)2ICKrWC}h5}Z6 ze#BDurpTzAgS^apY*<=tZNN@2wo#?iZou$vz1D)U;c6Xj{K}j+@L6p!5;UyuAcSv) zfxWolE*^v&Zx<@FpVH-%*;;nr5w+f@&J56HKvU4Mb#LdCgDjB zL1W50&6(MF&dd(L8B}9t5zeR@6Q+YCjhWqPzRbn*WlqDFv4XxYf;EMz=bc0NqRT@I zdNTL)`7svhiRR<^F|Xl=5yKDYn^k_yBmB@O-35-!KYflw`{spsjx1~!M;20XgwHSt z99h_Djx5G=WN`qFz}it&M-~x|Xs1&IhAi#`LmV<5Mn*+}Bm32ewlqjc`_Bx5<){&! zs6CH7ePrzyaNTk4h|w*^odLsJggYgKJ8|=6B40{7&KEJh#rQHvfD2(i!WS&UMfeg- z5Q==+k>ECBO=5J5@nw+k_7_mTVDYWm3-K9Ikt;hA+D5n{#2xi2zj2SQ!LiZDlvbus1)672p8${Z}(O|_CiWYCU8z7YnAp)bauL1JIrxUk?C zF)n$@hRB{B34~Frq)|`V^5tJ02F>31P84NLY;O4?t2Ao2vdO z3vmSY*u|ZW|Llag*d8P<#@U0##Rz+N;+L-m64|rk;jx_%9@~S2$7tULkW}qa`z~pD zmB^eOkCE+!7}*{qMn-)PKvJ2b`W|F0naH3W50~wPaM>OtT*i$H3zre&;@M!nXij9$ zj>pY*LfmW*5;x=egT>8={_wb&C(E5pkwH5iIok=5vpq=UjOz~;IV1WbFNGA@vm>$d zr^SuSa*l6HQ-Icwj|kZ`Ta3#iQ|Wu}$}^hQKIN5Dvi2h{i|AXttc{kPLEo0qx3lQm zIrQy3`gQ?*yND5tqHCiB+bF>{-U3RnjS_651lw~Mx-;0zGWK#7dx^F=kCj}&UM~7u z_hkk4r_^)0OCqgLiYgiz83nmjI~<}}Irpedr^HX((>MGFSu)Cw<%CrL?lqRy3pIbC z=FV2t9AAPmbvm+Uo`&-Rb~t|S<3QH8^xp}0V}(J zmtDllE}{~)5eehmgUkEUteTPR*R0}F3qxu^#BQSUO@$~ibHACBwSl!60T zz3rG#4sN?mEN55F!rj$yTDYdYoPR-W9z>PcE=ZU+XG4NNg+wT&!`EK+?zs@-o-}EN)hgc zeVG(A)Kek90D1R_TdCs9t;LxGe9NGl@ z^J(zsb~?QY_#94w59bhrV5NZ&aEL)Tk_Lj$p-u98l>A9%8}2PAm(??i;m!+hOYK~7 zUfJUccV6(1i;>N&$N165C1=hSZ@uvLtR5CW&k8lOG^lAib2AGP#0*^9;UDq}pUX*S zVH|Ujo7P-~@WeRbAzw-?Ve3j*-^`GBDEfPHBB#{`e4Mv}$HkpFV#6#vl_NG@XGd%- zt-QLWBcOk5KRVb|SMiV8osmKI>Bu1Nn3Fy(eVBvv(+>nv z&>DkY3g;6HO_76Z{9F#Msqc&&GCMDaK98Gugl^LD8LSme$l!r5nE9e1W|ytwcVjKf zI_Z6fZcyWXgA!V#H3^ar^{$#%lH7fKZ(uJa`?4fw7q+409QE7K7I;eMCng@T13?7b zRg-YU&Rmh*dXsd2j?Hf3NxDDBW;gL9-JdhF0ELq7&nXwl4-oQa20103?A5J5XjOX~ za5LGXPIY}9E*P3GFQ`G3_Gqu?@3OJ~KFBd*kx&OYODw!g?9a(%V=uAbXlexp*`hY7 zcZmGCJzZ>&Q-KFLW$Z(`M{Itk3_d5p56;ibL7@acIPZF|1V1=mE++fI`Ldnt2j|OW z^1sG3_lWfu%0v!7?B^b_;7~D^3QKp77wrR$7ZLx&Za!o zRUi5x=UQt|b^Z5)gk$_BLL?H~nPK*`o^smZ+gr_N<@VNZ!82WF+2yE7f!6E{`Q3kQ zkpFuy{zh~*uoGqv^Eo<8PP0o)owa(mdaqGiZLdFWx4bv*Ih5Yyo|MHlY00k!%egfb zS!g5V#Z9_MUUJ)o6C^cN`01JgG%wpeL(aNY;S>mMRS8};fCr0WHI?`}PywD7nQ?fq zU{qUGf|sE$kz+2Bu!vHr6gm3!0y*IpghNG@g3-ifid18u5f;=+c+Ept5%T@h7Ag1lJ@5ygO)jkumMujvRV|AsM~H( zv_j5%`06F-DYkOFstJdsp7ZKrg7i8}CVT1F*pHHeR|}J>pFHLVlZuDbdI~ei(4wda$L3GBetw7CC&RuqdtfrpNUHJ&skh^TBgQl(vQ9SU~4CrKA z05r9@^KcJvcGf?HmR_rM9@OsG%kR}zAFj9StzP$u0Vj%9*QP)2A-fE|{6>qtfkRx~ z59}ZDzUe%B0SRuT!U6x2T;;I$zKGRvp*y9bbpO&5!>@z_@++}x;-bj;mFyTuwyx4l zOr;rJrE*NAvaZr>Or=>}rMZ|&bGl0NF_q?Zl@?+uE$AvO##CBVDtStTipf(Vos{et zC4z6bKG-oz1m853Q0zR-v$g2OD3PsGB0ENj;F|`$7$vfGN@T|<5qy&&c~U}2@}#6v zMlsyTR2P;2n2sK($`L2bdSBAbHsi0o%m zh4s~y+Jo8$uWd9MaLwK7dlY>eIl(cyQ=p*RAW%3!a_iL|+#}GUaPRls?o4HR`clDq zmy9g4z5+eEQs3G*8CmN+Ua!$ik!>9q*{n7;YIN7qIINx5 zaqGo>LT>ds_11$op(~}rb3{(oZV%5mIM-<=pV3LsdYh2eM$>^)I6G@ypQF0MVjgw4 zKsgJ?mU56&AF3FJb)Ou##agPZy$08Gw5rY8wA(T&s^Xw zv5rgCJp97%`$BF6FQdZKqHS%k6V$PIhwfD(8Y|5(R|dRaISe#IhOEB@=P$T$qdr!- z2{jEw>(E`C1R0g%3MvzXRJ`~$V@mgz)zRGW5d>*|*{=-Fw-+Pfs#|u!7=FZP@;Rre zK|CU!D@J;Ly`c3&6Bg*p>ecI{j21`Mky$U)qI?ulGRKT+ffu&q@AoVcMo6uW+7c_8*AAdh{~BjG6~ z`gF-anajA~f1C?mj(-)wr-RDHj*ucFq&EiXBKW!>198u|Cvq9_M3Kk9jaa$3M>-U@ zejj)s?l=J4%8_9OCTh>x^m!Yef022M3p=oKdE}x9J1tx;ZiE~eCA|?Kljfq8iyMJG zk;~{OiaY^!#LC4ZheEN3TQ_C808eu_7<0VOW%_RRxXb}6~U*2%EgY5 zBC|(t4AMe^_l3>hCGl7zPv$b_i87A?3{jhKvS{RtpsSE=?$vvbA&sYv#{-GSf%u&7 z(~*%qChhpS6Pc;Fd4HArayez7NI4!Fmo9?3jEH9Up4erN!6;4Owi>3%*d|DJOgD^! zLzV?NVvH7U!7e2RY5L~7zC@A43ntN|NiLa{xJ>^#m#Li7QDm9~k4qInUq&T`RN2^D z;IR9F^F4V7aQ%a~~B z7yYkfscJMOqq{va6~ib&rYYhm4NcZ}5&h&FCR8tmO`heKd5TLHLMYAro+93~fGx|< zh#(U|GEy3*LBVwd0hEcwBWt_K1x_BIAC&p3w4n2@Ak{ZJ7*k?7i zV8UNbBeOF}x^k9CC*2s3fpiffWOVwMt{6`F(v@-E9_ixol)Z&`s>f{?$pB%}J<)Mf zwb;TIcVU&xOkB1<8d$EJ4pL;B1Tc^+LV%1+|B@9$DqpfP#@i!VJf^a@5K{$`MUp)+ zlEso&uHDMaOp>ge_0dUI3owu@LV%1+|B@90D_^oQ#@i!VJhBoF7n5m8>~FuD+bojL z!6ch{aiNR}F5+(l#w#az6cML}9LO3WNyaKZaVJeVwS^dKMod%rGM7n6lz9Slt0n@( zb5%1ba#cxSzLV4EQsv8+^dv9`=@ zTnm0O&;mKVq-a5E$bk+ZB*|FmO+=cAjMzUC$;s#^%0C8j)c)~k4z?F3%03Q6VE;&_ z36r~TgJ~Aq%RI&ymW2ZsVA%03Q6Aa^816(e{3IxoKMMdmOr_aN?97Ov~$o(wsVJ3^9-RlHe9 zv7c!BWi|FwNJx}@0`!ROm+|-y&U;9de-aoT_rpE_Wzi_6177@{xP3|HHLd|c+^_6H zP&6PV+(7;aIWkUq(~x8zv3oV{Q;<)TeGKHN-QzJ|IYrDD2ZHzeu#F>?%)P!h)#fvq z$GF^seqY&0qR71u*n#X3vShsUW|0v0m({pWAtF)!3DBdqkH>!H6tQ0tn85!d2{Fv} z{oWT#utnnjO_|%c7KHu2oG4SYAT{Jb2N05Etn?-l6ZgaEV8Ov7mY4HUHZ~>7KL&Et z{_&WvoFe9n0}# z<(~jOV*j&v>^GYt_Dcd2*guja!}P$n!sHFTGPjfLUrwAUT96uYpaTd=GFEyMiHZAX zk=Rd0KT-ZMkR$d#i^qMlDdN635PvdB_T7R#)_3gb9-MN~?l^4g6yJ`Oo=r!^3F=(o z$0qp}t*=ZjeR&f0xiMQhx-{n3xgIM{vA^fgXSI9-!HV_An-n+`Nx6B)Vk1K z@70_2&(}KB>`09`#6A)h^$ykOr?4GYxDdAN9;^{p)(*=?PS%L4Q-;0-&X=gJ*Qc8` z*ps;0oi0whbHRPra_1TQtW_EaR%tpB)(Wc5UdK6o>Bi05SFc@t_sWWM>+Ku2r$fFq z;I-ix`%nb3j?H$H$3>F%#( zN&A>9>=-Q;HB^e$Q6!cK5>wVm4aH%JaF zcX0fSuyu2f`nU0Rg7gk{AE!3TSDIvLE)L#*)a1e zoh++w_7}$^_byNcn(FO&=_A-TSwlAKx$Dj`Xtppu->2`V93g`kq0 z1qdo3%TR(!@{*CD60&X~s3fm52r3~H-a#dKa+)g@E z(kT%(D-~4Iq8Fn?O0GUYFGh)!1aD9&Mv0V@tAa`~N~D}z#ifLi? zVibNAj2yp;ViJB8j2OR)Vi0~6%pUwIiaGdIFl+FuD8}Gd!EC{=BAbHdo|@BSyw0}+ z&&~=L`Ob!MOdl8dUXw5K{V1vQcYUu|+J;Md)jkjF&F!N0oFL8lK{j0Li*!2_duf;b zt#(KsLA3|h*q8kCMr6aM_=z+29AB!4^j}oJFO1W!_H9(Nn=6Eb5u`uKF@&U$8;^3313A67ng8BztQPNq!%ABE)8VPlS94JdtcPz9*7j2lQY;kWUZxDWFFQ zkbHWS_aP*`C&GfN5R&Y1I3)Rf2nk^jrwREKLXx?}A<6GUNQk@PkdRLyB-!I|Nb>s- z65?(+B;->FN%lA#lKeh|gt!|H3HcO4l06QGB)<}$U^X;$gcw^2>*R3$d>?$ zEC3&h{5pVw^a4H<KTrjhjVG5Y275*m0Dwiy8l|rWr2Ji%_zL295;P{?5pghSisjl|wn{XaJ98C|iQ>0k~Q~eO4$-$r zO6g_ENsY@hsei*><-BRoGh|9r4lW_+)Vl4B&T0)xX3~}1mg`p+iqTsNUtGV+&w`iM z?(Uc;eR2KjW7VS$>h&@&ydL7-o@=l7%^dk5gPc{bUoAP_BG%Pvqv5a%Sm&dz?J(ij zoWlKf(VNWbH@ z4LS7|oCVRPqa6|5$k z%e4o&@V>7Q$4n98oCd2oL5^ucoDn)aKcD6b*7+IodzAc1hAfM!)2)4OgPo|&bffyX z)^S`bPtcyZ8Fo>tH=uU)Id6d4j(IQU%JYIBVCL>##LPb~3Vy(($9OX(XKt2V78>;* zhsDojp=LI$X+upJVul7a*=G?Zc2WH?7rAM6r&I0zM&l;jE9DwYZEYER-V@`5hkPlq zgsm%KeKSMip-`;@6d7%WOb#CxU)(7@SDdrHFiesJrDyWFSQQW!Qu?aE)YbAO!*sX2 zGctJgyOkdZq@Xnhy%f$T7@8sn*Z8>{TvOi}Ib?Red$;>=&@=OJ*&mkp@ENQXO~~K@ z(;*q_i$-KQ?eV*@mSvsvzQcl8jr$EsXpz<|4?9LozC4Vo*M0S|FwB-0hMgiO)Lc_G zvij?lBzGU*8`w+9zAVYvg>7g#NBuUm1uC~8VRJKm!se_dO4&ulUJNdbPY7ush)BLw zeh&5^oQA6gV}m2`(@-dzRHzJvMv@9KoJNxhEt20P7 zrk%FSKyVh9JR`)kMvcsHc4V~Q1lrpyf%f{!m2Jp^l*>DvL(4oXIkZ6{;z4q0A6E1^ zHwTekCLz+BTYy4I(dT>-3MECK^DIP7iazIAh?*3A&d*Jam_IeKX96zr-qVGx;TIB)xT3d6P?X_A1?7H4^+UwLLkjK8>zOi0-g&c4ku}HmD z@3A6h>1o|3;?=d-4%uIT8@#)<&L-S&?yS}BS2r3x`eO!J<_SPI%}C4Sm{bS~x4Jbi zt6nMYF)U>bK$deYLVT9+IG_1V|! zfLPeyPWa(4sLX**?ZHL^%6{Hm?BGuVKodDS2FIItXSt9G$kUpRm;ZH0!5pLKb}&LML^E?$#5!Craqv2%$7Jp~T!rM*-DVtwBt4Kh3hGPE&`-?vMmm+9LJ` zqSznQVSh-EeLjl);RyCtAJ`ueus@m#`(sh;kL$2Mp~wDY6#G*V?5Fy`{H0s@+^Qco@U$1q1 zIc{fLCljIlRt5~}EIrg}w;ar(iFuBWx@y)1m+%tJ`G+u(6Xtn(#PdVA!S6LM&|O{u zUS}KJNYrn{`htF($~kaAQ?dh6osa849@JVjIETvF00-@~@4XMvr!TsHkpUhRXpYIZ z<`m(NkMKWA_FwH|XpwdhvN%uwi9vqW_=RCOJ$d}KjYi{g=fN#lj7l$%Y_;>COHX}$ z_&71&_ZKh`^1kUx89F{P`hK=q?LMTtNDe%>TN3dj0+uYww^ASL6~R;vv|3AXZ$4u4~Kr~vOjFER@L z2pL8GH{;^VoNgB%x!z`lFp z!PxBLOeDLSH>Q5XAYTR*{6R?t5BMs$>+8cXgnECJX^K;fRkuk&-L^x^$Uirr>%2s> z?0h^xVw#S6$KSzo(SNqdm|3DD?%{Q0uW5tRs#QBJ78-V7hSth4&)&@6h6nKN&HOv? z0N%ZszXK28-<$bQ!UK5tX8u#~06xB%|1>;6t_bI^gxIn&w(Gycy^&0h`G{MS=2ZhWQ!Gsa<{Vih!}?_y zk3G}o98%<&^BWOnQoQd@%NdyBmH2 z#(qBzox@W|KWuksO8TM}toiw-_KTUNFKkb~Db5|+G=(V_FhwwDsw zQ<(%u#s21r5&4Zxc2{=!W&9+`p~?aOMKo1He%o03cLq5n4qwA0g`eei=Hbp9iHU#H zAYTI~0vm!NP6SR$4YP~=j9^bjl9Tl7!@J3@_eGff&-;0HwCyd)?#fOYq7SB^(Ot{#Y6YyC&L%W)R!_Ri0NNhS5!+w9+tQZ6TUBXu=1(F(DAtCFXKiL(QvCz zuMgY;38&?jOFt8CDsF*D|uOnZU<_~?`2MPff0o=)pOGV2Cg5!8oa z2CB9#*h5ZcOW3>XnFhiBtNo-+>)XWO1f5{UlO8WfaBc}NfpsbHf(xEgq=vrN2spL; cWcXDw{A75DEPi8nj+CRFS-S3RfS&FD12FMBwg3PC literal 0 HcmV?d00001 diff --git a/src/smlp_py/NN_verifiers/saved_model/fingerprint.pb b/src/smlp_py/NN_verifiers/saved_model/fingerprint.pb new file mode 100755 index 00000000..a80bb51c --- /dev/null +++ b/src/smlp_py/NN_verifiers/saved_model/fingerprint.pb @@ -0,0 +1 @@ +ޒвeߩ\ߕn ̫㝚H(2 \ No newline at end of file diff --git a/src/smlp_py/NN_verifiers/saved_model/keras_metadata.pb b/src/smlp_py/NN_verifiers/saved_model/keras_metadata.pb new file mode 100755 index 00000000..01426138 --- /dev/null +++ b/src/smlp_py/NN_verifiers/saved_model/keras_metadata.pb @@ -0,0 +1,7 @@ + +'root"_tf_keras_sequential*'{"name": "sequential", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": false, "class_name": "Sequential", "config": {"name": "sequential", "layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 4]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "dense_input"}}, {"class_name": "Dense", "config": {"name": "dense", "trainable": true, "dtype": "float32", "batch_input_shape": {"class_name": "__tuple__", "items": [null, 4]}, "units": 8, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}, {"class_name": "Dense", "config": {"name": "dense_1", "trainable": true, "dtype": "float32", "batch_input_shape": {"class_name": "__tuple__", "items": [null, null]}, "units": 4, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}, {"class_name": "Dense", "config": {"name": "dense_2", "trainable": true, "dtype": "float32", "units": 2, "activation": "linear", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}]}, "shared_object_id": 10, "input_spec": [{"class_name": "InputSpec", "config": {"dtype": null, "shape": {"class_name": "__tuple__", "items": [null, 4]}, "ndim": 2, "max_ndim": null, "min_ndim": null, "axes": {}}}], "build_input_shape": {"class_name": "TensorShape", "items": [null, 4]}, "is_graph_network": true, "full_save_spec": {"class_name": "__tuple__", "items": [[{"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, 4]}, "float32", "dense_input"]}], {}]}, "save_spec": {"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, 4]}, "float32", "dense_input"]}, "keras_version": "2.14.0", "backend": "tensorflow", "model_config": {"class_name": "Sequential", "config": {"name": "sequential", "layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 4]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "dense_input"}, "shared_object_id": 0}, {"class_name": "Dense", "config": {"name": "dense", "trainable": true, "dtype": "float32", "batch_input_shape": {"class_name": "__tuple__", "items": [null, 4]}, "units": 8, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 1}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 2}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "shared_object_id": 3}, {"class_name": "Dense", "config": {"name": "dense_1", "trainable": true, "dtype": "float32", "batch_input_shape": {"class_name": "__tuple__", "items": [null, null]}, "units": 4, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 4}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 5}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "shared_object_id": 6}, {"class_name": "Dense", "config": {"name": "dense_2", "trainable": true, "dtype": "float32", "units": 2, "activation": "linear", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 7}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 8}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "shared_object_id": 9}]}}, "training_config": {"loss": "mean_squared_error", "metrics": [[{"class_name": "MeanMetricWrapper", "config": {"name": "mse", "dtype": "float32", "fn": "mean_squared_error"}, "shared_object_id": 12}]], "weighted_metrics": null, "loss_weights": null, "optimizer_config": {"class_name": "Adam", "config": {"name": "Adam", "learning_rate": 0.0010000000474974513, "decay": 0.0, "beta_1": 0.8999999761581421, "beta_2": 0.9990000128746033, "epsilon": 1e-07, "amsgrad": false}}}}2 +root.layer_with_weights-0"_tf_keras_layer*{"name": "dense", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": {"class_name": "__tuple__", "items": [null, 4]}, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "Dense", "config": {"name": "dense", "trainable": true, "dtype": "float32", "batch_input_shape": {"class_name": "__tuple__", "items": [null, 4]}, "units": 8, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 1}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 2}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "shared_object_id": 3, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": null, "max_ndim": null, "min_ndim": 2, "axes": {"-1": 4}}, "shared_object_id": 13}, "build_input_shape": {"class_name": "TensorShape", "items": [null, 4]}}2 +root.layer_with_weights-1"_tf_keras_layer*{"name": "dense_1", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": {"class_name": "__tuple__", "items": [null, null]}, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "Dense", "config": {"name": "dense_1", "trainable": true, "dtype": "float32", "batch_input_shape": {"class_name": "__tuple__", "items": [null, null]}, "units": 4, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 4}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 5}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "shared_object_id": 6, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": null, "max_ndim": null, "min_ndim": 2, "axes": {"-1": 8}}, "shared_object_id": 14}, "build_input_shape": {"class_name": "TensorShape", "items": [null, 8]}}2 +root.layer_with_weights-2"_tf_keras_layer*{"name": "dense_2", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "Dense", "config": {"name": "dense_2", "trainable": true, "dtype": "float32", "units": 2, "activation": "linear", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 7}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 8}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "shared_object_id": 9, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": null, "max_ndim": null, "min_ndim": 2, "axes": {"-1": 4}}, "shared_object_id": 15}, "build_input_shape": {"class_name": "TensorShape", "items": [null, 4]}}2 +Iroot.keras_api.metrics.0"_tf_keras_metric*{"class_name": "Mean", "name": "loss", "dtype": "float32", "config": {"name": "loss", "dtype": "float32"}, "shared_object_id": 16}2 +Jroot.keras_api.metrics.1"_tf_keras_metric*{"class_name": "MeanMetricWrapper", "name": "mse", "dtype": "float32", "config": {"name": "mse", "dtype": "float32", "fn": "mean_squared_error"}, "shared_object_id": 12}2 \ No newline at end of file diff --git a/src/smlp_py/NN_verifiers/saved_model/saved_model.pb b/src/smlp_py/NN_verifiers/saved_model/saved_model.pb new file mode 100755 index 0000000000000000000000000000000000000000..e6b78b2ac875aad4494a4a120111fa58d1627d43 GIT binary patch literal 93990 zcmeHwYj7M#dKhN000xZ*!R7qOuviIOPtKuUL~J9DxEmeh*a zUEJ36<=IePF2o6JNBjG*i7tmE*Hmk5|6xDHuV-c#AVgiLtK$x4y8C;7{q@%`)1W{3yCdW$r|BP0k-KE(YNu1b z-+Zguu2eClf-P-I<1}dMlE-kPLq*db!(?KH?rCNV7oRqMZ5=^b^o-;G<>da zR=YK)^KNy!MzimrUF$#q$S381j}RU0)|#DGTcVAeCkH=K z?=Y^cwYERF*<7zxH>2EPBwiur_03wdTkn3rNUS$KQlfN?2Xv7gGJ2!hy|J@JM~P7d zqS@~|j1!Y&Y`fjs+}UthF-s=8?P{~L-RjhwYRJ~P$cJ}?561%_l^HT^Pfj!Tjb{%Y zoi!=>qCx15TKj(OEqm?V+Qx(JR=wHn&{=Y5Bly$VZrASB->-FYWArGYo3&22-mG@( zt)~AaH#T>i9N4UF)j-^>txadM-tGh#7;~q|q3YIF>!H)AgH|=~JNN2awXlGz5~C#h zM(gG_xk0kGsv8eL!A4yLW6F>>-iY#Tk{EB$VZii~Q84C0F3uew+3NfCj{D8Zq1zI)g2?oS6 zLnb|Cb(+P?Wn@2BkS9dKB4hX{Ke<>lOA;(nCg0^;{* z_TEl&0}6}+z4z+(1HYA_Lvt3H^xlGgLWQCKw~wp*D1c2T7J~NQY>F7*vEU`!{zoy@v>dKe80R|-MCk}S6W=IEgSS3|07GfWEz}1cwl>; zby)K~DpHm|N~Rr;uU_v5zK&!nL-`5bOJ)j29{L#%+tEsvX3R|fbd=B;D#0KPG8ct9 z&)kyxkKmgmUu8+HN)<@Z zA)c3^ry^HrDSt{Kc#??7M2HR~Tii3rX0?*#Fha@lH>pVVCClr?NH!Z$y->2n`Mr~@ zE8VMxvRhL{;xQ2$_R<-$rPoHkVN&6}htaZ6rumB07+)@6!e$S?X64y{{ly4mB02 zQSTck*+lO<2$_;(Q+i)PvVkA%OR{0D7|AAj-$4kLD)f21)C9QXJ`Mu>h8Y3=V>$x-!s!TTlBum)wcTXND;U_;b}NTdQ8^r-(dNm0 zay9}l6-m)H(h&wWv02-wez04DRAY9VWj5hHq$0pWB^}~_fsDRWgLJRGTg)ME3Z@;T z7fkB6OFly^4?-&3{aXDYziWHeAYTlSFZO|aF&*-sBiTA+iTdXIc+`s?iSixt^sH6# zCdskiRA{?3(jYxf_~6eE#QR1bN!Mrwe?EjD{NSN~npBM!e)u)g4;A>}4=Q@Y`2om7 z!jHI&lm{mWKLW_9wB`P)Uwv{&>`-Oj4nJ zQX{8AXsL)3YR5=WijKQH%e0r4s2}V2cG%#VChzZ5lbJENPn zvEop+BFwM^$@BC%S8>0j?;H46?#lRAFGZB6-ek}O8Mnub!|s3iDSs|#gbdq;0lsI} z*=TLCS+9Dp<}|9!>it^Vsc&{D%ub}`UM7EuSWX*e1DsZK>jQZ5X6ikGbYQN~v_`$T z#QbGxR_P=?PR7Xa+O0QdU|OCGzYGsUcIMTgW#fB|p?sdrQe9X-N6tFX+Pzx4*4%(e zeOMge&SN?c+tuxDz_?s2lSyw97pBBfp*@In$trx2hii?1%O4yUxQ}{r(L~~I@k3|r*`atx! zfanPs(UVa`Pw5antw;1s6w$L0MCbZI^qhd`c>&QYWCXZv6yWDDHHz?aoEkR#oS;St ze$G*&3_s_ovEWLe-}0IK5c&BGIc{E{vrSkhf|&?0FVa!hN;_a<&(N_(4d$9RI^?u@ zi5~Y18@Hf?=Cd$hb}gUR*#oqia(Mf=Da_aErRHaTe8Qs%)LV|vlZ z-GK+tj7IKFcmUmKg61lWLRlrmgoT~oB(<#=%1jXZei>lqx*-Ms+HqZZ9!1A zMN!q3l2mP3RkdCF%t>=lCwbANskP|_NnOU@hES?Uyc^R$sdDqNoZ? zH?E2=?$S>y(hH`%Vzh$?%JT^_e4E};+V}Gx-l*Ba^ur&5C$!D<1Dp-h4{$Z1e#q|v z{aB6Dk2OI*J|XJIwIuy`N!5>CM+cMUpy9wpbFMG_cv(~jrXR0}FJkl~7!6_GgMZwG zet4rRR6qQ=`2_u7az6&jVR$-{@j{?s2!R4ua^3s%Dpwy?1^fTY^s1u9ZeG%QHi*9G z-Hc`ePOs5xN<&^!xoCYXY|2eZ7n7K$Z_HL2vy&RTPS+I7ya2+=hlqJ;{sbaFLvJfh zMW-z+J8UvK1(MJ-CY-%^w+w@%*Tj&1%6wf6&&SOh)SFi%rWTr&3ZGq2_j=|`jZ}6>mpeq(_FG{(;Es|9?vo8#dJiWjkeJi zINibLU2r{yZoWtjl{fDt2d6t($9se7cyG5l-a~0FBs)Vv+w?WXhZT z_y{?s*;E26W3&?&5*CkVuEO$yRvvNs%NZ;`&XZ!aAf({SfGoc^_LgJ$vm#+{7wv)6 zejylF`hAqLiZq;O@k>Nku=O}k+^;=LhS1~n151%11SJBU^5!gHUd7$dbC#U8PmXz0 zIOxA)(TSb0q=_A!_MC~G$6-@Kea_!pVDmR;)=!ZW;`|NE$Zo*C7m{}^-=W(8fPJVQ>|C*qXyN!7_qOefE+gHd5R36qy>i=(qt zUZTJJm$T&i3Hfs)#q=YE9EC6qoipe$GPcuv&}=F7*TJHK45ms5Km0>&J>VPrZB%EXWEgu=$PT;>~ z2AJ{|*3SOI;7@Ua*^+TmjL((`dmaX2)Jwv0B3R z?O-Ffbw<;BSG>)Bm4Y>LmrZ@=$JjPrw#hH$>W7T=yf}4k%DWFf7I~ghx zJ;?W+2UtNTc7#nJ!?s7*Mb7qZvz?Z;_q!Apo<6w7796a@g6?{wmWLT`Z_NPTxCl0i zEY_$Dh5xhd{x!35#~h-!NJ{Qx$pv!Q*?~DCr?Xw#sNbve&7f0MRPo9=u;SC?q^9r) z9WECNP$-jB$c940NrlQ#D4SGh5ekhY6FUBs6xm+yx_i%|dHCST!O!Fq6JO3Q7|!)T(Z>H44>Dyp7POu92crzX)qx zQX=8`-WQ3DG?E^TETl12H0<M=@{-puD60i zI35`nm6G165lX>l<$@5w=o-C*lug~N-AA+bJld=UJHt~FM&;ej{pxmK3*D^03daD* z%<*ZU{&^ji=z**&n$nVdD(=}9um-B%j>5FQVvw}zqA;s(D^9)c(fT_E$tkq^3P<^2 zFVb8=+ zXN;PwD{-xF8047D5%3qDYyw3;Cl>*w-rA`MCzycv1Qqkd7qqiiCXls$%pmFAo5IWi zdXX42h^||;{=PvhnN6YR4GlrzWxyuE6a=vcyu^L@z{ZfZda8#5Tm*B#P~EjbWG&Pn zG#!K@LxD4XM3dBy#~6g$uSb?2TR(>x1V=6~hzp3MQ<(&BX|GHogPVk|YgX8)nnYqm z6Jrw5J>9rT1gOQC1ZPvMNdztJ9W&eL;7_%K0CdnX3_kw*TbalJ@X}pbw=`jv^^95) z53=$~!O$^DytAIg%BF5M;7f{Txs0WvPvSUtJ$>MnHbS#+r4D>k2FUhX{q&9LyHWd2 zAMs1-NC(AMr#@80R#z0oR-+q zCv`dl8^9mM!?H)l5!`8Z3I0LpnG-1KFRH1BXH0r%Sw7ROlPH=4N0k|?O*AY~C`{sf z`aq)>9x56PIA=VY(Amj2t5m9vj?_ga)&BaSE)XcTp)m?yfX1r;Lx0 zSxw5=E@;(4qb~Lpou%LgRJjR><=4_Dmd|LsE*t~k`4jNDd+wdd?~RckGRTDluLo-Y z%ZpLhrk&8rh?7j>gcjzlw3AG~J;)>z_aVsqQhwT-s2n3B_AqRKjLt{y-R8&hv6JR- zw5*jeGW|}RUuSbgpIBereed-D?QBv%u3?Z_q&+wGy^eLnVqN%?6xE&syIM!IH^l*Ppat$s`|6u=Io9N&fG2@=76C{KL-g4vwU!^W z}U}Wf7GOH9{bl8R^AU6wpmxt&%a!e_*K1#p7{$bvE-Ll|2uIFY?TLE#JJxYA&;==eKc;)~g^3I8Nq+KCr)LM(a?3m=-I^nX4MRHy(zvP$aN*ICT zsbK-bZaJu@K*~}S5|f;J2D-2A5lEggVvy`YP)~uBp%~|EZiPPQdZ{>Pb8~c|usZRa zvqMWn;OIGLbCWDO3M8C!HrE83b55ox14%4ZgKVw~-t3Gc(v*R_EhhM4LdRatjk)Uy zm3v$s8T6`jwTn}HWE@go7SOkNSsN`|Lf@9rw+raoMfB|w`gR$8yMhsnqHCiB+bF>{ z-U3RnjS_651lvHcDj#T8eF?3qFQHNOCE6zNP%Q~8R9~)Ge>P2y2KSE4ix-V(PWI7X zF>pehXC2G`1iw0ker0;~%hPZ&4)Yr)R(dnF)TyJr)@KC zFB{5B?i2@W5yLWt#QVlFQR1;JVpyh-cx;;kNj$d26VBQz*VbOUb@TcgcidxmZmzz1 zZS9Wp^7_?VuUJ0|>&e2d*FA$=xY293-S2DsY`xXh7lXccXmz0fS${Vp-cA-?RA!0e za_e+RjTlITD`?Jz1!QuALh?z1f5}jGMYPE_ySr^a{|`qc@~hO`omqza-?;Y^{ddgz zESd>B6cAtW?`f+PWMYH0r#XdanCV~yTR(3g7dR?I0V_T~VyPQRWK=GaGtRqgSXyoF zz}^D3tEAmxdknT~O&A+))Zxah%q0V#ffOS_!}>Nt_`Vg`3ma~^K{!&B(MZ9Oe+c&+ zszV%_h9`XR$p&}Up%6)}Ha;e<*zsJk2jI#tV_dNjt|aHW1kTv|&6!d>XG#Nb=9e(e zln~CGRx&VQ8bQ*S(th)$9M6~X0DM6;ri}1K)fk>8;YklcW6Jx@nT2@HEDXRIRAUwp z&Zrs_rh_DnS=evBEXMO?QNx$u#viSJA7sti*FOP7c9a>_!3MIihS9Z;5K4SVswk~Wsvap z7g4@o@vYhm@flH(EBg}KMz|uzwis6i32t$&V9_nY6@Lm?=DCTj6J&eHZGp~$0&QS05@@RUu4ccgt~?mVU8H< zV$2yN*!@+MIasutY9)inpnZvaBMcHlUyMP6#J;$3VZkqAT=J3)kv;no21nQ<#=#hS z1_^|5{lOw(M1R!vDI$aRJuJ2p!eV=nuo%@JfTSiiRsB&G;t1@q3;P}a*$Hv6JxE-P zvj>Zd5%%!JFJBELvS;7JV>=-{wg(B1(Y^~HsoJCVUDEO@kvaPwBijivvOP$QjQSpc zq%ueKJ;+)zkwN<&F53y=vOP$+j2jmgE+fXpv%!4PoXDPikDKj;xY-^gZpQToi<=Ss z;c+uhmODL>LHiy#+X<1gJxJt?>kk$=Bl;sRg%sJdFR}Ay#7&uUj&D~}fYwiq2-!4S zjLRca=?Cx1Gn&>R<&{&k4kIrM=v%z3jg~E;Z_DW01@!GA`gRF@yNteF!3ajtwNZj? zlwccg0VUW*3ARy!?L`dT685r;yxzS%Lj2^)cNgjn=0`6^)FH zg50Vd4$-VJ_gqY;$oCBC8~(#A8D(c^!YTmw8cFM^nm<)@XSZsOFF~0*7g;k;!+8Na z96$GQdg^QPR`?284i8H0@5w%&W(|*h37IJ>Qg!yIXy>t1c!47M@d949h?Ui}Yhz_? zyxkI3wuF~0V`a;D*#)fZ0$z3zE4zr7UBb#P;boVxvdehc6|C$EDq$OuFwR}b1*j6X zaS6i^94U)S7)FUmSq%=Dgl$~HFj7I<#U%`3J5m;xFhsCO*+rDQC4{>rv}746Sw>4P zASD;jl8Z=59M2`Bh;T z46To*Y28~WUO5j7Z(+Hp2KQ0#GS|vQII?~&ZUe5j9TUpIZL^8x?8*hWI~tDm)<7Ne zFNm#NhFhech0|2<)3@RD@W{5z$}@1wG+Y#}DLxL@2e!JfPP7OGr{W9xTQdRA>3BTd z7NOXI_+oB*Ajz2&lH~i08X(DoDJ1D#EYzqJ;BMCENM1udrSh#L4Z!D63VeJ$oM#XY zr-8sV5o%m9kEDUX@2-IIM^lvN9O?mwW>a&h2l(gG;Lq)Jz6bamOMwsP5QAW)fe>(r zK{%cUg3qDeo}5@#UWy*;^Yp|rZ+mUVl+F&_w}U;No>=yQr5y0_^u)3!?huY_T;Kj- zoLCk)&WN2@R*t$h^u#jH+J(z>#1qS~V~D=VPAtpj_x;4Oh@7fTc5g>_r|sSZvsq_% z!a!WM)#`L=9XLE-dB0CA%RVXh-=sUS%sZdwv7cC$=ER-k6U$0zPAn^}kpD17;JS-* z>Y;?fL9EQcVG2|{Z51_ewu@hX#};zVnAX7m-AD3BLi6z$_{;1_!qxQ`$qTVkN-*yw zd%w-w{AihX);)gfv72|AzC78yi)qZexI%uPl0VImHBoWGBj(eeNmwct)GMao&OHAp zpe6bCJJ$>Fj|-7Qcuw-``ijuytO#fNc;Ue@;qwYRq=_HjlchmT8xDVFSL%u9C}rT< zKmSf1_IXjLi5-S|(nW4wbKlw{(_DTqtC+6+ABN&l#&v zx$J(DlSsz9(1j^Fg;s;ltFN^yT=@Z?90-ycXpYYciRTuLxBMGmgHZh#oHBGgvE{kii2UzL~MUXo#y~tW$SmEz3INeTVd&#{C8*v`A|v zc0be$F<(e>_wl`fy_)RHlAK-GhL&^GZ$n$)Fs|mCB21EH%J*O1ypeNxHN6%ftQO z49oqwo6CB7XR$Gqf1M5bUS7YpC$al}kNloqZ0sIl)S*YnA>>kwxT$k59CGadTu;Kq z#-$RwvpC^g`cjEGu>_x4DzQ6@6MSZ=#O^Fk@|k6J*>aN4EZgLF|CK@h&rxzhbec}L zUEP3do;%fE%~@PnVvbT+OpqI+3*;QTd)V2mcdGAf)izq&AGDg@8~0dRZ?0b&9GbM~ zSA*Rhnu=`FUl=r+bcH{Z#;{UQIuDu`g*|F`(ePC0OXQT> zD(u%%ibl_dF8YFi4gG4-$WjDgjCZCp94GBI4yQt^RpfJ`gTk1k{Ur%O%bY^k53Olg zEs9ChZ8s=dA(uRS^%C?HTRCmngtL<`dUY{DdhUeDUOGkkd6M^PVN&(SWc-|B@yu#Z z!52Bcn4o!W1!|TCP;sdzoGZ zr!t!Xooq*trWSV|?s?8UlNIGIJ7Zc~TY(yKmz^}NsmEPN1ghsm0`BVd2s?qA&X7sw zu6yj@gl4Ck8;hJty|Cb)NlkCm+V^X3*=z6CHXdxZ>dkKFkpTw`S2yQB;UT*QzWipB zy@4~G-4E;^@gDRXdjaD{q{0#ZlU(JP_r8GDaiKfKf^-YzBg3zR0`e=d%W|X0`IYP# zNVcw0DW+0MSE(FRsjREC5L0PES7|Y((xR@?QcR^KU8UuiO3S)RD>0Q;luDiwp5X zk~}G?6nRxpg1jm!Jzf=*9IuK>jaLOF#;c;E1+@pSifRrX6x15LDylJfRZv^-s>r5b zJtT)&2ClxbUb|m=|D~O+Ex1Z><6VlrjU4D2-8huj?H9@)A!FOsE?g_pq;R9@-A<`8 zKYt}}y-h~eSYLsjWUFs&f{bi-KiIC(Oo8nr8tGNHc4~BLbpjUC4y}${*{-&`aM{q9 z+cxYZ@_?kpHPZTV-a1{Kaa$)&x7OI6cR_rQK}K)aVDQnU{;zcsV`P|Gopwl%$z^mBv|cBqxzli9Jlo#v_#D+0 z7SDDa&Qs39@zpUHehyU(!@5U~+-5D+HeZ5Udz#foZQgAe6;)wSI78MC8{~ilh{`Xl zLm>kL{#d7^Y94;!_d_8!f|pU@Y0R~xhiM&iMU92qGCKO3KNLZS+!v3w zILSl}kfIuzDC`)xaLQ2#N<1hcJj+>sU_~J6H~`=7iy4X4Ffpf1Cds_TMg28{c%qi0 zK}FPQ;c{^!XRC!X&Rid}!sL?HGgARqXm zN5WG~^y!j;GM90|{}dOz9RDhUPY0EY9U(FK_TxzsmLSy>p_WdnZQoXr^d%uXzJur|-!P$iF>LZIzsyryx)4HX=JyowrUh(S zenteD2$GS~FeNJJ`+`f3929%X%ZJF5N_!&d=#i^I{1kKcYY;zah_ZTI(kWA3$%`{& zcH%Pr>w%2r)R7|NWRQV$5h7%CG>nLmt{6u7(v@-E8|mV4l${ty={xj5x=3mWvtyst z*n$avHI2;9B+3gU%FyAcxdJCb)>d9vH8j&StHExJL>)*16Zn55i-hTc?}f3fRCKYn z%xzo?ekRZYIlZK4L2Af>4j?4SSm{kfnuv_pKN88w=qJiQ26EK?@n{Y<*(Az74n$!8 zNTvysyKjSO7Te1_#w8!xznp|pB;N<@K=ue(GG2PqKxQQ)_K(DSG7*XLPkV}2c-#kD;}T^b2O^L=lA?-{yMCP)-}WMN7?*nx_bUt6b#hOJ9LOCZNyaMP zETq^^wEeOg`za(O%02;l#P;Dhyl_4|%%+!9#D7U(c-#;B0F*_em=1XHcjERXnb)`m z1aZHz3qjF+z+QMgr?|?`4u7(<(~jOYWsNXS56W8 zC4mY2Kavo`Y~SyFu>@Nr?(fOm#F1 z$3TwSKOXayQ^b66AOibGl4F?MeH)Bhi`HMyAKJg1E>k4m2kb!h2wAf1^`?;!_b(uE zpG-ue{1c!@?0*4|{T5Qheo0^g`$v*wm>&36n7pA^=5~_(%ZW2Z3sOT4bO0eq#!7D@ zF>(I_68p*MC(1tta>V`@@VIXwMcfw$;!mf^q1&*>`Yn6D3#VMP+78<~#kXUnXVVFB zf_g0f6VrT))>o!izdQ~5+?Xw$SY7naetlT)zUw@!)$hOC?OZPK)Bmqry>n;%`s$l^ zuHBx0>+1UTtE;bHbKbi8`kU8Q%V<5e>UBP*76vW23m~Q*2F<&>qhDyjy#Ree^KS=; zsr77YyIXJ6zffz>vm-U)5c^nI)H_t8pTc%r;e6P#d$2}aSvxEnIawpFP8s?VIA5Z= zU7v5%U{B&kXTC7+&IR{f%bjQJvsQ5+SjG87SnH@ddmZQem7BNjT)%Ps?Q84K?bmPK znGgBah{OLD+NNhBgSCiZ8Jf>vZDLq*(;BQr49n2$25S?;lAGjUEn--Pj!)ZWZ_gzj z+aiW#3W>+IiSaT~;;}7aSf-G8Y?~OCi4u=(@r1MX%C)uEZr!~8#vQjOySe)6wY59W z%j;Kfy<+{e0Y}++{Vr@%-mW?9jP1@=ePj21?`yp4U0r=K=y-=#2l}7&Yoo9s-Nh3& zq`SYCCGBIbuw%4X)KDo{Cy-boNK9I1G!%y=!d*7uh3HoJGQ8(nco|+^EW8x13Jt5T zgn+`#Sy6x_ycIwRZv|MwTLGBx7KLV`&}6{e7_f?5*m?gIEI_yX)DIOb#sUVW%w^;eELJsNb;u zfkBW%lPL1enooy%s3qd5dNP)~Vmj>Q54V{th`Co0BwR|YVl3m~FN4xw8vLdR3eUVQ zgqc_AUL$?8zcdj!&KN^gdkg|xSk0DhOB>nw1_5M_!#e6#Q1-s_aWR}urKEg)=<3N; zK_v{3cXESMl$RX`m5>a407+iW8dO43w?QR&p=3}A$@K-5OPKmHtsi2Y;y%;4@a`gdvF-oK)c!NqYN~D}z6;z5*BIV>NE+vd4UrLHu_y=uZ z1o>4IqwuR>!$ zb(q)a*1I1#3k$-%-V0$I)5pEuH{^T0pC^_6Zu1sP+i+*M+UH@txn0nn6Qnsm$c9_K zk#2`#FYU6w)eh++sP^C*`-*?wh-~;2KXJxhl-|zKv>jbA^yFg7haj zhL9AJd`R;95E9~oIZeo?5R&WwIZg8W5E9}>I3(m#2ub!a9FqJ#goJn>4hi`bLXuq* zha|rbAt6qSLqa}A<6GUNQkfFkdRLyB-s&iNb>s-65=8`B;->FN%o%{lKeh| zgm_jC3HcO4lHD$cB)<8;^32`?Z67ng8 zBzqhVNq!$fLfj39gnSAi$sUJ8lHZ4r5O>2NA)i7>vd7_&~T0G z`F&tZh%9_tLcRpHBn!j0CHZv#1(Aaf1^E&{kpvV z;lB?B`4T{p1>i%GUk6Z-UciTfdW`E>vV@%}y( zAS9M30Y0F!JAOd$W`w?2}lbv}^-t&gT@-J2w- zoQIhPNQG+X9Hm({NnN=Jm$=Tw-8k*dPeZx!6y@y71(VRT=D5 z(YeAHi*k>b-m}YJ?G^RHvf%R9QZeT8*HTe?`D>}T4A-ATFTMz_a4k9|J90aGg|^1tcUwv z?UMM%KH0+4%U?Bj**(4db;!Lo{^{kfks;F*6|XRd`v8+JGAyRJ{1tjWXesF;!y>z* zFzF&gIP5I>B12g3lYEh(y+D4GkUufVi{d?oe0Pnv+H<42wdF1yYuCCv?Iyb?YqQ?1 zx0=;0XRFoe)H+TXmP!yZ>fZa<$r-djPBp6C#?F?*j$7(g+jVx^VQU-y?6h{;8#N_% zx=c>95oC39Gp33Of?#XBv(_Xr`CM7f_yNO!ySP@N`qfQ&?ZPAD2;2&DV`ocV)6gaO z2W2ln0wsCX9*D43s}uGag};?RIN(X9La-u}8r!#vO=L)Q_K{Z1&iM(fbi09?$<6~r2y$sfL zyAPQ0#gV6)_bjW1QbqCCm79=NdkwNH(~h@NklpCB4U5@O-8bX1Kc0`OTmn~ep3G@7 z5OCWQ_f$|alddHB=Y;&RL0(WjX?j0e8r+XE$7s^N{HXcS@S`qe;@%&2<<36&QFDL% zsI&w8!||i0wby>sgzaU)_GMus^B($9^P9qtW}ieqn*EUc=r0iR#SFP5`q579^E+&5 z6bm`4AHXDs8;k^py`8OFa0#FrT-P?&AnvxlJPdm{@NZQ2foB-p zuFm4DumbuD;bSp;w8KZp)^T2x;Uc`@J8(ze1R1x-jKl7K`6-gU1069PA;Y#|D+TLt^LeA%tlkG1)#1uLxWO%L$eMx0ZI&DTJFVu{2fRylZ_XI(;bnLjvNNv^EgRo!4CP-Z#y$G-`Zcna ztW<21jDDn3vRAAY>eJ3>#1urKgGP!O>-k8en7gZi^teMpTMh|fT1*WIU024J4%&AA z>RY)3!QQR3!Cp3QPSqR8y7~iy;iOSK>;X9k!QN?dQWF-A(BTqn2}vHs!w!+;Q9LZ` zOdiEAEJ2~sB%D_EAojGE?19JdPh(Fw6|=L~WB8}B=jLYMJ^LQRKaD+M2icU@dGg^h zI);B5dxoR^xe@bglQWZWZ}3b9cA~bOaNj9CuIYB2B4mM%=|0-anYq*C5NmDIX|y(L zTM!@Bn@($+nq<`L*z+%tBaR~$sW4f>`{6B-GTIsitF|U*jX` zc?W*n+iE?W_hHY&PQ=YRBm)1-+vImLBnRE{VeD7|G67%4+%K04P%MKMD?+ghFII$N z!)P%ZiVgE(5XS&r<2)_hA+zQPJ=AT3SXi3c2|pZzuJ=g0c7JCJQrKT`Q+|HR8=4rS z>JrPL9L+qFF9AR|%X>{eH6wx%L`BnV{pr z;S6LNRt=HhYiI5R1S;Koj)z7#Mxt5MfH?S945&X9C3IRx=m9;UGf_ehMhK;;2_>+E zXB1Fv*0zXg9;VrMAY(r_OAdr2xY8D}KN7|Ms1EyCJ@&aM_QxXFTYX@ET)_TBD(p{2 zu|K85{_O_xxEiLf$i7DMQCcM&HXes+|XPij08= zcS}R|0TLG@@aKbvEENHX9MkU~J9OgFut_r8Lxg=eNXUbSEbZXd8~*C>5QS63yjMXD zR*;g3YO~pblnk5yp~GJv9?HY}FNlnSKSD;4|HZiY9A_Nk;${9v(f*Ch9^@p7g*|8l zmdSl|c^|SDcptKt{7Yl?w+wPh%=-B5i3el$!b)VeKWEJSm_fb_D)?6<6+Ggr;M7-# zArl)%+pbCH8VUH{AZhtn?-+BGC644(#m6N zjCuA(?hZVFZ*Sz@ga`2Mjoe%C0RFv^`xHEYhi~LQ4G-Ys8@bQG1Jvr}-i8OL)ysVr z9-vk)_c?fgTD{!2!2{Il0 z?R%gr(8~xNQKI|rw|sJk^7mMWQjY6T7K9FEQS4Bbk~)-SwL{r;xHbt3vBf|%eSCHi z7G=-vy3hEJU^V6ncW?#!*9>w_G(T@F;q{{~Fi7H;?GjCCbbc;4&d2Yy(Sb(^=^=l{ z=&OYE{m$-Ze4l92LsH49$_W^fc{3s~*PJu#NmrWw8h4Vb9EI<0E)c#6yG{r*OYjyf zo&;Y}Ol8LXluv2K-x6oW&kJV!t)dyfkYvUmSIv0W>GHKY(6*X|d=J<-I6#F8_WQ@$y?<_zXK z?Kx(?p@di!mYHG3oDB@!-#p@)Gj#RLoS~ynXU_eUPifApapt@xnDZw@bH0{j&M&Fv zyz7Ddq&euw{h~RySLXb3sO-2o+p;;s(IG{iIlmH8h?{eS!uXlq^3oGdd`PTd z&dgvTBh1a2@7@y3nSJ&8ZO7ZGMUu^J0W96{3_j!KgN7j=l&TxQ?p1p=jh{$HL@L4Z z$WprLkBrqnFw*85;C2NuLk?pCmNN+ui(4W-8O*sMy~;F(SXQ$$`q0U&hI^n(IONj541iK#J&N` zDP9w&^ry_%#p(HR^9CJpPg#K}m^bK%dy>jrj?9D$9K4xhe{$n4w0ASw-mPeRpA6eu z_uGRtAh#)TKuI1vWYI)S`z-5v!yA6Rk!JXHS)wmtcKBfB01KPW3|}&ApX|shmwY!@ z^hpni^|VNcOxY78Bo~470a9+;A3yWcTN+|+n`gb*@|F39F!pv+X|{59jWG7k@^QI; zIfYo;9l7(`eB2K0hl#ihug%Y(ku&lV);%rOy+o{g0$dquu{MsiSX;)rM}IhZM)vb` z^w8nsC(fO>%8!PNB>!mm8FJxEnYrkKHg&aH&LC_?ey;fS;VUrJ7_3}7`=!jO@Gqqv z?# zC&+)C%psrnWck5&70F}-d}{Y3k*BgM^oc!^uDr@GNz>^Fw)nJ>ymI>xxAD7na!_l698d4*-8c;0z z{*_J6cD&A!r`C~~Kl?X%@8&Y?g6TK8W?823)cw+fpB?%ux&7_W4KM!jfuS22z^9?1 z!MK!bGorQ)tkR;l@Oc?aVvnAvb9aAxS*Q8vcWio+jZN5cUaRDCS){jT#~Gbu>$U?@ z*;w}Qr#5{%eME~VE_1*1+&Sm~~{?PM>n{!({g?ryECL&0uQ{*RY0jC;?W)a`aR47;fP=?y)5<6I}ZYv$kg z-?}F3o*<=?DDSJ(5iKtK)vGyKt<8*NxThb*Hai=T3IC z>(JO6ZdAJG+#SJglqKlVxAadvZmcrTy{W95_ZKrHo$YDnbh%tK@9)?zNAr3}*ZX|& zAc9S+o6GxSCZ)1^wMl?&wiRuht#6z%y@F8FFIc>WFmok^EAszLyq|nht1@gRI^H^_S+eeVUjEYcXrg2JP6CbfAR$_7 zn4vD%T$WydzS+g@g0{9zniapA&&!#Zox~w*n`U;!72fvN$FC5T_Wgy+uC3*BF-Q*; z7EfKHF~z^l%irCzoVfWZrqP_G`0p$EH+Vhd+xv{_UQ0Y*^)2r|>-I<5$4W10T8@p; zUQ<=+q&us9>u|i{FWRx4D|MNsle(>6#~MDHbbQ2DAL56lxEzy!;Z5{1#!f9YI~P++ z8FS-er(<$19tR~cyaay8F=~P2D+umNjYd2LpjWi zRvOwcaFjR@Yio9zE#_9njt|A+0q&DJvZa^=-zI@tGXHdj4wTNYfzwIwPbbkY9q`cU z;8+rfhVx=&ns5vo4k8ztX+faL{(+7dDA1IF1C{#+I?^vt#UKJr6$Cn}f1soN106F^ zplJgKI@UkX$NT~vH;6#T3j$3S1e$|K!P%DN!q;#hQNdR#kf`Bn6p&1Tukk>V2Vdzx zlFy?uSbb@B1io8}r(%zT6e~nO*cQRD3=qxlQU}~)nIOi(IPKT%cTNt7G&|vz$1x>{UdT9UN(I}wy=^rfi>W}=BBq{Rh9_cbIFSr( zx0~|fOJ^ovta}BVUf>)4M>PZ{xobrE0??u zHlUSDJ{LBil}kPkHlUSD{tRqDE0=seY(OiQoZ?eX!IQ8lV3>okyHE_hry0x#K>h@X z^2)*bP~IC6YI&hh%NL+lAVjTD3^h$4YL9$o#$(|gk{K9yXqtvWL!Ma5+$RBch&tP&8vh_N~trRN}46nM6o{{`DCROS@b)R;<(XVsKcEB(Fj zt2GCh0=(lM@91b(a7a9iNIfOc5Lg2Fj*?Q3=qWEg$Rya`lb|R7 zWB&eCOz>6TN^~EK#WVwy1UTxwz7-SuTzM|^@SUjxoTS<6N;HqB%qV99)SDA>p4z@VoN1q1%NSYxm>19}%usn~&%0ZOV=A@foWi=B-)!SOE zP9CG_(6YkcveMtOs?XBYYYBg;RRf%Auu@Bw()LEr4;;WUtIi<)u23J zem%h{@_auzmAyr?GFQpREjF`3o$_f0cgl;zoT8MfJSmxkDG4VmjFePAL`o{Rn#c^c7sK1N?o8iVC>#%jxVY# ziproPdn&H*$6XFw0plk4R|?ADS@+cn;g9|-fdAd_uRejVP6?5>w~`2dxq+hyVZp literal 0 HcmV?d00001 diff --git a/src/smlp_py/NN_verifiers/saved_model/variables/variables.index b/src/smlp_py/NN_verifiers/saved_model/variables/variables.index new file mode 100755 index 0000000000000000000000000000000000000000..8661b62f8721b1affbdab880b74838f01a4e28ec GIT binary patch literal 1779 zcmZQzVB=tvV&Y(Akl~Ma_HcFf4)FK%3vqPvagFzP@^W z$Fy=w!z%_8zUX9Pp8lj*I@lH#~@Fz zVPTFwp{^W^KvTXjX|R|ss}{5pQQ#}dFGpnU>bM8|BJM#kbgjkr>|#} zYfyZ!kAH}MenClQZe~?#k$x^wCb0lb@;O|1)kH?9j2I0!*nk?=zVwO_RTKfnPF^a3 zkmUe680=_`21e8RqrX1Mnu?R_WVl%h4a^|pj+)-hGm{Y~;BJJj8>}E*QeU__g%yPi zi41KPg#}C+0;Y4rcG^%F+CY<7fhK+5nWIizXtOB%UNrp$+!0!U9H+ zUn4)xe8LbUjwP`f>ch;VBsQRSv4HGS^xh;(Tx>ve{a^#>T6R!q4KOy0h>Q&;g$qm? z9Hue59G+4b8$gp-fhKMC*_lFIY%nRT-~ejaJ#FQBQAIIW=;14dU@4DDVFME|*F0|W zmX$RXqa?gRISFVQE6}vAOrd{fGGas~ekO$u4xlE^y(M!QwD}N)L{e%=V!R>#jL)F( z090`OeSQ2szYL!dA;nKXig&;Lb(2ehFC{fOv67I67qDWHOP(*Y1Xyz6Dcd=LW%~_K z3i!KfLIsxwUruUbQC?97#Nug80PZA#jf$OL@wgxSkKE362cF{e>Zfil)B#r0CX49@&Et; literal 0 HcmV?d00001 diff --git a/src/smlp_py/NN_verifiers/smlp_toy.onnx b/src/smlp_py/NN_verifiers/smlp_toy.onnx new file mode 100755 index 0000000000000000000000000000000000000000..97dfc7642c6a1461eeeddcea356569db34e024dd GIT binary patch literal 1777 zcmd;J7h*3-Gs@4)tB~R~)H5{GGgL4%O|~#Ju-eDVRm{bmlA2eX8lRb0P+G#JQJh*> znwnRVnV6#w7T5PpEb%SP(GN;ZObJUY%1lhkN%b$VG7yr)q0-7gN*srj5*x%Yu0}>K z+}gP`F|<2nCKfxUq+mKi3YSu#Dk3qh$lr9e|fF-!$XD6xP@ zu2x1aTpGEw&^5*z;tX49oN9qCzz8|GLGbYHWaPqU1eZ1z^Kpi;3~t3h*I)!9&^1t_ zAmJRrh1USgbOba28rB$L36+9|b~~0-1lNouyp2c*Z}Z4~WZ}BNAq&sdS|J=P983a?PMApy<_HXHycEKV=j^jP-)*Pf_1^B= zt4gZ`k5~H&c-8IC|MJ}X;J{5gEo-oum`Mfh3QR*eRxE#F+ji`?t=3jX4h{|$tZu@Y z?7bQo!!`=ro!9WU5xmT}-z4CTUCWr(Q2$U#PrwpTA1+ zKG7At`>hw++J$9i?3;J!pIt|Dx?NKEEW3`G+;$SOPi?iF!H&lCG(1sbdRpMf<+rxd zEDLQJfMDA@Ejz8vjKB!t0D7M2(iG@*n+u2SKJ2(=Xa4J(-Q9qlc5$=T*xBBdv+rB< zWgp94_5Ht=rQ6-$(YCwn@^jz8nZ9H6o0($DfV4}HfA7l_ooAVx}Ibg44^*5nuuUxG;8` literal 0 HcmV?d00001 diff --git a/src/smlp_py/NN_verifiers/test.onnx b/src/smlp_py/NN_verifiers/test.onnx new file mode 100755 index 0000000000000000000000000000000000000000..97dfc7642c6a1461eeeddcea356569db34e024dd GIT binary patch literal 1777 zcmd;J7h*3-Gs@4)tB~R~)H5{GGgL4%O|~#Ju-eDVRm{bmlA2eX8lRb0P+G#JQJh*> znwnRVnV6#w7T5PpEb%SP(GN;ZObJUY%1lhkN%b$VG7yr)q0-7gN*srj5*x%Yu0}>K z+}gP`F|<2nCKfxUq+mKi3YSu#Dk3qh$lr9e|fF-!$XD6xP@ zu2x1aTpGEw&^5*z;tX49oN9qCzz8|GLGbYHWaPqU1eZ1z^Kpi;3~t3h*I)!9&^1t_ zAmJRrh1USgbOba28rB$L36+9|b~~0-1lNouyp2c*Z}Z4~WZ}BNAq&sdS|J=P983a?PMApy<_HXHycEKV=j^jP-)*Pf_1^B= zt4gZ`k5~H&c-8IC|MJ}X;J{5gEo-oum`Mfh3QR*eRxE#F+ji`?t=3jX4h{|$tZu@Y z?7bQo!!`=ro!9WU5xmT}-z4CTUCWr(Q2$U#PrwpTA1+ zKG7At`>hw++J$9i?3;J!pIt|Dx?NKEEW3`G+;$SOPi?iF!H&lCG(1sbdRpMf<+rxd zEDLQJfMDA@Ejz8vjKB!t0D7M2(iG@*n+u2SKJ2(=Xa4J(-Q9qlc5$=T*xBBdv+rB< zWgp94_5Ht=rQ6-$(YCwn@^jz8nZ9H6o0($DfV4}HfA7l_ooAVx}Ibg44^*5nuuUxG;8` literal 0 HcmV?d00001 diff --git a/src/smlp_py/NN_verifiers/test_marabou.py b/src/smlp_py/NN_verifiers/test_marabou.py new file mode 100755 index 00000000..4263906a --- /dev/null +++ b/src/smlp_py/NN_verifiers/test_marabou.py @@ -0,0 +1,134 @@ +from src.smlp_py.NN_verifiers.verifiers import MarabouVerifier +from src.smlp_py.smtlib.text_to_sympy import TextToPysmtParser + +from pysmt.shortcuts import Symbol, And, Not, Or, Implies, simplify, LT, Real, Times, Minus, Plus, Equals, GE, ToReal, LE +from pysmt.typing import * +import tf2onnx +import numpy as np +from maraboupy.MarabouPythonic import * + + +if __name__ == "__main__": + from keras.models import load_model + + model = load_model("/home/ntinouldinho/Desktop/smlp/result/abc_smlp_toy_basic_nn_keras_model_complete.h5") + model_proto, external_tensor_storage = tf2onnx.convert.from_keras(model, opset=13, output_path="smlp_toy.onnx") + print("SAVING TO ONNX") + parser = TextToPysmtParser() + parser.init_variables(inputs=[("x1", "real"), ('x2', 'real'), ('p1', 'real'), ('p2', 'real'), + ('y1', 'real'), ('y2', 'real')]) + + mb = MarabouVerifier(parser=parser) + mb.init_variables(inputs=[("x1", "Real"), ('x2', 'Integer'), ('p1', 'Integer'), ('p2', 'Integer')], + outputs=[('y1', 'Real'), ('y2', 'Real')]) + + y1 = parser.get_symbol("y1") + y2 = parser.get_symbol("y2") + p1 = parser.get_symbol("p1") + p2 = parser.get_symbol("p2") + x1 = parser.get_symbol("x1") + x2 = parser.get_symbol("x2") + + x2_int = parser.create_integer_disjunction("x2", (-1, 1)) + p2_int = parser.create_integer_disjunction("p2", (3, 7)) + # alpha = (((-1 <= x2) & (0.0 <= x1) & (x2 <= 1) & (x1 <= 10.0)) & (((p2 < 5) & (x1 == 10.0)) & (x2 < 12))) + # beta = ((4 <= y1) & (6 <= y2)) + + + # with x as input: y1==6.847101329531717 & y2==10.31207527363552 + # with x as knob: y1==4.120704402283359 & + solution = And( + Equals(x1, Real(10)), + Equals(x2, Real(1)), + Equals(p1, Real(7)), + Equals(p2, Real(4)) + ) + + theta = And( + GE(p1, Real(6.8)), + GE(p2, Real(3.8)), + LE(p1, Real(7.2)), + LE(p2, Real(4.2)) + ) + alpha = And( + GE(x2, Real(-2)), + GE(x1, Real(0.0)), + LE(x2, Real(1)), + LE(x1, Real(11.0)), + And( + LT(p2, Real(5)), + Equals(x1, Real(10.0)), + LT(x2, Real(12)) + ) + ) + + beta = And( + GE(y1, Real(4)), + GE(y2, Real(8)), + ) + + not_beta = Or( + LT(y1, Real(4)), + LT(y2, Real(8)) + ) + eta = And( + GE(p1, Real(0.0)), + LE(p1, Real(10.0)), + GE(p2, Real(3)), + LE(p2, Real(7)), + Or( + p1.Equals(Real(2.0)), + p1.Equals(Real(4.0)), + p1.Equals(Real(7.0)) + ) + ) + # mb.apply_restrictions(x2_int) + # mb.apply_restrictions(p2_int) + # mb.apply_restrictions(beta) + # mb.apply_restrictions(alpha) + # mb.apply_restrictions(eta) + mb.apply_restrictions(solution) + + # mb.apply_restrictions(theta) + + witness= mb.solve() + print(witness) + +################## TEST PARSER ########################### +# if __name__ == "__main__": +# parser = TextToPysmtParser() +# parser.init_variables(inputs=[("x1", "real"), ('x2', 'int'), ('p1', 'real'), ('p2', 'int'), +# ('y1', 'real'), ('y2', 'real')]) +# +# mb = MarabouVerifier(parser=parser) +# mb.init_variables(inputs=[("x1", "Real"), ('x2', 'Integer'), ('p1', 'Integer'), ('p2', 'Integer')], +# outputs=[('y1', 'Real'), ('y2', 'Real')]) +# +# +# # (1<=(ite)) and (y<=4) and (y>=8) +# # ite_without_ite = Or(And(c, t), And(Not(c), f)) +# +# y1 = parser.get_symbol("y1") +# y2 = parser.get_symbol("y2") +# +# ex = parser.parse('(y1+y2)/2') +# +# c = y1 > y2 +# t = y1 +# f = y2 +# # ite_without_ite = Or(And(c, t), And(Not(c), f)) +# +# condition_true = Times(ToReal(c), y1) # y1 if y1 > y2 +# condition_false = Times(ToReal(Not(c)), y2) # y2 if y1 <= y2 +# +# # Combine them +# ite_without_ite = Plus(condition_true, condition_false) +# +# # Final expression (ite_without_ite >= 1) +# inequality = GE(ite_without_ite, Real(1)) +# +# # Combine with the inequality 1 <= ITE(c, t, f) +# # inequality = Real(1) <= ite_without_ite +# print(inequality) + + diff --git a/src/smlp_py/NN_verifiers/variables/variables.data-00000-of-00001 b/src/smlp_py/NN_verifiers/variables/variables.data-00000-of-00001 new file mode 100755 index 0000000000000000000000000000000000000000..496d9ce9b03efa36942169de7236b0106b4066e8 GIT binary patch literal 6806 zcmcJT4OCOt9l%NC?# zC&+)C%psrnWck5&70F}-d}{Y3k*BgM^oc!^uDr@GNz>^Fw)nJ>ymI>xxAD7na!_l698d4*-8c;0z z{*_J6cD&A!r`C~~Kl?X%@8&Y?g6TK8W?823)cw+fpB?%ux&7_W4KM!jfuS22z^9?1 z!MK!bGorQ)tkR;l@Oc?aVvnAvb9aAxS*Q8vcWio+jZN5cUaRDCS){jT#~Gbu>$U?@ z*;w}Qr#5{%eME~VE_1*1+&Sm~~{?PM>n{!({g?ryECL&0uQ{*RY0jC;?W)a`aR47;fP=?y)5<6I}ZYv$kg z-?}F3o*<=?DDSJ(5iKtK)vGyKt<8*NxThb*Hai=T3IC z>(JO6ZdAJG+#SJglqKlVxAadvZmcrTy{W95_ZKrHo$YDnbh%tK@9)?zNAr3}*ZX|& zAc9S+o6GxSCZ)1^wMl?&wiRuht#6z%y@F8FFIc>WFmok^EAszLyq|nht1@gRI^H^_S+eeVUjEYcXrg2JP6CbfAR$_7 zn4vD%T$WydzS+g@g0{9zniapA&&!#Zox~w*n`U;!72fvN$FC5T_Wgy+uC3*BF-Q*; z7EfKHF~z^l%irCzoVfWZrqP_G`0p$EH+Vhd+xv{_UQ0Y*^)2r|>-I<5$4W10T8@p; zUQ<=+q&us9>u|i{FWRx4D|MNsle(>6#~MDHbbQ2DAL56lxEzy!;Z5{1#!f9YI~P++ z8FS-er(<$19tR~cyaay8F=~P2D+umNjYd2LpjWi zRvOwcaFjR@Yio9zE#_9njt|A+0q&DJvZa^=-zI@tGXHdj4wTNYfzwIwPbbkY9q`cU z;8+rfhVx=&ns5vo4k8ztX+faL{(+7dDA1IF1C{#+I?^vt#UKJr6$Cn}f1soN106F^ zplJgKI@UkX$NT~vH;6#T3j$3S1e$|K!P%DN!q;#hQNdR#kf`Bn6p&1Tukk>V2Vdzx zlFy?uSbb@B1io8}r(%zT6e~nO*cQRD3=qxlQU}~)nIOi(IPKT%cTNt7G&|vz$1x>{UdT9UN(I}wy=^rfi>W}=BBq{Rh9_cbIFSr( zx0~|fOJ^ovta}BVUf>)4M>PZ{xobrE0??u zHlUSDJ{LBil}kPkHlUSD{tRqDE0=seY(OiQoZ?eX!IQ8lV3>okyHE_hry0x#K>h@X z^2)*bP~IC6YI&hh%NL+lAVjTD3^h$4YL9$o#$(|gk{K9yXqtvWL!Ma5+$RBch&tP&8vh_N~trRN}46nM6o{{`DCROS@b)R;<(XVsKcEB(Fj zt2GCh0=(lM@91b(a7a9iNIfOc5Lg2Fj*?Q3=qWEg$Rya`lb|R7 zWB&eCOz>6TN^~EK#WVwy1UTxwz7-SuTzM|^@SUjxoTS<6N;HqB%qV99)SDA>p4z@VoN1q1%NSYxm>19}%usn~&%0ZOV=A@foWi=B-)!SOE zP9CG_(6YkcveMtOs?XBYYYBg;RRf%Auu@Bw()LEr4;;WUtIi<)u23J zem%h{@_auzmAyr?GFQpREjF`3o$_f0cgl;zoT8MfJSmxkDG4VmjFePAL`o{Rn#c^c7sK1N?o8iVC>#%jxVY# ziproPdn&H*$6XFw0plk4R|?ADS@+cn;g9|-fdAd_uRejVP6?5>w~`2dxq+hyVZp literal 0 HcmV?d00001 diff --git a/src/smlp_py/NN_verifiers/variables/variables.index b/src/smlp_py/NN_verifiers/variables/variables.index new file mode 100755 index 0000000000000000000000000000000000000000..8661b62f8721b1affbdab880b74838f01a4e28ec GIT binary patch literal 1779 zcmZQzVB=tvV&Y(Akl~Ma_HcFf4)FK%3vqPvagFzP@^W z$Fy=w!z%_8zUX9Pp8lj*I@lH#~@Fz zVPTFwp{^W^KvTXjX|R|ss}{5pQQ#}dFGpnU>bM8|BJM#kbgjkr>|#} zYfyZ!kAH}MenClQZe~?#k$x^wCb0lb@;O|1)kH?9j2I0!*nk?=zVwO_RTKfnPF^a3 zkmUe680=_`21e8RqrX1Mnu?R_WVl%h4a^|pj+)-hGm{Y~;BJJj8>}E*QeU__g%yPi zi41KPg#}C+0;Y4rcG^%F+CY<7fhK+5nWIizXtOB%UNrp$+!0!U9H+ zUn4)xe8LbUjwP`f>ch;VBsQRSv4HGS^xh;(Tx>ve{a^#>T6R!q4KOy0h>Q&;g$qm? z9Hue59G+4b8$gp-fhKMC*_lFIY%nRT-~ejaJ#FQBQAIIW=;14dU@4DDVFME|*F0|W zmX$RXqa?gRISFVQE6}vAOrd{fGGas~ekO$u4xlE^y(M!QwD}N)L{e%=V!R>#jL)F( z090`OeSQ2szYL!dA;nKXig&;Lb(2ehFC{fOv67I67qDWHOP(*Y1Xyz6Dcd=LW%~_K z3i!KfLIsxwUruUbQC?97#Nug80PZA#jf$OL@wgxSkKE362cF{e>Zfil)B#r0CX49@&Et; literal 0 HcmV?d00001 diff --git a/src/smlp_py/NN_verifiers/verifiers.py b/src/smlp_py/NN_verifiers/verifiers.py new file mode 100755 index 00000000..78009ce6 --- /dev/null +++ b/src/smlp_py/NN_verifiers/verifiers.py @@ -0,0 +1,502 @@ +from abc import ABC, abstractmethod +from enum import Enum +from typing import List, Dict, Optional, Tuple + +from z3 import And, Or, Not, Implies, Bool +from maraboupy import Marabou +from maraboupy import MarabouCore +from maraboupy import MarabouUtils +import tensorflow as tf +from pysmt.shortcuts import Symbol, And, Not, Or, Implies, simplify, LT, Real, Times, Minus, Plus, Equals, Int, ToReal +from pysmt.typing import BOOL, REAL, INT +import numpy as np +from maraboupy.MarabouPythonic import * +from pysmt.walkers import IdentityDagWalker +from fractions import Fraction + +from src.smlp_py.smtlib.smt_to_pysmt import smtlib_to_pysmt +from src.smlp_py.smtlib.text_to_sympy import TextToPysmtParser +from tensorflow.python.framework.convert_to_constants import convert_variables_to_constants_v2 + + +_operators_ = [">=", "<=", "<", ">"] + +convert_comparison_operators = { + "=": MarabouCore.Equation.EQ, + "<=": MarabouCore.Equation.LE, + ">=": MarabouCore.Equation.GE + } + +class Verifier(ABC): + @abstractmethod + def add_disjunction(self): + pass + + +class Variable: + _input_index = 0 + _output_index = 0 + + class Type(Enum): + Real = 0 + Int = 1 + + class Bounds: + lower = -np.inf + upper = np.inf + + def __init__(self, form: Type, name="", is_input=True): + if is_input: + self.index = Variable._input_index + Variable._input_index += 1 + else: + self.index = Variable._output_index + Variable._output_index += 1 + + self.form = form + self.name = name + self.is_input = is_input + self.bounds = Variable.Bounds() + + @staticmethod + def get_index(direction="output"): + return Variable._input_index if direction == "input" else Variable._output_index + + def set_lower_bound(self, lower): + self.bounds.lower = lower + + def set_upper_bound(self, upper): + self.bounds.upper = upper + +class MarabouVerifier(Verifier): + def __init__(self, model_path=None, parser=None): + # MarabouNetwork containing network instance + self.network = None + + # Dictionary containing variables + self.bounds = {} + + # Dictionary containing MarabouCommon.Disjunction for each variable + self.disjunctions = dict() + + # List containing yet to be added equations and statements + self.unprocessed_eq = [] + + # List of MarabouCommon.Equation currently applied to network query + self.equations = set() + + # Error variable bounding around excluded values, + # e.g. var != val -> And(var >= val + epsilon, var <= val - epsilon) + + # List of variables + self.variables = [] + + self.model_file_path = "./" + self.log_path = "marabou.log" + + # Adds conjunction of equations between bounds in form: + # e.g. Int(var), var >= 0, var <= 3 -> Or(var == 0, var == 1, var == 2, var == 3) + self.int_enable = False + + # Stack for keeping ipq + self.ipq_stack = [] + + self.model_file_path = "/home/ntinouldinho/Desktop/smlp/result/abc_smlp_toy_basic_nn_keras_model_complete.h5" + + self.convert_to_pb() + + self.parser = parser + + + + def epsilon(self, e, direction): + if direction == 'down': + return np.nextafter(e, -np.inf) + elif direction == 'up': + return np.nextafter(e, np.inf) + else: + raise ValueError("Direction must be 'up' or 'down'") + + + def convert_to_pb(self, output_model_file_path="."): + model = tf.keras.models.load_model(self.model_file_path) + tf.saved_model.save(model, output_model_file_path) + # Load the SavedModel + model = tf.saved_model.load(output_model_file_path) + concrete_func = model.signatures[tf.saved_model.DEFAULT_SERVING_SIGNATURE_DEF_KEY] + + # Convert to ConcreteFunction + frozen_func = convert_variables_to_constants_v2(concrete_func) + graph_def = frozen_func.graph.as_graph_def() + + # Save the frozen graph + with tf.io.gfile.GFile('model.pb', 'wb') as f: + f.write(graph_def.SerializeToString()) + + self.network = Marabou.read_tf('model.pb') + ipq = self.network.getInputQuery() + self.ipq_stack.append(ipq) + print("converted h5 to pb...") + + + def init_variables(self, inputs: List[Tuple[str, str]], outputs: List[Tuple[str, str]]) -> None: + for input_var in inputs: + name, type = input_var + var_type = Variable.Type.Real if type.lower() == "real" else Variable.Type.Int + self.variables.append(Variable(var_type, name, is_input=True)) + + for output_var in outputs: + name, type = output_var + var_type = Variable.Type.Real if type.lower() == "real" else Variable.Type.Int + self.variables.append(Variable(var_type, name, is_input=False)) + + + def get_variable_by_name(self, name: str) -> Optional[Tuple[Variable, int]]: + is_output = name.startswith("y") + + for index, variable in enumerate(self.variables): + if variable.name == name: + if is_output: + index -= Variable.get_index("input") + index = self.network.outputVars[0][0][index] if is_output else self.network.inputVars[0][0][index] + return variable, index + return None + + def reset(self): + self.network.clear() + self.network = Marabou.read_tf(self.model_file_path, modelType="savedModel_v2") + + # Default bounds for network + + + if self.int_enable: + for variable in self.variables: + if variable.type == Variable.Type.Int: + possible_values = list(range(int(variable.bounds.min), int(variable.bounds.max))) + possible_values = map(lambda p: "{var} == {val}".format(var=variable.name, val=p), possible_values) + self.add_disjunctions(possible_values) + print("{var} is int type adding values in range {min} to {max}".format(var=variable, + min=variable.bounds.min, + max=variable.bounds.max)) + + def add_bound(self, variable:str, value, direction="upper", strict=True): + var, var_index = self.get_variable_by_name(variable) + if var is None: + return None + + epsilon_direction = "down" if direction == "upper" else "up" + value = self.epsilon(value, epsilon_direction) if strict else value + + if direction == "upper": + self.network.setUpperBound(var_index, value) + var.set_upper_bound(value) + elif direction == "lower": + self.network.setLowerBound(var_index, value) + var.set_lower_bound(value) + + + # TODO: CHECK IF MARABOU NATIVELY SUPPORTS INTEGERS: it does not + def add_bounds(self, variable, bounds=None, num="real", grid=None): + var, is_output = self.get_variable_by_name(variable) + if var is None: + return None + + # TODO: handle case when one of the two is None + if bounds: + lower, upper = bounds + self.network.setLowerBound(var.index, lower) + self.network.setUpperBound(var.index, upper) + + if num == "int": + # add all distinct integer values + grid = range(lower, upper+1) + + if num in ["int", "grid"] and grid is not None: + disjunction = [] + for i in grid: + eq1 = MarabouUtils.Equation(MarabouCore.Equation.EQ) + eq1.addAddend(1, var.index) + eq1.setScalar(i) + disjunction.append([eq1]) + + self.network.addDisjunctionConstraint(disjunction) + + def apply_restrictions(self, formula): + conjunctions, disjunctions = self.process_formula(formula) + + for conjunction in conjunctions: + self.process_comparison(conjunction) + + self.process_disjunctions(disjunctions) + + def transform_pysmt_to_marabou_equation(self, formula): + symbol, comparator, scalar = formula + symbol, is_output = self.get_variable_by_name(str(symbol)) + equation_type = None + scalar = float(scalar.constant_value()) + + if comparator in convert_comparison_operators: + equation_type = convert_comparison_operators[comparator] + else: + if comparator == '<': + equation_type = MarabouCore.Equation.LE + scalar = self.epsilon(scalar, "down") + elif comparator == '>': + equation_type = MarabouCore.Equation.GE + scalar = self.epsilon(scalar, "up") + + equation = MarabouUtils.Equation(equation_type) + equation.addAddend(1, symbol.index) + equation.setScalar(scalar) + return equation + + def create_equation(self, formula): + equations = [] + if formula.is_and(): + equation = [self.create_equation(eq) for eq in formula.args()] + return equation + elif formula.is_le() or formula.is_lt() or formula.is_equals(): + res = self.parser.extract_components(formula) + equations.append(self.transform_pysmt_to_marabou_equation(res)) + + return equations + + def process_disjunctions(self, disjunctions): + marabou_disjunction = [] + for disjunction in disjunctions: + # split the disjunction into separate formulas + for formula in disjunction.args(): + equation = self.create_equation(formula) + marabou_disjunction.append(equation) + + if len(marabou_disjunction) > 0: + self.network.addDisjunctionConstraint(marabou_disjunction) + + def process_formula(self, formula): + conjunctions = [] + disjunctions = [] + + def traverse(node, source=[]): + if node.is_and(): + # conjunctions.extend(node.args()) + for arg in node.args(): + traverse(arg, conjunctions) + elif node.is_or(): + disjunctions.append(node) + elif node.is_le() or node.is_lt() or node.is_equals(): + source.append(node) + else: + # Leaf nodes (symbols, literals, etc.) are not conjunctions or disjunctions + pass + + traverse(formula) + return conjunctions, disjunctions + + def process_comparison(self, formula): + if formula.is_le() or formula.is_lt() or formula.is_equals(): + symbol, comparison, constant = self.parser.extract_components(formula) + symbol = str(symbol) + constant = float(constant.constant_value()) + + if comparison == "<=": + self.add_bound(symbol, constant, direction="upper", strict=False) + elif comparison == "<": + self.add_bound(symbol, constant, direction="upper", strict=True) + if comparison == ">=": + self.add_bound(symbol, constant, direction="lower", strict=False) + elif comparison == ">": + self.add_bound(symbol, constant, direction="lower", strict=True) + elif comparison == "=": + self.add_bound(symbol, constant, direction="lower", strict=False) + self.add_bound(symbol, constant, direction="upper", strict=False) + else: + return + + def alpha(self): + # (((-1 <= x2) & (0.0 <= x1) & (x2 <= 1) & (x1 <= 10.0)) & (((p2 < 5) & (x1 = 10.0)) & (x2 < 12))) + # p2<5 and x1==10 and x2<12 + # (p2≥5)∨(x1#10)∨(x2≥12) + + p1, is_output = self.get_variable_by_name("p1") + p2, is_output = self.get_variable_by_name("p2") + x1, is_output = self.get_variable_by_name("x1") + x2, is_output = self.get_variable_by_name("x2") + y1, is_output = self.get_variable_by_name("y1") + y2, is_output = self.get_variable_by_name("y2") + + # + # self.network.setUpperBound(p2.index, 5-epsilon) + v = Var(p2.index) + + # self.network.addConstraint(v <= self.epsilon(5, "down")) + # + # self.network.setUpperBound(x1.index, self.epsilon(10,'up')) + # self.network.setLowerBound(x1.index, self.epsilon(10, "down")) + # + # self.network.setUpperBound(x2.index, self.epsilon(12, "down")) + # + # self.network.setLowerBound(y1.index, 4) + # self.network.setUpperBound(y2.index, 8) + + # p1==4.0 or (p1==8.0 and p2 > 3) + eq1 = MarabouUtils.Equation(MarabouCore.Equation.EQ) + eq1.addAddend(1, p1.index) + eq1.setScalar(4) + + eq2 = MarabouUtils.Equation(MarabouCore.Equation.EQ) + eq2.addAddend(1, p1.index) + eq2.setScalar(8) + + eq3 = MarabouUtils.Equation(MarabouCore.Equation.GE) + eq3.addAddend(1, p2.index) + eq3.setScalar(self.epsilon(3, "up")) + + self.network.addDisjunctionConstraint([[eq1], [eq2, eq3]]) + + # b1 = self.network.getNewVariable() + # + # # Define the epsilon value + # epsilon = 1e-5 + # + # # Constraint for (y1 + y2) / 2 > 1 when b1 = 1 + # # This is equivalent to y1 + y2 > 2 + # self.network.addInequality([y1, y2, b1], [1, 1, -2], -epsilon) # y1 + y2 - 2*b1 > 0 -> y1 + y2 > 2 when b1 = 1 + # + # # Ensure b1 is binary + # self.network.setLowerBound(b1, 0) + # self.network.setUpperBound(b1, 1) + def find_witness(self, witness): + answers = {} + for variable in self.variables: + _, index = self.get_variable_by_name(variable.name) + answers[variable.name] = witness[index] + return answers + + def solve(self): + try: + results = self.network.solve() + if results and results[0] == 'unsat': + return {} + else: # sat + return self.find_witness(results[1]) + except Exception as e: + print(e) + return None + + def add_disjunction(self,): + pass + + + + +if __name__ == "__main__": + parser = TextToPysmtParser() + # p2 is an int not a real + parser.init_variables(inputs=[("x1", "real"), ('x2', 'int'), ('p1', 'real'), ('p2', 'real'), + ('y1', 'real'), ('y2', 'real')]) + + mb = MarabouVerifier(parser=parser) + mb.init_variables(inputs=[("x1", "Real"),('x2', 'Integer'), ('p1', 'Integer'), ('p2', 'Integer')], outputs=[('y1', 'Real'), ('y2', 'Real')]) + + + def linearize(expr): + """ + Linearize the given expression, ensuring it is in a linear format. + """ + if expr.is_real_constant(): + return expr, 0 + elif expr.is_symbol(): + return expr, 0 + elif expr.is_plus(): + lhs, lhs_const = linearize(expr.arg(0)) + rhs, rhs_const = linearize(expr.arg(1)) + return Plus(lhs, rhs), lhs_const + rhs_const + elif expr.is_minus(): + lhs, lhs_const = linearize(expr.arg(0)) + rhs, rhs_const = linearize(expr.arg(1)) + return Minus(lhs, rhs), lhs_const - rhs_const + elif expr.is_times(): + const_part = 1 + var_part = None + for arg in expr.args(): + if arg.is_real_constant(): + const_part *= arg.constant_value() + else: + var_expr, var_const = linearize(arg) + if var_const != 0: + raise ValueError(f"Non-linear term detected: {expr}") + if var_part is None: + var_part = var_expr + else: + raise ValueError(f"Non-linear term detected: {expr}") + return Times(Real(const_part), var_part), 0 + else: + raise ValueError(f"Unsupported operation: {expr}") + + + def simplify_to_linear(formula): + """ + Simplify a given formula to a linear format if possible. + """ + if formula.is_lt() or formula.is_le(): + lhs, lhs_const = linearize(formula.arg(0)) + rhs, rhs_const = linearize(formula.arg(1)) + return LT(Plus(lhs, Real(lhs_const - rhs_const)), rhs) + elif formula.is_gt() or formula.is_ge(): + lhs, lhs_const = linearize(formula.arg(0)) + rhs, rhs_const = linearize(formula.arg(1)) + return LT(rhs, Plus(lhs, Real(lhs_const - rhs_const))) + elif formula.is_equals(): + lhs, lhs_const = linearize(formula.arg(0)) + rhs, rhs_const = linearize(formula.arg(1)) + return Equals(Plus(lhs, Real(lhs_const - rhs_const)), rhs) + else: + raise ValueError(f"Unsupported formula type: {formula}") + + y1 = parser.get_symbol("y1") + y2 = parser.get_symbol("y2") + p1 = parser.get_symbol("p1") + p2 = parser.get_symbol("p2") + + # formula = ( (-1 <= 5*x2) | ( (0.0 == x1) & (x2 > 1) ) ) + # Construct the left-hand side: 0.1 * (x1 - 0.2) + lhs = Times(Real(0.1), Minus(y1, Real(0.2))) + + # Construct the right-hand side: 0.3 * (0.4 * (x2 - x1) - 0.5) + inner_term = Minus(y2, y1) + scaled_inner_term = Times(Real(0.4), inner_term) + rhs_inner = Minus(scaled_inner_term, Real(0.5)) + rhs = Times(Real(0.3), rhs_inner) + + # Construct the inequality: lhs < rhs + inequality = LT(lhs, rhs) + # f = simplify_to_linear(inequality) + # formula = parser.parse("p1==4.0 or (p1==8.0 and p2 > 3)") + # formula = parser.parse("((3 <= p2) & (p2 <= 4) & (7656119366529843/1125899906842624 <= p1) & (p1 <= 8106479329266893/1125899906842624))") + # formula = "(let ((|:0| (- p1 7))) (let ((|:1| (- p2 4))) (and (and true (<= (ite (< |:0| 0) (- |:0|) |:0|) (/ 1 5))) (<= (ite (< |:1| 0) (- |:1|) |:1|) (/ 1 5)))))" + + # formula = ((3 <= p2) & (p2 <= 4) & (7656119366529843/1125899906842624 <= p1) & (p1 <= 8106479329266893/1125899906842624)) + formula = And(p1.Equals(Real(4)), Or(p1.Equals(Real(8)), And(LT(Real(3), p2), p1.Equals(Real(5))))) + var_types = { + 'y1': 'REAL', + 'y2': 'REAL', + 'p1': 'REAL', + 'p2': 'INT', + 'x1': 'REAL', + 'x2': 'INT' + } + # formula = smtlib_to_pysmt(formula, var_types) + # mb.apply_restrictions(formula) + + + # mb.add_bounds("x1", (0,10)) + # mb.add_bounds("x2", (-1, 1), num="int") + # mb.add_bounds("p1", (0, 10), num="grid", grid=[2, 4, 7]) + # mb.add_bounds("p2", (3, 7), num="int") + # mb.alpha() + # + # for var in mb.network.outputVars[0][0]: + # print(var) + # + exitCode1, vals1, stats1 = mb.solve() + print(exitCode1) diff --git a/src/smlp_py/marabou/fake_marabou.py b/src/smlp_py/marabou/fake_marabou.py new file mode 100755 index 00000000..caeb2dfd --- /dev/null +++ b/src/smlp_py/marabou/fake_marabou.py @@ -0,0 +1,36 @@ +from maraboupy import Marabou +import numpy as np + +# %% +# Set the Marabou option to restrict printing +options = Marabou.createOptions(verbosity = 0) + +# %% +# Fully-connected network example +# ------------------------------- +# +# This network has inputs x0, x1, and was trained to create outputs that approximate +# y0 = abs(x0) + abs(x1), y1 = x0^2 + x1^2 +print("Fully Connected Network Example") +filename = "../../test.onnx" +network = Marabou.read_onnx(filename) + + +# %% +# Set input bounds +network.setLowerBound(0,-10.0) +network.setUpperBound(0, 10.0) +network.setLowerBound(1,-10.0) +network.setUpperBound(1, 10.0) +network.setLowerBound(2,-10.0) +network.setUpperBound(2, 10.0) +network.setLowerBound(3,-10.0) +network.setUpperBound(3, 10.0) +network.setLowerBound(4,-10.0) +network.setUpperBound(4, 10.0) +network.setLowerBound(5,-10.0) +network.setUpperBound(5, 10.0) + +# %% +# Call to Marabou solver +exitCode, vals, stats = network.solve(options = options) \ No newline at end of file diff --git a/src/smlp_py/marabou/marabou.py b/src/smlp_py/marabou/marabou.py new file mode 100755 index 00000000..695ad657 --- /dev/null +++ b/src/smlp_py/marabou/marabou.py @@ -0,0 +1,141 @@ +import sys +sys.path.append('/home/Desktop/Marabou/maraboupy') +from maraboupy import Marabou, MarabouCore +from maraboupy.MarabouCore import * +import numpy as np + +import os + + +class ONNXNetwork: + def __init__(self): + filename = "../../test.onnx" + # filename = "test.onnx" + self.network = Marabou.read_onnx(filename) + + def beta(self): + # self.network.setLowerBound(4, 4) + # self.network.setUpperBound(4, 10) + # + # self.network.setLowerBound(5, 8) + # self.network.setUpperBound(5, 20) + + # BEST SOLUTION + self.network.setLowerBound(4, 0.24) + self.network.setUpperBound(4, 10.7007) + + self.network.setLowerBound(5, 1.12) + self.network.setUpperBound(5, 12.02) + + def alpha(self): + # p2<5 and x1==10 and x2<12 + # (p2≥5)∨(x1#10)∨(x2≥12) + + epsilon = 1e-12 + + eq1 = MarabouCore.Equation(MarabouCore.Equation.GE) + eq1.addAddend(1, 3) + eq1.setScalar(5) + + eq2 = MarabouCore.Equation(MarabouCore.Equation.GE) + eq2.addAddend(1, 1) + eq2.setScalar(12) + + eq3 = MarabouCore.Equation(MarabouCore.Equation.GE) + eq3.addAddend(1, 0) + eq3.setScalar(10+epsilon) + + eq4 = MarabouCore.Equation(MarabouCore.Equation.LE) + eq4.addAddend(1, 0) + eq4.setScalar(10 - 1e-12) + + self.network.addDisjunctionConstraint([[eq1], [eq2], [eq3], [eq4]]) + + def add_bounds(self, var, bounds, num="real", grid=None): + lower, upper = bounds + self.network.setLowerBound(var, lower) + self.network.setUpperBound(var, upper) + + if num == "in": + disjunction = [] + + for i in range(lower, upper+1): + eq1 = MarabouCore.Equation(MarabouCore.Equation.EQ) + eq1.addAddend(1, var) + eq1.setScalar(i) + disjunction.append([eq1]) + + self.network.addDisjunctionConstraint(disjunction) + + if grid is not None: + disjunction = [] + + for num in grid: + eq1 = MarabouCore.Equation(MarabouCore.Equation.EQ) + eq1.addAddend(1, var) + eq1.setScalar(num) + disjunction.append([eq1]) + + self.network.addDisjunctionConstraint(disjunction) + + + def run_marabou(self): + options = Marabou.createOptions(verbosity = 10) + + grid = [2, 4, 7] + for var in self.network.inputVars[0][0]: + # if var == 0: + # self.add_bounds(var, (0, 10)) + # elif var == 1: + # self.add_bounds(var, (-1, 1), num="int") + # elif var == 2: + # self.add_bounds(var, (0, 10), num="int", grid=grid) + # elif var == 3: + # self.add_bounds(var, (3, 7), num="int") + + # BEST SOLUTION + if var == 0: + self.add_bounds(var, (-0.8218, 9.546)) + elif var == 1: + self.add_bounds(var, (-1, 1), num="int") + elif var == 2: + self.add_bounds(var, (0.1, 10), num="int", grid=grid) + elif var == 3: + self.add_bounds(var, (3, 7), num="int") + + # self.alpha() + self.beta() + + exitCode, vals, stats = self.network.solve(options = options) + + # Test Marabou equations against onnxruntime at an example input point + # inputPoint = np.ones(inputVars.shape) + # marabouEval = network.evaluateWithMarabou([inputPoint], options = options)[0] + # onnxEval = network.evaluateWithoutMarabou([inputPoint])[0] + print(exitCode, vals, stats) +# ONNXNetwork().run_marabou() + +# if __name__ == "__main__": +# onnx_file = "/home/ntinouldinho/Desktop/Marabou/data/test.onnx" +# # property_filename = "/home/ntinouldinho/Desktop/Marabou/data/model_constraints.vnnlib" +# # onnx_file = "/home/ntinouldinho/Desktop/smlp/src/test.onnx" +# property_filename = "/home/ntinouldinho/Desktop/smlp/src/query.vnnlib" +# +# network = Marabou.read_onnx(onnx_file) +# network.saveQuery("./query.txt") +# +# try: +# ipq = Marabou.load_query("./query.txt") +# # MarabouCore.loadProperty(ipq, property_filename) +# exitCode_ipq, vals_ipq, _ = Marabou.solve_query(ipq, propertyFilename=property_filename, filename="res.log") +# print(exitCode_ipq, vals_ipq) +# +# except Exception as e: +# print(e) + +if __name__ == "__main__": + network = Marabou.read_tf("/home/ntinouldinho/Desktop/smlp/result/abc_smlp_toy_basic_model_checkpoint.h5") + + + + diff --git a/src/smlp_py/marabou/query.txt b/src/smlp_py/marabou/query.txt new file mode 100755 index 00000000..d7d7c6f5 --- /dev/null +++ b/src/smlp_py/marabou/query.txt @@ -0,0 +1,51 @@ +30 +12 +0 +14 +12 +4 +0,0 +1,1 +2,2 +3,3 +2 +0,28 +1,29 +12,0.0000000000 +13,0.0000000000 +14,0.0000000000 +15,0.0000000000 +16,0.0000000000 +17,0.0000000000 +18,0.0000000000 +19,0.0000000000 +24,0.0000000000 +25,0.0000000000 +26,0.0000000000 +27,0.0000000000 +0,0,0.0302654002,0,-0.1437274814,1,0.1061730981,2,-0.0870333686,3,-0.4645415545,4,-1.0000000000 +1,0,-0.0128128808,0,-0.5780213475,1,0.5900486112,2,0.5305879116,3,0.3497457504,5,-1.0000000000 +2,0,-0.0007434468,0,-0.6913169622,1,-0.1507877558,2,0.6713727117,3,-0.1944409609,6,-1.0000000000 +3,0,0.0132133318,0,-0.0250194836,1,0.1154540256,2,-0.4708667994,3,-0.4788347185,7,-1.0000000000 +4,0,0.0000000000,0,-0.5775315166,1,-0.1357627511,2,-0.3575040698,3,-0.5444254875,8,-1.0000000000 +5,0,-0.0274824426,0,0.5064330697,1,-0.6230512857,2,-0.4707299173,3,-0.3514576852,9,-1.0000000000 +6,0,-0.0666948929,0,0.2382464856,1,-0.3667045534,2,0.1506942511,3,0.5816352963,10,-1.0000000000 +7,0,-0.0941732377,0,0.5513240099,1,-0.2898558974,2,-0.6173257232,3,0.7682082653,11,-1.0000000000 +8,0,-0.1418720335,12,-0.4838356972,13,-0.2879619300,14,0.7321704030,15,-0.1759769320,16,-0.6894207597,17,0.3923604190,18,0.1298776269,19,0.0196413193,20,-1.0000000000 +9,0,0.0638892725,12,0.6492301226,13,0.2203886509,14,-0.2999479175,15,0.5637680888,16,-0.0014058352,17,0.3970938921,18,-0.6249498725,19,0.3417502046,21,-1.0000000000 +10,0,0.0540842414,12,-0.5587263703,13,0.2551203668,14,0.5727752447,15,0.4214664698,16,-0.2815549374,17,-0.5591005087,18,-0.0682931393,19,-0.7146613598,22,-1.0000000000 +11,0,-0.0614044182,12,-0.4547419548,13,0.5524955988,14,0.1784126908,15,-0.2580326498,16,-0.0371657610,17,0.5687257051,18,0.2459676862,19,0.7129698992,23,-1.0000000000 +12,0,0.0996098146,24,0.0004564780,25,0.8553681374,26,-0.3106124997,27,0.8385750651,28,-1.0000000000 +13,0,-0.1071689427,24,0.4485777915,25,0.1472451091,26,-0.8218147159,27,0.9324899912,29,-1.0000000000 +0,relu,12,4 +1,relu,13,5 +2,relu,14,6 +3,relu,15,7 +4,relu,16,8 +5,relu,17,9 +6,relu,18,10 +7,relu,19,11 +8,relu,24,20 +9,relu,25,21 +10,relu,26,22 +11,relu,27,23 \ No newline at end of file diff --git a/src/smlp_py/marabou/res.log b/src/smlp_py/marabou/res.log new file mode 100755 index 00000000..99c4c78f --- /dev/null +++ b/src/smlp_py/marabou/res.log @@ -0,0 +1,167 @@ +Engine::processInputQuery: Input query (before preprocessing): 14 equations, 30 variables +Engine::processInputQuery: Input query (after preprocessing): 26 equations, 35 variables + +Input bounds: + x0: [ 10.0000, 10.0000] [FIXED] + x1: [ -1.0000, 12.0000] + x2: [ 4.0000, 4.0000] [FIXED] + x3: [ 3.0000, 7.0000] + +Branching heuristics set to LargestInterval + +Engine::solve: Initial statistics + +10:56:59 Statistics update: + --- Time Statistics --- + Total time elapsed: 2 milli (00:00:00) + Main loop: 0 milli (00:00:00) + Preprocessing time: 2 milli (00:00:00) + Unknown: 0 milli (00:00:00) + Breakdown for main loop: + [0.00%] Simplex steps: 0 milli + [0.00%] Explicit-basis bound tightening: 0 milli + [0.00%] Constraint-matrix bound tightening: 0 milli + [0.00%] Degradation checking: 0 milli + [0.00%] Precision restoration: 0 milli + [0.00%] Statistics handling: 0 milli + [0.00%] Constraint-fixing steps: 0 milli + [0.00%] Valid case splits: 0 milli. Average per split: 0.00 milli + [0.00%] Applying stored bound-tightening: 0 milli + [0.00%] SMT core: 0 milli + [0.00%] Symbolic Bound Tightening: 0 milli + [0.00%] SoI-based local search: 0 milli + [0.00%] SoI-based local search: 0 milli + [0.00%] Unaccounted for: 0 milli + --- Preprocessor Statistics --- + Number of preprocessor bound-tightening loop iterations: 5 + Number of eliminated variables: 9 + Number of constraints removed due to variable elimination: 7 + Number of equations removed due to variable elimination: 0 + --- Engine Statistics --- + Number of main loop iterations: 1 + 0 iterations were simplex steps. Total time: 0 milli. Average: 0.00 milli. + 0 iterations were constraint-fixing steps. Total time: 0 milli. Average: 0.00 milli + Number of active piecewise-linear constraints: 4 / 5 + Constraints disabled by valid splits: 1. By SMT-originated splits: 0 + Last reported degradation: 0.0000000000. Max degradation so far: 0.0000000000. Restorations so far: 0 + Number of simplex pivots we attempted to skip because of instability: 0. + Unstable pivots performed anyway: 0 + --- Tableau Statistics --- + Total number of pivots performed: 0 + Real pivots: 0. Degenerate: 0 (0.00%) + Degenerate pivots by request (e.g., to fix a PL constraint): 0 (0.00%) + Average time per pivot: 0.00 milli + Total number of fake pivots performed: 0 + Total number of rows added: 0. Number of merged columns: 0 + Current tableau dimensions: M = 26, N = 61 + --- SMT Core Statistics --- + Total depth is 0. Total visited states: 1. Number of splits: 0. Number of pops: 0 + Max stack depth: 0 + --- Bound Tightening Statistics --- + Number of tightened bounds: 0. + Number of rows examined by row tightener: 0. Consequent tightenings: 0 + Number of explicit basis matrices examined by row tightener: 0. Consequent tightenings: 0 + Number of bound tightening rounds on the entire constraint matrix: 0. Consequent tightenings: 0 + Number of bound notifications sent to PL constraints: 30. Tightenings proposed: 0 + --- Basis Factorization statistics --- + Number of basis refactorizations: 2 + --- Projected Steepest Edge Statistics --- + Number of iterations: 0. + Number of resets to reference space: 1. Avg. iterations per reset: 0 + --- SBT --- + Number of tightened bounds: 11 + --- SoI-based local search --- + Number of proposed phase pattern update: 0. Number of accepted update: 0 [0.00%] + Total time (% of local search time) updating SoI phase pattern : 0 milli [0.00%] + Total time obtaining current assignment: 0 milli [0.00%] + Total time getting SoI phase pattern : 0 milli [0.00%] + --- Context dependent statistics --- + Number of pushes / pops: 0 / 0 + [0.00%] Pre-Push hook: 0 milli + [0.00%] Push : 0 milli + [0.00%] Post-Pop hook: 0 milli + [0.00%] Pop : 0 milli + [0.00%] Total context-switching time: 0 milli + --- Proof Certificate --- + Number of certified leaves: 0 + Number of leaves to delegate: 0 + +--- +Before declaring sat, recomputing... + +Engine::solve: sat assignment found + +10:56:59 Statistics update: + --- Time Statistics --- + Total time elapsed: 2 milli (00:00:00) + Main loop: 0 milli (00:00:00) + Preprocessing time: 2 milli (00:00:00) + Unknown: 0 milli (00:00:00) + Breakdown for main loop: + [21.85%] Simplex steps: 0 milli + [9.66%] Explicit-basis bound tightening: 0 milli + [0.00%] Constraint-matrix bound tightening: 0 milli + [0.00%] Degradation checking: 0 milli + [0.00%] Precision restoration: 0 milli + [2.10%] Statistics handling: 0 milli + [0.00%] Constraint-fixing steps: 0 milli + [4.62%] Valid case splits: 0 milli. Average per split: 0.00 milli + [0.84%] Applying stored bound-tightening: 0 milli + [0.00%] SMT core: 0 milli + [387.39%] Symbolic Bound Tightening: 0 milli + [0.00%] SoI-based local search: 0 milli + [0.00%] SoI-based local search: 0 milli + [7750732804079643648.00%] Unaccounted for: 0 milli + --- Preprocessor Statistics --- + Number of preprocessor bound-tightening loop iterations: 5 + Number of eliminated variables: 9 + Number of constraints removed due to variable elimination: 7 + Number of equations removed due to variable elimination: 0 + --- Engine Statistics --- + Number of main loop iterations: 8 + 5 iterations were simplex steps. Total time: 0 milli. Average: 0.00 milli. + 0 iterations were constraint-fixing steps. Total time: 0 milli. Average: 0.00 milli + Number of active piecewise-linear constraints: 4 / 5 + Constraints disabled by valid splits: 1. By SMT-originated splits: 0 + Last reported degradation: 0.0000000000. Max degradation so far: 0.0000000000. Restorations so far: 0 + Number of simplex pivots we attempted to skip because of instability: 0. + Unstable pivots performed anyway: 0 + --- Tableau Statistics --- + Total number of pivots performed: 5 + Real pivots: 5. Degenerate: 0 (0.00%) + Degenerate pivots by request (e.g., to fix a PL constraint): 0 (0.00%) + Average time per pivot: 0.00 milli + Total number of fake pivots performed: 0 + Total number of rows added: 0. Number of merged columns: 0 + Current tableau dimensions: M = 26, N = 61 + --- SMT Core Statistics --- + Total depth is 0. Total visited states: 1. Number of splits: 0. Number of pops: 0 + Max stack depth: 0 + --- Bound Tightening Statistics --- + Number of tightened bounds: 2. + Number of rows examined by row tightener: 5. Consequent tightenings: 1 + Number of explicit basis matrices examined by row tightener: 1. Consequent tightenings: 2 + Number of bound tightening rounds on the entire constraint matrix: 0. Consequent tightenings: 0 + Number of bound notifications sent to PL constraints: 60. Tightenings proposed: 0 + --- Basis Factorization statistics --- + Number of basis refactorizations: 2 + --- Projected Steepest Edge Statistics --- + Number of iterations: 5. + Number of resets to reference space: 1. Avg. iterations per reset: 5 + --- SBT --- + Number of tightened bounds: 13 + --- SoI-based local search --- + Number of proposed phase pattern update: 0. Number of accepted update: 0 [0.00%] + Total time (% of local search time) updating SoI phase pattern : 0 milli [0.00%] + Total time obtaining current assignment: 0 milli [0.00%] + Total time getting SoI phase pattern : 0 milli [0.00%] + --- Context dependent statistics --- + Number of pushes / pops: 0 / 0 + [0.00%] Pre-Push hook: 0 milli + [0.00%] Push : 0 milli + [0.00%] Post-Pop hook: 0 milli + [0.00%] Pop : 0 milli + [0.00%] Total context-switching time: 0 milli + --- Proof Certificate --- + Number of certified leaves: 0 + Number of leaves to delegate: 0 diff --git a/src/smlp_py/smtlib/__init__.py b/src/smlp_py/smtlib/__init__.py new file mode 100755 index 00000000..e69de29b diff --git a/src/smlp_py/smtlib/parser.py b/src/smlp_py/smtlib/parser.py new file mode 100755 index 00000000..3f407ae3 --- /dev/null +++ b/src/smlp_py/smtlib/parser.py @@ -0,0 +1,96 @@ +import ast +from pysmt.shortcuts import Symbol, And, Or, Not, Implies, Iff, Ite, Equals, Plus, Minus, Times, Div, Pow, Bool, TRUE, FALSE, Int, Real +from pysmt.typing import BOOL, REAL, INT + +pysmt_types = { + "int": INT, + "real": REAL, + "bool": BOOL +} + +class TextToPysmtParser(object): + def __init__(self): + self.symbols = {} + self._ast_operators_map = { + ast.Add: Plus, # Addition + ast.Sub: Minus, # Subtraction + ast.Mult: Times, # Multiplication + ast.Div: Div, # Division + ast.Pow: Pow, # Exponentiation + ast.BitXor: Iff, # Bitwise XOR (interpreted as logical Iff) + + ast.USub: Minus, # Unary subtraction (negation) + + ast.Eq: Equals, # Equality + ast.NotEq: Not, # Not equal + ast.Lt: lambda l, r: l < r, # Less than + ast.LtE: lambda l, r: l <= r, # Less than or equal to + ast.Gt: lambda l, r: l > r, # Greater than + ast.GtE: lambda l, r: l >= r, # Greater than or equal to + + ast.And: And, # Logical AND + ast.Or: Or, # Logical OR + ast.Not: Not, # Logical NOT + + ast.IfExp: Ite # If expression + } + + def add_symbol(self, name, symbol_type): + assert symbol_type in pysmt_types.keys() + self.symbols[name] = Symbol(name, pysmt_types[symbol_type]) + + def parse(self, expr): + assert isinstance(expr, str) + symbol_list = self.symbols + + def eval_(node): + if isinstance(node, ast.Num): + return Real(node.n) if isinstance(node.n, float) else Int(node.n) + elif isinstance(node, ast.BinOp): + return self._ast_operators_map[type(node.op)](eval_(node.left), eval_(node.right)) + elif isinstance(node, ast.UnaryOp): + return self._ast_operators_map[type(node.op)](eval_(node.operand)) + elif isinstance(node, ast.Name): + return symbol_list[node.id] + elif isinstance(node, ast.BoolOp): + res_boolop = self._ast_operators_map[type(node.op)](eval_(node.values[0]), eval_(node.values[1])) + for value in node.values[2:]: + res_boolop = self._ast_operators_map[type(node.op)](res_boolop, eval_(value)) + return res_boolop + elif isinstance(node, ast.Compare): + left = eval_(node.left) + first_comparator = eval_(node.comparators[0]) + result = self._ast_operators_map[type(node.ops[0])](left, first_comparator) + for op, comparator in zip(node.ops[1:], node.comparators[1:]): + left = eval_(comparator) + result = And(result, self._ast_operators_map[type(op)](left, eval_(comparator))) + return result + elif isinstance(node, ast.IfExp): + return self._ast_operators_map[ast.IfExp](eval_(node.test), eval_(node.body), eval_(node.orelse)) + elif isinstance(node, ast.Constant): + if node.value is True: + return TRUE() + elif node.value is False: + return FALSE() + elif isinstance(node.value, int): + return Int(node.value) + elif isinstance(node.value, float): + return Real(node.value) + else: + return node.value + else: + raise TypeError(f"Unexpected node type {type(node)}") + + return eval_(ast.parse(expr, mode='eval').body) + + +if __name__ == "__main__": + + parser = TextToPysmtParser() + parser.add_symbol('x1', 'int') + parser.add_symbol('x2', 'real') + parser.add_symbol('p2', 'real') + + formula = parser.parse('p2<5.0 and x1==10 and x2<12.0') + print(formula) + diff --git a/src/smlp_py/smtlib/smt_to_pysmt.py b/src/smlp_py/smtlib/smt_to_pysmt.py new file mode 100755 index 00000000..bc412854 --- /dev/null +++ b/src/smlp_py/smtlib/smt_to_pysmt.py @@ -0,0 +1,190 @@ +import re + +from pysmt.smtlib.parser import SmtLibParser +from pysmt.shortcuts import Symbol, simplify, get_env +from pysmt.typing import REAL, INT, BOOL +from io import StringIO +from pysmt.rewritings import CNFizer + +from pysmt.shortcuts import Symbol, And, LE, Real, Ite, Or, Not, LT, Equals, Plus, Minus, Times, Div +from pysmt.typing import REAL +from sympy import sympify + + +def smtlib_to_pysmt(smt_query, var_types): + """ + Converts an SMT-LIB query string to a PySMT formula. + + Parameters: + smt_query (str): The SMT-LIB query string. + var_types (dict): A dictionary mapping variable names to their types (REAL, INT, BOOL). + + Returns: + pysmt.shortcuts.FNode: The PySMT formula. + """ + # Initialize the SMT-LIB parser + parser = SmtLibParser() + + # Build the declarations for the variables + declarations = [] + for var, vtype in var_types.items(): + if vtype == 'REAL': + declarations.append(f"(declare-fun {var} () Real)") + Symbol(var, REAL) + elif vtype == 'INT': + declarations.append(f"(declare-fun {var} () Int)") + Symbol(var, INT) + elif vtype == 'BOOL': + declarations.append(f"(declare-fun {var} () Bool)") + Symbol(var, BOOL) + else: + raise ValueError(f"Unsupported variable type: {vtype}") + + # Join the declarations with the original SMT-LIB query + smt_query_with_declarations = "\n".join(declarations) + f"\n(assert {smt_query})" + + # Parse the SMT-LIB query + script = parser.get_script(StringIO(smt_query_with_declarations)) + + # Extract the formula from the script + formula = script.get_last_formula() + + # Simplify the parsed formula + simplified_formula = simplify(formula) + + return simplified_formula + + +def convert_fractions_to_floats(formula: str) -> str: + # Regular expression to find fractions in the format 'numerator/denominator' + fraction_pattern = re.compile(r'(\d+/\d+)') + + def fraction_to_float(match): + fraction = match.group() + numerator, denominator = map(int, fraction.split('/')) + return str(numerator / denominator) + + # Substitute all fractions in the formula with their float equivalents + formula_with_floats = fraction_pattern.sub(fraction_to_float, formula) + + return formula_with_floats + + +def convert_ternary_to_logic(formula: str) -> str: + # Regular expression to find ternary statements in the format 'condition ? true_expr : false_expr' + ternary_pattern = re.compile(r'\(([^()]+)\?\(([^()]+)\):\(([^()]+)\)\)') + + while ternary_pattern.search(formula): + formula = ternary_pattern.sub( + lambda match: f'(({match.group(1)}) & ({match.group(2)}) | (~({match.group(1)}) & ({match.group(3)})))', + formula) + + return formula + +def pysmt_convert_fractions_to_floats(term): + if term.is_constant() and term.is_real_constant(): + value = term.constant_value() + return Real(float(value)) + elif term.is_symbol(): + return term + elif term.is_plus() or term.is_minus() or term.is_times() or term.is_div(): + if term.node_type() == 12: + return Plus(*[pysmt_convert_fractions_to_floats(arg) for arg in term.args()]) + elif term.node_type() == 13: + return Minus(*[pysmt_convert_fractions_to_floats(arg) for arg in term.args()]) + elif term.node_type() == 14: + return Times(*[pysmt_convert_fractions_to_floats(arg) for arg in term.args()]) + elif term.node_type() == 15: + return Div(*[pysmt_convert_fractions_to_floats(arg) for arg in term.args()]) + return term + +def recursively_convert_ite(term): + if term.is_ite(): + condition = term.arg(0) + true_branch = recursively_convert_ite(term.arg(1)) + false_branch = recursively_convert_ite(term.arg(2)) + return Or(And(condition, true_branch), And(Not(condition), false_branch)) + elif term.is_and(): + return And(*[recursively_convert_ite(arg) for arg in term.args()]) + elif term.is_or(): + return Or(*[recursively_convert_ite(arg) for arg in term.args()]) + elif term.is_not(): + return Not(recursively_convert_ite(term.arg(0))) + elif term.is_le() or term.is_lt() or term.is_equals(): + left = recursively_convert_ite(term.arg(0)) + right = recursively_convert_ite(term.arg(1)) + if term.node_type() == 16: + return LE(left, right) + elif term.node_type() == 17: + return LT(left, right) + elif term.node_type() == 18: + return Equals(left, right) + else: + return pysmt_convert_fractions_to_floats(term) + +# node types: +# 12: + +# 13: - +# 14: / +# 15: * +# 16: <= +# 17: < +# 18: == +# 19: ITE + + +# Example usage +if __name__ == "__main__": + + + + # Define the SMT-LIB query as a string + # smt_query = "(let ((|:0| (* (/ 281474976710656 2944425288877159) (- y1 (/ 1080863910568919 4503599627370496))))) (let ((|:1| (* (/ 281474976710656 2559564553220679) (- (* (/ 1 2) (+ y1 y2)) (/ 1170935903116329 1125899906842624))))) (and (>= (ite (< |:0| |:1|) |:0| |:1|) 1) (and (>= y1 4) (>= y2 8)))))" + smt_query = "(and (and true (and (>= x1 0) (<= x1 10))) (and (>= x2 (- 1)) (<= x2 1)))" + # Define variable types + var_types = { + 'y1': 'REAL', + 'y2': 'REAL', + 'p1': 'REAL', + 'p2': 'REAL', + 'x1': 'REAL', + 'x2': 'REAL' + } + + # Convert SMT-LIB to PySMT + pysmt_formula = smtlib_to_pysmt(smt_query, var_types) + # + # pysmt_formula = recursively_convert_ite(pysmt_formula) + # + # print(pysmt_formula.serialize()) + + # pysmt_formula = pysmt_formula.serialize() + # # Print the PySMT formula + # print("Converted PySMT Formula:") + # print(pysmt_formula) + # + # pysmt_formula = convert_fractions_to_floats(pysmt_formula) + # print("Removed fractions:") + # print(pysmt_formula) + # + # pysmt_formula = recursively_convert_ite(sympify(pysmt_formula)) + # # pysmt_formula = convert_ternary_to_logic(pysmt_formula) + # print("Removed ITE:") + # print(pysmt_formula) + + + cnfizer = CNFizer() + cnf_formula = cnfizer.convert(pysmt_formula) + print("CNF PySMT Formula:") + print(cnf_formula) + + # Example: Add an additional constraint to the formula + # p1 = Symbol('p1', ) + # additional_constraint = p1 != 3 + # combined_formula = simplify(pysmt_formula & additional_constraint) + + # print("Combined Formula with Additional Constraint:") + # print + + +##################################################################################### diff --git a/src/smlp_py/smtlib/text_to_sympy.py b/src/smlp_py/smtlib/text_to_sympy.py new file mode 100755 index 00000000..27ac283c --- /dev/null +++ b/src/smlp_py/smtlib/text_to_sympy.py @@ -0,0 +1,491 @@ +import re + +from pysmt import * +from sympy.logic.boolalg import And, Or, Not +from pysmt.shortcuts import Symbol, And, Or, Not, Implies, Iff, Ite, Equals, Plus, Minus, Times, Div, Pow, Bool, TRUE, \ + FALSE, Int, Real, simplify, LT, LE, GT, GE, ToReal +from pysmt.shortcuts import Or, Equals +from pysmt.fnode import FNode +from pysmt.typing import BOOL, REAL, INT +from pysmt.rewritings import CNFizer +from pysmt.walkers import IdentityDagWalker, DagWalker +import ast +import smlp + +from typing import List, Dict, Optional, Tuple + +pysmt_types = { + "int": INT, + "real": REAL, + "bool": BOOL +} + + +class Equation: + + def __init__(self, variable, operator: str, scalar: float): + self.variable = variable + self.operator = operator + self.scalar = scalar + self._eq = [str(variable),operator,str(scalar)] + + def __str__(self): + return "{0} {1} {2}".format(self.variable,self.operator,self.scalar) + + def __eq__(self, o: object) -> bool: + return str(self) == str(o) + + def lhs(self): + return self.variable + + def rhs(self): + return self.scalar + + def op(self): + return self.operator + + def __hash__(self) -> int: + return hash(self.variable) * hash(self.operator) * int(self.scalar) + +class InequalityChecker(DagWalker): + def __init__(self, env=None): + DagWalker.__init__(self, env=env) + self.is_inequality = False + self.contains_and_or = False + + def walk_and(self, formula, args, **kwargs): + self.contains_and_or = True + return formula + + def walk_or(self, formula, args, **kwargs): + self.contains_and_or = True + return formula + + def walk_le(self, formula, args, **kwargs): + self.is_inequality = True + return formula + + def walk_lt(self, formula, args, **kwargs): + self.is_inequality = True + return formula + + def walk_ge(self, formula, args, **kwargs): + self.is_inequality = True + return formula + + def walk_gt(self, formula, args, **kwargs): + self.is_inequality = True + return formula + +def check_inequality(formula): + checker = InequalityChecker() + checker.walk(formula) + return checker.is_inequality and not checker.contains_and_or + +class TextToPysmtParser(object): + _instance = None + + def __new__(cls, *args, **kwargs): + if cls._instance is None: + cls._instance = super(TextToPysmtParser, cls).__new__(cls) + cls._instance._initialized = False + return cls._instance + + def __init__(self): + self.symbols = {} + self._ast_operators_map = { + ast.Add: Plus, # Addition + ast.Sub: Minus, # Subtraction + ast.Mult: Times, # Multiplication + ast.Div: self._div_op, # Division + ast.Pow: Pow, # Exponentiation + ast.BitXor: Iff, # Bitwise XOR (interpreted as logical Iff) + + ast.USub: Minus, # Unary subtraction (negation) + + ast.Eq: Equals, # Equality + ast.NotEq: Not, # Not equal + ast.Lt: lambda l, r: l < r, # Less than + ast.LtE: lambda l, r: l <= r, # Less than or equal to + ast.Gt: lambda l, r: l > r, # Greater than + ast.GtE: lambda l, r: l >= r, # Greater than or equal to + + ast.And: And, # Logical AND + ast.Or: Or, # Logical OR + ast.Not: Not, # Logical NOT + + ast.IfExp: Ite # If expression + } + + def _div_op(self, left, right): + # Ensure both operands are real numbers for division + left = ToReal(left) if not left.is_real_constant() else left + right = ToReal(right) if not right.is_real_constant() else right + return Div(left, right) + + + @staticmethod + def and_(*expressions): + return And(*expressions) + + @staticmethod + def or_(*expressions): + return Or(*expressions) + + @staticmethod + def eq_(*expressions): + return Equals(*expressions) + + @staticmethod + def ite_(*expressions): + return Ite(*expressions) + + @staticmethod + def to_cnf(formula): + cnfizer = CNFizer() + cnf_formula = cnfizer.convert(formula) + return cnf_formula + + @staticmethod + def conjunction_to_disjunction(formula): + if formula.is_and(): + negated_terms = [Not(arg) for arg in formula.args()] + disjunction = Or(negated_terms) + return simplify(Not(disjunction)) + else: + raise ValueError("Input formula is not a conjunction") + + def is_comparison(self, node: FNode) -> bool: + return node.is_le() or node.is_lt() or node.is_ge() or node.is_gt() or node.is_equals() + + def create_integer_disjunction(self, variable, values): + variable = self.get_symbol(variable) + if not variable: + return None + + lower, upper = values + value_range = range(lower, upper + 1) + + return Or(*(Equals(variable, Real(val)) for val in value_range)) + + + def split_disjunctions(self, formula: FNode) -> list: + if formula.is_or(): + comparisons = [arg for arg in formula.args() if self.is_comparison(arg)] + if len(comparisons) == len(formula.args()): + return comparisons + elif self.is_comparison(formula): + return [formula] + else: + raise ValueError("Input formula is not a valid disjunction of comparisons") + return [] + + def opposite_comparator(self, comparator): + # sympy only uses LE and LT + # GE and GT are described using LE and LT and reversing the order of the symbol and number + if comparator == "<=": + return ">=" + elif comparator == "<": + return ">" + else: + return comparator + + def decide_comparator(self, formula): + node_type = formula.node_type() + if node_type == 16: + return "<=" + elif node_type == 17: + return "<" + elif node_type == 18: + return "=" + else: + return None + + def extract_coefficient(self, symbol): + coeff = [] + # possible formats + # 1) x-5 + # 2) a*x - 5 + for arg in symbol.args(): + if arg.is_constant(): + coeff.insert(0, arg) + elif arg.is_symbol(): + coeff.append(arg) + else: + pass + + return coeff + + def extract_components(self, comparison: FNode): + left = comparison.arg(0) + right = comparison.arg(1) + comparator = self.decide_comparator(comparison) + + # if left.is_constant(): + # # so the formula is like const <= a*x + # # check if right is like a*x + # right = self.extract_coefficient(right) + # + # elif right.is_constant(): + # # check if left is like a*x + # left = self.extract_coefficient(left) + + + + if left.is_symbol() and right.is_constant(): + return left, comparator, right + elif right.is_symbol() and left.is_constant(): + return right, self.opposite_comparator(comparator), left + else: + raise ValueError("Comparison does not contain a simple variable and constant") + + def process_formula(self, formula: FNode): + components = [] + if formula.is_and(): + for arg in formula.args(): + components.extend(self.process_formula(arg)) + elif formula.is_or(): + print("Disjunction found, storing components:") + for arg in formula.args(): + if arg.is_and(): + components.extend(self.process_formula(arg)) + else: + components.append(arg) + elif self.is_comparison(formula): + components.append(formula) + else: + print("Other formula type encountered.") + + return components + + + def propagate_negation(self, formula): + """ + Apply negation to a formula and propagate the negation inside without leaving any negations in the formula. + """ + if formula.is_not(): + return self.propagate_negation(formula.arg(0)) # Remove double negation if exists + + elif formula.is_and(): + # Apply De Morgan's law: not (A and B) -> (not A) or (not B) + return Or([self.propagate_negation(Not(arg)) for arg in formula.args()]) + + elif formula.is_or(): + # Apply De Morgan's law: not (A or B) -> (not A) and (not B) + return And([self.propagate_negation(Not(arg)) for arg in formula.args()]) + + elif formula.is_equals(): + # not (A = B) -> A != B + A, B = formula.args() + return And(LT(A, B), LT(B, A)) + + elif formula.is_lt(): + # not (A < B) -> A >= B + A, B = formula.args() + return LE(B,A) + + elif formula.is_le(): + # not (A <= B) -> A > B + A, B = formula.args() + return LT(B, A) + + elif formula.is_plus() or formula.is_times(): + # Propagate negation inside arithmetic operations + return formula + + elif formula.is_symbol() or formula.is_constant(): + # Apply negation directly to literals + return Not(formula) + + else: + raise NotImplementedError(f"Negation propagation not implemented for formula type: {formula}") + + def simplify(self, expression): + return simplify(expression) + + def cast_number(self, symbol_type, number): + if symbol_type == REAL: + return Real(number) + elif symbol_type == INT: + return Int(number) + + def init_variables(self, inputs: List[Tuple[str, str]]) -> None: + for input_var in inputs: + name, type = input_var + self.add_symbol(name, type) + + def add_symbol(self, name, symbol_type): + assert symbol_type.lower() in pysmt_types.keys() + self.symbols[name] = Symbol(name, pysmt_types[symbol_type]) + + def get_symbol(self, name): + assert name in self.symbols.keys() + return self.symbols[name] + + def parse(self, expr): + assert isinstance(expr, str) + symbol_list = self.symbols + + def eval_(node): + if isinstance(node, ast.Num): + return Real(node.n) if isinstance(node.n, float) else Int(node.n) + elif isinstance(node, ast.BinOp): + return self._ast_operators_map[type(node.op)](eval_(node.left), eval_(node.right)) + elif isinstance(node, ast.UnaryOp): + return self._ast_operators_map[type(node.op)](eval_(node.operand)) + elif isinstance(node, ast.Name): + return symbol_list[node.id] + elif isinstance(node, ast.BoolOp): + res_boolop = self._ast_operators_map[type(node.op)](eval_(node.values[0]), eval_(node.values[1])) + for value in node.values[2:]: + res_boolop = self._ast_operators_map[type(node.op)](res_boolop, eval_(value)) + return res_boolop + elif isinstance(node, ast.Compare): + left = eval_(node.left) + first_comparator = eval_(node.comparators[0]) + result = self._ast_operators_map[type(node.ops[0])](left, first_comparator) + for op, comparator in zip(node.ops[1:], node.comparators[1:]): + left = eval_(comparator) + result = And(result, self._ast_operators_map[type(op)](left, eval_(comparator))) + return result + elif isinstance(node, ast.IfExp): + return self._ast_operators_map[ast.IfExp](eval_(node.test), eval_(node.body), eval_(node.orelse)) + elif isinstance(node, ast.Constant): + if node.value is True: + return TRUE() + elif node.value is False: + return FALSE() + elif isinstance(node.value, int): + return Int(node.value) + elif isinstance(node.value, float): + return Real(node.value) + else: + return node.value + else: + raise TypeError(f"Unexpected node type {type(node)}") + + return eval_(ast.parse(expr, mode='eval').body) + + +if __name__ == "__main__": + + parser = TextToPysmtParser() + parser.add_symbol('x1', 'int') + parser.add_symbol('x2', 'real') + parser.add_symbol('p2', 'real') + parser.add_symbol('y1', 'real') + parser.add_symbol('y2', 'real') + + # Example usage + y1 = parser.get_symbol('y1') + y2 = parser.get_symbol('y1') + # Original formula: not(y1 >= 4.0 and y2 >= 8.0) + original_formula = Not(And(LE(Real(4.0), y1), LE(Real(8.0), y2))) + + negated_formula = parser.propagate_negation(original_formula) + + print(f"Original formula: {original_formula}") + print(f"Negated formula with propagated negation: {negated_formula}") + + def separate_conjunctions_and_disjunctions(formula): + conjunctions = [] + disjunctions = [] + + def traverse(node, source=None): + if node.is_and(): + # conjunctions.extend(node.args()) + for arg in node.args(): + traverse(arg, conjunctions) + elif node.is_or(): + disjunctions.extend(node.args()) + # Or(*[recursively_convert_ite(arg) for arg in term.args()]) + # for arg in node.args(): + # traverse(arg, disjunctions) + elif node.is_le() or node.is_lt() or node.is_equals(): + source.append(node) + source.append(node) + else: + # Leaf nodes (symbols, literals, etc.) are not conjunctions or disjunctions + pass + + traverse(formula) + return conjunctions, disjunctions + + + x = Symbol('x', REAL) + y = Symbol('y', REAL) + + # formula = And(LT(x, Real(10.0)), Or(LT(y, Real(10.0)), LT(y, Real(10.0)))) + # formula = And(LT(x, Real(10.0)), LT(y, Real(10.0))) + formula = ((-1 <= y) & (0.0 <= x) & (y <= 1) & (x <= 10.0)) + conjunctions, disjunctions = separate_conjunctions_and_disjunctions(formula) + + # Define symbols + + # Create a formula + formula = And(LT(x, Real(10.0)), Or(LT(y, Real(10.0)), LT(y, Real(10.0)))) + + # Apply external negation + negated_formula = Not(formula) + + # Simplify the formula + simplified_formula = simplify(negated_formula) + + print(simplified_formula) + + + formula = parser.parse('(y1+y2)/2') + # formula = parser.parse('p2<5.0 and x1==10 and x2<12.0') + x = Symbol('x', REAL) + y = Symbol('y', REAL) + a = Symbol('a', INT) + b = Symbol('b', INT) + + form=parser.and_((x <= Real(5.0)), # x < 5.0 + Equals(y, Real(10.0)), + Equals(a, Int(3)), # a < 3 + Equals(b, Int(4))) # b = 4) + print(formula) + print(form) + ts = parser.conjunction_to_disjunction(form) + print(ts) + + x2 = Symbol('x2', REAL) + x3 = Symbol('x3', REAL) + + disjunction = Or(LT(x2, Real(12.0)), GE(x3, Real(5.0))) + + comparisons_array = parser.split_disjunctions(disjunction) + + for comparison in comparisons_array: + print(comparison) + + x = Symbol('x', REAL) + y = Symbol('y', REAL) + z = Symbol('z', REAL) + + # Example formula with conjunctions and disjunctions + # formula = And(Or(LT(x, Real(10.0)), GT(y, Real(5.0))), LE(z, Real(3.0))) + formula = And(LT(x, Real(10.0)), GE(y, Real(4.0))) + t = parser.to_cnf(formula) + + + def frozenset_to_formula(cnf): + clauses = [] + for clause in cnf: + literals = [] + for literal in clause: + if literal.is_not(): + literals.append(Not(literal.arg(0))) + else: + literals.append(literal) + clauses.append(Or(literals)) + return And(clauses) + + + # Reconstruct the formula + t = frozenset_to_formula(t) + t = parser.simplify(t) + # Process the formula + res = parser.process_formula(formula) + print(res) \ No newline at end of file diff --git a/src/smlp_py/vnnlib/__init__.py b/src/smlp_py/vnnlib/__init__.py new file mode 100755 index 00000000..e69de29b diff --git a/src/smlp_py/vnnlib/vnnlib_parser.py b/src/smlp_py/vnnlib/vnnlib_parser.py new file mode 100755 index 00000000..7da7f3e8 --- /dev/null +++ b/src/smlp_py/vnnlib/vnnlib_parser.py @@ -0,0 +1,356 @@ + +import re + +import sympy +from sympy import * +from sympy.logic.boolalg import And, Or, Not +from sympy.core.relational import Relational +from sympy.core.numbers import * +import ast +import smlp +import pysmt + +def variable_init_string(variable): + return f"(declare-const {variable} Real)\n" + + +def replace_let_vars(expression): + pattern = r"\(let \(\((\|:\d+\|) \((.+?)\)\)\) (.*)\)" + while 'let' in expression: + match = re.search(pattern, expression) + if not match: + break + + var_name, var_expr, rest_expression = match.groups() + + rest_expression = rest_expression.replace(var_name, f"({var_expr})") + + expression = rest_expression + + return expression + + +class VnnLibParser(object): + + def __init__(self, inputs=[], outputs=[]): + from maraboupy import Marabou, MarabouCore + + self.variables = {} + self.file_name = "query.vnnlib" + self.inputs = inputs + self.outputs = outputs + self.final_output = "" + self.sympy_expr = "" + self.symbols = None + self.symbol_dict = None + self.global_variable_constraints = {} + self.formula_list = [] + self.initialize() + + self.text_parser = TextToSympyParser(self.symbol_dict) + + def initialize(self): + self.initialize_variables(self.inputs, "X") + self.initialize_variables(self.outputs, "Y") + + symb = "" + for symbol in self.inputs: + symb += f"{symbol} " + + for symbol in self.outputs: + symb += f"{symbol} " + + symb = symb[:-1] + self.symbols = symbols(symb) + self.symbol_dict = {str(sym): sym for sym in self.symbols} + self.symbol_dict.update({ + 'And': And, + 'Or': Or, + 'Not': Not + }) + + + def replace_names(self): + for key, variable in self.variables.items(): + self.final_output = self.final_output.replace(key, variable) + + def initialize_variables(self, data, variable_name): + for i, arg in enumerate(data): + variable = f"{variable_name}_{i}" + self.variables[arg] = variable + self.final_output += variable_init_string(variable) + + def finalize(self): + self.replace_names() + with open(self.file_name, 'w') as file: + file.write(self.final_output) + self.call_marabou() + + + def add(self, expression): + expression = replace_let_vars(expression) + self.final_output += f"(assert \n ({expression}) \n )\n" + + def and_(self, exp1, exp2): + return sympy.logic.boolalg.And(exp1, exp2) + + def and_multi_(self, forms): + res = True + for form in forms: + res = form if res is True else self.and_(res, form) + return res + def or_(self, exp1, exp2): + return sympy.logic.boolalg.Or(exp1, exp2) + + def equal(self, var, num): + return sympy.And(var <= num, var >= num) + + def get_symbol(self, symbol): + return self.symbol_dict.get(symbol) + + def save(self, formula): + self.formula_list.append(formula) + + def not_(self, expression): + expression = replace_let_vars(expression) + + def sympy_and(self, exp1, exp2): + return f"And({exp1}, {exp2})" + + def add_global_constaints(self, variable, expression): + # check if the constraint exists in our list + assert variable in self.symbol_dict + + self.global_variable_constraints[variable] = expression + + + + def call_marabou(self): + onnx_file = "/home/ntinouldinho/Desktop/smlp/src/test.onnx" + property_filename = f"/home/ntinouldinho/Desktop/smlp/src/{self.file_name}" + + network = Marabou.read_onnx(onnx_file) + network.saveQuery("./query.txt") + ipq = Marabou.load_query("./query.txt") + MarabouCore.loadProperty(ipq, property_filename) + exitCode_ipq, vals_ipq, _ = Marabou.solve_query(ipq, propertyFilename=property_filename, filename="res.log") + + + + +class SymbolicExpressionHandler: + def __init__(self, variables): + self.symbols = symbols(variables) + self.symbol_dict = {str(sym): sym for sym in self.symbols} + self.symbol_dict.update({ + 'And': And, + 'Or': Or, + 'Not': Not + }) + + def parse_expression(self, expression): + return parse_expr(expression, local_dict=self.symbol_dict) + + def sympy_to_vnnlib(self, expr): + + def parse_expr(e): + # Handle logical operators + if isinstance(e, And): + return "(and {})".format(' '.join(parse_expr(arg) for arg in e.args)) + elif isinstance(e, Or): + return "(or {})".format(' '.join(parse_expr(arg) for arg in e.args)) + elif isinstance(e, Not): + return "(not {})".format(parse_expr(e.args[0])) + + elif isinstance(e, Relational): + left = parse_expr(e.lhs) + right = parse_expr(e.rhs) + op = e.rel_op + return "({} {} {})".format(op, left, right) + + elif isinstance(e, Abs): + return "(abs {})".format(parse_expr(e.args[0])) + + elif isinstance(e, (Symbol, Integer, Float, Zero, One)): + return str(e) + + else: + raise TypeError("Unsupported type: {}".format(type(e))) + + # Start the parsing + return parse_expr(expr) + + +class TextToSympyParser(object): + def __init__(self, map): + self.symbols = map + self._ast_operators_map = { + ast.Add: sympy.Add, # Addition + ast.Sub: sympy.Mul, # Subtraction (handles through Mul with -1) + ast.Mult: sympy.Mul, # Multiplication + ast.Div: sympy.div, # Division (true division, use sympy.Mul with sympy.Pow for reciprocal) + ast.Pow: sympy.Pow, # Exponentiation + ast.BitXor: sympy.Xor, # Bitwise XOR + + ast.USub: sympy.Mul, # Unary subtraction (negation, effectively multiplying by -1) + + ast.Eq: sympy.Eq, # Equality + ast.NotEq: sympy.Ne, # Not equal + ast.Lt: sympy.Lt, # Less than + ast.LtE: sympy.Le, # Less than or equal to + ast.Gt: sympy.Gt, # Greater than + ast.GtE: sympy.Ge, # Greater than or equal to + + ast.And: sympy.And, # Logical AND + ast.Or: sympy.Or, # Logical OR + ast.Not: sympy.Not, # Logical NOT + + ast.IfExp: sympy.ITE # If expression + } + + def parse(self, expr): + # print('evaluating AST expression ====', expr) + assert isinstance(expr, str) + symbol_list = self.symbols + + # recursion + def eval_(node): + if isinstance(node, ast.Num): # + # print('node Num', node.n, type(node.n)) + return sympy.Float(node.n) + elif isinstance(node, ast.BinOp): # + # print('node BinOp', node.op, type(node.op)) + if type(node.op) not in [ast.Div, ast.Pow]: + return self._ast_operators_map[type(node.op)](eval_(node.left), eval_(node.right)) + elif type(node.op) == ast.Div: + if type(node.right) == ast.Constant: + if node.right.n == 0: + raise Exception('Division by 0 in parsed expression ' + expr) + elif not isinstance(node.right.n, int): + raise Exception( + 'Division in parsed expression is only supported for integer constants; got ' + expr) + else: + # print('node.right.n', node.right.n, type(node.right.n)) + return self._ast_operators_map[ast.Mult](smlp.Cnst(smlp.Q(1) / smlp.Q(node.right.n)), + eval_(node.left)) + else: + raise Exception('Opreator ' + str(self._ast_operators_map[type(node.op)]) + + ' with non-constant demominator within ' + str( + expr) + ' is not supported in ast_expr_to_term') + elif type(node.op) == ast.Pow: + if type(node.right) == ast.Constant: + if type(node.right.n) == int: + # print('node.right.n', node.right.n) + if node.right.n == 0: + return sympy.Float(1) + elif node.right.n > 0: + left_term = res_pow = eval_(node.left) + for i in range(1, node.right.n): + res_pow = sympy.Mul(res_pow, left_term) + # print('res_pow', res_pow) + return res_pow + raise Exception('Opreator ' + str(self._ast_operators_map[type(node.op)]) + + ' with non-constant or negative exponent within ' + + str(expr) + 'is not supported in ast_expr_to_term') + else: + raise Exception('Implementation error in function ast_expr_to_term') + elif isinstance(node, ast.UnaryOp): # e.g., -1 + # print('unary op', node.op, type(node.op)); + return self._ast_operators_map[type(node.op)](eval_(node.operand)) + elif isinstance(node, ast.Name): # variable + # print('node Var', node.id, type(node.id)) + return symbol_list[node.id] + elif isinstance(node, ast.BoolOp): + res_boolop = self._ast_operators_map[type(node.op)](eval_(node.values[0]), eval_(node.values[1])) + if len(node.values) > 2: + for i in range(2, len(node.values)): + res_boolop = self._ast_operators_map[type(node.op)](res_boolop, eval_(node.values[i])) + # print('res_boolop', res_boolop) + return res_boolop + elif isinstance(node, ast.Compare): + # print('node Compare', node.ops, type(node.ops), 'left', node.left, 'comp', node.comparators); + # print('len ops', len(node.ops), 'len comparators', len(node.comparators)) + assert len(node.ops) == len(node.comparators) + left_term_0 = eval_(node.left) + right_term_0 = eval_(node.comparators[0]) + if type(node.ops[0]) == ast.Eq: + # if x==10 then x<=10 and x>=10 + if type(left_term_0) == sympy.Symbol and type(right_term_0) == sympy.Float: + res_comp = sympy.And(left_term_0 <= right_term_0, left_term_0 >= right_term_0) + else: + res_comp = self._ast_operators_map[type(node.ops[0])](left_term_0, + right_term_0); # print('res_comp_0', res_comp) + if len(node.ops) > 1: + # print('enum', list(range(1, len(node.ops)))) + left_term_i = right_term_0 + for i in range(1, len(node.ops)): + right_term_i = eval_(node.comparators[i]) + # print('i', i, 'left', left_term_i, 'right', right_term_i) + res_comp_i = self._ast_operators_map[type(node.ops[i])](left_term_i, right_term_i) + res_comp = sympy.And(res_comp, res_comp_i) # self._ast_operators_map[type(node.op.And)] + # for the next iteration (if any): + left_term_i = right_term_i + # print('res_comp', res_comp) + return res_comp + elif isinstance(node, ast.List): + # print('node List', 'elts', node.elts, type(node.elts), 'expr_context', node.expr_context); + raise Exception('Parsing expressions with lists is not supported') + elif isinstance(node, ast.Constant): + if node.n == True: + return True + if node.n == False: + return False + raise Exception('Unsupported comstant ' + str(node.n) + ' in funtion ast_expr_to_term') + elif isinstance(node, ast.IfExp): + res_test = eval_(node.test) + res_body = eval_(node.body) + res_orelse = eval_(node.orelse) + # res_ifexp = smlp.Ite(res_test, res_body, res_orelse) + res_ifexp = self._ast_operators_map[ast.IfExp](res_test, res_body, res_orelse) + # print('res_ifexp',res_ifexp) + return res_ifexp + else: + print('Unexpected node type ' + str(type(node))) + # print('node type', type(node)) + raise TypeError(node) + + return eval_(ast.parse(expr, mode='eval').body) + + +if __name__ == "__main__": + # variable_names = 'x1 x2 p1 p2 y1 y2' + # handler = SymbolicExpressionHandler(variable_names) + # expr_string = "Or(And(x1<=0, y1>9), And(x2<7, y2>10))" + # parsed_expr = handler.parse_expression(expr_string) + # b = Not(parsed_expr) + # b = simplify_logic(b) + # print(b) + # + from sympy import symbols, parse_expr, And + from sympy.parsing.sympy_parser import parse_expr, standard_transformations + + # Define symbols + p2, x1, x2 = symbols('p2 x1 x2') + + # Define the expression string + expression_str = "(p2 < 5) & (x1 == 10) & (x2 < 12)" + + # Standard transformations + transformations = standard_transformations + + # Parse the expression + expr = parse_expr(expression_str, transformations=transformations, local_dict={'p2': p2, 'x1': x1, 'x2': x2}) + + # Print the parsed expression + print(expr) + + + + + + + + + + + From 580e37a86ffb8562653406a1aa5094ec80589a09 Mon Sep 17 00:00:00 2001 From: konstantopoulos-tetractys Date: Wed, 24 Jul 2024 21:06:47 +0100 Subject: [PATCH 04/28] update workflow --- .gitignore | 2 + src/smlp_py/NN_verifiers/verifiers.py | 9 +- src/smlp_py/smlp_optimize.py | 7 +- src/smlp_py/smlp_terms.py | 171 ++++++++++++++++++++++---- src/smlp_py/train_keras.py | 4 +- 5 files changed, 164 insertions(+), 29 deletions(-) mode change 100644 => 100755 src/smlp_py/smlp_optimize.py diff --git a/.gitignore b/.gitignore index 4dcb2653..e7b53844 100644 --- a/.gitignore +++ b/.gitignore @@ -56,3 +56,5 @@ obj/table.c.o /data .idea src/logs.log +*.pb +/src/variables diff --git a/src/smlp_py/NN_verifiers/verifiers.py b/src/smlp_py/NN_verifiers/verifiers.py index 78009ce6..83dd10f1 100755 --- a/src/smlp_py/NN_verifiers/verifiers.py +++ b/src/smlp_py/NN_verifiers/verifiers.py @@ -2,7 +2,6 @@ from enum import Enum from typing import List, Dict, Optional, Tuple -from z3 import And, Or, Not, Implies, Bool from maraboupy import Marabou from maraboupy import MarabouCore from maraboupy import MarabouUtils @@ -101,13 +100,13 @@ def __init__(self, model_path=None, parser=None): # Stack for keeping ipq self.ipq_stack = [] - self.model_file_path = "/home/ntinouldinho/Desktop/smlp/result/abc_smlp_toy_basic_nn_keras_model_complete.h5" - - self.convert_to_pb() - self.parser = parser + def initialize(self): + self.model_file_path = "/home/kkon/Desktop/smlp/result/abc_smlp_toy_basic_nn_keras_model_complete.h5" + self.convert_to_pb() + def epsilon(self, e, direction): if direction == 'down': diff --git a/src/smlp_py/smlp_optimize.py b/src/smlp_py/smlp_optimize.py old mode 100644 new mode 100755 index 80ba3de1..7032e5d8 --- a/src/smlp_py/smlp_optimize.py +++ b/src/smlp_py/smlp_optimize.py @@ -13,6 +13,7 @@ import pandas as pd import keras import numpy as np +from src.smlp_py.smtlib.text_to_sympy import TextToPysmtParser # single or multi-objective optimization, with stability constraints and any user # given constraints on free input, control (knob) and output variables satisfied. @@ -46,6 +47,7 @@ def __init__(self): self._DEF_OBJECTIVES_EXPRS = None self._DEF_APPROXIMATE_FRACTIONS:bool = True self._DEF_FRACTION_PRECISION:int = 64 + self._ENABLE_PYSMT = False # Formulae alpha, beta, eta are used in single and pareto optimization tasks. # They are used to constrain control variables x and response variables y as follows: @@ -469,7 +471,10 @@ def active_objectives_max_min_bounds(self, model_full_term_dict:dict, objv_terms else: min_name = min_name + '_' + objv_name if min_name != '' else objv_name if min_objs is not None: - min_objs = smlp.Ite(objv_term < min_objs, objv_term, min_objs) + if self._ENABLE_PYSMT: + min_objs = TextToPysmtParser.ite_(objv_term < min_objs, objv_term, min_objs) + else: + min_objs = smlp.Ite(objv_term < min_objs, objv_term, min_objs) else: min_objs = objv_term diff --git a/src/smlp_py/smlp_terms.py b/src/smlp_py/smlp_terms.py index 4826612e..5ccac0fc 100644 --- a/src/smlp_py/smlp_terms.py +++ b/src/smlp_py/smlp_terms.py @@ -21,6 +21,10 @@ list_subtraction_set, get_expression_variables, str_to_bool) #from smlp_py.smlp_spec import SmlpSpec +from smlp_py.NN_verifiers.verifiers import MarabouVerifier +import pysmt + +from src.smlp_py.smtlib.text_to_sympy import TextToPysmtParser # TODO !!! create a parent class for TreeTerms, PolyTerms, NNKerasTerms. # setting logger, report_file_prefix, model_file_prefix can go to that class to work for all above three classes @@ -1476,12 +1480,33 @@ def feature_scaler_to_term(self, orig_feat_name, scaled_feat_name, orig_min, ori (self.smlp_var(orig_feat_name) - self.smlp_cnst(orig_min))) ####return self.smlp_div(self.smlp_var(orig_feat_name) - self.smlp_cnst(orig_min), self.smlp_cnst(orig_max) - self.smlp_cnst(orig_min)) ####return smlp.Cnst(smlp.Q(1) / smlp.Q(orig_max - orig_min)) * (smlp.Var(orig_feat_name) - smlp.Cnst(orig_min)) - + + + def pysmt_feature_scaler_to_term(self, orig_feat_var, parser, orig_min, orig_max): + parser.add_symbol(orig_feat_var, 'real') + orig_feat_var = parser.get_symbol(orig_feat_var) + if orig_min == orig_max: + return pysmt.shortcuts.Real(0) + else: + scaling_factor = pysmt.shortcuts.Real(1) / pysmt.shortcuts.Real(orig_max - orig_min) + orig_min_cnst = pysmt.shortcuts.Real(orig_min) + + # (orig_feat_var - orig_min_cnst) + scaled_expr = pysmt.shortcuts.Minus(orig_feat_var, orig_min_cnst) + # scaling_factor * (orig_feat_var - orig_min_cnst) + scaled_term = pysmt.shortcuts.Times(scaling_factor, scaled_expr) + return scaled_term + # Computes dictionary with features as keys and scaler terms as values - def feature_scaler_terms(self, data_bounds, feat_names): - return dict([(self._scaled_name(feat), self.feature_scaler_to_term(feat, self._scaled_name(feat), + def feature_scaler_terms(self, data_bounds, feat_names, parser=None): + if parser: + return dict([(self._scaled_name(feat), self.pysmt_feature_scaler_to_term(feat, parser, + data_bounds[feat]['min'], + data_bounds[feat]['max'])) for feat in + feat_names]) + return dict([(self._scaled_name(feat), self.feature_scaler_to_term(feat, self._scaled_name(feat), data_bounds[feat]['min'], data_bounds[feat]['max'])) for feat in feat_names]) - + # Computes term x from column x_scaled using expression x = x_scaled * (max_x - min_x) + x_min. # Argument orig_feat_name is name for column x, argument scaled_feat_name is the name of scaled column # x_scaled obtained earlier from x using min_max scaler to range [0, 1] (same as normalization of x), @@ -1562,6 +1587,18 @@ def __init__(self): # 'help':'Should terms be cached along building terms and formulas in model exploration modes? ' + # '[default {}]'.format(str(self._DEF_CACHE_TERMS))} } + + self.parser = TextToPysmtParser() + self.parser.init_variables(inputs=[("x1", "real"), ('x2', 'int'), ('p1', 'real'), ('p2', 'int'), + ('y1', 'real'), ('y2', 'real')]) + + self.verifier = MarabouVerifier() + self.verifier.init_variables(inputs=[("x1", "Real"), ('x2', 'Integer'), ('p1', 'Real'), ('p2', 'Integer')], + outputs=[('y1', 'Real'), ('y2', 'Real')]) + + self._ENABLE_PYSMT = True + self._RETURN_PYSMT = True + # set logger from a caller script def set_logger(self, logger): @@ -1836,15 +1873,30 @@ def compute_objectives_terms(self, objv_names, objv_exprs, objv_bounds, scale_ob orig_objv_terms_dict = dict([(objv_name, self.ast_expr_to_term(objv_expr)) \ for objv_name, objv_expr in zip(objv_names, objv_exprs)]) #self._smlpTermsInst. #print('orig_objv_terms_dict', orig_objv_terms_dict) + + if self._ENABLE_PYSMT: + pysmt_objv_terms_dict = dict([(objv_name, self.parser.parse(objv_expr)) \ + for objv_name, objv_expr in zip(objv_names, objv_exprs)]) + if scale_objv: - scaled_objv_terms_dict = self.feature_scaler_terms(objv_bounds, objv_names) #._scalerTermsInst + if self._ENABLE_PYSMT: + scaled_objv_terms_dict = self.feature_scaler_terms(objv_bounds, objv_names, + self.parser) # ._scalerTermsInst + else: + scaled_objv_terms_dict = self.feature_scaler_terms(objv_bounds, objv_names) # ._scalerTermsInst + #print('scaled_objv_terms_dict', scaled_objv_terms_dict) objv_terms_dict = {} for i, (k, v) in enumerate(scaled_objv_terms_dict.items()): #print('k', k, 'v', v, type(v)); x = list(orig_objv_terms_dict.keys())[i]; #print('x', x); print('arg', orig_objv_terms_dict[x]) - objv_terms_dict[k] = self.smlp_cnst_fold(v, {x: orig_objv_terms_dict[x]}) #self.smlp_subst + if self._ENABLE_PYSMT: + substitution = {self.parser.get_symbol(x): pysmt_objv_terms_dict[x]} + # Apply the substitution + objv_terms_dict[k] = self.parser.simplify(v.substitute(substitution)) + else: + objv_terms_dict[k] = self.smlp_cnst_fold(v, {x: orig_objv_terms_dict[x]}) #objv_terms_dict = scaled_objv_terms_dict else: objv_terms_dict = orig_objv_terms_dict @@ -1871,8 +1923,9 @@ def compute_stability_formula_theta(self, cex, delta_dict:dict, radii_dict, univ assert delta_rel >= 0 else: delta_rel = delta_abs = None - + theta_form = self.smlp_true + PYSMT_theta_form = pysmt.shortcuts.TRUE() #print('radii_dict', radii_dict) radii_dict_local = radii_dict.copy() knobs = radii_dict_local.keys(); #print('knobs', knobs); print('cex', cex); print('delta', delta_dict) @@ -1891,12 +1944,15 @@ def compute_stability_formula_theta(self, cex, delta_dict:dict, radii_dict, univ if delta_rel is not None: # we are generating a lemma rad = rad * (1 + delta_rel) + delta_abs rad_term = self.smlp_cnst(rad) + if self._ENABLE_PYSMT: + verifier_rad_term = float(rad) elif radii['rad-rel'] is not None: rad = radii['rad-rel']; #print('rad', rad) if delta_rel is not None: # we are generating a lemma rad = rad * (1 + delta_rel) + delta_abs rad_term = self.smlp_cnst(rad) - + if self._ENABLE_PYSMT: + verifier_rad_term = float(rad) # TODO !!! issue a warning when candidates become closer and closer # TODO !!!!!!! warning when distance between previous and current candidate # TODO !!!!!! warning when FINAL rad + delta is 0, as part of sanity checking options @@ -1920,11 +1976,22 @@ def compute_stability_formula_theta(self, cex, delta_dict:dict, radii_dict, univ else: # radius for excluding a candidate -- cex holds values of the candidate rad_term = rad_term * abs(cex[var]) elif delta_dict is not None: - raise exception('When delta dictionary is provided, either absolute or relative or delta must be specified') + raise exception('When delta dictionary is provided, either absolute or relative or delta must be specified') theta_form = self.smlp_and(theta_form, ((abs(var_term - cex[var])) <= rad_term)) + if self._ENABLE_PYSMT: + value = float(self.ground_smlp_expr_to_value(cex[var])) + PYSMT_var = self.parser.get_symbol(var) + type = pysmt.shortcuts.Int if str(PYSMT_var.get_type()) == "Int" else pysmt.shortcuts.Real + calc_type = int if str(PYSMT_var.get_type()) == "Int" else float + lower = calc_type(value - verifier_rad_term) + lower = type(lower) + upper = calc_type(value + verifier_rad_term) + upper = type(upper) + PYSMT_theta_form = self.parser.and_(PYSMT_theta_form, PYSMT_var >= lower, PYSMT_var <= upper) + # self.verifier.add_bounds(var, (value - verifier_rad_term, value + verifier_rad_term)) #print('theta_form', theta_form) return theta_form - + # Creates eta constraints on control parameters (knobs) from the spec. # Covers grid as well as range/interval constraints. def compute_grid_range_formulae_eta(self): @@ -1945,7 +2012,29 @@ def compute_grid_range_formulae_eta(self): eta_grid_form = self.smlp_and(eta_grid_form, eta_grid_disj) #print('eta_grid_form', eta_grid_form); return eta_grid_form - + + def pysmt_compute_grid_range_formulae_eta(self): + # print('generate eta constraint') + eta_grid_form = pysmt.shortcuts.TRUE() + eta_grids_dict = self._specInst.get_spec_eta_grids_dict; # print('eta_grids_dict', eta_grids_dict) + for var, grid in eta_grids_dict.items(): + # self.verifier.add_bounds(var, grid=grid) + eta_grid_disj = pysmt.shortcuts.FALSE() + var_term = self.parser.get_symbol(var) + symbol_type = var_term.get_type() + for gv in grid: # iterate over grid values + if eta_grid_disj == pysmt.shortcuts.FALSE(): + eta_grid_disj = self.parser.eq_(var_term, self.parser.cast_number(symbol_type, gv)) + else: + eta_grid_disj = self.parser.or_(eta_grid_disj, + self.parser.eq_(var_term, self.parser.cast_number(symbol_type, gv))) + if eta_grid_form == pysmt.shortcuts.TRUE(): + eta_grid_form = eta_grid_disj + else: + eta_grid_form = self.parser.and_(eta_grid_form, eta_grid_disj) + # print('eta_grid_form', eta_grid_form); + return eta_grid_form + # Compute formulae alpha, beta, eta from respective expression string. def compute_input_ranges_formula_alpha(self, model_inputs): @@ -1974,8 +2063,10 @@ def compute_input_ranges_formula_alpha(self, model_inputs): assert False return alpha_form - def compute_input_ranges_formula_alpha_eta(self, alpha_vs_eta, model_inputs): + def compute_input_ranges_formula_alpha_eta(self, alpha_vs_eta, model_inputs, specs=None): alpha_or_eta_form = self.smlp_true + smt_form = pysmt.shortcuts.TRUE() + if alpha_vs_eta == 'alpha': alpha_or_eta_ranges_dict = self._specInst.get_spec_alpha_bounds_dict elif alpha_vs_eta == 'eta': @@ -1992,6 +2083,13 @@ def compute_input_ranges_formula_alpha_eta(self, alpha_vs_eta, model_inputs): #print('mn', mn, 'mx', mx) if mn is not None and mx is not None: if self._declare_domain_interface_only: + if self._ENABLE_PYSMT: + symbol_v = self.parser.get_symbol(v) + form = self.parser.and_(symbol_v >= mn, symbol_v <= mx) + smt_form = self.parser.and_(smt_form, form) + + # self.verifier.add_bounds(v, (mn, mx), num=specs[v]['range']) + if self._encode_input_range_as_disjunction and alpha_vs_eta == 'alpha' and v in self._specInst.get_spec_inputs: rng = self.smlp_or_multi([self.smlp_eq(self.smlp_var(v), self.smlp_cnst(i)) for i in range(mn, mx+1)]) else: @@ -2005,6 +2103,15 @@ def compute_input_ranges_formula_alpha_eta(self, alpha_vs_eta, model_inputs): alpha_or_eta_form = self.smlp_and(alpha_or_eta_form, rng) else: assert False + + if self._ENABLE_PYSMT: + smt_form = self.parser.simplify(smt_form) + # return self.parser.simplify(smt_form) + if self._RETURN_PYSMT: + return smt_form + else: + print(smt_form) + return alpha_or_eta_form # alph_expr is alpha constraint specified in command line. If it is not None @@ -2028,6 +2135,12 @@ def compute_global_alpha_formula(self, alph_expr, model_inputs): raise Exception('Variables ' + str(dont_care_vars) + ' in input constraints (alpha) are not part of the model') alph_glob = self.ast_expr_to_term(alph_expr) + if self._ENABLE_PYSMT: + if self._RETURN_PYSMT: + return self.parser.parse(alph_expr) + else: + print(self.parser.parse(alph_expr)) + return alph_glob #self._smlpTermsInst.smlp_and(alph_form, alph_glob) # The argument model_inps_outps is the union of model input and output varaiables. @@ -2050,6 +2163,8 @@ def compute_beta_formula(self, beta_expr, model_inps_outps): def compute_eta_formula(self, eta_expr, model_inputs): if eta_expr is None: + if self._ENABLE_PYSMT and self._RETURN_PYSMT: + return pysmt.shortcuts.TRUE() return self.smlp_true else: # eta_expr can only contain knobs (control inputs), not free inputs or outputs (responses) @@ -2058,6 +2173,12 @@ def compute_eta_formula(self, eta_expr, model_inputs): if len(dont_care_vars) > 0: raise Exception('Variables ' + str(dont_care_vars) + ' in knob constraints (eta) are not part of the model') + if self._ENABLE_PYSMT: + if self._RETURN_PYSMT: + return self.parser.parse(eta_expr) + else: + print(self.parser.parse(eta_expr)) + return self.ast_expr_to_term(eta_expr) def var_domain(self, var, spec_domain_dict): @@ -2101,16 +2222,24 @@ def create_model_exploration_base_components(self, syst_expr_dict:dict, algo, mo # get variable domains dictionary; certain sanity checks are performrd within this function. spec_domain_dict = self._specInst.get_spec_domain_dict; #print('spec_domain_dict', spec_domain_dict) - + + self.verifier.initialize() # contraints on features used as control variables and on the responses - alph_ranges = self.compute_input_ranges_formula_alpha_eta('alpha', feat_names); #print('alph_ranges') - alph_global = self.compute_global_alpha_formula(alph_expr, feat_names); #print('alph_global') - alpha = self.smlp_and(alph_ranges, alph_global); #print('alpha') - beta = self.compute_beta_formula(beta_expr, feat_names+resp_names); #print('beta') - eta_ranges = self.compute_input_ranges_formula_alpha_eta('eta', feat_names); #print('eta_ranges') - eta_grids = self.compute_grid_range_formulae_eta(); #print('eta_grids') - eta_global = self.compute_eta_formula(eta_expr, feat_names); #print('eta_global', eta_global) - eta = self.smlp_and_multi([eta_ranges, eta_grids, eta_global]); #print('eta', eta) + alph_ranges = self.compute_input_ranges_formula_alpha_eta('alpha', feat_names, + spec_domain_dict); # print('alph_ranges') + alph_global = self.compute_global_alpha_formula(alph_expr, feat_names); # print('alph_global') + alpha = self.smlp_and(alph_ranges, alph_global) if ( + not self._ENABLE_PYSMT or not self._RETURN_PYSMT) else self.parser.and_(alph_ranges, alph_global) + beta = self.compute_beta_formula(beta_expr, feat_names + resp_names); # print('beta') + eta_ranges = self.compute_input_ranges_formula_alpha_eta('eta', feat_names, + spec_domain_dict); # print('eta_ranges') + eta_grids = self.compute_grid_range_formulae_eta() if ( + not self._ENABLE_PYSMT or not self._RETURN_PYSMT) else self.pysmt_compute_grid_range_formulae_eta() + eta_global = self.compute_eta_formula(eta_expr, feat_names); # print('eta_global', eta_global) + + eta = self.smlp_and_multi([eta_ranges, eta_grids, eta_global]) if ( + not self._ENABLE_PYSMT or not self._RETURN_PYSMT) else self.parser.simplify( + self.parser.and_(eta_ranges, eta_grids, eta_global)) self._smlp_terms_logger.info('Alpha global constraints: ' + str(alph_global)) self._smlp_terms_logger.info('Alpha ranges constraints: ' + str(alph_ranges)) diff --git a/src/smlp_py/train_keras.py b/src/smlp_py/train_keras.py index a24416da..d874754b 100644 --- a/src/smlp_py/train_keras.py +++ b/src/smlp_py/train_keras.py @@ -38,8 +38,8 @@ def __init__(self): # hyper parameter defaults self._DEF_LAYERS_SPEC = '2,1' - self._DEF_EPOCHS = 2000 - self._DEF_BATCH_SIZE = 200 + self._DEF_EPOCHS = 100 + self._DEF_BATCH_SIZE = 10 self._DEF_OPTIMIZER = 'adam' # options: 'rmsprop', 'adam', 'sgd', 'adagrad', 'nadam' self._DEF_LEARNING_RATE = 0.001 self._HID_ACTIVATION = 'relu' From be59d48000e83667991eb5786cff2aa87f711dd5 Mon Sep 17 00:00:00 2001 From: konstantopoulos-tetractys Date: Fri, 26 Jul 2024 09:06:36 +0100 Subject: [PATCH 05/28] add scaled/unscaled functionality --- .gitignore | 1 + smlp_toy.onnx | Bin 0 -> 1777 bytes src/smlp_py/NN_verifiers/test_marabou.py | 50 ++++++++++--------- src/smlp_py/NN_verifiers/verifiers.py | 58 +++++++++++++++++++---- src/smlp_py/smlp_terms.py | 4 +- 5 files changed, 80 insertions(+), 33 deletions(-) create mode 100644 smlp_toy.onnx diff --git a/.gitignore b/.gitignore index e7b53844..9586bdb4 100644 --- a/.gitignore +++ b/.gitignore @@ -58,3 +58,4 @@ obj/table.c.o src/logs.log *.pb /src/variables +/variables diff --git a/smlp_toy.onnx b/smlp_toy.onnx new file mode 100644 index 0000000000000000000000000000000000000000..910e9d0aa6c5d49a774cb8a113d1a19489181e9d GIT binary patch literal 1777 zcmd;J7h*3-Gs@4)tB~R~)H5{GGgL4%O|~#Ju-eDVRm{bmlA2eX8lRb0P+G#JQJh*> znwnRVnV6#w7T5PpEb%SP(GN;ZObJUY%1lhkN%b$VG7yr)q0-7gN*srj5*x%Yu0}>K z+}gP`F|<2nCKfxUq+mKi3YSu#Dk3qh$lr9e|fF-!$XD6xP@ zu2x1aTpGEw&^5*z;tX49oN9qCzz8|GLGbYHWaPqU1eZ1z^Kpi;3~t3h*I)!9&^1t_ zAmJRrh1USgbOba28rB$L36+9|b~~0-1lNouyp2c*Z}Z4~WZ}BNAq&sdS|J=P983a?PMApy<_HXHyc8l7C+}SH>0@F~AFTb|fp8vGPR%>$RP1#QuF-XL;H4thL+w^t3?7!&|nG zf9$no00OP=uWYq8GXf)o1L%38OH-iN->n5j z?`PSozJJBRBD$AFy}$In+1{A_!!|~F``%fBr@(;`B$=F_SC(2- zlA5BBR+OKsfGA0#lu_y(ArUSi4n`q9E)F5K(!A{Wcrz|04wfW&E?92BP+$&Jz~scj W#UQ}z#K*;zn5hS<;Pi5H1egF}t$CLK literal 0 HcmV?d00001 diff --git a/src/smlp_py/NN_verifiers/test_marabou.py b/src/smlp_py/NN_verifiers/test_marabou.py index 4263906a..b6c2069d 100755 --- a/src/smlp_py/NN_verifiers/test_marabou.py +++ b/src/smlp_py/NN_verifiers/test_marabou.py @@ -11,26 +11,30 @@ if __name__ == "__main__": from keras.models import load_model - model = load_model("/home/ntinouldinho/Desktop/smlp/result/abc_smlp_toy_basic_nn_keras_model_complete.h5") - model_proto, external_tensor_storage = tf2onnx.convert.from_keras(model, opset=13, output_path="smlp_toy.onnx") + # model = load_model("/home/kkon/Desktop/smlp/result/abc_smlp_toy_basic_nn_keras_model_complete.h5") + # model_proto, external_tensor_storage = tf2onnx.convert.from_keras(model, opset=13, output_path="smlp_toy.onnx") print("SAVING TO ONNX") parser = TextToPysmtParser() parser.init_variables(inputs=[("x1", "real"), ('x2', 'real'), ('p1', 'real'), ('p2', 'real'), - ('y1', 'real'), ('y2', 'real')]) + ('y1', 'real'), ('y2', 'real'), + ("x1_unscaled", "real"), ('x2_unscaled', 'real'), + ('p1_unscaled', 'real'), ('p2_unscaled', 'real'), + ('y1_unscaled', 'real'), ('y2_unscaled', 'real')]) mb = MarabouVerifier(parser=parser) mb.init_variables(inputs=[("x1", "Real"), ('x2', 'Integer'), ('p1', 'Integer'), ('p2', 'Integer')], outputs=[('y1', 'Real'), ('y2', 'Real')]) + mb.initialize() - y1 = parser.get_symbol("y1") - y2 = parser.get_symbol("y2") - p1 = parser.get_symbol("p1") - p2 = parser.get_symbol("p2") - x1 = parser.get_symbol("x1") - x2 = parser.get_symbol("x2") + y1 = parser.get_symbol("y1_unscaled") + y2 = parser.get_symbol("y2_unscaled") + p1 = parser.get_symbol("p1_unscaled") + p2 = parser.get_symbol("p2_unscaled") + x1 = parser.get_symbol("x1_unscaled") + x2 = parser.get_symbol("x2_unscaled") - x2_int = parser.create_integer_disjunction("x2", (-1, 1)) - p2_int = parser.create_integer_disjunction("p2", (3, 7)) + x2_int = parser.create_integer_disjunction("x2_unscaled", (-1, 1)) + p2_int = parser.create_integer_disjunction("p2_unscaled", (3, 7)) # alpha = (((-1 <= x2) & (0.0 <= x1) & (x2 <= 1) & (x1 <= 10.0)) & (((p2 < 5) & (x1 == 10.0)) & (x2 < 12))) # beta = ((4 <= y1) & (6 <= y2)) @@ -39,9 +43,9 @@ # with x as knob: y1==4.120704402283359 & solution = And( Equals(x1, Real(10)), - Equals(x2, Real(1)), - Equals(p1, Real(7)), - Equals(p2, Real(4)) + Equals(x2, Real(0)), + Equals(p1, Real(2)), + Equals(p2, Real(3)) ) theta = And( @@ -51,10 +55,10 @@ LE(p2, Real(4.2)) ) alpha = And( - GE(x2, Real(-2)), - GE(x1, Real(0.0)), + GE(x2, Real(-1)), LE(x2, Real(1)), - LE(x1, Real(11.0)), + GE(x1, Real(0.0)), + LE(x1, Real(10.0)), And( LT(p2, Real(5)), Equals(x1, Real(10.0)), @@ -82,12 +86,12 @@ p1.Equals(Real(7.0)) ) ) - # mb.apply_restrictions(x2_int) - # mb.apply_restrictions(p2_int) - # mb.apply_restrictions(beta) - # mb.apply_restrictions(alpha) - # mb.apply_restrictions(eta) - mb.apply_restrictions(solution) + mb.apply_restrictions(x2_int) + mb.apply_restrictions(p2_int) + mb.apply_restrictions(beta) + mb.apply_restrictions(alpha) + mb.apply_restrictions(eta) + # mb.apply_restrictions(solution) # mb.apply_restrictions(theta) diff --git a/src/smlp_py/NN_verifiers/verifiers.py b/src/smlp_py/NN_verifiers/verifiers.py index 83dd10f1..d76e744f 100755 --- a/src/smlp_py/NN_verifiers/verifiers.py +++ b/src/smlp_py/NN_verifiers/verifiers.py @@ -16,7 +16,7 @@ from src.smlp_py.smtlib.smt_to_pysmt import smtlib_to_pysmt from src.smlp_py.smtlib.text_to_sympy import TextToPysmtParser from tensorflow.python.framework.convert_to_constants import convert_variables_to_constants_v2 - +import json _operators_ = [">=", "<=", "<", ">"] @@ -44,10 +44,13 @@ class Bounds: lower = -np.inf upper = np.inf - def __init__(self, form: Type, name="", is_input=True): - if is_input: + def __init__(self, form: Type, index=None, name="", is_input=True): + if is_input and not index: self.index = Variable._input_index Variable._input_index += 1 + elif is_input and index: + # auxiliary scaled variable + self.index = index else: self.index = Variable._output_index Variable._output_index += 1 @@ -90,9 +93,12 @@ def __init__(self, model_path=None, parser=None): # List of variables self.variables = [] + self.unscaled_variables = [] + self.model_file_path = "./" self.log_path = "marabou.log" - + self.data_bounds_file = "/home/kkon/Desktop/smlp/result/abc_smlp_toy_basic_data_bounds.json" + self.data_bounds = None # Adds conjunction of equations between bounds in form: # e.g. Int(var), var >= 0, var <= 3 -> Or(var == 0, var == 1, var == 2, var == 3) self.int_enable = False @@ -106,7 +112,13 @@ def __init__(self, model_path=None, parser=None): def initialize(self): self.model_file_path = "/home/kkon/Desktop/smlp/result/abc_smlp_toy_basic_nn_keras_model_complete.h5" self.convert_to_pb() + self.load_json() + self.add_unscaled_variables() + + def load_json(self): + with open(self.data_bounds_file, 'r') as file: + self.data_bounds = json.load(file) def epsilon(self, e, direction): if direction == 'down': @@ -142,20 +154,50 @@ def init_variables(self, inputs: List[Tuple[str, str]], outputs: List[Tuple[str, for input_var in inputs: name, type = input_var var_type = Variable.Type.Real if type.lower() == "real" else Variable.Type.Int - self.variables.append(Variable(var_type, name, is_input=True)) + self.variables.append(Variable(var_type, name=name, is_input=True)) for output_var in outputs: name, type = output_var var_type = Variable.Type.Real if type.lower() == "real" else Variable.Type.Int - self.variables.append(Variable(var_type, name, is_input=False)) + self.variables.append(Variable(var_type, name=name, is_input=False)) + + + + def add_unscaled_variables(self): + for variable in self.variables: + unscaled_variable = self.network.getNewVariable() + self.unscaled_variables.append(Variable(Variable.Type.Real, index=unscaled_variable, name=f"{variable.name}_unscaled", is_input=True)) + + self.convert_scaled_unscaled() + + + def convert_scaled_unscaled(self): + for scaled_var, unscaled_var in zip(self.variables, self.unscaled_variables): + bounds = self.data_bounds[scaled_var.name] + min_value, max_value = bounds["min"], bounds["max"] + + scaling_factor = max_value - min_value + + # Create an equation representing (x_max - x_min) * x_scaled - x_unscaled = - x_min + eq = MarabouUtils.Equation(MarabouCore.Equation.EQ) + eq.addAddend(scaling_factor, scaled_var.index) + eq.addAddend(-1, unscaled_var.index) + eq.setScalar(-min_value) + + # Add the equation to the network + self.network.addEquation(eq) def get_variable_by_name(self, name: str) -> Optional[Tuple[Variable, int]]: is_output = name.startswith("y") + is_unscaled = name.find("_unscaled") + repository = self.unscaled_variables if is_unscaled else self.variables - for index, variable in enumerate(self.variables): + for index, variable in enumerate(repository): if variable.name == name: - if is_output: + if is_unscaled: + return variable, variable.index + elif is_output: index -= Variable.get_index("input") index = self.network.outputVars[0][0][index] if is_output else self.network.inputVars[0][0][index] return variable, index diff --git a/src/smlp_py/smlp_terms.py b/src/smlp_py/smlp_terms.py index 5ccac0fc..93dc8129 100644 --- a/src/smlp_py/smlp_terms.py +++ b/src/smlp_py/smlp_terms.py @@ -1596,8 +1596,8 @@ def __init__(self): self.verifier.init_variables(inputs=[("x1", "Real"), ('x2', 'Integer'), ('p1', 'Real'), ('p2', 'Integer')], outputs=[('y1', 'Real'), ('y2', 'Real')]) - self._ENABLE_PYSMT = True - self._RETURN_PYSMT = True + self._ENABLE_PYSMT = False + self._RETURN_PYSMT = False # set logger from a caller script From a5fddf88aed49ec5fc57649dcd33edb3ec2bf27b Mon Sep 17 00:00:00 2001 From: konstantopoulos-tetractys <37417801+ntinouldinho@users.noreply.github.com> Date: Tue, 30 Jul 2024 17:10:40 +0100 Subject: [PATCH 06/28] support unscaled variables --- src/smlp_py/NN_verifiers/test_marabou.py | 7 ++----- src/smlp_py/NN_verifiers/verifiers.py | 2 +- src/smlp_py/smlp_query.py | 1 + src/smlp_py/smlp_terms.py | 12 ++++++------ src/smlp_py/smtlib/text_to_sympy.py | 6 ++++-- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/smlp_py/NN_verifiers/test_marabou.py b/src/smlp_py/NN_verifiers/test_marabou.py index b6c2069d..e5a3c531 100755 --- a/src/smlp_py/NN_verifiers/test_marabou.py +++ b/src/smlp_py/NN_verifiers/test_marabou.py @@ -15,11 +15,8 @@ # model_proto, external_tensor_storage = tf2onnx.convert.from_keras(model, opset=13, output_path="smlp_toy.onnx") print("SAVING TO ONNX") parser = TextToPysmtParser() - parser.init_variables(inputs=[("x1", "real"), ('x2', 'real'), ('p1', 'real'), ('p2', 'real'), - ('y1', 'real'), ('y2', 'real'), - ("x1_unscaled", "real"), ('x2_unscaled', 'real'), - ('p1_unscaled', 'real'), ('p2_unscaled', 'real'), - ('y1_unscaled', 'real'), ('y2_unscaled', 'real')]) + parser.init_variables(symbols=[("x1", "real"), ('x2', 'real'), ('p1', 'real'), ('p2', 'real'), + ('y1', 'real'), ('y2', 'real')]) mb = MarabouVerifier(parser=parser) mb.init_variables(inputs=[("x1", "Real"), ('x2', 'Integer'), ('p1', 'Integer'), ('p2', 'Integer')], diff --git a/src/smlp_py/NN_verifiers/verifiers.py b/src/smlp_py/NN_verifiers/verifiers.py index d76e744f..105759ba 100755 --- a/src/smlp_py/NN_verifiers/verifiers.py +++ b/src/smlp_py/NN_verifiers/verifiers.py @@ -433,7 +433,7 @@ def add_disjunction(self,): if __name__ == "__main__": parser = TextToPysmtParser() # p2 is an int not a real - parser.init_variables(inputs=[("x1", "real"), ('x2', 'int'), ('p1', 'real'), ('p2', 'real'), + parser.init_variables(symbols=[("x1", "real"), ('x2', 'int'), ('p1', 'real'), ('p2', 'real'), ('y1', 'real'), ('y2', 'real')]) mb = MarabouVerifier(parser=parser) diff --git a/src/smlp_py/smlp_query.py b/src/smlp_py/smlp_query.py index 4061f38d..aeac9221 100644 --- a/src/smlp_py/smlp_query.py +++ b/src/smlp_py/smlp_query.py @@ -529,6 +529,7 @@ def query_condition(self, universal, model_full_term_dict:dict, quer_name:str, q candidate_solver.add(alpha) #candidate_solver.add(beta) candidate_solver.add(quer) + test = self._smlpTermsInst.smlp_simplify(quer) #print('eta', eta); print('alpha', alpha); print('quer', quer); #print('solving query', quer) self._query_tracer.info('{},{}'.format('synthesis' if universal else 'query', str(quer_name))) #, str(quer_expr) ,{} diff --git a/src/smlp_py/smlp_terms.py b/src/smlp_py/smlp_terms.py index 93dc8129..4881a15b 100644 --- a/src/smlp_py/smlp_terms.py +++ b/src/smlp_py/smlp_terms.py @@ -20,7 +20,7 @@ from smlp_py.smlp_utils import (np_JSONEncoder, lists_union_order_preserving_without_duplicates, list_subtraction_set, get_expression_variables, str_to_bool) #from smlp_py.smlp_spec import SmlpSpec - +import pysmt.shortcuts.Real as pysmtReal from smlp_py.NN_verifiers.verifiers import MarabouVerifier import pysmt @@ -1486,10 +1486,10 @@ def pysmt_feature_scaler_to_term(self, orig_feat_var, parser, orig_min, orig_max parser.add_symbol(orig_feat_var, 'real') orig_feat_var = parser.get_symbol(orig_feat_var) if orig_min == orig_max: - return pysmt.shortcuts.Real(0) + return pysmtReal(0) else: - scaling_factor = pysmt.shortcuts.Real(1) / pysmt.shortcuts.Real(orig_max - orig_min) - orig_min_cnst = pysmt.shortcuts.Real(orig_min) + scaling_factor = pysmtReal(1) / pysmtReal(orig_max - orig_min) + orig_min_cnst = pysmtReal(orig_min) # (orig_feat_var - orig_min_cnst) scaled_expr = pysmt.shortcuts.Minus(orig_feat_var, orig_min_cnst) @@ -1589,7 +1589,7 @@ def __init__(self): } self.parser = TextToPysmtParser() - self.parser.init_variables(inputs=[("x1", "real"), ('x2', 'int'), ('p1', 'real'), ('p2', 'int'), + self.parser.init_variables(symbols=[("x1", "real"), ('x2', 'int'), ('p1', 'real'), ('p2', 'int'), ('y1', 'real'), ('y2', 'real')]) self.verifier = MarabouVerifier() @@ -1981,7 +1981,7 @@ def compute_stability_formula_theta(self, cex, delta_dict:dict, radii_dict, univ if self._ENABLE_PYSMT: value = float(self.ground_smlp_expr_to_value(cex[var])) PYSMT_var = self.parser.get_symbol(var) - type = pysmt.shortcuts.Int if str(PYSMT_var.get_type()) == "Int" else pysmt.shortcuts.Real + type = pysmt.shortcuts.Int if str(PYSMT_var.get_type()) == "Int" else pysmtReal calc_type = int if str(PYSMT_var.get_type()) == "Int" else float lower = calc_type(value - verifier_rad_term) lower = type(lower) diff --git a/src/smlp_py/smtlib/text_to_sympy.py b/src/smlp_py/smtlib/text_to_sympy.py index 27ac283c..3eb19de9 100755 --- a/src/smlp_py/smtlib/text_to_sympy.py +++ b/src/smlp_py/smtlib/text_to_sympy.py @@ -309,10 +309,12 @@ def cast_number(self, symbol_type, number): elif symbol_type == INT: return Int(number) - def init_variables(self, inputs: List[Tuple[str, str]]) -> None: - for input_var in inputs: + def init_variables(self, symbols: List[Tuple[str, str]]) -> None: + for input_var in symbols: name, type = input_var + unscaled_name = f"{name}_unscaled" self.add_symbol(name, type) + self.add_symbol(unscaled_name, type) def add_symbol(self, name, symbol_type): assert symbol_type.lower() in pysmt_types.keys() From 93a86644289051f468bfa84e90b557642d131d03 Mon Sep 17 00:00:00 2001 From: konstantopoulos-tetractys <37417801+ntinouldinho@users.noreply.github.com> Date: Tue, 30 Jul 2024 22:42:51 +0100 Subject: [PATCH 07/28] support objectives --- src/smlp_py/NN_verifiers/test_marabou.py | 30 +++- src/smlp_py/NN_verifiers/verifiers.py | 73 +++++---- src/smlp_py/smlp_optimize.py | 4 +- src/smlp_py/smlp_query.py | 39 +++-- src/smlp_py/smlp_terms.py | 45 ++++-- src/smlp_py/smtlib/text_to_sympy.py | 185 ++++++++++++++++++++--- 6 files changed, 286 insertions(+), 90 deletions(-) diff --git a/src/smlp_py/NN_verifiers/test_marabou.py b/src/smlp_py/NN_verifiers/test_marabou.py index e5a3c531..56096a44 100755 --- a/src/smlp_py/NN_verifiers/test_marabou.py +++ b/src/smlp_py/NN_verifiers/test_marabou.py @@ -5,6 +5,13 @@ from pysmt.typing import * import tf2onnx import numpy as np +from pysmt.shortcuts import Symbol, Times, Minus, Div, Real +from pysmt.smtlib.parser import get_formula +# from pysmt.oracles import get_logic +from pysmt.typing import REAL +from z3 import simplify, parse_smt2_string +import z3 + from maraboupy.MarabouPythonic import * @@ -19,10 +26,28 @@ ('y1', 'real'), ('y2', 'real')]) mb = MarabouVerifier(parser=parser) - mb.init_variables(inputs=[("x1", "Real"), ('x2', 'Integer'), ('p1', 'Integer'), ('p2', 'Integer')], + mb.init_variables(inputs=[("x1", "Real"), ('x2', 'Integer'), ('p1', 'Real'), ('p2', 'Integer')], outputs=[('y1', 'Real'), ('y2', 'Real')]) mb.initialize() + smlp_formula = '(let ((|:0| (* (/ 281474976710656 2944425288877159) (- y1 (/ 1080863910568919 4503599627370496))))) (let ((|:1| (* (/ 281474976710656 2559564553220679) (- (* (/ 1 2) (+ y1 y2)) (/ 1170935903116329 1125899906842624))))) (>= (ite (< |:0| |:1|) |:0| |:1|) 1)))' + smlp_str = f""" + (declare-fun y1 () Real) + (declare-fun y2 () Real) + (assert {smlp_formula}) + """ + + smlp_parsed = z3.parse_smt2_string(smlp_str) + smlp_simplified = z3.simplify(smlp_parsed[0]) + ex = parser.parse(str(smlp_simplified)) + # ex = parser.replace_constants_with_floats_and_evaluate(ex) + marabou_formula = parser.convert_ite_to_conjunctions_disjunctions(ex) + print(marabou_formula.serialize()) + + + + + y1 = parser.get_symbol("y1_unscaled") y2 = parser.get_symbol("y2_unscaled") p1 = parser.get_symbol("p1_unscaled") @@ -85,9 +110,10 @@ ) mb.apply_restrictions(x2_int) mb.apply_restrictions(p2_int) - mb.apply_restrictions(beta) + # mb.apply_restrictions(beta) mb.apply_restrictions(alpha) mb.apply_restrictions(eta) + # mb.apply_restrictions(marabou_formula) # mb.apply_restrictions(solution) # mb.apply_restrictions(theta) diff --git a/src/smlp_py/NN_verifiers/verifiers.py b/src/smlp_py/NN_verifiers/verifiers.py index 105759ba..c18b436b 100755 --- a/src/smlp_py/NN_verifiers/verifiers.py +++ b/src/smlp_py/NN_verifiers/verifiers.py @@ -12,6 +12,7 @@ from maraboupy.MarabouPythonic import * from pysmt.walkers import IdentityDagWalker from fractions import Fraction +import smlp from src.smlp_py.smtlib.smt_to_pysmt import smtlib_to_pysmt from src.smlp_py.smtlib.text_to_sympy import TextToPysmtParser @@ -78,17 +79,8 @@ def __init__(self, model_path=None, parser=None): # Dictionary containing variables self.bounds = {} - # Dictionary containing MarabouCommon.Disjunction for each variable - self.disjunctions = dict() - - # List containing yet to be added equations and statements - self.unprocessed_eq = [] - # List of MarabouCommon.Equation currently applied to network query - self.equations = set() - - # Error variable bounding around excluded values, - # e.g. var != val -> And(var >= val + epsilon, var <= val - epsilon) + self.equations = [] # List of variables self.variables = [] @@ -101,18 +93,19 @@ def __init__(self, model_path=None, parser=None): self.data_bounds = None # Adds conjunction of equations between bounds in form: # e.g. Int(var), var >= 0, var <= 3 -> Or(var == 0, var == 1, var == 2, var == 3) - self.int_enable = False # Stack for keeping ipq self.ipq_stack = [] self.parser = parser + self.network_num_vars = None def initialize(self): self.model_file_path = "/home/kkon/Desktop/smlp/result/abc_smlp_toy_basic_nn_keras_model_complete.h5" self.convert_to_pb() self.load_json() + self.network_num_vars = self.network.numVars self.add_unscaled_variables() @@ -135,6 +128,7 @@ def convert_to_pb(self, output_model_file_path="."): # Load the SavedModel model = tf.saved_model.load(output_model_file_path) concrete_func = model.signatures[tf.saved_model.DEFAULT_SERVING_SIGNATURE_DEF_KEY] + print("converted h5 to pb...") # Convert to ConcreteFunction frozen_func = convert_variables_to_constants_v2(concrete_func) @@ -145,9 +139,7 @@ def convert_to_pb(self, output_model_file_path="."): f.write(graph_def.SerializeToString()) self.network = Marabou.read_tf('model.pb') - ipq = self.network.getInputQuery() - self.ipq_stack.append(ipq) - print("converted h5 to pb...") + def init_variables(self, inputs: List[Tuple[str, str]], outputs: List[Tuple[str, str]]) -> None: @@ -190,7 +182,7 @@ def convert_scaled_unscaled(self): def get_variable_by_name(self, name: str) -> Optional[Tuple[Variable, int]]: is_output = name.startswith("y") - is_unscaled = name.find("_unscaled") + is_unscaled = name.find("_unscaled") != -1 repository = self.unscaled_variables if is_unscaled else self.variables for index, variable in enumerate(repository): @@ -205,23 +197,18 @@ def get_variable_by_name(self, name: str) -> Optional[Tuple[Variable, int]]: def reset(self): self.network.clear() - self.network = Marabou.read_tf(self.model_file_path, modelType="savedModel_v2") - + self.network = Marabou.read_tf('model.pb') + self.unscaled_variables = [] + self.add_unscaled_variables() # Default bounds for network + for equation in self.equations: + self.apply_restrictions(equation) - - if self.int_enable: - for variable in self.variables: - if variable.type == Variable.Type.Int: - possible_values = list(range(int(variable.bounds.min), int(variable.bounds.max))) - possible_values = map(lambda p: "{var} == {val}".format(var=variable.name, val=p), possible_values) - self.add_disjunctions(possible_values) - print("{var} is int type adding values in range {min} to {max}".format(var=variable, - min=variable.bounds.min, - max=variable.bounds.max)) + def add_permanent_constraint(self, formula): + self.equations.append(formula) def add_bound(self, variable:str, value, direction="upper", strict=True): - var, var_index = self.get_variable_by_name(variable) + var, var_index = self.get_variable_by_name(f"{variable}_unscaled") if var is None: return None @@ -271,10 +258,8 @@ def apply_restrictions(self, formula): self.process_disjunctions(disjunctions) def transform_pysmt_to_marabou_equation(self, formula): - symbol, comparator, scalar = formula - symbol, is_output = self.get_variable_by_name(str(symbol)) + symbols, comparator, scalar = formula equation_type = None - scalar = float(scalar.constant_value()) if comparator in convert_comparison_operators: equation_type = convert_comparison_operators[comparator] @@ -287,20 +272,29 @@ def transform_pysmt_to_marabou_equation(self, formula): scalar = self.epsilon(scalar, "up") equation = MarabouUtils.Equation(equation_type) - equation.addAddend(1, symbol.index) equation.setScalar(scalar) + + for parameter in symbols: + coefficient, symbol = parameter + # TODO: do not enforce the unscaled variables + name = str(symbol) + if name.find("_unscaled") == -1: + name += "_unscaled" + symbol, index = self.get_variable_by_name(name) + equation.addAddend(coefficient, index) + return equation - def create_equation(self, formula): + def create_equation(self, formula, from_and=False): equations = [] if formula.is_and(): - equation = [self.create_equation(eq) for eq in formula.args()] + equation = [self.create_equation(eq, from_and=True) for eq in formula.args()] return equation elif formula.is_le() or formula.is_lt() or formula.is_equals(): res = self.parser.extract_components(formula) equations.append(self.transform_pysmt_to_marabou_equation(res)) - return equations + return equations[0] if from_and else equations def process_disjunctions(self, disjunctions): marabou_disjunction = [] @@ -336,8 +330,8 @@ def traverse(node, source=[]): def process_comparison(self, formula): if formula.is_le() or formula.is_lt() or formula.is_equals(): symbol, comparison, constant = self.parser.extract_components(formula) + _, symbol = symbol[0] symbol = str(symbol) - constant = float(constant.constant_value()) if comparison == "<=": self.add_bound(symbol, constant, direction="upper", strict=False) @@ -348,6 +342,7 @@ def process_comparison(self, formula): elif comparison == ">": self.add_bound(symbol, constant, direction="lower", strict=True) elif comparison == "=": + # TODO: add a marabou equation instead self.add_bound(symbol, constant, direction="lower", strict=False) self.add_bound(symbol, constant, direction="upper", strict=False) else: @@ -408,7 +403,7 @@ def alpha(self): # self.network.setUpperBound(b1, 1) def find_witness(self, witness): answers = {} - for variable in self.variables: + for variable in self.unscaled_variables: _, index = self.get_variable_by_name(variable.name) answers[variable.name] = witness[index] return answers @@ -417,9 +412,9 @@ def solve(self): try: results = self.network.solve() if results and results[0] == 'unsat': - return {} + return "UNSAT", {} else: # sat - return self.find_witness(results[1]) + return "SAT", self.find_witness(results[1]) except Exception as e: print(e) return None diff --git a/src/smlp_py/smlp_optimize.py b/src/smlp_py/smlp_optimize.py index 7032e5d8..6fd203b2 100755 --- a/src/smlp_py/smlp_optimize.py +++ b/src/smlp_py/smlp_optimize.py @@ -47,7 +47,7 @@ def __init__(self): self._DEF_OBJECTIVES_EXPRS = None self._DEF_APPROXIMATE_FRACTIONS:bool = True self._DEF_FRACTION_PRECISION:int = 64 - self._ENABLE_PYSMT = False + self._ENABLE_PYSMT = True # Formulae alpha, beta, eta are used in single and pareto optimization tasks. # They are used to constrain control variables x and response variables y as follows: @@ -825,7 +825,7 @@ def check_synthesis_feasibility(self, feasibility:bool, objv_names:list[str], ob self._opt_logger.info('Pareto optimization synthesis feasibility check: Start') self._opt_tracer.info('synthesis_feasibility') quer_res = self._queryInst.query_condition(True, model_full_term_dict, 'synthesis_feasibility', 'True', beta, - domain, eta, alpha, theta_radii_dict, delta, solver_logic, True, float_approx, float_precision) + domain, eta, alpha, theta_radii_dict, delta, solver_logic, True, float_approx, float_precision, self.verifier) #print('quer_res', quer_res) if quer_res['query_status'] == 'UNSAT': self._opt_logger.info('Pareto optimization synthesis feasibility check: End') diff --git a/src/smlp_py/smlp_query.py b/src/smlp_py/smlp_query.py index aeac9221..30594ca2 100644 --- a/src/smlp_py/smlp_query.py +++ b/src/smlp_py/smlp_query.py @@ -42,6 +42,7 @@ def __init__(self): self._trace_runtime = None self._trace_precision = None self._trace_anonymize = None + self._ENABLE_PYSMT = True def set_logger(self, logger): self._query_logger = logger @@ -98,11 +99,15 @@ def synthesis_results_file(self): def find_candidate(self, solver): #res = solver.check() - res = self._modelTermsInst.smlp_solver_check(solver, 'ca', self._lemma_precision) - if self._modelTermsInst.solver_status_unknown(res): # isinstance(res, smlp.unknown): - return None - else: + if self._ENABLE_PYSMT: + res, _ = solver.solve() return res + else: + res = self._modelTermsInst.smlp_solver_check(solver, 'ca', self._lemma_precision) + if self._modelTermsInst.solver_status_unknown(res): # isinstance(res, smlp.unknown): + return None + else: + return res def update_consistecy_results(self, mode_status_dict, interface_consistent, model_consistent, mode_status, mode_results_file): @@ -513,7 +518,7 @@ def smlp_certify(self, syst_expr_dict:dict, algo:str, model:dict, # Enhancement !!!: implement timeout ? UNKNOWN return value def query_condition(self, universal, model_full_term_dict:dict, quer_name:str, quer_expr:str, quer:smlp.form2, domain:smlp.domain, eta:smlp.form2, alpha:smlp.form2, theta_radii_dict:dict, #beta:smlp.form2, - delta:dict, solver_logic:str, witn:bool, sat_approx:bool, sat_precision:int): + delta:dict, solver_logic:str, witn:bool, sat_approx:bool, sat_precision:int, candidate_solver=None): # feasibility (existence) of at least one candidate feasible = None if quer_expr is not None: @@ -521,15 +526,21 @@ def query_condition(self, universal, model_full_term_dict:dict, quer_name:str, q else: self._query_logger.info('Querying condition {} <-> {}'.format(str(quer_name), str(quer))) #print('query', quer, 'eta', eta, 'delta', delta) - candidate_solver = self._modelTermsInst.create_model_exploration_instance_from_smlp_components( - domain, model_full_term_dict, True, solver_logic) - - # add the remaining user constraints and the query - candidate_solver.add(eta) - candidate_solver.add(alpha) - #candidate_solver.add(beta) - candidate_solver.add(quer) - test = self._smlpTermsInst.smlp_simplify(quer) + if not self._ENABLE_PYSMT: + candidate_solver = self._modelTermsInst.create_model_exploration_instance_from_smlp_components( + domain, model_full_term_dict, True, solver_logic) + + # add the remaining user constraints and the query + candidate_solver.add(eta) + candidate_solver.add(alpha) + #candidate_solver.add(beta) + candidate_solver.add(quer) + else: + candidate_solver.reset() + candidate_solver.add(eta) + candidate_solver.add(alpha) + candidate_solver.add(quer) + #print('eta', eta); print('alpha', alpha); print('quer', quer); #print('solving query', quer) self._query_tracer.info('{},{}'.format('synthesis' if universal else 'query', str(quer_name))) #, str(quer_expr) ,{} diff --git a/src/smlp_py/smlp_terms.py b/src/smlp_py/smlp_terms.py index 4881a15b..092161a5 100644 --- a/src/smlp_py/smlp_terms.py +++ b/src/smlp_py/smlp_terms.py @@ -20,7 +20,7 @@ from smlp_py.smlp_utils import (np_JSONEncoder, lists_union_order_preserving_without_duplicates, list_subtraction_set, get_expression_variables, str_to_bool) #from smlp_py.smlp_spec import SmlpSpec -import pysmt.shortcuts.Real as pysmtReal +from pysmt.shortcuts import Real as pysmtReal from smlp_py.NN_verifiers.verifiers import MarabouVerifier import pysmt @@ -1592,12 +1592,12 @@ def __init__(self): self.parser.init_variables(symbols=[("x1", "real"), ('x2', 'int'), ('p1', 'real'), ('p2', 'int'), ('y1', 'real'), ('y2', 'real')]) - self.verifier = MarabouVerifier() + self.verifier = MarabouVerifier(parser=self.parser) self.verifier.init_variables(inputs=[("x1", "Real"), ('x2', 'Integer'), ('p1', 'Real'), ('p2', 'Integer')], outputs=[('y1', 'Real'), ('y2', 'Real')]) - self._ENABLE_PYSMT = False - self._RETURN_PYSMT = False + self._ENABLE_PYSMT = True + self._RETURN_PYSMT = True # set logger from a caller script @@ -2224,6 +2224,8 @@ def create_model_exploration_base_components(self, syst_expr_dict:dict, algo, mo spec_domain_dict = self._specInst.get_spec_domain_dict; #print('spec_domain_dict', spec_domain_dict) self.verifier.initialize() + self.add_integer_constraints() + # contraints on features used as control variables and on the responses alph_ranges = self.compute_input_ranges_formula_alpha_eta('alpha', feat_names, spec_domain_dict); # print('alph_ranges') @@ -2378,11 +2380,13 @@ def smlp_solver_check(self, solver, call_name:str, lemma_precision:int=0): sat_model = self.witness_term_to_const(res.model, approximate=False, precision=None) if approx_lemmas: sat_model_approx = self.approximate_witness_term(res.model, lemma_precision) + return TextToPysmtParser.SAT #print('res.model', res.model, 'sat_model', sat_model) elif isinstance(res, smlp.unsat): #print('smlp_unsat', smlp.unsat) status = 'unsat' sat_model = {} + return TextToPysmtParser.UNSAT else: raise Exception('Unexpected solver result ' + str(res)) @@ -2486,22 +2490,37 @@ def get_solver_model(self, res): def check_alpha_eta_consistency(self, domain:smlp.domain, model_full_term_dict:dict, alpha:smlp.form2, eta:smlp.form2, solver_logic:str): #print('create solver: model', model_full_term_dict, flush=True) - solver = self.create_model_exploration_instance_from_smlp_components( + if not self._RETURN_PYSMT: + solver = self.create_model_exploration_instance_from_smlp_components( domain, model_full_term_dict, False, solver_logic) - #print('add alpha', alpha, flush=True) - solver.add(alpha); #print('alpha', alpha, flush=True) - solver.add(eta); #print('eta', eta) - #print('create check', flush=True) - #res = solver.check(); print('res', res, flush=True) - res = self.smlp_solver_check(solver, 'interface_consistency' if model_full_term_dict is None else 'model_consistency') + #print('add alpha', alpha, flush=True) + solver.add(alpha); #print('alpha', alpha, flush=True) + solver.add(eta); #print('eta', eta) + #print('create check', flush=True) + #res = solver.check(); print('res', res, flush=True) + res = self.smlp_solver_check(solver, 'interface_consistency' if model_full_term_dict is None else 'model_consistency') + else: + self.verifier.reset() + self.verifier.apply_restrictions(alpha) + self.verifier.apply_restrictions(eta) + res, witness = self.verifier.solve() + consistency_type = 'Input and knob' if model_full_term_dict is None else 'Model' - if isinstance(res, smlp.sat): + if res=="SAT": self._smlp_terms_logger.info(consistency_type + ' interface constraints are consistent') interface_consistent = True - elif isinstance(res, smlp.unsat): + elif res=="UNSAT": self._smlp_terms_logger.info(consistency_type + ' interface constraints are inconsistent') interface_consistent = False else: raise Exception('alpha and eta cosnsistency check failed to complete') return interface_consistent + def add_integer_constraints(self): + for symbol, values in self._specInst.get_spec_domain_dict.items(): + if values['range'] == 'int': + ranges = values['interval'] + integer_formula = self.parser.create_integer_disjunction(f'{symbol}_unscaled', (ranges[0], ranges[-1])) + self.verifier.add_permanent_constraint(integer_formula) + + diff --git a/src/smlp_py/smtlib/text_to_sympy.py b/src/smlp_py/smtlib/text_to_sympy.py index 3eb19de9..70d27e26 100755 --- a/src/smlp_py/smtlib/text_to_sympy.py +++ b/src/smlp_py/smtlib/text_to_sympy.py @@ -1,5 +1,6 @@ import re +import gmpy2 from pysmt import * from sympy.logic.boolalg import And, Or, Not from pysmt.shortcuts import Symbol, And, Or, Not, Implies, Iff, Ite, Equals, Plus, Minus, Times, Div, Pow, Bool, TRUE, \ @@ -83,6 +84,12 @@ def check_inequality(formula): return checker.is_inequality and not checker.contains_and_or class TextToPysmtParser(object): + SAT = "SAT" + UNSAT = "UNSAT" + types = pysmt_types + real = Real + true = TRUE + false = FALSE _instance = None def __new__(cls, *args, **kwargs): @@ -101,7 +108,7 @@ def __init__(self): ast.Pow: Pow, # Exponentiation ast.BitXor: Iff, # Bitwise XOR (interpreted as logical Iff) - ast.USub: Minus, # Unary subtraction (negation) + ast.USub: lambda l: -l, # Unary subtraction (negation) ast.Eq: Equals, # Equality ast.NotEq: Not, # Not equal @@ -114,7 +121,10 @@ def __init__(self): ast.Or: Or, # Logical OR ast.Not: Not, # Logical NOT - ast.IfExp: Ite # If expression + ast.IfExp: Ite, # If expression + ast.Call: And, + 'If': Ite, + 'And': And } def _div_op(self, left, right): @@ -219,25 +229,56 @@ def extract_coefficient(self, symbol): def extract_components(self, comparison: FNode): left = comparison.arg(0) right = comparison.arg(1) + + if not right.is_constant() and not left.is_constant(): + raise ValueError("The right-hand side of the formula must be a constant") + comparator = self.decide_comparator(comparison) - # if left.is_constant(): - # # so the formula is like const <= a*x - # # check if right is like a*x - # right = self.extract_coefficient(right) - # - # elif right.is_constant(): - # # check if left is like a*x - # left = self.extract_coefficient(left) + terms_subformula = left if right.is_constant() else right + terms = [] + def traverse(node): + if node.is_times(): + coeff, var = node.args() + if coeff.is_constant() and var.is_symbol(): + terms.append((float(coeff.constant_value()), var)) + elif var.is_constant() and coeff.is_symbol(): + terms.append((float(var.constant_value()), coeff)) + else: + raise ValueError("Invalid term structure in linear inequality") + elif node.is_plus(): + for arg in node.args(): + traverse(arg) + elif node.is_minus(): + left, right = node.args() + traverse(left) + if right.is_times(): + coeff, var = right.args() + if coeff.is_constant() and var.is_symbol(): + terms.append((-float(coeff.constant_value()), var)) + elif var.is_constant() and coeff.is_symbol(): + terms.append((-float(var.constant_value()), coeff)) + else: + raise ValueError("Invalid term structure in linear inequality") + else: + raise ValueError("Invalid term structure in linear inequality") + elif node.is_symbol(): + terms.append((1.0, node)) + elif node.is_constant(): + terms.append((node.constant_value(), Real(0))) + else: + raise ValueError("Unsupported node type in linear inequality") + + traverse(terms_subformula) - if left.is_symbol() and right.is_constant(): - return left, comparator, right - elif right.is_symbol() and left.is_constant(): - return right, self.opposite_comparator(comparator), left + if right.is_constant(): + scalar = float(right.constant_value()) + return terms, comparator, scalar else: - raise ValueError("Comparison does not contain a simple variable and constant") + scalar = float(left.constant_value()) + return terms, self.opposite_comparator(comparator), scalar def process_formula(self, formula: FNode): components = [] @@ -313,8 +354,9 @@ def init_variables(self, symbols: List[Tuple[str, str]]) -> None: for input_var in symbols: name, type = input_var unscaled_name = f"{name}_unscaled" - self.add_symbol(name, type) - self.add_symbol(unscaled_name, type) + # TODO: i replaced the type variable with real, make sure that's ok + self.add_symbol(name, 'real') + self.add_symbol(unscaled_name, 'real') def add_symbol(self, name, symbol_type): assert symbol_type.lower() in pysmt_types.keys() @@ -324,17 +366,92 @@ def get_symbol(self, name): assert name in self.symbols.keys() return self.symbols[name] + def replace_constants_with_floats_and_evaluate(self, formula: FNode) -> FNode: + def traverse(node: FNode) -> FNode: + if node.is_plus(): + left, right = node.args() + new_left = traverse(left) + new_right = traverse(right) + if new_left.is_constant() and new_right.is_constant(): + return Real(new_left.constant_value() + new_right.constant_value()) + return Plus(new_left, new_right) + elif node.is_minus(): + left, right = node.args() + new_left = traverse(left) + new_right = traverse(right) + if new_left.is_constant() and new_right.is_constant(): + return Real(new_left.constant_value() - new_right.constant_value()) + return Minus(new_left, new_right) + elif node.is_times(): + left, right = node.args() + new_left = traverse(left) + new_right = traverse(right) + if new_left.is_constant() and new_right.is_constant(): + return Real(new_left.constant_value() * new_right.constant_value()) + return Times(new_left, new_right) + elif node.is_div(): + left, right = node.args() + new_left = traverse(left) + new_right = traverse(right) + if new_left.is_constant() and new_right.is_constant(): + return Real(new_left.constant_value() / new_right.constant_value()) + return Div(new_left, new_right) + elif node.is_ite(): + condition, true_branch, false_branch = node.args() + new_condition = traverse(condition) + new_true_branch = traverse(true_branch) + new_false_branch = traverse(false_branch) + return Ite(new_condition, new_true_branch, new_false_branch) + elif node.is_le(): + left, right = node.args() + new_left = traverse(left) + new_right = traverse(right) + return LE(new_left, new_right) + elif node.is_lt(): + left, right = node.args() + new_left = traverse(left) + new_right = traverse(right) + return LT(new_left, new_right) + elif node.is_and(): + new_args = [traverse(arg) for arg in node.args()] + return And(new_args) + elif node.is_or(): + new_args = [traverse(arg) for arg in node.args()] + return Or(new_args) + elif node.is_constant(): + if isinstance(node.constant_value(), gmpy2.mpq): + return Real(float(node.constant_value())) + elif node.is_int_constant(): + return Real(float(node.constant_value())) + elif node.is_real_constant(): + return Real(node.constant_value()) + else: + return node + + return traverse(formula) + def parse(self, expr): assert isinstance(expr, str) symbol_list = self.symbols def eval_(node): if isinstance(node, ast.Num): - return Real(node.n) if isinstance(node.n, float) else Int(node.n) + # return Real(node.n) if isinstance(node.n, float) else Int(node.n) + return Real(float(node.n)) elif isinstance(node, ast.BinOp): - return self._ast_operators_map[type(node.op)](eval_(node.left), eval_(node.right)) + left = eval_(node.left) + right = eval_(node.right) + if left.is_constant() and right.is_constant(): + if isinstance(node.op, ast.Mult): + return Real(float(left.constant_value() * right.constant_value())) + elif isinstance(node.op, ast.Div): + return Real(float(left.constant_value() / right.constant_value())) + return self._ast_operators_map[type(node.op)](left, right) elif isinstance(node, ast.UnaryOp): - return self._ast_operators_map[type(node.op)](eval_(node.operand)) + operand = eval_(node.operand) + if operand.is_constant() and isinstance(node.op, ast.USub): + return Real(-operand.constant_value()) + return self._ast_operators_map[type(node.op)](operand) elif isinstance(node, ast.Name): return symbol_list[node.id] elif isinstance(node, ast.BoolOp): @@ -350,6 +467,14 @@ def eval_(node): left = eval_(comparator) result = And(result, self._ast_operators_map[type(op)](left, eval_(comparator))) return result + elif isinstance(node, ast.Call): + func = node.func.id + args = [eval_(arg) for arg in node.args] + if func in self._ast_operators_map: + return self._ast_operators_map[func](*args) + else: + raise ValueError(f"Unsupported function call: {func}") + elif isinstance(node, ast.IfExp): return self._ast_operators_map[ast.IfExp](eval_(node.test), eval_(node.body), eval_(node.orelse)) elif isinstance(node, ast.Constant): @@ -368,6 +493,26 @@ def eval_(node): return eval_(ast.parse(expr, mode='eval').body) + def convert_ite_to_conjunctions_disjunctions(self, formula): + def traverse(node): + if node.is_ite(): + condition, true_branch, false_branch = node.args() + return Or( + And(traverse(condition), traverse(true_branch)), + And(traverse(Not(condition)), traverse(false_branch)) + ) + elif node.is_and(): + new_args = [traverse(arg) for arg in node.args()] + return And(new_args) + elif node.is_or(): + new_args = [traverse(arg) for arg in node.args()] + return Or(new_args) + elif node.is_not(): + return self.propagate_negation(node) + else: + return node + + return traverse(formula) if __name__ == "__main__": From d94c6aa3db116e7556d03d790fb0022f315e0851 Mon Sep 17 00:00:00 2001 From: konstantopoulos-tetractys <37417801+ntinouldinho@users.noreply.github.com> Date: Wed, 31 Jul 2024 16:47:49 +0100 Subject: [PATCH 08/28] integrate marabou to smlp workflow --- src/smlp_py/NN_verifiers/verifiers.py | 42 +++++++++++-- src/smlp_py/smlp_optimize.py | 58 ++++++++++++++---- src/smlp_py/smlp_query.py | 63 +++++++++++++------ src/smlp_py/smlp_terms.py | 88 +++++++++++++++++++-------- src/smlp_py/smtlib/text_to_sympy.py | 16 +++++ 5 files changed, 201 insertions(+), 66 deletions(-) diff --git a/src/smlp_py/NN_verifiers/verifiers.py b/src/smlp_py/NN_verifiers/verifiers.py index c18b436b..4cd243aa 100755 --- a/src/smlp_py/NN_verifiers/verifiers.py +++ b/src/smlp_py/NN_verifiers/verifiers.py @@ -250,6 +250,7 @@ def add_bounds(self, variable, bounds=None, num="real", grid=None): self.network.addDisjunctionConstraint(disjunction) def apply_restrictions(self, formula): + formula = self.parser.simplify(formula) conjunctions, disjunctions = self.process_formula(formula) for conjunction in conjunctions: @@ -285,8 +286,28 @@ def transform_pysmt_to_marabou_equation(self, formula): return equation + def is_negation_of_ite(self, formula): + if formula.is_and(): + if len(formula.args()) == 2: + # this is a custom logical block for handling the negation of objectives which yield a formula that looks like + # Or(A,B,C) where C = And(Or(D,E),Or(F,G)) (1) , which needs to be translated into: + # let K = Or(D,E), then C=And(K, Or(F,G)), which is equivalent to: Or(And(K,F),And(K,G)) (2). + # Then, using (2): And(K,F) = And(F, Or(D,E)), which is equivalent to: Or(And(F,D), And(F,E)) (3) + # Same applied to And(G, K) = Or(And(G,D), And(G,E)) (4) + # Finally: Or(And(K,F),And(K,G)) = Or(Or(And(F,D), And(F,E)), Or(And(G,D), And(G,E))), + # Which can be simplified to Or(And(F,D), And(F,E), And(G,D), And(G,E)) + left = formula.args()[0] + right = formula.args()[1] + if left.is_or() and len(left.args()) == 2 and right.is_or() and len(right.args()) == 2: + eq_1, eq_2 = left.args()[0], left.args()[1] + eq_3, eq_4 = right.args()[0], right.args()[1] + return True, [eq_1, eq_2, eq_3, eq_4] + return False, [] + def create_equation(self, formula, from_and=False): equations = [] + formula = self.parser.simplify(formula) + if formula.is_and(): equation = [self.create_equation(eq, from_and=True) for eq in formula.args()] return equation @@ -301,8 +322,14 @@ def process_disjunctions(self, disjunctions): for disjunction in disjunctions: # split the disjunction into separate formulas for formula in disjunction.args(): - equation = self.create_equation(formula) - marabou_disjunction.append(equation) + res, formulas = self.is_negation_of_ite(formula) + if res: + for form in formulas: + equation = self.create_equation(form) + marabou_disjunction.append(equation) + else: + equation = self.create_equation(formula) + marabou_disjunction.append(equation) if len(marabou_disjunction) > 0: self.network.addDisjunctionConstraint(marabou_disjunction) @@ -402,17 +429,20 @@ def alpha(self): # self.network.setLowerBound(b1, 0) # self.network.setUpperBound(b1, 1) def find_witness(self, witness): - answers = {} + answers = {"result":"SAT", "witness":{}, 'witness_var':{}} for variable in self.unscaled_variables: - _, index = self.get_variable_by_name(variable.name) - answers[variable.name] = witness[index] + _, unscaled_index = self.get_variable_by_name(variable.name) + name = variable.name.replace("_unscaled", "") + scaled_var, _ = self.get_variable_by_name(name) + answers['witness_var'][scaled_var] = witness[unscaled_index] + answers['witness'][scaled_var.name] = witness[unscaled_index] return answers def solve(self): try: results = self.network.solve() if results and results[0] == 'unsat': - return "UNSAT", {} + return "UNSAT", {"result":"UNSAT", "witness": {}} else: # sat return "SAT", self.find_witness(results[1]) except Exception as e: diff --git a/src/smlp_py/smlp_optimize.py b/src/smlp_py/smlp_optimize.py index 6fd203b2..6183fe5d 100755 --- a/src/smlp_py/smlp_optimize.py +++ b/src/smlp_py/smlp_optimize.py @@ -265,9 +265,13 @@ def optimize_single_objective(self, model_full_term_dict:dict, objv_name:str, ob T = (l + u) / 2 #quer_form = objv_term > smlp.Cnst(T) quer_form = objv_term >= smlp.Cnst(T) + quer_form = self._modelTermsInst.verifier.parser.handle_ite_formula(quer_form) if self._ENABLE_PYSMT else quer_form quer_expr = '{} >= {}'.format(objv_expr, str(T)) if objv_expr is not None else None quer_name = objv_name + '_' + str(T) - quer_and_beta = self._smlpTermsInst.smlp_and(quer_form, beta) if not beta == smlp.true else quer_form + if not beta == smlp.true: + quer_and_beta = self._modelTermsInst.verifier.parser.and_(quer_form, beta) if self._ENABLE_PYSMT else self._smlpTermsInst.smlp_and(quer_form, beta) + else: + quer_and_beta = quer_form #print('quer_and_beta', quer_and_beta) 'u0_l0_u_l_T' self._opt_tracer.info('objective_thresholds_u0_l0_u_l_T, {} : {} : {} : {} : {}'.format(str(u0),str(l0),str(u),str(l),str(T))) quer_res = self._queryInst.query_condition( @@ -325,6 +329,13 @@ def optimize_single_objective(self, model_full_term_dict:dict, objv_name:str, ob #print('objv_term', objv_term, flush=True); print('stable_witness_terms', stable_witness_terms, flush=True) l_prev = l # save the value of l, it is for reporting only. #if objv_expr is not None: # the objective is not a symbolic max_min term, we may need its value, at least to see search progress + + # if self._ENABLE_PYSMT: + # substitution = {self.parser.get_symbol(x): pysmt_objv_terms_dict[x]} + # # Apply the substitution + # objv_terms_dict[k] = self.parser.simplify(v.substitute(substitution)) + # else: + objv_witn_val_term = smlp.subst(objv_term, stable_witness_terms); #print('objv_witn_val_term', objv_witn_val_term) #using objective values as lower bounds is not sound since objective value in sat model is the ceneter-point value # and the objective's value is not guaranteed to be a lower bound in entire stability region @@ -432,7 +443,9 @@ def optimize_single_objectives(self, feat_names:list, resp_names:list, #X:pd.Dat #assert scale_objectives objv_terms_dict, orig_objv_terms_dict, scaled_objv_terms_dict = \ self._modelTermsInst.compute_objectives_terms(objv_names, objv_exprs, objv_bounds_dict, scale_objectives) - + + pysmt_objv_terms_dict, pysmt_orig_objv_terms_dict, pysmt_scaled_objv_terms_dict = \ + self._modelTermsInst.pysmt_compute_objectives_terms(objv_names, objv_exprs, objv_bounds_dict, scale_objectives) # TODO: set sat_approx to False once dump and load with Fractions will work opt_conf = {} for i, (objv_name, objv_term) in enumerate(list(objv_terms_dict.items())): @@ -459,10 +472,11 @@ def optimize_single_objectives(self, feat_names:list, resp_names:list, #X:pd.Dat def active_objectives_max_min_bounds(self, model_full_term_dict:dict, objv_terms_dict:dict, t:list[float], smlp_domain:smlp.domain, alpha:smlp.form2, beta:smlp.form2, eta:smlp.form2, theta_radii_dict, epsilon:float, delta:float, solver_logic:str, direction, scale_objectives, objv_bounds, update_thresholds_dict, - sat_approx:bool, sat_precision:int, save_trace:bool): + sat_approx:bool, sat_precision:int, save_trace:bool, pysmt_objv_terms_dict=None): assert direction == 'up' eta_F_t = eta min_objs = None + pysmt_min_objs = None min_name = '' #print('thresholds t', t, 'objv_terms_dict', objv_terms_dict) for j, (objv_name, objv_term) in enumerate(objv_terms_dict.items()): @@ -470,14 +484,18 @@ def active_objectives_max_min_bounds(self, model_full_term_dict:dict, objv_terms eta_F_t = self._smlpTermsInst.smlp_and(eta_F_t, objv_term > smlp.Cnst(t[j])) else: min_name = min_name + '_' + objv_name if min_name != '' else objv_name - if min_objs is not None: - if self._ENABLE_PYSMT: - min_objs = TextToPysmtParser.ite_(objv_term < min_objs, objv_term, min_objs) + if self._ENABLE_PYSMT: + pysmt_objv_term = pysmt_objv_terms_dict[objv_name] + if pysmt_min_objs is not None: + pysmt_min_objs = TextToPysmtParser.ite_(pysmt_objv_term < pysmt_min_objs, pysmt_objv_term, pysmt_min_objs) else: - min_objs = smlp.Ite(objv_term < min_objs, objv_term, min_objs) + pysmt_min_objs = pysmt_objv_term + # else: + if min_objs is not None: + min_objs = smlp.Ite(objv_term < min_objs, objv_term, min_objs) else: min_objs = objv_term - + # When active_objectives_max_min_bounds() is called for the first time from # optimize_pareto_objectives(), the list t which represents the proven lower # bounds of objectives, is composed of None's, and the proven lower @@ -672,6 +690,10 @@ def optimize_pareto_objectives(self, feat_names:list[str], resp_names:list[str], objv_terms_dict, orig_objv_terms_dict, scaled_objv_terms_dict = \ self._modelTermsInst.compute_objectives_terms(objv_names, objv_exprs, objv_bounds_dict, scale_objectives) + pysmt_objv_terms_dict = None + pysmt_objv_terms_dict, pysmt_orig_objv_terms_dict, pysmt_scaled_objv_terms_dict = \ + self._modelTermsInst.pysmt_compute_objectives_terms(objv_names, objv_exprs, objv_bounds_dict, + scale_objectives) objv_count = len(objv_names) objv_enum = range(objv_count) @@ -705,7 +727,7 @@ def sanity_check_fixed_objv_thresholds(t:list[float], fixed_onjv_dict): self._opt_tracer.info('pareto_iteration,{},{},{}'.format(str(call_n), '__'.join(objv_names), '__'.join([str(e) for e in s]))) c_lo, c_up, witness = self.active_objectives_max_min_bounds(model_full_term_dict, objv_terms_dict, s, smlp_domain, alpha, beta, eta, theta_radii_dict, epsilon, delta, solver_logic, direction, - scale_objectives, objv_bounds_dict, call_info_dict, sat_approx, sat_precision, save_trace) + scale_objectives, objv_bounds_dict, call_info_dict, sat_approx, sat_precision, save_trace, pysmt_objv_terms_dict) #print('c_lo', c_lo, 'c_up', c_up); print('witness', witness); assert c_lo != np.inf @@ -770,12 +792,22 @@ def sanity_check_fixed_objv_thresholds(t:list[float], fixed_onjv_dict): self._opt_logger.info('Checking whether to fix objective {} at threshold {}...\n'.format(str(j), str(s[j]))) self._opt_tracer.info('activity check, objective {} threshold {}'.format(str(objv_names[j]), str(s[j]))) #print('objv_terms_dict', objv_terms_dict) - quer_form = smlp.true + quer_form = self._modelTermsInst.parser.true() if self._ENABLE_PYSMT else smlp.true for i in objv_enum: #print('obv i', list(objv_terms_dict.keys())[i]) - quer_form = self._smlpTermsInst.smlp_and(quer_form, list(objv_terms_dict.values())[i] > smlp.Cnst(t[i])) + if self._ENABLE_PYSMT: + quer_form = self._modelTermsInst.parser.and_(quer_form, + list(objv_terms_dict.values())[i] > self._modelTermsInst.parser.real( + t[i])) + else: + quer_form = self._smlpTermsInst.smlp_and(quer_form, list(objv_terms_dict.values())[i] > smlp.Cnst(t[i])) #print('queryform', quer_form) - quer_and_beta = self._smlpTermsInst.smlp_and(quer_form, beta) if not beta == smlp.true else quer_form + if not beta == smlp.true: + quer_and_beta = self._modelTermsInst.verifier.parser.and_(quer_form, + beta) if self._ENABLE_PYSMT else self._smlpTermsInst.smlp_and(quer_form, beta) + else: + quer_and_beta = quer_form + opt_quer_name = 'thresholds_' + '_'.join(str(x) for x in t) + '_check' quer_res = self._queryInst.query_condition(True, model_full_term_dict, opt_quer_name, 'True', quer_and_beta, smlp_domain, eta, alpha, theta_radii_dict, delta, solver_logic, True, sat_approx, sat_precision) @@ -825,7 +857,7 @@ def check_synthesis_feasibility(self, feasibility:bool, objv_names:list[str], ob self._opt_logger.info('Pareto optimization synthesis feasibility check: Start') self._opt_tracer.info('synthesis_feasibility') quer_res = self._queryInst.query_condition(True, model_full_term_dict, 'synthesis_feasibility', 'True', beta, - domain, eta, alpha, theta_radii_dict, delta, solver_logic, True, float_approx, float_precision, self.verifier) + domain, eta, alpha, theta_radii_dict, delta, solver_logic, True, float_approx, float_precision) #print('quer_res', quer_res) if quer_res['query_status'] == 'UNSAT': self._opt_logger.info('Pareto optimization synthesis feasibility check: End') diff --git a/src/smlp_py/smlp_query.py b/src/smlp_py/smlp_query.py index 30594ca2..b4c82ae0 100644 --- a/src/smlp_py/smlp_query.py +++ b/src/smlp_py/smlp_query.py @@ -100,8 +100,8 @@ def synthesis_results_file(self): def find_candidate(self, solver): #res = solver.check() if self._ENABLE_PYSMT: - res, _ = solver.solve() - return res + res, witness = solver.solve() + return witness else: res = self._modelTermsInst.smlp_solver_check(solver, 'ca', self._lemma_precision) if self._modelTermsInst.solver_status_unknown(res): # isinstance(res, smlp.unknown): @@ -172,13 +172,23 @@ def check_concrete_witness_consistency(self, domain:smlp.domain, model_full_term # theta x y /\ alpha y /\ ! ( beta y /\ obj y >= T) def find_candidate_counter_example(self, universal, domain:smlp.domain, cand:dict, query:smlp.form2, model_full_term_dict:dict, alpha:smlp.form2, theta_radii_dict:dict, solver_logic:str): #, beta:smlp.form2 - solver = self._modelTermsInst.create_model_exploration_instance_from_smlp_components( - domain, model_full_term_dict, False, solver_logic) + theta = self._modelTermsInst.compute_stability_formula_theta(cand, None, theta_radii_dict, universal) - solver.add(theta); #print('adding theta', theta) - solver.add(alpha); #print('adding alpha', alpha) - solver.add(self._smlpTermsInst.smlp_not(query)); #print('adding negated quert', query) - return self._modelTermsInst.smlp_solver_check(solver, 'ce', self._lemma_precision) + if self._ENABLE_PYSMT: + self._modelTermsInst.verifier.reset() + self._modelTermsInst.verifier.apply_restrictions(theta) + self._modelTermsInst.verifier.apply_restrictions(alpha) + negation = self._modelTermsInst.verifier.parser.propagate_negation(query) + self._modelTermsInst.verifier.apply_restrictions(negation) + res, witness = self._modelTermsInst.verifier.solve() + return witness + else: + solver = self._modelTermsInst.create_model_exploration_instance_from_smlp_components( + domain, model_full_term_dict, False, solver_logic) + solver.add(theta); #print('adding theta', theta) + solver.add(alpha); #print('adding alpha', alpha) + solver.add(self._smlpTermsInst.smlp_not(query)); #print('adding negated quert', query) + return self._modelTermsInst.smlp_solver_check(solver, 'ce', self._lemma_precision) #return solver.check() # Enhancement !!!: at least add here the delta condition @@ -518,7 +528,7 @@ def smlp_certify(self, syst_expr_dict:dict, algo:str, model:dict, # Enhancement !!!: implement timeout ? UNKNOWN return value def query_condition(self, universal, model_full_term_dict:dict, quer_name:str, quer_expr:str, quer:smlp.form2, domain:smlp.domain, eta:smlp.form2, alpha:smlp.form2, theta_radii_dict:dict, #beta:smlp.form2, - delta:dict, solver_logic:str, witn:bool, sat_approx:bool, sat_precision:int, candidate_solver=None): + delta:dict, solver_logic:str, witn:bool, sat_approx:bool, sat_precision:int): # feasibility (existence) of at least one candidate feasible = None if quer_expr is not None: @@ -536,10 +546,10 @@ def query_condition(self, universal, model_full_term_dict:dict, quer_name:str, q #candidate_solver.add(beta) candidate_solver.add(quer) else: - candidate_solver.reset() - candidate_solver.add(eta) - candidate_solver.add(alpha) - candidate_solver.add(quer) + self._modelTermsInst.verifier.reset() + self._modelTermsInst.verifier.apply_restrictions(eta) + self._modelTermsInst.verifier.apply_restrictions(alpha) + self._modelTermsInst.verifier.apply_restrictions(quer) #print('eta', eta); print('alpha', alpha); print('quer', quer); #print('solving query', quer) @@ -552,9 +562,11 @@ def query_condition(self, universal, model_full_term_dict:dict, quer_name:str, q # solve Ex. eta x /\ Ay. theta x y -> alpha y -> (beta y /\ query) print('searching for a candidate', flush=True) - ca = self.find_candidate(candidate_solver) - - if self._modelTermsInst.solver_status_sat(ca): # isinstance(ca, smlp.sat): + ca = self.find_candidate(self._modelTermsInst.verifier) if self._ENABLE_PYSMT else self.find_candidate(candidate_solver) + + condition = self._modelTermsInst.solver_status_sat(ca["result"]) if self._ENABLE_PYSMT else self._modelTermsInst.solver_status_sat(ca) + + if condition: # isinstance(ca, smlp.sat): print('candidate found -- checking stability', flush=True) #print('ca', ca_model) ca_model = self._modelTermsInst.get_solver_model(ca) #ca.model @@ -577,7 +589,11 @@ def query_condition(self, universal, model_full_term_dict:dict, quer_name:str, q else: ce = self.find_candidate_counter_example(universal, domain, ca_model, quer, model_full_term_dict, alpha, theta_radii_dict, solver_logic) - if self._modelTermsInst.solver_status_sat(ce): #isinstance(ce, smlp.sat): + + is_sat = self._modelTermsInst.solver_status_sat(ce["result"]) if self._ENABLE_PYSMT else self._modelTermsInst.solver_status_sat(ce) + is_unsat = self._modelTermsInst.solver_status_unsat(ce["result"]) if self._ENABLE_PYSMT else self._modelTermsInst.solver_status_unsat(ce) + + if is_sat: #isinstance(ce, smlp.sat): print('candidate not stable -- continue search', flush=True) ce_model = self._modelTermsInst.get_solver_model(ce) #ce.model cem = ce_model.copy(); #print('ce model', cem) @@ -601,9 +617,13 @@ def query_condition(self, universal, model_full_term_dict:dict, quer_name:str, q else: lemma = self.generalize_counter_example(cem); #print('lemma', lemma) theta = self._modelTermsInst.compute_stability_formula_theta(lemma, delta, theta_radii_dict, universal) - candidate_solver.add(self._smlpTermsInst.smlp_not(theta)) + if self._ENABLE_PYSMT: + theta_negation = self._modelTermsInst.parser.propagate_negation(theta) + self._modelTermsInst.verifier.add_permanent_constraint(theta_negation) + else: + candidate_solver.add(self._smlpTermsInst.smlp_not(theta)) continue - elif self._modelTermsInst.solver_status_unsat(ce): #isinstance(ce, smlp.unsat): + elif is_unsat: #isinstance(ce, smlp.unsat): #print('candidate stable -- return candidate') self._query_logger.info('Query completed with result: STABLE_SAT (satisfiable)') if witn: # export witness (use numbers as values, not terms) @@ -616,7 +636,10 @@ def query_condition(self, universal, model_full_term_dict:dict, quer_name:str, q assert quer_ce_val return {'query_status':'STABLE_SAT', 'witness':witness_vals_dict, 'feasible':feasible} else: - return {'query_status':'STABLE_SAT', 'witness':ca_model, 'feasible':feasible} + if self._ENABLE_PYSMT: + return {'query_status':'STABLE_SAT', 'witness':ca['witness_var'], 'feasible':feasible} + else: + return {'query_status':'STABLE_SAT', 'witness':ca_model, 'feasible':feasible} elif self._modelTermsInst.solver_status_unsat(ca): #isinstance(ca, smlp.unsat): self._query_logger.info('Query completed with result: UNSAT (unsatisfiable)') if feasible is None: diff --git a/src/smlp_py/smlp_terms.py b/src/smlp_py/smlp_terms.py index 092161a5..62714102 100644 --- a/src/smlp_py/smlp_terms.py +++ b/src/smlp_py/smlp_terms.py @@ -682,6 +682,8 @@ def ground_smlp_expr_to_value(self, ground_term:smlp.term2, approximate=False, p # Can also be applied to a dictionary where values are terms. def witness_term_to_const(self, witness, approximate=False, precision=64): witness_vals_dict = {} + if isinstance(witness, dict): + return witness for k,t in witness.items(): witness_vals_dict[k] = self.ground_smlp_expr_to_value(t, approximate, precision) return witness_vals_dict @@ -1874,16 +1876,8 @@ def compute_objectives_terms(self, objv_names, objv_exprs, objv_bounds, scale_ob for objv_name, objv_expr in zip(objv_names, objv_exprs)]) #self._smlpTermsInst. #print('orig_objv_terms_dict', orig_objv_terms_dict) - if self._ENABLE_PYSMT: - pysmt_objv_terms_dict = dict([(objv_name, self.parser.parse(objv_expr)) \ - for objv_name, objv_expr in zip(objv_names, objv_exprs)]) - if scale_objv: - if self._ENABLE_PYSMT: - scaled_objv_terms_dict = self.feature_scaler_terms(objv_bounds, objv_names, - self.parser) # ._scalerTermsInst - else: - scaled_objv_terms_dict = self.feature_scaler_terms(objv_bounds, objv_names) # ._scalerTermsInst + scaled_objv_terms_dict = self.feature_scaler_terms(objv_bounds, objv_names) # ._scalerTermsInst #print('scaled_objv_terms_dict', scaled_objv_terms_dict) objv_terms_dict = {} @@ -1891,12 +1885,8 @@ def compute_objectives_terms(self, objv_names, objv_exprs, objv_bounds, scale_ob #print('k', k, 'v', v, type(v)); x = list(orig_objv_terms_dict.keys())[i]; #print('x', x); print('arg', orig_objv_terms_dict[x]) - if self._ENABLE_PYSMT: - substitution = {self.parser.get_symbol(x): pysmt_objv_terms_dict[x]} - # Apply the substitution - objv_terms_dict[k] = self.parser.simplify(v.substitute(substitution)) - else: - objv_terms_dict[k] = self.smlp_cnst_fold(v, {x: orig_objv_terms_dict[x]}) + + objv_terms_dict[k] = self.smlp_cnst_fold(v, {x: orig_objv_terms_dict[x]}) #objv_terms_dict = scaled_objv_terms_dict else: objv_terms_dict = orig_objv_terms_dict @@ -1907,7 +1897,41 @@ def compute_objectives_terms(self, objv_names, objv_exprs, objv_bounds, scale_ob for objv_name in objv_names] #print('objv_terms_dict', objv_terms_dict) return objv_terms_dict, orig_objv_terms_dict, scaled_objv_terms_dict - + + def pysmt_compute_objectives_terms(self, objv_names, objv_exprs, objv_bounds, scale_objv): + # print('objv_exprs', objv_exprs) + if objv_exprs is None: + return None, None, None, None + + pysmt_objv_terms_dict = dict([(objv_name, self.parser.parse(objv_expr)) \ + for objv_name, objv_expr in zip(objv_names, objv_exprs)]) + + if scale_objv: + scaled_objv_terms_dict = self.feature_scaler_terms(objv_bounds, objv_names, + self.parser) # ._scalerTermsInst + + # print('scaled_objv_terms_dict', scaled_objv_terms_dict) + objv_terms_dict = {} + for i, (k, v) in enumerate(scaled_objv_terms_dict.items()): + # print('k', k, 'v', v, type(v)); + x = list(pysmt_objv_terms_dict.keys())[i]; + # print('x', x); print('arg', orig_objv_terms_dict[x]) + # if self._ENABLE_PYSMT: + substitution = {self.parser.get_symbol(x): pysmt_objv_terms_dict[x]} + # Apply the substitution + objv_terms_dict[k] = self.parser.simplify(v.substitute(substitution)) + # else: + # objv_terms_dict = scaled_objv_terms_dict + else: + objv_terms_dict = pysmt_objv_terms_dict + scaled_objv_terms_dict = None + + if scaled_objv_terms_dict is not None: + assert list(scaled_objv_terms_dict.keys()) == [self._scaled_name(objv_name) # ._scalerTermsInst + for objv_name in objv_names] + # print('objv_terms_dict', objv_terms_dict) + return objv_terms_dict, pysmt_objv_terms_dict, scaled_objv_terms_dict + # Compute stability region theta; used also in generating lemmas during search for a stable solution. # cex is assignement of values to knobs. Even if cex contains assignements to inputs, such assignements @@ -1972,14 +1996,20 @@ def compute_stability_formula_theta(self, cex, delta_dict:dict, radii_dict, univ # from relative radius based on variable values in the counter-exaples to candidate rather than variable values # in the candidate itself. if delta_rel is not None: # radius for a lemma -- cex holds values of candidate counter-example - rad_term = rad_term * abs(var_term) + if isinstance(var_term, smlp.term2): + rad_term = rad_term * abs(var_term) + else: + rad_term = rad_term * self.smlp_cnst(abs(var_term)) else: # radius for excluding a candidate -- cex holds values of the candidate - rad_term = rad_term * abs(cex[var]) + if isinstance(cex[var], smlp.term2): + rad_term = rad_term * abs(cex[var]) + else: + rad_term = rad_term * self.smlp_cnst(abs(cex[var])) elif delta_dict is not None: raise exception('When delta dictionary is provided, either absolute or relative or delta must be specified') - theta_form = self.smlp_and(theta_form, ((abs(var_term - cex[var])) <= rad_term)) + theta_form = self.smlp_and(theta_form, ((abs(var_term - self.smlp_cnst(abs(cex[var])))) <= rad_term)) if self._ENABLE_PYSMT: - value = float(self.ground_smlp_expr_to_value(cex[var])) + value = float(cex[var]) PYSMT_var = self.parser.get_symbol(var) type = pysmt.shortcuts.Int if str(PYSMT_var.get_type()) == "Int" else pysmtReal calc_type = int if str(PYSMT_var.get_type()) == "Int" else float @@ -1990,7 +2020,7 @@ def compute_stability_formula_theta(self, cex, delta_dict:dict, radii_dict, univ PYSMT_theta_form = self.parser.and_(PYSMT_theta_form, PYSMT_var >= lower, PYSMT_var <= upper) # self.verifier.add_bounds(var, (value - verifier_rad_term, value + verifier_rad_term)) #print('theta_form', theta_form) - return theta_form + return PYSMT_theta_form if self._RETURN_PYSMT else theta_form # Creates eta constraints on control parameters (knobs) from the spec. # Covers grid as well as range/interval constraints. @@ -2159,7 +2189,7 @@ def compute_beta_formula(self, beta_expr, model_inps_outps): if len(dont_care_vars) > 0: raise Exception('Variables ' + str(dont_care_vars) + ' in optimization constraints (beta) are not part of the model') - return self.ast_expr_to_term(beta_expr) + return self.parser.parse(beta_expr) if self._ENABLE_PYSMT else self.ast_expr_to_term(beta_expr) def compute_eta_formula(self, eta_expr, model_inputs): if eta_expr is None: @@ -2461,20 +2491,24 @@ def smlp_solver_check(self, solver, call_name:str, lemma_precision:int=0): return res def solver_status_sat(self, res): - return isinstance(res, smlp.sat) + return res=="SAT" if self._ENABLE_PYSMT else isinstance(res, smlp.sat) def solver_status_unsat(self, res): - return isinstance(res, smlp.unsat) + return res=="UNSAT" if self._ENABLE_PYSMT else isinstance(res, smlp.unsat) def solver_status_unknown(self, res): - return isinstance(res, smlp.unknown) + return res=="UNKNOWN" if self._ENABLE_PYSMT else isinstance(res, smlp.unknown) # we return value assignmenets to interface (input, knob, output) variables defined in the Spec file # (and not values assigned to any other variables that might be defined additionally as part of solver domain, # like variables tree_i_resp that we decalre as part of domain for tree models with flat encoding). def get_solver_model(self, res): - if self.solver_status_sat(res): - reduced_model = dict((k,v) for k,v in res.model.items() if k in self._specInst.get_spec_interface) + condition = self.solver_status_sat(res["result"]) if self._ENABLE_PYSMT else self.solver_status_sat(res) + if condition: + if self._ENABLE_PYSMT: + reduced_model = dict((k, v) for k, v in res["witness"].items() if k in self._specInst.get_spec_interface) + else: + reduced_model = dict((k,v) for k,v in res.model.items() if k in self._specInst.get_spec_interface) return reduced_model else: return None diff --git a/src/smlp_py/smtlib/text_to_sympy.py b/src/smlp_py/smtlib/text_to_sympy.py index 70d27e26..43330ec1 100755 --- a/src/smlp_py/smtlib/text_to_sympy.py +++ b/src/smlp_py/smtlib/text_to_sympy.py @@ -1,6 +1,7 @@ import re import gmpy2 +import z3 from pysmt import * from sympy.logic.boolalg import And, Or, Not from pysmt.shortcuts import Symbol, And, Or, Not, Implies, Iff, Ite, Equals, Plus, Minus, Times, Div, Pow, Bool, TRUE, \ @@ -304,6 +305,7 @@ def propagate_negation(self, formula): """ Apply negation to a formula and propagate the negation inside without leaving any negations in the formula. """ + formula = self.simplify(formula) if formula.is_not(): return self.propagate_negation(formula.arg(0)) # Remove double negation if exists @@ -366,6 +368,20 @@ def get_symbol(self, name): assert name in self.symbols.keys() return self.symbols[name] + def handle_ite_formula(self, formula): + smlp_str = f""" + (declare-fun y1 () Real) + (declare-fun y2 () Real) + (assert {formula}) + """ + + smlp_parsed = z3.parse_smt2_string(smlp_str) + smlp_simplified = z3.simplify(smlp_parsed[0]) + ex = self.parse(str(smlp_simplified)) + # ex = parser.replace_constants_with_floats_and_evaluate(ex) + marabou_formula = self.convert_ite_to_conjunctions_disjunctions(ex) + return marabou_formula + def replace_constants_with_floats_and_evaluate(self, formula: FNode) -> FNode: def traverse(node: FNode) -> FNode: if node.is_plus(): From fef60cc3d0b8c507c15fb9c22d0b27f899dde18a Mon Sep 17 00:00:00 2001 From: konstantopoulos-tetractys <37417801+ntinouldinho@users.noreply.github.com> Date: Thu, 1 Aug 2024 00:43:44 +0100 Subject: [PATCH 09/28] complete marabou integration --- src/smlp_py/NN_verifiers/test_marabou.py | 10 ++++ src/smlp_py/NN_verifiers/verifiers.py | 55 ++++++++++-------- src/smlp_py/smlp_optimize.py | 38 ++++++++----- src/smlp_py/smlp_query.py | 20 ++++--- src/smlp_py/smlp_terms.py | 72 +++++++++++++----------- src/smlp_py/smtlib/text_to_sympy.py | 45 +++++++++++++-- 6 files changed, 153 insertions(+), 87 deletions(-) diff --git a/src/smlp_py/NN_verifiers/test_marabou.py b/src/smlp_py/NN_verifiers/test_marabou.py index 56096a44..3e0bea83 100755 --- a/src/smlp_py/NN_verifiers/test_marabou.py +++ b/src/smlp_py/NN_verifiers/test_marabou.py @@ -11,6 +11,8 @@ from pysmt.typing import REAL from z3 import simplify, parse_smt2_string import z3 +from pysmt.smtlib.script import smtlibscript_from_formula +from io import StringIO from maraboupy.MarabouPythonic import * @@ -70,6 +72,7 @@ Equals(p2, Real(3)) ) + print(smtlibscript_from_formula(solution)) theta = And( GE(p1, Real(6.8)), GE(p2, Real(3.8)), @@ -108,6 +111,13 @@ p1.Equals(Real(7.0)) ) ) + script = smtlibscript_from_formula(eta) + + outstream = StringIO() + script.serialize(outstream) + output = outstream.getvalue() + smlp_parsed = z3.parse_smt2_string(output) + smlp_simplified = z3.simplify(smlp_parsed[0]) mb.apply_restrictions(x2_int) mb.apply_restrictions(p2_int) # mb.apply_restrictions(beta) diff --git a/src/smlp_py/NN_verifiers/verifiers.py b/src/smlp_py/NN_verifiers/verifiers.py index 4cd243aa..108d7aa1 100755 --- a/src/smlp_py/NN_verifiers/verifiers.py +++ b/src/smlp_py/NN_verifiers/verifiers.py @@ -249,14 +249,14 @@ def add_bounds(self, variable, bounds=None, num="real", grid=None): self.network.addDisjunctionConstraint(disjunction) - def apply_restrictions(self, formula): + def apply_restrictions(self, formula, need_simplification=False): formula = self.parser.simplify(formula) conjunctions, disjunctions = self.process_formula(formula) for conjunction in conjunctions: - self.process_comparison(conjunction) + self.process_comparison(conjunction, need_simplification) - self.process_disjunctions(disjunctions) + self.process_disjunctions(disjunctions, need_simplification) def transform_pysmt_to_marabou_equation(self, formula): symbols, comparator, scalar = formula @@ -304,7 +304,7 @@ def is_negation_of_ite(self, formula): return True, [eq_1, eq_2, eq_3, eq_4] return False, [] - def create_equation(self, formula, from_and=False): + def create_equation(self, formula, from_and=False, need_simplification=False): equations = [] formula = self.parser.simplify(formula) @@ -312,12 +312,12 @@ def create_equation(self, formula, from_and=False): equation = [self.create_equation(eq, from_and=True) for eq in formula.args()] return equation elif formula.is_le() or formula.is_lt() or formula.is_equals(): - res = self.parser.extract_components(formula) + res = self.parser.extract_components(formula, need_simplification) equations.append(self.transform_pysmt_to_marabou_equation(res)) return equations[0] if from_and else equations - def process_disjunctions(self, disjunctions): + def process_disjunctions(self, disjunctions, need_simplification=False): marabou_disjunction = [] for disjunction in disjunctions: # split the disjunction into separate formulas @@ -325,10 +325,10 @@ def process_disjunctions(self, disjunctions): res, formulas = self.is_negation_of_ite(formula) if res: for form in formulas: - equation = self.create_equation(form) + equation = self.create_equation(form, from_and=False, need_simplification=need_simplification) marabou_disjunction.append(equation) else: - equation = self.create_equation(formula) + equation = self.create_equation(formula, from_and=False, need_simplification=need_simplification) marabou_disjunction.append(equation) if len(marabou_disjunction) > 0: @@ -354,24 +354,29 @@ def traverse(node, source=[]): traverse(formula) return conjunctions, disjunctions - def process_comparison(self, formula): + def process_comparison(self, formula, need_simplification=False): if formula.is_le() or formula.is_lt() or formula.is_equals(): - symbol, comparison, constant = self.parser.extract_components(formula) - _, symbol = symbol[0] - symbol = str(symbol) - - if comparison == "<=": - self.add_bound(symbol, constant, direction="upper", strict=False) - elif comparison == "<": - self.add_bound(symbol, constant, direction="upper", strict=True) - if comparison == ">=": - self.add_bound(symbol, constant, direction="lower", strict=False) - elif comparison == ">": - self.add_bound(symbol, constant, direction="lower", strict=True) - elif comparison == "=": - # TODO: add a marabou equation instead - self.add_bound(symbol, constant, direction="lower", strict=False) - self.add_bound(symbol, constant, direction="upper", strict=False) + symbols, comparison, constant = self.parser.extract_components(formula, need_simplification) + + if len(symbols) > 1: + equation = self.transform_pysmt_to_marabou_equation((symbols, comparison, constant)) + self.network.addEquation(equation) + else: + _, symbol = symbols[0] + symbol = str(symbol) + + if comparison == "<=": + self.add_bound(symbol, constant, direction="upper", strict=False) + elif comparison == "<": + self.add_bound(symbol, constant, direction="upper", strict=True) + if comparison == ">=": + self.add_bound(symbol, constant, direction="lower", strict=False) + elif comparison == ">": + self.add_bound(symbol, constant, direction="lower", strict=True) + elif comparison == "=": + # TODO: add a marabou equation instead + self.add_bound(symbol, constant, direction="lower", strict=False) + self.add_bound(symbol, constant, direction="upper", strict=False) else: return diff --git a/src/smlp_py/smlp_optimize.py b/src/smlp_py/smlp_optimize.py index 6183fe5d..038e8ddc 100755 --- a/src/smlp_py/smlp_optimize.py +++ b/src/smlp_py/smlp_optimize.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # This file is part of smlp. - +import pysmt import smlp from smlp_py.smlp_terms import SmlpTerms, ModelTerms, ScalerTerms from smlp_py.smlp_query import SmlpQuery @@ -14,6 +14,7 @@ import keras import numpy as np from src.smlp_py.smtlib.text_to_sympy import TextToPysmtParser +from pysmt.shortcuts import Real # single or multi-objective optimization, with stability constraints and any user # given constraints on free input, control (knob) and output variables satisfied. @@ -47,7 +48,7 @@ def __init__(self): self._DEF_OBJECTIVES_EXPRS = None self._DEF_APPROXIMATE_FRACTIONS:bool = True self._DEF_FRACTION_PRECISION:int = 64 - self._ENABLE_PYSMT = True + self._ENABLE_PYSMT = False # Formulae alpha, beta, eta are used in single and pareto optimization tasks. # They are used to constrain control variables x and response variables y as follows: @@ -234,7 +235,7 @@ def eval_objv(row): def optimize_single_objective(self, model_full_term_dict:dict, objv_name:str, objv_expr:str, objv_term:smlp.term2, epsilon:float, smlp_domain:smlp.domain, eta:smlp.form2, theta_radii_dict:dict, alpha:smlp.form2, beta:smlp.form2, delta:float, solver_logic:str, scale_objectives:bool, orig_objv_name:str, objv_bounds:dict, call_info=None, sat_approx=False, sat_precision=64, save_trace=False, - l0=None, u0=None, l=(-np.inf), u=np.inf): + l0=None, u0=None, l=(-np.inf), u=np.inf, pysmt_min_objs=None): self._opt_logger.info('Optimize single objective ' + str(objv_name) + ': Start') self._opt_tracer.info('single_objective_u0_l0_u_l, {} : {} : {} : {} : {}'.format(str(objv_name),str(u0),str(l0),str(u),str(l))) #print('l0', l0, 'u0', u0, 'l', l, 'u', u) @@ -330,13 +331,15 @@ def optimize_single_objective(self, model_full_term_dict:dict, objv_name:str, ob l_prev = l # save the value of l, it is for reporting only. #if objv_expr is not None: # the objective is not a symbolic max_min term, we may need its value, at least to see search progress - # if self._ENABLE_PYSMT: - # substitution = {self.parser.get_symbol(x): pysmt_objv_terms_dict[x]} - # # Apply the substitution - # objv_terms_dict[k] = self.parser.simplify(v.substitute(substitution)) - # else: - - objv_witn_val_term = smlp.subst(objv_term, stable_witness_terms); #print('objv_witn_val_term', objv_witn_val_term) + if self._ENABLE_PYSMT: + substitution = {} + for symbol, value in stable_witness_terms.items(): + symbol = self._modelTermsInst.verifier.parser.get_symbol(symbol) + substitution[symbol] = Real(value) + # Apply the substitution + objv_witn_val_term = self._modelTermsInst.verifier.parser.simplify(pysmt_min_objs.substitute(substitution)) + else: + objv_witn_val_term = smlp.subst(objv_term, stable_witness_terms); #print('objv_witn_val_term', objv_witn_val_term) #using objective values as lower bounds is not sound since objective value in sat model is the ceneter-point value # and the objective's value is not guaranteed to be a lower bound in entire stability region #objv_witn_val = self._smlpTermsInst.ground_smlp_expr_to_value(objv_witn_val_term, sat_approx, sat_precision) @@ -481,7 +484,10 @@ def active_objectives_max_min_bounds(self, model_full_term_dict:dict, objv_terms #print('thresholds t', t, 'objv_terms_dict', objv_terms_dict) for j, (objv_name, objv_term) in enumerate(objv_terms_dict.items()): if t[j] is not None: - eta_F_t = self._smlpTermsInst.smlp_and(eta_F_t, objv_term > smlp.Cnst(t[j])) + if self._ENABLE_PYSMT: + eta_F_t = TextToPysmtParser.and_(eta_F_t, pysmt_objv_term > Real(t[j])) + else: + eta_F_t = self._smlpTermsInst.smlp_and(eta_F_t, objv_term > smlp.Cnst(t[j])) else: min_name = min_name + '_' + objv_name if min_name != '' else objv_name if self._ENABLE_PYSMT: @@ -541,7 +547,7 @@ def active_objectives_max_min_bounds(self, model_full_term_dict:dict, objv_terms r = self.optimize_single_objective(model_full_term_dict, min_name, None, min_objs, epsilon, smlp_domain, eta_F_t, theta_radii_dict, alpha, beta, delta, solver_logic, scale_objectives, min_name, objv_bounds, update_thresholds_dict, - sat_approx, sat_precision, save_trace, l0, u0, l, u) + sat_approx, sat_precision, save_trace, l0, u0, l, u, pysmt_min_objs) #print('r', r) c_up = r['threshold_up'] if 'threshold_up' in r else np.inf; #print('c_up', c_up) @@ -792,13 +798,12 @@ def sanity_check_fixed_objv_thresholds(t:list[float], fixed_onjv_dict): self._opt_logger.info('Checking whether to fix objective {} at threshold {}...\n'.format(str(j), str(s[j]))) self._opt_tracer.info('activity check, objective {} threshold {}'.format(str(objv_names[j]), str(s[j]))) #print('objv_terms_dict', objv_terms_dict) - quer_form = self._modelTermsInst.parser.true() if self._ENABLE_PYSMT else smlp.true + quer_form = pysmt.shortcuts.TRUE() if self._ENABLE_PYSMT else smlp.true for i in objv_enum: #print('obv i', list(objv_terms_dict.keys())[i]) if self._ENABLE_PYSMT: quer_form = self._modelTermsInst.parser.and_(quer_form, - list(objv_terms_dict.values())[i] > self._modelTermsInst.parser.real( - t[i])) + list(pysmt_objv_terms_dict.values())[i] > pysmt.shortcuts.Real(t[i])) else: quer_form = self._smlpTermsInst.smlp_and(quer_form, list(objv_terms_dict.values())[i] > smlp.Cnst(t[i])) #print('queryform', quer_form) @@ -808,6 +813,9 @@ def sanity_check_fixed_objv_thresholds(t:list[float], fixed_onjv_dict): else: quer_and_beta = quer_form + if self._ENABLE_PYSMT: + quer_and_beta = self._modelTermsInst.parser.simplify(quer_and_beta) + opt_quer_name = 'thresholds_' + '_'.join(str(x) for x in t) + '_check' quer_res = self._queryInst.query_condition(True, model_full_term_dict, opt_quer_name, 'True', quer_and_beta, smlp_domain, eta, alpha, theta_radii_dict, delta, solver_logic, True, sat_approx, sat_precision) diff --git a/src/smlp_py/smlp_query.py b/src/smlp_py/smlp_query.py index b4c82ae0..cd8998fc 100644 --- a/src/smlp_py/smlp_query.py +++ b/src/smlp_py/smlp_query.py @@ -42,7 +42,7 @@ def __init__(self): self._trace_runtime = None self._trace_precision = None self._trace_anonymize = None - self._ENABLE_PYSMT = True + self._ENABLE_PYSMT = False def set_logger(self, logger): self._query_logger = logger @@ -179,7 +179,7 @@ def find_candidate_counter_example(self, universal, domain:smlp.domain, cand:dic self._modelTermsInst.verifier.apply_restrictions(theta) self._modelTermsInst.verifier.apply_restrictions(alpha) negation = self._modelTermsInst.verifier.parser.propagate_negation(query) - self._modelTermsInst.verifier.apply_restrictions(negation) + self._modelTermsInst.verifier.apply_restrictions(negation, need_simplification=True) res, witness = self._modelTermsInst.verifier.solve() return witness else: @@ -547,9 +547,9 @@ def query_condition(self, universal, model_full_term_dict:dict, quer_name:str, q candidate_solver.add(quer) else: self._modelTermsInst.verifier.reset() - self._modelTermsInst.verifier.apply_restrictions(eta) + self._modelTermsInst.verifier.apply_restrictions(eta, need_simplification=True) self._modelTermsInst.verifier.apply_restrictions(alpha) - self._modelTermsInst.verifier.apply_restrictions(quer) + self._modelTermsInst.verifier.apply_restrictions(quer, need_simplification=True) #print('eta', eta); print('alpha', alpha); print('quer', quer); #print('solving query', quer) @@ -564,9 +564,11 @@ def query_condition(self, universal, model_full_term_dict:dict, quer_name:str, q ca = self.find_candidate(self._modelTermsInst.verifier) if self._ENABLE_PYSMT else self.find_candidate(candidate_solver) - condition = self._modelTermsInst.solver_status_sat(ca["result"]) if self._ENABLE_PYSMT else self._modelTermsInst.solver_status_sat(ca) + condition_sat = self._modelTermsInst.solver_status_sat(ca["result"]) if self._ENABLE_PYSMT else self._modelTermsInst.solver_status_sat(ca) + condition_unsat = self._modelTermsInst.solver_status_unsat(ca["result"]) if self._ENABLE_PYSMT else self._modelTermsInst.solver_status_unsat(ca) + condition_unknown = self._modelTermsInst.solver_status_unsat(ca["result"]) if self._ENABLE_PYSMT else self._modelTermsInst.solver_status_unknown(ca) - if condition: # isinstance(ca, smlp.sat): + if condition_sat: # isinstance(ca, smlp.sat): print('candidate found -- checking stability', flush=True) #print('ca', ca_model) ca_model = self._modelTermsInst.get_solver_model(ca) #ca.model @@ -637,17 +639,17 @@ def query_condition(self, universal, model_full_term_dict:dict, quer_name:str, q return {'query_status':'STABLE_SAT', 'witness':witness_vals_dict, 'feasible':feasible} else: if self._ENABLE_PYSMT: - return {'query_status':'STABLE_SAT', 'witness':ca['witness_var'], 'feasible':feasible} + return {'query_status':'STABLE_SAT', 'witness':ca['witness'], 'feasible':feasible} else: return {'query_status':'STABLE_SAT', 'witness':ca_model, 'feasible':feasible} - elif self._modelTermsInst.solver_status_unsat(ca): #isinstance(ca, smlp.unsat): + elif condition_unsat: #isinstance(ca, smlp.unsat): self._query_logger.info('Query completed with result: UNSAT (unsatisfiable)') if feasible is None: feasible = False #print('candidate does not exist -- query unsuccessful') #print('query unsuccessful: witness does not exist (query is unsat)') return {'query_status':'UNSAT', 'witness':None, 'feasible':feasible} - elif self._modelTermsInst.solver_status_unknown(ca): #isinstance(ca, smlp.unknown): + elif condition_unknown: #isinstance(ca, smlp.unknown): self._opt_logger.info('Completed with result: {}'.format('UNKNOWN')) return {'query_status':'UNKNOWN', 'witness':None, 'feasible':feasible} #raise Exception('UNKNOWN return value in candidate search is currently not supported for queries') diff --git a/src/smlp_py/smlp_terms.py b/src/smlp_py/smlp_terms.py index 62714102..ca41a101 100644 --- a/src/smlp_py/smlp_terms.py +++ b/src/smlp_py/smlp_terms.py @@ -130,8 +130,9 @@ def __init__(self): ast.LtE: self.smlp_le, ast.Gt: self.smlp_gt, ast.GtE: self.smlp_ge, ast.And: self.smlp_and, ast.Or: self.smlp_or, ast.Not: self.smlp_not, ast.IfExp: self.smlp_ite - } - + } + self._ENABLE_PYSMT = False + # set logger from a caller script def set_logger(self, logger): self._smlp_terms_logger = logger @@ -682,10 +683,17 @@ def ground_smlp_expr_to_value(self, ground_term:smlp.term2, approximate=False, p # Can also be applied to a dictionary where values are terms. def witness_term_to_const(self, witness, approximate=False, precision=64): witness_vals_dict = {} - if isinstance(witness, dict): - return witness + # if self._ENABLE_PYSMT: + # return witness for k,t in witness.items(): - witness_vals_dict[k] = self.ground_smlp_expr_to_value(t, approximate, precision) + if isinstance(t, smlp.term2): + new_value = self.ground_smlp_expr_to_value(t, approximate, precision) + elif isinstance(t, pysmt.fnode.FNode) and t.is_constant(): + new_value = float(t.constant_value()) + else: + new_value = float(t) + + witness_vals_dict[k] = new_value return witness_vals_dict # computes and returns sat assignment witness_approx which approximates input witness/sat assignment @@ -1598,8 +1606,8 @@ def __init__(self): self.verifier.init_variables(inputs=[("x1", "Real"), ('x2', 'Integer'), ('p1', 'Real'), ('p2', 'Integer')], outputs=[('y1', 'Real'), ('y2', 'Real')]) - self._ENABLE_PYSMT = True - self._RETURN_PYSMT = True + self._ENABLE_PYSMT = False + self._RETURN_PYSMT = False # set logger from a caller script @@ -1948,8 +1956,7 @@ def compute_stability_formula_theta(self, cex, delta_dict:dict, radii_dict, univ else: delta_rel = delta_abs = None - theta_form = self.smlp_true - PYSMT_theta_form = pysmt.shortcuts.TRUE() + theta_form = pysmt.shortcuts.TRUE() if self._ENABLE_PYSMT else self.smlp_true #print('radii_dict', radii_dict) radii_dict_local = radii_dict.copy() knobs = radii_dict_local.keys(); #print('knobs', knobs); print('cex', cex); print('delta', delta_dict) @@ -1967,16 +1974,16 @@ def compute_stability_formula_theta(self, cex, delta_dict:dict, radii_dict, univ rad = radii['rad-abs']; #print('rad', rad); if delta_rel is not None: # we are generating a lemma rad = rad * (1 + delta_rel) + delta_abs - rad_term = self.smlp_cnst(rad) + if self._ENABLE_PYSMT: - verifier_rad_term = float(rad) + rad_term = float(rad) + else: + rad_term = self.smlp_cnst(rad) elif radii['rad-rel'] is not None: rad = radii['rad-rel']; #print('rad', rad) if delta_rel is not None: # we are generating a lemma rad = rad * (1 + delta_rel) + delta_abs - rad_term = self.smlp_cnst(rad) - if self._ENABLE_PYSMT: - verifier_rad_term = float(rad) + # TODO !!! issue a warning when candidates become closer and closer # TODO !!!!!!! warning when distance between previous and current candidate # TODO !!!!!! warning when FINAL rad + delta is 0, as part of sanity checking options @@ -1995,32 +2002,33 @@ def compute_stability_formula_theta(self, cex, delta_dict:dict, radii_dict, univ # candidate; this is a matter of definition of relative radius, and seems cleaner than computing actual radius # from relative radius based on variable values in the counter-exaples to candidate rather than variable values # in the candidate itself. - if delta_rel is not None: # radius for a lemma -- cex holds values of candidate counter-example - if isinstance(var_term, smlp.term2): + + if self._ENABLE_PYSMT: + rad_term = float(rad) + else: + rad_term = self.smlp_cnst(rad) + if delta_rel is not None: # radius for a lemma -- cex holds values of candidate counter-example rad_term = rad_term * abs(var_term) - else: - rad_term = rad_term * self.smlp_cnst(abs(var_term)) - else: # radius for excluding a candidate -- cex holds values of the candidate - if isinstance(cex[var], smlp.term2): + else: # radius for excluding a candidate -- cex holds values of the candidate rad_term = rad_term * abs(cex[var]) - else: - rad_term = rad_term * self.smlp_cnst(abs(cex[var])) - elif delta_dict is not None: + elif delta_dict is not None: raise exception('When delta dictionary is provided, either absolute or relative or delta must be specified') - theta_form = self.smlp_and(theta_form, ((abs(var_term - self.smlp_cnst(abs(cex[var])))) <= rad_term)) if self._ENABLE_PYSMT: value = float(cex[var]) PYSMT_var = self.parser.get_symbol(var) type = pysmt.shortcuts.Int if str(PYSMT_var.get_type()) == "Int" else pysmtReal calc_type = int if str(PYSMT_var.get_type()) == "Int" else float - lower = calc_type(value - verifier_rad_term) + lower = calc_type(value - rad_term) lower = type(lower) - upper = calc_type(value + verifier_rad_term) + upper = calc_type(value + rad_term) upper = type(upper) - PYSMT_theta_form = self.parser.and_(PYSMT_theta_form, PYSMT_var >= lower, PYSMT_var <= upper) + theta_form = self.parser.and_(theta_form, PYSMT_var >= lower, PYSMT_var <= upper) # self.verifier.add_bounds(var, (value - verifier_rad_term, value + verifier_rad_term)) + else: + theta_form = self.smlp_and(theta_form, ((abs(var_term - cex[var])) <= rad_term)) + #print('theta_form', theta_form) - return PYSMT_theta_form if self._RETURN_PYSMT else theta_form + return theta_form # Creates eta constraints on control parameters (knobs) from the spec. # Covers grid as well as range/interval constraints. @@ -2410,13 +2418,13 @@ def smlp_solver_check(self, solver, call_name:str, lemma_precision:int=0): sat_model = self.witness_term_to_const(res.model, approximate=False, precision=None) if approx_lemmas: sat_model_approx = self.approximate_witness_term(res.model, lemma_precision) - return TextToPysmtParser.SAT + # return TextToPysmtParser.SAT #print('res.model', res.model, 'sat_model', sat_model) elif isinstance(res, smlp.unsat): #print('smlp_unsat', smlp.unsat) status = 'unsat' sat_model = {} - return TextToPysmtParser.UNSAT + # return TextToPysmtParser.UNSAT else: raise Exception('Unexpected solver result ' + str(res)) @@ -2540,10 +2548,10 @@ def check_alpha_eta_consistency(self, domain:smlp.domain, model_full_term_dict:d res, witness = self.verifier.solve() consistency_type = 'Input and knob' if model_full_term_dict is None else 'Model' - if res=="SAT": + if (self._ENABLE_PYSMT and res=="SAT") or isinstance(res, smlp.sat): self._smlp_terms_logger.info(consistency_type + ' interface constraints are consistent') interface_consistent = True - elif res=="UNSAT": + elif (self._ENABLE_PYSMT and res=="UNSAT") or isinstance(res, smlp.unsat): self._smlp_terms_logger.info(consistency_type + ' interface constraints are inconsistent') interface_consistent = False else: diff --git a/src/smlp_py/smtlib/text_to_sympy.py b/src/smlp_py/smtlib/text_to_sympy.py index 43330ec1..2bb41a30 100755 --- a/src/smlp_py/smtlib/text_to_sympy.py +++ b/src/smlp_py/smtlib/text_to_sympy.py @@ -13,6 +13,8 @@ from pysmt.walkers import IdentityDagWalker, DagWalker import ast import smlp +from pysmt.smtlib.script import smtlibscript_from_formula +from io import StringIO from typing import List, Dict, Optional, Tuple @@ -125,7 +127,8 @@ def __init__(self): ast.IfExp: Ite, # If expression ast.Call: And, 'If': Ite, - 'And': And + 'And': And, + 'Not': Not } def _div_op(self, left, right): @@ -227,7 +230,11 @@ def extract_coefficient(self, symbol): return coeff - def extract_components(self, comparison: FNode): + def extract_components(self, comparison: FNode, need_simplification=False): + if need_simplification: + smtlib = self.extract_smtlib(comparison) + comparison = self.handle_ite_formula(smtlib, handle_ite=False) + left = comparison.arg(0) right = comparison.arg(1) @@ -368,18 +375,44 @@ def get_symbol(self, name): assert name in self.symbols.keys() return self.symbols[name] - def handle_ite_formula(self, formula): + def remove_first_and_last_line(self, text): + # Split the text into a list of lines + lines = text.split('\n') + + # Remove the first and last lines + if len(lines) > 1: + lines = lines[1:-2] + else: + # If there's only one line or no line, return an empty string + lines = [] + + # Join the remaining lines back into a single string + return '\n'.join(lines) + + def extract_smtlib(self, formula): + script = smtlibscript_from_formula(formula) + outstream = StringIO() + script.serialize(outstream) + output = outstream.getvalue() + return self.remove_first_and_last_line(output) + + def handle_ite_formula(self, formula, handle_ite=True): smlp_str = f""" (declare-fun y1 () Real) (declare-fun y2 () Real) (assert {formula}) - """ + """ if not isinstance(formula, str) else formula smlp_parsed = z3.parse_smt2_string(smlp_str) smlp_simplified = z3.simplify(smlp_parsed[0]) - ex = self.parse(str(smlp_simplified)) + ex = self.parse(str(smlp_simplified).replace('\n','')) + if ex.is_not(): + ex = self.propagate_negation(ex) # ex = parser.replace_constants_with_floats_and_evaluate(ex) - marabou_formula = self.convert_ite_to_conjunctions_disjunctions(ex) + if handle_ite: + marabou_formula = self.convert_ite_to_conjunctions_disjunctions(ex) + else: + marabou_formula = ex return marabou_formula def replace_constants_with_floats_and_evaluate(self, formula: FNode) -> FNode: From a4c76818af63d82e50e3dd21b9426b2d9ab25b93 Mon Sep 17 00:00:00 2001 From: konstantopoulos-tetractys <37417801+ntinouldinho@users.noreply.github.com> Date: Thu, 1 Aug 2024 01:31:38 +0100 Subject: [PATCH 10/28] verbosity --- src/smlp_py/NN_verifiers/test_marabou.py | 35 +++++++++++++++++------- src/smlp_py/NN_verifiers/verifiers.py | 5 +++- src/smlp_py/smlp_optimize.py | 2 +- src/smlp_py/smlp_query.py | 2 +- src/smlp_py/smlp_terms.py | 6 ++-- src/smlp_py/train_keras.py | 2 +- 6 files changed, 35 insertions(+), 17 deletions(-) diff --git a/src/smlp_py/NN_verifiers/test_marabou.py b/src/smlp_py/NN_verifiers/test_marabou.py index 3e0bea83..690e8680 100755 --- a/src/smlp_py/NN_verifiers/test_marabou.py +++ b/src/smlp_py/NN_verifiers/test_marabou.py @@ -18,6 +18,21 @@ if __name__ == "__main__": + import numpy as np + from tensorflow.keras.models import load_model + + # Load the model from the .h5 file + model = load_model("/home/kkon/Desktop/smlp/result/abc_smlp_toy_basic_nn_keras_model_complete.h5") + + # Prepare your input data + input_data = np.array([[1.043789425, 0, 0.191919192, 0]]) # Example input data + + # Pass the inputs to the model and get the outputs + outputs = model.predict(input_data) + + # Print the outputs + print("Model outputs:", outputs) + from keras.models import load_model # model = load_model("/home/kkon/Desktop/smlp/result/abc_smlp_toy_basic_nn_keras_model_complete.h5") @@ -50,12 +65,12 @@ - y1 = parser.get_symbol("y1_unscaled") - y2 = parser.get_symbol("y2_unscaled") - p1 = parser.get_symbol("p1_unscaled") - p2 = parser.get_symbol("p2_unscaled") - x1 = parser.get_symbol("x1_unscaled") - x2 = parser.get_symbol("x2_unscaled") + y1 = parser.get_symbol("y1") + y2 = parser.get_symbol("y2") + p1 = parser.get_symbol("p1") + p2 = parser.get_symbol("p2") + x1 = parser.get_symbol("x1") + x2 = parser.get_symbol("x2") x2_int = parser.create_integer_disjunction("x2_unscaled", (-1, 1)) p2_int = parser.create_integer_disjunction("p2_unscaled", (3, 7)) @@ -67,7 +82,7 @@ # with x as knob: y1==4.120704402283359 & solution = And( Equals(x1, Real(10)), - Equals(x2, Real(0)), + Equals(x2, Real(-1)), Equals(p1, Real(2)), Equals(p2, Real(3)) ) @@ -121,10 +136,10 @@ mb.apply_restrictions(x2_int) mb.apply_restrictions(p2_int) # mb.apply_restrictions(beta) - mb.apply_restrictions(alpha) - mb.apply_restrictions(eta) + # mb.apply_restrictions(alpha) + # mb.apply_restrictions(eta) # mb.apply_restrictions(marabou_formula) - # mb.apply_restrictions(solution) + mb.apply_restrictions(solution) # mb.apply_restrictions(theta) diff --git a/src/smlp_py/NN_verifiers/verifiers.py b/src/smlp_py/NN_verifiers/verifiers.py index 108d7aa1..03a5f6c2 100755 --- a/src/smlp_py/NN_verifiers/verifiers.py +++ b/src/smlp_py/NN_verifiers/verifiers.py @@ -177,6 +177,7 @@ def convert_scaled_unscaled(self): eq.setScalar(-min_value) # Add the equation to the network + # self.add_permanent_constraint(eq) self.network.addEquation(eq) @@ -441,11 +442,13 @@ def find_witness(self, witness): scaled_var, _ = self.get_variable_by_name(name) answers['witness_var'][scaled_var] = witness[unscaled_index] answers['witness'][scaled_var.name] = witness[unscaled_index] + print(answers['witness']) return answers def solve(self): try: - results = self.network.solve() + options = Marabou.createOptions(verbosity=0) + results = self.network.solve(options) if results and results[0] == 'unsat': return "UNSAT", {"result":"UNSAT", "witness": {}} else: # sat diff --git a/src/smlp_py/smlp_optimize.py b/src/smlp_py/smlp_optimize.py index 038e8ddc..ffbcae8a 100755 --- a/src/smlp_py/smlp_optimize.py +++ b/src/smlp_py/smlp_optimize.py @@ -48,7 +48,7 @@ def __init__(self): self._DEF_OBJECTIVES_EXPRS = None self._DEF_APPROXIMATE_FRACTIONS:bool = True self._DEF_FRACTION_PRECISION:int = 64 - self._ENABLE_PYSMT = False + self._ENABLE_PYSMT = True # Formulae alpha, beta, eta are used in single and pareto optimization tasks. # They are used to constrain control variables x and response variables y as follows: diff --git a/src/smlp_py/smlp_query.py b/src/smlp_py/smlp_query.py index cd8998fc..dccc624c 100644 --- a/src/smlp_py/smlp_query.py +++ b/src/smlp_py/smlp_query.py @@ -42,7 +42,7 @@ def __init__(self): self._trace_runtime = None self._trace_precision = None self._trace_anonymize = None - self._ENABLE_PYSMT = False + self._ENABLE_PYSMT = True def set_logger(self, logger): self._query_logger = logger diff --git a/src/smlp_py/smlp_terms.py b/src/smlp_py/smlp_terms.py index ca41a101..71763867 100644 --- a/src/smlp_py/smlp_terms.py +++ b/src/smlp_py/smlp_terms.py @@ -131,7 +131,7 @@ def __init__(self): ast.And: self.smlp_and, ast.Or: self.smlp_or, ast.Not: self.smlp_not, ast.IfExp: self.smlp_ite } - self._ENABLE_PYSMT = False + self._ENABLE_PYSMT = True # set logger from a caller script def set_logger(self, logger): @@ -1606,8 +1606,8 @@ def __init__(self): self.verifier.init_variables(inputs=[("x1", "Real"), ('x2', 'Integer'), ('p1', 'Real'), ('p2', 'Integer')], outputs=[('y1', 'Real'), ('y2', 'Real')]) - self._ENABLE_PYSMT = False - self._RETURN_PYSMT = False + self._ENABLE_PYSMT = True + self._RETURN_PYSMT = True # set logger from a caller script diff --git a/src/smlp_py/train_keras.py b/src/smlp_py/train_keras.py index d874754b..b67c5376 100644 --- a/src/smlp_py/train_keras.py +++ b/src/smlp_py/train_keras.py @@ -38,7 +38,7 @@ def __init__(self): # hyper parameter defaults self._DEF_LAYERS_SPEC = '2,1' - self._DEF_EPOCHS = 100 + self._DEF_EPOCHS = 200 self._DEF_BATCH_SIZE = 10 self._DEF_OPTIMIZER = 'adam' # options: 'rmsprop', 'adam', 'sgd', 'adagrad', 'nadam' self._DEF_LEARNING_RATE = 0.001 From 32444b3dc2e9cfa59a19e8964ba85bf8f1eb0785 Mon Sep 17 00:00:00 2001 From: konstantopoulos-tetractys <37417801+ntinouldinho@users.noreply.github.com> Date: Thu, 1 Aug 2024 09:21:19 +0100 Subject: [PATCH 11/28] fix bug in input, output indices --- src/smlp_py/NN_verifiers/verifiers.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/smlp_py/NN_verifiers/verifiers.py b/src/smlp_py/NN_verifiers/verifiers.py index 03a5f6c2..7b59f795 100755 --- a/src/smlp_py/NN_verifiers/verifiers.py +++ b/src/smlp_py/NN_verifiers/verifiers.py @@ -170,10 +170,13 @@ def convert_scaled_unscaled(self): scaling_factor = max_value - min_value + _, scaled_var_index = self.get_variable_by_name(scaled_var.name) + _, unscaled_var_index = self.get_variable_by_name(unscaled_var.name) + # Create an equation representing (x_max - x_min) * x_scaled - x_unscaled = - x_min eq = MarabouUtils.Equation(MarabouCore.Equation.EQ) - eq.addAddend(scaling_factor, scaled_var.index) - eq.addAddend(-1, unscaled_var.index) + eq.addAddend(scaling_factor, scaled_var_index) + eq.addAddend(-1, unscaled_var_index) eq.setScalar(-min_value) # Add the equation to the network @@ -447,8 +450,7 @@ def find_witness(self, witness): def solve(self): try: - options = Marabou.createOptions(verbosity=0) - results = self.network.solve(options) + results = self.network.solve() if results and results[0] == 'unsat': return "UNSAT", {"result":"UNSAT", "witness": {}} else: # sat From 9e65fa74f06fba888beaf9ea63f847dacbd48089 Mon Sep 17 00:00:00 2001 From: konstantopoulos-tetractys <37417801+ntinouldinho@users.noreply.github.com> Date: Thu, 1 Aug 2024 11:22:33 +0100 Subject: [PATCH 12/28] delete files --- .../variables/variables.data-00000-of-00001 | Bin 6806 -> 0 bytes .../saved_model/variables/variables.index | Bin 1779 -> 0 bytes .../variables/variables.data-00000-of-00001 | Bin 6806 -> 0 bytes .../NN_verifiers/variables/variables.index | Bin 1779 -> 0 bytes 4 files changed, 0 insertions(+), 0 deletions(-) delete mode 100755 src/smlp_py/NN_verifiers/saved_model/variables/variables.data-00000-of-00001 delete mode 100755 src/smlp_py/NN_verifiers/saved_model/variables/variables.index delete mode 100755 src/smlp_py/NN_verifiers/variables/variables.data-00000-of-00001 delete mode 100755 src/smlp_py/NN_verifiers/variables/variables.index diff --git a/src/smlp_py/NN_verifiers/saved_model/variables/variables.data-00000-of-00001 b/src/smlp_py/NN_verifiers/saved_model/variables/variables.data-00000-of-00001 deleted file mode 100755 index 496d9ce9b03efa36942169de7236b0106b4066e8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6806 zcmcJT4OCOt9l%NC?# zC&+)C%psrnWck5&70F}-d}{Y3k*BgM^oc!^uDr@GNz>^Fw)nJ>ymI>xxAD7na!_l698d4*-8c;0z z{*_J6cD&A!r`C~~Kl?X%@8&Y?g6TK8W?823)cw+fpB?%ux&7_W4KM!jfuS22z^9?1 z!MK!bGorQ)tkR;l@Oc?aVvnAvb9aAxS*Q8vcWio+jZN5cUaRDCS){jT#~Gbu>$U?@ z*;w}Qr#5{%eME~VE_1*1+&Sm~~{?PM>n{!({g?ryECL&0uQ{*RY0jC;?W)a`aR47;fP=?y)5<6I}ZYv$kg z-?}F3o*<=?DDSJ(5iKtK)vGyKt<8*NxThb*Hai=T3IC z>(JO6ZdAJG+#SJglqKlVxAadvZmcrTy{W95_ZKrHo$YDnbh%tK@9)?zNAr3}*ZX|& zAc9S+o6GxSCZ)1^wMl?&wiRuht#6z%y@F8FFIc>WFmok^EAszLyq|nht1@gRI^H^_S+eeVUjEYcXrg2JP6CbfAR$_7 zn4vD%T$WydzS+g@g0{9zniapA&&!#Zox~w*n`U;!72fvN$FC5T_Wgy+uC3*BF-Q*; z7EfKHF~z^l%irCzoVfWZrqP_G`0p$EH+Vhd+xv{_UQ0Y*^)2r|>-I<5$4W10T8@p; zUQ<=+q&us9>u|i{FWRx4D|MNsle(>6#~MDHbbQ2DAL56lxEzy!;Z5{1#!f9YI~P++ z8FS-er(<$19tR~cyaay8F=~P2D+umNjYd2LpjWi zRvOwcaFjR@Yio9zE#_9njt|A+0q&DJvZa^=-zI@tGXHdj4wTNYfzwIwPbbkY9q`cU z;8+rfhVx=&ns5vo4k8ztX+faL{(+7dDA1IF1C{#+I?^vt#UKJr6$Cn}f1soN106F^ zplJgKI@UkX$NT~vH;6#T3j$3S1e$|K!P%DN!q;#hQNdR#kf`Bn6p&1Tukk>V2Vdzx zlFy?uSbb@B1io8}r(%zT6e~nO*cQRD3=qxlQU}~)nIOi(IPKT%cTNt7G&|vz$1x>{UdT9UN(I}wy=^rfi>W}=BBq{Rh9_cbIFSr( zx0~|fOJ^ovta}BVUf>)4M>PZ{xobrE0??u zHlUSDJ{LBil}kPkHlUSD{tRqDE0=seY(OiQoZ?eX!IQ8lV3>okyHE_hry0x#K>h@X z^2)*bP~IC6YI&hh%NL+lAVjTD3^h$4YL9$o#$(|gk{K9yXqtvWL!Ma5+$RBch&tP&8vh_N~trRN}46nM6o{{`DCROS@b)R;<(XVsKcEB(Fj zt2GCh0=(lM@91b(a7a9iNIfOc5Lg2Fj*?Q3=qWEg$Rya`lb|R7 zWB&eCOz>6TN^~EK#WVwy1UTxwz7-SuTzM|^@SUjxoTS<6N;HqB%qV99)SDA>p4z@VoN1q1%NSYxm>19}%usn~&%0ZOV=A@foWi=B-)!SOE zP9CG_(6YkcveMtOs?XBYYYBg;RRf%Auu@Bw()LEr4;;WUtIi<)u23J zem%h{@_auzmAyr?GFQpREjF`3o$_f0cgl;zoT8MfJSmxkDG4VmjFePAL`o{Rn#c^c7sK1N?o8iVC>#%jxVY# ziproPdn&H*$6XFw0plk4R|?ADS@+cn;g9|-fdAd_uRejVP6?5>w~`2dxq+hyVZp diff --git a/src/smlp_py/NN_verifiers/saved_model/variables/variables.index b/src/smlp_py/NN_verifiers/saved_model/variables/variables.index deleted file mode 100755 index 8661b62f8721b1affbdab880b74838f01a4e28ec..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1779 zcmZQzVB=tvV&Y(Akl~Ma_HcFf4)FK%3vqPvagFzP@^W z$Fy=w!z%_8zUX9Pp8lj*I@lH#~@Fz zVPTFwp{^W^KvTXjX|R|ss}{5pQQ#}dFGpnU>bM8|BJM#kbgjkr>|#} zYfyZ!kAH}MenClQZe~?#k$x^wCb0lb@;O|1)kH?9j2I0!*nk?=zVwO_RTKfnPF^a3 zkmUe680=_`21e8RqrX1Mnu?R_WVl%h4a^|pj+)-hGm{Y~;BJJj8>}E*QeU__g%yPi zi41KPg#}C+0;Y4rcG^%F+CY<7fhK+5nWIizXtOB%UNrp$+!0!U9H+ zUn4)xe8LbUjwP`f>ch;VBsQRSv4HGS^xh;(Tx>ve{a^#>T6R!q4KOy0h>Q&;g$qm? z9Hue59G+4b8$gp-fhKMC*_lFIY%nRT-~ejaJ#FQBQAIIW=;14dU@4DDVFME|*F0|W zmX$RXqa?gRISFVQE6}vAOrd{fGGas~ekO$u4xlE^y(M!QwD}N)L{e%=V!R>#jL)F( z090`OeSQ2szYL!dA;nKXig&;Lb(2ehFC{fOv67I67qDWHOP(*Y1Xyz6Dcd=LW%~_K z3i!KfLIsxwUruUbQC?97#Nug80PZA#jf$OL@wgxSkKE362cF{e>Zfil)B#r0CX49@&Et; diff --git a/src/smlp_py/NN_verifiers/variables/variables.data-00000-of-00001 b/src/smlp_py/NN_verifiers/variables/variables.data-00000-of-00001 deleted file mode 100755 index 496d9ce9b03efa36942169de7236b0106b4066e8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6806 zcmcJT4OCOt9l%NC?# zC&+)C%psrnWck5&70F}-d}{Y3k*BgM^oc!^uDr@GNz>^Fw)nJ>ymI>xxAD7na!_l698d4*-8c;0z z{*_J6cD&A!r`C~~Kl?X%@8&Y?g6TK8W?823)cw+fpB?%ux&7_W4KM!jfuS22z^9?1 z!MK!bGorQ)tkR;l@Oc?aVvnAvb9aAxS*Q8vcWio+jZN5cUaRDCS){jT#~Gbu>$U?@ z*;w}Qr#5{%eME~VE_1*1+&Sm~~{?PM>n{!({g?ryECL&0uQ{*RY0jC;?W)a`aR47;fP=?y)5<6I}ZYv$kg z-?}F3o*<=?DDSJ(5iKtK)vGyKt<8*NxThb*Hai=T3IC z>(JO6ZdAJG+#SJglqKlVxAadvZmcrTy{W95_ZKrHo$YDnbh%tK@9)?zNAr3}*ZX|& zAc9S+o6GxSCZ)1^wMl?&wiRuht#6z%y@F8FFIc>WFmok^EAszLyq|nht1@gRI^H^_S+eeVUjEYcXrg2JP6CbfAR$_7 zn4vD%T$WydzS+g@g0{9zniapA&&!#Zox~w*n`U;!72fvN$FC5T_Wgy+uC3*BF-Q*; z7EfKHF~z^l%irCzoVfWZrqP_G`0p$EH+Vhd+xv{_UQ0Y*^)2r|>-I<5$4W10T8@p; zUQ<=+q&us9>u|i{FWRx4D|MNsle(>6#~MDHbbQ2DAL56lxEzy!;Z5{1#!f9YI~P++ z8FS-er(<$19tR~cyaay8F=~P2D+umNjYd2LpjWi zRvOwcaFjR@Yio9zE#_9njt|A+0q&DJvZa^=-zI@tGXHdj4wTNYfzwIwPbbkY9q`cU z;8+rfhVx=&ns5vo4k8ztX+faL{(+7dDA1IF1C{#+I?^vt#UKJr6$Cn}f1soN106F^ zplJgKI@UkX$NT~vH;6#T3j$3S1e$|K!P%DN!q;#hQNdR#kf`Bn6p&1Tukk>V2Vdzx zlFy?uSbb@B1io8}r(%zT6e~nO*cQRD3=qxlQU}~)nIOi(IPKT%cTNt7G&|vz$1x>{UdT9UN(I}wy=^rfi>W}=BBq{Rh9_cbIFSr( zx0~|fOJ^ovta}BVUf>)4M>PZ{xobrE0??u zHlUSDJ{LBil}kPkHlUSD{tRqDE0=seY(OiQoZ?eX!IQ8lV3>okyHE_hry0x#K>h@X z^2)*bP~IC6YI&hh%NL+lAVjTD3^h$4YL9$o#$(|gk{K9yXqtvWL!Ma5+$RBch&tP&8vh_N~trRN}46nM6o{{`DCROS@b)R;<(XVsKcEB(Fj zt2GCh0=(lM@91b(a7a9iNIfOc5Lg2Fj*?Q3=qWEg$Rya`lb|R7 zWB&eCOz>6TN^~EK#WVwy1UTxwz7-SuTzM|^@SUjxoTS<6N;HqB%qV99)SDA>p4z@VoN1q1%NSYxm>19}%usn~&%0ZOV=A@foWi=B-)!SOE zP9CG_(6YkcveMtOs?XBYYYBg;RRf%Auu@Bw()LEr4;;WUtIi<)u23J zem%h{@_auzmAyr?GFQpREjF`3o$_f0cgl;zoT8MfJSmxkDG4VmjFePAL`o{Rn#c^c7sK1N?o8iVC>#%jxVY# ziproPdn&H*$6XFw0plk4R|?ADS@+cn;g9|-fdAd_uRejVP6?5>w~`2dxq+hyVZp diff --git a/src/smlp_py/NN_verifiers/variables/variables.index b/src/smlp_py/NN_verifiers/variables/variables.index deleted file mode 100755 index 8661b62f8721b1affbdab880b74838f01a4e28ec..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1779 zcmZQzVB=tvV&Y(Akl~Ma_HcFf4)FK%3vqPvagFzP@^W z$Fy=w!z%_8zUX9Pp8lj*I@lH#~@Fz zVPTFwp{^W^KvTXjX|R|ss}{5pQQ#}dFGpnU>bM8|BJM#kbgjkr>|#} zYfyZ!kAH}MenClQZe~?#k$x^wCb0lb@;O|1)kH?9j2I0!*nk?=zVwO_RTKfnPF^a3 zkmUe680=_`21e8RqrX1Mnu?R_WVl%h4a^|pj+)-hGm{Y~;BJJj8>}E*QeU__g%yPi zi41KPg#}C+0;Y4rcG^%F+CY<7fhK+5nWIizXtOB%UNrp$+!0!U9H+ zUn4)xe8LbUjwP`f>ch;VBsQRSv4HGS^xh;(Tx>ve{a^#>T6R!q4KOy0h>Q&;g$qm? z9Hue59G+4b8$gp-fhKMC*_lFIY%nRT-~ejaJ#FQBQAIIW=;14dU@4DDVFME|*F0|W zmX$RXqa?gRISFVQE6}vAOrd{fGGas~ekO$u4xlE^y(M!QwD}N)L{e%=V!R>#jL)F( z090`OeSQ2szYL!dA;nKXig&;Lb(2ehFC{fOv67I67qDWHOP(*Y1Xyz6Dcd=LW%~_K z3i!KfLIsxwUruUbQC?97#Nug80PZA#jf$OL@wgxSkKE362cF{e>Zfil)B#r0CX49@&Et; From a39bb9711327e277050b880ea4de4f46dfdd8e63 Mon Sep 17 00:00:00 2001 From: konstantopoulos-tetractys <37417801+ntinouldinho@users.noreply.github.com> Date: Thu, 1 Aug 2024 11:22:46 +0100 Subject: [PATCH 13/28] Update .gitignore --- .gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 9586bdb4..b9f3c71c 100644 --- a/.gitignore +++ b/.gitignore @@ -57,5 +57,4 @@ obj/table.c.o .idea src/logs.log *.pb -/src/variables -/variables +variables/ From 0cb8009beee35a63ccb1e309fc42d8349b03176d Mon Sep 17 00:00:00 2001 From: konstantopoulos-tetractys <37417801+ntinouldinho@users.noreply.github.com> Date: Thu, 1 Aug 2024 11:23:48 +0100 Subject: [PATCH 14/28] delete files --- src/smlp_py/NN_verifiers/fingerprint.pb | 1 - src/smlp_py/NN_verifiers/model.pb | Bin 5201 -> 0 bytes src/smlp_py/NN_verifiers/reconstructed_model.h5 | Bin 18024 -> 0 bytes src/smlp_py/NN_verifiers/saved_model.pb | Bin 93992 -> 0 bytes src/smlp_py/NN_verifiers/smlp_toy.onnx | Bin 1777 -> 0 bytes src/smlp_py/NN_verifiers/test.onnx | Bin 1777 -> 0 bytes 6 files changed, 1 deletion(-) delete mode 100755 src/smlp_py/NN_verifiers/fingerprint.pb delete mode 100755 src/smlp_py/NN_verifiers/model.pb delete mode 100755 src/smlp_py/NN_verifiers/reconstructed_model.h5 delete mode 100755 src/smlp_py/NN_verifiers/saved_model.pb delete mode 100755 src/smlp_py/NN_verifiers/smlp_toy.onnx delete mode 100755 src/smlp_py/NN_verifiers/test.onnx diff --git a/src/smlp_py/NN_verifiers/fingerprint.pb b/src/smlp_py/NN_verifiers/fingerprint.pb deleted file mode 100755 index b6744820..00000000 --- a/src/smlp_py/NN_verifiers/fingerprint.pb +++ /dev/null @@ -1 +0,0 @@ -Д=ߩ\ߕn ̫㝚H(2 \ No newline at end of file diff --git a/src/smlp_py/NN_verifiers/model.pb b/src/smlp_py/NN_verifiers/model.pb deleted file mode 100755 index 948b921d188fed21e03d1dd396fd1676a8ebfde8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5201 zcmeHLeQXnD9Nyb?TYbySc4KOe%`6kPdC1yz8?d>)$2u6y2wR43hTzJ(_1LP%UFlsl z^CK+Lm_>~kiGP3yVW>d-NQ}WqvR+Xa4S(oJM(_icNMJF+ZzYS2_;DS*yLR^Ey>;;~ zoAmy;YoGV|J-^@cymtrJL1%yyB+efcLJ^sAcJR258xZ*b7iLOejx>Nn997gA98Ijfs(iRt zLdQ-GD-E|^RHpJ?M2GLmjcqP;v;B=s?2BKD*!;(qu+clKn&>CT(1p1QI`5iDH|%l0It!1lZ7)8W>y?kn67t=(0EmQ zQ{e=N)pdN2I?rT{`p&TE!mlTlJ%0?L-UQdL+8Q@4py+SgTJBD2V%<9 z<6GFOt4}wb-}bdqed#RfWbS9*8(NMkcNMTZ?>MfsFO`(yqxrGB4|!2na6mcm(VuAe zseaVky$=oVb)wr!r@XM$;O|Mxf0?FIiH}aGWG^omoA|-I#D2&dSLFR483eZ&IJFv1 zgZ5XquR3WUY~^L$(tZVvKZh!x{RvH99&FsU;j$9SuVlw(+M?f_{0gOtj125L4ZBps zUhJb1v~1;$qem`zpZVmPH}(JP;j-3<(C6ALW0~6?;X80x4$47M-~ug}=UubTaH)gg zXj84CUK*s6J;UTdaNn#6TF8KEl7KYR7$746A~Pluw=)uFfO*%fXt9<+=7y5L!I{?B6bicVL;yrR3V6H555jyvjveB#{6|gv835O0iee^Ps9G!CxOmfmm5ZaOuiwh zMoJ2Sd1#?)PP%KlNuE+ zXb`MS*GO`>{t zY>yx5H9clO`p5li`UcS~IYQdoblOJ@pgO{YM6(r)jA@uPg|o!i7JZD|frD&s3%hqXIabA)a`)Q8t zzW3ZUD^VkXkOv4=nvnJZLmLug8lWLGv`LLqpn~on8mZF=ut}S?3N*gThpMg(7@_hv#8R~hj_(y%N}ae z_z+e+SZ1)Cu9FoHTh3st75kf!zp_kr4C2Jk?(6SooRauciPLfn8gdKS1%o*dQVkwqa&pIC%nv<-saur~6ro*1pjM{Hc=15o7U zY7%@T9wW)bm=Oj3>-fz;rgCy6SMWjP2U|4HuUYW3P0dIciD(2_R^s;pEDrhA^J+PJ z;Dh_}>j_z~O7`IR$0VVG^lnjG2srIQ2$w%EHdY>sQ9WVlyeZ0vAIku^p8J;6itpE3bo2 z+&8c_#P+~8EOB8QXgz!;a3Z@%a?Wqyo&3tkqUiUoJNMo#BmjfJFIqpr7-;tg+kK3Y znU#S8V>ZV%X^zE{kH~o>zpx*?>!^1mu345qz}uyG_v?q!dcroexOa=2}C|Dco6oc(dXqt zYxed4Ot(^+X#wGw7PoZXVQSH+KDwZT#FP3XxO*ojw{XD=*VhMn=8&jhE{qA|sSU^B zj<%VG8{!J0DZ3fX(5hRII#8wqmS59nr6iYNRUtYq9}U%>ZNXYg*+uk1h$4f5h*!}Jh>`c;dyba?g$(A9~vD}j6jiM#Epcm32W<`73r+Vlr0gn za&U5zpWEdn;%(u7<4Ftbzjs{M5)tcAS_50Gn`Tlx%5mK`jS=Bti(KqIe8xRu{D7&Y zQbnGY{Mxwmc%@v}m~clWZaq195X_J~pmBlna-i)5vFZGlx2VaKZH$B6#&ZXle1`|V zV^nkcBodT{;#`l#;l};SFj`4*%92~-_E5Q)km}mInGJI9XRPJ%ZLl#?AN#YI(UyZZbeNk7h+8EX9t&v z_jrlNpj@0vaadu$KSxLND9*(Yk(GCSQjFMd3PRbKI@hYF!T|5uFFs}>GCZC5bmv3I2 zpLl%ukRb-BlwVaoA)hJfh5LkS3jt?-mEyJ#=G_SmiaW_(4A4N8rL;%&IeT;(17G5Q z5%aFCC>aDxCH^q7UkoZ>r&X-2DbJX;&~nS7%*N}CmA-N z#aZS#xlxi#Ate<@sB%q8saJJ5yLfWT;$jdkG4JAZwKm#4_cQhU%de{)moBJp4!)v( z?y2voJ#XF|{@Qc@$kxwnn!WU$s5;+xn|ikI{p?SkwAANc&15^*zLPz-;<)aRD3n;yL>JoBlx?)8(W)VEJO*Yi;5SaziO>)jXnR?XCJTNB=M>V@nN zv=6eznrGClzj$7K{CBIv-%JJ6jUQdnv+S#HoOtM!PW2DJY*%;BeSYTq>({98EbjZ^ zjp~`Uz1=NmJ+r~VKdSp)V<&dK7FOT>+sW+X|C~}^s(Us2bknXdyZBD$)NjqKX51Q{ z{np>r4foGx&mDar-1hz#yWbf4O}72r*VTQS?+yQG>Na)LvF7lhTVBfU-DqXkOgGKm z{%ntWpApMG^V5H+nJ+}u;R8>pnI~7OtsBnuoI3lbo}22Q?EzVS{O8T;0qYxC^~_h) zO<(?tdf~l^uAzbVvZDoZh=Q1^PEgTwY@U diff --git a/src/smlp_py/NN_verifiers/saved_model.pb b/src/smlp_py/NN_verifiers/saved_model.pb deleted file mode 100755 index 88b09eca418034586178669a018beb74b183ab73..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 93992 zcmeHwYj7ONc^GD~zz!N9NH$-~;TwD$xjGWNgT-qtpN@cdcf3131o9~9By+L^mb^Xg z1KM4HN7;@aij|Zqu8M3)v6YGwGw~yerjV!%19rDlRAG zDkrIw^YzU1Oi%aMGqVd2N6I{vxPzJQ{@!1I{q@T<=#$?aA%A<4{?RUSm+ZOR?baW( z?o>PV>b*wo#yXuKW~bI|Z**2``TcZ~jBHjL8?~vCwcg|P+SD%MJH+t(E zJ@{j4ioi#=<;o#)z}{zMhO;B1=GgeeuF2hd_R@U^4^3Sp;{dMe+;4Z9)gB$~_B!>} z18bCKufOrh<;ob5%)ZR8Bu`|*00 zab>x^{`f{~rB+>wa)*(4iI`W{YOP+q_n47bZ+WCd=^6{@A{%7%TD5m=qd`ZBQ3aye z?>)we36fjywAVIPT~-_-r~{!J>kQV07zvInY1S+8T-bvdk-Em zDfyy7=(SqsLG6yc{9bMK;d;B?>UHTMvTrr`(^>D-?$%4S0zSC_Imrq zI=M!&H>;};LBU2{1!KxkFs?`WHbIQ*bQmx_ZxoHGkc(5hNw)eyz3YCnGWj90OvZ0E zs;jm40Iga_6}F_e`^ku>uHKK8tz_~Bd6i5uA?u96iZ=$P4Od8ZrPkP>yNG$eR_$$o zB{9+NA>-B6UVYPLCVMk=G!)ievfJL}Y5LgYUQqFI(k5do!2Nay#06S4>InwKvWHA~ z%IdVLO;=F+$sU&yr`lR`x{dlOdu#0_Ij>yT9(4`3JC!H?6D5s$w|Ab|?*+zJhIfOE z-l}d62#xj&?PCB9lxVdFd>UJAx7O-R$?}Am0#knWTVs5SIEPUD)_c{?T5at$ z&@jddW|;S?UH2LIH0H{;vz8wQ1tYDECi706wQhbG%*a2d`~38sK48-(yU0-4W;m=#b}S^JEI7wO;M@YMmBui@Z(9E!T~{32p&|GV0a? zMPe2i*3C_k&>+DPjdg45)lL;q<1JZ04*8lv#=(Z(0F7il3gdX#qp)f~Y_D2x)gslV z4wG>vq}rOR+hU87XPyE(gp$Au?`P^X+3orrsJd3$WF|l-TpjoNh6`4PFyAvZ(yeti zYlTt!QmK5YXdl`19xirPOBe0Z#RuiJ>gxTO`!jP3wFQIz?ElP?9+?E^4j$N^W*ye_ zCKV|w93+#D$5*fS17Alnm7&5o?7fn!94`6s5J&6a~~&@R7@b#BlT32R7w>{&>^0Y zp{F8OX{m5rA$Wp_$V7+^BwN}x$!4{Z4SP_p5>ROI@S4Qs_n)y7ED%7^g)hrx~s_`U^Kp=H;QiPa_$Vdw@5;da| z{lIrf+bBFr_rA>?@V?E^Mmf|z2<81m zGe)ABs6Sq?Q7sZsA6 zC)q^rI|!MQWK()yL9&4#?Mt#@tr*EBdf!0^mgJh?eVdZ^O-H2Yed8pW;C%-nPm*UU z@7q+oZ#pug-Z#lcC1UTAk=6D_t2cnt1uyP}yet?#80|cB)N7V}k{ov1Pi2H&om2rg zWVJ?m?OqiIaKb1gRaNGh;}Cb7Wr({E=@55|rh-oHk(|pBr?gen^HH9UQQk253_0S% zNQFMHmzn^V+^0c+-!LPUNHu1+S!NUNLn;D1RMH{-7s%+n8l-#etzr&&Q!woyy)1+#=@WZc>eyG3)e^AjI&JREy5`M&G zq&zr5_z^%(B{x5E=Gevrl3unbGWZK5=>_WpkSYa|S=}(+@gRhf^oIw1NYbkjBS~p! z5J-}T*#nR#Nz#kOb-6_jCzK?A(~l&1oft_XLxKh&VR~c$LM1txg(FdyGC_s*NsSy2 zp`{{Ds2w9gDLU@)EYo&cqJFI7+hLPun!LYLk>`(fyc#i*LlQ85ksM(ua~2J))$Ug} z8a?=B>2P;`fQnOYg_o4<>jAQ_vX_pVf5!haSY9}i{U{ve0XJ5Ze05kqJENPnvEop+ zBFwM^$@BC%S8>0j?;H46?#lRAFGWFfC7pUx$YwJM-qyg7N*vP@%wPsm`vPCZ`-|?S8FOYpuehJ}eG! z=P{i}o$C5JU_4)5AQRpsE=;krsoY<==+CiJZ>oK2*R(ecQvq0&Vw#RSSNPMH`E>mM zczNj$Up8}eXzBM}Ccl;;dDEc#8`Z}E`%%63p7W?yfAC(fd$CBy;Y-f_0)OPq3|g!N z#WK8D35pG)#cU`x%!}D%7ffJ`ZThPsy5x{KLifRZJj{o~dH^TPVIZ__A!8ckCk-gID@x9!j-1_k za`r^Y*&88;rY47&`{=0WjfiRPhxG$6vlb$HAd2Wg9ioTyh~}e+9*!Vt^?~RS0nwu} zqQ|0$9@imyLXYUlD59q#h)(r^=xG7bGXkQQ$Ov%TD8kQvYLwvT2sLc@IZBNg_&H6D zGW?vO#;hxWe#>VHL*(Z&{>prvki6`HN;rs>c>jYfkjWr7Oh&d%jToNhyu+4x4^|RKlTy+M+q3p zzP`*~u&)>SAMC5kasw>ql0T9n=CnVSErI(l2IJZ>vqZg-HrZ?1Qs%+RF}-N!Z^Hv< zMl=5oJb-RA^LO9@w4<5-Bs_qAH1nT=2hfma{?qUPwKntb!UNRW%zp+Rpw?#ov+w}5 zHuK*B4^V3}e-|F0)@I&uNrsh1<_z5}g%dyz1^p9L)Gds?V|4#8Q?+uOs?7?jHYcju ze3GgysH(PQpE+R;>LkyZG_^ME1{GQ4DpCYCu@@Eb#b@aqGI{6n%GJwDZ(nilTz>nV zD|`ymbHvu1XJ*k3mQM*?x_tZg%GITJZeO`I9c{&x)~D!-(z+N~mC?4{^%My$Gd%$H zV(tr+s}$&l{4UUqZ;R87j|#f+?V@hHlB63SQ*~p@-PD9RXuovMwEEJGS4CA|x^Y>2 zahHBdkzO$66{8(IP@YbZ;k)#<(!QVn@J7uRrXT(gJfUr-AK+}5et@e9^+SFa=*Lo= zek=?6@o`Z2FSp=3^oGLUoIlfLm4iX&v>!H47pL8i zNg9AQ>rWTr&3ZGq2_j=|`V$^mD9?vo8rF2B0jk2 zEMtkrL;g}u9ym3v6*Ttoy4C4RH*Vg(dhP1FS5}-`Z{N5*9a=;)Nm841U4iZAw%ov~ zX~A-MHD%(hiQ59)@1mOuTzS17PIVYB3{Le26fztr$NfCrG)^~rwXeoF z-HAG$2swuCK#6g>gU?TgZeNXYx)XIgk#6WpSyHv*ZdYt4PCn7QaMv1=}X)iTicak|AvJ`hlg$5P}kcPI+?{Ft6h7=Q%}A*vE3-6b|}t zE;_N3OPbixY0t^nc^oz+)aU%oSvG%ja^*NVD$d`qjO;4xi*aWS78gt8t1-H1ztGJv zj*vB{$#u)u9R4@g{->auFe~UPa=PBeWgz4zKK@XGMM(bg#{iv1S zUzvbmzq=ZN?Qejcu4DcurwAWS_~=e}!=z~Np(NDuY6`V%p;lI`H6zq=*RhDO%0jIS z+X+_(jM--4Jll8q#L#vIUmj%YTV#jDSz!hA6~o6;_-KcZGvQlB{J4W#d&rs ze^&#kuwPkw0UgNHu~6ISUcN~^zzTZ218lY#_DI5(bGFGF{^->{=uudu`tS-{ov;F{ zzw6Cf0j9&f1qOVVBlMEU%8trV_&?k3f8|EywmC#^l9b%ZlCxyLvjG!IPItYwTEAcC zJ4bg>(GDu7!PHNXW17Mvba-xI1`1`83e7^H;iN+IP$-*JXb}pHBotaKL7~y4LNnwy z3Hf8#0xYVx=Mzh!!;IJM`Gp0x%iQ1eYtSM&4o53AHyZ9r6TaA?y&fz#5j>vfCohr{ zEF`P0v9%18pRXifOlGg^xo;d$a*8f$7)uvsQ;_SfWsv#QB&tZPCS8)@FEG2S2vM+T#&c z@sFkeZ+09P)fQ;I>7Sb5o}FOr@!JnQRsYqE(ilhU8CobvZ zy1E`&{cHU-IDJKCj!y&ik4dt`SkAhnDJ_Ku1NTLy^%aApRTqU>eOqx%*Bo7LZv8!j zUMa#-dF_zO=qfuf(5i-1xq;fkPQp7?@xw#x*v){ht@y?axbIY2KGV+K-$ z`VS3a$!rQeZ)gY#F9S9SrXYwt;3e+E2R4Q*#MFw116%}iz);ea!5jgEQ3*0?a>LR9^ux&wo7Q&chE40K=Z4L1 ziQI7ZSz zpJol_tdIo)g-LJ4Nsf%z!?2S*y0B#XcCnv{UHt@Sgj*RS)9(tXt8B9W<10(pY-mJI z@jc$$%>19*nf`RsMxJXJWQM=5iJ&ybo;bNb?@pvAOzzJw&X6DcZu0*oQ!G@12_SZU zcyJ^>ohHYs?j;JsthX0rdCUBT$#O3s`Up~?!w4CsBlwcbr*v4Bx3ARy!ZIoae2v+3-&8jblWFmy6V1s&`YR5OiW3gG!jJK*Tj*CNm%cm+C-X4BadZjWxLo8by7jvn zm#{7CeB+V!umluZxMfIR4}-?H`{-+7w1pdW^z|@ke3Om7R{CAgOa1I}Rtm@MVqWTF zoAS~C`;V6f+F6U&O4?3|gF)Tx~ z8>~%?m)s-=YZ1dTbbQ)2+IIgrF-TRiG4 zzj0;xt(!NlUcc=g19W5Q%`3~do!3_`-+aURX;@ehcD?Qy=)#R+TkU?|;%ECUExj7_ zy+ca_{m=UQ8S!?s@S-$J9G6=sLTbc7BAiM-9Tt$u4GPI88U7_h*=W%r>+J5ho%}x> z)yS`Mb9aUr?tf$6PxRlM^%-=seP2L)#lI)6Qk01eR$b>6q9H-Y2)2ICKrWC}h5}Z6 ze#BDurpTzAgS^apY*<=tZNN@2wo#?iZou$vz1D)U;c6Xj{K}j+@L6p!5;UyuAcSv) zfxWolE*^v&Zx<@FpVH-%*;;nr5w+f@&J56HKvU4Mb#LdCgDjB zL1W50&6(MF&dd(L8B}9t5zeR@6Q+YCjhWqPzRbn*WlqDFv4XxYf;EMz=bc0NqRT@I zdNTL)`7svhiRR<^F|Xl=5yKDYn^k_yBmB@O-35-!KYflw`{spsjx1~!M;20XgwHSt z99h_Djx5G=WN`qFz}it&M-~x|Xs1&IhAi#`LmV<5Mn*+}Bm32ewlqjc`_Bx5<){&! zs6CH7ePrzyaNTk4h|w*^odLsJggYgKJ8|=6B40{7&KEJh#rQHvfD2(i!WS&UMfeg- z5Q==+k>ECBO=5J5@nw+k_7_mTVDYWm3-K9Ikt;hA+D5n{#2xi2zj2SQ!LiZDlvbus1)672p8${Z}(O|_CiWYCU8z7YnAp)bauL1JIrxUk?C zF)n$@hRB{B34~Frq)|`V^5tJ02F>31P84NLY;O4?t2Ao2vdO z3vmSY*u|ZW|Llag*d8P<#@U0##Rz+N;+L-m64|rk;jx_%9@~S2$7tULkW}qa`z~pD zmB^eOkCE+!7}*{qMn-)PKvJ2b`W|F0naH3W50~wPaM>OtT*i$H3zre&;@M!nXij9$ zj>pY*LfmW*5;x=egT>8={_wb&C(E5pkwH5iIok=5vpq=UjOz~;IV1WbFNGA@vm>$d zr^SuSa*l6HQ-Icwj|kZ`Ta3#iQ|Wu}$}^hQKIN5Dvi2h{i|AXttc{kPLEo0qx3lQm zIrQy3`gQ?*yND5tqHCiB+bF>{-U3RnjS_651lw~Mx-;0zGWK#7dx^F=kCj}&UM~7u z_hkk4r_^)0OCqgLiYgiz83nmjI~<}}Irpedr^HX((>MGFSu)Cw<%CrL?lqRy3pIbC z=FV2t9AAPmbvm+Uo`&-Rb~t|S<3QH8^xp}0V}(J zmtDllE}{~)5eehmgUkEUteTPR*R0}F3qxu^#BQSUO@$~ibHACBwSl!60T zz3rG#4sN?mEN55F!rj$yTDYdYoPR-W9z>PcE=ZU+XG4NNg+wT&!`EK+?zs@-o-}EN)hgc zeVG(A)Kek90D1R_TdCs9t;LxGe9NGl@ z^J(zsb~?QY_#94w59bhrV5NZ&aEL)Tk_Lj$p-u98l>A9%8}2PAm(??i;m!+hOYK~7 zUfJUccV6(1i;>N&$N165C1=hSZ@uvLtR5CW&k8lOG^lAib2AGP#0*^9;UDq}pUX*S zVH|Ujo7P-~@WeRbAzw-?Ve3j*-^`GBDEfPHBB#{`e4Mv}$HkpFV#6#vl_NG@XGd%- zt-QLWBcOk5KRVb|SMiV8osmKI>Bu1Nn3Fy(eVBvv(+>nv z&>DkY3g;6HO_76Z{9F#Msqc&&GCMDaK98Gugl^LD8LSme$l!r5nE9e1W|ytwcVjKf zI_Z6fZcyWXgA!V#H3^ar^{$#%lH7fKZ(uJa`?4fw7q+409QE7K7I;eMCng@T13?7b zRg-YU&Rmh*dXsd2j?Hf3NxDDBW;gL9-JdhF0ELq7&nXwl4-oQa20103?A5J5XjOX~ za5LGXPIY}9E*P3GFQ`G3_Gqu?@3OJ~KFBd*kx&OYODw!g?9a(%V=uAbXlexp*`hY7 zcZmGCJzZ>&Q-KFLW$Z(`M{Itk3_d5p56;ibL7@acIPZF|1V1=mE++fI`Ldnt2j|OW z^1sG3_lWfu%0v!7?B^b_;7~D^3QKp77wrR$7ZLx&Za!o zRUi5x=UQt|b^Z5)gk$_BLL?H~nPK*`o^smZ+gr_N<@VNZ!82WF+2yE7f!6E{`Q3kQ zkpFuy{zh~*uoGqv^Eo<8PP0o)owa(mdaqGiZLdFWx4bv*Ih5Yyo|MHlY00k!%egfb zS!g5V#Z9_MUUJ)o6C^cN`01JgG%wpeL(aNY;S>mMRS8};fCr0WHI?`}PywD7nQ?fq zU{qUGf|sE$kz+2Bu!vHr6gm3!0y*IpghNG@g3-ifid18u5f;=+c+Ept5%T@h7Ag1lJ@5ygO)jkumMujvRV|AsM~H( zv_j5%`06F-DYkOFstJdsp7ZKrg7i8}CVT1F*pHHeR|}J>pFHLVlZuDbdI~ei(4wda$L3GBetw7CC&RuqdtfrpNUHJ&skh^TBgQl(vQ9SU~4CrKA z05r9@^KcJvcGf?HmR_rM9@OsG%kR}zAFj9StzP$u0Vj%9*QP)2A-fE|{6>qtfkRx~ z59}ZDzUe%B0SRuT!U6x2T;;I$zKGRvp*y9bbpO&5!>@z_@++}x;-bj;mFyTuwyx4l zOr;rJrE*NAvaZr>Or=>}rMZ|&bGl0NF_q?Zl@?+uE$AvO##CBVDtStTipf(Vos{et zC4z6bKG-oz1m853Q0zR-v$g2OD3PsGB0ENj;F|`$7$vfGN@T|<5qy&&c~U}2@}#6v zMlsyTR2P;2n2sK($`L2bdSBAbHsi0o%m zh4s~y+Jo8$uWd9MaLwK7dlY>eIl(cyQ=p*RAW%3!a_iL|+#}GUaPRls?o4HR`clDq zmy9g4z5+eEQs3G*8CmN+Ua!$ik!>9q*{n7;YIN7qIINx5 zaqGo>LT>ds_11$op(~}rb3{(oZV%5mIM-<=pV3LsdYh2eM$>^)I6G@ypQF0MVjgw4 zKsgJ?mU56&AF3FJb)Ou##agPZy$08Gw5rY8wA(T&s^Xw zv5rgCJp97%`$BF6FQdZKqHS%k6V$PIhwfD(8Y|5(R|dRaISe#IhOEB@=P$T$qdr!- z2{jEw>(E`C1R0g%3MvzXRJ`~$V@mgz)zRGW5d>*|*{=-Fw-+Pfs#|u!7=FZP@;Rre zK|CU!D@J;Ly`c3&6Bg*p>ecI{j21`Mky$U)qI?ulGRKT+ffu&q@AoVcMo6uW+7c_8*AAdh{~BjG6~ z`gF-anajA~f1C?mj(-)wr-RDHj*ucFq&EiXBKW!>198u|Cvq9_M3Kk9jaa$3M>-U@ zejj)s?l=J4%8_9OCTh>x^m!Yef022M3p=oKdE}x9J1tx;ZiE~eCA|?Kljfq8iyMJG zk;~{OiaY^!#LC4ZheEN3TQ_C808eu_7<0VOW%_RRxXb}6~U*2%EgY5 zBC|(t4AMe^_l3>hCGl7zPv$b_i87A?3{jhKvS{RtpsSE=?$vvbA&sYv#{-GSf%u&7 z(~*%qChhpS6Pc;Fd4HArayez7NI4!Fmo9?3jEH9Up4erN!6;4Owi>3%*d|DJOgD^! zLzV?NVvH7U!7e2RY5L~7zC@A43ntN|NiLa{xJ>^#m#Li7QDm9~k4qInUq&T`RN2^D z;IR9F^F4V7aQ%a~~B z7yYkfscJMOqq{va6~ib&rYYhm4NcZ}5&h&FCR8tmO`heKd5TLHLMYAro+93~fGx|< zh#(U|GEy3*LBVwd0hEcwBWt_K1x_BIAC&p3w4n2@Ak{ZJ7*k?7i zV8UNbBeOF}x^k9CC*2s3fpiffWOVwMt{6`F(v@-E9_ixol)Z&`s>f{?$pB%}J<)Mf zwb;TIcVU&xOkB1<8d$EJ4pL;B1Tc^+LV%1+|B@9$DqpfP#@i!VJf^a@5K{$`MUp)+ zlEso&uHDMaOp>ge_0dUI3owu@LV%1+|B@90D_^oQ#@i!VJhBoF7n5m8>~FuD+bojL z!6ch{aiNR}F5+(l#w#az6cML}9LO3WNyaKZaVJeVwS^dKMod%rGM7n6lz9Slt0n@( zb5%1ba#cxSzLV4EQsv8+^dv9`=@ zTnm0O&;mKVq-a5E$bk+ZB*|FmO+=cAjMzUC$;s#^%0C8j)c)~k4z?F3%03Q6VE;&_ z36r~TgJ~Aq%RI&ymW2ZsVA%03Q6Aa^816(e{3IxoKMMdmOr_aN?97Ov~$o(wsVJ3^9-RlHe9 zv7c!BWi|FwNJx}@0`!ROm+|-y&U;9de-aoT_rpE_Wzi_6177@{xP3|HHLd|c+^_6H zP&6PV+(7;aIWkUq(~x8zv3oV{Q;<)TeGKHN-QzJ|IYrDD2ZHzeu#F>?%)P!h)#fvq z$GF^seqY&0qR71u*n#X3vShsUW|0v0m({pWAtF)!3DBdqkH>!H6tQ0tn85!d2{Fv} z{oWT#utnnjO_|%c7KHu2oG4SYAT{Jb2N05Etn?-l6ZgaEV8Ov7mY4HUHZ~>7KL&Et z{_&WvoFe9n0}# z<(~jOV*j&v>^GYt_Dcd2*guja!}P$n!sHFTGPjfLUrwAUT96uYpaTd=GFEyMiHZAX zk=Rd0KT-ZMkR$d#i^qMlDdN635PvdB_T7R#)_3gb9-MN~?l^4g6yJ`Oo=r!^3F=(o z$0qp}t*=ZjeR&f0xiMQhx-{n3xgIM{vA^fgXSI9-!HV_An-n+`Nx6B)Vk1K z@70_2&(}KB>`09`#6A)h^$ykOr?4GYxDdAN9;^{p)(*=?PS%L4Q-;0-&X=gJ*Qc8` z*ps;0oi0whbHRPra_1TQtW_EaR%tpB)(Wc5UdK6o>Bi05SFc@t_sWWM>+Ku2r$fFq z;I-ix`%nb3j?H$H$3>F%#( zN&A>9>=-Q;HB^e$Q6!cK5>wVm4aH%JaF zcX0fSuyu2f`nU0Rg7gk{AE!3TSDIvLE)L#*)a1e zoh++w_7}$^_byNcn(FO&=_A-TSwlAKx$Dj`Xtppu->2`V93g`kq0 z1qdo3%TR(!@{*CD60&X~s3fm52r3~H-a#dKa+)g@E z(kT%(D-~4Iq8Fn?O0GUYFGh)!1aD9&Mv0V@tAa`~N~D}z#ifLi? zVibNAj2yp;ViJB8j2OR)Vi0~6%pUwIiaGdIFl+FuD8}Gd!EC{=BAbHdo|@BSyw0}+ z&&~=L`Ob!MOdl8dUXw5K{V1vQcYUu|+J;Md)jkjF&F!N0oFL8lK{j0Li*!2_duf;b zt#(KsLA3|h*q8kCMr6aM_=z+29AB!4^j}oJFO1W!_H9(Nn=6Eb5u`uKF@&U$8;^3313A67ng8BztQPNq!%ABE)8VPlS94JdtcPz9*7j2lQY;kWUZxDWFFQ zkbHWS_aP*`C&GfN5R&Y1I3)Rf2nk^jrwREKLXx?}A<6GUNQk@PkdRLyB-!I|Nb>s- z65?(+B;->FN%lA#lKeh|gt!|H3HcO4l06QGB)<}$U^X;$gcw^2>*R3$d>?$ zEC3&h{5pVw^a4H<KTrjhjVG5Y275*m0Dwiy8l|rWr2Ji%_zL295;P{?5pghSisjl|wn{XaJ98C|iQ>0k~Q~eO4$-$r zO6g_ENsY@hsei*><-BRoGh|9r4lW_+)Vl4B&T0)xX3~}1mg`p+iqTsNUtGV+&w`iM z?(Uc;eR2KjW7VS$>h&@&ydL7-o@=l7%^dk5gPc{bUoAP_BG%Pvqv5a%Sm&dz?J(ij zoWlKf(VNWbH@ z4LS7|oCVRPqa6|5$k z%e4o&@V>7Q$4n98oCd2oL5^ucoDn)aKcD6b*7+IodzAc1hAfM!)2)4OgPo|&bffyX z)^S`bPtcyZ8Fo>tH=uU)Id6d4j(IQU%JYIBVCL>##LPb~3Vy(($9OX(XKt2V78>;* zhsDojp=LI$X+upJVul7a*=G?Zc2WH?7rAM6r&I0zM&l;jE9DwYZEYER-V@`5hkPlq zgsm%KeKSMip-`;@6d7%WOb#CxU)(7@SDdrHFiesJrDyWFSQQW!Qu?aE)YbAO!*sX2 zGctJgyOkdZq@Xnhy%f$T7@8sn*Z8>{TvOi}Ib?Red$;>=&@=OJ*&mkp@ENQXO~~K@ z(;*q_i$-KQ?eV*@mSvsvzQcl8jr$EsXpz<|4?9LozC4Vo*M0S|FwB-0hMgiO)Lc_G zvij?lBzGU*8`w+9zAVYvg>7g#NBuUm1uC~8VRJKm!se_dO4&ulUJNdbPY7ush)BLw zeh&5^oQA6gV}m2`(@-dzRHzJvMv@9KoJNxhEt20P7 zrk%FSKyVh9JR`)kMvcsHc4V~Q1lrpyf%f{!m2Jp^l*>DvL(4oXIkZ6{;z4q0A6E1^ zHwTekCLz+BTYy4I(dT>-3MECK^DIP7iazIAh?*3A&d*Jam_IeKX96zr-qVGx;TIB)xT3d6P?X_A1?7H4^+UwLLkjK8>zOi0-g&c4ku}HmD z@3A6h>1o|3;?=d-4%uIT8@#)<&L-S&?yS}BS2r3x`eO!J<_SPI%}C4Sm{bS~x4Jbi zt6nMYF)U>bK$deYLVT9+IG_1V|! zfLPeyPWa(4sLX**?ZHL^%6{Hm?BGuVKodDS2FIItXSt9G$kUpRm;ZH0!5pLKb}&LML^E?$#5!Craqv2%$7Jp~T!rM*-DVtwBt4Kh3hGPE&`-?vMmm+9LJ` zqSznQVSh-EeLjl);RyCtAJ`ueus@m#`(sh;kL$2Mp~wDY6#G*V?5Fy`{H0s@+^Qco@U$1q1 zIc{fLCljIlRt5~}EIrg}w;ar(iFuBWx@y)1m+%tJ`G+u(6Xtn(#PdVA!S6LM&|O{u zUS}KJNYrn{`htF($~kaAQ?dh6osa849@JVjIETvF00-@~@4XMvr!TsHkpUhRXpYIZ z<`m(NkMKWA_FwH|XpwdhvN%uwi9vqW_=RCOJ$d}KjYi{g=fN#lj7l$%Y_;>COHX}$ z_&71&_ZKh`^1kUx89F{P`hK=q?LMTtNDe%>TN3dj0+uYww^ASL6~R;vv|3AXZ$4u4~Kr~vOjFER@L z2pL8GH{;^VoNgB%x!z`lFp z!PxBLOeDLSH>Q5XAYTR*{6R?t5BMs$>+8cXgnECJX^K;fRkuk&-L^x^$Uirr>%2s> z?0h^xVw#S6$KSzo(SNqdm|3DD?%{Q0uW5tRs#QBJ78-V7hSth4&)&@6h6nKN&HOv? z0N%ZszXK28-<$bQ!UK5tX8u#~06xB%|1>;6t_bI^gxIn&w(Gycy^&0h`G{MS=2ZhWQ!Gsa<{Vih!}?_y zk3G}o98%<&^BWOnQoQd@%NdyBmH2 z#(qBzox@W|KWuksO8TM}toiw-_KTUNFKkb~Db5|+G=(V_FhwwDsw zQ<(%u#s21r5&4Zxc2{=!W&9+`p~?aOMKo1He%o03cLq5n4qwA0g`eei=Hbp9iHU#H zAYTI~0vm!NP6SR$4YP~=j9^bjl9Tl7!@J3@_eGff&-;0HwCyd)?#fOYq7SB^(Ot{#Y6YyC&L%W)R!_Ri0NNhS5!+w9+tQZ6TUBXu=1(F(DAtCFXKiL(QvCz zuMgY;38&?jOFt8CDsF*D|uOnZU<_~?`2MPff0o=)pOGV2Cg5!8oa z2CB9#*h5ZcOW3>XnFhiBtNo-+>)XWO1f5{UlO8WfaBc}NfpsbHf(xEgq=vrN2spL; cWcXDw{A75DEPi8nj+CRFS-S3RfS&FD12FMBwg3PC diff --git a/src/smlp_py/NN_verifiers/smlp_toy.onnx b/src/smlp_py/NN_verifiers/smlp_toy.onnx deleted file mode 100755 index 97dfc7642c6a1461eeeddcea356569db34e024dd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1777 zcmd;J7h*3-Gs@4)tB~R~)H5{GGgL4%O|~#Ju-eDVRm{bmlA2eX8lRb0P+G#JQJh*> znwnRVnV6#w7T5PpEb%SP(GN;ZObJUY%1lhkN%b$VG7yr)q0-7gN*srj5*x%Yu0}>K z+}gP`F|<2nCKfxUq+mKi3YSu#Dk3qh$lr9e|fF-!$XD6xP@ zu2x1aTpGEw&^5*z;tX49oN9qCzz8|GLGbYHWaPqU1eZ1z^Kpi;3~t3h*I)!9&^1t_ zAmJRrh1USgbOba28rB$L36+9|b~~0-1lNouyp2c*Z}Z4~WZ}BNAq&sdS|J=P983a?PMApy<_HXHycEKV=j^jP-)*Pf_1^B= zt4gZ`k5~H&c-8IC|MJ}X;J{5gEo-oum`Mfh3QR*eRxE#F+ji`?t=3jX4h{|$tZu@Y z?7bQo!!`=ro!9WU5xmT}-z4CTUCWr(Q2$U#PrwpTA1+ zKG7At`>hw++J$9i?3;J!pIt|Dx?NKEEW3`G+;$SOPi?iF!H&lCG(1sbdRpMf<+rxd zEDLQJfMDA@Ejz8vjKB!t0D7M2(iG@*n+u2SKJ2(=Xa4J(-Q9qlc5$=T*xBBdv+rB< zWgp94_5Ht=rQ6-$(YCwn@^jz8nZ9H6o0($DfV4}HfA7l_ooAVx}Ibg44^*5nuuUxG;8` diff --git a/src/smlp_py/NN_verifiers/test.onnx b/src/smlp_py/NN_verifiers/test.onnx deleted file mode 100755 index 97dfc7642c6a1461eeeddcea356569db34e024dd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1777 zcmd;J7h*3-Gs@4)tB~R~)H5{GGgL4%O|~#Ju-eDVRm{bmlA2eX8lRb0P+G#JQJh*> znwnRVnV6#w7T5PpEb%SP(GN;ZObJUY%1lhkN%b$VG7yr)q0-7gN*srj5*x%Yu0}>K z+}gP`F|<2nCKfxUq+mKi3YSu#Dk3qh$lr9e|fF-!$XD6xP@ zu2x1aTpGEw&^5*z;tX49oN9qCzz8|GLGbYHWaPqU1eZ1z^Kpi;3~t3h*I)!9&^1t_ zAmJRrh1USgbOba28rB$L36+9|b~~0-1lNouyp2c*Z}Z4~WZ}BNAq&sdS|J=P983a?PMApy<_HXHycEKV=j^jP-)*Pf_1^B= zt4gZ`k5~H&c-8IC|MJ}X;J{5gEo-oum`Mfh3QR*eRxE#F+ji`?t=3jX4h{|$tZu@Y z?7bQo!!`=ro!9WU5xmT}-z4CTUCWr(Q2$U#PrwpTA1+ zKG7At`>hw++J$9i?3;J!pIt|Dx?NKEEW3`G+;$SOPi?iF!H&lCG(1sbdRpMf<+rxd zEDLQJfMDA@Ejz8vjKB!t0D7M2(iG@*n+u2SKJ2(=Xa4J(-Q9qlc5$=T*xBBdv+rB< zWgp94_5Ht=rQ6-$(YCwn@^jz8nZ9H6o0($DfV4}HfA7l_ooAVx}Ibg44^*5nuuUxG;8` From d5fcf765980d78dc68f79e1fb04ee72c186c8f31 Mon Sep 17 00:00:00 2001 From: konstantopoulos-tetractys <37417801+ntinouldinho@users.noreply.github.com> Date: Thu, 1 Aug 2024 11:24:33 +0100 Subject: [PATCH 15/28] Update .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index b9f3c71c..3a0e3377 100644 --- a/.gitignore +++ b/.gitignore @@ -57,4 +57,6 @@ obj/table.c.o .idea src/logs.log *.pb +*.h5 +*.onnx variables/ From 6e8333dc21628a44a50162e7314a35cfa563e164 Mon Sep 17 00:00:00 2001 From: konstantopoulos-tetractys <37417801+ntinouldinho@users.noreply.github.com> Date: Thu, 1 Aug 2024 11:25:42 +0100 Subject: [PATCH 16/28] delete files --- .gitignore | 1 + .../NN_verifiers/saved_model/fingerprint.pb | 1 - .../NN_verifiers/saved_model/keras_metadata.pb | 7 ------- .../NN_verifiers/saved_model/saved_model.pb | Bin 93990 -> 0 bytes 4 files changed, 1 insertion(+), 8 deletions(-) delete mode 100755 src/smlp_py/NN_verifiers/saved_model/fingerprint.pb delete mode 100755 src/smlp_py/NN_verifiers/saved_model/keras_metadata.pb delete mode 100755 src/smlp_py/NN_verifiers/saved_model/saved_model.pb diff --git a/.gitignore b/.gitignore index 3a0e3377..fb0e8386 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,4 @@ src/logs.log *.h5 *.onnx variables/ +saved_model/ diff --git a/src/smlp_py/NN_verifiers/saved_model/fingerprint.pb b/src/smlp_py/NN_verifiers/saved_model/fingerprint.pb deleted file mode 100755 index a80bb51c..00000000 --- a/src/smlp_py/NN_verifiers/saved_model/fingerprint.pb +++ /dev/null @@ -1 +0,0 @@ -ޒвeߩ\ߕn ̫㝚H(2 \ No newline at end of file diff --git a/src/smlp_py/NN_verifiers/saved_model/keras_metadata.pb b/src/smlp_py/NN_verifiers/saved_model/keras_metadata.pb deleted file mode 100755 index 01426138..00000000 --- a/src/smlp_py/NN_verifiers/saved_model/keras_metadata.pb +++ /dev/null @@ -1,7 +0,0 @@ - -'root"_tf_keras_sequential*'{"name": "sequential", "trainable": true, "expects_training_arg": true, "dtype": "float32", "batch_input_shape": null, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": false, "class_name": "Sequential", "config": {"name": "sequential", "layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 4]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "dense_input"}}, {"class_name": "Dense", "config": {"name": "dense", "trainable": true, "dtype": "float32", "batch_input_shape": {"class_name": "__tuple__", "items": [null, 4]}, "units": 8, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}, {"class_name": "Dense", "config": {"name": "dense_1", "trainable": true, "dtype": "float32", "batch_input_shape": {"class_name": "__tuple__", "items": [null, null]}, "units": 4, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}, {"class_name": "Dense", "config": {"name": "dense_2", "trainable": true, "dtype": "float32", "units": 2, "activation": "linear", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}}, "bias_initializer": {"class_name": "Zeros", "config": {}}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}}]}, "shared_object_id": 10, "input_spec": [{"class_name": "InputSpec", "config": {"dtype": null, "shape": {"class_name": "__tuple__", "items": [null, 4]}, "ndim": 2, "max_ndim": null, "min_ndim": null, "axes": {}}}], "build_input_shape": {"class_name": "TensorShape", "items": [null, 4]}, "is_graph_network": true, "full_save_spec": {"class_name": "__tuple__", "items": [[{"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, 4]}, "float32", "dense_input"]}], {}]}, "save_spec": {"class_name": "TypeSpec", "type_spec": "tf.TensorSpec", "serialized": [{"class_name": "TensorShape", "items": [null, 4]}, "float32", "dense_input"]}, "keras_version": "2.14.0", "backend": "tensorflow", "model_config": {"class_name": "Sequential", "config": {"name": "sequential", "layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": {"class_name": "__tuple__", "items": [null, 4]}, "dtype": "float32", "sparse": false, "ragged": false, "name": "dense_input"}, "shared_object_id": 0}, {"class_name": "Dense", "config": {"name": "dense", "trainable": true, "dtype": "float32", "batch_input_shape": {"class_name": "__tuple__", "items": [null, 4]}, "units": 8, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 1}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 2}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "shared_object_id": 3}, {"class_name": "Dense", "config": {"name": "dense_1", "trainable": true, "dtype": "float32", "batch_input_shape": {"class_name": "__tuple__", "items": [null, null]}, "units": 4, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 4}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 5}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "shared_object_id": 6}, {"class_name": "Dense", "config": {"name": "dense_2", "trainable": true, "dtype": "float32", "units": 2, "activation": "linear", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 7}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 8}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "shared_object_id": 9}]}}, "training_config": {"loss": "mean_squared_error", "metrics": [[{"class_name": "MeanMetricWrapper", "config": {"name": "mse", "dtype": "float32", "fn": "mean_squared_error"}, "shared_object_id": 12}]], "weighted_metrics": null, "loss_weights": null, "optimizer_config": {"class_name": "Adam", "config": {"name": "Adam", "learning_rate": 0.0010000000474974513, "decay": 0.0, "beta_1": 0.8999999761581421, "beta_2": 0.9990000128746033, "epsilon": 1e-07, "amsgrad": false}}}}2 -root.layer_with_weights-0"_tf_keras_layer*{"name": "dense", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": {"class_name": "__tuple__", "items": [null, 4]}, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "Dense", "config": {"name": "dense", "trainable": true, "dtype": "float32", "batch_input_shape": {"class_name": "__tuple__", "items": [null, 4]}, "units": 8, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 1}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 2}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "shared_object_id": 3, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": null, "max_ndim": null, "min_ndim": 2, "axes": {"-1": 4}}, "shared_object_id": 13}, "build_input_shape": {"class_name": "TensorShape", "items": [null, 4]}}2 -root.layer_with_weights-1"_tf_keras_layer*{"name": "dense_1", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": {"class_name": "__tuple__", "items": [null, null]}, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "Dense", "config": {"name": "dense_1", "trainable": true, "dtype": "float32", "batch_input_shape": {"class_name": "__tuple__", "items": [null, null]}, "units": 4, "activation": "relu", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 4}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 5}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "shared_object_id": 6, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": null, "max_ndim": null, "min_ndim": 2, "axes": {"-1": 8}}, "shared_object_id": 14}, "build_input_shape": {"class_name": "TensorShape", "items": [null, 8]}}2 -root.layer_with_weights-2"_tf_keras_layer*{"name": "dense_2", "trainable": true, "expects_training_arg": false, "dtype": "float32", "batch_input_shape": null, "stateful": false, "must_restore_from_config": false, "preserve_input_structure_in_config": false, "autocast": true, "class_name": "Dense", "config": {"name": "dense_2", "trainable": true, "dtype": "float32", "units": 2, "activation": "linear", "use_bias": true, "kernel_initializer": {"class_name": "GlorotUniform", "config": {"seed": null}, "shared_object_id": 7}, "bias_initializer": {"class_name": "Zeros", "config": {}, "shared_object_id": 8}, "kernel_regularizer": null, "bias_regularizer": null, "activity_regularizer": null, "kernel_constraint": null, "bias_constraint": null}, "shared_object_id": 9, "input_spec": {"class_name": "InputSpec", "config": {"dtype": null, "shape": null, "ndim": null, "max_ndim": null, "min_ndim": 2, "axes": {"-1": 4}}, "shared_object_id": 15}, "build_input_shape": {"class_name": "TensorShape", "items": [null, 4]}}2 -Iroot.keras_api.metrics.0"_tf_keras_metric*{"class_name": "Mean", "name": "loss", "dtype": "float32", "config": {"name": "loss", "dtype": "float32"}, "shared_object_id": 16}2 -Jroot.keras_api.metrics.1"_tf_keras_metric*{"class_name": "MeanMetricWrapper", "name": "mse", "dtype": "float32", "config": {"name": "mse", "dtype": "float32", "fn": "mean_squared_error"}, "shared_object_id": 12}2 \ No newline at end of file diff --git a/src/smlp_py/NN_verifiers/saved_model/saved_model.pb b/src/smlp_py/NN_verifiers/saved_model/saved_model.pb deleted file mode 100755 index e6b78b2ac875aad4494a4a120111fa58d1627d43..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 93990 zcmeHwYj7M#dKhN000xZ*!R7qOuviIOPtKuUL~J9DxEmeh*a zUEJ36<=IePF2o6JNBjG*i7tmE*Hmk5|6xDHuV-c#AVgiLtK$x4y8C;7{q@%`)1W{3yCdW$r|BP0k-KE(YNu1b z-+Zguu2eClf-P-I<1}dMlE-kPLq*db!(?KH?rCNV7oRqMZ5=^b^o-;G<>da zR=YK)^KNy!MzimrUF$#q$S381j}RU0)|#DGTcVAeCkH=K z?=Y^cwYERF*<7zxH>2EPBwiur_03wdTkn3rNUS$KQlfN?2Xv7gGJ2!hy|J@JM~P7d zqS@~|j1!Y&Y`fjs+}UthF-s=8?P{~L-RjhwYRJ~P$cJ}?561%_l^HT^Pfj!Tjb{%Y zoi!=>qCx15TKj(OEqm?V+Qx(JR=wHn&{=Y5Bly$VZrASB->-FYWArGYo3&22-mG@( zt)~AaH#T>i9N4UF)j-^>txadM-tGh#7;~q|q3YIF>!H)AgH|=~JNN2awXlGz5~C#h zM(gG_xk0kGsv8eL!A4yLW6F>>-iY#Tk{EB$VZii~Q84C0F3uew+3NfCj{D8Zq1zI)g2?oS6 zLnb|Cb(+P?Wn@2BkS9dKB4hX{Ke<>lOA;(nCg0^;{* z_TEl&0}6}+z4z+(1HYA_Lvt3H^xlGgLWQCKw~wp*D1c2T7J~NQY>F7*vEU`!{zoy@v>dKe80R|-MCk}S6W=IEgSS3|07GfWEz}1cwl>; zby)K~DpHm|N~Rr;uU_v5zK&!nL-`5bOJ)j29{L#%+tEsvX3R|fbd=B;D#0KPG8ct9 z&)kyxkKmgmUu8+HN)<@Z zA)c3^ry^HrDSt{Kc#??7M2HR~Tii3rX0?*#Fha@lH>pVVCClr?NH!Z$y->2n`Mr~@ zE8VMxvRhL{;xQ2$_R<-$rPoHkVN&6}htaZ6rumB07+)@6!e$S?X64y{{ly4mB02 zQSTck*+lO<2$_;(Q+i)PvVkA%OR{0D7|AAj-$4kLD)f21)C9QXJ`Mu>h8Y3=V>$x-!s!TTlBum)wcTXND;U_;b}NTdQ8^r-(dNm0 zay9}l6-m)H(h&wWv02-wez04DRAY9VWj5hHq$0pWB^}~_fsDRWgLJRGTg)ME3Z@;T z7fkB6OFly^4?-&3{aXDYziWHeAYTlSFZO|aF&*-sBiTA+iTdXIc+`s?iSixt^sH6# zCdskiRA{?3(jYxf_~6eE#QR1bN!Mrwe?EjD{NSN~npBM!e)u)g4;A>}4=Q@Y`2om7 z!jHI&lm{mWKLW_9wB`P)Uwv{&>`-Oj4nJ zQX{8AXsL)3YR5=WijKQH%e0r4s2}V2cG%#VChzZ5lbJENPn zvEop+BFwM^$@BC%S8>0j?;H46?#lRAFGZB6-ek}O8Mnub!|s3iDSs|#gbdq;0lsI} z*=TLCS+9Dp<}|9!>it^Vsc&{D%ub}`UM7EuSWX*e1DsZK>jQZ5X6ikGbYQN~v_`$T z#QbGxR_P=?PR7Xa+O0QdU|OCGzYGsUcIMTgW#fB|p?sdrQe9X-N6tFX+Pzx4*4%(e zeOMge&SN?c+tuxDz_?s2lSyw97pBBfp*@In$trx2hii?1%O4yUxQ}{r(L~~I@k3|r*`atx! zfanPs(UVa`Pw5antw;1s6w$L0MCbZI^qhd`c>&QYWCXZv6yWDDHHz?aoEkR#oS;St ze$G*&3_s_ovEWLe-}0IK5c&BGIc{E{vrSkhf|&?0FVa!hN;_a<&(N_(4d$9RI^?u@ zi5~Y18@Hf?=Cd$hb}gUR*#oqia(Mf=Da_aErRHaTe8Qs%)LV|vlZ z-GK+tj7IKFcmUmKg61lWLRlrmgoT~oB(<#=%1jXZei>lqx*-Ms+HqZZ9!1A zMN!q3l2mP3RkdCF%t>=lCwbANskP|_NnOU@hES?Uyc^R$sdDqNoZ? zH?E2=?$S>y(hH`%Vzh$?%JT^_e4E};+V}Gx-l*Ba^ur&5C$!D<1Dp-h4{$Z1e#q|v z{aB6Dk2OI*J|XJIwIuy`N!5>CM+cMUpy9wpbFMG_cv(~jrXR0}FJkl~7!6_GgMZwG zet4rRR6qQ=`2_u7az6&jVR$-{@j{?s2!R4ua^3s%Dpwy?1^fTY^s1u9ZeG%QHi*9G z-Hc`ePOs5xN<&^!xoCYXY|2eZ7n7K$Z_HL2vy&RTPS+I7ya2+=hlqJ;{sbaFLvJfh zMW-z+J8UvK1(MJ-CY-%^w+w@%*Tj&1%6wf6&&SOh)SFi%rWTr&3ZGq2_j=|`jZ}6>mpeq(_FG{(;Es|9?vo8#dJiWjkeJi zINibLU2r{yZoWtjl{fDt2d6t($9se7cyG5l-a~0FBs)Vv+w?WXhZT z_y{?s*;E26W3&?&5*CkVuEO$yRvvNs%NZ;`&XZ!aAf({SfGoc^_LgJ$vm#+{7wv)6 zejylF`hAqLiZq;O@k>Nku=O}k+^;=LhS1~n151%11SJBU^5!gHUd7$dbC#U8PmXz0 zIOxA)(TSb0q=_A!_MC~G$6-@Kea_!pVDmR;)=!ZW;`|NE$Zo*C7m{}^-=W(8fPJVQ>|C*qXyN!7_qOefE+gHd5R36qy>i=(qt zUZTJJm$T&i3Hfs)#q=YE9EC6qoipe$GPcuv&}=F7*TJHK45ms5Km0>&J>VPrZB%EXWEgu=$PT;>~ z2AJ{|*3SOI;7@Ua*^+TmjL((`dmaX2)Jwv0B3R z?O-Ffbw<;BSG>)Bm4Y>LmrZ@=$JjPrw#hH$>W7T=yf}4k%DWFf7I~ghx zJ;?W+2UtNTc7#nJ!?s7*Mb7qZvz?Z;_q!Apo<6w7796a@g6?{wmWLT`Z_NPTxCl0i zEY_$Dh5xhd{x!35#~h-!NJ{Qx$pv!Q*?~DCr?Xw#sNbve&7f0MRPo9=u;SC?q^9r) z9WECNP$-jB$c940NrlQ#D4SGh5ekhY6FUBs6xm+yx_i%|dHCST!O!Fq6JO3Q7|!)T(Z>H44>Dyp7POu92crzX)qx zQX=8`-WQ3DG?E^TETl12H0<M=@{-puD60i zI35`nm6G165lX>l<$@5w=o-C*lug~N-AA+bJld=UJHt~FM&;ej{pxmK3*D^03daD* z%<*ZU{&^ji=z**&n$nVdD(=}9um-B%j>5FQVvw}zqA;s(D^9)c(fT_E$tkq^3P<^2 zFVb8=+ zXN;PwD{-xF8047D5%3qDYyw3;Cl>*w-rA`MCzycv1Qqkd7qqiiCXls$%pmFAo5IWi zdXX42h^||;{=PvhnN6YR4GlrzWxyuE6a=vcyu^L@z{ZfZda8#5Tm*B#P~EjbWG&Pn zG#!K@LxD4XM3dBy#~6g$uSb?2TR(>x1V=6~hzp3MQ<(&BX|GHogPVk|YgX8)nnYqm z6Jrw5J>9rT1gOQC1ZPvMNdztJ9W&eL;7_%K0CdnX3_kw*TbalJ@X}pbw=`jv^^95) z53=$~!O$^DytAIg%BF5M;7f{Txs0WvPvSUtJ$>MnHbS#+r4D>k2FUhX{q&9LyHWd2 zAMs1-NC(AMr#@80R#z0oR-+q zCv`dl8^9mM!?H)l5!`8Z3I0LpnG-1KFRH1BXH0r%Sw7ROlPH=4N0k|?O*AY~C`{sf z`aq)>9x56PIA=VY(Amj2t5m9vj?_ga)&BaSE)XcTp)m?yfX1r;Lx0 zSxw5=E@;(4qb~Lpou%LgRJjR><=4_Dmd|LsE*t~k`4jNDd+wdd?~RckGRTDluLo-Y z%ZpLhrk&8rh?7j>gcjzlw3AG~J;)>z_aVsqQhwT-s2n3B_AqRKjLt{y-R8&hv6JR- zw5*jeGW|}RUuSbgpIBereed-D?QBv%u3?Z_q&+wGy^eLnVqN%?6xE&syIM!IH^l*Ppat$s`|6u=Io9N&fG2@=76C{KL-g4vwU!^W z}U}Wf7GOH9{bl8R^AU6wpmxt&%a!e_*K1#p7{$bvE-Ll|2uIFY?TLE#JJxYA&;==eKc;)~g^3I8Nq+KCr)LM(a?3m=-I^nX4MRHy(zvP$aN*ICT zsbK-bZaJu@K*~}S5|f;J2D-2A5lEggVvy`YP)~uBp%~|EZiPPQdZ{>Pb8~c|usZRa zvqMWn;OIGLbCWDO3M8C!HrE83b55ox14%4ZgKVw~-t3Gc(v*R_EhhM4LdRatjk)Uy zm3v$s8T6`jwTn}HWE@go7SOkNSsN`|Lf@9rw+raoMfB|w`gR$8yMhsnqHCiB+bF>{ z-U3RnjS_651lvHcDj#T8eF?3qFQHNOCE6zNP%Q~8R9~)Ge>P2y2KSE4ix-V(PWI7X zF>pehXC2G`1iw0ker0;~%hPZ&4)Yr)R(dnF)TyJr)@KC zFB{5B?i2@W5yLWt#QVlFQR1;JVpyh-cx;;kNj$d26VBQz*VbOUb@TcgcidxmZmzz1 zZS9Wp^7_?VuUJ0|>&e2d*FA$=xY293-S2DsY`xXh7lXccXmz0fS${Vp-cA-?RA!0e za_e+RjTlITD`?Jz1!QuALh?z1f5}jGMYPE_ySr^a{|`qc@~hO`omqza-?;Y^{ddgz zESd>B6cAtW?`f+PWMYH0r#XdanCV~yTR(3g7dR?I0V_T~VyPQRWK=GaGtRqgSXyoF zz}^D3tEAmxdknT~O&A+))Zxah%q0V#ffOS_!}>Nt_`Vg`3ma~^K{!&B(MZ9Oe+c&+ zszV%_h9`XR$p&}Up%6)}Ha;e<*zsJk2jI#tV_dNjt|aHW1kTv|&6!d>XG#Nb=9e(e zln~CGRx&VQ8bQ*S(th)$9M6~X0DM6;ri}1K)fk>8;YklcW6Jx@nT2@HEDXRIRAUwp z&Zrs_rh_DnS=evBEXMO?QNx$u#viSJA7sti*FOP7c9a>_!3MIihS9Z;5K4SVswk~Wsvap z7g4@o@vYhm@flH(EBg}KMz|uzwis6i32t$&V9_nY6@Lm?=DCTj6J&eHZGp~$0&QS05@@RUu4ccgt~?mVU8H< zV$2yN*!@+MIasutY9)inpnZvaBMcHlUyMP6#J;$3VZkqAT=J3)kv;no21nQ<#=#hS z1_^|5{lOw(M1R!vDI$aRJuJ2p!eV=nuo%@JfTSiiRsB&G;t1@q3;P}a*$Hv6JxE-P zvj>Zd5%%!JFJBELvS;7JV>=-{wg(B1(Y^~HsoJCVUDEO@kvaPwBijivvOP$QjQSpc zq%ueKJ;+)zkwN<&F53y=vOP$+j2jmgE+fXpv%!4PoXDPikDKj;xY-^gZpQToi<=Ss z;c+uhmODL>LHiy#+X<1gJxJt?>kk$=Bl;sRg%sJdFR}Ay#7&uUj&D~}fYwiq2-!4S zjLRca=?Cx1Gn&>R<&{&k4kIrM=v%z3jg~E;Z_DW01@!GA`gRF@yNteF!3ajtwNZj? zlwccg0VUW*3ARy!?L`dT685r;yxzS%Lj2^)cNgjn=0`6^)FH zg50Vd4$-VJ_gqY;$oCBC8~(#A8D(c^!YTmw8cFM^nm<)@XSZsOFF~0*7g;k;!+8Na z96$GQdg^QPR`?284i8H0@5w%&W(|*h37IJ>Qg!yIXy>t1c!47M@d949h?Ui}Yhz_? zyxkI3wuF~0V`a;D*#)fZ0$z3zE4zr7UBb#P;boVxvdehc6|C$EDq$OuFwR}b1*j6X zaS6i^94U)S7)FUmSq%=Dgl$~HFj7I<#U%`3J5m;xFhsCO*+rDQC4{>rv}746Sw>4P zASD;jl8Z=59M2`Bh;T z46To*Y28~WUO5j7Z(+Hp2KQ0#GS|vQII?~&ZUe5j9TUpIZL^8x?8*hWI~tDm)<7Ne zFNm#NhFhech0|2<)3@RD@W{5z$}@1wG+Y#}DLxL@2e!JfPP7OGr{W9xTQdRA>3BTd z7NOXI_+oB*Ajz2&lH~i08X(DoDJ1D#EYzqJ;BMCENM1udrSh#L4Z!D63VeJ$oM#XY zr-8sV5o%m9kEDUX@2-IIM^lvN9O?mwW>a&h2l(gG;Lq)Jz6bamOMwsP5QAW)fe>(r zK{%cUg3qDeo}5@#UWy*;^Yp|rZ+mUVl+F&_w}U;No>=yQr5y0_^u)3!?huY_T;Kj- zoLCk)&WN2@R*t$h^u#jH+J(z>#1qS~V~D=VPAtpj_x;4Oh@7fTc5g>_r|sSZvsq_% z!a!WM)#`L=9XLE-dB0CA%RVXh-=sUS%sZdwv7cC$=ER-k6U$0zPAn^}kpD17;JS-* z>Y;?fL9EQcVG2|{Z51_ewu@hX#};zVnAX7m-AD3BLi6z$_{;1_!qxQ`$qTVkN-*yw zd%w-w{AihX);)gfv72|AzC78yi)qZexI%uPl0VImHBoWGBj(eeNmwct)GMao&OHAp zpe6bCJJ$>Fj|-7Qcuw-``ijuytO#fNc;Ue@;qwYRq=_HjlchmT8xDVFSL%u9C}rT< zKmSf1_IXjLi5-S|(nW4wbKlw{(_DTqtC+6+ABN&l#&v zx$J(DlSsz9(1j^Fg;s;ltFN^yT=@Z?90-ycXpYYciRTuLxBMGmgHZh#oHBGgvE{kii2UzL~MUXo#y~tW$SmEz3INeTVd&#{C8*v`A|v zc0be$F<(e>_wl`fy_)RHlAK-GhL&^GZ$n$)Fs|mCB21EH%J*O1ypeNxHN6%ftQO z49oqwo6CB7XR$Gqf1M5bUS7YpC$al}kNloqZ0sIl)S*YnA>>kwxT$k59CGadTu;Kq z#-$RwvpC^g`cjEGu>_x4DzQ6@6MSZ=#O^Fk@|k6J*>aN4EZgLF|CK@h&rxzhbec}L zUEP3do;%fE%~@PnVvbT+OpqI+3*;QTd)V2mcdGAf)izq&AGDg@8~0dRZ?0b&9GbM~ zSA*Rhnu=`FUl=r+bcH{Z#;{UQIuDu`g*|F`(ePC0OXQT> zD(u%%ibl_dF8YFi4gG4-$WjDgjCZCp94GBI4yQt^RpfJ`gTk1k{Ur%O%bY^k53Olg zEs9ChZ8s=dA(uRS^%C?HTRCmngtL<`dUY{DdhUeDUOGkkd6M^PVN&(SWc-|B@yu#Z z!52Bcn4o!W1!|TCP;sdzoGZ zr!t!Xooq*trWSV|?s?8UlNIGIJ7Zc~TY(yKmz^}NsmEPN1ghsm0`BVd2s?qA&X7sw zu6yj@gl4Ck8;hJty|Cb)NlkCm+V^X3*=z6CHXdxZ>dkKFkpTw`S2yQB;UT*QzWipB zy@4~G-4E;^@gDRXdjaD{q{0#ZlU(JP_r8GDaiKfKf^-YzBg3zR0`e=d%W|X0`IYP# zNVcw0DW+0MSE(FRsjREC5L0PES7|Y((xR@?QcR^KU8UuiO3S)RD>0Q;luDiwp5X zk~}G?6nRxpg1jm!Jzf=*9IuK>jaLOF#;c;E1+@pSifRrX6x15LDylJfRZv^-s>r5b zJtT)&2ClxbUb|m=|D~O+Ex1Z><6VlrjU4D2-8huj?H9@)A!FOsE?g_pq;R9@-A<`8 zKYt}}y-h~eSYLsjWUFs&f{bi-KiIC(Oo8nr8tGNHc4~BLbpjUC4y}${*{-&`aM{q9 z+cxYZ@_?kpHPZTV-a1{Kaa$)&x7OI6cR_rQK}K)aVDQnU{;zcsV`P|Gopwl%$z^mBv|cBqxzli9Jlo#v_#D+0 z7SDDa&Qs39@zpUHehyU(!@5U~+-5D+HeZ5Udz#foZQgAe6;)wSI78MC8{~ilh{`Xl zLm>kL{#d7^Y94;!_d_8!f|pU@Y0R~xhiM&iMU92qGCKO3KNLZS+!v3w zILSl}kfIuzDC`)xaLQ2#N<1hcJj+>sU_~J6H~`=7iy4X4Ffpf1Cds_TMg28{c%qi0 zK}FPQ;c{^!XRC!X&Rid}!sL?HGgARqXm zN5WG~^y!j;GM90|{}dOz9RDhUPY0EY9U(FK_TxzsmLSy>p_WdnZQoXr^d%uXzJur|-!P$iF>LZIzsyryx)4HX=JyowrUh(S zenteD2$GS~FeNJJ`+`f3929%X%ZJF5N_!&d=#i^I{1kKcYY;zah_ZTI(kWA3$%`{& zcH%Pr>w%2r)R7|NWRQV$5h7%CG>nLmt{6u7(v@-E8|mV4l${ty={xj5x=3mWvtyst z*n$avHI2;9B+3gU%FyAcxdJCb)>d9vH8j&StHExJL>)*16Zn55i-hTc?}f3fRCKYn z%xzo?ekRZYIlZK4L2Af>4j?4SSm{kfnuv_pKN88w=qJiQ26EK?@n{Y<*(Az74n$!8 zNTvysyKjSO7Te1_#w8!xznp|pB;N<@K=ue(GG2PqKxQQ)_K(DSG7*XLPkV}2c-#kD;}T^b2O^L=lA?-{yMCP)-}WMN7?*nx_bUt6b#hOJ9LOCZNyaMP zETq^^wEeOg`za(O%02;l#P;Dhyl_4|%%+!9#D7U(c-#;B0F*_em=1XHcjERXnb)`m z1aZHz3qjF+z+QMgr?|?`4u7(<(~jOYWsNXS56W8 zC4mY2Kavo`Y~SyFu>@Nr?(fOm#F1 z$3TwSKOXayQ^b66AOibGl4F?MeH)Bhi`HMyAKJg1E>k4m2kb!h2wAf1^`?;!_b(uE zpG-ue{1c!@?0*4|{T5Qheo0^g`$v*wm>&36n7pA^=5~_(%ZW2Z3sOT4bO0eq#!7D@ zF>(I_68p*MC(1tta>V`@@VIXwMcfw$;!mf^q1&*>`Yn6D3#VMP+78<~#kXUnXVVFB zf_g0f6VrT))>o!izdQ~5+?Xw$SY7naetlT)zUw@!)$hOC?OZPK)Bmqry>n;%`s$l^ zuHBx0>+1UTtE;bHbKbi8`kU8Q%V<5e>UBP*76vW23m~Q*2F<&>qhDyjy#Ree^KS=; zsr77YyIXJ6zffz>vm-U)5c^nI)H_t8pTc%r;e6P#d$2}aSvxEnIawpFP8s?VIA5Z= zU7v5%U{B&kXTC7+&IR{f%bjQJvsQ5+SjG87SnH@ddmZQem7BNjT)%Ps?Q84K?bmPK znGgBah{OLD+NNhBgSCiZ8Jf>vZDLq*(;BQr49n2$25S?;lAGjUEn--Pj!)ZWZ_gzj z+aiW#3W>+IiSaT~;;}7aSf-G8Y?~OCi4u=(@r1MX%C)uEZr!~8#vQjOySe)6wY59W z%j;Kfy<+{e0Y}++{Vr@%-mW?9jP1@=ePj21?`yp4U0r=K=y-=#2l}7&Yoo9s-Nh3& zq`SYCCGBIbuw%4X)KDo{Cy-boNK9I1G!%y=!d*7uh3HoJGQ8(nco|+^EW8x13Jt5T zgn+`#Sy6x_ycIwRZv|MwTLGBx7KLV`&}6{e7_f?5*m?gIEI_yX)DIOb#sUVW%w^;eELJsNb;u zfkBW%lPL1enooy%s3qd5dNP)~Vmj>Q54V{th`Co0BwR|YVl3m~FN4xw8vLdR3eUVQ zgqc_AUL$?8zcdj!&KN^gdkg|xSk0DhOB>nw1_5M_!#e6#Q1-s_aWR}urKEg)=<3N; zK_v{3cXESMl$RX`m5>a407+iW8dO43w?QR&p=3}A$@K-5OPKmHtsi2Y;y%;4@a`gdvF-oK)c!NqYN~D}z6;z5*BIV>NE+vd4UrLHu_y=uZ z1o>4IqwuR>!$ zb(q)a*1I1#3k$-%-V0$I)5pEuH{^T0pC^_6Zu1sP+i+*M+UH@txn0nn6Qnsm$c9_K zk#2`#FYU6w)eh++sP^C*`-*?wh-~;2KXJxhl-|zKv>jbA^yFg7haj zhL9AJd`R;95E9~oIZeo?5R&WwIZg8W5E9}>I3(m#2ub!a9FqJ#goJn>4hi`bLXuq* zha|rbAt6qSLqa}A<6GUNQkfFkdRLyB-s&iNb>s-65=8`B;->FN%o%{lKeh| zgm_jC3HcO4lHD$cB)<8;^32`?Z67ng8 zBzqhVNq!$fLfj39gnSAi$sUJ8lHZ4r5O>2NA)i7>vd7_&~T0G z`F&tZh%9_tLcRpHBn!j0CHZv#1(Aaf1^E&{kpvV z;lB?B`4T{p1>i%GUk6Z-UciTfdW`E>vV@%}y( zAS9M30Y0F!JAOd$W`w?2}lbv}^-t&gT@-J2w- zoQIhPNQG+X9Hm({NnN=Jm$=Tw-8k*dPeZx!6y@y71(VRT=D5 z(YeAHi*k>b-m}YJ?G^RHvf%R9QZeT8*HTe?`D>}T4A-ATFTMz_a4k9|J90aGg|^1tcUwv z?UMM%KH0+4%U?Bj**(4db;!Lo{^{kfks;F*6|XRd`v8+JGAyRJ{1tjWXesF;!y>z* zFzF&gIP5I>B12g3lYEh(y+D4GkUufVi{d?oe0Pnv+H<42wdF1yYuCCv?Iyb?YqQ?1 zx0=;0XRFoe)H+TXmP!yZ>fZa<$r-djPBp6C#?F?*j$7(g+jVx^VQU-y?6h{;8#N_% zx=c>95oC39Gp33Of?#XBv(_Xr`CM7f_yNO!ySP@N`qfQ&?ZPAD2;2&DV`ocV)6gaO z2W2ln0wsCX9*D43s}uGag};?RIN(X9La-u}8r!#vO=L)Q_K{Z1&iM(fbi09?$<6~r2y$sfL zyAPQ0#gV6)_bjW1QbqCCm79=NdkwNH(~h@NklpCB4U5@O-8bX1Kc0`OTmn~ep3G@7 z5OCWQ_f$|alddHB=Y;&RL0(WjX?j0e8r+XE$7s^N{HXcS@S`qe;@%&2<<36&QFDL% zsI&w8!||i0wby>sgzaU)_GMus^B($9^P9qtW}ieqn*EUc=r0iR#SFP5`q579^E+&5 z6bm`4AHXDs8;k^py`8OFa0#FrT-P?&AnvxlJPdm{@NZQ2foB-p zuFm4DumbuD;bSp;w8KZp)^T2x;Uc`@J8(ze1R1x-jKl7K`6-gU1069PA;Y#|D+TLt^LeA%tlkG1)#1uLxWO%L$eMx0ZI&DTJFVu{2fRylZ_XI(;bnLjvNNv^EgRo!4CP-Z#y$G-`Zcna ztW<21jDDn3vRAAY>eJ3>#1urKgGP!O>-k8en7gZi^teMpTMh|fT1*WIU024J4%&AA z>RY)3!QQR3!Cp3QPSqR8y7~iy;iOSK>;X9k!QN?dQWF-A(BTqn2}vHs!w!+;Q9LZ` zOdiEAEJ2~sB%D_EAojGE?19JdPh(Fw6|=L~WB8}B=jLYMJ^LQRKaD+M2icU@dGg^h zI);B5dxoR^xe@bglQWZWZ}3b9cA~bOaNj9CuIYB2B4mM%=|0-anYq*C5NmDIX|y(L zTM!@Bn@($+nq<`L*z+%tBaR~$sW4f>`{6B-GTIsitF|U*jX` zc?W*n+iE?W_hHY&PQ=YRBm)1-+vImLBnRE{VeD7|G67%4+%K04P%MKMD?+ghFII$N z!)P%ZiVgE(5XS&r<2)_hA+zQPJ=AT3SXi3c2|pZzuJ=g0c7JCJQrKT`Q+|HR8=4rS z>JrPL9L+qFF9AR|%X>{eH6wx%L`BnV{pr z;S6LNRt=HhYiI5R1S;Koj)z7#Mxt5MfH?S945&X9C3IRx=m9;UGf_ehMhK;;2_>+E zXB1Fv*0zXg9;VrMAY(r_OAdr2xY8D}KN7|Ms1EyCJ@&aM_QxXFTYX@ET)_TBD(p{2 zu|K85{_O_xxEiLf$i7DMQCcM&HXes+|XPij08= zcS}R|0TLG@@aKbvEENHX9MkU~J9OgFut_r8Lxg=eNXUbSEbZXd8~*C>5QS63yjMXD zR*;g3YO~pblnk5yp~GJv9?HY}FNlnSKSD;4|HZiY9A_Nk;${9v(f*Ch9^@p7g*|8l zmdSl|c^|SDcptKt{7Yl?w+wPh%=-B5i3el$!b)VeKWEJSm_fb_D)?6<6+Ggr;M7-# zArl)%+pbCH8VUH{AZhtn?-+BGC644(#m6N zjCuA(?hZVFZ*Sz@ga`2Mjoe%C0RFv^`xHEYhi~LQ4G-Ys8@bQG1Jvr}-i8OL)ysVr z9-vk)_c?fgTD{!2!2{Il0 z?R%gr(8~xNQKI|rw|sJk^7mMWQjY6T7K9FEQS4Bbk~)-SwL{r;xHbt3vBf|%eSCHi z7G=-vy3hEJU^V6ncW?#!*9>w_G(T@F;q{{~Fi7H;?GjCCbbc;4&d2Yy(Sb(^=^=l{ z=&OYE{m$-Ze4l92LsH49$_W^fc{3s~*PJu#NmrWw8h4Vb9EI<0E)c#6yG{r*OYjyf zo&;Y}Ol8LXluv2K-x6oW&kJV!t)dyfkYvUmSIv0W>GHKY(6*X|d=J<-I6#F8_WQ@$y?<_zXK z?Kx(?p@di!mYHG3oDB@!-#p@)Gj#RLoS~ynXU_eUPifApapt@xnDZw@bH0{j&M&Fv zyz7Ddq&euw{h~RySLXb3sO-2o+p;;s(IG{iIlmH8h?{eS!uXlq^3oGdd`PTd z&dgvTBh1a2@7@y3nSJ&8ZO7ZGMUu^J0W96{3_j!KgN7j=l&TxQ?p1p=jh{$HL@L4Z z$WprLkBrqnFw*85;C2NuLk?pCmNN+ui(4W-8O*sMy~;F(SXQ$$`q0U&hI^n(IONj541iK#J&N` zDP9w&^ry_%#p(HR^9CJpPg#K}m^bK%dy>jrj?9D$9K4xhe{$n4w0ASw-mPeRpA6eu z_uGRtAh#)TKuI1vWYI)S`z-5v!yA6Rk!JXHS)wmtcKBfB01KPW3|}&ApX|shmwY!@ z^hpni^|VNcOxY78Bo~470a9+;A3yWcTN+|+n`gb*@|F39F!pv+X|{59jWG7k@^QI; zIfYo;9l7(`eB2K0hl#ihug%Y(ku&lV);%rOy+o{g0$dquu{MsiSX;)rM}IhZM)vb` z^w8nsC(fO>%8!PNB>!mm8FJxEnYrkKHg&aH&LC_?ey;fS;VUrJ7_3}7`=!jO@Gqqv z Date: Thu, 1 Aug 2024 11:26:41 +0100 Subject: [PATCH 17/28] Delete smlp_toy.onnx --- smlp_toy.onnx | Bin 1777 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 smlp_toy.onnx diff --git a/smlp_toy.onnx b/smlp_toy.onnx deleted file mode 100644 index 910e9d0aa6c5d49a774cb8a113d1a19489181e9d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1777 zcmd;J7h*3-Gs@4)tB~R~)H5{GGgL4%O|~#Ju-eDVRm{bmlA2eX8lRb0P+G#JQJh*> znwnRVnV6#w7T5PpEb%SP(GN;ZObJUY%1lhkN%b$VG7yr)q0-7gN*srj5*x%Yu0}>K z+}gP`F|<2nCKfxUq+mKi3YSu#Dk3qh$lr9e|fF-!$XD6xP@ zu2x1aTpGEw&^5*z;tX49oN9qCzz8|GLGbYHWaPqU1eZ1z^Kpi;3~t3h*I)!9&^1t_ zAmJRrh1USgbOba28rB$L36+9|b~~0-1lNouyp2c*Z}Z4~WZ}BNAq&sdS|J=P983a?PMApy<_HXHyc8l7C+}SH>0@F~AFTb|fp8vGPR%>$RP1#QuF-XL;H4thL+w^t3?7!&|nG zf9$no00OP=uWYq8GXf)o1L%38OH-iN->n5j z?`PSozJJBRBD$AFy}$In+1{A_!!|~F``%fBr@(;`B$=F_SC(2- zlA5BBR+OKsfGA0#lu_y(ArUSi4n`q9E)F5K(!A{Wcrz|04wfW&E?92BP+$&Jz~scj W#UQ}z#K*;zn5hS<;Pi5H1egF}t$CLK From ed2b04a1bf843ae88332633d47ac2e437efb1ebd Mon Sep 17 00:00:00 2001 From: konstantopoulos-tetractys <37417801+ntinouldinho@users.noreply.github.com> Date: Thu, 1 Aug 2024 11:32:33 +0100 Subject: [PATCH 18/28] Delete fake_marabou.py --- src/smlp_py/marabou/fake_marabou.py | 36 ----------------------------- 1 file changed, 36 deletions(-) delete mode 100755 src/smlp_py/marabou/fake_marabou.py diff --git a/src/smlp_py/marabou/fake_marabou.py b/src/smlp_py/marabou/fake_marabou.py deleted file mode 100755 index caeb2dfd..00000000 --- a/src/smlp_py/marabou/fake_marabou.py +++ /dev/null @@ -1,36 +0,0 @@ -from maraboupy import Marabou -import numpy as np - -# %% -# Set the Marabou option to restrict printing -options = Marabou.createOptions(verbosity = 0) - -# %% -# Fully-connected network example -# ------------------------------- -# -# This network has inputs x0, x1, and was trained to create outputs that approximate -# y0 = abs(x0) + abs(x1), y1 = x0^2 + x1^2 -print("Fully Connected Network Example") -filename = "../../test.onnx" -network = Marabou.read_onnx(filename) - - -# %% -# Set input bounds -network.setLowerBound(0,-10.0) -network.setUpperBound(0, 10.0) -network.setLowerBound(1,-10.0) -network.setUpperBound(1, 10.0) -network.setLowerBound(2,-10.0) -network.setUpperBound(2, 10.0) -network.setLowerBound(3,-10.0) -network.setUpperBound(3, 10.0) -network.setLowerBound(4,-10.0) -network.setUpperBound(4, 10.0) -network.setLowerBound(5,-10.0) -network.setUpperBound(5, 10.0) - -# %% -# Call to Marabou solver -exitCode, vals, stats = network.solve(options = options) \ No newline at end of file From c72298ea0a8d2ddd2647f320990875de89e927e3 Mon Sep 17 00:00:00 2001 From: konstantopoulos-tetractys <37417801+ntinouldinho@users.noreply.github.com> Date: Thu, 1 Aug 2024 11:49:29 +0100 Subject: [PATCH 19/28] use relative paths --- src/smlp_py/NN_verifiers/test_marabou.py | 6 +++++- src/smlp_py/NN_verifiers/verifiers.py | 10 ++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/smlp_py/NN_verifiers/test_marabou.py b/src/smlp_py/NN_verifiers/test_marabou.py index 690e8680..66902c95 100755 --- a/src/smlp_py/NN_verifiers/test_marabou.py +++ b/src/smlp_py/NN_verifiers/test_marabou.py @@ -20,9 +20,13 @@ if __name__ == "__main__": import numpy as np from tensorflow.keras.models import load_model + import os + script_dir = os.path.dirname(os.path.abspath(__file__)) + relative_h5_path = os.path.join(script_dir, '../../../result/abc_smlp_toy_basic_nn_keras_model_complete.h5') + absolute_h5_path = os.path.normpath(relative_h5_path) # Load the model from the .h5 file - model = load_model("/home/kkon/Desktop/smlp/result/abc_smlp_toy_basic_nn_keras_model_complete.h5") + model = load_model(absolute_h5_path) # Prepare your input data input_data = np.array([[1.043789425, 0, 0.191919192, 0]]) # Example input data diff --git a/src/smlp_py/NN_verifiers/verifiers.py b/src/smlp_py/NN_verifiers/verifiers.py index 7b59f795..5f4f17f9 100755 --- a/src/smlp_py/NN_verifiers/verifiers.py +++ b/src/smlp_py/NN_verifiers/verifiers.py @@ -1,3 +1,4 @@ +import os from abc import ABC, abstractmethod from enum import Enum from typing import List, Dict, Optional, Tuple @@ -89,7 +90,7 @@ def __init__(self, model_path=None, parser=None): self.model_file_path = "./" self.log_path = "marabou.log" - self.data_bounds_file = "/home/kkon/Desktop/smlp/result/abc_smlp_toy_basic_data_bounds.json" + self.data_bounds_file = self.find_file_path("../../../result/abc_smlp_toy_basic_data_bounds.json") self.data_bounds = None # Adds conjunction of equations between bounds in form: # e.g. Int(var), var >= 0, var <= 3 -> Or(var == 0, var == 1, var == 2, var == 3) @@ -102,7 +103,7 @@ def __init__(self, model_path=None, parser=None): def initialize(self): - self.model_file_path = "/home/kkon/Desktop/smlp/result/abc_smlp_toy_basic_nn_keras_model_complete.h5" + self.model_file_path = self.find_file_path('../../../result/abc_smlp_toy_basic_nn_keras_model_complete.h5') self.convert_to_pb() self.load_json() self.network_num_vars = self.network.numVars @@ -121,6 +122,11 @@ def epsilon(self, e, direction): else: raise ValueError("Direction must be 'up' or 'down'") + def find_file_path(self, relative_path): + script_dir = os.path.dirname(os.path.abspath(__file__)) + relative_h5_path = os.path.join(script_dir, relative_path) + absolute_h5_path = os.path.normpath(relative_h5_path) + return absolute_h5_path def convert_to_pb(self, output_model_file_path="."): model = tf.keras.models.load_model(self.model_file_path) From 85bdf5e19f00b7b2a1a753c3dbf0904e800a63c0 Mon Sep 17 00:00:00 2001 From: konstantopoulos-tetractys <37417801+ntinouldinho@users.noreply.github.com> Date: Fri, 16 Aug 2024 20:40:22 +0100 Subject: [PATCH 20/28] update how input/output indexing is stored in memory --- src/smlp_py/NN_verifiers/verifiers.py | 261 +++++++++++++++----------- 1 file changed, 148 insertions(+), 113 deletions(-) diff --git a/src/smlp_py/NN_verifiers/verifiers.py b/src/smlp_py/NN_verifiers/verifiers.py index 5f4f17f9..8e15f50b 100755 --- a/src/smlp_py/NN_verifiers/verifiers.py +++ b/src/smlp_py/NN_verifiers/verifiers.py @@ -43,20 +43,12 @@ class Type(Enum): Int = 1 class Bounds: - lower = -np.inf - upper = np.inf + def __init__(self, lower=-np.inf, upper=np.inf): + self.lower = lower + self.upper = upper def __init__(self, form: Type, index=None, name="", is_input=True): - if is_input and not index: - self.index = Variable._input_index - Variable._input_index += 1 - elif is_input and index: - # auxiliary scaled variable - self.index = index - else: - self.index = Variable._output_index - Variable._output_index += 1 - + self.index = index self.form = form self.name = name self.is_input = is_input @@ -73,7 +65,7 @@ def set_upper_bound(self, upper): self.bounds.upper = upper class MarabouVerifier(Verifier): - def __init__(self, model_path=None, parser=None): + def __init__(self, parser=None, variable_ranges=None, is_temp=False): # MarabouNetwork containing network instance self.network = None @@ -86,6 +78,8 @@ def __init__(self, model_path=None, parser=None): # List of variables self.variables = [] + self.variable_ranges = variable_ranges + self.unscaled_variables = [] self.model_file_path = "./" @@ -95,19 +89,36 @@ def __init__(self, model_path=None, parser=None): # Adds conjunction of equations between bounds in form: # e.g. Int(var), var >= 0, var <= 3 -> Or(var == 0, var == 1, var == 2, var == 3) - # Stack for keeping ipq - self.ipq_stack = [] + self.input_index = 0 + self.output_index = 0 self.parser = parser self.network_num_vars = None + self.init_variables(is_temp=is_temp) + + if self.variable_ranges: + self.initialize() + + def initialize(self, variable_ranges=None): + if variable_ranges: + self.variable_ranges = variable_ranges - def initialize(self): self.model_file_path = self.find_file_path('../../../result/abc_smlp_toy_basic_nn_keras_model_complete.h5') self.convert_to_pb() self.load_json() self.network_num_vars = self.network.numVars self.add_unscaled_variables() + self.create_integer_range() + + def reset(self): + self.network.clear() + self.network = Marabou.read_tf('model.pb') + self.unscaled_variables = [] + self.add_unscaled_variables() + # Default bounds for network + for equation in self.equations: + self.apply_restrictions(equation) def load_json(self): @@ -148,17 +159,35 @@ def convert_to_pb(self, output_model_file_path="."): - def init_variables(self, inputs: List[Tuple[str, str]], outputs: List[Tuple[str, str]]) -> None: - for input_var in inputs: - name, type = input_var - var_type = Variable.Type.Real if type.lower() == "real" else Variable.Type.Int - self.variables.append(Variable(var_type, name=name, is_input=True)) + def init_variables(self, is_temp=False) -> None: + self.create_variables(is_input=True, is_temp=is_temp) + self.create_variables(is_input=False, is_temp=is_temp) - for output_var in outputs: - name, type = output_var + def create_variables(self, is_input=True, is_temp=False): + store = self.parser.inputs if is_input else self.parser.outputs + for var in store: + name, type = var var_type = Variable.Type.Real if type.lower() == "real" else Variable.Type.Int - self.variables.append(Variable(var_type, name=name, is_input=False)) + if name.startswith(('x', 'p', 'y')): + index = self.input_index if is_input else self.output_index + self.variables.append(Variable(var_type, name=name, index=index, is_input=is_input)) + if is_input: + self.input_index += 1 + else: + self.output_index += 1 + + def create_integer_range(self): + integer_variables = [variable for variable in self.variables if variable.form == Variable.Type.Int] + for variable in integer_variables: + int_range = self.variable_ranges.get(variable.name) + if not int_range: + raise Exception(f"Need integer rangers for variable {variable.name}") + ranges = int_range['interval'] + lower, upper = ranges[0], ranges[-1] + variable.bounds = Variable.Bounds(lower=lower, upper=upper) + integer_formula = self.parser.create_integer_disjunction(f'{variable.name}_unscaled', (lower, upper)) + self.add_permanent_constraint(integer_formula) def add_unscaled_variables(self): @@ -200,22 +229,15 @@ def get_variable_by_name(self, name: str) -> Optional[Tuple[Variable, int]]: if is_unscaled: return variable, variable.index elif is_output: - index -= Variable.get_index("input") + index -= self.input_index index = self.network.outputVars[0][0][index] if is_output else self.network.inputVars[0][0][index] return variable, index return None - def reset(self): - self.network.clear() - self.network = Marabou.read_tf('model.pb') - self.unscaled_variables = [] - self.add_unscaled_variables() - # Default bounds for network - for equation in self.equations: - self.apply_restrictions(equation) def add_permanent_constraint(self, formula): self.equations.append(formula) + self.apply_restrictions(formula) def add_bound(self, variable:str, value, direction="upper", strict=True): var, var_index = self.get_variable_by_name(f"{variable}_unscaled") @@ -232,32 +254,14 @@ def add_bound(self, variable:str, value, direction="upper", strict=True): self.network.setLowerBound(var_index, value) var.set_lower_bound(value) + def add_equality(self, variable, value): + var, var_index = self.get_variable_by_name(f"{variable}_unscaled") - # TODO: CHECK IF MARABOU NATIVELY SUPPORTS INTEGERS: it does not - def add_bounds(self, variable, bounds=None, num="real", grid=None): - var, is_output = self.get_variable_by_name(variable) - if var is None: - return None - - # TODO: handle case when one of the two is None - if bounds: - lower, upper = bounds - self.network.setLowerBound(var.index, lower) - self.network.setUpperBound(var.index, upper) - - if num == "int": - # add all distinct integer values - grid = range(lower, upper+1) - - if num in ["int", "grid"] and grid is not None: - disjunction = [] - for i in grid: - eq1 = MarabouUtils.Equation(MarabouCore.Equation.EQ) - eq1.addAddend(1, var.index) - eq1.setScalar(i) - disjunction.append([eq1]) + eq = MarabouUtils.Equation(MarabouCore.Equation.EQ) + eq.addAddend(1, var_index) + eq.setScalar(value) + self.network.addEquation(eq) - self.network.addDisjunctionConstraint(disjunction) def apply_restrictions(self, formula, need_simplification=False): formula = self.parser.simplify(formula) @@ -311,7 +315,7 @@ def is_negation_of_ite(self, formula): if left.is_or() and len(left.args()) == 2 and right.is_or() and len(right.args()) == 2: eq_1, eq_2 = left.args()[0], left.args()[1] eq_3, eq_4 = right.args()[0], right.args()[1] - return True, [eq_1, eq_2, eq_3, eq_4] + return True, [And(eq_1, eq_3), And(eq_1, eq_4), And(eq_2, eq_3), And(eq_2, eq_4)] return False, [] def create_equation(self, formula, from_and=False, need_simplification=False): @@ -385,64 +389,13 @@ def process_comparison(self, formula, need_simplification=False): self.add_bound(symbol, constant, direction="lower", strict=True) elif comparison == "=": # TODO: add a marabou equation instead - self.add_bound(symbol, constant, direction="lower", strict=False) - self.add_bound(symbol, constant, direction="upper", strict=False) + self.add_equality(symbol, constant) + # self.add_bound(symbol, constant, direction="lower", strict=False) + # self.add_bound(symbol, constant, direction="upper", strict=False) else: return - def alpha(self): - # (((-1 <= x2) & (0.0 <= x1) & (x2 <= 1) & (x1 <= 10.0)) & (((p2 < 5) & (x1 = 10.0)) & (x2 < 12))) - # p2<5 and x1==10 and x2<12 - # (p2≥5)∨(x1#10)∨(x2≥12) - - p1, is_output = self.get_variable_by_name("p1") - p2, is_output = self.get_variable_by_name("p2") - x1, is_output = self.get_variable_by_name("x1") - x2, is_output = self.get_variable_by_name("x2") - y1, is_output = self.get_variable_by_name("y1") - y2, is_output = self.get_variable_by_name("y2") - - # - # self.network.setUpperBound(p2.index, 5-epsilon) - v = Var(p2.index) - - # self.network.addConstraint(v <= self.epsilon(5, "down")) - # - # self.network.setUpperBound(x1.index, self.epsilon(10,'up')) - # self.network.setLowerBound(x1.index, self.epsilon(10, "down")) - # - # self.network.setUpperBound(x2.index, self.epsilon(12, "down")) - # - # self.network.setLowerBound(y1.index, 4) - # self.network.setUpperBound(y2.index, 8) - - # p1==4.0 or (p1==8.0 and p2 > 3) - eq1 = MarabouUtils.Equation(MarabouCore.Equation.EQ) - eq1.addAddend(1, p1.index) - eq1.setScalar(4) - - eq2 = MarabouUtils.Equation(MarabouCore.Equation.EQ) - eq2.addAddend(1, p1.index) - eq2.setScalar(8) - - eq3 = MarabouUtils.Equation(MarabouCore.Equation.GE) - eq3.addAddend(1, p2.index) - eq3.setScalar(self.epsilon(3, "up")) - - self.network.addDisjunctionConstraint([[eq1], [eq2, eq3]]) - - # b1 = self.network.getNewVariable() - # - # # Define the epsilon value - # epsilon = 1e-5 - # - # # Constraint for (y1 + y2) / 2 > 1 when b1 = 1 - # # This is equivalent to y1 + y2 > 2 - # self.network.addInequality([y1, y2, b1], [1, 1, -2], -epsilon) # y1 + y2 - 2*b1 > 0 -> y1 + y2 > 2 when b1 = 1 - # - # # Ensure b1 is binary - # self.network.setLowerBound(b1, 0) - # self.network.setUpperBound(b1, 1) + def find_witness(self, witness): answers = {"result":"SAT", "witness":{}, 'witness_var':{}} for variable in self.unscaled_variables: @@ -582,3 +535,85 @@ def simplify_to_linear(formula): # exitCode1, vals1, stats1 = mb.solve() print(exitCode1) + + +# TODO: CHECK IF MARABOU NATIVELY SUPPORTS INTEGERS: it does not + # def add_bounds(self, variable, bounds=None, num="real", grid=None): + # var, is_output = self.get_variable_by_name(variable) + # if var is None: + # return None + # + # # TODO: handle case when one of the two is None + # if bounds: + # lower, upper = bounds + # self.network.setLowerBound(var.index, lower) + # self.network.setUpperBound(var.index, upper) + # + # if num == "int": + # # add all distinct integer values + # grid = range(lower, upper+1) + # + # if num in ["int", "grid"] and grid is not None: + # disjunction = [] + # for i in grid: + # eq1 = MarabouUtils.Equation(MarabouCore.Equation.EQ) + # eq1.addAddend(1, var.index) + # eq1.setScalar(i) + # disjunction.append([eq1]) + # + # self.network.addDisjunctionConstraint(disjunction) + + +# def alpha(self): +# # (((-1 <= x2) & (0.0 <= x1) & (x2 <= 1) & (x1 <= 10.0)) & (((p2 < 5) & (x1 = 10.0)) & (x2 < 12))) +# # p2<5 and x1==10 and x2<12 +# # (p2≥5)∨(x1#10)∨(x2≥12) +# +# p1, is_output = self.get_variable_by_name("p1") +# p2, is_output = self.get_variable_by_name("p2") +# x1, is_output = self.get_variable_by_name("x1") +# x2, is_output = self.get_variable_by_name("x2") +# y1, is_output = self.get_variable_by_name("y1") +# y2, is_output = self.get_variable_by_name("y2") +# +# # +# # self.network.setUpperBound(p2.index, 5-epsilon) +# v = Var(p2.index) +# +# # self.network.addConstraint(v <= self.epsilon(5, "down")) +# # +# # self.network.setUpperBound(x1.index, self.epsilon(10,'up')) +# # self.network.setLowerBound(x1.index, self.epsilon(10, "down")) +# # +# # self.network.setUpperBound(x2.index, self.epsilon(12, "down")) +# # +# # self.network.setLowerBound(y1.index, 4) +# # self.network.setUpperBound(y2.index, 8) +# +# # p1==4.0 or (p1==8.0 and p2 > 3) +# eq1 = MarabouUtils.Equation(MarabouCore.Equation.EQ) +# eq1.addAddend(1, p1.index) +# eq1.setScalar(4) +# +# eq2 = MarabouUtils.Equation(MarabouCore.Equation.EQ) +# eq2.addAddend(1, p1.index) +# eq2.setScalar(8) +# +# eq3 = MarabouUtils.Equation(MarabouCore.Equation.GE) +# eq3.addAddend(1, p2.index) +# eq3.setScalar(self.epsilon(3, "up")) +# +# self.network.addDisjunctionConstraint([[eq1], [eq2, eq3]]) +# +# # b1 = self.network.getNewVariable() +# # +# # # Define the epsilon value +# # epsilon = 1e-5 +# # +# # # Constraint for (y1 + y2) / 2 > 1 when b1 = 1 +# # # This is equivalent to y1 + y2 > 2 +# # self.network.addInequality([y1, y2, b1], [1, 1, -2], -epsilon) # y1 + y2 - 2*b1 > 0 -> y1 + y2 > 2 when b1 = 1 +# # +# # # Ensure b1 is binary +# # self.network.setLowerBound(b1, 0) +# # self.network.setUpperBound(b1, 1) \ No newline at end of file From bbeefa2dd78cdb5160ffb8c07863434b46a04193 Mon Sep 17 00:00:00 2001 From: konstantopoulos-tetractys <37417801+ntinouldinho@users.noreply.github.com> Date: Fri, 16 Aug 2024 20:41:11 +0100 Subject: [PATCH 21/28] create temporary solver and add flag for z3 simplifier --- src/smlp_py/smlp_optimize.py | 2 +- src/smlp_py/smlp_query.py | 39 +++++++++++++++++------- src/smlp_py/smlp_terms.py | 16 +++++----- src/smlp_py/smtlib/text_to_sympy.py | 47 +++++++++++++++++++++++------ 4 files changed, 75 insertions(+), 29 deletions(-) diff --git a/src/smlp_py/smlp_optimize.py b/src/smlp_py/smlp_optimize.py index ffbcae8a..b4b9db12 100755 --- a/src/smlp_py/smlp_optimize.py +++ b/src/smlp_py/smlp_optimize.py @@ -266,7 +266,7 @@ def optimize_single_objective(self, model_full_term_dict:dict, objv_name:str, ob T = (l + u) / 2 #quer_form = objv_term > smlp.Cnst(T) quer_form = objv_term >= smlp.Cnst(T) - quer_form = self._modelTermsInst.verifier.parser.handle_ite_formula(quer_form) if self._ENABLE_PYSMT else quer_form + quer_form = self._modelTermsInst.verifier.parser.handle_ite_formula(quer_form, is_form2=True) if self._ENABLE_PYSMT else quer_form quer_expr = '{} >= {}'.format(objv_expr, str(T)) if objv_expr is not None else None quer_name = objv_name + '_' + str(T) if not beta == smlp.true: diff --git a/src/smlp_py/smlp_query.py b/src/smlp_py/smlp_query.py index dccc624c..ff122475 100644 --- a/src/smlp_py/smlp_query.py +++ b/src/smlp_py/smlp_query.py @@ -8,6 +8,8 @@ from smlp_py.smlp_terms import ModelTerms, SmlpTerms from smlp_py.smlp_utils import np_JSONEncoder #, str_to_bool +from src.smlp_py.NN_verifiers.verifiers import MarabouVerifier + class SmlpQuery: def __init__(self): @@ -100,9 +102,11 @@ def synthesis_results_file(self): def find_candidate(self, solver): #res = solver.check() if self._ENABLE_PYSMT: + print("PYSMT LOOKING FOR CANDIDATE") res, witness = solver.solve() return witness else: + print("FORM2 LOOKING FOR CANDIDATE") res = self._modelTermsInst.smlp_solver_check(solver, 'ca', self._lemma_precision) if self._modelTermsInst.solver_status_unknown(res): # isinstance(res, smlp.unknown): return None @@ -155,9 +159,12 @@ def check_concrete_witness_consistency(self, domain:smlp.domain, model_full_term solver.add(alpha); #print('alpha', alpha) solver.add(eta); #print('eta', eta) solver.add(witn_form); #print('witn_form', witn_form); print('query', query) + + printer = {'alpha':alpha, 'eta':eta,'witn_form':witn_form} if query is not None: + printer['query'] = query solver.add(query) - res = self._modelTermsInst.smlp_solver_check(solver, 'witness_consistency') + res = self._modelTermsInst.smlp_solver_check(solver, 'witness_consistency',printer ) #res = solver.check(); #print('res', res) return res @@ -175,12 +182,15 @@ def find_candidate_counter_example(self, universal, domain:smlp.domain, cand:dic theta = self._modelTermsInst.compute_stability_formula_theta(cand, None, theta_radii_dict, universal) if self._ENABLE_PYSMT: - self._modelTermsInst.verifier.reset() - self._modelTermsInst.verifier.apply_restrictions(theta) - self._modelTermsInst.verifier.apply_restrictions(alpha) - negation = self._modelTermsInst.verifier.parser.propagate_negation(query) - self._modelTermsInst.verifier.apply_restrictions(negation, need_simplification=True) - res, witness = self._modelTermsInst.verifier.solve() + solver = MarabouVerifier(parser=self._modelTermsInst.parser, variable_ranges=self._modelTermsInst.verifier.variable_ranges, is_temp=True) + # self._modelTermsInst.verifier.reset() + solver.apply_restrictions(theta) + solver.apply_restrictions(alpha) + negation = solver.parser.propagate_negation(query) + z3_equiv = solver.parser.handle_ite_formula(negation, handle_ite=False) + solver.apply_restrictions(negation, need_simplification=True) + print('PYSMT FORMULA',{'alpha': alpha, 'theta': theta, 'not_query': negation.serialize()}) + res, witness = solver.solve() return witness else: solver = self._modelTermsInst.create_model_exploration_instance_from_smlp_components( @@ -188,7 +198,7 @@ def find_candidate_counter_example(self, universal, domain:smlp.domain, cand:dic solver.add(theta); #print('adding theta', theta) solver.add(alpha); #print('adding alpha', alpha) solver.add(self._smlpTermsInst.smlp_not(query)); #print('adding negated quert', query) - return self._modelTermsInst.smlp_solver_check(solver, 'ce', self._lemma_precision) + return self._modelTermsInst.smlp_solver_check(solver, 'ce', self._lemma_precision, {'alpha':alpha, 'theta':theta,'not_query':self._smlpTermsInst.smlp_not(query)}) #return solver.check() # Enhancement !!!: at least add here the delta condition @@ -219,7 +229,7 @@ def validate_witness_smt(self, universal:bool, model_full_term_dict:dict, quer_n #candidate_solver.add(smlp.Var(var) == smlp.Cnst(val)) candidate_solver.add(self._smlpTermsInst.smlp_eq(smlp.Var(var), smlp.Cnst(val))) - candidate_check_res = self._modelTermsInst.smlp_solver_check(candidate_solver, 'ca') + candidate_check_res = self._modelTermsInst.smlp_solver_check(candidate_solver, 'ca', {'alpha':alpha, 'eta':eta,'quer':quer}) if self._modelTermsInst.solver_status_sat(candidate_check_res): #isinstance(candidate_check_res, smlp.sat): cond_feasible = True if universal: @@ -561,7 +571,11 @@ def query_condition(self, universal, model_full_term_dict:dict, quer_name:str, q while True: # solve Ex. eta x /\ Ay. theta x y -> alpha y -> (beta y /\ query) print('searching for a candidate', flush=True) - + if self._ENABLE_PYSMT: + print('PYSMT FORMULA', {'alpha': alpha, 'eta': eta, 'quer': quer.serialize()}) + else: + print('FORM2 FORMULA', {'alpha': alpha, 'eta': eta, 'quer': quer}) + ca = self.find_candidate(self._modelTermsInst.verifier) if self._ENABLE_PYSMT else self.find_candidate(candidate_solver) condition_sat = self._modelTermsInst.solver_status_sat(ca["result"]) if self._ENABLE_PYSMT else self._modelTermsInst.solver_status_sat(ca) @@ -621,9 +635,12 @@ def query_condition(self, universal, model_full_term_dict:dict, quer_name:str, q theta = self._modelTermsInst.compute_stability_formula_theta(lemma, delta, theta_radii_dict, universal) if self._ENABLE_PYSMT: theta_negation = self._modelTermsInst.parser.propagate_negation(theta) - self._modelTermsInst.verifier.add_permanent_constraint(theta_negation) + # self._modelTermsInst.verifier.add_permanent_constraint(theta_negation) + self._modelTermsInst.verifier.apply_restrictions(theta_negation) + print("PYSMT THETA ADDED ", theta_negation) else: candidate_solver.add(self._smlpTermsInst.smlp_not(theta)) + print("FORM2 THETA ADDED ", self._smlpTermsInst.smlp_not(theta)) continue elif is_unsat: #isinstance(ce, smlp.unsat): #print('candidate stable -- return candidate') diff --git a/src/smlp_py/smlp_terms.py b/src/smlp_py/smlp_terms.py index 71763867..621fc026 100644 --- a/src/smlp_py/smlp_terms.py +++ b/src/smlp_py/smlp_terms.py @@ -1599,12 +1599,10 @@ def __init__(self): } self.parser = TextToPysmtParser() - self.parser.init_variables(symbols=[("x1", "real"), ('x2', 'int'), ('p1', 'real'), ('p2', 'int'), - ('y1', 'real'), ('y2', 'real')]) + self.parser.init_variables(symbols=[("x1", "real", True), ('x2', 'int', True), ('p1', 'real', True), ('p2', 'int', True), + ('y1', 'real', False), ('y2', 'real', False)]) self.verifier = MarabouVerifier(parser=self.parser) - self.verifier.init_variables(inputs=[("x1", "Real"), ('x2', 'Integer'), ('p1', 'Real'), ('p2', 'Integer')], - outputs=[('y1', 'Real'), ('y2', 'Real')]) self._ENABLE_PYSMT = True self._RETURN_PYSMT = True @@ -2261,8 +2259,7 @@ def create_model_exploration_base_components(self, syst_expr_dict:dict, algo, mo # get variable domains dictionary; certain sanity checks are performrd within this function. spec_domain_dict = self._specInst.get_spec_domain_dict; #print('spec_domain_dict', spec_domain_dict) - self.verifier.initialize() - self.add_integer_constraints() + self.verifier.initialize(variable_ranges=spec_domain_dict) # contraints on features used as control variables and on the responses alph_ranges = self.compute_input_ranges_formula_alpha_eta('alpha', feat_names, @@ -2401,7 +2398,9 @@ def create_model_exploration_instance_from_smlp_components(self, domain, model_f return base_solver # wrapper function on solver.check to measure runtime and return status in a convenient way - def smlp_solver_check(self, solver, call_name:str, lemma_precision:int=0): + def smlp_solver_check(self, solver, call_name:str, lemma_precision:int=0, equations=None): + if equations: + print('FORM2 FORMULA', equations) approx_lemmas = lemma_precision > 0 start = time.time() #print('solver chack start', flush=True) @@ -2540,11 +2539,12 @@ def check_alpha_eta_consistency(self, domain:smlp.domain, model_full_term_dict:d solver.add(eta); #print('eta', eta) #print('create check', flush=True) #res = solver.check(); print('res', res, flush=True) - res = self.smlp_solver_check(solver, 'interface_consistency' if model_full_term_dict is None else 'model_consistency') + res = self.smlp_solver_check(solver, 'interface_consistency' if model_full_term_dict is None else 'model_consistency', equations={'alpha':alpha, 'eta':eta}) else: self.verifier.reset() self.verifier.apply_restrictions(alpha) self.verifier.apply_restrictions(eta) + print('PYSMT FORMULA',{'alpha':alpha, 'eta':eta}) res, witness = self.verifier.solve() consistency_type = 'Input and knob' if model_full_term_dict is None else 'Model' diff --git a/src/smlp_py/smtlib/text_to_sympy.py b/src/smlp_py/smtlib/text_to_sympy.py index 2bb41a30..3b364de7 100755 --- a/src/smlp_py/smtlib/text_to_sympy.py +++ b/src/smlp_py/smtlib/text_to_sympy.py @@ -18,6 +18,8 @@ from typing import List, Dict, Optional, Tuple +from z3 import Tactic, Goal + pysmt_types = { "int": INT, "real": REAL, @@ -103,6 +105,8 @@ def __new__(cls, *args, **kwargs): def __init__(self): self.symbols = {} + self.inputs = [] + self.outputs = [] self._ast_operators_map = { ast.Add: Plus, # Addition ast.Sub: Minus, # Subtraction @@ -128,7 +132,8 @@ def __init__(self): ast.Call: And, 'If': Ite, 'And': And, - 'Not': Not + 'Not': Not, + 'Or': Or } def _div_op(self, left, right): @@ -359,18 +364,22 @@ def cast_number(self, symbol_type, number): elif symbol_type == INT: return Int(number) - def init_variables(self, symbols: List[Tuple[str, str]]) -> None: + def init_variables(self, symbols: List[Tuple[str, str, bool]]) -> None: for input_var in symbols: - name, type = input_var + name, type, is_input = input_var unscaled_name = f"{name}_unscaled" # TODO: i replaced the type variable with real, make sure that's ok - self.add_symbol(name, 'real') - self.add_symbol(unscaled_name, 'real') + self.add_symbol(name, 'real', is_input=is_input, nn_type=type) + self.add_symbol(unscaled_name, 'real', is_input=is_input, nn_type=type) - def add_symbol(self, name, symbol_type): + def add_symbol(self, name, symbol_type, is_input=True, nn_type='real'): assert symbol_type.lower() in pysmt_types.keys() self.symbols[name] = Symbol(name, pysmt_types[symbol_type]) + if name.find("_unscaled") == -1: + store = self.inputs if is_input else self.outputs + store.append((name, nn_type)) + def get_symbol(self, name): assert name in self.symbols.keys() return self.symbols[name] @@ -396,14 +405,34 @@ def extract_smtlib(self, formula): output = outstream.getvalue() return self.remove_first_and_last_line(output) - def handle_ite_formula(self, formula, handle_ite=True): - smlp_str = f""" + def handle_ite_formula(self, formula, is_form2=False, handle_ite=True): + # smlp_str = self.extract_smtlib(formula) if not isinstance(formula, str) else formula + # smlp_str = f""" + # (declare-fun y1 () Real) + # (declare-fun y2 () Real) + # (assert {formula}) + # """ if not isinstance(formula, str) else formula + flag=False + if is_form2: + smlp_str = f""" (declare-fun y1 () Real) (declare-fun y2 () Real) (assert {formula}) - """ if not isinstance(formula, str) else formula + """ + elif isinstance(formula, str): + smlp_str = formula + else: + smlp_str = self.extract_smtlib(formula) + flag=False smlp_parsed = z3.parse_smt2_string(smlp_str) + if flag: + # Apply the tactic to the formula + goal = Goal() + goal.add(smlp_parsed) + t = Tactic('tseitin-cnf') + smlp_parsed = t(goal)[0] + smlp_simplified = z3.simplify(smlp_parsed[0]) ex = self.parse(str(smlp_simplified).replace('\n','')) if ex.is_not(): From 9554fd77ecb3830212a134e201d3f84aae7d0a52 Mon Sep 17 00:00:00 2001 From: konstantopoulos-tetractys <37417801+ntinouldinho@users.noreply.github.com> Date: Sat, 17 Aug 2024 17:37:24 +0100 Subject: [PATCH 22/28] abstract classes for solver agnostic communication --- src/smlp_py/smlp_operations.py | 259 ++++++++++++++++++++++++ src/smlp_py/solver.py | 357 +++++++++++++++++++++++++++++++++ 2 files changed, 616 insertions(+) create mode 100644 src/smlp_py/smlp_operations.py create mode 100644 src/smlp_py/solver.py diff --git a/src/smlp_py/smlp_operations.py b/src/smlp_py/smlp_operations.py new file mode 100644 index 00000000..897e799a --- /dev/null +++ b/src/smlp_py/smlp_operations.py @@ -0,0 +1,259 @@ +import pysmt.shortcuts +import smlp +import functools +import operator as op + +from pysmt.fnode import FNode + +USE_CACHE = False + + +def conditional_cache(func): + """Custom decorator to conditionally apply @functools.cache.""" + if USE_CACHE: + # Apply caching + return functools.cache(func) + else: + # Return the original function without caching + return func + + +class SMLPOperations: + @property + @conditional_cache # @functools.cache + def smlp_true(self): + return smlp.true + + @property + @conditional_cache # @functools.cache + def smlp_false(self): + return smlp.false + + @property + @conditional_cache # @functools.cache + def smlp_real(self): + return smlp.Real + + @property + @conditional_cache # @functools.cache + def smlp_integer(self): + return smlp.Integer + + @conditional_cache # @functools.cache + def smlp_var(self, var): + return smlp.Var(var) + + @conditional_cache # @functools.cache + def smlp_cnst(self, const): + return smlp.Cnst(const) + + # rationals + @conditional_cache # @functools.cache + def smlp_q(self, const): + return smlp.Q(const) + + # reals + @conditional_cache # @functools.cache + def smlp_r(self, const): + return smlp.R(const) + + # logical not (logic negation) + @conditional_cache # @functools.cache + def smlp_not(self, form: smlp.form2): + # res1 = ~form + res2 = op.inv(form) + # assert res1 == res2 + return res2 # ~form + + # logical and (conjunction) + @conditional_cache # @functools.cache + def smlp_and(self, form1: smlp.form2, form2: smlp.form2): + ''' test 83 gets stuck with this simplification + if form1 == smlp.true: + return form2 + if form2 == smlp.true: + return form1 + ''' + res1 = op.and_(form1, form2) + # res2 = form1 & form2 + # print('res1', res1, type(res1)); print('res2', res2, type(res2)) + # assert res1 == res2 + return res1 # form1 & form2 + + # conjunction of possibly more than two formulas + # @functools.cache -- error: unhashable type: 'list' + def smlp_and_multi(self, form_list: list[smlp.form2]): + res = self.smlp_true + ''' + for i, form in enumerate(form_list): + res = form if i == 0 else self.smlp_and(res, form) + ''' + for form in form_list: + res = form if res is self.smlp_true else self.smlp_and(res, form) + return res + + # logical or (disjunction) + @conditional_cache # @functools.cache + def smlp_or(self, form1: smlp.form2, form2: smlp.form2): + res1 = op.or_(form1, form2) + # res2 = form1 | form2 + # assert res1 == res2 + return res1 # form1 | form2 + + # disjunction of possibly more than two formulas + # @functools.cache -- error: unhashable type: 'list' + def smlp_or_multi(self, form_list: list[smlp.form2]): + res = self.smlp_false + ''' + for i, form in enumerate(form_list): + res = form if i == 0 else self.smlp_or(res, form) + ''' + for form in form_list: + res = form if res is self.smlp_false else self.smlp_or(res, form) + return res + + # logical implication + @conditional_cache # @functools.cache + def smlp_implies(self, form1: smlp.form2, form2: smlp.form2): + return self.smlp_or(self.smlp_not(form1), form2) + + # addition + @conditional_cache # @functools.cache + def smlp_add(self, term1: smlp.term2, term2: smlp.term2): + return op.add(term1, term2) + + # sum of possibly more than two formulas + # @functools.cache -- error: unhashable type: 'list' + def smlp_add_multi(self, term_list: list[smlp.term2]): + for i, term in enumerate(term_list): + res = term if i == 0 else self.smlp_add(res, term) + return res + + # subtraction + @conditional_cache # @conditional_cache #@functools.cache + def smlp_sub(self, term1: smlp.term2, term2: smlp.term2): + return op.sub(term1, term2) + + # multiplication + @conditional_cache # @functools.cache + def smlp_mult(self, term1: smlp.term2, term2: smlp.term2): + return op.mul(term1, term2) + + # TODO: !!! check that term2 does not evaluate to term 0 ??? + + # Do this before calling smlp_div, whenver possible? + @conditional_cache # @functools.cache + def smlp_div(self, term1: smlp.term2, term2: smlp.term2): + # return self.smlp_mult(self.smlp_cnst(self.smlp_q(1)) / term2, term1) + return self.smlp_mult(op.truediv(self.smlp_cnst(self.smlp_q(1)), term2), term1) + + @conditional_cache # @functools.cache + def smlp_pow(self, term1: smlp.term2, term2: smlp.term2): + return op.pow(term1, term2) + + # equality + @conditional_cache # @functools.cache + def smlp_eq(self, term1: smlp.term2, term2: smlp.term2): + res1 = op.eq(term1, term2) + # res2 = term1 == term2; print('res1', res1, 'res2', res2) + # assert res1 == res2 + return res1 + + # operator != (not equal) + @conditional_cache # @functools.cache + def smlp_ne(self, term1: smlp.term2, term2: smlp.term2): + res1 = op.ne(term1, term2) + # res2 = term1 != term2; print('res1', res1, 'res2', res2) + # assert res1 == res2 + return res1 + + # operator < + @conditional_cache # @functools.cache + def smlp_lt(self, term1: smlp.term2, term2: smlp.term2): + return op.lt(term1, term2) + + # operator <= + + @conditional_cache # @functools.cache + def smlp_le(self, term1: smlp.term2, term2: smlp.term2): + return op.le(term1, term2) + + # operator > + @conditional_cache # @functools.cache + def smlp_gt(self, term1: smlp.term2, term2: smlp.term2): + return op.gt(term1, term2) + + # operator >= + + @conditional_cache # @functools.cache + def smlp_ge(self, term1: smlp.term2, term2: smlp.term2): + return op.ge(term1, term2) + + # if-thne-else operation + @conditional_cache # @functools.cache + def smlp_ite(self, form: smlp.form2, term1: smlp.term2, term2: smlp.term2): + return smlp.Ite(form, term1, term2) + + # this function performs substitution of variables in term2: + # it substitutes occurrences of the keys in subst_dict with respective values, in term2 term. + # @functools.cache + def smlp_subst(self, term: smlp.term2, subst_dict: dict): + return smlp.subst(term, subst_dict) + + # simplifies a ground term to the respective constant; takes als a s + # @functools.cache + def smlp_cnst_fold(self, term: smlp.term2, subst_dict: dict): + return smlp.cnst_fold(term, subst_dict) + +class ClassProperty: + def __init__(self, fget): + self.fget = fget + + def __get__(self, instance, owner): + return self.fget(owner) + +class PYSMTOperations: + + @ClassProperty + def smlp_true(cls): + return pysmt.shortcuts.TRUE() + + @ClassProperty + def smlp_false(cls): + return pysmt.shortcuts.FALSE() + + @ClassProperty + def smlp_real(cls): + return pysmt.shortcuts.Real + + @ClassProperty + def smlp_integer(cls): + return pysmt.shortcuts.Int + + @conditional_cache # @functools.cache + def smlp_cnst(cls, const): + return pysmt.shortcuts.Real(const) + + # logical not (logic negation) + @conditional_cache # @functools.cache + def smlp_not(cls, form: FNode): + return pysmt.shortcuts.Not(form) + + # logical and (conjunction) + @conditional_cache # @functools.cache + def smlp_and(cls, form1: FNode, form2: FNode): + return pysmt.shortcuts.And(form1, form2) # form1 & form2 + + def smlp_and_multi(cls, form_list: list[FNode]): + return pysmt.shortcuts.And(*form_list) + + # logical or (disjunction) + @conditional_cache # @functools.cache + def smlp_or(cls, form1: FNode, form2: FNode): + return pysmt.shortcuts.Or(form1, form2) + + def smlp_or_multi(cls, form_list: list[FNode]): + return pysmt.shortcuts.Or(*form_list) + + def smlp_eq(self, term1: smlp.term2, term2: smlp.term2): + return pysmt.shortcuts.Equals(term1, term2) diff --git a/src/smlp_py/solver.py b/src/smlp_py/solver.py new file mode 100644 index 00000000..a4316cb5 --- /dev/null +++ b/src/smlp_py/solver.py @@ -0,0 +1,357 @@ +import functools +import types +from abc import ABC, abstractmethod +from enum import Enum + +import pysmt +import smlp +from pysmt.shortcuts import Real + +from src.smlp_py.NN_verifiers.verifiers import MarabouVerifier +from src.smlp_py.smtlib.text_to_sympy import TextToPysmtParser +import operator as op +from pysmt.shortcuts import Symbol, And +from src.smlp_py.smlp_operations import SMLPOperations, PYSMTOperations + +class ClassProperty: + def __init__(self, fget): + self.fget = fget + + def __get__(self, instance, owner): + return self.fget(owner) + +class AbstractSolver(ABC): + + # @abstractmethod + # def true(self): + # pass + + # @abstractmethod + # def GE(self, *args, **kwargs): + # pass + + # @abstractmethod + # def LE(self, *args, **kwargs): + # pass + + @abstractmethod + def create_query(self, *args, **kwargs): + pass + + @abstractmethod + def create_query_and_beta(self, *args, **kwargs): + pass + + @abstractmethod + def substitute_objective_with_witness(self, *args, **kwargs): + pass + + @abstractmethod + def generate_rad_term(self, *args, **kwargs): + pass + + @abstractmethod + def create_theta_form(self, *args, **kwargs): + pass + + @abstractmethod + def get_rad_term(self, *args, **kwargs): + pass + + @abstractmethod + def create_alpha_or_eta_form(self, *args, **kwargs): + pass + + @abstractmethod + def parse_ast(self, *args, **kwargs): + pass + + @abstractmethod + def create_solver(self, *args, **kwargs): + pass + + @abstractmethod + def add_formula(self, *args, **kwargs): + pass + + @abstractmethod + def check(self, *args, **kwargs): + pass + + + +class Pysmt_Solver(AbstractSolver, PYSMTOperations): + verifier = None + + def __init__(self, specs): + super().__init__() + self.specs = specs + self.create_verifier() + + def create_verifier(self): + symbols = [] + feat_names, resp_names, spec_domain_dict = self.specs + + for feature in feat_names: + type = spec_domain_dict[feature]['range'] + symbols.append((feature, type, True)) + + for response in resp_names: + type = spec_domain_dict[response]['range'] + symbols.append((response, type, False)) + + parser = TextToPysmtParser() + parser.init_variables(symbols=symbols) + + self.verifier = MarabouVerifier(parser=parser) + self.verifier.initialize(spec_domain_dict) + + @ClassProperty + def smlp_true(self): + return pysmt.shortcuts.TRUE() + + def smlp_var(self, var): + return self.verifier.parser.get_symbol(var) + + def create_query(self, query_form=None): + self.verifier.parser.handle_ite_formula(query_form, is_form2=True) + + def create_query_and_beta(self, query, beta): + return self.verifier.parser.and_(query, beta) + + def substitute_objective_with_witness(self, *args, **kwargs): + stable_witness_terms = kwargs["stable_witness_terms"] + objv_term = kwargs["objv_term"] + + substitution = {} + for symbol, value in stable_witness_terms.items(): + symbol = self.verifier.parser.get_symbol(symbol) + substitution[symbol] = Real(value) + # Apply the substitution + return self.verifier.parser.simplify(objv_term.substitute(substitution)) + + def generate_rad_term(self, **kwargs): + rad = kwargs["rad"] + return float(rad) + + def get_rad_term(self, **kwargs): + rad = kwargs["rad"] + return float(rad) + + def create_theta_form(self, **kwargs): + witness = kwargs["witness"] + var = kwargs["var"] + rad_term = kwargs["rad_term"] + theta_form = kwargs["theta_form"] + + value = float(witness) + PYSMT_var = self.verifier.parser.get_symbol(var) + type = pysmt.shortcuts.Int if str(PYSMT_var.get_type()) == "Int" else Real + calc_type = int if str(PYSMT_var.get_type()) == "Int" else float + lower = calc_type(value - rad_term) + lower = type(lower) + upper = calc_type(value + rad_term) + upper = type(upper) + theta_form = self.verifier.parser.and_(theta_form, PYSMT_var >= lower, PYSMT_var <= upper) + return theta_form + + def create_alpha_or_eta_form(self, **kwargs): + alpha_or_eta_form = kwargs["alpha_or_eta_form"] + mx = kwargs["mx"] + mn = kwargs["mn"] + v = kwargs["v"] + + symbol_v = self.smlp_var(v) + form = self.smlp_and(symbol_v >= mn, symbol_v <= mx) + return self.simplify(self.smlp_and(alpha_or_eta_form, form)) + + def simplify(self, expression): + return self.verifier.parser.simplify(expression) + + def parse(self, expression): + return self.verifier.parser.parse(expression) + + def GE(self, *args): + return args[0] >= args[1] + + def parse_ast(self, *args, **kwargs): + expression = kwargs['expression'] + return self.parse(expression) + + def create_solver(self, *args, **kwargs): + self.verifier.reset() + return self + + def add_formula(self, formula): + self.verifier.apply_restrictions(formula) + + def check(self, *args, **kwargs): + + + +class Form2_Solver(AbstractSolver, SMLPOperations): + verifier = None + smlp_term_instance = None + terms = None + + def __init__(self): + super().__init__() + # self.verifier = verifier + # self.smlp_term_instance = smlp_term_instance + + @property + def smlp_true(self): + return smlp.true + + def create_query(self, query_form=None): + return query_form + + def create_query_and_beta(self, query, beta): + return self.smlp_term_instance.smlp_and(query, beta) + + def substitute_objective_with_witness(self, *args, **kwargs): + stable_witness_terms = kwargs["stable_witness_terms"] + objv_term = kwargs["objv_term"] + + return smlp.subst(objv_term, stable_witness_terms); + + def generate_rad_term(self, *args, **kwargs): + rad = kwargs["rad"] + delta_rel = kwargs["delta_rel"] + var_term = kwargs["var_term"] + candidate = kwargs["candidate"] + + rad_term = smlp.Cnst(rad) + if delta_rel is not None: # radius for a lemma -- cex holds values of candidate counter-example + rad_term = rad_term * abs(var_term) + else: # radius for excluding a candidate -- cex holds values of the candidate + rad_term = rad_term * abs(candidate) + + return rad_term + + def create_theta_form(self, *args, **kwargs): + theta_form = kwargs["theta_form"] + witness = kwargs["witness"] + var_term = kwargs["var_term"] + rad_term = kwargs["rad_term"] + smlp_and_ = kwargs["smlp_and_"] + + return smlp_and_(theta_form, ((abs(var_term - witness)) <= rad_term)) + + def get_rad_term(self, *args, **kwargs): + smlp_cnst = kwargs["smlp_cnst"] + rad = kwargs["rad"] + + return smlp_cnst(rad) + + def create_alpha_or_eta_form(self, *args, **kwargs): + alpha_or_eta_form = kwargs["alpha_or_eta_form"] + is_in_spec = kwargs["is_in_spec"] + is_disjunction = kwargs["is_disjunction"] + is_alpha = kwargs["is_alpha"] + mx = kwargs["mx"] + mn = kwargs["mn"] + v = kwargs["v"] + + + if is_disjunction and is_alpha and is_in_spec: + rng = self.smlp_or_multi([self.smlp_eq(self.smlp_var(v), self.smlp_cnst(i)) for i in range(mn, mx + 1)]) + else: + rng = self.smlp_and(self.smlp_var(v) >= self.smlp_cnst(mn), self.smlp_var(v) <= self.smlp_cnst(mx)) + + return self.smlp_and(alpha_or_eta_form, rng) + + def GE(self, *args): + return args[0] >= args[1] + + def parse_ast(self, *args, **kwargs): + expression = kwargs['expression'] + parser = kwargs['parser'] + return parser(expression) + + def create_solver(self, *args, **kwargs): + create_solver = kwargs["create_solver"] + domain = kwargs["domain"] + model_full_term_dict = kwargs["model_full_term_dict"] + incremental = kwargs["incremental"] + solver_logic = kwargs["solver_logic"] + formulas = kwargs["formulas"] + + self.verifier = create_solver(domain, model_full_term_dict, incremental, solver_logic) + return self + + def add_formula(self, formula): + self.verifier.add(formula) + + +class Solver: + class Version(Enum): + FORM2 = 0 + PYSMT = 1 + + _instance = None + version = None + + def __new__(cls, *args, **kwargs): + version = kwargs["version"] + if isinstance(version, cls.Version): + cls.version = version + else: + raise ValueError("Must be a valid version") + + if cls._instance is None and isinstance(cls.version, cls.Version): + if cls.version == cls.Version.PYSMT: + specs = kwargs["specs"] + cls._instance = Pysmt_Solver(specs) + else: + cls._instance = Form2_Solver() + cls._map_instance_methods() + return cls._instance + + @classmethod + def _map_instance_methods(cls): + """Automatically maps all methods from the instance to the SingletonFactory class.""" + # for name, method in cls._instance.__class__.__dict__.items(): + # if isinstance(method, types.FunctionType): + # # Avoid overwriting existing methods in Solver class if not hasattr(cls, name): + # setattr(cls, name, cls._create_delegator(name)) + for base_class in cls._instance.__class__.__mro__: + for name, method in base_class.__dict__.items(): + if isinstance(method, types.FunctionType): + # Avoid overwriting existing methods in Solver classifnothasattr(cls, name): + setattr(cls, name, cls._create_delegator(name)) + + @classmethod + def _create_delegator(cls, method_name): + """Create a method that delegates the call to the _instance.""" + def delegator(*args, **kwargs): + return getattr(cls._instance, method_name)(*args, **kwargs) + return delegator + + + + + # + # @classmethod + # def _map_instance_properties(cls): + # """Automatically maps all properties from the instance to the Solver class.""" + # for name, attribute in cls._instance.__class__.__dict__.items(): + # if isinstance(attribute, property): + # # Map property to Solver class + # if not hasattr(cls, name): + # setattr(cls, name, cls._create_property_delegator(name)) + # + # @classmethod + # def _create_property_delegator(cls, property_name): + # """Create a property that delegates access to the _instance.""" + # def getter(self): + # return getattr(self._instance, property_name) + # + # def setter(self, value): + # setattr(self._instance, property_name, value) + # + # def deleter(self): + # delattr(self._instance, property_name) + # + # # Return a property with the mapped getter, setter, and deleter + # return property(getter, setter, deleter) From db0e37c049f753cca6c5f44d37ee00e8fb150619 Mon Sep 17 00:00:00 2001 From: konstantopoulos-tetractys <37417801+ntinouldinho@users.noreply.github.com> Date: Tue, 20 Aug 2024 11:54:24 +0100 Subject: [PATCH 23/28] move operations to external file, integrate solver in workflow --- src/smlp_py/NN_verifiers/verifiers.py | 9 +- src/smlp_py/smlp_flows.py | 10 +- src/smlp_py/smlp_operations.py | 8 + src/smlp_py/smlp_optimize.py | 30 +- src/smlp_py/smlp_query.py | 144 ++++---- src/smlp_py/smlp_solver.py | 4 + src/smlp_py/smlp_terms.py | 507 +++++++++----------------- src/smlp_py/smtlib/text_to_sympy.py | 2 + src/smlp_py/solver.py | 128 ++++++- 9 files changed, 408 insertions(+), 434 deletions(-) diff --git a/src/smlp_py/NN_verifiers/verifiers.py b/src/smlp_py/NN_verifiers/verifiers.py index 8e15f50b..871d8d1a 100755 --- a/src/smlp_py/NN_verifiers/verifiers.py +++ b/src/smlp_py/NN_verifiers/verifiers.py @@ -83,7 +83,6 @@ def __init__(self, parser=None, variable_ranges=None, is_temp=False): self.unscaled_variables = [] self.model_file_path = "./" - self.log_path = "marabou.log" self.data_bounds_file = self.find_file_path("../../../result/abc_smlp_toy_basic_data_bounds.json") self.data_bounds = None # Adds conjunction of equations between bounds in form: @@ -168,7 +167,7 @@ def create_variables(self, is_input=True, is_temp=False): for var in store: name, type = var var_type = Variable.Type.Real if type.lower() == "real" else Variable.Type.Int - if name.startswith(('x', 'p', 'y')): + if name.startswith(('x', 'p', 'y')) and name.find("_scaled") == -1: index = self.input_index if is_input else self.output_index self.variables.append(Variable(var_type, name=name, index=index, is_input=is_input)) @@ -200,6 +199,8 @@ def add_unscaled_variables(self): def convert_scaled_unscaled(self): for scaled_var, unscaled_var in zip(self.variables, self.unscaled_variables): + if scaled_var.name.find("_scaled") != -1: + continue bounds = self.data_bounds[scaled_var.name] min_value, max_value = bounds["min"], bounds["max"] @@ -222,8 +223,12 @@ def convert_scaled_unscaled(self): def get_variable_by_name(self, name: str) -> Optional[Tuple[Variable, int]]: is_output = name.startswith("y") is_unscaled = name.find("_unscaled") != -1 + is_scaled = name.find("_scaled") != -1 repository = self.unscaled_variables if is_unscaled else self.variables + if is_scaled: + return None + for index, variable in enumerate(repository): if variable.name == name: if is_unscaled: diff --git a/src/smlp_py/smlp_flows.py b/src/smlp_py/smlp_flows.py index 50c1c0f0..a97264e4 100644 --- a/src/smlp_py/smlp_flows.py +++ b/src/smlp_py/smlp_flows.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # This file is part of smlp. +import time # imports from SMLP modules from smlp_py.smlp_logs import SmlpLogger, SmlpTracer @@ -121,6 +122,8 @@ def __init__(self, argv): self.optInst.set_tracer(self.tracer, self.args.trace_runtime, self.args.trace_precision, self.args.trace_anonymize) self.queryInst.set_lemma_precision(self.args.lemma_precision) + + self.use_pysmt = self.args.use_pysmt # TODO !!!: is this the right place to define data_fname and new_data_fname and error_file ??? @@ -349,6 +352,8 @@ def smlp_flow(self): args.approximate_fractions, args.fraction_precision, self.dataInst.data_bounds_file, bounds_factor=None, T_resp_bounds_csv_path=None) elif args.analytics_mode == 'optimize': + start = time.time() + use_pysmt = args.use_pysmt self.optInst.smlp_optimize(syst_expr_dict, args.model, model, self.dataInst.unscaled_training_features, self.dataInst.unscaled_training_responses, model_features_dict, feat_names, resp_names, objv_names, objv_exprs, args.optimize_pareto, @@ -357,8 +362,9 @@ def smlp_flow(self): args.solver_logic, args.vacuity_check, args.data_scaler, args.scale_features, args.scale_responses, args.scale_objectives, args.approximate_fractions, args.fraction_precision, - self.dataInst.data_bounds_file, bounds_factor=None, T_resp_bounds_csv_path=None) - + self.dataInst.data_bounds_file, bounds_factor=None, T_resp_bounds_csv_path=None, use_pysmt=use_pysmt) + end = time.time() + print(f"TOTAL TIME IS {end-start}") #self.logger.info('self.optInst.best_config_dict {}'.format(str(self.optInst.best_config_dict))) if syst_expr_dict is not None: if 'final' in self.optInst.best_config_dict: diff --git a/src/smlp_py/smlp_operations.py b/src/smlp_py/smlp_operations.py index 897e799a..02ab0675 100644 --- a/src/smlp_py/smlp_operations.py +++ b/src/smlp_py/smlp_operations.py @@ -232,6 +232,8 @@ def smlp_integer(cls): @conditional_cache # @functools.cache def smlp_cnst(cls, const): + if isinstance(const, FNode): + return const return pysmt.shortcuts.Real(const) # logical not (logic negation) @@ -257,3 +259,9 @@ def smlp_or_multi(cls, form_list: list[FNode]): def smlp_eq(self, term1: smlp.term2, term2: smlp.term2): return pysmt.shortcuts.Equals(term1, term2) + + def smlp_q(self, const): + return pysmt.shortcuts.Real(const) + + def smlp_mult(self, *args): + return pysmt.shortcuts.Times(*args) diff --git a/src/smlp_py/smlp_optimize.py b/src/smlp_py/smlp_optimize.py index b4b9db12..341aee8b 100755 --- a/src/smlp_py/smlp_optimize.py +++ b/src/smlp_py/smlp_optimize.py @@ -15,6 +15,7 @@ import numpy as np from src.smlp_py.smtlib.text_to_sympy import TextToPysmtParser from pysmt.shortcuts import Real +from smlp_py.solver import Solver # single or multi-objective optimization, with stability constraints and any user # given constraints on free input, control (knob) and output variables satisfied. @@ -48,7 +49,7 @@ def __init__(self): self._DEF_OBJECTIVES_EXPRS = None self._DEF_APPROXIMATE_FRACTIONS:bool = True self._DEF_FRACTION_PRECISION:int = 64 - self._ENABLE_PYSMT = True + self._ENABLE_PYSMT = False # Formulae alpha, beta, eta are used in single and pareto optimization tasks. # They are used to constrain control variables x and response variables y as follows: @@ -267,10 +268,13 @@ def optimize_single_objective(self, model_full_term_dict:dict, objv_name:str, ob #quer_form = objv_term > smlp.Cnst(T) quer_form = objv_term >= smlp.Cnst(T) quer_form = self._modelTermsInst.verifier.parser.handle_ite_formula(quer_form, is_form2=True) if self._ENABLE_PYSMT else quer_form + # quer_form = solver.create_query() + quer_expr = '{} >= {}'.format(objv_expr, str(T)) if objv_expr is not None else None quer_name = objv_name + '_' + str(T) if not beta == smlp.true: quer_and_beta = self._modelTermsInst.verifier.parser.and_(quer_form, beta) if self._ENABLE_PYSMT else self._smlpTermsInst.smlp_and(quer_form, beta) + # quer_and_beta = solver.create_query_and_beta(quer_form, beta) else: quer_and_beta = quer_form #print('quer_and_beta', quer_and_beta) 'u0_l0_u_l_T' @@ -447,8 +451,6 @@ def optimize_single_objectives(self, feat_names:list, resp_names:list, #X:pd.Dat objv_terms_dict, orig_objv_terms_dict, scaled_objv_terms_dict = \ self._modelTermsInst.compute_objectives_terms(objv_names, objv_exprs, objv_bounds_dict, scale_objectives) - pysmt_objv_terms_dict, pysmt_orig_objv_terms_dict, pysmt_scaled_objv_terms_dict = \ - self._modelTermsInst.pysmt_compute_objectives_terms(objv_names, objv_exprs, objv_bounds_dict, scale_objectives) # TODO: set sat_approx to False once dump and load with Fractions will work opt_conf = {} for i, (objv_name, objv_term) in enumerate(list(objv_terms_dict.items())): @@ -798,14 +800,15 @@ def sanity_check_fixed_objv_thresholds(t:list[float], fixed_onjv_dict): self._opt_logger.info('Checking whether to fix objective {} at threshold {}...\n'.format(str(j), str(s[j]))) self._opt_tracer.info('activity check, objective {} threshold {}'.format(str(objv_names[j]), str(s[j]))) #print('objv_terms_dict', objv_terms_dict) - quer_form = pysmt.shortcuts.TRUE() if self._ENABLE_PYSMT else smlp.true + # quer_form = pysmt.shortcuts.TRUE() if self._ENABLE_PYSMT else smlp.true + quer_form = Solver._instance.smlp_true for i in objv_enum: #print('obv i', list(objv_terms_dict.keys())[i]) - if self._ENABLE_PYSMT: - quer_form = self._modelTermsInst.parser.and_(quer_form, - list(pysmt_objv_terms_dict.values())[i] > pysmt.shortcuts.Real(t[i])) - else: - quer_form = self._smlpTermsInst.smlp_and(quer_form, list(objv_terms_dict.values())[i] > smlp.Cnst(t[i])) + # if self._ENABLE_PYSMT: + # quer_form = self._modelTermsInst.parser.and_(quer_form, + # list(pysmt_objv_terms_dict.values())[i] > pysmt.shortcuts.Real(t[i])) + # else: + quer_form = self._smlpTermsInst.smlp_and(quer_form, list(objv_terms_dict.values())[i] > Solver.smlp_cnst(t[i])) #print('queryform', quer_form) if not beta == smlp.true: quer_and_beta = self._modelTermsInst.verifier.parser.and_(quer_form, @@ -902,7 +905,7 @@ def smlp_optimize(self, syst_expr_dict:dict, algo:str, model:dict, X:pd.DataFram quer_names:list[str], quer_exprs, delta:float, epsilon:float, alph_expr:str, beta_expr:str, eta_expr:str, theta_radii_dict:dict, solver_logic:str, vacuity:bool, data_scaler:str, scale_feat:bool, scale_resp:bool, scale_objv:bool, - float_approx=True, float_precision=64, data_bounds_json_path=None, bounds_factor=None, T_resp_bounds_csv_path=None): + float_approx=True, float_precision=64, data_bounds_json_path=None, bounds_factor=None, T_resp_bounds_csv_path=None, use_pysmt=False): self.objv_names = objv_names self.objv_exprs = objv_exprs self.feat_names = feat_names @@ -912,7 +915,12 @@ def smlp_optimize(self, syst_expr_dict:dict, algo:str, model:dict, X:pd.DataFram # output to user initial values of mode status with open(self.optimization_results_file+'.json', 'w') as f: json.dump(self.mode_status_dict, f, indent='\t', cls=np_JSONEncoder) - + + # initiliase Solver + solver = Solver(specs=(feat_names, resp_names, self._modelTermsInst._specInst.get_spec_domain_dict), version=Solver.Version.PYSMT if use_pysmt else Solver.Version.FORM2) + + + domain, syst_term_dict, model_full_term_dict, eta, alpha, beta, interface_consistent, model_consistent = \ self._modelTermsInst.create_model_exploration_base_components( syst_expr_dict, algo, model, model_features_dict, feat_names, resp_names, diff --git a/src/smlp_py/smlp_query.py b/src/smlp_py/smlp_query.py index ff122475..f9c135ca 100644 --- a/src/smlp_py/smlp_query.py +++ b/src/smlp_py/smlp_query.py @@ -9,6 +9,9 @@ from smlp_py.smlp_utils import np_JSONEncoder #, str_to_bool from src.smlp_py.NN_verifiers.verifiers import MarabouVerifier +from smlp_py.solver import Solver + +from smlp_py.solver import Pysmt_Solver class SmlpQuery: @@ -44,7 +47,7 @@ def __init__(self): self._trace_runtime = None self._trace_precision = None self._trace_anonymize = None - self._ENABLE_PYSMT = True + self._ENABLE_PYSMT = False def set_logger(self, logger): self._query_logger = logger @@ -100,18 +103,11 @@ def synthesis_results_file(self): return self.report_file_prefix + '_synthesize_results.json' def find_candidate(self, solver): - #res = solver.check() - if self._ENABLE_PYSMT: - print("PYSMT LOOKING FOR CANDIDATE") - res, witness = solver.solve() - return witness + res, witness = self._modelTermsInst.smlp_solver_check(solver, 'ca', self._lemma_precision) + if res == "unknown": + return None else: - print("FORM2 LOOKING FOR CANDIDATE") - res = self._modelTermsInst.smlp_solver_check(solver, 'ca', self._lemma_precision) - if self._modelTermsInst.solver_status_unknown(res): # isinstance(res, smlp.unknown): - return None - else: - return res + return res, witness def update_consistecy_results(self, mode_status_dict, interface_consistent, model_consistent, mode_status, mode_results_file): @@ -180,26 +176,23 @@ def check_concrete_witness_consistency(self, domain:smlp.domain, model_full_term def find_candidate_counter_example(self, universal, domain:smlp.domain, cand:dict, query:smlp.form2, model_full_term_dict:dict, alpha:smlp.form2, theta_radii_dict:dict, solver_logic:str): #, beta:smlp.form2 - theta = self._modelTermsInst.compute_stability_formula_theta(cand, None, theta_radii_dict, universal) - if self._ENABLE_PYSMT: - solver = MarabouVerifier(parser=self._modelTermsInst.parser, variable_ranges=self._modelTermsInst.verifier.variable_ranges, is_temp=True) - # self._modelTermsInst.verifier.reset() - solver.apply_restrictions(theta) - solver.apply_restrictions(alpha) - negation = solver.parser.propagate_negation(query) - z3_equiv = solver.parser.handle_ite_formula(negation, handle_ite=False) - solver.apply_restrictions(negation, need_simplification=True) - print('PYSMT FORMULA',{'alpha': alpha, 'theta': theta, 'not_query': negation.serialize()}) - res, witness = solver.solve() - return witness - else: - solver = self._modelTermsInst.create_model_exploration_instance_from_smlp_components( - domain, model_full_term_dict, False, solver_logic) - solver.add(theta); #print('adding theta', theta) - solver.add(alpha); #print('adding alpha', alpha) - solver.add(self._smlpTermsInst.smlp_not(query)); #print('adding negated quert', query) - return self._modelTermsInst.smlp_solver_check(solver, 'ce', self._lemma_precision, {'alpha':alpha, 'theta':theta,'not_query':self._smlpTermsInst.smlp_not(query)}) - #return solver.check() + theta = self._modelTermsInst.compute_stability_formula_theta(cand, None, theta_radii_dict, universal) + solver = Solver.create_counter_example( + create_solver=self._modelTermsInst.create_model_exploration_instance_from_smlp_components, + domain=domain, + model_full_term_dict=model_full_term_dict, + incremental=False, + solver_logic=solver_logic, + formulas=[alpha, theta], + query=query + ) + + + + return self._modelTermsInst.smlp_solver_check(solver, 'ce', self._lemma_precision, + {'alpha': alpha, 'theta': theta, + 'not_query': self._smlpTermsInst.smlp_not(query)}, temp=True) + # Enhancement !!!: at least add here the delta condition def generalize_counter_example(self, coex): @@ -546,22 +539,38 @@ def query_condition(self, universal, model_full_term_dict:dict, quer_name:str, q else: self._query_logger.info('Querying condition {} <-> {}'.format(str(quer_name), str(quer))) #print('query', quer, 'eta', eta, 'delta', delta) - if not self._ENABLE_PYSMT: - candidate_solver = self._modelTermsInst.create_model_exploration_instance_from_smlp_components( - domain, model_full_term_dict, True, solver_logic) - - # add the remaining user constraints and the query - candidate_solver.add(eta) - candidate_solver.add(alpha) - #candidate_solver.add(beta) - candidate_solver.add(quer) - else: - self._modelTermsInst.verifier.reset() - self._modelTermsInst.verifier.apply_restrictions(eta, need_simplification=True) - self._modelTermsInst.verifier.apply_restrictions(alpha) - self._modelTermsInst.verifier.apply_restrictions(quer, need_simplification=True) - #print('eta', eta); print('alpha', alpha); print('quer', quer); + # if not self._ENABLE_PYSMT: + # candidate_solver = self._modelTermsInst.create_model_exploration_instance_from_smlp_components( + # domain, model_full_term_dict, True, solver_logic) + # + # # add the remaining user constraints and the query + # candidate_solver.add(eta) + # candidate_solver.add(alpha) + # #candidate_solver.add(beta) + # candidate_solver.add(quer) + # else: + # self._modelTermsInst.verifier.reset() + # self._modelTermsInst.verifier.apply_restrictions(eta, need_simplification=True) + # self._modelTermsInst.verifier.apply_restrictions(alpha) + # self._modelTermsInst.verifier.apply_restrictions(quer, need_simplification=True) + + candidate_solver = Solver.create_solver( + create_solver=self._modelTermsInst.create_model_exploration_instance_from_smlp_components, + domain=domain, + model_full_term_dict=model_full_term_dict, + incremental=True, + solver_logic=solver_logic + ) + candidate_solver.add_formula(eta, need_simplification=True) + candidate_solver.add_formula(alpha) + candidate_solver.add_formula(quer, need_simplification=True) + + # res = self.smlp_solver_check(solver, + # 'interface_consistency' if model_full_term_dict is None else 'model_consistency', + # equations={'alpha': alpha, 'eta': eta}) + + #print('eta', eta); print('alpha', alpha); print('quer', quer); #print('solving query', quer) self._query_tracer.info('{},{}'.format('synthesis' if universal else 'query', str(quer_name))) #, str(quer_expr) ,{} use_approxiamted_fractions = self._lemma_precision != 0 @@ -571,21 +580,22 @@ def query_condition(self, universal, model_full_term_dict:dict, quer_name:str, q while True: # solve Ex. eta x /\ Ay. theta x y -> alpha y -> (beta y /\ query) print('searching for a candidate', flush=True) - if self._ENABLE_PYSMT: + if isinstance(candidate_solver, Pysmt_Solver): print('PYSMT FORMULA', {'alpha': alpha, 'eta': eta, 'quer': quer.serialize()}) else: print('FORM2 FORMULA', {'alpha': alpha, 'eta': eta, 'quer': quer}) - ca = self.find_candidate(self._modelTermsInst.verifier) if self._ENABLE_PYSMT else self.find_candidate(candidate_solver) + result, ca = self.find_candidate(candidate_solver) - condition_sat = self._modelTermsInst.solver_status_sat(ca["result"]) if self._ENABLE_PYSMT else self._modelTermsInst.solver_status_sat(ca) - condition_unsat = self._modelTermsInst.solver_status_unsat(ca["result"]) if self._ENABLE_PYSMT else self._modelTermsInst.solver_status_unsat(ca) - condition_unknown = self._modelTermsInst.solver_status_unsat(ca["result"]) if self._ENABLE_PYSMT else self._modelTermsInst.solver_status_unknown(ca) + # condition_sat = self._modelTermsInst.solver_status_sat(ca) + # condition_unsat = self._modelTermsInst.solver_status_unsat(ca["result"]) if self._ENABLE_PYSMT else self._modelTermsInst.solver_status_unsat(ca) + # condition_unknown = self._modelTermsInst.solver_status_unsat(ca["result"]) if self._ENABLE_PYSMT else self._modelTermsInst.solver_status_unknown(ca) - if condition_sat: # isinstance(ca, smlp.sat): + if result == "sat": # isinstance(ca, smlp.sat): print('candidate found -- checking stability', flush=True) #print('ca', ca_model) - ca_model = self._modelTermsInst.get_solver_model(ca) #ca.model + # ca_model = self._modelTermsInst.get_solver_model(ca) #ca.model + ca_model = Solver.get_witness(result=result, witness=ca, interface=self._modelTermsInst._specInst.get_spec_interface) #ca.model if use_approxiamted_fractions: ca_model_approx = self._smlpTermsInst.approximate_witness_term(ca_model, self._lemma_precision) #print('ca_model_approx -------------', ca_model_approx) @@ -600,18 +610,21 @@ def query_condition(self, universal, model_full_term_dict:dict, quer_name:str, q #print('ca_model_approx', ca_model_approx) feasible = True if use_approxiamted_fractions: - ce = self.find_candidate_counter_example(universal, domain, ca_model_approx, quer, model_full_term_dict, alpha, + c_result, ce = self.find_candidate_counter_example(universal, domain, ca_model_approx, quer, model_full_term_dict, alpha, theta_radii_dict, solver_logic) else: - ce = self.find_candidate_counter_example(universal, domain, ca_model, quer, model_full_term_dict, alpha, + c_result, ce = self.find_candidate_counter_example(universal, domain, ca_model, quer, model_full_term_dict, alpha, theta_radii_dict, solver_logic) - is_sat = self._modelTermsInst.solver_status_sat(ce["result"]) if self._ENABLE_PYSMT else self._modelTermsInst.solver_status_sat(ce) - is_unsat = self._modelTermsInst.solver_status_unsat(ce["result"]) if self._ENABLE_PYSMT else self._modelTermsInst.solver_status_unsat(ce) + # is_sat = self._modelTermsInst.solver_status_sat(ce["result"]) if self._ENABLE_PYSMT else self._modelTermsInst.solver_status_sat(ce) + # is_unsat = self._modelTermsInst.solver_status_unsat(ce["result"]) if self._ENABLE_PYSMT else self._modelTermsInst.solver_status_unsat(ce) - if is_sat: #isinstance(ce, smlp.sat): + if c_result == "sat": #isinstance(ce, smlp.sat): print('candidate not stable -- continue search', flush=True) - ce_model = self._modelTermsInst.get_solver_model(ce) #ce.model + ce_model = Solver.get_witness(result=result, witness=ca, + interface=self._modelTermsInst._specInst.get_spec_interface) # ca.model + + cem = ce_model.copy(); #print('ce model', cem) # drop Assignements to responses from ce for var in ce_model.keys(): @@ -642,11 +655,14 @@ def query_condition(self, universal, model_full_term_dict:dict, quer_name:str, q candidate_solver.add(self._smlpTermsInst.smlp_not(theta)) print("FORM2 THETA ADDED ", self._smlpTermsInst.smlp_not(theta)) continue - elif is_unsat: #isinstance(ce, smlp.unsat): + elif c_result == "unsat": #isinstance(ce, smlp.unsat): #print('candidate stable -- return candidate') self._query_logger.info('Query completed with result: STABLE_SAT (satisfiable)') if witn: # export witness (use numbers as values, not terms) - ca_model = self._modelTermsInst.get_solver_model(ca) # ca.model + ca_model = Solver.get_witness(result=result, witness=ca, + interface=self._modelTermsInst._specInst.get_spec_interface) # ca.model + + witness_vals_dict = self._smlpTermsInst.witness_term_to_const(ca_model, sat_approx, sat_precision) #print('domain witness_vals_dict', witness_vals_dict) # sanity check: the value of query in the sat assignment should be true @@ -659,14 +675,14 @@ def query_condition(self, universal, model_full_term_dict:dict, quer_name:str, q return {'query_status':'STABLE_SAT', 'witness':ca['witness'], 'feasible':feasible} else: return {'query_status':'STABLE_SAT', 'witness':ca_model, 'feasible':feasible} - elif condition_unsat: #isinstance(ca, smlp.unsat): + elif result == "unsat": #isinstance(ca, smlp.unsat): self._query_logger.info('Query completed with result: UNSAT (unsatisfiable)') if feasible is None: feasible = False #print('candidate does not exist -- query unsuccessful') #print('query unsuccessful: witness does not exist (query is unsat)') return {'query_status':'UNSAT', 'witness':None, 'feasible':feasible} - elif condition_unknown: #isinstance(ca, smlp.unknown): + elif result == "unknown": #isinstance(ca, smlp.unknown): self._opt_logger.info('Completed with result: {}'.format('UNKNOWN')) return {'query_status':'UNKNOWN', 'witness':None, 'feasible':feasible} #raise Exception('UNKNOWN return value in candidate search is currently not supported for queries') diff --git a/src/smlp_py/smlp_solver.py b/src/smlp_py/smlp_solver.py index 4528ce49..472874b8 100644 --- a/src/smlp_py/smlp_solver.py +++ b/src/smlp_py/smlp_solver.py @@ -12,6 +12,7 @@ def __init__(self): self._DEF_SOLVER = 'z3' self._DEF_SOLVER_PATH = None self._DEF_SOLVER_LOGIC = 'ALL' + self.use_pysmt = False #self._DEF_SOLVER_INCREMENTAL = True ''' @@ -35,6 +36,9 @@ def __init__(self): 'help':'SMT2-lib theory with respect to which to solve model exploration task at hand, ' + 'in modes "verify," "query", "optimize" and "optsyn". ' + '[default: {}]'.format(str(self._DEF_SOLVER_LOGIC))}, + 'use_pysmt': {'abbr': 'use_pysmt', 'default': self.use_pysmt, 'type': str_to_bool, + 'help': 'Solver to use in model exploration modes "verify," "query", "optimize" and "optsyn". ' + + '[default: {}]'.format(str(self.use_pysmt))}, #'solver_incr': {'abbr':'solver_incr', 'default': self._DEF_SOLVER_INCREMENTAL, 'type':str_to_bool, # 'help':'Should sover be used in incremental mode? ' + # '[default: {}]'.format(str(self._DEF_SOLVER_INCREMENTAL))} diff --git a/src/smlp_py/smlp_terms.py b/src/smlp_py/smlp_terms.py index 621fc026..ebc86fec 100644 --- a/src/smlp_py/smlp_terms.py +++ b/src/smlp_py/smlp_terms.py @@ -25,6 +25,11 @@ import pysmt from src.smlp_py.smtlib.text_to_sympy import TextToPysmtParser +from smlp_py.solver import Solver + +from smlp_py.smlp_operations import SMLPOperations + +from src.smlp_py.solver import Form2_Solver # TODO !!! create a parent class for TreeTerms, PolyTerms, NNKerasTerms. # setting logger, report_file_prefix, model_file_prefix can go to that class to work for all above three classes @@ -54,16 +59,7 @@ # to solver instance separately (as many as required, depending on whether all responses are analysed together). -USE_CACHE = False -def conditional_cache(func): - """Custom decorator to conditionally apply @functools.cache.""" - if USE_CACHE: - # Apply caching - return functools.cache(func) - else: - # Return the original function without caching - return func ''' def conditional_cache(func): @@ -81,7 +77,7 @@ def wrapper(self, *args, **kwargs): # Class SmlpTerms has methods for generating terms, and classes TreeTerms, PolyTerms and NNKerasTerms are inherited # from it but this inheritance is probably not implemented in the best way: TODO !!!: see if that can be improved. -class SmlpTerms: +class SmlpTerms(SMLPOperations): def __init__(self): self._smlp_terms_logger = None self.report_file_prefix = None @@ -131,7 +127,7 @@ def __init__(self): ast.And: self.smlp_and, ast.Or: self.smlp_or, ast.Not: self.smlp_not, ast.IfExp: self.smlp_ite } - self._ENABLE_PYSMT = True + self._ENABLE_PYSMT = False # set logger from a caller script def set_logger(self, logger): @@ -154,190 +150,8 @@ def ast_operators_smlp_map(self): #def set_cache_terms(self, cache_terms): # self._cache_terms = cache_tems - @property - @conditional_cache #@functools.cache - def smlp_true(self): - return smlp.true - - @property - @conditional_cache #@functools.cache - def smlp_false(self): - return smlp.false - - @property - @conditional_cache #@functools.cache - def smlp_real(self): - return smlp.Real - - @property - @conditional_cache #@functools.cache - def smlp_integer(self): - return smlp.Integer - - @conditional_cache #@functools.cache - def smlp_var(self, var): - return smlp.Var(var) - - @conditional_cache #@functools.cache - def smlp_cnst(self, const): - return smlp.Cnst(const) - - # rationals - @conditional_cache #@functools.cache - def smlp_q(self, const): - return smlp.Q(const) - - # reals - @conditional_cache #@functools.cache - def smlp_r(self, const): - return smlp.R(const) - - # logical not (logic negation) - @conditional_cache #@functools.cache - def smlp_not(self, form:smlp.form2): - #res1 = ~form - res2 = op.inv(form) - #assert res1 == res2 - return res2 #~form - - # logical and (conjunction) - @conditional_cache #@functools.cache - def smlp_and(self, form1:smlp.form2, form2:smlp.form2): - ''' test 83 gets stuck with this simplification - if form1 == smlp.true: - return form2 - if form2 == smlp.true: - return form1 - ''' - res1 = op.and_(form1, form2) - #res2 = form1 & form2 - #print('res1', res1, type(res1)); print('res2', res2, type(res2)) - #assert res1 == res2 - return res1 # form1 & form2 - - # conjunction of possibly more than two formulas - #@functools.cache -- error: unhashable type: 'list' - def smlp_and_multi(self, form_list:list[smlp.form2]): - res = self.smlp_true - ''' - for i, form in enumerate(form_list): - res = form if i == 0 else self.smlp_and(res, form) - ''' - for form in form_list: - res = form if res is self.smlp_true else self.smlp_and(res, form) - return res - - # logical or (disjunction) - @conditional_cache #@functools.cache - def smlp_or(self, form1:smlp.form2, form2:smlp.form2): - res1 = op.or_(form1, form2) - #res2 = form1 | form2 - #assert res1 == res2 - return res1 #form1 | form2 - - # disjunction of possibly more than two formulas - #@functools.cache -- error: unhashable type: 'list' - def smlp_or_multi(self, form_list:list[smlp.form2]): - res = self.smlp_false - ''' - for i, form in enumerate(form_list): - res = form if i == 0 else self.smlp_or(res, form) - ''' - for form in form_list: - res = form if res is self.smlp_false else self.smlp_or(res, form) - return res - - # logical implication - @conditional_cache #@functools.cache - def smlp_implies(self, form1:smlp.form2, form2:smlp.form2): - return self.smlp_or(self.smlp_not(form1), form2) - # addition - @conditional_cache #@functools.cache - def smlp_add(self, term1:smlp.term2, term2:smlp.term2): - return op.add(term1, term2) - - # sum of possibly more than two formulas - #@functools.cache -- error: unhashable type: 'list' - def smlp_add_multi(self, term_list:list[smlp.term2]): - for i, term in enumerate(term_list): - res = term if i == 0 else self.smlp_add(res, term) - return res - - # subtraction - @conditional_cache #@conditional_cache #@functools.cache - def smlp_sub(self, term1:smlp.term2, term2:smlp.term2): - return op.sub(term1, term2) - - # multiplication - @conditional_cache #@functools.cache - def smlp_mult(self, term1:smlp.term2, term2:smlp.term2): - return op.mul(term1, term2) - - # TODO: !!! check that term2 does not evaluate to term 0 ??? - # Do this before calling smlp_div, whenver possible? - @conditional_cache #@functools.cache - def smlp_div(self, term1:smlp.term2, term2:smlp.term2): - #return self.smlp_mult(self.smlp_cnst(self.smlp_q(1)) / term2, term1) - return self.smlp_mult(op.truediv(self.smlp_cnst(self.smlp_q(1)), term2), term1) - - @conditional_cache #@functools.cache - def smlp_pow(self, term1:smlp.term2, term2:smlp.term2): - return op.pow(term1, term2) - - # equality - @conditional_cache #@functools.cache - def smlp_eq(self, term1:smlp.term2, term2:smlp.term2): - res1 = op.eq(term1, term2) - #res2 = term1 == term2; print('res1', res1, 'res2', res2) - #assert res1 == res2 - return res1 - - # operator != (not equal) - @conditional_cache #@functools.cache - def smlp_ne(self, term1:smlp.term2, term2:smlp.term2): - res1 = op.ne(term1, term2) - #res2 = term1 != term2; print('res1', res1, 'res2', res2) - #assert res1 == res2 - return res1 - - # operator < - @conditional_cache #@functools.cache - def smlp_lt(self, term1:smlp.term2, term2:smlp.term2): - return op.lt(term1, term2) - - # operator <= - @conditional_cache #@functools.cache - def smlp_le(self, term1:smlp.term2, term2:smlp.term2): - return op.le(term1, term2) - - # operator > - @conditional_cache #@functools.cache - def smlp_gt(self, term1:smlp.term2, term2:smlp.term2): - return op.gt(term1, term2) - - # operator >= - @conditional_cache #@functools.cache - def smlp_ge(self, term1:smlp.term2, term2:smlp.term2): - return op.ge(term1, term2) - - # if-thne-else operation - @conditional_cache #@functools.cache - def smlp_ite(self, form:smlp.form2, term1:smlp.term2, term2:smlp.term2): - return smlp.Ite(form, term1, term2) - - # this function performs substitution of variables in term2: - # it substitutes occurrences of the keys in subst_dict with respective values, in term2 term. - #@functools.cache - def smlp_subst(self, term:smlp.term2, subst_dict:dict): - return smlp.subst(term, subst_dict) - - # simplifies a ground term to the respective constant; takes als a s - #@functools.cache - def smlp_cnst_fold(self, term:smlp.term2, subst_dict:dict): - return smlp.cnst_fold(term, subst_dict) - ''' destruct(e: smlp.libsmlp.form2 | smlp.libsmlp.term2) -> dict Destructure the given term2 or form2 instance `e`. The result is a dict @@ -1483,37 +1297,16 @@ def _unscaled_name(self, name): def feature_scaler_to_term(self, orig_feat_name, scaled_feat_name, orig_min, orig_max): #print('feature_scaler_to_term', 'orig_min', orig_min, type(orig_min), 'orig_max', orig_max, type(orig_max), flush=True) if orig_min == orig_max: - return self.smlp_cnst(0) #smlp.Cnst(0) # same as returning smlp.Cnst(smlp.Q(0)) + return Solver.smlp_cnst(0) #smlp.Cnst(0) # same as returning smlp.Cnst(smlp.Q(0)) else: - return self.smlp_mult( - self.smlp_cnst(self.smlp_q(1) / self.smlp_q(orig_max - orig_min)), - (self.smlp_var(orig_feat_name) - self.smlp_cnst(orig_min))) + return Solver.smlp_mult( + Solver.smlp_cnst(Solver.smlp_q(1) / Solver.smlp_q(orig_max - orig_min)), + (Solver.smlp_var(orig_feat_name) - Solver.smlp_cnst(orig_min))) ####return self.smlp_div(self.smlp_var(orig_feat_name) - self.smlp_cnst(orig_min), self.smlp_cnst(orig_max) - self.smlp_cnst(orig_min)) ####return smlp.Cnst(smlp.Q(1) / smlp.Q(orig_max - orig_min)) * (smlp.Var(orig_feat_name) - smlp.Cnst(orig_min)) - - def pysmt_feature_scaler_to_term(self, orig_feat_var, parser, orig_min, orig_max): - parser.add_symbol(orig_feat_var, 'real') - orig_feat_var = parser.get_symbol(orig_feat_var) - if orig_min == orig_max: - return pysmtReal(0) - else: - scaling_factor = pysmtReal(1) / pysmtReal(orig_max - orig_min) - orig_min_cnst = pysmtReal(orig_min) - - # (orig_feat_var - orig_min_cnst) - scaled_expr = pysmt.shortcuts.Minus(orig_feat_var, orig_min_cnst) - # scaling_factor * (orig_feat_var - orig_min_cnst) - scaled_term = pysmt.shortcuts.Times(scaling_factor, scaled_expr) - return scaled_term - # Computes dictionary with features as keys and scaler terms as values - def feature_scaler_terms(self, data_bounds, feat_names, parser=None): - if parser: - return dict([(self._scaled_name(feat), self.pysmt_feature_scaler_to_term(feat, parser, - data_bounds[feat]['min'], - data_bounds[feat]['max'])) for feat in - feat_names]) + def feature_scaler_terms(self, data_bounds, feat_names): return dict([(self._scaled_name(feat), self.feature_scaler_to_term(feat, self._scaled_name(feat), data_bounds[feat]['min'], data_bounds[feat]['max'])) for feat in feat_names]) @@ -1604,8 +1397,8 @@ def __init__(self): self.verifier = MarabouVerifier(parser=self.parser) - self._ENABLE_PYSMT = True - self._RETURN_PYSMT = True + self._ENABLE_PYSMT = False + self._RETURN_PYSMT = False # set logger from a caller script @@ -1773,7 +1566,8 @@ def _compute_model_terms_dict(self, algo, model, feat_names, resp_names, data_bo if self._tree_encoding == 'flat' and algo in ['dt_sklearn', 'rf_sklearn', 'et_sklearn', 'dt_caret', 'rf_caret', 'et_caret']: # rule_form: model_term = [self.smlp_cnst_fold(form, {feat_name: feat_term}) for form in model_term] else: - model_term = self.smlp_cnst_fold(model_term, {feat_name: feat_term}) #self.smlp_subst + model_term = self.smlp_cnst_fold(model_term, {feat_name: feat_term}) + # model_term = Solver.substitute(var=model_term, substitutions={feat_name: feat_term}) #self.smlp_subst #print('model term after', model_term, flush=True) model_term_dict[resp_name] = model_term #print('model_term_dict', model_term_dict, flush=True) @@ -1878,7 +1672,7 @@ def compute_objectives_terms(self, objv_names, objv_exprs, objv_bounds, scale_ob #print('objv_exprs', objv_exprs) if objv_exprs is None: return None, None, None, None - orig_objv_terms_dict = dict([(objv_name, self.ast_expr_to_term(objv_expr)) \ + orig_objv_terms_dict = dict([(objv_name, Solver.parse_ast(objv_expr)) \ for objv_name, objv_expr in zip(objv_names, objv_exprs)]) #self._smlpTermsInst. #print('orig_objv_terms_dict', orig_objv_terms_dict) @@ -1892,7 +1686,8 @@ def compute_objectives_terms(self, objv_names, objv_exprs, objv_bounds, scale_ob x = list(orig_objv_terms_dict.keys())[i]; #print('x', x); print('arg', orig_objv_terms_dict[x]) - objv_terms_dict[k] = self.smlp_cnst_fold(v, {x: orig_objv_terms_dict[x]}) + objv_terms_dict[k] = Solver.substitute(var=v, substitutions={Solver.smlp_cnst(x): orig_objv_terms_dict[x]}) + # objv_terms_dict[k] = self.smlp_cnst_fold(v, {x: orig_objv_terms_dict[x]}) #objv_terms_dict = scaled_objv_terms_dict else: objv_terms_dict = orig_objv_terms_dict @@ -1954,7 +1749,9 @@ def compute_stability_formula_theta(self, cex, delta_dict:dict, radii_dict, univ else: delta_rel = delta_abs = None - theta_form = pysmt.shortcuts.TRUE() if self._ENABLE_PYSMT else self.smlp_true + # theta_form = pysmt.shortcuts.TRUE() if self._ENABLE_PYSMT else self.smlp_true + theta_form = Solver._instance.smlp_true + #print('radii_dict', radii_dict) radii_dict_local = radii_dict.copy() knobs = radii_dict_local.keys(); #print('knobs', knobs); print('cex', cex); print('delta', delta_dict) @@ -1966,17 +1763,19 @@ def compute_stability_formula_theta(self, cex, delta_dict:dict, radii_dict, univ radii_dict_local[cex_var] = {'rad-abs':0, 'rad-rel': None} # delta for var,radii in radii_dict_local.items(): - var_term = self.smlp_var(var) + var_term = Solver.smlp_var(var) # either rad-abs or rad-rel must be None -- for each var wr declare only one of these if radii['rad-abs'] is not None: rad = radii['rad-abs']; #print('rad', rad); if delta_rel is not None: # we are generating a lemma rad = rad * (1 + delta_rel) + delta_abs - if self._ENABLE_PYSMT: - rad_term = float(rad) - else: - rad_term = self.smlp_cnst(rad) + # if self._ENABLE_PYSMT: + # rad_term = float(rad) + # else: + # rad_term = self.smlp_cnst(rad) + rad_term = Solver.get_rad_term(rad=rad) + elif radii['rad-rel'] is not None: rad = radii['rad-rel']; #print('rad', rad) if delta_rel is not None: # we are generating a lemma @@ -2001,29 +1800,35 @@ def compute_stability_formula_theta(self, cex, delta_dict:dict, radii_dict, univ # from relative radius based on variable values in the counter-exaples to candidate rather than variable values # in the candidate itself. - if self._ENABLE_PYSMT: - rad_term = float(rad) - else: - rad_term = self.smlp_cnst(rad) - if delta_rel is not None: # radius for a lemma -- cex holds values of candidate counter-example - rad_term = rad_term * abs(var_term) - else: # radius for excluding a candidate -- cex holds values of the candidate - rad_term = rad_term * abs(cex[var]) + # if self._ENABLE_PYSMT: + # rad_term = float(rad) + # else: + # rad_term = self.smlp_cnst(rad) + # if delta_rel is not None: # radius for a lemma -- cex holds values of candidate counter-example + # rad_term = rad_term * abs(var_term) + # else: # radius for excluding a candidate -- cex holds values of the candidate + # rad_term = rad_term * abs(cex[var]) + + rad_term = Solver.generate_rad_term(rad=rad, delta_rel=delta_rel, var_term=var_term,candidate=cex[var]) + + elif delta_dict is not None: raise exception('When delta dictionary is provided, either absolute or relative or delta must be specified') - if self._ENABLE_PYSMT: - value = float(cex[var]) - PYSMT_var = self.parser.get_symbol(var) - type = pysmt.shortcuts.Int if str(PYSMT_var.get_type()) == "Int" else pysmtReal - calc_type = int if str(PYSMT_var.get_type()) == "Int" else float - lower = calc_type(value - rad_term) - lower = type(lower) - upper = calc_type(value + rad_term) - upper = type(upper) - theta_form = self.parser.and_(theta_form, PYSMT_var >= lower, PYSMT_var <= upper) - # self.verifier.add_bounds(var, (value - verifier_rad_term, value + verifier_rad_term)) - else: - theta_form = self.smlp_and(theta_form, ((abs(var_term - cex[var])) <= rad_term)) + # if self._ENABLE_PYSMT: + # value = float(cex[var]) + # PYSMT_var = self.parser.get_symbol(var) + # type = pysmt.shortcuts.Int if str(PYSMT_var.get_type()) == "Int" else pysmtReal + # calc_type = int if str(PYSMT_var.get_type()) == "Int" else float + # lower = calc_type(value - rad_term) + # lower = type(lower) + # upper = calc_type(value + rad_term) + # upper = type(upper) + # theta_form = self.parser.and_(theta_form, PYSMT_var >= lower, PYSMT_var <= upper) + # + # else: + # theta_form = self.smlp_and(theta_form, ((abs(var_term - cex[var])) <= rad_term)) + + theta_form = Solver.create_theta_form(theta_form=theta_form, witness=cex[var], var=var, var_term=var_term, rad_term=rad_term) #print('theta_form', theta_form) return theta_form @@ -2032,20 +1837,20 @@ def compute_stability_formula_theta(self, cex, delta_dict:dict, radii_dict, univ # Covers grid as well as range/interval constraints. def compute_grid_range_formulae_eta(self): #print('generate eta constraint') - eta_grid_form = self.smlp_true + eta_grid_form = Solver._instance.smlp_true eta_grids_dict = self._specInst.get_spec_eta_grids_dict; #print('eta_grids_dict', eta_grids_dict) for var,grid in eta_grids_dict.items(): - eta_grid_disj = self.smlp_false - var_term = self.smlp_var(var) + eta_grid_disj = Solver._instance.smlp_false + var_term = Solver.smlp_var(var) for gv in grid: # iterate over grid values - if eta_grid_disj == self.smlp_false: - eta_grid_disj = var_term == self.smlp_cnst(gv) + if eta_grid_disj == Solver._instance.smlp_false: + eta_grid_disj = Solver.smlp_eq(var_term, Solver.smlp_cnst(gv)) else: - eta_grid_disj = self.smlp_or(eta_grid_disj, var_term == self.smlp_cnst(gv)) - if eta_grid_form == self.smlp_true: + eta_grid_disj = Solver.smlp_or(eta_grid_disj, Solver.smlp_eq(var_term, Solver.smlp_cnst(gv))) + if eta_grid_form == Solver._instance.smlp_true: eta_grid_form = eta_grid_disj else: - eta_grid_form = self.smlp_and(eta_grid_form, eta_grid_disj) + eta_grid_form = Solver.smlp_and(eta_grid_form, eta_grid_disj) #print('eta_grid_form', eta_grid_form); return eta_grid_form @@ -2100,8 +1905,11 @@ def compute_input_ranges_formula_alpha(self, model_inputs): return alpha_form def compute_input_ranges_formula_alpha_eta(self, alpha_vs_eta, model_inputs, specs=None): - alpha_or_eta_form = self.smlp_true - smt_form = pysmt.shortcuts.TRUE() + # self.ENABLE_PYSMT + # alpha_or_eta_form = self.smlp_true + # smt_form = pysmt.shortcuts.TRUE() + + alpha_or_eta_form = Solver._instance.smlp_true if alpha_vs_eta == 'alpha': alpha_or_eta_ranges_dict = self._specInst.get_spec_alpha_bounds_dict @@ -2119,35 +1927,36 @@ def compute_input_ranges_formula_alpha_eta(self, alpha_vs_eta, model_inputs, spe #print('mn', mn, 'mx', mx) if mn is not None and mx is not None: if self._declare_domain_interface_only: - if self._ENABLE_PYSMT: - symbol_v = self.parser.get_symbol(v) - form = self.parser.and_(symbol_v >= mn, symbol_v <= mx) - smt_form = self.parser.and_(smt_form, form) - - # self.verifier.add_bounds(v, (mn, mx), num=specs[v]['range']) - - if self._encode_input_range_as_disjunction and alpha_vs_eta == 'alpha' and v in self._specInst.get_spec_inputs: - rng = self.smlp_or_multi([self.smlp_eq(self.smlp_var(v), self.smlp_cnst(i)) for i in range(mn, mx+1)]) - else: - rng = self.smlp_and(self.smlp_var(v) >= self.smlp_cnst(mn), self.smlp_var(v) <= self.smlp_cnst(mx)) - alpha_or_eta_form = self.smlp_and(alpha_or_eta_form, rng) + # if self._ENABLE_PYSMT: + # symbol_v = self.parser.get_symbol(v) + # form = self.parser.and_(symbol_v >= mn, symbol_v <= mx) + # smt_form = self.parser.and_(smt_form, form) + # + # + # if self._encode_input_range_as_disjunction and alpha_vs_eta == 'alpha' and v in self._specInst.get_spec_inputs: + # rng = self.smlp_or_multi([self.smlp_eq(self.smlp_var(v), self.smlp_cnst(i)) for i in range(mn, mx+1)]) + # else: + # rng = self.smlp_and(self.smlp_var(v) >= self.smlp_cnst(mn), self.smlp_var(v) <= self.smlp_cnst(mx)) + # alpha_or_eta_form = self.smlp_and(alpha_or_eta_form, rng) + + alpha_or_eta_form = Solver.create_alpha_or_eta_form( + alpha_or_eta_form=alpha_or_eta_form, + v=v, + mn=mn, + mx=mx, + is_alpha=alpha_vs_eta == 'alpha', + is_in_spec=v in self._specInst.get_spec_inputs, + is_disjunction=self._encode_input_range_as_disjunction + ) elif mn is not None: - rng = self.smlp_var(v) >= self.smlp_cnst(mn) - alpha_or_eta_form = self.smlp_and(alpha_or_eta_form, rng) + rng = Solver.smlp_var(v) >= Solver.smlp_cnst(mn) + alpha_or_eta_form = Solver.smlp_and(alpha_or_eta_form, rng) elif mx is not None: - rng = self.smlp_var(v) <= self.smlp_cnst(mx) - alpha_or_eta_form = self.smlp_and(alpha_or_eta_form, rng) + rng = Solver.smlp_var(v) <= Solver.smlp_cnst(mx) + alpha_or_eta_form = Solver.smlp_and(alpha_or_eta_form, rng) else: assert False - if self._ENABLE_PYSMT: - smt_form = self.parser.simplify(smt_form) - # return self.parser.simplify(smt_form) - if self._RETURN_PYSMT: - return smt_form - else: - print(smt_form) - return alpha_or_eta_form # alph_expr is alpha constraint specified in command line. If it is not None @@ -2161,21 +1970,22 @@ def compute_global_alpha_formula(self, alph_expr, model_inputs): #alph_form = self.compute_input_ranges_formula_alpha(model_inputs) #alph_form = self.smlp_true if alph_expr is None: - alpha_expr = self._specInst.get_spec_alpha_global_expr + alph_expr = self._specInst.get_spec_alpha_global_expr if alph_expr is None: - return self.smlp_true + return Solver._instance.smlp_true else: alph_expr_vars = get_expression_variables(alph_expr) dont_care_vars = list_subtraction_set(alph_expr_vars, model_inputs) if len(dont_care_vars) > 0: raise Exception('Variables ' + str(dont_care_vars) + ' in input constraints (alpha) are not part of the model') - alph_glob = self.ast_expr_to_term(alph_expr) - if self._ENABLE_PYSMT: - if self._RETURN_PYSMT: - return self.parser.parse(alph_expr) - else: - print(self.parser.parse(alph_expr)) + + alph_glob = Solver.parse_ast(parser=self.ast_expr_to_term, expression=alph_expr) + # if self._ENABLE_PYSMT: + # if self._RETURN_PYSMT: + # return self.parser.parse(alph_expr) + # else: + # print(self.parser.parse(alph_expr)) return alph_glob #self._smlpTermsInst.smlp_and(alph_form, alph_glob) @@ -2188,20 +1998,18 @@ def compute_global_alpha_formula(self, alph_expr, model_inputs): # and are not dropped during data processing (see function SmlpData._prepare_data_for_modeling). def compute_beta_formula(self, beta_expr, model_inps_outps): if beta_expr is None: - return self.smlp_true + return Solver._instance.smlp_true else: beta_expr_vars = get_expression_variables(beta_expr) dont_care_vars = list_subtraction_set(beta_expr_vars, model_inps_outps) if len(dont_care_vars) > 0: raise Exception('Variables ' + str(dont_care_vars) + ' in optimization constraints (beta) are not part of the model') - return self.parser.parse(beta_expr) if self._ENABLE_PYSMT else self.ast_expr_to_term(beta_expr) + return Solver.parse_ast(parser=self.ast_expr_to_term, expression=beta_expr) def compute_eta_formula(self, eta_expr, model_inputs): if eta_expr is None: - if self._ENABLE_PYSMT and self._RETURN_PYSMT: - return pysmt.shortcuts.TRUE() - return self.smlp_true + return Solver._instance.smlp_true else: # eta_expr can only contain knobs (control inputs), not free inputs or outputs (responses) eta_expr_vars = get_expression_variables(eta_expr) @@ -2209,13 +2017,13 @@ def compute_eta_formula(self, eta_expr, model_inputs): if len(dont_care_vars) > 0: raise Exception('Variables ' + str(dont_care_vars) + ' in knob constraints (eta) are not part of the model') - if self._ENABLE_PYSMT: - if self._RETURN_PYSMT: - return self.parser.parse(eta_expr) - else: - print(self.parser.parse(eta_expr)) + # if self._ENABLE_PYSMT: + # if self._RETURN_PYSMT: + # return self.parser.parse(eta_expr) + # else: + # print(self.parser.parse(eta_expr)) - return self.ast_expr_to_term(eta_expr) + return Solver.parse_ast(parser=self.ast_expr_to_term, expression=eta_expr) def var_domain(self, var, spec_domain_dict): interval = spec_domain_dict[var][self._SPEC_DOMAIN_INTERVAL_TAG]; #self._specInst.get_spec_interval_tag @@ -2255,28 +2063,27 @@ def create_model_exploration_base_components(self, syst_expr_dict:dict, algo, mo else: raise Exception('Data bounds file cannot be loaded') self._smlp_terms_logger.info('Parsing the SPEC: End') - + # get variable domains dictionary; certain sanity checks are performrd within this function. spec_domain_dict = self._specInst.get_spec_domain_dict; #print('spec_domain_dict', spec_domain_dict) self.verifier.initialize(variable_ranges=spec_domain_dict) + + # contraints on features used as control variables and on the responses alph_ranges = self.compute_input_ranges_formula_alpha_eta('alpha', feat_names, spec_domain_dict); # print('alph_ranges') alph_global = self.compute_global_alpha_formula(alph_expr, feat_names); # print('alph_global') - alpha = self.smlp_and(alph_ranges, alph_global) if ( - not self._ENABLE_PYSMT or not self._RETURN_PYSMT) else self.parser.and_(alph_ranges, alph_global) + alpha = Solver.smlp_and(alph_ranges, alph_global) + beta = self.compute_beta_formula(beta_expr, feat_names + resp_names); # print('beta') eta_ranges = self.compute_input_ranges_formula_alpha_eta('eta', feat_names, spec_domain_dict); # print('eta_ranges') - eta_grids = self.compute_grid_range_formulae_eta() if ( - not self._ENABLE_PYSMT or not self._RETURN_PYSMT) else self.pysmt_compute_grid_range_formulae_eta() + eta_grids = self.compute_grid_range_formulae_eta() eta_global = self.compute_eta_formula(eta_expr, feat_names); # print('eta_global', eta_global) - eta = self.smlp_and_multi([eta_ranges, eta_grids, eta_global]) if ( - not self._ENABLE_PYSMT or not self._RETURN_PYSMT) else self.parser.simplify( - self.parser.and_(eta_ranges, eta_grids, eta_global)) + eta = Solver.smlp_and_multi([eta_ranges, eta_grids, eta_global]) self._smlp_terms_logger.info('Alpha global constraints: ' + str(alph_global)) self._smlp_terms_logger.info('Alpha ranges constraints: ' + str(alph_ranges)) @@ -2396,32 +2203,32 @@ def create_model_exploration_instance_from_smlp_components(self, domain, model_f eq_form = self.smlp_eq(self.smlp_var(resp_name), resp_term) base_solver.add(eq_form) return base_solver - + # wrapper function on solver.check to measure runtime and return status in a convenient way - def smlp_solver_check(self, solver, call_name:str, lemma_precision:int=0, equations=None): + def smlp_solver_check(self, solver, call_name:str, lemma_precision:int=0, equations=None, temp=False): if equations: print('FORM2 FORMULA', equations) approx_lemmas = lemma_precision > 0 start = time.time() #print('solver chack start', flush=True) - res = solver.check() + res, witness = solver.check(temp=temp) #print('solver chack end', flush=True) end = time.time() - if isinstance(res, smlp.unknown): + status = Solver.convert_results_to_string(res) + + if status == "unknown": #print('smlp_unknown', smlp.unknown) - status = 'unknown' sat_model = {} - elif isinstance(res, smlp.sat): + elif status == "sat": #print('smlp_sat', smlp.sat) - status = 'sat' - sat_model = self.witness_term_to_const(res.model, approximate=False, precision=None) + witness = res.model if isinstance(solver, Form2_Solver) else witness["witness"] + sat_model = self.witness_term_to_const(witness, approximate=False, precision=None) if approx_lemmas: - sat_model_approx = self.approximate_witness_term(res.model, lemma_precision) + sat_model_approx = self.approximate_witness_term(witness, lemma_precision) # return TextToPysmtParser.SAT #print('res.model', res.model, 'sat_model', sat_model) - elif isinstance(res, smlp.unsat): + elif status == "unsat": #print('smlp_unsat', smlp.unsat) - status = 'unsat' sat_model = {} # return TextToPysmtParser.UNSAT else: @@ -2495,16 +2302,16 @@ def smlp_solver_check(self, solver, call_name:str, lemma_precision:int=0, equati #print('res.mode;', res.model, 'assignment', assignment, 'assignment_approx', assignment_approx); #return res, assignment_approx #print('exit smlp_solver_check', flush=True) - return res + return status, witness def solver_status_sat(self, res): - return res=="SAT" if self._ENABLE_PYSMT else isinstance(res, smlp.sat) + return res == "SAT" def solver_status_unsat(self, res): - return res=="UNSAT" if self._ENABLE_PYSMT else isinstance(res, smlp.unsat) + return res == "UNSAT" def solver_status_unknown(self, res): - return res=="UNKNOWN" if self._ENABLE_PYSMT else isinstance(res, smlp.unknown) + return res == "UNKNOWN" # we return value assignmenets to interface (input, knob, output) variables defined in the Spec file # (and not values assigned to any other variables that might be defined additionally as part of solver domain, @@ -2531,27 +2338,39 @@ def get_solver_model(self, res): def check_alpha_eta_consistency(self, domain:smlp.domain, model_full_term_dict:dict, alpha:smlp.form2, eta:smlp.form2, solver_logic:str): #print('create solver: model', model_full_term_dict, flush=True) - if not self._RETURN_PYSMT: - solver = self.create_model_exploration_instance_from_smlp_components( - domain, model_full_term_dict, False, solver_logic) - #print('add alpha', alpha, flush=True) - solver.add(alpha); #print('alpha', alpha, flush=True) - solver.add(eta); #print('eta', eta) - #print('create check', flush=True) - #res = solver.check(); print('res', res, flush=True) - res = self.smlp_solver_check(solver, 'interface_consistency' if model_full_term_dict is None else 'model_consistency', equations={'alpha':alpha, 'eta':eta}) - else: - self.verifier.reset() - self.verifier.apply_restrictions(alpha) - self.verifier.apply_restrictions(eta) - print('PYSMT FORMULA',{'alpha':alpha, 'eta':eta}) - res, witness = self.verifier.solve() + # if not self._RETURN_PYSMT: + # solver = self.create_model_exploration_instance_from_smlp_components( + # domain, model_full_term_dict, False, solver_logic) + # #print('add alpha', alpha, flush=True) + # solver.add(alpha); #print('alpha', alpha, flush=True) + # solver.add(eta); #print('eta', eta) + # #print('create check', flush=True) + # #res = solver.check(); print('res', res, flush=True) + # res = self.smlp_solver_check(solver, 'interface_consistency' if model_full_term_dict is None else 'model_consistency', equations={'alpha':alpha, 'eta':eta}) + # else: + # self.verifier.reset() + # self.verifier.apply_restrictions(alpha) + # self.verifier.apply_restrictions(eta) + # print('PYSMT FORMULA',{'alpha':alpha, 'eta':eta}) + # res, witness = self.verifier.solve() + + solver = Solver.create_solver( + create_solver=self.create_model_exploration_instance_from_smlp_components, + domain=domain, + model_full_term_dict=model_full_term_dict, + incremental=False, + solver_logic=solver_logic + ) + solver.add_formula(alpha) + solver.add_formula(eta) + + res,_ = self.smlp_solver_check(solver, 'interface_consistency' if model_full_term_dict is None else 'model_consistency', equations={'alpha':alpha, 'eta':eta}) consistency_type = 'Input and knob' if model_full_term_dict is None else 'Model' - if (self._ENABLE_PYSMT and res=="SAT") or isinstance(res, smlp.sat): + if res == "sat": self._smlp_terms_logger.info(consistency_type + ' interface constraints are consistent') interface_consistent = True - elif (self._ENABLE_PYSMT and res=="UNSAT") or isinstance(res, smlp.unsat): + elif res == "unsat": self._smlp_terms_logger.info(consistency_type + ' interface constraints are inconsistent') interface_consistent = False else: diff --git a/src/smlp_py/smtlib/text_to_sympy.py b/src/smlp_py/smtlib/text_to_sympy.py index 3b364de7..e1ad5866 100755 --- a/src/smlp_py/smtlib/text_to_sympy.py +++ b/src/smlp_py/smtlib/text_to_sympy.py @@ -368,9 +368,11 @@ def init_variables(self, symbols: List[Tuple[str, str, bool]]) -> None: for input_var in symbols: name, type, is_input = input_var unscaled_name = f"{name}_unscaled" + scaled_name = f"{name}_scaled" # TODO: i replaced the type variable with real, make sure that's ok self.add_symbol(name, 'real', is_input=is_input, nn_type=type) self.add_symbol(unscaled_name, 'real', is_input=is_input, nn_type=type) + self.add_symbol(scaled_name, 'real', is_input=is_input, nn_type=type) def add_symbol(self, name, symbol_type, is_input=True, nn_type='real'): assert symbol_type.lower() in pysmt_types.keys() diff --git a/src/smlp_py/solver.py b/src/smlp_py/solver.py index a4316cb5..53a8a705 100644 --- a/src/smlp_py/solver.py +++ b/src/smlp_py/solver.py @@ -78,10 +78,48 @@ def add_formula(self, *args, **kwargs): def check(self, *args, **kwargs): pass + @abstractmethod + def add_not_query(self, *args, **kwargs): + pass + + @abstractmethod + def create_counter_example(self, *args, **kwargs): + pass + + @abstractmethod + def substitute(self, *args, **kwargs): + pass + + + def get_witness(self, *args, **kwargs): + result = kwargs["result"] + witness = kwargs["witness"] + interface = kwargs["interface"] + + condition = result == "sat" + + if condition: + reduced_model = dict((k, v) for k, v in witness.items() if k in interface) + return reduced_model + else: + return None + + def convert_results_to_string(self, res): + if isinstance(res, smlp.sat): + return "sat" + elif isinstance(res, smlp.unsat): + return "unsat" + elif isinstance(res, smlp.unknown): + return "unknown" + elif type(res) == str: + return res.lower() + else: + raise Exception("Unsupported result format") class Pysmt_Solver(AbstractSolver, PYSMTOperations): verifier = None + temp_solver = None def __init__(self, specs): super().__init__() @@ -144,6 +182,7 @@ def create_theta_form(self, **kwargs): rad_term = kwargs["rad_term"] theta_form = kwargs["theta_form"] + rad_term = float(rad_term) value = float(witness) PYSMT_var = self.verifier.parser.get_symbol(var) type = pysmt.shortcuts.Int if str(PYSMT_var.get_type()) == "Int" else Real @@ -179,16 +218,63 @@ def parse_ast(self, *args, **kwargs): return self.parse(expression) def create_solver(self, *args, **kwargs): - self.verifier.reset() + temp = kwargs.get('temp', False) + if temp: + self.temp_solver = MarabouVerifier(parser=self.verifier.parser, + variable_ranges=self.verifier.variable_ranges, + is_temp=True) + + else: + self.verifier.reset() return self - def add_formula(self, formula): - self.verifier.apply_restrictions(formula) + def add_formula(self, formula, **kwargs): + need_simplification = kwargs.get("need_simplification", False) + self.verifier.apply_restrictions(formula, need_simplification=need_simplification) def check(self, *args, **kwargs): + temp = kwargs.get("temp", False) + if temp: + result = self.temp_solver.solve() + self.temp_solver = None + return result + else: + return self.verifier.solve() + + def generate_theta(self, *args, **kwargs): + pass + def add_not_query(self, *args, **kwargs): + query = kwargs["query"] + temp = kwargs.get("temp", False) + def create_counter_example(self, *args, **kwargs): + formulas = kwargs["formulas"] + query = kwargs["query"] + + self.temp_solver = MarabouVerifier( parser=self.verifier.parser, + variable_ranges=self.verifier.variable_ranges, + is_temp=True) + for formula in formulas: + self.temp_solver.apply_restrictions(formula) + + negation = self.temp_solver.parser.propagate_negation(query) + z3_equiv = self.temp_solver.parser.handle_ite_formula(negation, handle_ite=False) + self.temp_solver.apply_restrictions(negation, need_simplification=True) + return self + + def substitute(self, *args, **kwargs): + var = kwargs["var"] + substitutions = kwargs["substitutions"] + + for x in list(substitutions.keys()): + temp = substitutions[x] + del substitutions[x] + substitutions[self.smlp_var(x)] = temp + + return self.simplify(var.substitute(substitutions)) + class Form2_Solver(AbstractSolver, SMLPOperations): verifier = None smlp_term_instance = None @@ -221,7 +307,7 @@ def generate_rad_term(self, *args, **kwargs): var_term = kwargs["var_term"] candidate = kwargs["candidate"] - rad_term = smlp.Cnst(rad) + rad_term = self.smlp_cnst(rad) if delta_rel is not None: # radius for a lemma -- cex holds values of candidate counter-example rad_term = rad_term * abs(var_term) else: # radius for excluding a candidate -- cex holds values of the candidate @@ -234,15 +320,12 @@ def create_theta_form(self, *args, **kwargs): witness = kwargs["witness"] var_term = kwargs["var_term"] rad_term = kwargs["rad_term"] - smlp_and_ = kwargs["smlp_and_"] - return smlp_and_(theta_form, ((abs(var_term - witness)) <= rad_term)) + return self.smlp_and(theta_form, ((abs(var_term - witness)) <= rad_term)) def get_rad_term(self, *args, **kwargs): - smlp_cnst = kwargs["smlp_cnst"] rad = kwargs["rad"] - - return smlp_cnst(rad) + return self.smlp_cnst(rad) def create_alpha_or_eta_form(self, *args, **kwargs): alpha_or_eta_form = kwargs["alpha_or_eta_form"] @@ -275,14 +358,37 @@ def create_solver(self, *args, **kwargs): model_full_term_dict = kwargs["model_full_term_dict"] incremental = kwargs["incremental"] solver_logic = kwargs["solver_logic"] - formulas = kwargs["formulas"] self.verifier = create_solver(domain, model_full_term_dict, incremental, solver_logic) return self - def add_formula(self, formula): + def add_formula(self, *args, **kwargs): + formula = kwargs["formula"] + self.verifier.add(formula) + def check(self): + return self.verifier.check(), None + + def generate_theta(self, *args, **kwargs): + pass + + def create_counter_example(self, *args, **kwargs): + formulas = kwargs["formulas"] + query = kwargs["query"] + + self.create_solver(*args, **kwargs) + for formula in formulas: + self.add_formula(formula) + + self.add_formula(self.smlp_not(query)) + return self + + def substitute(self, *args, **kwargs): + var = kwargs["var"] + substitutions = kwargs["substitutions"] + + return self.smlp_cnst_fold(var, substitutions) class Solver: class Version(Enum): From 1aa2f92d81acc58975ab2b2b284be8da4a066362 Mon Sep 17 00:00:00 2001 From: konstantopoulos-tetractys <37417801+ntinouldinho@users.noreply.github.com> Date: Tue, 20 Aug 2024 17:05:00 +0100 Subject: [PATCH 24/28] change file structure --- src/smlp_py/smlp_operations.py | 267 ------------- src/smlp_py/smlp_optimize.py | 2 +- src/smlp_py/smlp_query.py | 4 +- src/smlp_py/smlp_terms.py | 10 +- src/smlp_py/smtlib/text_to_sympy.py | 3 +- src/smlp_py/solver.py | 446 ---------------------- src/smlp_py/solvers/abstract_solver.py | 119 ++++++ src/smlp_py/solvers/marabou/operations.py | 59 +++ src/smlp_py/solvers/marabou/solver.py | 166 ++++++++ src/smlp_py/solvers/universal_solver.py | 73 ++++ src/smlp_py/solvers/z3/operations.py | 191 +++++++++ src/smlp_py/solvers/z3/solver.py | 118 ++++++ 12 files changed, 736 insertions(+), 722 deletions(-) create mode 100644 src/smlp_py/solvers/abstract_solver.py create mode 100644 src/smlp_py/solvers/marabou/operations.py create mode 100644 src/smlp_py/solvers/marabou/solver.py create mode 100644 src/smlp_py/solvers/universal_solver.py create mode 100644 src/smlp_py/solvers/z3/operations.py create mode 100644 src/smlp_py/solvers/z3/solver.py diff --git a/src/smlp_py/smlp_operations.py b/src/smlp_py/smlp_operations.py index 02ab0675..e69de29b 100644 --- a/src/smlp_py/smlp_operations.py +++ b/src/smlp_py/smlp_operations.py @@ -1,267 +0,0 @@ -import pysmt.shortcuts -import smlp -import functools -import operator as op - -from pysmt.fnode import FNode - -USE_CACHE = False - - -def conditional_cache(func): - """Custom decorator to conditionally apply @functools.cache.""" - if USE_CACHE: - # Apply caching - return functools.cache(func) - else: - # Return the original function without caching - return func - - -class SMLPOperations: - @property - @conditional_cache # @functools.cache - def smlp_true(self): - return smlp.true - - @property - @conditional_cache # @functools.cache - def smlp_false(self): - return smlp.false - - @property - @conditional_cache # @functools.cache - def smlp_real(self): - return smlp.Real - - @property - @conditional_cache # @functools.cache - def smlp_integer(self): - return smlp.Integer - - @conditional_cache # @functools.cache - def smlp_var(self, var): - return smlp.Var(var) - - @conditional_cache # @functools.cache - def smlp_cnst(self, const): - return smlp.Cnst(const) - - # rationals - @conditional_cache # @functools.cache - def smlp_q(self, const): - return smlp.Q(const) - - # reals - @conditional_cache # @functools.cache - def smlp_r(self, const): - return smlp.R(const) - - # logical not (logic negation) - @conditional_cache # @functools.cache - def smlp_not(self, form: smlp.form2): - # res1 = ~form - res2 = op.inv(form) - # assert res1 == res2 - return res2 # ~form - - # logical and (conjunction) - @conditional_cache # @functools.cache - def smlp_and(self, form1: smlp.form2, form2: smlp.form2): - ''' test 83 gets stuck with this simplification - if form1 == smlp.true: - return form2 - if form2 == smlp.true: - return form1 - ''' - res1 = op.and_(form1, form2) - # res2 = form1 & form2 - # print('res1', res1, type(res1)); print('res2', res2, type(res2)) - # assert res1 == res2 - return res1 # form1 & form2 - - # conjunction of possibly more than two formulas - # @functools.cache -- error: unhashable type: 'list' - def smlp_and_multi(self, form_list: list[smlp.form2]): - res = self.smlp_true - ''' - for i, form in enumerate(form_list): - res = form if i == 0 else self.smlp_and(res, form) - ''' - for form in form_list: - res = form if res is self.smlp_true else self.smlp_and(res, form) - return res - - # logical or (disjunction) - @conditional_cache # @functools.cache - def smlp_or(self, form1: smlp.form2, form2: smlp.form2): - res1 = op.or_(form1, form2) - # res2 = form1 | form2 - # assert res1 == res2 - return res1 # form1 | form2 - - # disjunction of possibly more than two formulas - # @functools.cache -- error: unhashable type: 'list' - def smlp_or_multi(self, form_list: list[smlp.form2]): - res = self.smlp_false - ''' - for i, form in enumerate(form_list): - res = form if i == 0 else self.smlp_or(res, form) - ''' - for form in form_list: - res = form if res is self.smlp_false else self.smlp_or(res, form) - return res - - # logical implication - @conditional_cache # @functools.cache - def smlp_implies(self, form1: smlp.form2, form2: smlp.form2): - return self.smlp_or(self.smlp_not(form1), form2) - - # addition - @conditional_cache # @functools.cache - def smlp_add(self, term1: smlp.term2, term2: smlp.term2): - return op.add(term1, term2) - - # sum of possibly more than two formulas - # @functools.cache -- error: unhashable type: 'list' - def smlp_add_multi(self, term_list: list[smlp.term2]): - for i, term in enumerate(term_list): - res = term if i == 0 else self.smlp_add(res, term) - return res - - # subtraction - @conditional_cache # @conditional_cache #@functools.cache - def smlp_sub(self, term1: smlp.term2, term2: smlp.term2): - return op.sub(term1, term2) - - # multiplication - @conditional_cache # @functools.cache - def smlp_mult(self, term1: smlp.term2, term2: smlp.term2): - return op.mul(term1, term2) - - # TODO: !!! check that term2 does not evaluate to term 0 ??? - - # Do this before calling smlp_div, whenver possible? - @conditional_cache # @functools.cache - def smlp_div(self, term1: smlp.term2, term2: smlp.term2): - # return self.smlp_mult(self.smlp_cnst(self.smlp_q(1)) / term2, term1) - return self.smlp_mult(op.truediv(self.smlp_cnst(self.smlp_q(1)), term2), term1) - - @conditional_cache # @functools.cache - def smlp_pow(self, term1: smlp.term2, term2: smlp.term2): - return op.pow(term1, term2) - - # equality - @conditional_cache # @functools.cache - def smlp_eq(self, term1: smlp.term2, term2: smlp.term2): - res1 = op.eq(term1, term2) - # res2 = term1 == term2; print('res1', res1, 'res2', res2) - # assert res1 == res2 - return res1 - - # operator != (not equal) - @conditional_cache # @functools.cache - def smlp_ne(self, term1: smlp.term2, term2: smlp.term2): - res1 = op.ne(term1, term2) - # res2 = term1 != term2; print('res1', res1, 'res2', res2) - # assert res1 == res2 - return res1 - - # operator < - @conditional_cache # @functools.cache - def smlp_lt(self, term1: smlp.term2, term2: smlp.term2): - return op.lt(term1, term2) - - # operator <= - - @conditional_cache # @functools.cache - def smlp_le(self, term1: smlp.term2, term2: smlp.term2): - return op.le(term1, term2) - - # operator > - @conditional_cache # @functools.cache - def smlp_gt(self, term1: smlp.term2, term2: smlp.term2): - return op.gt(term1, term2) - - # operator >= - - @conditional_cache # @functools.cache - def smlp_ge(self, term1: smlp.term2, term2: smlp.term2): - return op.ge(term1, term2) - - # if-thne-else operation - @conditional_cache # @functools.cache - def smlp_ite(self, form: smlp.form2, term1: smlp.term2, term2: smlp.term2): - return smlp.Ite(form, term1, term2) - - # this function performs substitution of variables in term2: - # it substitutes occurrences of the keys in subst_dict with respective values, in term2 term. - # @functools.cache - def smlp_subst(self, term: smlp.term2, subst_dict: dict): - return smlp.subst(term, subst_dict) - - # simplifies a ground term to the respective constant; takes als a s - # @functools.cache - def smlp_cnst_fold(self, term: smlp.term2, subst_dict: dict): - return smlp.cnst_fold(term, subst_dict) - -class ClassProperty: - def __init__(self, fget): - self.fget = fget - - def __get__(self, instance, owner): - return self.fget(owner) - -class PYSMTOperations: - - @ClassProperty - def smlp_true(cls): - return pysmt.shortcuts.TRUE() - - @ClassProperty - def smlp_false(cls): - return pysmt.shortcuts.FALSE() - - @ClassProperty - def smlp_real(cls): - return pysmt.shortcuts.Real - - @ClassProperty - def smlp_integer(cls): - return pysmt.shortcuts.Int - - @conditional_cache # @functools.cache - def smlp_cnst(cls, const): - if isinstance(const, FNode): - return const - return pysmt.shortcuts.Real(const) - - # logical not (logic negation) - @conditional_cache # @functools.cache - def smlp_not(cls, form: FNode): - return pysmt.shortcuts.Not(form) - - # logical and (conjunction) - @conditional_cache # @functools.cache - def smlp_and(cls, form1: FNode, form2: FNode): - return pysmt.shortcuts.And(form1, form2) # form1 & form2 - - def smlp_and_multi(cls, form_list: list[FNode]): - return pysmt.shortcuts.And(*form_list) - - # logical or (disjunction) - @conditional_cache # @functools.cache - def smlp_or(cls, form1: FNode, form2: FNode): - return pysmt.shortcuts.Or(form1, form2) - - def smlp_or_multi(cls, form_list: list[FNode]): - return pysmt.shortcuts.Or(*form_list) - - def smlp_eq(self, term1: smlp.term2, term2: smlp.term2): - return pysmt.shortcuts.Equals(term1, term2) - - def smlp_q(self, const): - return pysmt.shortcuts.Real(const) - - def smlp_mult(self, *args): - return pysmt.shortcuts.Times(*args) diff --git a/src/smlp_py/smlp_optimize.py b/src/smlp_py/smlp_optimize.py index 341aee8b..4e56738a 100755 --- a/src/smlp_py/smlp_optimize.py +++ b/src/smlp_py/smlp_optimize.py @@ -15,7 +15,7 @@ import numpy as np from src.smlp_py.smtlib.text_to_sympy import TextToPysmtParser from pysmt.shortcuts import Real -from smlp_py.solver import Solver +from src.smlp_py.solvers.universal_solver import Solver # single or multi-objective optimization, with stability constraints and any user # given constraints on free input, control (knob) and output variables satisfied. diff --git a/src/smlp_py/smlp_query.py b/src/smlp_py/smlp_query.py index f9c135ca..ae66cfab 100644 --- a/src/smlp_py/smlp_query.py +++ b/src/smlp_py/smlp_query.py @@ -9,9 +9,9 @@ from smlp_py.smlp_utils import np_JSONEncoder #, str_to_bool from src.smlp_py.NN_verifiers.verifiers import MarabouVerifier -from smlp_py.solver import Solver +from src.smlp_py.solvers.universal_solver import Solver -from smlp_py.solver import Pysmt_Solver +from src.smlp_py.solvers.marabou.solver import Pysmt_Solver class SmlpQuery: diff --git a/src/smlp_py/smlp_terms.py b/src/smlp_py/smlp_terms.py index ebc86fec..a5251f4f 100644 --- a/src/smlp_py/smlp_terms.py +++ b/src/smlp_py/smlp_terms.py @@ -17,19 +17,19 @@ import sys import smlp -from smlp_py.smlp_utils import (np_JSONEncoder, lists_union_order_preserving_without_duplicates, +from src.smlp_py.smlp_utils import (np_JSONEncoder, lists_union_order_preserving_without_duplicates, list_subtraction_set, get_expression_variables, str_to_bool) #from smlp_py.smlp_spec import SmlpSpec from pysmt.shortcuts import Real as pysmtReal -from smlp_py.NN_verifiers.verifiers import MarabouVerifier +from src.smlp_py.NN_verifiers.verifiers import MarabouVerifier import pysmt from src.smlp_py.smtlib.text_to_sympy import TextToPysmtParser -from smlp_py.solver import Solver +from src.smlp_py.solvers.universal_solver import Solver -from smlp_py.smlp_operations import SMLPOperations +from src.smlp_py.solvers.z3.operations import SMLPOperations -from src.smlp_py.solver import Form2_Solver +from src.smlp_py.solvers.z3.solver import Form2_Solver # TODO !!! create a parent class for TreeTerms, PolyTerms, NNKerasTerms. # setting logger, report_file_prefix, model_file_prefix can go to that class to work for all above three classes diff --git a/src/smlp_py/smtlib/text_to_sympy.py b/src/smlp_py/smtlib/text_to_sympy.py index e1ad5866..81147fa3 100755 --- a/src/smlp_py/smtlib/text_to_sympy.py +++ b/src/smlp_py/smtlib/text_to_sympy.py @@ -600,6 +600,7 @@ def traverse(node): parser.add_symbol('x1', 'int') parser.add_symbol('x2', 'real') parser.add_symbol('p2', 'real') + parser.add_symbol('p1', 'real') parser.add_symbol('y1', 'real') parser.add_symbol('y2', 'real') @@ -661,7 +662,7 @@ def traverse(node, source=None): print(simplified_formula) - formula = parser.parse('(y1+y2)/2') + formula = parser.parse("p1==4 or (p1==8 and p2 > 3)") # formula = parser.parse('p2<5.0 and x1==10 and x2<12.0') x = Symbol('x', REAL) y = Symbol('y', REAL) diff --git a/src/smlp_py/solver.py b/src/smlp_py/solver.py index 53a8a705..70100f20 100644 --- a/src/smlp_py/solver.py +++ b/src/smlp_py/solver.py @@ -11,453 +11,7 @@ from src.smlp_py.smtlib.text_to_sympy import TextToPysmtParser import operator as op from pysmt.shortcuts import Symbol, And -from src.smlp_py.smlp_operations import SMLPOperations, PYSMTOperations -class ClassProperty: - def __init__(self, fget): - self.fget = fget - def __get__(self, instance, owner): - return self.fget(owner) -class AbstractSolver(ABC): - # @abstractmethod - # def true(self): - # pass - - # @abstractmethod - # def GE(self, *args, **kwargs): - # pass - - # @abstractmethod - # def LE(self, *args, **kwargs): - # pass - - @abstractmethod - def create_query(self, *args, **kwargs): - pass - - @abstractmethod - def create_query_and_beta(self, *args, **kwargs): - pass - - @abstractmethod - def substitute_objective_with_witness(self, *args, **kwargs): - pass - - @abstractmethod - def generate_rad_term(self, *args, **kwargs): - pass - - @abstractmethod - def create_theta_form(self, *args, **kwargs): - pass - - @abstractmethod - def get_rad_term(self, *args, **kwargs): - pass - - @abstractmethod - def create_alpha_or_eta_form(self, *args, **kwargs): - pass - - @abstractmethod - def parse_ast(self, *args, **kwargs): - pass - - @abstractmethod - def create_solver(self, *args, **kwargs): - pass - - @abstractmethod - def add_formula(self, *args, **kwargs): - pass - - @abstractmethod - def check(self, *args, **kwargs): - pass - - @abstractmethod - def add_not_query(self, *args, **kwargs): - pass - - @abstractmethod - def create_counter_example(self, *args, **kwargs): - pass - - @abstractmethod - def substitute(self, *args, **kwargs): - pass - - - def get_witness(self, *args, **kwargs): - result = kwargs["result"] - witness = kwargs["witness"] - interface = kwargs["interface"] - - condition = result == "sat" - - if condition: - reduced_model = dict((k, v) for k, v in witness.items() if k in interface) - return reduced_model - else: - return None - - def convert_results_to_string(self, res): - if isinstance(res, smlp.sat): - return "sat" - elif isinstance(res, smlp.unsat): - return "unsat" - elif isinstance(res, smlp.unknown): - return "unknown" - elif type(res) == str: - return res.lower() - else: - raise Exception("Unsupported result format") - - -class Pysmt_Solver(AbstractSolver, PYSMTOperations): - verifier = None - temp_solver = None - - def __init__(self, specs): - super().__init__() - self.specs = specs - self.create_verifier() - - def create_verifier(self): - symbols = [] - feat_names, resp_names, spec_domain_dict = self.specs - - for feature in feat_names: - type = spec_domain_dict[feature]['range'] - symbols.append((feature, type, True)) - - for response in resp_names: - type = spec_domain_dict[response]['range'] - symbols.append((response, type, False)) - - parser = TextToPysmtParser() - parser.init_variables(symbols=symbols) - - self.verifier = MarabouVerifier(parser=parser) - self.verifier.initialize(spec_domain_dict) - - @ClassProperty - def smlp_true(self): - return pysmt.shortcuts.TRUE() - - def smlp_var(self, var): - return self.verifier.parser.get_symbol(var) - - def create_query(self, query_form=None): - self.verifier.parser.handle_ite_formula(query_form, is_form2=True) - - def create_query_and_beta(self, query, beta): - return self.verifier.parser.and_(query, beta) - - def substitute_objective_with_witness(self, *args, **kwargs): - stable_witness_terms = kwargs["stable_witness_terms"] - objv_term = kwargs["objv_term"] - - substitution = {} - for symbol, value in stable_witness_terms.items(): - symbol = self.verifier.parser.get_symbol(symbol) - substitution[symbol] = Real(value) - # Apply the substitution - return self.verifier.parser.simplify(objv_term.substitute(substitution)) - - def generate_rad_term(self, **kwargs): - rad = kwargs["rad"] - return float(rad) - - def get_rad_term(self, **kwargs): - rad = kwargs["rad"] - return float(rad) - - def create_theta_form(self, **kwargs): - witness = kwargs["witness"] - var = kwargs["var"] - rad_term = kwargs["rad_term"] - theta_form = kwargs["theta_form"] - - rad_term = float(rad_term) - value = float(witness) - PYSMT_var = self.verifier.parser.get_symbol(var) - type = pysmt.shortcuts.Int if str(PYSMT_var.get_type()) == "Int" else Real - calc_type = int if str(PYSMT_var.get_type()) == "Int" else float - lower = calc_type(value - rad_term) - lower = type(lower) - upper = calc_type(value + rad_term) - upper = type(upper) - theta_form = self.verifier.parser.and_(theta_form, PYSMT_var >= lower, PYSMT_var <= upper) - return theta_form - - def create_alpha_or_eta_form(self, **kwargs): - alpha_or_eta_form = kwargs["alpha_or_eta_form"] - mx = kwargs["mx"] - mn = kwargs["mn"] - v = kwargs["v"] - - symbol_v = self.smlp_var(v) - form = self.smlp_and(symbol_v >= mn, symbol_v <= mx) - return self.simplify(self.smlp_and(alpha_or_eta_form, form)) - - def simplify(self, expression): - return self.verifier.parser.simplify(expression) - - def parse(self, expression): - return self.verifier.parser.parse(expression) - - def GE(self, *args): - return args[0] >= args[1] - - def parse_ast(self, *args, **kwargs): - expression = kwargs['expression'] - return self.parse(expression) - - def create_solver(self, *args, **kwargs): - temp = kwargs.get('temp', False) - if temp: - self.temp_solver = MarabouVerifier(parser=self.verifier.parser, - variable_ranges=self.verifier.variable_ranges, - is_temp=True) - - else: - self.verifier.reset() - return self - - def add_formula(self, formula, **kwargs): - need_simplification = kwargs.get("need_simplification", False) - self.verifier.apply_restrictions(formula, need_simplification=need_simplification) - - def check(self, *args, **kwargs): - temp = kwargs.get("temp", False) - if temp: - result = self.temp_solver.solve() - self.temp_solver = None - return result - else: - return self.verifier.solve() - - def generate_theta(self, *args, **kwargs): - pass - - def add_not_query(self, *args, **kwargs): - query = kwargs["query"] - temp = kwargs.get("temp", False) - - - def create_counter_example(self, *args, **kwargs): - formulas = kwargs["formulas"] - query = kwargs["query"] - - self.temp_solver = MarabouVerifier( parser=self.verifier.parser, - variable_ranges=self.verifier.variable_ranges, - is_temp=True) - for formula in formulas: - self.temp_solver.apply_restrictions(formula) - - negation = self.temp_solver.parser.propagate_negation(query) - z3_equiv = self.temp_solver.parser.handle_ite_formula(negation, handle_ite=False) - self.temp_solver.apply_restrictions(negation, need_simplification=True) - return self - - def substitute(self, *args, **kwargs): - var = kwargs["var"] - substitutions = kwargs["substitutions"] - - for x in list(substitutions.keys()): - temp = substitutions[x] - del substitutions[x] - substitutions[self.smlp_var(x)] = temp - - return self.simplify(var.substitute(substitutions)) - -class Form2_Solver(AbstractSolver, SMLPOperations): - verifier = None - smlp_term_instance = None - terms = None - - def __init__(self): - super().__init__() - # self.verifier = verifier - # self.smlp_term_instance = smlp_term_instance - - @property - def smlp_true(self): - return smlp.true - - def create_query(self, query_form=None): - return query_form - - def create_query_and_beta(self, query, beta): - return self.smlp_term_instance.smlp_and(query, beta) - - def substitute_objective_with_witness(self, *args, **kwargs): - stable_witness_terms = kwargs["stable_witness_terms"] - objv_term = kwargs["objv_term"] - - return smlp.subst(objv_term, stable_witness_terms); - - def generate_rad_term(self, *args, **kwargs): - rad = kwargs["rad"] - delta_rel = kwargs["delta_rel"] - var_term = kwargs["var_term"] - candidate = kwargs["candidate"] - - rad_term = self.smlp_cnst(rad) - if delta_rel is not None: # radius for a lemma -- cex holds values of candidate counter-example - rad_term = rad_term * abs(var_term) - else: # radius for excluding a candidate -- cex holds values of the candidate - rad_term = rad_term * abs(candidate) - - return rad_term - - def create_theta_form(self, *args, **kwargs): - theta_form = kwargs["theta_form"] - witness = kwargs["witness"] - var_term = kwargs["var_term"] - rad_term = kwargs["rad_term"] - - return self.smlp_and(theta_form, ((abs(var_term - witness)) <= rad_term)) - - def get_rad_term(self, *args, **kwargs): - rad = kwargs["rad"] - return self.smlp_cnst(rad) - - def create_alpha_or_eta_form(self, *args, **kwargs): - alpha_or_eta_form = kwargs["alpha_or_eta_form"] - is_in_spec = kwargs["is_in_spec"] - is_disjunction = kwargs["is_disjunction"] - is_alpha = kwargs["is_alpha"] - mx = kwargs["mx"] - mn = kwargs["mn"] - v = kwargs["v"] - - - if is_disjunction and is_alpha and is_in_spec: - rng = self.smlp_or_multi([self.smlp_eq(self.smlp_var(v), self.smlp_cnst(i)) for i in range(mn, mx + 1)]) - else: - rng = self.smlp_and(self.smlp_var(v) >= self.smlp_cnst(mn), self.smlp_var(v) <= self.smlp_cnst(mx)) - - return self.smlp_and(alpha_or_eta_form, rng) - - def GE(self, *args): - return args[0] >= args[1] - - def parse_ast(self, *args, **kwargs): - expression = kwargs['expression'] - parser = kwargs['parser'] - return parser(expression) - - def create_solver(self, *args, **kwargs): - create_solver = kwargs["create_solver"] - domain = kwargs["domain"] - model_full_term_dict = kwargs["model_full_term_dict"] - incremental = kwargs["incremental"] - solver_logic = kwargs["solver_logic"] - - self.verifier = create_solver(domain, model_full_term_dict, incremental, solver_logic) - return self - - def add_formula(self, *args, **kwargs): - formula = kwargs["formula"] - - self.verifier.add(formula) - - def check(self): - return self.verifier.check(), None - - def generate_theta(self, *args, **kwargs): - pass - - def create_counter_example(self, *args, **kwargs): - formulas = kwargs["formulas"] - query = kwargs["query"] - - self.create_solver(*args, **kwargs) - for formula in formulas: - self.add_formula(formula) - - self.add_formula(self.smlp_not(query)) - return self - - def substitute(self, *args, **kwargs): - var = kwargs["var"] - substitutions = kwargs["substitutions"] - - return self.smlp_cnst_fold(var, substitutions) - -class Solver: - class Version(Enum): - FORM2 = 0 - PYSMT = 1 - - _instance = None - version = None - - def __new__(cls, *args, **kwargs): - version = kwargs["version"] - if isinstance(version, cls.Version): - cls.version = version - else: - raise ValueError("Must be a valid version") - - if cls._instance is None and isinstance(cls.version, cls.Version): - if cls.version == cls.Version.PYSMT: - specs = kwargs["specs"] - cls._instance = Pysmt_Solver(specs) - else: - cls._instance = Form2_Solver() - cls._map_instance_methods() - return cls._instance - - @classmethod - def _map_instance_methods(cls): - """Automatically maps all methods from the instance to the SingletonFactory class.""" - # for name, method in cls._instance.__class__.__dict__.items(): - # if isinstance(method, types.FunctionType): - # # Avoid overwriting existing methods in Solver class if not hasattr(cls, name): - # setattr(cls, name, cls._create_delegator(name)) - for base_class in cls._instance.__class__.__mro__: - for name, method in base_class.__dict__.items(): - if isinstance(method, types.FunctionType): - # Avoid overwriting existing methods in Solver classifnothasattr(cls, name): - setattr(cls, name, cls._create_delegator(name)) - - @classmethod - def _create_delegator(cls, method_name): - """Create a method that delegates the call to the _instance.""" - def delegator(*args, **kwargs): - return getattr(cls._instance, method_name)(*args, **kwargs) - return delegator - - - - - # - # @classmethod - # def _map_instance_properties(cls): - # """Automatically maps all properties from the instance to the Solver class.""" - # for name, attribute in cls._instance.__class__.__dict__.items(): - # if isinstance(attribute, property): - # # Map property to Solver class - # if not hasattr(cls, name): - # setattr(cls, name, cls._create_property_delegator(name)) - # - # @classmethod - # def _create_property_delegator(cls, property_name): - # """Create a property that delegates access to the _instance.""" - # def getter(self): - # return getattr(self._instance, property_name) - # - # def setter(self, value): - # setattr(self._instance, property_name, value) - # - # def deleter(self): - # delattr(self._instance, property_name) - # - # # Return a property with the mapped getter, setter, and deleter - # return property(getter, setter, deleter) diff --git a/src/smlp_py/solvers/abstract_solver.py b/src/smlp_py/solvers/abstract_solver.py new file mode 100644 index 00000000..09d40a34 --- /dev/null +++ b/src/smlp_py/solvers/abstract_solver.py @@ -0,0 +1,119 @@ +import functools +from abc import ABC, abstractmethod +import smlp + +USE_CACHE = False + + +def conditional_cache(func): + """Custom decorator to conditionally apply @functools.cache.""" + if USE_CACHE: + # Apply caching + return functools.cache(func) + else: + # Return the original function without caching + return func + + +class ClassProperty: + def __init__(self, fget): + self.fget = fget + + def __get__(self, instance, owner): + return self.fget(owner) + + +class AbstractSolver(ABC): + + # @abstractmethod + # def true(self): + # pass + + # @abstractmethod + # def GE(self, *args, **kwargs): + # pass + + # @abstractmethod + # def LE(self, *args, **kwargs): + # pass + + @abstractmethod + def create_query(self, *args, **kwargs): + pass + + @abstractmethod + def create_query_and_beta(self, *args, **kwargs): + pass + + @abstractmethod + def substitute_objective_with_witness(self, *args, **kwargs): + pass + + @abstractmethod + def generate_rad_term(self, *args, **kwargs): + pass + + @abstractmethod + def create_theta_form(self, *args, **kwargs): + pass + + @abstractmethod + def get_rad_term(self, *args, **kwargs): + pass + + @abstractmethod + def create_alpha_or_eta_form(self, *args, **kwargs): + pass + + @abstractmethod + def parse_ast(self, *args, **kwargs): + pass + + @abstractmethod + def create_solver(self, *args, **kwargs): + pass + + @abstractmethod + def add_formula(self, *args, **kwargs): + pass + + @abstractmethod + def check(self, *args, **kwargs): + pass + + @abstractmethod + def add_not_query(self, *args, **kwargs): + pass + + @abstractmethod + def create_counter_example(self, *args, **kwargs): + pass + + @abstractmethod + def substitute(self, *args, **kwargs): + pass + + def get_witness(self, *args, **kwargs): + result = kwargs["result"] + witness = kwargs["witness"] + interface = kwargs["interface"] + + condition = result == "sat" + + if condition: + reduced_model = dict((k, v) for k, v in witness.items() if k in interface) + return reduced_model + else: + return None + + def convert_results_to_string(self, res): + if isinstance(res, smlp.sat): + return "sat" + elif isinstance(res, smlp.unsat): + return "unsat" + elif isinstance(res, smlp.unknown): + return "unknown" + elif type(res) == str: + return res.lower() + else: + raise Exception("Unsupported result format") diff --git a/src/smlp_py/solvers/marabou/operations.py b/src/smlp_py/solvers/marabou/operations.py new file mode 100644 index 00000000..8fb20248 --- /dev/null +++ b/src/smlp_py/solvers/marabou/operations.py @@ -0,0 +1,59 @@ +import pysmt.shortcuts +import smlp +from pysmt.fnode import FNode +from src.smlp_py.solvers.abstract_solver import ClassProperty, conditional_cache + + +class PYSMTOperations: + + @ClassProperty + def smlp_true(cls): + return pysmt.shortcuts.TRUE() + + @ClassProperty + def smlp_false(cls): + return pysmt.shortcuts.FALSE() + + @ClassProperty + def smlp_real(cls): + return pysmt.shortcuts.Real + + @ClassProperty + def smlp_integer(cls): + return pysmt.shortcuts.Int + + @conditional_cache # @functools.cache + def smlp_cnst(cls, const): + if isinstance(const, FNode): + return const + return pysmt.shortcuts.Real(const) + + # logical not (logic negation) + @conditional_cache # @functools.cache + def smlp_not(cls, form: FNode): + return pysmt.shortcuts.Not(form) + + # logical and (conjunction) + @conditional_cache # @functools.cache + def smlp_and(cls, form1: FNode, form2: FNode): + return pysmt.shortcuts.And(form1, form2) # form1 & form2 + + def smlp_and_multi(cls, form_list: list[FNode]): + return pysmt.shortcuts.And(*form_list) + + # logical or (disjunction) + @conditional_cache # @functools.cache + def smlp_or(cls, form1: FNode, form2: FNode): + return pysmt.shortcuts.Or(form1, form2) + + def smlp_or_multi(cls, form_list: list[FNode]): + return pysmt.shortcuts.Or(*form_list) + + def smlp_eq(self, term1: smlp.term2, term2: smlp.term2): + return pysmt.shortcuts.Equals(term1, term2) + + def smlp_q(self, const): + return pysmt.shortcuts.Real(const) + + def smlp_mult(self, *args): + return pysmt.shortcuts.Times(*args) \ No newline at end of file diff --git a/src/smlp_py/solvers/marabou/solver.py b/src/smlp_py/solvers/marabou/solver.py new file mode 100644 index 00000000..9bc4e584 --- /dev/null +++ b/src/smlp_py/solvers/marabou/solver.py @@ -0,0 +1,166 @@ +import pysmt + +from src.smlp_py.NN_verifiers.verifiers import MarabouVerifier +from src.smlp_py.smtlib.text_to_sympy import TextToPysmtParser +from src.smlp_py.solvers.abstract_solver import AbstractSolver, ClassProperty +from src.smlp_py.solvers.marabou.operations import PYSMTOperations +from pysmt.shortcuts import Real + + +class Pysmt_Solver(AbstractSolver, PYSMTOperations): + verifier = None + temp_solver = None + + def __init__(self, specs): + super().__init__() + self.specs = specs + self.create_verifier() + + def create_verifier(self): + symbols = [] + feat_names, resp_names, spec_domain_dict = self.specs + + for feature in feat_names: + type = spec_domain_dict[feature]['range'] + symbols.append((feature, type, True)) + + for response in resp_names: + type = spec_domain_dict[response]['range'] + symbols.append((response, type, False)) + + parser = TextToPysmtParser() + parser.init_variables(symbols=symbols) + + self.verifier = MarabouVerifier(parser=parser) + self.verifier.initialize(spec_domain_dict) + + @ClassProperty + def smlp_true(self): + return pysmt.shortcuts.TRUE() + + def smlp_var(self, var): + return self.verifier.parser.get_symbol(var) + + def create_query(self, query_form=None): + self.verifier.parser.handle_ite_formula(query_form, is_form2=True) + + def create_query_and_beta(self, query, beta): + return self.verifier.parser.and_(query, beta) + + def substitute_objective_with_witness(self, *args, **kwargs): + stable_witness_terms = kwargs["stable_witness_terms"] + objv_term = kwargs["objv_term"] + + substitution = {} + for symbol, value in stable_witness_terms.items(): + symbol = self.verifier.parser.get_symbol(symbol) + substitution[symbol] = Real(value) + # Apply the substitution + return self.verifier.parser.simplify(objv_term.substitute(substitution)) + + def generate_rad_term(self, **kwargs): + rad = kwargs["rad"] + return float(rad) + + def get_rad_term(self, **kwargs): + rad = kwargs["rad"] + return float(rad) + + def create_theta_form(self, **kwargs): + witness = kwargs["witness"] + var = kwargs["var"] + rad_term = kwargs["rad_term"] + theta_form = kwargs["theta_form"] + + rad_term = float(rad_term) + value = float(witness) + PYSMT_var = self.verifier.parser.get_symbol(var) + type = pysmt.shortcuts.Int if str(PYSMT_var.get_type()) == "Int" else Real + calc_type = int if str(PYSMT_var.get_type()) == "Int" else float + lower = calc_type(value - rad_term) + lower = type(lower) + upper = calc_type(value + rad_term) + upper = type(upper) + theta_form = self.verifier.parser.and_(theta_form, PYSMT_var >= lower, PYSMT_var <= upper) + return theta_form + + def create_alpha_or_eta_form(self, **kwargs): + alpha_or_eta_form = kwargs["alpha_or_eta_form"] + mx = kwargs["mx"] + mn = kwargs["mn"] + v = kwargs["v"] + + symbol_v = self.smlp_var(v) + form = self.smlp_and(symbol_v >= mn, symbol_v <= mx) + return self.simplify(self.smlp_and(alpha_or_eta_form, form)) + + def simplify(self, expression): + return self.verifier.parser.simplify(expression) + + def parse(self, expression): + return self.verifier.parser.parse(expression) + + def GE(self, *args): + return args[0] >= args[1] + + def parse_ast(self, *args, **kwargs): + expression = kwargs['expression'] + return self.parse(expression) + + def create_solver(self, *args, **kwargs): + temp = kwargs.get('temp', False) + if temp: + self.temp_solver = MarabouVerifier(parser=self.verifier.parser, + variable_ranges=self.verifier.variable_ranges, + is_temp=True) + + else: + self.verifier.reset() + return self + + def add_formula(self, formula, **kwargs): + need_simplification = kwargs.get("need_simplification", False) + self.verifier.apply_restrictions(formula, need_simplification=need_simplification) + + def check(self, *args, **kwargs): + temp = kwargs.get("temp", False) + if temp: + result = self.temp_solver.solve() + self.temp_solver = None + return result + else: + return self.verifier.solve() + + def generate_theta(self, *args, **kwargs): + pass + + def add_not_query(self, *args, **kwargs): + query = kwargs["query"] + temp = kwargs.get("temp", False) + + + def create_counter_example(self, *args, **kwargs): + formulas = kwargs["formulas"] + query = kwargs["query"] + + self.temp_solver = MarabouVerifier( parser=self.verifier.parser, + variable_ranges=self.verifier.variable_ranges, + is_temp=True) + for formula in formulas: + self.temp_solver.apply_restrictions(formula) + + negation = self.temp_solver.parser.propagate_negation(query) + z3_equiv = self.temp_solver.parser.handle_ite_formula(negation, handle_ite=False) + self.temp_solver.apply_restrictions(negation, need_simplification=True) + return self + + def substitute(self, *args, **kwargs): + var = kwargs["var"] + substitutions = kwargs["substitutions"] + + for x in list(substitutions.keys()): + temp = substitutions[x] + del substitutions[x] + substitutions[self.smlp_var(x)] = temp + + return self.simplify(var.substitute(substitutions)) \ No newline at end of file diff --git a/src/smlp_py/solvers/universal_solver.py b/src/smlp_py/solvers/universal_solver.py new file mode 100644 index 00000000..374ea908 --- /dev/null +++ b/src/smlp_py/solvers/universal_solver.py @@ -0,0 +1,73 @@ +from enum import Enum +import types +from src.smlp_py.solvers.z3.solver import Form2_Solver +from src.smlp_py.solvers.marabou.solver import Pysmt_Solver + + +class Solver: + class Version(Enum): + FORM2 = 0 + PYSMT = 1 + + _instance = None + version = None + + def __new__(cls, *args, **kwargs): + version = kwargs["version"] + if isinstance(version, cls.Version): + cls.version = version + else: + raise ValueError("Must be a valid version") + + if cls._instance is None and isinstance(cls.version, cls.Version): + if cls.version == cls.Version.PYSMT: + specs = kwargs["specs"] + cls._instance = Pysmt_Solver(specs) + else: + cls._instance = Form2_Solver() + cls._map_instance_methods() + return cls._instance + + @classmethod + def _map_instance_methods(cls): + """Automatically maps all methods from the instance to the SingletonFactory class.""" + for base_class in cls._instance.__class__.__mro__: + for name, method in base_class.__dict__.items(): + if isinstance(method, types.FunctionType): + if not hasattr(cls, name): + setattr(cls, name, cls._create_delegator(name)) + + @classmethod + def _create_delegator(cls, method_name): + """Create a method that delegates the call to the _instance.""" + def delegator(*args, **kwargs): + return getattr(cls._instance, method_name)(*args, **kwargs) + return delegator + + + + + # + # @classmethod + # def _map_instance_properties(cls): + # """Automatically maps all properties from the instance to the Solver class.""" + # for name, attribute in cls._instance.__class__.__dict__.items(): + # if isinstance(attribute, property): + # # Map property to Solver class + # if not hasattr(cls, name): + # setattr(cls, name, cls._create_property_delegator(name)) + # + # @classmethod + # def _create_property_delegator(cls, property_name): + # """Create a property that delegates access to the _instance.""" + # def getter(self): + # return getattr(self._instance, property_name) + # + # def setter(self, value): + # setattr(self._instance, property_name, value) + # + # def deleter(self): + # delattr(self._instance, property_name) + # + # # Return a property with the mapped getter, setter, and deleter + # return property(getter, setter, deleter) diff --git a/src/smlp_py/solvers/z3/operations.py b/src/smlp_py/solvers/z3/operations.py new file mode 100644 index 00000000..195bfb4e --- /dev/null +++ b/src/smlp_py/solvers/z3/operations.py @@ -0,0 +1,191 @@ +import smlp +import operator as op +from src.smlp_py.solvers.abstract_solver import conditional_cache + + +class SMLPOperations: + @property + @conditional_cache # @functools.cache + def smlp_true(self): + return smlp.true + + @property + @conditional_cache # @functools.cache + def smlp_false(self): + return smlp.false + + @property + @conditional_cache # @functools.cache + def smlp_real(self): + return smlp.Real + + @property + @conditional_cache # @functools.cache + def smlp_integer(self): + return smlp.Integer + + @conditional_cache # @functools.cache + def smlp_var(self, var): + return smlp.Var(var) + + @conditional_cache # @functools.cache + def smlp_cnst(self, const): + return smlp.Cnst(const) + + # rationals + @conditional_cache # @functools.cache + def smlp_q(self, const): + return smlp.Q(const) + + # reals + @conditional_cache # @functools.cache + def smlp_r(self, const): + return smlp.R(const) + + # logical not (logic negation) + @conditional_cache # @functools.cache + def smlp_not(self, form: smlp.form2): + # res1 = ~form + res2 = op.inv(form) + # assert res1 == res2 + return res2 # ~form + + # logical and (conjunction) + @conditional_cache # @functools.cache + def smlp_and(self, form1: smlp.form2, form2: smlp.form2): + ''' test 83 gets stuck with this simplification + if form1 == smlp.true: + return form2 + if form2 == smlp.true: + return form1 + ''' + res1 = op.and_(form1, form2) + # res2 = form1 & form2 + # print('res1', res1, type(res1)); print('res2', res2, type(res2)) + # assert res1 == res2 + return res1 # form1 & form2 + + # conjunction of possibly more than two formulas + # @functools.cache -- error: unhashable type: 'list' + def smlp_and_multi(self, form_list: list[smlp.form2]): + res = self.smlp_true + ''' + for i, form in enumerate(form_list): + res = form if i == 0 else self.smlp_and(res, form) + ''' + for form in form_list: + res = form if res is self.smlp_true else self.smlp_and(res, form) + return res + + # logical or (disjunction) + @conditional_cache # @functools.cache + def smlp_or(self, form1: smlp.form2, form2: smlp.form2): + res1 = op.or_(form1, form2) + # res2 = form1 | form2 + # assert res1 == res2 + return res1 # form1 | form2 + + # disjunction of possibly more than two formulas + # @functools.cache -- error: unhashable type: 'list' + def smlp_or_multi(self, form_list: list[smlp.form2]): + res = self.smlp_false + ''' + for i, form in enumerate(form_list): + res = form if i == 0 else self.smlp_or(res, form) + ''' + for form in form_list: + res = form if res is self.smlp_false else self.smlp_or(res, form) + return res + + # logical implication + @conditional_cache # @functools.cache + def smlp_implies(self, form1: smlp.form2, form2: smlp.form2): + return self.smlp_or(self.smlp_not(form1), form2) + + # addition + @conditional_cache # @functools.cache + def smlp_add(self, term1: smlp.term2, term2: smlp.term2): + return op.add(term1, term2) + + # sum of possibly more than two formulas + # @functools.cache -- error: unhashable type: 'list' + def smlp_add_multi(self, term_list: list[smlp.term2]): + for i, term in enumerate(term_list): + res = term if i == 0 else self.smlp_add(res, term) + return res + + # subtraction + @conditional_cache # @conditional_cache #@functools.cache + def smlp_sub(self, term1: smlp.term2, term2: smlp.term2): + return op.sub(term1, term2) + + # multiplication + @conditional_cache # @functools.cache + def smlp_mult(self, term1: smlp.term2, term2: smlp.term2): + return op.mul(term1, term2) + + # TODO: !!! check that term2 does not evaluate to term 0 ??? + + # Do this before calling smlp_div, whenver possible? + @conditional_cache # @functools.cache + def smlp_div(self, term1: smlp.term2, term2: smlp.term2): + # return self.smlp_mult(self.smlp_cnst(self.smlp_q(1)) / term2, term1) + return self.smlp_mult(op.truediv(self.smlp_cnst(self.smlp_q(1)), term2), term1) + + @conditional_cache # @functools.cache + def smlp_pow(self, term1: smlp.term2, term2: smlp.term2): + return op.pow(term1, term2) + + # equality + @conditional_cache # @functools.cache + def smlp_eq(self, term1: smlp.term2, term2: smlp.term2): + res1 = op.eq(term1, term2) + # res2 = term1 == term2; print('res1', res1, 'res2', res2) + # assert res1 == res2 + return res1 + + # operator != (not equal) + @conditional_cache # @functools.cache + def smlp_ne(self, term1: smlp.term2, term2: smlp.term2): + res1 = op.ne(term1, term2) + # res2 = term1 != term2; print('res1', res1, 'res2', res2) + # assert res1 == res2 + return res1 + + # operator < + @conditional_cache # @functools.cache + def smlp_lt(self, term1: smlp.term2, term2: smlp.term2): + return op.lt(term1, term2) + + # operator <= + + @conditional_cache # @functools.cache + def smlp_le(self, term1: smlp.term2, term2: smlp.term2): + return op.le(term1, term2) + + # operator > + @conditional_cache # @functools.cache + def smlp_gt(self, term1: smlp.term2, term2: smlp.term2): + return op.gt(term1, term2) + + # operator >= + + @conditional_cache # @functools.cache + def smlp_ge(self, term1: smlp.term2, term2: smlp.term2): + return op.ge(term1, term2) + + # if-thne-else operation + @conditional_cache # @functools.cache + def smlp_ite(self, form: smlp.form2, term1: smlp.term2, term2: smlp.term2): + return smlp.Ite(form, term1, term2) + + # this function performs substitution of variables in term2: + # it substitutes occurrences of the keys in subst_dict with respective values, in term2 term. + # @functools.cache + def smlp_subst(self, term: smlp.term2, subst_dict: dict): + return smlp.subst(term, subst_dict) + + # simplifies a ground term to the respective constant; takes als a s + # @functools.cache + def smlp_cnst_fold(self, term: smlp.term2, subst_dict: dict): + return smlp.cnst_fold(term, subst_dict) diff --git a/src/smlp_py/solvers/z3/solver.py b/src/smlp_py/solvers/z3/solver.py new file mode 100644 index 00000000..fb4986b8 --- /dev/null +++ b/src/smlp_py/solvers/z3/solver.py @@ -0,0 +1,118 @@ +import smlp +from src.smlp_py.solvers.abstract_solver import AbstractSolver +from src.smlp_py.solvers.z3.operations import SMLPOperations + + +class Form2_Solver(AbstractSolver, SMLPOperations): + verifier = None + smlp_term_instance = None + terms = None + + def __init__(self): + super().__init__() + # self.verifier = verifier + # self.smlp_term_instance = smlp_term_instance + + @property + def smlp_true(self): + return smlp.true + + def create_query(self, query_form=None): + return query_form + + def create_query_and_beta(self, query, beta): + return self.smlp_term_instance.smlp_and(query, beta) + + def substitute_objective_with_witness(self, *args, **kwargs): + stable_witness_terms = kwargs["stable_witness_terms"] + objv_term = kwargs["objv_term"] + + return smlp.subst(objv_term, stable_witness_terms) + + def generate_rad_term(self, *args, **kwargs): + rad = kwargs["rad"] + delta_rel = kwargs["delta_rel"] + var_term = kwargs["var_term"] + candidate = kwargs["candidate"] + + rad_term = self.smlp_cnst(rad) + if delta_rel is not None: # radius for a lemma -- cex holds values of candidate counter-example + rad_term = rad_term * abs(var_term) + else: # radius for excluding a candidate -- cex holds values of the candidate + rad_term = rad_term * abs(candidate) + + return rad_term + + def create_theta_form(self, *args, **kwargs): + theta_form = kwargs["theta_form"] + witness = kwargs["witness"] + var_term = kwargs["var_term"] + rad_term = kwargs["rad_term"] + + return self.smlp_and(theta_form, ((abs(var_term - witness)) <= rad_term)) + + def get_rad_term(self, *args, **kwargs): + rad = kwargs["rad"] + return self.smlp_cnst(rad) + + def create_alpha_or_eta_form(self, *args, **kwargs): + alpha_or_eta_form = kwargs["alpha_or_eta_form"] + is_in_spec = kwargs["is_in_spec"] + is_disjunction = kwargs["is_disjunction"] + is_alpha = kwargs["is_alpha"] + mx = kwargs["mx"] + mn = kwargs["mn"] + v = kwargs["v"] + + if is_disjunction and is_alpha and is_in_spec: + rng = self.smlp_or_multi([self.smlp_eq(self.smlp_var(v), self.smlp_cnst(i)) for i in range(mn, mx + 1)]) + else: + rng = self.smlp_and(self.smlp_var(v) >= self.smlp_cnst(mn), self.smlp_var(v) <= self.smlp_cnst(mx)) + + return self.smlp_and(alpha_or_eta_form, rng) + + def GE(self, *args): + return args[0] >= args[1] + + def parse_ast(self, *args, **kwargs): + expression = kwargs['expression'] + parser = kwargs['parser'] + return parser(expression) + + def create_solver(self, *args, **kwargs): + create_solver = kwargs["create_solver"] + domain = kwargs["domain"] + model_full_term_dict = kwargs["model_full_term_dict"] + incremental = kwargs["incremental"] + solver_logic = kwargs["solver_logic"] + + self.verifier = create_solver(domain, model_full_term_dict, incremental, solver_logic) + return self + + def add_formula(self, *args, **kwargs): + formula = kwargs["formula"] + + self.verifier.add(formula) + + def check(self): + return self.verifier.check(), None + + def generate_theta(self, *args, **kwargs): + pass + + def create_counter_example(self, *args, **kwargs): + formulas = kwargs["formulas"] + query = kwargs["query"] + + self.create_solver(*args, **kwargs) + for formula in formulas: + self.add_formula(formula) + + self.add_formula(self.smlp_not(query)) + return self + + def substitute(self, *args, **kwargs): + var = kwargs["var"] + substitutions = kwargs["substitutions"] + + return self.smlp_cnst_fold(var, substitutions) From cbca5c70107cf5b8239620a7a866400bd1aa9a66 Mon Sep 17 00:00:00 2001 From: konstantopoulos-tetractys <37417801+ntinouldinho@users.noreply.github.com> Date: Fri, 23 Aug 2024 19:10:01 +0100 Subject: [PATCH 25/28] continue merging of solver --- src/smlp_py/NN_verifiers/verifiers.py | 25 ++++--- src/smlp_py/smlp_operations.py | 0 src/smlp_py/smlp_optimize.py | 78 +++++++++++----------- src/smlp_py/smlp_query.py | 29 ++++---- src/smlp_py/smlp_terms.py | 25 ++++--- src/smlp_py/smtlib/text_to_sympy.py | 80 ++++++++++++++++++----- src/smlp_py/solvers/abstract_solver.py | 26 ++++---- src/smlp_py/solvers/marabou/operations.py | 7 +- src/smlp_py/solvers/marabou/solver.py | 34 ++++++++-- src/smlp_py/solvers/z3/solver.py | 31 ++++++++- 10 files changed, 225 insertions(+), 110 deletions(-) delete mode 100644 src/smlp_py/smlp_operations.py diff --git a/src/smlp_py/NN_verifiers/verifiers.py b/src/smlp_py/NN_verifiers/verifiers.py index 871d8d1a..5f59c282 100755 --- a/src/smlp_py/NN_verifiers/verifiers.py +++ b/src/smlp_py/NN_verifiers/verifiers.py @@ -95,6 +95,8 @@ def __init__(self, parser=None, variable_ranges=None, is_temp=False): self.network_num_vars = None self.init_variables(is_temp=is_temp) + self.applied_equations = [] + if self.variable_ranges: self.initialize() @@ -115,6 +117,7 @@ def reset(self): self.network = Marabou.read_tf('model.pb') self.unscaled_variables = [] self.add_unscaled_variables() + self.applied_equations = [] # Default bounds for network for equation in self.equations: self.apply_restrictions(equation) @@ -269,6 +272,7 @@ def add_equality(self, variable, value): def apply_restrictions(self, formula, need_simplification=False): + self.applied_equations.append(formula) formula = self.parser.simplify(formula) conjunctions, disjunctions = self.process_formula(formula) @@ -325,7 +329,7 @@ def is_negation_of_ite(self, formula): def create_equation(self, formula, from_and=False, need_simplification=False): equations = [] - formula = self.parser.simplify(formula) + formula = self.parser.simplify(formula) if not need_simplification else self.parser.z3_simplify(formula) if formula.is_and(): equation = [self.create_equation(eq, from_and=True) for eq in formula.args()] @@ -333,6 +337,10 @@ def create_equation(self, formula, from_and=False, need_simplification=False): elif formula.is_le() or formula.is_lt() or formula.is_equals(): res = self.parser.extract_components(formula, need_simplification) equations.append(self.transform_pysmt_to_marabou_equation(res)) + elif formula.is_not(): + negation = self.parser.propagate_negation(formula) + res = self.parser.extract_components(negation, need_simplification) + equations.append(self.transform_pysmt_to_marabou_equation(res)) return equations[0] if from_and else equations @@ -342,13 +350,12 @@ def process_disjunctions(self, disjunctions, need_simplification=False): # split the disjunction into separate formulas for formula in disjunction.args(): res, formulas = self.is_negation_of_ite(formula) - if res: - for form in formulas: - equation = self.create_equation(form, from_and=False, need_simplification=need_simplification) - marabou_disjunction.append(equation) - else: + formulas = formulas if res else [formula] + for formula in formulas: + # formula = self.parser.z3_simplify(formula) equation = self.create_equation(formula, from_and=False, need_simplification=need_simplification) - marabou_disjunction.append(equation) + if equation: + marabou_disjunction.append(equation) if len(marabou_disjunction) > 0: self.network.addDisjunctionConstraint(marabou_disjunction) @@ -374,6 +381,7 @@ def traverse(node, source=[]): return conjunctions, disjunctions def process_comparison(self, formula, need_simplification=False): + formula = self.parser.z3_simplify(formula) if formula.is_le() or formula.is_lt() or formula.is_equals(): symbols, comparison, constant = self.parser.extract_components(formula, need_simplification) @@ -414,6 +422,7 @@ def find_witness(self, witness): def solve(self): try: + # Options(verbose=0, cores=5) results = self.network.solve() if results and results[0] == 'unsat': return "UNSAT", {"result":"UNSAT", "witness": {}} @@ -424,7 +433,7 @@ def solve(self): return None def add_disjunction(self,): - pass + return diff --git a/src/smlp_py/smlp_operations.py b/src/smlp_py/smlp_operations.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/smlp_py/smlp_optimize.py b/src/smlp_py/smlp_optimize.py index 4e56738a..2ac56af2 100755 --- a/src/smlp_py/smlp_optimize.py +++ b/src/smlp_py/smlp_optimize.py @@ -2,6 +2,7 @@ # This file is part of smlp. import pysmt import smlp +from oauthlib.uri_validate import query from smlp_py.smlp_terms import SmlpTerms, ModelTerms, ScalerTerms from smlp_py.smlp_query import SmlpQuery from smlp_py.smlp_utils import (str_to_bool, np_JSONEncoder) @@ -266,14 +267,15 @@ def optimize_single_objective(self, model_full_term_dict:dict, objv_name:str, ob else: T = (l + u) / 2 #quer_form = objv_term > smlp.Cnst(T) - quer_form = objv_term >= smlp.Cnst(T) - quer_form = self._modelTermsInst.verifier.parser.handle_ite_formula(quer_form, is_form2=True) if self._ENABLE_PYSMT else quer_form + quer_form = objv_term >= Solver.smlp_cnst(T) + quer_form = Solver.handle_ite_formula(formula=quer_form) # quer_form = solver.create_query() quer_expr = '{} >= {}'.format(objv_expr, str(T)) if objv_expr is not None else None quer_name = objv_name + '_' + str(T) - if not beta == smlp.true: - quer_and_beta = self._modelTermsInst.verifier.parser.and_(quer_form, beta) if self._ENABLE_PYSMT else self._smlpTermsInst.smlp_and(quer_form, beta) + if not beta == Solver._instance.smlp_true: + quer_and_beta = Solver.smlp_and(quer_form, beta) + # self._modelTermsInst.verifier.parser.and_(quer_form, beta) if self._ENABLE_PYSMT else self._smlpTermsInst.smlp_and(quer_form, beta) # quer_and_beta = solver.create_query_and_beta(quer_form, beta) else: quer_and_beta = quer_form @@ -335,16 +337,19 @@ def optimize_single_objective(self, model_full_term_dict:dict, objv_name:str, ob l_prev = l # save the value of l, it is for reporting only. #if objv_expr is not None: # the objective is not a symbolic max_min term, we may need its value, at least to see search progress - if self._ENABLE_PYSMT: - substitution = {} - for symbol, value in stable_witness_terms.items(): - symbol = self._modelTermsInst.verifier.parser.get_symbol(symbol) - substitution[symbol] = Real(value) - # Apply the substitution - objv_witn_val_term = self._modelTermsInst.verifier.parser.simplify(pysmt_min_objs.substitute(substitution)) - else: - objv_witn_val_term = smlp.subst(objv_term, stable_witness_terms); #print('objv_witn_val_term', objv_witn_val_term) - #using objective values as lower bounds is not sound since objective value in sat model is the ceneter-point value + objv_witn_val_term = Solver.substitute_objective_with_witness(stable_witness_terms=stable_witness_terms, objv_term=objv_term) + # if self._ENABLE_PYSMT: + # substitution = {} + # for symbol, value in stable_witness_terms.items(): + # symbol = self._modelTermsInst.verifier.parser.get_symbol(symbol) + # substitution[symbol] = Real(value) + # # Apply the substitution + # objv_witn_val_term = self._modelTermsInst.verifier.parser.simplify(pysmt_min_objs.substitute(substitution)) + # else: + # objv_witn_val_term = smlp.subst(objv_term, stable_witness_terms); #print('objv_witn_val_term', objv_witn_val_term) + + + #using objective values as lower bounds is not sound since objective value in sat model is the ceneter-point value # and the objective's value is not guaranteed to be a lower bound in entire stability region #objv_witn_val = self._smlpTermsInst.ground_smlp_expr_to_value(objv_witn_val_term, sat_approx, sat_precision) #assert objv_witn_val >= T @@ -486,21 +491,22 @@ def active_objectives_max_min_bounds(self, model_full_term_dict:dict, objv_terms #print('thresholds t', t, 'objv_terms_dict', objv_terms_dict) for j, (objv_name, objv_term) in enumerate(objv_terms_dict.items()): if t[j] is not None: - if self._ENABLE_PYSMT: - eta_F_t = TextToPysmtParser.and_(eta_F_t, pysmt_objv_term > Real(t[j])) - else: - eta_F_t = self._smlpTermsInst.smlp_and(eta_F_t, objv_term > smlp.Cnst(t[j])) + eta_F_t = Solver.calculate_eta_F_t(eta=eta_F_t, term=objv_term, val=t[j]) + # if self._ENABLE_PYSMT: + # eta_F_t = TextToPysmtParser.and_(eta_F_t, pysmt_objv_term > Real(t[j])) + # else: + # eta_F_t = self._smlpTermsInst.smlp_and(eta_F_t, objv_term > smlp.Cnst(t[j])) else: min_name = min_name + '_' + objv_name if min_name != '' else objv_name - if self._ENABLE_PYSMT: - pysmt_objv_term = pysmt_objv_terms_dict[objv_name] - if pysmt_min_objs is not None: - pysmt_min_objs = TextToPysmtParser.ite_(pysmt_objv_term < pysmt_min_objs, pysmt_objv_term, pysmt_min_objs) - else: - pysmt_min_objs = pysmt_objv_term + # if self._ENABLE_PYSMT: + # pysmt_objv_term = pysmt_objv_terms_dict[objv_name] + # if pysmt_min_objs is not None: + # pysmt_min_objs = TextToPysmtParser.ite_(pysmt_objv_term < pysmt_min_objs, pysmt_objv_term, pysmt_min_objs) + # else: + # pysmt_min_objs = pysmt_objv_term # else: if min_objs is not None: - min_objs = smlp.Ite(objv_term < min_objs, objv_term, min_objs) + min_objs = Solver.smlp_ite(objv_term < min_objs, objv_term, min_objs) else: min_objs = objv_term @@ -699,9 +705,9 @@ def optimize_pareto_objectives(self, feat_names:list[str], resp_names:list[str], self._modelTermsInst.compute_objectives_terms(objv_names, objv_exprs, objv_bounds_dict, scale_objectives) pysmt_objv_terms_dict = None - pysmt_objv_terms_dict, pysmt_orig_objv_terms_dict, pysmt_scaled_objv_terms_dict = \ - self._modelTermsInst.pysmt_compute_objectives_terms(objv_names, objv_exprs, objv_bounds_dict, - scale_objectives) + # pysmt_objv_terms_dict, pysmt_orig_objv_terms_dict, pysmt_scaled_objv_terms_dict = \ + # self._modelTermsInst.pysmt_compute_objectives_terms(objv_names, objv_exprs, objv_bounds_dict, + # scale_objectives) objv_count = len(objv_names) objv_enum = range(objv_count) @@ -808,16 +814,14 @@ def sanity_check_fixed_objv_thresholds(t:list[float], fixed_onjv_dict): # quer_form = self._modelTermsInst.parser.and_(quer_form, # list(pysmt_objv_terms_dict.values())[i] > pysmt.shortcuts.Real(t[i])) # else: - quer_form = self._smlpTermsInst.smlp_and(quer_form, list(objv_terms_dict.values())[i] > Solver.smlp_cnst(t[i])) + quer_form = Solver.smlp_and(quer_form, list(objv_terms_dict.values())[i] > Solver.smlp_cnst(t[i])) #print('queryform', quer_form) - if not beta == smlp.true: - quer_and_beta = self._modelTermsInst.verifier.parser.and_(quer_form, - beta) if self._ENABLE_PYSMT else self._smlpTermsInst.smlp_and(quer_form, beta) + if not beta == Solver._instance.smlp_true: + quer_and_beta = Solver.smlp_and(quer_form, beta) else: quer_and_beta = quer_form - if self._ENABLE_PYSMT: - quer_and_beta = self._modelTermsInst.parser.simplify(quer_and_beta) + quer_and_beta = Solver.z3_simplify(quer_and_beta) opt_quer_name = 'thresholds_' + '_'.join(str(x) for x in t) + '_check' quer_res = self._queryInst.query_condition(True, model_full_term_dict, opt_quer_name, 'True', quer_and_beta, @@ -916,9 +920,9 @@ def smlp_optimize(self, syst_expr_dict:dict, algo:str, model:dict, X:pd.DataFram with open(self.optimization_results_file+'.json', 'w') as f: json.dump(self.mode_status_dict, f, indent='\t', cls=np_JSONEncoder) - # initiliase Solver - solver = Solver(specs=(feat_names, resp_names, self._modelTermsInst._specInst.get_spec_domain_dict), version=Solver.Version.PYSMT if use_pysmt else Solver.Version.FORM2) - + # initialise Solver + Solver(specs=(feat_names, resp_names, self._modelTermsInst._specInst.get_spec_domain_dict), + version=Solver.Version.PYSMT if use_pysmt else Solver.Version.FORM2) domain, syst_term_dict, model_full_term_dict, eta, alpha, beta, interface_consistent, model_consistent = \ diff --git a/src/smlp_py/smlp_query.py b/src/smlp_py/smlp_query.py index ae66cfab..3c5cfc1b 100644 --- a/src/smlp_py/smlp_query.py +++ b/src/smlp_py/smlp_query.py @@ -565,7 +565,7 @@ def query_condition(self, universal, model_full_term_dict:dict, quer_name:str, q candidate_solver.add_formula(eta, need_simplification=True) candidate_solver.add_formula(alpha) candidate_solver.add_formula(quer, need_simplification=True) - + print("IN QUERY CONDITION") # res = self.smlp_solver_check(solver, # 'interface_consistency' if model_full_term_dict is None else 'model_consistency', # equations={'alpha': alpha, 'eta': eta}) @@ -646,14 +646,14 @@ def query_condition(self, universal, model_full_term_dict:dict, quer_name:str, q else: lemma = self.generalize_counter_example(cem); #print('lemma', lemma) theta = self._modelTermsInst.compute_stability_formula_theta(lemma, delta, theta_radii_dict, universal) - if self._ENABLE_PYSMT: - theta_negation = self._modelTermsInst.parser.propagate_negation(theta) - # self._modelTermsInst.verifier.add_permanent_constraint(theta_negation) - self._modelTermsInst.verifier.apply_restrictions(theta_negation) - print("PYSMT THETA ADDED ", theta_negation) - else: - candidate_solver.add(self._smlpTermsInst.smlp_not(theta)) - print("FORM2 THETA ADDED ", self._smlpTermsInst.smlp_not(theta)) + Solver.apply_theta(solver=candidate_solver, formula=theta) + # if self._ENABLE_PYSMT: + # theta_negation = self._modelTermsInst.parser.propagate_negation(theta) + # # self._modelTermsInst.verifier.add_permanent_constraint(theta_negation) + # self._modelTermsInst.verifier.apply_restrictions(theta_negation) + # print("PYSMT THETA ADDED ", theta_negation) + # else: + # candidate_solver.add(self._smlpTermsInst.smlp_not(theta)) continue elif c_result == "unsat": #isinstance(ce, smlp.unsat): #print('candidate stable -- return candidate') @@ -671,11 +671,12 @@ def query_condition(self, universal, model_full_term_dict:dict, quer_name:str, q assert quer_ce_val return {'query_status':'STABLE_SAT', 'witness':witness_vals_dict, 'feasible':feasible} else: - if self._ENABLE_PYSMT: - return {'query_status':'STABLE_SAT', 'witness':ca['witness'], 'feasible':feasible} - else: - return {'query_status':'STABLE_SAT', 'witness':ca_model, 'feasible':feasible} - elif result == "unsat": #isinstance(ca, smlp.unsat): + return {'query_status':'STABLE_SAT', 'witness':ca_model, 'feasible':feasible} + # if self._ENABLE_PYSMT: + # return {'query_status':'STABLE_SAT', 'witness':ca['witness'], 'feasible':feasible} + # else: + # return {'query_status':'STABLE_SAT', 'witness':ca_model, 'feasible':feasible} + elif result == "unsat": self._query_logger.info('Query completed with result: UNSAT (unsatisfiable)') if feasible is None: feasible = False diff --git a/src/smlp_py/smlp_terms.py b/src/smlp_py/smlp_terms.py index a5251f4f..593c48c2 100644 --- a/src/smlp_py/smlp_terms.py +++ b/src/smlp_py/smlp_terms.py @@ -5,6 +5,8 @@ import numpy as np import pandas as pd import keras +from Cython.Compiler.TreePath import operations +from pysmt.fnode import FNode from sklearn.tree import _tree import json import ast @@ -1294,21 +1296,22 @@ def _unscaled_name(self, name): # x_scaled obtained from x using min_max scaler to range [0, 1] (which is the same as normalizin x), # orig_min stands for min(x) and orig_max stands for max(x). Note that 1 / (max(x) - min(x)) is a # rational constant, it is defined to smlp instance as a fraction (thus there is no loss of precision). - def feature_scaler_to_term(self, orig_feat_name, scaled_feat_name, orig_min, orig_max): + def feature_scaler_to_term(self, orig_feat_name, scaled_feat_name, orig_min, orig_max, allow_solver=False): #print('feature_scaler_to_term', 'orig_min', orig_min, type(orig_min), 'orig_max', orig_max, type(orig_max), flush=True) + operations = Solver if allow_solver else self if orig_min == orig_max: - return Solver.smlp_cnst(0) #smlp.Cnst(0) # same as returning smlp.Cnst(smlp.Q(0)) + return operations.smlp_cnst(0) #smlp.Cnst(0) # same as returning smlp.Cnst(smlp.Q(0)) else: - return Solver.smlp_mult( - Solver.smlp_cnst(Solver.smlp_q(1) / Solver.smlp_q(orig_max - orig_min)), - (Solver.smlp_var(orig_feat_name) - Solver.smlp_cnst(orig_min))) + return operations.smlp_mult( + operations.smlp_cnst(operations.smlp_q(1) / operations.smlp_q(orig_max - orig_min)), + (operations.smlp_var(orig_feat_name) - operations.smlp_cnst(orig_min))) ####return self.smlp_div(self.smlp_var(orig_feat_name) - self.smlp_cnst(orig_min), self.smlp_cnst(orig_max) - self.smlp_cnst(orig_min)) ####return smlp.Cnst(smlp.Q(1) / smlp.Q(orig_max - orig_min)) * (smlp.Var(orig_feat_name) - smlp.Cnst(orig_min)) # Computes dictionary with features as keys and scaler terms as values - def feature_scaler_terms(self, data_bounds, feat_names): + def feature_scaler_terms(self, data_bounds, feat_names, allow_solver=False): return dict([(self._scaled_name(feat), self.feature_scaler_to_term(feat, self._scaled_name(feat), - data_bounds[feat]['min'], data_bounds[feat]['max'])) for feat in feat_names]) + data_bounds[feat]['min'], data_bounds[feat]['max'], allow_solver=allow_solver)) for feat in feat_names]) # Computes term x from column x_scaled using expression x = x_scaled * (max_x - min_x) + x_min. # Argument orig_feat_name is name for column x, argument scaled_feat_name is the name of scaled column @@ -1672,12 +1675,14 @@ def compute_objectives_terms(self, objv_names, objv_exprs, objv_bounds, scale_ob #print('objv_exprs', objv_exprs) if objv_exprs is None: return None, None, None, None - orig_objv_terms_dict = dict([(objv_name, Solver.parse_ast(objv_expr)) \ + orig_objv_terms_dict = dict([(objv_name, Solver.parse_ast(parser=self.ast_expr_to_term, expression=objv_expr)) \ for objv_name, objv_expr in zip(objv_names, objv_exprs)]) #self._smlpTermsInst. #print('orig_objv_terms_dict', orig_objv_terms_dict) if scale_objv: - scaled_objv_terms_dict = self.feature_scaler_terms(objv_bounds, objv_names) # ._scalerTermsInst + # allow_pysmt = isinstance(next(iter(orig_objv_terms_dict), FNode) + + scaled_objv_terms_dict = self.feature_scaler_terms(objv_bounds, objv_names, allow_solver=True) # ._scalerTermsInst #print('scaled_objv_terms_dict', scaled_objv_terms_dict) objv_terms_dict = {} @@ -1686,7 +1691,7 @@ def compute_objectives_terms(self, objv_names, objv_exprs, objv_bounds, scale_ob x = list(orig_objv_terms_dict.keys())[i]; #print('x', x); print('arg', orig_objv_terms_dict[x]) - objv_terms_dict[k] = Solver.substitute(var=v, substitutions={Solver.smlp_cnst(x): orig_objv_terms_dict[x]}) + objv_terms_dict[k] = Solver.substitute(var=v, substitutions={x: orig_objv_terms_dict[x]}) # objv_terms_dict[k] = self.smlp_cnst_fold(v, {x: orig_objv_terms_dict[x]}) #objv_terms_dict = scaled_objv_terms_dict else: diff --git a/src/smlp_py/smtlib/text_to_sympy.py b/src/smlp_py/smtlib/text_to_sympy.py index 81147fa3..fdfa4ac0 100755 --- a/src/smlp_py/smtlib/text_to_sympy.py +++ b/src/smlp_py/smtlib/text_to_sympy.py @@ -175,7 +175,7 @@ def conjunction_to_disjunction(formula): raise ValueError("Input formula is not a conjunction") def is_comparison(self, node: FNode) -> bool: - return node.is_le() or node.is_lt() or node.is_ge() or node.is_gt() or node.is_equals() + return node.is_le() or node.is_lt() or node.is_equals() def create_integer_disjunction(self, variable, values): variable = self.get_symbol(variable) @@ -220,6 +220,18 @@ def decide_comparator(self, formula): else: return None + def apply_comparator(self, comparator, a, b): + match comparator: + case '<=': + return a <= b + case '<': + return a < b + case "=": + return a == b + case _: + return None + + def extract_coefficient(self, symbol): coeff = [] # possible formats @@ -236,9 +248,9 @@ def extract_coefficient(self, symbol): return coeff def extract_components(self, comparison: FNode, need_simplification=False): - if need_simplification: - smtlib = self.extract_smtlib(comparison) - comparison = self.handle_ite_formula(smtlib, handle_ite=False) + # if need_simplification: + # smtlib = self.extract_smtlib(comparison) + # comparison = self.handle_ite_formula(smtlib, handle_ite=False) left = comparison.arg(0) right = comparison.arg(1) @@ -383,7 +395,10 @@ def add_symbol(self, name, symbol_type, is_input=True, nn_type='real'): store.append((name, nn_type)) def get_symbol(self, name): - assert name in self.symbols.keys() + # assert name in self.symbols.keys() + if name not in self.symbols.keys(): + self.symbols[name] = Symbol(name, pysmt_types['real']) + return self.symbols[name] def remove_first_and_last_line(self, text): @@ -407,6 +422,19 @@ def extract_smtlib(self, formula): output = outstream.getvalue() return self.remove_first_and_last_line(output) + def z3_simplify(self, formula): + if isinstance(formula, str): + smlp_str = formula + else: + smlp_str = self.extract_smtlib(formula) + smlp_parsed = z3.parse_smt2_string(smlp_str) + smlp_simplified = z3.simplify(smlp_parsed[0]) + ex = self.parse(str(smlp_simplified).replace('\n', '')) + if ex.is_not(): + ex = self.propagate_negation(ex) + + return ex + def handle_ite_formula(self, formula, is_form2=False, handle_ite=True): # smlp_str = self.extract_smtlib(formula) if not isinstance(formula, str) else formula # smlp_str = f""" @@ -414,7 +442,7 @@ def handle_ite_formula(self, formula, is_form2=False, handle_ite=True): # (declare-fun y2 () Real) # (assert {formula}) # """ if not isinstance(formula, str) else formula - flag=False + if is_form2: smlp_str = f""" (declare-fun y1 () Real) @@ -425,15 +453,13 @@ def handle_ite_formula(self, formula, is_form2=False, handle_ite=True): smlp_str = formula else: smlp_str = self.extract_smtlib(formula) - flag=False smlp_parsed = z3.parse_smt2_string(smlp_str) - if flag: - # Apply the tactic to the formula - goal = Goal() - goal.add(smlp_parsed) - t = Tactic('tseitin-cnf') - smlp_parsed = t(goal)[0] + # if flag: + # goal = Goal() + # goal.add(smlp_parsed) + # t = Tactic('tseitin-cnf') + # smlp_parsed = t(goal)[0] smlp_simplified = z3.simplify(smlp_parsed[0]) ex = self.parse(str(smlp_simplified).replace('\n','')) @@ -574,12 +600,23 @@ def eval_(node): return eval_(ast.parse(expr, mode='eval').body) def convert_ite_to_conjunctions_disjunctions(self, formula): - def traverse(node): - if node.is_ite(): + def traverse(node, from_ite=False, value=0, position=None, comparator=None): + if from_ite: condition, true_branch, false_branch = node.args() + true_branch = traverse(true_branch) + false_branch = traverse(false_branch) + condition = traverse(condition) + not_condition = traverse(Not(condition)) + + + true_branch = self.apply_comparator(comparator, true_branch, value) if position == "right" else self.apply_comparator(comparator, value, true_branch) + false_branch = self.apply_comparator(comparator, false_branch, value) if position == "right" else self.apply_comparator(comparator, value, false_branch) + + true_branch = self.z3_simplify(true_branch) + false_branch = self.z3_simplify(false_branch) return Or( - And(traverse(condition), traverse(true_branch)), - And(traverse(Not(condition)), traverse(false_branch)) + And(condition, true_branch), + And(not_condition, false_branch) ) elif node.is_and(): new_args = [traverse(arg) for arg in node.args()] @@ -589,6 +626,15 @@ def traverse(node): return Or(new_args) elif node.is_not(): return self.propagate_negation(node) + elif self.is_comparison(node): + left, right = node.args() + comparator = self.decide_comparator(node) + if left.is_constant() and right.is_ite(): + return traverse(right, from_ite=True, value=left, position='left', comparator=comparator) + elif right.is_constant() and left.is_ite(): + return traverse(left, from_ite=True, value=right, position='right', comparator=comparator) + else: + return node else: return node diff --git a/src/smlp_py/solvers/abstract_solver.py b/src/smlp_py/solvers/abstract_solver.py index 09d40a34..363ccfaa 100644 --- a/src/smlp_py/solvers/abstract_solver.py +++ b/src/smlp_py/solvers/abstract_solver.py @@ -25,18 +25,6 @@ def __get__(self, instance, owner): class AbstractSolver(ABC): - # @abstractmethod - # def true(self): - # pass - - # @abstractmethod - # def GE(self, *args, **kwargs): - # pass - - # @abstractmethod - # def LE(self, *args, **kwargs): - # pass - @abstractmethod def create_query(self, *args, **kwargs): pass @@ -82,15 +70,23 @@ def check(self, *args, **kwargs): pass @abstractmethod - def add_not_query(self, *args, **kwargs): + def create_counter_example(self, *args, **kwargs): pass @abstractmethod - def create_counter_example(self, *args, **kwargs): + def substitute(self, *args, **kwargs): pass @abstractmethod - def substitute(self, *args, **kwargs): + def handle_ite_formula(self, *args, **kwargs): + pass + + @abstractmethod + def calculate_eta_F_t(self, *args, **kwargs): + pass + + @abstractmethod + def apply_theta(self, *args, **kwargs): pass def get_witness(self, *args, **kwargs): diff --git a/src/smlp_py/solvers/marabou/operations.py b/src/smlp_py/solvers/marabou/operations.py index 8fb20248..d00e646c 100644 --- a/src/smlp_py/solvers/marabou/operations.py +++ b/src/smlp_py/solvers/marabou/operations.py @@ -26,6 +26,8 @@ def smlp_integer(cls): def smlp_cnst(cls, const): if isinstance(const, FNode): return const + elif isinstance(const, str): + const = float(const) return pysmt.shortcuts.Real(const) # logical not (logic negation) @@ -56,4 +58,7 @@ def smlp_q(self, const): return pysmt.shortcuts.Real(const) def smlp_mult(self, *args): - return pysmt.shortcuts.Times(*args) \ No newline at end of file + return pysmt.shortcuts.Times(*args) + + def smlp_ite(self, *args): + return pysmt.shortcuts.Ite(*args) diff --git a/src/smlp_py/solvers/marabou/solver.py b/src/smlp_py/solvers/marabou/solver.py index 9bc4e584..ccf6d0aa 100644 --- a/src/smlp_py/solvers/marabou/solver.py +++ b/src/smlp_py/solvers/marabou/solver.py @@ -54,7 +54,7 @@ def substitute_objective_with_witness(self, *args, **kwargs): substitution = {} for symbol, value in stable_witness_terms.items(): symbol = self.verifier.parser.get_symbol(symbol) - substitution[symbol] = Real(value) + substitution[symbol] = Real(float(value)) # Apply the substitution return self.verifier.parser.simplify(objv_term.substitute(substitution)) @@ -97,6 +97,9 @@ def create_alpha_or_eta_form(self, **kwargs): def simplify(self, expression): return self.verifier.parser.simplify(expression) + def z3_simplify(self, expression): + return self.verifier.parser.simplify(expression) + def parse(self, expression): return self.verifier.parser.parse(expression) @@ -134,10 +137,6 @@ def check(self, *args, **kwargs): def generate_theta(self, *args, **kwargs): pass - def add_not_query(self, *args, **kwargs): - query = kwargs["query"] - temp = kwargs.get("temp", False) - def create_counter_example(self, *args, **kwargs): formulas = kwargs["formulas"] @@ -163,4 +162,27 @@ def substitute(self, *args, **kwargs): del substitutions[x] substitutions[self.smlp_var(x)] = temp - return self.simplify(var.substitute(substitutions)) \ No newline at end of file + return self.simplify(var.substitute(substitutions)) + + def calculate_eta_F_t(self, *args, **kwargs): + eta = kwargs["eta"] + term = kwargs["term"] + val = kwargs["val"] + + return self.smlp_and(eta, term > self.smlp_cnst(val)) + + + def handle_ite_formula(self, *args, **kwargs): + formula = kwargs["formula"] + + return self.verifier.parser.handle_ite_formula(formula, is_form2=False) + + def apply_theta(self, *args, **kwargs): + formula = kwargs["formula"] + solver = kwargs["solver"] + + theta_negation = self.verifier.parser.propagate_negation(formula) + # self._modelTermsInst.verifier.add_permanent_constraint(theta_negation) + solver.verifier.apply_restrictions(theta_negation) + print("PYSMT THETA ADDED ", theta_negation) + diff --git a/src/smlp_py/solvers/z3/solver.py b/src/smlp_py/solvers/z3/solver.py index fb4986b8..215440cf 100644 --- a/src/smlp_py/solvers/z3/solver.py +++ b/src/smlp_py/solvers/z3/solver.py @@ -89,12 +89,19 @@ def create_solver(self, *args, **kwargs): self.verifier = create_solver(domain, model_full_term_dict, incremental, solver_logic) return self - def add_formula(self, *args, **kwargs): + def simplify(self, *args, **kwargs): formula = kwargs["formula"] + return formula + + def z3_simplify(self, expression): + return expression + + + def add_formula(self,formula, **kwargs): self.verifier.add(formula) - def check(self): + def check(self, *args, **kwargs): return self.verifier.check(), None def generate_theta(self, *args, **kwargs): @@ -116,3 +123,23 @@ def substitute(self, *args, **kwargs): substitutions = kwargs["substitutions"] return self.smlp_cnst_fold(var, substitutions) + + def calculate_eta_F_t(self, *args, **kwargs): + eta = kwargs["eta"] + term = kwargs["term"] + val = kwargs["val"] + + return self.smlp_and(eta, term > self.smlp_cnst(val)) + + def handle_ite_formula(self, *args, **kwargs): + formula = kwargs["formula"] + + return formula + + def apply_theta(self, *args, **kwargs): + formula = kwargs["formula"] + solver = kwargs["solver"] + + solver.add(self.smlp_not(formula)) + + From 2084bc61df9c3f79acf20b2e7e9d75a64b9dd8fc Mon Sep 17 00:00:00 2001 From: konstantopoulos-tetractys <37417801+ntinouldinho@users.noreply.github.com> Date: Mon, 26 Aug 2024 23:00:09 +0100 Subject: [PATCH 26/28] add profiler --- src/smlp_py/solvers/marabou/solver.py | 6 +++--- src/smlp_py/solvers/z3/solver.py | 10 ++++------ 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/smlp_py/solvers/marabou/solver.py b/src/smlp_py/solvers/marabou/solver.py index ccf6d0aa..56812a0c 100644 --- a/src/smlp_py/solvers/marabou/solver.py +++ b/src/smlp_py/solvers/marabou/solver.py @@ -5,7 +5,7 @@ from src.smlp_py.solvers.abstract_solver import AbstractSolver, ClassProperty from src.smlp_py.solvers.marabou.operations import PYSMTOperations from pysmt.shortcuts import Real - +from memory_profiler import profile class Pysmt_Solver(AbstractSolver, PYSMTOperations): verifier = None @@ -137,7 +137,7 @@ def check(self, *args, **kwargs): def generate_theta(self, *args, **kwargs): pass - + @profile def create_counter_example(self, *args, **kwargs): formulas = kwargs["formulas"] query = kwargs["query"] @@ -149,7 +149,7 @@ def create_counter_example(self, *args, **kwargs): self.temp_solver.apply_restrictions(formula) negation = self.temp_solver.parser.propagate_negation(query) - z3_equiv = self.temp_solver.parser.handle_ite_formula(negation, handle_ite=False) + # z3_equiv = self.temp_solver.parser.handle_ite_formula(negation, handle_ite=False) self.temp_solver.apply_restrictions(negation, need_simplification=True) return self diff --git a/src/smlp_py/solvers/z3/solver.py b/src/smlp_py/solvers/z3/solver.py index 215440cf..90e23d58 100644 --- a/src/smlp_py/solvers/z3/solver.py +++ b/src/smlp_py/solvers/z3/solver.py @@ -1,17 +1,14 @@ import smlp from src.smlp_py.solvers.abstract_solver import AbstractSolver from src.smlp_py.solvers.z3.operations import SMLPOperations - +from memory_profiler import profile class Form2_Solver(AbstractSolver, SMLPOperations): verifier = None - smlp_term_instance = None - terms = None def __init__(self): super().__init__() # self.verifier = verifier - # self.smlp_term_instance = smlp_term_instance @property def smlp_true(self): @@ -21,7 +18,7 @@ def create_query(self, query_form=None): return query_form def create_query_and_beta(self, query, beta): - return self.smlp_term_instance.smlp_and(query, beta) + return self.smlp_and(query, beta) def substitute_objective_with_witness(self, *args, **kwargs): stable_witness_terms = kwargs["stable_witness_terms"] @@ -107,6 +104,7 @@ def check(self, *args, **kwargs): def generate_theta(self, *args, **kwargs): pass + @profile def create_counter_example(self, *args, **kwargs): formulas = kwargs["formulas"] query = kwargs["query"] @@ -140,6 +138,6 @@ def apply_theta(self, *args, **kwargs): formula = kwargs["formula"] solver = kwargs["solver"] - solver.add(self.smlp_not(formula)) + solver.add_formula(self.smlp_not(formula)) From 727d7d1ac1b200f75fe0d980b0d0341e1e01810c Mon Sep 17 00:00:00 2001 From: konstantopoulos-tetractys <37417801+ntinouldinho@users.noreply.github.com> Date: Fri, 30 Aug 2024 09:56:25 +0100 Subject: [PATCH 27/28] dynamically load nn and specs --- src/smlp_py/NN_verifiers/verifiers.py | 10 ++-- src/smlp_py/smlp_flows.py | 10 +++- src/smlp_py/smlp_query.py | 71 +++++++++++++++---------- src/smlp_py/smlp_terms.py | 28 +++++----- src/smlp_py/solvers/marabou/solver.py | 6 ++- src/smlp_py/solvers/universal_solver.py | 4 +- 6 files changed, 77 insertions(+), 52 deletions(-) diff --git a/src/smlp_py/NN_verifiers/verifiers.py b/src/smlp_py/NN_verifiers/verifiers.py index 5f59c282..d1c91dbd 100755 --- a/src/smlp_py/NN_verifiers/verifiers.py +++ b/src/smlp_py/NN_verifiers/verifiers.py @@ -65,7 +65,7 @@ def set_upper_bound(self, upper): self.bounds.upper = upper class MarabouVerifier(Verifier): - def __init__(self, parser=None, variable_ranges=None, is_temp=False): + def __init__(self, parser=None, data_bounds_file=None, model_file_prefix=None, variable_ranges=None, is_temp=False): # MarabouNetwork containing network instance self.network = None @@ -83,7 +83,9 @@ def __init__(self, parser=None, variable_ranges=None, is_temp=False): self.unscaled_variables = [] self.model_file_path = "./" - self.data_bounds_file = self.find_file_path("../../../result/abc_smlp_toy_basic_data_bounds.json") + self.model_file_prefix = model_file_prefix + # self.data_bounds_file = self.find_file_path("../../../result/abc_smlp_toy_basic_data_bounds.json") + self.data_bounds_file = self.find_file_path('../../'+data_bounds_file) self.data_bounds = None # Adds conjunction of equations between bounds in form: # e.g. Int(var), var >= 0, var <= 3 -> Or(var == 0, var == 1, var == 2, var == 3) @@ -105,7 +107,7 @@ def initialize(self, variable_ranges=None): if variable_ranges: self.variable_ranges = variable_ranges - self.model_file_path = self.find_file_path('../../../result/abc_smlp_toy_basic_nn_keras_model_complete.h5') + self.model_file_path = self.find_file_path('../../'+ self.model_file_prefix +'_model_complete.h5') self.convert_to_pb() self.load_json() self.network_num_vars = self.network.numVars @@ -169,7 +171,7 @@ def create_variables(self, is_input=True, is_temp=False): store = self.parser.inputs if is_input else self.parser.outputs for var in store: name, type = var - var_type = Variable.Type.Real if type.lower() == "real" else Variable.Type.Int + var_type = Variable.Type.Real if type.lower() in ["real", "float"] else Variable.Type.Int if name.startswith(('x', 'p', 'y')) and name.find("_scaled") == -1: index = self.input_index if is_input else self.output_index self.variables.append(Variable(var_type, name=name, index=index, is_input=is_input)) diff --git a/src/smlp_py/smlp_flows.py b/src/smlp_py/smlp_flows.py index a97264e4..54d3bd45 100644 --- a/src/smlp_py/smlp_flows.py +++ b/src/smlp_py/smlp_flows.py @@ -19,6 +19,9 @@ from smlp_py.smlp_optimize import SmlpOptimize from smlp_py.smlp_refine import SmlpRefine +from src.smlp_py.solvers.universal_solver import Solver + + # Combining simulation results, optimization, uncertainty analysis, sequential experiments # https://foqus.readthedocs.io/en/3.1.0/chapt_intro/index.html @@ -298,7 +301,12 @@ def smlp_flow(self): # sanity check that the order of features in model_features_dict, feat_names, X_train, X_test, X is # the same; this is mostly important for model exploration modes self.modelInst.model_features_sanity_check(model_features_dict, feat_names, X_train, X_test, X) - + + Solver(specs=(feat_names, resp_names, self.modelTernaInst._specInst.get_spec_domain_dict), + data_bounds_file= self.dataInst.data_bounds_file, + model_file_prefix= self.dataInst.model_file_prefix, + version=Solver.Version.PYSMT if args.use_pysmt else Solver.Version.FORM2) + if args.analytics_mode in self.model_exploration_modes: if args.analytics_mode == 'verify': if True or len(self.specInst.get_spec_knobs)> 0: diff --git a/src/smlp_py/smlp_query.py b/src/smlp_py/smlp_query.py index 3c5cfc1b..1fbed730 100644 --- a/src/smlp_py/smlp_query.py +++ b/src/smlp_py/smlp_query.py @@ -150,17 +150,29 @@ def get_model_exploration_base_components(self, mode_status_dict, results_file, # just for small potential speedup. def check_concrete_witness_consistency(self, domain:smlp.domain, model_full_term_dict:dict, alpha:smlp.form2, eta:smlp.form2, query:smlp.form2, witn_form:smlp.form2, solver_logic:str): - solver = self._modelTermsInst.create_model_exploration_instance_from_smlp_components( - domain, model_full_term_dict, True, solver_logic) - solver.add(alpha); #print('alpha', alpha) - solver.add(eta); #print('eta', eta) - solver.add(witn_form); #print('witn_form', witn_form); print('query', query) + # solver = self._modelTermsInst.create_model_exploration_instance_from_smlp_components( + # domain, model_full_term_dict, True, solver_logic) + # solver.add(alpha); #print('alpha', alpha) + # solver.add(eta); #print('eta', eta) + # solver.add(witn_form); #print('witn_form', witn_form); print('query', query) + + candidate_solver = Solver.create_solver( + create_solver=self._modelTermsInst.create_model_exploration_instance_from_smlp_components, + domain=domain, + model_full_term_dict=model_full_term_dict, + incremental=True, + solver_logic=solver_logic + ) + candidate_solver.add_formula(eta, need_simplification=True) + candidate_solver.add_formula(alpha) + candidate_solver.add_formula(witn_form, need_simplification=True) printer = {'alpha':alpha, 'eta':eta,'witn_form':witn_form} if query is not None: printer['query'] = query - solver.add(query) - res = self._modelTermsInst.smlp_solver_check(solver, 'witness_consistency',printer ) + # solver.add(query) + candidate_solver.add_formula(query, need_simplification=True) + res = self._modelTermsInst.smlp_solver_check(candidate_solver, 'witness_consistency', equations=printer ) #res = solver.check(); #print('res', res) return res @@ -208,28 +220,31 @@ def validate_witness_smt(self, universal:bool, model_full_term_dict:dict, quer_n self._query_logger.info('Verifying assertion {} <-> {}'.format(str(quer_name), str(quer_expr))) else: self._query_logger.info('Certifying stability of witness for query ' + str(quer_name) + ':\n ' + str(witn_dict)) - candidate_solver = self._modelTermsInst.create_model_exploration_instance_from_smlp_components( - domain, model_full_term_dict, True, solver_logic) - - cond_feasible = None - # add the remaining user constraints and the query - candidate_solver.add(eta); #print('adding eta', eta) - candidate_solver.add(alpha); #print('adding alpha', alpha) - #candidate_solver.add(beta) - candidate_solver.add(quer); #print('adding quer', quer) - #print('adding witn_dict', witn_dict) + + candidate_solver = Solver.create_solver( + create_solver=self._modelTermsInst.create_model_exploration_instance_from_smlp_components, + domain=domain, + model_full_term_dict=model_full_term_dict, + incremental=True, + solver_logic=solver_logic + ) + candidate_solver.add_formula(eta, need_simplification=True) + candidate_solver.add_formula(alpha) + candidate_solver.add_formula(quer, need_simplification=True) + + for var,val in witn_dict.items(): #candidate_solver.add(smlp.Var(var) == smlp.Cnst(val)) - candidate_solver.add(self._smlpTermsInst.smlp_eq(smlp.Var(var), smlp.Cnst(val))) + candidate_solver.add_formula(Solver.smlp_eq(Solver.smlp_var(var), Solver.smlp_cnst(val))) - candidate_check_res = self._modelTermsInst.smlp_solver_check(candidate_solver, 'ca', {'alpha':alpha, 'eta':eta,'quer':quer}) - if self._modelTermsInst.solver_status_sat(candidate_check_res): #isinstance(candidate_check_res, smlp.sat): + candidate_check_res, _ = self._modelTermsInst.smlp_solver_check(candidate_solver, 'ca', equations={'alpha':alpha, 'eta':eta,'quer':quer}) + if candidate_check_res == "sat": #isinstance(candidate_check_res, smlp.sat): cond_feasible = True if universal: self._query_logger.info('The configuration is consistent with assertion ' + str(quer_name)) else: self._query_logger.info('Witness to query ' + str(quer_name) + ' is a valid witness; checking its stability') - elif self._modelTermsInst.solver_status_unsat(candidate_check_res): #isinstance(candidate_check_res, smlp.unsat): + elif candidate_check_res == "unsat": #isinstance(candidate_check_res, smlp.unsat): cond_feasible = False if universal: # Assertion cannot be satisfied (is constant False) given the knob configuration and the constraints. @@ -255,20 +270,20 @@ def validate_witness_smt(self, universal:bool, model_full_term_dict:dict, quer_n # checking stability of a valid witness to the query witn_term_dict = self._smlpTermsInst.witness_const_to_term(witn_dict) - ce = self.find_candidate_counter_example(universal, domain, witn_term_dict, quer, model_full_term_dict, alpha, + ce, ce_witness = self.find_candidate_counter_example(universal, domain, witn_term_dict, quer, model_full_term_dict, alpha, theta_radii_dict, solver_logic) - if self._modelTermsInst.solver_status_sat(ce): #isinstance(ce, smlp.sat): + if ce == "sat": #isinstance(ce, smlp.sat): if universal: self._query_logger.info('Completed with result: FAIL') #self._query_logger.info('Assertion ' + str(quer_name) + ' fails (for stability radii ' + str(theta_radii_dict)) #status = 'FAIL' if cond_feasible else 'FAIL VACUOUSLY' - ce_model = self._modelTermsInst.get_solver_model(ce) + ce_model = self._modelTermsInst.get_solver_model(ce, ce_witness) return {'assertion_status':'FAIL', 'asrt': False, 'assertion_feasible': cond_feasible, 'counter_example':self._smlpTermsInst.witness_term_to_const(ce_model, approximate=sat_approx, precision=sat_precision)} else: self._query_logger.info('Witness to query ' + str(quer_name) + ' is not stable for radii ' + str(theta_radii_dict)) return 'witness, not stable' - elif self._modelTermsInst.solver_status_unsat(ce): #isinstance(ce, smlp.unsat): + elif ce == "unsat": #isinstance(ce, smlp.unsat): if universal: self._query_logger.info('Completed with result: PASS') #self._query_logger.info('Assertion ' + str(quer_name) + ' passes (for stability radii ' + str(theta_radii_dict)) @@ -422,15 +437,15 @@ def validate_witness(self, universal:bool, syst_expr_dict:dict, algo:str, model: self._query_logger.info('Verifying consistency of configuration for assertion ' + str(quer_name) + ':\n ' + str(witn_form)) else: self._query_logger.info('Certifying consistency of witness for query ' + str(quer_name) + ':\n ' + str(witn_form)) - witn_status = self.check_concrete_witness_consistency(domain, model_full_term_dict, + witn_status, _ = self.check_concrete_witness_consistency(domain, model_full_term_dict, alpha, eta, None, witn_form, solver_logic) - if self._modelTermsInst.solver_status_sat(witn_status): #isinstance(witn_status, smlp.sat): + if witn_status == "sat": #isinstance(witn_status, smlp.sat): if universal: self._query_logger.info('Input, knob and configuration constraints are consistent') else: self._query_logger.info('Input, knob and concrete witness constraints are consistent') mode_status_dict[quer_name][CONSISTENCY] = 'true' - elif self._modelTermsInst.solver_status_unsat(witn_status): #isinstance(witn_status, smlp.unsat): + elif witn_status == "unsat": #isinstance(witn_status, smlp.unsat): if universal: self._query_logger.info('Input, knob and configuration constraints are inconsistent') else: diff --git a/src/smlp_py/smlp_terms.py b/src/smlp_py/smlp_terms.py index 593c48c2..90ff648a 100644 --- a/src/smlp_py/smlp_terms.py +++ b/src/smlp_py/smlp_terms.py @@ -1394,14 +1394,14 @@ def __init__(self): # '[default {}]'.format(str(self._DEF_CACHE_TERMS))} } - self.parser = TextToPysmtParser() - self.parser.init_variables(symbols=[("x1", "real", True), ('x2', 'int', True), ('p1', 'real', True), ('p2', 'int', True), - ('y1', 'real', False), ('y2', 'real', False)]) - - self.verifier = MarabouVerifier(parser=self.parser) - - self._ENABLE_PYSMT = False - self._RETURN_PYSMT = False + # self.parser = TextToPysmtParser() + # self.parser.init_variables(symbols=[("x1", "real", True), ('x2', 'int', True), ('p1', 'real', True), ('p2', 'int', True), + # ('y1', 'real', False), ('y2', 'real', False)]) + # + # self.verifier = MarabouVerifier(parser=self.parser) + # + # self._ENABLE_PYSMT = False + # self._RETURN_PYSMT = False # set logger from a caller script @@ -2072,7 +2072,7 @@ def create_model_exploration_base_components(self, syst_expr_dict:dict, algo, mo # get variable domains dictionary; certain sanity checks are performrd within this function. spec_domain_dict = self._specInst.get_spec_domain_dict; #print('spec_domain_dict', spec_domain_dict) - self.verifier.initialize(variable_ranges=spec_domain_dict) + # self.verifier.initialize(variable_ranges=spec_domain_dict) @@ -2321,13 +2321,9 @@ def solver_status_unknown(self, res): # we return value assignmenets to interface (input, knob, output) variables defined in the Spec file # (and not values assigned to any other variables that might be defined additionally as part of solver domain, # like variables tree_i_resp that we decalre as part of domain for tree models with flat encoding). - def get_solver_model(self, res): - condition = self.solver_status_sat(res["result"]) if self._ENABLE_PYSMT else self.solver_status_sat(res) - if condition: - if self._ENABLE_PYSMT: - reduced_model = dict((k, v) for k, v in res["witness"].items() if k in self._specInst.get_spec_interface) - else: - reduced_model = dict((k,v) for k,v in res.model.items() if k in self._specInst.get_spec_interface) + def get_solver_model(self, res, witness): + if res == "sat": + reduced_model = dict((k, v) for k, v in witness.items() if k in self._specInst.get_spec_interface) return reduced_model else: return None diff --git a/src/smlp_py/solvers/marabou/solver.py b/src/smlp_py/solvers/marabou/solver.py index 56812a0c..aec03889 100644 --- a/src/smlp_py/solvers/marabou/solver.py +++ b/src/smlp_py/solvers/marabou/solver.py @@ -11,9 +11,11 @@ class Pysmt_Solver(AbstractSolver, PYSMTOperations): verifier = None temp_solver = None - def __init__(self, specs): + def __init__(self, specs,data_bounds_file, model_file_prefix): super().__init__() self.specs = specs + self.data_bounds_file = data_bounds_file + self.model_file_prefix = model_file_prefix self.create_verifier() def create_verifier(self): @@ -31,7 +33,7 @@ def create_verifier(self): parser = TextToPysmtParser() parser.init_variables(symbols=symbols) - self.verifier = MarabouVerifier(parser=parser) + self.verifier = MarabouVerifier(parser=parser, data_bounds_file=self.data_bounds_file, model_file_prefix=self.model_file_prefix) self.verifier.initialize(spec_domain_dict) @ClassProperty diff --git a/src/smlp_py/solvers/universal_solver.py b/src/smlp_py/solvers/universal_solver.py index 374ea908..4887f65e 100644 --- a/src/smlp_py/solvers/universal_solver.py +++ b/src/smlp_py/solvers/universal_solver.py @@ -22,7 +22,9 @@ def __new__(cls, *args, **kwargs): if cls._instance is None and isinstance(cls.version, cls.Version): if cls.version == cls.Version.PYSMT: specs = kwargs["specs"] - cls._instance = Pysmt_Solver(specs) + data_bounds_file = kwargs["data_bounds_file"] + model_file_prefix = kwargs["model_file_prefix"] + cls._instance = Pysmt_Solver(specs, data_bounds_file, model_file_prefix) else: cls._instance = Form2_Solver() cls._map_instance_methods() From 6e4e13acb36645d9782abbfa063b9bcf3155669e Mon Sep 17 00:00:00 2001 From: Konstantinos Konstantopoulos <131668015+konstantopoulos-tetractys@users.noreply.github.com> Date: Sun, 10 Nov 2024 23:40:26 +0200 Subject: [PATCH 28/28] Documentation --- ...ion___Konstantinos_Konstantopoulos (3).pdf | Bin 0 -> 275162 bytes Marabou-Abstract_Solvers.md | 36 ++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 Extending_SMLP_with_solvers_for_neural_network_verification___Konstantinos_Konstantopoulos (3).pdf create mode 100644 Marabou-Abstract_Solvers.md diff --git a/Extending_SMLP_with_solvers_for_neural_network_verification___Konstantinos_Konstantopoulos (3).pdf b/Extending_SMLP_with_solvers_for_neural_network_verification___Konstantinos_Konstantopoulos (3).pdf new file mode 100644 index 0000000000000000000000000000000000000000..124c1e95661120d605bf23df134aac4cdb6e0927 GIT binary patch literal 275162 zcmeFYW0Yh~x2|2btINuAciFaW+eVjd+qRAFvTfV8ZGH9Zcfa4!9_Q!zwR4OdDO!GR?6JQSl`~<-Pi~~|6Kww zurkxLFaT)90Ga>>CPq2{6FoCP8$c@yU||Q)3IkZ^m|0l?v@!sOZzH8|b0%gwMgR{F zjIoW;KhFgGzk9*3u>4O0QF8lj44_q0FgE!fhOv#QqZxpW4L~bmZsllf|9!R6cQh6< zHncVRw))pqPT$_z0q~Cqg0@yp);10RCIGFRy|IzGp`)!mfS%!dSO8kpZz1Rb4Bzi> z6~t|fj9tGgz9nQ}X82c)i7kMI@%xJ?0B99#Z5;tDtp6$~ev1lVr2iJ{`KW*%no+5Nx;$jdmZq# z>4_ml4neGf7l46-0To|B7yL8F|JlX=``P~AmHY4Ve=G3c3jDVM|E<7(EAW3)0Omj9 z@xO6LM&H`l;hSClA&2jPE&JQphTqW9+|~xbz()U{bxw{}-%#{zMgN~Qa{8vm(*FSt z$_~b||8A!Jr}`f&x_{mGF8~qLchtACHT|E6h=uLD{ononmpG#MjR}nZAQt04#PXl; z!pOk(zlL-@z3+g&U1_MNuczlZ1?UC>*JGhXVxX&)lKXD)huX>+EpEACtDrrTY7DUQ znBu72V7M`fF|9FbFitU&-Ip?LGUS}(NE*mK0iM?Ksi-(*ys}q=kxGxvN1_`UK6uVc z7N_9_G0|$;Q@Uw{X|pk|m1}Ae8Xh*D)4DKF5A!u2a<-G85Pp^3SXnCA`Bj$qK0TsX zCelalvDq(P?$D6nCqd>mZaNBq-C%62i)d=q9RlYtH=03;MAl(dJa z4(k~Uv%gwsYID!-;sH-Ip;#)JvK*1hBs)uMY0_E0bq4EK*a0*b;yj8!?j%z>gRD^t z2)qq1L~uql+;%e(7ZlJmid*Ubi7?+-`#+|K-^Rw)(cvHTWn=h{S!H1R7uqqfG5tsR zn*jxFZ5)klzO7jQbK<4HiBVs`_8%_%r-qrGnTCOt1;EBcPs7Ik4kYhVVTv(6)aLNN!#kZK>z^CzdIyklehEb zrHN~!=;cr4sL_+_CaJhL)$GQvs+ba{HSWeX-U7rd%q*riL5+U-zh7rJPulIAKNjer zQLfCc5Ibadk2ybwv(p*TeFhM`2`j6t}q37KirhB%m;I(#m?r z-zM2b?Jau(Kjt~{qUXYBa|pupou;z8abi*-?e1r+S9lF6M~G)M@k+B5F_VQI7lWVS zm~yeB^si6&7^)J9ttX-%@88`eadoTtKf};A_MvbL7N)NsO-rL>-B+2sutlwLRf}O= zftj>hXfuixr?=L@CT^`uwWWFE$UlIsH>~|KBUXXb=PUSO@X>|p7K(O{x!~cey{Au9 z1Vy2ZsYv~?vF3S3EP9}9O7ilOdNLW@n_q{CpyQgMAdx5jGP%?0|N1yP73NSf8Xk2s zQ&;6(`IL>2j93>elPTyciMa|R4Dof;iVI^;M*Ep1G^ZvHK(R9Xy?spxF<6OKmE&%) z&W?fOQx6ShTc)3x+0kLSc+0#d)W>~z*Br{DN73X$mUV9}`06F%5M9{KcByiBi?vu+Z#62c%c)8xaR58?v%5F4EHZ}`WO6Z zfX*y28H<1rH!01pcCl=9F{|-mv_p9mt^mTS+~J8koFBiorUAfznQm7%5-66phrnxsv4O*_EbeFY$82;>pHEUD)cL$rD&-L=f3C%1V=H0;y>4rp z;9k54)bq5a57q4!mj>zhzekNkEIcC_v-5+oeYuSwioX3l9MuiJ1?gwQ62YWu@9|y= zh}#EBRZF07q>^&xPohn?lCvj7|ts{}hZN)s@ zIe@k&JXIEq1Rsc?iZNk!b}sH55I*N9C^@>6IoyFUgd=!QE^fguLYBdah@ggiIvs&> z0FN5+{`4)pQGfVYFTH%7aqErk%E~}SU4Qwl1FQ>D+q;y~d1boh@F}u1nmjdANt1G! zw7?cPFMQrwcU_?V0yzNI&T=wbg(8$3F3*i z!okpM;+0;h-<7FIz*nZv-PvA-4{#^Fq2&J=>RA8bnSaNVe?=Tw0}Dk*>;E4*7}&ol z=)caH$oywI#ooaYz{<@0zl56$4Rc$ZA*|0$U3vmku+0`)0Mxc*v?hU{b)ignZf;@2 zcrX!KxP9j5E6>z^HWyZ6HjnPq8yR=PP>?hQDoJ$O2mF(bhE3z zBhoLZy#g_LHDe)(UJgnHbacJcJt=f7OQChFod%2*n5%$c86jig!Ce>^wc#EAAfV)g zX_&P`s`^33svj&U>5~lu5h~EIf0BDhO4tlJsIx zFZQJ$=7=hX4GzF!%?&o3WQ{~)hCtMh+$@7qZ3Vp8ED)sFOF7KX39pXq2AQWmd-;>) zcA0%AfIJc-`|n)pHX}QP639!5pYb%b=iq6NFX}2KGL_w+Mvp-`gs4Ag3-&k+&F^g? zD4}tWV+hxRKhkjfm`qf`-Rk1~VvG7+Mxma*1`1k7hC&_Bu~K54hSDffwQ5Pt$GeI>=(`n;y?*7#a7wcN^Gy)0Ofc_VLmx!}}B zmxaeplF{Kmb|d3XH;#HE@}UZVzpa6ovO2qLDq%S^0=+c|S0SP~dg17;t1Sd8fve4- zZ@0@N0GE5?l-L0IU!hssqE4SoT=e`>yJOiF3_RGyhVcM>tt|gx3$lp;|EzjdY^ple zI-B&NF}x(equ#C#BsL5gabVuj@9tQP&3GWVerz9+aTVlAv0w3OG*AMA+sH(qN-7xf zl!TbJsyjmr{Q0nOa~cz&F>E^bQX|m$Ywzvu#b6=Nu|Bm1Mn&{qw#9fKSsL?n3Mc=e z&))8~^|1IhL{wC*^s^a+;wYi+mGH^!?e)lXJA{~NV?J*eQCX>vrYGO}l|GOW zxxQQ2HL2;R_tex_#h&Cjul9J4B4c#LrZR*MR`y@}us{6Djsueex1O}smjdHa{8VSV zlx8$?Rt5U-k>D&C)ND$tYQHK}RxR^G`W9ELPW`yTJyBb6rp(;hg4cJStp2oFQ(Th< zhSO<)i%jKGz>Zgi;4vDBxI!~eD%tE{^z7n|Cy0hQ+!AI+y5J~G4ZJp0j2z8$>4oRB z2j~eS2wfxXEU{~|2QK%)8Nyf5=Q9fjhKZ}8Awk8~C`&=W?;;9wgHZEARGm}or{k%p zR8SBHJj1C4(L4LPin6X27&|2)O(6)Jbqygvy!2S(8-udEQ4k3~BD_RyBu~NRjZhwj ziuuP)H*@A04&2WW0oOgsm>K3)id~`AZl%{$@l*ap7F4)L3lDB96*^Wbdi5okg1oZI#-0~g12@0LFQd~p*Gr++; z$9S0E65%3vh~K9w<~K@NybR9;_;j7Hl> zG%^&}V}Y|CyK%tRnV{qT78VY(*eDgG9K{KZiYw%2(RY>d_3b4f04R&aL!KcGloA6t zM+uhAC_ar0P8bD8fDIXXTh%VGgq-!6B|2OM1{qEFb$aepL*&YaVQ?vr;zFZtevD6| zJqKN@2)pw(m%^2pD$(fgOA^FZw~h0=w^IgtV(E4qwXFHQ3}k0)*(nL%qa-$6#5E68 zhg4@!bB`MFv3y*!$2>o0B-F$z0Uo5ZbBIlocr;x18=s2XtzUc6`*_?N7VGV+?Ken? zsD>K9z!P1oKDRpch+_=n%6@q~DW};l*K?y&Z#~u6Hja)4JX?k}f^I@(P=QZ@` zQtVKHPW5OE=JInpj>AA!l}C))aPqQLpG99N9}C5rO=}cbuEdKYZ2$aq=W$xe`qL6> zFZ%QLD?}DL)@=}dAB2Iqu~hPnCX^yvmivgUz^h9cUY*nL%rW{cmn(+FtHB(9hZPKJ zp)FRPsp975V))dScmM5mf8^wi}a-_3sy9+zR+^uJ^ zjnP{hcgvI71vQ>ciEnOro$Y^<3UdbnUTrnkVR)!*HLO4wIXd^acnHU7+9(LXXLP(6 zolmiCX9*zBp6U%k5I%jRpPt+I#PeA9b)p4doR;!DJ=WLk)&*Qb_F~Be-7ws$UqrcM zg&_Z+Z%2eGRb;diZU|yIa~OO8hyi*sRR;302b?eqIw3|6`|ytuek8#<>l<|5|JA$pl@(vfO4r7BS6l`!{t zNtKm~je&m3H6|Ap;RGh8$b0>zemE0y+Bin7IBU>JuywgHo$75q`Z1P(eNe1exnQLMIK|4 zP$#*Q{`%MxFd(W_^b{#L=BJD6eodD=WeWA1DQls(VW`9N+Q421_T?$(>v8)c!WN&$ z=N%n_D5sjN?quZS1B*|G_?G!+c2+#4}ujLWv| zc>XyJjws^FnXKVJBji!rQPQC*IePGx5-NAE!35`3i=H?FEJ-o3ragb$9oc3lgw^=8 z3hSHgLQ+QF2=cfSgm|{o5{wMpmU$9x4To{jM7}lXJV^C7>+eBRnKh<1IXhBVsxMkH zUuJo-e#nivL{t=UomHK8czz8Hyt&hq+^Vk7=}sO9&>S|jFXt5n?c6q143rm(rQ9m$ zEE3cXN>p#so@rEj>kd=C@Ovlv4ErpN4WzTZ(~uXLccBw@-?h1S(7?A5Myc}Rb$C8c zM4Y_bEF9boysvbqKCCQs3=yb4yx-@HTsdmK(!BhXIO5G%Jiz=fRs;WJxc}W1HLzeS^m95_}&Qp-tlGlck!Q$*1t2B^vsM*bpOk`pvu+IweTX@#%ir>S!Bp$ zs4**A@#OncqqWoWpBZ}2M(Zrkt!!?OYOllhG)CUj)=kqJoim*&_lj_TA^9Nw<@wEJ zsoqV5;qKnyanRVNzs$~dDveD^O-ISbv5brjAfBV_+dqE_WsOJrsysRyi10Du`v#}| z;u{|a>mG&C+c`W8E&vK${%Q~c*ffGIWILZfdJA-2Oh7o8VAaU3D=mtjR&$2cVqS~Ov_43h=b>w8CV`1 z1p`>kC^+c7eNd_M=C#@918HP|zF#qgDFY5Cal#mzkDrMBXA%IW!Wk>xi@ zCZmGD*KMjIOdsa64W@w&{55)E^-u<}oZ+L9zM9eet~|TdIz^YGksFtk#Rz0}2=Pvt z@&~JXesra0aQK7q$&?O~yrN6LnN4#={T&vVyL)83cX(ueaD3&3aoqI0tM#si1Nf`R zofr}l=c%jX;;Zph7EXoFlo7{4gbjUhDwC{xY8_q0`L~qMM0kBx3%7633xT~NMu^X8 zX9oMn2H>>IB=`_DFs`g4D~hvwejGPid=vjU4TbN6*D&qICuaW(Jp4-)>QguG=__X4 z=jgagYxisG?foi`RLjb0Kgx<1a+`n?;=08s@*F6~0|Eg1Y&p}s{HaSL(x}36`9)je z)3iwU2C0GUfivf`Bn}$Iqe*0CdLs_YNPNc|s4|J7Vq;PquEK{Z>Dv-FVcPYXT(Odt z&c33ylIC^$>LLjYM>RhFRjk-h&+uaMgaM5EC6~66_R0A!TmA_#MN&~tQdG_Nv_-Yl z1&_uX0#tf>c^!7aD-l=6`Z3j~t1l#Uz5$dCN0q4s1i|8*Jk|CD9h#A_^42B(;sQwa z$vTMil~J()%9?6TQ~#{~Yny}P@AFMhq1 z@>O^0ll_(L#>mLZh}wJr?t!`qIGc}T(xY~s+-^>lv1NsQ)~gj153tbh)NSxev9(wx za|zn9l%_M>BboB3czgfTw|1$z+OcCpg;QQ_RCmV(V#|=8?K#gp`bg{O0W9@O8fs3C zEuW3y@D~SZHFNybLvv4J^mU*{E->N3N`?*tk;&U=>+XYY_zEx9Q%d2~kd9Lhn}6B@ zHd^*4Dgwb}*{f(U1Y~P9z{C0G^>z0-n`$e>!N<746H|Gs_Yj1D{v6 zcx9nxkuqcyAYLlb*FLTbprMdC{`w%YJ_Mx&l%d~JY8qR^IyDQkx<4aBfzk_=MdEmz zheGmFiq2jMd~p~OAiSso@7;HZ)lIHt6U*VpAzd;;RXN5KNq*@Y8I~pyP1y6KQtlG$ zMqNbEu%Rk-=9R5*{cTyN#Gg+&>l+j-+io$F6KHC;yI~?upHa>jG3r@pTpb)j5 zi?GGHOiZs{oK(kBy2ck-`+1?eEM4AyZ5kk^0+$zwBd2wu)xvD=@u!4tZ9R7HZ_2BG zlz}-Mbi$Oxiq>A$%?r4>Wfs25^>C0ztB{gPYIJ5Do`k1Q+ROdCgqe5}|z^Oe37 zXh%WX7wuO5&u}h*HdtzA9WuG~DocwRyM#u-8l@iD-qjit8 z+J1NouR(5mCf(gunh73Ktqn@FgzI=i1BCa}aUIjjy_|Lk*Z}8{7CT9HFxpI^gPdp3 zgqpP2HoZJGO)+dRD>S4eG?#%cq#Lr$)u@d{W1h>`QDP~M+hv?a9vMUMow!1#K?ay= z$SU4NukCS5DV>nQNR;P$RcV!0Ban(W^ZwuzvlV= zb1AM@%zVHIa=~}II|lq$EnJ$3&n62bGW@(SKeS1FGqagdZuWNExR{f;yP7X-AgPF_ zRR!J@!VYVXfmXK_w`c(kn$&JcEm&cugEfDLA8(z)0uQ zAE$QnzFU>hhM;zrD3^=p$(>IsB*g$?qYAAKM*6traoS+P#DhN$Ldd`#$o1EkUES$FE1b45= z0aN^sZ5jIb@>QCGT_T^VHmVOeegsW87ivOQi9Gfp!4;jAkKzd)3eIX(!4=yPdx$kI za&V0%d#xD|G?QV8iSAv8WkOcz3$~MenNOW~?r@x=9KvL-@4W?E7|2a*J&5%Rs7D=` zzyiL_)H}kxc#S-4jPWSXgEw3^Hl?SWdJQB5>2puHNqkD+s#j1-onjcxI5x>ip|JZOF%~pm#iCwc0Skm zNaHtgi#+bMppkZ$aDbDxh*GVy%b(fu6QAOxQnI|G6}0{|J)MwYXIJG{s#Qk^?f3!L zdJ%CI$hv{r&s&&b&&48@r>AAvWh88@Iuel>V+v-nZ!M+<3FNot@GG7U%)MX1q;=1{B~-`p^R9M#axI;S4@YjF!mC_vXcC+TF{dB;2Sl&9 z$X582UU0Xt4n{nQmMOAb^G698d(~RG9{2xa>_3i;ws8q}0 z6XiotVR^d-{p)RyPZIjfo2SrGYYAw|0j-S8uR)k6pzJXt``nGe*eE>thldl~1zzFF z@51>rl7s~mokz*tcT@?#FCCo+_(|p3a-j6re#~|V^oXb1Le%0zmg$2MwDS&o&mds7 z)39^VFU>tWCJ^;=H?AdviwPC}r9BsBbnKm300W1AsQ)WljlkHhmPVAJ z_J9vPPpf#zen{1$tOloYc;BSsoI02AlTiFKRi%2Qnihr?*idvq*gPFGG(-aR;lA6@ zufbHbhZ_|(e$a@UTKo!WSrkY=kBM>vND2slI-m-yv&GRV0$VUXFAaYQ0XA?##}_|X zzh4UWINQRjaPIi5FIn%G3Mv#r8|Nh>uKN`KPWVGwWIm)z5EY$VL@^H|%Bt1I_Wh%S z_JMGtYlJck7wY{1YS`N6Y+8b;58=?ZF5IB1Wdq8VJhQFAu|xlZikUNiH^+ zilw!7py<4&pneRvSDe{nkPrbc{R*pJt(c9t;8|*NY*n8bGKnLUXNkelZeJ7El1PJ| zH$)La5t?G{G z(~wHr*lRMx>48v@l@?hTxpl5@!(`23;echUcfyu5GeoHq72e4?%e$3S$PXtJfUtBc zZjs1=WD>g&Qknul@YDA-kAI)2KSm<=3tufSpQ*v1Rn3+zXvqbsym>3cE2oxb{{r&q zI8538A@iuH&ntsqX2Z*iaFe6fyAyk!#9?OxXnj46fuJp7L`Z8`ld6eV9Na%IAdkX$ zmm=V8&|7xsQjxHPII@vcv3PvOD|2g(TWMcTJYKizQyCh#WM8%~KXBVAH=&*D;HiQ< zyOx>Xk8|CnSJ_!Jyt*g1&J3f#vXh<=uEkPvmPC*weAnu({G7AcV-t45|D}WQ?W(Uu z-9310lk?1Vf@dfZK6}8#dBMKD z-3%$diAVvhEk|J#DZx<4d-D{-WRwh{s%j@E`b|G?Hoe)Dz@Mi_1>~``173hyw%ZcH zc(o?%>X!PLJ5R5EEj7<|xeIo6a~{SkoL^!p4!I$ccWR+8!_^$h*5QbxLpUAcp$JPS zpfY7#{C7T`m`Z=6!V{n#0P$;UGPL2~Vji-6=6nv~_XuCe3@4!+csS>MH}4>?5nUaz zGI-s4WG=bxJ#S~T8BI#{LHjR_(ycqp1kUW>l)@Fz;)2W%h9nr|z3&4qi25^liBV+< zQ@{Q;{`KQ9EEEN6icZ|y>4b!8ediiON1Lbxo2KqxpjbcOs+gr*ZquW)QPlXMGAin1 zP@uUz0_#z#A$r|CmAJBMpD(v9iErOAVi>$J9=FrYD2O^wj^ zg7MZux-qNjLtCHUBIIkBpZ9o=Rpe^xW9(_dT~$?J5 zm6U>_TCznoPvwZ@R*i^Y(NJE5vlE_y+GBa7BGDV|Bob8!<4jz&d?%AF0gjH+JX`8TZ9z!x^m zlZC9rDtY|!eUB#F0S#7wTsI-9%}nFc$$5%c)qO%p3S$D9z%#SsS;KMD5ua_u0IRkE z{huKxm)fj@1kRGTcg1b{qbu6TxNepj&P&3D-9MhezDJM>Qjb@)G+=3Y!%!07BD|iG zi60ib(>}PYpMwG=)A`mS6iIU@tRGbx=N<~On`a^papAF-kxlNj!B?wM5S+d**s7H$ zog~t~iRZyVMb@>?Gp!P~`nUMtioipy>eX~}4hg5wENTVbtZ&#FXK4Mn-MKJ^yrHJC zkV5K}C<Q7b=xY6#+CP`94~qfT+2^ccpN7fpY$7)gg+W>K+H$m z+5T2eU^ztB2l|rsKLw6$sL!O}b9SaY#!j}|8cc0yqlMl*WCWp2Gx{O0MEqK;e-M+U zt@D#`MYWpIT{KR55iYik;t^^(ZUMdl^<3pFXo8MgfE%cF3x?hy+%YVySHBo5C*63; znC=#6SiK8=05MeGy4j5A#npu7sUB}QWVEbT3FU4#3zp}4W0_7D)Oa_JzbTlEfaH3K zJdqho+R)6_xuZYHEg!c>DM{mzS4jay@?LG^7(~%_v6OL*VH7{Xt70Duj4f&TnQeyY zg&BSezhN-pKC6f{TeeX>j*tQ)sJF0*jY#10lVd3!>U*F1h)@{N8UT15;!~*szNGLSD_Qi z42jO}sRmWSA|J0m&-oNgl4La5p95n;Zmhe2}0}7ftpIJTtUjD9Iij6 zudl1^Amhwkz8qqYFlTa7<@G z+1y(P)(CK>)n!5$Bmd;P1(Xt3WnE%PtZ~U4I@b$FtzR|^#^XX4E1z3Lr+VNQ@X(An zlVSyM@F4815KbKLkBbbMi#npag*nQ?B^FDZ7uoLCIv_n;MoGP5W^24UZjbE{lM}ox zt=Yh=RlczJ^_kEQOUrI7s?nHu6k_k{WK{;@3CiaPg=v2@;J?TA8B8EsxK3 zGM)J*bFk`SM4LfYbY{w#d^QcANU>#4!i274SSW>j+wnsm->I@-5#CUg8&tW2rkDp2 zRm%$EN>NH>5i7mHTupavx7r!3-rA<0-1$R5drgZgzt77q-Why;rTXq@R12F-50y}* z)(R{MK-jr~>teG(#In`P@F2_~oN!n3KbhV@PZIV>po=eGi={=Px^|+qR1?DIfyZgt zckK6xD8}{Br8{PwΠMBr9E~ii{D#$*FuLZT#?KsL7}gZlwf~4r3fL@w{(^miF5xpO^xy)9`g?fior2!zI3B?M+E zJ-s&7b$N~SK=h+Zx{} z>c07nS_6eL;Zo{zr?ee7C0SJE$WLpb1Qpm9hqVR|cl)JE`76U`PD)x+hsdoKz5G+glZm=DA!<;a)%%97Ti2x7UNbjuM^ zwzv_I@~iaznWyxwK3CPVVpbDWYv!OKl>6LeuMRc?gv04PzEe&MVB+SUKLMaWnrQ&A-AP_?EC_b72<=M!Uey{ zl>nyZE7Faa z@Lb)S3O>#ZM2tuk?CI0XgV(yV)*HVQha29jq!`n9Zu_04g)@rdCT_&Ws2nGWQe|xFvgz$5x%FPY z3KC+oMY7seX^lM=I7_I%d=ViM{LA#ND@m@&d&qms^L6|MRTi3CZLU+bT}}t;?Apq8 z^r99mV^xO^jE78FZDmC^0cNYZ@z9jTQ-6J3Um4s8$a0!B40<$F1(|mEnagZe{*-l~ z$Pln#KP~u)2A9$|=S*S&Cxr>C?Zk;|V-K^4Sx=`kpJPm*&{j^DAU4gg>s#WbLZhiE;MSVP=q*Mo<1`KYzv$clKo|FZLi6|6ex}ge; z!}Btg32d2SzPpcm#FFM6yznby=<};PekIlRua+)p$3tJ%ZxaS7YlPFG!;jlB0dO8$L>W(3cVZ zO_Xy&@8@*YNzsJkPvh|C-FOv_fr8UyB7)PiP1+qyItT%tU?gJbm7f|jZv#3z#U^6G z3_75&xa(l}DthSj*)ePIHjIVtffmkXA`qCMrGgk#8f1oEe^jD5;d#%aFaT8Za(6XN zJF>}60TsyF;A@gnyH}=LGL*AbbSi}TQ_D*R{eor)(lVo0% zp_Gj3LQgu;r;2Sva`(-=s(eTZS<7fB@ zHaJMMB_!>9o-3@P&0dhGLMl^ zJgQUg{HGaO@ti6?^tC7 zT#;UITVXu7YHSROSW$>8B~AV%jAN^#k>!^w|IN@sC{{Dvmj0)I%zo5}bzUU9 znI^FFBweF20%koK2k9N2q@?*339dirymt=zh5#juxJHlL9}7MkhIjYLe`W+SW9?7w zi!`{W`KPtjR-O^!y$pCr%zJL5Ww+oQ&^B?}bC3hKf!QRu7=CM~V$MW|X2fB?$6Ef) zB#iztvBCJ>Uc$`fpP42w;`eiHh*!XFcK^&x$+lp&?$5o5nPX$uQ@2(S4OLIIizcaZ zw;4(Oj5Jn5Sdtu1wS2?{yOMLAI_R7I#=!gK8rr9%%ZDH`{lbU~`%HR=Kni8N;*N`{ z1bd%ViPN<|iMMvbs%}sq{K<9?DCEl~b}pN6d}5VLH9voT(07hDoWwXnr3*qt-%<(X zT;3 zN594M()qo^iuD(c1S}Ild=rfqCiD1t!RGe8{RFC!4D#+8{HrC|z1NypK6_u(j<_0lh_$^$dZd(6bRy#ok>P#MbJ`Rt@##oMs_hVhUTzECRI zMn5+MPRNyS3cQ4+c1HwIg~w&3-!Hv}+slMj?JRqwe(bptOuldI;<#a~5J(RFh)&Rs zzz7W$fR*Etkuk38?P~Q1B}e_>IGkX~u;CJ7|4A)k8q|7(>Az!oL(~OfS?n=|`Yhgh z&Qe@B91Lu0YntUwQcQDq7@NmZ0zZPse`zR#CuKj!6{fu%8aM-XxJtiN>=i5qn)R{Q zgiZoZe}f{AyXHht%`xPK`BoCe*tL!nyD&;X81o>xhFt@4r(GG=EP{ych{hIhJ}Mfe zS7?mZ;K_^;pNNY)jl_Qa^SYV(F0J&GW_DP}j>7EZ;i#k5WTvcpBe`}>gIN+|bN}&T z1A*J1Y^<($jZ%2v@(XZ_uv>h#tHB#r?W(?bF+b7V`^)K;iu+in_5jPpd32!jT`T%Q zaATjuU3W_5=NLyzz|GkPWj6;JUIxWx~4D&aai+&$h-MJ<QtF+=d`#}Y$Q%!RpI90&40oqi}kb5(qb7G6RDBGTn2%3L!O?2 zn`*gvr1K`*OUmgo#AfNQl=C~ql)^me@Q##^GoJ>bID*sTgX@t=Snyqy_ePG*3YU$F z-_~+^>vTru4ax)BaWaom!SW`%@6F6BiBVr-xO$WethFkPk{J$~M zHoW@6ICS>}8D$zRcPV3LAf5}4TKNk#@I4Suqd3*-KIWl+(gKCv_2X{ak0B<}6DSNa zNK!n5Y}!b zy}m?e7{L#>yS|8^mK`t}-d5-RVk-)QX|QGAlyPJ{kSFZPv3+wClm=0#tP(+#8s z-eTBcu&xR`$s8mkbk_OFf5Nw=1ul@u1TrEfL{Nn1cGYvy%et*M^6U4JX?9vli${7u z**u_jgvUm&m~no2)%3wqKW~TPpwE_EZS+|U6PB7JO}O?Pdi{c>hQ+6RVvmJOznQY9Ct2U)&S7Oe9sbHC*92lF zv(5;q=ThLO&@8_S$*h@|Zvb=bDFWn|H+%rg3>rOp6IUm^QNA5fF}=fx&JCu^2e3D6 z+vdJ~Qx66Mr{sl>Be~OSZUPLq7F_ar!s#~p{OpPE0?~5H;YW6fa%i*ZMn9ctBlL$1 z!duoFj)f%#`hCPg+1n#12*Tc!(EJwe>I!She2X=~VA|EEV%fz(GnHOmTEYt4g`O|( zd}O1lR47>b*=Y?A+gh6t=1$zhYGLG1zicO2(zw@N^xQ9gxn8gA zM7BdJ%|&mrW>bZlF|B!8_{`28(hj4F=lMNt78e=UXogGVcmVz%ccA&@R2+#cVRY} zcobU11j$?E9pr2+$B4FKDw%OJYXGkwUp-x$@>FF|3N)$smG^swRSJdy`?C@u_*Rt# z+~>J}Q&GdOvyPKzY+Bjv7e9yrc-^*`*;9ax`ooxrqdWbJlBUlVKm3m00VJZxU*K7z;Pu#3#8HUs$_Y?_&jZ;H}tXA-~CJ9GwW-mxB& z8p|l?O%q7#Yfu zYw+6XH>$P#PGD}^vcX5<5h&h4Q$_bKTq3&C*0iVU^7_S=nz&dR z<$(eKU5xOW_WkEAMr0;6t%69!+MhOVW{UX?R8J?`=->~Ch^aHiLnVq)-6`(A7&M}9 zEi}xVzB;osm-{(qq5TG=F4sLIS@xtEDnoB#L8;{8YhDFKo>b683-w?GNNn$|_1ie% zThTABXVZD3yq*Xy#uf?Yy_LouQyse&Rwk`ldrV+mTWzL35RL>ZwXbX`NZuVQYG8hgGqm0Cy+6zv-US zj~&+X$Ax)ENULbDb-;Ul{L~<@I^ZTQc5D&shLY9lX(aK^x<{5m)VlC?17=>2$L3Q4 z>@{=;Hu>0NmHvRL`~b@XaV}G?{<0;k%_?l3!@>l8Lr#LXIOie1rdjfihuKF^oc0T2ED4}UL{xlT*C<;~ z^57UsOf^1zQyC5O&mwSP#SbAO7DQDL+}EiYbo9&~QJrrIwE0RS)G&Wf>`4Ci;V{i9j9KmBfu-oC0Y$cq1eDcQKf|XoPrY4A;4|@ce7}HQctM`}G5i zS8i<}Pwm-$-t<$b5H32uJ_cWWecPvzxN)YU6=|iygf8X+r8Xxir_CQ@+v4!GhahRD zGcd262Ux5sGUsiQQ+>A!8zuaoLXXT<{l8_ zUdm7;9rT{SP-n~i_Tmg0<8jklGnd|^KSA(%huGxuPiLk^Y%I6AcUQe$)GRGBlU z(^>bSjQF2UE$S$fXRaa$6u>p%UG#gFevM@>WJ7}3jLUnKCc%OJKl85I-#-N;mAV_&!yb#>x%iJ zyN30Ld@}Du1|t@#B%KwB;M3>IK^1~^nPK|Uy@5h!44;=WWSitL#HUM^pIqR5gnn_+ zT4c>=D#nQB?!@5&QfjI*FGWuM!x8(>>BQy^y=c&0xr(LikLoJxY=>dw2!uoO?n5Db ze_(3&4#J`p6+HtpXzDMgJjfcm29ex(%#CU2;bOY6K!s3VxT=Qd%ft`i$Is|87uc6F z!_=uE@!4-$fX#DwGBSVBIm&VXt>Am>hioSw4L!iwiJa$_lU$9+$Bh|c1O1Ii>~7qC zHB(x1yStl*3(WPOw$r!QjP)VwIl6ZNVl6F!{JT^4T1Bfy6ts8NRGxkwl5gQiVo(`2 ziW}ATWbIYm!tf>_eIop_t$+MwH8L<$?@+Eq8|{s4Dk|-wtxKTYH{@dua!q6O|a8S~G%Ki=dWzDjT3yvEueGHUC(j#F)@t}LhWDyJ`p z6jDsCAQ-@nHJ#A-o5HPQ*^GLW7;vqU7O-Dz3S?;>6Mpk0yi>MNOU=?eSC$v~3et15 z-d3MGBmnYdKyKJf*EaT}Kply?fMyb!JY}esxX67rt*wLpYtX{<-Sx+CvWAgWo-LIw z<voKXf$^zji- zax0R`kTBgfv$IUAdLJTNv;*T=%&*02~ zk&xZi^O>dHmC4XvQ13OY{0aY~6n#s))-6lHxz3{-b5NTY$2(N0d+4JElNetr!Ft=%b8YF4IH7SdlZwAMV|)WD?UbiYb*(7k&; z#!C!7&r}*rZFC;7BJz;!SF}LXh(1r>nKcHLW5`TLBo^;Sb(MHgbg*j5+)QaLVmy(B zCYsZpfCs@YHJ)V`=fY}iE3cDAel+o!*xuH1d3M(DR@${6ct=9y+BX?-a8};hVSTd% z!+0p>c~$PC8zG6HX^=G9$%X1|kJ?QG?a92z40K^#|Lu})%~Utcfcr+)+9tH?6-6B{ zOENET?)ub;XUEy3!>-Y-*1WPU=;~w6dEG$?O=?`bZY?3%*+4YaNb8qVs#ZbmL?!yU zr#z6?JWmj%76KyT+sKyF$ke!$3plPVh$n7dPbDs|^IC7=w}5GNQx-&#i87OJpyvpHrG|in zE>7#8xmt)z=0_3UxvivXp1%yP=j^nmZGGn?EBCEF|H&VC!lp1!pXz+8L|0`|;j7h` zEir8&Y}(UL)Hz9+gogF#g}|8^NIQK$;RCR$%)sDtHv9T{CDZ? zy~uq{8ubDhT7G1Q%xXqeOfy+X`Yf+je%NGonm7o*Lm~XUSHa5#(r0Bmgw!kq!|w9) z)Gb?Ws%V?s#aj-~k8K4(*h}EIPIop1MvB9F!3ZmtKnI;j9`ECZm4rZm;Q8dDmAot9 zj(1IBf6K-O*~%%~twc zSpPFE+FU&&psKS5hI?4D8o%^AwrDn_)e&#pRA~`T2F-Ee2qb&-bWF zt|#mj3ij`CS_)MPfxdGC5i~mApN*dn#&vBIh0uFMkuO$Nf-&R$EG#TEyV`)fu=FR{ z_4<*Vh}$UrMGm8QN9+j8mH_v#4NcBj1wT>Rlq@wfQ{WI+X&W@F;ytZk=-M|o zMa+$BDgCcVmYj+Y(R5_*W3-tI^s|p!zZwj)#5Aa}ahVv2ojLnbgf-L0EW^I5eH6!7$UK({?mZc)N|{ zXqo#iTxki_Gx6Da8U(Jy!Qz9htTN|Y>Cwr*?Bm#gnP(P(9?8Q2XD=+Y9m25BbCq_W zKK6&Ym94k5Nh`yPfuuqlli6g{H&|{yW?^@e@Yf*8A~ET)-hNRmMj~=_532B9rkJE+ z&_2PP9dT+4?$XDLHdRKRd9Sa#t6r4e?fEKDe2B?8@*F+Gfxj&{w_Bo!w5vUIUn)qO z>vM7+NiEk#+T9{=TN2)rw*x|J)z_&rQm2tW`{7+$UkA~Uu(k4iIOweS%rGj!1qCcz ziuxcTRa`Lnb{aEbG^{@GH|#q>iQ){k7^xbRBzt4?$wve_cLWCfo&Tr>hm4xCNDHBC(A5tHkpuDnx|V+!N; z(QsH2ZWYDfZz)*_?`5AD6%XO4ZArxghPE>6qfo(B!_+?IPlpPViIhYp9nCyx*=_2c zC*{p;5>`I-J(bjgK$jyI?Y>h|mMbT%k;hPtS^N#i-%`rtYSgC7tv;(`RVF~+m@wCK z3HBU3(>?U6w#+3;aj?xAPQ~B$s~Uv{(!lv5jix+0j(%qmj`Eeb0;7P&1BFo` z<%zG$CLGl5J3E$w>fQ58p1@2&e96nvCMrG~fszj!ex`^83#!)<4*8g6@(xj?282F} zUtGgoL&|qWP?5aXxp%v&3ZgX2TeJSmy9~J?l+4lFi!}G|+f>)${)9TsPL345HIRbi zC6$e*ADpcagmFHanhl&33k?AZ*ciHvNt({8_|DAz);eeryxtnj4mSh{xQCZRJ>{>1TsC8l>VCPwq1 zT0>tKJiBK)j5iS_OqPJ*absOGjUHSZF=Gh0Po-dz7^2$Er^nuT7sTCe--Jx4mjYbq z1jsTkc6F_Y;*Ae*<-NMRvHm%i)0CgrXC&u4s&+akt?8Ye9VE9B3LjT5b7+VU%6=RZ zpcd-_E%F6+>;1uU&QGf;xg(kWc#THwF{m?&@%J>4jeeDh#)Y{0&=4xemg_?nHj^kW z#Kj$Ifm+;NBoJ!g47va$>jdggRfSKs!lY*5M<+?t47&)U;}*VT z3I$N6vg|D57{iMebNL`G!5_X-Ndix^|udDMs7XD%vkyD{|!!hf(&HVch`q+*`) zFQO1Dy2l6N=FOh8v|%&#kha!)mtTKjYI}X3YN4Qgqoni0M?+o12z*f>XHu}{;UxMI z<5})asX+z&DB0vEWw1!1cMFY;(jPGwz|E@?5>AYZEs`zPM8(QpG6AdD4f%?}EO3de zJY}yCEhETTzjaWGC>uDNkW*^#T^vDwIkx*y!5h1!;k?)5B*^cY<{UozB$U=nJCM$z zK%<^AL;IcxoUDfp~LiDW0X&e(~~`9JBuyy!b!S;QuFH z{7*Ce-w^RXyqJyY|Gw6wJB zjDBI+-JylS>FrE@ivP4_5g{Qjd?deV08!jOLXWG9;{#Ku{6DXwC}!5Cw!lr!KhXES z*JUsuKv~oN07}r7`24~;0!osyK>QR0rBHDI8$dX))qqg6qARE)02w~fYLIB7-^Xi9n+44FW=6>){X!>x^%nbLBo?h|v_Ce?! zng5mB1UBXJ3$CtjjvVKM{7TN;_@*N86zl9oW%YIT?MJ@K%F2$yp}EQi^evY| zGj$X?^)=wq_)-DBlK$1qSi|W2Qd;PB^40&b$cCk{$_AL>cm4MQ0}zlDRrX~i1lQN_ z7qro}=}irjRu;9zDQb1mred_SMbgky8GAm=tnK)Cspd}*CY3r zs;Moqxj8F;dKd8RH3snOFJnlHU+;er#-{V?Se!qtJfErB%PSyfX4d-9Uz69p8&G-w zl-pl)oG6aahE$ur;(DL0P-d17x>RpctGy4V2xuy*NGYZ7yVPsG7?fW)Af_nrpS|th zu@*jS>0h&heDnhYE;j&loTM(Y060u!i@Tq%Yy`9>s=2>1W4?e4->2`P`Hh9aT>PK5 z+fmZL6+gaSzwQg)bo2%-BQB)BQ85jT`PJjQ=AXF%zDpytF)_XpKdV4*TF<^&zbvo( zd0cboHqPTSX=H1REJV}oU|gnVg36t>p^#qWLp?=0#q_(q6E&N3EMB3NTYKG{K|T5r zpN-3g_ftQI{23p1Mc|Ve1zZ?z$*sN&X9H^S8}iR~L?hVFx~LdjcA=J-c32@?dJ zuRU@LmNzRmSCS}((85`0^l7{|mT?q2!S`i&3rSZUAn!E5lXx_#my;Pq&OiQ5ddfU& zbNrP!?gfoXKQxUp>>P!kWS=^O@IGLB=NTxLv0EzqEOhQL3`^goC<|UsdaxN!V_OzqqkH)#T^(w zz13~iR-=a3wz4Fp6TW2whGX?ob3|*2<*laI32{C@m_MV<)yk%xkJKLpi7F8fDZQSJ zAmp8rC4OXwMbB-EZ#ZghC{?xO8qy;UUHCN^pUH7CY%^{oA^#*eL<{hZcMz^q&r7mm zA!qu{?S2s4G|O6$%hlWnyr4_RzC%$9;zRk7)_}tzD;#;|PYOP#N*XwgOkxH_E z^sjqH1jy@=wmVq%AkZwQ%H2OWwI|mWfU8xtP|JR_+@Pvgbt(wrx-wjve08TV*L|Ty z1c#7E>2^YQyBoB8CCBaZiyb>P${7ne-SjqNN8ko;Unieb1zFIHH7kbWPnW(HT}WKo zEf^-xufIInt!HP@$J>~_S@+M>re*8cN<4#FNBGn&jUngA=zuNZ&GNk(m7AyMsnGX` zfjb5NNe(z9FEjO%@y5Im_WXmD^2W*IuwbdI?;nP2-JL3qkFq#)6GxQFlj6Zyu{5M` z=q9V6qHcp`^08JglR~Did=x=#g!z$pzcY&N&4c zcgJ4ND{9Nj@8aphY1+Y!yr#oAs!Z9qoZcx28F!6k0y{jt;z%TQ{A~uX=HG>*^0ynR zw-MVsB^xYn=1`SS9>VunIx^w z^%KjpuGr&oeG((z!iDChldiAQk?OrEVX;MbM(@k1u7X#BA&bKB&DrmnBz4oscPFBc z+>6P66x1O=;&66`d}|lV(&(jT?xh9^KjE;JM~K;ts|1lJNmf#|LFS zkHslwnM|cE7`h=OKd=_u|hzsLw@VczNL=l-Rbbp4}6^XkV*(iOAGh< z>7dr6U4&`W6EY4bZkV@SD}8kPHapWLcgh5BizPvq*S>o%=nN?wI1_?`Fay|Y?zfA1 zg^4|bA^JGD9a@w@dsAg~XIqi%yHj^FZUt z%uZCo2Ln^NB&Fm~U(13nt9oBFx`i6fXJPGfW?N*M!&=f;aU2dUrpS#=ul)4eVLZ{< zqhnW*D8@(5&wJG>8dXbx?E6icUhH1B#h0O*)%nSftCTNchsz~Hl5dK)YXZKe%k)g( z@S_GrN`>+mYcHXMNO0w644_$*28-iFulOyJ@`gz9FgZ@QM2L!$k!_WP+vNcKDeDMF zP39Q^1MXvNuBnc^`+B{r9WPlh9?nOPprADSNbqsO@ux_5?KKNsA4g9DJb&ljvTq33 z@_RNHfWg#+FoD8b^#gO2z9k4+_B@s+DMz-S>agZB+|HZ;Y?p>pHQGpGJmmfUi><|X z*T%!YmrVCq0hAs@J%U|)Kd2Z?y{7!ZJbMZ9iMk>1$0eG~>u4&55#P@v2*2~XE3wsjeoS(WA-0hySWk>=l3pdOw9 z^6QG5tVLE@Qt51AR8p~=(O==YBl6;+R@u4=T@OyZe`umHqDN^n0lMq*)sUV2;nO1W zp2BPsPiLa|44Wfa5ZG99z@S6so;$7%uszw`1m|AEizP+O1ddK3`Uw*nHuw~#)G%0u z^;VD+sVJ-Y^h0q}OsJVa0pNFr8@y+m;g#?Bj7$sgnZ^hA^Cz=*kPQibsjD92#XXUE zA*EHtX8PM~3a=qQX&HlVPfh|C_j>2f{w0_wGHV8EqZ>` zK%$xX;&y%#%n0Hzw4KOr%Mkn1bgJjK(qRdJ^e+sKNEt+3 zSLTAA@j9D@=oVs}9usNEO(LriRchjB-%wTk3GR+O4zctGHFfW8US7W;`3xzKRK!v> zNm!k4L6l%3n-rndp;n-N$8p4$KCNY7h@#5?e_i!A> z*YQz*{~-ednmrvE(i$s%g3qNRh`gcxFayvMkst21;EBr7B#63|wi~%e3ih^R@4_ai zr`%CNkiEjYxglXH^RTmAO9C0i-`cjdexn!emP!S~6FxZ~KEe2_e5Gz7glP8nYUVtc z_B;zUyrmZ{aU8oP>akH~;hs^T!$JImk%tC`#?}^hyrQS~ma{I|G!6WTTiYr(YAgQ#L3&D+i@ECfiS(Mnuk8~k15&(DnU|1}FLMJN& zKRPznIJ#OKiJ5YDhxiPAg61JUgIQb00>IG-O^|_C<^E;*-w-8l=>Q4_Nf4U?e%c&0 zX3Sz4!6<|aCVqN^IUEP+>iUj^DK~BF!Xt73 zd9>H|?xDK@u&dYVK)De0X;#^x6qTy;+RJ`})lIwm=S2PJzMzgW+2i5ttGO#Z2^YaR9aLltvimW65mTv58Knok@pLdi@?w7FV#PSc?_L za)k&i*6WeT;rG__J$Nt)?{rPR!e{hA=;S)59(w>CxroF}z^?Dv-~)$k znjO@PJQ1NAs#RHtyu*6S&{;}_LsyaJoinoNP)!6I?2G-(1jhOqH7AXZOj0cQpIbmd z*5`>}v@iXB?Q||8RkecnF1MVRFZ=t1nQ-S&2db@*dq)e8AP}>9|@oX&=KrR9)p2L*;6A2^McMk+)5yzFsgH z3*V%0NCV+j4j8$20kJp&%(=UY_3nb3J5pDU*M zQp*6jIR{B1HE(p;OCgK%N*~OagOGp9@(ScfMwy^;Nl=I}{d@}flJ zkMJ_#U=XVrF^}Zez3mDqDY-YWqSR2?E*Xli#{cYA+z^@5cq&*-ypcjCidN??keEa* zilj5Gg{Okg{{(4B_&cAISTeYiZ z<4NOxX@jab^%=>AwLYsE3_4f?0gm<{!z(n7M-}E+-F2<`I=3e8LB(RlElI%-8V%?{ zjqvrmOOYJxjhhJyhc6=7XDQ@MwKBA}D^s5ZcMMn7G*5Im2M<~Lf@|LfCPKMEb(jbW zb}sZoRV2D7JU?UFRjp`*E*7WRZ@Pk@eRrqygtU#MY1`waXu4k)$YdV^iH_S{@=UWL zV64fmR4Q1j^E^qMTuIln^TtmPi+q<~7M0;>Ky~5VOB-_kU4Wj2vfX&qsmk5;U+ao( z{09GQ#{P6rq>3f;n5+$){nI69Gf|*FY5f4D|OEJJ!Mt4XW#6FX#{Ho5@P!(b;u8&m%CYF*+!u9N0R)2OR~-@w=52 zWY<2>8K3}5rrwhb^EsyzG}LYuNt6C6C0Q&JqxW_?p8&@wr#7*(-TRI6>zaG6o8tO@ z5+C{po!fjNx1kw#;+Fe2a~X94X}{9vGxFf`v0I*HyJY*0N&NsONJtS^2+6n5O$#N5y^**vi<$ZR*Ad>L}t=uD_oiro3aE`o(;An~>b*&;g3P{W;hf+~p=KO*Z%yR~WZ_jKk1ZN{ER zY)v7jB{t%WSL5JhvqD$|3)hRv_%Y0C;ep)%qX+TtM(qGN$v}>^6;)h z{jf_k`~bD~Z3L73&ym+JYFuhvO(h4c6js}!I?ogm7P9YTrwOB#f?8lQbE=pUY=5TS zez2EXTY3+*8)NYlti*bt&KypGp*Rf-J*#jhpb}NJs88R1BDj6k`-7g)0LFgyEAnoI zmIo~;9QC<~LWz$K@EtK_DJ%*Eu2E_g% zEp1c}L4xvXXFU};!R!nDO&DzKWu;Ku^b-u$r^rXh7y*`dT88~4c@qM%sHeyQ{4(F4 z@cdMMZE^3lO)3^VniHiZQ6f$l__nH7c~nF828%t3ZBZF1-yI-FWN@5D^MNiNX+Oc@ zEh0JTje!9hy;)}Pbr@+9Fc;kr_Gf=o@B3l{y1AAY`B5mL1i8sWvQ}24DxGh`GMZ?Qs^||*n z;bJrKv+t&$FYuHeHr-kX`RkI^q)8fV?+;3u;x|vXK$_QuOtyOX+IN>e~bZbBs`L;wbw`SQN!oGf{ab?g@+SyC8)!f(4 zj5&%5L#I>eg%=2}^z7LbXmv4CZ^;S#HIDpeb3PXRU4@nu2U{UM795mY! zN3)Orq)cR)ek~u4DQma5^Sn~~k;kd+?xY|lYncu0C*XGt3$H1ra<-DE#)(z4_*&Eh z8kRw%ZuH}3c7F~|h4hm6tc7zoBibL79|fR)VzXf+LBmk^ zm`3@)tz`}xLt!)b%!M38Bobd0U4oExR4~bh3!DJ4xSm7b=UxN>BMF8uUtM!`zTef& zvIW|zj8?&gy9p@8?>)Z0NwSUC4f4EGdIkO|H49z@x-m~a6`?xKz#X{h{Foe`k}~>!vxFGO*!W@w0jNQCT!*wB|0=7jyhAef`xAEd{zQH-xB0zWOfoLYYMhxi zaF4-sE4hC@$_>=HJ4t`pVFF}@jQvY&L>WZTNq@1JEbKCtf4N ze*f@eP^}Pf_TDJdNp|p?*z$m@A1dDgyA5KvAPJI;HUlP0;vhCkxBRh^^RQRP4%bma`JVd(fb*DDr#SYHy6r7$=sTYAmb<5wS+9J*V%q`FD+#WK^_KERB~t8l7fA*DrT_sm`8WSrjcJY^ z3;y##Y_4tGR(0VWr#wT1H$BN6JPg?))|u%k7Oy#qcg3fYn+WiQXY=@U8fszhy-o0B zu`JPrXbVS%nH0A<4X>@>C-PB+gQf@jWY;m?0`m*w!RjZ!qYA|lnsiPwUV-5r6K}&t z01-B8!xS0l#KM#;%Jd!17)3%Qmxx2h(qsZ?;kV=r=PP%9Z$odvz+MR?`T{-&@=t$O zaI4R{!9)v^fs#Sal;x(Xw1T7J+YHMhGM?W#|9*8X)uF#QUow(aJb@4CHjEoat3@MpCN%4kdD{Sx7iY^10P6X`k(QG3DB^g3p`T zF=yFE_vS$%y) zaK7AP0gZKxofu>ob0s?#E+on&%4v}hF^CNe6BC=_YDHn<20 z)!~Fwlo?8JTlT=ojvH!DE*s2AXE9CC;xc>Ja$5bezx2JtaMI8P0r*c8%@wwqJaBx zL|<3~e>$ylP3t$0@i)w3G9oD<*NLL-{cV4pEen-A*t1qu$yB zuXxF7eVNuDX39yG1(fxrh|2--OVpb@`j63HdzM8dw>zK1DHBn?)yEL^?thgcp`i2e z=;Qg9&fTifwJGHXCoe3PbMgQIOJ3P~*h&9YbqpDkbldT#q8KC=ihwS@ds&z`UQPt3 zX<2Dt#l%;lN7wjp67nNiQeI;AAVt-!R46IYmH-E6a_cwce5GV;p*^nUM!;2z&!yX7 zL)MuF6<$F1i`GCjPbAPzEDv2`)8Kp70SokCY@-|wnr5_)x{U) zs2LPa`BD1`RiV=g=vi0ck1oE8iJ*;uL@)1!#|%x=uNAC$`{2E6s^J=iQq#d*yZhv^ z(6giByUxy@juz5{my2)v+wWYOyIF6Wt?ftM(S0no<2A>#RBYGp+60gbo5rC3QE1cc)pjZr-l|Z0xjZ|Oxt`xJ@Vhz?pwI>k(r}l~)jaPoR zL1y!#Gck!UH!Md#u+JWlrN|b24XIew3fa<0<~qy(((_^hi+DJj{gRe5N~(OAACBt| zw$jWsv)`S~ILhTQF7%2@kgJj1E|H%x$a%5tZ|JsRAN}zG&;ogC==K{{9M_gN5T;cO zupU^bLNc)fDpZcS*!M43+>(KzsP>`OjCPr4QL`rgk%59 zINwC)>sT$Uzo=(ArUE8SHoiMqd}{MhU!l|rIh>=3k(El`oJIn#-1j_|$X76OC$Gy# zNic?n=H@>{f9Y552PEj};SR~+&~AKgS~rfHSLH=h6zl+XgGi_l+d+hAQvDv}Fi!4; zctBoW02T#DMQ?8r>MF|rMrycN{MW{T7m}bjqcY}MZOC|yDP&x6^p5zj( zNcKA5=rIq7=Mu&+%NE>2%O@T6P+|%7d&!#J?$wtD8etMz`Pyc1Th=5~_(G`#fbQ_0 zWXKj9_@;QN%2W@Z7obriM1d}-rc}9cUmggU%Hk>sO>Q!K1d|M0dsP84p}t5rXrFue zjoYcP5f(T6oEWlIjzibPHCYtdr@krh3$Z5vmr5_`S%+h`ld!t6uX!}5-4*S;0iF!Y zMvCaF&U>)c{G$WFT#+`TnC36X+akySol%EC2dl{^4qkBC3?#$HU98Ctk_8A3PIFB< zE$0uvJ9_bY5a(!v@WE{A=AV8M=)a%Q02bpVfxHhNBWS@Q-m@A9P0vV`X8=Mo#K}s0Q__zNBSmAsv32BQxY|qCD z;68$jRLl`F365)hv+kqf@CZOwTF-B5f5{@+1SGkOfyw)0bRXkfG%D3(67H{$07|qS zZIkFpZeR|j2pON|gwnsCB|Eo!8x_%*)WuieP?W|a%NpgE*>&boh~b4>cJ!34g<{Lb zK&Qgo*SBSC^p82rEYeT$R75(HV?u=0OQ^|v(Pd6mK`>&st)DWaXJ?X+$hJFb%Xdi3 z)ag(176qTCnu{xWavOr~OB?jG5z+7G(Birem+B?H~iFJP0>TUVhP$skj-f(J=oOYGBpp^ z3(QamT_c&-0PNhq+8x1DBN1J8SXmo&AuOBPThk1POZ_FUc#Zh#{_aVeGqZ;!SwB{) z9rx|NvC=acg4Ys9U9qMefg`{-Z13~bLLO2-KBC=(M3F3MZ`^)mnSrIFZZ{uLSB#aE zLrw~Wgm*lk>MX($k#gbeMViI--8Rape@gGcS7I9d7hs#lYa!TG|Jv-ibrgih0;h#7 zsKTPn4G$+b8mA|mvjWim7N4GYO9-ZJzEVD9V>sgy%#sOUhyRuCarX8HZU6N9@VSW! z$bNs;JUGm&NrX=*@Gx*+VW20c+$r+-#(sY?v$Nw0o`JZgvpD;7DVW4 zIlj7i2omx3Iq*k^*5 zWy}Gi{wHCIiVviaypSyw8+H9r!3miW+#o=)IjwR|%!FESYo4_PpK;T1h?rXHVY;Vj z-nlm=V_s9T%)SDK#%&=#l7+i-^7{!WRa)ifA~xrSrLq8E>($FyJyAhi-}oVG@2Iv@ z?(g=xPlu~-zb@?_LaFY?IYikDv{l3@2LypF^#h2RvJmeE&4B5yulSz@Z`)#e!1A8e z_2yi%qW%UX0f}>#s}pv(U&nbeb<7hzo4o;%T-`!CxM{Ml&a1q);*y&LE*Wk^&dxuC zEwP!+@^X$Y@%&WrVf||N*YamQd&yY{D8*|v`Mql>x(!st50|4gYSv)!(fp5Q9IsrV z^-uBhG09F+{_B6iip~U7*jJC6Nt{HjpB7(({^;3sk*%Ot8M;8LAn~GjU~sqWgirRp zje%xea=EOn*-A$zKhf2M7kO?`Z>Z4g($wOWiK|A^5B%1bA+)jW;upgxUS(wz?;QBx zp`=gov?ORY&HRfTAi-Y$Al(wgF6kCkkwKnaP#BA@rc3Y)ne}jUv|%c7SP2#?P zAL8OynORlQ5vBHVk=ImAF341UQ%I-5pA?89S*mJeVt+t|*}}CjXb_pp6i!R<0LH0iiONAeW554Wmp@@SBDtKwQ1RiQY4*(SsHKh%Hbq> zB%X^eyWgsdcaefPOlFuQLHI+N7PGrZ?t9FfGg^E&$_86tlGdiZ`%w7ZL%mjh_NP49 z2Y z4U{dco$Tp|e6EZI?JVfTIMAk&Ryj$`xBvJ(XlQ@5y;P$D*V#KIS2m~QnUN%4KO%Pr zlClN84>*(4eTXhL!0V^EDhItmf{Qs}<%|dc$kr2JecUi7&+&-jBXcAy=pgF8lil=z zZX}si{_JxyWu;3{9PjiT=xQ{T_FHQW<|4d-NDdv3&U1Pea{#G75XXUsh+1f zr+g|Zp*qo*rqKSCHf9D64+5NU$LAMmUG2j2D^;) zZ@1$580B||FWz5>*4QGUQ2n3*;R zdK=$Au}_zE;H$t+xnV9B6q%py2Rswkj`~0M!55$%X}yFZL?EjMfx=Mcrqnp9rg?2;BX;Swl{VQV^b2v?H<9 z5hoEs_y9%?3CppxwU^kp`ph^LtQ($C+f<_x?FG#n|4n9n$d7daCG@CIQzCZeUM8j* zvcVOV$)`P&(+{P`Z35X`Ny*fM2)7f^1-f+@aF?&o$5IH7Ffsi6!^IdM44vim3rEnV zpjA}u$m}Sy`5mwU6G=4(D42KDpIEdEJ3aEsbyyKI8N2}`$u0FhNmz>}!E;R|qFB>T zfR4|9v?jcf_6-=vS*MpEZ-2^4`k`mX6Lt4sMHGxmlVR0d2 zohKdGV(zL|t9yx#r{!{PpaBju&j=*N#Rjh}VeiQ`H8*Ikp$5MzC&AF$tINQRr>x&r zq-)IlZoDres5n!qeoxxkk2-&H7n{DcY|SBSf5TIw*qr`?kZ=*-ZNuT{G)(2)x#mVtetBQur#+m zAg4hX7nQ!>XK_u6bHR5PZZ_j@=Y5;# z?-I=zQ3t~kXY5pu$6I8SsTe*x*eHHR)&?K&ikwOF1gAz^W@Iu(E;9r*yF5!edr115 zMk2BJ1-=|nTRn7FSszXx!pKi3L?^Bk$Z11HQpR#3+zNrVPN^!*Lc>oHR7s$QZ;Ygy z&02EK;MvVO^$=54*(KCL-(-6d=eWD|dr7>CZe@H0afemH|2fYp`5?6wkgfI=oYssL z=uSk#-RBm^rBY*l`nfj3>_Mh&uQIzw>c~V>#VSQ4vQ$3#MI>E3%3k865CT|LhRsck z*?PjIy+ed^%#DMFCmv<4*qx;czN(OU2g<_Ad1mH{VJ{!$As}2lE(1#PAQOuFI%gpp z`Wmt>9Ek0~>&7E>utJiSbIVp%5(#@>u)3Ro`zyXTIrbglc@7Ry-THC|DKUFt7ozh+ zIXbnXjywMF3-G3>mH)Dnh{f9W<=7~F13*7?+6^*QNf=?JYd*4<2vyxgIqgrJvz~jd zU`qeXHsGy>Q1bYWHQ(M9XmH~>uJ_4No>Y9H5xVA^Wjk>uubN9mW z&kw0OJ3rjP8!2C;0+hO+-8ZD+{`M#!(Q?BK?P8u6#;ec8XU=_TeT$^Cl0Dw9Odd(- zf;fXn`ZT6rS_L+wN~x;L4`h2vNuS_7dEPCeSYH~Vg8?<0+SY*JrwbbzzKs+FHd_2P zm~^cP%_)!O-3>-!(4hmx>|Neb76LQ4;FZ%u7v57TK0<~MQ!@i2z1nc%d_G@>(iUp_QePBj(PYaohA2@$h%nkb1>Rf~ZFHnalmvi5TaP!5 zDob=);U`ElNgb3fTjev1y+@}|d%rXTun6|4ib`|RA5ub zt>7qza~({DTD4;0R$vVToZO#vCp&*OXrAcu;ZEjcm@FH5;`M444wfQA5IG*euz*oY z{@Tj-^{6AQqE4fS;e94FEP6-fcdVi6D(Fo%@cSusblsxdCXD_2J#+MQ)w%3wKOS>d z$tB}X(B~qZKsPOeQ_60aw5vQP31vJwK_Q8Jd0hpiyjPkZO_sokXDrc^A-BaoSoNQQ zx9id?ZB{73iv8zYT_F}$_ofIQ_n#5$ds6kH_895j$gn&R)Sp)he< zn$OHOCX^K#`ZfUxD;Eg8yHYQHNi=DH=A=0E?->=$H$<>TyoE5*&IP(n@mtdyvLWXI zaEryL)tr#j08_4p>zYT2X^YH~aDUY{XU0VsV7V{N(e_d)i}$vb@J;N~yLK^YI|T*h z2y?{tm1yT>HgS$mjk+r-nZR@qiIX8SCf5FgLpySNRP3a3nG@Or%K(KS(T`TE_y}k zn`E1Oz2?*nT$TMFU{g#yS^NA&aPmpo_SC4CE>@%tOxs$-YjP)EAe=U85sR6dN>cKN zVBdpY=>R=DBjmf zR=iq-zyx_&cO~fYUW6#Z|3gYDHEocDDWmO+b6SLkIi_gsGBep z_dsad>w}?)d_eO&vXIkXZ>h{%O>Nxo+5Lg{rkSm-hF}c#@ZBJ=7kM4sL9I|l?U44? z+HwNF{Ao#dqQ-Nr$5L}YH`!gsLhUFGRpNLZFT)P?Kd`xkka?=4R0&eVmi5-6L2YK#Me_1O6+K*^$zAwDbayj&@F zn>uOi&H_nVfa``DDDArK=T;k+#b1gu;vrX6`j?&tFEvjHNyDVHz^R@=)Vp>(lNQH4 zhq=%?Zq&X2ovE1+nceuNW8S0*4jr`igeBU|RsoNn!olFPU-_!whZI7kOsqJPDc}n> z*Z+>A^nH|t`4$x_{~;v`Ta{f+Q%Gi0@R;L)6E$~szuln&yyQMO8$8v_+CEI$o-Hhe z&IF|4n`Ccn*O0(SrDxm8#xygFZu$RlgPs1fm$t7C!RopI_y1Re*9}S*B(amo3tiwg z)*6bdj9r{F>@&PYK>JF@auNInG~Maqh@wvMr7F>)DzDF(k$=s-g|kGEc^i#)VbIhl z&c{r@>XEQMdcR&v0fu>uNf)h`re*rV=A#NF7|unFvuPP>p2UFdQA@r&`dRQ>A@xwx zm1tSmg7-a>*e$H>T$ABPm$^WumNIN(ZGY(~uY@z#p;HD*MCPn4;4|bWi9Kzztcx|O z>>w>{yg5d^-74TDJ$1##LF?wRSp%y6>iH!FuiqIxrnH#=<%Kds!{{VeQj z!F_rB4-?Ho;q|V8>N^wjw50On)sP=F{W16Ldrl#>kIycP>j@0gbgc&}SjPuy{}ORY zu4Vb&LU5F`kQ|J}-aAFT3fW;%($2gEhCz)f$!D4B5Ot9P2wv~KgMr(UAGaZOv_^OA z?WQjZ%l3}(%ZH_Q@(I#jc{XZbfDiM-MF9a7o-a(NJ<6cmXGfWvH9}9<6q*xjx?WT9-zxs8aHOUi-$MPF9m>bdq^=VIgv-<<=313o!%J4oaK zMmMPkCotEO%C>p(U*^QP5hbQeL+C*oT#{AC*!X!D^=5N>pW2{^b%`bPqYxcDO=u(yF8|tissJ+2a7BJRj*P-nnPaz zAMNqe3>tLp>F0r~ARo*<_Y*f?rcT)34u<`-!Fw5eV<9X?!*DLEY6O-*ZK-g`89=^T zGU!#~nvo~1Z9ZU~+H(!)xz4psJtupUW5zzc zVl~7+B5P)%*%mWWE6vL$(I=4n%ZiM2e^S^dRQ4>)u%#HhbS#qHBnNkh9tC$L@vMcLIzqG{mPhjz=`)D$_NB~gfeD?Eq#Vq=^n(?oigre zM~q&WD}iMd?TBCa;`w48#S{*`R>o~<-R)}Q&kaJN@GOLA328*XpcfIshPU*7^*>34*Xx2^1--xX~o-g0Gb{(V{tWm?lfL4P2avU;CUN2s2G$g44}{(V)f zV18}*W_^go@zc(E@gT*fY5UQ6#8-1|9RT<0V^J>jO%W4LSN6j4CE3AEa`NXi5H`2K z4j)%yZq;PD%|vnlcJ;GJryojQj|i_N-?<>N;)!r5Xv(%rcl$j6+SfaBkrLPbXB`*F z@&V>^!}291YV}*hIL>F8M;_Ue9Vz$v1w);<^B+41++S`%@&A$LdSZxNc=n8;lgOmm zRS8W8ZFE#NckaC~Gq<{h;u|fs)a<_mf-k2cr%M2Cyg=^#LAkX00s5-lpx{=Wk^p>e zs^`0D7ZHF@t=cJu$>;iK){d+ab?#mgziW*Y>Yz$LUaC4*L6z}kmuzDKUF+ltt9hV{ zEo=Gi2@C04#ohbmc))&kQGTNt-&1Ko=nZd~7!{MPrwLJyrKsBOcZf{2(#^Yw4_3PR z{jvffT59uFbE)$S^lhh?JGWdvOi7;w=<kGr$*Qd{TSy=}2ANqDQ8%GVt_n^#aE z67lSRSB&{AC2Y*A`N3#0LLjzirR`9?d0VM{8aiJRlAWUe_ehSNDLBU634))V3k3XMelSSCErf6lnNIc%|*+2F|V zI`~F&zz_uze+wRoB>l>S=Wlf=j?9>qRA*1nr8dyv(FpKuI7geHl-9T4#Rjtx$_`;# zqTQ}V+}trH+hU@qb*uv#s|L{W^es7n(CvknbgB`HRPN6(C6-V{ry14*b*wsmR8i>) zYsL^btbMZ#YEQ2Q=&GoMo!Q`yiu^k24OD-IU#}33LCRlCmZ2n+1fK}b8oP1Mcv`z3 zm6Dp1$FI8Wr57vNCWEaa{N@lsC z3eR8cKh0ScXr+@Jsyea)_-<9-yKAYQvXnV)UDuEs?g|VxxUj_}zM?H&jnA>j#%htj?oYWO&YbY5T0Kg-1AB zW0=%jC{j53^LqD~+I9(?r8WmF;sJN{@J<^6)7qgLf9^9jQk***@ho-@dz!f&4Q+k3 z?xgx1O+ob|6Ng5(U0sBhD~#-i0hSg;N*B z#uMXsHh;`o7b+;f2RRjn!%qYxo z)E%3U;+R^iWBJscUnv5Awqm2i)xmXYVNlKnlo&n}y6OKrY_IG$0w`t{rdXyx#srxjjhpHXO zwaqAHxt?pf3hUSW3LyUc0XIR(KKo!BG69ZtOa<6p658~jiAoJ?@>f7R)e#EL%M!~0 zL1%JP5ciss86xAa^GMFX&4euVPE`4E7XD)Fe#E%;1$P*4WnIlL5KiFPoqCDMTmw3w zWSvpeA72}{uWrZP&}tr2yj~2;=Q1lK1A0z$=!93S1mxiH*jsX zk32C(-V{Lw-M1W4x}6pbu6A;#->2K6NgE32LZ&6&k|7KUc*>)3ZojIuymmCZ$U#fi ze;n<$n`vudXH~g2A_4vG^_je4uKCoeWRe*Exa^w|X8jc?K9mi)Rr1gQh@C!-CwC%3 z7E?$ATDep4B>-!b?U#LyQ3NU+gBXZ;yK%*!Zsy#tOQ4&~#iZ#KQI)AS82 z|M78a+P`n+Y0y-yOyNqI8m|fqIWQu;B!?^6S0}X27@UhiatFliX2MOfh+zMy1O}-2 zZu;yYP5fDK(OOsnkD1I9P2bg=VH^ppnN@`h79ohKYRXPiDE59* z0(d&RF$+P@nb$i6CwM7s);foWGAtS+b$AwTt(+CMasy0X}0^G3^z z!r_e!dAL!@FYcv}EKPAF>4$$aZo+Z~%@yNY8lpK5?Ttm2?(_ln{i0MoSh_GCxx-}5 z8f*-pV@+FCAeD6`D4{vnZV{6_!-zlFmW$3$GB;N+c5OSaRUxVt=a!6g6a9FsNvocQ zh>9|!{_oM0HYR;1AN%r&o0+^GnItghLTeSL8bFL1s}K=%c_T)V$+y3(jy2E5k{Z|< z#aDD%`?ce*e;OcYi4C2*cLN;E$>-t4#cLqXTY9B$lcejH3NxAw14s+fM5*8(Ch6_B z=mh&pFf~#;x%e*vqo{)p!_?|-4?_pYfIYxMsXJqQ%)`dttw{sfqgxVKwrGMF*FFN(7umQVV z`^MYtvN=c&@&=>!`8B_-aBrLo>Q`^R{iT*4hB@D^vQDpco4)obVfG_S(T=s!Q-Z@_ zaoAfQtZN=>%8W7vU7S+oKBpA`MKh(XE&I-$q@mInBk*Yh*H!Rz;;m3&=8e2FccQ5M z8gc=o9e=GkK6AlgBdH=BoUP%3xks0%Vb76EEet#IM0S}_;X2p|90Y{*-*)^SOEFQQ z-Nzoe+SZ~Lpy2594e}>R>2`AarI!`aSdH{D&WO<;9&3kh4k`&QiS^RMgC#} z>}zD0)fq+G!4j8u0PKZz!Z$Yl)u5J?36MiDz@YgD!+tX~-1Rcn1q1T9`mp&@3yp)! z%@xkG@+(E;94P<)X!attEcD}P2`Xx@OR(`k(A0A{(l zQM?4gyX#uAhByS@J%eT#R9xW4KCp5$dJ`9Nws^)V^*st%$>D3=b=DdZUJPmCNC3`r2*gIprcCG{iJ;|2&=0~v{m z>OpnIs?o=T%99e$6#7&CA4DCN|9_$m3;X{?)L~?1=J?<1|D^K#FV+qLBNHPBD*+!L z)c+ys)Vi6f;Or#PA`lGj)ud?P$LB zelXknxzzKuR=cU@>Xxf1Cs0)~MF8B!2qL+)8GxaQndt=tg~m2ECr9PR2I0oz1Vc^C zXkyuzziO&EUWk zsHrcO6c!i%rJvHH2Q+^1C%#FLLo;gwoPNQhC$#4F!cL%l%RTbLselFo7&s#gto&Co zLr_&&N>fV!QIMvp045P^BN%@pOUa3?rIi72LL(#4CP#oBSeZjK_=yJ|7@0$}cvQER zi@Db!k^>+PNamdwIpT#mF#~G)gDwK!2*$Cc$;sLC2d2pZFpCpwJ9y&;*aoDrk*&!M z^hbTLV{iTHCioAm;|`zgev>ORIR|-Ta%5o$57uL@ij@2=o@?Mt{Gi{NOS5MfkbBup zzA!iPQJ?lZwr9Dg*pL`niwpqc1pJpejR69n#j&Z4-PwQcNB%#lI^8hlc;J?sxom=lVZEzSHkwZ*FUM|Ki%dwy*iE8vwWiYh*DGH8u8~vjX;= zvo$w{n15L5m)gqQ0MO3-Hv;Eg|3-8K;{5dyUi6uTpY=iz7v|QsZ~;IvhnOih0{4}H zD*nu=n1AmhzxxRu|Dp!|sGolSBERmje$vIi|Gx76atV+Z2W?f`Vn#LUdl1ian*`~A8u{jp;? zY9n(W=sTA{u{DAA#qarJ{RYYA;NtG1{&{2fxc|;S@5cjxI00u4(9>^ZK7eYLnUw@u zJ2ZihkGObuYx2Q>ZKJPOqn>=IZn}Py`E4IjvisD*nSC=3er+A``P=+8kqdY8@5I{5 zuBO6p&6EBkE54i*)-Ja>F4~4SV4@I{;$DpT*OQUF{htWLWNl*~Pg#wT!Yhn4%fi_C z+r?4-So|#MJY3SxJpRTwNeMl8HSQA_sF&>TJ%s6S{Rda|Z(`v%&el12UQI!enKcTc_pQ?m9gw zv4bJpM6E(qEs_aA<-U)Dn*H}_bUbE$RtqE^_Ds%IG*M-J z@^uA$D@u9v=Mxz|o4qOW_EWlQLfTKl_8S>H%M}5%OI+u5dGWkkd7Pd~wW>@eXuP%B zb4K(uZ8}jMZ{GDxM~Uyhs8TLQCkFE#*v}Gbf}U`H=3TZ1R`t_JOno<;C_9&fbbq~r z9H_dJP%gR8c{%EDa*0o~KS(VmuB)bM3v4(S{6h3IN_}QmBDFL!jhER7Cz>r93tj#A zU}e4zOpYS-2Z_2EL`karU^FYdve*+Nj-6QjE%rlMtLJmeryn?Tmn%mSwct9(7>c@NOtc{8ig!~jE zxQj%`$`9IEo#0eg9O)o;`(#h!u@>prlfY110dP-$P99Bor z`e^|QX1hWPN7-DRqUWSEW!|KK&V5s@IBme_DDG2Lkb#q?@qKGEtBJ&WAH@@l*oHbC z8u$@I12*r@`1j1fR0SbiYbLJbJs=mIbmr6vIk_quCt!2pgSK3~m2{r#jrC@!b2q?48S~*8 z1B9^skP(`aOHejg%pEq9j2gb^(NA|hgZOPlYeDKzdcdhWd-ij#7~cp8LJK>F6q5(k zZdNt)2W1lRjuQ-w3HM#R5@jWgxryPkrCcF}rzuj+9BZ^#nhKjQYE0cgoEv2*8s|%} zvx;*FyBFwn4Sl&K*20Q8g*ou+tX+Vp)8P~DHw-4|-z_~^`4XH{L4!TJ8k!H* zo2?-3yfL-s^OKCRs-{)(Qp-x^h~OWJ!_-M685GI~N`)8(xiW3CFQeir%QQ}#8KwR5 z3WTI1m{uwSJPR$x6Jo)$PnUtII?CzA`_Ti&Gt-OY04J(@{@Ilg*O?_|g&1$z-~RC4 z^nl?t5L;A9TFX3+OMaDqaV}nSST4!`&SsJhb6}}Jl~KNl)F93cb4GYM;l*hgt~`x> zOMhjn{iMQUtdKrGDnv5VQTbQ&TVm>w9a}BbgJGyol5->a%ea5ZpwkXnoOlJdP>O~3h)0amfyDLvB_dYNp{}e`EJ}d@Q=Cm92lc*<&~Lq zw)_&~^vKVEWcpM_xstl*TJadCz=~L{W^n69x9%OsjG;du=Q&e35N1c{8AUb$6&h7T z!AG$tez#}2Ie~kqUpi1Om0zf*63eIxNw>ahh7&zVLV~cI-iu@R5A61o>N==V7I`1t zB0Ps;`{18HT^t(v-V{~g+T}K<3`Pd#_ilP8?Gl$E06ja)ut6WFJveOJp=@Z)%d;U4 zWmmK25Vk{A99wM3j`U!0Vnh*Cn`?MtH`-RjM&9^PQ?s`wWnH+)U!(~UPMc+KFekWu zeX4qjmdK4iZC9fh9K;wIrGIO;Ft!UTQbh%Fic%SMj_q6v2V0Y<{?qyjS6E^!%r9J!i3(+i zoDiGy#O4A~X%RBG%w*C*uSj~nWhWzC(iScyc9_sT1enJegl5TJBHIHhmR-g*U}bQu zCe%l{>|xZ+zM~xjffoO0y-X>atd$*-V0*z9;#9*-nXYHOcOY)M{@iIyu$9^~P7n`5 z#=+L4+M%7bGZw=*=0tm`8ahhpP*wSf#HC_wb%5s-EpbcvPAR2C#u~S0dedS7jM>mJ z?;HY^^hu)7LW1guv8mt=LAqzLyLEHWEl_D-YK~vC% z!4UFHwah2*+orfDUMfRU)M39_?*-(=nHr$auHAO3iR4gY1A0jQcTt+*O!BjPQ>4Q% z?$ka6O`U9$z|J(F;hN@)`afp+!WgAsVXjnMZOjbskS z&a}*g3+52p{#ilQl5%K7bT3nb();L$`ynmB`u9Lz))Gty9NSgyY^E36Di;$_Ph?PH zx3gn&!cEfRt2vJ&sS=B|WLMbLgDJJtE7`!m{=J6<^$7CE|o5!aKT&+d=kw&#O$*-eNf@|tQpub{6;$m7d%G!avb5j|{ z$F2m|wJIm1_zJ7HM3EQHuAC2V2Axn&^{7Xr{%prOTvgp7+?_H*2jc3ytm*j$??25d z5}tP7pAE+92-e2&dCMmgA-#tcU7P7F?A=UrrtN4?!Uz}#(ImMyaNlc}sp(%RTSit> z97-40*QX`i-aXG)vD8k>fArwLTXnBqpIP54cGv!uDs-?9JpZkzk9(R%(gy@SR~a>; z9jyP)g2WZ5?S#qO`n3Xwo@9GwnH-2L6`>J2e5zjU#A@5$cH3-w^7gS6%KIu$3u;0F8Me~REwBC<@#0q z4}9LHkL1G^RB59HP+z#m<78Mo>^N2%T4lBf_=tw`+-P;KuMTCZUJnKqv1VcNhe!}g zgy%n@kTN@j272&pMM%=vbbr$ZGEXI6gk^M|yDYZKPEM{*Q8na72XzgwefUTJf~3i6 z&lv+Ns=l+d-(~aJGFQr4oZs5O(YH!3QVfOaU0-lsl7#BA&5aXVucQs6;@=T#f$)N! zOJ}K{dqXf%R-a5dej^L#N3M$}*c*AW;b$J0A->~92#|M%N395bph=cd<>rQSL53KO zq6Jb<1lh;5NzqF~Ve;jA#L-pVzKgu60nagpz902;hH#O6Ot`GFFUQn8+5FtV)Q zpd|rBLgV5OpCsaQV!^VHbuoF0dU^D+IxR3Yk_>iEmo!*`XcZphieVu8Rl z$bwXx{1(QHp301z_4z0%bJ;LgRO*^}QFh2Uk4FYHVupV!EI+9XcmqUcVS?p^-ORfr z84MN-M7T@4yZv4Nv>|=x5btUL*{pyzorwOmwwzWqzZa)O*U($x26q1p;#(Lx(wxn` zVNmrA>WxN>;LeGX?RoYM5aB1ppv?vrjDc}6dIx+P#=qKRp*CqVNlHAUu)b-3(c_rU z!u9)oF(P^g1}(Qv#33v9Sw!bNiHT)oKQbREW4v1Kk;RO&+$>vq_C~InC@4Tz{p204 zz=mBtg}(|n|2pxlMz%`yc8n-e9+;E{W|0_IpFK2;(7idbYV{=J(mAmDS7nvMDEIt~ zGlROGB@vQHo0&&G$*w0e0(R%y4Kit30xS~hAOEQR!l41%9!=b_dXfijOn12jfP?xn zBPJ#M$XBL*R-u(yJU7GYkZPCXcVlwP_bG)p>(mPeroD9QEH_rs%uglSb*qoy^GO|= zRXv+x!#f9BF(7AjV~(ro`*nNBWH`Bm)$2nl=OGS)k$uUnTv5FwUgW19<{6c;>oxiC zMC3OLSJTgcRlQ{hJGbN{7ae$>nG7iXBX?jDS1`s*sSZnOVbUjPOHSJ@hi#=g<}Zc| z;Wxiv+S9z9k>z;z=l@-(GsT5^4sLu6K1=tV=~mUKMVs=T30pQ+^1Er#Ny@mph61DH zYOe6g4_IV(e%Fs%@Gh)|;HooSyX&aydWqZDzMln6-99yB(P9>tC2=e;?W49~^CL2l z`bZGLAAld#JX<4=kW=AO$2%}Kxh|=^6J0YbF21JTS|!_p_FvH z0TVTqjO_4@^%>nPP_Zh|>3l)8V7X%)%wuW#RW_Q?0KPt{bBIHRLWa)cf0fsh65{Z$ zy8Wl(-PM;@?skG*Up0f(f9g3%@8e=|L(J0fV$E8os&l2b}$ z8t7wMaAej7#_VHPF_bbQK9uH4amu#5yl|-6!AADpN-v5H1QHV#vVQL9GhRoy!-;=p z_m9C@M$1%Ybd`VXH2)cvf7zfX%Kngy`sj*!2Lsq1MuSR@pc6E|6Q7b?>u+b-{Q*CD z>a3m{;pc?p4(Z*|eVlC(O=lW6qjl>C=y4lQ`|k{UlvfY63=YMUFlDfe8ZuCrKnVe^ zI9*RbRpL{;rpnkc{uvnijB(7jqZha`yBA6msgt;IVwBQkjrE~L@FrzG34q-s?by9D zD@D^EbP_Z13rQ5Mmp}(q@*v+g^k=u8&^=1(V*jWrMw1S8X}kz)eB?%hTdE3FTIHC1 z)u0g0z@J8cpkuI}?OAPQ08ybd5hLSW|sH)z;+zRK3JZ3t?uyNVaV_}$F~IwJm| zqPP=fO-la0wOUEq<1e97C&R8ED}eXaCex`h7WU3 z^}vJlQ;qmu)9+cS);0^$8m}HG9~xHFY=~j*4`EDL5L@vU+s|)r1vteFTpkHx0}Dou zSzMfXjHktK;0b!i8RjBlYjgOp#e#9=!SuE5*4+VTmqd!o6i1o&dq$3dT>pMA(Q_iw z@8L0Z+(R6(Gl#?AaCEI_J$6j@n1a|OgLaxAGHHGaTmLQZWX(0MvF(q>2w-~RXU+*T zz5{Z{GO`l0IyuY(;1qh}XQZm*w6Z-|r|?dMm}m;aIf+lf@6Y^mh9Mm&y8NBc0Om$clC+c57DWkS-VHSRjPU zroi)ougKH(z_)$-J1W8&5~iHsEY1bDE`AB(CvDn;&0idDrP|?Qp_(M%4hgo9U)X-$ zJ8D4}yeC%wuqw)u_%Cp@I-mFlHH3E~6eo`o!+$CU#)L2|%p0ZS|X+pw7K$YkQQCkfU zPw-mD@X`SHO<91AcIHEl1K3hEUgurx)ywbGPmwnUU#;hv(D|&y7lA4>VLukO3c!DkUaauws-tf;2to8Jt0EwLXc) zLp_oLxEA{U_28lMb{XVVGzg({p3!V1MqN?%4coKGg60Hky(p!&=Fw1+l%7gSi6tJ} zikbLfol|J!2}@G3aW)$)%oPVF-u7C4zL*X}qjMy}gp>gKBz7DncqV3AiY(3j&@%N1 zUO?eG&&_||uttW;nZb!A%lJ4)HD}UVm=obEEBMaf51I+74Dj#g)0D)&l$u#B6?siu zH_8vU(_xlDx~irJ7m(VR>dkzJ*x2{>C%5ola7(>$RXg^UpPeqjZhWxV9y9bM3yuwT z{`JyT4{}G!YcHb~2f0y=I97Zvx4W+mAJX#kFmt^3N*1qzfnFwc`ARpGBY{oM<+s@W z@fm~USE8d7jv3O3wWf1Jf`toi=odKa>{odS)MoZnsTG#aTVk~TH%I%^n5ShS#zNAK zAS9gOvG1Qhn*hT%19Fo>?34ns>?gCTM|D!3!U4!Q|8uf(q%V40m7|w{ZYBS@$5ZNe z3H@~8VIC#3l7Rnq>zlA=N&<*tyjBAg0vC5_*95<{!PEizf2x=Tr0gqY*cmtP#f-09 z0zNWYU8;pgNuW*6zT;@}qYUae{=0Fpb)(E*3pbbBbJwE^5Uay;A93=EY=&EB%7d~V z484R0=w8hyKcNNHp<2Lp{dpb496_J$u8EWYzV)I%^dfDVxUM$EDGs!C$z?I&2;3h9 z-3fS$iM7tj)gw)9`pZh3uoa+el{U(Qf0nv|bnZ&9PlT zpqtYYFJ^F`9jj#h*Un)n=oRk^p=gQyg9?2oNam2O!HvPvMDd?d`C^LjAC5f=dLgm` zPl`;xEyc;w$-Zq90jQlke={-qcvoI3i>TfNHE`OUR>6W#|~(BVqlqmi#;$!u== zpy0A|b_usnEk`D9tGACk34U$%huhRi?dzTr17W~Rw=7*zgnc0jK3P^E|F zrKkA7e7Vw715jvKu1j^1rqdn0iAdgM@(Xb)PpAQ6v~oUMnAV8(a*So{4^l~s7wE-- z!9I6->0+dDqh4Qi@+K$N@~u_a;Jntd=V#E9Re(&u=cGGyZ4L47aMdLOM~Mv^ujYw@ z863-J@*VEHyW~(j)E$CBwxHu zWIi(AHCa^3GWi9wAA<|wlrj};v?3{=eb8g;r^6GyWvGB?)_KMsJ=YL-K&&Ug3M_$22($8B{vm{wm6$(IJa zCP{tNx>q6MKjTcs43_u;e~&l%CWfn+ajm$T1c+?>E8CAq5`+uz%LZ7`fRR@iZYxI` zrLBPTjgdq88AtJaHHo-fE8D%V<1kl)(U{OQ_Nk_ladK0f_KZo);QJ3P7DAVwu#A7^ zCm9AqB`xpDu_|2^^nuxz-d+5~V!SP+{Xv;ifmqu#dqN@QVp~AJG7vUR6YE-bWPwuL zDt-wxb5OqowAg|q%>f`;uQRe8`SHuNzI~VNa4PE)Q$QjQkwNL1t~PMX$P+ZRd5+;D ztIZc9gLIMUHrtG0PHEh;4Ik(Ad^{7}ttS`kAvqRSn zw>Uexn+SYs#C{In}N*;*O7tJ^+X!rFg%wEqBaSq3RyxZ49j$>P*_r(>l^Vn~@J zm8ZkWcAD>iXD#eOQ>miuCDCo>cQf}~@Hwqt9!>`-Vxlo?LHFG%O;Hr;dd^QMhPR9K zpm*1Wl16&MiqZ*ufiC(b&awtkVR{Kl8Y-QI7*Ohe8W%aG`c~o4egf#{p4@!aRCjm1 zeqC3@{~9fkO}JoKC?zdW_pN;Eeu8(pyJUe1SORW$KUy|j{xG1cZ#8KT^+LiEBNelMaJu~nZkw5VwfYKtC*pk8sX2xb zk?Te$7TjwCd=!(KTED@VLOm4i3+8R8f;w6Ojilgra_*&Z#%*;YrupXAJP3|;Gg6Bm zC+j}5#a3LJ7)WDx=eAT;yMyl~g0BQBfH2nq5F2(45PmC(wUUbegQ!0Sh}l_JXI>B0 zeSlBdo9dO{2V46liWv}efL-QSQScMT)CRhdi1|mqO5P4DvlZga2yZbk+r#DMi-x=j zluiQ8fW_ZT`;tP9-Qv3GVHhrN+6NO?eulIsUAFLwc)#WQEY>r;u7X@G{tO-FB7zGx z$8Eou*+qlF9o30U!~4#Sz_Oh@ThS%DYIJfh#n$m3%`~zX@zcA*QO({jfj{JPY$nnO zi<9Mt@yeo5ZQ{&jWHra8TJX|*TRyst(%2OEhI_T}+QCW*Sq?RGCqad55>*n_HJYy+ zpS7&EZl+@wYls*7$^rJa7sA%nP|0nL&m<9MU4sWK58aU%ubNO5pV^Mu>`-{>$RgeE z`li{1*;CIIAWdL>!w?^}D4Xg%{`3a<&`EZew)Ez2x&n2jpr0;s4Ap=(=7_x-hIXJs8%$eM z_6{d-Bx}Mc8J|z^RanQ{e7w=O@7r+1OxE!g_fl(K%EcmsAddJ+Zj)M>go<1)S|-dx z>h^L(nuMG~;T9a#X7$}kmG5qzEt_n_o*QMUcZdOgCm)gv#Raqb!W^-#XR~gjKVlmm zL}?Hb$>j&dih!n^&X^Oja=A5`#g@4R)^wN%NlWnH`lygQyEeIyQ~i_bvN8po=se`6Q1>4H*dQ^du+?z1Fm!&Uhz{Y8|w#4?g7IZS?r(0X7e*DewRwm>(Z zU1!$AVA2UaK1mDR_4PcMRILhgKP|mt^m3oZZvy)~p0fF{*Bp5A+urEGB@Cr6dEfQz z4WzHUA;q$`dHT(4F1f6WD1N`yqa_?H3$TovDqb;V7s)F}F~LiNhOg0n#7`9(`=fxL z-mQI@1rqL{MWZn*RAuoGsZ8a!&qn0ya8^Z*+v76GHs2S!!w#6+ft$RF*UvHOVXgO< z>t(8a1E@zdvlIhd?QH?#yC4i^y;d7OVc^-0my6}N`5g-T?4H0~`C;NVN1}=!e_eLF z6#H@|;A0i+M&zIw$EZ49)s;Kzv=eE z`K%TcvG2#aEP^IkrztMI=IUEg*?eItl8&o7Ctp^*mIio=^6T`0Di!_sEW z>m7=YrLDj$&o;yn4rbz6^!Hr6wmhV-&g#AEaTmXW8p15da91pz#OZQ-m_vI!#CS*a z@VSG}ht-!J!v5G+Idh_wf&G+=WD=5>M1_= z{d29gloFg zd>$m+)@8^N3tpViXX{^_Gf9piw~*1=E}cb zIbIOX+N*^~UVH`Ukt@@{M7Aa9xiMc{0HHD~$Br+)I2wbxj6N65F(0peSfzm)FQ zZt7-eTIGT`a$Pr(HI78oX=3+6#A-W|CaY<2nqXfq&y1bK5I>1ba40ifhfjp?2y3W2!&VoXj&z&Tb4byg}+>wc|{ceO};R}$)&3*6NZw{mWQS|-eAG)ArJ0gg$Ij)hw)FUps7MbL0$RqsKP>7VgmVji4~B` z!f#^*G~-!AaQ7-!y&r>=L2#__g(fNmvg<1nb7;hkO(xZTeFH(^K(2Pe8bH)G-gqoV zGk=05qO%-Ci!Q!?jFEK_ETq6g&zYRJExtgw(d6g(-hn<#cyN_+`h4_$okd?&muow;7pGg(Di0<)`7n*x&Dp3rY6|Gk{)6eWCA9 znFABEJi}YR5m1sSo=5Fwm9_4)Jv-e(qY~YoU`rfDOv_BMtAgIGvCLunIs8Fl5J;n%GIIM|o5GVHY%+489 z{iE7iJ%#9`xLEnw;6t!G`eY!{?(?DQv~^=aJ7!7MyvKb(T}A<%ZQnS2l7|3EmX$Bq zJ!aVFYFie(8)vE-LZgR;q|-TuUn&z)^~hJRmFbd9(x8KMbN+khG)Md)4XrAQqyhtI0Q!!ZX|_Mst;J2RvQ>s+xD80<(7h7F6%Z8=-NH zP76MaG^AA&WN9EeM3* z?(PJ4cXxMpcXxM(KyW8Wa0w9H-GjTkTX5^j@BZ)Yd&j$Py!S?Te`)woP^apgU3;&& z=3Mhs?OkH7iMY)(Z}XXdh{eOd)XM)2$8SeO>1`)y4&3^vdy|luJZM@6X_S+V*HHGA z9jQpPQm(`__WN=~*F35=UOB#1m0O(%@8z(=0wtuEjXlO5>q7%HfvNiIa7mcHgk(8B z{17q;8Xil3zenI7Uh+o@BFK^fx(Vx`tEvO*rBZl^S^b+c)CDdBLQ-3Qijn(r zbKg-d5wMs>ere#@;SJV@iM{fog~Vxw$0CXL0fpg^iZg?n<82cSE8U>ev{X*mGT^gi zBW9*`J*avh{)wLFfffLNTcmr`cp#kB;;r5Gcp$?`m%6UUW3BLDe*S?a<**%rs#4q& z#-JP{UFpYU)O^r|^>IYiZZ!I3Ig!J)u_0O4Mmqffb7;sooJQ>q-29W_;;bl-0DX12 zS>A!q_g!>QE_*2!@ps3l?0lhD*8n=QZ}u_$MK4szc<|`aDr|NlD#}M8iUSF0AGHd? z=zo%R%{_v60|=Ztai5EEUunRD3!UbsNG^Udcg!#rMDSY7afbR=P7!rGefg|jIoB0d zTy0(16YU1cXHI34`RjGNLdSs8VG**Yhyc0hJBDg!m_-^)WEq?08R6{8qVD~ccu_xj z)r(jWB{y6shLVp(#AaDbxUTscw-cGsX77YYmvpg2c^Ud@zq-{)0@s(wu+t1v5ym5b zmj;QDluA4%zj)v{Qb5~f3M~E> z!PId`u0UGK@L-*9r5)Ak#-uEsjejtmA_+JfG^L;Uu0vKI?xdEMv{8%IJy^RaBn4hi z4q0I2iagRwr(DT(k(wJ7A|htja1xVn&b1V;@2+CeeB_X)XCe#b}9Zd_@%K^ zv9?kVw^aoO9G617ZS)GW;gkQfuH9TeoV1+GLr~4pz+8QEGUYk#3~^3BUFD0%vU9W{ z`?wginpKBM=E+AfIa7g%M*0V$z%m@dw5Q7(a<--BG>sa!!k}=4I~w%34T5;FN*t`F z-{_z8xpXtL^!02(8P5m`C+NLoN);sGO~fV#m*_xIsbW|AK$0sr%d>&8=0(Zq@7g`Tig&Mr2)fs+%9Q{NGD?|B>sG>jEa^!jxca-05 zgy4jF6eq62k}GBaqbWdNVIdp4F+S#H8^#3YQ^v3V5P)Jd{tu;%oM@W z{4*QPS3%godgR9QmVqvNa7c%W%O5kJ=@M5aPanxd^f3hLW5GrB)gHH5)bAdzeWN7N z-?fxAnSNd23P2Ga&U9ojKiCRBjHmp>qV$h5oZf0kXfaA#>uygOB*XggAicEm7-+~4 zq2HD`;2h7YJ~za}74nTlVB*jQS1*+y0~N<$Q`0p4k;`tUQ1xZhvRhaGbn3fF3iD-s zDY#_nL&kC{>V{BSrlmS7WRc)Rk|v^4%)T3uLNgJboc9 z4G9@=7ukpI=wxbF$O#ln3=4lHIpl7%4VUnmm8*y-qTjvlz`a3CsYw3#q*lpJ^WQuR z!1llQEC3NZ7uSEe!{1J|$G(sespA%XL!|ir>tMK`EV>v8QW4&-s!Z#~MeuBTPa+|(YUfkQGtlQk}$gg{qu91DToQ7m&eFtXTT8vMa zoUiUY%O#k*{jH|MQNHS{V!LAAUbrWx(~)Q6G4dQRGupv+ZUN&tcV6+rcRaMafq?pK zwb#ME%X1#EsobEx*tj!iV9&{>_mV%}v5*<)rgONC+^Wzs=TdPjHc?=8+5@ZTGYlXVlSo7hd4jaA-|ZdhB#ea4eZRsbl2uk=Q__bCK#xy){ zaJF{uG0SB@;$2g&QY{0gBZLXh|kKdPf%nX@=JeT) z)24Vf2L{lS+23R2!3G%BCAb!!7f0&?4u7LRdqBR6%EUfZ_G7bZ_VI~48i|Yq5vE!& zQ?Y6WQ07kzIvX?oKxc4?e_}m>!m%A}tQ@KakAN71?CW1ceI7Bc)*Fi**Ky_znVfSW z7Pz~NLo}yujP;>^{N_#nM~V+~(oQ?lVM{uLW|C_tKzVBv<~nYw)bmI{BjEXJd&jv& zA9CIc>!@T4H_`#r_e1WqI!$%+u5?FW(?0D5wc-c!BP{tJXN&&^1ML4FV8F6*5$%70 z0sVEd*2R;7oYq^#7O8iaO-K$7F^!yrYJ9?+3O&DNKEBe!f{I`Es!dNNOAQmA5+hl^ zyag67lI}i=QhaPyrqHfRS9AWsfMEpm8|#hx#H0MnW)oV~yjvCLT^g>u+#)}vOHJb! z!}SF(kI)Mh#=A9ZZzUJY74IvsIhsB2FRE@TasC5SCz25Po-a-~>rS%-t%h}qvzRNq zaea_9ZLlQ@BIn^Hekx5GYN7U1ejgY4g2#p}s*OB~ESYys0j}21ex;}P7lj_wdyU9! zY8q`ST{Ua94x75QzrD*a%v#`dbplpd>XO?D=v$!oPq+xo%~QcEk<;MDVZmaHmx}&C z^_hwbM&1)IS^>c;t_x!bEeSW^Vqb*H+Bp2n5W1Kwj#AzNE*%eqS$TvJ85G;wlpUMF@1MtAXtOV2&G^fJ!2D;j)-^Enezb1060+_nh{!Gehg||C#TBWGUDE+M>ox) zE3#xIl;>OSXI+u@R3URmi6Lc3e>-x0VtMjOq#Ckt%bfc9FQN09*LceCw^>kyxCe-& zgI$gknp2&X9sH;tF+zn>d+@z}ESU1mmdx>Nw>97v^3!4zLB?1Wh1oW5Hv3CmE=MqL zZJz?ZlMni;@4(?yZs!ue))wq_BaX3Tpy5;sp#3&C;bKakjZWhl^2KojhiBi_UNF%J z775u8F*3e@elulStTq!rpySFFvN&n^n)m!L2+5M7J;;ao3dH|&Q7-HuJ6%7AWAQlp zVfL;7m7`hMS|MD*e{C(P05=1oY)=R5z<3vqOJc>haZcu^EeOAnV9fKb#vAIlcnb$_GwLfj=5qxG!_x5Jv zgI(dU;q4+zv2 zF}1)Q6#PC8h=l$D!Kw;J=7Yb7qxG|0b;85cWrqrJP25pg>k9sh-hQ~=%Smw9f=naj znAz6KbDqbS)TFdGE{+0SzM;PH@uWbuounWyp};z{tYiYC^l>PtY5qNKIVRBINEuLC z(5M2>$vD^G5{zXG+JfXFj*1P&m>U|E!6s80b!i4Y0xI z0}LY@NrptdYOMTp5Ysf0SYs?#cajAQi*kXsAYYI!gc!8kP-bc20DXxxwtgFIQwR@N zTDPr5^&Rvb)g8ENY2(NRE*=|U%_{6E;(5ldD8^Q!cShq^RFB}hv)L73aZ?RA961sm zsD}w%&h76boD1V!DA`i;v{W|5inHWhCw zpXE+>SmCYnx<|G+)oJQ;fbvurf7xMgiyZ7Vye^f?0N3H80*tN}apyHQI^x^mi2@9b`moq+L&}kD%3+~oS zUDQ-c9EXfJ6kQc_x-)j;vhej@3~bJyj`U>jU)72n&6wgs4UDE02Ie49t%Urxs4(^* z-gZJ?L4GJ%%G3Pzl>U4tlBWvR`sJIkvh<@Q+c#-LOB@z#70ogsO(r+_bXnySlMGoZ z;YVfU+F?p#X`H2=h?(zSz?&Iam1Q$ZGa)(D4zb$FOqpd-MNp<~Xe^t}>XPOt%<4el zpbd!NzYwQKkys}-Gr1O~CqndvnTv<;Q5l=d3g-ybeQrXLm$NQQFpE;0qxgJ^9EMv* zHV8%aSq6nJ5+wYQu}xJm^vh>78nnj#ab*@KQ#0!K>s5!?t|60P*=A$kfY$V6TL~6_ z)@)3(6ugve+CmDV(p>r*LNjv7yFX08^Y>x}YEvA2&dGficJFTw`()HB%u*<96hZlM zQeW{Wq(oEdPobp!6TqNm%Qs98M^~ z>YuWI_w)66x-jOvpo^|Jel*U^J;@O z<)ry>d(znPz43(f3SxgTM*qnr^%La#&Xf7G6?5ojb`^}AS@pCp0e^A`*p?a3N@2)p zMF-Nk+Ixd1-wK(aOwBcYL&>KqO9>zEtLvMyhR^3x8F8)1HnU({e&jt;kfsdyC*;9Y z3K58(Ul3hlnh`4W#TywG^>I7dl^Lw%@0ff>W8}CeH#hO~2pfLax$upC*(U^ZJGv`c z3Pd~m&dtf>%tPU{LchB{GV!*5-FC{H#cO&SahkVV|AqQ2fJ12jcX=LO#JC#*2I?vTrxZ{2SbgDYUjXRjQ&a-vB>VLdQo>pPY~sj! ziE-TF44;n0 z1%BPNNmRI(g}E>=yOC%wBVv_+GiN z4^z|{_oIcxD5=9?eemQdVCTsP{&>}qpuG7!}?BFA_r$kZUQp+5|nl~ zGE?}I@b4Z@sP}Kz4uVsUSWH0>ze5UQKk@GLXr77WVt-6>Qa(ceO7HHmBahteY{H7W z`MKf9yMN2ne;fyc(uv`%29c`M@m=8@B{A2kbPx!cHwLPx-V@5qeG%CKjuGH*Qq58q0g^Hsc|@ z5qRR@-!b9eRvC5X*QrS!6n3gGPN|D49yn|-#e`)zofm=)V|Zlwm<+he%SBn5wo?LY zd%!O~mLc?%ieu=%dFj`VjMww6z!UDGh57{*M8p}y`Z;QVU5H;bDVtriMSJ9?1*K$B zH)+G!b?lR~+THhRi4@oQhAOaX4W+hJ)c}9F--0vy-}M$F;LFN>D@~lDfL$~YQ35zUZeh5yi%L@OnC2*3y18afme&ndxsj2$}H5v|wrEx!o%(rm5>vZZPQ) zi&mzL6?9};YhV`HU|NJHYZPV1c_Gq~TDVSVigoGAqSXcPK%)to= zVJxu;3@N(uWntyk1o6|6(AxFk9dg#?@Yw#;pU2-+QwcPa7ISiRVUNpsUyGt&qAb|5 zWYs8GvFQp1=zb4VHykEOMrV;pkx<$9YOva&c9evdq@`nM&q2eXwNu@^Gw*A6V>{1~ z5fy|Vu%@DvNoh;NBP_3Q6%QuqI7pW@nXk1Hss$TQ+cdAj-%OpJ^51jdOt1RwA-t(} zyH-T~ZZUDOBzbn5vnC<_vk-d4J!Z}4B^RC9-pvjnne$7QmZo_O4_X#{(T+tJm1!13 zQjM_6vQ8E0yhf}PDrK{%PjnT}aP^O3%1=9?QfjDK>jI(`vN<&*J5lw=nJU;H)j z`cjSMZ$}5CZy029j3l!jS2+R6u6jhdf1>x#djmq2 zHtC$-zok(W;0J&zoP$}`3i^BsCCKIr8Vq5fV*IluoBSK}@M~9~%N89k=TLt|H&pK&KArWri(}^U9*pmbrrX4(Hz4=w8Z4WQQtPH4Xn9-bbu+;e$8xQ05@x0nNOp^@Hf?4*bwx9IVkW+Xy2nom!?TE(Bll$@#!u<4mm zE+@%04A&5v3>eI7lcQqGjuo5dKttPo@H{~(ViD0!0kB`jM zz!xUfWX+ZiLgAun7q=U}azTO7%?TN&%nOXf%1^m%rkB5IU(}^t)|Z)w znM-!Tr`19VHCJ`<951KIYBC>}9K=nR^3!v)N~d(1hqM6e540|lXX9{>R-<_^*`C+K z56PWZu0@SrGvN)a_3{uXQxpf4TifcostoD8unAWlXNmplIW!lWOmuErE=nZV{%rY? z?2CwogY1)Umy>OS-mI>PPZiqId|8oS>OGE@0cqEFkv`Tc4i%U2AUL@aXY>5TH#E4d z+2X-lDI#Dir*ok#(7>u;Km;L?b4jeum-ICkZ*7{sda*RErdnSTPAB;!J$`GCGhog940L45ejE0QlH{GuzCA_ssJIA{uVGuD@n0dv zezKbcL2Q9so;lUA|6J`u!P zf9giTpfTIFx;ja@YcJr}&Lu^kLe2ws@RhWG>z(lNjXu%!iHml)NI~_E%73&bZ6s&= z$DN&WlU_qB1y`pNwZFTZBWjz3ZV)f#;@C+Vo+P3QPBHS2; z@PcZBm-$(`;OfbKjSDPp_FoqM6)>S$^q+} zH+jX;6gw0*UtG3X?K+*0XA6rb9Y~mmWOgIo50ekm(R-c+n5V7^-Q{baNFH?Qi0Voi zB8X=w=OMekMq_UWxFh=+2bUdda`cFAD^4`>CmD4b>zq#fPNzGj4goz{ZR&PE4+NiyEvvl7)g|K@dn##>}M1G-(=VX^g&&Ip`ApzT};4 zgGcB>8y%^f^bzfTz;w67^5NB8DctBmMoTWmDWPL?OEo0UACQwCO@MW~S(lPh&Mm%8C4+M>zCHLMKG-lw!_+b_D z&HVP{&I~Unte{IC#p%~_ZdLy>wbs)qCD&@Q#fPWywb|{<=@?lyGkGFm(Y01LtwR7$ z7qEO|on|!Im2g0v>`XV$;rYbS=hrva>!2o&fK5C4YOlA8or{x2T>ZSw_T7Q`yYq;c z)0nIYWBj04M!1fN`5|2Pou!%LMq_M9e>A9H#a1@+dmYy9L<}(Pr@Ds^=iOOrTMkEe zJ|m&>6(x##`q%38z5Op6GV7euq>vM*NDFTye+V#*wi=&G4s`STM(Vo2-Y zt#SCbGxso!iIex8WYGbP-@3 znW6d{$<0Xm(+hXMDX+AJ!inIMDJOi(kHa^z*N&a-D)A6dX&YEY_BmJ2DI=jxVlCGd zz>;G@wIjQA2(WXt5={2GxxMMYu1<;WW=Z6D^Ok}*q!xw}I>1BK(3#%g(m>9l=`A)^ z{Wk7GKP`p&+a?f_t8jZp+InwjzOXPX4(Q))tqAGB2q0@EZ_lG)EzFHS*^5p|IhQGsnpN@8ZAUvRGdZxBl^F~n@^ z##Nym<(zb9%qsD8okY8R5Ya6~H;oaTwLzuy9u=css za7|IC9dzXfE^0qmgVPfGe5^LBL@TNek; zV@FcRD=|6NgPhZ?m~pAFGHbB9^Yc*u5o!LcL}~T| zW>he`(UcmksC0mc?*^0@qve+0LR!CZVzP8bzm|h8^s6zkXgeB2zEV0UjPsm?*C&cs zY9gl>p9243<61i^P7ppQ-Cd~m)gNuA@C*qe7@?^BM25mlQL`@hAV>}s`?sp~3ycsu z<85{DM9~_xxvU_(uc9oWcGGK7&GkK6Ke)9(`Qb&sz>{1~!Q7yGf*_iBC8!7Me7SOo z5619kR2-ZPRpsG7YJUXH{nc6(#O6d00^{NcrdYRDtMY9NiIJIvZ|``|pHVouxK0M; z4}t5q@I}qx>#raJ9Xcl08D&Nui-h*ri<2p+vaOR)v+08QF^o}8IG?bcc4jTZnW6TK zK!SWJc zsF5F*NPHzl3F!D$uw1Ls&zXFc>Gvgjs-M$VRLYgtc59n|h^vo{xCl$@@HaTjhFk>$ z8zb3DI`pJS>zo>1Jt?f_yl%T?bMJQkd~}Ql4*o{aL<$S}6+z}bK~JvW*v1u6xarZU z8-fa=opiMH_D`~WP>=vS4rk*^OJYTOfxrpYpBDnz@?Ewrs3`FXjc9^_OuJkCwf(%&hF*g!A0vBfWzmor`5kyJb9+%f`u9cEEW083h!J7); z4bGS)h-f5uK%anS%ve?6EJ`pCwFCj-lU6QDoyOoKV zvV`#eL@h{~Sy@;DG6eV`Vo;GYahiX@~c8ZC58Bo-|hH>X(izOs zZ(*)5Io@*6Y2h(tjZ=q|n3xy>iHHCVUAJs&zsFU#+;e0Er{_loO*{#Pig$}2KuSWg zC9B~5E|0+DaPw6-|Wq_(Z=H!cHnudU!%g<3U)? za4gfr`TVNn#AW#9GgzFHp+(l5VR1pbvXbTDDog*TI|jW} zJRh&N#bPk-?(Zkl7&dhrI@~Td_*CyNHwkMzZ}y}*mRnp7XG-K^yuE(Oq|wLoc>Os- zUa|Q3jmzbrySqCt;GJs)RXT*XpTSt!Twkp!(yq@?l%Sg=`&_p)lwwd@+Le1f8Y1lr(KQx3>J%`cGHn0+vT5n z&5k{Fa!vN@+-@hXhm-iph7*7fYY7HYvnXe+Ez84EX3U$+Dg=r_-zz4DdXeqna*>h#0cS^HZ-- zui6=i5gEQYLlk&bERn^Y3k?sC9zjn2WxtLCh6n|v+2(FH8rReH?;m)%R7YyFPzki- z_?O*kK2HY$GakDw7T9Em&%@4k6e`7NDjkpI@BIFXpRUI<6&f|}wk$@yq3vF`^4Xl% zSKEE@cRhL?28LUi>RR_ zl{;;qZFB_)v`1LNQ?a0!@|A~D55;27{!k#C@E35|8%~T;-S3M)Ok6x1&$vCAub9@i z*&j`r$Ycw_g+-%ebUpe#lt5_Xu6D6pFM&gJquc6woXzEe!)i&y2K2CW9zUP!&F+49xR+)8>wM z0Wc_AS^)t8d+V8cDbLT&@VM;ROeW%*npnMKy;Fj4QJko4ZoXL~Qi>@+yk@h8As>w7 z2!dL1I-8znP%D)x6%JNv)t9T51>d-kZ%sG598LnAWvJI-K3A$_x6&|uR{r|-uvv`4 z&;Y~*5RmM4E37L7NNOZ^sW%1zZ*Cv&Z$%%gPXE?ZGid&AcZ*r9R^PiEW(q_i)T^|u z5X!UJ?QoLzrM`W!8c`u2*dP_`G&<#YloAmw)1?hRXO zaUtaMMoZ!GxbVH~cu@2_%<k?~BMk1Rp$s6W{HSGe%m&c%?>pjD zeq7z1E^c}q=J-C{n2jcviAJHIQpn35AOIK5>afw?-v9>-8*`5=C(+y2_ig#xVc0)V zZl`*Eyxs+-C!MR(@c??=<8lM&YXd(&Km0TU88DywvoeD&{|1ey#(iLR1^cyTCc#n>cy`FCNKS3fC z$)w4r?g1%^xIaudfq-wXf=_Z!=a+3%7m$TgN>P3S&^!Gu7Smv1@QOSQ}HaO(A^D1 z)EH2qY+wR!%XJ}v@az?by_}m z#vuTNKfKXGnaWU^Ox@pZvqo&fDpn`g}!HU#_;c~qV6A7u-W|6+= zL~^MPa_An2Mj$ln%neAPa($1B(;7V0yxyMgfMmVi>F2Zh%WfizBTa4bU_4{CREd-V zK1>`)!J|$mXyNhR_h(wQhC&`59(&UO1XI1eAlEiG1sE9>{hw&m)h^ar>Wv4Y*|VxF zr@x0lzzfT%s-h)%@SZNc10erRs(h*4Xl=LI$p=6T_{12A6iNU|m}aFvfyiDmN+c2c zohK53z~w;s&j@Ar4gwEWkg4@i*}~yj0ND9|V+Vi`Kw4B^gzbomivvkt;~^L+$+cAa z6BHt1f?JA-u%vZR{SpXcFCc>4-58*Vr6in1AW|nkk&ff0gi?ZhlKFvxg3|jn3F))o zQpmBa4tEF=PxZxTVlp!%9WXTAbxg6}5p(nS#GyrG=_|vicI>#lV^F{mO6A~vkp+9nT&NStCdQh2KJ4n z4=o#pa``2PmNy`z4{UF5{}Y#@qE{FZNs1{8p#Rdox_=Mf8g%I7BrN~uR6#YIPeWQ# zrP3PLN7JO^Wk8O@x{v+`9_mSCr8g^$Ht_~=|Z7vPe%>_vWCk}KlG4a=i^SybGXG+Zp!HM-Km6_*udTT zI3TiyW>*1)V|a_l_yo|-tF!S87A5=n@)TBmKwJ?lRO{jH z1F6t#Di=QvNUiGt`E6|(@_zQ?^MBR6&lp62qgwTU+E4SOEDsGe@C#Qprno_Bi>s)( zJ(!TmVyDA-d3jl^&|m;k4J9Nv_*%OcOU5n+u#()$K)Lj*&B)p_kPC%bk zFaw&_dt%WbATAS7+?Mx-Wk4qLYz8FG-9;Y4ACrimjPLklWZFsSnFt67BEMy^*|?v~l|4fCAj=z>Y;n>^ zMVC4QEiDzop+{`8T@H?5vz`kqRP=mU_bBEkApt`qE(QdY1Z=ly9@HzKjsYbdn7P{I zKx%feP^FWk)&i6*tTamB!IMihNOIe~@5+<6rC3W93wqO;Osp3wqnKx0=009;rzi_4 z9M=H_%xp0U*Y*x*DNmRWG=lGMKtS5E z3s9Gyk@=WrPv&v2m`Q;j*F%IT9i%V=BJJL<4_1Xe|7!L7DmU0T(X1|~#|55l^fN=} zBgx7^1Oh$|k2`IWL{z_F5g;*7%XJ&A(P1R#iD6i~RixDSH4Oa1%;|J~#olQ}B1}C$ zKcBEUZ-Q{6EV4Ol=n^9FCbRMS{5c~2sOjqZ04iZ}4vdB+)nz~e_)SSx0B?whzPm!q zpyKys$@ylvXnl9LAjiw4u|~P7)mC@V)Ex%;e*nGT*3JP^2$Oj^ z29rVD=8MM_pjB}f{Q;@)339YUR4)aI2yC)iukBzg?Xq!E|JyUryP+Yj^Gu)-I{vTM z5}c5U1-umkVeMFKwRz1K^BM*z&wx@p=HRWXxa@pEz)8cy)=PPKHYr!$e4Q&(iTswR z>#$S{DnvAM->g~|;AlBpBImi(V5I`Ibna~W5;&&7=_#r@Fc1XV4^<>s1?yN~=S0@s z5gM1nUYF$IwY~lAel<5Ymqxu(a(R8Hq31)h!`p30Ivb*NB$tl|L0wVtH&A0LMah_s zkG%m+|5N2C3aIh427AR6=_HZ|Y7_DjYM>a1>XzbHC;M|UKQ_^c(Nrz1N`jD_$M|@^ z>hP^_Gy;(@y5|q#UOoiGH;p+|$Hdn-T#hTkmbNyNCZ{r#tjmoKohdJg1cLMXGdW5x zVyVGGyH4kQX9M^`z_gPPi(Xi-U^6pjL= zqBQn0-*d>A^lKK`Io&&uXD>Fp37(-V!pAUNp*3V-gRP*8Qqu2q0;wKPa#rw#~dZWvFx;@+_ z6g`NK0EFVe21=o{E`#Fz?Jcf=e|!H3o6Q0oB4XM@isP}6W9}9w)KsxF$hR%+@ZQNB z9$-8p1zQ1krR((u*4l9d3Jy+G9vKDYW~=8@|A!|=kY$_u`Nc*@41~bzl?Xtb(U=Sb zI5_1oGL{RK*zr8MfOti(R_BD#*VpG^xydVmbqqXy7M6-dQvgFiHSQ!j8t|pnsKz^^ zurtIO_h`0XsMMNfhrB5$1cU)8(!fW|bI_GC5}i$204 z&Z|chHu$!e+S5hbB>>H#%(sP5-*#(&!W6U39|YtBfz}nYyB;h5*#qiMECskr02M&G z_|4an$m4#d`Bq(nP|f-Hz~@U@sPy$6ZKIX#X zkYXVQ&|(s&$XW9VaqJv}{#Ci}4(wX1 z8?456ORv^y>7sFf@kF+P6pWPbuK`#Pqd>rPS8V+xVfzFD=lOEpRICe-2-$EvPk$rC zf>=Y;I?1d1c#jjCkj2-ct<9l*l=sEkxBgO;vetJ5{CA;-1mFqXzZn5sZO1ppOD4NH z8Yp(jZ#3KZrdGK-J3EK3i~RwwCDPWw`>H4ad9M5qxUZEmOTK#f;h z%h;bCmKRhC=Tx!_3(&*{rbx-z?rtH6GDuBRm`QYonhaTAI&v)g zz8`LZ(L$bo9148&sGzi2N2%-&Cz9d|e2xH7BN_LUbj+5r4pl04lZ>UtM|IpjTds#1 zd4IiCNZ#hqONk^aY-|StufzKeh_Q&A;q{PaN7dqViSN>U8zQ+ezFgPkXLDt!DCcTZ zg5~sFkC|*X+^$Dg51Rop_4qvQhDhSESj32^*lOY7;nL*QV3v7XO~z%?fZR?k{ zo9z8g7wFEHdA5dk#!_gStY%9rd1A|?C*IYO@6RqT39+!6fnJ)hKkhGe1{@RnkN2M; zzN#SRvuBqEU4VXu(yBKb!$3o8@Vqwe(BdtRjgIaOAp0OC83)|ir>Cd+YCWCKQgi_T z>VVu2`p3eS7FPR0u#34Xr}dAuT}k#ubssQVfgZl^g=H`u{z`1n={tPvez|d8K1qZu z^XzH6mCW>WeNQVIFiC5BGd4;%7x^dBm~7wu^RNY$Kr5JJNU94LAXP;yNHL6(Ce;sEj8MyFrfE2Cl_zaP8(+EM8!K=}EAAkBdJ03f`Z zM!4l>79EiHDW#8qKko$~UWs0{RPiQ&i{$BFJu6!*t!1SAp_dEb^uZD2{S~l)i+cEY zBB`&h00=qJ4Zd7$iyWcJxA{EKuK>>K=R4rY#k86E%i!J_K=9qv^Z)$wT&`BpU}1wM z19nk5%49N_Um0NR2n?*}QjLK?^2@_z!p&(-7rjDHhG%I2%t~0ZHO!cWGq|oU7#P@& z!KUr=pJRY3`sX2`!RW(oo@Dz!Zsv;sUXZE(cz@_BO`O6XctueOwn5`W1VUZ5bQFV5 zd$8uWyQ7PO@OSVy;knK z8^@uzAP`1LgCYYdn(~Yv`1LCi?^=^Xo=^xDN3~jUr5n5Jk)naYCacxVFf$cXBKThu z?e{T&O()Jh2`14L#;VTIeZbj`j8NRK(rGe9Zv^s5A^Z$@^4&GZUljp#=+ho!Z>{BV zIju6dN|X#xi~O1h0;;{iMvUKp@kh?&VVeNLa4_V*_f}Zb_5M61@R4lD=C4L#DXNta zZ>l7%<)o)t_K~iAzS6k9qqnofs9PcZ_HE-Y%M&2ru$BTevHa7_=V~kH{>&q7FoBSP zX7|{`BLEPWy5WGYPhCD2P<;f{L^m{IipB53GY3EN{Fz@3UNHe!0?0^b12RNk2D{^y zpkE^Pp23YiqAWRd^ZP!t=odx8U(ps!`k~~K`(!qK)r^oex-BSjDI5+P{UQnq3SL-f zXnXgzN}rYX0sJ;Bq5RE|2JG03hJjgr1BP+su|lkBfaF-6c#lb~H=hs|>P(+)axf^p z{Bt~ex>%D|feV2efh(c-3kAOOVZ-OA>=4K1|Ha&!fK%DG>%)}>ndgYi<7!ZnC_{!! zNv4&G$k?PZRc6W@8Cz0hR?AYR6iLdEsf1FY0ijYVQzFxM-t@fteg5D3-~0IXbbRk| z?BjX1hqdne{{4Q}bzbLrUf1mtkHyDL8B=O_@%Wlj8X9fOafXt z_rBStf!r=}9C0ByKVNpkCN{g`)RB8$Kfjc8`5nLAfwd1kVk#e|8CY%y&84(>cdF_& zns|tf2AM;u=K#6t0?Hq2KmtUsu^WA#HukOPV6V<}MDTle|LkkE^^$6biV^5r51E;n zz0Nz>f8*`56qQ|Mj_QpRQva>P60r_~2P2Y{zLU4PqS7z@4mj81AEvIlBuILVUL|ON zn7zUQ5Ph!a=}(raW4pB`5?@aubKJ1iARlX1UCQ{RL9-?vnD^)CsN31IzsAQ;wxpf& z87G?)Qo~T&qjPjyXmPys}PV}OLzS<-9TP9ek?us}k?fJIE6AU`q+J;FA zP1V(!s?jtiNB+ZcqR2ayPNp6xNBK58-n6{u`qY}X`(@&ZWW5XEtX3@W0W0ybep%KM5pi*GQPHZG4~&xQ>vfZwK7O%^xfBZ4%Pf0wP-rL^{MMHbgk`{+ ztx=3%UL^K(AX6OQ6!SLbT=wWA`cds{nh_=8{JCt?l`B{Fyfx;gJvR{^taQJ={+fE& z8gRIZjI`&+!qzWl$on24`dRY}I5#N`@&Pu369TqLO(8>aRZ+R#mq?WxZ9m(KD>)F~ zy-w6%Whk1EHI6OO9Ql}aI*k;5#if_Ou|~j%Qu7r5C;q?qPbb9`zbIPYQ9Vm~vffi| zh~On9CCNb<`j?Em|NJmyTI=Th`}Z$jzI<~l!j_lghnhRM!l0nwV5Dco*==;qdLIr@ z^=&oS_KB`IYnGkst#y+HpB8R+sDU%CRDc$j$XWO3`Ac6Bkjo#Jos-bix8=a^ALzT~ zj7JzVhsVEPSXAIM!aSBpJB2f8O{Aby#LhI*t?=3O?_pu|iZ5Qi)AQBpBs>8<28Mk7 z>7@0#VQW^)WyeeiRJotl3F8FB3-Nf4m1gcV@uceS64@$s@EzlpbL+Qvy5PT?rjgbM zCnfQZp8RyK(BV)H3CsM*#$nz9kgN*ub6~~O4`7C28E_F574?YFB(Pe*+nA1$+cqqD z#IHN|ZnaKkCC~q98VU0MO=7QX^kWcGbWXE+;N>~eH#|Ph!oY9<(zWG!DTBloNQ_)X zj!j-4pYk=$-ezy@#6#0d+PA6s#J&YEcxajnRU`suC*%=4UDSyjM}`p23h%a&J%>N+ zJjzs8ziubAImNpWef*uBH?BK) z-QWta>*`5a#szp-IU(~FG)CV$L`*lXkMAFNr*F}#HNUxU@r|RK%m`%;<;Bi7NqLPp z#h~X{gTX;TUuAE-{$7y%=#Cwa&M#v$QuP>0yKzG*Meg%M-{pVqz>h(!AX(Yd_2TL+ z-G%TqfJr!ZhV0m!a&Wb>*BE_gP*aLI1*Ly%BOo_^(d3l>^n?pu-rCnY;sl}H#!hqg zH8d(S4uqLfX9fWERfQ7rAWzRwi=2C^ew#utSGS!lrzxKqOf=Kqf9m1suX}}rgnZ+H z?oeL7XqdBv21Aqwft98cNPYkSXKq4GokbW+YXkz->xSa$JN5-jB9pBjPKphAa+^XT z%gvsV_{TcTfj~6+09IS-$C0_;U-RM809p|;yRzZnHoaP`odwNIW_|60Mx@rvCb%Qk zoaK!s{T%m52lzxfvA%uyx5s0A_3BCM<7U|zL~NXpK_S$MT7QB;!+ZGgDkOru4(5-a zKK%mt{5>^=YTgG6a=zx$8F`7oA5G23k zl`10#SVSz}F-qCSxk$<|BH|gy1*hvT} ze^5Ds$G^RKA~@35V0zneH!_|+F>>zSpF=|dceJAg*LVGDh!xWTNdt?>LHGOE6gp7?u_w=dZV@3kc7fHa zzm)C^b+j(L1=V6FY9GtejaOB?#&jc2+!5XA+E=d;%m7)PMMyQ7R;-ASp?jWG2|~J{ zr^w-nJD;lO2*Hr!;y_k>24huH@(UNqX_l?ang@pVASyiLTFF%#EFV2dcLmx+H8RAl z(YTURvU`6$+x?g1@JAfY2{NVxemu`>EOUI)b`Ho>vlp;+HD%t(A+&eTV^C!(tFiP!8Bm|fyw_dJ2O@A6%_ZK_PuL=!^<;#p;UXvt%+J*Yv*99k zd3%E{6H-gIBV>KE?8gg5{L1)4)wqL7<7@lh1&BYKW?_~XH47eQO8|889B(p#IGUvX7LbGcI-_(AvgE_0y#HQpc0 zF_D-~YPb9?F#05JqOR58Sj?0ug@*tPdPe;?xanbtb?T~7a$1K%dqx^vh+Bc{F*mH-XEsAqFCav%HUY}15bnCiT0SuP;`@_^kcu!9;fRE9OAg9bN4%{ zc{aB030Bd#AUcW8K|n%F+Vsd1S=<(B%V!SqQ5PJ=q4FOA@L{pVRqhqjiE0xt8H8;l z4QSC9F=E-h^pQFte|Iy94=T`-L0isVJNsV!$?7e~j<`@qK(fVHAy@(*THHFYWzU{w zaIQDOTQpZWg1f(PCcg-AyFR$fXLPp;Uo@F=Bp_|o6!y=Ph%*9nl?m%DqCjl~<3_n< zOqP0<@g$Lk*U1fI18CCP5fX?E?eV^uEP8G|3@UI}WW6^IF}-eZ&Yn$$XWc%czHChi z&J=!wI<>kaCaH#6NHe*eugb;C`}zBKD>!X%mXD7cDM(98!^+X_R($qRU|?XYikh05 z^Z_sKTv9Uvb(%We;Npv#8X7DDig;R8sY8N!OA3Y&f^52xt!m2-uy5eOK76c-wA)=O z;Bp~yv)(}>kmi7BcVqBdszTNzlL$Zwe*|b7)kBXK4GOhhkGB%#R{zI;0KpLLw1#gi zm~mC|7)K8MLeTn`k#IrvBr;D{J(sHQZ+8TU@&^wfBhxKef^aiY?rr3KPvIiXj0?Xb z?#}?i=wL;2JK#1PGka-fD;<2-q-bS~yPjB0&XFd+ zpcOV~a29STPq>h9eAnuQgf22^3i_zCJ^W_y*Lsw?#&gfM|ERBq{8)PbzF0!U>7F}g z#g-P&|KyH_)T;dQ5r#9^(MKO>7vFQ~)%GPb4_TbUm&b&YSe_RcOp(WdiVpCa* zmj|we)vLV-&&jXPt(sqszoZ54GmIwn8fQ0L{^v{L)jwxbkP60tNkDOu@u1ky4miAp1H&ej#ORUNGoE5;*hyEnjEDB^gfJjlRB-R4NtLvwrZC zpsOI%d`Fp5tVMm^_vDTQ^B>F=Cv$V!^@Jqf;kY7VhcKD8Gd#I-#RjDZ!rh9^1~zNg zy><*imSn+p(a_KU9z^1N44VLicz7=KfG@v73*X`4;Q{*~U}oG1QI*X5H~xUh24=o} z+gKwvo-(5`K$Yj+w~YdL`)myv$aPtjpuyqJ=X^o!E~jO~ZUg3qv~#3&f2oi@ubTBo z)L&NFRx5J1!--y2Y_POUCx?ScEW}Vd&Qu%U{Y13tl5_n zbTL3T6Ist-$-nc@kluk$RQ;5MnqXjfC)7aR-ri62q-g?9bL`rG{kT4Z6IMFt4I?XS z0g5(rO69?cCMmw)h!`GyuPhaQJe?H2QvFc4z*}QV-goxGZDtaiq@vDA?9u(;3UF^I zK)1{MS1?Q^f2}1r z))Rita(Vm?inL`ttlC-Ye$z(&vO090++%&))Wk&J(<^R}vK2EA zy@sGqrAo^q>k$q$82bni2oynh5f3;SJ0Mv7>?#@nKRJGV6UfLE#PA;dT1!tCzN>5@%pmu)9=p zs>`qY(PAz-Wx0lwxkuyBFgYZJ5&`^H>Ys`N(NL8JI71oF@9;$fu->xFW zZQGf_FlXZr5AZQ!ciU{KBUp%?d{tVLa2!Mnsf7pvyR2Q0N=w4K`eVy$tflP(^Nw`Q z6_;S@48r4~JV{s|u_R+R^#?deugjp_zV`PAtH6bz=H+mc_VntOwO;41f_=TA7r#7e zckCvYpg&7H$QAMyj0%j7UWZBu#>bY*@m%vBIQD{gr}Ik$^`Fd5e^N8)$UC@Q9v;2} z=V{6EM>!f{wh7q?Coab=H55)~yXt2Vpa`_Uxxvp7Fz`o1?Nq>cUhb-5960wTcTDcFfod%#CS_$SzyMSn%#@m??u zJ`CfMgy&qD0iMlAk{S9SBFhu3Jn;m8&$T-$?=}bt{Yi&NnIRVCgJa*`oW)Kv5<0Wp zwNLk(5_zQx>R43ER|>3qYcJkWR9)9c1=D!z;ZPMLc%sPhkO@MQ*m}wje)&sqT@zM8 zF8F$}7@<{wIPfxGkhYX#bDWVxt|L)QtKyOgh z2m1R@PmO;gpy|#1oZEBu6@CC9k)K+{il=)Aeq7kIN^v<5i7)QY)S(QWfQfnLTk)wb ztG&-3rQFz^v=?PgG2bRzt(#NswE+vbX2oHH9>g2o0dS1c)uA!zj&q0hKyr~iSk{Jz zAFVdayMFV4)o-w_m_}qXYQN`pn6SJQqrk)|SkbHz z2sQ6L*6Qi%gA~S98)^j8ER`8oG)8=Er0n83vE6&z36`55Z|`xsvC60l|AA)CwOlD6 zqT!Ir@X~V6<{u6|^n2hlxtfHh#Ls}GZ@juCHzj<ytF;O0NT>sewI8`nDX z3BIt64)1P*vM#%SA60r`C1F#KZt> zpZ)rZd6Nxsn}CGz%e)=thSCS;u!<%*m+ii9MmaT44d7c47Xh!1KK05XqKwXJxMY-? z^7Qws>fX+`u|jV~le#rx@VWaRyuy#a&Z`W<*^JoV%S*;#J&N)A;gV&H zJZa7Y&$F@3L@%c*x_n4e4@~JCczbX28SdNK=ENC{br;ju;_0H1*M>XV(d0e~n)vLH zrhwSoYK6GRh`V}B`LIBW;WTlHedc&TfO7$mpAvD#<}ZI z)+o6y_1tvk!>3PMPk%9Mnsrm6*Mfdnpob+EX-?d-scUiFN26iXeRtZ;_}o~Z@&0vp zRWeNE#{2L01}sGC(!CV=am$}ymj5X+sQPw)*6`n|g#X~c{`XpD4|aZ z(CIJMdci=9-k8i>Z*>zvA~5jqFVaqYp+nG@!uLezc6~a>{_&#zJnsBsBEFXiIy3uk zHeweZ!bkcyBe5ZhJK@Am#TsawJq|%&yQRtJZE9K?bX^;z3CdP0k%Q%4?57nWeNB3R z!yKD?VJ8k(BKT}zLjv#p|Lvb0&dM_aM_~}z<2ApTd zdD9w8-g~Vb;X_`!a5oefxEW+!FpA@k&8c&hfh#tVi6$AM`2_rxJCr{3n50J@I*lOS zxtT1e=q!A&%zflJHT1Vpz%TuTb)G=`z^s?oTkP$;Od{)N=y#}zJ`V^8AY%B(?#e(o z8Npdc{qrU0>l7Vs{$8r|G+gELfE(e=Fd_<;+WS&Pn&V9cUKn>3`_={;ytr?F;4x^-CjY`T~ ze7>Zrd1!pxs?7Z;xWH6OWLVgalnrl)BI!t)618Z@xg{f)tSa0GI7YtfQ%LFY5ODYu zoDz97Dc}uH%fnN^jI_u1^?e-O!cKAi_CQ~H!-3?S$7k~88faSLE6J*U)4UFYq`hQ) zvmBkfL!{?tXmR4p5r^@_kNgOjgX=QN8$|Q@w^CY5+wY#R=zAhVMXMsgs_nh~aJMPe zGxiKL-mK6f|E%|r*pt=#D;!JQv77U)9_W+fY6v|v@CUh;NiJGI33SSmt-mKHcVF8^ z-qlKVMRCzi-mznm?XIjj6zpNVp-g-q1_tImM?V5!MqjafIkxoPsS?2)eGXU){xdS_ zzpQ8fOTz4b;uAvwYaj*A3v^;)Clr$Y%6LYUDx1UMIoE&N=eKfBmh9E}WD`O0*flf*{)Pgh6RyYU(S6AW)B`?4p1$i25o+lT*^#UV$?GPLv?r&wPlG%}Rz$jNg&Uy}W53+EWPMtdlmGwp%}kBc8<{ zg?QvuSiA`h^_Uig&>hhF6dk<>B{d?&eRvoaI zOAjFN=LTkcJUp*ZAt^iD?m)M-l;;nf!Y{>r>777 zg18oAV`D1ZPjX*+HYJ`weSuW5`|^f08K!+dpnzAKBCEBfE4G-X0iZ`ry?C_e5RpAi#dft-mqY<@@vOh%m-Y zQk}8~is|XXo(QolB?X>8?|ihnpS2&>%|^1pZP|th&49oNDmjXU#S}c=m)H5$sBEFJ zDs@=FZ2LgX&j?9dZ4sN z8A}){Smwi`bzyCh)@|&J(B$kRf$u&9mAidn`Qq{o)swua#h8%n7e%il$ug$bQEhLq zxI2j658lQ8+flka1Yd87$*W@7X5on~&CU5pbyQC@S3AFIg7j=o7#-1V>|&d_j)6Oz zeCq&rKWhJmJeD=vf*heGZEq~P_te(7pKt~?5j`}a9`CRbG6JN{zWBp6sm#Q}@`|dz zLLmAScG&O`NSIAYKouTp*pEIN1 z4q9&v)idH%bUvh|r3FGEgjFc7LS?FagtG)6%0=4C}=`1x{Icle-emv~atmm9L_uw>HG+*Zlx=?-+_R^Ir z8)8K0M%>HgNodgzcrRM9lW%U_(lCDkbBi&y*S4skm5fib73rJZgM$Ej8d!{9HYkeF zt6eT`HjYN4esKc)8sHAPpGS^bf7wYvht`hdfUsq&ju>dR_1=7ul4Bxy&9V79^xUw` zeAge}yLjV0)HQV7-!Ql$+LUIito3L!nxVPa%ynlVc#3zo%|Ad{`SYP5XD+<@2$Vzh zNAG)|4f?yB>4ldXOJwMO>##W8<@;+*a$pq{cH{jSS}W~`rFZCWOj;U0lx|xisz+w7#&&1E^Eji&OLlHCEzQ~=i8mvEhA7I zrNI&JH{59~VGyA|!PXD2piugjw^586@}9*FYH6BFjLQsf-iL1R^Dj$luTXUuXs%dR z4)g9Gc&hzW{Y@{?zcz!XYb&_ZG+WSdN2Huh`YZIgX`5#IeYN_qSlqTfy>He;HlAJM zRm$#1kSMKE;@&3HE>)`e{w}r>i;7eB0!TIqUnIzivn)!hiG7)})7`aOn8%(?_kwYt zSa!!MQB9@XHI^S^gpZ$b8R&{M%z}dg3=AuWy{xRPrImEX<{HygU{&5;7#d}qJ-fFTl(|b;?UrX+ zDker0=_e}V`09b5MiS37wY;SLXD2^FBg=dE@ZsCr(ErTj$#;*mYGi0>MS?HW=JSs| z47q+!)lLJUnq=wfhPbOQ^R0zg(W{B)n+RLamv}}}!hI9aZ~+HK`=8v{BaqwU;d831 zVm-TR$m(GjiVEJ`?$WN276}TJkld5PG@vIZc$vgLo(Uz~a7N`S@NU|2oi7-9kpo{z zbBYSu=YRB6htjj8Uq14~2sf5hOHD;*j$z2)nKZc|f7g(-%|{|fSFCrb5DBcE3^CQ{ zqBEo`)~cEXpPsoNdCsK*ot8DpVL@nc4BX5-cRy)Hj4w(>D`S0X>7(v69}G3|xC4I& z@1yx;Z={(x#@pLRt(>^Py*8%1JBY9!v-S4Kef$1BYEbbi3637wnq&_KsW~JEb`4?5 zRA2la@v54mZy`!6M>IqQ87rsm)&HOEm*nSaq&m1ZC+?}s z(JAhY2oMrmo-G^?MylnW170S67X4|3! z(|HdE190HycOV z(>O*qEZc6ksMYY)+6B$~j^1@r?7PxzQjF5|?$6l_Zxw@?;d@gMjS)YU^5wj|r_Q5= zc^CUn#O3JQ`g!;ewm1c|?2(}fqE(oHT0O&C;Pvm6PHO~)>9Gd69-X`0*2KWo2eVXq zkA6a}c(PfwFy2GJ{J>@@sIni#Ir1Bk2ixFEN3t$z(Z{~HHr#(1s(jhwafyD9m_7F! zGPH+H(cinDzs@Alo?#WyH5V(2oa;L6Z1=&%@@P{sJWb*8x> z{Czig+zkKN>@X$vRiwtO-);RN-nCG2t5b6HFB=uw?wlDt5M}IebM3>P&rq%DDDdO> zBU+-1P3T1S9gkNAZX%Anhd(Hz+YR`oaVsZW2f|x`zApWkes*QE&TW5FI`uUgO-y`p zT%^(pgX$05IPuyaj*+u9$X_yb_XYSS5Q??zEzYlNKx)^ z$=+fq-)p_OiJWMY96pLB9fzFv91{BV$MTkZ(3{zOVMu&6S`%#G@+wNq@>#Y|HSK$E zFWqZ6_(+(6QScRcAo#|yHtM<1BhgCd+ZXi%T~=OXyPV|9EGln-6w-)AtW80W(RG!n zxCyCLpCw11th#_xKZ0y7ZTH&@0OZ7>Q_9Jk?3RB#fGdAH_=J&wT-QRyV>v#&Y+b=C zXcO(Wy!|Iuy;rf({O)OT?VXV~00m%Cru!N=>wXA+Ic_Un>UcGGV}l~6R4gwUwKV66 z80p8q*BKs7E<=;pL+76Jj&nxa1T|Ep!|XQcv@G8feiPy6R>$kQkhk_nzk-SXjwwb< zT9+(i_`I^zDT>#`nv3zan6k2EoC};O$`KtO?2}{diAiz8 zJulb1P9ihK8}ua7B9HfP4T>UqWuIS!dFY;sg>j?HWZJ}Q12^u4&7RqqU-6saggxJv z#2xM)dtcRa^wn~?i{~AOa@z(it-YzDB4*sFN$=j|ns3A3$$%D^r88VAT-JimTyIt@?WViakReH@Vhw@)lG{ zFb-uN!=f813Tbk)TV7ZO>(yRS46mX7y$M$5)$Yf)f14H!iVsTVdH;NAo^6$|^%8$d zJGU)ejdz@w*cw%@F)A!UKb!Vk+eX3oA@SEpR*&4{UwYY*vu7(s_Ma$4`%>paa!yBP z7cihkZ+klWDM>4RUsj8oDqbhuQQYgd*5dBEybvfH@R@C9K-2f5hU@voAx#TAasC)b zl7v)!q?f-?iwRuK`?RT|Ej=aR2}u(@0FSN=B2pJiZy?)_LcaK_^aGOy-lfc(xm6w` z-rK4G1HFV`1w2((XL8>b9ryvYEp-p$DW1}N_!SM08X+TuapVVyY76G7J0hA z1V~)k^ALh@+pR3t2b+viKZmaHJJ*Hcy%I>@7p8|^U2s1bX z!KY|xzyI8n_4<&n?&K;n1dw0D!;=82@CaUf%Gfam3aCYU@0c&-*Wrzw9|fRCaZ}MOk*og zhlysKVCmD;0dt`(Tn6{7`*K#8uHt#gqP^N2p_dt%m};sCGvPu@W*+j=#19i&#U}cB zW9UoCVVZajm!LIT_;YvLn#rVg7zxg-nB}UzXxh$(YFaG889o=zKZ)RP^>&@(?S>yx z95<>wvxuw_6ckjy=RU_LcHCuGsssodQZw>{Qp7c8ZQkC&FII%Rp2S*o>SMRzGP?-9 z8uq>XAO(a~v{rxrVkOO3%Ogho(kBo-OYOrV*P9AN9)?eUfjij7B&f`Lx5m2SNxZ=! z-N^KHqJm+Y|T#3E(vh2&Uhqhi@w>(dgS5r66xa@R{<_~<-h^&Vp$8B{o2WORA zTp}}Qem2nmPB6~|4JC^^VnB{$&?oUt{2={B;EL&h>ans%N$=@$_;Bc(^=uUgWBmX5 z5lic2O}D4SPseT9M2A0Ii3Y9r@864RD@olGT(*go_)*c0yam&JasuMjr-lVrkLJF8 ziSRA-u2#jc5kg_{{3kkQE>KE*6E`e4@Yf6CYe@?hoS3;l;sWW3i6z91+!i?%4B{hb znrL0vp+(;ghm(^+62GUR)0w`|)0V#C;pz1&yK(P^1QiBXX=I;T0a~5?43Ei_bZE^mI4W3SvPP z3)ucOh6Em_es$f3`7tDuJkg!l5`p^#1O389$@E^ImCSCuO;vj=;@#Pi>QAb;C=|Hky}nXW@e=nW9wKY#rKEpCT- z3GdO~1JPS*@=n|kb0gH$R3$e@NX%%TEmWXqWc-SF79VK^bnZfnB)ZZ-HJtv^=HfJjx=yQt?Pd~4oKN3Us2mFSF~H;|Gs2!5#D`Ds zZi#{7hyA?5@vNg|ew>eDbQ|t7#Uttm3KDh$(O1upqr+4Q9|n>qihpY!E}BpZe*e^* z!S_I#u*%R1$5knN3@zG%_IP4$m%K%heS@{%1E&>)ISs1pL2jrH~QC?9$|7AAk0xP_56J2Y}}b4z%~jKIU#QVruUrV3UA zMBax8EeOY7ljB+p7hK{vts@O{TpK*6{uJEop|$f|#47GgDO8TRFsye*qCsrS#2&(b zF&zz1050qu!gsvFGp)tg9^s!R2xp3mubU^vOd&E6z9FJDI)D}K%_w?KRg4Z3CE?pP z!cxlxr|F`B^nd{7ifY17HJ>B-TEgUbe&ZppYK`*2zgB6k!m=mN~@Axo_C zV56Dfk0ufG4Z)%yAFSMhnjUP1b`K~wLQqP(1M-MV+UO7zec%`h)x@7$UYix=4Re0g zIZRR8L;^J=@tSCO_`q=f2oJw#tu{aP4(KK1GQ&0 z&ncNC_6kvfcZAd5bDB0bHg@SMIU3_)ZX+^8j#g=yVXc{j?aslyojW=asT(Bw>r`C!Y^%R2z^fwF)%yJ;hK?5Sut^o;V~eEiJI69vP>am=oMcbPGGP5zz>R z=BM(YeZx2ex~FjD>0CsF(Db}5StfPMc^_=ZfGQNxop(<#N_@#jdvi4o`4eXCBWy{E zv?6pXk8owJVPgyMI!Hku1RMvNHV6a2N*8xgkfCJ&iA;W=*vD79~owEv$?vA#m(HM$z=OqI0vWz(x~(k^c1*gf0y^0Na4mu_aZz zhTebx&wL>I7Nl^&WPwff$fsr@KE52htzlo??E(S$Y$fsTA^y%b8kuoR%XZN>`^YP& zegc)lo9ChE_>{M82TfH~)q{6*jk7!J_Eit+;8*V>1VD?i8}RK#Rqd%#j5i}&+L+F|D>dl1B7WdEfwmW)p>vQBHs<&y>rEq;$J3&JAXcI7;2YH(#K5-b%@|_# zA7y~vtllYza^7BEd5Dd}N%jbec|VL9pvcT%?oIr=Ew3LfL~_M}3D60cku)=05`H(p z;yew})D{GtNtzqW=Prq}3duTzQV_j4>J+v*CYo53dklkFP)~$u42udw6;Yeie zLx&E54R^3%UUQHF^S&0bu0bmH%_$F{E>OpQ{-oSC?j)opgr(DG&*o#;9%03tdR}9N6c30pT)dM#=#QA#K8fSJH{a^w?v_W6x6urs@>oZ+TiJw2g30ba4tiEK zPB52^tf|GQiz+*au_4MfI}J9;SQR_c>TchD71 zV$1`=4h9fN`O9Luw9V=SRWHwyedP=YnoaVK4*F;v9%D22O zHo#l~;@I3d9!d;{IWnG~7eTT+AwwxT(*i>p<{E^AorR<8ca_$&fKgq7U~r+4poEZe zwHU&_J3dQZNd}5m`zVPv<2GK0tRZPx%hDEO3Lb(sU&Uid-DG3D=dp_z4TEx}gL~u% z3jOsCbH7Ee41w_tHjW3m3p=5d<8XucpPj(Vh5?H9uoU4I21+jki-ntKCpzP})}^?B z2gDzwjUZDV*@&hB;?vgr%J1;kSfZi~S0ngiR$l2w;O(^7xG_}8yrtV~RoO0wdw>t90Ioq%sfO{41z5D;T} zS;ka5A?K3NvTDOL({NUa$59<}2h5bPaf#628G49NlOtPOsQNDnu`RooAK70mX#%25 z&m6WxOs@BsKUMx2hDBtLNW1yOp6&G?=)ALbm`wHQi{K2SL+zCshW{df12+7F2vh!rMm(n}iiMqKZ@@y&uZ?aF1ZD2Od!@FSW$*mES4x}E=nYQvh5t<-zLcs~ttH|HDmqgPx z&!$GDa5)AP7W4H+Q35(Wh>qTWSzO5dSBi_u)R&TyaE@{PX8l?kYcH&p{{q7N>@+Hd z`FcvMMXj}|>zIM$bgMQaQKrq(mu~LfbIt-C0Dp4-)6H9V6{J2ys^{3T)zS)|_r^uK{te)fkIkSQFCm~{LbLu-_1BM4l- z57>L`jV2MjU4*Y@%T(xlb~j?i-BsUfR!L1n06_N-a?+|T1@1d`?7-d3PIb*;5Sr{> zG6v*sM$bGjD4U!W`%j3gxYlQswXf%n|0u7{q;~!i5^p;1u8S{bMhI|JICe z-Xvs}Jz!k-3;F_Fl!{&HEJubg!I-Lyc8h1eVf070I0@6H$6oBg?Gw$4yeqw0X&mGG zki+kG+_KnOZ$y^hfx7C1fiZVab)_IiL*%mULCWrV3A|f*$eGJ5@?~oX5F^n*`t1!N zP_vLH+f4VYw<<41rkA!`-}V~jRbk;paiP}>c&4w5r|WlUAfXlhLiS9LB*%5i8bryF z3Ppb}^=?yGenQ$j?=9A#MagIeWywW-kJC_$F03zm26wT?zI0L(nv}fShKcG803-V? zk|*dy{X)L$Fxi^miFz4J@JzuHmF)`NIh@1!ia%XX!ibj9$o_s5acEgfuVLUO?RZkO z=7{O}hE-M`>1ZGarhhyV!+os#*wR!Maz~~P>m|6$AjFKr_`OEWPk@4M9WeCNFh%(a zH8or&owPf8TZQhS*yqbTcPzdlLyFbV4_b~rC`QvdlcTBtbJYYFjYF8{@k5l2@%|2F zKa&+O_c5kVlu`R)O|o##Qh|Z;=_D|}=%+!$95=M;5ox7VLF(hw4o7brnX2zXFa)=IZRE~bijHhYulYj7j0&p?B z1bPx9{phRxje5WS4whbqQm?s_61E6M?K%p^_vogR&)A0X9~mu7GIbaq-;F*Vvkxnhh8Bkd0U6< zdA+Q5!2O7*D5!X)cQ=KR*w5&DqCE)M+#LQS4f0h%K5J>?rix{K-DtDyzyFpmeCVQS4@=05T%%;> zU^4Z17sxBfF6ckkk>KyFJ#<#LjFW?-zzdNHh4yu%(qNjrid+SmW`??@@a`-&NuE*q z5{8XCQj|R}@T_Ohtm3DtU-}8`n^^>A8YX*_&F+=yBus@oXT)5|Pl5;(P6$H6Lz?GV zb<|VnSsc!bwckV}1osiPlD!^CmVAF06vs|gWv#&20=o||>Z$_g5hSDqSD?Va;^Nw8|%j0ojdsK;q!7wwQezvuwagZH1uyd`~9s+4y_0?dt*opK>X(y;ov-Y z@W7$!LbK}BIuc30l}dCIBWIa7MKxWP6;ej;*>m7cWt?OP)i^Ni7>4!ceJS=a>|A4| zZay{-TUv~=UIL2IuUr|+JAxz_Yu@t+90^Y_%3+mXZ{6^v9O`KXfoEo?Ju$xF@%2WG zl+^DBT?9hP9TS9&`|(%dm(#R5=ra(hOHN{*pa1MM%I6BVPf>5)7cn%8%`eQ>=u>#Q z59em5;2we)c(71KTg9oppizO1=T*M7bzu(2%A$4oG*?)fFx64(Sya$A#PAEdDK<7X zUlK4b-UY3!#{DRa3JoSWyAKCl+PB=_&enDb9UbP=US`^(+ zJ1P5?s~XUl+YiEx8asP<`pt}*hu{@rYn6KXK=D8S2ychr7&mp=>jZUx0rwOv)vq1> zfP@DcR?oIy;rTZ>e|LQ}3?*Tkulp4aX2J-Bz|PVA4hRxGHe6u8+F&RqC@Z*$!Bj+_ z0OPRx+M~BBL&|c*#&?;UNZ4?2bx!f6o=ucMMGus=jrV$PQax}u&fnw_0>#gq>>*Qf@zm1a*8sn+=d?w|JBa>A`MTv=NBU271 z13$(7e>+&tKJVzJI*6Sc+WS?oz4>DZ!VId?5z}yS%^pvC;zV-de?me(2KFWv;zo4Ll;8;Xt>;8Jcadajw(lX5cn zsCrjMsLDHrxI%am7xRhQ?YbN|8(G*h5E8>gOa_24Ibi+p<+Gm8S2BgGYVG;(wnt|{ zR#2>Cw-RL-5gk}$E-)q{-XQ&MTYZan88@hR!Fa7xMii~kp+s>nXb9Va>cm^2A+(Ap zONPG(ZfEJ)q&Ef)!JR%@FJmiJKSR^Qfa3w!`!!W|ai1YWd}jM!_z1(6*|GHT#J)YE z94hdLUSnIeYSrr1B-=}NaSsJ|jnU!ys-WpPOJl0<@@2~!YzDr4YYL42ycE|Liun{1 zlaroD59#W;qa%5YZpp5`?gM16vYkQG{vJCx4yH2`lEe%?nD|$fcRU)iti(+T7uf60 z-~3p}TZUimYw0IpTYec3##ZpSrUo7{k61Gf8@dN|X|IpfFYX@L%eFP$FK4@|s^GIR zcelto8UD{IVWQS0Jx~twQJ&Qkm+kqdd{`8}!+S(o#kIOc8RK0H1)Ob;!c8+EYZsn?;fi*#QC~XU@*fLd)1SIi90`?jOIP zC7v@o1%26O5=Sz<9C#L(XX^KFysGCL;YUMkV6X!8^T84T!!2hOyg6+_L7E3mV+qVg zywBsMLlV6y3W-!P9hUN-Wmz1Kwc17<0Ljo&jwi|cOgK9Y!O=HAIMuP zOH3X~+A2FgD&s}`O%lP7sPT;=24y6}F9GKM8tSatsK=$ckhNh`mP0{@_K7_M2TUa>Sj^RPhnE?E--bc}{-e&V z_>aZA(ed7~ivaN(vO-5tbrjU069R^~*-#-r5BP}+bmZUk2%;T?^QIgutTLEdvb&Y~ zyh5ej6|QTAQkb%U#T^^sg69Wn4CqZBrYssq%8nhWF@_UZXA&5(2{R6+Whl6xof4vZ zE_GonEt$gB3t-qIQN9h-lbIC)ET*(bgQr|V$Sy&ic~%cP(P!#pbiU*dvhS2sGVaIp zlM95^kMQlG8rsO;SMJ=SF}Q2*+0r^>&-4>)!dFAh(ICvdGgE(RTmbLniBONDs?%Sf zQvO`wUP^0gEB^c^n(~nEX_8qZZ=&ee6)w^`CueKO{+eLQmziBTLIy(hpIGrmoj0Tr zD_Va>Jkwm=l5{pUAUZlaH8pjdEJeE{Z=HJ|_d!UTT^QX^a{TpoOk`#GRf;qNzpBVX z?Wd9E#JJk|%MqT^Zo)2dI4#*7gaaotce4n;x6cDit~7LO&VESzLf%WxZ(zS1bPb3| zo$nrvv?(Y}x_s*UeP_ukHREij6&lqf-8Zq5Ao5o?Y=0WU{qiCbaXYw&4nXvrPSRm-^H{kL4aIwpuh&>vwn! zPlD#6R~Ud)6NAo~b0#sN7f9GarD%;u36&msWH*I(Fh~PE9z(~_+7|}wgV)aHdQCIV~aEyRmp0p0G-{ zNEQ)JY+uYDZH#v(tl;&Y>Ff=A$l;aQMr#vrp})ab;4L))J6ITZ><34vuHqw6D5|0E zv~+`pHr#N5TJgndnDYh6#v=zvUUWz4)*R{x4k8*N*fLZHPeM&8S~jIpD zD7n@4>_R+9$j>~!l8MRSn5WbqG^E!sQ56F^U7P4=ZC3e}Y|BH24Tj#Ia0<$_Ym(!* z2t0Ax2V-pI(NE0P*HZz5c*Zm~79hQTMk6Gw8_-3-USL^6Nl?6m<)_yhr3E}qN_T*F zWMBL1*CjyNUsQ&XUTwFxYe2QdXvbW|s|mYv8^MFCO%2}Orz$7sipsDkmT(9l1j5+W zzEhNJb^i>rH|Al-O;2G+3-rw)C<%~=p_LjzlttuE_bu=H@k0$BPRw+`dCxH>Iv6rC zGYjnjNudqj!UTQJ0WHGhA9S+CNLYK6It&Kw5lm8 z$AKhN@t1nEse~I-*%EtW=q7NMyqdSHHKJD=TbK$5;#6i{bLWw){y5b+4MS&3t1a6Cs zArt!7)0|%Ct>OA)M4^9b{}}4wjw78XW8zM zHW!jmR{O_0Lq)rPNSac8F++ojq2Rd>NW^4%n}-+H3nnPo$rdp_{`iP!%46f;$S~ZH zrA?tv%67y&jxa}5UOiShS}ki|RECpUcYfK`UV*`3cu|VZ#nr)KT&KEJGe0e$;LJ7F zSr<_k+=AYfCGx`d(K%vrDpmjIZT+d{^)^zxO!}3zsBA>sMr&(ogxPi%C(aZ*(MXSj zJoFsZH2Er%4AzP##1Hs#>4q1UwOzkDw7gIQgW+8ebqQe)p4f3 zO^D{)#Eg0bL#{VBE&-PaZF8q)827_FT(IcrM@tH7?xCMQA6skLqDUGY9zJ*8p5rbE z2(e6^7BXf}6B}DCH|VOUm{@uZM(e`v?h2~*{CN;qOV5Fi5ma!aHe%L=@Cp=93B-GG z0-d>*C34z~0xfat5U#(teVCSc?+l4^2u!{_&f^7dKcL z2+!erMh5!hu!ThVxwSXf_@G+SneRLr_mbNxYv?cg#w_N7Hl==3Y@BA~GcwWCiNjVU zUSAqp|3ta$5M)Lp(RB_wAtS-bM)fQ_V1V*jTHxw_U zi@Hd3l71iQJt!<$SGCbcG<0;K^!yJZ`oH8{{xi7S*4FkzZ!gNccsCF&=rjDQR7@wm z+hJ*G>Ev|thK0HuMsJJ_tvf^axrbd$+oOzIoQ}v)eIZQiHnWUv+#~t|joOQsC-CVT z7%*w-&;NfxDvH;RK8X~r4Iw^2fc9LJm=~LJGD6Sje%lLXk?#R0A{Fn%N@;6p371Ef z^bGLuakz_>8G6NemYg0EeI5s4~~UWCw=&+}qb3Yw$( zO0>WZT?yyZ&=r-$T((c2Zp`#M5bx89qbJ^{H7WKpi0y;GiG`LyfsBPJkiJ!R-mIn( z=GM2j6c0_;wmj_d(CDaU96x5|4TSJUr=D#vy5RZi~hx*=S}~CDEplLc-0iI4Ky$ZkCcfz^s0fOz*(63eIWA;ic@tEg_oq6gWKlUu+mXuOeO%ZZQB*+-yt@h*BS~srP zO4N+lw2SexKiq^tOY_LP8;Oa$N7MpS&bZLAD-3P0oEj)}dK=7->L2Vyb!RNJ7dU0VShQA-u~(wpAJQ<{oBx*X-yU_awNP-aeW1xS-(y2(^VUz{SG;9jk%M+FhtFu!RujB7l9(e? zwSkk&H%~Re27<1<@VCXZa#&}8XmHLuiqOF2i+^v74`%>u5VqI{%gOAwm$|D53)T)# zyi*XpDwssIvkA&HRISi&D97#tN?-)J8C740cZ^FxVEMPhvd5JhD6{#}_EZuw%rHbM znau;(c}>^;`yLChFi zA;#pzz|SH!emgs*PyZ8f6Z&{XRZ!4UMdt4sZ8vBm_*i;LV^3%yzyUt!+M-&QI;V4z z{eWww#R6(Z=7oXwP5=;N)wAD|%XskY*!S3o+1B{oZHu=Y@W~))lCMbY;^BgsAp9Uv zGo<>CU!jg5V~ux(td_IaEuuMf7Uz`B;JptRiYN}xUcm+{wBb`V$lI!ddn49jdKG=| znu=|JxtFcHWQ-jS9qI;dDc%C#|D~P;f0uh&X98B=O!*jMQ-ks@_2bKf=h?|ebENJ) zzY=&~oq|q+b*MHCV<0hwk%+EQ4d9>+%tQm=Ji}VF%s6*11@@ZDmuoVRhUV}xo(hC! z+Vhng1gs<9dH;B@ijPmG&{xz1uNdJ+!AkLeeRF@Hzv;YROX=nf(u)xGH_47;`T9f9 z2JK|&m)u#YxTFe8*z*!vnjdDfhwX#N7P1UgLw}?P+8^RfvVGbnVZ_NO$Lm8cWoBk# z9ya?1fi9G2*A!hcASGD)3mqPR>JCv%46vs(dvUmwkRC`V}(C z;5;8cMa(IT}Zlv^>**j>m?xj<-Z z^ULGFF6d66z`X2Bz)6Ay5HwyXwf62hCW|EW3N;cIibg=8vwEL+jkjw%xq0a}Po0h? zEf(F;ary150SuOJ1R~d_5^umsUX4mwOC>V&HwZ%UK_EbFzACdpGXmjGV`SzX1l~*Q z_RE1!ZYLT7bM(C}w^R^0o7H#+Zx@Ds=KRqN!%q_audJnH(eQe3-%c|5*g+ju6y(dm z7Z_q1?9ZP0|FHKKP+9Hox+pCmA`Jo}T_P=lG}7H5(jhI4fRso|mmo@rfTVPabR!|5 zgea*gD(BxP@bz$m_ZSP-~=ba2u{#57pxcG3CM&GNEpI3z#%`d52$0kJm^bM zssa2no2{&4$7B9t^a-7I#1oUEJ^)%61bWhiZ;|L#+v&XYtC)aBS}BS3F|v#SeAbEj zm9i4#lVR?F451asNlB}rTu025tZ_dm=u9mXZu_m4)aI#W#-Qzmplm{S&uSsfSaSi! zGHs@<;^BumSUfBT4@wn-1K^!FPF66riF1^Juvmks23i)YqpzPq?WXPxyU7R*kn+#N zl3pK6v(r4kJCEcAJ5vb?HkX+aM_?-PMc9XolMHWI&U#-dk2rhETF~c-SplHVI8!Q6bcL z;*BXc?R5>UWh&|U_w8x0Ij-vz1`qVDi_|BWCqp&giw1jQq))*lgh0Vu7zlqIINAq4 zN5cxo%sT|$T)0|D)ZLsSC)v2NbIVHIkhclIdcTeOA7cSPSP3ge1qXdE$@f>E2olVL zNnaVIj^Zp7{%|wzSE!)v0==KXeXpWGvq==9#%9}dn8X&Q$RSd$ga-h8|voD|>2;J96}w3!bb;A?UnTc(j+e;E!=hbUEvtofhJ z%a|!qTuH2xwYXB<&}FnZ0o2|Q@JB4c+;WM4ysu6bK2ZFAn6QvejV)3fHt- zX(2mA(+#V4@nPduG-WXSXVkO;izUJs`d6?tj|vyAMAzF_ir21X!VYb?SI8$HLU~!X z2s@O@>4QGNGYPtOgqDo(E~z7#TX*48hq*?4s?h1_YYMIN#|sc^;t}yC`TKLvrS&qQ zbEJ=I1X3SJ@-pG!fe8_6)dYse@;PQC>7x$_IYJVszD6nBhV8fY1jN`FRW& z)Nz}O3p-T$Ml*iwX11M$JOtQ^^X*#+2FNqeblxD@Qg%(*fw4iPdlb1a9Wk8>g3sVT}UMJez}b)Wi^*-qQa} zQdoj&n;o!`D_20x8v$TtPXY|E+4Rdy0d0lwnJk~YGdq6any)%U7MK?!WOF(?)O|oE zkw@b6?kz+(6kpq!()hx2=&yPJib9e3vK1kIUFGZs!sNO16M8vVtE!pD8c zb*kyV3sKOj0cKVBHy?bV(8~m(;=*lU2z%Z&rb#^c_UA0j{w~C+s=^L-*XrMpKSuuu zM2G*K+*Rw@p&}50jhJlEW*OfXuJ=2xVQd5Nr@NqJLU|ml06ciY3WJx`pK*O^I{Uq< zDP%KD+j-w038L7Fbn*&1K5+ejuwS8!h9Vwb6rJ4%!l5}XddsD6g-LQGpnD2el~|0j zZ-uN0Ym$t>ZFvkq<{+gaVX_YzQJEjd(-PXT9xm%!6$4))Q?FfJT3?TD@c^|?854;Q z_^x#x$fu_`&YJ}9L9o-)iO`AT!oa|gNc|ZQi^asCItNqy1v^*|x3ap5uuY*psI%*r z=hqa}k7$xk8Z61Z=2k?+0{(Swxlvn}<6oIy1k9oMZmuF}s$zyYB$6s$Gi5Tje8nrg z`}Nu{k6CFu`fSmc;Mu>ek$d|GW6-sio7m<+v&72nX^P5L4ECl;HjW4 z+uW;^-3JjAHM$kM@AIo0k^O3OQ2s1Hgsyoo^BZ8+0&sHZ6O>`;S4nR92mz8*07+CKvdJ<)Gbae*Nz~|ULCgWiTU$u;j&J09hUbg)$`Y~MT$I( zq3AARoW$m107y3I= z6N2fET*WgPvSGLbP|>kiYmVJn!-9hWwE6+A^+DPiY`aPR;5jNE598u-1wWooP+N5T z4Bf?-!&kkvzXc`;S;aS@?kKJpkj?I$toi{%)p~-ZdG|Z|nqXt}(l4SWRGaW4J|!#y z_zZ@xQC_M2UkVL@osqBsHLzg~(9MNmoDIx?`9-O-NOY74I#o?~-Zr`@%m$FN9cpFw z!Q%~vP3$l>Kj!Hh66|UaZB`8?)3poIqOmRfJ z;z4hcs(w_<&LbfhMTko;Sgcb-W(OPLBMsjHa=ff?a-8|+{%ncM7;jF%_otu3PzQz>p4II?6eX`aLfd_gnV_ z4#d%l71l{Qs16D*Ddf?zRTeX4#tJ=usiMQA$vugUFM{zbq6W*X?`LR#RpK={)ysJr zxe*^a;GX#0xTt{02K@8EKz9C_>I+8zqUrHp5Wdsg^ zrw07(X4oS+Ax1(;P0evwov4|+v#7`7teE#ll#(PR6F>Z@#kxkE(djZdGE*Vy&c`Tp>)YhbS`03I->> zOGV{hJ+Dk9lYfVOD{?0#z&8vY5QCTmjEuxgw0Jcs7z&YsUI%u!RhO$;c`^w6|B#9Tb*RkZ6 zKzRD{BL!4RbfTMj{PS&H%Ve_2jCXj<#i{yzKFYs&%43GFBf|gHF#yXDi-$ll3z?Y6Z=7YodE;gn z#Wslmz3kc#{wkp*;(}f+tTK9CTUg|+EQ{E~!aM^Qt!m{1C8+3tJAUrHW4u%M33O!A z$-wICWZxkvdGxi^b0(=+%x;Xel*P?39|SHJl9$Vq*5qlT+B)UW{GjD+bhw-~*N4_Q z$K>uxImJcJc7)6S)_wxn8aPsRmDr*~Ya*PYJae?TFG)FvIhIoq4RmVKP)x&%By-$# zg+?|XNctTRIxZ{R4(L%zAH*(1wxK#0LTAY@?zDt;F8Tr{A9n0ES+XO^X>6< z87vR5-*G>HJ~uN|!2TzAe5Sk(TMK{sPd!t~uEiUlrkf0-3%HUY7ow595x{_fg80cH zjOW)*=(xNMmC6e=J>#MB;a3Y>$iQfXs#4z-@S~kg7~{OpI59ni#Wk>Z)C!}^OW13Z z*Gf10r)M=t-W6@2w#&uBB+K+xeZ{lPCmTbg zz&mj1#~XksG`1GN!4qTN7DqP&f{wjIiQX}M@ z78<_uyz#X<#j&Or#sl^`Y~pSJ_?Q)Ig7MV;?!Ar`rNU_BDm#BS2mX#S{*T6ELu zMdvbH*3X3_y&^5VncOvj7q1mZ@=A^JKg>G&r(6eH_4bIdEIA<^CU0rwAnHayp`Nba zB2GvL5~oj!ynTHe-`q2E zC0L}`4zibi|K>t%!48aYwkrHgubU7}z|uzZ7Ya019naqJ)1D~p;aKA8u&pTZJ93ye zgXKD`X{8tKmQi6=c7ChKvWpVQK7Ng|2Ocf}xzmKz|EMQFKw(om$KK=Z^82()UhIs= zn=rgh#PJ%pEoMZObHDKibV_DGc6>V1VZ)}Kt4wv>#OxGCB3ac??1nEdrpzu5zN7Sh z%D%+e=!dr9Ok-M9Fp_0mUM_LD|DzSh`Jsj5@e}kd?d-hdMdrsw{m^IubE#DoK9`;J zqN)O9$0{qb!!C_q;r?NokXzo#AbvLDWYLxjBnYBz=wC&4Kk#LDKpeL@|D=ivG!fsy zD`6+|8E@{zli^LR@h_#l>9;nZP5aK4(R9S;+P-;kVYij!T_ zKt>P>!p7QGcAu&Z9tugH9?=j#(W3G;2lTwH--{~aK|G!0t?bntmP=_5i}csO!0LcL zLPX3GkMec7C9dn#nU~O*O!#ng*WYT-Xo5m&< z=cG3HGPlp&-zw&u^Op0dY>fQD=#ez(Ure_{({w;U|NQf1tyz2v@2LrhArL;?Pl4hs z#uTLtlE1%HRX$-%x%fX0zOQ`1s3$9;A#=`=8(~QOuJAU>D_gNodw!_xgDAnKzj${8(S+S8f4!}Gp7~#z=j=TGTe8$9f3fYpcZDy&ZbcyA zjeMbS@hv###+1cKwK%iPULaxmoqu8I5m}MsEAE;}13V?o^76I^Zdi6{ zsgt^T9Q89uFd~8~5rkpMg^a(=+JbR_h+s@fp_xX+a61hC2=y_Ot8)pv=nQ=V9fgaf z_di^{G}80GZ;+zW_i4Dn_?mlub_e*2=!oNho5lGnIrWq{pHI$^%QjNcA|L(*CTorI zxUgFk&?tH;zjHsmfA*S6+EFO|II;`j7;zB4e;Z)GM ztY)E+tc(7mj$9l^`GRvEcA{S@d(;^}0QPJhiBaJrmxC4V<>9^-kCp*nPoI&H@Eb6{ zV=!BOQx4wOf$h8k0-r`pxJeOeA8xxT{4w5V(}ern!-Bf-eOj$8ywwzkwj3~!quq>& zh}d?&u8i%IoN1dwy&x8C)gh3`4}8aNZ61a!cCARs%&u(q8*SL|K!o6!b?1>i5kp=U zBje+hU`gRmK|^-EmFPVsBL@-h9XiGrB;3O!hipzQI$~s=-1t@eiH&OIk=ptS2*_c z+VrpXCmJag0?V`t+g9u~q?kSEb6EE6AdLZ8Y8N^LXhLo5sX2WCg@t-D_thl@9`FFB zey`v-dQ|i_sF0sio2i`TiS`lD#lXJN=5N|dP1jr-jPK`QE;2eCn&Zsb{PsHj# zQ1BOwHfZ>52*^O2t_{?ONYd+{fB^*smhGX}@quG8SmFjb!e-@jUP=cqV57iw1kVy* zK*+$x;i1gyK7;`h0%SGPWEi9(+uQ(Xtfn7;wgp_D;H2p9$(%;Np+3$BOvM}ktuki- zLC;AibKuPKuEatVP0jNLav4qw0ptVsT@PAfd4zEfy^MB7IAQAxz=8(==`6=9_Bh-y z%C`lAh!>7-Qobuhdar%pWKNJ23xj{cIt?NS03a3+vCi)9_GsWq%PoPB*(FP=3=#W6 zPm~SY>xpziq41GHy>8wM1D@9>h!Lg^jtp4(#=>jDrB+FO zGJR+;3X>-$pmKpmxp-#gsEvhv1n=6>unD(85z+*#v`+xxGHEi=>)(A3W?N;qaMd~K zEejA7K*#nbIF7_BX$t}2sX)75p#U)!l+Q-Ep@2QvDAr^Z5>4{#tU6Pv8zS2R9yNly z0SxItz2L#iFHk&IWr{^yJlOq_-;pm>IT zxrWJ6`k)eUKp+M~IuL!J%S5k`uW8^Q>X zf}8rLu&m@4?1Maoe}R-hv@3wJIa^>w+M*YZq5+Jt``L2qu<(Rb`UN0je<@=65{KC1 zYY>C)r+4P@((SB%K{xXz!-pM=y36N*{*_FUPvmszqAy7OPKq^QE1x28I6z9UH1^=& zA250<84nq08E|&L@?;&wfrqMBT?fdX;A_qSV=22!9AR!3Q0dJ3VY+aemDL6iFStd( z?+g|LFpyD!NeR8IKFQoNv_DmkI#$4~p)FxTTUq%7u-{0huWkXCimIGRAmqd#V^C?& ziY~a~S1t5UJ5CjaA`l5+&;Z8WF}0)M%=#J3702c<#0I}Ha9#lyQDaNXMQ8{#tM5K6 zbcO4LEFe`9fIRsqiu9!JLPsvbbxWeN#?ZE%2>0ujN`O&hr!7?0Y%ep{uxAl6*7ZTa zt?6#cBl_}TAYC1($h z7{a|l*a8y{L z3A{~pxP$s) zYWtLiQ;>W=-2iy_(r29lO_}nL@O)qJ)rxmFt;`SDTaNc_3F^oJZ4X$+06yfIuh>gD z{N1VfRJW-muoKbE!Ce-9_1;@>uSArR8KGaD>&nEslj_I{M=}6Q1?ns(EiEl0BSyK_ zRK{w2j0&Ryi93ZRyXw*wkh>+tSS3`T%8Z{SK-|(pILd5Cnd>XD-oPkX4Y3}HxENj7 z2NP92kW`g-J_psy&UQ7xPvWMV(6GX0+jST-h3WFNkhdlplnPo3ZF1^GYDC_soM=fKi>^yNiMZpV}I1?tsP) z`q|tG|4pJ}#VQKkhrhTHw7bHxwXNrOdcG!+cZ4e43bN`Rc(bb7z~$|Tgtrk4ZzBOZ zf{%3Xi^Y(i8j z?9Ugedb~yCecK?YMW<;HhKf3kPep!1&mpU%jy~GgVr5kleenru_*B*W55W`u>w zSEMyFb)etzsWHP>Sg{O%9bk;dx;U3Fi#yMvXkP(h`Ocz3=Yit{$UxZybYOd_Ub%{V z(624%Aq~#imj$(_ms}Qmmam-1xKDu`6;K9<$E~(DnsC?6-4;Zwgf*XBh0jN|Z6ESW zj$v{*BM79lp}FUO4YNjVCWRZ?R#tPFsucr_zzI$z`}4VQB4ed6Rch#p0d1V0pAcu8UVLGKqJDt0S%kt?1i|W z77JypZJ*(8KsNEdYj1|U6{Za~yTD#daMfY25yB|LojL%v@2insGc|}2?zw$A_ig#; zRh#s#P*JU(9SE{ivj}UmJ$zp+k-Srqk;4jD{ReQB0SbZtQXeS^JUG#Z*Lfd;LHe3Y zGdm$;Y}J`|Hl&l)tu!ldU9ZVJOz>6`X@h41 zxJ<}2*`^YbDk}rMiXE)Vome>IYk-&r_?|3EL{u?wX?Rt9oSnB`I5pugWjA?k&x0^R ziitiBk#(7X3$YGxfPrTdolgs)8=fauK_jY>TMF+EHotN4#QH@~XR%kq$Q8Za2kJ5Y znjunOXzQWPl%M-Zl$6e)0GCZ9ThU`gpyR2l96adKP0xH`Ei#xA?mv16HAM(%e0gCJ z%#hK2+y#y9!92_uJeC|JXHwv7?pk(GEh?;ShqI5Dnn8pyra28|hnnDN8}yVw5g!Wy zdu@3B98QvLxpk;jKOaERx4$t%W%lNFd)fKBJFn`H1BR%+dH+ah+*bVk1yaP7X1=sJPFHzs*AAm^0EdX~hHQ z(SCqEQ)BriLzSg2RNA>Nu)Rio1qX{%OHHXOqj2@0CxS4^84`q zE{wbNt*ah$>vjVn10+Ep?`l9fjVmk&YTg{i-#}3KySWZNxrQhKB@kxhx!mtkJ?X~z z1)m+e3XLpN;+H~;s#3L8=a+ZH9$l<<@I(`)9X7s`F!?fN> zM74)cAd!M9Dqi!Vk8;BxLWB&31y9gbicF58`^1(bm2^qy7k)F5=QP8QdO9Yd@Kb_c-Uf-=TLS$5KDgYt5)4`CmGcrQHaK){q??1i;vyMI~)d9Ha261ZK#=?bC1!x>NxVQjoe}X}qCUlgrm=)UeejrUGEzw$@9*wj z2f@(6{;zEd?eDOU5z05HL%TQ?Q#cVDz^Z*l#2aH=zMkDI3TXyH{#H%!N}A$m0pA8Y zx%uq3JuIm}<}m5P082jbLTtK0aYY5bQBW<+j>F8^yc~E0Fv+J3fh!d;3`zTIo(k1T zb*aWfz{8CRXMsNfHL3}8gxU|Ls&EhjBVsrPWNGtGxXicpmY^91#!!?`&lZq73fT$u zvqR#J;4*FrXt0f5wYXtOUbY4QKJ_D5hLKmo5^)6gn5?2U&h;~3%o5Ck|H-zKynGK- zk?Ml}CxUI4QKAMrN>FNmkQs_qz&M+UdJP%60w>GyE*( zor9a~`O?AwT2y8ILxE?jozrD=>rKVRMXrNIBjl-U=^^} zd{cn!p+y|dKw@hg17n)10!@GPt4?xcvU#!hdknD6a}l#H@BvY|r$)33k6UyS%PhP4 z5(qMrSTu4eF%U~GXcoiSpCPKwAYLt!o-ZiIiKzSZ)1~WM2<} zza-pKP=u%Fj2q}k<>*s{AcVst6@kJe#o$(OxlO5+&`HQaJ`^AA2-8vkNWfH4`Uvh< ze0=;+Dws@0ZXvF1-1lJaL&z#+FaMQ^V5ZXgvbi`3GRB7o<`u&Bgh?{&kNw;vJF@_3 z9FUD|@**H7T`J#zN=v}$Yg89-lw6>-vtd6$`eZwuh#mswKi^+CA=OTBHGG$DhHu+s zwY^>y_ebkwDA8a)<0-I5vzZs55I%oo<^HWIxDPW;&3en;1b9vX1wbST95lBzHwEvA z!AK44V&vam{gnUZ-NRBK`)xUbX$~yG0vWq&K_)mffNy42D`-Ff-Ms#%2_64>Xh=xw zG=%*Y|E2rGP?49F{WX6joX2m)k=qGG5L?u0;>&mTa{kE;aejOk)L529U*251gG6`& z##s4w$RFrWz9xRV#DbW^GWT4&axV{cPe1A}9RmQxh#M$a^L|}YAfq69&)7^e7|U7` zL0&EZPI`bu?7Lsz1Dc&Wux3Gk2oeASwRzin#0eBM&p=DSalSJNutsdU-t$#T73Zsz z{;NM0lsPo<<<0JwHDW9A_C|JE^KkV$(&$9l8FcU$FE4I}e^!^A& ze4Q)RCz8@sD!v5TgA)C%8^ls0IN2OUA^Kj;cpNx%Y~2;x$yVP4=@>8RbLWsyAm1F` z$#6Zg*-uIyx7@9N&XkEip0CW?g z(B;Di(zrLG^y?6iWN>m(4z2Nou{PMu)Cc_WS0n>scqW9Nmt#eC_TvX{FxDt9N$@2Y zCBT{Y+q~uLfwMs`^Hlx_QW7AZNS}G~2;w`G2IezhLk$5jD-R+kn~Qv!H)sR_cm^eU zYTHtJk)~tG)ENnMm(8GZd&M(IZUJxHR$gGI-g`GMHWo;cXAwJ)mJNV`3ppC#SGJmb zWe#?YNg#5@AtcY99tcN%fcOErwF(d&S{XPnfmGp|{u*G=!1;K*s$L1y@G&Ibz|ZS5 ziHJ~NY=3`nbE*p%Q~B^jXm+WVKjc8TvH-fX3Osp*7Xf{x9Rzdebs)X#JTYUlzgHxi zd~k091lT|)1$V5VUsnKeEKP~#2l`~RLck%B6rQA_SS}#Ax`u|-xB#ez=rlasY%*;{ zI}!HUnnh@rX&5yS7Kva>ufPo;Fazm?wOvV{C7NnD>J}Dkw zeSk{n@QmXw}_~ec#BvyMD0A7(~~ShH0>6WLf@PcMXXJ-U&ikg=sEK0pfj*DQ^J01r%X*hx!2E z@Io~R6fh_6pWt=?6ESD7H-IV(tl)DLlksa&blRS4J~4WYswrC)QTI6{T^al14Q#fB zg@r^T2LNEO(ZKcy!U*^{dcm5;Ul$_kOG`H)4J9#ZPZnsg+%@WR=4h+jVlmairrl;g zgLV-d<`7@{iI8^6WW-`xJixdQZoxZ-7tH8!ML|0P)uB%8V`i1K53n^&r_j9)GoNVX z8KvV9P$X=W6>r}~>rdiP!2Za>hUb+fXZa3p6)=G+7$afRT&x@tyYq$VPu}_Z^v9Z4!E3mLh)>Y`_vI^H)a4I4$MOe3#p+L0YG3e7y4%52zFve zy3;~SZH4fU3P&uqa6C{>VI?Z@l1X?j2!#+1RDh07wx-ypNlm5U%v-AyW=7n#t z$VT-{M+fBkJz%~7y6M~6OEBq)yW4IAHg*t0^c)^qBhvaGEfYO?bZW@kfxCvO)`Q^q z{dO0XK4K^Zh?O{4ctiDb%e-{dtEt%13*(&zP^7~QysApXkQzb z#BL4vaer7*LkdB*Ta-s6+1=*fR&+x_f@Gf~Ey*q9Auw>OFCreY5Nu9>RgvKs>Whg$ z5`*M?>fJ+#y=y@bE;17^1z4CvaR1QLRhbHzQ+AZ6}oaAr8J1xr{NWIqE6{oBEGOmvKd zQCWXyp%%zOY&dlIX4JWDZ)t-WG0~Ms5fLB@cbet|PH^0!sc8YcXmAcnWVbe1z;SRK ziTU>VVnN0beSwOKH7b_i9MpcSXUHi9Z9&_~nBREm7}II%b}DG_*c3ER#BZFvd4p^9 zz|zg#)ym8f`JI!w9qvt9Zd&A55fNH0Sz8BpD_8i{!OY!C+RDPo(u$T#+3JC{yA3Tb zKR2zI81COc?Df(}T}7>(8}I!Oy~PRYd7+3I^#@f-cL}pTV&UKjZZievJyi&AR+}p3 zHZw8KNdIB|S)YYSJQ~7m?g+`}C)(Inww5yLQ^ZGA z#F=OlRGx0_gfEq(!(!RUJjd6g3}fe+m*|VgjZRq%)mVQ0h~e))HCS0<}Od6Y0k zM4(&$sj1m--DKzcPwh8~!cRJ#J?aO()G7|S{Mim%mbDTh4D$;>u}{^i{g2m8;NM+0 zDO-0pbt~69PL9q_53C-z(+bjZ-Eneoa@BG+vw%w|ZRKfeVI}8k=KX(jjpeOut!>)+j*cbp!$!!79s_ary1CM}nW zm8Gqjl#>_jJ@}fN_NLHHPTpGrw1WIRoPt8MMzmb&X0FIB%p=72*AHk~xjA{bT3ET! z^1^*~e#T#Sq&jkULP+>KkRa#(cLE8IfB^5m+^*GnGfvY21Ob~SX1{WI35rN$Rv7MG zbT4{mWs{K2gN5gK&GLn6Ro^pN1;>+bn?1t42^^(YKfRxLvk-qWbGZFxDbcz#M0{K1 z*0Th)XVVHjEU)lpCQ=fY?(cjhtc>mdmht|Xv7Y>I&WjzvuX=`>n|$9HHoh3Bw~YBQ z_?_2uQ0b(xKD~8wGxF9i@5AmHSC{C$qRyFdu3PfWjou+0w{l zc@yN7UblBxaPGGEenBVRCE#IM^=P`D;0I<;cdR{H{*XMzYODA0j4WPJPepIX#!p$Z zGZ`aoA?5IVu!<{&!o%U!ai^prt%m=_ohie|K_m5gKFq0i?tLv_v{wr+4#m1fB{5j> z<>hIo#P@`UPV*yf?iI~?bqQ;?dioP8`tU0y(uzJPHgpEq%P@R=_?(o@g=up15a<3Q zOx&5(q~S$(*H?R-H~QT%@Av+GDxEJM`ObZPr_qJr@(e3=mv#c_lG=6 zM?+p2-%H9nWb^`#2+*Q4 z(nMLQI@{NB&7j)?eYNGP<<+dI4$c>(A<{8IWyKFUO|Lzr67F zJIC4QK5aRj9uIu88J+YC zy6pDR>)oXOEsCa`@jHtB4?jD76-$3UC2!9+Ec`h$i}km@usmuBXE}d^Bmr?_q8+|q ziLmb<-|vzRG&9?uy&PP=x?RYydD&Knj;ZnEg^#1kMTdq>n~NvRS}4^Td2O2-##K87 z#!u*O`r8)k6{sk>gcOiiM3B-Y{{!N4EI+~Y@#{e@p8Rs)-; z`g-n8J;QKFcNj^_i9NnmXt?U0JB67b7F6>{IBVb``qYTU>FmTGiQLp$_AC7zHB3Cj zy_T*7tj@2d1l5I88mr#om;zJ;5@Jw1?n${eM<>sRx8dT~4CXVrIcsol1R zMpMRkFPU>Lk(Xdfzql?-U#^m-z$?mO!A_?BT*-BnTh|+fIEmzm>7`C_%4F!~EVlob@Yqe_1Z-?Ya*mf$UydT4a(5-tAt606K0e(r9gXyx^x zhGFq&)Qef{ky@r#BYLdnK1w@`Nz**^{-tx8O?bMfF5X=9jpel=%4b&|&l`sg%lrC5 zLwX>(Wccjp6KC>`cV@9job&_X{eElWR)Lb> zk{Fjo!_Rk<$*%CISt>1BJVY^<33}f_>goEu+#x|XN_j9eGLbKdpRR7V@KfGII#s5} z`_hs01`3KrF|~@vC9Wsmm}1$YLhJ-N@N`x@OYeUfw_aPRr@Q!j;9~#dhdY82_m2g( zq-$eSa|oudcFD%B@H?Dl`i!wu}RzO ztuX`rZC`9=a~A$;R+a%heBev|AxfTL&RF{WUPY_X{-3Qcf?^r* zZfosoew?-}35*`JP5HowS4U}VF#0iUy`JOLlQwoi!hzzFOCWe2KS(!jvL>fF{v~d9 zy{r6rKF!UKtTmD4p7``H>IG7`Kik(l*P~mTC!ePmd98WN+m`!G!D>B@(u*YLFD<9jFe++GZvv@;)$|rL6 zp0DfI684cqxG_Ptmc#|IxL;obeY~DN4#Bvwn1;_;%SR2TdOHm{$d3Jc zhknFnG{(70vGwL!CLR$~=23h7&2~@jdVOc=<=^5GPlL$;r-wdzU5k zfJp{}^VxLgs&}lvr}x2*xxc?afSb42soU9TB6-l|kZ}D3ea1wJBPF?ZX4;pL9U~t^ z2vLH&*RpV54BAjX*|KHNuF2nTST6icDf94ph(w8}WBGyeL%v*>v8n(+(Iwj)!uOm9 zSVaa8Hz-fe250V8Q(YYq$I4|?Fn_F=zCNWpKI0>_Pu0QoLf3YPa0}bv#eUO+`?6-Q=|Zq8 zNN#@7)@4@99!QcMkSe(;oz$(c=b0PD&G{M)#|IPjraA#vx$JfLXGba^UxAl44I* zK~j7H;dALBsWh&nDax?Oug%z(Px>w5%D(AZl^btvH#yiDO-j7bs_ZD$A#WO7!U|T# zx+?8pIABXzQrIaq{zP~^lQUGrYi#XyyJfyhC?$guzfuQ(+H_lA%u}3=o7wLvo_)-> zxmxn|t>YMK2S27M=CHwyIL+(C6{=Sv2bRn~o3mLrC864klkHq($6KLRWSITxIGv3XoNYJ#7jxBKhDRnj~%@x>7x9h;^25(b^n{Tqz7lLeiuM_AIr zCDgc;v68>Ob@CKFQHZ|JOpsrwD-&6E13j_WrUER&w(>k)(%F0=H|cLeL9JzNXiKWK z{3x^D>oAV09i)>-n~1k|z$$08d&%`%Z~KsIt#Q~M*1W+hD&@*x-_gR&qW7-{j_&6g zc#LHj&ENI1H4mvxrl}~SwY$)gyp!NQsjFw~{HobQ)F2A$F1tMQZf_J{0`ps~&3N9c z8bK-%dzH)YXt{7L*mfD6t`ZB;&oK_V!dA;XI@zE_wq5C3Gb?k7>oM~$vUqs{Gv?X( zGqU2lg75#FIH={seHnxq+Huo-B?t}0^4`6f`P`oZSU;Y&?{7^La~&UClbayyfBRDl z<*ShpZ#4g@u$CqErHs2oYvj)5&By$4(e1vJUBUYKF}=Jb+0BjQZ2Et_fG zva_3hffAHfn#UdS?S*&~Cu;MT@mwKWk$F?Iq96AL+U1` z?l_JtKKorRnB5d5*2e4a1~8O^dEdS(oPI}%Q}BR&s9@u3os&y;whM*x@~Iowcg?%i z8xuX~gWog>3avcw##9Vi3jVmJZorG5qT5z^`Ps8zdNOo@F%I3T(q9@l z8gXp-EMG|^LphjvmC&AUTXg>Jnq}1#)8$ZmG@hBk%Nda6GS@{LK`=~#!N4y!A9=fM zUrg*l&)M{l`L&Vsrgx9pH(~>~6{=sh%7{Km*Agmizl5*Z-e3HV^Ew4n0?R_p{$}O; ztC(YXT`vO*%fM>`;U9joSgp$I1|0jT6Qf)f&l0oN-A)|z_Mv~*Sxz~hL-~eSYmsru zJt@`gh40Jl*Oo4I2JL5?*k5>uOczYBtn(}hk}d9a-e(38%M5AoY(^G=Q#eMwi6oI9km!vaq#u&t2YUd z+sEFg=B9n}DgJ=-OW58JcK>k>$DU2v@=RW47wQHtQxc7^97?wcPkPI5=D`;IvdMu+ z_Np*Rv9tgi>imuBr!yEBL3(tF=xE&nd}urr1ehrr3wu!?A923+OQ>VU@R8c0z#EYK zIYj$9_qeo2nA5TaO+RpeC^eC>w~H(aU1>xx*fBXn8XKM8w*V78G`cD86XHB3lG=vROQ#n&sc zi47){T~cP{J+_{kb#D%19_=t~uq^mjIkoaKiZ{9SUf48T*qZd~J$P`&4&4+9eNZ~dg?g9lE~!rr6h z#zQII$S<({{JdTBQNmZNzy>L;``SmZ7>Z zRO)GEUSHwi-Pe(LZ{6VY+Z?x9d?;geW!1!F$eN|$Z&mkK=;!F{lh_l5RA>h1lQ32J-(CEX?vVa$6?a@#^om!woWW30 zaZ}Gu0heiyin#u-Cz=^7*Nu8dRqAhi*3*)GpM9d#{7Yro`nj=Gimfq=hi>Zk1m1PA zw+Warc=i(uS!?<6r>J$qTA z`9yJ2j!7>;eoCcgRh60E1!b%IC-$#c2~nahtar~ZetX(sCh}8B#~`HSVY#R0Zm!bh zOtw@*4hLq1@T;3TEZ-mJsm=TxxPHA(g5qe8D zUZ^LDYUjO!d+i-lI_WqzNtZlQ&}W2GtT=O-aN@LTjr=yoo=a^@zP+4WRI7^n>v8?~ zQ!~u0lDKTnlB}DA(Y%(Qzum6)yrN0Sw&~dYp23YhIxlIv^~=yN7rgc>&%FW@o2`kj?@(BH9W_g~LR=``1LM+kHAB^>m@*?P%u7=` z2L+?)n*2)p3${Ec80*8?xwCi1h=*yf-BLIst+|pVKN7A^7uSjE-8l5e(H#v_o%Wv7 zr-p3uyz#uFuJ`6b1e|?qj-NUx^!d7+-F7g(-g?2Yp?NuWocmmrqm-e#jHQZ3Xn%Q+jo-(?ri$3gKfh&yp#N(N z_Kg`8p>{F4Eio;D;}2W{I`4k|eqm~!-=}Sj`I8|wr=o6<%)sf8OGSeB{B#Y~E3>23T#nZ^LQO=PnB*h@G zmk!a!FH$0EbzSae#5V8W^ZZ8WY>M$x(Y$Lz7B7&vEc8;>wTlmZr&Rr>RO|KIL+fhx z$!bNr*&f~IOWkNva$auC2rH%TL+1&6%PB8}`bA;=!q#*9z&eGhp9_3?4_hed_AuQ9 zhs$^LzuwLmGMuTPkY{MR<+xp5MH>COq`a8)>eCvr297xDoApbN@=Cfl%ezLNoj8r^ z@~72$GF?+ObR}`=eMF?kx^|a3e6aW17e#3!w3G(Qk89S)9rv%zUL}~9KqnFq-)^1H zb9VniLe2lc#|fiK*k0^ui|rRfG3nVVzT`g)FG~I>eW}7MOAK)L^kX?Rf5>!UjgR-B zVmhK#UZg8dq2pd4U)#Nfc28o@id2mq1~8k3-*6F+6eU!%J=c;q>iyt$w^$dGdGKvo zF==D{$Q41>Trb{7(Jb$i2itLMcbwHHXejSkC4b4WIyU|ol{3&QcKB4KL~FKWSl4lO zQg+qf?!K1Cn^%>sXD?pd8IclyM?7`bYCAo>m0#uhD)!<>lvY98pLw$xnNBpSRz_%J z*g~Zw`ssuWX5Y+Pc8zd_{qZ_5`J(4;V%fgsX`H0o+8gU$`2GE*(5>$3Ncz&WCo2uE zw%m1N+?lebT8$c6jk{F_Nt1V{X2ga|g-K@bJwfyRW|@5)v|f$oYio49CAMVxmCYFn zw{m4Ixrc|68;auI=DXqvZKEH9tLV0qmhNbc-jCI$!m7Pr4Sjw>6Bu-Az0QFV7TwV` z(`zYPl8U`)orU(BDD0EHt+7!m5C0W9RvxaqV|Oi{${M}o`7S>7Q+@c^Sb>Q;3$A2o zYh#gsirNS@odee^XTMM@G0uK-(O)9&7@t{fHf|612us}~x0=@0VL77rvlFUhikK!@ zmi)lK;krxTJB^XUVYzTu)bOB2s@5s) zUESVbhcUNL7`51gFb67`3;9vRgU{d3<9>cB%-x&ST^_Gq6&-RYhjJLnb~#Dt3JMze z)Xc}r3t58m#Q4v&pX@!Q*dvo#UOIe#nygh4m|tT2I&dW_?t>uQPgt3iLV1dVPF# z#$w+*6!+yw2xFYolZh95nIZB0k-C+}Mp3bs;LXc_A`<)>zKN2B3I5(GEzMMW*IgC< zI`+@W{;pG|nD5N#;a>mcYr765jIoh^FR#&<-!)12&~7msG-2(+?)Ii=pNw*sx8hw+ z_!+hA>O+M^qi^1GFElXoiq^w#OOM*z{+8A&z`M{;kn%jhx^OL^@_7(G$8v~KELl;o zb?{AtsS!`&{{E!QnZ2GjG1q8_Q|WIk)qVdmVn@AKa4n>zAiRg+TkW@HT+c70G8mD$ zI@Mn;@2)aPD!-MH9^Dd^*ZU*OVsarWc|*AM-nWT^2P%Z=>*H%9g1^5nhI@~RJI;V4 zWt?YaO(gBvGVb19qtbl-wLftRTAx$X9gZu6mrE*^?~DpIJhcrurCF=fyxb;KQPpL{ z6X@2+TB^drXMG*-A(f9>3d{X(IBr7ZoXuhf8Zqw$WI|PXN$&baIb28?(@pg+t3DQu z88u*bArUH1Ir1kt!+ulu@$0OA?7OAzJ6ox5ecq$oc=w$Axr@=vVDt>{YJKLr(8b-S zrP3x%nxVxmGu&!z$2Cf9*wi6OVW^e7&Enrb{r1nuq7C1@A&GtD?#?PO2c@k`q6^u%k4_~2*;VZX0y)8Euj7xpu zx9R>Iiy2RN&uPy>CE23zs%_?@?U3dUwZXIrCja<)PFdy_(e| zT|R>*fk1zdPD{hweJQ`wlb@05b6E24i{B87^wSUUacN434Es-Bm_yOiMS1IPyzmYe6vcTIX>9XXdXma`9_+^` zwm8l$r*0FQM=L8uJ&PR7RyD}-~BxPR6g|;c#ZmfqZa?I+n=l;Wh|kj ztyT9W-T*<_YfUE{$_L^UK2zj#^s;Y0j-!+^-PSSutmfHRU=V}qTS~X$Ek4^D5;Tj7=qU-^8%aPUp4`Cuego4W8Ae*Q<9#gDG_U)O zxzFz>pWTS7_Wr+!dlRsl_r70PgEY}RYHnFgq)|z;rd2JPq!iJhnMx9(Ii{9|{jEu*FiZ+4rVaQJ56IUt_s(bq8h z(tS%1zx!LmC)#>07nmMqs1+rU31M9RH%_k2-92`weUCqr^r}vtU-1%bd|I-?ZM6Dy zN9Vg=CyEaRoXg%(Vw0`cAp2pm^_QOV5cTRrzg<~i?4aCGNN0YBIBDT#+4YaU`+heVKhv_g z_ z3pTG)lP3GwzO2`bJh1&lIq^&3rDZ2V?epC}ls_j#HKlvG9#>+{sd*SIu>8^`?`+9J zzU|vyiqCz{P8&P~Lt%v{Lup%Q&y%^J3D13SoMyoz2PjbD6xm?pQo zK1b%gR-;;9Z+gTN89@yxDy92;=a;nxn}<6+4BRg68DA353glNiyld*mT7$T9qoPlD zBfk8Q8(7ihd!nfI*T$q%>phpNPi8I_3YqQr$mM(Sw$mrM%6BRH+b%|NblHcO&scEi zRe0YC=NhK;& zlfABA2@=`I@bR6CY<=yNXu;-0sm;W0HelfA(A z>TkWB-ipn>44yCY6uV=>Z>MdcF>=Ojy!O&h{!;7q;vX}{7DED}2bZf{wC%0*>Atga z&$??DM0EEN3n>NFH6^d#e*RcKcTI*NVwk7%nSjPA`7m+{Z%UVwd&=$I>uo)@ zr>xWr^GID&Y_cY7$;tf_uc^_<&u5oTU;6RX^@ZWb-7kMGwfg53;;-BJhxdhrZ7r3F z<2GiM{-6JYwCTn}*baq)yJ2vA_ru7IRV+Pf4Yhy%Ad&gHtZN0NW1-9K?5oMTf|=Q( zfPcnKv?{Zl|6}$*`JbPVwTg8UZMf zt3EbylZlfryw56KCU<`I_umY@JTc7nYF{o-HF>g2%Eoc4XB^fEGS=Uwwpl5*j@3i8 z`J%<4Z;XtL^>0sepIfW8dBv9Qa&p2AwQjcFW!rm()e?Kw^zurZ76;b^Qv6 zE$5cyPZi~AT&3QB((#U!D)-59waF2&^!rbEjIEh9leq0E(KT!BH)h4Tp6a+7GFix! z_1j|Wv#||{^BINV{aZ#KpI75Crt$dkN|zDO@5rxze9E^_dgn8X;>?nHKgNr6d>qFR z!PuH?c1>+KMx@B+?itnxrz3}tFjbVcJ(XZ7J_R);X=_c&eYU9vbxXny#a@ zvoT_kMx%v5W%fHSr=$miwK?zN+6GWi)k&zv7 z6IqZ)+5-r>je+GkKYR$T zhGJDmXJ?OhA1HS(uaZ={fSHz6;y(zCbz)LVipdqQMxkZ@ zZ!ge*!0oI>KY+kM4zHxNmH{EXXAq!Ql|JbZcz-BI`7aE0m|+zfU<&kdw12fc%3r;SH_yF3f{fbX<1qFf%pX3CdmO z8q4gKff5&1h=JTEHtsknb9+brJhid^&P3_!X4*@qvx*gx1-#N7;T0ral{maghZz?Q)eU0dG_ zxzgVb znl*U*`cN`feJ#5oNz(3qWfzU62B{UrYjGNoejFtW2-+4IN!PAv{)q^*^Q3OcPVP+j zI+#pCUSx+)mbUp#AD)Nib!wb%CIgKf&dy(@vfoe9ztgZJT0cf!Q zStEoUq(umpBLm%QMnPuG_(%$T)+jOo+a*cpMLd1FNkzq`^b~gmPlN}U4&b=$A8WYw zwtWO=q_zy=0vk7Ogx|p%xI|a&xb_wZcrYCIdw9e)({BchctXOzyv=D^x~560R)f5w z(3P0(m;Hr5ARN-Yf}vsrVG_T3Vg`y2FD56cV!P9aFF?Kkyo3su8y9P8))6QLo>4>B z)h=pm;F_Ghk)pg@nqZGl1-tpHkP5WnQ7A($oNRguc^Am(?%BVeOFH=A!9tZiF-v>T z%x-O%;07oj$};R}YS6_23x3n4O>XeIedRbB`VQw(#{Xdz=n?B4MryJ_O%XcWE6hWK zg7Vb8SGhlfzch#-pbqJ6A)sS(idSoCuW+h~F#bx3sG2H^-h~kcQA6{CR zt?j`3fNU}SJi?{GqFjt>=rUgtR!8|5!XACeLz%X`42I?I=xpVk1qOoFtg2BHtIlxG%?j`HVksg${>AVTOoqI&l;7rMM<(Z> zs|D`uK-%N*4}Jw~HlH)|xsCM$D@bv91^h1gO^1v)$aAx^`HRuGG*9wmiEGxBmYk`w zE+ZjeKqVr_jIiYcA~zh+4#QaTj%R;42b*-4Az7wlh&tRm|Fb^t>(*zi2ybw514+%T z&SwB>3TEc!vLWzHv4Fww5;RbYw1Z{vzaU5s^OoGyRPuSVR2XPEK^!xH*WdsX$#4<} z_JaP>gFXa>ZE9po3yY|8dB zZ7$hkIpR-Inf(1J^7rO(Fvj?OL1hgL2-q!dECKSZ!lbl-jcm>lzvZ*wr>|=>SejvK z!>8S9OMz@2*jFqsA1rR}8gYSt<8Xh^?-vd0Aj$E)z#(|MM+(I}*)iK)srN zqEteHaOLPPhWb`|w%7=fvdmWy@oZ^czpIgZE~-Xm7jF@wu*~sSefg{}*0&{+Y$-u+ zL^b_}pFA;Aue}D|``o#=qoSgsJE|NMmQF*V7`8&NQ&G^

`2r%}4g}x?d#j3$mTu zeVe0hf#fVPyUi3Fk&5rd+qgHB#M}MkS8}wzNUGSQKW4_ND)HwnE;uGktX7jED^0te zChVn9uiewv*MA`3yMc7%*WFqoTQ2LWRgRUC@`I14VDdb9`qTiVDd@0dF8Q&hvkg;MwJ;UhxHX4sETBNe&tAe9!8Y4y{@OMB>5 zv+19^in!}5PY?M0YerGD-iKb1t2<5(&OV>kxLqkp2z84fHA8?_XqD{qyY1~T zogW2{o$kZNxKdODjIex%WglulB}Ta^2KVWqRi(xWbKZ-aH*b#r{Ql_;uGgi~wn(x= z&$J-j(AIViABbMsR%s}xszUvRNALzhG(_c{cI@!0CfoT=%S*?-i7hZeoO4lXw}FF9 z_fy@M}r8;D+J`zo{oj1#^;dq#4`F=M4SHlH(VIL~hsWAo6Pn;4VvNK8$I<6?>B zPtKiCu>1M*=SqU0Rhy4x9CzWLTO)aigz$W3UPWRyp#q8m=elebo0sO2C+2k|)d_Lj z0(i~A&#E;khvI3F1E820rT-2X$u$_b9jfF^j{f|=vrBZR8f`32yE4Uud{O=lSmzc0- z787}pKA!ew6s0eDOSx}z`jdVX{`%#Iy^b|II=6Lz;WM58=QY@eS0mk2kWR)Lq+7A_ z{P}d1eqMMm55K(-?Fgv#AjGl$`8{?Idl#tN653Dii{obd%7O$jbo~mH@a6)SiLjp4 zAjDbx`NO{9`*|OCA@M0D=d*-lMvGS@Qv zRw}p5iNDVX#X3nVs&849Xl~N_=fpUlj7kj!Y3V(!GOGvyLaEgTv?;c0evDa zi5rNhbjB!3dW05@z!3*GcZa))JhD)%Ls;8u{Ypm!Ji|;GQNQRKLbzx9ObB2v|%u&}P&Zy@PwOcgU+ybBczJL`la5DW|e zHym};A*gX3>&eQ>O5l>D+aVu4`U64dPDe-gI}?71v!oZv-&AWj+qKHItxp8mx=J~y z9n9uFM=qmL5nmx@W-ZHGKsdaLcXx`qohfq`ajfw$-`+XMhCD~Fk~M>1DkyKW8x&HO zuOG&4eM4KBI;y3rI!o6AIe5@is?*Mhf;KCD+1wpA33x*y3V-K?DBG&6*x-%#3J{PE zzKN!i(EP+5F%p0ar!;CfARWOQNr0*|wy zSH-ES1))&SCd7)g#n`+?EKD3l^@#a`h8?~E&hoF&3Pbid9}aChidi9^SXV#+HH)ld zdy2+kTWTX~dx4)|oT%SdMgoC4_`m@rT=T9%B!{XYy=^!NJ6&D>gnv#F^uzkl=%LAf zxv5u^ZRq08P0*B!h5$t9$5D|Tj0|}wFNYfrf8KlO&>^u3)a~vKhBU&`2U=;euJ@EZ z+AGciG+Ze1l}t*()RoT?4*CsFHiRASbajb76Kdn;=C(m0fI3^Y332FLc*wXMbX79$ zSQOOZVSspRxV__}_D>WLdzkN#?iJ~|5cDH|>Ha#NLX>XD;UeLp1iCDV>xS3XbBh{! z)Uri`H=NuoYo69&S_!w)UNK3@^(V8>oJr}{@S!0w1gDxT;0<8#@yMe1r)@rTOwQ^b zLyM2n<>mA=o3rT@n&H_ABU2-f@tvXt)SZ+EbOLiy%}n5m6)Tt@U2;ojn)tr!W{ByC zki>Di$CVioX?LiCPu)Muu=-4EZU5@cT|qyX=yKbsKPy$QFl4&WZLg8V*r@=$Tnf%*_wI9b$`AeI=SlZmW;@K0l34*}bo=xtrNpdIrbK zT^=)>^&KqjFX3areG~q=I3`t|1#!|y3^j9SR3|-Z;ZCDQ!=&Oj`i?|SPR{r6mb28m z|G87hEis$rE2z@Pp(xW)ne^iOk01VL+E6jtFwF^*C>q_TwICMh!2^MjJ&4S~W;EhW zVynp9FQJq<9obB@Fb{>_T)splRh~mJ8MK9UkFl{ae(~iY)0YKxkXK7RzaFZc-h_)n z%IAWbK{GZwK6PSn(bloDOW8}~1im*HZLBCZ+^DSYs1}TcitvjopFSjasS3%BqhqEP{10%jh($SOrhd`(~l*Gt=V?6JCjF zklX_AByt8Em6d2IqMGbS=!Q0Reh)1>I=Z?hhl+P)5ur_M<6&uDTJ(Am!hRWM*0oW% z%1Jj@RdnK7nNXvTsm9#B8v!e6sQQ#Tt!B}f#xz656%s0ouH=!ulp?UX(FJ+?(W*(b zM!^oI5T!uNrvbb$xUVTKUDLNh$kwMu>_a-cqH}{`y?ng}-5zLXl5lVh8bvp<@x?P| z&cp>+jc<Su^=B{)4%9whvzobMw zUDnSbXTgE&yZ+Sxfl~wb_d;dY!-v~O9jz=Z@^lXRA<(KsBu5X=1n`D`pZ7%#fXtny zGCGZ`BXb;HEO5v^tLDYdV&hBi6FFP6h}$Y`CZGbB^E!obcWrxAvPM=NBzVeDdRP5^ zrmtSR-YjS_f|bS1#pNl#Qb!b#be90_1G?^`ulPp<+Z)IduQg zlP6E~A{J~$=2flgw|)1f{QCLLjlZiU?E%3x8-3;k6sM*J&3#XkV-k(6I&LmO>Vf+v ztc`px^rxS^PK02xkfMV{#BsVeXYarxLTZ6Yf{ni~i(QlkYSr*xOE+~xFu_H3Fum6p z`MU`v>PYfEeFieG{7(j**|p6fcek?kEajCmMSh=G=-RY>`*tXW)z;P`+NMe9eecUx zM+5$*wD8QCu5`(C&xHc6JMy;Y;w-<0=FLANL}l`9rV0%1p= znbYalGb+BKLvJ21v{9fs!TA%yvHY_Zv2?rW__pw)N0Xl{#5%hBGas*ucpgrMx<# z!5XsNF{d%YDRr;o&b(IF1oYUT}jL?k+zmqZ4F*ek%oYXz8C$ zq?3t~N&}0ZB}Xg_9MAFWKTfvf@ylY_`|C2Vx~!g-+%xrz35peM^Wg|-f;3^Xjm?TJ ziN?0>N|fJcPJ~~Gdctp7QPB0SsPBM}G3$U~lDM{Xukk>Qb%9&fu3o~kz4=X#2+sL4 z=QJ8U&Tt!BN1xjiWX#_xH`v>8_jbG{NniF^fWj_!h1;C2ja;rxxry1wcaguxYc6K< zc&D;T=cjHhW0E5b&Q)GloBn1Kx5OM_*v-{v{_C@!jNSLcn$tp$w7pDNd-@`(@PqSL zTvsz@wUvM0!ltSj;01$ao8f3acfa9Y=n9q8GRbuedB!_-J$xLOu{EpuK{Q|1W&HOk zf&NVHtmS)Nn`j@VThK7nt5M*Ay^Pb&wsQ<~6MJSZ>{)X(<;qY+OQs=8)_Jr;|IRC~r`!~7* zQMSh0w?BONfb!DEGg|!&J|}Hf?5gp>%(^`d_gM@MzA{SsQcZuWDE8uy`_$%v1qzRGRLdia~?1zP5Ksl;Dn$0Gt9pRpvJgT&Rjr4=Y=<4U?Fx;M$c^UM_cXCn`=<sIp8^6wf78#X}V=OVUA6{n!t^7ZXQj!dLv`B*bc`IBFnv#BH$GRY|^bTM`xA0K#= zoYV>!R&nA!8H`6vz`&8EYPr6wFAXCDhtgme?D(lEI_YUpTZ zzkDHY*|NpbGDhW$?qOaz4yGabW_|LG#_(YDYLFYkQ`|#?V*O?xCL=XU&YkeM7=O^Nu`v$K9lW;82U@mLY zGBAKeoSn&}NyqJt6}c#l8F5RaE=UE#_QwJ-zF7H@TF}~#L(??Vg6Z$D)g_Q)sJ`oi zXl7@L6QicWj~VZnW_C3DKA@t@OggOYmu2^#ZV#qzP~ECV0e$Aqqep@=C3cr2zkd66 zVlAv^pEVq`Dz&0urW_mW=3RtYuKRiwj6*OqxG+>eAWUEjfP%oCzNp62thDpbqH@dM z$FdAF2D&Pzed>h^{&?=&{K#(~)tt6?1gpa-XUmlx)>BFLQT$2FxMXp`6OFZ`hUF{_ zu{IMySe!;8K2+_c6#HeZnrI}DjUC<3^{IT)O?;$tO?pP7L|c5V+#|D=SF7 z(FG=b%QQ*T@|TNpoc0#h-@Hk01Y|h1xC;o0>FJQXPH}pBzJ_#Z*X=8Hb^QLm%$F}; zeozFs*(3S$5?zlrAMoKdi3bLR4r)N?tG$Ch`38h8)>{ExX78UH`huQ`r^Tx$k zstr@^)^8E?Pjz_vvK0mz?M_Q@&_IE8!)M?eM)--6X!@N_&(FQITIfQZy3Ndo26Zd$ zdPuMp7K%HpFe98&lI7FRs&AEhOvO8vkqq|O2SI=LqH?I2O_n`HNSt8fPKRhSL=2sr zoEndQgGl{hxUOz>#3V{mOh-pYlx58`kD#e;9*bnlW+KL;H?S+Ly#qNqosAoN_BYVg zT#MAU`BQ9`1ygu(m+2@5suh!fICy&d=#73dV?R-0xgW18z!0V9JB~QI=O4TR^aKV8 zDOHFy5_oLFz0HfSR-2pOQS<6;Vegonf~uY2IS6ky!%R^1vaAzESTLKw=yhBUa~{F; z(?$VQxiPpFtg$PrAfalkz?@@_C%LOc#zxi?L7Pud(6P?Pc53}&f(o{Mv+}O=wYP3~ zvEAzAy0}|VuS5rk1&*R_4XQ}(T>OI3?I1toRa^-nt|yv+FnDoraecxd$A-1=6B#D< zgaCGl#4ME6vbgp%f{G;u{i+yPE@_O_(rh9G_unFId*?{L(l_?Xc#oxA)P zQ30MUp5s^SDA|mfNqG!@Su|UBJ*CTi`650vTmu9Bvq+d$>zP#)FjnR%2t{9l#O4t; zP);)Ys4Se9V3N_A@gc*1P9f~cBo z|2@xGCtm4huSxmG9X8H3Md|gA_r-T^Eefpj|DO2sxxF$4c8L7rtBaFQM)mT4{H>J@ zQoFk?I;wZxx{h87M@Fso1!gp`A#p+P@FMb#^QIMAT3TWsu|(hMvWgd29wj>}rbzqe z*R+UWzr6|3(zhG?;1@v6V6c2JEX+(_(zfq6m*u+;9|Em=(UM+8meAID>xb}Jsnd%J z5oRr^eCr*nhatopm4`93=iO5uP$bD1VF&tSewt5AEagHFf@kVjP6bKZD}tE$8@Io- z8%Y|LZ34=!!y_Xjot<7-X(S@DeDdCrNRGi>vz)~WT;8pT9!1Cw0emGL{WeKne?8Mv zBL{Sk%)d!D_~m2mNXI!?qr^aVf{(F})P2+|#;ez^TQ^uuLRbJ2=cp)sC1d+*zmt=& z8r>AQX&(6^S}|MRR%T@ZzoaC68;i*g#$g8qvr5pGHQ-%n6p6KlW2uf;bdubTB znTc9cTZq;h>vhBDA2H0;o;`c~JKGweTuo)=i^%A-HJke}bK2PyvGF?Qo~|C>(gQ5J zBHz+7QQbj1@A2T^0a(S_dE&c=15P{0}|IY zt{j=0Vsl?5p{orH0QR&}dl4cN^RYz=LrMA@xzW_J-W>AH9h*fDsDAX1&^qz^O^V4R zMoR8e^mMp|+!mCu{!@1&LORyrJek^b0mx@SEGOlB#+ZoJrju2)vxImnIE2_zPZhcZ3aIZ)V~QILf|DM?+VFg zov`=v=|a1Q#W#jf^P(DpCK$hVI;GE4*q+C$hhg9zXDcD*gZwwlF0HZj#A4l_hk(BR z;D-?b?;Qxyq;hHFT5%qfo2nOI{QmXLz*i_j5fxM2Ro;U6u|v<3BGbh-#zan(Dt@^p zzpW~}uy7}o1miX&-(9i&j=&vr#$3)F$?0phNG6%CS_sEqQoJKX_TtOi8y=gLec*g~ zcI8Uv-a0Sh6h#^hoW_|K(4X#ja7i`5*E2qI`E_n_@zi}{9!l=0*kUwLRW$!{95w(ZW&{Uoq@^)2^K_Ha>JdoLV0&IgYL8`&PkP_E*X!LJ1-)d~@UpaRs1VUhG1 zVF86eW!=I;ndelw^6i#VV2>~e`HTYnXa4&p7igo-TxxDMn0L_^ms7E$z|wvF(x!-o z8C0zk@+uLDeAcLRw^VWEa-EGvIq-rT+vDh42`5fOe!izw|K{7nmeODzoO7<|Z(qJV z9KPF9y}O{Nudt-cdvjK4pX**@{gsH=R_z;4UfNI7P0|P`&C7Yy1Dx17yGxWz;u_`< zvRV=5mF^vPg3jAhcH1OktY2x7Gy|-`HoxVxUCQ+3v`j0`?z&&5qFAKx?8B$ViKHGC z2_ahw2nSx4O=BEv2_imcJ{ql`q-iT`nE46|lXG3rP2FySlKz+amoepfc1CW3Nmpm5 z(L|_UR{xPMh52`nx1N8cX=sYsEyKPto^&RUT={cgsV_#y28AHSd-mNQ;1X8n9ie-|6Tzr?|Yj%sG=v zmFnc$`qyjLva-jKMLYR^R@Dp0(C9$7xK44sVD&9uFfVV_u#69Ro6@1;y?_66Dhb7B zTFbWGhvrVpKA;vCONsF>7kpP;n_Hop8Gr(fbw8x`fzQnP#PH$tzVB~EJTBs-{KBrX zx}Renu>u0Um?TK0o;`cE>k%qqXn50|NQO^(DqaiSlajFEO&@v7kz;+d8GMVi;)~H! z-^MzUU=n&9Zcf7*yIYniJOeHbzyhU<+PYVfLxCMbn{Gzd*|h0`zN@~k-70s^(%;rK zkLn}p*@KoX`TP4Pw;N^gq$MUk7*-ln9T3nEFZ&VT$&b}LEwS~6aYVq(MC+Ypk^`v7 z&s%Bew|{-SyR+nTEE^PPhgND z-oQWQF{KPlWzc~GE-*9b7g`RlTXvnz$C9A!f8y$9SYPanU(MQe|k#wI5Px@;nBWWtJjd9SkJU6wdr)_2T}v31f2XeQi?q;oorQ*)J|G zjvnYt!e_PRXPuO+%=PzfP=IaCmpzBO@SLbMzD@vlWM~);Qs%Pn6U7aVi%p=dzA(g2 z{i%BE;5$KH3$4%$7P4L3Y7L)d7h3CyzsOHVsK8oSb#CS}bK2q04C|cqJbI&V9=&j^ z%#A|77f)7vHoBC!k6cdleXP#UGXa>D^AB2aZY&iVz>?)pO09&3a=IMz1Vk$>KkTU0 z4h^ny@UuJVYu6EQl|*ma`>jSP%iny{w|{og1*iq5jyjcFY``o9aO*HEQ5|e{1hO z(M7FFv{fw(SPT`R;bjE;VaM$w*HmvAQaZ#RqU(Ya?a+noT+bat%XAOE%q}45j=aW{xq>g zi>IU5_I!xR*>ncma^ju`9XHrQ`DX+p7)+!O`em8vC4W<=&(q(om`ZmqVa${Zu=Hrn zTBgU_#B6jf=;o}R#hWv4+sRrBITPyO59=p2Fiktr@kdXT8pMzp0Jx9jsGPnXI#@Y@!WZTwFYCeqdR6yS zts%h7I6fk%UvjZJ^dJrEFt$5}i=9#&`&+rw^RO~3`4Phc5U<4Bj~_oOyFHM9Oo&P^ z&3kGh6m9#EOh)xNaJ6#CEllUo$M_d4|FU#ACG#G5A@o;duvUpzAq{dLeT_>*A1 z#a+NJA)&la>u&37=Ud6AG%7LsTu6S#buC)I8N;}l71<@|5Zybanl`nVsmN1}C^Ktp zckE->54C0$M%3(^g&L5oeHqD2hZzfV5{PSScnbB7Zn*df7?7uRzV=uZ0777ylkUIH zP3WX5$MfevMJ1idYz~(=QINez`mAn#PVOu`oJRD`4o@eicXiywlx5IHvCtn4+J3#D z@D$AlES0J6h6p)F*lvWXZy613Kk=l$e-(N4*=;$GG;+h0`-=%*zkb!BZ$M-f0dp|hmNM{$%~toYqs}`KyK>w6=r}XOjGk)f-bqvD z*~wkSQ)mX6=FZWkS{sihw&M}@YOg8#fC%aE?QMQJVK4phR_nyT?H9Ch%RL>fL9&-_ z4EfnPIPf=JKQ9;eA#p!6oWq8=B_c9At|gRD$V!`O<$Q0C`Y7bp@^MoAY>sBn(pICg z^$X`1&f8S3V|+Z1JLyx?)YRlI6AVueV8uGVnar;irGqVwL~+H2hYQ{|K_V>q*3EEE zEo92VX_lrFS3$yFko{QN=e^)-EAL>{jHk9Rb zV;Q=gnEm(OTo|ymdZ**yx6SN)m*&Hd`IV&nGwFw8Q{VIqOp$Fa&X=G77rxq&yWS@& zJi#J3uck=#qxaoYpft4qny1P_kpIj z_EZXfRvv1WlRYaw^daPJ!dFo^Q>o2F{Tl)%M^|`dxcT^ zBAk3OSQf2bcDzZ?YEkYOlWYYL>A~xYA*~eL(vB1j4N~7?#n6q5Gp2^6A01{*H=9h} zEu@m1IDiO&dIQoE{_Z_ZX$Rz4DnA29x_9NXz#N^+SEj-T>r9DR?a zXpjX67PIR^g|-VSpS;vJvS?4qPR=awY*^nYnI5jpxze>XvPyv$9UQFm&kw+MHWd5!5*Gz`E$qK#q}aaIoBecP8LvIopzc? z{yp!r%*V$&{3#O40(a;v-^we7uptl-8KW zuEm_HYa~>We0*NO2;^*)EDhYib}`YFOe^5`>F&AD=-f{>%oz>2@;)poD&w2k(w5Fp2h&O;1 zHHaDO*LVsaD;UBt3ROs&^|>UJb^a2K#^IM0vgaVrv45>{`UBpqp2gdNdqwG|oylES z1D|eq5!q}1{W9-i&jg>WJ=RsErOUjg?Qdni|7#hBA?0o*D}id|b6o$L(#FYiT)*Pb z`v3cH3(z_K`{y@UefXLX<>Y{$xqI*4z24p`Ao3BH1q_zuUjFy>@#Eu3T3ZCS8~k&5 zO#SBa-!6`wzke93tySUTN`Ldl$Kjf8zM12Hv_6X|s z{I^T)o=-MMcU&SVnkj3R5)=|bcQM8o^y04DBmZSE^XJL z*MARUl}vkkyDdv37W_cfkkLr}vr1Z(Z0HWuJ3S$2j$(&wkaP#xSXHAk>CFbey@NVtn z*jHdMJ)x`kWo0G8oOIOec8?9dAdjdh6&?otF9`Gy)+Ll$Hi?*S%LMG-ZvZ|C^C{(> zCz)EAxf6tvo=2mR{deRgFtU>Mf>}v7d8QI{dGh{3Y?@4=yp$3K4_|#HdcbBqN-<_oZyB~5O@eQdiGsywXSLV3QzCtT zr7`#g!&g!PGih+wI5kzD+;%fr<>K5?J+Si zsYBO^t#XhAn!Cg}%iG6?pFfwpGWG7=QPAabl*u6PadjOlJiG(!%G)i#&4Vj-5i8=1 zI4tXq9655b1}=#K~w7Tsig>L3=+*yGV>=9kbE%+OIkj_lHO#xl!_73G!`G_-5)5htkF z!G++#14|hM=PCVzbf(d1JJ3&EaH&fpr7z4a-_Gclb?zUTZ41-?=Gyov-Yy-u2qQ{5 zzE6CU%<-A=R`);}I@F2P*itx(l#ozdZypDZ2IaAhxNxS^#T4sti%FKVu9Pses}UWk zH?bT&dAJBJ*C$W!!(?WyQ>|2otg32oM@tCfsIf0!^?nv}E$qwZPVd90D7)PH;Ru1>aMZJ7O~#|$7dZWBc(*VtuHiWh ztO;fWNk_x9mpg`e7LsQk;wX2->7(~Q89Pza z*<7EQnL)|S8vkJN85J+lRyH0jPr{^-k%?&y*|^lVF{WjPvM;2Vqxus6HLXY-#9_tlVS;#hO!?-!rp zi*{|^co~I$CQC?IP{ij8Urp90tA!le2#6WRp<6%fQuNDeTTJ2pIb&ZQY8|?qd;*Xl zR{$9IFK%I)S>bddoZ3&dD8Q^<&O_pW{q$qP3pXzX9sz+h>t0$4fNaAmJ%FuYYZtaodw zwkz$$H7fPQ>6DO=KuW}(a{-IQ@_IpSOH4D$t@C4q7cVzqQw>}Nih$MjJFd3={iWBR zL5BQ;*?R^-jAze|WA=h~1Q~~W7-y=Sz~MCGEyE{%C)HKLVQB-h=2Efw}NC z`Gt8le)CRXXt7lum)6$o=B+a+BK|uA=)B<{22cxHkvl0O2LO-4y77(mVFX1nQJuI0 zCYF+tk|xM++&U}#(U4`^?fM%Jcm%g~c0bJ0(6i5DmVmG#^HVZ%sU#SC_%K@vgj1*- z&gD#hLF>vX6nkJpur7)b=DenNWZuvk_%4-&CdfXZG(%*O(K1=ypj6=I$%1G4=jjTWMuROw z+uBRN@lPohhF}3W^I_5`vq5wJGqJSt;I-hF#%0XQybB%2kS%tLRMdn-tR~}Mpu6}| z!kRTJCH0pu5=Wnht<9t1%P6(bRX*zHK5B$!*cE#d%%O&VqLf0tE2K7v*iAQp?dmda z8E3;xAGNox#(R{O5YdoZ$g$KbWjbVh1?ao*yEs@p&c%k&*dGXb@hx|{9*}Ptcm!qI zi7{zCBw-?ZV$Y9euhJdux+2hwol;+(o*>LV^L-AG| z1KuYwNKj<_*Ytf(#DEAoWOl%+&zHaH8z(+5~-@#rje(}`^Ss@zPfD;Y6L!<=CSIYuQh4}g7nzO+O!pao=QCN?7Uo3%W9wrlN*7~2*l z>aG1KL~+uDM-4}#c^fgtV;&D67ibvmMl<~E*)t$NX=x(`p%_@Z9#cKMyjak5h?$vr z{5gJD_n=#SP>DmOU4p-pmklN8`ya{;##42XzmLnwmE(C+^KLRTuSfrYgpNEr^vseB z)uF6io`tI-FtrvgzTMcUtEY$it1P3s3pi5lSO8alK5sg9!JdLuPV|7=rgnT!n{;$6 ze*<~X+fz)++LD6MfM6lxieJGWHKt5miH_D+RD6rbW0MSq%ejDBepL=;avPDkR>;!j zEmM$6w#Y9uK}5A^R#3JsmWhIFu!l9{EVj;$RSs+-Cr`VM6!7!$F>Q@qn-mHA$6x47 z&O%@YmmTxC9`EpCRX`#X9Ir5t8;wTF?Af zJ~9=YZUM^~>JzwG0y#%W*Qul!5Q{VsR&gE6iN}6?xPfnYE-xb^15=ojPYAvZET`sR zp;}r&NAn5t*?{$5v4koIS9qD9^xlT=Uq2`T9=Iagu^=jom4kz#!4^*^*KJ{#0P)OM zn9=nD4oc+DFS*x=l|$;gk@#uanj{KJVdc)qMxv>SNn#Kx?anGHM&vCA#`h}hRf?`^ zm!|gIKy0%e^~?q{_s;{$T;)2-5u4v)W|kt1@IH%@AHlZOGmBnVc>~jfXs=QMSmS%t zq&!Y58HMj~2q+GryhEgPAH+j|lWP!~$oJa-ItC6xYNeeFP-wXde!O!+d{Q>_9Xm5{ zEE*Ul<$UjQ1g5#^$Hz@M))-4}+vdJ}2#+1%kDX*lqMqJ2BXaq=i1`T#v@+840YmcY zcw_^-+?|pMLWay9ANkLK3wCa7n-^4eWi>E~H~fs<-0_8p6*;GfB>{7l>PA(MhE z4B4ao{U3o;lCJ@V@@p^pkKaGHza~iZyJI9)pC*3*iBxFa4r(U&$gI+Y+ub_i;Rrp_ zDgaM`FB1f8E$AY#!GbZ-1I++|V}s+Kb`C5%o9w@8ti&YooFgKb6iS+~+D*K8A$$9w zx$g8UwBR>Gye_M3G9`x`Jh&2AJ>|rW0Q7CZ=CJq46A?A0*5RgON0EGsl|sl3$A2C{ zHJr)vL)rO?NokRT5#!}1(T~iUZA5+$cdLQD`~yP#GSOJ9C!b)s?x^-4G%qTC{fHX@ z(7pEjyrukf@P?ISEUZLAx16_JsXJZ9py7)#9avavov%>2Ph%rw>{#Q4@aWpavBdxp zcf{G$)D%$MHs0Ve%yS+dCo_QM{ij?SJ!42gT%oPGKo=^AU$(ax-v=_8m1;f%nfA@1 zkfFnUZpiqr<5SC|CHH{f?fp8mhb}EsdF1xg9jRUG74-PLmoXKDqD*%X*FgOU?!dNN zte|*v8`C(##$OS{u_QV^mN9GKn6mh_;CP$?f6c@fnv9i<*G3hKFNg z;=iFz($;;|M&>_=E|G)>Nq@>LV~Uu#xHMt%dFW#*5W0Y2f$76Dfw~0%u#vpm#Uu;R7Uo*rKW zME6`eOZKD`cza+_$zpX+Mb+!>A0DoK-7k9y^(Yd}4J-?o?~CY$t1yaEwkzyCh{=>it_6C-fC?c-lt!Iyp7MTlv4dI2 z*L+u3xBE^hn<%$xDW@yz%mBK*di7~~xv9ImB?|IeGRp#^bfuR4o*t^?`?c@M6_62W zDDpEG&|}*6Xpp>%faj00ktVylpNoy%jNHj{7{fDZ13ZhUqvM9dBZw-hzAr#)P|sYo z9WpKGxbJc->v?kXK)D(8u(A$7j){XK8HA0K9yc2rEU{|zV z;}^#=*rbUu;`I#pvez-97(|_5TPYU?Dhj53r3cM{nt!2Bt+2?HL7vOGPxO7wMvnQK zNGBjbafU4#pw-tsloioH4r~SUG+^xZQLQFfsqr^&uH&MI%kXA2ll~a#nh=#gAU1Q`eDGeN8-}L@6}I!4&CmhT@v}ou zvqz2fPG{%DXZiuStgy0xs(t~S7_{sickaA<`j4N(aFpk64yJ}|XJ_Z;j2BRhqrpZWfIfNR{MV1CqY(G-OgfHX7~OgA8m1|JqoFQ0 zLrNQGDtDaD);k*ZKS+D?a4P%teYnh1Wu7V-mSsxjM97@QQpO00k`N`SBqj5lVUZz~ zd1{g*^eAIV5uzd$l4z7vNWAA=J?&>d-{0`Q$GiX8$Fa9q>%Kq3b)D1Ide+ouFM5`4 zGLCC(5MaK`X}!2uaAYuS>)Cb>aT?)*%dj#IG%j^o1qvRGL=S}V|@Tt(4X(&J3p z-k6XdmridYc)Z(u=>XF!R$P~2Fd-kE9HAI8F)=wfI=%)mlqss{*2RVb4cQM zR=nxETHI=A0pZU3vLd8}XrUOp&$`}MvLf({7j7-!4+r<}S8`W4mgP3`!4D#REXs3h zw!+#GUH}KY*l765YVMFiT@yTcU4z@Qi<<6+d$)@ezLk76lT&zunL4iL&wczW51S zq@4Wc9~_ZLzU8_75(UaliL@9_+TXuVhueeu9g9`LYz~D*0wEAdT}BckJW;=Ye-LRx z+=NTVWb)PtYiv2;--yWMCm3npTwQ1^#j{=X8!?#)C}5Rj-6YAigX!Z`{L(e=|z+-gS;kKVBk#x!jC& z0OOJg;P)?X9NsLUtf6rPa2Z%?Kn~E)32OR#P^tGfikQsDZrce+)c)4-x~Y$Vfi2?2 z3?h*__b}rbgodu3p5q@r9RDz&fkPH#?UqujS14$$LV;ol$XGFi{NbOnFK9&p>>i57 zG3tR)>+^MB+_4i7po$xWNR@0>{ipj;^IlFefnozx1i|dtnbUylNsSwB8jA zBHgC1^MqlGrRm6^6m~hvCxeE(ygaLcl{ZplZB4^*0kDjN{Bb7G_>jU>dB-ihs1eLi zQ<&qaAk9Q83YgWX@k;|{V*G!p{eGwS=!b||;<)3jvtyH{567Shx%i{2+GVrY4$a(M*q?gk!CgwzmT z$P8l?w4lQ@(C9!&Id=mPoaY^=Y7lD#P12TF01jVp_P2& z93>C*==E;o&1d5?a2bCa>!R-)?u5ZP>}ln`7sz9xE7}hc3O+j#1L+6_vHa8^03KAc z#vKTF@;!JE$m52S_J8UDK~^RXLCzH`R zqR4_me6WGuTR#U6ZgK@6XE0W58NFx)Ei+L71S2}?3BwQor#5H+%K1EjWE^y+I}(cQ z0?6++H%CQ8AUxHJWMN5DUUhDS!j&r(;pF?jT^OiSwwoPz1C*ek|ab zV=Fid5d^8%4}R&mZbt-{ZF3MTOnDFg_Vmx#6Wk5Q4jBVWh53><_33;dD> zWg)jlgNtv=3+`s*2w@K}B zZh%`1e%AA-YkftgPY4DNsi1l50Re^E?FVA+*nN>PMKB_ksP35G-h zjA33d>Y^;uL^5S-!-deWFx=IzA-D!n^=xNs7B4^lsgqgZ-29f-)xJYu<-~}RrDCu`tBV+9uohFVcEwr+nY*_g-878&n89vBtXSK7xdqe+p)+xvKEuaZig`6q@o~k(Sof3`B>p zT6?H5$DUk~7*_Y`51&GsUW@L>*RP}8WVdDExc>{+gw%P!Ob-@z ze*CMcOtWh3@h)j2Jzd6eBU6t|Za@+})oa+y5 zc&Em7G(jzP37?_i*KazzwLk2Bt=gfd+oAFHThxPWJR|o9)%+g;qS8Nzyf4P#VYj$I z1JYqSBt*ebp*wE6%L@PT^;Pu4`4c&8rk=&d)8*yEo$IxB9PJzs#I(5Z=V7}eT?_5L52fGC`7lIoQ16yh;Gj2ft@^lFeeLE7mrAxHS7+%5b_7QIy8lAUt(O`=PzF(&hPj0 z6W(xq4hJt|pDN3-@zL?|nfKRU#8zzFMAi$z8fRi`Fvm`O$X&`W?=bj=xZdR*M#jd* zUcVOL;RzcUEX9-L-PH|AQZjv`g=w{=Q{yopcqP0w>RaC2^0$ta%>?DFJnOK1sYfvm zjeH8ClRV|GuNd*`OizHTms$0&>_1BW{ zhrpc0Y!m}ip6At2vo-|l9N)1*fRAsNw>MQJvPvy1QJ+hi_Y%_wX#P!crG*_Z2&|T{ z-w*F?LVUbpTazwF3bKZbk!nj6ZroTH%VXlIwoUO?@otMcYVmlyD?h4QY|#obufxsj zBXTsG#pei)o^D_r`Yk#IdvSYmn(EYKZfpp+2ZY8h6I2`okSDl=^5Vvsb^G%mND&Oq z^#QZ99`V!r9c=ONYB`ek1F%dXm~a^G_@^GgCFTr~=|di><=lPeJu zyl3xT5l+tXL+>3v)vditN{yI6$;DYD^itvGbf|NY!G5un?X`S@zdvscGO>IVSP>WQ z$-d_cuRR&U>9pspUhFIAaA`&KE>V1~!Np)vkWFP{ zW6L{=?;^jgaY;@z!cUTz@>)Toz9dUaHX}YvIGLsC>nEPc)mmCBb|q8AU+SATe;ES0 z3Zf#fUU@v<)vW7aHS?Df*lP=~H!Pq_pMWMnm7<`1p`hgL7U>CBNl8xZiXDUKt_c&< z^iCz9gxou;Gj|wP(bz3+R(80>|%Q9*xZ#* z6*C|&Snzr~?7A`g;qa;g~&jnX2Ey z$;x`?C><}y&xf7l8Y^5^sf@3Z=Z=r(q251oj)+Ox0kZ4F1n7<};I?pP;2l0Lwcd@A zY)3`tUD6;V@XvP3tXb31-cIC=gFm0L-F8!X9G6iEM;YmO#2pVt$Jk3v+O3L43C<3g z&N`{{T;QURV6=G~d)U9er0F?u7tN3<@})4sKAZ_l zzlk>Sss(!P3Y@XvupTQf9Mc>Km3E>&w_}1KeW%BW{hh>X9S;E`LFz-7b)X4)!jyHC}9&}=uphe?g8Sm z}#Bi0Mg8W1uW1kk;H6m zZ4E&i$v6~0)ZHhE8m_yq0QVt`Z+7^cI5vqa+Lh&IHCV$-uS+z+lWtK1T;!0SpMDLV zxjtq=I0VxD_k*>uq@6kyd5E_DoTjK~4&Gu-O%0R>4ZQ%Uw*M7y1@{EYDxDxAPM~u` zZf-6P|BFauV3jomS^ov(*NeKN0$I$NqDni^?x&`wZ`^$b3LK{tv9MlzyiCl(PaE^MTfr2sG6FZLVv?r!IQ_eFfn8> zXL!Q7Hb!|I7d8vnkrQnZ!$`Dic%S+dXBMZ>s#S>-sNzgK4vuxYV)=uXSo8c)x7`1D zjaI8wqaY zljv8jqs@Ar)6t>eu zmcktJFb$p%9>WA`ULPrz0iaySJlAg8h#PlfjWVLx35}2;vb|oM%WL-+`PbpFV^Jux zIc$0xb~Hz$vPDNM;vgw6Z$2$bgTmt~UL+o4w@hWH7CINKK^?fsY;0_hV^DGRYGI>0 zq>H5EzUk-B%Qa$kKe&4r0U}t-{2x*rPrG?~#xM{8wzg%wbtkN4Y3TUSj!t|8L<(ch zR@^CQ;)1(RJ+%Gzdtqi--miWax>EGVh~8yxp*dwD95OD>48m#TjZr=&t%2^&WEJin zghdRUYh)^86=$CkGdt-}YKa2KwMd@95YIk9KBeW++=Ge1(oB z)p)lC1-bq`GuEnU|6ZqGJjH5Kva)7Nz^!|9lVJ$YR6UqAZ) z<_v35j9B`w9jt`==1gmAD^7sSj0{xn=p6YSyWkavz*Vklm_z7aFLHPU`xcCf2kzzd zCzq?QQB+J?YcBi7@4$frpY{QlXn9}V`x&oT{8IASf1iojz%p}|K&_#_A=kOFsR>|& zSnJvl!)l=DkiO4AX(3j^iDS}?xd%y(b|n_B1aN=Bx)@@2b7yCfMt`@b5Jq#|z6B|r zySqDf@7Aqbqj9}~{&{P+{a7zFUsxCOX2up4oNMrL&V%c~mExWfo|Napv7oQthFI!Q^bh{1 zCmC?2#kS(}YAGsmhM+bU6cXwpaaSh6%_>Tx;EkmgPq{d?9m4x&G2k7#jW)-61oDO0 zLdmHW|9;5~bWA=0;Nq1cNj!DyPs99gH2?j@!tDnr)n&DhEEz%;6wvl()%)p{@Er`O zK+U<&wb+JiepXA!6?YFQQp8{yswx|PFRKgkdBw!DU)Uj7G&a){ z={!voV^YAO^p```nS2}srv7atbHj_D@dEx?IodDbek64^|I2t|V2(bhKnqn@3siZ{|#kUsp(!p2F2JK`c# z-3bjgSl}B;Nu1%iM;QA0Nw{KBdMG$mbB0Da*O#VHaAa&mvP9OSong2emnh+q0Q$Wh zwNT{Jm9lfz|*d79_ zR$ZRz#-;)J{OHoQ^HYTCoOqi8TT#v64&?Xa>#e{SuHoi_DA;W^{g(xRMwc;}XByIi zPmndpchrur`SbOSpo00_Yqwpd4uC=x_E{@Op`BSY)$8%4XRxXkDkV|SsuudXyN|*+ zf&dG-a%wlzuP4^rnZ1BZ1V8#79L3wWH!y;wd7$>-c_dpx!30zoX*@7v7ebZ=OoVV6 z9sY=y2DQoLAV67WRJF)Hs8z!=Yw`}2e*=ay1dCY0*cSdJwebw52S~I8kJYn&fg`j`{yQNvZYUjnoMB`#X@pob4J3<-Q5dnUs+;C{Q>cubjgP}?EG4%f`_ zK+D~#5_=it1<=mvu595<292I8J2wujX?>)8ZSOPT5cn7@#!HAnJ&a4-tU!o(_VF2J zp_Ow>>CU!XoMoV*%hC38^)+^82g5mp(h;xhW== zOjQIH)z|An@CHn^tIbDmt?jNIJ5tcqZ0Kz*z}{L0)zP(s2M#n{b*lE&ORB57T3&9C zI2LdqV@Ko#9Rdc)_xZ|Swd{ZYf$gRcb&afbN2YTXmJLpEPAGdtbb;1|A-sRsgN>}9kc~BMLOQUgL zm%50KQv=mA78g8P*DERlkuEKcke?c4+UfNvtuh%L8DS{BdhGilQ))>H1pCO@;RWY$?jj?)HN-hjk5Oz1 z9qt>>qo{sbC$2uw$-lTdH@##E*P!ef^Uz@&2l;h!XdiWuP6e9|Wmh*SIm=nVtBie@ z(om!cq~0YJ%Db-UiJxAJ2FL zK=%TDf^rNErJ9?2o`T*{1yb1Qhs!?cvR_SWBXNxtV(D(VV2>i$I*m%q)-&q(dGK3# z>Tv2q9LcK#nRH#9HXRfV{`>dey?5{XaGZikw=Ux^l(6CPV|7VMqoli}EIn?wmEYd< z$}xz#jMb~DDYII!rRfocOteiZ1wBUb)DHV`GxZjJl@%P#KMyq-rr|jhGl;cMJXYoRV_vUNEv0S;`mnbgY)L+Q$)f) z+ehT$h>D)@1EkcCn+RvSg{0>!HiF~Ut#yd)25gEaij@oI8HEdvDve^122_Gqh?`s7 zlw9-Qp9-CKz?&W!1{@MQU0rd=FKx|5M|$(fXQpHnMC3x?*-M=C2f^O?hrIzop<9SX-$(BcT2h*Z%IBML=clk?l1=j3LTLHq139_4*k&)A%8t=wU2e}t+OF}Ed*#7gv$ckqA#zq0pX zQ9Jl30YSmBk&$pg2epgSh0EQnkfD35oT){BPcdVkoV-7C-F6$H#qApjwShkcSjQwwaMCBiF6k81zZo? zQJQsEqnU_IZtS=<9YPh~DtQNv8+hSydB@!{WhDQ(LRuF8eMA#Kn5VwS3Z?Q*Bx0U9 zoe2U@DQVe@Kg~f)VjeXRvxCt5dw57+oO3l5{ZrTfzi4Z=yWq+oKPyT>>wX*ssi|28 zM2w?HHf1O_An7oW_|6+D&SD8}2hg<9z41?n!j2|8zI)r{4aPKGvdnP?zy(xU-anr! zcI4HoCGfhyliboGsi~QDC}!Z=o^&|dF?{0k<&QwhF+g;WkB`ruJ!Y?aV{tSRv{>Eq z=g*-XL)P}vnLRzR;Z{zR>PHw~okyWwH+c~zW^60}r=WCY?YjcWwHQil30D^B5=XeD4K zu~VWmV}sj_5B%r16Q;c#VI5n-Jd)Q}A3}cuc@Mgi$4=yJe+5`(o8zJ@ob4kOD&M_* z+c?{tfExK0*L;C$gW>tjV|(1(7>EbvXI6c{&(FdXSecsllE~kE^OG4lZhn4(VAQK3 zZU!Idy-~IUbT9BhVanQ0pQ#FX(@Fn6G!RgfZTbdW*Mf}l56Q%GjX>Se5>{62p}|&8 zP|ysp`&c0kNVXB_kEkqJ26zALu6XyX0d5Oo0Qn3ODI1~HG$V*fZQ z8Ph$Rlv!){!h~J2NJI?F7?*WxE;qs{(K`+$r>QbU18Z`B_N*bfM~l2SzX8wR?Az zS4sbYQkVbwt44V4ND)ZT)z_yZ#fq+PPB^)M08t|(l+Hgx3JdrhLRH3Xv&Gr@?1bJW z490klA)hs3WO@1a#zt{%?QL0MR+*`wR`=~g_S2ehOd~uL6ti?6Vk;2wMUst5=gzELQ#sPO--4h@_a0DlvqryF$T~~L^ z*qG^V8chG|)~}Dd3q2H8$f=Ls-H_2D-=O)NUtA@7q0>KbCDWS2AJ4|b+y#EY6}dXu zk%a(BnVF-oGtQ?|av&tdFAMMF-hHq+v!uSh9-AR(7xx=cw6Q@mR$iro~Y6HsU z3L1@PjaVkG21r)863?MTN1s3&&9Nb1MA0bR`Qn6dn`N!q9va)055UjGW@5`^zA*j` z-QALf5D&fxuSXZ~&7Pw|hzt+!yuJ@OIG`yQaP+gl=OJit{knCc14z2FlCi3xgBU*1 z_|)*D6f`E3q0!Rd z-|)IYxx5FXdiq{VUga19hlHFU$(5yw@#5g<4gzQ-SaP9A7spPvy=#8n=>LZwxI}jq z=S@upHZxI+Dp|FA<Qh560GTQ50jv^_$@0_8A(EcAg9_6sN;*Em#~tnmY)HvBTj z2`L~zGBE5^U`NfW#O^FJLS=+Dj1EeX#xRq^PFUnw&4ch zpKL20xkuXneXq|)LVXE!X6}(lNdj#zZvHBIw*Ugf`#GWga;r6Q4iR;z*gu`csf4T$0l>;2ZcZuxL78?X+|_AcGwwvd%Y0 zbp2wWli-t$EE9%{oBQ?ehMzX$>NoC2kpUwF1mV$$7;e}906$&L`Yh#-)i92~H- zQfiXO;(sGycRGx*RGt$6ZT-R#y+h4D04qOFP9i}vaj}V)msen*=2YT90y=!rLXWkSaaIWu z?X1o*wx?1X9|LX$J|J`XoLqon_V=| z$W}%FH=> z%Xi(consW>5q_dIbTc6gxWtWv5X%Ea2SeWtser5T@f_St7-bGe;Ei>!%#O@MREZreJ4S~tjjj#-m^@hjJFNjmd3L9y<>{t4%OV!KOj0#mYLudc2( zIWM-9UW^N#S>h(kiVoW;R(z2h+$yg(s?Rn5Ab4KlqJ|-tFs-P(Q`lrtVeGjb7Y#GR z*BZiowAeL@s8}zby zwL0Z8wHCo^W#?nRi8B?tXhm2Scgsk85jy1yZNM4sr(q--Ay}SX+^ylK93wu&%o&+T zB%+5$VNA&iE>(J={8`BXL-^Ghn1-RTIdKk+TZp~>mQboZ6;p9Gv3~LuMX5}?r?Pr= z{M@o#X4dvzZrwyO>>XC@>x@TRSFZD2Gz7f&6fI-|V7WR1VQ+4s>auL3$uwfRh4n$(-a4EWhAk64M3Bk%9OGkuZZemf zduXZitdJ0|ro&NgGy)~xC1)}qJiSY>)gxowa4vEfH^#(1bbsM(d3=eX-pb4?5c|6& z{sOV&5|b%YlV`GJZ(CSM5EDq|(CJ+|wn#Y4&;)8bhCb>#C%HOn+VQv7LGA{sNI%a= zOEcobX`{y0r~)%k$=&S<)c%?SQZbr6AxmLv~Mp_wXJce+`bqA)a?@)ABR zWc{^ubVPnZA_?14hTDS6m&fUCIU5f+w2>Z`H9C>V(Lzi{Vr~~LNupq5iEhtz_u=NX zgld1{Gt_*;DJM`TGNlk^#A;9@ijE#Bpr@?2PPIAPc8KToG~`e}O*xj9b-DSZq{0D* ztL-QXV^b9y2Pam7-ScWZ$A1K_9b^)l5K@#By>etXx4X>dJo zL}LhUh#|`ys(|HVHAuOU0$bg1T2BXz#w@sMY>ViP`z_X^t!sS)MD*Y;JU&Zuj2a%C zcFZc9C%wIVuFduNjSG*~*U!H{< zF`*9IXqO2v)d+89=n(Em+0oTfH0aX9r$FjT)9sY5mYBoc-R!%0TG5WaE+kM~ZZ^x_ z;)w>^gOd(=twRk%-i(=tygloSJs3tf_AQHvUOsdo zY0*w?6%`d;v2TTrKxZyqy!igzI~@I~nVF?!iIv}jE#gz{9i=mU4=5Ree3+`PPW+_< zv2jD8Hbs8>a7&mWi+se6*u}pJXI&@;_8(8Tyx+pd_Ol4jNd*}S3jEQXJ9nNxpQr6R zd2#nqr}UsF{FR}n;|QXkkc6fpeQ6~iLsWf;s0w;h3ruV8fB%S^V5pCtTy>2g(MkEA z30C6x9J1NxWG#cb-A$((7)^PP%i`AYV>Jg8(W%8HCJJ0W1z*tY-KTiJsqSY#vQ~Ln z1Xo{-)WGAuSIF_uwR;%-@kcMl1z37%jv3?pt4HnrZWqwzb8q%qa)xJ7A8`d-@Flq*2oStJeuDmDKB^v zqEpd~0y)n~I4w#W?KtyxeB5*QZgrs9dPP+5v7_(a#j|mUXxW>0hIR<+#5x||8NgPlUm?yjTx`67L zihU2r1T~|+3rdiz`(&Rd+FGw?Y^%RP^4 zhVI1A-#;59_nGC47+xW3a>B*h)|7HPN#1C~pvbS~z)@2ps_5asLu}k`MO{XC>{Ekz_Ea;{9>rch9Ie zG8zcSG4`d4iwkHx3@)!*`d_%9wUt%4{uy&C!&C~O6SgHw&{(gL-Rb3Js^XADLBpc& zg4hoNYj8xy1u$19Ju;0Ti<7I-y!_Y)71EYVP81~sMO)IYMh9l{3JH}KO~!uHKb<=c z_WK6_Ky#7pG_1`?Z<*P}5D^pYFBWGnG6Gvk4jLRrV%NPq{s4*!BALL7zhKs|g&U+o zYWQ56&lR=8m4dfmY-rdiWbA@SF)u%UO9di6<5JP2>BnR9a~fF)yj)U9ME13QEOfkC zn99k~XEj9+ZLk=&Y-7@CIg}S2xjKhGteC%$D5OHh2_r2Xt+NX88RYpQ12XXG)J=s! zpN1m>M(Xl7P5+F>TOA99^#u66MkVPCR`}|4+>>>U!lwE3E7ef zzij1ReC*!zy$)jX|KpsFZnP>LsbsnfD=b@N(z=C&!sTg-_Y4wsi z6?cExIL801uJ)JmUD_zk+H}(Q+dYZc34jVgqJjddraGVhIIHNP;_vVZ$ExqRx=F^?iZo5cVXPDP`~EqYZJs_ z#``2O1D7tGy*N*!(jc!Qq|2#`L|nj8`|V}>wBg4TkN1-2qWF)hsy`TcAT;4T(=Qh0 zi`!y`zDk;%GTDjK@FCC+y#_yEwPx=&l~ZqymBcGZVo?L8W9XP$7Rv= z4->wIV>kW;Xw%M`>5Fp9ANlxPyRgsnA+kW}3r#M^Dr3B_1E(ho_?~)pF<+!5Nb_i3 z;Qz36l3rF?PL6&4Jqn8?%DH1-g&0^bU2?eAh7CoGu|XpIAW*Rq?$orh>_VLMoX3Ie z7cp7Za$WB}sx%kA5oYvE_-27(+U_cGBw9z!{xHP@aDixUSC)~9xxXST8N`$6qHS>0 zTN9Sg=XY&TqUXA!0Ud!`i6&P zr8TohhYljS3|!B(w({miY~?YJWsUb;ckxC>iK#3g8v!TFcp0F4py7F^LoddBY*F-w zVWv*>Dlr&YFVi}7x}n_clK=jtTfJ%ovkv0-N1siXVe;?c*TH#UHRwv{MRkP)uXV+% zIz2??0=QoEs7Z))UF8c z5Lc&CmPt_hMq>P;5ZglK0NNQMM6X07LniR=@5#;0je=@iy9<59i1R?xdX07KvUQF* zl)h^iTuQH5GoYGI5qt%KSf>|-W!Vz6LXnlu1dHG42~0g9I$Ad}m17(*pHUr^dzq7U z>~P)0d2AhG{Fksrw(~=+CT{j$+mJ;iN?cs!K}W}J>rTg`Pq@a#HJ>;b`djz6krsWF z3S5>4B8wmhCN|$Yqqc|t9_ftn??uLVDF+xwsJD!=sugtJjey|`H-AUc`b)6l$5K*L z87vJ??!nVioe$N*2)G~s=ZIzn$LHr)apT7N+kv6pMj;u_2;&UA#iR~6GHysrb(!*A zJ2Lu1Vm4iLOk(0U@TtG=!Z0rk=qhR3>af`IjqYP@jBmT7XZWJzZhrxq;reeMFt6U6 zJx)lL-!A3P%Mt9??PaOBvOH7c2^6D*!0h7rywg?+v{X%{5mrhSEZ-b_Tq-#IF+&li z26ns=Ad#QV)c<_XSLlqAj4_K8a!3s*gz(`ICBOMx4AN8dzv`Z6QMRLJju1ON@j&4* zqK8i`mq(8k4L_qOy|lA!pnlLeKOY~XC1IlgYNG!}lo?P`zRC1`&?ZpWh=G;2O{$wm z%oLB8h@jv){E10gtcIDKsOid{3ABpcW>)x?>3e6y+bynCrx3;}oS!V242h)OqoMEQ zmq%_G7RZm?Yg$LOdX<`U?akxMiCpIj3wJ?$)!p4K?d(;9j_0N1fM&wU{>Eb5pIn&` z9-;u7x>lVSjytHUOQhh~hE(TSpl(J&*4e0*^x{87E~<_D`yb3;_kKZ7^!ul6uEo8> zKEWmEF!x0a^r;9J*QZj`-8$SWuSINTR2LvL&Obk9i47ST^ou)pUh{0a4F&}5Z`Lxo zF3aCb{ijCyrw{BGgV92WpnqD3{;mNCMi4*XlKfL|oahX@Qq)?7ADi*0bI3G&kCG$F z*WsN*Me^`D%Z##TH>qFnGoYSgA7??XpS_w@V&40(`1p8`T-ZqHl5WFuJOtd}Zs9pa z*HHKudN!17-B5Mu*3;lUbueQP?JB)+bljAO_UHd5@ z$(DzXj|A!h5>40M{t4J>)eDhg^*<|_HG#A1{;KPL5$VdUcf0E#Lqv(NXs?lVSbE;4}lFLfi8Ql{)@wEt2!* zT~V;&el9o<3QI*z&3yA_2+uA7<-c*`sEG;7nE+z6oPYpC+bU?(z%>gVx3I84tD$I; z{Yxsx{c!K~aN$e@&~iY|NF-X%MLRa>J?SLV$ExwVne_qHwx}g5m!inT&Jftujg7#U zIJee~sC`{dzlF36B!BrHr0adU;G^*%r{^nt5N`YSEej#thS)1vF{#K-l7`uM;)j`H zC-m;uvl7{)D5;N2K(4WA7MFYzq8`PXEFM5Vi_^Rq*rMVLiXAW*|3j)5jpj`1ES)Dj z7HkiSWC=v}E?^j)B?N;}3q`R%TTRwc96pwu;^P6lOOj9J&1`FlX>f>rUaxIZ?+#6c z$cE$biUay^=ZS7Y$YZNK(hz1bz&w38k?(GUXE{%^z-vERLBJRvc*L7QYnizeFwwN=&5il z9DO`TRZZlwRnM>P-pqiSiIIk!+?!Spay-8KxywamIHytd_k%M$1yU(BJNw<+x9p2g zojH@RpI_q~(n(r1rfE3bg1+|c_P-!MMI;F^W>U*A864E7yE{2H((rD3dxXIWtHxfG zAjpR|1^)GBSIV_(Us{tT#x25dyX6VbJZHa#_(WyYbj0e?3gA@oc*C|}xXF-WCX*?6 zp>S-;Kp2B}6^?LTe9~oSj!!~PPJ4L5Y%8>Gk;etEx=^Er3IZhDxZk_S+;Dlu%evb{ z>U~)X+v6aYSG-NOIrrmrUw-$wRB}s83u{IC zv9SxNEmp-C z&f=S?DJTA{f))cR>fR8`G|}Zp5P#=d$@uXTFuP_jd}R2DzdprQL$(i_vP6sYvJl9p zfV-ads}Jx26B)YpDrI6rH(L@lOZ#Y5K=!3>VeXkW()4SfUJ{~Gw7_p4%&lGHvE?P1 zo;v;MefOeEE-jT==~*x)#Oo6b-n!h74fNuMRHzL1!yBg;29Zk=m8cD}4P0DA%NXd3 zA3G*d@^0$yPVOg;UQadNB|paA;6sW_NaTKNKK!Hp{(UN5HRNw7*~@Cn-Z*=7U

=o4&SP^UyQnW##=DT@7 zMN%k&AAs)dSP}z>RhIH{3$k+nI*RGeN9{`C{q-gx7Mn$XD9y1xO=P1S@iK2=AhrDt zW8+Aty{i;xgOZ}|kUepQ9YwEA_-M|fyTs(^34FqqJ`#znUZ}b|IcQf(R{L?jj}g}T z^GRNK)B^kNe=9saYh_Buv_~e8{+sCH(=8@Rks*rC`#&9m-YxsgnQXn%(EC8c2=vzxiw5W-EfswW&+1kFz?$-TadTTzotYP+RDmPq#)+1= z+~brg-{{aHU!+jqPm<1q+>cdR9SXsZAjOmkbt9C4&KC~zKEmB z@{JRsmcb3hE|FvtG)Fk4kg^+;9NhxxS)!-%Dwj$w4!g!%7jiV|L^}aU*omRFf;Do!qs20X)nl)b6-~us_5z z86K0iHD0uD!pQ_TAI^p{=xbT#;O~p}_w`-pKbF*eCzVDqIf5RC0k8)lI?&%|`;0&z zacUt-;a7wL^Xo+vdWleGiz$x0_hz>&?zOcEYX??BWDkrynRNxDv9UjedbxTZH`USN zd*SjC9<4~&tAG&g*dcz~8sy=dT~;U+27M7fKI97BkI{=VP!bJwIe{DICc?mVvZ}OX zw{#noUd18R(tf5unL<-|#ujNG+pU?(!12gxis^$M-P-e05~&zjf#A&@m*N9ZN|{X^lG^5SP8KUpyDZ( zc*MQHLIh96w!qMffw9}kP|hNF%UVfCUHvg8Xr)7GpX<6M>mDH(>kszV4BGyYfId2>U-q%NXJX>2LhI80pl26bI5}UJam%7!PC!eGYU9J0Vbp zRBjr!EaRG|*oJWokWX&KOwN`$pjZHZP#re67NU$q)5!1UsE45m5>Z%aDA$syRs-uX zi1P?|*tZb*RS>Z7yyiZHGX^^CgaRxjB?bTR)yI#C!n3opdl5#EO(N%3Qy}*8u0b%2-q%BcoEqm2#kpsHG9 zbmu`c+{vhNoB%cEyer{C5;vp0I2{Zvp^x%fvCs%at?DQ$^N&E5xT2$@Mbqe%7D?#_ zkEMMuOvnS-9Z&R0N58yDqtU_!=BHmSYbK%?r=KFfEupQ`Of`VDUkhy+2VW~ZN|6M1 z^Zwo9x^OnpMRyAQ*OsE<1U_iINRvT0Ox$usuaJHof)pSZ1q2t#9OiN&eB#q5^rb_6 zO2c6;VFh~iXdo6$)dek6DG;n~?C{+cN?owGx6ctkwM+LR&aX?xH_)H`E=fzKrw80T zLmwrb;1xw5Ci&Y)PKFrr+Q8nkIpH{{G9v>d6;ojWLyDJut^YVyVA(zfwXu>yMe%{K z@Vr}^$7U-N5|;REV&56=DtoZlSJ{7cKg7I42R*mnfqJQP)X4OVe56C?COCm6OeUCo z!=i~4^!VgWZ1r_XuA%R8)Qy4}&lexqoY5G8UF=_3<6;is9Pii3Nx;O4$9I&6-RkPT z_0CkDP}P}`TdT(FeaDe^!tbU`(MljD0LaR+E&eSWcJuWmym`LshwH#BjS(##A zZCZnb)gZ_SaMTFw3#T@YQIj-y{F|Y`;yjs{SUix5YVZr1AP=>rw>{8^afiZ^Tponi#H zu*tJoao+2pkny4xe6}esN>ZJ^{yfVHt$Qxdo+)0g-k5c^{__rA%16y{X=&)q-;`3wVz8%$!t7g?{L}ilG%rtX zk*1eSJj*xH+@9^k8Kw+kgNNTRpVjDa**sOfP?=RDRnG3@YJtUpVN96}>L#qs@Kp{M2pp=9ZRMi5uoEWND&;V=a9+ z@?n`g58cCUC)L5wgK=?Uh}M@`)6t+P4~}WxS3+_=>$My$#|lv-60OdHUF?CatbmKh zQ(Iy7o9GEu&J1@@16JU?MqB$?Q&U*keh}^8;>x^iTTesFCDDlv%)v<1$da5(=9EH* zz~|O~yT4!frA0i0(p>XHJv!GM+3RarZ>EaOKyDc796<|@H{1$~F{%w^W#u*mlh7z$ zXB|>0q4$Sgt})6C-?FXTW$`=>0nD3v!}f&!-g70BcBm~wZiy=t_naw9r{rDb`_YT1 z-lD$O5CJ3J-WRMg3wpM@7Ti^M7EAZ*rf~R0Jk5{Zh9&2LI>(Z)>!J6{s5yNaLy(AD z0fH3ed7a_QqymKNv@6zHX1A<_BbG_QbQkvs=U>8|U zU7EhIY(_v{vM1DAvH+T;vM8^I%I<*V)U!^X#vF0|mtCstHar(U@Q0W+@Ms^IG_)kw z9seQROXPN;IcpX%zp4Nz6~gW^2V0ws(l(!D`-PN?rkFj1w2)2qQ{;Ov&}!Y?&Q6Qz z>nI>{og7}0j*mx9l`LaQ8&2Olk!?{1lit9M!wTZbr?aV6pL<{rR~)q+P-pdF>0{b| z{!OWx;YRrgt|1YU&5Yc%6$oc&#Pq%%jLchat3Wf{EjKI2!SM=lEr_8=Fx{1672KO^ z!0g<(jrXV2_wgP2>-p7#Ta$cxSz;u=ZT(Tq7^Ze{ZSQKq;XT!WbGqEq0v7?FYtlBs ztHFXsBigckl^UX6W7Qd?*?tziZ(my09@**a{9OG-5TR+=Y$a*yga|L^E&MCdA%C5k zx_|$E5boURnOh>iKs#&^TWG{De8FveuE4aczPtnLPUrGYwqHQpYyj^wLgDG-?G44M z+KWJkeXwT0k98|QKR+(+9w3Viqc1DCe^GBH(zECP>0G5O0Z;-WZnVktXF2hIRWB<} zjG`sq78LZ?jv&}>0f)-kl(@LN6KPnub9sZG6-a9z`wj#i&FA&2SLN_pi;G`_Phu~3 znnO`t**5!kN!@)X`=H$M{vuXW0ke`LwTL20m$8o2af7|r18%VQf7uiWRu_&f9tf+A znC2^I84%UHGPS{y)&#;2#zLFA^03dv<|gp=@Lk53bl3n0$c^9&&{-(!Z83fDwbuyG zy|MoETOeWG&&V;T8|@t&5byE<#oP4GIv7*|QiD{aJmmfSZBK0RK4eo?VS!ml5Yaji z9e~x;)vd?-k}#JzzP~6kH8mBcewKVCw_&-K8)Nut@ygFoLW@1tf#`z5~Zx7cOQdn?| z2JAn|DFZKz5c&`0sc+vBPoBIjSP;GGMdT*mLx)bCeHB}5pxnde3R|0&=w#53*=w^~ zLJfh$(aR-e zzQ*NItrq;PGD)p#Ip^P9eGnUXv0B72HUzDQVSPm7Mm5iFhKlfxU11?{Diagjf-Uu@ zporXq_M35l(O4bOdYA+x6|3jKRy=@3Hpg+02NATQt%s;MBPTz9CIsE~NwS#?0zDTU z)dM)(7{8gVVcFz3K-Ikx8S1^!Y3{rYd8JlB(#cvLxAp7$o;ddlC<=?9W}YR37xN?kp~eVIXtBBElo`H+A%4J7R&TBT3@QoH5`Z9f zz@ZQ+B(8bv1`zLI)^c&_EI*9a>G&ph75gc#3LR6DDhq60!`sIx>9FHIV2m<8NjG~3 z;kyrz@5*f=swEofGJNUfaMjb{vx@>Wy}VkEqw;-oZj&rcSw$s^Lx@hhbdlIbDNK?6lUl)n*kC#(y77R3A% zFNODq1cesp>G-Y|&*rq*&DMtdbKeN?O=@w_$M@CkZRC^x;E=hJM^O1jo(lRT?{5a z)ZAMQ{2lA@*L0%$xh}jM2yVr}S13W)3S|YF=VBeitwX=yh!dm!9jWSdB>pID)_y+1 zHP&f<;t@v2hSsI9#>)n>`qdRD>t2HHie}G>;Sn26VX|htMEtBTVDiUx50mH-&sD8wI`@>V z);vq;j5~{@B_&}fgJB4*HXgMuU3DFF+o*cDs9%8h%`<-+f_)aoJjMMMm=VY95W)N~ zU^w3Te76i#&$~dBA^k*VS41EVMbNz0Zx&B@}iQ^y>40`>r38grjEtwkf?!`FBYbx zuuRUZp=!1d$fzZm_8c4?SAjPR$20TL7TQ6UCk+5q=4;wq|Li0XJwc&dNb*Z=?QPN9 zOlyQEDGj-&%TgD}Wk2`_buC8oOZCvF;C=xnQI zuw;KVkQ|9m>~Q6um;t>6;>f%_&H(!wq0k_< zn7{c2Lp2l$RoZW0B|U=R2X1?i&3cJD0oi>?YhC~#u-cd?$ogj;b8@*%oA~%stg~&i zR$Xp$%La8ouI3YS0H-k6+uXW`B7{NW`!obtOhFg!%YO!4Q!`QM7<61;IE`^Yh|aN0 z?PoJ|NkcLBE0OCU7B=0jay!nK)Pj;i@y8{+%}XRJXi4BYRM)^sFC0&fP~kBDi&vK4 zB={Ed5jQUOlLqW~M=oHBz#bs1!$CQg0Qt@8@FS?m9hpTR0QX{cnhjrue)R+p70-wN zb~9eE&mv>W%Y%A4l`Q$YJppB6K`6x#UtsC&AO}w+Rg9s7s=0`qM_B zRnjbWAhHHvF`3XlPQq%oVISLJCU8>jNkgr_xxIaT2ph{3Et03F0QDT0Z|IM~;jT2t z#Teb0JGw9mk8xzbIZICs&6kmN7w9fpS-Z#!eX1-9Vto|9O;2dBbRoKBv=Ed9z=J}c z0xr}uBz0=(%8-ceYfAJM!?BHri`$6=P(ihao$Axt40@d7%Yn$KH@D7bynQwR7?-!| zqZQyzxWa9IQl5h}P}BI#rWCBu6L*}Z@Ccu&!$;W*QhjZ8b#mK{FUrcwdu@Og(Zf+u z(h=29DXol7|pJ<|Nby$NKt;K^gbvK>cMLH{t%I-abLrd+3j4SS~ zs5=RU)o4`U%R&z(X325v$@xc&vmP+Hq$dlyK8WB9T=^L0)Ya^a?T6`rf{Km*Wo5y^ z;}6}oB`r|1BFsDajo-lL8eZqHMTW!bzbID^ju<`zkaqxyGrxcSK6v*aQ_fCyLf>?D zVowpB6D|=z7rnD)(L|u>J>iibUz?$`IsoUAGEH(E;@L}XMt{_1*mN%WrMu5fbosns z(~~g+?^^#Jm@y?Ah*6U+WVrwrX(0xq+_?GD?(0FrX3%(V%bgHt ze};^x{7|nWZW5r((ozR#b6t&FSUEewgN*eV94MiEC`!%QNL6f=xP{Uu01W~;Ml{so zQJe??s|tXAMo~Gim4n~Av?I-iSkbafj9(cZmb2c7AwFtw^=m<98~Ub^WD9|=l*P0m@cAZ6;Cvy z%d%xndZA}p53(5`OfSvP>j2wSO-(JTyUMO)4YD?RDysCfv?9?JkemW63tmI;IyAf@ zh33sCLcIu?5daL>Q$ueWloS7CH8I0x(AfYbLV9A9mbafDjAGYoK$`?p zQQ%PZg*XU`FN&S$I>XuSpEg3+nj=`NRfVJQaQ{8WwFMR(NO9nGLyzT2-+BW#EXpYu zXhUp~gi7=!QZRplwE7h45<1p5FCY_CF~mW07AQG>F$(=w1PbGw3cY8cLJ&_lw(m}{*vwF@;495=xK zKa7ReJL0MnNx1?Cpc9m~!SbVPsw_}LyNSdvfV6O48GEW{88{LFC;&3S+>~GofwWJ9 z-uL`xAzUhn?B8cUw&fejkCGPyq`y79X=6Xr>pWGZYINE)n~;he6y;U5qb@n|7IW+> zY&eZWoqBT=<0$j0m#LGrC>a!94NZTBxI|rbU!+irCyjmGD0Ag_q2v3TFSB#L*IRP$ zG_5QqH-E~_y)(2jc6{g4T$QQk9%LNE}1Cj={?m#&=GNO_)*Mb=`y}DYTnu^8%gZL4b45wZ>pRFDrdU zJVRZ89ipIdqfRnji7fO4x=-|_W+^6sxK&84Ah@cKutqq z24#NwP7a|@H%(b!Si2t}wg%VrF?oG`JwhkDI9ChopuhmS{5923d1z;F?{kGdh|Rtc zM!Dk9w}ATYJDBqDa&y<#U8{w`u4WnPkRignXe26kcUpe}TtO@H#}c8qY{40z8rfFx zuHhKWlQj*@Z*D$(6^^A?<{Gd+qQ)O!IKj<48Nu-)xVMC@=m!Y(EZFZ`a5=5Mge2Mf z;!>u%v|fqW*FuEp8QAu%uhP@g8=PB1jRPE4*PF_|9jGB~VX6WhFQ?4t1|~7dUhreQ zVnO|Ar|tXo02{;7@>^Re%b0c)-a-*~RI}MzdulFIe<#!+;o%Yr|v437@gzNxh)z-5+oe6Uc{N z8ExA2;J$RwksgRLzo*xC?ZUg|WH8$3-%@XJK(NqI?6tH~`D{lzQ4{zLD!wlJ)eHA+ zQGCFd<7v(jxW{W~YCEt&HFVm_keSBy4H6D-5=C!YwGp5OQsZ|EwfHehZ@d?~koj@DyL)vS@uH-Y4 zcB!?#jD`%$_5~qC^DQf^hDJu>Vq$Z5e$GMoe(rBjuS)aEm&vg_tCm?m)?`}uNcwoQ zK2pX`uVb-&K7z=5hh|Kt#UVhCgE@kHlQm%*x<}I+7d_f^sZ%(_0DxQH+%&kyYk*c= zt8S>$nl}K#=+EDq>EAG+@lZ@m?8nW^h^?x(B5UDzjGh$|Xi@AwmEFkhkDMpb#hYUbh+^xyVMrpw)QCi9O^Q0 zuj9Etw)vsQ@s+U#cY=o152}-blje&e+-YR5@qdt)NE(ZCU33LBpx1)hn8nTAy<@v& zV^fVs*S+JHy;~bF0#$u>Tv`rHIAkG}b&p3ulR5GRPX&N5w4d%S^`^Lc%`nbzc(fm6 zo}$cO?c`Dp~HR&$+=_J9_!3lcdUWl6xzNR5u78y#|IoGvuC#y0~Y1ae(Oia-tjqo zJWz}{zLBy1rlm#NU#w4c8+ZnfL%#Jf{zO$geX9OqW7?%=TvrRkBp~ncWAE>S7tP@m z+dB?5+V-UyTLgB03f!!H%VL~1`-7*?pzrtLV)DwP+e6^mTjCEBG-E^g)4Y?7GoH~i zd^&_(^0D~6`C_4#5;p6kTDsI+0um(a?Q_41o{j+o|tO{AJD|t5P=5- zTG8bDoMOO&C!f!Q(OC`M$kdIMJ11Pd)pk2_^?r7GsP>IxSc=a@)E#Ev#B!8BQq4Vj z9K8Jkah%<}#r(NlJtHFnNG3b$>w4{9syE=EYp<+)S6y3Xb_0-PbI(t8c8Pbc6#W!Y zADjf6Ad9lU%PgH?xD}&u2uW5FR@H9KX(DN07f{gr9^pK?G@Pt^8XY_YY|W|N74WV?6?wVJ!8-+b4l~30k*HsgLz4SmiYr@o#7DSx~!Y}V(rPp zeeFSsj4y^U8JU^o^8_9>YkG&u&yh19gs$<-HToBgYWkWVUUGZVm&f50Xufz8c5;&DA?eJhKAc|=LSpDT@MnWRj$)m z^1;Ik4v0(zP`S8BG~{3Umor^ASa3+HZK)?t~Cvw20+2u3cRG;1ypfE zef^@YfVGkbat9yuJ1oHI_f0ie^%SVefR6g7bg+V3;JfQvLCM zLFB)DEPZg>5~wpq-SPjv?LXf!Qa>>RoL4xvVi4ERYl@Z(FJX z1O?D^)vNpkC%)q7zc*oJ4IhZIGHHFl;3}RYBWBF&QM&ohLpe>F|C`#5x#QrHhT1Sw z|MNMaYR>=yf&y^M=+^G81dX2J-+TW(2s<$$0WI5TFb!Ai+%Q=U{pUAo)Sdxl3IzS| z8>n4rnj@w;oG!)U!=VlGVy(^#OTK*j@-0Mhk;uK=304yW1N20(|9FoHj1h3y1lD2u z>gHw$V9Z;C;8~9L`J5K`jsBgNtdoDXSO_qW8ZJgCc78TN;=q%Ffb%X#jJNlrCJqkH z4|_wyaU#JslymDYDXcm8dQcHF!9fV0^!m-8U-^4cW8|Cb>;1MOkYPGWU`NK5)}m$c`Yap~#*E&I|8M>vMWi6(gvWcGh!vmC8WV%; z^;1ltu5%cO2z{(uaX1(X*rm9y$cQk)@UNeT568eDe?*U=%!h%&A}NLuV}gO9;^~Y* zT#JDbH~#_SfA1T_)nhyw^$-6MO6pgdN@(Y9d)vp`&dLq_HxFxPLSYGn5CZ)d85xA& zRVP;;J8$^Y)yl_C#m>gV)(#=4W#{hTj97M7|q?D5ho7)X<6`G z*S}UNzEJK*rua~f`S;oz6Rn?G@NPb_sj6GjGpPUl6AQoJPtSkl3MDos8Iyciknx~^ zhaP#Cmw`q$_Bcmx8sb5q0pSIMjd?+%s8HjVmY;eL4Uh?O4o>Y}5QSG$DLO{#Llc zooRa{eyT91+mn;jdS7JF)}SX)XUG1teTr6Hu}>^#@s)ley;PaJtOw6q*_yWSXh9JDv^KC5+M0Cio_kWu`xM%EM2aJx1q!tWvH%#du!N$3tn!+uA*#*GJX6 zp1bWj<{xywZ9kvnfV|D;0CU?STk+!Pba{AO#>Y##jU)YeCWz`eaSLzxoICQXM6-57 zbkSI&H^)s^cu1FpF?jX}zTXX_Rehb~UiFjyv@^@M=??RDkmwNcsC^WdAH9T>{#;J; zK{i97f(>VDmcjyY*XZXS4lTCTSC5^UrPvfM$oVz|d|kz{vut!r&Uqgj6@9@RwpU%=DOuDBntiLV!_m}QajO^UZ;Jweed%0ks~d*s^`jR~rWC=H!omdW3R-*l zBshs?ssq&$7Otg+67qOzI-l-sKcNemyC{nxLM2Wju0f*gXk(u8J)rZB+&B)mGd&_? zY>9#4)0XLZw@BTuQjz)Uwh8f(Mp;_Sfr&-RWQXU>J#p`oJzSfxxj)FIaLu>be$(n8 zxo3%W=e6j(jJmDX7N-AX)b_jA7|l_XY*hQRy9ei7a)rc*2ldrT)XaUT#)W9Im~%H+ zBt3;xZr@ry%b7XA=I_8qDw31@iaD7HIjfYATdnrH%#mn58}HNj(Y|!zJWH<^M)cgU zw0@UK0ho_!UFH1xyEJ?P@y6+W!x)_Z0j>)uGGI!uc zk2F1VbDyOH76}1|0>c%$Zke)h?eExew~L5>joDz{{Dj*=nrDP(<^l55WG;# zg~+0M`z6*WbrcU9Z~Km99JAyjNp(#3<)+7Plr|@Dl&+rj%uc=9rSI*TU&eQK%aC1Z zOt$ch)f0L)f@|4IWr{kyI>9QV>w|NUM;W`eIoHYGaK4G<6KC`OC9L1Z?XT_#?rw5< zhDx``RtMOg=EU`B9BV`J65lL zyr{M4#8Ul@&9#6}EUkY?t_p-2r7HHdU)%5TG>D}Q?c`NvawV1whH{8 zggd02lY^rVLQF^)A*k|_gpQ9=@4`A?7( z6%zSRF>>p*FeU~DMrZ!%Q-|}U`S92O|G)ol?!fi)PJ<&I|AWON^)HJ@*~8rj*34~$ zh!Fg%9zyV%ovo9Vl1Bi-6n+*$h)IbFh)9SdBt?Y@%hiO`{x7xJIAhwkeK*?T2dKw=E+N<-|mE}zD&tn zb!B+_OOm+&$9*R%(+&rV6C2q~`M$wpO#N}1(7Tr}d5ophB`){{YD@_jXKb{7>z+$| zP?Ir9KyIv20s=LRdb)fiYA^Qj?dqu$hIq;QKk+g+M_M0uoGrVDA`L0pPk*#}+zt4? zetc5nZ}~DLw@)mEPuDB;YircD@x8R|u;zE($+z*57~m;HYqWNXNM(z^8*=<(w}&x) zuC*P>vW2_YM90ypob&cZCV7T2mYK%Ds%y*0k6b6u8>&r3I}bS9X-DiHIh7dGBUQ~h z$8H?LSas{w*T2FapEKt+K8NLTg{)p}ChJH151%nCCeV0Jf9QEN%_Xl1os&b7& zONxxiZISze$4*hC+l2X9JVNc`Z$A>}J7=5ijLg}{`(Bf*4qsYr*t+^;^Idl~=Im=1 zrnn%>Fsu;U9FJXr zqS)96;o`Q**i7+Ze)op?TYn8I-(nrU^E0a8#a_azQ+<}i5XDt9Y1p8K+F%E&N@pJviPYuj&aRq#s z{EJcyJcgKh7l`5%llO5nEWeJ7TsX;_r7$|b_Oyt4?K$CkY}iDDeIF+MlSL~#hRK4J zg_6F#dQ|$@D(mLdo6g~^fdr0jM>78d4vG@32yVG4Uh4A=IPTKpi%Y~jrF=zdS7kS@ z%L{FOjjoZHWkOt&9qrli+xIdd^rVQ>yUNMLN9o6m!5Bmlq`KLwcHj32BeSv_osFAM zwtb~^)V%sf=CaJn*ke=1u;cofBhRLz;>g6-B zpQiSFa4XMsHeQ@xRv;S1Q;Z24BHrf;b08WXy?k>~fH8uXD5)zyG=()h+vu`XjUx*N z`z?MRPS@URt8?`d1=6QWt7asYtf|^>0{fRIRx~$q+26ORtvI1jG!q2p#M?YpSP=So zhu>wrPsHx3Mc1^glfJ_B;z-c>Wzl}(Tp^=GFsMh48kUIv!I~Afpt3BIY3Q}+kxsmsqZ$;1_zosQcT-m+_T+y@MsQWUl4U z)7NTPr?=L~i8N?C%;&Worgliw3`~unkG!}(v8U66e^Me_jg%*sjV_K{_BRP&mf-rl znNY~`p*77z%#ft}cVTi<<056%o$|LQE(c@FK0e6cd~J~Qm_EQ{iHpcJ@I5V^ zZ^-ND%RDx8_maZyi(fXDd|@l(oJ~USRBgtR-q96|S^o2N?f69c0}R!5=GxCxtvYV6 z9;}cNpJxsG&i3WY0b4<8m3rq++An;z3EfFSuigMAMZaq{Sq0~5wL|>m3!E0JYN?jq zBYL}>_cLUYPcK>w*L_V}<-PLTf8ANWouOm1HE@so`&#Jr)m*&=q_Y~4$M^DBv8Qq~ zcV$@2t_QvPCRm^(FR~rjsf(peJ`>&jhViD*sF;B742ASwSgxVN;tiLMIJd=^pV1s5 z8R<9XHTAKUJa%RCF4%tX|B~gv@M3c6O?~dlJ*=65-m5OFY>|UjA9cP1htyuIE6$!(H8YiQWSydZSi2Bg^lf*{9UX{ zN=aT?I7~Eub49J&H*5RElEyQ}F26b=*+_zwaI2tGKbsG!nHt;e@%ir0i;A}vdx98$ z^RAM2Ce<7gE3?Mt%Z zoD5p?wCt1{l)ICB4;()|9M)X@JhYWddbULVB{?x*nKLdwkPZhwR52!x^zeCd^RIRh z{kC6jtc9;A2@^)#-=7%>ttoby{(j%cys_AK^y<9^S9AK%z-limXPB96zu~toYEGOm z(L0@%a@uh+tb{U<>+Zx3HjgoxDt^VCP`?`Qp! zF8`l1ACZ5#&i`JvuBWE-zfRUgMA6CmHLKe$2$6q2`@faCi;De6>MkrSAujo!GOev# z{{X7-;~x^~D1|tJcFG_R3Po;m>|$Pigsdnvc6y;6PO<-*YH=ijNA+;@p$`>RF`)8- z;U{xDa)P(-o7!`>J~V&SE+4w-3r(Qx+xCxF@>U*YOWBB1wu6gE3%J4o^#Uz+kBbu` zx8}Pw{>;tf5R5BBw%oeH;-Wf7cZn{}G<&gnhnRCK$q-l4H=QSn%~UejOH{QqPN2ig z%ItcK(y^R~CfmUrh%L;lpWkRforynV*WlW7C>F#OyhVGj8CT%#GbTSb{BT;KLWBUu zfN-{!fvytvTJ8ud@=^XEnRWF3NQgPnx5owGEVJ27W}PT=y0UQ-b%xdcF`Zt&ghoTVE=BeUL1AzR8|}gb=trFjxcFzvqvyB#LUi-cgMc zSe4>0z0g1qB?^t=H-<2!aAwU2n6sW$F;sdBx93^g;uSQ`ES$}yPX1N`u*!b zY$Isz%t7t$4Shbm^P3N)<%7RcN)@}BdqPY~x-VF2OQ{|KHwd(n1QcwJFl%!v8wG=7_JP>0ohPGczqio+eW`D;;I={> zrKFZdL;OZAN4+W%n04(Y(Nh#txVX3)m_9AFGjPFiD4s~o_3P3L>YHj)1yK*Cm4cuf z1~_p{bTqqqDt3WW5`)yN>lNlBuh;r7D!Yso!*=4dKq zo|FJ^*eFzquZ(H{eIyc`c^R_-ye~mfcqlQm(+6lE3p^$bq5TbZ*yf;?1Cdm{!}}gX zBoQ?a6$M2dSlWRYlld|ak4C#85L&)kj6-kY^D|F)TmR|nad)|jRj{-PxM5xt%{kRd z7`>i;(Zq{-6x=oJ`ciQSrNHM8cnSTw=~bZ8#4U+dk#PA4VnDmea?ra9h(AP0qOW6E z27d%gHri3)J3@tYpQ;dZFp^AmSU(O|lvHV7L7Vx1@B(Gd(31mW7^V8=%^OhTR-1nV z<_SpA0>Q?1h`-7EY#6ABphG1lA-N9HBzV@-Pj)nA$sTCj>83_dUl?x z(p+4tz$v#Mm~}VdyV+3ogAr^7AB`%32}v$vl4L}ll4yS5Y5{B|N!`_F5BY^1Krz$u z3rJ1f)-bcHnAk=22^fc?-O{2Gr&oG@_)kwgZ{xIh`RM)JcL)mQRTo#$E5hE@U=u z_WS}4e>`7v3AtHjfO!p~Oud8>?Z=GLA>cYzVGS!h0rgz7&pCW}!iwKL`@vB_B^u_9aZ1H0sbT>oG&(6{p(`GmYaRz2*P{(E zapXXB3ItUu5`6s0m!U6%wqWPk%7dmgq~ZM>j?R5uAj#rVLm=EKNoEnZi9;rQVt z=nR@Y)W!}X90^iPnIbRtw|U4b_4(y$pVRAfgZ{pz*@|`+ig}y3>TLlq!CM9IRiml;SDR zD(xt6(3taJev;A`n705?p?pHA#ZF$M<|V9g6QB(g^7WfniP-f^PKSbZlWsoHG~#?q zs6nLpVa(YY=g*S&Oqh%FfdJ8)vXbKbq0Fi=ULS1;y$vI&((y`w7bgY|q1iMu2|!vZ zdIhYp4~ZhJjZ-&Nj=;?Cf+#B35F|HhK@e2EWH6V`6#s0ON7h%aCgcl%>;r5nhM$8= zu+2Dx=JAJ;)Os&QlF6V6egvToY~<8b7-d2nD8C51y%h$Y1usJr3IS=9oK8CbeCMpZfZ z+(8Q_n-evU$4S7A=`zU8TA9nJp@*BA6bkZfI|HK&2(g#+7Ib3H>2q%Kzm_N~-9`$d z`ILUt($UlN>|9Ne{QF(@^Y11>ASywYB2$zCjSYMZJH!nOlyK6{59|2@TNNzmy3^Ry zQ-R0;F_`yD#W#jEFc;l)&jOlFcu=>1Fdi~PZspCT$v?n){ zntCdmIvf#8acKLUQ|P>-hJbR^!!XiyX_Z?AD<{^REU9C!=OV|$z%AEoQ71hJ88DKB z0enF%k!YSfxF3Sgvh)dldJYIk`-rcLNQFWKLr-2*EQYXPb--A`idGcD$_4{_MDuv4 zqtc&cw;3T&|80~^MGOY`t(WcrdsJ{|M7aYb=e5B6`d-2LBbRUu)(5PD9A0CAcur0T zUoeYc=D7xD`mD2QV8&C0@N;zmYoqMk+%>;IN`bLPD*sz&aTQ`B(fRWkRWSZzXY|FM z{y*Pj>1)uhB?fRK!LIQW| z7o0@zGOga9`w_5`>tE?70+c`3hqA$bH`=@wV~`6V)OK&ua!c?rG0EyN(<_T)`M`8% zKc%AdEmOgt1H&)HY5POkB6>c86!_%w>J~<+i#66oG@<;~Mpc)1As)hs3aX7vqDJX} z-TX~#YS6=5z<0JqYewzI@$#EkZA1z{ax258vNzJ1j^k$8x0cXJt7%5g$4;K9C8w7fsm`U4D-bKy*`weD8o ze|fxp<)jg;@121(#N7RDBn?@EXLtZMnM5D(utVw_O zw_k0Gi@fi;0Uuaz(vHkS(rKF)7;g;T_d0V*K>rs3CU~+Bco#onv<5CCsC>a-+4~U> zM^I9)zV$N18m^Ng_*duIMj*6MSUXWEFz`x24e(|GEI1boFkeRhIlCuSPZ+p6Nx~0&W(zO+_`2O!$sOf3qJ0sqY8$;+T4#ZB21Yd|(XXS88dy1uAOi#C1O)A(zU|FjST&Fu z#7y;);GL&us~IeGl8l9CkZPTG;=m1g=Fh=5>uC(Aq-W2bVG~e*<)e1+o5@KS^Kh)> zxy0;OL*N7kGGF6Zioujc(Bv6{W@Yf<%2Vaujlv-NmuIk|38{ICW%`(El!DMcicX{R zTpk_2I8Ft#FZebcy^Mqzw)J&A?opdIJsx zwO~r5!}kOZ&5tm-&5ZxU^|Ko%I>>lj_ng=xLygz#C-em2cY*8yRm#BcY}hL4Qc_>= zMP%!`8j_+6g1w8+;Sf%myBMbo7pdS;@}Qa4jVQUGedO{r?_wL41C3u08f=8Y{L`;Q zkFe;uK%!$&pU4WuLLtx@LGQER(>HsFfyunU#r3cWl?%kC_;v-t96ty1+TpALpE*3GIhs=iaS`M4#6&zp$Ki?d7KB|N> zMKkIV-;Fwu>N;sXO}KOl7EDmwWT<9aVK`8yl-J>#Q(wG&^nN{7Nki0gOs10KXIP4j zFan{FjcF01&0GrcB>Yw(KypArD))yYV%l<{%&;PJt`Q_CwR7VT9K8@-?>qw=g(AQJ zDx2Z`9GjOb7~|T>p}_U`W>KdRQ^7G18?`bqz@?)Nl6@ZiC-)waHR4~P*1K;HzAio6 zJZ8@-{k#{t2;!(k@o3LIGISDI&}&trBCFx)cm0`82ue%UaBz8E3RZPlph+pqTEC)6}XebbGFKFPB0b#@od^*bDm|M3jRp+`!ZwqB&PY^+~b zgvFb9a&+(pO{*KfGD4%L?FDHOB+=Nn!1gF($D=&jfPo?6mj4WpllvXRR5Va3#RaG) z_AiHpSR!M&zYKRMU|LhJYao%jHQT}F@b%~q`U<{w(_AQoVgyX@b8ORtZ6kJJG+jH5 zm`QF{SA9*#=Dqir0|)@UWeu-K>;fIVk0b+QoCZ4+ILX99|4Sg*hLx9_^H+7A92?1Lro zL6~%U>C&b11Yk0)Dlw%S_*AMoNM!pGHTBoBVtgkU#Rf%=D(-kqrL~8*O)aGILC&q9 z3f~_VTOKKg6$MQEYni&12Wn;H1gy6~@Kv|E|9G4rI^A*pNubG{RAolV6sbuZ@=9^?Ac~r< zWOaWLuvggJM7(gGL2}q#SdE?w~1SnkyYfQLQzQ>}x_WvIeo zg!B_*<31=iSY;iC7+S|EnN{D_xbm)O z7!*L5589%Ns=9X}lGpXdr3Wxc3@WeCp{cNTwUFkRyn{?Oh|^+q`pr&xMQ;W#ZBiDy zCQUK8;f0+8kLg#y zgD-3cpF~p?B+mBh%1^Fwe5*cz)BGKko}l;9zEg$&mh9nsJvdXhAI5HRN-Y_x1kNuV z(CV!Px10ssqt8AepiPJLOc&W6^X{}*CVyipG!u@c{J{Sr1^X{w0KQ$Q-TMnJu0E{tSmaZjJC1xip#Ou@q1y0ZEO=(eMn#p$*S*pd+S5i~m?!hq z<0PeW(Iig3s|2o$sO|q!3XpK}X69;f4UXkR=bP1ioLwl; zxCCED#l!$Y>I8XHoyH78*0CCZ^@re`T?}B5Dd1W^f2x})gkWYaHV6XZoC`Mdofys& z)pL;5P_SztWWl9Z@)G#72Lrwn@upf z#G$}Z{4UHZhN34j6Y%2Q(6i&mQ`w3^!)C1@#{h6k{S0zwK&xAikJ$te9r-Ixnea_t zyFy{WnFvjdk&%%@rj(A5Um~LbvT1?q1w>xNi(Zwsgl{l`r#FrBJ%}gJRs-K(vg)zX zOStm|zv4lXG{t;f^EYTcA#PuyphyLT7%ax% z6DYDFfX9_U$axGoJ{T!5$p%qngDe`XW2MUPcR~)>?z5I!`o8*g3gi76moHZP@>A?e zgx}LtC;ecC4q)?xXGv04Q26dSU zfdC{1uO_GjA%UrtX2}H@;^^LF8_as(HL6q?g3?mQ73AMsSn}W!y$p{!h4iahuCAod?8&N_`d>59D&a+f@ z%^Y9+r{=e#3s_FpN*@cDHW3Ts{_ul#K^c27wVL9qeEc@hTje4sCz~z z6gx{rF06@QlGHVRKD5Gw$Xe4Dih9~Lw2q||b<<4@Fc&sxA4I%^A%L$75yjTVqm9OV zw;|I;qOC)FLyq>jtJ`4EN9G9>@5V_PmmRK~eI3lyQDm@!U4s8?PB!Ci{NDD!sz*u{ zN+S-R(vX_dDOZO!lzQ4P?*;R=NMr5Kd5 z3H#usV|O-eWvj%WvJJYlcZ>=5U)g~4Y1O#p&bQz6YxI)*MlckN3Ucd5sjP)9P?9W> z(u*QR0JL%T)fIo)k_9b_Wa;P+&b+^V^DbRCXKGdAVF#s`fZcFOL`v3~ED7n%KpH10 z1t3=4h8HJXTvM5qV_+CUYY52+4dK(9NSIzvMMmZcPJG=6a$SXscKmEZnb5gJ!NYj>6qz_M+Jw`%(=Hs&umeA(XZYnjX z>sdmW-5>~KPi)Toa#nt`ktgVUubPA6H>k--BXwEy&NFw1n%CQ3tGBBZf!-j!k~Ac< z1lLjmp~3+TkK%hV7QH^ud8m7O3OPIa-O3&%>>>b2<6Z6*i6C{xE2Vd_q7BGL!7iT- zVhL!xgwJN~+3+JoqZ?VjLwlzGrrBN!os>6Q0_1O3O_3MHUE|KJCNj4}ClNgFZJuBM z)2kf7&thect|*KQh1Cy%fb}*QFXc%kpT=Hij-H<sG!=t5SYIP#08+T$_X0gaf7>3mynIyu(DIq7Js z`OsM5`7rw@79nfPDF9+J z0Y)d-gikRjl^*o}kb;kR3<#?%CMp*#aLL* zS?B=34Z3Sq^zgK}t7N2z1|1J*(^tZsE;QOD4|M9(RUs4{%H+S;5(|SK_44|d1@bMC zb9k3-r>!vda>~7Z9R0zL4S z7l22|?R8=CM$+C*l{Tozn(yuLy$#@J`4q0CVb&9nY12kSzN5g*E?^@wAZ(Es=bw+F zcp#kjm99p$>k8C619b1b?@h2QT;eM3vb*OX)#F0GM*^G&k^kcDU+XI!LBCD#DB+|BV~G@k?8SZ<0;&hy8XB69Fn#u}aMu8xw%?x*kzp?ep_CQ0StvNP=!}_R z!YEr4KeQIAP;g&-VHtd5*J-VUlLl@xJgAP~pYG5{&+rJL+_@z)_2&!hrlU^LM!n-S zqfjf`4L$k=&?*v|CI(V!&%{j%y%EDVDX)WJ54Lp!f>Xo5@+_Bs(@;13W+1+c&SjnN~5 zC}D!3b9X%Nzul5HzT$kCQUHj>=xgI)no*FWiMh{8LT3SPsQ!vF>TA(;y%_SR@X-Fb z+#AS|r^#_~j&Nh(tQ!B76y*bQVQ2_&8&yt1)r~%<8e!!1k8e1p7^;SHr>bp{d$xZT z1@mdtKAcr(2X(`6I21wWz6U85y4`)OA7Rc6wJ!QlhPT~V^@K}+Mmdr60URbl8&%Jt zDZ55l8wmstfBT9s3bb!bBYF#td0aj zHnj;6D;tK*G zbgOk8#?ZfeERTl%!4Su(3;I0nD$8(R$**dJgHy(U!Vc`5(V55X&IT{@GHn7tdi}`D-_j$bH6M&*RHFUje$81 zE->Vn&iHHKS}5W$#J6@>og8LDE)oE%6*jCAKUfDkx0>#cjWA!!;2DRb%57pHf|`u% z8O#VTe!zDl6*{67ZkKI)Jy(aBE2?%&;n7Jz^hD?Wpite+yASEed~rb--$EUW-~ogv z2?$qR&rvLXsqcVs!`!Ub-^_#wxFvczsJ*7~O=V<31HK~rrJ4g>s4X5s--->|Gdw)& zkV`b-&_KHj%!&cqZEzgv^y81BI^tV{%(Viqg#0R?>y6O&Ng_Gf{i58!4r=SB3bMw9 zovZ1INDFVY0u_2^54nrGS|Lm>m?FEiUqg%-*WP%+ZHE5?R!k5W+|^%ZhC$q`Q1-yx zs7R*o3V`OSTrmMXkci%bh0%1T6Ug&nX#=|N&y_c;tc%60DsO%Rs)aRp1*#c5)!+tz z?daC(Ff>fIKUQ6g7zMM6es|2?%{X#aLRta7=f-d)csIoLk@5b2@5je7U1bsQ$S^O{ zVJ^A%(A=r_mZPKNO(H8RD@8>`^jwFze&P!vD*VNJ222MW_(8xjV_5?L_(l~DR69_B zIG$spYCneVhADLziY14-N?9TCY!yBSTwSgTNdbwibxKEsdZ?~|^l8|t#YwnU?%nt&z zk9K_w$~yq~z>|a6DJmDvf&#;ESgOTvy8|;fTOr zBTrMF^cqP~^vHG8DssRfMxTV&{o`DWpjOm5guMvRp&2+(p7x_8zr0r}XKF+_%VaV! z5z`6HLl=FWA?m`a*}$Uqr$iPwVpck$ zFA2s!kmY;Mp$9vS_2>JC4csq=CX@lh`eyOi)hMNMyhcJ!J!6ZDCt({s>O^p-5Qh zA|$zoKJ}STnCkf)R`mN*$Tex}BgajHRA_elc__hTh_+h%3YpaVTxm|k&wc+Hus3-WjU+y@3d!cuR{tZ0&LtZRn(kM$>y_z8{nB8`L9!8%FPiZ~A-elWjwxBci zDYi@~+z%P}_d+cC2g>-Ie@NjG|yN+0_2z^xnUB8S8QV{fdHzFCPvHCEQXa<{>PZ zT#+YmVh=3S6yhwz{m$R{u`&3H6Tf39F`ct`QF19W#A=)O^~ZHF4aMz{v)^fGOA9G>sgFMts%yU8Nf0$sA#= zYY8gx&JutD#kOzcMV{sH<=~it)l)zmb&w_+SYj)sD|ZOwI^i1+nTzx;68Kj^2JcNYnzt;V6FbHJ)18aU{QWV zJm_+mgG7qirJwV|@(9YN2Pq4qqBQH0Yiekuxhsb0)#uAbw_7n#m)!&kDgqbwhHE2s z{bl;u&srGA39fV5_MUPDO)&kRFhI|nE#rJDLS5Zu5G}}MjaUq`tZXcRH3Yu6@~eHl zsT*44%@FB$Dv5T=k(V}0U;?1@NeH3Nhq0=ao8MI^0oL!sJL!A`-?<}E``3^ukeHQJ zM&0)wWXkr^ZoV;_SAmZ^84Fc7$CM+;fG)(e-6+Jzhe($_=$+s-%z%kfT?aC1!IVrB z6UNhnJu+K+Drt7%eGmLy!O==YQTekAUU9Z9Igw~Ml|%f32tj_VYc;J=f!^CkQOfqt zCL9cCk>z_Y+F*k|kf5}csjVUK_pr3(`FUIr<)LaN;Jy@Sn6>JYGv5@$Eq5)p^|o|x z>5M2DINrGypL_D1e7Nq;%v-(y)-Pae$O!?Zg?+{Fz}AS7v_%;Ps-RK3G(E~IJea1$Olg%mJu5qjFN|davC=Zyx?AdL_n^T>EpULpA`m}^OW?tIdJpmYP})oiM~5&s zXUPgd4`{$LeWz@{$Of-$!5!NC-X>FLiJl?LAH z)=2>qzjc)l=Ji>R6;lpm&Jky~-^ebOERrxuU73KL91H(%3?|+$~^X3 z5hf5^$hGECsL*{|fPUtXf$yFkbvIXR&Ez8ztgwqbkq4R@K*^lgHuYN(BDRvV%Aoc# zdrKc=+cK;v4ql$gYVObwD%c0lBOyIcP}yG`+JE?f{(flxYc`|*=TLw@eMbLED8N5C zib7DKOf^z)DQvG36|NBz?yK{`|k6;)@BCbEe6%_xly8nk^ zjQ!t!`rnx2x&Cn7fBNhH+8h7J{r~ ziVE7%+73dO2e^Vpd@0SBbAq(i578kSZMwuh?kN7Zlbe@;K4!Y@!>t{G^^co{g27n*Sy{C{TBRhGqQvxWe=o!wp&cP>{yxP8kw!s|jhb|hTCxB5~xJnOXt zEzCjwfSfsV-5=eX361yB`_UlRj2c(E6*TFf)4E0yN~!`oti=l?o$li|6HC5MKh23F z8~(EIE0ihDa}xDk#skl8-0flm#@!R^=11y;FKUwgCgGt^K_l}DR^g8g zn79u72lq8bXwyk%x69#^{53k?BSo$Wbu?$1l^`Qrix>RpQhak7>Re+74^yADMOXV- z(uX-T?%s+l2N`lwAz@8k+0D7$jc)Ma&S@uD}W8Z{-7Z9c3cTZq@G#L;T zR7ZKS_t{)-u0&jF{uJ?%`o7r1^a$kx2`2~z*Iuzy^Qg;~4$;N{htGWGRybCtwg-rxk)X7A6Ye8lD5v%u#CRS?-`7v;Pwx{7_+@!OEa=(p%qFiStMf*tOZ$ z6=hkcjAe(QpHHK+N(I@w57^;$t7KFtEytfYfIK~szeql%@r^^Z>V-CQ=VWuW^gEJz zfcweS?@w1c1OKjiT685X)B<5Cu?&mwg9imEVHVatv0;f#O61$I;fQx zWs^EtT(s^1V9w+>;L#xOWvMDFZ1SIYS^YkazATrze7KJKeq%mI2h@Inop>~Z@677# zri+F8O~H#4rX>%0w~*3CoJY4PeoCGduZ;8Bb>W>nS|Wo;ZZO9oya()}!q+^h#L7^$ zS3WF8IJz{LuPMqzkKq+bycaltAQfB^Z6Em`3uYidJ#gW@+*L zPv#k9*>yIB5 zw8nOTuAXA|aKwYQzhop;s;js@d1EqCwZ{b{TW+gplvjRV&%>iV>_3r$<|eg3uPm{< zF{v?riXL<`Yn<}fcD&v1wev{>UoiUCltowYl&~Tur7PL13clQFs6`ReAIyJe^moGn^JB%#o!o1otaEZQbn?i-_@pK(vH+NpvFXZy8~>-R6tr9K%&v!^ccVPH8W zjS|_%#866}VJz~~wt9G&Q9?ZmoO5o=O=u_^PK87Pw0lV*%~=oM&B&s|&5)NEji>SU zCPN%oXLj(kfT#7KaZD31rT18se0_wSXY_*T8cN>#V;mma2Ah)N!rwj9+ZO9mK^%1C zbZN1!%0@=FX`U^sw>r2P#9CN40DBIQ^#T*R3q7u`FSA0tw9)E&N0@HLR^63*U#huH z4W*AEeRidwyJaH7d)4z-u2$!RmmxM_PKhcZ41bdsh<3npIi`qRG#lp0w(bFbs`8xW z-zJkK`!e;qXd(qdHmCLbOA`|e5XW=N`Iwb+67$}iy6948WEJT@nGnDU-L|O<#Wc5} zVO%Rd+Pmw1_s}F7e3^1pVE16{=91RH>+mtt%(1k z?RxF&?b;(+)!@=HH1dsL6_X2j2&WC_rxXUCw2o^O*(yy^GKaRvgpQmGiQ_}GE z@9pn6<}&8W3QdlT_n)8$k;TVKsgw)S6arLj<`o=N7187=C=Cj6k$LJCp;&_mt`C3e zSsaBaa`O$o9VZQ_eSj_{s;vD5n)=5;y?;kjOn(aq`X?OqC$#$C;3%fgys7^NNd03l z)_)JAxc)iYZ%xC}eq|8;{rr=-$TyZt^W2kJHXCq0xLpnPG%GhNY+rSx3X59GmJDQY ze7k_9O^m0CjY8#FNby2WQb&T-!|d{(Ydon{v#!&G>L&TVYv9=mwM)VX93HPx))@Ya zmpDnF^5gb65_te1&UAhPPMLY(#p&JH@i<~f4*vkARS_}KZtvFqZVTP%-R>e&$>H6p zp$nLLdm*#Ep_f{6B zgH7`Zg=(IZhzXdaZ8f>RGvhE1WEnR(BrX&zb+HsWc|y`!g&v@6x=qduh65~D82X~- zu&MRpn2c_l=F8Dea!8SU+5kR7s!y$D+@1|d(3j(>uwnNJ_)dUsm&jL3dT!|>zR-Onq!qY64-s8fm9%Y26U>v^XcG|CJ=p4) zVMf9>W0Emps2M}&`Od)lh+-u@=5?O?LT&yK_}iawyr3kyz;wsUrnd zz%;5PV2ly#iwh+=y$GWPsVA9M(OE{pfmTxCN^qmeyOgfHb(F258ENW8op6`0x&pV1 z^Bf#=jxiqcp5A&7B!LD*AP~1!BY1 zHpLAmgcn&XExnyh|jCWjLd zbi$h{77QSQ_&QgV9+8P|PqO_r4aiW48C*lDSdV6tNVG2rJe?oD$tGPeIQ;3WxTo^F zy)+!REPm)#+_=Dc!O&*E1AXVPIDewhTZmAKYv`DA3|Di)-BQ|6XB8g55r;fW(UcF% z)sSe?{sKJ(aNvQfnaFB;_3O zO5o+7N(gX|D0LtiE$7{+l@Dg*FEEVr+##^8({rX_YO6x363c8wE9xn*2c@vJfj z2$BuCIsM>+lcW??K<$eng+Zwua zGMJUV!Lw$u!92Cz#15TYXBRb%vmGGkA?5%J@Oed?gM?BzflJS9KLDT*))~+Pctm_d zTSU%0yj$c{1qFMX;~>vGxdD?*B?xP`%fW>%-R%3b!7>cBjR=A@jD6yCRx$`N(O6Z$ zQF)DSX_g_?(pOd89t+1?bbc!MO#%&OcY>_WNkpw9!K8#uC%{QWEvtMj&{WXF7OR0> z!F65lD7Ldj^_aIq7SAbN0XS|QRh<^8-2o&4AZtn70S+;{7W{oet}A)q@DEJFnk3xx zZo4)WVo<|sw_c0ERCWp|#pj$&@@vAVSn>#U7UbNyM(y~Cq^}mAJ zR64pJ91Ae$EN{`y3?mT?KjQ2gGvx8E0oZZBlZ1Su*7#C+^yz%9)59fYdM%F&b}MDq zsgxeUg|qkl=bS)57&Ky3OOj5u(j?DrFUeZ#W+TgQH~I0^@1lzWFGyI#;lB^IkfG~! zB!-7!gm0kM@iU5vH9p~-sHvad!Yru zAH9J1@*SWy66QyOfWW%LknlB8$O|C#Dl+I27^NPLBqLgXwtfv7a+k1GD`eybFpQix zfE5Matp}4Uo2v!*nku#Ik%;BwYWg!@)bM6>+Ef7hkUKkgZoMeCcWY|(Z}L&EfMm^O zMG@hDL8zH3XF>uj)d%w1E)%Z_Nfya;gl19Oos!BZB-3RxYBVnX-lWR(W5r*r!X!f2 zz5hTZoBVp`XbR>=S61a+=YUCR1$0n7mZl$xcese&yrnX-MPb~sE!EaNRk40lsdzDK z(f+blsd{Q`&Dfz@jf=fj*s$?oo^WnoVfj8mga1AU482PZga1A=1%CgLGv4k)T`}r8 z9b2fv#iq5lzw-nujpw%0Nl7u<^A|YlAA=kJ0cZW0|HH)iXZj)&9vtsXfaA)s%_gX?B&mpQrjhRD;<(9J@7AF3w+X#(3e zg7G2Z5|Q2+7T&xR8>Mb~bH0ESazrhP+NrD@Xuy5X@0SHhop5Zt3?OYD&rn%daLJBT zN0*G~cVAAYr2$#QZfdf4nu>FpRN7c4>vF^hdCicqD^%K5%Mn{S8p#(9c>~EEIJ>b1 z22`kU8e{LJfTPGS0>(qkmHu%zS3?J=mQ5#m!BmK@MB-4sU>1xizuG7D@V`6(brCD6PgOn@CF34z>h3~dLSsVXach)`d{~Gyi%t>)-IK&6kIWWs#KB9%BN_H*nA~6-vv% z%ieMsWY>9#9 zY$5a;)T}-W6p;o~fW!A552814)>R{(+*!E;>+_jG5$!Q(=IiM)-;yRB$f*z zKSQ`U0>gWmd5MOF+nywK0D7G{4HCHvg9Mw&s%Jnap9L;|VZ}0iu4S^SrR#j|Qpudf z>s`t`dJ&T^s3QI?qX9-9h0P)L;CgFai=BIo`l#HoLd zVUrBTsj=3Blc&q+4-1pEx7F&5t$w>Q{a^(=U&4eNyMSuWtG4 zh55oTw-xnA?|`QtO5dChid5&@*Nj{I4v-aN2-0WOKi}i$8Z4+;-E)>pXSd_gGoL>- zvn>a3GgR)Ym*QnCp8WQc5tfP(Bs@{4g^Wz1Fkw8YPNb+tz%LN0nLJaNy))H;m(RGG z(RP*`QgfJ>U#RzFqolL}_>)k+tVbZGt<^z0*@T{%!JRYFJxE)XoS=Bl@@xD*Pm{Pm zOq9;zq)!uh#;60R71dXd0+31Jq%`=yWoreOu%2i`pSZ$*{pA2w~a`83AS7bwr_4yv1HMWXNyH z5Nbu>!H9zmYmOG7Z_JdoDhS2P{*e@urnfgI^B(P9cY7x+<%7I%@}v9q_2GR<`GArj1l~iIlwMyUNxJ~Fu2H7uuBE06UMkIce&s8GYUw#k<7r0XJyYh0Q zAiT7b@YFU>i3x$O?2yJFq74-b>Iln%jMqm=Sgrl-&>WUag+rKf#jq;1 z!x%;*{(?amS)1kFt5Stgb)}Cjp~q;!Hxtz0q>y+tZ+~K2=se0ps$uIKsER@3r`n7W zpoL$Ko7>3acy(_@O++x7^-P(szWH|3bM_pbkL2q}A2pr<%SZ31=`oADc1XZBVL#1| zDP)O86L5|WCW464Z5C6gV7Upi3|49K2*NwV z_D;;ZD2T;h^`%x%#%O8!&CF-bcCGtTnCT9;W3|jXIOpef7E3876r<7xi0vWca8zvh zD)r~c?U#6r$36tmFdMKCc$N|gWxRnBP5%pgm#&6y5;hzbR9?g3sc3uoeyC|S5u>r2 z=B;z;5-3;Db_Z-?5Hw(GX%u^|rg0<+<@o8cLs-TVOfM-b6r{L)y$$_5*XGb7@_+zqJ^=f}Af4GMX=a70Zv| z+LV2i7*-+J-`oWY3J^ly`GQA05piCQ6A_1KvjMVQjcVo}HqeCJkyzNVso15tfQ#|H z);B69DQOG>jQ6NOAB|uKgitl5n)7>qB`OB23z*;}FH3Eup~yx65W=E{w}kv1WJ>Mx z4(_xU(5!p4T8-zfu5Hj_4krUEuahlupmiRXFZ9_r;eg}=l}r4kN;%IpBIS1RdFDj~ zkG1Y$<#n)dW7{CVjDECJ*8McWMU$0MQ2rP{)aVil-Anz-sKmh@J@*`TkJ_Z!5Uux9 z(}Wc|tcgZ|Nf4m%U1dlZAr?JWTnppTw53O?75fW9$@+-u3MhzJDvL3RZGg|9Tjirq zDuH5}i6%67j9*#ztRRYtQ>lNB13&&DLLtj*41IO51{J(P``4^5)f!b)KzbH7H%Ef5 zY0qFFzawmbbqCt(hRL;^3{o8Iy)~FszWCdS5Ta@$?a`iD^#=p77Sk)zm@x^F8@WhK z%LmIp=YP`*oVscerv=?NoDfRDeq^!yx&Sd>u~Kn zWjHZE8xyk%3G^E?&h{5haYVR2z7_0rg9avhvFfLS#T@{C#kPCus|Rghi}iZrG@C&a zbE?Pi`!nGc{U-n5Z0^cFpW}puO>zjTd`oABv^VJJ?@r_paLwCk@;o*32k4>{u-!fu zvfa7S*C_kj4*Q5Yuz(9E2UfM|&%blwQ_Ac4r5DxbYerHR+RX7W^(a8~({zpi^4Oa) z7=(CM3DN2lTLMUdGa}rYr^P@b(x<#5h)Gp#Bt)ki6|SLc|Ud`e45*V+1UhpZu#8Z2CpW%&}G$f>{t-zO7h|Ti1Vt-r{&eZ z>A?AO0qrbDF1WD6tB5!=;a-~J5G~$wmcT zD0HHf!15Wao_;Z9{+;kTw!n4uGu?C|9dfOOe~Z5Mq<%Zik%P4sXpSm0gEEJvf9_a& zWaz>xSVH}!faUNvZS;HIUZJgBYTPA3UKV*+HZVyox6|?npApl>zAy5`?Wtai5}k7J zi;M_LH?7|wS+@u$kJzf-m5*0Ek>k~h--V+GFmFf1h~B{HVs3ITw9#Sn`&vYO_dRPB zSXH;fO16_8{6=hIc-e2t{gDY-Ai_J&f^hR*_d(9o;*Uu_!8evO{Vc3qDC<(QObP(H z`lwm=q$amY5BmVra)R3 z{nywjwX!S;Oj8;{hht@=xvn}ln!hFDlY1d^;UNUuKN%lBFvweb(NC+S_B(zvEKLH1 zyTKA)F~D6YQExw0^A>#1Yr(vncDBNI_!O(sN zp+_5Agge2r^NPq@D_pOm>Q0Q+wwDxc_DGfm$LB5YmO*X+UT|8z;AQhhB|NI1fj$L% z|D0S0tP++3k#NjeDXn^Br0q+m@WeYUJM(MeO$RXuI(YqRQ-i1dWT3fw^WDD|9%r2; z?o2eIw&%zKV&!rb{QJ+O%l?E_ERLtmMB|?I%(`^!ahJ+9bk37_c%Tgk-b_eXPBb-> zZ9DZqQrr_GJDXIrd@P@*c5$8E%-b%~sOajo+R+S2Y`!NTah`8`&V9O<=H3SvVRIc~ zJ>q~$J9I#F96(O2W-*yE^4tZ#Vv3Nb3EEgbL-c}|C0)pU5})mN@l-{9f3{FpSIMfM z-j_1W1?$9|YT!~3$X=+T{pYtutdt+G@dhHpw0RAlo8Ej&`@OuXl99$Yn^)t&q@6xm|H6HER+` zF(FK3Lj$Og^uqg5EtvG9yf+^~7;ekSyn_jpU8}mDrgOf>!&La>qju?jK^pwBtD&A! zF$8Oh8$0(jHI-lkG08+ie+&`b#EhJ!jzExp?(ymT-y=hO*YLky!jH(SPjyF4LWYQW zb%fhGj{nYCovAU@D+!fTfTP^!sIjl?H%&`l;+8P5qFjPx_cQp{xmWszQa@1A(RN_xVI`}c5f)Jv3PY>B>A;3md= z0TO9VDQ?gWzH1o4uArDW&h-??l(PiP)iwu`0XYm-@6Qpra?mO&RM~h>X*Cs1iP{Ve zzCQVAkbO7GVIb#|m|eda*bVEJ=auVhq)vSkif{bSae7W_HGRT)h($|zCy*A zWvuFd8E5l;P0@o1=Aqr)A##8{G7AxALzWgBia zQ020DA$!3YmBiYZfp1ZzOKY24o|UWQDrF0WOQhO(R8~DA=n_jd1zFX$)cLgNio>Yz zW<54k=8W<)+7}+AmU~>8PEX9IsqeCf$D?6#Qty%oUNhbJ77!dB#)Clg;WC}n&6%0U zR3z%DLg%SM?m34c{V3_28%lJJv7Lw$m;=KHH0cD9r8^JY-bEX+-e%ct16-CsbJ49y zP3s_r5}*E+`$uvuZ3PPY^LnFo(r9l_7buDnKdqiicRd!>fR0TwS$r>fm{zjP&md&* z4j8f2`1f(}^VYM;Vrs#moR$mMJoWjNLBxH(NnppDIVkF0G5k?PGaeR(nCzEXUYs{k zv~|l{i&m4jz=0ONdu39hw;Pi1cP_?^^vI2yd}q*n&LyP)@c^XzF z@=p1p6_B9G&$%RVg`p}~em|hinFe&gX;tJl*RdX&jM5cxOyKB=^N)zkVGRfTrr$q< zhNEcE8UVRTGkIcJ1Zeu0RVJI8?HM|wn%_@3vmH2(sEpcy?W=mz6}{Gzh2z3IvIob> zh<#o$VHo#o@@Sjdy6H=ItspyF3Bt*ua6IsZnDtJdhEiDOA$N$RMjkpB5GNRT!N_{T zqvn5sEFi>=XK~s5{^<{TpI8M*%EU$k1fR_6`hC-HbwB(&yNr@S3$m*KH zPCghYw;e)op&pO8Ks^=#PijVLV#T&2BRqwhvM?{UkN1$-2+QD+H%>rRcGp$Ld`P@c zlYp69M|a{=NyvTMb_6RPi9Kht7jNe8UdE)e`_(PaLfB#E#8e|}7xWvGiC!SNrY`-| zHQv#kr(8;RWlW7peI|?)G`dlMA-;~W@|YEcYy5T+ig^o@(|fO}cas5HS*|FWT?!$L_?m zkFwqxwp^SHdd^G(R&4Dlim%o(-1r4bqaYeL4mHqnR%eQf(n35*{hnR;fcz=`*Pk{! zqgLM7mn`^%n;(!ghP3y8fv5gbIn2cT=R^snzs+L!CtUR><@7($y!=;wiTTgg&R>g% zS^jvyzr-)Gf6i_D$DT_$&Qk1(__=3?u(gaznAb`EjdU5_84gu_(*-`h9AZtk3QJ9IZ*n*dThq}_>MP;U-pQafJsp*ugCT}5UP zKi;Z3TxE8wt@zvC4|s6ApU!b{+Pq%g$HrFDoKK8OfP#@oG&|q7@R^tl9nqWA&BoD^ zdPi)N<`5K)$XYglyj)uQvm?)kk8L!=On=SQ*_^QN5qGtZm7F&4P8lj;I4T*r zPR%ZWEuVs{ek4wdY9yF6x~t77){fxd7}3oQua(Ex#>`0jd^Hm01<~s3w@mu-vwzBS z{q)+epFjYCH|_=iMS;2!)kjXiX3X1hes=O~G`3EyLO0<2J654MZ zj1TgaEQZTVCH&}Ci-+&enwoWC2omNl?v#e+$}flRRq=QD9N8;qYWPxvr48fJi+n_H zlchsgsTjH@q4j)F#1<`TZL@+b)#3H@t}8w(1P~NCzRjMBU;u_9;fMil)^3;FX10f7 zoDd525T&{f4x5W1t0b<3KsFADa*PWMg?17v%`nAAfaSF+-kw-?n%1hkBV2@uWDn7Wu1nS(rtY!|K zBUAOSj4{m<5AHawx?QD+1D9F6=GeiQQ3t+){0W=Psa;vkq#R?iA{z>i zPr=I1=83kSF~^OZVA*GQr=8ZCaydR3fb}2Q6pz1~^-eRt$cb;i4V2o-gqv7*caiC_ zbe`db^5SOEa11dQx3BL>I4I>VyNeV^F*eRRc}y`YT#+@@rC{EE+|lz0?~-6a)qHhT zv9EDdzmbcMxf_xwY>kVyZQ6_U3Tw|6X4*|e6ZBWu>zXy(z^5dwyyb9nH8-spyW(Gk zG$kY;_K9SWTf*rU-NfbLecWbPy@d37BdUmgeWt1)e7ei^5!=$fJvR2d8%O}+^&n=< zo>{J0><=H?IzG^SGJri13@o0dKj)j(&n`EVres`a;0L_@xNVz`_+fkG>NETGgp>Du zPSgNeLYj87<|}z>)P2{l>olCiFjq+!`sDiWsoT`D#EQmy0OfprDefG%bk5?*FmiMH z5^f`Cc?sphaU4q8dHZj&XyJV!$I<6w)P{H6YIcmPlo5mK%qe6yj35uVas6=EuIWfE zbxf%16#H4J(iGcFbI{o$F$TPEH_DE`Xvh*HD6a;~3fE)c9vvPM`+1;7rm@!3cm-i_ z)S(&G#T2kkDIPfvtz`Cei#@Pv_6{P}|8z zsuypu*M3v))=z0yF0UYy`_t8kD;S`bq&+#??R7Zo#*jH;laqv_F{^fZDu^!r>10qlkj?TCYsDB zNE`JrUIF=4iOV|WVI@qn6-c@Y$XkFH!o*IUx_mXjMS_l*{`T1`AnVqVMLzu8DpW$1 zyWk#ty`XDt7|SL2)2Hy2sU845zIA5i-tWhj5!~Ph55QOv98LOK)wNawI$FP15E}2t zpvr8l8B_l zZ=yQ;Oy3BO_?la=r2yL0HJ;tS^`` z`)hhYiyEG#Hu^F#RV}!|T;t42lRr=PNwyfb627XL+zcAT=X-Sosez0E| z7Rq>M%(No0L^kB1aF!M~JZL`|jM(b@EsSJM2f9|(=73qz@@$JKK^BR+dw?PajnIAT z+8F_edsA|4J}Ua7?&%nL>Olu>G}WEr_y`*uNK9h^ z#_1e+58f7znub9FoQ-VDth9Z16QXeph?e_L9)F{hLa)0eo$pYq|w`Fv8xw zZ$_)&A*#;tB_w+}yu09RX6>2BNKl+@@f5j;{53;7_3cap4`ymUPXfE#(u042s{dL+ z{^=t8L+Z`M{HJ&BpHTJxSV{h0QT5-|P=Bpt`q!1@|HQF8K$(VlFCF;o$YNr;EpTAwhu$2Wc{G!{3I-QFVkS}?s|e5 zI;7?@6!lXIp3k?C4N?4tXLxCMZqK(|9x4R`ZJSmKHPAeqKYHF4Ha?nO9(p!9A1Oi? zPTqM#?_ab>7NyFgK=+L$X}vxs{V5A_Qsu5GSZSP59lgam4C!Rg3tx zt9FZ%DV2qjdOR6cp(aQ&#qHvAb|gA=_icaekYsXoR%IYy1wnK5O0Vol+}A$rpYGSmfRi|wZPu!&5YZar*V!ve#y*d>b2=cBcc-tri9~M(#H-YNW*>8b=bN`xSyBUslifaVWR~wO%7WuTG5sD9%uLF2A-?weP8u&I9q?5m zSTGb=5o}1xPyZkxhl|tD&&rcV4uq>NnL>6 z>KD`QX~)Ra99oOp%6PC{ScBKJK(mk})^DQ0AnlXch6~B(h$Wn|2%dVLP#LV)9(X&-Lx@NPbCqb5eE_Sto zG+UA_&@bfj+r7B;m?Qz)q9%1=kCM(h)`!o30UJ{~94>5#1?NF7^)~YTf~l_RKz~VJ z2Gw%0f7cXtK*EIN7XFKwSl>(mzRuXHF-6?!IPBu^5+$w0**iYG6#K)8NsF z+{cS758n4dUzdOnbwCUJ?ul=J=RbS0BLeP=LE9BKm*o-q}<;N#(XWDDgi zI*`pPiY9@Al!2{nd>;I({TFJhC4kJ2yJ=rMJ^4+|=8~07Le5FedA4za1ye;Ta+nv* z<#&D@qZ~7&ys3N*V^rK;e*Eb@%o5Eb2ILPD@?Q#j{3+d5`gqa~LPSp7oWD*s848bo zQx_^IL=fJPC;fm4mb3^7iA2NOY<3A~U!vI%GoD%JQ>7YN`6Cez71gRpTagK+X0YBWi8Cd)$e23d3+p%GDqP0 zLagKW8j-TvPEm|-adK_LbwwnMtRA-1fYbdS*3LP|(k@H)CvDrVw6ju`wr#W0wr#u8 zwrv}gwr$&at9yFxbjN&iqbK72x#OIOH{!e}_S$h%(nVwlQ)ydh%9l(I0wyY`iz*ky}z%yOxI7V^Eu*v4c-ywX`iXiL^s&Q`uL z`1J`wm{5G)wdS#9)DJJIB>{xEn%)?bo+WvC`%97;5H99Q*S7ItT4C;56v!@#p!jbg zoN(@o$47O847)Vo;5FA^&UvrBEXC`c+fRKJ%H zLAeJISt2?__pB#jQ^dT(byK_9m!f4%DBSSXTmcSY&pj?>weKR9+Bx?$APV5qze``> z(_Eeux7vOah|IaihMQ=iomdXbBuaV|{*~34@hA2BC5G|7u5MCIf|fbB%maB=8&ktw zo9(UARBrsXv~5Md3|(^Wj4U9nTW8DTG(oOxF)Jj?HH$f9ZsJ0^RFr0th6l|f3Fk3$ z51|w0n&(1$25@Q$>%%#M?2K@oY#SL3-T3xRq zm+&sb!kY6dIp4cgU2RZ3)c@T4^-Lv`)u;N+y#6`w)$%hmSBSq2eYBq@E9u?Df}TRZ zvFk(0UU5Jq?iCMb=BqjCJz9Fe=Z^zq7w6}Wbk5?RU&>3pBDoIp8USpMh*joJN!yiV z@^YJZM=*0N2n=j>7GB(OFW(V5G8K{~2>&edCu-eQepd@M&xv_{PCYA}E>3@_G64NP zi<50nD`;l+_j9U1ed8sLkT`$#S&9LjjKpVPSZz2D(CzBNg6d>EFRI*Hl%P+BdG)@V z`H#f0DsS^hj7wKJb7FU=XeAp(&f==d@+J+5DuL4p1Rsy*YhE~mZ+{Vc%lyw6`2V#i zVEO-mb~FD&0{+Vb(cd5NZ?zd2nE%xis8j8C{9X%O*OcNjdZ~Ful8|`v6YI)_^tbOD z%t?<$R7t=6XlvJ`#uEh<)=s#*0KkdHDTiqhVE%-VyGJC)y!brt*OFwc!f_X8ZR@yM zZ0+{-b58^Mb+9vHV@|za8)ZgJiH!tXSL$)VpG9EUJ@bo|-L372hG=zl*-P zASVTrhs@MGLPvsM*uk89*NE|~mUWfUT9QR?)~ros?IBo@Zcs^l+ZT7Tw|CvHGiC#6 zUb@sxIH5C`%IB62DCk{@KAPDrzzufMcr_4ZIsY9OdgvtcrK&%vENw5g#)YNceGkn= zTE@z_m()nBP7P6A;m151H)AgK2SM6ncxe#Ra}dqQml}VU5P9rWj((3ILU;*6S#1?Z zLdmpVkjH9X28N%ThYGH$A~M#Yv{$7fWt=!>Y+uCXK2CUKdk&e_E^|;5cr!0ZV)cP7jO28}&Pd0=Fl(+y=#yPf`%Wco~m#4zmv=*C2IcLm@ zABw0POUb8*8C%*#8boHZ+&I1>R2kCxcT=s1fEiFR3WF;2_6W_`2x}sljsWwXAE;ycwxVtqdFAd)% zw_`W*hiBXRDj$Z9GlD}W0f0@ZSYmT~`X-oO8P1gvmAO?^G-;5r&^u3J6A48)zylwn z4-s{Nz=FRAZr2AJ4dg+4IOq!z(2vJlUvcqu-3`P+K`da8)}Q}^#q369Hm|b}ZSd*0 zFlwmI4QJC+*S@Oua)YE6_OL>Z?!SP@x`gD7Tf$R*3O7M|+rKE$Q$>HQPp&@|*!FL? zgW@Aph2})R{kSM;CpiHCs%qGC1@>w8B-3g?yI?-v-GuT38t7Ae^S3Q6wYKV=>ksxEn- z(dZ+#6TIg|=(@H5olXo2G0fAwwY@*SduSA@RY1D=8YJe8&4DEN**=aI(Nlrt4rj>e z;SjuZ?KNc7A#5M=s|7Gue!cb5>;ai2zF|jmsiOd?ndVYXyIaBw17foo1EX^5Jf4-y?Aob6T_ z{*ebUk|7#I4$1>u?B;ku zWwPBnB+g!8_?=idaIipn-z~+SVM2*wnlv=2pcfc)pLgrij6{}qfoI-a@nS2s%p6_6 zNo!~0xQpAL>Y+q?;fHmo2&i)i&ulJ)b(_!V0t8Xy$2=v4LRVssBjY1^&XEqQLZZ}J z?b|L1>||DsY-|<;-CKUK%wxB0>Qut#2>Re$`)ZI;ed5L!reTG>A=k^_yiGT4#rb5t z{rWLsx#9 z*@;8)5xe2d1~_QOjK5BIacvg*xuJ=?R6@B?75z+7=(yrdxXV922d?&w1>ZB#(>^_C z1Utr)*-&3yZ*n$_jpbF}u`A6>ZeRE{ZyX-i-u6Crirl*0cMYDt?ES8iOsaamlEZ}; zhd*H`TKJXUK@4NG$$bPlckWRf+z&X*vnc@le#Aghx)=PTM(QjbU5~YJKttkAh};)4 zbX8=L71ad8<=c4f--zPx05;KCreX#g)Oj2ov1L!XT}WttVAm4UTeJg5n;N9UWEf}+ z0Gh?y_2;}kPXm&GuHf@UdP?>ldMQ6<&z2va&jgS_+2#Me#q%ex{=Zbp{{qqeE?)kt z*%He?IyL`3i1r`iEB!wWsr@;D{LdxgS*G-4_OoxJHG7b9M{0Mf{?mnqgUuF{2O zhDd70vaSG^@Od0ZU2#LcN*0OE5An%;{!@8zL64b*mJu%6-_tH4cuM(`mMh9$KOhao zMnb;&1wkWqBxXooP7fxjpgOFBXu}b-tFO zRF?5gjaX#;0#BK*Ki(716=-3xzZcJEfK2dzpc5nMz|{s^#r5Cfv0DPNN39!VOfHw} z=NYymC{$%vyG&MCVfnP8upu+@la0uPq;ERQ%F{}NODfnOEERAE_2L6nS*(#2AVL%3 z8lFk?gXb%W6sGE2WCOJb@jNzUqH=y0W|Dd!iY$6|HlYL|*#?iA_WRzOwG&qSC$4-Gj7qqn_M5fDlk;l1G<(ueD!=SmO()_XH z`Zy@mY%i;NUyy#=XG!d4l5iRDKwii-Iry%-J$d!}kxy)li>N&34BDT1oluF#FG9l{ z0x_#lGcrNgj+qW@h+FQ(&fMd6R^0MS*vf!X_3tcgN5u9Si&s|2%K4Ca<~DP4uT5@tLifIjTR@T(Aq0qmk09Q?(zL2fK+4EX5?3R;TA+Jq z$GC7KFG%%IpT)7?3+XO+uH<9OHnc6q zOJb-UDFRb5+b4~*!tY0vSk_Mj)S;vVQUB;URNBwXk)=KGrJmpzyMW+c0>6KM7MqOG z>J1poD_GutSmJV5H-X0aQ5u6DgVtD(d8P)o1(yic5Nhl26edJBX~-fugnVse^6fVp zgD-Rlv|c~u6u+miGL_KuNEyDZW>nK^|6&&;PjOpdRMj3>inA#1HBc}ZB&#$+WzS7D z$z2B__gV54IrC%Xt1K>d@ME>}M9U>}&$E@LS_Tj|(SQK&V?xa}59C@N(4+6Aekv1P z+KKWTu;_2LtBh2~95h*al|l`?n;jD1AHeeq_OWq`As#3{-(zz2)Uaa4$#wZTiJ zzw}JtB^{M#d9t$=3BYDJR7w(*l$a+NNHerSPI_=Fidk7&`ukddB?6t1MYFK2JVNO; zsF|o3L+^Y;kk`}o_!KI!wV&?l*&i-#e7Z<-(S3y#AjDBU+#(m*_SRO4O8b{cY3qb2*~AFsw8ctIQ*Q3gE{HM@v>@2#EX!TJXn z+Jn3@|3dL&XbMx{{6wq=rIi(0y9WgH=@15-K*M!@E!Taiz6c@l**i)q4qN)D;@F+f z<+n+gvwYkR1P*ftLi$dAQ|9Jo+@ChKEYba=bX=OSZJ5L>ebGl$aApQ5ST2FQYRgUh zdgGE!=_;Pq(SWMr2aym(psrzudTmE!ehQY)622K82}YTw2jYZTRE)GiQ5wz1YH(I% zUjC@^6Vl}30r5 zb*20+`)HrebRib5B;yjU<&d|w&0;{P&R@hU_LCWFXTa`a28-_?Ya=hDUxwgrp9XUo zH?LZ5?$~Q)2LF3Q;g8Jnj}`*s-%Iu1 z*8fq68Cm`TH?jOjdw`MUAA{yB|9zP5A7A-jz)t_YcC-9HYxlnkkux$gGXD$gl%y)@ z@W+|>PwhTdu`kMl;t{H4BbQ%wG#FO2Sp8GH38CYs6+mJ9?cT52d*~=;n-cGdY0w7@ z2)20c-9JuGwUZ|?BgVL7f1Yhri2W4S5i(AjT3lPn=0^_ksr2b`PpQ9Ab;BNvuteWu zv+-{0(yaD|Iugf+pn*PmA%?HA^LS$n+|pGgu=V(Q`*3#(xL;fNcq6ch>+14~7 zM<`EUxZtEwjP1pQS-j%i%tKaGJ~G3mog1{w8kEN-FBvZ;9qo5h8S*PRkZ0vG@{9Y8 zUqCKC9U6Pk%QFFDU0@zts*$_V)c%@){MsnFo-T{WtPq6gV2adpe^WJ>jEx$C*3} zeqIjnV}#=X(^`tDgFtkiD5CES#8`e_dTe=19ghueHCNOh2uwU{p5P2C6USL$W}5MR z9S(jM=0b;76BUMyU5E@8%g&Iq8+#wW-4&2*ndSF2<=QZ!uESCajC&$`T~(nVrdPtIep0yt(hJu-E#W{ zHL`yu5F||V&;$PSyt?%eJ#(lzSt*;E^`Du7+0M}04$^=WrSAo1(`yxk8HJWm)@5c2 zc+sH9D5lG@KlHB$WXAJlcR>4YAq7WfK9Tgquh;k&@wgdA{`7p<=7MFgIHITcuNnL% zQmo%)te!2TQdsyM6tC#?VoCQ_=pqz&qd+W_$Z*r=inS5~JM+9pN@B=m7YkaBV|{)`~4Z75h-zI>i>|MvGmb z24pJ2EW%~>Ap+B-dZEWh!3P#Q#Ip#YTGL(aaL`U{#?4%<7x7%6g zRrPh{ywt^UM4i*R6kFznBU`Farp)R+pJKycsZ<%MvdfnjkpJpA=VTZm04Syhw7!ygBz9 zjir~OvGHoLF-z@zjshlryiCYdT*_wF-gs1SVSjlOp{0$9kpx?F;cZcO~`Fm2UOkRBUl$gniOS z9(JODreN6BSfCmU(Xy3@{8vUzRgPzJwwWhvUStv;020P@DHD-*5S;ENLloi*xcQo;bGk8@-ae)n$`>C)5W zjbT7U*hd#<;f^5A)#hX40z%4690%kU9h)&Ta`JeivmjdHDE{GUV5kL{!}lDl=Sf!n zQ>YW-X0ddP+m^)gOGwAB*b0om)OOfqoT4H*ay1_)l!-mTA_;gX?cRLmuD5sf@Zq?z z=B`;#onH*DoPnt@!+eXguHBe?Em)=&dCS`Cya89t<#`d$lXf{}uR2uF#) z2+9>&#k!72*9HrPO*~S-LWuzG8eVks(lDtkF(3f;wxhCJ<%k2^lp>gw2U0NID&>!< zXQvWfXiK5sZ-ai}oatmCxvnEt|ywLp6Pon4JX%d^%h3JYNF z!^Nt2Sck2}s*6qf3Ft&+f65bm-a#qM2-y|&TvC1UvjU_e8qSrd5jRLS+MPq60SV+x_qQR7WIkh6L+Gn7|qQMj^ZOVDu<~nClit%_e$Qh^zaE^xf^e z4gsxImHU{yr5g;bCx^(IbBAnG7*wpTotJ{r^tdoKmyO?F&S8o6f3cnYGh_5$+u8q` zBl_>#+4sM-w*IH7N0xt=BVuOw*Ym2CJhx3B1LDRD8opIn%j3!MbIiuF?KC+7Jy zli6{m)mcnjerRF`3plZ8>;$1G4YILfrAkgS&0Jy5xZVf-c#cv7^y6zKDy!5i$9UrO z{TcQ-U2gee)<$pETKd^KsHV>^z9QI*j~9cF!IveSJFZK?HXf&b1EUlU?JZgL z2Fb{M0m4zzTV#gR^@q&07y*&#gpKA&w$fv4oQ}I;CAb7n&$OP%Tu*4eZg9e=ZGj!k z0l~FkSr-pzhM)#4|GSiVH(!`(;Feg8v@hVKqHPYYgTpXLgZH9g?$imel<)yx4ph!4 zAR*gFovBZNZlsIMU%c*rR!RRK)257n?m4Usg!EFz)+UaoOoXf~EFAy2`rE7i#|MOL zEKL8ZYDPJODJ5zzv1$X)<_-X6IUHRiT%s9ZL0}m{`WXfgXYq&&9m8fT#l)$hf#abe zktLuNW(i)OJFeb3zPcBmDm^M&uj;EUuIiswmUt}KS0bgrlrs>c<)lGz$skdYs35q!}4M84@AxZGJ!%gQ1>)2=*9%1>^`?-b%>qp4d z*>erRTnF2Rz4HZz{r(Ur_elW+z`VSVfPB2>*Ln+>(fb9le3N$l!-;`UaS#*GRdp@v z#e+yX0dqjI4|4acoPS--uryo&MpQDK@hL|QraIYH}cR50r_QC^#gYYV)(t& zKDakC!zXEH|BTEMKE&-!z-j)=wo01xtErVw9sq@eB~K|X4w%OQJcP7Bf35b=^uWC0 zfPOORE+IzV2)F`eRFC_D_h;lT(IZD9V8Vgr#CA>VP51zRX~hBj0OXjUV77s6J=>&T z(cV_UoL`Mym%PY_2sLP}$pnamw`VU)7{?m=GLS8zJ>4hWnlzSX`Q_%uJzvH9K7M6o zkUjtrIVD^|3MvvxU_iKJC`dtp5K!+ItPyDNhxk5khST;Wr0<08iw?p37DxV9sPEBK z;5G0k);Pe9R-3~2XiK6QB@WR~>kIHT<^FYX`<3ugP4QJe`}KoFS{ET^i+xHr@CyRM zqmK*tlcB2cETRn`#B1Pzq30{z9O}cu`5{+^L3ip)#7S=Z%udd)-Sz#9P@gA_!KWkv z2^P%mLz2;F<+so8GzeJu?*h*~-WzlP{+!SeAN?f|aJ%j`lB2yrhE>5^5KjyQm!=Pb<8|Md%^GQqZs&DRd_yX?MAwod`fjj+yM$yR(@)xp)n zl_W6qLx_Ob&tgQxRj zY}_dhlfS#dC-K(l=&@?1LmKMvuvq_q;cWKNV`^FzVPckGyY6ySEUe8={Utsv30%*iH0H4^4qD*0+qlB_K{ z$DuS+4N4AZYS?p$^M{QLF1}X|m8;L7`x6ka__a#B8rINHd6pL=Cflwtsre(cJvCFh`^Y@vKP^q^0t>+cL8E}uOn5Ni#oq^$?N>en5 zvLnP^`UfrF#`Ewc$-*CwW=8E_uu=V#R70->C~Xp>weqIiT}!$KcEvUZXNi_rP={Qa zca;0xcJUt>YHpFTl~R-FSFh{{)D2!N?`1KnN&~6x?(nd^AIv3_>OKdjP@-toCa41v zMBF~@$ubCy@C)rvh*@`(?SYKr(Dg-r?|xh?ARf)N@5ZIiGLvIEwiSKy-%L$s>t@v- z1uA(B%C22|L{_~ru_HN%!&q*1oUhII_7OjHSR8mqEaTCq{rb2d zYrpzZ>ZvDx$!fqfWm;?<;vP6H$`i9L7H4p}Rw2Ju`*EtS-IH0d`$NogwrH(hpzq<+ zx_{Py12r%t+c_DP1Q^Yx(rG=BgB5dJ7{>eEk4}A`n~j-T{>adJZU%0KTUrYYSjirpCUse)DKe8hgrjp`74lC_@z`rMc^MJZHbQx8@JN57%W3J@Q`UU#sv@+7z0GU@!I^^gRU)lqM&PbjeouIB zbWFugeIJn6OTOvNKTl+H(kTIlm*wGL_S(e83o`UW$1;tBxA9{AyzPV4)*DZe-33HE zmz7Lo&bSldC2vueGgNm`cTge9VYkDb-^bq@rkZulj#1Gmwe#qmX*RZKzV1{6Rk--q zWykoE$giu*=F||m!8K8G@Jjj8IV|4NvaA|>4_>gzU(lVIREM%=7;bnNRNanR@aAqI zlhvuWqzxRKl+>PrX^P|SMbr9bJLX;qnpJ(^JQV>c#J@;;g&Qj5dX##B z_79sFjtiYRr8t7!S7&Wt>#Xj37vhdX;^t^0TMWfsot{k=Z#MIC)@NVL0Z*knm2Pty zAPgczEORLjRwSawaY$D>BZh8rk5#D4bAgDZdJ=}?%v8opX!VU}$%vC-r{`X==Er(y zKUtNhZK-f6J7pt8Q07OBXmSb3l-0=N5&Gd6mj)WTuMd5vO}=ro)SptZae6DbHGRTo z`*yP)y0ojUC>B@72QN1OK49sb%u@PI0whUM?vt<)z+0I_3S~gUo?(=VGMxiUZOH?7 zL}0#GvC&MfN`~V`IoR((@=7z7t)hA>|Jv#zTYWP~*Erlk;{|t|(Hue_W_-9%n(ysy>ipjFvewHnf z*V~)aUFU+^Lq1gNm!Ksfz*1`qNdTa?O1AZZ)hs)uc@J#0Rt|R6tdD( zpw&t3NAIb2GfD95+c5t_dOy$w)gUGxqay)1t;baJ50~4NJ4e1#<`s}6wDm#CI0TU{ zf>MjYU&!}ZTUgy!L0;_!Omz`*3ZJKEQ*DTS+RhITbSa~rgujXL41i^S>AH7+vlO?R z}SS)=+bRUYgq1_51!ONrotd_VFG!rl&EIK@d>t z#9{m0;9R@&Hn3-#HOD*cWk`hGWYU-wB<=Xh0=iNPoWIacm#ePQJC z*BoW$7|W!o57yq$uXX&)HP_ZT#_$>!k6Xv6KJz_>-&+^s7Qta&iRzZQWrI%`+Qg)7xAqxg;s-WW zCv@QhNuA9D9C;7O-aERAjb%zUJM;vzhrh{-#vayeFqv_kATEfRktJdNO0 zlIbzB+%flWn?Y}85;6HrKlz`F?8%fnXuZZ-f{vY_*1X2e_I$_nfwowHW3u66w8#79 zvji{8uUJ#qMHklHW->pUeXAOHq)`EiVSeSKlxF8BO?1KA5clvi_Th3VgMf1}W&Ji! z@Qik%^oji>%r`r_;T2Dl0r5i%8p6WL34sq| zV4XIAThiW!>zr1FnavdO7md*?O^T!LO1V&c39MWQT{f_E_X zp)>^0uXt$vfP=ZB3^dMx+m{R#VpwYy{pm=ri367w7REA56;E!X=E^OxfqS#gi>O7U zh6dlUl|Xq;8?m$* zV)?K|sV$kw^)3%(dg=zBvtS%ZMx28_t@RMnWjEeQRqXrv@{OqHT1if`Us^zs+2lQL zaqhzlB6ty~7E6>B-|q@X=-wVRa7t~&vl4nh2=ZfOGu^bE6Ak<98epA35SYa-t1CLq z$yjN-n$x}6v%dA1Go56#E!daN7^WyXD1-J_8H(~UPq|Z*x4&*Ix=!u)mrcMX{76m^(zV$!~7>u(>+1te{WwR4Jftl8vcbwgc+PLiOgs zt1^`FOcg@Bvt!k>{;Z)P$jwzjnq{)ZGA!={DfdbkPaE- zL33T>0^+7-sipC|!{_PXh!-tx^cwjb3c+c+heio&83DC#MPbRaeL`_8(ohP3RdTNt zA#UWeKb!>PT$tPoD_NCbyZL!F5*r|$DK*O88gUbyJxENwq^$e8w-NZq0+kX(S_^LD z+DBvtNjP&E&jg_yobP?49^~Y91R{7HfI5A6h(OR#oTTu!RGJpbJi#0)VA`>`WfsW1 z@!oVBs8TY6*((@?KgbD!xxl zqu68q!yAqJhP}S#@F>gIfTK@PUIE;>@DmO>Ho%K}gBb?^lG}g;)3otA;-~9OXM?F2 z2wN;gME2nwtyoZMbPSrW9Is&ONc?5rwTW|9?6S25BL|)Z>qTM$W_jCUFo9#886$v$ z$xYo1#uAjNQkS|%{%fVN(3dxluRk?QX zvN|Q~05(^KK7OLJrh184Qm6d$B%?d&Z)ZuW`A<^Q6C*`VWbfbH1F{v-_c;GJ0QMY` ze?HwtDlLkBtL6z&s{EK(`fWJ)BwU!P!);p=rZ&%Hr^h&Mi?lX-iLN`Aeuy7%nik8U zG!fK#bi-S%rKhEllj8`ZVV;yGNYiA7gNfv-I9RB#>4ZgwNPw`Rqf679QA6%M$Vj+K zlF|M4u@^c0y2-^paa|I%H#hy#AB(kj{L>t5I8! zpY*c-l9;BtxD2nT5Yz+=%_mYddJnFcpJd2)>azLd(OpecS*q=feR`jeE(P8VNgk_c z<(PoGtl620rEYe%F7#?&?)i8dq;S{Wd7vj)6>>x1S&E8$u z)XltDEhpm}a)%PXpDKO^m8EPlNeg?KPEVAno*|PK@*%A(i!sItER+53Am%Ohe z``_jVqegaxz46V7^E>*Ht z`Hp}u5EqlIG0PQirwnI3gh32ba5r>#R?!_y2x5e+h4PxjN?29@l4@BTSAmF(ZP8I3 zCLbJzikB@zRHzi<(7Va@gX|inj_E=}hS^OAT&ixSP>ZYe$=DesMYOu(09;PMauE&> z#dRB-cao)*uKts&^P4yAS<^gf2(&t2%c(iK3A4QU%WE&k z#s%oJodI;&*PobP)1$q**w+k-@m@+Z146M1$$L>r_$Db%JXUt6I1lW_W9C3>VE%Vy zX=#|h4eF!%W^2?^npI-2XI`t|R}@47C-kI(@qUumxj&U{B%#ncj6~&;WU!gY7gAbK zF<(w?hqEtq+@5Arys;1wTBX*sr7qYvl&-w$)DzOY?NIaYDZq_C$~#2`_b@_bd%_bj z13M80g^BHZdMbCGuVlXHeL|9Mxi=cSn|eY(UVLtB%k5k(bM| zQd8u(i_BV*ii&ZS%{W1VhXJX@nd=mO%$@W|LW(}t-gxGk&D|WWm~xCSMO>y@q)B^> zIpFzp1`Xfo8iz4f$=mhhr{Vs1_UpbTP(QxwR1-y`V%o;8yzd-<-oX$}z%r1Xw9?b? zS}_yN%W^9GSk~@fI|TJR$@~H!8NFCYz{zSD?L;iW9a0A$R059oj^%SbB^Sbr`p^_N z=%88UQx{z~Z*p8RR~d^FTZQME7cY5QKX~9}9dhmc%Cyo4w}sgQRgUCP(ye@F|W?Tl)P!w;#1@4SI%CG2~CoTYGQj&dYMb}5SSTM#e;u} zi^61YT7!y(zlXjNgRY*Q){o7f{o)4cmji_#BhoL}oGN>m`nFlMaKq&d8V9OQ+gCaPB~G(7Z!j_cl#3*}sk4uTf)-hlk5iU;MMY2F?^3p7wY0!y zBN5Gj)e>qV9Aeb{04kfdxOYG8!2bJ=_aHG0b!^p;%>-4d(}}~5X1{}bVr&g7T+O;Y zUMZLT&^3rU9wse_^(P{@!{Mhnl_xoN6Gdu4JK^l4*;CJK8{B^31WG*U0Y`q^n3 zDqS=*eq{9&Z0cg@M=797!6Uy?zk7ctk!fWoLVmJi2v`HOeCK`VsE z*H6I`+A~|;E4a*pr!q=^b}FstQB37*Re`f-2bg5WPHxPLjv&AcV))}|=I!azhJ%12 zxlU;&t?IJqRS?1BgHZ%i|2deSxrh0NW@j2U)N${2+#!(dalk<4QM?UJi3n?S=tYuJpyah$T^g|y`)N&?S=g`Tlg&m?$C;2sUz3%tprJePtw0@oR%6VenlW$X z_|-1JY>QOZ!%IORkj9_uv`~9Z>Yg<2!9UrggnfgrzlGeYjd%i)`BS}0rNBd!+2gdR zwM2(Li$ClnKZd$`wzb|v2#q+fixgBQdq%*JIK8!%! zkq1O-Z8kCSNL5SDcsUOZPa}3ANV&nb3zlaLT@mA+rw$LP$vfqpJLh^Z9k(ga ziE5XogEOFmV?>-SEx>r9(~3(Gu?a!2Hw526AE6uf%CADzC2Y7@5_4XW6eCnBxI>bN zNQ)OM2Pl`aC@v<=7_i{lTvVKknBGu~0=F531eQf8{E|aJ-kj0{!X|eJplze2W_c(O zIZnIU7|m}6&5Lu?MLA2Iqj}o>GeiUX>$yYf^&+lif^ZfPO{Ndpm4Gf*Q>8+bJP@jG zg)dV0WBuvbN%Y652Q5{I5V<&wBRXPD3Ne0X29-EY>v02H4@Fp&r38a}s#W<-tHdw3ZgWRLZMK$t z1m{HYR(E+_*$|4}BU@^h-WvDStdbfUeI!$ao6*aI%l3NZ`S%6vCb=%NN%@f3tD)rH zFYxMf86-N)9WeC}FYKzk`lzsSWZ{{6MgyV%^+T-7J!t2X5>T_bqKtaY6gjdru2_Yl z0`mQi;)5OZ#UZcLkX*FZ=HDRQ((o83 zb1(UvxibFA;{IpDx+qr&rG&Lv z*4cs*r8pU(ZMi_w`%Mot$Q%VhNJI?*WXfDg8;L9FQsUWmmO|)I1yKGnR7z+GdSG3- zsm`bMFWs{@l|_loj|t{_lSi(4RpiTbmq@Ur^VlX^rve-mL>ka|@RH*bGe3R+qL>gQ zD00#EVK8A0-F3Gew6!8Tb^u?S_!opA7+i?K%`Ym{2=Y^+4Ztdb7C;XPpr(aQLj(s5 z6v$6l+8rj=z`}=i>Q@7D3XKf1$)(nk&5GBISZgBJV{sFlibOY3*pn%-$(+ZNP zwPOkjdkQ`sV1Tz%^^B)4hvCbJ9PDe@`Gw@Kx`-4ZEDqwAm6atRry7Zfd_g$ng0mgg z!wDR@jmf?WVhzzkft7>c6!b+Y9qNw*uoKer#)*u3RFkE^hVKEPgT!2i3vhAU{s|!p za@`IR+2{xaPSLVvQ+H|8XVsJ459Axv^9B7h{6>t}cag%33NF}|mt(yN$p+!e-3JEG zAh(3p=dN!9l4I?RAyfhb9`+RC&WA)Nr^&UWdMfAJ7Ec5Sul}RCVq#+Z^%(>*K&C`s z`(fx`4_6%iEcSPF3ToG9M8$Y1C7!m${9Nz!^GQ29559N2d+{)iv8}J~rs%%}f;Qa^ zw)FulDSY(z3Zi}u;clz>Vu~pU2r0k`OhwnT(HewokLFqQP;q~ z0EzqX&I*j;i$dEMSas*}^Xk!_LAiFM!kKQilw36QQoQxSQ3cmqyIZf1i(T)!$E zVPY{MgMC20vL8Oj55CCnBL1PbK0wgXb)R>1Lw&{Iorc@qeD?Q9J^`Wgg1d{oGqilL zEx>+yI6rJVheA|+SuOMl-0bD-ANr;=5UMFblD^`I>dhnR*tY+Gq?x#Bw)Ecnh5>Tq zZyT`1Z-9uQ3cCpd`bv-EJ*GGLYyldubZCQu8bH8tUM-?ivGbh!YJzWW%}}s0cn=j3 z29jgSGM^5-<@qiFyA3iFQ;N5KmeK~qN5n>oeg)jKbPiD?nny{8P5TWDuq(QZ<_`2F zFc1XvC-!g+&WGWvj}MSDF0A1LVh|4?gx*pr*q}cqpzOuZnR(9eF@ws!+B(y12En}q z3z3(KCeddil35_44M~!q$?_i;tRvMl3vmeo3xTc}so5|mS%iTaT2WWzEIdun)^nmR zcJ?xU>d#mS!^~@#{l9QqyP?LLE)h__OMz z$h?XU=VWlSY_=5Fwx(F=~Q#D4P+tKq^6xbgH3aXjQ`R`Ey1?dld1R_u9}PJ3)l zdLOc}OGO!W2zkM3&t0IQ24=k67x%|O6IPRpYdO5D(%Q+Lmuv(_g~o}khrsL2MakG7 z)S9oSs>2)DXZi#p%kQej(7)r;0D|aSZP{MebNF;A=OYE2$h~t&iTk@Of-7*wjtl-K#85fcM9@K8n{633NM^t*YKaD}_(|p&QxVAp{BEag zJ0;WsxlQ(0tc}cf$a!=pWnj20?6CP5wwt-9mj);lMEp^um)ZiLLcESxKv}7Ru?SvkosC ztO2@Iv!48$^$xiRiwmw6T%brh5b!9DXK_;_aRjn&V?m#1@+Lt=7cYh@0n-U~7U9tCMBvGn1Zwz-1b(+PvUw#G`cNY|CIo^H}VCkNNbhQrw^Krtl=Ump6t-9cwC# zO;A`{b|BJ5a$|mS@)pLkgHD{tt0xIrxrW!tJ=dtekIBlae>yGneNY%BpNcQuiRH1< zzj(hv{Z2D2Qa&h4VQttdejGaS}AW_3* z%eHOXwr$(CZC7>Kwr$(&vTfU%{w8APA1!A2_L(mu^4@b#R#rxh=i-wA*LdxZnQwkf z&e@`m1mSXV`qTA@G5m)%L(bUO;vo}IyZb8r?w^YM5nWfXEeZyloX4xw@Xpt@DEeUI zme{N?mvT=b)V*7NoZ%_a>P`0LDL?mx_jilrsg8M5|I|&d)s(43E?q0ddTI{)_b_G^ zcY*-V4+_|a8@RPoUj$s{9&Np&B6!XVWxm$s#Gh5EVO2fa6ui&&dG5f>MQr`?%U zttZ@cMgXKnq zEt#2Z$KwV`H|MkNxGxPFhPvv5j3{a+J4}{g-IG_4=ku^_bbN6z#dnI0@+r04O3_`^ z{je^#)y{~O0*Ef}A+^V-YgZ_fa%u=f%#{m*;~O$kwzmmL@7tRl)I!5qbgudqRBNax z-S<3uKcV;`C(y5Imp}Gku|3d0^`tO>x@+j$oN1dz-qCXY=lkCkgY!|p zDW$U}DRm<^W8Mk5F0RvF*zZJb@0Xp0Ji}ihQqUz)b}A=MQNr3*mbC-7HKuPPh>`rr zFN=_1wpv+gOfWq$9bc9R8Z4nqr-R!%#G^i05lDPiie96~AztyT7@rf}S<5XF1eJQpY3^C;%kNzVhuFiP{f)5G>9J_?Iv;6sHvkn_UlP#Yg2gL*ftl zR0Evb-leb7wtZ1MGmQ8kycjM<@#@&lwQ$wFo^4&i?6oyNcarwql2*rlCqx4DTgMoM z6C0;&PA56*pzP=8oWGHfszkZ7Q9@x+4lwUpP5@Lii)m^a4zhyGAICfs(~?;DGW}7m zhC%|ALK!=THPV;2OOBE{qPBr#_<>v@B6P?v2YNS;o;F82D9gEis_?9=fvjI!V|g}7 zImpz0J5>*2UOjd0RJ$_IlXYx%vc!#dwTI4K1%I=RJ3kbQ@i)7~9MfiI<8@5ipMr?cVRJv$^Jbn!9%b(u%i*?tW%4$X>ExujQa* z`P7>C{d91%3fBDm07B-~R^ci5Nls)wQ|<+3|2|d>WPk*(%|&R_EKDG`mutQk?=&0g@Vi+d#|4z5c0q}mqjZRIo$>igzkO<1~4 zK0RQ~v)2fYdldiX1x%)A&uzfkB~t|8SA!rc+QL-Fy3Boa+xKlR_pUg{eKExYpHG#< zxjtD{eSj-Ht&Jl7f+2TA@TzE+w~`GBH3NeSQl>ntGb*QIT~(ltSHvwEHG%eHAkS*gmt@eBsrj9<1!9ga_Y#s4N?8;v+0I zmfnTsj~>vu&bEn!{qdm*$tA5KG1^sR_9v52aof0az1^^KkVL3uHGOC7%nq^g^$V#| z=tx#dL%=|N741~aFob8|;~qdRA4p_o0`Vh&YaCt!Q6#w{`!r)Ae^6tLIGII#to2o| zf_EhcZW+V)XK-v`X0Cc_3kR6t19H5pocoTS=zU)1-E?3|u zPf4R^%&NLVZ~u(nPzFq?f}RRU|2g4{a92A*eJ*0!C{Lxav9g=<(TnyV8LArAg%gB# zEPjW^ECX5|Ujk{ z(rp*m(wk+MS6wIpE?Mc-fQRgTa(-FGRr`9Vxq4m(gzccKI*r9&eb1(5%0f6wY%RJE zkHVhV9BVRs2_i1*3 z4dDzGz}`VF2^a9ON6H<~pq@uFyG%spEy9~}OX{J>G}NlyEm znoed>K3;MDT6S0+qzB{AAW`d-;rY6w9qZaQ3`JgwhCCc{$mPue@W}>^<-isz#A~Pe zA$O(`GpZ2QDz+AH(I?B}9BXHZwWQ#TbEv7>j)wy^UY12~-XJ-vr;;4?&(PYeup}EI zq(u9tK3TuQ;TqMS@v)DKApsIu1=~yyPFAEhWX&(v5?#B}!s2(GDtn3S_p2&4txLju z{|npNhH*1`x~2WKcgPHR3o-voumiQlVx?)aa}C>`8ap+c!}r1m=wE4wSQlGP$PDgyK9t9AQ@Hu9ML{Vziwuc%>A|e1Oz<}R z$>O15^W2&)-Ej{rdTnMQB->2EY3Y=5)VI89ayninIEwGx1K{jahBzs83QLckBsSy{+52kCc6S8?qA?P%<)2b&52tWH;{l4u<)!MdOymk$QwBL>W}wVXtD@`e z(b1QZOVY8e3mLUIvp!X3y&}aX@l)D(VV=hNBQxB-L0C%P#S@g%!ii)NQH4wE4|Ork zx?89nl&=D|V-B>GDKYNCs%oX96^&JW`yO^m*6$J+!gDXpem5c&D}*w{_cgta_9)fj z9#`g%Aem|uHfD+k!IQ+)sK?!*XUkGv9hfd#^|F&}zNyil(0Ybb880rr9MY|`R!HSx z*HuFI5K!4pcfku(Y_^|R->z6$+&`nII8I#5r%+bn)Lb+f89b~&+zIa%&~%1QbdQl3 z`KugK#eWPAA9<6=O>YKBT)#rtJT3a0fJrCOBgqnZd$z|p{7_DyF6UKS27S>_jkHDC zg#&X0F^k?X+Ok6=3~${HQDF#^p8c-q)m>NkF4Bd9?0M5n^dcrC#VRD*JYURDVVKJY zEGUtkxu>{}n_pIlqFcOed;26ir_7I9ShMQU#Fvgm>DXvkjy0U3t2I@VEP36Lmc${d z9{rbx^L0H;8LJ(s%M~_q(hJJw=C#dBUu1N}UUochKL6(Z)T58KI+Os8=)zA^=(QLsCHu?ktQ);JzW|V_yjCREo_&|+1hW88Bc zgMHWhShe3uZq(Y2n7_D8Qfq8GOzcxBSow$CcFjMt0lW2bzP?T^T})^^pakm-`f%1= z44&8}o>f%gM^~$N9?v3I$b%|Ti@_A94kdqqVdiNl(0OPpZl^vr#(F+)g&Z9p#&(b& zfSK&eO)5txLKQ#}>Zhto>Wv}|H)Zl~JgdoO+VovnshRjv53pf*eM%Rd() z1BQMGc{9`m32m$+pp;4EB@)d=wHaNqhs_ni2No9>gy2$2?EM&DF)){)n?B6u_7ed2 ziF~J?a9;`wuj`7{gO3u_Q(`z(oFqMhH>h-Vr5~cqh^r2`&6-g}n!4K;A+GDafYM8K z=nl}tK^){^-JX3@-Ll#q7@Qk-q27YLBZTnl$c~+QOt~;Cq0QhF=IL??!%qNKm2Y_V zUm)Cp6i!s;=8ysbbXN!Y}rWkVZ*il_F}Tb(H5umG*^jhG{vQ%BY} za%L0`pGu6%#l3jeX4V!WTKN8zdMr2KR%Cc8b;yMG%~P(AN4bk)no+MuDwUSSY1D=$ zES=aMet(_+i(r=1bcV+qE*XQ``xl{omodHyN0}9UxtDHLVswrf10=zN?i?Qh&z;)V zq_=FF^u2{-?Aa^!;$}rm{5mPDP~Edy-ZRH&&Nl5G^T?-xk*3X%IFew)E?#;_&GcG3 zrrNTJTu_tVgXDY<5}bd`*tK-;T4f+{R-RRy1(2gy!$E)l=@F}oiO8xYU^vMySMDPm zU(!1Y8V=YcD+EQLg*gZxtt`e?r51rF*&$q*2?`_1)E1gjk;2Ax=t_bV&B)9Ed6erx zS7p3EdWw4B5_%?1Jlq{S<6#mEhs2fTtcG2Xl|6Vgp{t#!Y40`|7~)({sR=F6_CO^k zGxa6!5W#4UevaX0KfaM|Ed1cC;@Ngv1kbX`TTz$x{60W5d?AQSm-{_^+(9U3DnlJr zUW8A>2J2E_ zd(%!=u91CUHZXMW>^W(-0~sB`VfD)+8|>uhyU5E*4d8V)Ftv zjvirE9qg^ghd+5aqmDzcF)_mWVTZPfU3ihIpK=4w@xgG>@M(#=uF20Jb|U5=Hb*`O zR9sC$WghgMyV(BCyj1dF?@U^ya$j;l=}# z$IAr2M{#`&anG)c`F264io4-AvCi|<$M7q%E_WzeC7oE3Hi1vn^)k)VUPT*|v9gZA zu1OJ<8uMkabA5#G2)ujhAufL1C;8}vVtx`Oo6{EA9+vn=`kVUV=fU1k3O=Tf;0&+? zZjosb786~-pGcufZJAMIl0l(xCGGG65%QG54EU94emB2~Fb0yY*^HU3;o-oM!G;rd4 zyY#G_9i_L-0E1ibo3@e^L zxKM{-S%Wf5PlZasVcZu~bG6K1PY-3AeBiFJPZ8EJqO4lLxU$MzfteE1CxBOLA#b}& z-R|LTx%u;kboFI+dRt~NkAWH`ZCP8j^tM9n4e7RZKk4NmI>s2Lt(oa2@F>DO{sQHG zVZy{jQrn)?MBQ=b+#2q}5KyP0?DDp+$fh{;I$ZAzgHS_9!hc^VSjKwmJ`$C64wM%Z zSj`QXR6P(7DOoD1F98Bce_eJ;l)2lA*Y-C0N1ZFkV{!U<6yCq}-oYpIJ*P|dPAgbLgj zrb3-upxm*usPpA{%Ra?mYU^Tor|M)iU=xgfv@C6_J64+prw zQxeYn-JSsej)D6}^Rh^U69ffNP*yM`6G12?A|)>TVMz=s9_AkHdHvqK)xP{(seaCK z&i%gfec}6D4^Nuu*HD?_+>0 z!vu*8BHCm6jm{4W37FTAL5&%Ga6xnkNJWPMKmi6IH4RW=A|e7%L_|FIM?{ps44_)W zzJU3Q3S>dJhkycE1~oVY59#VEY#jH#iqwBN3=JS9Ee-kQ22Rl-9IrsmfRhg`)+M;r zK+Y9t2w)t7j1u(nDGq_h$K*X0EeXCD~;t;;cRFh|!9b^q#^&;}~p%S(_&^;fALVZxt5 z9RiDhkcfzglol+&1Bjqk&Tg=8Fzd}zzz;jH&%g#$5D*UnJ^;EELIPn4H0YZcQXsI0 z5P)R?8VTjD5B7KKKlG=0IWpEBgk$KSo#QSN2l>>RmV;v%LN<2tFfnK^_nIO-lYjODbb~mXF9FUcr$72pAW)!SFleyFSZ7H-N zEASx4AZ}@s=baQU3|Le-JD|y3xtrOK3W3RO$yJ~?F?K>I z7WlZTArWhAmiYGvIubmsUm(p-E?9nWKSyt{nr_o#CfokV%01k?(eE|?@jur#`s8UI zMg*X4@}%9-gN{$}$PR<|o*ZH8MM?By@Q*unyeg5lxz4VLQ(8t@+>A+(kAJVJ-Lal@ zuVs&lRG50Tw*-KS>R>)~fqJi~8%=DwrT;Le%xxfB+MyO-SD97BpA=2qYO-t{K2BFx zjbQv1<7xMzd%rqH*MHU#_AsBge$ogtwE#WUkYd|>Lx_x}fZNu-ymE=cmku51wu#dv zq8*ILUZFgQa7aB8>!|3WLRRX-R>X{`IBpb$nCvduE7fSnQR%D@)gu&PSl6p4T#hRW z9o=(i0hK@`ljRXgxZFI zg92Bo$Z9Tr>EQ%fnahN``M?^F1QySLzbOEhV{s@QpD-a^PAh^|W`}Rn1=X(Q=J4 zM#(UzTVGw~TK2W1RNBbe!zJL9&to!kUI6PC?tQwFGe#Gi#NU&Dg#Vye?`HA!~tUHW~&cKBq-G(Y3+knY$!%HBWQwkeXNMdO6% zOP;e;9z2d)m}7H_56*S*%_rxR$|}0Qc?Nj$njVGapX|=N=9H@IHUQ$>q%x2_l1wQl zGOFu~ByK3P>;zz%ps^6b3%5Pwz87A-VejNFE6T?8H;L%n{^_-|Jt@`Giyn(*Zu@Dr zaug`_nG%p)AK>1#J4bBW5+Z-OF0-R^&9$w!qm6ncJx(lT9zD5Zj>9QRrk7xk+gW0_ zdnY%GCy(aJX)P7h`iMmVc_zh>QTc#XZ3R&;W$5pXgo2?;W6Hy6T$4fLF<@Snr030& ztOi2AV2I?Eh$9ER1JMl_hAcrMh1Q1bnMdmN%0)yfw1fB;?7YjG(nycjT&BOjl_E3{Xc`fpeYv9|}ur zOK9v%!TX%Ibg^KXf@iTDYhl7(@>RY0P3>3h^bv)>u6SPz(-j2FjO|=caC4lbflm-5 zo;q@jVZ_1PiE~y-Jr;fBVmxkp;h+pixWX`TV@TJix@ROK7_29-{X4=xFr~gMFZOyb zivEpMfQ@&CsoaN@)JQJx+_=0#g(kpej6K5zreQ|Smu6;-$Q`i0j6>`b6jRTkPfh6g zK(Ow&sWA;9_AR2KoxR=W&h~-{xXliwuw0B2u3zs5mPwjt$Qba`E#0KnllRh^Uo_g+ zTxKk;Pfatc(aQDhgz-1Bx55IEPlaWghS!%nLIOyWHv?{XlkbvXU!}i`XoZ)i@7AD9 z#xh*%KQOKG6fX=C3u}WsGVN`-R?6A6oyjcWrEpJ))S8xq&oAm6G7IqRTG1SU#ri$; zWwPCkmVedre?=rCd&jt`I6ZElY2vJFK8Br4yQ91t@yC3u97waPIOR!HupK4@lIUPa z!{q$#iQJ~mY*Yz5HrH?JiWNCNlwi_^ex6@U;sm&$1*A~|_{?%^AAEZ6kP~P5{6tw6nlh$UW%2Qrvs>v4> zn0lJ?-s3Cve7{`ni9^Cg&%vt0tSrFK7@Sw7&-$bN%`FqZuLShmE z^A4kTNgsev#TZvc1T5h|-r=rds`>N4+x0s6|J1#(K5|#*7Gg{qwe=kS#{axq!{2_R zHPW1vmsyG<wy7R(qVbpV7_k0r#9Ga9p#ua}G7W&+LjN z8W!X;;|VQ9j(LGM)H#@<_o8v>r?&oVjmlDWx$@1eB2d4ctKKw~z*Ry}!SGeSr!?!d zcmZWZWeOLsi{)+#8M||n|1ospFGn!^yVlDgf`~UAC~0GYkA!T^!4{8~w$m6TE-8NK z#7&dZ5N}IZHZ5=Zqy*L0sW23tL&;hl6+*Yq&UBqVV@}*5GU7`SZTM4Mg+p9;(xsBZ z*OZXg`c#5j_n^M^tlv|7an5Xi2&}j<3)SazH^jr{BaO3|V`ThR{md2t&kW33@Qv4J zR|d}E)H=873PB&ibi(3Urd~#yq98bIpUl>3`iEwkYQh+20z#Jkaj4H^mUx2{kcbS)7=i*gTdt{8ae27;Btfk1PU%8J_iHHpqb)_AEi=2q? z(V*&8yNjlZ$i-m+R6I^XJ$V4y9BzOxt0?Lx0fr8jGcInG!g|6`g&l~WXS%$z%a3en zJNF}Zx1?`zw{Dy}`?<=uq|x6~{%NWlzYuDN7*-_2V;2q!ZmejN#$n*y#vM}(dP`EN ztms_Vmu$T@2_Z+KCLn?q^*hEE-!IN5GEL15N|++fHkk~$snpD~DKfiubvQ_A&~AD* z?;53QCb-6UP)82NXGEBV$k_s@PnVs)E-^>V9J8p`RC0u_^h?<2VbACJs45Kh(;<>| zACjSk-*!_{=vD;lkdU6LM77SxQQ;5L=x(I%?c`(8TH2_>9+}oor_a5eej`_95t~Zb za)j`PT4E;k$@F$zHoDdf!sT|4W8d?5H-%i;w#0hux&$W7XZ!i*ET8!nlXR()(5vf6 z%lmSU<;C{erHcH;)OeLWIQlnAuRcP@U&f3nG?vr&#-@r%(^-^-P)$7M=C^*qa>ie; zKi%jxnhGldI9se#ES3{bx{g3{%0V(croYC}e8?M8A}huIy&-A~#0X1-!DXYo>IC(9 zMJ-Kn*={9rm44~6pOk`ZJAubDdu|D>PET98B*ovpQ!v&ei&{wU$*ybS!f|DaF{GXY zL)|lqw@q?Y(66C~rghlN4!iZA4irr>y`|4Dy$9@8yNT}mRy;%(yp7+kEQ?{e9`aG# z%6RDx{%cdcq*=6SXP(%64Ogg9Vdv!lNwgb)NWL+1l012_ZQg#AFsd4{*iYb<7?14C z$JhXyc#Q&`_c5ZM5+hR0O0H#IM7aYV|GwPL2n|buov`X5+q_RC%_R;q^aKro-L{L! z&U+evmKs5;;D=BT^zKUq@wqovH~df#m9l`s*7(Y__%=VnoG00Wsw7#AI+R$x5(TrI zw<{EC3f-cQ%}05UC-h`DHwi~W`i=W@g7>+g+Om39!W6meFr)`k>cdCJrTObY1XlZ% zdOTbdQbj+sc#^JL4hXrVaW>i|L?h#i*WU^8#1fOH%N~j~ZTDHXE3saS9_?rgEaF)V zdzmXps)TADl!J{?Q9FR)bJOGpHcUH|sBgw=WJ~(i!$}@GVQ7QDm~pJQtWh3PS#lgu z{;;+y9oIr&CA!rLcJFhzR|qu;7|O6oo7poO;ezimk+Sjkg^_054t*`H%h|m5dV^FL z#sYVy8OQZPb%8|Al$qK1`7L)0*Xm_UDYlMY7e-@s@(o%U+IbNBj``}-|E=rVT1@I`ar>S&=7N#vBGT%lmM01Brcs+~Vvks}Z}TIua( zxMARjXoGF#xgkJ@?0Yh&aW6Sq4?58Wq3<--sN|gD!C^PQ*nk!vAy%q(ul20n0_wF9 z!{#__F8K%C#;Z0ZH5@rQxfNa2rtQLjJMp`|>}8-wO2;ReWA}+f?+Uj@Yi`b0H*J#7 zXtT?LPC8`$*yr2@BHJ3`ftyIozcaTG8nhlfI>LHGP6IEt zqMQ2CVYj^1Tfg9j)UYns@w?ay>Ft7?Q|s*>;`~_1=k@?^BIvECt)Iyw=38F%od90w zM-@G$su>&nQBx7Tw&tMrreY{$I}QF~L%b1*Gz63{C+m4Nl{F!^OG?y;PRj5x6-A+L zA%123xb=IqV7rDsIdxncd%7}QoC)$?d6s7N57fOm;DnS#6n0inx(J}v&IlJnX@1wN zt)sw(*HOzVYQjBMPqQ16Fo%`EQ3+;Nq-LGeuL(P;r?)uAru|tNum^qGgDSdawX3NA6>T`ZY>(CY zdktRL$C>w|c=R*B%7dWd0|w{YodQ|H)V_h%3lvV%<$XkpjYjrr2T(8^p4O>-s{w#vJ`j7mVt}VD{fUqi}d6lr$&g|8g9g0}N z?5FcLId|5hxQ`@@F?txUe)JQLF421#cIF9DBjVNYeNvrlB=%Q0oK9cIOJG_aiI-}i zQ`o}MMmf#wmzgzVNw9~x53Gl7t+@ID6k1_xP2_CW;k|lnj7K|DX z;VTTGjNE%CU3VeHAHKTmj;x_ZJK>nGWe>gA<)OOxP+Mlzn)&`yS68$nLE$>p_y%2< zJekhq&J{`BGx|HdBk~dqkO`#WMtF7$eAwM{f2%}gXJi*2v!+oGM;rFUn7Qc)d8#b2 zEu6O*hJh9S!Rw7T+HAlv`AnqxJ>M6y4ui7=TS_81!p&sV6uXxn8<*Zio%HrkV z18>KNI(NpjL{rCKH8A~6c6MB5dr}11w(38S1$>f;)BZ3cW|7+VzgcL1dt zN9lC5Ob_#*F>7-f29j|k$XCct1cE;e3stH`m?{I`ijFc?l34a402MFxxSSiv)^A3_ z4}L&BzMhuvs)K}qa8G~vhT2H^aCTX^8DckyDmt0Dgme?OT&_}!$tqy2wT3sQ6TUiR z+A_Z7yY1fR)fap+i3FR6mf5ZVky%8#j7pV%@_OC9X@i+`@4U&OiQr*RqP;wpk&JH= z!*jCG%yQrFQg1tyM*K?eaS}8gdi~4pghyaxV>Vsqh%2kjPmW}?X-v{AFdGR;uz8*;3dUB>9rB3AvQIY zHfr8QmVyf5i;dj^Qw6eNZuSse)@`ce#5c#(D)-XI7=diF^v$<(y^LBx-bLnwzi{%V z>bG)8IlBb($FL6;)Fx>wEm6kXsJ5%dzEu^7^(_u$G=wzdr)u7Du!`v82<-9u8V4`q zWJagUd6&_%!;uSQ5hn~Pj)-NZw0|e!-9xa!8s>obdCNN zi_2*4Ug@PN>^n5?~>Pk+mqPO?R|P2f~P z6md{vRZSh}aPSunJB#kIQA8a%3?Z{N(&V#Wd`)w&QlVf+*%h3$6?7`4db@oulr5Jg zDTM#9?O@xo#Xna??0&>(s!wg7TJ^ELuTey?oSH3%tndgeiY>na+7V6`VKiMH6I@Mw zN>taGgrUqVHPcmh(dxyW@PRpuy@`4{IpC#@&f;y{C&L+}F-|7JBnQokKjjTC38GOl zrFV7^W+h?8y?*utJ8j0$6zkN@t>aXNV5W3fur~Ulpvn;y9YZIN^4=|a~B zxrtsT9klU7U`ARef%8%_v8aH##D{QM3Y|&YWuR)s)BX%wZ>pyw%SN;<-eyhcRR1mb z66jnNoHV5I$POXrFU`M*X+K)X8M;VVSLC6)eBs*e@8^7!vS#mU2R;?BXgNvp`eMz( zdcjSE9d?9rRSEP}4krU;&BX05&^HkkQKXtlfm{Uw8BDxF*-P<^juxXK0Dg3%X5N~8 zHggX&w$AxG@?3V3u|F9P>ZGCfe&S+urzh`@ND`{^hwHW8)9&!{7vPcxisS#0TK>nn z@&A`vvN1FMS8B=5!piJ2yzK7CqSU|#RDi{CPuW!BPe-_$$Io0+fFZ3=RUss5Ws*y@yw2RP7134`aVbizzZ0_ zh6MObW}+_W1?DlJqoa7{`{M`V5ZD1I^beu*>DvaB!c(Bfzz6|23sk68fPacLK)^8w z0vY5X_P*=)c`sJ^%>7cTtdmo&*d3 zczOshaLxkAK4oygrIZ)|0+&$#i$^f;0UpI2pdE&{IxJBCP(l1AETF=mr@H_mg`JST z2jx&t!GSYpc!aM-9e0ogc*LjINzAYSj$bO^gF{M-uz>B?K&y(M{d~efe+G04C<3r# zBy@CSAc0%p{2v4L0e$RrFb^SrZ2zG%ETJHtgj@h~7tFs1>C7nKf*41z51~K`dVGK5 z^MBdCyhSNUKmz*vDFTp}K*I>W$holovsx?mx(oepwHC1Om>dM)$Lsra0mt*Ip|K2BZh;wUCivKmvMt1pva%)w2`~0tpD-6om3$dq!Xo zPy`8-4bXsEMv%RjC!()q4MoBLf(__%aL1tEWya?BHcc&jQ%rP{<-idISup zaIRl=)g$s-hU4w$Dlv}0gmoUpCY9qOO(qyX_&tI$Zmc5k-vcp z=x_bV4={1hPjDfycRvE(&hHTz!jni@Xh^<&rM~AEvN{Ux>LPU5%@_TVei_uu{e#e= z{}jIOk$;{4uDL=vKZ*BLHNJme^MR&P+;p^e9Pjpw~25z1aD+YPk!P_0nxGMT> z!|0N)skp@Tc7x4O$7K4%x`9_NJ^JmJt@qh_NIijp?~yudmprZgJwMS_yxE8dDQ zHCn6cQh5rk<^nE#77o5LZPUGVb$ETT^YN_6#x39_n`B23K7a2lKGrlEC*8_<9dg~D zg8UMgB{ND4Y3$MxshL%~=g~a{G;ZaJX&J*3+UJOUjZwE`joxfJJg*EcnuEow!a5|s z&Aovvv^=rmwQ3!~-3P|tA+g+Zcfo@$Tg&@epXYU($tzccn5OycAudD@Lw-0`WPr8? z?|U(OEKNN}yOQXZzljmA1WLm{kHg#0W8!hjp&{2y$9suGdAXF)*n=57W7)K5wId#;A2Y+1_8r9;Lf1APLJ>NB;cQ&CbyN$(-Szb%>?N7Go_A*DR9SU#6Wf!ll9w zFF%;E2@~kdDYJdk^R>^`mg_@?T+g10unY0tuxG!-_tI0_R#yYvB0SEHPDRqJTZTT2 zhZ-b|Vun{G{UP%V$6%SqA>@fh=y!0nBist0KBG}Uzg@5AwUH*U|0>w`bXl3ZODHS~ z6jbo$K=6sx#VVllJh!H3e3JaTAn@fHfYeyFKNZ|V%N_msREg+y-R`JXNPSGsXnQoG zHorxlGiW{u5O|I5a51gDr_u{xO z&iKYPM?mbEGj@A&d5z1!9+o|XiqX$A^mhP1(A zTh52^`@re)hTZrf53bx2I%6xBUEev(7p3DywZLO1Sy^V6%{vnxrTgbPOA$iCM7akj zHfP2AKyz-g#v_a~` zxP3gfcH(I?mKJ^8O{;0m&rafdy4k|pUTR%~)QJwITyH`pH6iUpb0M()0K>BepJm!7 z?uEGA>>Vv>4YaU#b>*l|-}@MfDMAh6jDS1cU57;RuPK!Bpu}76M0rMmT9I@7Y}aJl zj?27={*KnND;h4u&F2eQ+$Ktk{w!n5c_TIo<;}5`*l6)?f-c*U%z__VE5=0@*AV;g z%J7AiT5XxbMtfO(=FJ2%HtD|fL!g-5yuvuQiEc29Zi5KF?DUTW3Gf-fm6)S~-A$&` z$=Z$0c$~WMBk0cHY&I1+;FR?<+|XQE9`#0G2FW5ef(z0sC$S~J@x`!iWGD9cJkD3O zS~e?iu`12LJ0BL&G9KNckK346-jpy=pg;bmSsU=L4 zWcMLbMq?cZlTJs`BfSwLI*YDf8(T+V_i`&|6OU3*+=5$eUj>AMj!DAAQK5^P-SC)r zLlO*i)C2kMxe*B?&!y+`%z~=JA1R0!;k8#gyH#$nPyiU$_-fD6<-}}k*3;;0JFwy} z0aW;#Id;_3cZi0#INSwMd*__BCI=PlV2-`MaL4VV-1b-{N-_X1YhmCtV!5XvJ(N$U z*Lnx1IXz4ze^xX%d4;iion@46L;#N@mu5080dtvLDv9w|0MG53!G+$D7sN*u-B|G- zvw>JuP&C$wg9`AazcpI18+W4kv8Y+uw#+`aHt2j8pL4y7sU@#>w%jN~no9kf!gKq2 zLeE+l!U(GL9bz2Yn0$%+ zS5-n=V}(YgoGsZ>B52N!!<-!r)wPXN$P>qBe z-LM3l?D)KOJ!ccaSqzJ!V`Y;8pS^J177Jw;r#@nWg>+2*-3z6{eP<(?-^BJl?^ejt zjinuozxw8t;#MnbNnF&^1`T>^=TYXVYZ7dTb#wbISC`JX!H`>aPKlzGp>v|M@@N0F z7r{I2Tu3=W8>o*6bM(Wa@RuP7cAVptxx1osR|jy#`BctlZzA>RuYrSn1<%naZU57K z(cO=kbKL&idR6o5)NXru%tC2yG2U#?g+~k}s>kJVsb%)0Q~4gig`oI_f!uVB6AAI0 zV5`bO{OViG$`DbpcFua>EM{n*wqiH{ApXrJv>tiE$2z)xRwiw z(>FXXOc~)FLK1{F>G87qoDw3>i?dhSl0H}a33zwD)DzFH>Z!As=NClPJ{2r*iW%9k zG{%SvQijw3ZzDmu=Yin2WP84?kPU zqTBA&Bpcr0W85#5^-*Uh5gT6HSlxZ_Y{vfA@Cfh^bbY^%hr#zOa_1?P- zJV#$vR2nyA_lzH}GI{BO^@jX6t*^!vb;mN2gxFcZqP6Kw!gGNVG z#E3|qyJis!a7`DPO{W;+LZEIS%hYG*;u~WDBbpS4U(2x|G=OSg{1ge)xVG?-hHJ`a zqt)T3_vqIeCP_xVLt1KYe}?937gWk6-ZK%G<>Hu<-p_$si78tS;4VYN+qP}nwrxLS+qP}nwrAgSlHKek`(gVhbUNvz>Z-bN zBt}GLrC0jeW!_k#_!MV%cE`8WE6aPEHeLTl-WM<^pC+qLBp1xOK}y$?QX=Vv=3sO0 zddbT<53*S?X;Aoe@483sm#d1m$zF`>>W;{ZW>3SW#97LC3xChBY}iAe6-&B~fCxoz z?(F&|ENTigvF_CTv$Fwf?LBg^2;JLS*D^!Fk~7V#tE2$b9j4&j33fFagQXq)eh-cW zj2yJnjr#b))5P5d#>^knXzF;G3iZWN_Xa8?Y3wx0rd3(m=<_6c-ODm1dq}Yz66@}S zVJ7>xXlu@~P$;muhJ`NJQ{jdOLiH0`I;br-Mx2(;brZr`Gf@0{8z!^FS(Ze321jOO z^A5`Q8{~~}_=QB$kTk=NmLj8O5U?0Kbk`%-VWNKvQ1#LgT8@pm-h-5!nmi7La>veim(BEr= zVIC2|`Db!rjy(OR9mQv;U9~#2LtE|DWx9;>6XI$Klw_MojfA z81U>ZHOgK&n21%L@(uf9-V6L-^kj4&f>Hm9?_NCJ^76tBnTb^#ehc^E)VdU#Q&nRV zeP+~hsp-M~k%9FMQmW|T<{rJGhk?d>eB`gFp<(jQ*eUp(xgJb-DVgocz}UTU!3QfW^RM0_IXa9ZU-1r~l8WXhMZA6g-a^M5!>p{ClR|#aT9lp3uhp_- z;({z;uj^UHfwOvw#$ju1%kq(t9u$91k?lG3O)I_pZs5!fI%Cx9H*w=GIbiDQ+@6Z) zJs>>M*lzl0B7)836kZzz|CD93<w8b zUEtGmVTu?EKM_y$5_i)ptvx@fko3h=CX#%z-Kdk?fm$-1BZi=FCz(iN&|XRUPhs3h z3ke*dedkiPC*If0tnGn^N!D(W>EOQ*JC=r0bx1=!LobRCfR0X#ycY*@)azUD*J0gq zK`+v}Fd`1~sSyLs(-p|ln~M*f1BhTdj#WLAL_@A{5Gc_o*VBK96#tHhIvtokA|y*! z!n0k=6i>&JB+bvp-Pz!QHZY`a$ARcY+MYlmU^ph9?PAcSvXL!%P$QM2n}RoNa_6Vh8(mwP*n zQY0)`p5i>deOjF;46L192yo|`85BjYrG}{!Yl$6rtZ9@ZX=hS!35y4>A@#Wd_`ZpE;tOBP2w6Cg*B0r{JeQ2 z$05YmUII!|`(CZzCuzvJbxaxTDmCHiffgdwG;zRmu2vg%6#Rh>oTT$gdr4~N!lu(D z`&k*7-N5jP?T}R->nVPfYzYbQ?rrY}dRjUgp{!<_W0)U@(8idqe3YvB)tVpC@So91 zoe~jSMQ`9ph>aSaMcdxbF8WPmiaBYS2vn4MtdU{FKd?IAIub=D^94JOMdDw`0~(uK z!uk_XdVUf`RH5@9i+s(4Q3EZLTwfE{;m^e!I}@f8s@M!ZM1{2h-Q0ZpW)f5_ZaPz0 zYgb9NIrk(FJNw-S>BY*Jrr!Tdf+t-P?|6A8+eER5eG%vzHnh~6!zA47p>Saw3i)yO zkeSZvd$b&5AP&-tn2>NNc9yL}!cZxuU*O`eeQH`3Xbje-SE4p_3&&7d=RJ+Y#$+~>SfjgW zL?xz9pX*nlR;>cBMTL=_lmSK4z$ZC4sieg`#(t% zL+Fv>&bb*9v>LS|o~+$5Al8A&7%XU_Mdo7|x9sJ7-#Y!3q0|p(+D1r^Cad0_l{3Ew z3*>c$y$!@nuIs&f0R;}c)p^Ici#<&7E;$8ls!=r$TdFj=)T-8l{#@pJC>*NN$7Xe# zd)v88uv8RzYxyK?irwTNiQiXr#V~61*mB;XfjJNMjr3_*1xy(eq zQ>zn<*}>_a{X;Hm-+Sn6KY>PV#8bs31LoP>)i~p!Ll@=eK4W^?Zs5z_nFsfd6uR7V z&$yOM74fl#2wQ`B^eMZ<&1v%F<~G31pns=p=Q}MKF26Ca_e*=Wl%|}s z0NuIt&kOlISEy}(SpbSwmmrUx7Hq?5eG}LX5Gq$d+jp`kq`ao;#o4Mwm#5dYyyG6F zBL@#QDJVOMUk97V=C?w`D{9t8&!bV+PlV!j=*mTH5sx@Jt}GF3ZI4V@)Ds4Xo*Sg! zmfUVX#)>+wDz&7e!!hdkN4JAFk{eUa-8p-fP7nX)kus7{tVL}ZnF;g>wRV@}zd^jM zz;4j2XHfsj>F!}7!37#!YccBxz#rj)7&)c)4YFpOy(jB?_rbtyy`B8ZmZ?mZAtn3J ze4~rC%~y)`XeYmkJ-136PtMuzC}l14$%Q#&9y_VT)al0Na|dCU(oA7}|HS#f@;Cb(1iRfUc6NblhLx26Wz##%iGjtp?^UKtGAxy8Q)61ErODqOH1V;E)w7( zzNO$WWG2QnvmLlTA_tSjw#U(R(~{<2AYBi%q+i$%y|L&-Tdtei9#V*OEc70-UvWoO zdcKr>{pT^5zQ>^>}`x^6cX`EAkTxHfqq@i_b){keEk3N4*l<&xFG^rFT7Scup!nz*u1y*h$WW&+#x5oyOuYrSRw0B zHZ+rqwymdJ=}L54GfaIY6RTg7(j!-&lT5@0Q?=q7kAa)a^Wb+oY&d45IW4Xq42Njd z9#63zWJhvdcol-Ma}q%|-2{jq)7N{d%55iMV&2V*^;8f^JTq9a!#f zZ*8%P^OSnWsl%u%f^!>F3OY<=^|oVV_kIqQHyBcLH1bCpnnZ9_=XkfBdBN{hDsgMw zWMR=d-TLq*wwrqWGL~tVy+mV?C0_>j%x_&OY~S1iOQOZ;b2`Y6?-v`R zWfiCA8ipKv!+5Gzp^t|fpngJE#UqR~uyQ>2qP-Nuimf9U;t`4+D1 zJc_Dv%NCy=P2R8hYZ5JM#k0^RMA@zk7Xa!4(~4iAU1;zKP0WNSll~`d*`q1+-2``x z($g2}ZPX}e<{z64)(%8%YE9o57O6F>Zg1iNfvv4S1{adzAq^l^P=oO0Hl`{NpDbX} z=Y(xED!)e?s+jA79t7pEw!mX#+uV=BA=IYOmIMOG9n~87M9*YBbWyQ5=(mR?G&|vw z8`mIUGzQ)E-*}Q~UO;TF?HYD`Qw7p~g4TrL!oi~-dACa{zIEK(J&qq3PdH*zLJG(r zVp?kWlTicTH>jfBWkgts+#S5ocJEmF?@?hj$vKKyhdevq}O6Bc`I zTi+8(kd%A%gQ|~xo9tBu&W5DVb{Y7MUGi3_)ly6ajCtA~PQ}dc){G5PklZs!S&IZC zi`|F1KG->#4ijC#9ZtBJC0(e2HO~LIs@Yamwas=oPj_85(82O;YQVKkO+ zD?A@2??_BI6Ga4;nbJ1$`WiQeYgE#xCih7l7?yIT@!*Iz+PDt9o`e{pqVn*!Bt!MYpEXX2T14NNt^X@66Dt({%9sT&J$F zt1g4dn;RU{B^6Ra*hhYO(=|vUIdLPif|1XJz9)iuza4N~u?mxJcMi3qW7ae4aH#U- zclLUmS1ar$oasffC-nR(I`8smN_agh*&+AT#pidm9G0hlMCC_b!+{e$LJdp9;>1|t zkwjcxA?|6wCtm0J9CW#iBIcu9ycv*$ul<$EG+2>xO=51uzx1nPFO=8q+frcUps06+ zzcIGS7;pW5Tb0A0UW-OL53}`gc(Kjdvy-7hpB!Vur56Cc*9%^r+NM| zg3mKPI#HXFW_kW5yhObFz4V-G(6zyi{zCrkPpU&!Nhn7Za^3TG`BXQFhT-$0UQfp1VAj{^fUk&9UYwpQ$T%gPj+x0wx4Vz;qyhGPE^; zfQFu+vXUH*03}@+6ax4r5H3s&Ae5ZgD%vOjCbV%u4RHdLfz=s6{qGo%qC!H8{BZssYjB0GPprwdcj*1>69jwT-RG zllvfi)@5&g$xQAH-W&%{hu#274bC8(SlsIC!2$UisUjkNg2~6X5uVlU#G~Qa`)8kz ziX(Fymh?$~ZoUFO?t@@xZEyhz3aHQPbi@P#B18Z6v=nZM-@-F^gGcvSa&T&`XZ<7t z()I&dn4KFK!MeD72YmGZR&oDTBj4{;vNyN2xqWeO-rgzyjSK)CLo~CMgqs@uj$6Wf zj@z0ULQmbV6-aDmZ31Rs{28)4HvY-yGtVE~fhqp7GOOT|z{p;h@`gAk6 z0bus)+sCvN4*j`q&yNmn-uOejuLWQB`zL(+?@I*1@MjTPSicfN5c3}K$DqX-q16f? zgF+`XaP}|HfBsYU-%_=e4S-ukYwd>@{j(zrh>J{o(D$J_MH^LX^>4|$!KW*zndnRU zL%6WNgh7(ZGI|_j`Fku~jO+~JD|+Vq0}d2CFoqGJvU|nH`TIwu$({VS`3*`=VsZv` zU+~o)<1$Ty1MnIzAxB15_ec1!#~|~!J%ts7L!$$@iNC_kyL7u|J65!3*iEaHAF|hj`;+lRcclo zXvOI47ZYvq_QoXm3(-h#sX#gXR?~X*APKv3Sjpf~0a;|9bL?mBoXy|rzmr_Fi+?*( zUwSSHmUnjGnW*q`LR^>Z?l6D-#eAMhP@G#X?xpjcv=OpsWUjKcho`QxNbUC#!oM+a zcz<#TpGqI;+)Iyj(ZgHR<|yS0<3+ASj3H$^@BryKxRCE1HISIkALdnC@b&6o*=TGq z?g7KtlN=X`BxnZ)*9JRkKQBLvL$}2f1t^i4@Jt?Alc+aDFbF+EI6wwZF@RtAi}Nv) zreR{q9L2X0T)=dK19O^K#G^k5+CWx0@hdi31sU)=f-6G|gNePOCLIi%bz6$aVtbm7 z)?|Ne{k7mzTjE`dSl!hsJJK_y1~~&z0azz6EnQ% zYY1gOnE_g7Z-S$y2?Iai9hfsB^Hy?n=qj6?$Gg#>(uYx5Y>a=uYGHC~VHkAs-@baA zJzq|jeHj3OXWnr9XpZL21YwJ7ga6R5Wrd_##l&4ME-jz$jM*oM5MYDFqRRDdQ}Niw zXOyu|wI;KM5DPV8S~hn^2T6ckxXN!RInw5Py)Tt&6g`18=9%orRQ?NJu4g1a%5+kJ zm%i1$tOs(-c+A`|c@W2(ALMHec{L;!!~v~7P7EXcfyj|Pd{plF3p#w&ZDcV+uJ05! z$HlQP7}eWvBxe*ytF9<@wS^N0UmWw;%WOmOFHot)VAOp2*;1=moJp$9a!wH3wOo^H#cz=13Vuxpm*2 zZWl?ko1h-JuXN%lf2a~?yB-FFlwss{Xx%2-yPl7j_b_XSqMqF9!hdDZG>DqD48QWl zqa%1v$_K8Y(>qcQp4xigxOwH#wO+^}~ZZzP>(HcpzNrjAiRrfG)S;@2`sDfibcd0bO-LL_a|< zi&-W%15@2l?LH3D?f?lc_UKZ!q~Aii(o8|^Sv=0EZUS&foEL&v@L^zmi0D!Cl+v#Y zz2HJa)8mE0?RVs__g^@)%fY)KVJ48bdJuvpK(_>Uuc_Kw8t{%r012gEC(@1T}P32JAULk>`rtIW6 z8Fbud;D(bNNh)GKqOvfl20%XG5N`A#-cWq6@?^zB8!38f zq(=nzX%%gf;EO9B+s@H95_dO%!7COwLZo;6Kpt)GBWA1da+yALs}az~5#nYjud7zx{8(aJDab6)MDp9?GS?A6^>y#SxLJC#4PUJ9bprYZBjaj8Bf z_$eA=;Q!W7I;ISG3=ykcVSk6zclniH&$|ant#zFii21-~e%JF-OxA>iq?BBgqF4^v z%iAsyPUKFf=0EwYL~sF*d!_2}c7 z)AWar9I!$7@sUa|llh9!NbGC`zsPVLgI;?N4)9?~wCRNn!1PRV6lK*YVKB6E4<$v( zAo3!32{4=6;p6{v;`iMj;I?rx35}jTXZz{2q&lAL{TkC!#fh_7@kUSwOhajLbi{;) z!q)ZxySUdSSO2;yj5eo~p6O5$1gXZ7n<-qQS#*-5qtUh!dn6EB}3VSSX* zEcd-+C1sN8uSMXP{~AzPS_1-+@Zv_w9Gf3Ugv4I-+M8|OFA{AAueQrvy?(~u;=It$ zo~LXY*>r9KhPQ{t^KcR4txwsViDM*SK1Y!Tc&A2cl(6`08yIkl>?A4WF>c4cMm4N6 zjED2)j*d*Px)N}-a%iD|V_Wq0qrU6+<0Ryhq`;b7Q5}M%zS?uaW`^^q&Ju>y0`29# zJ>S|ARo^#?x_0;f>*!1NRy-JKH_Zm&C^7a|l_8_JjXF?x5&k?=7QPou{{$j5xT4B| zAk}a)7{Xe$m(DZP`EcR(3;)hN>@9B?)=?0KT6gOJEv`C84U{d$dCd9f=o1xi(N~QB zc^o>Zu9Uf4G}hqd$y!8m2mBP(Hw+>Gk<%G+nox6@vaAvX=kyM%oPMApWUBh=cKC6rv4i@vLI;?^;lI?;hvAgIKX&T4h>{H1wcpt)UHTm z5#4hf^Q}JfH5SvT9|;*3C84WvbXF%lU@xhva~1?sNp@|fe;pKg{;b(XET zFX5|=KG{D4OBGQXsWm*B;>kw&v|ARd*$3$JrWDe1iH4!lg-m~G$h39~WDo9w9SS`L zRIb!u*b!%EQbS_wtw(;mcAq>VljUJm5SoNQBko{h)J%EO{kBMYQwdBrDMx{bBqijJ zm8eYdt5ToQ`wCYv)+pJ}js((dlzkKaMq$B|G-2lZX``(auYzvnYQETl;+2d7Wtu>E zZFJ2)o@t<>`$z_Ec}e6b$~@MW!}T=w-L(gCh4c$<6dit3+4K2(F13KC5?U{sUEsmkg}N#!f%NXg#gFf$Fu z6fomA?A?lx8zg}U_Jz&G31R47^DI2OC~hKQ1z&^Gs>WU70w{N@pl&X^GE5d8GHK<} z@4#a@=G=aC`yb@nA_37R&lp+Gtpf*jL3Y?eEq^kcqm|X)E7Ue6v^rg?4H&auZp==_ z^k0c>QVT{#sdX**W;h31ZOCpr^TXB2BD66qpZ`m{+Y+|TZW-7JuT2KRI2szKYTGmh zd@vULT*nZm`qH)HE`ymd&awV>H7SQB`R<$>h#x4#z*iXV0sqtYswwwRq8n0_-ovF% zU2_tOls58~_RYKT<&(n8sR04(Nf@9;* zUWbgVxcO7(a5mpq^QjcTbr@*tBjt$a;-`G8C1`dh8l7jY$%qMg?|BXYlN_(q+%pzw z^)nykAYocG_bp_^wKj2jLg)5ZS@@cROigR;byv`YzEz5;;E_GpGn@Siz4N-S&q$p^ zwd6?`GSY(bISX3SHYUmhW8O}sxZeTU_oL{OULfW*Y z(?n&CDsy1gsjQ3ikH{!an5MhVf7v5YBq;Qc%%lX+RXJLl0Ubtg(+g>GEq2~9Ou3~y z>Ri>ThLtvcs?iAR#5!F>zK{Okg_L-Qyd=K2cJRDLWBw;MIOwy`G8i!t;nz{ zy56B_OBv08djqe$O@Q9xhTxTEVh&(P>8i=FBg*3H?yz`IDD(xh*XC8dz4Kv-!JREF zdLmZgD9urBQQZ`_Ka;+nNz3=E_!;(NQoJ)522~nS7%nuP7f)+hLH-uJ^UjR0*3-mG zHWM(p)kcRM?j=9oy$Q+ajlF5e%DHcyuf*DujmnmHyEjl(MQ;>6QB3}1hiAgRkQDX* zpl_jXb%l51zhv<@gU`hl#Ns{ITZNQB2>))Qg$onCuB2Rw_NSBNYFi{HG7Y=Syf+_fRRe>VZsuh8+Nx>aa6TRB^!z zNsWg(qJSGhax0;GYoO{(7QKS{NASqDNX>eVVr^$Obgh{K+Fe{0D|jcnA~P|PjOL(< zt>Z>2uq@dfNGi&&ESQW3JBysb;B5C}9#cyq@+)``a7(X#GJ>)+q4P@#%umu3DG9PY zzA1&|dNoC|j>%Z1dyuZTs#ZJFZ>W}j&6!HjQbk)`U6!6H39w4k)Z0GZz3R5AYq>lBE_{;+EN5JE@Ka%UKV#5cW^DnCW#^v5Dwqe%uO3EiXyU3jwresw;JVr^3b+q$$*tZfmZIp>sqwZ-7oU>=w!xQErZjI+*?SIw`@tJg56o*rjGxWDErV%J~`RVodjzrLDn&$|cG27ZkFN-l0ueHp5RL?_ZXKLN+xBm>CS7Ap}Q! zqBZVTvzB4td47K;UF#_g5byIW{os5P6u7t*)f7B_GM(Y}SV2e3dpZn&HZ{VUY;w9Kz6dO3lsH9=|MWs+g;fRj7Z$83UjqCFL`~G|EFW_~1dV9D zk`ki0&@P4;;%t0#7}F%@&pzvrmO1|Uzwalqf*|5NGV<7;HrWd4ugH&mxX*+l`xzR) zsT=JJv^hJ0t=T;t!h5KA#j$yK9m1T%W4Yx-khH1fW&AQ0oxpKPecXIT1By$?|YF+p(hFk9Nq zu7U}|Jo9Brm!&Xx_X09gGoc!ji%V7MSfWl;4z0di+d^CUL`^#JiCZT4uMD z&ZFh9zdnP=4GDY457$Pj5Fp(E^L{CXtJ9VrenNmh=~^U4>auKa>kQ@U%DG=%#VAto zOti7{?|rnpJQFb&spHGW!e*ITG6c1(^lr*4e|5Yp9^%r;)qN{PoiUc)jk_Y7G#Bpa zAUUU$JO(La4yzA8O)!c6*IfpI9PK5~h0uIZ=O$gFl`LXiNt?jgVYOcQZ|P1CtNVK@ z_muJPf$w5&uU|mdzx1hd-%zD(2u+Z#k0^e5c7TMw%`NPBJ_bNwxN~FLs;1KEkpp~+ zMp8iB>>fxqkpA)s2j+T_a`$H@JQsr2NUg-2tIQRAQV@MpPaHI@K00R_aI*IH zOFOMTzq|Z<31#BPbt{v#{Q>!q_K0xK7>Q#?Z0hP;_!?ev-$^U(<9OoLybAVDsthX2 znurhG%phZz`x&#H_H019*_!v{sH-k-Cvc2YLw9cI_`+oILHfhkyCK`u`fBTFU!DAz z=ZJFkOiF!u!t4nRF_QLO9zm*~3$z#b^RQr#7nvat#vj~b2Ilgp_q@f`C)?SU6OY;| zvB=&u2mN_NBa6u%;BK`@bjYv45>x%WQzH@%IoPQgQRkV9)OaE1ip|>SNz~oEirzKD zHiX?L*PF_05f-lwt{k&jI&bZNCgQl%S*+5S*y=@0a$yZ7DA(-;-ZugR`B_5GQ1AExwpsplSr=9y?=|+bR0}znB*Dy3D_@NNoV|uQZhNJi68( znVb5MUbdT+D~Y;%D5->njNG?uvg{VGw>8W`6#Z9LkwOG%$3M#Jo5Aj>t(`z9Q4XS; zR--|Htx>xw%4&pfB28%OF0`Fqp}gs_G52t_L1={>g)~udx51sgiVCCasVqDFHCV6? zaiz|r;l7`D$EhGteAKqZ6$&zD=&&tO&(Q7ckvb;;&^yIIo+`83-b?&;EJd#Li}hhe z6C1{bKEiiRrYD^_hpb$hQKYdStW7oEL}XO~quP3f?l|yr1n%0FC?S3Q*Opi?CNh_R zJ)^U8UD|Q^fiR6FA>9&7db5tqlvQaNAb&S!U`{e+&EaUrfGJ-J=Z^1FIUUG(qZC61 zoxax^6Eu2#zHRyf5CL&$a6T2#*Vh2b+3q8wHTPGq->?}L^&-(D;^O)Fih_?X>jiMb zMYXr4L8x+eb(NgOA-Hj6cS6zqpAMWDed}N{Y{zveP$pT^B`@emFzsvfP_y%AFZ;5U zhlq$oYjX52v}1(McldZwi@RDkBMXS6*Vsi8TaixII|G|RR56o&6a4FBtp?ADDR;`s z!4%$_oCoPvRkKw2bA+LANUDkK^vR*O=~n5I5|dMFE?`<-pvFuYXBxAD)}pnG?OA-} zIYU?gTH<)dxivGWSR}6(9dlzn8^63IJcW;Z)Ul+5LyFe-4pmkz6cNLH9S3KQd>YU4 z)%C#DYB~~3wY`Ed+x2gC(}zWqJ0os*WOKVVz$uPi^{GUUwa&wnm#RY=q%l)aM_G^~IE>s>>`&Z~2a17QQl7pR^^2A(7-mxcij>l1)S0|XeDWYlO3c@P1f!hkA}lQSaddSD=iak^Lgur zfbkZY(zYYAVqBW!KcPFd6b@mpOTVN+RtDe)*c2mj%i?HSq!I@!4pLvqrrm+ezH95Y zk;uchf%5c(?to>>a1_X5WmG3f80IE8Vy5`Lt!c8`OrX;u@oPcRtGJ28jokoMF!d2-MVly>TDpTy8|j1LHV)&v~1oW2kQT03mbV)D%FQ0hROK|hd#t%gO6 zy`*F{d~=J4NGG8!>B4>;LIiUa*BoV=Gg*^KxmCp>Qf@Jv*YB{QjzB3#TtUHhZ?rMw zf=$&U?OeU>9Yo}~fweU((IRQqn~xRRCXaf;!7kc$aOaV982EWrBvW!Vf}UJWyb9Pq z$@Qub$3K!TY#OM_&Kn8Wy@AoN5E1neG1Zwg;~$Sd`&on5GDB>sv4>;p0rfQSbbZFo@Z<&(*!?gqD!2VCCu#){8Hz;uSSGr>LfF%BcMCDCmbq-ncb}VCkJUcy$)dUVX6Gtl6Wc9}ptFTvtG>BZh ztagcGq?bsnH?QwEx5zbj;PP|I(+@k8^PI_s)5bWf(9yBt_H!(kf=stoyv29=4;Ty! zP1Vg*4LZm$PUwJM;N2V+%2d^S;{D2@({g=cW1kuL!7z*ma93MoG~TT0CGl$$GNpkT zs65Fh1L^wXS@%h|;Nq08;@Hq@+Kq6riTv01j7fko+UCeu4XuLL(^i zkUNdD#W?ORx9%|NeIQsJVcbQg7|4tGW+Zkb9Sp$g77D*p4r_r>&}s3bLjh%B%`^`y zFQ{go#Mm~>)h0D@iyQ3)^FaSZiFq-(cKfls!z5|%mzk5Js-PDBP%Pub$Cs_g%=Oi-!)_?pX6HG770`+6810S;8^nR+~Hs{uh&MYztiSQKl@5fS<~_ zQ!NT2e1?0Ca7qz_TT+0km}x@KGr@O$KR@wkq8o$Vl@+$6E0n1Q1IBy*mNM1If0={d z64(bsfo(XVQcXnH;AwRDp-4Y*(Wb>F>BtQ7<9P+^dPk!{>sZW_I;8-Uhn>Z?AlZ@` z7BZT?I3nA8ZP6>KfWh$OvL|Y#%*;W&YK80gfPA63XdX^uhbO=3Ao6EC(9nCiriH*4(N%{VnFYnIMsA`49Tsyijjc!N7dPi2~i z1=Efhrc*c7x}(v4ab6jBZD;unkn9v?4&FwdVG^uD?rByaKco;Uhp1h!V}|4>^LqR* z*f_cp<~>a*Ap}daN-*S|$4tVTW+Z@TrP}OtxC~uP+d!DdEU|ko`PT4;%X_p3&)RYe zz*Hp)Ckyy)P#6m-M&_+?x8qtaCkQPC5-Q-T;6c!IyTSf=mj^K11-R30NhRmOEYXqBEPBxlIDa z3o86yCgGZe=UBbkeF9CNf70i#GR=OcYt~=pmL)!#KRpaF?Iq_fw)G^WpsO zw68GSFS#H?!87%l0qW-ONA?i|5Y{jx?pS55(3ZjF zC0x1ZTA6;_t%uUCpp7{~bwwcdbLHKdJ{npkSdYx0?Mchuf0LCsoa2-S`268B7qzGg z8_KU6(PzQ27ejx$j2=A|Rhc@vy0?9PDAyy4t(f%YT=!nM`_gV$`cB!Nkpq4Sc9sfd z!2bSc6_jXD3MUcM{}nJDda#SAz{jpw!)*vWXjWh} zZ4;658J`f6G!H|P{!34eIRcUF10cDOS$+5f05n zWk0@raK`Icj7Gl_eSM)$MCN1~{Os{ujg1a5h(efT^LSe+y(QC?VV(#MbfY_pM?G?E z8hZ+*o6~`z9}sSY^YXUqB!eUg9J7FBC(^6fo<)~5bFwDJDXbuhn>!3%ruvo(N#?q# z&P#020B}?{!v2ciBTFLsV0}N=?}=RcYVG3p5Uo|xJXq@GfH&iN0&PtrQ6BU8X+@TM z8wcph75xU+g7ma|o2u{!(PFP662}l2X=^v{Nh5X%sP0E7|ML-OTI%01Ov&D0u z^kw5B6Ka?(wcYR9fcuh%|Mr9N-KQ_gc0vkyHAGhoes(y&85kgJk|p$1Yw_~0SwYSm zy4;^ib$GGl?FtL++yj!~MGp_#7ngk&?YaiiLNjEb94hY46%Il+0p4cyv;{Z$Pff3j zY10o4#X!K%P&tkz>wSq`lSUXW)*nMTa&e)5_{&T|-dPowx=q+xwF%j)X5+UY{n><) zDbH}Y4wrALvR2^XC=vQ*rPWy0;mA}^b*jt@R*5JHbVVAh!$Bhr_-H{QK@IBx-!y@h z=w}(~xBe9SqkhHM&s4AUY#mR|%j4=`_4br$H{!kp$Z(lu{eE=cM736$p-N9aRwFyu2@LmMG)~{}MvYX`QuL1Bvrp)D>;!|u(FE?v`HB&%m3kUJ@=HIwUbj(-yhm!8 zZS28mEnN^^V!@0{xO5Ve@W9tT)-~F!VH7Pxo-ci1jfgQ7&M%P_S^jnjs2?Xk-H*AB zYueW?5+HVk4awnFqMoOF(RgDXGUJG!tl@dQW9TlC7n4|CO9*3?8e?n^7UQt&{t*)_ zk6n=3MU5JaUdX53>%EJT2>YxegePAQuu*>cEccMDtJwglKn37j5;u>b^+~rDH0y*z z#~`jhG8-oXH{q8%n%)Tf?c$Z5x(LzxJiJ`b$gB($u()gY*Q4^=Y!0* z+KfXg2fGVl=3_I3%lF#qPk&Ih7z9(Rn)I#B-fh4q98CzpMWfM7=}-=|f6Y%5SPcaV zqYFo1hrKZo;OR##1cwG;OUu$Uv{9VNIO^tr(Mht|e?3jYsD1IaN`Yo(@XG@iMvZV2 z;N?AJniN|x_2SM0C|Pe?_T6deG3e=$M4aXgBE3^ zR*NG}LvYUV;1%W*%5qoJ`iX1DVQdRut|86P*{t99gC=RIewqUwdO_m`gGa6OXP29c zM6D#lqCv1@7mSpLg# zdz7hL=L|Nq^Rvue_WusU$b}rdP5#8>ruUE}WBS~N!&&A$Qg+^q^oZPdq;^9f%<;7O z7skg@S2EIkD^{G-;o~VI+7@N=Qf_TBB%_R|+wT#{7!sh|_sadgs$!o(jwg5Iujvjy z7UIgyhDr}ix-VLfSXLuvb0n5WTQSEjNIpdZ0nv<&Z8qA5a2P~%oor_gE4Bsud3pY^ zksdgrYrZ$1Yo_5RNp9$Wtw@4B=G&1eBV(NgY{;K-|6lrcW~j|B^5UWI9XVPvSD8fGAf(5YOwwre4)tt0U*=_?)$wz^QQ**(qXu9MNcD!rjx;rC^zUQSSn!2I0hb8(JTtLTTj$jKqnBK&&FZne`8**?OKHFB4LT$^ zXwSU|Jg&qb;Ksx5G12zENQlNlUN|g=Vu`RZ%mJcnaV(ys;v=cB%Ed2Y)@i}Q9khGl z`)QS#EHY+&%rP56lY4cp54_{a;jP6!vbP{d_7EBDT##A7uhNpKO$YVgRvrAH>XT+a zPtt<7{iNY2Ti;`pE96Tyr_tHUDWI*0?Gt2OA2zEQ@A_bqDL)_ozXf2-7^;bYN~67r znUkC3{|xvgB08m6{t^6`(&G%(fANRjCe!lP7%p|XNHcZcss|URC5o=$ZFTj5RjiG% zQa!J$z(7|OJ7*HrBlUsiUdf`#n4KL#+f%GD7!v;$%Ne`8l1k3QE*{a=)0ir4T8BnM zSSjDV{rC-oLpW$YFAJ!+`?FS^Qlu^4TS#4Cl76t8LsR z%Y*&WQdZxqNBOsW0OO3QUOuvI2zD52-x?8%+RIBqE}bf%Yv(k$mSzA|86-SKDuJFR zx4+sH{DQmqnsaW^-b(ZLHZ$N5SpgRqpq$|_bBXdrN7Q;T1M^mR#|Ru_H6=|@9v?9cr?@R^IEwR#<>)?07w3Cq>x(84%Xt-P4~p&7np z=U^;8s66(?tR!5^!oUn%yRp6^-E8(0!w-H@Vpwj0k#I+H{?jQ%p-PZ6t#Q558KJKa z1;h2>?#EAVFpuwXeeIIMFA|f6T^d9HPkS7z$rT;(DXu4EG~rnFdw{9gt7)3d5o)&p zF`f4utMgKrnCD;*wQ^~G{k)85;a|cr?e+@F0g){wrefdS{kYuWTvlzdI*AVxIaw|P zTRz^`fxul87xumt&i?cVI(Juq2rYQLWSm=o8-idCWRe6Lp~4h z*pwjmrYkx&W3+xoljB?Z_PVno<<2}dr`BR5zJIVhDqx*~)3X{X{vaz`@N)j%a;-Y| z217tkBbJ}hKw22kZwCRhw%`vu%ckUPq2)v;q+Z=_5BZHP z5ge)cdVZQyfZ&-dEjCnHYg{!#xSiV*kCpQd2`?oT97?lES0i|Rw{mzFH!F|qV{f*( zoJ;HrF`8sE&GlD#`HlEDCYpVG3KY*Jl6zl-b!A>cgq4MdLusDGrfOY6T9|6Z78)R7 zrNM?>P2w+ zbvMcZRQ}PWo$eBYT@QBrGbt-{M;y(v05fMqKhu<&G3zGj~g%f=d-Bamn5FF#g@qA@8!+-m$cNrq%IFXDqEA zwM5%!N*oQV8_C;fIUpgEzV>taZz%oZm zB`5Q4dOTi;XNCT?|9pIEEebl8R*ds^>F>a zG1el@!73;GV-&|P(9M8>&_3i8{H5aCUro+OcWI4^Oc?suxC;{5aa;Vtp!;u0_7Uwy z@A2P3ioX-UpdnzXs#pvKdWNyjhT$2ZuYfdnschtiWQwgVmEM5klk)0BIbc(@7G<=oeHrE_ENx02FG7V zaKK_$z~I^Q5uiiC)Rp0X3Fl%Dywcwp^xo?PlxiJYC%blj-r`CBsC%g5^OQ#MN=8jv zm3|TU{C)-hFEJa||M!>;$NyY$lcn5E zcD?93Z_7T2XakAV8~fIzc2fy(OY;+4T(=CN$Zk3_ z$RR}D6qx-NH2O8T{Hib74cvd82j~M#@qSu=NSzV?d^_YMc$%Op@OwW*^F)zQ(j~wE z1dDLLV%#9@YnGLGyjYkO5y`>Q4M-t;8S8KTD zW_vr12~d$x(a<%vHk6TtE;36d%OnSw0d`{JWEV zoQZ&=g=UD?u*Ez3HSy0CEL&(RGwcErmN_8oP_Y?1X{`lisj@BRUvC1pgnsA}@b}Rq znVxUr;H61A!QVaTl6#3E$zrNSE~ZMOqKZO4r?3cT?frR@UgQZSUwBsck4Z<}?d-WX z6+I#$l9lZr7Auml!oXJY1&fIT9SX;JL`o zHA>1-92sD!AK7hm$~$@=HSfSHJkoQKxOSV9XJSyRBdKji_f-#W3z9S1;#K}Ei=cFr z^O{}0{MoZY+8maC@7bKupT1EgIg(#3ZHwQ1Y4bUi+A-9+sCY19*gYGG#eN9!^xfY% zgr6Y7$PU==A8Ud1$R7D8nYi-QS**)p&2p=QlgQU}^Ipy<7Hxk~;7Co%HqtOlQO@8s zH+`r^=CfhGdt&}b_EXBCv0RDnwxgQg`U23zI$}$}tjiA2-d7yX+&!M^e7&&P6^yR# zvm;=DAOVT^aG~=0eTZMLpMGO|rTCIiymT{itT~^yjD^*`6w4^}K-w#) zw|R|NMrUPhUpij%w##ymYXDtoCbz42H@do{ifRR(GZqCtwQlq6buL$nf1s3@WkwCy zf~W!7*NAK0e*Q)21u-p{)29i0oF15^@i}`Y{pUZeI)-&RLDi&Sc)xUIcs?h1xR`ra zpFd@%Ahh;v(~*bSwPJ2sq#zl8P@?#K`C<3)g*kj0#3{ZO%sI7OW!kB=IB}@WGAdP* z)&bt>UlLEox9BibFDK|N`A%TE_AYEjqCK~_5EARg#Y~|Ho4L>iCtA)*DFJ$ zRKME4&<=Hqhm_Wtvo|G^hCj^|zIr5q_&JoeM|HG%v~#WryIL2H8u-NPfQd;DZ(RE_ znkW94*Q#79_nR0DhTn;5cY6`^lfNI;^*TsPM*h6mca{g(H#?d-{3bCHL$cW*HB>fdO|3^Ju!dlzwcn)xLhK~f~uQy64ZA!>Y%uvn# z-0XeFoMs)9ku|^y^Wb)9n5=ZtS!Z!T>F>G#OXf^|u*$U|bbXKCMSCf@5xKUQ0I6kEjApfQC4s%DZrzwZ!KC8{DwV?d|jfG8gD9Pl70}X|}ZtVT!%3Tg`bF#SiO^z=KUb14PL~?!hv<{8e=V}VHCI88| z?9hUqcK7sZnmy@{<<&mgz2RHCfBY5iupls#{<|}z6a=kAI?@&v=aDRnihW(+10G0K zeN|nsvO|J=p~1d5Lwp!8nvgIs{A#bkRFG+Qz|}Yf^H?!yupKe?1+WL#3X5XyHJ_@? zZBv8NwA5u#*b?o44!xD%$k)y~Q4!j#W)bkD=?d9ILmz%jho6&+h{#3m*j5e5-x%!E z`Mcun#OY%PY}?W~fntr)Xx7tn;(%MX)IRCA%e;AS*EQ<_Fhyo?J5uek6DK3waJv&T zW0E1ZklA-mrdiek2MsUua@tW(W^<)GebsM5eDzu<6Y^Jf;{HwK?PK=%xOZ1czIK<@ zWRD6brQIyF|G*}$HAJ{j0s!FXEe2*k(Zdqr5y*%Nts;A|j<@aO_!DAG;& zIFu-(F2)ZMcDAJ)8oq6A3_MRAf;Tm$8Qyll?zUN1}f*7wDa4O z)OD1j43|Q;bjz$;`MNkoYgS?q6mVolRdeZNvV?f#Srst8?0OF5WtIL~mM8qErOv4@ zoS3SF+o+d5oG;YgjLtTRKg*_PQ+-8c&(cbJvtN9EUG(J3E%H5b{)Y_Ix5jrch8j`a ze#fqFXMLdrNoXS*cZ5!g|g`T)Hcky#GPO8Ozy;Z`%*MYpZU|M4LgfxhAC<75>X5 zkok@+Qki3};b40^bGc%P?}$NPzz|oR>vvL$y87t@>-(VzwkTF5C@&C1zZ)Uw=Hzb% z0PUu`h-;2O)rsVCwTrI2l*T;lIqF4!!FJB2^&rQ)@z+noy!Uzu2l@8Udw1Qr7S^6` zl~%bt$EBPr)w*CY;jMIzgW3&1`IG&DbF=#OY1O47sxjG66Jzf(?FIgR!kk3!Y1>o) zvw~Ahfbli2^8B%F0Wy%Sgw3u4uwbOS%|y(t#j-yS^mj02 zAW_JpmNECF-J7i>_+XKH_-R*CfE_u>mv@;D68qvANwoj&_44S}I+P1!F_Z7b^K zzG8WfoJd2__8s5-n2mX!AP&uAU24(6s62$rLA6q~8rtv&g%M}O|6gPp+yA>v`>*AI zLDtmH+{J>Kkc*S+|A@B#Tc$BFacxM6f# zH>e9G2t_9d;DB%wDCpqMe3WBESry$G+|vk}Pu#>kKZ$l{bt+z1n&tqJB`gSUAf*D8Ku`eM3?vCM1B{Cc zk7^YV5?oZqGCqk7Qm~}FT4-)!@=Q7vkM^8w@7;y4D1ZQ!&XMq4F zqDS?!8iy@N2xJ5YrWte<0Q3hdAJb#V8Km&8n;N5F!aA50MLCW8-jFYfk$&}eAN!PLx;ECS zvN!tR`*(RD?mA6m2Le|4Ko!Bhv{>CrXMbnLmhnwa-vvCsFKzMmcF%thOq>vxR$C(i zYxGJCPmK2uOb~ZZ2_rh#`WKgPTI~JTTl&!j&3Ogw*<*!bb;cYhV``XbhTJ=K{*IfHqD zAZ%!DY_95x{r%Q|hJV6=@AmaAiu+IC(fuK$K#ibVJ#u^eF@O3`YG-Hbl73s@^)7$s z9`);hpq_xTifDb^88M(5q~<0AmV0M#aFbcjuFb;nU|nfzR%z*B%iYc%6yn#8su(>h z|Cm~4?D<%u1HEJprfKg#x#$a>Skaomjfwd+0Oa6x3Ts`{6B; zUhnSAzaSvM!Mo%Ry?zmzO~*=_^8?cfc);g?36h1cWISj(@0Hi zN59>7(O|8(Zjr-#k1|nItk^T1;ju5=n%zU+Q0yDpA8n%LrT*g;ulUFqnm9<+(M_A9>d@ zh!QU=51uXjZ2=xSo&fwMYY@Li)K?}glrN{AdFo~03a^|F7ADj@(R|^1EvKOnVpme) z%VrE4%D@?hd}glYOtcju&RA}N@VS3*%%02~AoCDSd>UA&^%PzIJ2wiq zY&3-+qL0(0-LQ@~u5n4c3JkXKT=WN)_VNb$Z}FAk!3=ua7h3+P1`-**Ve_|P63v#z zBLJ_+vK-Gn!FOw`@dV}JvGR`6om=uWUqq)1r$?n=_d+fmllz}JXRUlEkx5-Kxh6*T zx*toOUs*CVWNZ-^{Z`>WN(G&h#M6^P%Pzm7j2iX}Al2f3kw}VnhF6tI4F%Qncz5YF zG)mKn)d#I(%?PlcHnpIiF90$QqT$mud1MWInjfQy{4m7Gm zylpTOL*tjD}Q@)B?CjN_96?zQw5IjDZ{Y!klU`LZ$kBL(vRuF&gZDw7G(t`7u;XB+H5)#`3mluHdH_Ca?{$sJtY;ibWG9tX2@#E6$Wdw8irI3LI`@+)y#-E=;nAmsh>rHy(MdcqJgpYC z)SA)`4H^0`MRap5pq~D^k{**-{QTs?v^C`ZX{-LEUHp?|22!u!JE4`d7F#jr#;+&1 z7M6|Epp2JF$3ojuuv1$&SE`lEg5V$Z&HH%{A+z9r{DZ$CzAqIpH?@3Qv%RUwz&E2l z{vGMIYBt9=D|v};vN@So^kK~jF>3MG@mngGKemLi^ka`irKvAm^mS|X$P*Rsdct5# z$Z2fC`t}JJxc-p{sF|4VX11%tqk8s;xI8H~m(c9{hV1;ue4CkQQi2IMeitvOB37j% zGv4kmIRxETFY?RQa!UuR!S?`Ttvzq8uUgWjVjXUJBWnb8Wzrc%K6A?qK<}Q% zT)Bgh^bxwf=&c#zANP~F?kIMG8w}lL;qYTU~_uAO&$X2mIU+8m5CtNrG zU1P-YzqH=gYfCZ73t&^fY4(@kdRT z_-GD4=SZsP8sP_iu4hU`6=G9~!slhORrTSDKv8ul8-m_{>+aplL=7L(Tbozgy*y_N z|BB>a%mlV-!sB!>S9{q2#ipDIrdHhzjiecG35uRXYr-?bZfCePSd1iu0?h0R&H78 zT^i1$L7zN#mP9nzK9YRVOnP#h1zrsR2&O6#O#x_yfT$W1@H?MX58eaPbFKLjOFDSkm^2it!pmVS-I6~p{M zIQFk{j-7=?LY}8T7o}oW?t8sF#MUxr=6%`sMXk6O9jImOk33*NW1Xu&^+Zy>@qr9nIZv1|1*Ty0C%Y`1`eX^fW%ZHPi@5$>j~wB{E+X5_X~-@NUKEHbT#OSfhAwkbCKJ>(2T}F7a5fs$z;fTKk=*nonP9dgFp$vf?WLv4LB=1 z7tBc3(D8|kVZ5ov{$rr*p7in(RIO+^gd13Lc?GH+aj~J}9Z_XNHIMi}!1<86y zJ(bf*jl%f37k$5ZM4G{waD!3Ebv-)GenaXiM>VTmBUO$|CXfX6N<-~Eg^J=U=HAj| z|A8ooG@!MBI5!DA6m?c0gdtUg83ak~pufTG`t9YoQnTw0fpbh?>^n<;=8 znYF9FTU(}S9`FmPWF1q1nO+)iP8Cuf^DfM4RNm|5TuD+ zRg;tY4ny2;zF#iVO*w*oNN!1={pX;D@pbV7uyEoaPdx~kyU)|sjNn1I6W5KsMOI|R zb4wRbSF;Bx|DSgwr(9gZit80({VuH9ykNtpw+6&LyXb*Vdx2!kO8AkWR`ekvnnK-a*HlZuH1g!XRmKN;`*sH(-rF% z&$uIo%Jdz~i$D}+EFc%=8<)?z;A^p#HXG zk6~wr{Wv9%i3BkePDGG=WTV<1zu2sOPBspPklpVGIofIqX&^Zcdk&T3$mEY%BLq{v zcV2TmQ#8UXM({KDp;i^t_&@9DAm|n=(rPu)wfHgd4qYJ}P;2xY2I40pcZoN@Q<{p)nc7YP1GV!t&mZ)RSjMrKTbC|h}i1Hh@ z8@}qgX|jLF}17O=&e?P0_A9b~kH zI(o&L(-8bPPxqXgTd5dD8AZ4dGU6c(UF01^k%{!4PkpU1cPYhzaeZWg>wu1 zUs zwLO%3ydXhh)@Zg38R19LoAF3m>CbVVw^|ckd?+@bnw?Co713E1(+yU-vE_aCg%OW2 z!eXE4z7*ZZ!H;^<&s|ygiC!k?Ys#8ds1{!4g>9TE7vi4-HqkV>XP5Lkl@xk8aw_epenCC2<=rj=(+eW6aekaFl+b)T>UvbNn0aLrpC*AL|jwm z@j@kY!7y?V(>RzELUVRvkTLvAWzqbRVhABT-t1d7EHw@F!N$J?e!a$&RNBR`H_==1 zPi)jC`=qmd63(soopyrsmMcIzMj{YFz>E_9P6|lhtU3=Yv5=!~az@_SBbCnc_ z=Z7xgryq_*?9*<@*?_6UnVeAGjYDn0Tp<^}gNg;6b2Q7cd`6oekwH9dGv(>Ue|S&XD*C0b23?*Sn3ai_He_s{!NdYXBXCk_sG%_SjLbNA%T~ zP3botcpigkHVgG68y;!afSS5c>DMwkX27R^iSh_Vg!*^hOPeKK$cxU5MdiO|^=(~p zyYio_!TEM$U`@eFJrc`nTtL~Rk$Ijfd{IHDq(9rZhI+LqOfhPoO7xuphPcPepJr{W z<1H!o?(%|T#q7QLeeDdH>&sXtb~C3i8-8o^keQ;Fsvf~h#@YW?l)CwN09*qbwz1}D z90>v!imUcy5c5&|0_V1_*ZL@7V$YUlv+i^NTN3rma&VH6#94*#K+MkVtqijKS8!+! zON$>U!S-=zoKnL__|s=V=(eLqj4#VS>E=mVHeP*9r#fM{15FCINtoz*MW63$W97tz(SGlhP)%# zn9tJwz0bi`r=hReZhGJrYer$8rYqi%wyeb4L4zHg0*N?v^H7IgvF!Ts@h@B8O&^na`4C2!26Kx9+x^?X=sm(q9^!6Y zw||E8(Oar9?cd)Cub=xwDI+W0Rs=LU&qMHs+U^RO>Ybukx1HG`NKLRp%*C1G4)?yq zP8)pcVJZ@%vvvUt9Z%Lg5XjSFB=27nBVL3Q2OB>pK{)uHT-(z5T+zNc=A*k^A}9a` z^{6823dQcls2z3*QSTC1R`XdI7aPjQLR{6+S~em!7I*pKD+ff!2d8vo zdB2R1{XzV!QWlI%UQY(+wf#i7a&#ceewGbnY_Xi>%S4(HU?6`X7N|#7+zdu(H;lYIkV7`EWsd&+{yftsJs2*}&KX>h zsf|!!kLU7yu(3Ag>A`^N5eXlqeoGlweoYPE-P0bH=i@Bnlviyx!7H*7J2i}6NrPc! zeOkRAn16U<+nkR%{hW$rJck^cu0`u$Z|c8uC&lpB)-4VYO%v zx8*b&nZyESO#JUNpXqR3RFV1HPu`UOi`l*uzz?{Q{O6S`(89rSU~i1 z`XxLp%4Wo@^t%~lQp~&D>G?ETFC4|*=6QXHsWWL{(}@}zq?yonXBtrOES7ISmRwxo zRwJP;X7ES7#F?>A5DwFw*uzNJIjM&SSzmd4S9YB_ovL|kim$d*hCkQ7BZURmR3=5o z!ERNlbM-OBz&_~LW*9C+?D)K`%FTDZx)>P_CBv9%ZQ;5dut}$1BsJ?5u=E&{Df0UW z#ePjj_zGCgiWW*2~qJA4;`-$u3g z-zm`l5$)fIA^JyIq9sdmhuHIkMvtF`=m;cw7pgSA4ozFiwbg0!BqvD>ibEBUGK&oJ z^K;@&{NZ^@I3|~*T!DjdUk{)eEUeXap0NARn}NlD!^#aG%Dsn|WOt-!Px4T>U|mS% zE-Mz>ku)S}hwLrV*P5@=-6sw@nd{%qO%b^T~?~-0IqLcF*qGdNdUPGkI$OBhco(ExFdrvL?Gx&TW1U4~w zA9k3$Hfz70*G>6M+DF^&h`v+;m4b6Cg5>}W*-*z%N>5u7%BUN+Ch+AU04@|jfJ4qb zDP&C`s6RqZRbH-1*I+eQh=cZT_q3ONhh3x6(|s4ZXz`Y47cm2VXIU(EHoYNo?b;1!nMt4Q=+`-rz~+a5VcZ#b%LVOZ+Z)pr(al`_lc z;J#oYdf!=rM2`~)LjK9{!YY)Cb#K{6eT7-d=YSO&jAOYbYGrg5Bs6SLVe;(V9#YCV zUjeUaW$>0kJ96EGT$m0@ zbhZUVa`pWbzMtN})qf}AT7)^xqyY>eU~Og>`_kK}71MUtHn~l=P{rrnvWOdr_+yhg zpkE&W>qM`>GS`)O^Y5F(2XKBAuaXeJyiORuBppl9z4dn6MPmTGu|qr(-Qe`;_}9TP z7L$lKGT@RCox33-vYz{E{S_tZE^fQy_+5uCbd%p3AXsReuB!<{`PY*@HT>7h!DG*C z$g_8DD>5#wAdxG>+V4Mz{f}Et$sPQIZQ^0s*WH-Fty;^+;7b|IaE=&`0v1 z@apwtgrED*ZLoXr-`%KF+3(h@6re_83;;ctDP3F-EQ{n_GE)ZV&IKn;x?Rl&z8NrR zr_558v7O0FuMC?{9yKs@Ujme{nJj5J=R~xgOAizT8kBqnH`z0H%|d#*ohuaBIY-uF zEWhf(-GP+#C~1ZqBSEs3gLOGuOb5pVAqV+0WQ%kWUxI}TaN`VkfdzvuG3Zof^Q>#< zKhKJ)N@~b5yoMRh&DSGw)IA~}N_c$yLdRBF8gy#+*V^zhqF5yp3+6R9a1aPijPT46j1d_v$PRwASFIe|nmziYMV|xFT+DnMm$e(|d4*M4Pkr7gMFQ~Zc zy{_wFhO^8u|4Az4>2j^&z2#Jef)HX zNOQ4MH`tlyMDt?>vB_b*#ZIUCb(7?XP;#D4*_pKVY@t!@T$wom(JwCq>nK@mov2vm z8byP-#Ut6`k+Y-z|K4XfxvQ$E*Mv&@5^xqURUrMuTH{yn$p^ix?Kcq_MHmDhb5h#1 zxRfP&`a8rXMnkf|>vBe|@!XnYIh{(p$kKMq=ut=0`xGKGrkZ`l#k~_hfyZSCSnG}DRuLL|W7Uwyi z`aNm9iW?WHel;Fl8uEiczIX*ZpYm{LgAJ0Tu_PX;IVE*290_Yw9_X=U3$pm(fD-i>YGK$DT3D8o1_KKu2kSe|NXk11a?%KdzF4&-w$IRg2Ey{FgGh2A1zB3($dX~k zmm=H6fpPg()?9jHQJK^WTp_lrd7YeWST=CZ^gn8Vhn1z0PD8Q6DY+;AclrDaigyJS z$4Sw|^!fB~cdX0)+N}AHR9_(~co3zOA#*Fknd*fr1eUtX(AZ?tz;!r-X?4JTI4kcvA1CKAhnd-@q zc*G{fTut9hqg!Ls^-1(TwxVI5`Dzl$ZK|m5+MR(9Eq!ZO%6(&KU1LHKc*1x3b2jw? zZd1YSwA`l2%2Sl8fB{&QX^`jvN+iu%a=%rl*9<_2I91sqFz&#uPbNX3V7btdA3bN5 z_0%U?-AekD`+hQ9web<&?$Gz+d`8oX}cR1W>M z-3`TZIM3P~t)u-JPFXD921f~8RDPS%<`MBe^^)YT3hc@Ew0_Ex<}wqM+||IMdm1A! z;7n|ES@4}VLWsBNiwMyeN$|MQTMpU}HJ=n`0pm!Z+Ungk@afT4m3Doi4>O}a%Zz@e zDB|h%6{C|MFm#jyVw{E7XEu6I5n-JOz1qBGvM$4BRoPmedJT^-aFQOfA2OW5e_HEYqkn@hU6(cpZ!5&D^Gam`T z3~$!7QX!k7>RFf#jjw936gII=MugJz?Sd;yQE1#3;YCq6PIa4Th<<=T^8Pc;lz%I(go-3hIMk)Q+7Yn8GZLdv0O_oORB2kimCx%_(9L( z#VMbUjj@!qa1=?^Ck6SSZ6g)8F)+1p=r`N@GmXf~PLF^Kv4*H_!-dlgq3l6Y)JHp& zs8Gn(kE{W}?8l_&Y@2Nb$QtCVHmk5R8@n|wm~`>BZ&s)L21J4Ty+W(&(Ig zO|G`4KVD%P>}jSKY)nJlGQ{x+UTs*#tfL*SsOJy*ZlPnEd2i4POOAG~;pFM&(uwg| zN87!(jsjI{TM0$6oONgIN$3^v1M0R(J3>V@T&k7UN#8M~+!qcd0l^qZ^Kvw`K}hgW zWujX#FE}JKOzh{|`*(r`+4B+#h+)pllUGthAF?Z(==I`lgzZJHcHUt_iFNX%H$)ZE zFuB_-wYptvxoO-ihCRP}{XaIzu2$;bxtK*-1$S(PLeFm6w|`A^=(4A@c&4-H{t-sC zuP~bO#8>PttU~SgwUZapqy~b_&GbGciAJVVApMIe(g8e@oEEp-!#4#CdnjuzZM;m#17SW+tcwkJ!B5^#DSf^Ad#{qv*)cD>jTTU&B7P+ z*l~@+(2ut*Iia;)TG3(C+pOP_`Bbh{WXLI;Pz?JMP3jno<|C43!T!qLB5cr`01|c! zq2d8ns!YNP@zcl}^Er3$@z|PF-~<~ZFMgYouVe*c*Tct(FKx|L3Z4$JKYww=uj_{{ zAtd4{A?C$2YWnNZ4(~;G`^YTEr`2-&ZG}RXT^BXwU=dSHD<6h3f3^yLer7<0Q#3N# z`@>~d6Qbk12xTZ6pq!2>0$levI(XGm14ef@Hb@WUs`SaRn%bp>g*%nBNw6QhMe^UB zTS$q14Aq~?&j^xI&@GgT1+d6VK7jq+(HY`MI|;YL+c4|}{N_~|EHPH?OxG*+9?RvO zg}SwxCytM~ou6$Q>o5Jf&yv5nQGos#DqEV6jrO|gv14dLq@utsp74V8J5t`U1X`u{ zt<^Fo)Qps}?|Cxz(E>sHbk*ux>}=l!HvyN?#Bl)N2U}dqcFUOP^Lm`9fwz(r`A7Yp z{o_+i6Sq@gHycQZ{!*qE51U-#xl!9*0L3>u4OzgxOqz6QC|2@K4hG7J`i(V(cd6vS zrsb#>*qL+Xqn)-AomgEFeGfBhg$zd5!_TA9T8O#h{yg96uTbbC#ez&5x zisio_KP9_^*0z;_XdY3&Z+c+0`okD1u|m2;2ly>njx;rzJm4=e_vtv%E+z}z?Rx9y zob-(A;bwyA(W}Vcfj$c)?+;&IV<_pmE~V0qh- zM^v`~=f+;|SQyv%n9*MOUZQQJ-PJe2{Bl@Tu|G4WjA1I3v)0xHbpY3!N_-A#nNA+v zZT(f&Z1-q%Q_$87HJ~w4jo+0__%x^Fcx}jT`efQ;vin?g$7{vS<>WTkeo^4ix;NdM z)0mM({9Af(pp!}lOlzh9lsMD5v)nWOS%D7ubr!jeeHYnf+~;gPZh9H z#CU{XD-qYLFRrXj&@psr6_p}cZ`tSm0MxbA4Tq7DAKmhTC*uwrRMeXG9b zdwEXzm-~T6Utwt{d#VKRazUL(o7#e72j6(sST=cf*F6sHr;jq9`Z{Aso&e~-`#c6X z;^B@)W|P)hIML%5)48L|Y~fn|zruthSY38=P81gqSI!TuvmtUvs|!k;1JRXt>&|}D z5Az+uEe2hIquCSvM?!`lR7&NCjMuC(Lv)+eTXUGM3!`CqIS)%y3snka3?{U_9-h_FRzl60?+svhz_WJVh zRNG@%XCdDFLCciZ*0;vGIg9%LgK#&TC<<`d;tuCsf7HZd>J*LU#l{Qfu3S?-;wyse ztSI!vegM-UH-i$SXnztA^LTUpFDIw$|8FO!gsfcu6RTyLEN5TDh!}SJ1$|4T>{XBv z@1q74IV?13xLFyg-b4OEDh3UKmB9Y%$A>Suq6RM%Z@z=o2bw%l_u7+R^#SJhHN6hm z>}pT?mc2VRnk_3z_X+Lt3SA%=yZ#c|ldXMIXuLEdf90~yr*$tsWnE{W%7ozhKV^F*3jZV%t}$QeBC z52ShymGjqbs_@-q)eyvq@zxqe&9Rl>`MqoaLFdi6ujSgpHb~?NKa%+ey>UW0fcp&I zo$?q{B9`b`u;SlQc;JFxY-CtsWJ$m~YhG(KJ>p*T#f{AntGjyama7Gihv9q_aR4~V zIG{NS$sn1F(7n!i0)GfP;Ce{n6xBnrPjEiK^pv~0*+7&S57H#Gk3=~Nx)IKw*dt(@ z*z=F%eH-ga8u%T2fVM886t(C2=Y+pcJLuRgEI{mn9rrCJ>Uc$EWR#;pI(()zeJ0n% z?`*VcE_P+*xI-CpGPLSoPtZqtv+XNoVjr_j(3WFI=RhP4_xf#n0a3LzNB8rlFth~N z{{{t<#SQ;o#)jknactPR*#E}?vaPC|{Q)Clt7Lh1(S2mxszG|~E>L{1w(s#S}I zq$~%rlp<94&*w7^O0_y4qFmnYboV`s^XsK6&w^T88_M(edN!YXdT(>`)$0xF_V%%^ zr=<-KF95J^#mgtqu-Z%0=4SA^PQKgxe785A-O8&+&C}>dj`gY378t~T^|Sqxn3+dXTpeIqh8$zsj)Eg+x=gq=nv;%?BkDTqwHeWUFT$Nn$NSDH|E`}4 zSPhtB$MJ-uBARske3F+`bcp-uG3((S5I8vcNTW*U+<2$|ue<9GYHI8Dp$k%^3S7{r zpdb*E&=P^rq=^&}2qIBh=p>X-f^-lR5s==S3eo~7h;)%AQlfw$5IRx>fe@P17r*b_ znRnlJ-^~5{ojGUD%r0yH_L}+awfCO2*Iwp(_#%U0hGBuPeKFEz-7VI9zd1s0N};Pp zXG%d^c=>=Y;-DF0bG8U^ncex*xA>ZQ@2t>@MggwU! z*CQCY^v7*!QOEo6bpHmic!W(WBvlBy!6+W~hq?J}>7tX^0 z=-x}y4LxGm{cG$&%OZO(2d7hqTcxv^T4Gp((ls|ExD{HKdJ3eOPh3~?#|Olu1J`$$ zr6er4&ChtdU=LP@vg0*XEF$I>I>z-lkC;4uhPVF>0bu`-01C3;zY@T-6*CLwX-d^v?AG;Aw6bQD(<#EvV7;N{Q=^Jr}}r z$9aE&#A}~$OsZ*}Kn?ROCKgBC?@THbayYoMGa%P7%({x0u*<9$JSU02eYVTXNo#ml z#BH*4>~uKbcKLZn(}m_@(uVDIoZeX6Tei~a~w|ERtF(jtj( z*gIYRir-TBvL;hz>?d|pl+Rg+D~D5Pd|6ra4AgU~qx&2qA68|*HA{XwpE*TOKCmY7 z=yM?KAxG<0orFH}>-y@kR(kem&iIaFUKrV`#^xpbTjcOT#YA`IvM*yrQf=HYc)2so zgje5b-v^n7LAGRCb^nT%q6F}!)gNL(`Md{hI{K2yL^5xQBd;Xdu`1@)G~YPwWexp@ zby5*^9h|(9>?u{1uv-KxkII|i^cRlp4E|zUMo(5iW+ShF>2cA~Ee&f0*-@t@=Nu(P z<3dHh=7&T@$Zf=I2*L`4VeU7-ZP%@?4yzaO3~Y;Vd&F>b7N3^}_aDMK{Ei;0cy5jh z`hCEkxzMlvK3ut*-kj^~egx(HHtRQheDNHN0~&6oHj?N5@nU{GZO))Ym&c^v0u0CG z;>X#a`Dprnd1D}gpCau40Mz=+|{pLVpIMi zN()PT1mlE#$v9NkNJ^C6ih+~!FLb|R4^ls54dfbFUNiO>cuX<*u&AY*`v>nn=~X)M zX>N41F1c5-50qY#7R7lHhQ(X>8xA9iUM&H; zdR^lYzFYf0cozw8I0_!PPH043A(ZrOD`PVGN=aSH^K(;-T15N?2buE9=x@jZ|A%CO zQ!n*IAXvO`o@i?~7QZYvQ+@3CwKI(?$jT_da;uz*h%tzj?u8v4e2Y$hV}lUeKF|CJ zt?`!Q*KXV!mrS>38Zlqp>l77*B7Jj%5d8ry!Lc$F&FK5&cPfUcEV-$-QL}4=3ExGO zkl~c=s%GcjHlH&Pats2(8PnH7eZJr+M!T=D7h z6?Gsk&is^(k$F5{G_#eGfUrEv%o7|ICV#|TG*@}Pg=JgwMtaEhH-!V?P8c8MG%utj z;ekMBQ2hoc(F!RhwV$tuYF?@x7getOMLzP2ZiphI?@OjqqkBr7GaOX(IJE2yz8RPm zNj4>&=t@Z`=Ws;To)T2&D@|US(<@0^IBA~Hgq*l{-t8YexWJ-66^-{kK%Cb>5$n_4mK{-ztxOIUsHp88SvIMq@(%jVpv<8sOJL zGfhK3SwNGQ9d)h6!$cZ zU#{p@8cBvDLo`pV^3Gt0e|~=k^*pB4Q4ZE*8g#FI+X9=xB|bTn-{_qMHPbR_Tj1#^^-s(;;%v)hVeb;w=^N+2A#uM@ebyz;QV~)!QDoLX!7wk@=E+4c&@F|6jeLB>Mp4@TWoOpfH9~1bDfkgFn z-XhEb0uqUH3#Yv5;{dEH0f57kA+o#Z%agMO|O3=o<{5HQ8 zJ?E;W8nXi#af)VE)v`Ek#r%=A!wd65Mz-eTTdT(}MGVFI9az)208r0vFf$DU)tPTs z?DKQg7H81=j1ZP;m+n`p4FquQsk<8fPgngr03Gze20m+^w!Amo#R$d0!%6>5n!{@Z zPd~06F#+X91CYz35$H$OwI!1u?<>mNCEDxS8D^8)&y?%=47XQW$lOyeu?|YK!nIM4 zkL+DpV_g|7^8V~F&;oIkO%G6ehh6xvk@NyLcN*4x{%@s_xMeRsoCO;Y0k?t+t^ki z(}fi2U2o*Q2tPsGMkOK5_pItnR|$k5cP)kBqX6?v>l2>gQDpKVOV<&8t!?b6NwExr zIc!#GdAPi#9C3j)wWtf*O~g;BMBMgVSvxwC&1t^Acu*Xy%rP^&U6;rvJ-3y6dQ+q} zZf0tGW)D%3+dsGBlr_vkh?`rMs}I`<9!6-5hP{FaYk&9QmElSu8zhuQEb%FKo!@Zzn~h6cM+#AnjOM}fP134K?-2Kp&ApI0yV{@{lG zQP8QbHUb9)Z437BjVQwx;yZO z@kfTb=UXLaKPHbd=K zd5M~{ud3V&;BB)!mLYr6W98F~`=<%+p& zAUPHeibd5gsJY`S^M;4V(KU)g_aLd-gy=dUSgGh)=Zdr*T}iJyWCv7wTSAXeGG}I; z(!P_Op~+>RPB!h}m|o;}o3kXQd1%2ExAyz?w^CkB3{^iHk*&V<98z5s)wwB5SA^QU zI1IwiV7->(v^jnzoIR=%-DSg<&zoAfE{FeUpYlY0VOm+DQ%;di-y)IAdfE*-48AxN zIXzbaJ!d=DrJUO2J#^{88lSR_UQ@l6RP6dKspFyB&+m0jl;)3;k4p0UC`SplBSrH` zgK9pi0$U`?kz85>vF>D=dd{G-G{@gKD>P@lCS0C){E?)5u%2I(-1sm&SOPBtuN^CG z{AoJ5a~0dr%MiGdV!nY>@|s61L1$l;_TX*Q43TFHQwQ}7BuKa3>8pTRj+Kbydr1;V zkdmeM91jBXooX<}ekHCPZuZbj1>s6nEH$`i|7NU!{=>229~;cKuXuKV#fpnXAMWEdYVe2Y%Sx>6_YQbHJS!q;M-G`+4`^zbx9#e2r_GA0O5KYTn zQeUjqGV9VEQ*>!yF67Yo8Ce);I&$_7J=QSrWC5tQ zY*lAcGk#kkV>igx-Fe@Q5VuQQZ@PV<;^|Ux{h06p>U$*W;78cQVZ9o$9`y9sey;E3 zl!VkHxI(_Aa#ve#Z$gONY(gkrEu@MkGwGalTJNb-Ul8-x^dU4KAL-FZ(mc-h>wk(1 zE94vHlgv+g!P&7KLz76Ss&!T1@Y+g=?UTah6nSq1ekDxTWyvxLnQ zbi;;wTez~|nrE}=qpizgJqg?6pQ@(3mROD^LshF7JrubUHYcA3ECX!>x6Rz zI^x~Yo}OrXYPKF4?~C=c^ZJ)GeXQO8Ey&13(;TGb?dob{jlrO)0!((={hz@fs7T<{N=~jgYGcw`u2eN%N84iU zsKx&MH?sD0^8)_Xx(3$O+YRFdl=}@2ZRcc*qiPlMJB}Yv8VS4$gaBozABu|5$sY}r zrxw$_4!oWQc?CJJf;Acnv$M9d2gBrL!C-_E7-lacCxe!Q+e2YC z))0ujJz8D?CSxrp4^e=?U~mNLFH;~}8vxB^tw@E_0%`pG{0?C{JO1DX1|29Nj_!?= zst85sOt%q>nU|2{Jx2S}p&eJkb=dDz`oYuOSnP?k>0SX>ANBe3Ux$ufEM}&CgACDW z(g!BENtj>%f*d-`3eDtydLWT`OVQQdxZ;J$QN<8=?-tAKbFuymzf^ z%Gx<4Ga4NBHQe;I{UuT-SE!YYS%^*@*BS9aj*IBpP@R{FzyX->^-U?JQGzM!&T}}d zoxgQqDQwiCRYc+&pD=yqmjoKXii`;0KL>o^|C@)p6Jza(^P@721qzX85f#-m)MEKB DEiE!w literal 0 HcmV?d00001 diff --git a/Marabou-Abstract_Solvers.md b/Marabou-Abstract_Solvers.md new file mode 100644 index 00000000..2807caae --- /dev/null +++ b/Marabou-Abstract_Solvers.md @@ -0,0 +1,36 @@ + +# INFO: File Structure + +- src/smlp_py/NN_verifiers: contains scripts to test the marabou models and verify the validity of the conversion of pb files into h5 files. +- src/smlp_py/marabou: helper files that contain examples of maraboupy commands. +- src/smlp_py/smtlib: + - parser.py & smt_to_pysmt.py contain helper functions. They are not used in SMLP's pipeline. + - text_to_sympy.py: Contains all the logic of converting, simplifying and reformatting expressions to a state that is easily translated into marabou expressions. +- src/smlp_py/solvers: This folder will contain all the logic of the external neural network verifiers that are going to be integrated into the SMLP pipeline. +- src/smlp_py/vnnlib: Contains the logic for for the need in the future to utilise the VNNLIB format (solver agnostic, several solvers support this format) to interact with the solvers. + + +# INFO: The abstract solver +abstract_solver.py defines an abstract solver class that is used to interface all the functionalities that all integrated solvers must support. This is because the main flow has been updated to reference the abstract solver functionalities, and thus all functions must be overridden by every new solver. + +Some methods are optionally overridden as their content usually cover most use cases. + +# HOW: Integrating multiple solvers +In the solvers/ folder, each solver must have it own subfolder. Currently, z3 and marabou are the only supported solvers. Each solver must have 2 files: +1) Operations.py : this file contains the functions that are utilised during the formula building and processing part of the workflow. For example, the method #smlp_and contains the conjunction logic utilised within the formula building phase, in marabou's case, the conjunction operations take place by using pysmt. Whereas is z3's case, the operators library is used to manage the formulas. +2) solver.py : This file contains the class that extends our abstract solver and operations class. Consequently, it will have to override all the functions mentioned in the abstract solver class in order to function properly. + + +# INFO: How the universal solver class is used in the main worklfow +The current PR contains changes in multiple files that contain core functionalities used in the main SMLP workflow. Re-usable parts of the flow or certain cases that are handled differently for each solver have been moved into the z3 solver and have been replaced with the universal solver class's functions. +The universal solver (universal_solver.py) acts as the intermediary and eventually point to the specified solver (marabou, z3), depending on the given "version" argument. Possible values are: + +``` +Solver.Version.PYSMT +Solver.Version.FORM2 + +``` + +# INFO: Pysmt processing +The file #text_to_sympy .py contains all the logic required to transform the formulas into marabou queries. +My dissertation can be used as a reference point to understand the underlying methodologies used inside each function. \ No newline at end of file