From c7a600176184a3d013195e0bd3de30d5d10de824 Mon Sep 17 00:00:00 2001 From: Samarth Uday Date: Sat, 29 Aug 2026 21:13:52 +0530 Subject: [PATCH] research: Add unseen-entity generalization experiment Network features (out-degree, counterparty HHI, lifetime counts) drive most of the ablation lift (PR-AUC 0.09 -> 0.94), which raises an obvious question: does that hold for accounts the model never saw during training, or is it propped up by account history that only exists for already-observed entities? scripts/unseen_entity_evaluation.py reuses the existing artifact (no retraining) and partitions the held-out test set by whether both sender and receiver appeared in the training window: - Standard out-of-time (full test set): PR-AUC 0.9859, lift 198.6x - Both parties seen during training: PR-AUC 0.9918, lift 198.6x - At least one party unseen: PR-AUC 0.7912, lift 198.4x PR-AUC and precision drop sharply for unseen entities, but ROC-AUC (0.9998), recall @ 0.5% (~99.2%), and lift over base rate (~198x) are essentially unchanged across all three partitions. The PR-AUC drop tracks an 8x lower positive prevalence in the unseen-entity subset, not a loss of ranking ability -- PR-AUC's baseline is the prevalence itself, so a rarer-positive subset produces a lower score even under identical discrimination. Full writeup: docs/research/unseen_entity_generalization.md Also notes what this does NOT test: a connected-component/network holdout (colluding clusters never seen together) is a stricter and still-open question, left as further work. Co-Authored-By: Claude Sonnet 5 --- README.md | 1 + docs/assets/unseen_entity_generalization.png | Bin 0 -> 35550 bytes docs/assets/unseen_entity_results.json | 43 +++++ docs/research/unseen_entity_generalization.md | 75 ++++++++ scripts/unseen_entity_evaluation.py | 160 ++++++++++++++++++ 5 files changed, 279 insertions(+) create mode 100644 docs/assets/unseen_entity_generalization.png create mode 100644 docs/assets/unseen_entity_results.json create mode 100644 docs/research/unseen_entity_generalization.md create mode 100644 scripts/unseen_entity_evaluation.py diff --git a/README.md b/README.md index 38846bc..22e2e14 100644 --- a/README.md +++ b/README.md @@ -243,6 +243,7 @@ curl -X POST http://localhost:8000/api/predict \ - Calibration may drift under distribution shift. - Risk probabilities are model estimates, not financial-loss probabilities. - The online feature-store layer is outside the current research scope. +- Network features are strong (see ablation), which raises the question of whether they transfer to accounts absent from training. See [Unseen-Entity Generalization](docs/research/unseen_entity_generalization.md) for a dedicated experiment on this. ## Dataset Reference diff --git a/docs/assets/unseen_entity_generalization.png b/docs/assets/unseen_entity_generalization.png new file mode 100644 index 0000000000000000000000000000000000000000..a34ec8e8dbc56e2b8cc1e48e87edc2e292429863 GIT binary patch literal 35550 zcmeFaXH=9~7d2QmEhr)wK$3vA0!l_C$JREG5y_#*g5(^Eq^*Erzy=9Q7Lc5g927)^ zLO>)($&y2nF)eX3czSUe@Yvk)rB(?sM-s`|Q2X>BlNcH;$4skYg~I zqp~;uP{m*lHexU&E{90rZ?0$+zkvUUIbYLqzHN8M+0Dq&45Mh|Y;SGnY<<`GjH{WW z(_K4T0UmyS9)7Mf7S7K0PGY>gHvf4AkDa4AuVW9%I=sta`|{AY_=z5KC`;&Pu!un9AU+8-{3A*IA=!vAZ+jQ~vpZ^5Tb z;zHZY-;|9umnY+8L)eUxLN&lcplUhOvOcj?Sr z^nTcyRWknCe&m_(c%%5z=W{i$%wr}W>wJh!&#v6$@YRp<9*TE^%u$3 z_k_$EsH+Y&)3o9D7bzSq(xbo5-S9tq)&C)@d>FN-=7m2UMn3teafAwFUk#vK*>wAQ zj76ZuK1lYt2S$bN7blDbX254_)_&mw*~C%2Zh^&LzfOwJ6`TBMPuFa%^7{0Yekjte z@~~aSiUysarCJ1^(Un`Pk=5Gf(Jl;q-@4(=+HUmPRe2|Q&n8@d#^C4gqu|&`a<3Y; zkPHc*^N*Gs!w-Kt5=IPT)&_ljwQSzCFAdDCRqc5fPo<>?fA{LjF;XW#U%9(Ez3?L; z$iLITS5o2i<%^9`q5=gry+ZVVT=r+)nk_PLthls6y5b2ddSz>MHpFJYi|}RWT_;w) zvaoWgfivKXP%qbMw>}4>XM!DCQ@ib-qt(W^O*3#CC<<2Ue))E}Bn;)pCiTnLQ^?>KRe$G@)!)j6KgV|^F0&E*`Nd;3< zpNi++-zN!~y!-8KsIE^PliS0+?Fng70x1RE3xb5^7=QT6#zbsD9kVBww%yWby-`Po zcD!PwP`uMblhHRt;iF~qm7YHat9)-6R0%HX(QJyQa9U2e9q;u05g!n2a?{IuZ=29u zznE*)nN=^e+GP@y^MRDHt!FW%r8j-HbD{NP=U^+Yw{zJfb<#vN_qm53Zii;c?iWlT z&p}L^6>s^%mF-=(5b@dk!SUA{j22= zlW!yM*cFXD7uUcInz4MlMyvl)C(pG0(tO=g#@e*OUTMRY#^#CG&Q|h8I^W%m3A2Hq z)@DMUgz(pw7jLVgM_4kbUx|kk7qvN+o^83aqWjks2~9sA$~2BaE8AwtwTxwk{X@x z`i0I>A%ZX7Vrw0Cq~yFS=~0H475Wq+VS?16ghm0)l=tPwf^vmw5+0vqe_QG@U8u0u z9)C5U&2zp&(<*zgfqJ-vW7$b1oP;p7&12kEl+nC0TQrCrsY{?4qSFiig)rv9wQ`Y< zcXT>Q#rGseCHzq1fw-6_H2NEVX4O8Sr8V6;Ld&C&S+%=4HkaTwKalDiCz)32!ff zJB^>amCiDoWQa8Pt+DJb1UOE!ID9-lwz^ogw=;i9dVD?ev6T1bOI_Zkvc27{kk0E~ zGmJI`w@O?z^j+H98ea)mZLgGFZuicz92QN%-}i9wp%tIWx!NS=u;sgfH&9Ku5u_B8 zJQ3r`BWT@~)25+o6RJP)iP0|O>&VwYdy1?%t2b~=Qe8T_$T?FaENXlW*qb~7^oMD$pp(dY&O-*j|NtME|@ zY0&s?kFZ=#R*H5Uht<;<>)2ZDwZnf&&-AwL&NFl07z?k7g*ayVB3~qB3gy1*&riS7 zPe$jizsa_2R>Ky1x;T^WPGiJz*8z`-g?3GYP+iy0X9FI}^Hvn}ZPQHhKd0k=(^ozc zBwg8Ob&k>!Hg9sa(`)oZ7UPFwsn9^6BYCVPA;8 zjb)HAvJGqp2+h(g{uK0iZCgE%#v8s&x5Qs9@i1etvY9iXpeOWK%7*mqIqpDg9fG}2 zkzKwdw+`Rb6x{t5O!eC-Zk?B0yj$^qEbpuh2K1bYEt6dBI_I_g^~PA(Z0U5?5G)rJ zD=VvqUDvI04SQBpG z^QvMO0uAej;qM24BejTEwWpkuXK*n`mw+NpG#+M+y8vMX_YE=*Q2f z9a(6bj?gR<+%=OwSB++fmol7o6XwTju5F!VYxWdE+RT?)DWgUXmu;2%G3t zQrzZg5}k-z0PHg1^@A?`1zw}Oj*7$Nf@k$%_R)v=ep1Ythw8px>4N2|0c2$`DW+fQ zoVv6+H8p6WnEtp^-er#s6UTD`K!T*O;rn}hwtdBLuWSl8(|kz-XGDLu}@h>rwWz{Pq!Y7vwZqN1> zWiNSaOt^im^4%NNJ(=rQ>(nf%Ib60nBClT$k%ozI$7Pgw>em@qdix6{LO1lT<#kL0YL|>BM3} zMQ9j3atOCJ#{|c$Fwxg-JLfm7KNSme;AAH{A<GWi_OX)-NdJ zO3IC0EN0YoR-f53ZGvR@V zhx8jUb;&U}j*Q$-44m60T>7D>*V1RCst8TuMhVY^IEr=w1nCY5X{gouP!1RAAOP)1 za^zEBJ9klEUf-s}`}bbPEMq;Vq-8k7FBn@JIefoQ#x_{}`4KlC)@!kLR7CB^$Sr-* z%3R}=$xi$dF+e6(pT-+UL~lNWl7R zj77<+&Tp*xjqvcSVjlFZ(cSBZ7Wtf90|jN&3VhsS$FXSJsN^&msz;}QlW?F za?914>FlbQic0&T>hoGvfOH!&%N9Q|cIxKlN>q!H-V)v%7FOnq&`Pj#kG9Ff)A1Cg+6|shC+hm6nRFz$Iy5J;um8J%{{g z9Sg-hmPXa4J2IU*E?&GSyECcmE9VXQN_1N3)g{&y0KUl(9fK{uzPXk+=QilAX=}hJ zY^yit135v*w&L(fHpuLA^4hh1bx)zXdJB&kJ0}ET#dQioSp9%o`Jh+x7K2sBUv6dU z{Xt2YwDaPFyMmT&7gk!XQmL;OTaNG#z>1G`Jj?#p5Hp`9CA_#c2*Ay8nhDT&s&@Vz z*6GGbVP22Sm-_6sdyprx7CYzjv|=XG*kT@-IRHr4bT5Bg_eQW&Phfu6jOW~q=Nb0B ze!KcJu#xKrKic^aBHJK3$8JImgtsw&fA7Fpq57s>N(=E6mUH>Qu?+9M6Dee)I-QZ< zq{^FJv8t0P5%yYwt$lg*$bZrbs&&p78+vpZ1*p@BsjsLC@dYt^jQ&+_*EW*my|r4; zxI@V#HmZ-+HB!u7&E;Ux`RI*>tQp+$8+H($tfeI1-V&adFsDl)8+J<7>`MD<+|$Li zs1p52`TRRyR3$xE8|Ug$ZY6|Gmv}%qMX;6j-*eca@q33Vh4dBi)WYll8Yng?rFU^^R^XRKOjW?$L1W8JU*mk@?v&3yq zA~+JP*tt@PC0{}D<}oVzB95QCe*VK~9_ewYfR**(tcVU-+(qFdO}EHaeE_Ss7RZ@( zlUuJir=$9{@r()NWJUQEhSeEwT0Q4{irZu4)xtyCEe= z-_f=z6}AF!AQ^&xMq+i=vJDyK>4;JYY9THfqwmYvm`$jCX=(MHZ?(oGuvI5mj5ofL zJwwf@hc67f+nOkA8KNj8yR$RztK37a_O|Q(#uBkQ3De9tCIuHq@JXSamo9V_nJxGh zJdwo##oHnjGl~9`Os~B}sC#d{+|l?11*9r1*4=0-Io2mF^zYlYibH_bp^W@llQv;Wx)Z)qMqO*Xc$njq$DdDk{SGmm`pD4|D4lHa5kIyV;@| z$f?7or{J!Z(-*;TPwB8iY~@5=+Ax7Bb`7?$S?yqFkn}Dc#K-Wc4F~(Lp&B}=XQ~pQ zKw)1yzQ;B^Slz&x9W`U1kK56>rcT_ze#Z>XEzUdbZg13Ay8dV}dNp4qno5#-acl<2hPDA;&$%)E0>L)+h zrs06`n0-#_Ck-WY#ZQ#TLqQpU{(DC8f06GW9((P63WJH8(=M>MwGKsZb%u66TFldE z5pryrBlyn&!Vx|jD(Ba(q08+&)0L}XD-O7i&tdrE%u1n6um4JiZnj)eG*t2HfJJMz zHwafD(p5u5KCV+}tv$9yin*7jVnm6-xO}FP-aZB=q__o2o}3OHE5G|BWNbqV@WISH zmFvTZi5VMMg$PwW9%&czun%&SPnY3?qrqEHVJlWcY{s4|+8GT|t~S_R3OB6)V8Pz| zS?@VuV!3&z#o^i^xONKIm$Gw3zI(g%t{wVr{pT+aI^wx&P*pVMBL#Px5v;~_6dqLI z+!Nl)Ll4NPC=GmfTpYa-KL;_Ag2i{+LDhk|c8tFq+c7moSGc#)?AwH$?|Els0T5QT zHpb!WHCjKXCeh@$BFUZQZ-{LBOQB9~Bi3j0VyaRQ3ittTKdc+%fNA*DF=Wo zE5lT=P<>rwHkQpJJ1UXFGfo2JU!U-jF z3MR43cRfzj7IIZ>_0Q+wv!MDXJ5~4_p8Q6pX`>TU|F+G^;k%fo1h>rCk9CPX)L;Ww~&1`a6I(%P#kkXEhw@ z8Nth{&w3wqDvOT2H1&{j(1su#hVz(x*GXR4jA_${y=LFEqA1*dc>*_iPlVEISNeuAZ=z0(|{|Dspp1_UanUskoEu{x8}xi*R@RykH- zFqf@~HJ?1=4Cm_k3gDO=$BArMXP!y1J*qZ=VU)alH_tkt09Zun8DRB{fMXLSm^Vey zBYGF1F?+y$ti8O`@a&FW^`|&owY?yM49!t?qO&~WMr|VAf;5lLphCNc8sg~V$IVvOyCA*EE zaFgSD&1lF(pY1g7Ee83X?;P2%frX(25OH*Gcde?XE7v4s1ESzjcpLS1^Hdnjl#QPU z#d8E=7iRLC*$kR9vgifwJS-Z(aZZ-eCExFHLMXjviSE%JaA|TJ8S_cS>gIsmqFG* zy2j6HgnUx5gvv*5oQg$CVg?GV`pYU#sOpa5GYnvI{jAN**?7rV(9Mg0RbSM))q|?9 zXj^~)T2`6w@;VK0lSxs+!?~JPlYC{?oDr>^Be=x}g;!FVq)1fO%t#9wFZ8aaTgzZw zcN&)G2c@6V@&C5Pow9CZYt4O1dYQXFLE_x0IO^fT_TywbPY?0|nV^tWx93|&_1uIr ztXXLN`oXk;9DDuS^vo7QCW$t){qeGoWS#vDYFL=w2J81nQ6 zv&VVS)sIwPjgxfCV~%$|X}70&oM~3-JQ=JOP=|ddK40!sJQk+AROpaOytPbFPeKcY z9uMlIS??r09&Fl#eH5}Mus1GTCA(X7dZO0u;tm|WM$y=hFUnPl^s%`&St)H_FNRlb zvoiM7*=g1m-l69&4w^8A$e)E1_%R{WYaOCT&Po{-J{fyD9WNl%XJ3Z3(4qsJ_yirR4(7=8{=wkndIPZB;Fxe+{K3$B;*hxo~vBb1Sle zvKNt2jo3Yi1&+^m7IEDx_!!#ZxUO4JNI33N>6U&zU&zmuzI-OyL2NLfqP=8xsZftA zb6J#saK&uH7eG+k7CGTc;DcE+4)rxSmyUQ7K}CvB#w+6>?*--rwD7%T`XeK+gMFhT zTVci*Tl$+gDQ1y#+<}pQ^!WoaO^tpg&Vk|Y2}ztE1dKeWC+dFx6Ymo(bvvG!jARE@ zy$@OVFC!X1q)?fDQI|U@z3WIW#C@B9(~dWlvKK_+oF|43xReJKpy=?P(5B zMwusMX+`iL*&kn{?5nH{Gv(;$0&DWaO}#YsJiFs5UHm4OC%a6x64#K7t_5 zyFAs-nJicS#&Xr&>(S0P0lU7D#WOxO5~KqVJAFg|8H9|~z!82`V1d$>dx-ln~>g(7$irSKQLn*wPT zlOD_ie&g+pl%!l{YJT;KDppx)WAuD63SS|MPvnYjJSE6!mXcuJFLB6@$!kyDuUPHj zOl>HE6j%*gtWu%qSp^=^jfs212wS*CA0Q!ph@cC-sRi&x>CT>zbiRm7uik&^O{ZQX z0?~yDnC5;ejk2qJM1P!%1EK{u>siBEpM(pNyJqwd)K{) z1G2Z?*()ErX*Il9^A6aNJK1}1+J>N*Xg(}s9JG}7$bh}sE4uPcFHf@+NsjTOJ;B&Z zW^yGyWAyx7K`d>BAX)Y0t!E~zE7o6#Ve9~HFluZuO8aY3knuV**)*4Uj>}nF#wuAF zmfPwtB|Gb!Y`2ez#msY@=xzQ;Far>f?#u!^+798wCEteD?#4V7qnI0SYp-;>QYD#H z@Mixk$@zAuICe_jR6U{Ex-AQ zoZbt1J-^QUV5HIGe3pT0^m@GGc8ef(c+W`QPI$mT!cL7^L35(;6_smx+OkibuvqRJ zNh#XJZN@77Q}gUjF;6Bs-S-ytengRBWL{=V&gC78nE3iuF*0fOY7-YeL-&;k=C)br z1Pyn9KabA|I?hj*GBu5*J9=%)xXUC~4^%+_44$N8{&!Fd^5@Jd{>>Bilt-&9``GdcX;7H+J4d9Ly_dg=;zuxX@+a4LR};Ct zXCBY$r7dy4OoEh;!N^=~Df-ivMcu5WS>h5mlQAYVTa~p5zih-MPIp{IB)46b!bhHp zOUSYq@`1=a8a}>iGx}2V?iqzitHIa|Z~=EBx7v0xlu3xw3WM3to2m~he@FOZq{+YQe&oW z69_Ut0Gh&gT~2V8n6SD~JQVJ+NU(3ab;IugrPzh}GbZv*U>9KZ-O9ZpPWkdwdIe|$ zvDR*%U!1MK_rYxIukftp0YID8(oiAs0cn5!^YhT*+&2!FB=2~AQq*yLSj^OO5%;;J zZI%<5ltDRFGyF+S75)?)sNwmYfC;t}*XHTXqmH}w+)Xyc22e5utE5`941=&KC9yS= zFWUQrCV{^zFmyC`=gM2@MWFeP$G8taPNZ5w(iX*)g7Rg)fUywz8LPX)ai&SV9NaeV zbJJ{#^0stia>ShIurb2c?fvF}{HWdy?@41)JVBm_p=QQlR^$gQpEGLPaWsyc@jxbo zl8tbES8d0~#)IEm<8fxN1jpwPz1>QH^r^r@4;+UA?xH3;7g?BrI>|5{mRJ6se*bq5 z56-gm+ui56ke}s36z*h1T^xr-pyr(iM}odD-z`O60F(@8VntEh&olo?>=PgVr$p0ZiKan-0Xc zKEp?}+T6nl{b1*O`_^sS;812SwWlzK;jL(XP4zifj#VtYP$v2;UvjZd(P37bo=t8H z_5E8_P*|S{XCD9Gy~|~q|9+1a3DE)#oq{G zHiCMT9N>ZOl&dsQV+(D!mHklNB5qf6RG5N{$yp_-2w8sUByrA;gYc5u7Kr!-8Kb&G z*NzEcGY(0%INYzq9YS4eQ`CQ|No3^d`3pQ&3*SE&tuGA4d#=sl3%;<&dR^)RcY(T& zus^|!gaDkPKbUZYkX$gO62=JRDGA@!GjQOKj^(?mR$sPLA`gL6jOE27{m4KfH@Z#k1z>tSktwu5Znfx zW(a6c$`E#CC5~=8+Owz zpM=eIPlqP`-9&S2tZ?yMYm&TB;`2EGQRls~vb5eY_r5bNux#gua&DCsot6Nc*gc7C zUNd#TMMUP$xpeAl&Pf?}G1V-Yz3aXc>4yj@fO|6|EPLy9Q68`4oa657)9xwi4lkNp z8wyn(w|CIz9+k6pQPuC%b5gvSq}n}wgULm$M1wbpY!VWwNmCwB7vM=S9V9d&nC1zH zO!HG@oxN7K4u^ud~c$I*j5ud0P}t@Q3-f z=>kw+jjx2kE@Kb6Ci^*#y6gcp_3hD1tIxo&6YDr$cb{dLmD$3;V=S~KSvmI4S!uW& z8E;Kh)nL6#v$y1mWa}^JcSo0eSqkhe1&52%c&+hgFXXl0a~%D=0GN-hlM0NhiEbV< zMab3|u3Pm+h{JCIT#J5^YkdM-*m4n11BaZiUAjyuqAPE!w`*w)6sng#J!1?=&no5E zNW6p{{_)9S0K`Y#ENKn1Cit!MT=O6sF13e?TyiS^I4Q!*EP^D<&L1Yfz~!ab8L#V? zuE=JL@}$p{LqY9PGC!YN$9%-VXTxlD6X~;2%#%qj^8=Fg5r|b8#ZffCS^n$ckKAYS z84EnzlqxzWjsF@4A58v(=ud%LTC=;zFnE2ADkb2)2St4X)HH123P(bLw-zm7=9c%{ zK*jF%)B9T>Er;7Io=6smmAlh{)StL1IkJu;r?@V3I~VEcJpQWdwS0_X=)hNz<1 zp3$#{xEEEi3$DF~5Z{E7nPjhX89udfk}reUoxf9KB6acz8^Hr2d<~eUURG41bCy{& zP%5`YN7l(D`Tox7W?SIWx~FsX<>MuRw~!555{a`|PB+-$IE)vdnV0#JPD?Y+-}T&} z{VuTfTEHIQxeb6Jyi)g^DMXozT_Hjx$>!4sDMmp{YGTDE0in5`Jv#TAe3aDC!;`^6 zyz*OrfW;hhZ|ONiffh(IGb<4JE{g9U*Xl&9uatRzE4SXI>AK<1?)AcS=GWTCkP9I6 z%5=op&{MhuX55eXA$VO=q4DEax6U2~K9m0G<&$A`RKFO@@Nxz8sPh7BXZO0*Ev1@- zoC>nvB|*r_0ox*vOJr`W`u4B$Ag6eC!5TS+5h*YbX`1LoOD+L}k;(UC9@L6L7WJ#g zZY1*T2&R2vt$G;D>hz>k8FN%UZ8SWmxQ$;_6&mq276rwxlIh|Og*|+0 zlto}}qt89I2tn%xG2z&OOt80uGy`CIkeB7%U$7>jq zIzyJalU?^49xRl`siKXrvm>|rEE1jtQ~oNwy>QhXjKldhCdMtkP-n4yGb>twoXa{r zK3%g5wCMS35%jqnQm5Rn=i1@64da$UXCk9i_%zDNc}MofX@@_Of(|g`At|*kf|M!3 zoKogXq;R%I_@l9`4YX>+&fo-A8$f91X~`z={HsrSir?e8>i)(@Kl zvH;23(+RKr`aATT?#-Id3x@uu!Roh$GdrfuU0sfcb=0p~f?DA26%=W167N8z6k zw(t4#_>bXx^QiB=PNLB!p=ca2JchzcF-WQ*v5Trbh;S$bo|D6F9$EIQmq5i~lg0;f z8xD)1o5i$It-J~F_{18ncVZ5tG1Io;b-@SjkH5i2Hc(~7YFO8t=q(v#6m@}9O3C82 z@cX4d9W#ClI^Y4`sE|pK`5G!|g)2DXTTKPvZX3nqf}5E=JdGGGQeJ2LLYAcj+WR2Q%hU3v|30t(`SZ$@O)&~#QzS=CN)gb_q6NY}Do(97d5FIdH}~r}ct`Gv zKzgZx>MdG8`eX$1$qoTysqhvIDU0A54|v@}gSG?af|B7bgt!1;Hu)PP_!FvK7ywtH z9syHeAio-c=Pc#1KJlIJ4O482y3O_FfDc@QA?yZl`8KYb;8x~Ywk5uoi~|V)xk|lO z+BN0%?Vx?BDfQ-0?vExy8fxk8Nz{5)%~viT&eJ`~;J=$asb-ve3vG^6FcN>R()Zr0U0w5wQqXD9OY>r>Y8(`dLR_AZH|P8Yz)k{R zT5O6}?(b}NR#A@B1u4G;y=$4zZlFABj7nlbrWitf4fqm6oc7-@MU!S6bZsqH99YAK zo(eR5)|N8;{30qxLTr1X=HH*1o>@qa>ff~TUC7qQga+J4{&i15;OBmRUc}%v;AF9F z)UBjAgK7~BhFY2r75nb1-R4N#qwmmj`r?T9T$!Rlgnpu+Jlvg?F)i`6am5dO?p(I& z;EHl=xL9w7ssTWR_1hq@BfnMV+c_a<|2a#qhKyIpgtfb~6_5}lCAK`*?^#l`fi?lC zVA5Vo_3ZU4mEK!y6+NB6FZ?pMAMaN<_~xM*&=m&|l9H$mzvwi*8MFBP0hRw%D#^?A z)|W?+@(Vssn_}Q}LJr&ruLb3;8b}}(WPXm?0AxyzIBs|70t-tu5GP*u(F=UBlAf+w zIflPq@*aN7?%#6>=n7P)%bCz4$zTFhX*+9M}BZ7<{6jZlD+*ks}YcrH8N zpW8oaSc*-0&Jux3A3SW_fv*UO_1*jy+rTXj2sUr}W97KI15!MsO}emRT+NT#r%Abp zR=*T!a^{Fw=pB(g=;{nu*t|EtEfMir3ejSM1r}d_1EnLSZ%HH@#L6M?R+1xZqFPks zzBr7`3hdty)P6RDR4$Ma%30+p5=YKNLb>8YC+cXb1uqV{FJNL(0kD;Q=RH9iYNxO* zDR#xzFSlm~Dm*RviXBSE7Nth=0E4klXMulRz8^bN%S^0OV4KO}!s<30RKB4CFE!B) z2)NMIY`eiqnpJ&Nw?qip=vZ7BCiD;$y8l_};sK46_4hj4`aH7LEf^WNZO1Q?UigzW z{%*u{xn}xAh4z=o09Hjg%T=WVpM4cT?`^@$yCiV(p(b19y*IA2J*+!;oNu30t!}f#%}}$c}xcXwWAf3Ns@(8AR8gf!E@- zE~u$$px6jm7RW|hiX&4ZNPYU~HHf{{H(0!1Rt#Kj=X!bBQoYUl7mF0t)N4jYsV0sR z&9z4a-3F`}@AzGRri%+a5NIeHL3>AZzJ1Qa?)UVT%3N}%d&U+8XK180Eo>inRdp6K zw2!T@@y$a}UUF)9d66b7L_18o=n`sO^nu1M>C<4_My;KAo?d8Ls>XWG(rm8Hvj7QF zF?tD3!Y>l2a~oV&?QyZSE#OZW4S4{O^=!%L5j^6G5>H5Em0bpxY(438ZfSW2@}65m zzh%91#${q~HmucLASb|+4e05B1Dk_NcGPs-2NwDvFi19OGf!M|h}OtV>)15`yS*|g z_ZNx_Iyy7v%p~g;;B@5pY(l((xR`_Ey0#@|7l_>H;YOsdAfIaymHMR5s$YClx;O$Z?}^vbGMJXS+*4|;7Wf$Ot6 zk+KG?V}VsCj{>-f8(#}@YEJ-ps)-Dx7>tT?n}y*nobLcK7B9YXpjEq1vBBm)0zuHF z3)bQYgOD)v0-zX#qWvMT{NE6|5M%`G7Yj3xpn0azW)9H(>IZz26Za(XGTE};SgM`vJ~bAm`SxDVIL`MCVooW$a@LOJP*`P z^1wvnJn1Bab>Cv}uT?|xQEAx<`rtuP9nR)IxR`et`Swtg+Wl4H8r&``Q*o#fb#O<=00{D%Rs_;iyg$zj|F6V45A3h&CWK6? zRJF7|F$b((*w=Sfo&R}7QNYG2UF+lEdCfQ7ga}WCxK5GL7gw(|$4G|6ZKKwoVmLuH z>x(1#Mcv?D{#R0$@E>u=X?{xg6ghgcmW2YCaTf7z#8Cp6z}{dp;8%~pKf0lvLDON- z)K9$VscVMJkQ4NcpkqLH3J0spv;WBO@5`ax5u5|qLpA{ccx1S|4&1+_Qf_e5)zj{iI}ja6GB@p59I3^OSjD1!67wijOB7N0t*% zU}PLcf5biD(6g)BS;c43+U;L6+ukZtsTgl**@26a#7H;Hj34@;O#wWZeQc@^zrQez9%A4d_q7y=zY z{w*R^ka}Vd`8WXChtxP|z+3si9?xAgT440}=WpR?B2eS+5I8OI*gZJE)zIRtHfe+S z!hg00E8d^cmW~owN{(!nnk58X|FsYajJCyRtziuT>LQ^E0iS;ZH%L_OfTcqCyugsf zQzX&!#6}l}2GHmf+3KGxBR&*N?W|GESLn{P2X6~&I6M%c5(IS+W`nN8cW>vwQ4e2J z8irUMiqC$d#P2Q_hj9`Akn<)rl$@|Zm9vjQH?P_R5tJ=OryVV6T4Qw?W!d9YI%tp7!N>dXE=kk!xS@2$)jQTKfd zJgUuD7C;wGOF*or>1o9*qCg3qO45kUi-!iY_Wqv*1(u3{JX_)wz=ryx^y(=IXcO%B zm94#yQ`d~3{l2|_|9D@pHI#Cl>6(G3#S)!A1CQ_sv_UF1l_PKphW_zJPk@AJW&3*! z0Q@#^<)4-6uwzNcnkw3f)Uw3icB0(|4x{ehd;%zBn*IOO0|MqvjY?}^o-^-;cDzU@ zWca6qR*m{)S@!6c(bCWZ0oHdnA;fOLhSSg&n~|>RjP-U-%y7f!c;#OQDD1U#nh;kTv4vTU~fc zMuVev$B6VV?AQEGvZD-Y|5`#=Eu;t?MZVP%Gyd~c-~T*P{W}@cO95YCWsQca{knU@ zZ90GMG(9Rp3e4Uhnm}PcyCQ8i0*baDeI-uoU+jEBiljHdLBxP+9@A1lxba}9^XvN0 zcEI8=E$-T{L6)Lk`WeyIxrw?mCQj=Arco_0ibeKpjNedAKtOlCIr}&045NaaZ=W z;b)~ml(^vV%2#}W^Y$tqJ847wy!Yhv5s1j_o`hgaDLVSXzLCG~A)tohzEX#QTQ`_! zC}DHKy$^1W3;(?bQ^MHV%RCN&bY-Q5T05Z5c^&PBREKcr05j{&M1u&7K+Yet9R%?( z2VC1~|JGmYza@Wi5L4_uZDV$`^nwFTkkApPMi{ zYy%frcIBViI!FoXd()O?=!E2Q=`U8&6CU^pP1FsUYY#|Q!9WmVQ3yhGXS~aFN4J>+ z4ibq%Sgp%&De>M_>~Im`?NBY-gWG^L+ydm&`sOfh-BX)c#Gl8_k&c@kZXo6=3;Vw} z@&}EGc?d`9I@yVd=OO&FIZw8-T0_qt(eZ$~Z|C}nSy`t4P$LV7QDde0zys7kvmw(e zh&@wt`=>3Vm|Gh)GTXmDK=u&MXTDY$@l!D_2>~7)q2%l_?#8HGCUfN z7^->lGy7h)Z>Mae<i=2w2R=e~RaX?z^)Ao$ z3YQev_KOk8hHbhx`&awGx*i!hOp|>fDHzZGC$8E7PvI(7XtkfRN$Deyw_dy-4AB31 z5ij2!Otc_6)GG!ctQpVz{B7HoW+>8&8m4!V&jHCF0F;hG7!~&g5S2b>OT1%5Jplqs zDUCdYD4BQrc^<=QN=0~zG#BfFuh(mTk~|E>Z%D^IXL4^8Lv0dRvcJn!BEq=vZ(y_% zafS^tclYF=9)@=wYx@t{>HY@F0R>ptdgw8`%o7n7X6lZ77)(}Bn^e7}B?bM-d@%Uq z@0gt7F4+iX1r|i&BUI}@*IyO@U0Y>ze@B2xqg_-Pp-=?Rkry#=Lh}s7PYi_28^#HB zhuevlK`_(g?cX1$WXP`P9vNv}@d3XqdTj$69lqb_q(HNdICI?JQHMD02ON}r7 zjLXi3mF3);ZNkzX8BJ_L%gb@t%bNeX<8VM& zA0CheASKZY0129Vp-j5_l)#w}@xc!m5K#zJRfi$PdjAcBrj)1%LN+}Q&pj9U4YkI} zj%|PxMGF+^t)b1_YBXwwf<4+<_3S?<&3tlT90;kwYFY!YTS;Nu``d^Y#^)4T?;iv} zp_<8NKcg&E0C|7ENI}tZLuR(M`Obw!806M8F)fPoX@q&<$?^gHmi}_W6XiiZCuXh?I!2`vH0K z?;v8CTwR`l`nk|H0YXv`5zEU~#J7vv{dGJDm<>89FZXrm_qYNT?bW^Aof=>U88%m@ z!@uFi;210OKYwa_dbbAXE$pK*A_Z`- zw7q5<8K%IsBfZ1}Mc^$FC}4h;|5W~xT$h&tjS zubf0qum!+M|D;WBZ6H_PIkpeOv<$^xmH=3@69CeM=PQAury2kwwGueN5yDov3G-bj z;Hs_Z#9ggVjMibNa!V_qrf(lug%HIM11zfPCfLKge}K*Tz$&r)zlS>>;S0v&?nx3V z3BEpXd&FORa;hE^i}<`^Q0SaGg3c2At6Cs3s#xSIZvx4~Bn38(s10DLDXAovRhlq~ zc+1t9?)s4}Xu=AahIY1}sHp8+yKVg??eG;l(2B~APDMO22lv|%AM2DXgq%3jkM^Yn zc|%KJ8ep_+KzuYCaEfZ)b*@~RdJfl$fibB;R+h#m+{VM@W6)e`_Yl=c+fX1=q6tsy zmBa_GLiG!wwPE!zuV)3w0P%18AYRlrquK-wK@*o&s`K~?5g=cW<Td_@rPzAneT-+p;pNqcUr{ct0@QdGpY9`Gnv(={?1xPx}uObiHg z5p^!<3=wrcKalqx>8CXiuPghuh|x=il%GyJT1f0AsHsw`)ix`sBZ z9u%_eJyl&-C1dFZ)_#0Fmx{ByfUJ?|98?SxkA|z{ncCi3I~*4;7rK4oR5(}%KIGCu zDrxghoBwJ1=o0S01RBX&FpP`Mw@-Q4X%NPZWJ7{qSKP)dv4lh>U#apr!Z5m3flL?C zHyNA_u{834hu*FLpFECi^k{5N0BUmov`P##_e$fjzmC89U224#m8PDHQ}T?(H8$m# z(`b$WAE*~OaP~Ak_h8#=-W}V>TUImpT8mx!x7opqkqKjo<#>02i_1m97C0%#`0*xB zo0^Q46bE@_ewpK3hY8hO6BRv@^V}9RJ;EABlqwX@P9*|s`o#uVUIxvC((*I+H$X`_ z8EAtjE*~W{1P^9WAw*%m0hK}xkW3-@wr0jh!oV!R7CHv(mFg$rZIO(A^!w5Gf!V6ay|W@r?4N24K72GVB1w zk!c@|FoTjL@nK@rlq)RiPcQ*%h5DoAIjV~w9n74G1&c(p#^X8{IZuEEAxex6dvdx6U_c47m>mc+Y@MEv+R&^!RF$cG zQv%kS*u|?p&i@&axgi@t5=%E*Ly&q#g_@I_Y~VzmL_0FkGv5MY_ytQ5QAV^UmgS@vF+8Os-vg_xMX11uMK*07JT>Cd}UuS zkM6eHBAkhyb=;@3%l5;xLfC*;FZsZ*uWlc<^Ub)>L&BVm#3sNsJoGzyZ$s~sT5(Jf8doN zLP&R#O@bDHrnq$X$q8N*ZM5ztg)U|7=T0yu;1gv-^1h)tdZ-sJzf%}m{ZXJ!oOkPX zA!ge+ta=2{dg9_0u*1Q6(mS{-B#w-cQv*!ID`@-jULsV^EZIcRLmNKHap3RYJ2+1K zxeAL~S%Ak~R98D@3xGG~%G`dUq~_(59(#E^%#-P-HKI20r&;2E)Uc`F1fXp{@|c*% z4E;1`-+KXEz0ep#`~;cfcYm^V;JH9wH*fD#)0qE$|KIia?|%4y?>wA+$hNYm4Y2t- z5!FIzjr^6w*}ZmFbA2U4NGd>;_79NA^C}>F(mMMR!_I|DoVt+|gc)Kh9BPC>VT^Vp6(`gdNlri&(qX(wjwG18(fHtpAe4%ieDnZDloI~v z8Cpnlbp`pDky?ffvy=$0yF&u20ePYZ{A`&YfdF$b)V6j3{rt6U4YlCAB4R$4=~n^cXP2%7Wm|WWe7Nb0<~%W% zDe6+!Sq9ZqHS}Q9TnzZ_5qA+{Nq}iD?{s>J$u({+h4z6Df|(l|;8RYkb{Wp=dLWTM zw*ZPw; zeFm&(di<|HiKKCf#W1A(UlYuq-s+CyL<|mC$khi!S6F!~=9WZ|cnT~}4dTbpIGnM8 zeYV0yxgv$n4#>^bZ!>|WVP+lr8VG6-=M(2_1F}!Q9K~w|4uMdOn7T0jqu0C!5G6FT z@;3EOeB!P2H*+Utu7=8SlOK_13PBy5dMPCYY%d?@Wy`+S+Y~Cl?mJzxU7J0R==iNZ zQwKBX)r5no3tw+IF#=Bg)SkkVEqs3io^IIoHYE0?u)aDENU7?V5TVcet#HI92u4!d z*TK+XsPQD9f>-Azl;rk)sNgbMA0_8f@%f zG>;L~y|O*oryW0^9ReN*zxq27CC2Th(sFCRL0S}Q7$VNuL^7WUsKOx|xnfA>QA1;N zqIyyD5aNf)QD?=8D075xDVQZh%HgVvz`{g(%yh2#X9J76==_Fvfg z|EKeil1wv>YBn?s7?CHzcr;rc0+Y6R;XY5&-*&$39O`o`1_s7y98t5UVPrj#3`Tu* zw~BtgP96H;!F^SGz7NJ2IiR$J#>fTZLHUSoECbJ94S)gNf>|QAQYmVWd=EyxuYhJk z@Tj{^4-s^uxB@PkA-@4=vWG7hU5NR#yx7yoDrC9|vLk2FHkI%C)g^(feDEUZ4E}tb zLB$UvDX<=!du3L#l%fXxwe>9H`87MPAvU*o4$-!^#deh@iOrS}Vb+cEd z^u&?s6U|~W4vpI1UuB!+kK>BSRJfjhI4Z%1#)mBB!(>XDtbDj1wgYQ`5yKhmX>t4G zOnhOqE&6{tAf78eUMtTO$qO5Mv5NtNr<~j{4CSqLqrX2O1wYwTZFuw4jECI^{yna# zUCd?YCeL@C1B?r_EGJ2*Z(o(1_lBv4h~GLY&KK1F{g%=(-RscQ6&wp=`7>8$)e}K& z+8C?-{SU_ft+nrtr@HMBGiXca5p`v&Q1_{rWsMdfrltFIgK zfZ%2jtmFBsK1gQ*<0X5F~=)iFn06fA8K&g74p2uaYORU%c=zI7+ ztC7d=@5m^I7_UkY#y&*VWg1J6toLHN;h6|@ut&O}h;fiV7MlP2#%;+Yk3!zWW@ak zPc{EeQ~ED-fAD|rr;s~~SJ9lGVN_72t{1noLb zb(BCTU=4#XN?gFCn&F_OIzQRNJ7%?*_Zs+#5AYcrBcQW8a zmADH_{PJA);A%0fsCeycgWtUm2@AomNFUa?$$`$ zZo-{HmwFi?bjT4jf6fY4^zXvzkU(#NmG;!SKlAVfA4Gj$&*AI^-1E<-(=_lq$wYb5@EEUyZoVb zcj^(NG4Ou?BLZ=*%h z+c}}aD*p)N??wU}g>k*UeZ@$lp8+@E)9O2&AR#Yf0hyQ;Vj8=1_bJqWp}`xOt?L7b zErDAgu)mHIpiH(3Zw!Ww$5&j{X*Rtbn0Bp;o`{}DY8j+?{Wby+rK2lE>4kWGhoKEr zxCrrvRbDW%<#7ol3o5uLRc5#v+V`!L*E>1{aY$c-QHhJYiJ5%ZGTU30E*5IZLe*ciRG}W5|%yoYIB3!YU48ztD zU^WywJ)n(Nr{{3JLpE&+hlHg~L}a7^!4T2A1QM zCjiD)LavQ68z7+ZNOnpCbU-1?7T9<3No}(Mv|Y7>;w-`vI;DW zZ83X|FkD)0(>cW~LsEyP=2jo(#Q-@yE9!(g_}61Emca&Pl4LEO0$;Gl}5({N#3N+OeECDhhH(eHti_Gk)3E*GDqi8oJBk z)=_>O4LV^8aj3)%1f;(c$wxY?;Gn<*3hz!?v*#QtxnJYAw`pO6?%hwI{fdUWJId&E^WnZy#{~rhaGd+2$dP<)eL$apOkJ21Pb~A^usg1xjs<2z&1Z`Y_9GQzw{3 zkUYdYozp0#MHkAka7kTh6BffGAQpJj{thtY59)A@#N{Ch&|>$IDtwH}i5omx&B*PZ z7V;W`r!H2Tp9qxJ@~v;}Ra{m@84oXgKMi`-T2|^@;25h9?w?PiRKBsR*{hIzl?n&F z`>+&Zt9EJz?zT5s;0*7F?F-aOj>P1EBy!nZBZjPoQv53Knud7DdG<FCkBpfh7q= zSbpCAz`6B8!xbo^QV>K8XP$>0xByHmD1JY}qg+L5_C~H$zo^z3lh0sYAc3OCBvubH zn>r%WYA7xiP=wI{_TyX5OuI#L&LAKq3A~W_^xSq=|EAo8&e%5RcIeS-*S7Lmm{*Ii zDRobT$|Q}Bv*j>y-Ay!4#Ost5t;A>_ z$Y7oOApys)b9u6>xs!iRc4Pe!%^Z6YT@CwKR!p4JjaOC+S_TFNz8AO5GhvtZ>r8u- zLW+!xRGaz?JHtgwCk@CF%i~S+DZ- zZ_QpTYkF!_|CN|aX3w!dS3~EO8TT~?M7?>Ve{$OuDmN;?;Zy`FxfA{6euC8gPJ1be zVmK!#D41{1dKmWIaxCp3*u3IaklwWC>wmkPIjdeECKCUR|PHgACe zzla2L!}4R@M}KssZoi&u*nv5>-@jbzw9A!_A~Hiq=fY~@qtFio;`xHb`;CTK&d~@o z70rIcRdLzh3Vxk<39M|v%IZhWF51S^P=#R`OwvL2Q4H?DWpdc_8vzpRp|~3_JwVFX zF)}hz0t&6bZ zDBp7=T&AtxjBJg$sii*OH*uj_H-ve6jplT^|q>N{&P|k{l69f!$2vtN&Zm*mk#Q3zU zd0^WQKC3%)KBVDkE;hFBYQ3hUI(yd-_R)QhqR!uaP(jerQx;azoGAcVxEy_{p^Kmp zPIWERZr=l3{w*%FLdVexonY|gqU0l&(KM9YPG!zh4gsvf&JZafW#;A?iKmEFeCx^y zJ+5hZzWtZx2SeM@Fji%?-B%yXm6M~xe^D?a6)zi?)38|TjOvq)VEy(8y5I!|Iq{DwF;Wu(8mZqQ zn%uqYP*vX5k}oi8fKexf|FLUJ)99e;w7pvWs+zPt4eqDN7V+-(v@8nJ|=#SmM zaBUVO=Q?I31_x9B>5FykP4Q5B?8Y_NF`r`|qB7By9qR$Y`+#<}M0rP{gguoYJ^m)4 z(SCJE#d#k_{($V_+rI_t5}Y+wD(jNf?))It0ewZx?}GYw=5~RlDSQ9c=`Q!#pDK-> z=bnbrsg~ZT3fbv1f9t?hT)epq)-6-NOq6`&JtvPR`07$G(jPqWE|v|?`t4LEH5a*; z@mc!fADR2&$|q^Nf}Wj@y<$J;RwjyFdCK*Ji+eTnrb-@pCx&ten}L14Z%#HPtcx1t zx8w;oHU95{DnnP#2zgxC{QK;$)1cN!r8rblk+!W_v{#Fq?DV(=ev`Q z*&H12kH*B*rL5`BzZj1Jb#Az-pA07M7c{(80?&?!C0jQEZt=H7X~qfATIR7Y*1a#3nAnrSwL zVwD^N)wG^bc&kZleq%5&tU(A$fx!i->K0XrK(c;lMFXO(DO6nwl-Y%R+ zY0MKU>3yZc2yXrJa<4TmKJv{Q;$pyqcqevTUIL>MSyCkAmTCL++*klEX}=Ht z<4&K$g>WCTff(7>=U>vEq5HMz(ig7X~$oCnCdw{KTli-8Sqip43PAEIS#qy zo0TAo-%4-Z5tw38Dpvwdm}**Z+$QKC zCq1F2QX6X0$16R%h1Qbo!O3{zb*6xHwCt`$aOJrruOL(0|4%J%$R{< zHl9d-h(pTElg)w>Ev{C-ze*)cMNu(8U>W4D9>+5i)OO95C324rR z3mY0V@X}-WoDpAxno#7`a(<5%u9(6My-yX88^=v`YrTR>7#0A$5X^EOS~lpRh>l zm~w60$syu4_Joo9Y@@D)0n@`QH3hbd=b}U|^_!LP2!CAymYHd_f3QPL1_u9-CNrkn zHu*{kUF246@P-tvtII4m?EUB2n6j=Kp{*zxD6Kbz}zb*#JWwlOmPHm(Meo#-QmVczIO@-qR|}2f^f$$LT;P zIQ{e2&Ik)Pw>=jRrpYZ37y-{qpim4g;4Y>l=X?OKbo62v@6aIjM_us#M zKMObZ-@nIw_%r($vTO37c>MQ&eW2d>_k+R(gU9~;koZh@Qe3Lp5uOrf5ROJl+7I4n zAb~1ax!)3@1!B z+XSe%6b*E=0h70L&_(driox8678yhfo9o@x%+P7o%p!aX6E^JEht{`~@eoGtF1xrx zE!zSh7tW9nk&{d)agdV3RPP2ik-k>Z8%1~b(t>2X>2O1^4a6YW0K+~FudMvd4HFvO zg!@IsoHOE8iE)CW&Jm=%PRB*9-zhsFG~ZwJLO9e2AT!%z72pQOUsZ|O(Sb%TBhM( zhrRdaGpMX{#pW!_d9Q;^vk$<;k-51!DhN=9DG^`*`{H#@zqEZ?T>jxc=7h>yPapkC zyLA1{2Dk|M-+h@mJy7CkhRZsRv7243m)-hd1Ila{T04Kb2f+WiO*AY2d*?yD9#9D?1Q99DrhN~3p@e!Dr71+;ln@=g-!@=wNTP& z{5yb9W3v0Jw4FjDE)Uce{FpWYj<3letu9KCgPDXuK?+uG3r0LW& zM}F40(;_ovm%;4 zl;Q5a6o=LmTx5@-itn11mjVwmSZyfJ1s3LFoL+ z3tHE9niEvOMCc4VWi(%e)7S#tJ0%JNHF&XazY+>8!_Zm3xdmp0ZYe@=No&|meeOb- zf;WxexKszAxYA&mZjJR65|8HvCTm=@e4H#nFX?lwB`Fs3Vn6I^i;ZD|DSm1F0LnAM zBqB){caTX$P66aN!_uu)<>jg84G(du1JGRsM6`ESEzb*|F&=IisRBLdH8(<+E<;dr?Nf#&j<;9`Ew+L4X(%`@VMC0 zK~9;UFxj@c3=nUk{zzk31(?g*)PJphsr4`gW9*MsA>qP)Dk+A-6xsvrLGT00G& z+_L4)f>Kz@LD0)e1Igu?A%L40I^OFQTJz(-fostXdXIbw-g23Ay`n5wLe!+tW0KAp z#8_y7#fUv3GswXi#;^YNnHX>&)$WjsH~?pLX8+NPCi3x|IB+^-e)eY;EQe$b4=57$ zcgT{(kR$cZc7wanecd9PH&gQ<$nG70(_O%+1d6+EFBEC>okZ><8@Ffa_rgFdrfsJ< zKgPQUPg=lTF;?sHcYw4VS6Wy0j2ApmWpGF?1FZE0ob2b~9R)1)8HrH~>T08~YK6~F zq$K1qT<^*(&u-kNacVPmx_`))8bP~+(3@uof87{sbz`2$l$Vgrl{@^@T*Yr%d&0Ie z+t9WvDm?riqu18yq=EJRWh$JMKV7hV=C2bDUAlBhs7C< zhF!cACJog))W$aN)uh8bOewuddO#CY5{I}3$pKbpj{DWc1_1la7mIm> z)EB6rUomTSMTUl)#uc!9V->M)gVl?u4grBJc>Tmw*O4U69#ITt@LQQnb^-T4<{^`u zo8vdtCXu7&_GY4*a#X1LV%YObcWZVblqS7wI7N0dt+&`d5gQ=*K#(OklZz zJ4naNQRwPo;^+!~9nJaaEZnJDgDb7)sII%tXuPIl0VhWM5*j9C?TwY^WxG2)NiE6C z3}Tk{Jm;UFwF1J| z28&wOB;sL-o~v^sLPpornEW>)pb>UMBQ-QkK0pGC&^7(w5hyG7Ol6y6`@A6HP&q$DhDdi1MgKxFx9TUCW#^QGZErq5!T(N(m$#lKzkLU6!P9l=GCg2=791?g$-*@QiVEHXLc(_=dW40nIOHEt2WD&K8+Ygf}Fl zdJnEfp%>JizFmw=cMV=Sc)iw~DzYmLx5_H{*kgx1cIv=M(mq!Ay_kn_<`?%icd-P; zMtl*D5b!^&y-RN;L2CR5JnY7@FkX`esj|kwd;OJxEKb&g@wX#eTFT2yi}7g<)>@4(@6gMr)H;=seR; z1f|%Mo7^E#rtbJnzrA}#V=D-{PktQZPUE+_$zQdy8fW!aAbDR(aJi(*wA1F=BEuK8 z-Ek%t3OVH3R@n2R{>nQqay6wkO6O9f;Vnckr3&kAC7q4BKT@^5neC}d18P(Qjayp} zm-*Y!r2Rr%{8-LeLUD^kCbxyrD=7C$P4H~5P-3ZN1toFe3Un3zJXo!Rpgez!UHV}N~p}){ueG73_!Yr0GO#I(KBjn^U33gWKf93Mm#1n#o=a4OglF-ZaBSZM)Z0dUsgL7f*tNsKA))+@>ke#-{oX+ZC z=pjP;3t8=(*x{1X%q3OKeXO;z+AYrBz2NIOa@%FNfpPs<Vxn`?uxuRr%VTVxM(0TaZ*GEvL(5%FRMXUn9e7hS*DE{91($rEOJmvPyB+$NfJDgIxS%UJ( zv)~Xs<0+2uVIsv?y((oJ@s~K{eJmrQPEb4RYyWzZplOI`l$&xP^swCsmpLy?I&>2m)ZaU1T z;MiDB$Ja6gmw6wdjR0R z&mf|1QX;#&$EVGAk0VB5@%6yQN0g=J;3=pQ&9ej*+G}hQyR>^d&n^!Ltoh0CfJdaY z)+89R#6_FZGdI$s?C^#s^Q(ALo_)r0#F!ECgPu>0$#&TNswtoF1_fqcweL>iprZa5 zkgvqxQ7F?3DBdI%6zD0Mv1ybjYz>J$g&rL@i2gRJG0xK0M3<}x>{ckjX`BI>E zcdMNHX*eC`r0&vmpA|{ztflN4cf#R#h1^XauH6q;xgDVFKy^fbzA{mzpo+=*Eh7b% zxKS;OYL6@Cg$W#BbFdkN`iYD}ZF^(}>LZsstH7MQp|V`S5kkYP`@aDFGxvA>0R2`*f=f22&J+KIA-l9OX8dr zp=u%aL0z{qXrjoshW(0NH8lCR+wHqD^~`#apnKf|36up`v%q0gI0aM1hY}GM zHUjf!*{-NPdsJYBZ@s~{rXNf{%hS_7yJ_wgJFQ>}!{QlTLe#zG?-WB2P)}jQSjy90 zsESRR<+yikL{Irsweyku=)!aE zuaJBE?_0%S%%GV4_xmu=`u+P6js3s;!FG1j+jLp0Qv^i=iE8u;3RvYES=a77_ dict: + subset_y = y_test[mask] + subset_prob = calibrated_prob[mask] + + positives = int(subset_y.sum()) + result = { + "label": label, + "transactions": int(mask.sum()), + "positives": positives, + "prevalence": float(subset_y.mean()) if len(subset_y) else None, + "metrics": None, + } + + if len(subset_y) == 0 or positives == 0 or positives == len(subset_y): + logger.warning( + f"{label}: insufficient class variance ({positives} positives / " + f"{len(subset_y)} rows) -- metrics skipped" + ) + return result + + metrics = evaluate_model(subset_y, subset_prob) + budget = precision_recall_at_alert_rate(subset_y, subset_prob, alert_rate=ALERT_RATE) + result["metrics"] = { + "pr_auc": metrics["pr_auc"], + "roc_auc": metrics["roc_auc"], + "precision_at_alert_rate": budget["precision"], + "recall_at_alert_rate": budget["recall"], + "lift_at_alert_rate": budget["lift"], + } + return result + + +def main(): + logger.info(f"Loading artifact from {ARTIFACT_PATH}...") + artifact = joblib.load(ARTIFACT_PATH) + preprocessor = artifact["preprocessor"] + model = artifact["model"] + calibrator = artifact["calibrator"] + + logger.info(f"Loading features from {FEATURE_PATH}...") + df = pd.read_parquet(FEATURE_PATH) + + train, _, _, test = temporal_split(df) + + seen_accounts = set(train["Sender_account"]) | set(train["Receiver_account"]) + logger.info(f"Accounts observed during training: {len(seen_accounts):,}") + + sender_seen = test["Sender_account"].isin(seen_accounts).to_numpy() + receiver_seen = test["Receiver_account"].isin(seen_accounts).to_numpy() + both_seen = sender_seen & receiver_seen + unseen_entity = ~both_seen + + logger.info( + f"Test set: {len(test):,} transactions | " + f"{both_seen.sum():,} both-parties-seen ({both_seen.mean():.2%}) | " + f"{unseen_entity.sum():,} involve an unseen account ({unseen_entity.mean():.2%})" + ) + + X_test_processed = preprocessor.transform(test[MODEL_FEATURES]) + raw_prob = model.predict_proba(X_test_processed)[:, 1] + calibrated_prob = calibrator.predict(raw_prob) + y_test = test[TARGET].to_numpy() + + standard = summarize(np.ones(len(y_test), dtype=bool), y_test, calibrated_prob, "Standard out-of-time (full test set)") + seen_result = summarize(both_seen, y_test, calibrated_prob, "Both sender and receiver seen during training") + unseen_result = summarize(unseen_entity, y_test, calibrated_prob, "At least one party unseen during training") + + results = { + "alert_rate": ALERT_RATE, + "accounts_seen_in_training": len(seen_accounts), + "standard_out_of_time": standard, + "both_parties_seen": seen_result, + "unseen_entity": unseen_result, + } + + logger.info("=" * 70) + logger.info("UNSEEN-ENTITY GENERALIZATION RESULTS") + logger.info("=" * 70) + for key in ("standard_out_of_time", "both_parties_seen", "unseen_entity"): + r = results[key] + logger.info(f"\n{r['label']}:") + prevalence = f"{r['prevalence']:.4%}" if r["prevalence"] is not None else "n/a" + logger.info(f" Transactions: {r['transactions']:,} | Positives: {r['positives']:,} | Prevalence: {prevalence}") + if r["metrics"]: + for metric_key, value in r["metrics"].items(): + logger.info(f" {metric_key:30s}: {value:.6f}") + + OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True) + OUTPUT_PATH.write_text(json.dumps(results, indent=2) + "\n") + logger.info(f"\nResults written to {OUTPUT_PATH.relative_to(PROJECT_ROOT)}") + + chart_labels = { + "standard_out_of_time": "Standard\nOut-of-Time", + "both_parties_seen": "Both Parties\nSeen", + "unseen_entity": "Unseen\nEntity", + } + labels = [] + pr_aucs = [] + for key in ("standard_out_of_time", "both_parties_seen", "unseen_entity"): + r = results[key] + if r["metrics"] is None: + continue + labels.append(chart_labels[key]) + pr_aucs.append(r["metrics"]["pr_auc"]) + + if pr_aucs: + FIGURE_PATH.parent.mkdir(parents=True, exist_ok=True) + fig, ax = plt.subplots(figsize=(10, 6)) + colors = ["#06b6d4", "#10b981", "#f59e0b"] + bars = ax.bar(labels, pr_aucs, color=colors[: len(labels)], edgecolor="black", linewidth=1.5) + ax.set_ylabel("PR-AUC Score", fontsize=12, fontweight="bold") + ax.set_title("Generalization to Unseen Entities", fontsize=14, fontweight="bold", pad=20) + ax.set_ylim([0, 1.0]) + ax.grid(axis="y", alpha=0.3) + for bar in bars: + height = bar.get_height() + ax.text(bar.get_x() + bar.get_width() / 2, height, f"{height:.4f}", + ha="center", va="bottom", fontsize=10, fontweight="bold") + fig.tight_layout() + fig.savefig(FIGURE_PATH, dpi=150, bbox_inches="tight") + plt.close(fig) + logger.info(f"Saved comparison chart to {FIGURE_PATH.relative_to(PROJECT_ROOT)}") + + +if __name__ == "__main__": + main()