From c176649a6abec31d69998f2cd02f576caedc4654 Mon Sep 17 00:00:00 2001 From: Samarth Uday Date: Sat, 29 Aug 2026 22:18:17 +0530 Subject: [PATCH] fix: Refresh artifact under corrected methodology, fix silently-ignored CSVs Completes verification of the temporal-validation and walk-forward fixes against real data, per review checklist. **Real-data verification (all confirmed on the full 9.5M-row dataset):** - Retrained via the corrected train_model.py: test-set metrics are byte-identical to before (PR-AUC 0.9859, ROC-AUC 0.9998) as expected -- the trained model/calibrator don't change, only where the threshold is selected from. Decision threshold moved from 0.001075 (old, selected from the calibration split) to 0.001160 (new, selected from the true held-out validation split). Calibration diagnostics are now honestly slightly worse (calibrated Brier 0.000078 vs 0.000043 previously) since they're now measured on data the calibrator never saw, rather than on its own fitting set. - Walk-forward backtest: 4 real windows spanning the full timeline, PR-AUC 0.938 -> 0.990 as the training window expands (previously: 0 windows, RuntimeError). Added to README as a new Temporal Stability table. - Unseen-entity evaluation re-run against the fresh artifact; added the fixed-production-threshold table to docs/research/unseen_entity_generalization.md -- one threshold (0.001160) applied to both subgroups lands each within noise of the intended 0.5% alert budget independently, recall is indistinguishable (99.3% vs 99.2%), precision differs 11x consistent with the subgroups' 12x prevalence gap. - `python scripts/run_experiments.py --generalization` run end-to-end against a real 284K-row sample of SAML-D (fast mode) to verify the full 6-stage orchestration and --features/--artifact passthrough work correctly; real committed reports/ were untouched (sample used isolated /tmp paths) and diffed byte-identical against docs/assets/ afterward. **Bug found while verifying the above:** docs/assets/ablation_results.csv and docs/assets/typology_results.csv (and their reports/ counterparts) were never actually committed, despite prior commit messages claiming otherwise -- a blanket `*.csv` rule in .gitignore (meant for large raw datasets) was silently swallowing them since the first commit that tried to add them. This meant /api/results would return empty ablation/typology tables on any truly fresh clone, even though the PNG figures and JSON files (unaffected by the csv rule) worked fine -- the fallback-to-docs/assets resilience only half-worked. Fixed: `!docs/assets/*.csv` exception added, both files committed for the first time. All 63 tests pass, ruff clean, API smoke-tested against the refreshed artifact (health/model-info/results all correct, decision_threshold and PR-AUC match the retrained artifact). Co-Authored-By: Claude Sonnet 5 --- .gitignore | 3 +- README.md | 18 +++++ docs/assets/ablation_results.csv | 5 ++ docs/assets/calibration_curve.png | Bin 51143 -> 51147 bytes docs/assets/model_metrics.json | 8 +-- docs/assets/typology_results.csv | 29 ++++++++ docs/assets/unseen_entity_results.json | 25 +++++++ docs/assets/walk_forward_results.csv | 5 ++ docs/research/unseen_entity_generalization.md | 21 ++++++ reports/model_metrics.json | 8 +-- reports/unseen_entity_results.json | 68 ++++++++++++++++++ 11 files changed, 181 insertions(+), 9 deletions(-) create mode 100644 docs/assets/ablation_results.csv create mode 100644 docs/assets/typology_results.csv create mode 100644 docs/assets/walk_forward_results.csv create mode 100644 reports/unseen_entity_results.json diff --git a/.gitignore b/.gitignore index 721f485..2ef5452 100644 --- a/.gitignore +++ b/.gitignore @@ -44,4 +44,5 @@ logs/ htmlcov/ # Large data -*.csv \ No newline at end of file +*.csv +!docs/assets/*.csv \ No newline at end of file diff --git a/README.md b/README.md index 22e2e14..1a4d6c4 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,24 @@ The model detects specific money-laundering patterns with high recall: - **98%+ detection**: Behavioural Change, Cycle, Deposit-Send, Scatter-Gather, Gather-Scatter, Stacked Bipartite, Single Large - **Limitations**: Over-Invoicing (86% recall) detected less reliably due to low transaction prevalence +### Temporal Stability + +Walk-forward backtesting splits the full timeline into 14 chronological blocks +and evaluates 4 expanding out-of-time windows (train grows, calibration/test +roll forward each time): + +| Window | PR-AUC | Precision @ 0.5% | Recall @ 0.5% | Lift @ 0.5% | +|---|---:|---:|---:|---:| +| W1 | 0.9379 | 18.39% | 97.05% | 194.1x | +| W2 | 0.9658 | 21.61% | 98.19% | 196.4x | +| W3 | 0.9774 | 20.54% | 99.08% | 198.1x | +| W4 | 0.9901 | 23.49% | 99.38% | 198.7x | + +Performance is stable-to-improving across expanding out-of-time windows, not +just in the single held-out test split — later windows benefit from more +accumulated transaction history for the network features (degree, lifetime +counts, counterparty concentration). + ### Analysis Visualizations ![ROC Curve](docs/assets/roc_curve.png) diff --git a/docs/assets/ablation_results.csv b/docs/assets/ablation_results.csv new file mode 100644 index 0000000..98156a9 --- /dev/null +++ b/docs/assets/ablation_results.csv @@ -0,0 +1,5 @@ +model,pr_auc,recall_at_0.5% +Base,0.08984612760924429,0.12868949232585597 ++ Behavioral,0.20015470040725364,0.45218417945690675 ++ Network,0.943473551428709,0.9769775678866588 +All,0.9859286355765932,0.9929161747343566 diff --git a/docs/assets/calibration_curve.png b/docs/assets/calibration_curve.png index 0aa2b25035d97fcb2ad42a6f2b551a61e23917c7..68235034a21acb8aab5557a8beb6a9c3901a07b7 100644 GIT binary patch delta 11420 zcmb_?cTiK``z8niDosHVrHX(cMVdh9Aiau!5Fi1O5-{{4^&+6yDAIfHRq34%q5{%{ z0HLE&Llpu+`ks8gGdsIGv$Mb1-9MO++;h)8_dRcUp65MXQDnVQWYG|*RRVD_}IiwF%$@971 za;Urb^#iEPZBoz_#|yf2V- zbe@ESB<$8X<*@gH=5tci0669h}wv<_uY@3<(6DYRbX!pnGK~ zS3S#z6pW<0Ys9Sj!>q>P12v130l0&4#gh2@=NsH%RQ&j*{#d{`>2@3lzXk!6aE> z^rRmYprC_YjK1yJ>4{;!*42u0;68eiIQm=q>*JreI%`7CU%Enlj%a_<2a2yP@;AuU zBW~DIBq6JwUa-Fj22?};>(?t%t`tf^G$P-_9PDX!hDQl53bzUeEThAYMWEptM>pK$$DBBUw9HZ}u)L=#arrcD^o%hvF@1v( zHzl=xXew}t`Zsjz$EphD+b5m;Ec4hQeD{cKH?MKvthVcqxuyDc!J*n7BmBK6`KIk; z$xKX;h>b}^L}ZLm&sEz8T`)tA`!Mw>;tt#r)Uz7D@b?In}*I7u>@U1KiDnm+sh1cZ3pa^8SN< z^VJyTRKv4pPDx~x`cQ_%=!5DflO>h~<(yzR$sBRG6VKBko|FXVo=5nB?>IanC3AJB z^la6FN5&A^Md*&)w3PQ+tYASifg4`m2D_a`;8}eT`cGUBYUj7l^5R$&W7PL zuQbUo=j7%FEQSYJG37K#W7kF3e)6Gw58N57!AO8Q^;l%Qb$IkM1WF%QCR}0?(kO@5 z=BY=q85SGnXK4tJ3qZN`;xKWQoSf4XyR)F3G;UU&!k*!D$a_{4I#OlE&-?nW!|6KZ zSE3mc-Tr(rGiyF-D4PoGJGN*)N@CH)Pop9#JvA;8U#{)KTCnSAiSh6!8}*5Lm(1Q> z$UZaBz1}O8Y>t`nJ`JY}q-CcgcaO$#EIL|=b;y=3M@^YC{Zpwy+hEGBC4QDBl;pW=g z8SfY<8a{etJCG(lTy3vk{HQQ;q}&{mJs*y&W#D%wpRUg9%?}-~L`?jjasq0>BMuyP#_V&{93o^<) zqhdqmgkpp2koAdbb(0DUzMh^Q-;@kVP#%_+!IpDZS{&L#IScq7cCK?h3ZWhS;ly<~ zUE>w5-(#Vk3~Dek_e*CNayz(`9IUSZ-zj`yb&zvj%n~~@vHNb=Y5HO&MWKt~;zj1U z`cK7W%=}=SPL7EIV1Qzdj=p`3&=3ANcMp^vLQ9Gg7;- z@is>yW6sKT5Ym?SX=-ws)7cdRMdwCQHQ31Oy@o#R#kshsLy4Z)lc6k^A2g%=#QG5oJ$I111|7b= zCg>Q%Cg*1)WO-ORnOm)J`nS_|qD4nI>FHRAk^kkrYt*ZL)Z5?=lCZju)KtB3d^RPn zj?L}m=JkF8{ZmzogPfNg$I6nIapz;%wLSa9@9aKEJiFIQqasVZiBm{`OnJ1*RvYhe zxQk<63Ow1v7L+3!-4_EkzkG$}aOdCg(CK>~2Q~il!}nA;LT$*Km}{xrtZ&k<25+w} zCL-^vC`s*KKkVj;N}BWE-`p5}_?=GSn8UKk^Fy{u_#tQY_W%$E9+ZD21^vn`Fx_fDr=+T&|##ETKFO;8!U_WO`bk=)gLzop*OTfNJxU1#?Vy%{x^G>p6D)$VYfJl z05F(J_1}>sm)`~xXCS%7NXkjXA0!X%oKq(L@Y;YpjQB~PP4({#9LoQd8Ndzy4--IO z4gb>&4-;(7Ud&!NE(Vbl1JVho7V~k$A@}OgQ#_wOeRAJf(A<{Pq6*ar2xw{i(Gi2& zC5v|XrPMAh5pTc%J{e`!`%f6EvjD+4Gm)s8r|mA2IvFS+LTN_*ehlzA$}sr zcHaBM5zPbrAMEq*+WeS_*c}+~!vC_^|Ej(Cd49+Qw}?Tf0m%|h2`o~sidqQ*d;mF0 zLYY>YbREn4b5LSWEN|~vxjDCxkX}q|Y(8-Gn53kn#TsJj1a&zM4Da2Z@5=m7R2!Of-DuLfvtD=`E$*LnIckXsdL`Fu|A%YGTS@Hw7f2)?d&3}s&52t4SU4NqX z(*kPp?71Q3n$_js>4mklXH^K2?L}=)*7i0LCY|I8%xWD+Eu?7|$}8Irzia8}^tlFB zHvGQm^y~9WGp)6So>&ttt+!prz!J85D?_2nDfB+4$G-ToY%J#BaMrz7a6%6c0J=%% zD;l|_7ZF%?|x@%&4 zGZoq;to|W!nr?98>4yM$g|$%v?h{|y6*ld))W__>s+}l=Zj1HXTaBL&{%G0k%@ZMN z``uKynG1-eQr%8KQED6GEy0=x! zT#U#*u!TGVChMyO5mMN@iz(DMmv9;r$0K|fuV&7*IYK=jczY9t>z~-z==!~55+?Mv zaoqzjkrmlMNz3Azyg%|tKiL;}`Jl;U3bgwY10LxVaa|-N_@0gfI_AmOx5;YSW4 zb*a7}z&v^~J0kULKTr21i!B}>;0rb$G<$DYECHweUW54o`;i$)FjE{>`7`8=^jPW> zX^&!&9=1hfE5CgD_s_4TeL#t!gE)k;a)?ohQO|q}0A;D(Z?vWkI6Nl~tbb%F6|^RZ zp>rQ*DUzBsdzT*{u2q`p`2#(IXR8(XOVAXOd=|vx0?-Z&%i$74K5e-)T5kTh$rBfo z@8AEcGkh^ON?x?x2{A+{-K>eAQ%=10R3}q@X8UAe`kOcpj~bNWyOb=j$b9wYO{bvA zIn{ynIxB)Gm2QfDjbKq7&0aX_EU|sJEq`Cwo{W+}Wnp;kLZ|ted`@cPEWxl3r+y&2`HGrpcjbzxfEsKA%hvN;8=5j9V9LkPJoG z{=TJ-&U@TF;gO7&I9u_)jp;_ru_qL2U0t}=F9y-$IR9E~-gsPOptQ3*=)%@g6F%X- z_}$juJQF%?d^TC5c|*dHF{Ra=Q+xjvcn^ErY!;nolhcM-G zgwnsPb9@q*GpfH~l?~>zgG_JlQ=WlB0MGE>Y-HJ#4mmjDD?LyP?dDW@m6TT~4)*Q4l^dG(OAF9NF4xf%wm=(lRnOpRdKIua$YNPSv{zqfn?NQr#&!&Kg`| zMjdV_+!gAV%$QBFTb#}ERblU|iTI&HCl`13=>0#$HP*2jfz&HPx9ykt3FRvFZ2TT( zqAy9r2Kc`C;M09XE}F8Ireuboy|*3k%b$RbgZza=OmFZehk!7NVf;? zAD6yflb@HT(G$aM3npL6AP-xsz6TOJCYvM9q__IrNA%%G1)Fcwc>mr6L9!UOkSKX0 zRALVxbz(E0{nwnOOR=W-SKhQa`ISF+QQ5T0kFU~@;l?M}duniP9Iv~tBUEeg*Ecl2VFG0Scy7L6{7`PpGv_mTZ4ZzP ziT3aj54|i(@Fo{4-xXN$T-FE~eREsfYg`k!mF&BW9EoH zWoSTV%ai&}86*s^xic7f`P64`^l|FiM6Nm={^W^2_|m`o_wVzypBYqErK8=g4< z%O-IZKY%6eT0!nGUmi&J&{1&x89bV&$so1`Fu`xbSNCLG;)Id2(|4bx-bYuxWfFE? zYCYO)7Ft``k6%h=LOk!VJp_2L>=-)0r+SGarI(cgBZurc(-V`DRz&Ht`x9T_qS)jd zciso-Wy)`ex`2(248WfrZOIg5N_%_*q$@Ug7LdxjarE?RB!N<(hG+I?wF;mxdp*fI z9aQ6F-;={VkL0$119$b=hNToI7D?0n!S~E!JAbG?+}!1yZW{xNWF5?N#mds{>(wQf zlLN5Lwp6#gGJ-xGI(~V5*)ARAt%c8*IVuD9XM9X7EYjvlk?s0G%w#!m*j*W79xFz+ zw_&$ZK(6`9Z&C&&B_j)Tm`ef5S?bUs(?045S{p0(_4Mqfu^-IvS|DxpnSMSWVFG?E z2ed}49MtSFkb^D8e1YoQLP2u&w@ZQLKo_qAYM`#4w`(ywjq>G5AU7>OpLG~1`Y`V& zqY$1Z<@WhgOB;24WjjUqCdjnXT><@3ep|!4(5_3MTbtjWnprkIqzOYgV)s})#Q2y3bh9TGNDf1%CnNPkT?dRPwsA-F*v?|$UcS$Brh`Lsya zD$v{zzt@o|b56O)rS0HbiA7_Tb_P)o>9YFcPMSRorvfT_pKBBf)T&zN85pGJcb&)( zd_`~pfwixc6NN0xfC9nQxs!Z8435F&CnDT0kZeTBFSi^lrTTgrb?c%hYZ})*x&iYK zprwkJuOAmqUDpRGt_%c0*!0);!_YZP^i^M#DIh`d@STQp|=4dJv%<(z{Y z%JP08j?$i!IYGf|H#Fw{LghCZK+R=8BJYX{c%73?&E>4T##ZG1r_#4$;c?~fR^uZMOxQl z(CLA_832=Ug61YEk}jLIl;{(-scQQu%}M|_3tDf!UXVFuHr8Lo9PTqJDGxex;TQzVq?S&v$TkZrQ+7*dXC2uKYa zbSEI=Exj;M06}3}(7QSQsq<4A#jS1`g|p-8Y2W4aoh|5W%JxvL?@FCbCid)jSwT95mgEH^!;f+@#juGM-)diu8sICMj@XI~H!~V}9=$Y& zAb?RCi3x0wz>qQH+DQW$g6QKjDl&-F8v;;P2lWtS8gA(HIBk>OxaGMUZo08r96c$n z-pda2)VL;$5P*)R=JFoqsx{Wsll)``@&&mfV1tW*3yONLn?~i|2NT5NFoiM+Dz487 z6ZR)D0}T4IoTVn^F@QTKlQ3i#gXkQEQL~(i7G|nTmvTxZVZH4D&4NOOt-I*)UI==)2W^ z)<#4jI?z_@zNoIcHhnGE`&{*?6n+%e!b7*f_{LzSf-sRn9D_u9g@J)fQxnoO1aL5_ zcU5$U}oXHnSdTYX-T4GzPG`u|=H(Xh3cFPdYcjcc|lecbmHeB=f%D@T53PgvADA1 z7mz|(`$c!~lcXQ5XXakY165Vq1~%!LNRy*cXtllREQLw93$xQiQ!Z5Ey*Z6D3@JKm z-%WEEb#G0WElG5lX&Oz_POt1GRZK7?36{lg)X%~F*3iRwZ@gB1aNZRX8d&|29oj1j z8YuUBsc(pP!y8<)4_bZkOU%PCP+CSSkqUS3-yajOfqu`=m)mFferp1z>D>X~4*Env z%i;iR)Mx8^Ld^7Ik*qSJ&*lNm6vL~(ziu7DDl^7!n5Xe-PuvnIUAVtmS_waY>6)kJ z)|k0VZ6urAs#iJ|3C-|ZG<>u<0QtMi9SNP(#>d6Q{Tg4zR=Nm+Q0r{7AnF{_?s@T` zFWFTEFoFqa0GIpXp{3+@8K2Gdsz?N7dYcorf*OY-fMSzseFRc!6|s?#UjF?sJLGZ z{6!n&K;G9U?KAFVsf0Lil-k35~5 zszJpf>vFNi%ODR6PGPp%j+`rfp8)C_=_NZu_6+!^Xm7yT$vz^Hl*EOYNEAu4SBLZ8 zT_E?~n%yu}3%zft7E0+t`U^;FtR3ML#;|rECz$~0syyIeCmEL#DF(@0*VNGXSkZRq zc$b^|tK(44ORhVZ@PlgmK|ktm;#5PKM5)Oc@b*$rc_TNbJqG3E+1t}ISXbis9(~Kh zg6@*KA4qE+7wk5?Du@8F`q@O49V?A<+tH>>^?Qm}ghS&JBasdbz)1Q|CzW;p^tL3Z zhT0u+oNwWDkjfHqAG?f1PKD|3g$ zZW32(3oN5=ydxcPw?8+<-cunhRA8TE0HI!AxVGZedP11ejm{SzuQVx7F;hc6z0J#; zns^bEGtWF6o5SP_csk6hPg>|e9JU_B(fI9nLy%)6hAe#+^v2eJo(Boxfi%a2|4~j? z3ltPD9qbiF!9@&ON+#&I;R2hu9L=+|iH@Wo#PeV06BW^?#|L&d4?nISN2f6_@4ljD zev+;L&5F2y-gcy7kMtc^#4;!q&G8u-8P;cBtzW)8@Q_cpY|TxDRG_fUk2Uc9tS+5H(xml9)cf7hh!Bj^;&i|UebS|Z4pe0+UnlRt&Z(2Nl) zTGuX6M9s)3s9jq)T-#m|U@%LhVuSx`aUlz7sfS{z@U6}Evmmj`=i9%>KzmV?+>y{r zx!h9cy#X%;mD_GsTex*0sRB!+soj4d6?|JkHq^W3>zR~Jl6~LS zTs!p2mvAYM?@_^X3nmhwa6MZRj$oo6Vlo_na#oNYU&lVorgpTqmWcBsa$9tb=LzwY z0zj*M%RuyquXVzvIB!VEp(~%38WqQ&>Wk1ZH8ZniX4N|S1(TcFQ>-Oh?ap%zj@Cpc(7~d zU^z3@+XHLqlYR`g3W^1Llz1>XTLBQTd?C}yyj$V&bYq|eK+Y|vhiitlj>bU#jq2*y zL^jq`~3ZWx3VFx zeD?-5o9uWEe&mb}8MqgpTNiKCPj>ahS?`d z63yz#PewwT*I@=aiTx_|-U8X_Ju52D04Zsy$W2dKaXnuG@>_BN^yjBi*A|m7A`Uvm zXYPP1ltS{&d@|$t+C3Ha2cy6bZr_F1J;PnGfJNa8Yq+lXU=jZ1xa(iiy$iB;vmvBH zvZl7S7~SFIg>XDUb|XB47r7Ems@27~8?MM6B|~#OgW{+pEuBHVr|2B;%#hh&RF@GO zb{a3xy%~i27T4Y8WItHJ>*Gd{cC0udWat$Vy-_G#qQ(6*@Nz= zyrLGtNL(MuGAL|rze7rbblWF^f=Muj&C4171F|DT#^OFGw=d#~~9 zKx;59yZm(AcN|$GL9K{8_V8z^QZXT@C|`8)ZZ7Yv-$)IaxL`!z+psY5C`|1_@8*S3 zOTgpB^Z)zHNRTW?P+iKl*jaAL#)z4gAQW?`(NDkTY8;n*x%A-%2t|0VERd ztFVq6ILfy1Up0-=(q^r-f7rivcttsM1?<27e_qG$=L^cdKxOT0V)4L3TLABhc6osc zK>0M2B#aWYv8e8`Ix1;f*Q^#cS$GYBlQ+rXj)4wioP2Mu-d)0Y*T75U>79%lfzsLB zmkRSLfw~<18ovq|d(>NZQs5tzb{%|Y^Hwgk1IfEtzTBn|YCQS#Qm!o2q;oI6FlN9? zn$0K04xkW^fO7G{)sJ`JPYStHU}3MDiK=5}#w(?5Z)$>w%vSi4m%k|;>#Z*n^>aZ^ z=agNls#u{O0fliQwW6qC2Zw5kh02V5ATK14{1@=x1c7We4FHp*g8eH1bME3WT{YO= z_X!m>=&7-|1_yr;L9guxU0c=Kk)F)x)FO20Iw9mfx)A8Fm70iQ?{|c8Cty8e`}jis zm$NHm?eYdrd(wi|dk+TVBUP6XG&`Ym@~+7hpznEixpTIyZp&SP=V%&oGMBL@Tn}n0 zw~}G=$yJN^ZoprFbpiXwa|ax$dO3)iI3k{7O0{vlZ$H3-H+-NiSG!tNX!Q_jA%ziN z_bwdpRSVS%JjgbVok_$x<|X%h_de#XKK|KP!C$f=Ses{ne*!@p1FuErn)HkXjwsF5iWPv2XxkKJHWAtGP;1h9dV_xL3|-D&Mp|xi z;Jq23enz+EH5Q|9DbLNV>NryLYyX6l-dbpijt?+|SLqFDeG~;qaEOn&Z)z%QY_hi9 z1AB1kn!r69dcPa=0icAig>$>IL5#Bu8}i8IHMfJ$dRs){X`ZHO>mBV})=_eNpaE>R zBTFfi#b&<|@*F?!??9JId+-y*;md0G3!vbEVW}|KGi8t=<($p;HKRI`Y3Hrz7=EFw z4*$g{DjF*Ww*gG=j&=skN>OU-CzZMLPyO)VF~nBU&T|VGWYu~^&qHvf2f~ZklqXVs zA5b`{24+de2GGE2>8NYdbSaNdq& zCpzJy!>EBE=g!N7i?|*5#YH{sDA0ki(=xBI%u1+26|%saa=5wR@q>-%m;?lhE)pJ- zFUJizUU&c~KVR#xmzNCOab8De{s%5&(w~oYR#}^Suc1m=+xnUy{5Csu{E4rBWHkpa zT9B_|`#r8_Sbls1Q^TYy=!UscL_Eulfsc0c0m!(EfzKZDhnVW@c9n&XH2q< zG-5|vU~EXw)sdb!0jH}bf)B@Jfh6cJHq*M1N?V!D4Nu;CUYs8S=QgTrEhxnzrl=j? z6c(-yPX;~Qx71eJ0*xrK_0WVc$E_4Uq20eVp3`Q)&ixWU4Alj_#vGgBO;ADpr>4P- zQ+-=|8Td}o+R9mIn$BZzYE*LBxb(u>M=rRo@%}`523PZ!IrGXYlIW`sZ(OxQ(U6c- zX_iVkdTF4e7A;<0=}<4d>$og@gWh+|m%&%x|3=_avb|2NC5;6sN8ue#aQ=Ym3G9A; zg6HZW6Ea{ZP8{J(ZAkLxDk%gy>WcK3)jZg%q7C9v57x`1!3y-CHNW#-c!TsZO8|=Q zb<4v_gdBi)@A#NU5cbu>Tp}bEwEsEfz@aR1k9hj(|Mm<5IREfJs3HG9#gPB~ISP_D cI%n+2p?UM8EQp49n1n=ARqsKCvQ@}`12r}|q5uE@ delta 11416 zcmch7XHb(}*k-UG7K#-GX(A{{uhN^+A<{dD)Bpjb_Xh+;REqRo1cG#h0Mf+>i zjmMMAWP}XaQ(@LYJ^qJ1W20TpZ7AeHP^R6`QoNv`MEH>J99rhYL*0M53TM4J-xoB> z3xmO!Z(gRkW4|;~i#-#HH}Bp41<|{-y}rzPSZ4u7Wk20ve)Qx|ooBJfLVx=ueoOy@ zpEO5sR>r4ZN zWxJ;|>B+2)5BILn(>uP96As+>5Zq=4Bf%qbvaf^JW|Ay^FV~Fo(DIrKWxc(|ac4Y$ z7}*b%Ta1s_lY_^?xK7FMj;4#b=h^jSx)CsJL@{=D#T#6P%%UT%q0Qcu;28?UnXt8D zxXh8R-_8nm60e0w)>~R$p0_u+W4)H~&uvL>GgXwOXigD9rruJ$BP=`+z++UAN6YFC>MZbJ%c@tk_9i9BYUU1_3o@4yo<-!RL2F>_oUu{|;; zBMrqBHXXlX+0^nZzrRW+ErOX^Oa8Sa?d8iX=g*(VR5=noncI-p}8B@38!+tP%c7hJTRlrW39V$_O~x!f6BZIR$=iy~6vuKxR2k*kAK^ z?b{?alxz^fEH{4B!Tj;>YusOlPJ^XYuzbTX%F@u#3E4j#t3eL4ld-WGSs&kgpJn$t zFF!9Ui&u5<2H~#g&z__oKOWWoeMN6Q*5G^P>eaXz4Rd9cgk9fZ>+O9RHa6kF8r%><^PuOU*t$liEof4?00fE8=o z8UO9&MY{(#mt=fPIj(QknG_~=n%2XEWV|#{mz-3dP=HCS$xdTgt&9bdwfN?z>pmhk zhO!-oE|4cAUb4M*NgO*fIfv{S+y>_l3wC`hOR;(-`cO}!Uqky18=SX^d5+ijBeLQ7 zz6Fx9a^8OOtd2%MZ6$716iG>1;O?tnnlWU0YFSq{dB|tM*y;RL-gacxT@|-CCQm+g zxa?ZY2{fthBAQ!8LT~lWTpe8Q*$^Qen(OrqBn3Q6np@DeGlr60q-~}HO3HU0t;5u~ zSQHvpImqVG+7C8}$r|5IdvN(O6j(j3H~Cntu1h&q??Ah2j?G`&W)hw>y}mf&cW`Kx zpnT9O*x03(gq^O*owGj3sT8nUZsJ4wa>z`DMpri1GA`J=~qMXvto{Jyd0M zWXK4zWQhItR#1h5eNC`D6ly&266%}Nmh%?BdltL|HXi0A>$AON6cHJj@I9Pb4U5vS zS(|9l{`6E%MOiuK?8O_(%|St?#w-V=U8dx;8hgYBvd z_x-(wM^KDl^s&aVuyfAc{U;6uQx^t#Omu~QBE!4)11^ahO8w{F?9bVPw1;N(sYFbm z$VPA$5O;ffyY|N?Ig)VRCndg9<~ka>wXNPtJq0QSg(?+W^CvG_l#CTSpt>6|6ud1H zY!9@5F}bisOiJD4GQY37U~r#(>%omU$Zy{~C%K=Kp0#g}m}0It+=T8g;LM{_-*zx( z!*-A&j^b}HA4v7pPD9U*SNaQe>h`u42I5o`INL$tu|C*d#*`RUN^0=NpdU*2Sf#Zn zL_lXlC8ADvZw54Ipc^nl?tJ!1Jl&eh%&qxfto#OwFG#$Q2`o4GoUF@b4)r?98dclN zML6mk*_L$$l;=F}DpF4Q;LI>g+B!u~FgV3f_SbJCBHn{U>}QIy?Mh`29R97&UR+W`6@7XI$4|DK0Vp$cLvNpdd$#6SZ1a1k;*)nj40xt}{C{QPWI zil3jq2YZp+3a{94EQr_`_v8>LQy|+@3b!wB$xh%^EkOn5_i>ji=ibLHppCTnIJH40 z;8-VCr^(Gp=`jsBd@vUH*pkgd4PNloo8=abqT=y&<~-CfK1#P_a;(|9ZEgKoN1k zw%3=z?n!uJ@7SZN)3=a<%EDUUHO&6<0?b@mAHsZPJQw2g^7B=i0toc&84sU-vl(ho z=Z*1MFzQD^jj#PKkS}lPR|i?pRrQRvObE|o$>wf(FoG%Cd>QMmo)4PoND|e1ST#Cc zW>znmIJ*qc%b{6;W(M%Qm_yyV`tvIRIUP_uCL&Xclx3J{W65$H#+w>_vjpul>wL$9 z4oVS=2A`k(RQH~Y#S+fG(T`SlKMc(O+Ut7^m2*!PR-6YfJ5w7&N#Xg`XY2Qa^0qq9 zMN_{^ft!6wUE`<8FS!!#&$Kc@@n60?yd~_Q8CDIgu zlBjZWeAG4mz@SV>PeRyd@z3X6F7t*za}(SJ*o${@9!uPp*X^v41c|%c%jK}%Ai`Yv zg|R!WOfsBs>cH3kpalx(GK-nKy@cQ9M<+bG-Q)N}MpOSst?yHA;-3ydV#)VepTyL< z&p8#)s0~H8-!Ytne|1-n4YyTHB{J%{Afi?Uy z!*xre#F^EMuNH4^aC35SaA;TBYBXITt?JjxXgwPzrz)u7Yz12KH)Y>k<9Md&k!>v= zNH$IHS}Moxeb`hLJ16Jw9~WvvuE_iJXUm-?Hi2+zVzH*%BX7FX6D%3p@U@D3 z`qt`wmVaMK+BIqa|LbjU{%c#yH&_0#_5aqA|Lo#_G^B12vQ{XT(Z?GRdTWfHLZb!n)|G3k+#l3Jy0S01R)+6_L>v#`cB&KX=L)#{*@ zu-bK}zu%bY=%5rGVIHmXjB9IaGaZ7a8!yuGD9g*gI9&Tq?RI?VTd0-$YI9}OWC2&6 zwT!ND8oIN-zCOG9=a1nnzNfNOOZ=x=fLqzDjMTcVO~`Q=K^1u(OMkSS>z75^lXx9_ zO1V1(nzaC+7Cm0eyU}kZ)_W?w9EG;&TjPUz-`1DJCb-i?6Lyc z!KWwz(ks+{?6b=d+Nc?Y7J@6(TG6+6AYqG z`!1%rG<|+U%bU4!Xzn?nHwcPQfa(K&{=PSnkz2_YckbLN0akFcxH`>aR%g8;ycr;# zHE!b4DBOWylnEMf7;C6)l_dVS3O8(d70h=@a&J10J#L}vNCz%uJjqN!yE$8fCN4&ibi!_DUe0xAbm$sOcrEcQ&cD>*jlFtL zQ-~0GlmFXgR}uT(uXO=?Ze>igZqejUI8*mXv8;h2y~l( z)%|>Py;3of&SvAU|7xS%XyDGMK1i?b%Z4$c$GcP1#jbz9&EEE&?`_$i2&FFZ9uLHU zlSxQHg-rha9xj-XDQ$|`x^V6OQ_5QcKT;)qD^%PkzJ^4Nw?MnQyQWg?(`D1s*6xge zq}T<;?eDB+EeC&6OX6XV1TJsF+qC;(3&e>DwbW)z0R`Sy|cjaDq|H?T5`67A}LF6-Hq0sv#VY`_5_ZjCyAY zdo3BB`bLverJ|b1h3KABVhWrVZ~<108mF^xxGTB#^>kDkdbHlVAg$SDyy^Ck_ihh> zFI{3yd$Va>q4($TYDheqpQsqkm_gV?_Z#A%zHCg!aW|m>z?zvq$E>Re>Ak(VERD!B z`7I`1r@kdTFi739diJaYRM9S*v9R-3I~(3ye^58wBdb&(z~T(J4T&pby*>bQ@esv5 z=AYsb_6+kR>dFfejD3AMz`n`Nt&#L|0_EV81Z+ER5_gt{Dg_hF)z#lYx~0YgQw<>2 zQ@tWohxeF$hW0G~e6mC}CJG{RwE%n`LL=f1w})M%3#6>uXySIvp`)$a!xp>an0&Pv z00&8lnna~dXN4X5l_w`B1L_8E%LJK9AMaWbYi6$_n-?^PcbR<0KK7Jm~TmzR$a*P!xa< zTtF`16E(+wcOr#ZLn&~0`D&MO9&TeaMYnnF1h&c&3`m>NZ@yaTQ({UmajyNj$@scA z7TbK!q{dlzmE=@%EdYjF|NZuI);OO!9lg^`PDwZY=lDD1tH@;Nw-BQ5ct^KLx1FnC zk6n=ZJvwiFzAsn62Ks$XM?z687Jb+4`zuP6(h>;Q4y$OX#0RaR6KLKut1r(}D2xCE z(&^v#;jvUV?%ob3NPZ;OG9}y|CT_AnkeKsrjbd;+Ioe6%%->gdH<9lvSNvvfv&s@6 znb`DnuGI#jCe<>>DHY9=O#3Yz2}AJa@LM<9kA9t`Kz5n$Y0Z6D()^=jj{`S^$L(pv z+@_VUUUSIo3#SkXe4VWatLi^TG2?IP=JUF;)tjha)zcKwvGkLt0IM)jixKONAeOXD zXo!wl7flE3%_M092jE)aJ{3ZI7%S$9?EU-!rJ3PS^v2II^vy&fu$WgwUuTM-hKYUS zdt54XZvTl~F$1(UV6f2x9qTOg7qS9(>i9e*>v|%MNPAQolD(%i{J=4@Z`0FM9vN5U ztJIq6zA8%g_kjSgPN*IHJsXfbT!pH4ew10c@FwB@HBCWY-os2kn;A*RjD3KcRkgKa zYvr)p3KY)!xdlkSO?FdaWI+K6YBHr4a+!}zslH6d|LjrOlHeh40r0%#Y;oU>yxIud z;yaDd?|r#SY2CK$n(4~2H$V8Y7vt!V5+9LIma8os06jtlTJEfj&a|9kikI@={vvrO zzsm`5f4f~OH2@k6S9ZR6a9*7(Yua~6s=mHL;&EeJ5rr}sg^ABS(C6x9HSi~DQLw1P zk4*dPc#mm6v^es(>P=4d0Q1dD3}U{>-AycfW@j7{-77IDH%Eoi93;fVJWcE?GTZlE z`tw=L?{=Gn*pa_>jk6iCtZ_vATxj+2(LvgdS-qEyZ`NN6FZ14Nqs*cyME$l^YgEct zZb60w#27ZsuhbS=x_t@%%*rZXJw>1bItg;uI3>Sjo*c~ggYbk#wRmRe@gI@Kp)3jS zvc3AigKbinE4dV*JEpM(@HFSYjh;12i(-QfIT@k$2+ZWqw=^nf)~@dxK@vrBu5-5<8UHj=+RYi~b|9)` zD6fq`S?R+!Z*c0~PxGgO`m`rNQBDnijOR$Hk<3+~L(JJ3Jlo+c%4oG$63Cn#4k>V_ z&ZBT)5ZGLzt=O{+0 zg^KQ}Xc^~?F0c(uJ}r=U2kE5)zId)rK5pFvkk+u&xJrPy(sbagX+u*_{{U@y%Doe_Lkn>4)u#plm0di;TKe*XQi}C&WmexBZ zd?GO=gaFSFx7m2t)SV@>D&PXpTvPxNIQK);y?gger2IDdu){SjE@!Wv`o@kS9xSfJXsV4KXYOA*b!zcARl?huk&(qMbS!LmuFj?-QNPA{ zw4}JK?svc-2!|6XGnzu;X1H+WuP6q`!CPW();zb#Ht=A`<*|~Ig!9_gP9D%Fq5_00 zzlC`$3L!8*7G1gk;BrXdGQxE7S~N941NmyS?pCOf{6MQ_5UrBp;?u7Hrhm8*Y%bwk zM_?}o5u>lG8=o>*=jpfz;1JJZ!~R0C`RU;1lcUwBLzFKH)IfGAi7)+0v}WvgZmzsD zNX^*ms|ryJ5{ZQ1CxpPaN1xh*2FlEajceU(dNVpHj#|eD^xuo3HC|S?_w9G#m5uB5 z2SubZ?N_74?*-;6LlQy{!VyM z{v`vr(^)erh}kRXZ@-zvc%N}4iU zDD?U@1(9C+^Rvg%9Y-4-ycNJSq<#V#IXvh{jo+8>n)pfncg2s$9DOkgQte*)5YHV8I_?PEaUy-fH z<{-f0N(rzDIM5_w!Zp*OhnCJ1TcG+{*nC+6tr3O}2j&+I#17u(HZ$*0w(`d9)Q(M( zr`y(Lwmx~7`Sg?XsCeZnz?;th%m+pwJ4bQYu1-+1GXv~fCvcBW${Zw+B>@nSitcb& z<459!h~9v;+5ph!Tb;GG^o!ml9z>EFNUc!i)^%>Py8ieZo~vVKfT4WFG2{VllMSTI z_`y;WJI6f`fb9TsPrE<|X=wYtJ4xF&V3Mprwvf4RjQ)6+o&9&7N}OaOK3zJ{c(yZT z%8)ALFM#kp?>#aDHdKt7gBovxWPT=@LbJ*dC&IQnWEKP#Gb`68wKFmW`B0pJ`M%`L zg-x8>TEr!RLh~CaU0gr`A0u=q|8PAdRe zCHXQ1tcAqi*ZQUo0SqRnuB1j&#rjUCKr@9C0Wul#l^z|hS>4^{qwTaqz@lPhpddtV zb_fX!PE|m4@2X2NFntTUJb+RLok%H^RL8TThNK!R?RrvU1vCs~42t!+bq=z#s#47W zj>aq>fE>mNB-ljI9V|4eL;)Ib?|Y!Q`|P_z;D2s=8#Y8UkotcvR^StfyX#YKi={QH zIr1+hdBBv6mDGtg?cl_@K}v?ahX#Sp+F7fOkJ9a_{0&^EE+A&OHSRs2{;5RwRt9fn zQ07P-(Dh0C+1u%1*%F zOF)T8X$FUz##0Katw|_3WyOaNA36oLNfdw3y+_KplpHBzmwKhWnI^-FcO6)O-8I`k=ik|WPU81))P4uY0jLm3e` zvLVPeo?KZmF%yO>!!FHLpf47)@+VaY@eDNW*hiay@M|>nE3y)}_JpCxojczaum#ao zKI_kjwV=<+11-+`_7Z^X0Z+j2Pz$n3rx zPv|-XUMERzMHe>j4^T`A(0+y-M(e7qBzDI9T~N{Enmtq=z(-v$j-cAHHi{teU(Qr+ zi-TU8(CT$gUA{$oS0d0R(rkqI&^QtkJ+D~jL3p-}qlyJu+++nWozSr8Se+)W%&gRJ zYoXQ&dQeIB7pQ6)YxzW>rvt=d!T^weFLxG^Ae+!~wrNjD&>#TajkgIi1MAj5s8%*j z5sU+pxd(r5Ti2d*-4?HtGE?eVWbhube8H9964aF{+$FHjoD1aX*&KNwHr__molEt~ z)^839Br+uNnBhyX4ZiNJ-(!HrZafF|WOV;Ui4e z);1DimuWOX7x}WQCHkuI*>J(W2UyC!sFL5l2>JZh$f8OZ@PHRuw>Tu+vZ z^a(NRL?|mRHn!vno2nrRPVFsmGxx`y&Ikf2$^Bl|wq_W?V`vIA~3@7B_d&HPwVuB3}KL+joJr_N^*qZ}|N^bS**rrv3x~U2f zk$S;He0LWuUc}*Wcm2xs>odtH1;=*`5`%}O2Qy^=yrGid$enrh;inF1l4d8M5xxOx zokY{cBa`|K+?X-A-LX^=wc-tcvh|PGmw5;Xi6$0>SJwN6O}mp9vCb=ipCyq5NRq;x zw1ylEk}8sp9vuG0Z6H8oAwEiu$W!LuM|{A)ZqFxN|651`1(OEBxR8NXq{ObAzzr(a?2 z*ir^HdpiNYtAKG>lrP#Bu#209fD8(MMPQQh>tFW4 z)}rnViF-}SHqM7!k zY+g~_%JxuiNuOAqdR`FHEI~Of{Ip6UvhV4ily3Kej7fVtML#MD&xL9JE_&xLEnHKC zLq|p~?L1`^@g2D<2{NDO3+!5Ka=OPm?=9#t|3pxP03^!0fTIavg>Uv}KPjk}xlb0{ zIk`Xv3qs|4@istfFKP=9K+%~2G|h8Ul2E#%kRKB4-xIl#?oY-5usuftmfmeS@`)Pz z)-Qul?d78#3N5naMV>-gys3%Ki+|hi;M#7hn6V51rJJ0y=^g?<%JujLjH{0qbKsxV;LY z)OyhUOZ6^gOfuU3bk$gx*tj#+(J$_`Z-9^P{1ool>+mIIrZ7+NNmOeF$_QW>r>qBE zmEv4)68|6~P*g0F&OmjX9mVq9AB$>{ww*ibZE&Vu%uxt?2w>Z)y6k);?{GWEt(kKs z1WbO{*R42q4S|7|RSRV+ivTeQK!M;&5!_yZ0!AjQCI}iK9DqGGBg&I5bDI%HYAZRO z_iaBjUYLxC8n;WjO%T9~XbtNREuTf=HJ7(=7D)|x%Qu!kv_uulm#a{yxde3_gYfS^mKLF%r&0MFV|bmJ z8>KIbT8jrJgk%S*qdEI)cYOz%+jnBmY=qM@(b zP^cFplB%kvEc~!f)7#?!FRgYi0Jo|sajkDctP;In3XyN@OF2kX*00ocI9W7%k9vV- znwShgOztJQq3PwuRjQ1En_BKXOGWi{tH;ue-t;0)F|%E1YT!DFI>!JgqU!Bv46z_T zC?%iaDxt|yNfOVZ1+ZM#-k@X&n}zCdzr#7@6}0NgsP{NGsTaN{y_pOo3dFY3=^`yC zS5Yi+V4<20?J%rZ8e4ksF|^6iP8|TU9#^KoAx+Ibhf=>|VdlY;VPieR)5fJ6%OiVH z$4rC761J?m04JxC{xUO()}3>*$+gd9^1$^sdap@U0lMTMpo`e@xwwQHJK~PJ&E^0F z=zk|SkC0t!kGto8JMpXU6CXJG01C(`Z%w%6*p?&oo&$=o z`@7M(V~i#3QjV09_;hGN$h2qjOYbq?)`I60o`1r~PLunLRIkJ_h6*zLYGGvmhW)zf z>nVBbb9hW|j`D>YT**A603+wqfBRvs)Y`r|8wZ#Ee1Vo{Y5j}GSHB$=b!h1?GZ3n+ zs|ig}E5o2IVK&&ezVESnVaLWZRVpm>;H0p`>0ZIBj3)pTYSd^jnEx96N8(yNhR(t& zUVz4In}YGT4|)n!VfO{my2?p7?fc%werW$4XC3^9=vVCE6Vub@D zgE4$X0bCN$;4R5)qCTSXl-i-PM$?t+39DQbx`rU;5%D z5!c~J>V=U-2Xh9vEx2p>J)X<)x<0qopyVL9Y$!O|xSU2+p2KE*Ys;rF|AjG|Zcihs zc+PzIl0!s6!2$*!q_tzFCdcG6K_<8qni%G=n!0Z`zm6b$sRWcF**!Zy%6OqBXe75f5g29;0(Gir)0gf0A+>I zeD!crX44W$Hwk`3@bm(^gQxEE;8`*{c}O;zsef_$EpVO|Lvy3zZV@~ e=iw*ky50VqYWv*N285RLFqOxekILkqzW8s4n{TrK diff --git a/docs/assets/model_metrics.json b/docs/assets/model_metrics.json index 9434212..c135119 100644 --- a/docs/assets/model_metrics.json +++ b/docs/assets/model_metrics.json @@ -33,9 +33,9 @@ "training_prevalence": 0.001003416857597292, "test_prevalence": 0.0011921912179000402, "calibration": { - "raw_brier_score": 0.0016988252755254507, - "calibrated_brier_score": 4.299887950764969e-05, - "raw_log_loss": 0.006217300426214933, - "calibrated_log_loss": 0.0002347317640669644 + "raw_brier_score": 0.0016964440001174808, + "calibrated_brier_score": 7.803709740983322e-05, + "raw_log_loss": 0.006314944475889206, + "calibrated_log_loss": 0.00040054231067188084 } } diff --git a/docs/assets/typology_results.csv b/docs/assets/typology_results.csv new file mode 100644 index 0000000..818de64 --- /dev/null +++ b/docs/assets/typology_results.csv @@ -0,0 +1,29 @@ +typology,transactions,positives,recall_at_0.5% +Behavioural_Change_1,62,62,0.9838709677419355 +Behavioural_Change_2,61,61,1.0 +Cash_Withdrawal,240,240,1.0 +Cycle,49,49,0.9795918367346939 +Deposit-Send,163,163,0.9754601226993865 +Fan_In,36,36,1.0 +Layered_Fan_In,88,88,1.0 +Layered_Fan_Out,100,100,0.98 +Normal_Cash_Deposits,33430,0,0.0 +Normal_Cash_Withdrawal,45245,0,0.0 +Normal_Fan_In,304332,0,0.0 +Normal_Fan_Out,344619,0,0.0 +Normal_Foward,6336,0,0.0 +Normal_Group,78557,0,0.0 +Normal_Mutual,18740,0,0.0 +Normal_Periodical,31681,0,0.0 +Normal_Plus_Mutual,34939,0,0.0 +Normal_Small_Fan_Out,518275,0,0.0 +Normal_single_large,3065,0,0.0 +Over-Invoicing,7,7,0.8571428571428571 +Scatter-Gather,46,46,0.9782608695652174 +Smurfing,151,151,1.0 +Stacked Bipartite,128,128,1.0 +Structuring,363,363,1.0 +Single_large,34,34,0.9705882352941176 +Bipartite,26,26,1.0 +Gather-Scatter,82,82,0.9878048780487805 +Fan_Out,58,58,1.0 diff --git a/docs/assets/unseen_entity_results.json b/docs/assets/unseen_entity_results.json index 28d2e21..98643d9 100644 --- a/docs/assets/unseen_entity_results.json +++ b/docs/assets/unseen_entity_results.json @@ -39,5 +39,30 @@ "recall_at_alert_rate": 0.992, "lift_at_alert_rate": 198.36478273217836 } + }, + "fixed_production_threshold": { + "decision_threshold": 0.0011600581929087639, + "both_parties_seen": { + "label": "Both parties seen", + "metrics": { + "threshold": 0.0011600581929087639, + "precision": 0.3958333333333333, + "recall": 0.992989165073295, + "lift": 180.84663798597833, + "alert_rate": 0.005490780343675977, + "alerts": 3936 + } + }, + "unseen_entity": { + "label": "Unseen entity", + "metrics": { + "threshold": 0.0011600581929087639, + "precision": 0.035137432700481724, + "recall": 0.992, + "lift": 197.91510342873337, + "alert_rate": 0.005012250115399638, + "alerts": 3529 + } + } } } diff --git a/docs/assets/walk_forward_results.csv b/docs/assets/walk_forward_results.csv new file mode 100644 index 0000000..35d2f21 --- /dev/null +++ b/docs/assets/walk_forward_results.csv @@ -0,0 +1,5 @@ +window,pr_auc,recall_at_0.5%,precision_at_0.5%,lift_at_0.5% +W1,0.9379450719376109,0.9705197827773467,0.18386243386243387,194.10153168240834 +W2,0.9658401319348904,0.9819156061620897,0.21606484893146646,196.36922824815977 +W3,0.977403742060776,0.9907735982966643,0.2053847285567162,198.140726093725 +W4,0.9901104009196656,0.99375,0.23489437139902497,198.7222531023785 diff --git a/docs/research/unseen_entity_generalization.md b/docs/research/unseen_entity_generalization.md index e571761..652a991 100644 --- a/docs/research/unseen_entity_generalization.md +++ b/docs/research/unseen_entity_generalization.md @@ -63,6 +63,27 @@ synthetic dataset. That is a plausible property of real laundering rings too "the model doesn't know what to do with new accounts" from "new accounts are inherently rarer positives here." +## Under the actual production threshold + +The table above gives each subgroup its own top-K -- the best possible +ranking within that population alone. That answers "how good is ranking +within this population," not "what happens to unseen accounts under the +policy actually deployed." Applying the artifact's single fixed +`decision_threshold` (0.001160, chosen on the validation split) to both +subgroups instead: + +| Population | Alert rate | Precision | Recall | Lift | +|---|---:|---:|---:|---:| +| Both parties seen | 0.549% | 39.58% | 99.30% | 180.8x | +| Unseen entity | 0.501% | 3.51% | 99.20% | 197.9x | + +One threshold, applied identically to both, lands each subgroup within noise +of its own 0.5% alert budget on its own -- the calibration transfers across +subgroups even though it was never tuned per-subgroup. Recall is +indistinguishable (99.3% vs 99.2%). Precision differs by 11x, entirely +consistent with the 12x prevalence gap between the two populations, not a +sign the threshold behaves differently for unseen accounts. + ## What this doesn't test Network features (out-degree, counterparty HHI) are computed per-transaction diff --git a/reports/model_metrics.json b/reports/model_metrics.json index 9434212..c135119 100644 --- a/reports/model_metrics.json +++ b/reports/model_metrics.json @@ -33,9 +33,9 @@ "training_prevalence": 0.001003416857597292, "test_prevalence": 0.0011921912179000402, "calibration": { - "raw_brier_score": 0.0016988252755254507, - "calibrated_brier_score": 4.299887950764969e-05, - "raw_log_loss": 0.006217300426214933, - "calibrated_log_loss": 0.0002347317640669644 + "raw_brier_score": 0.0016964440001174808, + "calibrated_brier_score": 7.803709740983322e-05, + "raw_log_loss": 0.006314944475889206, + "calibrated_log_loss": 0.00040054231067188084 } } diff --git a/reports/unseen_entity_results.json b/reports/unseen_entity_results.json new file mode 100644 index 0000000..98643d9 --- /dev/null +++ b/reports/unseen_entity_results.json @@ -0,0 +1,68 @@ +{ + "alert_rate": 0.005, + "accounts_seen_in_training": 693879, + "standard_out_of_time": { + "label": "Standard out-of-time (full test set)", + "transactions": 1420913, + "positives": 1694, + "prevalence": 0.0011921912179000402, + "metrics": { + "pr_auc": 0.9859286355765932, + "roc_auc": 0.9998338278230887, + "precision_at_alert_rate": 0.23673469387755103, + "recall_at_alert_rate": 0.9929161747343566, + "lift_at_alert_rate": 198.5710767896297 + } + }, + "both_parties_seen": { + "label": "Both sender and receiver seen during training", + "transactions": 716838, + "positives": 1569, + "prevalence": 0.002188779054681811, + "metrics": { + "pr_auc": 0.9917797464630843, + "roc_auc": 0.9998019722862765, + "precision_at_alert_rate": 0.4345885634588563, + "recall_at_alert_rate": 0.992989165073295, + "lift_at_alert_rate": 198.55296153774358 + } + }, + "unseen_entity": { + "label": "At least one party unseen during training", + "transactions": 704075, + "positives": 125, + "prevalence": 0.00017753790434257713, + "metrics": { + "pr_auc": 0.7912056121187283, + "roc_auc": 0.9997998835144541, + "precision_at_alert_rate": 0.03521726782164158, + "recall_at_alert_rate": 0.992, + "lift_at_alert_rate": 198.36478273217836 + } + }, + "fixed_production_threshold": { + "decision_threshold": 0.0011600581929087639, + "both_parties_seen": { + "label": "Both parties seen", + "metrics": { + "threshold": 0.0011600581929087639, + "precision": 0.3958333333333333, + "recall": 0.992989165073295, + "lift": 180.84663798597833, + "alert_rate": 0.005490780343675977, + "alerts": 3936 + } + }, + "unseen_entity": { + "label": "Unseen entity", + "metrics": { + "threshold": 0.0011600581929087639, + "precision": 0.035137432700481724, + "recall": 0.992, + "lift": 197.91510342873337, + "alert_rate": 0.005012250115399638, + "alerts": 3529 + } + } + } +}