diff --git a/.gitignore b/.gitignore index 721f485..2ef5452 100644 --- a/.gitignore +++ b/.gitignore @@ -44,4 +44,5 @@ logs/ htmlcov/ # Large data -*.csv \ No newline at end of file +*.csv +!docs/assets/*.csv \ No newline at end of file diff --git a/README.md b/README.md index 22e2e14..1a4d6c4 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,24 @@ The model detects specific money-laundering patterns with high recall: - **98%+ detection**: Behavioural Change, Cycle, Deposit-Send, Scatter-Gather, Gather-Scatter, Stacked Bipartite, Single Large - **Limitations**: Over-Invoicing (86% recall) detected less reliably due to low transaction prevalence +### Temporal Stability + +Walk-forward backtesting splits the full timeline into 14 chronological blocks +and evaluates 4 expanding out-of-time windows (train grows, calibration/test +roll forward each time): + +| Window | PR-AUC | Precision @ 0.5% | Recall @ 0.5% | Lift @ 0.5% | +|---|---:|---:|---:|---:| +| W1 | 0.9379 | 18.39% | 97.05% | 194.1x | +| W2 | 0.9658 | 21.61% | 98.19% | 196.4x | +| W3 | 0.9774 | 20.54% | 99.08% | 198.1x | +| W4 | 0.9901 | 23.49% | 99.38% | 198.7x | + +Performance is stable-to-improving across expanding out-of-time windows, not +just in the single held-out test split — later windows benefit from more +accumulated transaction history for the network features (degree, lifetime +counts, counterparty concentration). + ### Analysis Visualizations ![ROC Curve](docs/assets/roc_curve.png) diff --git a/docs/assets/ablation_results.csv b/docs/assets/ablation_results.csv new file mode 100644 index 0000000..98156a9 --- /dev/null +++ b/docs/assets/ablation_results.csv @@ -0,0 +1,5 @@ +model,pr_auc,recall_at_0.5% +Base,0.08984612760924429,0.12868949232585597 ++ Behavioral,0.20015470040725364,0.45218417945690675 ++ Network,0.943473551428709,0.9769775678866588 +All,0.9859286355765932,0.9929161747343566 diff --git a/docs/assets/calibration_curve.png b/docs/assets/calibration_curve.png index 0aa2b25..6823503 100644 Binary files a/docs/assets/calibration_curve.png and b/docs/assets/calibration_curve.png differ diff --git a/docs/assets/model_metrics.json b/docs/assets/model_metrics.json index 9434212..c135119 100644 --- a/docs/assets/model_metrics.json +++ b/docs/assets/model_metrics.json @@ -33,9 +33,9 @@ "training_prevalence": 0.001003416857597292, "test_prevalence": 0.0011921912179000402, "calibration": { - "raw_brier_score": 0.0016988252755254507, - "calibrated_brier_score": 4.299887950764969e-05, - "raw_log_loss": 0.006217300426214933, - "calibrated_log_loss": 0.0002347317640669644 + "raw_brier_score": 0.0016964440001174808, + "calibrated_brier_score": 7.803709740983322e-05, + "raw_log_loss": 0.006314944475889206, + "calibrated_log_loss": 0.00040054231067188084 } } diff --git a/docs/assets/typology_results.csv b/docs/assets/typology_results.csv new file mode 100644 index 0000000..818de64 --- /dev/null +++ b/docs/assets/typology_results.csv @@ -0,0 +1,29 @@ +typology,transactions,positives,recall_at_0.5% +Behavioural_Change_1,62,62,0.9838709677419355 +Behavioural_Change_2,61,61,1.0 +Cash_Withdrawal,240,240,1.0 +Cycle,49,49,0.9795918367346939 +Deposit-Send,163,163,0.9754601226993865 +Fan_In,36,36,1.0 +Layered_Fan_In,88,88,1.0 +Layered_Fan_Out,100,100,0.98 +Normal_Cash_Deposits,33430,0,0.0 +Normal_Cash_Withdrawal,45245,0,0.0 +Normal_Fan_In,304332,0,0.0 +Normal_Fan_Out,344619,0,0.0 +Normal_Foward,6336,0,0.0 +Normal_Group,78557,0,0.0 +Normal_Mutual,18740,0,0.0 +Normal_Periodical,31681,0,0.0 +Normal_Plus_Mutual,34939,0,0.0 +Normal_Small_Fan_Out,518275,0,0.0 +Normal_single_large,3065,0,0.0 +Over-Invoicing,7,7,0.8571428571428571 +Scatter-Gather,46,46,0.9782608695652174 +Smurfing,151,151,1.0 +Stacked Bipartite,128,128,1.0 +Structuring,363,363,1.0 +Single_large,34,34,0.9705882352941176 +Bipartite,26,26,1.0 +Gather-Scatter,82,82,0.9878048780487805 +Fan_Out,58,58,1.0 diff --git a/docs/assets/unseen_entity_results.json b/docs/assets/unseen_entity_results.json index 28d2e21..98643d9 100644 --- a/docs/assets/unseen_entity_results.json +++ b/docs/assets/unseen_entity_results.json @@ -39,5 +39,30 @@ "recall_at_alert_rate": 0.992, "lift_at_alert_rate": 198.36478273217836 } + }, + "fixed_production_threshold": { + "decision_threshold": 0.0011600581929087639, + "both_parties_seen": { + "label": "Both parties seen", + "metrics": { + "threshold": 0.0011600581929087639, + "precision": 0.3958333333333333, + "recall": 0.992989165073295, + "lift": 180.84663798597833, + "alert_rate": 0.005490780343675977, + "alerts": 3936 + } + }, + "unseen_entity": { + "label": "Unseen entity", + "metrics": { + "threshold": 0.0011600581929087639, + "precision": 0.035137432700481724, + "recall": 0.992, + "lift": 197.91510342873337, + "alert_rate": 0.005012250115399638, + "alerts": 3529 + } + } } } diff --git a/docs/assets/walk_forward_results.csv b/docs/assets/walk_forward_results.csv new file mode 100644 index 0000000..35d2f21 --- /dev/null +++ b/docs/assets/walk_forward_results.csv @@ -0,0 +1,5 @@ +window,pr_auc,recall_at_0.5%,precision_at_0.5%,lift_at_0.5% +W1,0.9379450719376109,0.9705197827773467,0.18386243386243387,194.10153168240834 +W2,0.9658401319348904,0.9819156061620897,0.21606484893146646,196.36922824815977 +W3,0.977403742060776,0.9907735982966643,0.2053847285567162,198.140726093725 +W4,0.9901104009196656,0.99375,0.23489437139902497,198.7222531023785 diff --git a/docs/research/unseen_entity_generalization.md b/docs/research/unseen_entity_generalization.md index e571761..652a991 100644 --- a/docs/research/unseen_entity_generalization.md +++ b/docs/research/unseen_entity_generalization.md @@ -63,6 +63,27 @@ synthetic dataset. That is a plausible property of real laundering rings too "the model doesn't know what to do with new accounts" from "new accounts are inherently rarer positives here." +## Under the actual production threshold + +The table above gives each subgroup its own top-K -- the best possible +ranking within that population alone. That answers "how good is ranking +within this population," not "what happens to unseen accounts under the +policy actually deployed." Applying the artifact's single fixed +`decision_threshold` (0.001160, chosen on the validation split) to both +subgroups instead: + +| Population | Alert rate | Precision | Recall | Lift | +|---|---:|---:|---:|---:| +| Both parties seen | 0.549% | 39.58% | 99.30% | 180.8x | +| Unseen entity | 0.501% | 3.51% | 99.20% | 197.9x | + +One threshold, applied identically to both, lands each subgroup within noise +of its own 0.5% alert budget on its own -- the calibration transfers across +subgroups even though it was never tuned per-subgroup. Recall is +indistinguishable (99.3% vs 99.2%). Precision differs by 11x, entirely +consistent with the 12x prevalence gap between the two populations, not a +sign the threshold behaves differently for unseen accounts. + ## What this doesn't test Network features (out-degree, counterparty HHI) are computed per-transaction diff --git a/reports/model_metrics.json b/reports/model_metrics.json index 9434212..c135119 100644 --- a/reports/model_metrics.json +++ b/reports/model_metrics.json @@ -33,9 +33,9 @@ "training_prevalence": 0.001003416857597292, "test_prevalence": 0.0011921912179000402, "calibration": { - "raw_brier_score": 0.0016988252755254507, - "calibrated_brier_score": 4.299887950764969e-05, - "raw_log_loss": 0.006217300426214933, - "calibrated_log_loss": 0.0002347317640669644 + "raw_brier_score": 0.0016964440001174808, + "calibrated_brier_score": 7.803709740983322e-05, + "raw_log_loss": 0.006314944475889206, + "calibrated_log_loss": 0.00040054231067188084 } } diff --git a/reports/unseen_entity_results.json b/reports/unseen_entity_results.json new file mode 100644 index 0000000..98643d9 --- /dev/null +++ b/reports/unseen_entity_results.json @@ -0,0 +1,68 @@ +{ + "alert_rate": 0.005, + "accounts_seen_in_training": 693879, + "standard_out_of_time": { + "label": "Standard out-of-time (full test set)", + "transactions": 1420913, + "positives": 1694, + "prevalence": 0.0011921912179000402, + "metrics": { + "pr_auc": 0.9859286355765932, + "roc_auc": 0.9998338278230887, + "precision_at_alert_rate": 0.23673469387755103, + "recall_at_alert_rate": 0.9929161747343566, + "lift_at_alert_rate": 198.5710767896297 + } + }, + "both_parties_seen": { + "label": "Both sender and receiver seen during training", + "transactions": 716838, + "positives": 1569, + "prevalence": 0.002188779054681811, + "metrics": { + "pr_auc": 0.9917797464630843, + "roc_auc": 0.9998019722862765, + "precision_at_alert_rate": 0.4345885634588563, + "recall_at_alert_rate": 0.992989165073295, + "lift_at_alert_rate": 198.55296153774358 + } + }, + "unseen_entity": { + "label": "At least one party unseen during training", + "transactions": 704075, + "positives": 125, + "prevalence": 0.00017753790434257713, + "metrics": { + "pr_auc": 0.7912056121187283, + "roc_auc": 0.9997998835144541, + "precision_at_alert_rate": 0.03521726782164158, + "recall_at_alert_rate": 0.992, + "lift_at_alert_rate": 198.36478273217836 + } + }, + "fixed_production_threshold": { + "decision_threshold": 0.0011600581929087639, + "both_parties_seen": { + "label": "Both parties seen", + "metrics": { + "threshold": 0.0011600581929087639, + "precision": 0.3958333333333333, + "recall": 0.992989165073295, + "lift": 180.84663798597833, + "alert_rate": 0.005490780343675977, + "alerts": 3936 + } + }, + "unseen_entity": { + "label": "Unseen entity", + "metrics": { + "threshold": 0.0011600581929087639, + "precision": 0.035137432700481724, + "recall": 0.992, + "lift": 197.91510342873337, + "alert_rate": 0.005012250115399638, + "alerts": 3529 + } + } + } +}