diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index da50e2cd..12de86ff 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -9,6 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Hardening +- **Same-window interleaved A/B harness (`--pk-ab`) (P3c)** - new harness mode runs two default- + config variants as alternating rep pairs (A1,B1,A2,B2,...) and reports the per-rep paired median + ratio B/A per phase, so machine drift affects both arms of a pair and cancels out. Smoke (1 rep, + pure default vs `plain`/NoEncryptMode=true): UPDATE 1.41x, DELETE 1.28x, INSERT 1.21x, READ 1.10x + in the same window. Use `SHARPCOREDB_PK_AB_ARM_A` / `SHARPCOREDB_PK_AB_ARM_B` / + `SHARPCOREDB_BENCH_REPS`; documented in `docs/benchmarks/default-config-pk.md`. + - **Default-config benchmark follow-up: FullSync hypothesis falsified (P3b)** - single-knob isolation via `SHARPCOREDB_PK_DEFAULT_VARIANT` (`async`, `bufferedio`, `novalidate`, `noadaptive`, `hsinsert`, `plain`, `tuned`) disproved the earlier “FullSync dominates the diff --git a/docs/benchmarks/default-config-pk.md b/docs/benchmarks/default-config-pk.md index b1739def..2a61d5c2 100644 --- a/docs/benchmarks/default-config-pk.md +++ b/docs/benchmarks/default-config-pk.md @@ -39,13 +39,26 @@ Observed (2026-09-04 afternoon): Machine drift is significant: SQLite's own UPDATE varied 275K-315K across these runs. Single-knob deltas below ~1.3x are not reliably attributable outside a same-window interleaved A/B. +## Same-window interleaved A/B (--pk-ab) + +Run: `dotnet run --project tests/benchmarks/SharpCoreDB.Benchmarks.Comparative -- -c Release -- --pk-ab` +Arm names: `SHARPCOREDB_PK_AB_ARM_A` / `SHARPCOREDB_PK_AB_ARM_B` (defaults: `""` pure default vs +`plain`); reps via `SHARPCOREDB_BENCH_REPS` (default 3). Each rep runs A then B back-to-back and +the per-rep **paired ratio** B/A per phase is reported (median), so slow-machine windows affect +both arms of a pair and cancel out — this is the reliable way to attribute default-vs-tuned deltas. + +Preliminary smoke result (1 rep, 2026-09-04, default vs `plain`): UPDATE **1.41x**, DELETE **1.28x**, +INSERT 1.21x, READ 1.10x — direction consistent with the earlier medians, now measured inside a +single window. Re-run with the default 3 reps before quoting final numbers. + ## Honest conclusion 1. The earlier claim that the default `WalDurabilityMode.FullSync` dominates the gap is **wrong** (disproven by the `async` variant). Do **not** implement a “FullSync commit-flush optimization” based on it. -2. The default path is correct and engages the fast paths; part of the remaining gap correlates - with `NoEncryptMode` (record/at-rest toggles and file-format decisions), part is machine drift. -3. Next step: add a **same-window interleaved A/B** mode to this harness (arms round-robin within - one process) so default-vs-tuned deltas are attributable, then re-open the optimization only on - a measured knob. +2. The default path is correct and engages the fast paths; the `--pk-ab` smoke (same window) shows + `plain` (NoEncryptMode=true) ahead by UPDATE 1.41x / DELETE 1.28x, so the remaining gap + correlates with `NoEncryptMode`; the exact mechanism (record/at-rest toggles and file-format + decisions) still needs a code-level explanation before any change. +3. The same-window A/B tooling now exists; re-run with 3 reps to quantify the paired ratio, then + investigate the `NoEncryptMode` mechanism in code and only then consider a change. diff --git a/tests/benchmarks/SharpCoreDB.Benchmarks.Comparative/Program.cs b/tests/benchmarks/SharpCoreDB.Benchmarks.Comparative/Program.cs index aa3d5f8a..4fd752b0 100644 --- a/tests/benchmarks/SharpCoreDB.Benchmarks.Comparative/Program.cs +++ b/tests/benchmarks/SharpCoreDB.Benchmarks.Comparative/Program.cs @@ -79,6 +79,21 @@ static async Task Main(string[] args) return; } + // Optional: --pk-ab → same-window interleaved A/B: runs arm A and arm B as alternating + // rep pairs (A1,B1,A2,B2,...) and reports the PER-REP median ratio B/A per phase, so + // machine drift affects both arms of each pair equally. Arms are config variants named by + // SHARPCOREDB_PK_AB_ARM_A / SHARPCOREDB_PK_AB_ARM_B (defaults: pure default vs 'plain'). + if (args.Any(a => a.Equals("--pk-ab", StringComparison.OrdinalIgnoreCase))) + { + var engineArgAb = args.FirstOrDefault(a => a.StartsWith("--engine=", StringComparison.OrdinalIgnoreCase)); + var engineTypeAb = engineArgAb is not null + && engineArgAb.Substring("--engine=".Length).Equals("pagebased", StringComparison.OrdinalIgnoreCase) + ? SharpCoreDB.Interfaces.StorageEngineType.PageBased + : SharpCoreDB.Interfaces.StorageEngineType.AppendOnly; + RunPkAbComparison(engineTypeAb); + return; + } + // Optional: --engine=appendonly (default) | --engine=pagebased // PageBased is the v2.0 in-place-update engine (WP10-WP13 storage engine roadmap). var engineArg = args.FirstOrDefault(a => a.StartsWith("--engine=", StringComparison.OrdinalIgnoreCase)); @@ -395,6 +410,34 @@ static DatabaseConfig BuildConfig(SharpCoreDB.Interfaces.StorageEngineType engin }; } + /// + /// Builds the variant DatabaseConfig used by the --pk-default arm. The variant name comes from + /// SHARPCOREDB_PK_DEFAULT_VARIANT or the --pk-ab arm selector; "" is the pure default config. + /// + private static DatabaseConfig BuildPkDefaultVariantConfig( + SharpCoreDB.Interfaces.StorageEngineType engineType, + string? variant) + { + return variant switch + { + "async" => new DatabaseConfig { StorageEngineType = engineType, WalDurabilityMode = SharpCoreDB.Services.DurabilityMode.Async }, + "bufferedio" => new DatabaseConfig { StorageEngineType = engineType, UseBufferedIO = true }, + "novalidate" => new DatabaseConfig + { + StorageEngineType = engineType, + SqlValidationMode = SharpCoreDB.Services.SqlQueryValidator.ValidationMode.Disabled, + StrictParameterValidation = false, + }, + "noadaptive" => new DatabaseConfig { StorageEngineType = engineType, EnableAdaptiveWalBatching = false }, + "hsinsert" => new DatabaseConfig { StorageEngineType = engineType, HighSpeedInsertMode = true }, + // "plain" == the tuned harness config with NoEncryptMode=true (BuildConfig default); + // "tuned" == the same knob set but NoEncryptMode=false (isolates that flag). + "plain" => BuildConfig(engineType, fixedWidth: true), + "tuned" => BuildConfig(engineType, fixedWidth: true, noEncrypt: false), + _ => new DatabaseConfig { StorageEngineType = engineType }, + }; + } + static BenchmarkResult RunSharpCoreDB(SharpCoreDB.Interfaces.StorageEngineType engineType) { var dbPath = Path.Combine(Path.GetTempPath(), $"bench-sharpcoredb-{Guid.NewGuid()}"); @@ -862,7 +905,8 @@ data TEXT static BenchmarkResult RunSharpCoreDBPk( SharpCoreDB.Interfaces.StorageEngineType engineType, bool fixedWidth = false, - bool useDefaultConfig = false) + bool useDefaultConfig = false, + string? defaultVariant = null) { var dbPath = Path.Combine(Path.GetTempPath(), $"bench-sharpcoredb-pk-{Guid.NewGuid()}"); var result = new BenchmarkResult(); @@ -877,25 +921,9 @@ static BenchmarkResult RunSharpCoreDBPk( DatabaseConfig config; if (useDefaultConfig) { - var variant = Environment.GetEnvironmentVariable("SHARPCOREDB_PK_DEFAULT_VARIANT")?.ToLowerInvariant(); - config = variant switch - { - "async" => new DatabaseConfig { StorageEngineType = engineType, WalDurabilityMode = SharpCoreDB.Services.DurabilityMode.Async }, - "bufferedio" => new DatabaseConfig { StorageEngineType = engineType, UseBufferedIO = true }, - "novalidate" => new DatabaseConfig - { - StorageEngineType = engineType, - SqlValidationMode = SharpCoreDB.Services.SqlQueryValidator.ValidationMode.Disabled, - StrictParameterValidation = false, - }, - "noadaptive" => new DatabaseConfig { StorageEngineType = engineType, EnableAdaptiveWalBatching = false }, - "hsinsert" => new DatabaseConfig { StorageEngineType = engineType, HighSpeedInsertMode = true }, - // "plain" == the tuned harness config with NoEncryptMode=true (BuildConfig default); - // "tuned" == the same knob set but NoEncryptMode=false (isolates that flag). - "plain" => BuildConfig(engineType, fixedWidth: true), - "tuned" => BuildConfig(engineType, fixedWidth: true, noEncrypt: false), - _ => new DatabaseConfig { StorageEngineType = engineType }, - }; + var variant = defaultVariant + ?? Environment.GetEnvironmentVariable("SHARPCOREDB_PK_DEFAULT_VARIANT")?.ToLowerInvariant(); + config = BuildPkDefaultVariantConfig(engineType, variant); } else { @@ -1127,6 +1155,85 @@ static void RunPkDefaultComparison(SharpCoreDB.Interfaces.StorageEngineType engi Console.WriteLine($"\nResults saved to: {path}"); } + /// + /// P3c: same-window interleaved A/B comparison of two default-config variants. Each rep runs + /// arm A then arm B back-to-back, and the result reports the per-rep paired ratio B/A per phase + /// (median), so slow-machine windows affect both arms of a pair and cancel out. + /// + static void RunPkAbComparison(SharpCoreDB.Interfaces.StorageEngineType engineType) + { + var armA = Environment.GetEnvironmentVariable("SHARPCOREDB_PK_AB_ARM_A")?.ToLowerInvariant() ?? string.Empty; + var armB = Environment.GetEnvironmentVariable("SHARPCOREDB_PK_AB_ARM_B")?.ToLowerInvariant() ?? "plain"; + int reps = 3; + if (int.TryParse(Environment.GetEnvironmentVariable("SHARPCOREDB_BENCH_REPS"), out int envReps) && envReps > 0) + { + reps = envReps; + } + + Console.WriteLine("╔══════════════════════════════════════════════════════════╗"); + Console.WriteLine("║ Same-window interleaved A/B (default-config variants) ║"); + Console.WriteLine("╚══════════════════════════════════════════════════════════╝"); + Console.WriteLine(); + Console.WriteLine($"Arm A variant: '{armA}' Arm B variant: '{armB}' reps: {reps}"); + Console.WriteLine("(each rep: A then B back-to-back; per-rep ratios cancel drift)"); + Console.WriteLine(); + + var listA = new List(reps); + var listB = new List(reps); + for (int r = 0; r < reps; r++) + { + Console.WriteLine($"── rep {r + 1}/{reps} · A='{armA}' ──"); + listA.Add(RunSharpCoreDBPk(engineType, useDefaultConfig: true, defaultVariant: armA)); + Console.WriteLine($"── rep {r + 1}/{reps} · B='{armB}' ──"); + listB.Add(RunSharpCoreDBPk(engineType, useDefaultConfig: true, defaultVariant: armB)); + } + + static double Med(IEnumerable xs) + { + var sorted = xs.Where(x => x > 0).OrderBy(x => x).ToArray(); + return sorted.Length == 0 ? 0 : sorted[sorted.Length / 2]; + } + + static int Ops(IEnumerable runs, Func select) + { + var arr = runs.Select(select).Where(x => x > 0).OrderBy(x => x).ToArray(); + return arr.Length == 0 ? 0 : arr[arr.Length / 2]; + } + + static double Ratio(IEnumerable a, IEnumerable b, Func sel) + { + var ratios = a.Zip(b, (x, y) => sel(x) > 0 ? sel(y) / (double)sel(x) : 0.0); + return Med(ratios); + } + + var aUpdate = Ops(listA, static r => r.UpdateOpsPerSec); + var bUpdate = Ops(listB, static r => r.UpdateOpsPerSec); + var aDelete = Ops(listA, static r => r.DeleteOpsPerSec); + var bDelete = Ops(listB, static r => r.DeleteOpsPerSec); + + Console.WriteLine("║ phase │ A median ops/s │ B median ops/s │ median B/A (per rep) ║"); + Console.WriteLine($"║ UPDATE │ {aUpdate,13:N0} │ {bUpdate,14:N0} │ {Ratio(listA, listB, static r => r.UpdateOpsPerSec):F2}x"); + Console.WriteLine($"║ DELETE │ {aDelete,13:N0} │ {bDelete,14:N0} │ {Ratio(listA, listB, static r => r.DeleteOpsPerSec):F2}x"); + Console.WriteLine($"║ INSERT │ {Ops(listA, static r => r.InsertOpsPerSec),13:N0} │ {Ops(listB, static r => r.InsertOpsPerSec),14:N0} │ {Ratio(listA, listB, static r => r.InsertOpsPerSec):F2}x"); + Console.WriteLine($"║ READ │ {Ops(listA, static r => r.ReadOpsPerSec),13:N0} │ {Ops(listB, static r => r.ReadOpsPerSec),14:N0} │ {Ratio(listA, listB, static r => r.ReadOpsPerSec):F2}x"); + + var summary = new Dictionary + { + ["armA"] = armA, + ["armB"] = armB, + ["medianUpdateRatio_B_over_A"] = Ratio(listA, listB, static r => r.UpdateOpsPerSec), + ["medianDeleteRatio_B_over_A"] = Ratio(listA, listB, static r => r.DeleteOpsPerSec), + ["medianInsertRatio_B_over_A"] = Ratio(listA, listB, static r => r.InsertOpsPerSec), + ["medianReadRatio_B_over_A"] = Ratio(listA, listB, static r => r.ReadOpsPerSec), + }; + + var dir = "results"; + Directory.CreateDirectory(dir); + var path = Path.Combine(dir, $"pk_ab_{armA}_{armB}_{DateTime.UtcNow:yyyyMMdd_HHmmss}.json"); + File.WriteAllText(path, JsonSerializer.Serialize(summary, new JsonSerializerOptions { WriteIndented = true })); + Console.WriteLine($"\nResults saved to: {path}"); + } + // ══════════════════════════════════════ // LiteDB // ══════════════════════════════════════