diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index da50e2cd..12de86ff 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -9,6 +9,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Hardening
+- **Same-window interleaved A/B harness (`--pk-ab`) (P3c)** - new harness mode runs two default-
+ config variants as alternating rep pairs (A1,B1,A2,B2,...) and reports the per-rep paired median
+ ratio B/A per phase, so machine drift affects both arms of a pair and cancels out. Smoke (1 rep,
+ pure default vs `plain`/NoEncryptMode=true): UPDATE 1.41x, DELETE 1.28x, INSERT 1.21x, READ 1.10x
+ in the same window. Use `SHARPCOREDB_PK_AB_ARM_A` / `SHARPCOREDB_PK_AB_ARM_B` /
+ `SHARPCOREDB_BENCH_REPS`; documented in `docs/benchmarks/default-config-pk.md`.
+
- **Default-config benchmark follow-up: FullSync hypothesis falsified (P3b)** - single-knob
isolation via `SHARPCOREDB_PK_DEFAULT_VARIANT` (`async`, `bufferedio`, `novalidate`,
`noadaptive`, `hsinsert`, `plain`, `tuned`) disproved the earlier “FullSync dominates the
diff --git a/docs/benchmarks/default-config-pk.md b/docs/benchmarks/default-config-pk.md
index b1739def..2a61d5c2 100644
--- a/docs/benchmarks/default-config-pk.md
+++ b/docs/benchmarks/default-config-pk.md
@@ -39,13 +39,26 @@ Observed (2026-09-04 afternoon):
Machine drift is significant: SQLite's own UPDATE varied 275K-315K across these runs. Single-knob
deltas below ~1.3x are not reliably attributable outside a same-window interleaved A/B.
+## Same-window interleaved A/B (--pk-ab)
+
+Run: `dotnet run --project tests/benchmarks/SharpCoreDB.Benchmarks.Comparative -- -c Release -- --pk-ab`
+Arm names: `SHARPCOREDB_PK_AB_ARM_A` / `SHARPCOREDB_PK_AB_ARM_B` (defaults: `""` pure default vs
+`plain`); reps via `SHARPCOREDB_BENCH_REPS` (default 3). Each rep runs A then B back-to-back and
+the per-rep **paired ratio** B/A per phase is reported (median), so slow-machine windows affect
+both arms of a pair and cancel out — this is the reliable way to attribute default-vs-tuned deltas.
+
+Preliminary smoke result (1 rep, 2026-09-04, default vs `plain`): UPDATE **1.41x**, DELETE **1.28x**,
+INSERT 1.21x, READ 1.10x — direction consistent with the earlier medians, now measured inside a
+single window. Re-run with the default 3 reps before quoting final numbers.
+
## Honest conclusion
1. The earlier claim that the default `WalDurabilityMode.FullSync` dominates the gap is **wrong**
(disproven by the `async` variant). Do **not** implement a “FullSync commit-flush optimization”
based on it.
-2. The default path is correct and engages the fast paths; part of the remaining gap correlates
- with `NoEncryptMode` (record/at-rest toggles and file-format decisions), part is machine drift.
-3. Next step: add a **same-window interleaved A/B** mode to this harness (arms round-robin within
- one process) so default-vs-tuned deltas are attributable, then re-open the optimization only on
- a measured knob.
+2. The default path is correct and engages the fast paths; the `--pk-ab` smoke (same window) shows
+ `plain` (NoEncryptMode=true) ahead by UPDATE 1.41x / DELETE 1.28x, so the remaining gap
+ correlates with `NoEncryptMode`; the exact mechanism (record/at-rest toggles and file-format
+ decisions) still needs a code-level explanation before any change.
+3. The same-window A/B tooling now exists; re-run with 3 reps to quantify the paired ratio, then
+ investigate the `NoEncryptMode` mechanism in code and only then consider a change.
diff --git a/tests/benchmarks/SharpCoreDB.Benchmarks.Comparative/Program.cs b/tests/benchmarks/SharpCoreDB.Benchmarks.Comparative/Program.cs
index aa3d5f8a..4fd752b0 100644
--- a/tests/benchmarks/SharpCoreDB.Benchmarks.Comparative/Program.cs
+++ b/tests/benchmarks/SharpCoreDB.Benchmarks.Comparative/Program.cs
@@ -79,6 +79,21 @@ static async Task Main(string[] args)
return;
}
+ // Optional: --pk-ab → same-window interleaved A/B: runs arm A and arm B as alternating
+ // rep pairs (A1,B1,A2,B2,...) and reports the PER-REP median ratio B/A per phase, so
+ // machine drift affects both arms of each pair equally. Arms are config variants named by
+ // SHARPCOREDB_PK_AB_ARM_A / SHARPCOREDB_PK_AB_ARM_B (defaults: pure default vs 'plain').
+ if (args.Any(a => a.Equals("--pk-ab", StringComparison.OrdinalIgnoreCase)))
+ {
+ var engineArgAb = args.FirstOrDefault(a => a.StartsWith("--engine=", StringComparison.OrdinalIgnoreCase));
+ var engineTypeAb = engineArgAb is not null
+ && engineArgAb.Substring("--engine=".Length).Equals("pagebased", StringComparison.OrdinalIgnoreCase)
+ ? SharpCoreDB.Interfaces.StorageEngineType.PageBased
+ : SharpCoreDB.Interfaces.StorageEngineType.AppendOnly;
+ RunPkAbComparison(engineTypeAb);
+ return;
+ }
+
// Optional: --engine=appendonly (default) | --engine=pagebased
// PageBased is the v2.0 in-place-update engine (WP10-WP13 storage engine roadmap).
var engineArg = args.FirstOrDefault(a => a.StartsWith("--engine=", StringComparison.OrdinalIgnoreCase));
@@ -395,6 +410,34 @@ static DatabaseConfig BuildConfig(SharpCoreDB.Interfaces.StorageEngineType engin
};
}
+ ///
+ /// Builds the variant DatabaseConfig used by the --pk-default arm. The variant name comes from
+ /// SHARPCOREDB_PK_DEFAULT_VARIANT or the --pk-ab arm selector; "" is the pure default config.
+ ///
+ private static DatabaseConfig BuildPkDefaultVariantConfig(
+ SharpCoreDB.Interfaces.StorageEngineType engineType,
+ string? variant)
+ {
+ return variant switch
+ {
+ "async" => new DatabaseConfig { StorageEngineType = engineType, WalDurabilityMode = SharpCoreDB.Services.DurabilityMode.Async },
+ "bufferedio" => new DatabaseConfig { StorageEngineType = engineType, UseBufferedIO = true },
+ "novalidate" => new DatabaseConfig
+ {
+ StorageEngineType = engineType,
+ SqlValidationMode = SharpCoreDB.Services.SqlQueryValidator.ValidationMode.Disabled,
+ StrictParameterValidation = false,
+ },
+ "noadaptive" => new DatabaseConfig { StorageEngineType = engineType, EnableAdaptiveWalBatching = false },
+ "hsinsert" => new DatabaseConfig { StorageEngineType = engineType, HighSpeedInsertMode = true },
+ // "plain" == the tuned harness config with NoEncryptMode=true (BuildConfig default);
+ // "tuned" == the same knob set but NoEncryptMode=false (isolates that flag).
+ "plain" => BuildConfig(engineType, fixedWidth: true),
+ "tuned" => BuildConfig(engineType, fixedWidth: true, noEncrypt: false),
+ _ => new DatabaseConfig { StorageEngineType = engineType },
+ };
+ }
+
static BenchmarkResult RunSharpCoreDB(SharpCoreDB.Interfaces.StorageEngineType engineType)
{
var dbPath = Path.Combine(Path.GetTempPath(), $"bench-sharpcoredb-{Guid.NewGuid()}");
@@ -862,7 +905,8 @@ data TEXT
static BenchmarkResult RunSharpCoreDBPk(
SharpCoreDB.Interfaces.StorageEngineType engineType,
bool fixedWidth = false,
- bool useDefaultConfig = false)
+ bool useDefaultConfig = false,
+ string? defaultVariant = null)
{
var dbPath = Path.Combine(Path.GetTempPath(), $"bench-sharpcoredb-pk-{Guid.NewGuid()}");
var result = new BenchmarkResult();
@@ -877,25 +921,9 @@ static BenchmarkResult RunSharpCoreDBPk(
DatabaseConfig config;
if (useDefaultConfig)
{
- var variant = Environment.GetEnvironmentVariable("SHARPCOREDB_PK_DEFAULT_VARIANT")?.ToLowerInvariant();
- config = variant switch
- {
- "async" => new DatabaseConfig { StorageEngineType = engineType, WalDurabilityMode = SharpCoreDB.Services.DurabilityMode.Async },
- "bufferedio" => new DatabaseConfig { StorageEngineType = engineType, UseBufferedIO = true },
- "novalidate" => new DatabaseConfig
- {
- StorageEngineType = engineType,
- SqlValidationMode = SharpCoreDB.Services.SqlQueryValidator.ValidationMode.Disabled,
- StrictParameterValidation = false,
- },
- "noadaptive" => new DatabaseConfig { StorageEngineType = engineType, EnableAdaptiveWalBatching = false },
- "hsinsert" => new DatabaseConfig { StorageEngineType = engineType, HighSpeedInsertMode = true },
- // "plain" == the tuned harness config with NoEncryptMode=true (BuildConfig default);
- // "tuned" == the same knob set but NoEncryptMode=false (isolates that flag).
- "plain" => BuildConfig(engineType, fixedWidth: true),
- "tuned" => BuildConfig(engineType, fixedWidth: true, noEncrypt: false),
- _ => new DatabaseConfig { StorageEngineType = engineType },
- };
+ var variant = defaultVariant
+ ?? Environment.GetEnvironmentVariable("SHARPCOREDB_PK_DEFAULT_VARIANT")?.ToLowerInvariant();
+ config = BuildPkDefaultVariantConfig(engineType, variant);
}
else
{
@@ -1127,6 +1155,85 @@ static void RunPkDefaultComparison(SharpCoreDB.Interfaces.StorageEngineType engi
Console.WriteLine($"\nResults saved to: {path}");
}
+ ///
+ /// P3c: same-window interleaved A/B comparison of two default-config variants. Each rep runs
+ /// arm A then arm B back-to-back, and the result reports the per-rep paired ratio B/A per phase
+ /// (median), so slow-machine windows affect both arms of a pair and cancel out.
+ ///
+ static void RunPkAbComparison(SharpCoreDB.Interfaces.StorageEngineType engineType)
+ {
+ var armA = Environment.GetEnvironmentVariable("SHARPCOREDB_PK_AB_ARM_A")?.ToLowerInvariant() ?? string.Empty;
+ var armB = Environment.GetEnvironmentVariable("SHARPCOREDB_PK_AB_ARM_B")?.ToLowerInvariant() ?? "plain";
+ int reps = 3;
+ if (int.TryParse(Environment.GetEnvironmentVariable("SHARPCOREDB_BENCH_REPS"), out int envReps) && envReps > 0)
+ {
+ reps = envReps;
+ }
+
+ Console.WriteLine("╔══════════════════════════════════════════════════════════╗");
+ Console.WriteLine("║ Same-window interleaved A/B (default-config variants) ║");
+ Console.WriteLine("╚══════════════════════════════════════════════════════════╝");
+ Console.WriteLine();
+ Console.WriteLine($"Arm A variant: '{armA}' Arm B variant: '{armB}' reps: {reps}");
+ Console.WriteLine("(each rep: A then B back-to-back; per-rep ratios cancel drift)");
+ Console.WriteLine();
+
+ var listA = new List(reps);
+ var listB = new List(reps);
+ for (int r = 0; r < reps; r++)
+ {
+ Console.WriteLine($"── rep {r + 1}/{reps} · A='{armA}' ──");
+ listA.Add(RunSharpCoreDBPk(engineType, useDefaultConfig: true, defaultVariant: armA));
+ Console.WriteLine($"── rep {r + 1}/{reps} · B='{armB}' ──");
+ listB.Add(RunSharpCoreDBPk(engineType, useDefaultConfig: true, defaultVariant: armB));
+ }
+
+ static double Med(IEnumerable xs)
+ {
+ var sorted = xs.Where(x => x > 0).OrderBy(x => x).ToArray();
+ return sorted.Length == 0 ? 0 : sorted[sorted.Length / 2];
+ }
+
+ static int Ops(IEnumerable runs, Func select)
+ {
+ var arr = runs.Select(select).Where(x => x > 0).OrderBy(x => x).ToArray();
+ return arr.Length == 0 ? 0 : arr[arr.Length / 2];
+ }
+
+ static double Ratio(IEnumerable a, IEnumerable b, Func sel)
+ {
+ var ratios = a.Zip(b, (x, y) => sel(x) > 0 ? sel(y) / (double)sel(x) : 0.0);
+ return Med(ratios);
+ }
+
+ var aUpdate = Ops(listA, static r => r.UpdateOpsPerSec);
+ var bUpdate = Ops(listB, static r => r.UpdateOpsPerSec);
+ var aDelete = Ops(listA, static r => r.DeleteOpsPerSec);
+ var bDelete = Ops(listB, static r => r.DeleteOpsPerSec);
+
+ Console.WriteLine("║ phase │ A median ops/s │ B median ops/s │ median B/A (per rep) ║");
+ Console.WriteLine($"║ UPDATE │ {aUpdate,13:N0} │ {bUpdate,14:N0} │ {Ratio(listA, listB, static r => r.UpdateOpsPerSec):F2}x");
+ Console.WriteLine($"║ DELETE │ {aDelete,13:N0} │ {bDelete,14:N0} │ {Ratio(listA, listB, static r => r.DeleteOpsPerSec):F2}x");
+ Console.WriteLine($"║ INSERT │ {Ops(listA, static r => r.InsertOpsPerSec),13:N0} │ {Ops(listB, static r => r.InsertOpsPerSec),14:N0} │ {Ratio(listA, listB, static r => r.InsertOpsPerSec):F2}x");
+ Console.WriteLine($"║ READ │ {Ops(listA, static r => r.ReadOpsPerSec),13:N0} │ {Ops(listB, static r => r.ReadOpsPerSec),14:N0} │ {Ratio(listA, listB, static r => r.ReadOpsPerSec):F2}x");
+
+ var summary = new Dictionary
+ {
+ ["armA"] = armA,
+ ["armB"] = armB,
+ ["medianUpdateRatio_B_over_A"] = Ratio(listA, listB, static r => r.UpdateOpsPerSec),
+ ["medianDeleteRatio_B_over_A"] = Ratio(listA, listB, static r => r.DeleteOpsPerSec),
+ ["medianInsertRatio_B_over_A"] = Ratio(listA, listB, static r => r.InsertOpsPerSec),
+ ["medianReadRatio_B_over_A"] = Ratio(listA, listB, static r => r.ReadOpsPerSec),
+ };
+
+ var dir = "results";
+ Directory.CreateDirectory(dir);
+ var path = Path.Combine(dir, $"pk_ab_{armA}_{armB}_{DateTime.UtcNow:yyyyMMdd_HHmmss}.json");
+ File.WriteAllText(path, JsonSerializer.Serialize(summary, new JsonSerializerOptions { WriteIndented = true }));
+ Console.WriteLine($"\nResults saved to: {path}");
+ }
+
// ══════════════════════════════════════
// LiteDB
// ══════════════════════════════════════