Stabilize the SOS test harness across platforms

src/SOS/Strike/util.cpp

@@ -3148,7 +3148,7 @@ BOOL GetSOSVersion(VS_FIXEDFILEINFO *pFileInfo)
                 UINT uLen = 0;
                 if (VerQueryValueA(pVersionInfo, "\\", (LPVOID *) &pTmpFileInfo, &uLen))
                 {
-                    if (pFileInfo->dwFileVersionMS == (DWORD)-1) {
+                    if (pTmpFileInfo->dwFileVersionMS == (DWORD)-1) {
                         return FALSE;
                     }
                     *pFileInfo = *pTmpFileInfo; // Copy the info

src/tests/Debuggees.proj

@@ -19,6 +19,8 @@
     <SOSSingleFileDebuggee Include="SOS.UnitTests/Debuggees/SimpleThrow/SimpleThrow.csproj" />
     <SOSSingleFileDebuggee Include="SOS.UnitTests/Debuggees/ReflectionTest/ReflectionTest.csproj" />
     <SOSSingleFileDebuggee Include="SOS.UnitTests/Debuggees/SosHarnessScenarios/SosHarnessScenarios.csproj" />
+    <SOSFrameworkDebuggee Include="@(SOSSingleFileDebuggee)" />
+    <SOSFrameworkDebuggee Remove="SOS.UnitTests/Debuggees/DynamicMethod/DynamicMethod.csproj" />
   </ItemGroup>
 
   <Target Name="PublishSOSSingleFileDebuggees"
@@ -42,4 +44,16 @@
              Properties="Configuration=$(Configuration);BuildProjectFramework=%(_SOSSingleFileRuntime.TargetFramework);RuntimeIdentifier=$(TargetRid);RuntimeFrameworkVersion=%(_SOSSingleFileRuntime.Runtime);SelfContained=true;PublishSingleFile=true;SOSSingleFileRuntimeVersion=%(_SOSSingleFileRuntime.Runtime);ArtifactsObjDir=$(ArtifactsObjDir)SOSSingleFile/%(_SOSSingleFileRuntime.TargetFramework)/$(TargetRid)/" />
   </Target>
 
+  <Target Name="BuildSOSFrameworkDebuggees"
+          AfterTargets="Build"
+          Condition="'$(OS)' == 'Windows_NT' and '$(SkipSOSFrameworkDebuggees)' != 'true'">
+    <PropertyGroup>
+      <_SOSFrameworkPlatformProperty Condition="'$(TargetArch)' == 'x86'">PlatformTarget=x86;</_SOSFrameworkPlatformProperty>
+    </PropertyGroup>
+    <MSBuild Projects="@(SOSFrameworkDebuggee)"
+             Targets="Restore;Build"
+             BuildInParallel="true"
+             Properties="Configuration=$(Configuration);BuildProjectFramework=net462;$(_SOSFrameworkPlatformProperty)DebugType=full;DebugSymbols=true;ArtifactsObjDir=$(ArtifactsObjDir)SOSFramework/$(TargetArch)/%(Filename)/" />
+  </Target>
+
 </Project>

src/tests/SOS.TestHarness/BoundedProcess.cs

@@ -0,0 +1,139 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Runtime.InteropServices;
+using System.Threading.Tasks;
+
+namespace SOS.TestHarness;
+
+internal sealed record BoundedProcessResult(int ExitCode, string StandardOutput, string StandardError);
+
+internal static partial class BoundedProcess
+{
+    private const int KillSignal = 9;
+    private static readonly TimeSpan s_terminationTimeout = TimeSpan.FromSeconds(5);
+
+    public static BoundedProcessResult Run(
+        ProcessStartInfo startInfo,
+        TimeSpan timeout,
+        bool isolateLinuxProcessGroup = false,
+        TimeSpan? outputDrainTimeout = null)
+    {
+        if (!startInfo.RedirectStandardOutput || !startInfo.RedirectStandardError)
+        {
+            throw new ArgumentException("Standard output and standard error must both be redirected.", nameof(startInfo));
+        }
+
+        ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(timeout, TimeSpan.Zero);
+
+        string command = $"{startInfo.FileName} {string.Join(' ', startInfo.ArgumentList)}".Trim();
+        bool hasLinuxProcessGroup = isolateLinuxProcessGroup && OperatingSystem.IsLinux();
+        if (hasLinuxProcessGroup)
+        {
+            WrapWithSetSid(startInfo);
+        }
+
+        using Process process = Process.Start(startInfo)
+            ?? throw new InvalidOperationException($"Failed to start '{command}'.");
+        int processGroupId = process.Id;
+        Task<string> stdoutTask = process.StandardOutput.ReadToEndAsync();
+        Task<string> stderrTask = process.StandardError.ReadToEndAsync();
+
+        if (!process.WaitForExit(TimeoutMilliseconds(timeout)))
+        {
+            Terminate(process, hasLinuxProcessGroup, processGroupId);
+            BoundedProcessResult output = DrainOutput(
+                process,
+                stdoutTask,
+                stderrTask,
+                command,
+                s_terminationTimeout);
+            throw new TimeoutException(
+                $"'{command}' did not exit within {timeout}.{Environment.NewLine}" +
+                $"stdout:{Environment.NewLine}{output.StandardOutput}{Environment.NewLine}" +
+                $"stderr:{Environment.NewLine}{output.StandardError}");
+        }
+
+        // Linux single-file dump helpers can survive the target while retaining its redirected handles.
+        // End the isolated group before waiting for stream EOF so those descendants cannot wedge drainage.
+        if (hasLinuxProcessGroup)
+        {
+            KillProcessGroup(processGroupId);
+        }
+
+        return DrainOutput(
+            process,
+            stdoutTask,
+            stderrTask,
+            command,
+            outputDrainTimeout ?? s_terminationTimeout);
+    }
+
+    private static BoundedProcessResult DrainOutput(
+        Process process,
+        Task<string> stdoutTask,
+        Task<string> stderrTask,
+        string command,
+        TimeSpan timeout)
+    {
+        Task outputTask = Task.WhenAll(stdoutTask, stderrTask);
+        if (!outputTask.Wait(timeout))
+        {
+            throw new TimeoutException(
+                $"'{command}' exited with code {process.ExitCode}, but its redirected output did not close " +
+                $"within {timeout}.");
+        }
+
+        return new BoundedProcessResult(
+            process.ExitCode,
+            stdoutTask.GetAwaiter().GetResult(),
+            stderrTask.GetAwaiter().GetResult());
+    }
+
+    private static void Terminate(Process process, bool hasLinuxProcessGroup, int processGroupId)
+    {
+        if (hasLinuxProcessGroup)
+        {
+            KillProcessGroup(processGroupId);
+        }
+
+        if (!process.HasExited)
+        {
+            process.Kill(entireProcessTree: true);
+        }
+
+        process.WaitForExit(TimeoutMilliseconds(s_terminationTimeout));
+    }
+
+    private static void WrapWithSetSid(ProcessStartInfo startInfo)
+    {
+        string setSid = File.Exists("/usr/bin/setsid") ? "/usr/bin/setsid" :
+            File.Exists("/bin/setsid") ? "/bin/setsid" :
+            throw new FileNotFoundException("Could not locate setsid for Linux process-group isolation.");
+
+        string executable = startInfo.FileName;
+        string[] arguments = startInfo.ArgumentList.ToArray();
+        startInfo.FileName = setSid;
+        startInfo.ArgumentList.Clear();
+        startInfo.ArgumentList.Add(executable);
+        foreach (string argument in arguments)
+        {
+            startInfo.ArgumentList.Add(argument);
+        }
+    }
+
+    private static void KillProcessGroup(int processGroupId)
+    {
+        _ = KillUnix(-processGroupId, KillSignal);
+    }
+
+    private static int TimeoutMilliseconds(TimeSpan timeout) =>
+        (int)Math.Min(timeout.TotalMilliseconds, int.MaxValue);
+
+    [LibraryImport("libc", EntryPoint = "kill", SetLastError = true)]
+    private static partial int KillUnix(int processId, int signal);
+}

src/tests/SOS.TestHarness/ChildEngineClient.cs

@@ -23,6 +23,7 @@ public sealed class ChildEngineClient : ILiveDebuggerHost
     private readonly StreamWriter _stdin;
     private readonly BlockingCollection<string> _lines = new();
     private readonly Thread _reader;
+    private readonly Task<string> _stderr;
 
     public string Name { get; }
 
@@ -92,6 +93,7 @@ private ChildEngineClient(string name, string mode, IReadOnlyList<string> modeAr
 
         _process = Process.Start(psi) ?? throw new InvalidOperationException("Failed to start EngineHost");
         _stdin = _process.StandardInput;
+        _stderr = _process.StandardError.ReadToEndAsync();
 
         _reader = new Thread(ReadLoop) { IsBackground = true, Name = $"enginehost-reader-{name}" };
         _reader.Start();
@@ -144,10 +146,10 @@ private void WaitForReady(TimeSpan timeout)
     {
         while (true)
         {
-            if (!_lines.TryTake(out string? line, (int)timeout.TotalMilliseconds, HarnessCancellation.Token))
-            {
-                throw new TimeoutException("EngineHost did not become ready in time.");
-            }
+            string line = ReadLine(
+                timeout,
+                "EngineHost did not become ready in time.",
+                "before becoming ready");
 
             if (line == EngineProtocol.Ready)
             {
@@ -161,10 +163,10 @@ private string DrainToEnd(TimeSpan timeout, string command)
         StringBuilder sb = new();
         while (true)
         {
-            if (!_lines.TryTake(out string? line, (int)timeout.TotalMilliseconds, HarnessCancellation.Token))
-            {
-                throw new TimeoutException($"EngineHost did not return output for '{command}' within {timeout}.");
-            }
+            string line = ReadLine(
+                timeout,
+                $"EngineHost did not return output for '{command}' within {timeout}.",
+                $"while running '{command}'");
 
             if (line == EngineProtocol.End)
             {
@@ -186,12 +188,48 @@ private string DrainToEnd(TimeSpan timeout, string command)
         return sb.ToString();
     }
 
+    private string ReadLine(TimeSpan timeout, string timeoutMessage, string exitContext)
+    {
+        if (_lines.TryTake(out string? line, (int)timeout.TotalMilliseconds, HarnessCancellation.Token))
+        {
+            return line;
+        }
+
+        if (_lines.IsCompleted || _process.HasExited)
+        {
+            throw CreateEngineHostExitException(exitContext);
+        }
+
+        throw new TimeoutException(timeoutMessage);
+    }
+
+    private InvalidOperationException CreateEngineHostExitException(string context)
+    {
+        bool exited = _process.HasExited || _process.WaitForExit(1000);
+        string exitDescription = exited
+            ? $"exited with code {_process.ExitCode}"
+            : "closed its standard output";
+        string stderr = exited && _stderr.Wait(TimeSpan.FromSeconds(2))
+            ? _stderr.GetAwaiter().GetResult().Trim()
+            : string.Empty;
+        string details = stderr.Length == 0 ? string.Empty : $"{Environment.NewLine}{stderr}";
+
+        return new InvalidOperationException($"EngineHost {exitDescription} {context}.{details}");
+    }
+
     private void ReadLoop()
     {
-        string? line;
-        while ((line = _process.StandardOutput.ReadLine()) is not null)
+        try
+        {
+            string? line;
+            while ((line = _process.StandardOutput.ReadLine()) is not null)
+            {
+                _lines.Add(line);
+            }
+        }
+        finally
         {
-            _lines.Add(line);
+            _lines.CompleteAdding();
         }
     }
 

src/tests/SOS.TestHarness/DbgEngCapturer.cs

@@ -71,7 +71,7 @@ string Run(string command)
                 }
                 else // Crash
                 {
-                    RunToBreak(control, "second-chance crash");
+                    RunToBreak(control, "second-chance crash", requireSecondChanceException: true);
                 }
 
                 Run($".dump /o {DbgEngDumpOption(dumpKind)} \"{dumpPath}\"");
@@ -102,7 +102,7 @@ string Run(string command)
         _ => throw new ArgumentOutOfRangeException(nameof(dumpKind), dumpKind, "Unsupported dump kind"),
     };
 
-    private static void RunToBreak(IDebugControl control, string what)
+    private static void RunToBreak(IDebugControl control, string what, bool requireSecondChanceException = false)
     {
         const int MaxResumes = 40;
         for (int i = 0; i < MaxResumes; i++)
@@ -111,7 +111,10 @@ private static void RunToBreak(IDebugControl control, string what)
             control.WaitForEvent(TimeSpan.FromSeconds(60));
             control.GetExecutionStatus(out DEBUG_STATUS status);
 
-            if (status == DEBUG_STATUS.BREAK)
+            if (status == DEBUG_STATUS.BREAK
+                && (!requireSecondChanceException
+                    || (control.GetLastEvent(out DEBUG_LAST_EVENT_INFO_EXCEPTION exception, out _, out _)
+                        && exception.FirstChance == 0)))
             {
                 return;
             }

src/tests/SOS.TestHarness/DbgEngLiveHost.cs

@@ -114,7 +114,9 @@ public SosOutput RunToCrash()
                 Control.WaitForEvent(TimeSpan.FromSeconds(60));
                 Control.GetExecutionStatus(out DEBUG_STATUS status);
 
-                if (status == DEBUG_STATUS.BREAK)
+                if (status == DEBUG_STATUS.BREAK
+                    && Control.GetLastEvent(out DEBUG_LAST_EVENT_INFO_EXCEPTION exception, out _, out _)
+                    && exception.FirstChance == 0)
                 {
                     return; // second-chance crash break
                 }

src/tests/SOS.TestHarness/DumpGenerationRequirements.cs

@@ -34,8 +34,7 @@ namespace SOS.TestHarness;
 /// </summary>
 internal static class DumpGenerationRequirements
 {
-    private static readonly string s_root = RuntimeInformation.ProcessArchitecture == Architecture.X86 ? @"SOFTWARE\WOW6432Node\" : @"SOFTWARE\";
-    private static readonly string s_settingsNode = s_root + @"Microsoft\Windows NT\CurrentVersion\MiniDumpSettings";
+    private const string SettingsNode = @"SOFTWARE\Microsoft\Windows NT\CurrentVersion\MiniDumpSettings";
     private const string DisableCheckValue = "DisableAuxProviderSignatureCheck";
 
     // Read the registry value at most once per process (cheap, read-only; reading HKLM needs no elevation).
@@ -68,7 +67,7 @@ internal static DumpKind ResolveCaptureKind(Flavor flavor, DumpKind dumpKind)
         if (dumpKind == DumpKind.Mini)
         {
             HarnessSkipException.Now(
-                $@"Mini dump capture requires HKLM\{s_settingsNode}\{DisableCheckValue}=1 so dbghelp can " +
+                $@"Mini dump capture requires HKLM\{SettingsNode}\{DisableCheckValue}=1 so dbghelp can " +
                 "load the unsigned test DAC");
         }
 
@@ -90,12 +89,19 @@ private static bool ReadSignatureCheckDisabledWindows()
     {
         try
         {
-            using RegistryKey? key = Registry.LocalMachine.OpenSubKey(s_settingsNode);
+            using RegistryKey localMachine = RegistryKey.OpenBaseKey(
+                RegistryHive.LocalMachine,
+                RegistryViewForProcess(Environment.Is64BitProcess));
+            using RegistryKey? key = localMachine.OpenSubKey(SettingsNode);
             return key?.GetValue(DisableCheckValue) is int value && value == 1;
         }
         catch (Exception ex) when (ex is SecurityException or UnauthorizedAccessException or IOException)
         {
             return false;
         }
     }
+
+    [SupportedOSPlatform("windows")]
+    internal static RegistryView RegistryViewForProcess(bool is64BitProcess) =>
+        is64BitProcess ? RegistryView.Registry64 : RegistryView.Registry32;
 }