AB#12334: Prototype duration-balanced host build tests on Helix

build-tools/automation/HostBuildTestsHelix.proj

@@ -0,0 +1,44 @@
+<Project Sdk="Microsoft.DotNet.Helix.Sdk" DefaultTargets="Test">
+  <PropertyGroup>
+    <Language>msbuild</Language>
+    <EnableAzurePipelinesReporter>false</EnableAzurePipelinesReporter>
+    <FailOnTestFailure>true</FailOnTestFailure>
+    <HelixType Condition="'$(HelixType)' == ''">tests/android/host-build-tests/</HelixType>
+    <HelixTestRunName Condition="'$(HelixTestRunName)' == ''">Xamarin.Android.Build.Tests - Helix</HelixTestRunName>
+    <HostBuildTestsHelixWorkItemTimeout Condition="'$(HostBuildTestsHelixWorkItemTimeout)' == ''">01:30:00</HostBuildTestsHelixWorkItemTimeout>
+  </PropertyGroup>
+
+  <Import Project="$(HostBuildTestsHelixWorkItemsProps)" Condition="Exists('$(HostBuildTestsHelixWorkItemsProps)')" />
+  <Import Project="$(HostBuildTestsHelixCorrelationPayloadsProps)" Condition="Exists('$(HostBuildTestsHelixCorrelationPayloadsProps)')" />
+
+  <ItemGroup>
+    <HelixTargetQueue Include="$(HelixTargetQueueForItem)" Condition="'$(HelixTargetQueueForItem)' != ''">
+      <TestRunName>$(HelixTestRunName)</TestRunName>
+    </HelixTargetQueue>
+  </ItemGroup>
+
+  <ItemGroup>
+    <HelixCorrelationPayload Include="@(_HostBuildTestHelixCorrelationPayload)" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <HelixWorkItem Include="@(_HostBuildTestHelixWorkItem)">
+      <Timeout>$(HostBuildTestsHelixWorkItemTimeout)</Timeout>
+    </HelixWorkItem>
+  </ItemGroup>
+
+  <Target Name="_ValidateHostBuildTestsHelixInputs" BeforeTargets="Test">
+    <Error Text="HostBuildTestsHelixWorkItemsProps must name a generated work-items props file."
+           Condition="'$(HostBuildTestsHelixWorkItemsProps)' == '' or !Exists('$(HostBuildTestsHelixWorkItemsProps)')" />
+    <Error Text="HostBuildTestsHelixCorrelationPayloadsProps must name the generated correlation payload props file."
+           Condition="'$(HostBuildTestsHelixCorrelationPayloadsProps)' == '' or !Exists('$(HostBuildTestsHelixCorrelationPayloadsProps)')" />
+    <Error Text="HelixTargetQueueForItem is required."
+           Condition="'$(HelixTargetQueueForItem)' == ''" />
+    <Error Text="No host-build-test Helix correlation payloads were generated."
+           Condition="'@(_HostBuildTestHelixCorrelationPayload->Count())' == '0'" />
+    <Error Text="No host-build-test Helix work items were generated."
+           Condition="'@(_HostBuildTestHelixWorkItem->Count())' == '0'" />
+    <Message Text="Submitting @(_HostBuildTestHelixWorkItem->Count()) host-build-test work items with @(_HostBuildTestHelixCorrelationPayload->Count()) correlation payloads to $(HelixTargetQueueForItem)."
+             Importance="High" />
+  </Target>
+</Project>

build-tools/automation/azure-pipelines-public.yaml

@@ -31,6 +31,38 @@ parameters:
   values:
   - 'true'
   - 'false'
+- name: enableHostBuildTestsHelixPrototype
+  displayName: Enable host build tests in Helix prototype
+  type: boolean
+  default: false
+- name: hostBuildTestsHelixTargetMinutes
+  displayName: Host build tests Helix target minutes
+  type: number
+  default: 15
+- name: hostBuildTestsHelixDurationParallelism
+  displayName: Windows host build tests timing parallelism
+  type: number
+  default: 1.3
+- name: hostBuildTestsHelixLinuxDurationParallelism
+  displayName: Linux host build tests timing parallelism
+  type: number
+  default: 1.65
+- name: hostBuildTestsHelixFallbackTestDurationSeconds
+  displayName: Host build tests fallback seconds per test
+  type: number
+  default: 60
+- name: hostBuildTestsHelixTimingBuildId
+  displayName: Prior build ID containing host build test timings
+  type: string
+  default: ''
+- name: hostBuildTestsHelixWorkItemTimeout
+  displayName: Host build tests Helix work item timeout
+  type: string
+  default: '01:30:00'
+- name: hostBuildTestsHelixOnly
+  displayName: Run only the host build tests Helix prototype after builds
+  type: boolean
+  default: false
 
 # Repository resources
 resources:
@@ -162,6 +194,61 @@ stages:
         nugetArtifactName: $(LinuxNuGetArtifactName)
         use1ESTemplate: false
 
+# Guarded additive prototype. The existing Windows and Linux test jobs remain enabled.
+- ${{ if eq(parameters.enableHostBuildTestsHelixPrototype, true) }}:
+  - stage: host_build_tests_helix
+    displayName: Host Build Tests Helix Prototype
+    dependsOn:
+    - mac_build
+    - linux_build
+    condition: and(succeeded(), or(ne(variables['SkipTestStages'], 'true'), eq('${{ parameters.hostBuildTestsHelixOnly }}', 'true')))
+    jobs:
+    - job: windows_host_build_tests_helix
+      displayName: Windows > Host Build Tests > Helix
+      pool:
+        name: $(NetCorePublicPoolName)
+        demands:
+        - ImageOverride -equals $(WindowsPoolImageNetCorePublic)
+      timeoutInMinutes: 300
+      workspace:
+        clean: all
+      steps:
+      - template: /build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml
+        parameters:
+          platform: windows
+          helixQueue: Windows.Amd64.Server2022.Open
+          testFilter: ''
+          testRunTitle: Xamarin.Android.Build.Tests - Windows Helix Prototype
+          timingBuildId: ${{ parameters.hostBuildTestsHelixTimingBuildId }}
+          timingArtifactName: Host Build Test Timings - windows
+          targetMinutes: ${{ parameters.hostBuildTestsHelixTargetMinutes }}
+          durationParallelism: ${{ parameters.hostBuildTestsHelixDurationParallelism }}
+          fallbackTestDurationSeconds: ${{ parameters.hostBuildTestsHelixFallbackTestDurationSeconds }}
+          workItemTimeout: ${{ parameters.hostBuildTestsHelixWorkItemTimeout }}
+
+    - job: linux_host_build_tests_helix
+      displayName: Linux > BuildTest > Helix
+      pool:
+        name: $(NetCorePublicPoolName)
+        demands:
+        - ImageOverride -equals $(LinuxPoolImageNetCorePublic)
+      timeoutInMinutes: 300
+      workspace:
+        clean: all
+      steps:
+      - template: /build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml
+        parameters:
+          platform: linux
+          helixQueue: Ubuntu.2404.Amd64.Open
+          testFilter: 'class == Xamarin.Android.Build.Tests.BuildTest'
+          testRunTitle: Xamarin.Android.Build.Tests - Linux BuildTest Helix Prototype
+          timingBuildId: ${{ parameters.hostBuildTestsHelixTimingBuildId }}
+          timingArtifactName: Host Build Test Timings - linux
+          targetMinutes: ${{ parameters.hostBuildTestsHelixTargetMinutes }}
+          durationParallelism: ${{ parameters.hostBuildTestsHelixLinuxDurationParallelism }}
+          fallbackTestDurationSeconds: ${{ parameters.hostBuildTestsHelixFallbackTestDurationSeconds }}
+          workItemTimeout: ${{ parameters.hostBuildTestsHelixWorkItemTimeout }}
+
 # Package Tests Stage
 - ${{ if ne(variables.SkipTestStages, 'true') }}:
   - template: /build-tools/automation/yaml-templates/stage-package-tests.yaml

build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1

@@ -0,0 +1,161 @@
+Set-StrictMode -Version Latest
+$ErrorActionPreference = 'Stop'
+Import-Module (Join-Path $PSScriptRoot 'HostBuildTestsHelix.psm1') -Force
+
+function Assert-Equal
+{
+	param (
+		[Parameter(Mandatory)]
+		[object] $Expected,
+
+		[Parameter(Mandatory)]
+		[object] $Actual,
+
+		[Parameter(Mandatory)]
+		[string] $Message
+	)
+
+	if ($Expected -ne $Actual) {
+		throw "$Message Expected '$Expected', got '$Actual'."
+	}
+}
+
+$testRoot = Join-Path $PSScriptRoot '.test-output'
+Remove-Item -LiteralPath $testRoot -Recurse -Force -ErrorAction Ignore
+New-Item -ItemType Directory -Force -Path $testRoot | Out-Null
+
+try {
+	$emptyHistory = [System.Collections.Generic.Dictionary[string,long]]::new([StringComparer]::Ordinal)
+	$fallbackPlan = New-DurationBalancedWorkItems `
+		-TestNames @('F', 'E', 'D', 'C', 'B', 'A') `
+		-TimingHistory $emptyHistory `
+		-TargetMinutes 1 `
+		-DurationParallelism 1 `
+		-FallbackTestDurationSeconds 30
+
+	Assert-Equal 'count-fallback' $fallbackPlan.DurationMode 'No-history mode should be explicit.'
+	Assert-Equal 3 $fallbackPlan.WorkItems.Count 'Equal fallback durations should create three full bins.'
+	foreach ($workItem in $fallbackPlan.WorkItems) {
+		Assert-Equal 2 $workItem.Tests.Count 'Each fallback bin should contain two tests.'
+		Assert-Equal 60000 $workItem.EstimatedDurationMs 'Each fallback bin should target one minute.'
+	}
+
+	$history = [System.Collections.Generic.Dictionary[string,long]]::new([StringComparer]::Ordinal)
+	$history.Add('Oversized', 720000)
+	$history.Add('Eight', 480000)
+	$history.Add('Six', 360000)
+	$history.Add('Four', 240000)
+	$historicalPlan = New-DurationBalancedWorkItems `
+		-TestNames @('Four', 'Six', 'Eight', 'Oversized') `
+		-TimingHistory $history `
+		-TargetMinutes 10 `
+		-DurationParallelism 1
+
+	Assert-Equal 'historical' $historicalPlan.DurationMode 'Complete history should be reported.'
+	Assert-Equal 3 $historicalPlan.WorkItems.Count 'Best-fit packing should create three bins.'
+	$oversized = @($historicalPlan.WorkItems | Where-Object Oversized)
+	Assert-Equal 1 $oversized.Count 'Only the over-target test should be oversized.'
+	Assert-Equal 1 $oversized[0].Tests.Count 'An over-target test must remain isolated.'
+
+	$partialHistory = [System.Collections.Generic.Dictionary[string,long]]::new([StringComparer]::Ordinal)
+	$partialHistory.Add('KnownOne', 10000)
+	$partialHistory.Add('KnownTwo', 30000)
+	$partialPlan = New-DurationBalancedWorkItems `
+		-TestNames @('KnownOne', 'KnownTwo', 'NewTest') `
+		-TimingHistory $partialHistory `
+		-TargetMinutes 1 `
+		-DurationParallelism 1
+
+	Assert-Equal 'partial-history' $partialPlan.DurationMode 'Missing tests should be reported as partial history.'
+	Assert-Equal 20000 $partialPlan.MissingTestDurationMs 'Missing tests should use the selected-history average.'
+
+	$zeroTimingPath = Join-Path $testRoot 'zero-timing.xml'
+	Set-Content -LiteralPath $zeroTimingPath -Value '<tests><test name="FastTest" duration="0" /></tests>'
+	$zeroTimingHistory = Get-HostBuildTestTimingHistory -Path $zeroTimingPath
+	Assert-Equal 1 $zeroTimingHistory['FastTest'] 'Sub-millisecond TRX timings should retain a minimal positive load.'
+
+	$specialNames = @("Namespace.Test('value')", 'Namespace.Path(C:\repo\file)')
+	$runSettingsPath = Join-Path $testRoot 'special.runsettings'
+	Write-NUnitRunSettings -TestNames $specialNames -Path $runSettingsPath
+	$roundTrippedNames = Get-NUnitRunSettingsTestNames -Path $runSettingsPath
+	Assert-Equal ($specialNames -join '|') ($roundTrippedNames -join '|') 'Runsettings names should round trip.'
+
+	$singleRunSettingsPath = Join-Path $testRoot 'single.runsettings'
+	Write-NUnitRunSettings -TestNames @('OnlyTest') -Path $singleRunSettingsPath
+	$singleTest = Get-NUnitRunSettingsTestNames -Path $singleRunSettingsPath
+	Assert-Equal 1 $singleTest.Count 'A one-test runsettings file should remain an array.'
+	Assert-Equal 'OnlyTest' $singleTest[0] 'The single test should round trip.'
+
+	$emptyRunSettingsPath = Join-Path $testRoot 'empty.runsettings'
+	Set-Content -LiteralPath $emptyRunSettingsPath -Value "<RunSettings><NUnit><Where>test == 'dotnet-slicer-dummy-test-name'</Where></NUnit></RunSettings>"
+	try {
+		Get-NUnitRunSettingsTestNames -Path $emptyRunSettingsPath
+		throw 'The slicer dummy test should have caused an error.'
+	} catch {
+		if ($_.Exception.Message -notlike '*matched no tests*') {
+			throw
+		}
+	}
+
+	$secondFallbackPlan = New-DurationBalancedWorkItems `
+		-TestNames @('F', 'E', 'D', 'C', 'B', 'A') `
+		-TimingHistory $emptyHistory `
+		-TargetMinutes 1 `
+		-DurationParallelism 1 `
+		-FallbackTestDurationSeconds 30
+	Assert-Equal `
+		($fallbackPlan | ConvertTo-Json -Depth 8 -Compress) `
+		($secondFallbackPlan | ConvertTo-Json -Depth 8 -Compress) `
+		'Work-item generation should be deterministic.'
+
+	$payloadDirectory = Join-Path $testRoot 'payloads'
+	$propsPath = Join-Path $testRoot 'work-items.props'
+	Write-HostBuildTestWorkItemPayloads `
+		-Platform windows `
+		-Plan $fallbackPlan `
+		-WorkItemsDirectory $payloadDirectory `
+		-WorkItemsPropsPath $propsPath `
+		-TestAssemblyRelativePath 'bin\TestRelease\net10.0\Xamarin.Android.Build.Tests.dll' `
+		-NUnitWorkers '-1' `
+		-Configuration Release
+
+	[xml] $props = Get-Content -LiteralPath $propsPath -Raw
+	Assert-Equal 3 @($props.Project.ItemGroup._HostBuildTestHelixWorkItem).Count 'Props should contain every generated work item.'
+	foreach ($item in @($props.Project.ItemGroup._HostBuildTestHelixWorkItem)) {
+		if (-not (Test-Path -LiteralPath $item.PayloadDirectory -PathType Container)) {
+			throw "Payload directory '$($item.PayloadDirectory)' was not created."
+		}
+		$command = Get-Content -LiteralPath (Join-Path $item.PayloadDirectory 'run-host-tests.cmd') -Raw
+		if ($command.Contains('__CONFIGURATION__') -or
+			-not $command.Contains('bin\Release\dotnet') -or
+			$command.Contains('set "CONFIGURATION=Release"') -or
+			-not $command.Contains('set "RUNNINGONCI=true"') -or
+			-not $command.Contains('set "TEST_ANDROID_NDK_PATH=') -or
+			-not $command.Contains('set "BUILD_STAGINGDIRECTORY=%HELIX_WORKITEM_ROOT%\o"') -or
+			-not $command.Contains('dotnet-test-slicer.exe" retry') -or
+			-not $command.Contains('tar.exe -a -c -f "%HELIX_WORKITEM_UPLOAD_ROOT%\diagnostics.zip"')) {
+			throw 'The generated command did not apply the requested build configuration.'
+		}
+		Assert-Equal 'results.trx;retry-results.trx;console.log;slice.runsettings;work-item.json;diagnostics.zip' ([string] $item.DownloadFilesFromResults) 'Windows results should be downloaded.'
+	}
+
+	$linuxPayloadDirectory = Join-Path $testRoot 'linux-payloads'
+	$linuxPropsPath = Join-Path $testRoot 'linux-work-items.props'
+	Write-HostBuildTestWorkItemPayloads `
+		-Platform linux `
+		-Plan $fallbackPlan `
+		-WorkItemsDirectory $linuxPayloadDirectory `
+		-WorkItemsPropsPath $linuxPropsPath `
+		-TestAssemblyRelativePath 'bin/TestRelease/net10.0/Xamarin.Android.Build.Tests.dll' `
+		-NUnitWorkers '-1' `
+		-Configuration Release
+	[xml] $linuxProps = Get-Content -LiteralPath $linuxPropsPath -Raw
+	foreach ($item in @($linuxProps.Project.ItemGroup._HostBuildTestHelixWorkItem)) {
+		Assert-Equal 'bash run-host-tests.sh' ([string] $item.Command) 'Linux work items should explicitly use bash.'
+		Assert-Equal 'results.trx;retry-results.trx;console.log;slice.runsettings;work-item.json;diagnostics.tar.gz' ([string] $item.DownloadFilesFromResults) 'Linux results should be downloaded.'
+	}
+
+	Write-Host 'HostBuildTestsHelix tests passed.'
+} finally {
+	Remove-Item -LiteralPath $testRoot -Recurse -Force -ErrorAction Ignore
+}