From 1f0faf3bd26e164d572f3e9a943432cf2725c5ed Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Fri, 21 Aug 2026 15:44:15 -0700 Subject: [PATCH 01/23] Prototype host build tests on Helix Add a guarded public-pipeline path that prepares shared host toolchains and generates configurable duration-balanced Xamarin.Android.Build.Tests work items for Windows and Linux. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../automation/HostBuildTestsHelix.proj | 46 ++ .../automation/azure-pipelines-public.yaml | 79 +++ .../HostBuildTestsHelix.Tests.ps1 | 135 +++++ .../HostBuildTestsHelix.psm1 | 468 ++++++++++++++++++ .../host-build-tests-helix/README.md | 24 + .../prepare-host-build-test-helix-payload.ps1 | 256 ++++++++++ .../run-host-build-tests-in-helix.yaml | 176 +++++++ 7 files changed, 1184 insertions(+) create mode 100644 build-tools/automation/HostBuildTestsHelix.proj create mode 100644 build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 create mode 100644 build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 create mode 100644 build-tools/automation/host-build-tests-helix/README.md create mode 100644 build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 create mode 100644 build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml diff --git a/build-tools/automation/HostBuildTestsHelix.proj b/build-tools/automation/HostBuildTestsHelix.proj new file mode 100644 index 00000000000..db68078dd58 --- /dev/null +++ b/build-tools/automation/HostBuildTestsHelix.proj @@ -0,0 +1,46 @@ + + + msbuild + true + false + true + tests/android/host-build-tests/ + Xamarin.Android.Build.Tests - Helix + 00:45:00 + + + + + + + $(HelixTestRunName) + + + + + + %(Identity) + + + + + + %(_HostBuildTestHelixWorkItem.PayloadDirectory) + %(_HostBuildTestHelixWorkItem.Command) + $(HostBuildTestsHelixWorkItemTimeout) + + + + + + + + + + + diff --git a/build-tools/automation/azure-pipelines-public.yaml b/build-tools/automation/azure-pipelines-public.yaml index 44eae1ceb96..143ca87f891 100644 --- a/build-tools/automation/azure-pipelines-public.yaml +++ b/build-tools/automation/azure-pipelines-public.yaml @@ -31,6 +31,30 @@ parameters: values: - 'true' - 'false' +- name: enableHostBuildTestsHelixPrototype + displayName: Enable host build tests in Helix prototype + type: boolean + default: false +- name: hostBuildTestsHelixTargetMinutes + displayName: Host build tests Helix target minutes + type: number + default: 15 +- name: hostBuildTestsHelixDurationParallelism + displayName: Host build tests timing parallelism + type: number + default: 2.5 +- name: hostBuildTestsHelixFallbackTestDurationSeconds + displayName: Host build tests fallback seconds per test + type: number + default: 60 +- name: hostBuildTestsHelixTimingBuildId + displayName: Prior build ID containing host build test timings + type: string + default: '' +- name: hostBuildTestsHelixWorkItemTimeout + displayName: Host build tests Helix work item timeout + type: string + default: '00:45:00' # Repository resources resources: @@ -162,6 +186,61 @@ stages: nugetArtifactName: $(LinuxNuGetArtifactName) use1ESTemplate: false +# Guarded additive prototype. The existing Windows and Linux test jobs remain enabled. +- ${{ if eq(parameters.enableHostBuildTestsHelixPrototype, true) }}: + - stage: host_build_tests_helix + displayName: Host Build Tests Helix Prototype + dependsOn: + - mac_build + - linux_build + condition: and(succeeded(), ne(variables['SkipTestStages'], 'true')) + jobs: + - job: windows_host_build_tests_helix + displayName: Windows > Host Build Tests > Helix + pool: + name: $(NetCorePublicPoolName) + demands: + - ImageOverride -equals $(WindowsPoolImageNetCorePublic) + timeoutInMinutes: 300 + workspace: + clean: all + steps: + - template: /build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml + parameters: + platform: windows + helixQueue: Windows.Amd64.VS2026.Open + testFilter: '' + testRunTitle: Xamarin.Android.Build.Tests - Windows Helix Prototype + timingBuildId: ${{ parameters.hostBuildTestsHelixTimingBuildId }} + timingArtifactName: Host Build Test Timings - windows + targetMinutes: ${{ parameters.hostBuildTestsHelixTargetMinutes }} + durationParallelism: ${{ parameters.hostBuildTestsHelixDurationParallelism }} + fallbackTestDurationSeconds: ${{ parameters.hostBuildTestsHelixFallbackTestDurationSeconds }} + workItemTimeout: ${{ parameters.hostBuildTestsHelixWorkItemTimeout }} + + - job: linux_host_build_tests_helix + displayName: Linux > BuildTest > Helix + pool: + name: $(NetCorePublicPoolName) + demands: + - ImageOverride -equals $(LinuxPoolImageNetCorePublic) + timeoutInMinutes: 300 + workspace: + clean: all + steps: + - template: /build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml + parameters: + platform: linux + helixQueue: Ubuntu.2404.Amd64.Open + testFilter: 'Name = BuildTest' + testRunTitle: Xamarin.Android.Build.Tests - Linux BuildTest Helix Prototype + timingBuildId: ${{ parameters.hostBuildTestsHelixTimingBuildId }} + timingArtifactName: Host Build Test Timings - linux + targetMinutes: ${{ parameters.hostBuildTestsHelixTargetMinutes }} + durationParallelism: ${{ parameters.hostBuildTestsHelixDurationParallelism }} + fallbackTestDurationSeconds: ${{ parameters.hostBuildTestsHelixFallbackTestDurationSeconds }} + workItemTimeout: ${{ parameters.hostBuildTestsHelixWorkItemTimeout }} + # Package Tests Stage - ${{ if ne(variables.SkipTestStages, 'true') }}: - template: /build-tools/automation/yaml-templates/stage-package-tests.yaml diff --git a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 new file mode 100644 index 00000000000..3d6bb23cf40 --- /dev/null +++ b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 @@ -0,0 +1,135 @@ +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +Import-Module (Join-Path $PSScriptRoot 'HostBuildTestsHelix.psm1') -Force + +function Assert-Equal +{ + param ( + [Parameter(Mandatory)] + [object] $Expected, + + [Parameter(Mandatory)] + [object] $Actual, + + [Parameter(Mandatory)] + [string] $Message + ) + + if ($Expected -ne $Actual) { + throw "$Message Expected '$Expected', got '$Actual'." + } +} + +$testRoot = Join-Path $PSScriptRoot '.test-output' +Remove-Item -LiteralPath $testRoot -Recurse -Force -ErrorAction Ignore +New-Item -ItemType Directory -Force -Path $testRoot | Out-Null + +try { + $emptyHistory = [System.Collections.Generic.Dictionary[string,long]]::new([StringComparer]::Ordinal) + $fallbackPlan = New-DurationBalancedWorkItems ` + -TestNames @('F', 'E', 'D', 'C', 'B', 'A') ` + -TimingHistory $emptyHistory ` + -TargetMinutes 1 ` + -DurationParallelism 1 ` + -FallbackTestDurationSeconds 30 + + Assert-Equal 'count-fallback' $fallbackPlan.DurationMode 'No-history mode should be explicit.' + Assert-Equal 3 $fallbackPlan.WorkItems.Count 'Equal fallback durations should create three full bins.' + foreach ($workItem in $fallbackPlan.WorkItems) { + Assert-Equal 2 $workItem.Tests.Count 'Each fallback bin should contain two tests.' + Assert-Equal 60000 $workItem.EstimatedDurationMs 'Each fallback bin should target one minute.' + } + + $history = [System.Collections.Generic.Dictionary[string,long]]::new([StringComparer]::Ordinal) + $history.Add('Oversized', 720000) + $history.Add('Eight', 480000) + $history.Add('Six', 360000) + $history.Add('Four', 240000) + $historicalPlan = New-DurationBalancedWorkItems ` + -TestNames @('Four', 'Six', 'Eight', 'Oversized') ` + -TimingHistory $history ` + -TargetMinutes 10 ` + -DurationParallelism 1 + + Assert-Equal 'historical' $historicalPlan.DurationMode 'Complete history should be reported.' + Assert-Equal 3 $historicalPlan.WorkItems.Count 'Best-fit packing should create three bins.' + $oversized = @($historicalPlan.WorkItems | Where-Object Oversized) + Assert-Equal 1 $oversized.Count 'Only the over-target test should be oversized.' + Assert-Equal 1 $oversized[0].Tests.Count 'An over-target test must remain isolated.' + + $partialHistory = [System.Collections.Generic.Dictionary[string,long]]::new([StringComparer]::Ordinal) + $partialHistory.Add('KnownOne', 10000) + $partialHistory.Add('KnownTwo', 30000) + $partialPlan = New-DurationBalancedWorkItems ` + -TestNames @('KnownOne', 'KnownTwo', 'NewTest') ` + -TimingHistory $partialHistory ` + -TargetMinutes 1 ` + -DurationParallelism 1 + + Assert-Equal 'partial-history' $partialPlan.DurationMode 'Missing tests should be reported as partial history.' + Assert-Equal 20000 $partialPlan.MissingTestDurationMs 'Missing tests should use the selected-history average.' + + $zeroTimingPath = Join-Path $testRoot 'zero-timing.xml' + Set-Content -LiteralPath $zeroTimingPath -Value '' + $zeroTimingHistory = Get-HostBuildTestTimingHistory -Path $zeroTimingPath + Assert-Equal 1 $zeroTimingHistory['FastTest'] 'Sub-millisecond TRX timings should retain a minimal positive load.' + + $specialNames = @("Namespace.Test('value')", 'Namespace.Path(C:\repo\file)') + $runSettingsPath = Join-Path $testRoot 'special.runsettings' + Write-NUnitRunSettings -TestNames $specialNames -Path $runSettingsPath + $roundTrippedNames = Get-NUnitRunSettingsTestNames -Path $runSettingsPath + Assert-Equal ($specialNames -join '|') ($roundTrippedNames -join '|') 'Runsettings names should round trip.' + + $secondFallbackPlan = New-DurationBalancedWorkItems ` + -TestNames @('F', 'E', 'D', 'C', 'B', 'A') ` + -TimingHistory $emptyHistory ` + -TargetMinutes 1 ` + -DurationParallelism 1 ` + -FallbackTestDurationSeconds 30 + Assert-Equal ` + ($fallbackPlan | ConvertTo-Json -Depth 8 -Compress) ` + ($secondFallbackPlan | ConvertTo-Json -Depth 8 -Compress) ` + 'Work-item generation should be deterministic.' + + $payloadDirectory = Join-Path $testRoot 'payloads' + $propsPath = Join-Path $testRoot 'work-items.props' + Write-HostBuildTestWorkItemPayloads ` + -Platform windows ` + -Plan $fallbackPlan ` + -WorkItemsDirectory $payloadDirectory ` + -WorkItemsPropsPath $propsPath ` + -TestAssemblyRelativePath 'bin\TestRelease\net10.0\Xamarin.Android.Build.Tests.dll' ` + -NUnitWorkers '-1' ` + -Configuration Release + + [xml] $props = Get-Content -LiteralPath $propsPath -Raw + Assert-Equal 3 @($props.Project.ItemGroup._HostBuildTestHelixWorkItem).Count 'Props should contain every generated work item.' + foreach ($item in @($props.Project.ItemGroup._HostBuildTestHelixWorkItem)) { + if (-not (Test-Path -LiteralPath $item.PayloadDirectory -PathType Container)) { + throw "Payload directory '$($item.PayloadDirectory)' was not created." + } + $command = Get-Content -LiteralPath (Join-Path $item.PayloadDirectory 'run-host-tests.cmd') -Raw + if ($command.Contains('__CONFIGURATION__') -or -not $command.Contains('bin\Release\dotnet')) { + throw 'The generated command did not apply the requested build configuration.' + } + } + + $linuxPayloadDirectory = Join-Path $testRoot 'linux-payloads' + $linuxPropsPath = Join-Path $testRoot 'linux-work-items.props' + Write-HostBuildTestWorkItemPayloads ` + -Platform linux ` + -Plan $fallbackPlan ` + -WorkItemsDirectory $linuxPayloadDirectory ` + -WorkItemsPropsPath $linuxPropsPath ` + -TestAssemblyRelativePath 'bin/TestRelease/net10.0/Xamarin.Android.Build.Tests.dll' ` + -NUnitWorkers '-1' ` + -Configuration Release + [xml] $linuxProps = Get-Content -LiteralPath $linuxPropsPath -Raw + foreach ($item in @($linuxProps.Project.ItemGroup._HostBuildTestHelixWorkItem)) { + Assert-Equal 'bash run-host-tests.sh' ([string] $item.Command) 'Linux work items should explicitly use bash.' + } + + Write-Host 'HostBuildTestsHelix tests passed.' +} finally { + Remove-Item -LiteralPath $testRoot -Recurse -Force -ErrorAction Ignore +} diff --git a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 new file mode 100644 index 00000000000..f51626c48b9 --- /dev/null +++ b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 @@ -0,0 +1,468 @@ +Set-StrictMode -Version Latest + +function Get-NUnitRunSettingsTestNames +{ + param ( + [Parameter(Mandatory)] + [string] $Path + ) + + [xml] $runSettings = Get-Content -LiteralPath $Path -Raw + $where = [string] $runSettings.RunSettings.NUnit.Where + $prefix = "test == '" + $names = [System.Collections.Generic.List[string]]::new() + $offset = 0 + + while ($offset -lt $where.Length) { + $start = $where.IndexOf($prefix, $offset, [StringComparison]::Ordinal) + if ($start -lt 0) { + break + } + + $index = $start + $prefix.Length + $name = [Text.StringBuilder]::new() + $closed = $false + while ($index -lt $where.Length) { + $character = $where[$index] + if ($character -eq '\') { + if ($index + 1 -ge $where.Length) { + throw "Invalid escaped test name in '$Path'." + } + $index++ + [void] $name.Append($where[$index]) + } elseif ($character -eq "'") { + $closed = $true + $index++ + break + } else { + [void] $name.Append($character) + } + $index++ + } + + if (-not $closed) { + throw "Unterminated test name in '$Path'." + } + + $names.Add($name.ToString()) + $offset = $index + } + + if ($names.Count -eq 0) { + throw "No tests were found in '$Path'." + } + + return $names.ToArray() +} + +function Get-HostBuildTestTimingHistory +{ + param ( + [string] $Path + ) + + $durations = [System.Collections.Generic.Dictionary[string,long]]::new([StringComparer]::Ordinal) + if ([string]::IsNullOrWhiteSpace($Path)) { + return $durations + } + if (-not (Test-Path -LiteralPath $Path -PathType Leaf)) { + throw "Timing history '$Path' does not exist." + } + + [xml] $balance = Get-Content -LiteralPath $Path -Raw + foreach ($test in @($balance.tests.test)) { + $name = [string] $test.name + $duration = 0L + if ([string]::IsNullOrWhiteSpace($name) -or + -not [long]::TryParse([string] $test.duration, [ref] $duration) -or + $duration -lt 0) { + throw "Timing history '$Path' contains an invalid test entry." + } + if (-not $durations.TryAdd($name, [Math]::Max(1, $duration))) { + throw "Timing history '$Path' contains duplicate test '$name'." + } + } + + return $durations +} + +function New-DurationBalancedWorkItems +{ + param ( + [Parameter(Mandatory)] + [string[]] $TestNames, + + [Parameter(Mandatory)] + [System.Collections.Generic.IDictionary[string,long]] $TimingHistory, + + [Parameter(Mandatory)] + [ValidateRange(1, 1440)] + [int] $TargetMinutes, + + [Parameter(Mandatory)] + [ValidateRange(0.1, 128)] + [double] $DurationParallelism, + + [ValidateRange(1, 86400)] + [int] $FallbackTestDurationSeconds = 60 + ) + + $uniqueNames = [System.Collections.Generic.HashSet[string]]::new([StringComparer]::Ordinal) + foreach ($name in $TestNames) { + if ([string]::IsNullOrWhiteSpace($name)) { + throw 'Test names cannot be empty.' + } + if (-not $uniqueNames.Add($name)) { + throw "Duplicate test name '$name'." + } + } + if ($uniqueNames.Count -eq 0) { + throw 'At least one test is required.' + } + + $knownDurations = [System.Collections.Generic.List[long]]::new() + foreach ($name in $uniqueNames) { + $duration = 0L + if ($TimingHistory.TryGetValue($name, [ref] $duration)) { + $knownDurations.Add($duration) + } + } + + if ($knownDurations.Count -eq 0) { + $missingDuration = [long] $FallbackTestDurationSeconds * 1000 + $durationMode = 'count-fallback' + } else { + $missingDuration = [long] [Math]::Round( + ($knownDurations | Measure-Object -Average).Average, + [MidpointRounding]::AwayFromZero + ) + $durationMode = if ($knownDurations.Count -eq $uniqueNames.Count) { 'historical' } else { 'partial-history' } + } + + $tests = [System.Collections.Generic.List[object]]::new() + foreach ($name in $uniqueNames) { + $duration = 0L + $isEstimated = -not $TimingHistory.TryGetValue($name, [ref] $duration) + if ($isEstimated) { + $duration = $missingDuration + } + $tests.Add([pscustomobject] @{ + Name = $name + DurationMs = $duration + IsEstimated = $isEstimated + }) + } + + $comparison = [System.Comparison[object]] { + param ($left, $right) + $durationComparison = $right.DurationMs.CompareTo($left.DurationMs) + if ($durationComparison -ne 0) { + return $durationComparison + } + return [StringComparer]::Ordinal.Compare($left.Name, $right.Name) + } + $tests.Sort($comparison) + + $targetLoadMs = [long] [Math]::Round( + $TargetMinutes * 60000 * $DurationParallelism, + [MidpointRounding]::AwayFromZero + ) + $bins = [System.Collections.Generic.List[object]]::new() + + foreach ($test in $tests) { + $selectedBin = $null + if ($test.DurationMs -le $targetLoadMs) { + foreach ($bin in $bins) { + if ($bin.Oversized -or $bin.DurationMs + $test.DurationMs -gt $targetLoadMs) { + continue + } + if ($null -eq $selectedBin -or + $bin.DurationMs -gt $selectedBin.DurationMs -or + ($bin.DurationMs -eq $selectedBin.DurationMs -and $bin.Id -lt $selectedBin.Id)) { + $selectedBin = $bin + } + } + } + + if ($null -eq $selectedBin) { + $selectedBin = [pscustomobject] @{ + Id = $bins.Count + 1 + DurationMs = 0L + Oversized = $test.DurationMs -gt $targetLoadMs + Tests = [System.Collections.Generic.List[object]]::new() + } + $bins.Add($selectedBin) + } + + $selectedBin.Tests.Add($test) + $selectedBin.DurationMs += $test.DurationMs + } + + $workItems = [System.Collections.Generic.List[object]]::new() + foreach ($bin in $bins) { + $sortedNames = [System.Collections.Generic.List[string]]::new() + foreach ($test in $bin.Tests) { + $sortedNames.Add($test.Name) + } + $sortedNames.Sort([StringComparer]::Ordinal) + + $workItems.Add([pscustomobject] @{ + Id = $bin.Id + DurationMs = $bin.DurationMs + EstimatedDurationMs = [long] [Math]::Ceiling($bin.DurationMs / $DurationParallelism) + Oversized = $bin.Oversized + Tests = $sortedNames.ToArray() + }) + } + + return [pscustomobject] @{ + DurationMode = $durationMode + KnownTestCount = $knownDurations.Count + EstimatedTestCount = $uniqueNames.Count - $knownDurations.Count + MissingTestDurationMs = $missingDuration + TargetLoadMs = $targetLoadMs + TargetMinutes = $TargetMinutes + DurationParallelism = $DurationParallelism + TotalDurationMs = ($tests | Measure-Object -Property DurationMs -Sum).Sum + WorkItems = $workItems.ToArray() + } +} + +function Write-NUnitRunSettings +{ + param ( + [Parameter(Mandatory)] + [string[]] $TestNames, + + [Parameter(Mandatory)] + [string] $Path + ) + + $settings = [Xml.XmlWriterSettings]::new() + $settings.Indent = $true + $settings.Encoding = [Text.UTF8Encoding]::new($false) + $writer = [Xml.XmlWriter]::Create($Path, $settings) + try { + $filters = foreach ($name in $TestNames) { + $escaped = $name.Replace('\', '\\').Replace("'", "\'") + "test == '$escaped'" + } + $writer.WriteStartElement('RunSettings') + $writer.WriteStartElement('NUnit') + $writer.WriteElementString('Where', ($filters -join ' or ')) + $writer.WriteEndElement() + $writer.WriteEndElement() + } finally { + $writer.Dispose() + } +} + +function Write-HostBuildTestCommand +{ + param ( + [Parameter(Mandatory)] + [ValidateSet('windows', 'linux')] + [string] $Platform, + + [Parameter(Mandatory)] + [string] $TestAssemblyRelativePath, + + [Parameter(Mandatory)] + [string] $NUnitWorkers, + + [Parameter(Mandatory)] + [string] $Configuration, + + [Parameter(Mandatory)] + [string] $Path + ) + + if ($Platform -eq 'windows') { + $assemblyPath = $TestAssemblyRelativePath.Replace('/', '\') + $command = @' +@echo off +setlocal +if "%HELIX_CORRELATION_PAYLOAD%"=="" exit /b 2 +if "%HELIX_WORKITEM_ROOT%"=="" exit /b 2 +if "%HELIX_WORKITEM_UPLOAD_ROOT%"=="" exit /b 2 +set "REPO=%HELIX_CORRELATION_PAYLOAD%\repo" +set "DOTNET_ROOT=%REPO%\bin\__CONFIGURATION__\dotnet" +set "ANDROID_HOME=%HELIX_CORRELATION_PAYLOAD%\android-toolchain\sdk" +set "ANDROID_SDK_ROOT=%ANDROID_HOME%" +set "JAVA_HOME=%HELIX_CORRELATION_PAYLOAD%\jdk" +set "NUGET_PACKAGES=%HELIX_CORRELATION_PAYLOAD%\nuget-packages" +set "GRADLE_USER_HOME=%HELIX_CORRELATION_PAYLOAD%\gradle" +set "DOTNET_CLI_HOME=%HELIX_WORKITEM_ROOT%\dotnet-home" +set "TEMP=%HELIX_WORKITEM_ROOT%\temp" +set "TMP=%TEMP%" +set "BUILD_STAGINGDIRECTORY=%HELIX_WORKITEM_UPLOAD_ROOT%" +set "CONFIGURATION=__CONFIGURATION__" +set "PATH=%HELIX_CORRELATION_PAYLOAD%\dotnet-tools;%DOTNET_ROOT%;%ANDROID_HOME%\platform-tools;%JAVA_HOME%\bin;%PATH%" +set "DOTNET_CLI_TELEMETRY_OPTOUT=1" +set "DOTNET_NOLOGO=1" +set "DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1" +set "DOTNET_GENERATE_ASPNET_CERTIFICATE=false" +set "DOTNET_CLI_WORKLOAD_UPDATE_NOTIFY_DISABLE=true" +set "DOTNET_SDK_VULNERABILITY_CHECK_DISABLE=true" +set "DOTNET_SYSTEM_NET_SECURITY_NOREVOCATIONCHECKBYDEFAULT=true" +set "NuGetAudit=false" +set "DOTNET_DbgEnableMiniDump=1" +set "DOTNET_DbgMiniDumpType=2" +set "DOTNET_DbgMiniDumpName=%HELIX_WORKITEM_UPLOAD_ROOT%\dumps\%%e.%%p.dmp" +if not exist "%DOTNET_ROOT%\dotnet.exe" exit /b 3 +if not exist "%REPO%\__TEST_ASSEMBLY__" exit /b 3 +mkdir "%HELIX_WORKITEM_UPLOAD_ROOT%\dumps" 2>nul +mkdir "%DOTNET_CLI_HOME%" 2>nul +mkdir "%TEMP%" 2>nul +copy /y "%~dp0slice.runsettings" "%HELIX_WORKITEM_UPLOAD_ROOT%\slice.runsettings" >nul +copy /y "%~dp0work-item.json" "%HELIX_WORKITEM_UPLOAD_ROOT%\work-item.json" >nul +pushd "%REPO%" +"%DOTNET_ROOT%\dotnet.exe" test "%REPO%\__TEST_ASSEMBLY__" --settings "%~dp0slice.runsettings" --logger "trx;LogFileName=results.trx" --results-directory "%HELIX_WORKITEM_UPLOAD_ROOT%" -- NUnit.NumberOfTestWorkers=__NUNIT_WORKERS__ > "%HELIX_WORKITEM_UPLOAD_ROOT%\console.log" 2>&1 +set "testExitCode=%ERRORLEVEL%" +type "%HELIX_WORKITEM_UPLOAD_ROOT%\console.log" +"%DOTNET_ROOT%\dotnet.exe" build-server shutdown >nul 2>&1 +popd +exit /b %testExitCode% +'@ + $command = $command.Replace('__TEST_ASSEMBLY__', $assemblyPath) + $command = $command.Replace('__NUNIT_WORKERS__', $NUnitWorkers) + $command = $command.Replace('__CONFIGURATION__', $Configuration) + [IO.File]::WriteAllText($Path, $command, [Text.Encoding]::ASCII) + } else { + $assemblyPath = $TestAssemblyRelativePath.Replace('\', '/') + $command = @' +#!/usr/bin/env bash +set -euo pipefail +: "${HELIX_CORRELATION_PAYLOAD:?}" +: "${HELIX_WORKITEM_ROOT:?}" +: "${HELIX_WORKITEM_UPLOAD_ROOT:?}" +REPO="$HELIX_CORRELATION_PAYLOAD/repo" +export DOTNET_ROOT="$REPO/bin/__CONFIGURATION__/dotnet" +export ANDROID_HOME="$HELIX_CORRELATION_PAYLOAD/android-toolchain/sdk" +export ANDROID_SDK_ROOT="$ANDROID_HOME" +export JAVA_HOME="$HELIX_CORRELATION_PAYLOAD/jdk" +export NUGET_PACKAGES="$HELIX_CORRELATION_PAYLOAD/nuget-packages" +export GRADLE_USER_HOME="$HELIX_CORRELATION_PAYLOAD/gradle" +export DOTNET_CLI_HOME="$HELIX_WORKITEM_ROOT/dotnet-home" +export TMPDIR="$HELIX_WORKITEM_ROOT/temp" +export BUILD_STAGINGDIRECTORY="$HELIX_WORKITEM_UPLOAD_ROOT" +export CONFIGURATION=__CONFIGURATION__ +export PATH="$HELIX_CORRELATION_PAYLOAD/dotnet-tools:$DOTNET_ROOT:$ANDROID_HOME/platform-tools:$JAVA_HOME/bin:$PATH" +export DOTNET_CLI_TELEMETRY_OPTOUT=1 +export DOTNET_NOLOGO=1 +export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 +export DOTNET_GENERATE_ASPNET_CERTIFICATE=false +export DOTNET_CLI_WORKLOAD_UPDATE_NOTIFY_DISABLE=true +export DOTNET_SDK_VULNERABILITY_CHECK_DISABLE=true +export DOTNET_SYSTEM_NET_SECURITY_NOREVOCATIONCHECKBYDEFAULT=true +export NuGetAudit=false +export DOTNET_DbgEnableMiniDump=1 +export DOTNET_DbgMiniDumpType=2 +export DOTNET_DbgMiniDumpName="$HELIX_WORKITEM_UPLOAD_ROOT/dumps/%e.%p.dmp" +test -x "$DOTNET_ROOT/dotnet" +test -f "$REPO/__TEST_ASSEMBLY__" +mkdir -p "$HELIX_WORKITEM_UPLOAD_ROOT/dumps" "$DOTNET_CLI_HOME" "$TMPDIR" +cp "$HELIX_WORKITEM_ROOT/slice.runsettings" "$HELIX_WORKITEM_UPLOAD_ROOT/slice.runsettings" +cp "$HELIX_WORKITEM_ROOT/work-item.json" "$HELIX_WORKITEM_UPLOAD_ROOT/work-item.json" +cd "$REPO" +set +e +"$DOTNET_ROOT/dotnet" test "$REPO/__TEST_ASSEMBLY__" --settings "$HELIX_WORKITEM_ROOT/slice.runsettings" --logger "trx;LogFileName=results.trx" --results-directory "$HELIX_WORKITEM_UPLOAD_ROOT" -- NUnit.NumberOfTestWorkers=__NUNIT_WORKERS__ 2>&1 | tee "$HELIX_WORKITEM_UPLOAD_ROOT/console.log" +test_exit=${PIPESTATUS[0]} +set -e +"$DOTNET_ROOT/dotnet" build-server shutdown >/dev/null 2>&1 || true +exit "$test_exit" +'@ + $command = $command.Replace('__TEST_ASSEMBLY__', $assemblyPath) + $command = $command.Replace('__NUNIT_WORKERS__', $NUnitWorkers) + $command = $command.Replace('__CONFIGURATION__', $Configuration) + [IO.File]::WriteAllText($Path, $command, [Text.UTF8Encoding]::new($false)) + } +} + +function Write-HostBuildTestWorkItemPayloads +{ + param ( + [Parameter(Mandatory)] + [ValidateSet('windows', 'linux')] + [string] $Platform, + + [Parameter(Mandatory)] + [object] $Plan, + + [Parameter(Mandatory)] + [string] $WorkItemsDirectory, + + [Parameter(Mandatory)] + [string] $WorkItemsPropsPath, + + [Parameter(Mandatory)] + [string] $TestAssemblyRelativePath, + + [Parameter(Mandatory)] + [string] $NUnitWorkers, + + [Parameter(Mandatory)] + [string] $Configuration + ) + + Remove-Item -LiteralPath $WorkItemsDirectory -Recurse -Force -ErrorAction Ignore + New-Item -ItemType Directory -Force -Path $WorkItemsDirectory | Out-Null + + $propsSettings = [Xml.XmlWriterSettings]::new() + $propsSettings.Indent = $true + $propsSettings.Encoding = [Text.UTF8Encoding]::new($false) + $propsWriter = [Xml.XmlWriter]::Create($WorkItemsPropsPath, $propsSettings) + try { + $propsWriter.WriteStartElement('Project') + $propsWriter.WriteStartElement('ItemGroup') + + foreach ($workItem in $Plan.WorkItems) { + $name = 'host-build-tests-{0}-{1:d3}' -f $Platform, $workItem.Id + $payloadDirectory = Join-Path $WorkItemsDirectory $name + New-Item -ItemType Directory -Force -Path $payloadDirectory | Out-Null + + Write-NUnitRunSettings -TestNames $workItem.Tests -Path (Join-Path $payloadDirectory 'slice.runsettings') + $commandName = if ($Platform -eq 'windows') { 'run-host-tests.cmd' } else { 'run-host-tests.sh' } + $helixCommand = if ($Platform -eq 'windows') { $commandName } else { "bash $commandName" } + Write-HostBuildTestCommand ` + -Platform $Platform ` + -TestAssemblyRelativePath $TestAssemblyRelativePath ` + -NUnitWorkers $NUnitWorkers ` + -Configuration $Configuration ` + -Path (Join-Path $payloadDirectory $commandName) + + [pscustomobject] @{ + name = $name + durationMode = $Plan.DurationMode + estimatedDurationMs = $workItem.EstimatedDurationMs + durationLoadMs = $workItem.DurationMs + oversized = $workItem.Oversized + targetMinutes = $Plan.TargetMinutes + durationParallelism = $Plan.DurationParallelism + testCount = $workItem.Tests.Count + tests = $workItem.Tests + } | ConvertTo-Json -Depth 5 | + Set-Content -LiteralPath (Join-Path $payloadDirectory 'work-item.json') -Encoding utf8NoBOM + + $propsWriter.WriteStartElement('_HostBuildTestHelixWorkItem') + $propsWriter.WriteAttributeString('Include', $name) + $propsWriter.WriteElementString('PayloadDirectory', $payloadDirectory) + $propsWriter.WriteElementString('Command', $helixCommand) + $propsWriter.WriteElementString('EstimatedDurationMs', [string] $workItem.EstimatedDurationMs) + $propsWriter.WriteElementString('TestCount', [string] $workItem.Tests.Count) + $propsWriter.WriteEndElement() + } + + $propsWriter.WriteEndElement() + $propsWriter.WriteEndElement() + } finally { + $propsWriter.Dispose() + } +} + +Export-ModuleMember -Function ` + Get-NUnitRunSettingsTestNames, ` + Get-HostBuildTestTimingHistory, ` + New-DurationBalancedWorkItems, ` + Write-NUnitRunSettings, ` + Write-HostBuildTestWorkItemPayloads diff --git a/build-tools/automation/host-build-tests-helix/README.md b/build-tools/automation/host-build-tests-helix/README.md new file mode 100644 index 00000000000..905b40f0cc6 --- /dev/null +++ b/build-tools/automation/host-build-tests-helix/README.md @@ -0,0 +1,24 @@ +# Host build tests in Helix prototype + +This guarded prototype stages the already-prepared .NET SDK/workload, Android SDK, +JDK, NuGet/Gradle caches, selected repository inputs, and test assemblies into one +Helix correlation payload per host OS. Work item payloads contain only a runsettings +filter, a small command script, and generation metadata. To control payload size, the +source copy omits `src/Mono.Android`; host build tests consume that product through the +prepared workload packs instead. + +`prepare-host-build-test-helix-payload.ps1` discovers NUnit tests with the repository's +`dotnet-test-slicer`, reads its `balance.xml` format, and uses deterministic best-fit +decreasing bin packing. The target is `15` estimated wall-clock minutes by default. +`durationParallelism` translates summed per-test elapsed time into expected wall time +while preserving the current NUnit worker setting. The public prototype defaults this +factor to `2.5`, uses `60` seconds per test for the explicit first-run fallback, and +allows `45` minutes before Helix terminates a work item. + +If no timing history is provided, generation explicitly reports `count-fallback` and +uses the configured fallback duration per test. It does not label count-only slicing +as duration-balanced. Each completed run publishes a new `balance.xml`; pass that +run's build ID through `hostBuildTestsHelixTimingBuildId` on the next queued build. + +The public pipeline keeps the existing Azure Pipelines jobs and adds this path only +when `enableHostBuildTestsHelixPrototype` is `true`. diff --git a/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 b/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 new file mode 100644 index 00000000000..23c779f4764 --- /dev/null +++ b/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 @@ -0,0 +1,256 @@ +[CmdletBinding()] +param ( + [Parameter(Mandatory)] + [ValidateSet('windows', 'linux')] + [string] $Platform, + + [Parameter(Mandatory)] + [string] $RepositoryRoot, + + [Parameter(Mandatory)] + [string] $CorrelationPayloadDirectory, + + [Parameter(Mandatory)] + [string] $WorkItemsDirectory, + + [Parameter(Mandatory)] + [string] $WorkItemsPropsPath, + + [Parameter(Mandatory)] + [string] $ResultsDirectory, + + [Parameter(Mandatory)] + [string] $TestAssemblyRelativePath, + + [Parameter(Mandatory)] + [string] $TestSlicerPath, + + [string] $TestFilter = '', + + [string] $BalanceFile = '', + + [ValidateRange(1, 1440)] + [int] $TargetMinutes = 15, + + [ValidateRange(0.1, 128)] + [double] $DurationParallelism = 2.5, + + [ValidateRange(1, 86400)] + [int] $FallbackTestDurationSeconds = 60, + + [string] $NUnitWorkers = '-1', + + [string] $Configuration = 'Release', + + [Parameter(Mandatory)] + [string] $AndroidHome, + + [Parameter(Mandatory)] + [string] $JavaHome, + + [string] $NuGetPackages = '', + + [string] $DotNetToolsDirectory = '', + + [string] $GradleUserHome = '' +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +Import-Module (Join-Path $PSScriptRoot 'HostBuildTestsHelix.psm1') -Force + +function ConvertTo-NativeRelativePath +{ + param ( + [Parameter(Mandatory)] + [string] $Path + ) + + $separator = [string] [IO.Path]::DirectorySeparatorChar + return $Path.Replace('\', $separator).Replace('/', $separator) +} + +function Copy-PayloadDirectory +{ + param ( + [Parameter(Mandatory)] + [string] $Source, + + [Parameter(Mandatory)] + [string] $Destination, + + [switch] $Optional + ) + + if (-not (Test-Path -LiteralPath $Source -PathType Container)) { + if ($Optional) { + Write-Warning "Optional payload directory '$Source' does not exist." + return + } + throw "Required payload directory '$Source' does not exist." + } + + Write-Host "Staging payload directory '$Source'." + New-Item -ItemType Directory -Force -Path $Destination | Out-Null + if ($IsWindows) { + & robocopy $Source $Destination /E /COPY:DAT /DCOPY:DAT /R:2 /W:1 /NFL /NDL /NJH /NJS /NP + if ($LASTEXITCODE -gt 7) { + throw "robocopy failed with exit code $LASTEXITCODE while copying '$Source'." + } + } else { + & cp -a "$Source/." "$Destination/" + if ($LASTEXITCODE -ne 0) { + throw "cp failed with exit code $LASTEXITCODE while copying '$Source'." + } + } +} + +$repositoryRootFullPath = [IO.Path]::GetFullPath($RepositoryRoot) +$correlationPayloadFullPath = [IO.Path]::GetFullPath($CorrelationPayloadDirectory) +$workItemsFullPath = [IO.Path]::GetFullPath($WorkItemsDirectory) +$workItemsPropsFullPath = [IO.Path]::GetFullPath($WorkItemsPropsPath) +$resultsFullPath = [IO.Path]::GetFullPath($ResultsDirectory) + +Remove-Item -LiteralPath $correlationPayloadFullPath -Recurse -Force -ErrorAction Ignore +Remove-Item -LiteralPath $workItemsFullPath -Recurse -Force -ErrorAction Ignore +New-Item -ItemType Directory -Force -Path $correlationPayloadFullPath, $workItemsFullPath, $resultsFullPath | Out-Null + +$payloadRepository = Join-Path $correlationPayloadFullPath 'repo' +New-Item -ItemType Directory -Force -Path $payloadRepository | Out-Null +Get-ChildItem -LiteralPath $repositoryRootFullPath -File | Copy-Item -Destination $payloadRepository + +$payloadDirectories = @( + @("bin\$Configuration", "bin\$Configuration", $false), + @("bin\Test$Configuration", "bin\Test$Configuration", $false), + @('build-tools', 'build-tools', $false), + @('eng', 'eng', $false), + @('external', 'external', $false), + @('.github', '.github', $true), + @('samples', 'samples', $true), + @('src-ThirdParty', 'src-ThirdParty', $true), + @('tests', 'tests', $false), + @('tools', 'tools', $true) +) +foreach ($directory in $payloadDirectories) { + Copy-PayloadDirectory ` + -Source (Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath $directory[0])) ` + -Destination (Join-Path $payloadRepository (ConvertTo-NativeRelativePath $directory[1])) ` + -Optional:([bool] $directory[2]) +} + +$sourceRoot = Join-Path $repositoryRootFullPath 'src' +foreach ($sourceDirectory in Get-ChildItem -LiteralPath $sourceRoot -Directory | Where-Object Name -ne 'Mono.Android') { + Copy-PayloadDirectory ` + -Source $sourceDirectory.FullName ` + -Destination (Join-Path (Join-Path $payloadRepository 'src') $sourceDirectory.Name) +} + +Copy-PayloadDirectory -Source $AndroidHome -Destination (Join-Path $correlationPayloadFullPath (ConvertTo-NativeRelativePath 'android-toolchain/sdk')) +Copy-PayloadDirectory -Source $JavaHome -Destination (Join-Path $correlationPayloadFullPath 'jdk') + +if ([string]::IsNullOrWhiteSpace($NuGetPackages)) { + $NuGetPackages = Join-Path ([Environment]::GetFolderPath('UserProfile')) (ConvertTo-NativeRelativePath '.nuget/packages') +} +Copy-PayloadDirectory -Source $NuGetPackages -Destination (Join-Path $correlationPayloadFullPath 'nuget-packages') + +if ([string]::IsNullOrWhiteSpace($GradleUserHome)) { + $GradleUserHome = Join-Path ([Environment]::GetFolderPath('UserProfile')) '.gradle' +} +foreach ($gradleDirectory in @('caches', 'wrapper')) { + Copy-PayloadDirectory ` + -Source (Join-Path $GradleUserHome $gradleDirectory) ` + -Destination (Join-Path $correlationPayloadFullPath (ConvertTo-NativeRelativePath "gradle/$gradleDirectory")) ` + -Optional +} + +if (-not [string]::IsNullOrWhiteSpace($DotNetToolsDirectory)) { + $payloadTools = Join-Path $correlationPayloadFullPath 'dotnet-tools' + New-Item -ItemType Directory -Force -Path $payloadTools | Out-Null + $apkDiffShim = if ($Platform -eq 'windows') { 'apkdiff.exe' } else { 'apkdiff' } + $apkDiffShimPath = Join-Path $DotNetToolsDirectory $apkDiffShim + if (Test-Path -LiteralPath $apkDiffShimPath -PathType Leaf) { + Copy-Item -LiteralPath $apkDiffShimPath -Destination $payloadTools + } + Copy-PayloadDirectory ` + -Source (Join-Path $DotNetToolsDirectory (ConvertTo-NativeRelativePath '.store/apkdiff')) ` + -Destination (Join-Path $payloadTools (ConvertTo-NativeRelativePath '.store/apkdiff')) ` + -Optional +} + +$stagedTestAssembly = Join-Path $payloadRepository (ConvertTo-NativeRelativePath $TestAssemblyRelativePath) +if (-not (Test-Path -LiteralPath $stagedTestAssembly -PathType Leaf)) { + throw "Test assembly '$stagedTestAssembly' was not staged." +} +if (-not (Test-Path -LiteralPath $TestSlicerPath -PathType Leaf)) { + throw "dotnet-test-slicer '$TestSlicerPath' does not exist." +} + +$discoveryRunSettings = Join-Path $workItemsFullPath 'all-tests.runsettings' +$sliceArguments = @( + 'slice', + "--test-assembly=$stagedTestAssembly", + '--slice-number=1', + '--total-slices=1', + "--outfile=$discoveryRunSettings" +) +if (-not [string]::IsNullOrWhiteSpace($TestFilter)) { + $sliceArguments += "--test-filter=$TestFilter" +} + +& $TestSlicerPath @sliceArguments +if ($LASTEXITCODE -ne 0) { + throw "dotnet-test-slicer discovery failed with exit code $LASTEXITCODE." +} + +$testNames = Get-NUnitRunSettingsTestNames -Path $discoveryRunSettings +$timingHistory = Get-HostBuildTestTimingHistory -Path $BalanceFile +$plan = New-DurationBalancedWorkItems ` + -TestNames $testNames ` + -TimingHistory $timingHistory ` + -TargetMinutes $TargetMinutes ` + -DurationParallelism $DurationParallelism ` + -FallbackTestDurationSeconds $FallbackTestDurationSeconds + +Write-HostBuildTestWorkItemPayloads ` + -Platform $Platform ` + -Plan $plan ` + -WorkItemsDirectory $workItemsFullPath ` + -WorkItemsPropsPath $workItemsPropsFullPath ` + -TestAssemblyRelativePath $TestAssemblyRelativePath ` + -NUnitWorkers $NUnitWorkers ` + -Configuration $Configuration + +$summaryPath = Join-Path $resultsFullPath 'work-item-generation.json' +[pscustomobject] @{ + platform = $Platform + testFilter = $TestFilter + durationMode = $plan.DurationMode + targetMinutes = $TargetMinutes + durationParallelism = $DurationParallelism + fallbackTestDurationSeconds = $FallbackTestDurationSeconds + testCount = $testNames.Count + knownTestCount = $plan.KnownTestCount + estimatedTestCount = $plan.EstimatedTestCount + workItemCount = $plan.WorkItems.Count + totalDurationLoadMs = $plan.TotalDurationMs + workItems = @($plan.WorkItems | ForEach-Object { + [pscustomobject] @{ + id = $_.Id + testCount = $_.Tests.Count + durationLoadMs = $_.DurationMs + estimatedDurationMs = $_.EstimatedDurationMs + oversized = $_.Oversized + } + }) +} | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath $summaryPath -Encoding utf8NoBOM + +Write-Host "Generated $($plan.WorkItems.Count) $Platform Helix work items for $($testNames.Count) tests." +Write-Host "Duration mode: $($plan.DurationMode); known timings: $($plan.KnownTestCount); estimated timings: $($plan.EstimatedTestCount)." +Write-Host "Target: $TargetMinutes minute(s); duration parallelism: $DurationParallelism." +if ($plan.DurationMode -eq 'count-fallback') { + Write-Warning "No matching timing history was available. Work items use an explicit count fallback of $FallbackTestDurationSeconds second(s) per test." +} + +Write-Host "##vso[task.setvariable variable=HostBuildTestsHelixCorrelationPayload]$correlationPayloadFullPath" +Write-Host "##vso[task.setvariable variable=HostBuildTestsHelixWorkItemsProps]$workItemsPropsFullPath" +Write-Host "##vso[task.setvariable variable=HostBuildTestsHelixWorkItemCount]$($plan.WorkItems.Count)" diff --git a/build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml b/build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml new file mode 100644 index 00000000000..6132358d98e --- /dev/null +++ b/build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml @@ -0,0 +1,176 @@ +parameters: + platform: '' + helixQueue: '' + testFilter: '' + testRunTitle: '' + timingBuildId: '' + timingArtifactName: '' + targetMinutes: 15 + durationParallelism: 2.5 + fallbackTestDurationSeconds: 60 + workItemTimeout: '00:45:00' + nunitWorkers: '-1' + +steps: +- template: /build-tools/automation/yaml-templates/setup-test-environment-public.yaml + parameters: + useAgentJdkPath: false + installTestSlicer: true + use1ESTemplate: false + +- template: /build-tools/automation/yaml-templates/cache-gradle.yaml + parameters: + xaSourcePath: $(System.DefaultWorkingDirectory) + +- task: DownloadPipelineArtifact@2 + displayName: Download Xamarin.Android.Build.Tests assets + inputs: + artifactName: $(TestAssembliesArtifactName) + downloadPath: $(System.DefaultWorkingDirectory)/bin/Test$(XA.Build.Configuration) + +- ${{ if ne(parameters.timingBuildId, '') }}: + - task: DownloadPipelineArtifact@2 + displayName: Download prior ${{ parameters.platform }} host-test timings + inputs: + buildType: specific + project: $(System.TeamProject) + definition: $(System.DefinitionId) + buildVersionToDownload: specific + pipelineId: ${{ parameters.timingBuildId }} + artifactName: ${{ parameters.timingArtifactName }} + targetPath: $(Pipeline.Workspace)/host-build-test-history-${{ parameters.platform }} + +- pwsh: | + $ErrorActionPreference = 'Stop' + $platform = '${{ parameters.platform }}' + $historyRoot = '$(Pipeline.Workspace)/host-build-test-history-${{ parameters.platform }}' + $balanceFile = '' + if ('${{ parameters.timingBuildId }}') { + $balanceFile = Get-ChildItem -LiteralPath $historyRoot -Recurse -Filter '*.xml' | + Select-Object -First 1 -ExpandProperty FullName + if (-not $balanceFile) { + throw "Build '${{ parameters.timingBuildId }}' did not provide timing XML in '$historyRoot'." + } + } + + $testSlicer = Join-Path '$(Agent.ToolsDirectory)' $(if ($platform -eq 'windows') { 'dotnet-test-slicer.exe' } else { 'dotnet-test-slicer' }) + $stagingRoot = '$(Build.StagingDirectory)/host-build-tests-helix/${{ parameters.platform }}' + $resultsDirectory = Join-Path $stagingRoot 'results' + $correlationPayload = Join-Path $stagingRoot 'correlation' + $workItemsDirectory = Join-Path $stagingRoot 'work-items' + $workItemsProps = Join-Path $stagingRoot 'work-items.props' + $adbName = if ($platform -eq 'windows') { 'adb.exe' } else { 'adb' } + $androidHome = @( + $env:ANDROID_HOME + $env:ANDROID_SDK_ROOT + $env:AndroidSdkDirectory + $(if ($env:USERPROFILE) { Join-Path $env:USERPROFILE 'android-toolchain\sdk' }) + $(if ($env:HOME) { Join-Path $env:HOME 'android-toolchain/sdk' }) + ) | Where-Object { $_ -and (Test-Path (Join-Path $_ "platform-tools/$adbName")) } | Select-Object -First 1 + if (-not $androidHome) { + throw 'Could not locate the prepared Android SDK.' + } + + & '$(System.DefaultWorkingDirectory)/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1' ` + -Platform $platform ` + -RepositoryRoot '$(System.DefaultWorkingDirectory)' ` + -CorrelationPayloadDirectory $correlationPayload ` + -WorkItemsDirectory $workItemsDirectory ` + -WorkItemsPropsPath $workItemsProps ` + -ResultsDirectory $resultsDirectory ` + -TestAssemblyRelativePath 'bin/Test$(XA.Build.Configuration)/$(DotNetStableTargetFramework)/Xamarin.Android.Build.Tests.dll' ` + -TestSlicerPath $testSlicer ` + -TestFilter '${{ parameters.testFilter }}' ` + -BalanceFile $balanceFile ` + -TargetMinutes ${{ parameters.targetMinutes }} ` + -DurationParallelism ${{ parameters.durationParallelism }} ` + -FallbackTestDurationSeconds ${{ parameters.fallbackTestDurationSeconds }} ` + -NUnitWorkers '${{ parameters.nunitWorkers }}' ` + -Configuration '$(XA.Build.Configuration)' ` + -AndroidHome $androidHome ` + -JavaHome $env:JAVA_HOME ` + -NuGetPackages $env:NUGET_PACKAGES ` + -DotNetToolsDirectory '$(Agent.ToolsDirectory)' ` + -GradleUserHome (Join-Path ([Environment]::GetFolderPath('UserProfile')) '.gradle') + displayName: Prepare duration-balanced ${{ parameters.platform }} Helix work items + +- template: /build-tools/automation/yaml-templates/run-dotnet-preview.yaml + parameters: + command: msbuild + project: $(System.DefaultWorkingDirectory)/build-tools/automation/HostBuildTestsHelix.proj + arguments: >- + /restore + /t:Test + /p:TreatWarningsAsErrors=false + /bl:$(Build.StagingDirectory)/host-build-tests-helix/${{ parameters.platform }}/results/send-to-helix.binlog + displayName: Submit ${{ parameters.platform }} host build tests to Helix + continueOnError: false + taskTimeoutInMinutes: 240 + env: + HostBuildTestsHelixWorkItemsProps: $(HostBuildTestsHelixWorkItemsProps) + CorrelationPayloadDirectory: $(HostBuildTestsHelixCorrelationPayload) + HostBuildTestsHelixWorkItemTimeout: ${{ parameters.workItemTimeout }} + HelixResultsDestinationDir: $(Build.StagingDirectory)/host-build-tests-helix/${{ parameters.platform }}/results + HelixSource: pr/dotnet/android + HelixType: tests/android/host-build-tests/${{ parameters.platform }}/ + HelixBuild: $(Build.BuildNumber) + HelixConfiguration: ${{ parameters.platform }} + HelixTargetQueueForItem: ${{ parameters.helixQueue }} + HelixTestRunName: ${{ parameters.testRunTitle }} + HelixAccessToken: '' + WaitForWorkItemCompletion: 'true' + Creator: dotnet-android + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + +- task: PublishTestResults@2 + displayName: Publish ${{ parameters.platform }} Helix TRX results + condition: always() + inputs: + testResultsFormat: VSTest + testResultsFiles: '**/*.trx' + searchFolder: $(Build.StagingDirectory)/host-build-tests-helix/${{ parameters.platform }}/results + mergeTestResults: true + testRunTitle: ${{ parameters.testRunTitle }} + continueOnError: true + +- pwsh: | + $ErrorActionPreference = 'Stop' + $resultsDirectory = '$(Build.StagingDirectory)/host-build-tests-helix/${{ parameters.platform }}/results' + $timingDirectory = '$(Build.StagingDirectory)/host-build-test-timings-${{ parameters.platform }}' + $trxDirectory = Join-Path $timingDirectory 'trx' + New-Item -ItemType Directory -Force -Path $trxDirectory | Out-Null + + $trxFiles = @(Get-ChildItem -LiteralPath $resultsDirectory -Recurse -Filter '*.trx' -ErrorAction Ignore) + if ($trxFiles.Count -eq 0) { + Write-Warning "No TRX files were available to produce next-run timings." + Write-Host '##vso[task.setvariable variable=HostBuildTestsHelixHasTimings]false' + exit 0 + } + + for ($index = 0; $index -lt $trxFiles.Count; $index++) { + Copy-Item -LiteralPath $trxFiles[$index].FullName -Destination (Join-Path $trxDirectory ('{0:d4}.trx' -f $index)) + } + + $testSlicer = Join-Path '$(Agent.ToolsDirectory)' $(if ('${{ parameters.platform }}' -eq 'windows') { 'dotnet-test-slicer.exe' } else { 'dotnet-test-slicer' }) + & $testSlicer balance --input-directory=$trxDirectory --outfile=(Join-Path $timingDirectory 'balance.xml') + if ($LASTEXITCODE -ne 0) { + throw "dotnet-test-slicer balance failed with exit code $LASTEXITCODE." + } + Remove-Item -LiteralPath $trxDirectory -Recurse -Force + Write-Host '##vso[task.setvariable variable=HostBuildTestsHelixHasTimings]true' + displayName: Produce next-run ${{ parameters.platform }} timings + condition: always() + +- task: PublishPipelineArtifact@1 + displayName: Publish ${{ parameters.platform }} host-test timings + condition: and(always(), eq(variables['HostBuildTestsHelixHasTimings'], 'true')) + inputs: + artifactName: ${{ parameters.timingArtifactName }} + targetPath: $(Build.StagingDirectory)/host-build-test-timings-${{ parameters.platform }} + +- task: PublishPipelineArtifact@1 + displayName: Publish ${{ parameters.platform }} Helix diagnostics + condition: always() + inputs: + artifactName: Host Build Test Helix Results - ${{ parameters.platform }} + targetPath: $(Build.StagingDirectory)/host-build-tests-helix/${{ parameters.platform }}/results From 7b59cb7e7ad1859b41d111670d16571db0ff4c6d Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Fri, 21 Aug 2026 15:45:51 -0700 Subject: [PATCH 02/23] AB#12334 Link Helix prototype tracking Document the Azure DevOps task that tracks the guarded host build test prototype. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- build-tools/automation/host-build-tests-helix/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build-tools/automation/host-build-tests-helix/README.md b/build-tools/automation/host-build-tests-helix/README.md index 905b40f0cc6..a4951251cd3 100644 --- a/build-tools/automation/host-build-tests-helix/README.md +++ b/build-tools/automation/host-build-tests-helix/README.md @@ -1,5 +1,7 @@ # Host build tests in Helix prototype +Tracking: [AB#12334](https://dev.azure.com/dnceng/internal/_workitems/edit/12334) + This guarded prototype stages the already-prepared .NET SDK/workload, Android SDK, JDK, NuGet/Gradle caches, selected repository inputs, and test assemblies into one Helix correlation payload per host OS. Work item payloads contain only a runsettings From c41b6aa050b1145daa26ddfa83d4a19dd38ba75f Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Fri, 21 Aug 2026 17:20:56 -0700 Subject: [PATCH 03/23] AB#12334 Fix Linux Helix test discovery Use the NUnit class filter for BuildTest, fail clearly when discovery returns the slicer dummy test, preserve single-test arrays, and report staged payload size. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../automation/azure-pipelines-public.yaml | 2 +- .../HostBuildTestsHelix.Tests.ps1 | 17 +++++++++++++++++ .../HostBuildTestsHelix.psm1 | 5 ++++- .../prepare-host-build-test-helix-payload.ps1 | 3 +++ 4 files changed, 25 insertions(+), 2 deletions(-) diff --git a/build-tools/automation/azure-pipelines-public.yaml b/build-tools/automation/azure-pipelines-public.yaml index 143ca87f891..946a5959a5c 100644 --- a/build-tools/automation/azure-pipelines-public.yaml +++ b/build-tools/automation/azure-pipelines-public.yaml @@ -232,7 +232,7 @@ stages: parameters: platform: linux helixQueue: Ubuntu.2404.Amd64.Open - testFilter: 'Name = BuildTest' + testFilter: 'class == Xamarin.Android.Build.Tests.BuildTest' testRunTitle: Xamarin.Android.Build.Tests - Linux BuildTest Helix Prototype timingBuildId: ${{ parameters.hostBuildTestsHelixTimingBuildId }} timingArtifactName: Host Build Test Timings - linux diff --git a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 index 3d6bb23cf40..2d87470c77e 100644 --- a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 +++ b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 @@ -80,6 +80,23 @@ try { $roundTrippedNames = Get-NUnitRunSettingsTestNames -Path $runSettingsPath Assert-Equal ($specialNames -join '|') ($roundTrippedNames -join '|') 'Runsettings names should round trip.' + $singleRunSettingsPath = Join-Path $testRoot 'single.runsettings' + Write-NUnitRunSettings -TestNames @('OnlyTest') -Path $singleRunSettingsPath + $singleTest = Get-NUnitRunSettingsTestNames -Path $singleRunSettingsPath + Assert-Equal 1 $singleTest.Count 'A one-test runsettings file should remain an array.' + Assert-Equal 'OnlyTest' $singleTest[0] 'The single test should round trip.' + + $emptyRunSettingsPath = Join-Path $testRoot 'empty.runsettings' + Set-Content -LiteralPath $emptyRunSettingsPath -Value "test == 'dotnet-slicer-dummy-test-name'" + try { + Get-NUnitRunSettingsTestNames -Path $emptyRunSettingsPath + throw 'The slicer dummy test should have caused an error.' + } catch { + if ($_.Exception.Message -notlike '*matched no tests*') { + throw + } + } + $secondFallbackPlan = New-DurationBalancedWorkItems ` -TestNames @('F', 'E', 'D', 'C', 'B', 'A') ` -TimingHistory $emptyHistory ` diff --git a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 index f51626c48b9..ecfe96052bf 100644 --- a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 +++ b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 @@ -51,8 +51,11 @@ function Get-NUnitRunSettingsTestNames if ($names.Count -eq 0) { throw "No tests were found in '$Path'." } + if ($names.Count -eq 1 -and $names[0] -eq 'dotnet-slicer-dummy-test-name') { + throw "The test filter matched no tests in '$Path'." + } - return $names.ToArray() + return ,$names.ToArray() } function Get-HostBuildTestTimingHistory diff --git a/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 b/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 index 23c779f4764..6d9fefb795c 100644 --- a/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 +++ b/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 @@ -177,6 +177,9 @@ if (-not [string]::IsNullOrWhiteSpace($DotNetToolsDirectory)) { -Optional } +$payloadBytes = (Get-ChildItem -LiteralPath $correlationPayloadFullPath -Recurse -File | Measure-Object -Property Length -Sum).Sum +Write-Host ('Staged correlation payload size: {0:N2} GiB.' -f ($payloadBytes / 1GB)) + $stagedTestAssembly = Join-Path $payloadRepository (ConvertTo-NativeRelativePath $TestAssemblyRelativePath) if (-not (Test-Path -LiteralPath $stagedTestAssembly -PathType Leaf)) { throw "Test assembly '$stagedTestAssembly' was not staged." From 112d25143fbd141895dc6daeeeb7d25930aa5949 Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Fri, 21 Aug 2026 17:33:46 -0700 Subject: [PATCH 04/23] AB#12334 Preserve Helix work item metadata Let MSBuild copy generated Command and PayloadDirectory metadata when converting private work items to HelixWorkItem entries, while adding the shared timeout. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- build-tools/automation/HostBuildTestsHelix.proj | 2 -- 1 file changed, 2 deletions(-) diff --git a/build-tools/automation/HostBuildTestsHelix.proj b/build-tools/automation/HostBuildTestsHelix.proj index db68078dd58..92dfcdeb5ce 100644 --- a/build-tools/automation/HostBuildTestsHelix.proj +++ b/build-tools/automation/HostBuildTestsHelix.proj @@ -25,8 +25,6 @@ - %(_HostBuildTestHelixWorkItem.PayloadDirectory) - %(_HostBuildTestHelixWorkItem.Command) $(HostBuildTestsHelixWorkItemTimeout) From 911c6b3cc5a8472b10d8f16158d78753985e8602 Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Fri, 21 Aug 2026 18:32:30 -0700 Subject: [PATCH 05/23] AB#12334 Partition Helix correlation payloads Split large toolchain trees into correlation payloads capped at 1 GiB, preserve their destinations through generated MSBuild items, and omit the duplicated NuGet cache to avoid the Helix SDK in-memory ZIP limit. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../automation/HostBuildTestsHelix.proj | 13 +- .../HostBuildTestsHelix.psm1 | 7 +- .../host-build-tests-helix/README.md | 12 +- .../prepare-host-build-test-helix-payload.ps1 | 230 +++++++++++++++--- .../run-host-build-tests-in-helix.yaml | 5 +- 5 files changed, 220 insertions(+), 47 deletions(-) diff --git a/build-tools/automation/HostBuildTestsHelix.proj b/build-tools/automation/HostBuildTestsHelix.proj index 92dfcdeb5ce..1545f015658 100644 --- a/build-tools/automation/HostBuildTestsHelix.proj +++ b/build-tools/automation/HostBuildTestsHelix.proj @@ -10,6 +10,7 @@ + @@ -18,9 +19,7 @@ - - %(Identity) - + @@ -32,13 +31,15 @@ - + + - diff --git a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 index ecfe96052bf..e1dafe4f35c 100644 --- a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 +++ b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 @@ -293,7 +293,7 @@ set "DOTNET_ROOT=%REPO%\bin\__CONFIGURATION__\dotnet" set "ANDROID_HOME=%HELIX_CORRELATION_PAYLOAD%\android-toolchain\sdk" set "ANDROID_SDK_ROOT=%ANDROID_HOME%" set "JAVA_HOME=%HELIX_CORRELATION_PAYLOAD%\jdk" -set "NUGET_PACKAGES=%HELIX_CORRELATION_PAYLOAD%\nuget-packages" +set "NUGET_PACKAGES=%HELIX_WORKITEM_ROOT%\nuget-packages" set "GRADLE_USER_HOME=%HELIX_CORRELATION_PAYLOAD%\gradle" set "DOTNET_CLI_HOME=%HELIX_WORKITEM_ROOT%\dotnet-home" set "TEMP=%HELIX_WORKITEM_ROOT%\temp" @@ -316,6 +316,7 @@ if not exist "%DOTNET_ROOT%\dotnet.exe" exit /b 3 if not exist "%REPO%\__TEST_ASSEMBLY__" exit /b 3 mkdir "%HELIX_WORKITEM_UPLOAD_ROOT%\dumps" 2>nul mkdir "%DOTNET_CLI_HOME%" 2>nul +mkdir "%NUGET_PACKAGES%" 2>nul mkdir "%TEMP%" 2>nul copy /y "%~dp0slice.runsettings" "%HELIX_WORKITEM_UPLOAD_ROOT%\slice.runsettings" >nul copy /y "%~dp0work-item.json" "%HELIX_WORKITEM_UPLOAD_ROOT%\work-item.json" >nul @@ -344,7 +345,7 @@ export DOTNET_ROOT="$REPO/bin/__CONFIGURATION__/dotnet" export ANDROID_HOME="$HELIX_CORRELATION_PAYLOAD/android-toolchain/sdk" export ANDROID_SDK_ROOT="$ANDROID_HOME" export JAVA_HOME="$HELIX_CORRELATION_PAYLOAD/jdk" -export NUGET_PACKAGES="$HELIX_CORRELATION_PAYLOAD/nuget-packages" +export NUGET_PACKAGES="$HELIX_WORKITEM_ROOT/nuget-packages" export GRADLE_USER_HOME="$HELIX_CORRELATION_PAYLOAD/gradle" export DOTNET_CLI_HOME="$HELIX_WORKITEM_ROOT/dotnet-home" export TMPDIR="$HELIX_WORKITEM_ROOT/temp" @@ -364,7 +365,7 @@ export DOTNET_DbgMiniDumpType=2 export DOTNET_DbgMiniDumpName="$HELIX_WORKITEM_UPLOAD_ROOT/dumps/%e.%p.dmp" test -x "$DOTNET_ROOT/dotnet" test -f "$REPO/__TEST_ASSEMBLY__" -mkdir -p "$HELIX_WORKITEM_UPLOAD_ROOT/dumps" "$DOTNET_CLI_HOME" "$TMPDIR" +mkdir -p "$HELIX_WORKITEM_UPLOAD_ROOT/dumps" "$DOTNET_CLI_HOME" "$NUGET_PACKAGES" "$TMPDIR" cp "$HELIX_WORKITEM_ROOT/slice.runsettings" "$HELIX_WORKITEM_UPLOAD_ROOT/slice.runsettings" cp "$HELIX_WORKITEM_ROOT/work-item.json" "$HELIX_WORKITEM_UPLOAD_ROOT/work-item.json" cd "$REPO" diff --git a/build-tools/automation/host-build-tests-helix/README.md b/build-tools/automation/host-build-tests-helix/README.md index a4951251cd3..4f35a3076b2 100644 --- a/build-tools/automation/host-build-tests-helix/README.md +++ b/build-tools/automation/host-build-tests-helix/README.md @@ -3,11 +3,13 @@ Tracking: [AB#12334](https://dev.azure.com/dnceng/internal/_workitems/edit/12334) This guarded prototype stages the already-prepared .NET SDK/workload, Android SDK, -JDK, NuGet/Gradle caches, selected repository inputs, and test assemblies into one -Helix correlation payload per host OS. Work item payloads contain only a runsettings -filter, a small command script, and generation metadata. To control payload size, the -source copy omits `src/Mono.Android`; host build tests consume that product through the -prepared workload packs instead. +JDK, selected Gradle caches, repository inputs, and test assemblies as shared Helix +correlation payloads. Individual payloads are capped at 1 GiB to avoid the Helix SDK's +in-memory ZIP limit. Work item payloads contain only a runsettings filter, a small +command script, and generation metadata. To control total payload size, the source +copy omits `src/Mono.Android` and the NuGet package cache; host build tests consume the +product through the prepared workload packs and use isolated per-work-item NuGet +caches. `prepare-host-build-test-helix-payload.ps1` discovers NUnit tests with the repository's `dotnet-test-slicer`, reads its `balance.xml` format, and uses deterministic best-fit diff --git a/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 b/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 index 6d9fefb795c..9dbcece4b34 100644 --- a/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 +++ b/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 @@ -10,6 +10,9 @@ param ( [Parameter(Mandatory)] [string] $CorrelationPayloadDirectory, + [Parameter(Mandatory)] + [string] $CorrelationPayloadsPropsPath, + [Parameter(Mandatory)] [string] $WorkItemsDirectory, @@ -48,8 +51,6 @@ param ( [Parameter(Mandatory)] [string] $JavaHome, - [string] $NuGetPackages = '', - [string] $DotNetToolsDirectory = '', [string] $GradleUserHome = '' @@ -105,8 +106,167 @@ function Copy-PayloadDirectory } } +function Get-DirectorySize +{ + param ( + [Parameter(Mandatory)] + [string] $Path + ) + + $size = 0L + foreach ($file in Get-ChildItem -LiteralPath $Path -Recurse -File) { + $size += $file.Length + } + return $size +} + +$correlationPayloads = [System.Collections.Generic.List[object]]::new() +$filePayloadIndex = 0 +$maximumPayloadBytes = 1GB + +function Add-CorrelationPayload +{ + param ( + [Parameter(Mandatory)] + [string] $Source, + + [Parameter(Mandatory)] + [string] $Destination, + + [Parameter(Mandatory)] + [long] $SizeBytes + ) + + $script:correlationPayloads.Add([pscustomobject] @{ + Source = [IO.Path]::GetFullPath($Source) + Destination = $Destination.Replace('\', '/').Trim('/') + SizeBytes = $SizeBytes + }) +} + +function Add-FileCorrelationPayloads +{ + param ( + [Parameter(Mandatory)] + [IO.FileInfo[]] $Files, + + [Parameter(Mandatory)] + [string] $Destination + ) + + $currentFiles = [System.Collections.Generic.List[IO.FileInfo]]::new() + $currentSize = 0L + foreach ($file in $Files | Sort-Object Name) { + if ($file.Length -gt $script:maximumPayloadBytes) { + throw "File '$($file.FullName)' exceeds the maximum correlation payload size." + } + if ($currentFiles.Count -gt 0 -and $currentSize + $file.Length -gt $script:maximumPayloadBytes) { + New-FileCorrelationPayload -Files $currentFiles.ToArray() -Destination $Destination -SizeBytes $currentSize + $currentFiles.Clear() + $currentSize = 0L + } + $currentFiles.Add($file) + $currentSize += $file.Length + } + if ($currentFiles.Count -gt 0) { + New-FileCorrelationPayload -Files $currentFiles.ToArray() -Destination $Destination -SizeBytes $currentSize + } +} + +function New-FileCorrelationPayload +{ + param ( + [Parameter(Mandatory)] + [IO.FileInfo[]] $Files, + + [Parameter(Mandatory)] + [string] $Destination, + + [Parameter(Mandatory)] + [long] $SizeBytes + ) + + $script:filePayloadIndex++ + $payloadDirectory = Join-Path $script:correlationPayloadFullPath ('files-{0:d3}' -f $script:filePayloadIndex) + New-Item -ItemType Directory -Force -Path $payloadDirectory | Out-Null + foreach ($file in $Files) { + $target = Join-Path $payloadDirectory $file.Name + try { + New-Item -ItemType HardLink -Path $target -Target $file.FullName | Out-Null + } catch { + Copy-Item -LiteralPath $file.FullName -Destination $target + } + } + Add-CorrelationPayload -Source $payloadDirectory -Destination $Destination -SizeBytes $SizeBytes +} + +function Add-DirectoryCorrelationPayloads +{ + param ( + [Parameter(Mandatory)] + [string] $Source, + + [Parameter(Mandatory)] + [string] $Destination, + + [switch] $Optional + ) + + if (-not (Test-Path -LiteralPath $Source -PathType Container)) { + if ($Optional) { + Write-Warning "Optional correlation payload directory '$Source' does not exist." + return + } + throw "Required correlation payload directory '$Source' does not exist." + } + + $size = Get-DirectorySize -Path $Source + if ($size -le $script:maximumPayloadBytes) { + Add-CorrelationPayload -Source $Source -Destination $Destination -SizeBytes $size + return + } + + $files = @(Get-ChildItem -LiteralPath $Source -File) + if ($files.Count -gt 0) { + Add-FileCorrelationPayloads -Files $files -Destination $Destination + } + foreach ($directory in Get-ChildItem -LiteralPath $Source -Directory | Sort-Object Name) { + Add-DirectoryCorrelationPayloads ` + -Source $directory.FullName ` + -Destination "$($Destination.TrimEnd('/'))/$($directory.Name)" + } +} + +function Write-CorrelationPayloadProps +{ + param ( + [Parameter(Mandatory)] + [string] $Path + ) + + $settings = [Xml.XmlWriterSettings]::new() + $settings.Indent = $true + $settings.Encoding = [Text.UTF8Encoding]::new($false) + $writer = [Xml.XmlWriter]::Create($Path, $settings) + try { + $writer.WriteStartElement('Project') + $writer.WriteStartElement('ItemGroup') + foreach ($payload in $script:correlationPayloads) { + $writer.WriteStartElement('_HostBuildTestHelixCorrelationPayload') + $writer.WriteAttributeString('Include', $payload.Source) + $writer.WriteElementString('Destination', $payload.Destination) + $writer.WriteEndElement() + } + $writer.WriteEndElement() + $writer.WriteEndElement() + } finally { + $writer.Dispose() + } +} + $repositoryRootFullPath = [IO.Path]::GetFullPath($RepositoryRoot) $correlationPayloadFullPath = [IO.Path]::GetFullPath($CorrelationPayloadDirectory) +$correlationPayloadsPropsFullPath = [IO.Path]::GetFullPath($CorrelationPayloadsPropsPath) $workItemsFullPath = [IO.Path]::GetFullPath($WorkItemsDirectory) $workItemsPropsFullPath = [IO.Path]::GetFullPath($WorkItemsPropsPath) $resultsFullPath = [IO.Path]::GetFullPath($ResultsDirectory) @@ -115,13 +275,11 @@ Remove-Item -LiteralPath $correlationPayloadFullPath -Recurse -Force -ErrorActio Remove-Item -LiteralPath $workItemsFullPath -Recurse -Force -ErrorAction Ignore New-Item -ItemType Directory -Force -Path $correlationPayloadFullPath, $workItemsFullPath, $resultsFullPath | Out-Null -$payloadRepository = Join-Path $correlationPayloadFullPath 'repo' +$payloadRepository = Join-Path $correlationPayloadFullPath 'repository' New-Item -ItemType Directory -Force -Path $payloadRepository | Out-Null Get-ChildItem -LiteralPath $repositoryRootFullPath -File | Copy-Item -Destination $payloadRepository $payloadDirectories = @( - @("bin\$Configuration", "bin\$Configuration", $false), - @("bin\Test$Configuration", "bin\Test$Configuration", $false), @('build-tools', 'build-tools', $false), @('eng', 'eng', $false), @('external', 'external', $false), @@ -145,26 +303,8 @@ foreach ($sourceDirectory in Get-ChildItem -LiteralPath $sourceRoot -Directory | -Destination (Join-Path (Join-Path $payloadRepository 'src') $sourceDirectory.Name) } -Copy-PayloadDirectory -Source $AndroidHome -Destination (Join-Path $correlationPayloadFullPath (ConvertTo-NativeRelativePath 'android-toolchain/sdk')) -Copy-PayloadDirectory -Source $JavaHome -Destination (Join-Path $correlationPayloadFullPath 'jdk') - -if ([string]::IsNullOrWhiteSpace($NuGetPackages)) { - $NuGetPackages = Join-Path ([Environment]::GetFolderPath('UserProfile')) (ConvertTo-NativeRelativePath '.nuget/packages') -} -Copy-PayloadDirectory -Source $NuGetPackages -Destination (Join-Path $correlationPayloadFullPath 'nuget-packages') - -if ([string]::IsNullOrWhiteSpace($GradleUserHome)) { - $GradleUserHome = Join-Path ([Environment]::GetFolderPath('UserProfile')) '.gradle' -} -foreach ($gradleDirectory in @('caches', 'wrapper')) { - Copy-PayloadDirectory ` - -Source (Join-Path $GradleUserHome $gradleDirectory) ` - -Destination (Join-Path $correlationPayloadFullPath (ConvertTo-NativeRelativePath "gradle/$gradleDirectory")) ` - -Optional -} - +$payloadTools = Join-Path $correlationPayloadFullPath 'dotnet-tools' if (-not [string]::IsNullOrWhiteSpace($DotNetToolsDirectory)) { - $payloadTools = Join-Path $correlationPayloadFullPath 'dotnet-tools' New-Item -ItemType Directory -Force -Path $payloadTools | Out-Null $apkDiffShim = if ($Platform -eq 'windows') { 'apkdiff.exe' } else { 'apkdiff' } $apkDiffShimPath = Join-Path $DotNetToolsDirectory $apkDiffShim @@ -177,12 +317,40 @@ if (-not [string]::IsNullOrWhiteSpace($DotNetToolsDirectory)) { -Optional } -$payloadBytes = (Get-ChildItem -LiteralPath $correlationPayloadFullPath -Recurse -File | Measure-Object -Property Length -Sum).Sum -Write-Host ('Staged correlation payload size: {0:N2} GiB.' -f ($payloadBytes / 1GB)) +Add-DirectoryCorrelationPayloads -Source $payloadRepository -Destination 'repo' +Add-DirectoryCorrelationPayloads ` + -Source (Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath "bin/$Configuration")) ` + -Destination "repo/bin/$Configuration" +Add-DirectoryCorrelationPayloads ` + -Source (Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath "bin/Test$Configuration")) ` + -Destination "repo/bin/Test$Configuration" +Add-DirectoryCorrelationPayloads -Source $AndroidHome -Destination 'android-toolchain/sdk' +Add-DirectoryCorrelationPayloads -Source $JavaHome -Destination 'jdk' + +if ([string]::IsNullOrWhiteSpace($GradleUserHome)) { + $GradleUserHome = Join-Path ([Environment]::GetFolderPath('UserProfile')) '.gradle' +} +Add-DirectoryCorrelationPayloads ` + -Source (Join-Path $GradleUserHome (ConvertTo-NativeRelativePath 'wrapper/dists')) ` + -Destination 'gradle/wrapper/dists' ` + -Optional +Add-DirectoryCorrelationPayloads ` + -Source (Join-Path $GradleUserHome (ConvertTo-NativeRelativePath 'caches/modules-2')) ` + -Destination 'gradle/caches/modules-2' ` + -Optional +if (Test-Path -LiteralPath $payloadTools -PathType Container) { + Add-DirectoryCorrelationPayloads -Source $payloadTools -Destination 'dotnet-tools' +} + +Write-CorrelationPayloadProps -Path $correlationPayloadsPropsFullPath +$payloadBytes = ($correlationPayloads | Measure-Object -Property SizeBytes -Sum).Sum +$largestPayloadBytes = ($correlationPayloads | Measure-Object -Property SizeBytes -Maximum).Maximum +Write-Host ('Prepared {0} correlation payloads totaling {1:N2} GiB; largest payload is {2:N2} GiB.' -f ` + $correlationPayloads.Count, ($payloadBytes / 1GB), ($largestPayloadBytes / 1GB)) -$stagedTestAssembly = Join-Path $payloadRepository (ConvertTo-NativeRelativePath $TestAssemblyRelativePath) -if (-not (Test-Path -LiteralPath $stagedTestAssembly -PathType Leaf)) { - throw "Test assembly '$stagedTestAssembly' was not staged." +$testAssembly = Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath $TestAssemblyRelativePath) +if (-not (Test-Path -LiteralPath $testAssembly -PathType Leaf)) { + throw "Test assembly '$testAssembly' does not exist." } if (-not (Test-Path -LiteralPath $TestSlicerPath -PathType Leaf)) { throw "dotnet-test-slicer '$TestSlicerPath' does not exist." @@ -191,7 +359,7 @@ if (-not (Test-Path -LiteralPath $TestSlicerPath -PathType Leaf)) { $discoveryRunSettings = Join-Path $workItemsFullPath 'all-tests.runsettings' $sliceArguments = @( 'slice', - "--test-assembly=$stagedTestAssembly", + "--test-assembly=$testAssembly", '--slice-number=1', '--total-slices=1', "--outfile=$discoveryRunSettings" @@ -254,6 +422,6 @@ if ($plan.DurationMode -eq 'count-fallback') { Write-Warning "No matching timing history was available. Work items use an explicit count fallback of $FallbackTestDurationSeconds second(s) per test." } -Write-Host "##vso[task.setvariable variable=HostBuildTestsHelixCorrelationPayload]$correlationPayloadFullPath" +Write-Host "##vso[task.setvariable variable=HostBuildTestsHelixCorrelationPayloadsProps]$correlationPayloadsPropsFullPath" Write-Host "##vso[task.setvariable variable=HostBuildTestsHelixWorkItemsProps]$workItemsPropsFullPath" Write-Host "##vso[task.setvariable variable=HostBuildTestsHelixWorkItemCount]$($plan.WorkItems.Count)" diff --git a/build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml b/build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml index 6132358d98e..e6c95925e8a 100644 --- a/build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml +++ b/build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml @@ -57,6 +57,7 @@ steps: $stagingRoot = '$(Build.StagingDirectory)/host-build-tests-helix/${{ parameters.platform }}' $resultsDirectory = Join-Path $stagingRoot 'results' $correlationPayload = Join-Path $stagingRoot 'correlation' + $correlationPayloadsProps = Join-Path $stagingRoot 'correlation-payloads.props' $workItemsDirectory = Join-Path $stagingRoot 'work-items' $workItemsProps = Join-Path $stagingRoot 'work-items.props' $adbName = if ($platform -eq 'windows') { 'adb.exe' } else { 'adb' } @@ -75,6 +76,7 @@ steps: -Platform $platform ` -RepositoryRoot '$(System.DefaultWorkingDirectory)' ` -CorrelationPayloadDirectory $correlationPayload ` + -CorrelationPayloadsPropsPath $correlationPayloadsProps ` -WorkItemsDirectory $workItemsDirectory ` -WorkItemsPropsPath $workItemsProps ` -ResultsDirectory $resultsDirectory ` @@ -89,7 +91,6 @@ steps: -Configuration '$(XA.Build.Configuration)' ` -AndroidHome $androidHome ` -JavaHome $env:JAVA_HOME ` - -NuGetPackages $env:NUGET_PACKAGES ` -DotNetToolsDirectory '$(Agent.ToolsDirectory)' ` -GradleUserHome (Join-Path ([Environment]::GetFolderPath('UserProfile')) '.gradle') displayName: Prepare duration-balanced ${{ parameters.platform }} Helix work items @@ -108,7 +109,7 @@ steps: taskTimeoutInMinutes: 240 env: HostBuildTestsHelixWorkItemsProps: $(HostBuildTestsHelixWorkItemsProps) - CorrelationPayloadDirectory: $(HostBuildTestsHelixCorrelationPayload) + HostBuildTestsHelixCorrelationPayloadsProps: $(HostBuildTestsHelixCorrelationPayloadsProps) HostBuildTestsHelixWorkItemTimeout: ${{ parameters.workItemTimeout }} HelixResultsDestinationDir: $(Build.StagingDirectory)/host-build-tests-helix/${{ parameters.platform }}/results HelixSource: pr/dotnet/android From 39d7209de44d4caad6eb4d2e4ff344c16322ac42 Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Fri, 21 Aug 2026 19:30:24 -0700 Subject: [PATCH 06/23] AB#12334 Exclude device-only Android payloads Keep emulator images, skins, and sources out of host-only Helix correlation payloads so large system images are neither uploaded nor downloaded by build-test work items. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../prepare-host-build-test-helix-payload.ps1 | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 b/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 index 9dbcece4b34..776e7049929 100644 --- a/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 +++ b/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 @@ -324,7 +324,20 @@ Add-DirectoryCorrelationPayloads ` Add-DirectoryCorrelationPayloads ` -Source (Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath "bin/Test$Configuration")) ` -Destination "repo/bin/Test$Configuration" -Add-DirectoryCorrelationPayloads -Source $AndroidHome -Destination 'android-toolchain/sdk' + +$androidRootFiles = @(Get-ChildItem -LiteralPath $AndroidHome -File) +if ($androidRootFiles.Count -gt 0) { + Add-FileCorrelationPayloads -Files $androidRootFiles -Destination 'android-toolchain/sdk' +} +$excludedAndroidDirectories = @('emulator', 'skins', 'sources', 'system-images') +foreach ($androidDirectory in Get-ChildItem -LiteralPath $AndroidHome -Directory | Sort-Object Name) { + if ($androidDirectory.Name -notin $excludedAndroidDirectories) { + Add-DirectoryCorrelationPayloads ` + -Source $androidDirectory.FullName ` + -Destination "android-toolchain/sdk/$($androidDirectory.Name)" + } +} + Add-DirectoryCorrelationPayloads -Source $JavaHome -Destination 'jdk' if ([string]::IsNullOrWhiteSpace($GradleUserHome)) { From 7802c120276fe1bc139436e256b806ac99cf4696 Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Fri, 21 Aug 2026 21:09:16 -0700 Subject: [PATCH 07/23] AB#12334 Report Helix payload composition Log and persist the largest correlation payload destinations so setup size can be reduced based on measured CI data rather than aggregate bytes alone. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../prepare-host-build-test-helix-payload.ps1 | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 b/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 index 776e7049929..bf10c33c0a9 100644 --- a/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 +++ b/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 @@ -360,6 +360,13 @@ $payloadBytes = ($correlationPayloads | Measure-Object -Property SizeBytes -Sum) $largestPayloadBytes = ($correlationPayloads | Measure-Object -Property SizeBytes -Maximum).Maximum Write-Host ('Prepared {0} correlation payloads totaling {1:N2} GiB; largest payload is {2:N2} GiB.' -f ` $correlationPayloads.Count, ($payloadBytes / 1GB), ($largestPayloadBytes / 1GB)) +Write-Host 'Largest correlation payload components:' +$correlationPayloads | + Sort-Object SizeBytes -Descending | + Select-Object -First 20 | + ForEach-Object { + Write-Host ('- {0:N2} GiB -> {1}' -f ($_.SizeBytes / 1GB), $_.Destination) + } $testAssembly = Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath $TestAssemblyRelativePath) if (-not (Test-Path -LiteralPath $testAssembly -PathType Leaf)) { @@ -417,6 +424,14 @@ $summaryPath = Join-Path $resultsFullPath 'work-item-generation.json' estimatedTestCount = $plan.EstimatedTestCount workItemCount = $plan.WorkItems.Count totalDurationLoadMs = $plan.TotalDurationMs + correlationPayloadCount = $correlationPayloads.Count + correlationPayloadBytes = $payloadBytes + correlationPayloads = @($correlationPayloads | ForEach-Object { + [pscustomobject] @{ + destination = $_.Destination + sizeBytes = $_.SizeBytes + } + }) workItems = @($plan.WorkItems | ForEach-Object { [pscustomobject] @{ id = $_.Id From c6114de22c3e35515d4b55fdbf1c9234e998045f Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Fri, 21 Aug 2026 21:58:01 -0700 Subject: [PATCH 08/23] AB#12334 Trim host-test payload inputs Exclude SDK docs/extras and the Java.Interop checkout, stage only the net10 host test assembly inputs, and add an opt-in host-only iteration mode while preserving full dual execution for parity runs. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../automation/azure-pipelines-public.yaml | 6 ++++- .../host-build-tests-helix/README.md | 3 ++- .../prepare-host-build-test-helix-payload.ps1 | 23 ++++++++++++++----- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/build-tools/automation/azure-pipelines-public.yaml b/build-tools/automation/azure-pipelines-public.yaml index 946a5959a5c..c162f29acc9 100644 --- a/build-tools/automation/azure-pipelines-public.yaml +++ b/build-tools/automation/azure-pipelines-public.yaml @@ -55,6 +55,10 @@ parameters: displayName: Host build tests Helix work item timeout type: string default: '00:45:00' +- name: hostBuildTestsHelixOnly + displayName: Run only the host build tests Helix prototype after builds + type: boolean + default: false # Repository resources resources: @@ -193,7 +197,7 @@ stages: dependsOn: - mac_build - linux_build - condition: and(succeeded(), ne(variables['SkipTestStages'], 'true')) + condition: and(succeeded(), or(ne(variables['SkipTestStages'], 'true'), eq('${{ parameters.hostBuildTestsHelixOnly }}', 'true'))) jobs: - job: windows_host_build_tests_helix displayName: Windows > Host Build Tests > Helix diff --git a/build-tools/automation/host-build-tests-helix/README.md b/build-tools/automation/host-build-tests-helix/README.md index 4f35a3076b2..476dcc348d4 100644 --- a/build-tools/automation/host-build-tests-helix/README.md +++ b/build-tools/automation/host-build-tests-helix/README.md @@ -9,7 +9,8 @@ in-memory ZIP limit. Work item payloads contain only a runsettings filter, a sma command script, and generation metadata. To control total payload size, the source copy omits `src/Mono.Android` and the NuGet package cache; host build tests consume the product through the prepared workload packs and use isolated per-work-item NuGet -caches. +caches. Device-only Android SDK content, SDK documentation, the Java.Interop checkout, +and unrelated test assemblies are also excluded. `prepare-host-build-test-helix-payload.ps1` discovers NUnit tests with the repository's `dotnet-test-slicer`, reads its `balance.xml` format, and uses deterministic best-fit diff --git a/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 b/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 index bf10c33c0a9..7dc5d590fdb 100644 --- a/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 +++ b/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 @@ -282,7 +282,6 @@ Get-ChildItem -LiteralPath $repositoryRootFullPath -File | Copy-Item -Destinatio $payloadDirectories = @( @('build-tools', 'build-tools', $false), @('eng', 'eng', $false), - @('external', 'external', $false), @('.github', '.github', $true), @('samples', 'samples', $true), @('src-ThirdParty', 'src-ThirdParty', $true), @@ -321,15 +320,28 @@ Add-DirectoryCorrelationPayloads -Source $payloadRepository -Destination 'repo' Add-DirectoryCorrelationPayloads ` -Source (Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath "bin/$Configuration")) ` -Destination "repo/bin/$Configuration" -Add-DirectoryCorrelationPayloads ` - -Source (Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath "bin/Test$Configuration")) ` - -Destination "repo/bin/Test$Configuration" + +$testOutputRoot = Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath "bin/Test$Configuration") +$testAssembly = Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath $TestAssemblyRelativePath) +$testAssemblyDirectory = Split-Path -Parent $testAssembly +$testAssemblyDestination = "repo/bin/Test$Configuration/$([IO.Path]::GetRelativePath($testOutputRoot, $testAssemblyDirectory).Replace('\', '/'))" +Add-DirectoryCorrelationPayloads -Source $testAssemblyDirectory -Destination $testAssemblyDestination +$testOutputFiles = @(Get-ChildItem -LiteralPath $testOutputRoot -File) +if ($testOutputFiles.Count -gt 0) { + Add-FileCorrelationPayloads -Files $testOutputFiles -Destination "repo/bin/Test$Configuration" +} +foreach ($testDirectory in Get-ChildItem -LiteralPath $testOutputRoot -Directory | + Where-Object { $_.Name -eq 'Expected' -or $_.Name -like 'android-*' }) { + Add-DirectoryCorrelationPayloads ` + -Source $testDirectory.FullName ` + -Destination "repo/bin/Test$Configuration/$($testDirectory.Name)" +} $androidRootFiles = @(Get-ChildItem -LiteralPath $AndroidHome -File) if ($androidRootFiles.Count -gt 0) { Add-FileCorrelationPayloads -Files $androidRootFiles -Destination 'android-toolchain/sdk' } -$excludedAndroidDirectories = @('emulator', 'skins', 'sources', 'system-images') +$excludedAndroidDirectories = @('docs', 'emulator', 'extras', 'skins', 'sources', 'system-images') foreach ($androidDirectory in Get-ChildItem -LiteralPath $AndroidHome -Directory | Sort-Object Name) { if ($androidDirectory.Name -notin $excludedAndroidDirectories) { Add-DirectoryCorrelationPayloads ` @@ -368,7 +380,6 @@ $correlationPayloads | Write-Host ('- {0:N2} GiB -> {1}' -f ($_.SizeBytes / 1GB), $_.Destination) } -$testAssembly = Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath $TestAssemblyRelativePath) if (-not (Test-Path -LiteralPath $testAssembly -PathType Leaf)) { throw "Test assembly '$testAssembly' does not exist." } From b0bec43312276c80d2c67de38dbdb0c76ae58363 Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Mon, 24 Aug 2026 07:41:11 -0700 Subject: [PATCH 09/23] AB#12334 Align Helix test environment and results Avoid forcing generated projects to Release, set the CI environment, use the Server 2022 host queue, allow first-run fallback batches 90 minutes, and download TRX plus diagnostic artifacts from Helix. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- build-tools/automation/HostBuildTestsHelix.proj | 2 +- build-tools/automation/azure-pipelines-public.yaml | 4 ++-- .../host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 | 7 ++++++- .../host-build-tests-helix/HostBuildTestsHelix.psm1 | 8 ++++++-- build-tools/automation/host-build-tests-helix/README.md | 4 +++- .../yaml-templates/run-host-build-tests-in-helix.yaml | 2 +- 6 files changed, 19 insertions(+), 8 deletions(-) diff --git a/build-tools/automation/HostBuildTestsHelix.proj b/build-tools/automation/HostBuildTestsHelix.proj index 1545f015658..ff2d19cebbb 100644 --- a/build-tools/automation/HostBuildTestsHelix.proj +++ b/build-tools/automation/HostBuildTestsHelix.proj @@ -6,7 +6,7 @@ true tests/android/host-build-tests/ Xamarin.Android.Build.Tests - Helix - 00:45:00 + 01:30:00 diff --git a/build-tools/automation/azure-pipelines-public.yaml b/build-tools/automation/azure-pipelines-public.yaml index c162f29acc9..43bc5523f78 100644 --- a/build-tools/automation/azure-pipelines-public.yaml +++ b/build-tools/automation/azure-pipelines-public.yaml @@ -54,7 +54,7 @@ parameters: - name: hostBuildTestsHelixWorkItemTimeout displayName: Host build tests Helix work item timeout type: string - default: '00:45:00' + default: '01:30:00' - name: hostBuildTestsHelixOnly displayName: Run only the host build tests Helix prototype after builds type: boolean @@ -212,7 +212,7 @@ stages: - template: /build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml parameters: platform: windows - helixQueue: Windows.Amd64.VS2026.Open + helixQueue: Windows.Amd64.Server2022.Open testFilter: '' testRunTitle: Xamarin.Android.Build.Tests - Windows Helix Prototype timingBuildId: ${{ parameters.hostBuildTestsHelixTimingBuildId }} diff --git a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 index 2d87470c77e..68d07afef95 100644 --- a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 +++ b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 @@ -126,9 +126,13 @@ try { throw "Payload directory '$($item.PayloadDirectory)' was not created." } $command = Get-Content -LiteralPath (Join-Path $item.PayloadDirectory 'run-host-tests.cmd') -Raw - if ($command.Contains('__CONFIGURATION__') -or -not $command.Contains('bin\Release\dotnet')) { + if ($command.Contains('__CONFIGURATION__') -or + -not $command.Contains('bin\Release\dotnet') -or + $command.Contains('set "CONFIGURATION=Release"') -or + -not $command.Contains('set "RUNNINGONCI=true"')) { throw 'The generated command did not apply the requested build configuration.' } + Assert-Equal 'results.trx;console.log;slice.runsettings;work-item.json;diagnostics.zip' ([string] $item.DownloadFilesFromResults) 'Windows results should be downloaded.' } $linuxPayloadDirectory = Join-Path $testRoot 'linux-payloads' @@ -144,6 +148,7 @@ try { [xml] $linuxProps = Get-Content -LiteralPath $linuxPropsPath -Raw foreach ($item in @($linuxProps.Project.ItemGroup._HostBuildTestHelixWorkItem)) { Assert-Equal 'bash run-host-tests.sh' ([string] $item.Command) 'Linux work items should explicitly use bash.' + Assert-Equal 'results.trx;console.log;slice.runsettings;work-item.json;diagnostics.tar.gz' ([string] $item.DownloadFilesFromResults) 'Linux results should be downloaded.' } Write-Host 'HostBuildTestsHelix tests passed.' diff --git a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 index e1dafe4f35c..eca6506d6ca 100644 --- a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 +++ b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 @@ -299,8 +299,8 @@ set "DOTNET_CLI_HOME=%HELIX_WORKITEM_ROOT%\dotnet-home" set "TEMP=%HELIX_WORKITEM_ROOT%\temp" set "TMP=%TEMP%" set "BUILD_STAGINGDIRECTORY=%HELIX_WORKITEM_UPLOAD_ROOT%" -set "CONFIGURATION=__CONFIGURATION__" set "PATH=%HELIX_CORRELATION_PAYLOAD%\dotnet-tools;%DOTNET_ROOT%;%ANDROID_HOME%\platform-tools;%JAVA_HOME%\bin;%PATH%" +set "RUNNINGONCI=true" set "DOTNET_CLI_TELEMETRY_OPTOUT=1" set "DOTNET_NOLOGO=1" set "DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1" @@ -324,6 +324,7 @@ pushd "%REPO%" "%DOTNET_ROOT%\dotnet.exe" test "%REPO%\__TEST_ASSEMBLY__" --settings "%~dp0slice.runsettings" --logger "trx;LogFileName=results.trx" --results-directory "%HELIX_WORKITEM_UPLOAD_ROOT%" -- NUnit.NumberOfTestWorkers=__NUNIT_WORKERS__ > "%HELIX_WORKITEM_UPLOAD_ROOT%\console.log" 2>&1 set "testExitCode=%ERRORLEVEL%" type "%HELIX_WORKITEM_UPLOAD_ROOT%\console.log" +powershell.exe -NoLogo -NoProfile -Command "$files = Get-ChildItem -LiteralPath $env:HELIX_WORKITEM_UPLOAD_ROOT -Recurse -File -Include *.binlog,*.dmp,*.log,*.txt; if ($files.Count -gt 0) { Compress-Archive -LiteralPath $files.FullName -DestinationPath (Join-Path $env:HELIX_WORKITEM_UPLOAD_ROOT 'diagnostics.zip') -Force }" >nul 2>&1 "%DOTNET_ROOT%\dotnet.exe" build-server shutdown >nul 2>&1 popd exit /b %testExitCode% @@ -350,8 +351,8 @@ export GRADLE_USER_HOME="$HELIX_CORRELATION_PAYLOAD/gradle" export DOTNET_CLI_HOME="$HELIX_WORKITEM_ROOT/dotnet-home" export TMPDIR="$HELIX_WORKITEM_ROOT/temp" export BUILD_STAGINGDIRECTORY="$HELIX_WORKITEM_UPLOAD_ROOT" -export CONFIGURATION=__CONFIGURATION__ export PATH="$HELIX_CORRELATION_PAYLOAD/dotnet-tools:$DOTNET_ROOT:$ANDROID_HOME/platform-tools:$JAVA_HOME/bin:$PATH" +export RUNNINGONCI=true export DOTNET_CLI_TELEMETRY_OPTOUT=1 export DOTNET_NOLOGO=1 export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 @@ -373,6 +374,7 @@ set +e "$DOTNET_ROOT/dotnet" test "$REPO/__TEST_ASSEMBLY__" --settings "$HELIX_WORKITEM_ROOT/slice.runsettings" --logger "trx;LogFileName=results.trx" --results-directory "$HELIX_WORKITEM_UPLOAD_ROOT" -- NUnit.NumberOfTestWorkers=__NUNIT_WORKERS__ 2>&1 | tee "$HELIX_WORKITEM_UPLOAD_ROOT/console.log" test_exit=${PIPESTATUS[0]} set -e +(cd "$HELIX_WORKITEM_UPLOAD_ROOT" && find . -type f \( -name '*.binlog' -o -name '*.dmp' -o -name '*.log' -o -name '*.txt' \) -print0 | tar --null -czf diagnostics.tar.gz --files-from=-) || true "$DOTNET_ROOT/dotnet" build-server shutdown >/dev/null 2>&1 || true exit "$test_exit" '@ @@ -452,6 +454,8 @@ function Write-HostBuildTestWorkItemPayloads $propsWriter.WriteAttributeString('Include', $name) $propsWriter.WriteElementString('PayloadDirectory', $payloadDirectory) $propsWriter.WriteElementString('Command', $helixCommand) + $diagnosticsArchive = if ($Platform -eq 'windows') { 'diagnostics.zip' } else { 'diagnostics.tar.gz' } + $propsWriter.WriteElementString('DownloadFilesFromResults', "results.trx;console.log;slice.runsettings;work-item.json;$diagnosticsArchive") $propsWriter.WriteElementString('EstimatedDurationMs', [string] $workItem.EstimatedDurationMs) $propsWriter.WriteElementString('TestCount', [string] $workItem.Tests.Count) $propsWriter.WriteEndElement() diff --git a/build-tools/automation/host-build-tests-helix/README.md b/build-tools/automation/host-build-tests-helix/README.md index 476dcc348d4..f75aacaeb8e 100644 --- a/build-tools/automation/host-build-tests-helix/README.md +++ b/build-tools/automation/host-build-tests-helix/README.md @@ -18,12 +18,14 @@ decreasing bin packing. The target is `15` estimated wall-clock minutes by defau `durationParallelism` translates summed per-test elapsed time into expected wall time while preserving the current NUnit worker setting. The public prototype defaults this factor to `2.5`, uses `60` seconds per test for the explicit first-run fallback, and -allows `45` minutes before Helix terminates a work item. +allows `90` minutes before Helix terminates a first-run work item. If no timing history is provided, generation explicitly reports `count-fallback` and uses the configured fallback duration per test. It does not label count-only slicing as duration-balanced. Each completed run publishes a new `balance.xml`; pass that run's build ID through `hostBuildTestsHelixTimingBuildId` on the next queued build. +TRX, console output, runsettings, work-item metadata, and a diagnostic archive are +downloaded from each Helix results container for publication in Azure Pipelines. The public pipeline keeps the existing Azure Pipelines jobs and adds this path only when `enableHostBuildTestsHelixPrototype` is `true`. diff --git a/build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml b/build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml index e6c95925e8a..3b0fe7a01d3 100644 --- a/build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml +++ b/build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml @@ -8,7 +8,7 @@ parameters: targetMinutes: 15 durationParallelism: 2.5 fallbackTestDurationSeconds: 60 - workItemTimeout: '00:45:00' + workItemTimeout: '01:30:00' nunitWorkers: '-1' steps: From 83778074b7ddc00bc62dc56f7c20611c874566c1 Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Mon, 24 Aug 2026 11:44:11 -0700 Subject: [PATCH 10/23] AB#12334 Add required NDK and local packages Stage the prepared NDK and nuget-unsigned feed, expose the NDK test environment, keep Gradle cache post-jobs valid, and preserve diagnostic directory structure in Windows archives. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../HostBuildTestsHelix.Tests.ps1 | 4 +++- .../HostBuildTestsHelix.psm1 | 8 +++++++- .../automation/host-build-tests-helix/README.md | 3 ++- .../prepare-host-build-test-helix-payload.ps1 | 15 ++++++++++++--- .../run-host-build-tests-in-helix.yaml | 9 +++++++++ 5 files changed, 33 insertions(+), 6 deletions(-) diff --git a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 index 68d07afef95..127d0d2187c 100644 --- a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 +++ b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 @@ -129,7 +129,9 @@ try { if ($command.Contains('__CONFIGURATION__') -or -not $command.Contains('bin\Release\dotnet') -or $command.Contains('set "CONFIGURATION=Release"') -or - -not $command.Contains('set "RUNNINGONCI=true"')) { + -not $command.Contains('set "RUNNINGONCI=true"') -or + -not $command.Contains('set "TEST_ANDROID_NDK_PATH=') -or + -not $command.Contains('tar.exe -a -c -f diagnostics.zip')) { throw 'The generated command did not apply the requested build configuration.' } Assert-Equal 'results.trx;console.log;slice.runsettings;work-item.json;diagnostics.zip' ([string] $item.DownloadFilesFromResults) 'Windows results should be downloaded.' diff --git a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 index eca6506d6ca..18337345355 100644 --- a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 +++ b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 @@ -292,6 +292,8 @@ set "REPO=%HELIX_CORRELATION_PAYLOAD%\repo" set "DOTNET_ROOT=%REPO%\bin\__CONFIGURATION__\dotnet" set "ANDROID_HOME=%HELIX_CORRELATION_PAYLOAD%\android-toolchain\sdk" set "ANDROID_SDK_ROOT=%ANDROID_HOME%" +set "TEST_ANDROID_NDK_PATH=%HELIX_CORRELATION_PAYLOAD%\android-toolchain\ndk" +set "ANDROID_NDK_LATEST_HOME=%TEST_ANDROID_NDK_PATH%" set "JAVA_HOME=%HELIX_CORRELATION_PAYLOAD%\jdk" set "NUGET_PACKAGES=%HELIX_WORKITEM_ROOT%\nuget-packages" set "GRADLE_USER_HOME=%HELIX_CORRELATION_PAYLOAD%\gradle" @@ -324,7 +326,9 @@ pushd "%REPO%" "%DOTNET_ROOT%\dotnet.exe" test "%REPO%\__TEST_ASSEMBLY__" --settings "%~dp0slice.runsettings" --logger "trx;LogFileName=results.trx" --results-directory "%HELIX_WORKITEM_UPLOAD_ROOT%" -- NUnit.NumberOfTestWorkers=__NUNIT_WORKERS__ > "%HELIX_WORKITEM_UPLOAD_ROOT%\console.log" 2>&1 set "testExitCode=%ERRORLEVEL%" type "%HELIX_WORKITEM_UPLOAD_ROOT%\console.log" -powershell.exe -NoLogo -NoProfile -Command "$files = Get-ChildItem -LiteralPath $env:HELIX_WORKITEM_UPLOAD_ROOT -Recurse -File -Include *.binlog,*.dmp,*.log,*.txt; if ($files.Count -gt 0) { Compress-Archive -LiteralPath $files.FullName -DestinationPath (Join-Path $env:HELIX_WORKITEM_UPLOAD_ROOT 'diagnostics.zip') -Force }" >nul 2>&1 +pushd "%HELIX_WORKITEM_UPLOAD_ROOT%" +tar.exe -a -c -f diagnostics.zip --exclude=diagnostics.zip TestRelease dumps >nul 2>&1 +popd "%DOTNET_ROOT%\dotnet.exe" build-server shutdown >nul 2>&1 popd exit /b %testExitCode% @@ -345,6 +349,8 @@ REPO="$HELIX_CORRELATION_PAYLOAD/repo" export DOTNET_ROOT="$REPO/bin/__CONFIGURATION__/dotnet" export ANDROID_HOME="$HELIX_CORRELATION_PAYLOAD/android-toolchain/sdk" export ANDROID_SDK_ROOT="$ANDROID_HOME" +export TEST_ANDROID_NDK_PATH="$HELIX_CORRELATION_PAYLOAD/android-toolchain/ndk" +export ANDROID_NDK_LATEST_HOME="$TEST_ANDROID_NDK_PATH" export JAVA_HOME="$HELIX_CORRELATION_PAYLOAD/jdk" export NUGET_PACKAGES="$HELIX_WORKITEM_ROOT/nuget-packages" export GRADLE_USER_HOME="$HELIX_CORRELATION_PAYLOAD/gradle" diff --git a/build-tools/automation/host-build-tests-helix/README.md b/build-tools/automation/host-build-tests-helix/README.md index f75aacaeb8e..835d836950d 100644 --- a/build-tools/automation/host-build-tests-helix/README.md +++ b/build-tools/automation/host-build-tests-helix/README.md @@ -10,7 +10,8 @@ command script, and generation metadata. To control total payload size, the sour copy omits `src/Mono.Android` and the NuGet package cache; host build tests consume the product through the prepared workload packs and use isolated per-work-item NuGet caches. Device-only Android SDK content, SDK documentation, the Java.Interop checkout, -and unrelated test assemblies are also excluded. +and unrelated test assemblies are also excluded. The prepared NDK and the local +`nuget-unsigned` feed remain shared payloads because host tests consume both directly. `prepare-host-build-test-helix-payload.ps1` discovers NUnit tests with the repository's `dotnet-test-slicer`, reads its `balance.xml` format, and uses deterministic best-fit diff --git a/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 b/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 index 7dc5d590fdb..4dc6c30e245 100644 --- a/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 +++ b/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 @@ -48,6 +48,9 @@ param ( [Parameter(Mandatory)] [string] $AndroidHome, + [Parameter(Mandatory)] + [string] $AndroidNdkHome, + [Parameter(Mandatory)] [string] $JavaHome, @@ -320,6 +323,9 @@ Add-DirectoryCorrelationPayloads -Source $payloadRepository -Destination 'repo' Add-DirectoryCorrelationPayloads ` -Source (Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath "bin/$Configuration")) ` -Destination "repo/bin/$Configuration" +Add-DirectoryCorrelationPayloads ` + -Source (Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath "bin/Build$Configuration/nuget-unsigned")) ` + -Destination "repo/bin/Build$Configuration/nuget-unsigned" $testOutputRoot = Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath "bin/Test$Configuration") $testAssembly = Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath $TestAssemblyRelativePath) @@ -350,15 +356,18 @@ foreach ($androidDirectory in Get-ChildItem -LiteralPath $AndroidHome -Directory } } +Add-DirectoryCorrelationPayloads -Source $AndroidNdkHome -Destination 'android-toolchain/ndk' Add-DirectoryCorrelationPayloads -Source $JavaHome -Destination 'jdk' if ([string]::IsNullOrWhiteSpace($GradleUserHome)) { $GradleUserHome = Join-Path ([Environment]::GetFolderPath('UserProfile')) '.gradle' } +$gradleDistsDirectory = Join-Path $GradleUserHome (ConvertTo-NativeRelativePath 'wrapper/dists') +New-Item -ItemType Directory -Force -Path $gradleDistsDirectory | Out-Null Add-DirectoryCorrelationPayloads ` - -Source (Join-Path $GradleUserHome (ConvertTo-NativeRelativePath 'wrapper/dists')) ` - -Destination 'gradle/wrapper/dists' ` - -Optional + -Source $gradleDistsDirectory ` + -Destination 'gradle/wrapper/dists' + Add-DirectoryCorrelationPayloads ` -Source (Join-Path $GradleUserHome (ConvertTo-NativeRelativePath 'caches/modules-2')) ` -Destination 'gradle/caches/modules-2' ` diff --git a/build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml b/build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml index 3b0fe7a01d3..2897791d7f2 100644 --- a/build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml +++ b/build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml @@ -71,6 +71,14 @@ steps: if (-not $androidHome) { throw 'Could not locate the prepared Android SDK.' } + $androidNdkHome = @( + $env:TEST_ANDROID_NDK_PATH + $env:ANDROID_NDK_LATEST_HOME + $(Join-Path (Split-Path -Parent $androidHome) 'ndk') + ) | Where-Object { $_ -and (Test-Path $_ -PathType Container) } | Select-Object -First 1 + if (-not $androidNdkHome) { + throw 'Could not locate the prepared Android NDK.' + } & '$(System.DefaultWorkingDirectory)/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1' ` -Platform $platform ` @@ -90,6 +98,7 @@ steps: -NUnitWorkers '${{ parameters.nunitWorkers }}' ` -Configuration '$(XA.Build.Configuration)' ` -AndroidHome $androidHome ` + -AndroidNdkHome $androidNdkHome ` -JavaHome $env:JAVA_HOME ` -DotNetToolsDirectory '$(Agent.ToolsDirectory)' ` -GradleUserHome (Join-Path ([Environment]::GetFolderPath('UserProfile')) '.gradle') From 75a5cbdd86884df469fcc89373753583bc9ba258 Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Mon, 24 Aug 2026 14:30:57 -0700 Subject: [PATCH 11/23] AB#12334 Calibrate duration packing and Windows builds Use the observed 1.65 effective parallelism, omit Windows-only NDK/local packages from Linux, and serialize Windows inner MSBuild work to avoid the repeated multi-RID packaging race. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../automation/azure-pipelines-public.yaml | 2 +- .../HostBuildTestsHelix.Tests.ps1 | 1 + .../HostBuildTestsHelix.psm1 | 3 +-- .../host-build-tests-helix/README.md | 2 +- .../prepare-host-build-test-helix-payload.ps1 | 15 +++++++++------ .../run-host-build-tests-in-helix.yaml | 19 +++++++++++-------- 6 files changed, 24 insertions(+), 18 deletions(-) diff --git a/build-tools/automation/azure-pipelines-public.yaml b/build-tools/automation/azure-pipelines-public.yaml index 43bc5523f78..76992423ad1 100644 --- a/build-tools/automation/azure-pipelines-public.yaml +++ b/build-tools/automation/azure-pipelines-public.yaml @@ -42,7 +42,7 @@ parameters: - name: hostBuildTestsHelixDurationParallelism displayName: Host build tests timing parallelism type: number - default: 2.5 + default: 1.65 - name: hostBuildTestsHelixFallbackTestDurationSeconds displayName: Host build tests fallback seconds per test type: number diff --git a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 index 127d0d2187c..be48052c9a5 100644 --- a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 +++ b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 @@ -130,6 +130,7 @@ try { -not $command.Contains('bin\Release\dotnet') -or $command.Contains('set "CONFIGURATION=Release"') -or -not $command.Contains('set "RUNNINGONCI=true"') -or + -not $command.Contains('set "BuildInParallel=false"') -or -not $command.Contains('set "TEST_ANDROID_NDK_PATH=') -or -not $command.Contains('tar.exe -a -c -f diagnostics.zip')) { throw 'The generated command did not apply the requested build configuration.' diff --git a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 index 18337345355..da78f0f251a 100644 --- a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 +++ b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 @@ -303,6 +303,7 @@ set "TMP=%TEMP%" set "BUILD_STAGINGDIRECTORY=%HELIX_WORKITEM_UPLOAD_ROOT%" set "PATH=%HELIX_CORRELATION_PAYLOAD%\dotnet-tools;%DOTNET_ROOT%;%ANDROID_HOME%\platform-tools;%JAVA_HOME%\bin;%PATH%" set "RUNNINGONCI=true" +set "BuildInParallel=false" set "DOTNET_CLI_TELEMETRY_OPTOUT=1" set "DOTNET_NOLOGO=1" set "DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1" @@ -349,8 +350,6 @@ REPO="$HELIX_CORRELATION_PAYLOAD/repo" export DOTNET_ROOT="$REPO/bin/__CONFIGURATION__/dotnet" export ANDROID_HOME="$HELIX_CORRELATION_PAYLOAD/android-toolchain/sdk" export ANDROID_SDK_ROOT="$ANDROID_HOME" -export TEST_ANDROID_NDK_PATH="$HELIX_CORRELATION_PAYLOAD/android-toolchain/ndk" -export ANDROID_NDK_LATEST_HOME="$TEST_ANDROID_NDK_PATH" export JAVA_HOME="$HELIX_CORRELATION_PAYLOAD/jdk" export NUGET_PACKAGES="$HELIX_WORKITEM_ROOT/nuget-packages" export GRADLE_USER_HOME="$HELIX_CORRELATION_PAYLOAD/gradle" diff --git a/build-tools/automation/host-build-tests-helix/README.md b/build-tools/automation/host-build-tests-helix/README.md index 835d836950d..c4bf8e5d228 100644 --- a/build-tools/automation/host-build-tests-helix/README.md +++ b/build-tools/automation/host-build-tests-helix/README.md @@ -18,7 +18,7 @@ and unrelated test assemblies are also excluded. The prepared NDK and the local decreasing bin packing. The target is `15` estimated wall-clock minutes by default. `durationParallelism` translates summed per-test elapsed time into expected wall time while preserving the current NUnit worker setting. The public prototype defaults this -factor to `2.5`, uses `60` seconds per test for the explicit first-run fallback, and +factor to `1.65`, uses `60` seconds per test for the explicit first-run fallback, and allows `90` minutes before Helix terminates a first-run work item. If no timing history is provided, generation explicitly reports `count-fallback` and diff --git a/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 b/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 index 4dc6c30e245..37aa4948994 100644 --- a/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 +++ b/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 @@ -48,8 +48,7 @@ param ( [Parameter(Mandatory)] [string] $AndroidHome, - [Parameter(Mandatory)] - [string] $AndroidNdkHome, + [string] $AndroidNdkHome = '', [Parameter(Mandatory)] [string] $JavaHome, @@ -323,9 +322,11 @@ Add-DirectoryCorrelationPayloads -Source $payloadRepository -Destination 'repo' Add-DirectoryCorrelationPayloads ` -Source (Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath "bin/$Configuration")) ` -Destination "repo/bin/$Configuration" -Add-DirectoryCorrelationPayloads ` - -Source (Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath "bin/Build$Configuration/nuget-unsigned")) ` - -Destination "repo/bin/Build$Configuration/nuget-unsigned" +if ($Platform -eq 'windows') { + Add-DirectoryCorrelationPayloads ` + -Source (Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath "bin/Build$Configuration/nuget-unsigned")) ` + -Destination "repo/bin/Build$Configuration/nuget-unsigned" +} $testOutputRoot = Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath "bin/Test$Configuration") $testAssembly = Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath $TestAssemblyRelativePath) @@ -356,7 +357,9 @@ foreach ($androidDirectory in Get-ChildItem -LiteralPath $AndroidHome -Directory } } -Add-DirectoryCorrelationPayloads -Source $AndroidNdkHome -Destination 'android-toolchain/ndk' +if (-not [string]::IsNullOrWhiteSpace($AndroidNdkHome)) { + Add-DirectoryCorrelationPayloads -Source $AndroidNdkHome -Destination 'android-toolchain/ndk' +} Add-DirectoryCorrelationPayloads -Source $JavaHome -Destination 'jdk' if ([string]::IsNullOrWhiteSpace($GradleUserHome)) { diff --git a/build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml b/build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml index 2897791d7f2..44987393059 100644 --- a/build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml +++ b/build-tools/automation/yaml-templates/run-host-build-tests-in-helix.yaml @@ -6,7 +6,7 @@ parameters: timingBuildId: '' timingArtifactName: '' targetMinutes: 15 - durationParallelism: 2.5 + durationParallelism: 1.65 fallbackTestDurationSeconds: 60 workItemTimeout: '01:30:00' nunitWorkers: '-1' @@ -71,13 +71,16 @@ steps: if (-not $androidHome) { throw 'Could not locate the prepared Android SDK.' } - $androidNdkHome = @( - $env:TEST_ANDROID_NDK_PATH - $env:ANDROID_NDK_LATEST_HOME - $(Join-Path (Split-Path -Parent $androidHome) 'ndk') - ) | Where-Object { $_ -and (Test-Path $_ -PathType Container) } | Select-Object -First 1 - if (-not $androidNdkHome) { - throw 'Could not locate the prepared Android NDK.' + $androidNdkHome = '' + if ($platform -eq 'windows') { + $androidNdkHome = @( + $env:TEST_ANDROID_NDK_PATH + $env:ANDROID_NDK_LATEST_HOME + $(Join-Path (Split-Path -Parent $androidHome) 'ndk') + ) | Where-Object { $_ -and (Test-Path $_ -PathType Container) } | Select-Object -First 1 + if (-not $androidNdkHome) { + throw 'Could not locate the prepared Android NDK.' + } } & '$(System.DefaultWorkingDirectory)/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1' ` From 44ce9e0ab6f5ca968861980a0ff9fd4e4b35849d Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Mon, 24 Aug 2026 17:45:55 -0700 Subject: [PATCH 12/23] AB#12334 Isolate file-sharing manifest tests Run DebuggerAttribute and AllForegroundServiceTypes non-parallel because their parameterized cases share project files or multi-RID package outputs and repeatedly raced on Helix. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Tests/Xamarin.Android.Build.Tests/ManifestTest.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ManifestTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ManifestTest.cs index 0f5d6141d80..372c772e547 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ManifestTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ManifestTest.cs @@ -809,6 +809,7 @@ public void ManifestPlaceHoldersXA1010 ([Values ("legacy", "manifestmerger.jar") } [Test] + [NonParallelizable] [TestCaseSource (nameof (Get_DebuggerAttributeCases_Data))] public void DebuggerAttribute (string debugType, bool isRelease, bool expected, AndroidRuntime runtime) { @@ -1117,6 +1118,7 @@ void AddTestData (string serviceType, string expected, AndroidRuntime runtime) } [Test] + [NonParallelizable] [TestCaseSource (nameof (Get_AllForegroundServiceTypes_Data))] public void AllForegroundServiceTypes (string serviceType, string expected, AndroidRuntime runtime) { From 5ed46cc48c2ecfdaf8fc022ca52757e10d93e04d Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Mon, 24 Aug 2026 20:49:51 -0700 Subject: [PATCH 13/23] AB#12334 Remove remaining Windows test races Disable parallel runtime-identifier inner builds for Helix and isolate the move-and-rebuild test that showed a transient project-file lock. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 | 1 + .../automation/host-build-tests-helix/HostBuildTestsHelix.psm1 | 1 + .../Tests/Xamarin.Android.Build.Tests/BuildTest2.cs | 1 + 3 files changed, 3 insertions(+) diff --git a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 index be48052c9a5..9dc6e10ee11 100644 --- a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 +++ b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 @@ -131,6 +131,7 @@ try { $command.Contains('set "CONFIGURATION=Release"') -or -not $command.Contains('set "RUNNINGONCI=true"') -or -not $command.Contains('set "BuildInParallel=false"') -or + -not $command.Contains('set "_AndroidBuildRuntimeIdentifiersInParallel=false"') -or -not $command.Contains('set "TEST_ANDROID_NDK_PATH=') -or -not $command.Contains('tar.exe -a -c -f diagnostics.zip')) { throw 'The generated command did not apply the requested build configuration.' diff --git a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 index da78f0f251a..96c8a3f9c53 100644 --- a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 +++ b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 @@ -304,6 +304,7 @@ set "BUILD_STAGINGDIRECTORY=%HELIX_WORKITEM_UPLOAD_ROOT%" set "PATH=%HELIX_CORRELATION_PAYLOAD%\dotnet-tools;%DOTNET_ROOT%;%ANDROID_HOME%\platform-tools;%JAVA_HOME%\bin;%PATH%" set "RUNNINGONCI=true" set "BuildInParallel=false" +set "_AndroidBuildRuntimeIdentifiersInParallel=false" set "DOTNET_CLI_TELEMETRY_OPTOUT=1" set "DOTNET_NOLOGO=1" set "DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1" diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs index bc880f626a5..dd8c547404c 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest2.cs @@ -159,6 +159,7 @@ public void IncompatiblePlatformTargetAndRuntimeIdentifiersFailsBuild () } [Test] + [NonParallelizable] public void BuildBasicApplicationThenMoveIt ([Values] bool isRelease, [Values (AndroidRuntime.CoreCLR, AndroidRuntime.NativeAOT)] AndroidRuntime runtime) { if (IgnoreUnsupportedConfiguration (runtime, release: isRelease)) { From 280a2edee27a6a63b82bf12fe1499def537c898f Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Mon, 24 Aug 2026 23:53:04 -0700 Subject: [PATCH 14/23] AB#12334 Serialize the manifest test RIDs Disable runtime-identifier parallelism on the one manifest test that races its intermediate APK while leaving all other Helix test builds at their normal parallelism. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 | 2 -- .../automation/host-build-tests-helix/HostBuildTestsHelix.psm1 | 2 -- .../Tests/Xamarin.Android.Build.Tests/ManifestTest.cs | 1 + 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 index 9dc6e10ee11..127d0d2187c 100644 --- a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 +++ b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 @@ -130,8 +130,6 @@ try { -not $command.Contains('bin\Release\dotnet') -or $command.Contains('set "CONFIGURATION=Release"') -or -not $command.Contains('set "RUNNINGONCI=true"') -or - -not $command.Contains('set "BuildInParallel=false"') -or - -not $command.Contains('set "_AndroidBuildRuntimeIdentifiersInParallel=false"') -or -not $command.Contains('set "TEST_ANDROID_NDK_PATH=') -or -not $command.Contains('tar.exe -a -c -f diagnostics.zip')) { throw 'The generated command did not apply the requested build configuration.' diff --git a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 index 96c8a3f9c53..e7cf1cac6b6 100644 --- a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 +++ b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 @@ -303,8 +303,6 @@ set "TMP=%TEMP%" set "BUILD_STAGINGDIRECTORY=%HELIX_WORKITEM_UPLOAD_ROOT%" set "PATH=%HELIX_CORRELATION_PAYLOAD%\dotnet-tools;%DOTNET_ROOT%;%ANDROID_HOME%\platform-tools;%JAVA_HOME%\bin;%PATH%" set "RUNNINGONCI=true" -set "BuildInParallel=false" -set "_AndroidBuildRuntimeIdentifiersInParallel=false" set "DOTNET_CLI_TELEMETRY_OPTOUT=1" set "DOTNET_NOLOGO=1" set "DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1" diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ManifestTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ManifestTest.cs index 372c772e547..3d9a1f5f9b4 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ManifestTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ManifestTest.cs @@ -1130,6 +1130,7 @@ public void AllForegroundServiceTypes (string serviceType, string expected, Andr IsRelease = isRelease, }; proj.SetRuntime (runtime); + proj.SetProperty ("_AndroidBuildRuntimeIdentifiersInParallel", "false"); proj.Sources.Add (new BuildItem.Source ("TestActivity.cs") { TextContent = () => $@"using Android.App; From 8fbd7426449a4c7d6940b50f73980e1e083dab43 Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Tue, 25 Aug 2026 02:56:16 -0700 Subject: [PATCH 15/23] AB#12334 Shorten Helix test output paths Keep build-test staging under the short work-item root and archive it afterward, avoiding the repeated long-path package failure while retaining diagnostics in the Helix upload root. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../HostBuildTestsHelix.Tests.ps1 | 3 ++- .../host-build-tests-helix/HostBuildTestsHelix.psm1 | 13 ++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 index 127d0d2187c..c4a721f3030 100644 --- a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 +++ b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 @@ -131,7 +131,8 @@ try { $command.Contains('set "CONFIGURATION=Release"') -or -not $command.Contains('set "RUNNINGONCI=true"') -or -not $command.Contains('set "TEST_ANDROID_NDK_PATH=') -or - -not $command.Contains('tar.exe -a -c -f diagnostics.zip')) { + -not $command.Contains('set "BUILD_STAGINGDIRECTORY=%HELIX_WORKITEM_ROOT%\o"') -or + -not $command.Contains('tar.exe -a -c -f "%HELIX_WORKITEM_UPLOAD_ROOT%\diagnostics.zip"')) { throw 'The generated command did not apply the requested build configuration.' } Assert-Equal 'results.trx;console.log;slice.runsettings;work-item.json;diagnostics.zip' ([string] $item.DownloadFilesFromResults) 'Windows results should be downloaded.' diff --git a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 index e7cf1cac6b6..5cdf782af97 100644 --- a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 +++ b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 @@ -300,7 +300,7 @@ set "GRADLE_USER_HOME=%HELIX_CORRELATION_PAYLOAD%\gradle" set "DOTNET_CLI_HOME=%HELIX_WORKITEM_ROOT%\dotnet-home" set "TEMP=%HELIX_WORKITEM_ROOT%\temp" set "TMP=%TEMP%" -set "BUILD_STAGINGDIRECTORY=%HELIX_WORKITEM_UPLOAD_ROOT%" +set "BUILD_STAGINGDIRECTORY=%HELIX_WORKITEM_ROOT%\o" set "PATH=%HELIX_CORRELATION_PAYLOAD%\dotnet-tools;%DOTNET_ROOT%;%ANDROID_HOME%\platform-tools;%JAVA_HOME%\bin;%PATH%" set "RUNNINGONCI=true" set "DOTNET_CLI_TELEMETRY_OPTOUT=1" @@ -320,15 +320,14 @@ mkdir "%HELIX_WORKITEM_UPLOAD_ROOT%\dumps" 2>nul mkdir "%DOTNET_CLI_HOME%" 2>nul mkdir "%NUGET_PACKAGES%" 2>nul mkdir "%TEMP%" 2>nul +mkdir "%BUILD_STAGINGDIRECTORY%" 2>nul copy /y "%~dp0slice.runsettings" "%HELIX_WORKITEM_UPLOAD_ROOT%\slice.runsettings" >nul copy /y "%~dp0work-item.json" "%HELIX_WORKITEM_UPLOAD_ROOT%\work-item.json" >nul pushd "%REPO%" "%DOTNET_ROOT%\dotnet.exe" test "%REPO%\__TEST_ASSEMBLY__" --settings "%~dp0slice.runsettings" --logger "trx;LogFileName=results.trx" --results-directory "%HELIX_WORKITEM_UPLOAD_ROOT%" -- NUnit.NumberOfTestWorkers=__NUNIT_WORKERS__ > "%HELIX_WORKITEM_UPLOAD_ROOT%\console.log" 2>&1 set "testExitCode=%ERRORLEVEL%" type "%HELIX_WORKITEM_UPLOAD_ROOT%\console.log" -pushd "%HELIX_WORKITEM_UPLOAD_ROOT%" -tar.exe -a -c -f diagnostics.zip --exclude=diagnostics.zip TestRelease dumps >nul 2>&1 -popd +tar.exe -a -c -f "%HELIX_WORKITEM_UPLOAD_ROOT%\diagnostics.zip" -C "%BUILD_STAGINGDIRECTORY%" . >nul 2>&1 "%DOTNET_ROOT%\dotnet.exe" build-server shutdown >nul 2>&1 popd exit /b %testExitCode% @@ -354,7 +353,7 @@ export NUGET_PACKAGES="$HELIX_WORKITEM_ROOT/nuget-packages" export GRADLE_USER_HOME="$HELIX_CORRELATION_PAYLOAD/gradle" export DOTNET_CLI_HOME="$HELIX_WORKITEM_ROOT/dotnet-home" export TMPDIR="$HELIX_WORKITEM_ROOT/temp" -export BUILD_STAGINGDIRECTORY="$HELIX_WORKITEM_UPLOAD_ROOT" +export BUILD_STAGINGDIRECTORY="$HELIX_WORKITEM_ROOT/o" export PATH="$HELIX_CORRELATION_PAYLOAD/dotnet-tools:$DOTNET_ROOT:$ANDROID_HOME/platform-tools:$JAVA_HOME/bin:$PATH" export RUNNINGONCI=true export DOTNET_CLI_TELEMETRY_OPTOUT=1 @@ -370,7 +369,7 @@ export DOTNET_DbgMiniDumpType=2 export DOTNET_DbgMiniDumpName="$HELIX_WORKITEM_UPLOAD_ROOT/dumps/%e.%p.dmp" test -x "$DOTNET_ROOT/dotnet" test -f "$REPO/__TEST_ASSEMBLY__" -mkdir -p "$HELIX_WORKITEM_UPLOAD_ROOT/dumps" "$DOTNET_CLI_HOME" "$NUGET_PACKAGES" "$TMPDIR" +mkdir -p "$HELIX_WORKITEM_UPLOAD_ROOT/dumps" "$DOTNET_CLI_HOME" "$NUGET_PACKAGES" "$TMPDIR" "$BUILD_STAGINGDIRECTORY" cp "$HELIX_WORKITEM_ROOT/slice.runsettings" "$HELIX_WORKITEM_UPLOAD_ROOT/slice.runsettings" cp "$HELIX_WORKITEM_ROOT/work-item.json" "$HELIX_WORKITEM_UPLOAD_ROOT/work-item.json" cd "$REPO" @@ -378,7 +377,7 @@ set +e "$DOTNET_ROOT/dotnet" test "$REPO/__TEST_ASSEMBLY__" --settings "$HELIX_WORKITEM_ROOT/slice.runsettings" --logger "trx;LogFileName=results.trx" --results-directory "$HELIX_WORKITEM_UPLOAD_ROOT" -- NUnit.NumberOfTestWorkers=__NUNIT_WORKERS__ 2>&1 | tee "$HELIX_WORKITEM_UPLOAD_ROOT/console.log" test_exit=${PIPESTATUS[0]} set -e -(cd "$HELIX_WORKITEM_UPLOAD_ROOT" && find . -type f \( -name '*.binlog' -o -name '*.dmp' -o -name '*.log' -o -name '*.txt' \) -print0 | tar --null -czf diagnostics.tar.gz --files-from=-) || true +tar -czf "$HELIX_WORKITEM_UPLOAD_ROOT/diagnostics.tar.gz" -C "$BUILD_STAGINGDIRECTORY" . || true "$DOTNET_ROOT/dotnet" build-server shutdown >/dev/null 2>&1 || true exit "$test_exit" '@ From aa0f9d93c63b8daf0db3fcb971aa873364e15592 Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Tue, 25 Aug 2026 06:00:08 -0700 Subject: [PATCH 16/23] AB#12334 Stabilize slow Helix host tests Limit the foreground-service manifest test to one ABI and allow apkdiff up to two minutes on CI, matching the test intent without Helix path/rate sensitivity. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Tests/Xamarin.Android.Build.Tests/ManifestTest.cs | 1 + .../Tests/Xamarin.Android.Build.Tests/Utilities/BaseTest.cs | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ManifestTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ManifestTest.cs index 3d9a1f5f9b4..365f5580d41 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ManifestTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ManifestTest.cs @@ -1131,6 +1131,7 @@ public void AllForegroundServiceTypes (string serviceType, string expected, Andr }; proj.SetRuntime (runtime); proj.SetProperty ("_AndroidBuildRuntimeIdentifiersInParallel", "false"); + proj.SetRuntimeIdentifiers (["arm64-v8a"]); proj.Sources.Add (new BuildItem.Source ("TestActivity.cs") { TextContent = () => $@"using Android.App; diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/BaseTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/BaseTest.cs index 5c0fd14fc3c..436393b612b 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/BaseTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/Utilities/BaseTest.cs @@ -110,7 +110,8 @@ protected static (int code, string stdOutput, string stdError) RunApkDiffCommand }; using var stdOutput = new StringWriter (); using var stdError = new StringWriter (); - using var cancellationTokenSource = new CancellationTokenSource (TimeSpan.FromSeconds (30)); + int timeoutInSeconds = TestEnvironment.IsRunningOnCI ? 120 : 30; + using var cancellationTokenSource = new CancellationTokenSource (TimeSpan.FromSeconds (timeoutInSeconds)); int processId = -1; int exitCode; try { @@ -123,7 +124,7 @@ protected static (int code, string stdOutput, string stdError) RunApkDiffCommand ).GetAwaiter ().GetResult (); } catch (OperationCanceledException) { exitCode = -1; - stdError.WriteLine ($"apkdiff timed out after 30 seconds (PID {processId})."); + stdError.WriteLine ($"apkdiff timed out after {timeoutInSeconds} seconds (PID {processId})."); } var result = (code: exitCode, stdOutput: stdOutput.ToString ().Trim (), stdError: stdError.ToString ().Trim ()); From d5f35a76111ff10f088c1617b2569a6bb89d27c1 Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Tue, 25 Aug 2026 09:05:11 -0700 Subject: [PATCH 17/23] AB#12334 Shorten the foreground-service project path Give the foreground-service manifest case a compact explicit project directory so native Android tools remain below Windows path limits on Helix. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Tests/Xamarin.Android.Build.Tests/ManifestTest.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ManifestTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ManifestTest.cs index 365f5580d41..46983cb3866 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ManifestTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ManifestTest.cs @@ -1140,7 +1140,8 @@ public void AllForegroundServiceTypes (string serviceType, string expected, Andr [Service (ForegroundServiceType = {serviceType})] class TestService : Service {{ public override Android.OS.IBinder OnBind (Android.Content.Intent intent) {{ return null; }} }}" }); - using (ProjectBuilder builder = CreateApkBuilder ()) { + string projectDirectory = Path.Combine ("temp", $"ForegroundService_{expected}_{runtime}"); + using (ProjectBuilder builder = CreateApkBuilder (projectDirectory)) { Assert.IsTrue (builder.Build (proj), "Build should have succeeded"); string manifest = builder.Output.GetIntermediaryAsText (Path.Combine ("android", "AndroidManifest.xml")); var doc = XDocument.Parse (manifest); From 92174cc1f27f192b813ec338b9d7086512e137bb Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Tue, 25 Aug 2026 12:08:26 -0700 Subject: [PATCH 18/23] AB#12334 Sanitize the short manifest-test path Replace the combined foreground-service separator before using it as a Windows directory name. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Tests/Xamarin.Android.Build.Tests/ManifestTest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ManifestTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ManifestTest.cs index 46983cb3866..4a591c86508 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ManifestTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/ManifestTest.cs @@ -1140,7 +1140,7 @@ public void AllForegroundServiceTypes (string serviceType, string expected, Andr [Service (ForegroundServiceType = {serviceType})] class TestService : Service {{ public override Android.OS.IBinder OnBind (Android.Content.Intent intent) {{ return null; }} }}" }); - string projectDirectory = Path.Combine ("temp", $"ForegroundService_{expected}_{runtime}"); + string projectDirectory = Path.Combine ("temp", $"ForegroundService_{expected.Replace ('|', '_')}_{runtime}"); using (ProjectBuilder builder = CreateApkBuilder (projectDirectory)) { Assert.IsTrue (builder.Build (proj), "Build should have succeeded"); string manifest = builder.Output.GetIntermediaryAsText (Path.Combine ("android", "AndroidManifest.xml")); From 59f4a5254a93716176533356fc8b30dae8c3c656 Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Tue, 25 Aug 2026 14:42:32 -0700 Subject: [PATCH 19/23] AB#12334 Publish Helix results from the parent job Disable the per-work-item Azure Pipelines reporter because downloaded TRX is published centrally, avoiding duplicate reporting and transient DevOps reporter deadletters. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- build-tools/automation/HostBuildTestsHelix.proj | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build-tools/automation/HostBuildTestsHelix.proj b/build-tools/automation/HostBuildTestsHelix.proj index ff2d19cebbb..fff5bfc550d 100644 --- a/build-tools/automation/HostBuildTestsHelix.proj +++ b/build-tools/automation/HostBuildTestsHelix.proj @@ -1,8 +1,7 @@ msbuild - true - false + false true tests/android/host-build-tests/ Xamarin.Android.Build.Tests - Helix From 8b6f8656e37bc325aa82cac0ea131765ba44d93b Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Tue, 25 Aug 2026 20:20:18 -0700 Subject: [PATCH 20/23] AB#12334 Isolate shared restore-state tests Run the local-feed and Maven dependency-verification tests non-parallel after concrete Helix-only restore-state flakes across otherwise passing runs. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Tests/Xamarin.Android.Build.Tests/BindingBuildTest.cs | 1 + .../Tests/Xamarin.Android.Build.Tests/BuildTest.cs | 1 + 2 files changed, 2 insertions(+) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BindingBuildTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BindingBuildTest.cs index 7e3cf931004..09afbb2d19e 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BindingBuildTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BindingBuildTest.cs @@ -1152,6 +1152,7 @@ public void AndroidMavenLibrary ([Values (AndroidRuntime.CoreCLR, AndroidRuntime } [Test] + [NonParallelizable] public void AndroidMavenLibrary_FailsDueToUnverifiedDependency ([Values (AndroidRuntime.CoreCLR, AndroidRuntime.NativeAOT)] AndroidRuntime runtime) { bool isRelease = runtime == AndroidRuntime.NativeAOT; diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs index b428d8b2858..9db2e315748 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BuildTest.cs @@ -1087,6 +1087,7 @@ public void MinorApiLevelFallbackThrowsXA5207 () } [Test] + [NonParallelizable] public void BuildWithDeviceLockedStateListener () { var stableApiLevel = XABuildConfig.AndroidLatestStableApiLevel; From 84e659234b7130777440673b172cf642344eef59 Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Wed, 26 Aug 2026 07:56:19 -0700 Subject: [PATCH 21/23] AB#12334 Include Linux local feed and serialize Maven tests Ship the local nuget-unsigned source on Linux and keep all Maven dependency-verification cases out of NUnit's parallel queue. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../prepare-host-build-test-helix-payload.ps1 | 8 +++----- .../Tests/Xamarin.Android.Build.Tests/BindingBuildTest.cs | 2 ++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 b/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 index 37aa4948994..4bcd83814e4 100644 --- a/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 +++ b/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 @@ -322,11 +322,9 @@ Add-DirectoryCorrelationPayloads -Source $payloadRepository -Destination 'repo' Add-DirectoryCorrelationPayloads ` -Source (Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath "bin/$Configuration")) ` -Destination "repo/bin/$Configuration" -if ($Platform -eq 'windows') { - Add-DirectoryCorrelationPayloads ` - -Source (Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath "bin/Build$Configuration/nuget-unsigned")) ` - -Destination "repo/bin/Build$Configuration/nuget-unsigned" -} +Add-DirectoryCorrelationPayloads ` + -Source (Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath "bin/Build$Configuration/nuget-unsigned")) ` + -Destination "repo/bin/Build$Configuration/nuget-unsigned" $testOutputRoot = Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath "bin/Test$Configuration") $testAssembly = Join-Path $repositoryRootFullPath (ConvertTo-NativeRelativePath $TestAssemblyRelativePath) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BindingBuildTest.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BindingBuildTest.cs index 09afbb2d19e..fbd392ab70b 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BindingBuildTest.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/BindingBuildTest.cs @@ -1182,6 +1182,7 @@ public void AndroidMavenLibrary_FailsDueToUnverifiedDependency ([Values (Android } [Test] + [NonParallelizable] public void AndroidMavenLibrary_IgnoreDependencyVerification ([Values (AndroidRuntime.CoreCLR, AndroidRuntime.NativeAOT)] AndroidRuntime runtime) { bool isRelease = runtime == AndroidRuntime.NativeAOT; @@ -1209,6 +1210,7 @@ public void AndroidMavenLibrary_IgnoreDependencyVerification ([Values (AndroidRu } [Test] + [NonParallelizable] public void AndroidMavenLibrary_AllDependenciesAreVerified ([Values (AndroidRuntime.CoreCLR, AndroidRuntime.NativeAOT)] AndroidRuntime runtime) { bool isRelease = runtime == AndroidRuntime.NativeAOT; From 2c6d9436cda608eedebba92119a4075055767200 Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Wed, 26 Aug 2026 08:00:15 -0700 Subject: [PATCH 22/23] AB#12334 Retry failed Helix tests once Ship dotnet-test-slicer in the shared payload and reuse its existing retry flow so transient failures are rerun once while persistent failures still fail the work item. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../HostBuildTestsHelix.Tests.ps1 | 5 ++-- .../HostBuildTestsHelix.psm1 | 27 +++++++++++++++++-- .../host-build-tests-helix/README.md | 2 ++ .../prepare-host-build-test-helix-payload.ps1 | 18 +++++++------ 4 files changed, 40 insertions(+), 12 deletions(-) diff --git a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 index c4a721f3030..80e743e89a2 100644 --- a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 +++ b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.Tests.ps1 @@ -132,10 +132,11 @@ try { -not $command.Contains('set "RUNNINGONCI=true"') -or -not $command.Contains('set "TEST_ANDROID_NDK_PATH=') -or -not $command.Contains('set "BUILD_STAGINGDIRECTORY=%HELIX_WORKITEM_ROOT%\o"') -or + -not $command.Contains('dotnet-test-slicer.exe" retry') -or -not $command.Contains('tar.exe -a -c -f "%HELIX_WORKITEM_UPLOAD_ROOT%\diagnostics.zip"')) { throw 'The generated command did not apply the requested build configuration.' } - Assert-Equal 'results.trx;console.log;slice.runsettings;work-item.json;diagnostics.zip' ([string] $item.DownloadFilesFromResults) 'Windows results should be downloaded.' + Assert-Equal 'results.trx;retry-results.trx;console.log;slice.runsettings;work-item.json;diagnostics.zip' ([string] $item.DownloadFilesFromResults) 'Windows results should be downloaded.' } $linuxPayloadDirectory = Join-Path $testRoot 'linux-payloads' @@ -151,7 +152,7 @@ try { [xml] $linuxProps = Get-Content -LiteralPath $linuxPropsPath -Raw foreach ($item in @($linuxProps.Project.ItemGroup._HostBuildTestHelixWorkItem)) { Assert-Equal 'bash run-host-tests.sh' ([string] $item.Command) 'Linux work items should explicitly use bash.' - Assert-Equal 'results.trx;console.log;slice.runsettings;work-item.json;diagnostics.tar.gz' ([string] $item.DownloadFilesFromResults) 'Linux results should be downloaded.' + Assert-Equal 'results.trx;retry-results.trx;console.log;slice.runsettings;work-item.json;diagnostics.tar.gz' ([string] $item.DownloadFilesFromResults) 'Linux results should be downloaded.' } Write-Host 'HostBuildTestsHelix tests passed.' diff --git a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 index 5cdf782af97..d305c23aa4b 100644 --- a/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 +++ b/build-tools/automation/host-build-tests-helix/HostBuildTestsHelix.psm1 @@ -284,7 +284,7 @@ function Write-HostBuildTestCommand $assemblyPath = $TestAssemblyRelativePath.Replace('/', '\') $command = @' @echo off -setlocal +setlocal EnableDelayedExpansion if "%HELIX_CORRELATION_PAYLOAD%"=="" exit /b 2 if "%HELIX_WORKITEM_ROOT%"=="" exit /b 2 if "%HELIX_WORKITEM_UPLOAD_ROOT%"=="" exit /b 2 @@ -327,6 +327,18 @@ pushd "%REPO%" "%DOTNET_ROOT%\dotnet.exe" test "%REPO%\__TEST_ASSEMBLY__" --settings "%~dp0slice.runsettings" --logger "trx;LogFileName=results.trx" --results-directory "%HELIX_WORKITEM_UPLOAD_ROOT%" -- NUnit.NumberOfTestWorkers=__NUNIT_WORKERS__ > "%HELIX_WORKITEM_UPLOAD_ROOT%\console.log" 2>&1 set "testExitCode=%ERRORLEVEL%" type "%HELIX_WORKITEM_UPLOAD_ROOT%\console.log" +if not "!testExitCode!"=="0" if exist "%HELIX_WORKITEM_UPLOAD_ROOT%\results.trx" if exist "%HELIX_CORRELATION_PAYLOAD%\dotnet-tools\dotnet-test-slicer.exe" ( + "%HELIX_CORRELATION_PAYLOAD%\dotnet-tools\dotnet-test-slicer.exe" retry --trx="%HELIX_WORKITEM_UPLOAD_ROOT%\results.trx" --outfile="%~dp0retry.runsettings" + if not errorlevel 1 ( + findstr /C:"dotnet-slicer-dummy-test-name" "%~dp0retry.runsettings" >nul + if errorlevel 1 ( + copy /y "%~dp0retry.runsettings" "%HELIX_WORKITEM_UPLOAD_ROOT%\retry.runsettings" >nul + "%DOTNET_ROOT%\dotnet.exe" test "%REPO%\__TEST_ASSEMBLY__" --settings "%~dp0retry.runsettings" --logger "trx;LogFileName=retry-results.trx" --results-directory "%HELIX_WORKITEM_UPLOAD_ROOT%" -- NUnit.NumberOfTestWorkers=__NUNIT_WORKERS__ > "%HELIX_WORKITEM_UPLOAD_ROOT%\retry-console.log" 2>&1 + set "testExitCode=!ERRORLEVEL!" + type "%HELIX_WORKITEM_UPLOAD_ROOT%\retry-console.log" + ) + ) +) tar.exe -a -c -f "%HELIX_WORKITEM_UPLOAD_ROOT%\diagnostics.zip" -C "%BUILD_STAGINGDIRECTORY%" . >nul 2>&1 "%DOTNET_ROOT%\dotnet.exe" build-server shutdown >nul 2>&1 popd @@ -377,6 +389,17 @@ set +e "$DOTNET_ROOT/dotnet" test "$REPO/__TEST_ASSEMBLY__" --settings "$HELIX_WORKITEM_ROOT/slice.runsettings" --logger "trx;LogFileName=results.trx" --results-directory "$HELIX_WORKITEM_UPLOAD_ROOT" -- NUnit.NumberOfTestWorkers=__NUNIT_WORKERS__ 2>&1 | tee "$HELIX_WORKITEM_UPLOAD_ROOT/console.log" test_exit=${PIPESTATUS[0]} set -e +if [[ "$test_exit" -ne 0 && -f "$HELIX_WORKITEM_UPLOAD_ROOT/results.trx" && -x "$HELIX_CORRELATION_PAYLOAD/dotnet-tools/dotnet-test-slicer" ]]; then + if "$HELIX_CORRELATION_PAYLOAD/dotnet-tools/dotnet-test-slicer" retry --trx="$HELIX_WORKITEM_UPLOAD_ROOT/results.trx" --outfile="$HELIX_WORKITEM_ROOT/retry.runsettings"; then + if ! grep -q 'dotnet-slicer-dummy-test-name' "$HELIX_WORKITEM_ROOT/retry.runsettings"; then + cp "$HELIX_WORKITEM_ROOT/retry.runsettings" "$HELIX_WORKITEM_UPLOAD_ROOT/retry.runsettings" + set +e + "$DOTNET_ROOT/dotnet" test "$REPO/__TEST_ASSEMBLY__" --settings "$HELIX_WORKITEM_ROOT/retry.runsettings" --logger "trx;LogFileName=retry-results.trx" --results-directory "$HELIX_WORKITEM_UPLOAD_ROOT" -- NUnit.NumberOfTestWorkers=__NUNIT_WORKERS__ 2>&1 | tee "$HELIX_WORKITEM_UPLOAD_ROOT/retry-console.log" + test_exit=${PIPESTATUS[0]} + set -e + fi + fi +fi tar -czf "$HELIX_WORKITEM_UPLOAD_ROOT/diagnostics.tar.gz" -C "$BUILD_STAGINGDIRECTORY" . || true "$DOTNET_ROOT/dotnet" build-server shutdown >/dev/null 2>&1 || true exit "$test_exit" @@ -458,7 +481,7 @@ function Write-HostBuildTestWorkItemPayloads $propsWriter.WriteElementString('PayloadDirectory', $payloadDirectory) $propsWriter.WriteElementString('Command', $helixCommand) $diagnosticsArchive = if ($Platform -eq 'windows') { 'diagnostics.zip' } else { 'diagnostics.tar.gz' } - $propsWriter.WriteElementString('DownloadFilesFromResults', "results.trx;console.log;slice.runsettings;work-item.json;$diagnosticsArchive") + $propsWriter.WriteElementString('DownloadFilesFromResults', "results.trx;retry-results.trx;console.log;slice.runsettings;work-item.json;$diagnosticsArchive") $propsWriter.WriteElementString('EstimatedDurationMs', [string] $workItem.EstimatedDurationMs) $propsWriter.WriteElementString('TestCount', [string] $workItem.Tests.Count) $propsWriter.WriteEndElement() diff --git a/build-tools/automation/host-build-tests-helix/README.md b/build-tools/automation/host-build-tests-helix/README.md index c4bf8e5d228..1e6563e2aa2 100644 --- a/build-tools/automation/host-build-tests-helix/README.md +++ b/build-tools/automation/host-build-tests-helix/README.md @@ -27,6 +27,8 @@ as duration-balanced. Each completed run publishes a new `balance.xml`; pass tha run's build ID through `hostBuildTestsHelixTimingBuildId` on the next queued build. TRX, console output, runsettings, work-item metadata, and a diagnostic archive are downloaded from each Helix results container for publication in Azure Pipelines. +If an initial test invocation fails, the existing `dotnet-test-slicer retry` flow +reruns only the failed tests once; persistent failures retain a nonzero work-item exit. The public pipeline keeps the existing Azure Pipelines jobs and adds this path only when `enableHostBuildTestsHelixPrototype` is `true`. diff --git a/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 b/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 index 4bcd83814e4..8de2b320b44 100644 --- a/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 +++ b/build-tools/automation/host-build-tests-helix/prepare-host-build-test-helix-payload.ps1 @@ -307,15 +307,17 @@ foreach ($sourceDirectory in Get-ChildItem -LiteralPath $sourceRoot -Directory | $payloadTools = Join-Path $correlationPayloadFullPath 'dotnet-tools' if (-not [string]::IsNullOrWhiteSpace($DotNetToolsDirectory)) { New-Item -ItemType Directory -Force -Path $payloadTools | Out-Null - $apkDiffShim = if ($Platform -eq 'windows') { 'apkdiff.exe' } else { 'apkdiff' } - $apkDiffShimPath = Join-Path $DotNetToolsDirectory $apkDiffShim - if (Test-Path -LiteralPath $apkDiffShimPath -PathType Leaf) { - Copy-Item -LiteralPath $apkDiffShimPath -Destination $payloadTools + foreach ($toolName in @('apkdiff', 'dotnet-test-slicer')) { + $toolShim = if ($Platform -eq 'windows') { "$toolName.exe" } else { $toolName } + $toolShimPath = Join-Path $DotNetToolsDirectory $toolShim + if (Test-Path -LiteralPath $toolShimPath -PathType Leaf) { + Copy-Item -LiteralPath $toolShimPath -Destination $payloadTools + } + Copy-PayloadDirectory ` + -Source (Join-Path $DotNetToolsDirectory (ConvertTo-NativeRelativePath ".store/$toolName")) ` + -Destination (Join-Path $payloadTools (ConvertTo-NativeRelativePath ".store/$toolName")) ` + -Optional } - Copy-PayloadDirectory ` - -Source (Join-Path $DotNetToolsDirectory (ConvertTo-NativeRelativePath '.store/apkdiff')) ` - -Destination (Join-Path $payloadTools (ConvertTo-NativeRelativePath '.store/apkdiff')) ` - -Optional } Add-DirectoryCorrelationPayloads -Source $payloadRepository -Destination 'repo' From 188932a9692f15ecb49e47446c0f24b1e914c512 Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Wed, 26 Aug 2026 10:34:39 -0700 Subject: [PATCH 23/23] AB#12334 Calibrate Windows and Linux independently Use measured platform-specific effective parallelism so Linux remains centered near 15 minutes while Windows moves from a 19-minute median toward the target. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- build-tools/automation/azure-pipelines-public.yaml | 8 ++++++-- build-tools/automation/host-build-tests-helix/README.md | 5 +++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/build-tools/automation/azure-pipelines-public.yaml b/build-tools/automation/azure-pipelines-public.yaml index 76992423ad1..52b180b4f7f 100644 --- a/build-tools/automation/azure-pipelines-public.yaml +++ b/build-tools/automation/azure-pipelines-public.yaml @@ -40,7 +40,11 @@ parameters: type: number default: 15 - name: hostBuildTestsHelixDurationParallelism - displayName: Host build tests timing parallelism + displayName: Windows host build tests timing parallelism + type: number + default: 1.3 +- name: hostBuildTestsHelixLinuxDurationParallelism + displayName: Linux host build tests timing parallelism type: number default: 1.65 - name: hostBuildTestsHelixFallbackTestDurationSeconds @@ -241,7 +245,7 @@ stages: timingBuildId: ${{ parameters.hostBuildTestsHelixTimingBuildId }} timingArtifactName: Host Build Test Timings - linux targetMinutes: ${{ parameters.hostBuildTestsHelixTargetMinutes }} - durationParallelism: ${{ parameters.hostBuildTestsHelixDurationParallelism }} + durationParallelism: ${{ parameters.hostBuildTestsHelixLinuxDurationParallelism }} fallbackTestDurationSeconds: ${{ parameters.hostBuildTestsHelixFallbackTestDurationSeconds }} workItemTimeout: ${{ parameters.hostBuildTestsHelixWorkItemTimeout }} diff --git a/build-tools/automation/host-build-tests-helix/README.md b/build-tools/automation/host-build-tests-helix/README.md index 1e6563e2aa2..01baf2cf5b4 100644 --- a/build-tools/automation/host-build-tests-helix/README.md +++ b/build-tools/automation/host-build-tests-helix/README.md @@ -18,8 +18,9 @@ and unrelated test assemblies are also excluded. The prepared NDK and the local decreasing bin packing. The target is `15` estimated wall-clock minutes by default. `durationParallelism` translates summed per-test elapsed time into expected wall time while preserving the current NUnit worker setting. The public prototype defaults this -factor to `1.65`, uses `60` seconds per test for the explicit first-run fallback, and -allows `90` minutes before Helix terminates a first-run work item. +factor to `1.3` on Windows and `1.65` on Linux, uses `60` seconds per test for the +explicit first-run fallback, and allows `90` minutes before Helix terminates a +first-run work item. If no timing history is provided, generation explicitly reports `count-fallback` and uses the configured fallback duration per test. It does not label count-only slicing