From 0def8c92b23210150cedf22183cba14c5014ec5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Wed, 5 Aug 2026 12:30:27 +0200 Subject: [PATCH 1/5] Prepare affected-test selection rollout Use the public SDK build containing affected-test commands and prepare a credential-free Azure Pipelines Cache transport with safe full-test and coverage fallbacks. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3baa2cd4-db32-4f9f-b6b7-8cc190f3799f --- azure-pipelines.yml | 10 ++ docs/README.md | 1 + docs/affected-test-selection.md | 63 ++++++++ .../steps/test-windows-debug-coverage.yml | 112 ++++++++++++-- eng/validate-affected-tests.ps1 | 143 ++++++++++++++++++ global.json | 41 ++++- 6 files changed, 356 insertions(+), 14 deletions(-) create mode 100644 docs/affected-test-selection.md create mode 100644 eng/validate-affected-tests.ps1 diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 44bb5beb88..5c1226ec9b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -290,6 +290,10 @@ stages: condition: and(succeeded(), eq(variables._BuildConfig, 'Debug'), eq(variables['MSBuildCacheBuildSucceeded'], 'true')) - template: /eng/pipelines/steps/test-windows-debug-coverage.yml + parameters: + # Enable only after the prerequisites in docs/affected-test-selection.md are satisfied. + enableAffectedTests: false + affectedTestsMode: collect - task: PublishBuildArtifacts@1 displayName: 'Publish cache seed build binlogs' @@ -326,6 +330,8 @@ stages: - checkout: self fetchDepth: 0 clean: false + - pwsh: ./eng/validate-affected-tests.ps1 + displayName: Validate affected-test rollout - bash: | set -euo pipefail @@ -710,6 +716,10 @@ stages: - ${{ if eq(parameters.SkipTests, False) }}: - template: /eng/pipelines/steps/test-windows-debug-coverage.yml + parameters: + # Enable only after the prerequisites in docs/affected-test-selection.md are satisfied. + enableAffectedTests: false + affectedTestsMode: run # Integration tests are redundant in Release—they spawn child processes that already cover # both Debug and Release configurations. Use --test-modules to run only unit tests. diff --git a/docs/README.md b/docs/README.md index 6a0d5ba7bb..252f1943cb 100644 --- a/docs/README.md +++ b/docs/README.md @@ -39,6 +39,7 @@ For technical reasoning and implementation details, you can refer to the list of ## Guides +- [Affected-test selection rollout](affected-test-selection.md): disabled adoption layout and activation prerequisites. - [Testing WinUI apps](winui-testing.md): packaged (MSIX) vs unpackaged WinUI, and how the test host is started for each. ## Design notes diff --git a/docs/affected-test-selection.md b/docs/affected-test-selection.md new file mode 100644 index 0000000000..e763fe05e9 --- /dev/null +++ b/docs/affected-test-selection.md @@ -0,0 +1,63 @@ +# Affected-test selection rollout + +This repository is prepared to adopt the experimental affected-test workflow from +[dotnet/sdk#55574](https://github.com/dotnet/sdk/pull/55574). The workflow is Microsoft.Testing.Platform-only and +builds on the composable filter-provider support from +[testfx#10235](https://github.com/microsoft/testfx/pull/10235). + +The rollout is intentionally disabled. SDK `11.0.100-rc.1.26406.108` contains the affected-test commands, but the +affected-test extension package and its public local-filesystem storage contract are not available yet. Ordinary test +commands therefore remain unchanged. + +## Prepared layout + +- `global.json` defines the repository-specific `test.affectedTests` change and instrumentation scopes. +- The trusted main-branch Windows Debug test is the future `--collect-test-map` entry point. +- The Windows Debug PR test is the future `--affected-tests` entry point. +- Both pipeline call sites pass `enableAffectedTests: false`. The inactive template branches restore the map through + Azure Pipelines `Cache@2` and set `DOTNET_CLI_ENABLE_AFFECTED_TESTS=1` only for the affected-test commands. +- `eng/validate-affected-tests.ps1` protects the disabled state and verifies that the public SDK gate and command names + do not drift. + +`DOTNET_CLI_TEST_AFFECTED_TESTS_MODE` is an SDK-to-extension authorization marker. Repository scripts and pipeline +definitions must not set it. + +## Storage design + +The map should use the extension's local-filesystem provider rooted at +`$(Pipeline.Workspace)\affected-test-map`. Azure Pipelines `Cache@2` transfers that directory between runs without +credentials: + +- trusted main builds can restore the previous map and publish a new immutable cache entry; +- PR and fork-PR builds can read the target branch's cache scope but cannot write to it; +- the cache prefix includes its manual compatibility version, OS, architecture, and configuration; +- the unique build ID suffix lets every successful main collection publish a new map; +- prefix restore selects the newest compatible map. + +Azure Pipelines caches expire after seven days without activity. A cache miss is therefore an expected state, not a +test failure: the PR lane runs the unchanged full test command. The same fallback runs when the extension rejects a +missing, stale, or incompatible map, and scheduled or manual builds always keep full validation. + +Selected-test runs do not publish their partial coverage as the repository coverage report. Collection and full +fallback runs still publish complete coverage. + +Pipeline artifacts should contain only non-secret diagnostics or a mapping snapshot suitable for troubleshooting. +They are not the cross-run source of truth because artifact lookup and retention are tied to individual builds. + +The `storage` property is deliberately absent from `test.affectedTests` until the extension package publishes the exact +local-filesystem provider schema. Adding an invented provider or path setting now would create configuration that +cannot be validated. + +## Activation checklist + +1. Add the publicly available affected-test extension package through `Directory.Packages.props` and the test project + infrastructure, following the repository's normal dependency-flow and package-source policy. +2. Add `test.affectedTests.storage` using the package's published local-filesystem schema and point it at the Pipeline + Cache directory. +3. Update `affectedTestsCacheVersion` whenever the persisted map format or its compatibility dimensions change. +4. Enable `collect` in the main-branch cache-seed call site and publish non-secret diagnostics as an Azure DevOps + artifact. +5. After a compatible map exists, enable `run` in the PR call site. Keep the full test command available as an explicit + rollback by setting `enableAffectedTests` back to `false`. +6. Validate a documentation-only change, a product change with a narrow affected set, a force-all change, a missing or + incompatible map, a fork PR without secrets, and a collection failure before making selection required. diff --git a/eng/pipelines/steps/test-windows-debug-coverage.yml b/eng/pipelines/steps/test-windows-debug-coverage.yml index 3c630588a0..90e07a61ef 100644 --- a/eng/pipelines/steps/test-windows-debug-coverage.yml +++ b/eng/pipelines/steps/test-windows-debug-coverage.yml @@ -3,22 +3,112 @@ # Assumes the consuming job declares: # - the matrix variable `_BuildConfig` (Debug or Release) # - the job-level environment variables from eng/pipelines/variables/test-env-vars.yml +parameters: +- name: enableAffectedTests + type: boolean + default: false +- name: affectedTestsMode + type: string + default: disabled + values: + - disabled + - collect + - run +- name: affectedTestsCacheVersion + type: string + default: v1 + steps: # Because the build step is using -ci, restore is done in a local .packages directory. # NUGET_PACKAGES must point to that directory so test project evaluation imports the restored package props/targets. # # -p:TestingPlatformCaptureOutput=false streams each test executable's output to the AzDO console instead of # buffering it until failure, preserving live progress during long-running test sessions. -- script: | - echo ##vso[task.setvariable variable=TestStepRan]true - dotnet test -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\TestResults\$(_BuildConfig)\TestStep.binlog -p:UsingDotNetTest=true -p:TestingPlatformCaptureOutput=false - name: Test - displayName: Test - condition: and(succeeded(), eq(variables._BuildConfig, 'Debug')) - env: - # Secret variables are not automatically exposed to scripts. Fork PR builds do not receive this token, - # so report-azdo history queries no-op there; trusted branch builds exercise them end-to-end. - SYSTEM_ACCESSTOKEN: $(System.AccessToken) +- ${{ if eq(parameters.enableAffectedTests, false) }}: + - script: | + echo ##vso[task.setvariable variable=PublishCoverageReport]true + dotnet test -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\TestResults\$(_BuildConfig)\TestStep.binlog -p:UsingDotNetTest=true -p:TestingPlatformCaptureOutput=false + name: Test + displayName: Test + condition: and(succeeded(), eq(variables._BuildConfig, 'Debug')) + env: + # Secret variables are not automatically exposed to scripts. Fork PR builds do not receive this token, + # so report-azdo history queries no-op there; trusted branch builds exercise them end-to-end. + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + +# These branches are intentionally disabled at their call sites. They can be enabled only after the affected-test +# extension package and its local-filesystem storage schema have flowed into this repository. The SDK owns +# DOTNET_CLI_TEST_AFFECTED_TESTS_MODE; pipelines must not set it. +- ${{ if and(eq(parameters.enableAffectedTests, true), eq(parameters.affectedTestsMode, 'collect')) }}: + - task: Cache@2 + displayName: Restore and publish affected-test map + inputs: + key: '"affected-tests" | "${{ parameters.affectedTestsCacheVersion }}" | "$(Agent.OS)" | "$(Agent.OSArchitecture)" | "$(_BuildConfig)" | "$(Build.BuildId)"' + restoreKeys: | + "affected-tests" | "${{ parameters.affectedTestsCacheVersion }}" | "$(Agent.OS)" | "$(Agent.OSArchitecture)" | "$(_BuildConfig)" + path: '$(Pipeline.Workspace)\affected-test-map' + cacheHitVar: AffectedTestsMapCacheRestored + condition: and(succeeded(), eq(variables._BuildConfig, 'Debug')) + + - script: | + echo ##vso[task.setvariable variable=PublishCoverageReport]true + dotnet test -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\TestResults\$(_BuildConfig)\TestStep.binlog -p:UsingDotNetTest=true -p:TestingPlatformCaptureOutput=false --collect-test-map + name: Test + displayName: Test and collect affected-test map + condition: and(succeeded(), eq(variables._BuildConfig, 'Debug')) + env: + DOTNET_CLI_ENABLE_AFFECTED_TESTS: 1 + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + +- ${{ if and(eq(parameters.enableAffectedTests, true), eq(parameters.affectedTestsMode, 'run')) }}: + - task: Cache@2 + displayName: Restore affected-test map + inputs: + key: '"affected-tests" | "${{ parameters.affectedTestsCacheVersion }}" | "$(Agent.OS)" | "$(Agent.OSArchitecture)" | "$(_BuildConfig)" | "$(Build.BuildId)"' + restoreKeys: | + "affected-tests" | "${{ parameters.affectedTestsCacheVersion }}" | "$(Agent.OS)" | "$(Agent.OSArchitecture)" | "$(_BuildConfig)" + path: '$(Pipeline.Workspace)\affected-test-map' + cacheHitVar: AffectedTestsMapCacheRestored + condition: and(succeeded(), eq(variables._BuildConfig, 'Debug'), eq(variables['Build.Reason'], 'PullRequest')) + + - pwsh: | + dotnet test -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\TestResults\$(_BuildConfig)\TestStep.binlog -p:UsingDotNetTest=true -p:TestingPlatformCaptureOutput=false --affected-tests + if ($LASTEXITCODE -eq 0) { + Write-Host "##vso[task.setvariable variable=AffectedTestsSucceeded]true" + exit 0 + } + + Write-Host "##vso[task.logissue type=warning]Affected-test selection failed; running the full test suite." + exit 0 + name: TestAffected + displayName: Test affected changes + condition: and(succeeded(), eq(variables._BuildConfig, 'Debug'), eq(variables['Build.Reason'], 'PullRequest'), ne(variables['AffectedTestsMapCacheRestored'], 'false')) + env: + DOTNET_CLI_ENABLE_AFFECTED_TESTS: 1 + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + + # Scheduled and manual runs keep their full-validation contract. PRs also fall back when the cache is absent or the + # extension rejects a missing, stale, or incompatible map. + - pwsh: | + Write-Host "##vso[task.setvariable variable=PublishCoverageReport]true" + + $testResultsDirectory = "$(Build.SourcesDirectory)\artifacts\TestResults\$(_BuildConfig)" + if (Test-Path $testResultsDirectory) { + Get-ChildItem $testResultsDirectory -Filter *.coverage -File -ErrorAction SilentlyContinue | + Remove-Item -Force + } + + dotnet test -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\TestResults\$(_BuildConfig)\TestStep.binlog -p:UsingDotNetTest=true -p:TestingPlatformCaptureOutput=false + exit $LASTEXITCODE + name: Test + displayName: Test (affected-test fallback) + condition: and(succeeded(), eq(variables._BuildConfig, 'Debug'), or(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['AffectedTestsMapCacheRestored'], 'false'), ne(variables['AffectedTestsSucceeded'], 'true'))) + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + +- ${{ if and(eq(parameters.enableAffectedTests, true), eq(parameters.affectedTestsMode, 'disabled')) }}: + - pwsh: throw "affectedTestsMode must be 'collect' or 'run' when enableAffectedTests is true." + displayName: Reject invalid affected-test configuration # Publish a portable, compact report separately from the large TestResults artifact so external consumers can # retrieve it without downloading every test result and binary .coverage file. @@ -52,7 +142,7 @@ steps: } Write-Host "##vso[task.setvariable variable=CoverageReportReady]true" - condition: and(succeededOrFailed(), eq(variables._BuildConfig, 'Debug'), eq(variables['TestStepRan'], 'true')) + condition: and(succeededOrFailed(), eq(variables._BuildConfig, 'Debug'), eq(variables['PublishCoverageReport'], 'true')) - task: PublishBuildArtifacts@1 displayName: 'Publish Cobertura coverage report' diff --git a/eng/validate-affected-tests.ps1 b/eng/validate-affected-tests.ps1 new file mode 100644 index 0000000000..78fdd7286b --- /dev/null +++ b/eng/validate-affected-tests.ps1 @@ -0,0 +1,143 @@ +[CmdletBinding()] +param() + +$repoRoot = Split-Path $PSScriptRoot -Parent +$globalJsonPath = Join-Path $repoRoot "global.json" +$pipelinePath = Join-Path $repoRoot "azure-pipelines.yml" +$testTemplatePath = Join-Path $repoRoot "eng/pipelines/steps/test-windows-debug-coverage.yml" + +$configuration = Get-Content -LiteralPath $globalJsonPath -Raw | ConvertFrom-Json +$affectedTests = $configuration.test.affectedTests + +if ($null -eq $affectedTests) { + throw "global.json must define test.affectedTests." +} + +foreach ($path in @( + "changes.ignore", + "changes.forceAllTests", + "instrumentation.include", + "instrumentation.exclude" +)) { + $value = $affectedTests + foreach ($segment in $path.Split(".")) { + $value = $value.$segment + } + + if ($value -isnot [System.Array] -or $value.Count -eq 0) { + throw "global.json test.affectedTests.$path must be a non-empty array." + } +} + +$minimumAffectedTestsSdk = [System.Management.Automation.SemanticVersion]"11.0.100-rc.1.26406.108" +$configuredSdk = [System.Management.Automation.SemanticVersion]$configuration.sdk.version +if ($configuredSdk -lt $minimumAffectedTestsSdk) { + throw "global.json must pin SDK $minimumAffectedTestsSdk or newer for affected-test command support." +} + +$globalJsonText = Get-Content -LiteralPath $globalJsonPath -Raw +if ($globalJsonText -match '(?i)"[^"]*(token|secret|password|connectionString|sas)[^"]*"\s*:') { + throw "global.json must not contain affected-test credentials or secret-bearing settings." +} + +$pipeline = Get-Content -LiteralPath $pipelinePath -Raw +$collectCall = [regex]::Match( + $pipeline, + '(?s)enableAffectedTests:\s*(?true|false)\s+affectedTestsMode:\s*collect') +$runCall = [regex]::Match( + $pipeline, + '(?s)enableAffectedTests:\s*(?true|false)\s+affectedTestsMode:\s*run') +if (-not $collectCall.Success -or -not $runCall.Success) { + throw "The pipeline must define explicit main collection and PR selection call sites." +} + +$affectedTestsEnabled = + $collectCall.Groups["enabled"].Value -eq "true" -or + $runCall.Groups["enabled"].Value -eq "true" +if ($affectedTestsEnabled -and $null -eq $affectedTests.storage) { + throw "Affected-test pipeline execution requires test.affectedTests.storage." +} + +$testTemplate = Get-Content -LiteralPath $testTemplatePath -Raw +foreach ($requiredText in @( + "Cache@2", + "DOTNET_CLI_ENABLE_AFFECTED_TESTS: 1", + "--collect-test-map", + "--affected-tests", + '$(Pipeline.Workspace)\affected-test-map', + "AffectedTestsMapCacheRestored", + "enableAffectedTests", + "affectedTestsMode", + "affectedTestsCacheVersion" +)) { + if (-not $testTemplate.Contains($requiredText)) { + throw "The affected-test template is missing '$requiredText'." + } +} + +$disabledBranch = [regex]::Match( + $testTemplate, + '(?s)- \$\{\{ if eq\(parameters\.enableAffectedTests, false\) \}\}:.*?(?=\r?\n# These branches)') +if (-not $disabledBranch.Success) { + throw "The ordinary test fallback branch is missing." +} + +if ($disabledBranch.Value -match 'DOTNET_CLI_ENABLE_AFFECTED_TESTS|--collect-test-map|--affected-tests') { + throw "The ordinary test fallback must not enable affected-test behavior." +} + +$pipelineVariables = Get-Content -LiteralPath (Join-Path $repoRoot "eng/pipelines/variables/test-env-vars.yml") -Raw +$outerPipelineConfiguration = $pipeline, $pipelineVariables -join "`n" +foreach ($variableName in @( + "DOTNET_CLI_ENABLE_AFFECTED_TESTS", + "DOTNET_CLI_TEST_AFFECTED_TESTS_MODE" +)) { + if ($outerPipelineConfiguration.Contains($variableName)) { + throw "$variableName must be scoped to the affected-test template." + } +} + +$templateWithoutComments = $testTemplate -split '\r?\n' | + Where-Object { -not $_.TrimStart().StartsWith("#") } | + Join-String -Separator "`n" +if ($templateWithoutComments.Contains("DOTNET_CLI_TEST_AFFECTED_TESTS_MODE")) { + throw "DOTNET_CLI_TEST_AFFECTED_TESTS_MODE is SDK-to-extension plumbing and must not be set by the pipeline." +} + +$affectedTestsGateCount = [regex]::Matches( + $templateWithoutComments, + 'DOTNET_CLI_ENABLE_AFFECTED_TESTS').Count +if ($affectedTestsGateCount -ne 2) { + throw "DOTNET_CLI_ENABLE_AFFECTED_TESTS must appear exactly once in each enabled affected-test branch." +} + +$cacheTaskCount = [regex]::Matches($templateWithoutComments, 'task:\s*Cache@2').Count +if ($cacheTaskCount -ne 2) { + throw "The collect and run branches must each define one Cache@2 map task." +} + +$collectBranch = [regex]::Match( + $templateWithoutComments, + "(?s)eq\(parameters\.affectedTestsMode, 'collect'\).*?(?=\r?\n- \$\{\{)") +$runBranch = [regex]::Match( + $templateWithoutComments, + "(?s)eq\(parameters\.affectedTestsMode, 'run'\).*?(?=\r?\n- \$\{\{)") +if (-not $collectBranch.Success -or + -not $collectBranch.Value.Contains("DOTNET_CLI_ENABLE_AFFECTED_TESTS: 1") -or + -not $runBranch.Success -or + -not $runBranch.Value.Contains("DOTNET_CLI_ENABLE_AFFECTED_TESTS: 1")) { + throw "The affected-test gate must be scoped to the enabled collect and run branches." +} + +$runFallback = [regex]::Match( + $templateWithoutComments, + "(?s)displayName:\s*Test \(affected-test fallback\).*?condition:.*?Build\.Reason.*?AffectedTestsMapCacheRestored.*?AffectedTestsSucceeded") +if (-not $runFallback.Success) { + throw "The run branch must retain a full-test fallback for non-PR runs and affected-test failures." +} + +if (-not $testTemplate.Contains("PublishCoverageReport")) { + throw "Coverage publication must be limited to full-test and collection runs." +} + +Write-Output "Affected-test configuration and rollout wiring are valid." diff --git a/global.json b/global.json index d89d8406f6..3f6f67dc34 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "tools": { - "dotnet": "11.0.100-rc.1.26402.102", + "dotnet": "11.0.100-rc.1.26406.108", "runtimes": { "dotnet": [ "8.0.29", @@ -24,7 +24,7 @@ } }, "sdk": { - "version": "11.0.100-rc.1.26402.102", + "version": "11.0.100-rc.1.26406.108", "paths": [ ".dotnet", "$host$" @@ -34,7 +34,42 @@ "rollForward": "latestFeature" }, "test": { - "runner": "Microsoft.Testing.Platform" + "runner": "Microsoft.Testing.Platform", + "affectedTests": { + "changes": { + "ignore": [ + ".github/**", + "docs/**", + "**/*.md" + ], + "forceAllTests": [ + "global.json", + "Directory.Build.*", + "Directory.Packages.props", + "NuGet.config", + "TestFx.slnx", + "*.slnf", + "Build.cmd", + "Test.cmd", + "build.sh", + "test.sh", + "eng/**", + "test/Directory.Build.*" + ] + }, + "instrumentation": { + "include": [ + "src/**" + ], + "exclude": [ + "artifacts/**", + "samples/**", + "test/**", + "**/bin/**", + "**/obj/**" + ] + } + } }, "msbuild-sdks": { "Microsoft.DotNet.Arcade.Sdk": "11.0.0-beta.26405.4", From 7c2156ff69d0376e236f00c7c162b8444540a468 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Fri, 7 Aug 2026 02:12:36 +0200 Subject: [PATCH 2/5] Avoid broken affected-test SDK daily Keep the stable SDK pin until a daily containing the TransactionalAction fix is published, and enforce that corrected SDK only when rollout activation is enabled. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3baa2cd4-db32-4f9f-b6b7-8cc190f3799f --- docs/affected-test-selection.md | 22 +++++++++++++--------- eng/validate-affected-tests.ps1 | 13 +++++++------ global.json | 4 ++-- 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/docs/affected-test-selection.md b/docs/affected-test-selection.md index e763fe05e9..a92873a872 100644 --- a/docs/affected-test-selection.md +++ b/docs/affected-test-selection.md @@ -5,9 +5,11 @@ This repository is prepared to adopt the experimental affected-test workflow fro builds on the composable filter-provider support from [testfx#10235](https://github.com/microsoft/testfx/pull/10235). -The rollout is intentionally disabled. SDK `11.0.100-rc.1.26406.108` contains the affected-test commands, but the -affected-test extension package and its public local-filesystem storage contract are not available yet. Ordinary test -commands therefore remain unchanged. +The rollout is intentionally disabled. SDK `11.0.100-rc.1.26406.108` contains the affected-test commands but fails +`dotnet tool restore` on clean agents because it predates +[dotnet/sdk#55595](https://github.com/dotnet/sdk/pull/55595). The repository remains on the stable SDK until a fixed +daily is published. The affected-test extension package and its public local-filesystem storage contract are also not +available yet. Ordinary test commands therefore remain unchanged. ## Prepared layout @@ -50,14 +52,16 @@ cannot be validated. ## Activation checklist -1. Add the publicly available affected-test extension package through `Directory.Packages.props` and the test project +1. Update `global.json` to an SDK newer than `11.0.100-rc.1.26406.108` that contains dotnet/sdk#55595, then validate + `dotnet tool restore` on a clean agent. +2. Add the publicly available affected-test extension package through `Directory.Packages.props` and the test project infrastructure, following the repository's normal dependency-flow and package-source policy. -2. Add `test.affectedTests.storage` using the package's published local-filesystem schema and point it at the Pipeline +3. Add `test.affectedTests.storage` using the package's published local-filesystem schema and point it at the Pipeline Cache directory. -3. Update `affectedTestsCacheVersion` whenever the persisted map format or its compatibility dimensions change. -4. Enable `collect` in the main-branch cache-seed call site and publish non-secret diagnostics as an Azure DevOps +4. Update `affectedTestsCacheVersion` whenever the persisted map format or its compatibility dimensions change. +5. Enable `collect` in the main-branch cache-seed call site and publish non-secret diagnostics as an Azure DevOps artifact. -5. After a compatible map exists, enable `run` in the PR call site. Keep the full test command available as an explicit +6. After a compatible map exists, enable `run` in the PR call site. Keep the full test command available as an explicit rollback by setting `enableAffectedTests` back to `false`. -6. Validate a documentation-only change, a product change with a narrow affected set, a force-all change, a missing or +7. Validate a documentation-only change, a product change with a narrow affected set, a force-all change, a missing or incompatible map, a fork PR without secrets, and a collection failure before making selection required. diff --git a/eng/validate-affected-tests.ps1 b/eng/validate-affected-tests.ps1 index 78fdd7286b..232b98716b 100644 --- a/eng/validate-affected-tests.ps1 +++ b/eng/validate-affected-tests.ps1 @@ -29,12 +29,6 @@ foreach ($path in @( } } -$minimumAffectedTestsSdk = [System.Management.Automation.SemanticVersion]"11.0.100-rc.1.26406.108" -$configuredSdk = [System.Management.Automation.SemanticVersion]$configuration.sdk.version -if ($configuredSdk -lt $minimumAffectedTestsSdk) { - throw "global.json must pin SDK $minimumAffectedTestsSdk or newer for affected-test command support." -} - $globalJsonText = Get-Content -LiteralPath $globalJsonPath -Raw if ($globalJsonText -match '(?i)"[^"]*(token|secret|password|connectionString|sas)[^"]*"\s*:') { throw "global.json must not contain affected-test credentials or secret-bearing settings." @@ -54,6 +48,13 @@ if (-not $collectCall.Success -or -not $runCall.Success) { $affectedTestsEnabled = $collectCall.Groups["enabled"].Value -eq "true" -or $runCall.Groups["enabled"].Value -eq "true" + +$lastUnsupportedAffectedTestsSdk = [System.Management.Automation.SemanticVersion]"11.0.100-rc.1.26406.108" +$configuredSdk = [System.Management.Automation.SemanticVersion]$configuration.sdk.version +if ($affectedTestsEnabled -and $configuredSdk -le $lastUnsupportedAffectedTestsSdk) { + throw "Affected-test execution requires an SDK newer than $lastUnsupportedAffectedTestsSdk with dotnet/sdk#55595." +} + if ($affectedTestsEnabled -and $null -eq $affectedTests.storage) { throw "Affected-test pipeline execution requires test.affectedTests.storage." } diff --git a/global.json b/global.json index 3f6f67dc34..4a261d6303 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "tools": { - "dotnet": "11.0.100-rc.1.26406.108", + "dotnet": "11.0.100-rc.1.26402.102", "runtimes": { "dotnet": [ "8.0.29", @@ -24,7 +24,7 @@ } }, "sdk": { - "version": "11.0.100-rc.1.26406.108", + "version": "11.0.100-rc.1.26402.102", "paths": [ ".dotnet", "$host$" From 95afe48f15c890ce89b0d751f17dc70522a04b3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Fri, 7 Aug 2026 10:42:11 +0200 Subject: [PATCH 3/5] Preserve affected test failures Keep exit code 2 from selected-test runs and force full selection when nested Directory.Build files change. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3baa2cd4-db32-4f9f-b6b7-8cc190f3799f --- .../steps/test-windows-debug-coverage.yml | 8 +++++++- eng/validate-affected-tests.ps1 | 15 +++++++++++++++ global.json | 3 ++- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/steps/test-windows-debug-coverage.yml b/eng/pipelines/steps/test-windows-debug-coverage.yml index 90e07a61ef..eee512819b 100644 --- a/eng/pipelines/steps/test-windows-debug-coverage.yml +++ b/eng/pipelines/steps/test-windows-debug-coverage.yml @@ -73,11 +73,17 @@ steps: - pwsh: | dotnet test -c $(_BuildConfig) --no-build -bl:$(BUILD.SOURCESDIRECTORY)\artifacts\TestResults\$(_BuildConfig)\TestStep.binlog -p:UsingDotNetTest=true -p:TestingPlatformCaptureOutput=false --affected-tests - if ($LASTEXITCODE -eq 0) { + $exitCode = $LASTEXITCODE + if ($exitCode -eq 0) { Write-Host "##vso[task.setvariable variable=AffectedTestsSucceeded]true" exit 0 } + if ($exitCode -eq 2) { + Write-Host "##vso[task.logissue type=error]One or more affected tests failed." + exit $exitCode + } + Write-Host "##vso[task.logissue type=warning]Affected-test selection failed; running the full test suite." exit 0 name: TestAffected diff --git a/eng/validate-affected-tests.ps1 b/eng/validate-affected-tests.ps1 index 232b98716b..c2b430694d 100644 --- a/eng/validate-affected-tests.ps1 +++ b/eng/validate-affected-tests.ps1 @@ -29,6 +29,16 @@ foreach ($path in @( } } +foreach ($requiredForceAllPattern in @( + "Directory.Build.*", + "src/**/Directory.Build.*", + "test/**/Directory.Build.*" +)) { + if ($requiredForceAllPattern -notin $affectedTests.changes.forceAllTests) { + throw "global.json test.affectedTests.changes.forceAllTests must include '$requiredForceAllPattern'." + } +} + $globalJsonText = Get-Content -LiteralPath $globalJsonPath -Raw if ($globalJsonText -match '(?i)"[^"]*(token|secret|password|connectionString|sas)[^"]*"\s*:') { throw "global.json must not contain affected-test credentials or secret-bearing settings." @@ -130,6 +140,11 @@ if (-not $collectBranch.Success -or throw "The affected-test gate must be scoped to the enabled collect and run branches." } +if (-not $runBranch.Value.Contains('$exitCode -eq 2') -or + -not $runBranch.Value.Contains('exit $exitCode')) { + throw "The affected-test run must preserve exit code 2 when selected tests fail." +} + $runFallback = [regex]::Match( $templateWithoutComments, "(?s)displayName:\s*Test \(affected-test fallback\).*?condition:.*?Build\.Reason.*?AffectedTestsMapCacheRestored.*?AffectedTestsSucceeded") diff --git a/global.json b/global.json index 4a261d6303..11a7510966 100644 --- a/global.json +++ b/global.json @@ -54,7 +54,8 @@ "build.sh", "test.sh", "eng/**", - "test/Directory.Build.*" + "src/**/Directory.Build.*", + "test/**/Directory.Build.*" ] }, "instrumentation": { From 238ef1056500ca6fa48cab54774255560f3c266a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Fri, 7 Aug 2026 11:11:59 +0200 Subject: [PATCH 4/5] Preserve all-skipped affected test failures Keep exit code 8 from affected-test runs so the full-suite fallback cannot mask the SDK's all-skipped safeguard. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3baa2cd4-db32-4f9f-b6b7-8cc190f3799f --- eng/pipelines/steps/test-windows-debug-coverage.yml | 4 ++-- eng/validate-affected-tests.ps1 | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/pipelines/steps/test-windows-debug-coverage.yml b/eng/pipelines/steps/test-windows-debug-coverage.yml index eee512819b..07a278d47a 100644 --- a/eng/pipelines/steps/test-windows-debug-coverage.yml +++ b/eng/pipelines/steps/test-windows-debug-coverage.yml @@ -79,8 +79,8 @@ steps: exit 0 } - if ($exitCode -eq 2) { - Write-Host "##vso[task.logissue type=error]One or more affected tests failed." + if ($exitCode -in 2, 8) { + Write-Host "##vso[task.logissue type=error]Affected-test execution failed with exit code $exitCode." exit $exitCode } diff --git a/eng/validate-affected-tests.ps1 b/eng/validate-affected-tests.ps1 index c2b430694d..8802e11856 100644 --- a/eng/validate-affected-tests.ps1 +++ b/eng/validate-affected-tests.ps1 @@ -140,9 +140,9 @@ if (-not $collectBranch.Success -or throw "The affected-test gate must be scoped to the enabled collect and run branches." } -if (-not $runBranch.Value.Contains('$exitCode -eq 2') -or +if (-not $runBranch.Value.Contains('$exitCode -in 2, 8') -or -not $runBranch.Value.Contains('exit $exitCode')) { - throw "The affected-test run must preserve exit code 2 when selected tests fail." + throw "The affected-test run must preserve exit codes 2 and 8 for failed or all-skipped selections." } $runFallback = [regex]::Match( From 39d863db74ae900dae390a487d225714a9a586b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Fri, 7 Aug 2026 12:42:23 +0200 Subject: [PATCH 5/5] Validate the bootstrapped affected-test SDK Require the global.json SDK pins to match and gate activation on the tools.dotnet version installed by CI. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3baa2cd4-db32-4f9f-b6b7-8cc190f3799f --- eng/validate-affected-tests.ps1 | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/eng/validate-affected-tests.ps1 b/eng/validate-affected-tests.ps1 index 8802e11856..cd5cb0422b 100644 --- a/eng/validate-affected-tests.ps1 +++ b/eng/validate-affected-tests.ps1 @@ -59,9 +59,14 @@ $affectedTestsEnabled = $collectCall.Groups["enabled"].Value -eq "true" -or $runCall.Groups["enabled"].Value -eq "true" +$bootstrappedSdk = [System.Management.Automation.SemanticVersion]$configuration.tools.dotnet +$selectedSdk = [System.Management.Automation.SemanticVersion]$configuration.sdk.version +if ($bootstrappedSdk -ne $selectedSdk) { + throw "global.json tools.dotnet and sdk.version must match." +} + $lastUnsupportedAffectedTestsSdk = [System.Management.Automation.SemanticVersion]"11.0.100-rc.1.26406.108" -$configuredSdk = [System.Management.Automation.SemanticVersion]$configuration.sdk.version -if ($affectedTestsEnabled -and $configuredSdk -le $lastUnsupportedAffectedTestsSdk) { +if ($affectedTestsEnabled -and $bootstrappedSdk -le $lastUnsupportedAffectedTestsSdk) { throw "Affected-test execution requires an SDK newer than $lastUnsupportedAffectedTestsSdk with dotnet/sdk#55595." }