Skip to content

Test job refactor prototyping #41502

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
David Bennett (dkbennett) wants to merge 2 commits into
base: master
Choose a base branch
from
+118 −45
Draft

Test job refactor prototyping #41502

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0e443fd
Test job refactor prototyping
dkbennett Sep 2, 2026
c6c7f12
Fix shard failures, remove full wsl2 tests
dkbennett Sep 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 11 additions & 5 deletions .pipelines/test-job.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,13 @@ parameters:
- name: branch
type: string

- name: version
- name: test
type: string

- name: testDisplayName
type: string

- name: testMap
type: string

- name: image
Expand All @@ -16,8 +22,8 @@ parameters:
default: ''

jobs:
- job: test_${{ parameters.branch }}_${{ parameters.version }}
displayName: "${{ parameters.version }} tests - ${{ parameters.branch }}"
- job: test_${{ parameters.branch }}_${{ parameters.test }}
displayName: "${{ parameters.testDisplayName }} - ${{ parameters.branch }}"
dependsOn: []
condition: and(succeeded(), eq('${{ parameters.run }}', true))
variables:
Expand All @@ -34,10 +40,10 @@ jobs:
steps:
- task: CloudTestServerBuildTask@2
inputs:
DisplayName: "${{ parameters.version }} tests - ${{ parameters.branch }}"
DisplayName: "${{ parameters.testDisplayName }} - ${{ parameters.branch }}"
connectedServiceName: "CloudTest-PROD"
cloudTestTenant: "wsl"
testMapLocation: 'testbin\x64\cloudtest\wsl-test-image-${{ parameters.image }}-${{ parameters.version}}\TestMap.xml'
testMapLocation: 'testbin\x64\cloudtest\wsl-test-image-${{ parameters.image }}-${{ parameters.testMap}}\TestMap.xml'
pipelineArtifactName: "drop_wsl_build"
pipelineArtifactBuildUrl: '$(System.TaskDefinitionsUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)'
buildDropArtifactName: ""
Expand Down
28 changes: 21 additions & 7 deletions .pipelines/test-stage.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,24 @@ parameters:
type: string
default: ''

- name: versions
- name: test_suites
type: object
default:
- wsl1
- wsl2
- wslc
- id: wsl1
display_name: WSL1 tests
map: wsl1
- id: wsl2_unit
display_name: WSL2 unit tests
map: wsl2-unit
- id: wsl2_e2e_1
display_name: WSL2 E2E shard 1
map: wsl2-e2e-1
- id: wsl2_e2e_2
display_name: WSL2 E2E shard 2
map: wsl2-e2e-2
- id: wslc
display_name: WSLC tests
map: wslc

- name: test_images
type: object
Expand All @@ -40,14 +52,16 @@ stages:
${{ else }}:
dependsOn: [build_x64]
jobs:
- ${{ each version in parameters.versions }}:
- ${{ each suite in parameters.test_suites }}:
- ${{ each image in parameters.test_images }}:
- template: test-job.yml@self
parameters:
branch: "${{ image.name }}"
version: ${{ version }}
test: "${{ suite.id }}"
testDisplayName: "${{ suite.display_name }}"
testMap: "${{ suite.map }}"
image: "${{ image.image }}"
# Skip the WSLC suite on Server images: it triggers a Hyper-V VMBus
# PowerOff hang during VM teardown that times out the job.
run: ${{ and(or(not(parameters.rs_prerelease_only), eq(image.name, 'rs_prerelease')), or(ne(version, 'wslc'), ne(image.server, true))) }}
run: ${{ and(or(not(parameters.rs_prerelease_only), eq(image.name, 'rs_prerelease')), or(ne(suite.id, 'wslc'), ne(image.server, true))) }}
pool: "${{ parameters.pool }}"
86 changes: 79 additions & 7 deletions cloudtest/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,27 +34,99 @@ endif()
# When the package stage is included (release + nightly), test the real bundle from that stage.
# PR builds test the installer.msix directly from the build artifact (no bundle needed).
if (INCLUDE_PACKAGE_STAGE)
set(TEST_PACKAGE_PROVIDER "[package]")
set(TEST_PACKAGE_PATH "bundle\\Microsoft.WSL_${PACKAGE_VERSION}_x64_ARM64.msixbundle")
set(TEST_PACKAGE_FILE "Microsoft.WSL_${PACKAGE_VERSION}_x64_ARM64.msixbundle")
set(TEST_PACKAGE_BUILD_FILE " <Copy Src=\"[package]\\bundle\\${TEST_PACKAGE_FILE}\" Dest=\"[WorkingDirectory]\" IsRecursive=\"false\"/>")
set(PACKAGE_ARTIFACT_PROVIDER " <Provider Type=\"PipelineArtifacts\">
<Properties>
<Add Name=\"CloudTest.ProviderCustomName\" Value=\"[package]\" />
<Add Name=\"PipelineArtifactBuildUrl\" Value=\"https://${VSO_ORG}.visualstudio.com/${VSO_PROJECT}/_build/results?buildId=${PIPELINE_BUILD_ID}\"/>
<Add Name=\"PipelineArtifactName\" Value=\"drop_wsl_package\"/>
</Properties>
</Provider>")
else()
set(TEST_PACKAGE_PROVIDER "[drop]")
set(TEST_PACKAGE_PATH "testbin\\x64\\release\\installer.msix")
set(TEST_PACKAGE_FILE "installer.msix")
set(TEST_PACKAGE_BUILD_FILE "")
set(PACKAGE_ARTIFACT_PROVIDER "")
endif()

function(add_test_group image version suffix filter)
cmake_parse_arguments(
PARSE_ARGV 4
TEST_GROUP
"INCLUDE_UNIT_TESTS;INCLUDE_TEST_DATA;INCLUDE_TEST_PACKAGES"
""
"")

set(UNIT_TESTS_BUILD_FILE "")
set(UNIT_TESTS_ARGUMENT "")
if (TEST_GROUP_INCLUDE_UNIT_TESTS)
set(UNIT_TESTS_BUILD_FILE " <Copy Src=\"[drop]\\testbin\\unit_tests\\*\" Dest=\"[WorkingDirectory]\\unit_tests\" IsRecursive=\"true\" Writable=\"true\"/>")
set(UNIT_TESTS_ARGUMENT " /p:UnitTestsPath=[WorkingDirectory]\\unit_tests")
endif()

set(TEST_DATA_BUILD_FILE "")
if (TEST_GROUP_INCLUDE_TEST_DATA)
set(TEST_DATA_BUILD_FILE " <Copy Src=\"[drop]\\testbin\\${TARGET_PLATFORM}\\test_data\\*\" Dest=\"[WorkingDirectory]\\test_data\" IsRecursive=\"true\" Writable=\"true\"/>")
endif()

set(TEST_PACKAGES_BUILD_FILE "")
set(TEST_PACKAGES_PROVIDER "")
if (TEST_GROUP_INCLUDE_TEST_PACKAGES)
set(TEST_PACKAGES_BUILD_FILE " <Copy Src=\"[test_packages]\\*\" Dest=\"[WorkingDirectory]\" IsRecursive=\"false\"/>")
set(TEST_PACKAGES_PROVIDER " <Provider Type=\"VSODrop\">
<Properties>
<Add Name=\"CloudTest.ProviderCustomName\" Value=\"[test_packages]\" />
<!-- When updating the drops for this make sure the retention is set to never expire -->
<Add Name=\"DropURL\" Value=\"https://${VSO_ORG}.artifacts.visualstudio.com/_apis/drop/drops/WSL/${CLOUDTEST_TEST_PACKAGES}\"/>
</Properties>
</Provider>")
endif()

set(DIR ${OUT}/${image}-${suffix})
file(MAKE_DIRECTORY ${DIR})

configure_file(${CMAKE_CURRENT_SOURCE_DIR}/TestMap.xml.in ${DIR}/TestMap.xml)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/TestGroup.xml.in ${DIR}/TestGroup.xml)
endfunction()

set(NON_WSLC_FILTER "not (@TestCategory='WSLC')")

string(CONCAT WSL2_UNIT_CLASSES
"("
"@Name='UnitTests::UnitTests::*' OR "
"@Name='FilesystemUnitTests::FilesystemUnitTests::*' OR "
"@Name='StringUnitTests::StringUnitTests::*' OR "
"@Name='WindowsUpdateTests::*' OR "
"@Name='NetworkTests::GnsCallbackResultTests::*'"
")")

string(CONCAT WSL2_E2E_1_CLASSES
"("
"@Name='NetworkTests::MirroredTests::*' OR "
"@Name='MountTests::MountTests::*' OR "
"@Name='NetworkTests::ConsommeTests::*' OR "
"@Name='Plan9Tests::Plan9Tests::*' OR "
"@Name='PolicyTest::*' OR "
"@Name='SimpleTests::SimpleTests::*'"
")")

set(WSL2_UNIT_FILTER "${NON_WSLC_FILTER} AND ${WSL2_UNIT_CLASSES}")
set(WSL2_E2E_1_FILTER "${NON_WSLC_FILTER} AND ${WSL2_E2E_1_CLASSES}")
set(WSL2_E2E_2_FILTER "${NON_WSLC_FILTER} AND NOT ${WSL2_UNIT_CLASSES} AND NOT ${WSL2_E2E_1_CLASSES}")

foreach(image ${CLOUDTEST_IMAGES})
add_test_group("${image}" "1" "wsl1" "not (@TestCategory='WSLC')")
add_test_group("${image}" "2" "wsl2" "not (@TestCategory='WSLC')")
add_test_group("${image}" "1" "wsl1" "${NON_WSLC_FILTER}" INCLUDE_UNIT_TESTS INCLUDE_TEST_PACKAGES)
add_test_group("${image}" "2" "wsl2-unit" "${WSL2_UNIT_FILTER}" INCLUDE_UNIT_TESTS)
add_test_group("${image}" "2" "wsl2-e2e-1" "${WSL2_E2E_1_FILTER}" INCLUDE_UNIT_TESTS)
add_test_group(
"${image}"
"2"
"wsl2-e2e-2"
"${WSL2_E2E_2_FILTER}"
INCLUDE_UNIT_TESTS
INCLUDE_TEST_DATA
INCLUDE_TEST_PACKAGES)
if (NOT "${image}" IN_LIST CLOUDTEST_SERVER_IMAGES)
add_test_group("${image}" "2" "wslc" "@TestCategory='WSLC'")
add_test_group("${image}" "2" "wslc" "@TestCategory='WSLC'" INCLUDE_TEST_DATA)
endif()
endforeach()
10 changes: 5 additions & 5 deletions cloudtest/TestGroup.xml.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,15 @@
</ResourceSpec>
<Setup TimeoutMins="30">
<BuildFiles>
<Copy Src="${TEST_PACKAGE_PROVIDER}\${TEST_PACKAGE_PATH}" Dest="[WorkingDirectory]" IsRecursive="false"/>
${TEST_PACKAGE_BUILD_FILE}
<Copy Src="[drop]\testbin\${TARGET_PLATFORM}\release\*" Dest="[WorkingDirectory]\" IsRecursive="true" Writable="true"/>
<Copy Src="[drop]\testbin\${TARGET_PLATFORM}\test_distro.tar.xz" Dest="[WorkingDirectory]" IsRecursive="false" Writable="true"/>
<Copy Src="[drop]\testbin\${TARGET_PLATFORM}\test_data\*" Dest="[WorkingDirectory]\test_data" IsRecursive="true" Writable="true"/>
${TEST_DATA_BUILD_FILE}
<Copy Src="[drop]\testbin\test-setup.ps1" Dest="[WorkingDirectory]\" IsRecursive="false" />
<Copy Src="[drop]\testbin\CloudTest-Setup.bat" Dest="[WorkingDirectory]\" IsRecursive="false" />
<Copy Src="[drop]\testbin\wsl.wprp" Dest="[WorkingDirectory]\" IsRecursive="false" />
<Copy Src="[drop]\testbin\unit_tests\*" Dest="[WorkingDirectory]\unit_tests" IsRecursive="true" Writable="true"/>
<Copy Src="[test_packages]\*" Dest="[WorkingDirectory]" IsRecursive="false" />
${UNIT_TESTS_BUILD_FILE}
${TEST_PACKAGES_BUILD_FILE}
<Copy Src="[dump_tool]\DumpTool.exe" Dest="[WorkingDirectory]" IsRecursive="false" />
</BuildFiles>
<Scripts>
Expand All @@ -22,6 +22,6 @@
</Setup>

<TestJob Name="CloudTest.Taef" TimeoutMins="240">
<Execution Type="TAEF" Path="[WorkingDirectory]\wsltests.dll" Args="/p:bugReportDirectory=[LoggingDirectory]\BugReportOutput /errorOnCrash /testmode:etwlogger /EtwLogger:WPRProfileFile=[WorkingDirectory]\wsl.wprp /EtwLogger:WPRProfile=WSL /EtwLogger:SavePoint=ExecutionComplete /EtwLogger:RecordingScope=Execution /p:SetupScript=.\test-setup.ps1 /p:Package=[WorkingDirectory]\${TEST_PACKAGE_FILE} /p:Version=${version} /p:AllowUnsigned=${ALLOW_UNSIGNED_PACKAGE} /p:UnitTestsPath=[WorkingDirectory]\unit_tests /p:DistroPath=[WorkingDirectory]\test_distro.tar.xz /p:TestDataPath=[WorkingDirectory]\test_data /p:DistroName=test_distro /logOutput:High /p:RedirectStdout=[LoggingDirectory]\stdout.txt /p:RedirectStderr=[LoggingDirectory]\stderr.txt /p:KernelLogs=[LoggingDirectory]\dmesg.txt /p:DumpFolder=[LoggingDirectory] /p:WerReport /p:LogDmesg /p:PipelineBuildId=${PIPELINE_BUILD_ID} /p:DumpTool=DumpTool.exe /select:&quot;${filter}&quot;" />
<Execution Type="TAEF" Path="[WorkingDirectory]\wsltests.dll" Args="/p:bugReportDirectory=[LoggingDirectory]\BugReportOutput /errorOnCrash /testmode:etwlogger /EtwLogger:WPRProfileFile=[WorkingDirectory]\wsl.wprp /EtwLogger:WPRProfile=WSL /EtwLogger:SavePoint=ExecutionComplete /EtwLogger:RecordingScope=Execution /p:SetupScript=.\test-setup.ps1 /p:Package=[WorkingDirectory]\${TEST_PACKAGE_FILE} /p:Version=${version} /p:AllowUnsigned=${ALLOW_UNSIGNED_PACKAGE}${UNIT_TESTS_ARGUMENT} /p:DistroPath=[WorkingDirectory]\test_distro.tar.xz /p:TestDataPath=[WorkingDirectory]\test_data /p:DistroName=test_distro /logOutput:High /p:RedirectStdout=[LoggingDirectory]\stdout.txt /p:RedirectStderr=[LoggingDirectory]\stderr.txt /p:KernelLogs=[LoggingDirectory]\dmesg.txt /p:DumpFolder=[LoggingDirectory] /p:WerReport /p:LogDmesg /p:PipelineBuildId=${PIPELINE_BUILD_ID} /p:DumpTool=DumpTool.exe /select:&quot;${filter}&quot;" />
</TestJob>
</TestJobGroup>
23 changes: 2 additions & 21 deletions cloudtest/TestMap.xml.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,27 +9,8 @@
<Add Name="PipelineArtifactName" Value="drop_wsl_build"/>
</Properties>
</Provider>
<Provider Type="PipelineArtifacts">
<Properties>
<Add Name="CloudTest.ProviderCustomName" Value="[package]" />
<Add Name="PipelineArtifactBuildUrl" Value="https://${VSO_ORG}.visualstudio.com/${VSO_PROJECT}/_build/results?buildId=${PIPELINE_BUILD_ID}"/>
<Add Name="PipelineArtifactName" Value="drop_wsl_package"/>
</Properties>
</Provider>
<Provider Type="VSODrop">
<Properties>
<Add Name="CloudTest.ProviderCustomName" Value="[test_distro]" />
<!-- When updating the drops for this make sure the retention is set to never expire -->
<Add Name="DropURL" Value="https://${VSO_ORG}.artifacts.visualstudio.com/_apis/drop/drops/WSL/${CLOUDTEST_TEST_DISTRO_DROP}"/>
</Properties>
</Provider>
<Provider Type="VSODrop">
<Properties>
<Add Name="CloudTest.ProviderCustomName" Value="[test_packages]" />
<!-- When updating the drops for this make sure the retention is set to never expire -->
<Add Name="DropURL" Value="https://${VSO_ORG}.artifacts.visualstudio.com/_apis/drop/drops/WSL/${CLOUDTEST_TEST_PACKAGES}"/>
</Properties>
</Provider>
${PACKAGE_ARTIFACT_PROVIDER}
${TEST_PACKAGES_PROVIDER}
<Provider Type="VSODrop">
<Properties>
<Add Name="CloudTest.ProviderCustomName" Value="[dump_tool]" />
Expand Down