Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 16 additions & 11 deletions eng/helix_xharness.proj
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,14 @@
<HelixType>test/devices/</HelixType>
<HelixBuild Condition="'$(HelixBuild)' == ''">$(BUILD_BUILDNUMBER)</HelixBuild>
<HelixBuild Condition="'$(HelixBuild)' == ''">default</HelixBuild>
<HelixTargetQueues Condition="'$(TargetOS)' == 'ios'">osx.15.arm64.maui.open</HelixTargetQueues>
<HelixTargetQueues Condition="'$(TargetOS)' == 'maccatalyst'">osx.15.arm64.maui.open</HelixTargetQueues>
<HelixTargetQueues Condition="'$(TargetOS)' == 'android'">ubuntu.2204.amd64.android.33.open</HelixTargetQueues>
<!-- Public queues (default) -->
<HelixTargetQueues Condition="'$(TargetOS)' == 'ios' and '$(HelixInternal)' != 'True'">osx.15.arm64.maui.open</HelixTargetQueues>
<HelixTargetQueues Condition="'$(TargetOS)' == 'maccatalyst' and '$(HelixInternal)' != 'True'">osx.15.arm64.maui.open</HelixTargetQueues>
<HelixTargetQueues Condition="'$(TargetOS)' == 'android' and '$(HelixInternal)' != 'True'">ubuntu.2204.amd64.android.33.open</HelixTargetQueues>
<!-- Internal queues -->
<HelixTargetQueues Condition="'$(TargetOS)' == 'ios' and '$(HelixInternal)' == 'True'">osx.15.arm64.iphone.maui</HelixTargetQueues>
<HelixTargetQueues Condition="'$(TargetOS)' == 'maccatalyst' and '$(HelixInternal)' == 'True'">osx.15.arm64.iphone.maui</HelixTargetQueues>
<HelixTargetQueues Condition="'$(TargetOS)' == 'android' and '$(HelixInternal)' == 'True'">ubuntu.2204.amd64.android.33</HelixTargetQueues>
<TargetsAppleMobile Condition="'$(TargetOS)' == 'ios'">true</TargetsAppleMobile>
<Creator Condition="'$(HelixAccessToken)' == ''">maui</Creator>
<IncludeDotNetCli>true</IncludeDotNetCli>
Expand Down Expand Up @@ -110,7 +115,7 @@
<WorkItemTimeout>02:00:00</WorkItemTimeout>
<TestTimeout>01:15:00</TestTimeout>
<LaunchTimeout>00:10:00</LaunchTimeout>
<AppBundlePath>$([System.IO.Directory]::GetDirectories('$(ScenariosDir)Controls.DeviceTests/Release/$(TargetFrameworkToTest)-ios/', '*.app', System.IO.SearchOption.AllDirectories))</AppBundlePath>
<AppBundlePath>$([System.IO.Directory]::GetDirectories('$(ScenariosDir)Controls.DeviceTests/Release/net11.0-ios/', '*.app', System.IO.SearchOption.AllDirectories))</AppBundlePath>
<CustomCommands>xharness apple test --target "$target" --app "$app" --output-directory "$output_directory" --timeout "$timeout" --launch-timeout "$launch_timeout" --set-env="TestFilter=Category=%(Identity)"</CustomCommands>
</XHarnessAppBundleToTest>
<!-- Controls.DeviceTests - all other categories run as one work item -->
Expand All @@ -119,27 +124,27 @@
<WorkItemTimeout>02:00:00</WorkItemTimeout>
<TestTimeout>01:15:00</TestTimeout>
<LaunchTimeout>00:10:00</LaunchTimeout>
<AppBundlePath>$([System.IO.Directory]::GetDirectories('$(ScenariosDir)Controls.DeviceTests/Release/$(TargetFrameworkToTest)-ios/', '*.app', System.IO.SearchOption.AllDirectories))</AppBundlePath>
<AppBundlePath>$([System.IO.Directory]::GetDirectories('$(ScenariosDir)Controls.DeviceTests/Release/net11.0-ios/', '*.app', System.IO.SearchOption.AllDirectories))</AppBundlePath>
<CustomCommands>xharness apple test --target "$target" --app "$app" --output-directory "$output_directory" --timeout "$timeout" --launch-timeout "$launch_timeout" --set-env="TestFilter=SkipCategories=$(ControlsTestCategoriesToSkipForRestOfTests)"</CustomCommands>
</XHarnessAppBundleToTest>
<!-- Core.DeviceTests - single work item (no category splitting) -->
<XHarnessAppBundleToTest Include="$([System.IO.Directory]::GetDirectories('$(ScenariosDir)Core.DeviceTests/Release/$(TargetFrameworkToTest)-ios/', '*.app', System.IO.SearchOption.AllDirectories))">
<XHarnessAppBundleToTest Include="$([System.IO.Directory]::GetDirectories('$(ScenariosDir)Core.DeviceTests/Release/net11.0-ios/', '*.app', System.IO.SearchOption.AllDirectories))">
<TestTarget>ios-simulator-64</TestTarget>
<WorkItemTimeout>02:00:00</WorkItemTimeout>
<TestTimeout>01:00:00</TestTimeout>
</XHarnessAppBundleToTest>
<!-- Graphics, Essentials, BlazorWebView - single work item each -->
<XHarnessAppBundleToTest Include="$([System.IO.Directory]::GetDirectories('$(ScenariosDir)Graphics.DeviceTests/Release/$(TargetFrameworkToTest)-ios/', '*.app', System.IO.SearchOption.AllDirectories))">
<XHarnessAppBundleToTest Include="$([System.IO.Directory]::GetDirectories('$(ScenariosDir)Graphics.DeviceTests/Release/net11.0-ios/', '*.app', System.IO.SearchOption.AllDirectories))">
<TestTarget>ios-simulator-64</TestTarget>
<WorkItemTimeout>02:00:00</WorkItemTimeout>
<TestTimeout>01:00:00</TestTimeout>
</XHarnessAppBundleToTest>
<XHarnessAppBundleToTest Include="$([System.IO.Directory]::GetDirectories('$(ScenariosDir)Essentials.DeviceTests/Release/$(TargetFrameworkToTest)-ios/', '*.app', System.IO.SearchOption.AllDirectories))">
<XHarnessAppBundleToTest Include="$([System.IO.Directory]::GetDirectories('$(ScenariosDir)Essentials.DeviceTests/Release/net11.0-ios/', '*.app', System.IO.SearchOption.AllDirectories))">
<TestTarget>ios-simulator-64</TestTarget>
<WorkItemTimeout>02:00:00</WorkItemTimeout>
<TestTimeout>01:00:00</TestTimeout>
</XHarnessAppBundleToTest>
<XHarnessAppBundleToTest Include="$([System.IO.Directory]::GetDirectories('$(ScenariosDir)MauiBlazorWebView.DeviceTests/Release/$(TargetFrameworkToTest)-ios/', '*.app', System.IO.SearchOption.AllDirectories))">
<XHarnessAppBundleToTest Include="$([System.IO.Directory]::GetDirectories('$(ScenariosDir)MauiBlazorWebView.DeviceTests/Release/net11.0-ios/', '*.app', System.IO.SearchOption.AllDirectories))">
<TestTarget>ios-simulator-64</TestTarget>
<WorkItemTimeout>02:00:00</WorkItemTimeout>
<TestTimeout>01:00:00</TestTimeout>
Expand All @@ -148,7 +153,7 @@

<!-- MacCatalyst and Android: Use the original MAUIScenario approach (no category splitting) -->
<ItemGroup Condition="'$(TargetOS)' == 'maccatalyst'">
<XHarnessAppBundleToTest Include="$([System.IO.Directory]::GetDirectories('%(_MAUIScenarioSearch.PayloadDirectory)/Release/$(TargetFrameworkToTest)-maccatalyst/', '*.app', System.IO.SearchOption.AllDirectories))">
<XHarnessAppBundleToTest Include="$([System.IO.Directory]::GetDirectories('%(_MAUIScenarioSearch.PayloadDirectory)/Release/net11.0-maccatalyst/', '*.app', System.IO.SearchOption.AllDirectories))">
<TestTarget>maccatalyst</TestTarget>
<WorkItemTimeout>02:00:00</WorkItemTimeout>
<TestTimeout>01:00:00</TestTimeout>
Expand All @@ -158,7 +163,7 @@

<!-- Android: All scenarios run as single work items (no category splitting) -->
<ItemGroup Condition="'$(TargetOS)' == 'android'">
<_apks Include="%(_MAUIScenarioSearch.PayloadDirectory)/Release/$(TargetFrameworkToTest)-android/**/*Signed.apk" />
<_apks Include="%(_MAUIScenarioSearch.PayloadDirectory)/Release/net10.0-android/**/*Signed.apk" />
<XHarnessApkToTest Include="@(_apks)">
<WorkItemTimeout>02:00:00</WorkItemTimeout>
<TestTimeout>01:00:00</TestTimeout>
Expand Down
37 changes: 28 additions & 9 deletions eng/pipelines/arcade/stage-device-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,9 @@ parameters:
- name: checkoutDirectory
type: string
default: $(System.DefaultWorkingDirectory)
- name: HelixAccessToken
type: string
default: ''

stages:
- stage: devicetests_build
Expand All @@ -71,7 +74,7 @@ stages:
jobs:
- job: builddevice_tests
displayName: Build Device Tests (Mono)
timeoutInMinutes: 60
timeoutInMinutes: 120
variables:
- name: _BuildConfig
value: ${{ parameters.BuildConfiguration }}
Expand Down Expand Up @@ -145,7 +148,11 @@ stages:
- template: /eng/common/templates-official/steps/send-to-helix.yml
parameters:
HelixProjectPath: ${{ parameters.helixProject }}
HelixProjectArguments: /p:TargetOS=ios /p:TestRunNameSuffix=_$(_BuildConfig) /p:TestRunNamePrefix=DeviceTestsIOS_
${{ if eq(parameters.runAsPublic, true) }}:
HelixProjectArguments: /p:TargetOS=ios /p:TestRunNameSuffix=_$(_BuildConfig) /p:TestRunNamePrefix=DeviceTestsIOS_ /p:HelixInternal=False
${{ else }}:
HelixProjectArguments: /p:TargetOS=ios /p:TestRunNameSuffix=_$(_BuildConfig) /p:TestRunNamePrefix=DeviceTestsIOS_ /p:HelixInternal=True
HelixAccessToken: ${{ parameters.HelixAccessToken }}
HelixConfiguration: $(_BuildConfig)
IncludeDotNetCli: true
DisplayNamePrefix: DeviceTestsIOS
Expand Down Expand Up @@ -176,7 +183,7 @@ stages:
jobs:
- job: device_tests_maccatalyst
displayName: Run DeviceTests MacCatalyst (Mono)
timeoutInMinutes: 60
timeoutInMinutes: 240
variables:
- name: _BuildConfig
value: ${{ parameters.BuildConfiguration }}
Expand All @@ -195,7 +202,11 @@ stages:
- template: /eng/common/templates-official/steps/send-to-helix.yml
parameters:
HelixProjectPath: ${{ parameters.helixProject }}
HelixProjectArguments: /p:TargetOS=maccatalyst /p:TestRunNameSuffix=_$(_BuildConfig) /p:TestRunNamePrefix=DeviceTestsMacCatalyst_
${{ if eq(parameters.runAsPublic, true) }}:
HelixProjectArguments: /p:TargetOS=maccatalyst /p:TestRunNameSuffix=_$(_BuildConfig) /p:TestRunNamePrefix=DeviceTestsMacCatalyst_ /p:HelixInternal=False
${{ else }}:
HelixProjectArguments: /p:TargetOS=maccatalyst /p:TestRunNameSuffix=_$(_BuildConfig) /p:TestRunNamePrefix=DeviceTestsMacCatalyst_ /p:HelixInternal=True
HelixAccessToken: ${{ parameters.HelixAccessToken }}
HelixConfiguration: $(_BuildConfig)
IncludeDotNetCli: true
DisplayNamePrefix: DeviceTestsMacCatalyst
Expand Down Expand Up @@ -243,7 +254,11 @@ stages:
- template: /eng/common/templates-official/steps/send-to-helix.yml
parameters:
HelixProjectPath: ${{ parameters.helixProject }}
HelixProjectArguments: /p:TargetOS=android /p:TestRunNameSuffix=_$(_BuildConfig) /p:TestRunNamePrefix=DeviceTestsAndroid_
${{ if eq(parameters.runAsPublic, true) }}:
HelixProjectArguments: /p:TargetOS=android /p:TestRunNameSuffix=_$(_BuildConfig) /p:TestRunNamePrefix=DeviceTestsAndroid_ /p:HelixInternal=False
${{ else }}:
HelixProjectArguments: /p:TargetOS=android /p:TestRunNameSuffix=_$(_BuildConfig) /p:TestRunNamePrefix=DeviceTestsAndroid_ /p:HelixInternal=True
HelixAccessToken: ${{ parameters.HelixAccessToken }}
HelixConfiguration: $(_BuildConfig)
IncludeDotNetCli: true
DisplayNamePrefix: DeviceTestsAndroid
Expand Down Expand Up @@ -272,7 +287,7 @@ stages:
jobs:
- job: builddevice_tests_coreclr
displayName: Build Device Tests (CoreCLR)
timeoutInMinutes: 60
timeoutInMinutes: 120
variables:
- name: _BuildConfig
value: ${{ parameters.BuildConfiguration }}
Expand Down Expand Up @@ -346,7 +361,11 @@ stages:
- template: ${{ iif(eq(parameters.runAsPublic, 'true'), '/eng/common/templates/steps/send-to-helix.yml', '/eng/common/templates-official/steps/send-to-helix.yml@self') }}
parameters:
HelixProjectPath: ${{ parameters.helixProject }}
HelixProjectArguments: /p:TargetOS=android /p:TestRunNameSuffix=_$(_BuildConfig)_CoreCLR /p:TestRunNamePrefix=DeviceTestsAndroid_CoreCLR_
${{ if eq(parameters.runAsPublic, true) }}:
HelixProjectArguments: /p:TargetOS=android /p:TestRunNameSuffix=_$(_BuildConfig)_CoreCLR /p:TestRunNamePrefix=DeviceTestsAndroid_CoreCLR_ /p:HelixInternal=False
${{ else }}:
HelixProjectArguments: /p:TargetOS=android /p:TestRunNameSuffix=_$(_BuildConfig)_CoreCLR /p:TestRunNamePrefix=DeviceTestsAndroid_CoreCLR_ /p:HelixInternal=True
HelixAccessToken: ${{ parameters.HelixAccessToken }}
HelixConfiguration: $(_BuildConfig)
IncludeDotNetCli: true
DisplayNamePrefix: DeviceTestsAndroid_CoreCLR
Expand Down Expand Up @@ -394,7 +413,7 @@ stages:
- template: ${{ iif(eq(parameters.runAsPublic, 'true'), '/eng/common/templates/steps/send-to-helix.yml', '/eng/common/templates-official/steps/send-to-helix.yml@self') }}
parameters:
HelixProjectPath: ${{ parameters.helixProject }}
HelixProjectArguments: /p:TargetOS=ios /p:TestRunNameSuffix="_$(_BuildConfig)_CoreCLR" /p:TestRunNamePrefix="DeviceTestsIOS_CoreCLR_"
HelixProjectArguments: /p:TargetOS=ios /p:TestRunNameSuffix=_$(_BuildConfig)_CoreCLR /p:TestRunNamePrefix=DeviceTestsIOS_CoreCLR_
HelixConfiguration: $(_BuildConfig)
IncludeDotNetCli: true
DisplayNamePrefix: DeviceTestsIOS_CoreCLR
Expand Down Expand Up @@ -444,7 +463,7 @@ stages:
- template: ${{ iif(eq(parameters.runAsPublic, 'true'), '/eng/common/templates/steps/send-to-helix.yml', '/eng/common/templates-official/steps/send-to-helix.yml@self') }}
parameters:
HelixProjectPath: ${{ parameters.helixProject }}
HelixProjectArguments: /p:TargetOS=maccatalyst /p:TestRunNameSuffix="_$(_BuildConfig)_CoreCLR" /p:TestRunNamePrefix="DeviceTestsMacCatalyst_CoreCLR_"
HelixProjectArguments: /p:TargetOS=maccatalyst /p:TestRunNameSuffix=_$(_BuildConfig)_CoreCLR /p:TestRunNamePrefix=DeviceTestsMacCatalyst_CoreCLR_
HelixConfiguration: $(_BuildConfig)
IncludeDotNetCli: true
DisplayNamePrefix: DeviceTestsMacCatalyst_CoreCLR
52 changes: 44 additions & 8 deletions eng/pipelines/ci-device-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,36 @@ variables:
- template: /eng/common/templates/variables/pool-providers.yml@self
- template: /eng/pipelines/common/variables.yml@self
- template: /eng/pipelines/arcade/variables.yml@self
# Include Helix access token for internal builds
- ${{ if eq(variables['System.TeamProject'], 'internal') }}:
- group: DotNet-HelixApi-Access

parameters:

- name: windowsPool
# Internal pools (dnceng)
- name: windowsPoolInternal
type: object
default:
name: NetCore1ESPool-Internal
demands:
- ImageOverride -equals 1es-windows-2022
image: 1es-windows-2022
os: Windows

- name: macOSTestPoolInternal
type: object
default:
name: Azure Pipelines
vmImage: macOS-15

- name: macOSPoolInternal
type: object
default:
name: Azure Pipelines
vmImage: macOS-15

# Public pools (dnceng-public)
- name: windowsPoolPublic
type: object
default:
name: NetCore-Public
Expand All @@ -59,13 +85,13 @@ parameters:
image: 1es-windows-2022-open
os: Windows

- name: macOSTestPool
- name: macOSTestPoolPublic
type: object
default:
name: MAUI
vmImage: MAUI

- name: macOSPool
- name: macOSPoolPublic
type: object
default:
name: MAUI
Expand All @@ -84,10 +110,17 @@ stages:
# Use Helix for iOS / Android and MacCatalyst Device Tests
- template: /eng/pipelines/arcade/stage-device-tests.yml@self
parameters:
buildPool: ${{ parameters.macOSPool }}
testPool: ${{ parameters.windowsPool }}
runAsPublic: true
TargetFrameworkVersion: net11.0
# Select pools based on pipeline - internal vs public
${{ if eq(variables['System.TeamProject'], 'internal') }}:
buildPool: ${{ parameters.macOSPoolInternal }}
testPool: ${{ parameters.windowsPoolInternal }}
runAsPublic: false
HelixAccessToken: $(HelixApiAccessToken)
${{ else }}:
buildPool: ${{ parameters.macOSPoolPublic }}
testPool: ${{ parameters.windowsPoolPublic }}
runAsPublic: true
TargetFrameworkVersion: ${{ targetFrameworkVersion.tfm }}
prepareSteps:
- template: /eng/pipelines/common/provision.yml@self
parameters:
Expand All @@ -104,7 +137,10 @@ stages:
# Just use the old way for Windows Device Tests
- template: common/device-tests.yml
parameters:
windowsPool: ${{ parameters.windowsPool }}
${{ if eq(variables['System.TeamProject'], 'internal') }}:
windowsPool: ${{ parameters.windowsPoolInternal }}
${{ else }}:
windowsPool: ${{ parameters.windowsPoolPublic }}
targetFrameworkVersion: ${{ targetFrameworkVersion }}
windowsVersions: [ 'packaged', 'unpackaged' ]
skipProvisioning: true
Expand Down
74 changes: 62 additions & 12 deletions eng/pipelines/ci-uitests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -62,44 +62,85 @@ parameters:
type: boolean
default: false

- name: androidPool
# Internal pools (dnceng)
- name: androidPoolInternal
type: object
default:
name: MAUI
vmImage: MAUI
demands:
- Agent.OSArchitecture -equals ARM64

- name: androidPoolLinux
- name: androidPoolLinuxInternal
type: object
default:
name: NetCore1ESPool-Internal
demands:
- ImageOverride -equals 1ESPT-Ubuntu22.04

- name: iosPoolInternal
type: object
default:
name: MAUI
vmImage: MAUI

- name: windowsBuildPoolInternal
type: object
default:
name: NetCore1ESPool-Internal
demands:
- ImageOverride -equals 1es-windows-2022

- name: windowsPoolInternal
type: object
default:
name: NetCore1ESPool-Internal
demands:
- ImageOverride -equals 1es-windows-2022

- name: macosPoolInternal
type: object
default:
name: Azure Pipelines
vmImage: macOS-15

# Public pools (dnceng-public)
- name: androidPoolPublic
type: object
default:
name: MAUI
vmImage: MAUI

- name: androidPoolLinuxPublic
type: object
default:
name: MAUI-DNCENG
demands:
- ImageOverride -equals 1ESPT-Ubuntu22.04

- name: iosPool
- name: iosPoolPublic
type: object
default:
name: MAUI
vmImage: MAUI
demands:
- Agent.OSArchitecture -equals ARM64

- name: windowsBuildPool
- name: windowsBuildPoolPublic
type: object
default:
name: NetCore-Public
demands:
- ImageOverride -equals 1es-windows-2022-open

- name: windowsPool
- name: windowsPoolPublic
type: object
default:
name: NetCore-Public
demands:
- ImageOverride -equals 1es-windows-2022-open

- name: macosPool
- name: macosPoolPublic
type: object
default:
name: Azure Pipelines
Expand All @@ -109,12 +150,21 @@ stages:

- template: common/ui-tests.yml
parameters:
androidPool: ${{ parameters.androidPool }}
androidLinuxPool: ${{ parameters.androidPoolLinux }}
iosPool: ${{ parameters.iosPool }}
windowsPool: ${{ parameters.windowsPool }}
windowsBuildPool: ${{ parameters.windowsBuildPool }}
macosPool: ${{ parameters.macosPool }}
# Select pools based on pipeline - internal vs public
${{ if eq(variables['System.TeamProject'], 'internal') }}:
androidPool: ${{ parameters.androidPoolInternal }}
androidLinuxPool: ${{ parameters.androidPoolLinuxInternal }}
iosPool: ${{ parameters.iosPoolInternal }}
windowsPool: ${{ parameters.windowsPoolInternal }}
windowsBuildPool: ${{ parameters.windowsBuildPoolInternal }}
macosPool: ${{ parameters.macosPoolInternal }}
${{ else }}:
androidPool: ${{ parameters.androidPoolPublic }}
androidLinuxPool: ${{ parameters.androidPoolLinuxPublic }}
iosPool: ${{ parameters.iosPoolPublic }}
windowsPool: ${{ parameters.windowsPoolPublic }}
windowsBuildPool: ${{ parameters.windowsBuildPoolPublic }}
macosPool: ${{ parameters.macosPoolPublic }}
# BuildNativeAOT is false by default, but true in devdiv environment
BuildNativeAOT: ${{ or(parameters.BuildNativeAOT, and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'devdiv'))) }}
RunNativeAOT: ${{ parameters.RunNativeAOT }}
Expand Down
Loading
Loading