Updates

.gitignore

@@ -88,6 +88,9 @@ dlldata.c
 # Benchmark Results
 BenchmarkDotNet.Artifacts/
 
+# BenchmarkDotNet generated builds, written beside the benchmark project
+.bdn/
+
 # .NET Core
 project.lock.json
 project.fragment.lock.json

docs/benchmarkdotnet.md

@@ -294,7 +294,7 @@ M00_L00:
 
 The `--runtimes` or just `-r` allows you to run the benchmarks for **multiple Runtimes**.
 
-Available options are: Mono, wasmnet70, CoreRT, netcoreapp3.1, net6.0, net7.0, net8.0, and net9.0.
+Available options include: net8.0, net9.0, net10.0, net11.0, and the versioned forms of the other runtimes: mono11.0, nativeaot11.0, r2r11.0, monowasm11.0, monowasmaot11.0, corewasm11.0.
 
 Example: run the benchmarks for .NET 7.0 and 8.0:
 

docs/benchmarking-workflow-dotnet-runtime.md

@@ -206,11 +206,11 @@ validated individually.
 
 #### Run the benchmarks with AOT
 
-Essentially, add `--aotcompilermode wasm` to the `--bdn-arguments=".."`:
+Essentially, add `--wasm-runtime-flavor MonoAOT` alongside `--wasm`:
 
 ```cmd
-/path/to/dotnet/performance$ python3 ./scripts/benchmarks_ci.py --csproj src/benchmarks/micro/MicroBenchmarks.csproj -f net11.0 --dotnet-path </path/to/dotnet/runtime/>artifacts/bin/dotnet-latest --wasm --run-isolated --bdn-artifacts artifacts/BenchmarkDotNet.Artifacts
-    --bdn-arguments="--category-exclusion-filter NoInterpreter NoWASM NoMono --aotcompilermode wasm --logBuildOutput --buildTimeout 3600 --filter <filter>"
+/path/to/dotnet/performance$ python3 ./scripts/benchmarks_ci.py --csproj src/benchmarks/micro/MicroBenchmarks.csproj -f net11.0 --dotnet-path </path/to/dotnet/runtime/>artifacts/bin/dotnet-latest --wasm --wasm-runtime-flavor MonoAOT --run-isolated --bdn-artifacts artifacts/BenchmarkDotNet.Artifacts
+    --bdn-arguments="--category-exclusion-filter NoInterpreter NoWASM NoMono --logBuildOutput --buildTimeout 3600 --filter <filter>"
 ```
 
 #### Run CoreCLR WASM ReadyToRun with a nightly or VMR SDK

docs/microbenchmark-design-guidelines.md

@@ -411,9 +411,12 @@ However, **`[Params]` are applied to all the benchmarks in given class**. So if
 
 `[ArgumentsSource]` is xUnit's `[MemberData]` counterpart.
 
+How the source maps onto the benchmark's parameters is read from its declared element type.
+
+For a **single argument** the source yields the parameter's own type:
+
 ```cs
-// for single argument it's an IEnumerable of objects (object), not IEnumerable of arrays of objects (object[])
-public IEnumerable<object> Cultures()
+public IEnumerable<CultureInfo> Cultures()
 {
     yield return new CultureInfo("fr");
     yield return new CultureInfo("da");
@@ -427,6 +430,21 @@ public DateTime Parse(CultureInfo cultureInfo)
     => DateTime.Parse("10/10/2010 12:00:00 AM", cultureInfo);
 ```
 
+For **several arguments** it yields one element per case holding all of them, a `ValueTuple` naming each parameter's type:
+
+```cs
+public IEnumerable<(string input, NumberStyles styles)> Inputs()
+{
+    yield return ("1234", NumberStyles.Integer);
+    yield return ("1,234", NumberStyles.AllowThousands);
+    yield return ("0x4D2", NumberStyles.HexNumber);
+}
+
+[Benchmark]
+[ArgumentsSource(nameof(Inputs))]
+public int Parse(string input, NumberStyles styles) => int.Parse(input, styles);
+```
+
 **Note:** If you need to use the argument in the setup method, then instead of using `ArgumentsSource` you should use `ParamsSource`.
 
 **Note:** `[ArgumentsSource]` methods are only called once per execution of the benchmark, and the cached results are used per invocation.
@@ -445,7 +463,7 @@ public void Bitmap_FromStream(ImageTestData format)
     }
 }
 
-public IEnumerable<object> ImageFormats() => return new [] 
+public IEnumerable<ImageTestData> ImageFormats() => new [] 
 {
     new ImageTestData(ImageFormat.Bmp),
     new ImageTestData(ImageFormat.Jpeg),

eng/Versions.props

@@ -11,7 +11,7 @@
     <MicrosoftNETILLinkPackageVersion>11.0.0-rtm.26473.104</MicrosoftNETILLinkPackageVersion>
     <SystemThreadingChannelsPackageVersion>11.0.0-rtm.26473.104</SystemThreadingChannelsPackageVersion>
     <MicrosoftExtensionsLoggingPackageVersion>11.0.0-rtm.26473.104</MicrosoftExtensionsLoggingPackageVersion>
-    <BenchmarkDotNetVersion>0.16.0-nightly.20260801.595</BenchmarkDotNetVersion>
+    <BenchmarkDotNetVersion>0.16.0-nightly.20260923.641</BenchmarkDotNetVersion>
     <MicrosoftNETRuntimeEmscripten3156Nodewinx64Version>11.0.0-preview.5.26261.101</MicrosoftNETRuntimeEmscripten3156Nodewinx64Version>
     <MicrosoftDotNetXHarnessCLIVersion>11.0.0-prerelease.26473.1</MicrosoftDotNetXHarnessCLIVersion>
   </PropertyGroup>

scripts/benchmarks_local.py

@@ -399,13 +399,12 @@ def generate_single_benchmark_ci_args(parsed_args: Namespace, specific_run_type:
         os.environ['RestoreAdditionalProjectSources'] = os.path.join(get_run_artifact_path(parsed_args, RunType.WasmInterpreter, commit), "wasm_bundle")
 
     elif specific_run_type == RunType.WasmAOT:
-        benchmark_ci_args += ['--wasm', '--dotnet-path', os.path.join(get_run_artifact_path(parsed_args, RunType.WasmAOT, commit), "wasm_bundle", "dotnet")]
+        benchmark_ci_args += ['--wasm', '--wasm-runtime-flavor', 'MonoAOT', '--dotnet-path', os.path.join(get_run_artifact_path(parsed_args, RunType.WasmAOT, commit), "wasm_bundle", "dotnet")]
         bdn_args_unescaped += [
             '--anyCategories', 'Libraries', 'Runtime',
             '--category-exclusion-filter', 'NoInterpreter', 'NoWASM', 'NoMono',
             '--cli', os.path.join(get_run_artifact_path(parsed_args, RunType.WasmAOT, commit), "wasm_bundle", "dotnet", "dotnet"),
             '--wasmEngine', parsed_args.wasm_engine_path,
-            '--aotcompilermode', 'wasm',
             '--logBuildOutput',
             '--generateBinLog'
         ]

scripts/micro_benchmarks.py

@@ -145,8 +145,8 @@ def __get_bdn_arguments(user_input: str) -> list[str]:
         dest='wasm_runtime_flavor',
         required=False,
         default='Mono',
-        choices=['Mono', 'CoreCLR'],
-        help='Runtime flavor for WASM benchmarks: Mono (default) or CoreCLR'
+        choices=['Mono', 'MonoAOT', 'CoreCLR'],
+        help='Runtime flavor for WASM benchmarks: Mono interpreter (default), MonoAOT, or CoreCLR'
     )
 
     parser.add_argument(
@@ -298,22 +298,22 @@ def __get_benchmarkdotnet_arguments(framework: str, args: Any) -> list[str]:
     if framework.startswith("nativeaot"):
         run_args += ['--runtimes', framework]
     if args.wasm:
-        if framework == "net6.0":
-            run_args += ['--runtimes', 'wasm']
-        elif framework == "net7.0":
-            run_args += ['--runtimes', 'wasmnet70']
-        elif framework == "net8.0":
-            run_args += ['--runtimes', 'wasmnet80']
-        elif framework == "net9.0":
-            run_args += ['--runtimes', 'wasmnet90']
-        elif framework == "net10.0":
-            run_args += ['--runtimes', 'wasmnet10_0']
-        elif framework == "net11.0":
-            run_args += ['--runtimes', 'wasmnet11_0']
-        else:
+        wasm_moniker_prefix = {
+            'Mono': 'monowasm',
+            'MonoAOT': 'monowasmaot',
+            'CoreCLR': 'corewasm',
+        }[args.wasm_runtime_flavor]
+        wasm_version = {
+            'net6.0': '6.0',
+            'net7.0': '7.0',
+            'net8.0': '8.0',
+            'net9.0': '9.0',
+            'net10.0': '10.0',
+            'net11.0': '11.0',
+        }.get(framework)
+        if wasm_version is None:
             raise ArgumentTypeError('Framework {} is not supported for wasm'.format(framework))
-        if args.wasm_runtime_flavor != 'Mono':
-            run_args += ['--wasmRuntimeFlavor', args.wasm_runtime_flavor]
+        run_args += ['--runtimes', wasm_moniker_prefix + wasm_version]
 
     # Increase default 2 min build timeout to accommodate slow (or even very slow) hardware
     if not args.bdn_arguments or '--buildTimeout' not in args.bdn_arguments:

scripts/run_performance_job.py

@@ -554,7 +554,7 @@ def get_bdn_arguments(
             bdn_arguments += [
                 "--runtimes", "monoaotllvm",
                 "--aotcompilerpath", "$HELIX_CORRELATION_PAYLOAD/monoaot/mono-aot-cross",
-                "--customruntimepack", "$HELIX_CORRELATION_PAYLOAD/monoaot/pack", 
+                "--customruntimepack", "$HELIX_CORRELATION_PAYLOAD/monoaot/pack",
                 "--aotcompilermode", "llvm",
             ]
         else:
@@ -583,10 +583,7 @@ def get_bdn_arguments(
             bdn_arguments += ["\\\"--wasmArgs=--experimental-wasm-exnref\\\""]
 
         if is_aot:
-            bdn_arguments += [
-                "--aotcompilermode", "wasm",
-                "--buildTimeout", "3600"
-            ]
+            bdn_arguments += ["--buildTimeout", "3600"]
 
     if runtime_type == "wasm_coreclr":
         category_exclusions += ["NoWASM", "NoWasmCoreCLR", "NoMono"]
@@ -612,13 +609,13 @@ def get_bdn_arguments(
         ]
         if os_group == "windows":
             bdn_arguments += [
-                "--runtimes", "r2r11_0",
+                "--runtimes", "r2r11.0",
                 "--customruntimepack", "%HELIX_CORRELATION_PAYLOAD%\\r2r_interpreter\\runtimepack",
                 "--aotcompilerpath", "%HELIX_CORRELATION_PAYLOAD%\\r2r_interpreter\\crossgen2",
             ]
         else:
             bdn_arguments += [
-                "--runtimes", "r2r11_0",
+                "--runtimes", "r2r11.0",
                 "--customruntimepack", "$HELIX_CORRELATION_PAYLOAD/r2r_interpreter/runtimepack",
                 "--aotcompilerpath", "$HELIX_CORRELATION_PAYLOAD/r2r_interpreter/crossgen2",
             ]
@@ -753,6 +750,7 @@ def get_work_item_command(
         bdn_artifacts_dir: str,
         wasm_coreclr: bool = False,
         wasm_ready_to_run: bool = False,
+        wasm_aot: bool = False,
         only_sanity_check: bool = False,
         wasm_workload_source: Optional[str] = None):
     if os_group == "windows":
@@ -800,6 +798,8 @@ def get_work_item_command(
                     "--wasm-workload-source",
                     wasm_workload_source,
                 ]
+        elif wasm_aot:
+            work_item_command += ["--wasm-runtime-flavor", "MonoAOT"]
 
     work_item_command += ["--bdn-artifacts", bdn_artifacts_dir]
 
@@ -1496,6 +1496,7 @@ def get_work_item_command_for_artifact_dir(artifact_dir: str):
             artifact_dir,
             wasm_coreclr,
             wasm_coreclr and args.r2r_run_type == "r2r",
+            wasm_aot,
             args.only_sanity_check,
             helix_wasm_workload_source)
 

src/benchmarks/micro/Program.cs

@@ -5,6 +5,7 @@
 using System;
 using System.Collections.Generic;
 using System.Collections.Immutable;
+using System.Runtime.Versioning;
 using System.Threading.Tasks;
 using BenchmarkDotNet.Running;
 using System.IO;
@@ -15,6 +16,31 @@ namespace MicroBenchmarks
 {
     class Program
     {
+        // The MonoAotLLVM toolchain cannot build without these. Failing here reports an actionable message
+        // through the ArgumentException handler below, instead of passing an empty path into the generated
+        // project and failing deep inside MSBuild many minutes later.
+        private static string RequireMonoAotLLVMValue(List<string> values, string parameter)
+            => values.Count > 0
+                ? values[0]
+                : throw new ArgumentException($"{parameter} must be specified when running with --runtimes monoaotllvm");
+
+        // The TFM this assembly was compiled for. Environment.Version reports the runtime that actually
+        // loaded, which can roll forward to a newer major than --customruntimepack points at, producing an
+        // opaque restore/publish failure in the generated project.
+        private static string GetTargetFrameworkMoniker()
+        {
+            if (AppContext.TargetFrameworkName is string frameworkName)
+            {
+                var parsed = new FrameworkName(frameworkName);
+                if (parsed.Identifier == ".NETCoreApp")
+                {
+                    return $"net{parsed.Version.Major}.{parsed.Version.Minor}";
+                }
+            }
+
+            return $"net{Environment.Version.Major}.0";
+        }
+
         static async Task<int> Main(string[] args)
         {
             var argsList = new List<string>(args);
@@ -24,6 +50,7 @@ static async Task<int> Main(string[] args)
             List<string> categoryExclusionFilterValue;
             List<string> msBuildArguments;
             bool getDiffableDisasm;
+            MonoAotLLVMToolChain monoAotToolchain = null;
 
             // Parse and remove any additional parameters that we need that aren't part of BDN
             try
@@ -35,6 +62,58 @@ static async Task<int> Main(string[] args)
                 argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--msbuild-arguments", out msBuildArguments);
                 CommandLineOptions.ParseAndRemoveBooleanParameter(argsList, "--disasm-diff", out getDiffableDisasm);
 
+                // Extract monoaotllvm args not recognized by BDN and build the toolchain.
+                // BDN spells this option "-r, --runtimes" and matches monikers case-insensitively, so accept both forms here.
+                string runtimesArg = argsList.Contains("--runtimes") ? "--runtimes" : "-r";
+                argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, runtimesArg, out var runtimesValues);
+                int monoAotLLVMIndex = runtimesValues.FindIndex(runtime => string.Equals(runtime, "monoaotllvm", StringComparison.OrdinalIgnoreCase));
+                if (monoAotLLVMIndex >= 0)
+                {
+                    runtimesValues.RemoveAt(monoAotLLVMIndex);
+                    if (runtimesValues.Count > 0)
+                    {
+                        argsList.Add(runtimesArg);
+                        argsList.AddRange(runtimesValues);
+                    }
+                    argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--cli", out var cliPathValue);
+                    if (cliPathValue.Count > 0)
+                    {
+                        argsList.Add("--cli");
+                        argsList.AddRange(cliPathValue);
+                    }
+                    argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--packages", out var packagesPathValue);
+                    if (packagesPathValue.Count > 0)
+                    {
+                        argsList.Add("--packages");
+                        argsList.AddRange(packagesPathValue);
+                    }
+                    argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--aotcompilerpath", out var aotCompilerPathValue);
+                    string aotCompilerPath = RequireMonoAotLLVMValue(aotCompilerPathValue, "--aotcompilerpath");
+                    argsList.Add("--aotcompilerpath");
+                    argsList.AddRange(aotCompilerPathValue);
+
+                    argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--customruntimepack", out var customRuntimePackValue);
+                    string customRuntimePack = RequireMonoAotLLVMValue(customRuntimePackValue, "--customruntimepack");
+                    argsList.Add("--customruntimepack");
+                    argsList.AddRange(customRuntimePackValue);
+
+                    argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--aotcompilermode", out var aotCompilerModeValue);
+                    monoAotToolchain = new MonoAotLLVMToolChain(new MonoAotLLVMRuntime(Environment.Version), new()
+                    {
+                        CliPath = cliPathValue.Count > 0 ? new(cliPathValue[0]) : null,
+                        PackagesPath = packagesPathValue.Count > 0 ? new(packagesPathValue[0]) : null,
+                        TargetFrameworkMoniker = GetTargetFrameworkMoniker(),
+                        CustomRuntimePack = customRuntimePack,
+                        AotCompilerPath = aotCompilerPath,
+                        AotCompilerMode = aotCompilerModeValue.Count > 0 ? Enum.Parse<MonoAotCompilerMode>(aotCompilerModeValue[0], ignoreCase: true) : MonoAotCompilerMode.mini
+                    });
+                }
+                else if (runtimesValues.Count > 0)
+                {
+                    argsList.Add(runtimesArg);
+                    argsList.AddRange(runtimesValues);
+                }
+
                 CommandLineOptions.ValidatePartitionParameters(partitionCount, partitionIndex);
             }
             catch (ArgumentException e)
@@ -43,11 +122,6 @@ static async Task<int> Main(string[] args)
                 return 1;
             }
 
-            // Use RunAsync (not Run) so BDN does not install its single-threaded
-            // BenchmarkDotNetSynchronizationContext on the entrypoint thread. The sync
-            // entrypoint installs that context before benchmark discovery, which
-            // deadlocks any sync-over-async work performed by [ParamsSource]/[ArgumentsSource]
-            // callbacks (e.g. SslStreamTests.GetTls13Support).
             var summaries = await BenchmarkSwitcher
                 .FromAssembly(typeof(Program).Assembly)
                 .RunAsync(argsList.ToArray(),
@@ -59,7 +133,8 @@ static async Task<int> Main(string[] args)
                         exclusionFilterValue: exclusionFilterValue,
                         categoryExclusionFilterValue: categoryExclusionFilterValue,
                         getDiffableDisasm: getDiffableDisasm,
-                        msBuildArguments: msBuildArguments))
+                        msBuildArguments: msBuildArguments,
+                        toolchain: monoAotToolchain))
                 .ConfigureAwait(false);
 
             return summaries.ToExitCode();

src/benchmarks/micro/libraries/System.Buffers/Base64Tests.cs

@@ -99,7 +99,6 @@ public void SetupBase64IsValid()
     // To make the results stable the Iteration needs to last at least 100ms, this is why we are using bigger value for NumberOfBytes
     // Due to limitation of BDN, where Params have no Target and are applied to entire class the benchmarks live in a separate class.
     [BenchmarkCategory(Categories.Libraries, Categories.NoMono)]
-    [WarmupCount(30)] // make sure it's promoted to Tier 1
     public class Base64EncodeDecodeInPlaceTests
     {
         [Params(1000 * 1000 * 200)] // allows for stable iteration around 200ms

src/benchmarks/micro/libraries/System.Collections/Concurrent/AddRemoveFromDifferentThreads.cs

@@ -2,29 +2,18 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using BenchmarkDotNet.Attributes;
 using BenchmarkDotNet.Attributes.Filters;
-using BenchmarkDotNet.Configs;
-using BenchmarkDotNet.Filters;
 using MicroBenchmarks;
 
 namespace System.Collections.Concurrent
 {
-    internal class AddRemoveFromDifferentThreadsDisabledConfig : ManualConfig
-    {
-        public AddRemoveFromDifferentThreadsDisabledConfig() => AddFilter(new SimpleFilter(_ => false));
-    }
-
     [BenchmarkCategory(Categories.Libraries, Categories.Collections, Categories.GenericCollections, Categories.NoWASM)]
     [GenericTypeArguments(typeof(int))] // value type
     [GenericTypeArguments(typeof(string))] // reference type
-    [MinWarmupCount(6, forceAutoWarmup: true)]
-    [MaxWarmupCount(10, forceAutoWarmup: true)]
     [AotFilter("It hangs. https://github.com/dotnet/runtime/issues/66987")]
-    [Config(typeof(AddRemoveFromDifferentThreadsDisabledConfig))]
     public class AddRemoveFromDifferentThreads<T>
     {
         const int NumThreads = 2;