Table of Contents

Sample: IntroNuGet

You can set specific versions of NuGet dependencies for each job using MsBuild properties in your csproj. It allows comparing different versions of the same package (if there are no breaking changes in API).

Source code

using System;
using System.Collections.Immutable;
using System.Linq;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Jobs;

namespace BenchmarkDotNet.Samples
{
    /// <summary>
    /// Benchmarks between various versions of a NuGet package
    /// </summary>
    /// <remarks>
    /// Only supported with CsProj toolchains.
    /// </remarks>
    [Config(typeof(Config))]
    public class IntroNuGet
    {
        // Setup your csproj like this:
        /*
        <PropertyGroup>
          <!-- Use 9.0.0 as default package version if not specified -->
          <SciVersion Condition="'$(SciVersion)' == ''">9.0.0</SciVersion>
        </PropertyGroup>
        <ItemGroup>
          <PackageReference Include="System.Collections.Immutable" Version="$(SciVersion)" />
        </ItemGroup>
        */
        // All versions of the package must be source-compatible with your benchmark code.
        private class Config : ManualConfig
        {
            public Config()
            {
                string[] targetVersions = [
                    "9.0.0",
                    "9.0.3",
                    "9.0.5",
                ];

                foreach (var version in targetVersions)
                {
                    AddJob(Job.MediumRun
                        .WithMsBuildArguments($"/p:SciVersion={version}")
                        .WithId($"v{version}")
                    );
                }
            }
        }

        private static readonly Random rand = new Random(Seed: 0);
        private static readonly double[] values = Enumerable.Range(1, 10_000).Select(x => rand.NextDouble()).ToArray();

        [Benchmark]
        public void ToImmutableArrayBenchmark()
        {
            var results = values.ToImmutableArray();
        }
    }
}

Output

Method Job Arguments Mean Error StdDev
ToImmutableArrayBenchmark v9.0.0 /p:SciVersion=9.0.0 1.173 μs 0.0057 μs 0.0086 μs
ToImmutableArrayBenchmark v9.0.3 /p:SciVersion=9.0.3 1.173 μs 0.0038 μs 0.0058 μs
ToImmutableArrayBenchmark v9.0.5 /p:SciVersion=9.0.5 1.172 μs 0.0107 μs 0.0157 μs