Imported Upstream version 6.8.0.73

Former-commit-id: d18deab1b47cfd3ad8cba82b3f37d00eec2170af
2019-12-10 18:00:56 +00:00
parent bceda29824
commit 73ee7591e8
1043 changed files with 16271 additions and 22080 deletions
--- a/external/linker/eng/common/performance/perfhelixpublish.proj
+++ b/external/linker/eng/common/performance/perfhelixpublish.proj
@@ -5,8 +5,14 @@
    <CliArguments>--dotnet-versions %DOTNET_VERSION% --cli-source-info args --cli-branch %PERFLAB_BRANCH% --cli-commit-sha %PERFLAB_HASH% --cli-repository https://github.com/%PERFLAB_REPO% --cli-source-timestamp %PERFLAB_BUILDTIMESTAMP%</CliArguments>
    <Python>py -3</Python>
    <CoreRun>%HELIX_CORRELATION_PAYLOAD%\Core_Root\CoreRun.exe</CoreRun>
+    <BaselineCoreRun>%HELIX_CORRELATION_PAYLOAD%\Baseline_Core_Root\CoreRun.exe</BaselineCoreRun>
    <HelixPreCommands>$(HelixPreCommands);call %HELIX_CORRELATION_PAYLOAD%\performance\tools\machine-setup.cmd</HelixPreCommands>
    <ArtifactsDirectory>%HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts</ArtifactsDirectory>
+    <BaselineArtifactsDirectory>%HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts_Baseline</BaselineArtifactsDirectory>
+    <ResultsComparer>%HELIX_CORRELATION_PAYLOAD%\performance\src\tools\ResultsComparer\ResultsComparer.csproj</ResultsComparer>
+    <DotnetExe>%HELIX_CORRELATION_PAYLOAD%\performance\tools\dotnet\$(Architecture)\dotnet.exe</DotnetExe>
+    <Percent>%25%25</Percent>
+    <XMLResults>%HELIX_WORKITEM_ROOT%\testResults.xml</XMLResults>
  </PropertyGroup>

  <PropertyGroup Condition="'$(AGENT_OS)' != 'Windows_NT' and '$(RunFromPerfRepo)' == 'false'">
@@ -24,14 +30,24 @@
    <CliArguments>--dotnet-versions $DOTNET_VERSION --cli-source-info args --cli-branch $PERFLAB_BRANCH --cli-commit-sha $PERFLAB_HASH --cli-repository https://github.com/$PERFLAB_REPO --cli-source-timestamp $PERFLAB_BUILDTIMESTAMP</CliArguments>
    <Python>python3</Python>
    <CoreRun>$(BaseDirectory)/Core_Root/corerun</CoreRun>
+    <BaselineCoreRun>$(BaseDirectory)/Baseline_Core_Root/corerun</BaselineCoreRun>
    <HelixPreCommands>$(HelixPreCommands);chmod +x $(PerformanceDirectory)/tools/machine-setup.sh;. $(PerformanceDirectory)/tools/machine-setup.sh</HelixPreCommands>
    <ArtifactsDirectory>$(BaseDirectory)/artifacts/BenchmarkDotNet.Artifacts</ArtifactsDirectory>
+    <BaselineArtifactsDirectory>$(BaseDirectory)/artifacts/BenchmarkDotNet.Artifacts_Baseline</BaselineArtifactsDirectory>
+    <ResultsComparer>$(PerformanceDirectory)/src/tools/ResultsComparer/ResultsComparer.csproj</ResultsComparer>
+    <DotnetExe>$(PerformanceDirectory)/tools/dotnet/$(Architecture)/dotnet</DotnetExe>
+    <Percent>%25</Percent>
+    <XMLResults>$HELIX_WORKITEM_ROOT/testResults.xml</XMLResults>
  </PropertyGroup>

  <PropertyGroup Condition="'$(UseCoreRun)' == 'true'">
    <CoreRunArgument>--corerun $(CoreRun)</CoreRunArgument>
  </PropertyGroup>

+  <PropertyGroup Condition="'$(UseBaselineCoreRun)' == 'true'">
+    <BaselineCoreRunArgument>--corerun $(BaselineCoreRun)</BaselineCoreRunArgument>
+  </PropertyGroup>
+
  <PropertyGroup Condition="'$(WorkItemCommand)' != ''">
    <WorkItemCommand>$(Python) $(WorkItemCommand) --incremental no --architecture $(Architecture) -f $(_Framework) $(PerfLabArguments)</WorkItemCommand>
  </PropertyGroup>
@@ -57,20 +73,29 @@
    <Partition Include="$(BuildConfig).Partition4" Index="4" />
  </ItemGroup>

+  <PropertyGroup Condition="'$(Compare)' == 'true'">
+    <FailOnTestFailure>false</FailOnTestFailure>
+  </PropertyGroup>
+
  <!-- 
    Partition the Microbenchmarks project, but nothing else
  -->
  <ItemGroup Condition="$(TargetCsproj.Contains('MicroBenchmarks.csproj'))">
    <HelixWorkItem Include="@(Partition)">
      <PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
-      <Command>$(WorkItemCommand) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --artifacts $(ArtifactsDirectory) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)"</Command>
+      <PreCommands Condition="'$(Compare)' == 'true'">$(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)"</PreCommands>
+      <Command>$(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)"</Command>
+      <PostCommands Condition="'$(Compare)' == 'true'">$(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults);$(FinalCommand)</PostCommands>
      <Timeout>4:00</Timeout>
    </HelixWorkItem>
  </ItemGroup>
+
  <ItemGroup Condition="!$(TargetCsproj.Contains('MicroBenchmarks.csproj'))">
    <HelixWorkItem Include="$(BuildConfig).WorkItem">
      <PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
-      <Command>$(WorkItemCommand) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --artifacts $(ArtifactsDirectory)"</Command>
+      <PreCommands Condition="'$(Compare)' == 'true'">$(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument)"</PreCommands>
+      <Command>$(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument)"</Command>
+      <PostCommands Condition="'$(Compare)' == 'true'">$(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults)</PostCommands>
      <Timeout>4:00</Timeout>
    </HelixWorkItem>
  </ItemGroup>
--- a/external/linker/eng/common/performance/performance-setup.ps1
+++ b/external/linker/eng/common/performance/performance-setup.ps1
@@ -1,8 +1,9 @@
 Param(
    [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY,
    [string] $CoreRootDirectory,
+    [string] $BaselineCoreRootDirectory,
    [string] $Architecture="x64",
-    [string] $Framework="netcoreapp3.0",
+    [string] $Framework="netcoreapp5.0",
    [string] $CompilationMode="Tiered",
    [string] $Repository=$env:BUILD_REPOSITORY_NAME,
    [string] $Branch=$env:BUILD_SOURCEBRANCH,
@@ -12,11 +13,13 @@ Param(
    [string] $Csproj="src\benchmarks\micro\MicroBenchmarks.csproj",
    [string] $Kind="micro",
    [switch] $Internal,
+    [switch] $Compare,
    [string] $Configurations="CompilationMode=$CompilationMode"
 )

-$RunFromPerformanceRepo = ($Repository -eq "dotnet/performance")
+$RunFromPerformanceRepo = ($Repository -eq "dotnet/performance") -or ($Repository -eq "dotnet-performance")
 $UseCoreRun = ($CoreRootDirectory -ne [string]::Empty)
+$UseBaselineCoreRun = ($BaselineCoreRootDirectory -ne [string]::Empty)

 $PayloadDirectory = (Join-Path $SourceDirectory "Payload")
 $PerformanceDirectory = (Join-Path $PayloadDirectory "performance")
@@ -29,11 +32,17 @@ $HelixSourcePrefix = "pr"
 $Queue = "Windows.10.Amd64.ClientRS4.DevEx.15.8.Open"

 if ($Framework.StartsWith("netcoreapp")) {
-    $Queue = "Windows.10.Amd64.ClientRS4.Open"
+    $Queue = "Windows.10.Amd64.ClientRS5.Open"
+}
+
+if ($Compare) {
+    $Queue = "Windows.10.Amd64.19H1.Tiger.Perf.Open"
+    $PerfLabArguments = ""
+    $ExtraBenchmarkDotNetArguments = ""
 }

 if ($Internal) {
-    $Queue = "Windows.10.Amd64.ClientRS5.Perf"
+    $Queue = "Windows.10.Amd64.19H1.Tiger.Perf"
    $PerfLabArguments = "--upload-to-perflab-container"
    $ExtraBenchmarkDotNetArguments = ""
    $Creator = ""
@@ -56,6 +65,10 @@ if ($UseCoreRun) {
    $NewCoreRoot = (Join-Path $PayloadDirectory "Core_Root")
    Move-Item -Path $CoreRootDirectory -Destination $NewCoreRoot
 }
+if ($UseBaselineCoreRun) {
+    $NewBaselineCoreRoot = (Join-Path $PayloadDirectory "Baseline_Core_Root")
+    Move-Item -Path $BaselineCoreRootDirectory -Destination $NewBaselineCoreRoot
+}

 $DocsDir = (Join-Path $PerformanceDirectory "docs")
 robocopy $DocsDir $WorkItemDirectory
@@ -80,7 +93,9 @@ Write-PipelineSetVariable -Name 'TargetCsproj' -Value "$Csproj" -IsMultiJobVaria
 Write-PipelineSetVariable -Name 'Kind' -Value "$Kind" -IsMultiJobVariable $false
 Write-PipelineSetVariable -Name 'Architecture' -Value "$Architecture" -IsMultiJobVariable $false
 Write-PipelineSetVariable -Name 'UseCoreRun' -Value "$UseCoreRun" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'UseBaselineCoreRun' -Value "$UseBaselineCoreRun" -IsMultiJobVariable $false
 Write-PipelineSetVariable -Name 'RunFromPerfRepo' -Value "$RunFromPerformanceRepo" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'Compare' -Value "$Compare" -IsMultiJobVariable $false

 # Helix Arguments
 Write-PipelineSetVariable -Name 'Creator' -Value "$Creator" -IsMultiJobVariable $false
--- a/external/linker/eng/common/performance/performance-setup.sh
+++ b/external/linker/eng/common/performance/performance-setup.sh
@@ -2,20 +2,23 @@

 source_directory=$BUILD_SOURCESDIRECTORY
 core_root_directory=
+baseline_core_root_directory=
 architecture=x64
-framework=netcoreapp3.0
+framework=netcoreapp5.0
 compilation_mode=tiered
 repository=$BUILD_REPOSITORY_NAME
 branch=$BUILD_SOURCEBRANCH
 commit_sha=$BUILD_SOURCEVERSION
 build_number=$BUILD_BUILDNUMBER
 internal=false
+compare=false
 kind="micro"
 run_categories="coreclr corefx"
 csproj="src\benchmarks\micro\MicroBenchmarks.csproj"
 configurations=
 run_from_perf_repo=false
 use_core_run=true
+use_baseline_core_run=true

 while (($# > 0)); do
  lowerI="$(echo $1 | awk '{print tolower($0)}')"
@@ -28,6 +31,10 @@ while (($# > 0)); do
      core_root_directory=$2
      shift 2
      ;;
+    --baselinecorerootdirectory)
+      baseline_core_root_directory=$2
+      shift 2
+      ;;
    --architecture)
      architecture=$2
      shift 2
@@ -72,6 +79,10 @@ while (($# > 0)); do
      internal=true
      shift 1
      ;;
+    --compare)
+      compare=true
+      shift 1
+      ;;
    --configurations)
      configurations=$2
      shift 2
@@ -102,7 +113,7 @@ while (($# > 0)); do
  esac
 done

-if [[ "$repository" == "dotnet/performance" ]]; then
+if [ "$repository" == "dotnet/performance" ] || [ "$repository" == "dotnet-performance" ]; then
    run_from_perf_repo=true
 fi

@@ -114,6 +125,10 @@ if [ -z "$core_root_directory" ]; then
    use_core_run=false
 fi

+if [ -z "$baseline_core_root_directory" ]; then
+    use_baseline_core_run=false
+fi
+
 payload_directory=$source_directory/Payload
 performance_directory=$payload_directory/performance
 workitem_directory=$source_directory/workitem
@@ -123,6 +138,19 @@ queue=Ubuntu.1804.Amd64.Open
 creator=$BUILD_DEFINITIONNAME
 helix_source_prefix="pr"

+if [[ "$compare" == true ]]; then
+  extra_benchmark_dotnet_arguments=
+  perflab_arguments=
+
+  # No open queues for arm64
+  if [[ "$architecture" = "arm64" ]]; then
+    echo "Compare not available for arm64"
+    exit 1
+  fi
+
+  queue=Ubuntu.1804.Amd64.Tiger.Perf.Open
+fi
+
 if [[ "$internal" == true ]]; then
    perflab_arguments="--upload-to-perflab-container"
    helix_source_prefix="official"
@@ -132,7 +160,7 @@ if [[ "$internal" == true ]]; then
    if [[ "$architecture" = "arm64" ]]; then
        queue=Ubuntu.1804.Arm64.Perf
    else
-        queue=Ubuntu.1804.Amd64.Perf
+        queue=Ubuntu.1804.Amd64.Tiger.Perf
    fi
 fi

@@ -156,21 +184,33 @@ if [[ "$use_core_run" = true ]]; then
    mv $core_root_directory $new_core_root
 fi

+if [[ "$use_baseline_core_run" = true ]]; then
+  new_baseline_core_root=$payload_directory/Baseline_Core_Root
+  mv $baseline_core_root_directory $new_baseline_core_root
+fi
+
+ci=true
+
+_script_dir=$(pwd)/eng/common
+. "$_script_dir/pipeline-logging-functions.sh"
+
 # Make sure all of our variables are available for future steps
-echo "##vso[task.setvariable variable=UseCoreRun]$use_core_run"
-echo "##vso[task.setvariable variable=Architecture]$architecture"
-echo "##vso[task.setvariable variable=PayloadDirectory]$payload_directory"
-echo "##vso[task.setvariable variable=PerformanceDirectory]$performance_directory"
-echo "##vso[task.setvariable variable=WorkItemDirectory]$workitem_directory"
-echo "##vso[task.setvariable variable=Queue]$queue"
-echo "##vso[task.setvariable variable=SetupArguments]$setup_arguments"
-echo "##vso[task.setvariable variable=Python]python3"
-echo "##vso[task.setvariable variable=PerfLabArguments]$perflab_arguments"
-echo "##vso[task.setvariable variable=ExtraBenchmarkDotNetArguments]$extra_benchmark_dotnet_arguments"
-echo "##vso[task.setvariable variable=BDNCategories]$run_categories"
-echo "##vso[task.setvariable variable=TargetCsproj]$csproj"
-echo "##vso[task.setvariable variable=RunFromPerfRepo]$run_from_perf_repo"
-echo "##vso[task.setvariable variable=Creator]$creator"
-echo "##vso[task.setvariable variable=HelixSourcePrefix]$helix_source_prefix"
-echo "##vso[task.setvariable variable=Kind]$kind"
-echo "##vso[task.setvariable variable=_BuildConfig]$architecture.$kind.$framework"
+Write-PipelineSetVariable -name "UseCoreRun" -value "$use_core_run" -is_multi_job_variable false
+Write-PipelineSetVariable -name "UseBaselineCoreRun" -value "$use_baseline_core_run" -is_multi_job_variable false
+Write-PipelineSetVariable -name "Architecture" -value "$architecture" -is_multi_job_variable false
+Write-PipelineSetVariable -name "PayloadDirectory" -value "$payload_directory" -is_multi_job_variable false
+Write-PipelineSetVariable -name "PerformanceDirectory" -value "$performance_directory" -is_multi_job_variable false
+Write-PipelineSetVariable -name "WorkItemDirectory" -value "$workitem_directory" -is_multi_job_variable false
+Write-PipelineSetVariable -name "Queue" -value "$queue" -is_multi_job_variable false
+Write-PipelineSetVariable -name "SetupArguments" -value "$setup_arguments" -is_multi_job_variable false
+Write-PipelineSetVariable -name "Python" -value "$python3" -is_multi_job_variable false
+Write-PipelineSetVariable -name "PerfLabArguments" -value "$perflab_arguments" -is_multi_job_variable false
+Write-PipelineSetVariable -name "ExtraBenchmarkDotNetArguments" -value "$extra_benchmark_dotnet_arguments" -is_multi_job_variable false
+Write-PipelineSetVariable -name "BDNCategories" -value "$run_categories" -is_multi_job_variable false
+Write-PipelineSetVariable -name "TargetCsproj" -value "$csproj" -is_multi_job_variable false
+Write-PipelineSetVariable -name "RunFromPerfRepo" -value "$run_from_perf_repo" -is_multi_job_variable false
+Write-PipelineSetVariable -name "Creator" -value "$creator" -is_multi_job_variable false
+Write-PipelineSetVariable -name "HelixSourcePrefix" -value "$helix_source_prefix" -is_multi_job_variable false
+Write-PipelineSetVariable -name "Kind" -value "$kind" -is_multi_job_variable false
+Write-PipelineSetVariable -name "_BuildConfig" -value "$architecture.$kind.$framework" -is_multi_job_variable false
+Write-PipelineSetVariable -name "Compare" -value "$compare" -is_multi_job_variable false