Imported Upstream version 6.6.0.89

Former-commit-id: b39a328747c2f3414dc52e009fb6f0aa80ca2492

external/linker/eng/common/PSScriptAnalyzerSettings.psd1

@@ -0,0 +1,11 @@
+@{
+    IncludeRules=@('PSAvoidUsingCmdletAliases',
+                   'PSAvoidUsingWMICmdlet',
+                   'PSAvoidUsingPositionalParameters',
+                   'PSAvoidUsingInvokeExpression',
+                   'PSUseDeclaredVarsMoreThanAssignments',
+                   'PSUseCmdletCorrectly',
+                   'PSStandardDSCFunctionsInResource',
+                   'PSUseIdenticalMandatoryParametersForDSC',
+                   'PSUseIdenticalParametersForDSC')
+}

external/linker/eng/common/SigningValidation.proj

@@ -3,7 +3,7 @@
   <!--
     This MSBuild file is intended to be used as the body of the default 
     publishing release pipeline. The release pipeline will use this file
-    to invoke the the SignCheck tool to validate that packages about to
+    to invoke the SignCheck tool to validate that packages about to
     be published are correctly signed.
   
     Parameters:

external/linker/eng/common/cross/build-rootfs.sh

@@ -203,6 +203,7 @@ if [[ "$__LinuxCodeName" == "alpine" ]]; then
       -X http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/main \
       -X http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/community \
       -X http://dl-cdn.alpinelinux.org/alpine/edge/testing \
+      -X http://dl-cdn.alpinelinux.org/alpine/edge/main \
       -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \
       add $__AlpinePackages
     rm -r $__ApkToolsDir

external/linker/eng/common/init-tools-native.ps1

@@ -98,12 +98,18 @@ try {
       }
 
       Write-Verbose "Installing $ToolName version $ToolVersion"
-      Write-Verbose "Executing '$InstallerPath $LocalInstallerArguments'"
+      Write-Verbose "Executing '$InstallerPath $($LocalInstallerArguments.Keys.ForEach({"-$_ '$($LocalInstallerArguments.$_)'"}) -join ' ')'"
       & $InstallerPath @LocalInstallerArguments
       if ($LASTEXITCODE -Ne "0") {
         $errMsg = "$ToolName installation failed"
         if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) {
-            Write-Warning $errMsg
+            $showNativeToolsWarning = $true
+            if ((Get-Variable 'DoNotDisplayNativeToolsInstallationWarnings' -ErrorAction 'SilentlyContinue') -and $DoNotDisplayNativeToolsInstallationWarnings) {
+                $showNativeToolsWarning = $false
+            }
+            if ($showNativeToolsWarning) {
+                Write-Warning $errMsg
+            }
             $toolInstallationFailure = $true
         } else {
             Write-Error $errMsg

external/linker/eng/common/init-tools-native.sh

@@ -70,8 +70,7 @@ function ReadGlobalJsonNativeTools {
   # Only extract the contents of the object.
   local native_tools_list=$(echo $native_tools_section | awk -F"[{}]" '{print $2}')
   native_tools_list=${native_tools_list//[\" ]/}
-  native_tools_list=${native_tools_list//,/$'\n'}
-  native_tools_list="$(echo -e "${native_tools_list}" | tr -d '[:space:]')"
+  native_tools_list=$( echo "$native_tools_list" | sed 's/\s//g' | sed 's/,/\n/g' )
 
   local old_IFS=$IFS
   while read -r line; do
@@ -108,6 +107,7 @@ else
     installer_command+=" --baseuri $base_uri"
     installer_command+=" --installpath $install_bin"
     installer_command+=" --version $tool_version"
+    echo $installer_command
 
     if [[ $force = true ]]; then
       installer_command+=" --force"

external/linker/eng/common/internal-feed-operations.ps1

@@ -0,0 +1,135 @@
+param(
+  [Parameter(Mandatory=$true)][string] $Operation,
+  [string] $AuthToken,
+  [string] $CommitSha,
+  [string] $RepoName,
+  [switch] $IsFeedPrivate
+)
+
+$ErrorActionPreference = "Stop"
+Set-StrictMode -Version 2.0
+
+. $PSScriptRoot\tools.ps1
+
+# Sets VSS_NUGET_EXTERNAL_FEED_ENDPOINTS based on the "darc-int-*" feeds defined in NuGet.config. This is needed
+# in build agents by CredProvider to authenticate the restore requests to internal feeds as specified in
+# https://github.com/microsoft/artifacts-credprovider/blob/0f53327cd12fd893d8627d7b08a2171bf5852a41/README.md#environment-variables. This should ONLY be called from identified
+# internal builds
+function SetupCredProvider {
+  param(
+    [string] $AuthToken
+  )    
+
+  # Install the Cred Provider NuGet plugin
+  Write-Host "Setting up Cred Provider NuGet plugin in the agent..."
+  Write-Host "Getting 'installcredprovider.ps1' from 'https://github.com/microsoft/artifacts-credprovider'..."
+
+  $url = 'https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.ps1'
+  
+  Write-Host "Writing the contents of 'installcredprovider.ps1' locally..."
+  Invoke-WebRequest $url -OutFile installcredprovider.ps1
+  
+  Write-Host "Installing plugin..."
+  .\installcredprovider.ps1 -Force
+  
+  Write-Host "Deleting local copy of 'installcredprovider.ps1'..."
+  Remove-Item .\installcredprovider.ps1
+
+  if (-Not("$env:USERPROFILE\.nuget\plugins\netcore")) {
+    Write-Host "CredProvider plugin was not installed correctly!"
+    ExitWithExitCode 1  
+  } 
+  else {
+    Write-Host "CredProvider plugin was installed correctly!"
+  }
+
+  # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable 
+  # feeds successfully
+
+  $nugetConfigPath = "$RepoRoot\NuGet.config"
+
+  if (-Not (Test-Path -Path $nugetConfigPath)) {
+    Write-Host "NuGet.config file not found in repo's root!"
+    ExitWithExitCode 1  
+  }
+  
+  $endpoints = New-Object System.Collections.ArrayList
+  $nugetConfigPackageSources = Select-Xml -Path $nugetConfigPath -XPath "//packageSources/add[contains(@key, 'darc-int-')]/@value" | foreach{$_.Node.Value}
+  
+  if (($nugetConfigPackageSources | Measure-Object).Count -gt 0 ) {
+    foreach ($stableRestoreResource in $nugetConfigPackageSources) {
+      $trimmedResource = ([string]$stableRestoreResource).Trim()
+      [void]$endpoints.Add(@{endpoint="$trimmedResource"; password="$AuthToken"}) 
+    }
+  }
+
+  if (($endpoints | Measure-Object).Count -gt 0) {
+      # Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}'
+      $endpointCredentials = @{endpointCredentials=$endpoints} | ConvertTo-Json -Compress
+
+     # Create the environment variables the AzDo way
+      Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $endpointCredentials -Properties @{
+        'variable' = 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS'
+        'issecret' = 'false'
+      } 
+
+      # We don't want sessions cached since we will be updating the endpoints quite frequently
+      Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data 'False' -Properties @{
+        'variable' = 'NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED'
+        'issecret' = 'false'
+      } 
+  }
+  else
+  {
+    Write-Host "No internal endpoints found in NuGet.config"
+  }
+}
+
+#Workaround for https://github.com/microsoft/msbuild/issues/4430
+function InstallDotNetSdkAndRestoreArcade {
+  $dotnetTempDir = "$RepoRoot\dotnet"
+  $dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*)
+  $dotnet = "$dotnetTempDir\dotnet.exe"
+  $restoreProjPath = "$PSScriptRoot\restore.proj"
+  
+  Write-Host "Installing dotnet SDK version $dotnetSdkVersion to restore Arcade SDK..."
+  InstallDotNetSdk "$dotnetTempDir" "$dotnetSdkVersion"
+  
+  '<Project Sdk="Microsoft.DotNet.Arcade.Sdk"/>' | Out-File "$restoreProjPath"
+
+  & $dotnet restore $restoreProjPath
+
+  Write-Host "Arcade SDK restored!"
+
+  if (Test-Path -Path $restoreProjPath) {
+    Remove-Item $restoreProjPath
+  }
+
+  if (Test-Path -Path $dotnetTempDir) {
+    Remove-Item $dotnetTempDir -Recurse
+  }
+}
+
+try {
+  Push-Location $PSScriptRoot
+
+  if ($Operation -like "setup") {
+    SetupCredProvider $AuthToken
+  } 
+  elseif ($Operation -like "install-restore") {
+    InstallDotNetSdkAndRestoreArcade
+  }
+  else {
+    Write-Host "Unknown operation '$Operation'!"
+    ExitWithExitCode 1  
+  }
+} 
+catch {
+  Write-Host $_
+  Write-Host $_.Exception
+  Write-Host $_.ScriptStackTrace
+  ExitWithExitCode 1
+} 
+finally {
+    Pop-Location
+}

external/linker/eng/common/internal-feed-operations.sh

@@ -0,0 +1,142 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Sets VSS_NUGET_EXTERNAL_FEED_ENDPOINTS based on the "darc-int-*" feeds defined in NuGet.config. This is needed
+# in build agents by CredProvider to authenticate the restore requests to internal feeds as specified in
+# https://github.com/microsoft/artifacts-credprovider/blob/0f53327cd12fd893d8627d7b08a2171bf5852a41/README.md#environment-variables. 
+# This should ONLY be called from identified internal builds
+function SetupCredProvider {
+  local authToken=$1
+  
+  # Install the Cred Provider NuGet plugin
+  echo "Setting up Cred Provider NuGet plugin in the agent..."...
+  echo "Getting 'installcredprovider.ps1' from 'https://github.com/microsoft/artifacts-credprovider'..."
+
+  local url="https://raw.githubusercontent.com/microsoft/artifacts-credprovider/master/helpers/installcredprovider.sh"  
+  
+  echo "Writing the contents of 'installcredprovider.ps1' locally..."
+  local installcredproviderPath="installcredprovider.sh"
+  if command -v curl > /dev/null; then
+    curl $url > "$installcredproviderPath"
+  else   
+    wget -q -O "$installcredproviderPath" "$url"
+  fi
+  
+  echo "Installing plugin..."
+  . "$installcredproviderPath"
+  
+  echo "Deleting local copy of 'installcredprovider.sh'..."
+  rm installcredprovider.sh
+
+  if [ ! -d "$HOME/.nuget/plugins" ]; then
+    echo "CredProvider plugin was not installed correctly!"
+    ExitWithExitCode 1  
+  else 
+    echo "CredProvider plugin was installed correctly!"
+  fi
+
+  # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable 
+  # feeds successfully
+
+  local nugetConfigPath="$repo_root/NuGet.config"
+
+  if [ ! "$nugetConfigPath" ]; then
+    echo "NuGet.config file not found in repo's root!"
+    ExitWithExitCode 1  
+  fi
+  
+  local endpoints='['
+  local nugetConfigPackageValues=`cat "$nugetConfigPath" | grep "key=\"darc-int-"`
+  local pattern="value=\"(.*)\""
+
+  for value in $nugetConfigPackageValues 
+  do
+    if [[ $value =~ $pattern ]]; then
+      local endpoint="${BASH_REMATCH[1]}"  
+      endpoints+="{\"endpoint\": \"$endpoint\", \"password\": \"$authToken\"},"
+    fi
+  done
+  
+  endpoints=${endpoints%?}
+  endpoints+=']'
+
+  if [ ${#endpoints} -gt 2 ]; then 
+      # Create the JSON object. It should look like '{"endpointCredentials": [{"endpoint":"http://example.index.json", "username":"optional", "password":"accesstoken"}]}'
+      local endpointCredentials="{\"endpointCredentials\": "$endpoints"}"
+
+      echo "##vso[task.setvariable variable=VSS_NUGET_EXTERNAL_FEED_ENDPOINTS]$endpointCredentials"
+      echo "##vso[task.setvariable variable=NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED]False"
+  else
+    echo "No internal endpoints found in NuGet.config"
+  fi
+} 
+
+# Workaround for https://github.com/microsoft/msbuild/issues/4430
+function InstallDotNetSdkAndRestoreArcade {
+  local dotnetTempDir="$repo_root/dotnet"
+  local dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*)
+  local restoreProjPath="$repo_root/eng/common/restore.proj"
+  
+  echo "Installing dotnet SDK version $dotnetSdkVersion to restore Arcade SDK..."
+  echo "<Project Sdk=\"Microsoft.DotNet.Arcade.Sdk\"/>" > "$restoreProjPath"
+  
+  InstallDotNetSdk "$dotnetTempDir" "$dotnetSdkVersion"
+
+  local res=`$dotnetTempDir/dotnet restore $restoreProjPath`
+  echo "Arcade SDK restored!"
+
+  # Cleanup
+  if [ "$restoreProjPath" ]; then
+    rm "$restoreProjPath"
+  fi
+
+  if [ "$dotnetTempDir" ]; then
+    rm -r $dotnetTempDir
+  fi
+}
+
+source="${BASH_SOURCE[0]}"
+operation=''
+authToken=''
+repoName=''
+
+while [[ $# > 0 ]]; do
+  opt="$(echo "$1" | awk '{print tolower($0)}')"
+  case "$opt" in
+    --operation)
+      operation=$2
+      shift
+      ;;
+    --authtoken)
+      authToken=$2
+      shift
+      ;;
+    *)
+      echo "Invalid argument: $1"
+      usage
+      exit 1
+      ;;
+  esac
+
+  shift
+done
+
+while [[ -h "$source" ]]; do
+  scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+  source="$(readlink "$source")"
+  # if $source was a relative symlink, we need to resolve it relative to the path where the
+  # symlink file was located
+  [[ $source != /* ]] && source="$scriptroot/$source"
+done
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+. "$scriptroot/tools.sh"
+
+if [ "$operation" = "setup" ]; then
+  SetupCredProvider $authToken
+elif [ "$operation" = "install-restore" ]; then
+  InstallDotNetSdkAndRestoreArcade
+else
+  echo "Unknown operation '$operation'!"
+fi

external/linker/eng/common/native/CommonLibrary.psm1

@@ -59,9 +59,38 @@ function DownloadAndExtract {
                                           -Verbose:$Verbose
 
   if ($UnzipStatus -Eq $False) {
-    Write-Error "Unzip failed"
-    return $False
+    # Retry Download one more time with Force=true
+    $DownloadRetryStatus = CommonLibrary\Get-File -Uri $Uri `
+                                             -Path $TempToolPath `
+                                             -DownloadRetries 1 `
+                                             -RetryWaitTimeInSeconds $RetryWaitTimeInSeconds `
+                                             -Force:$True `
+                                             -Verbose:$Verbose
+
+    if ($DownloadRetryStatus -Eq $False) {
+      Write-Error "Last attempt of download failed as well"
+      return $False
+    }
+
+    # Retry unzip again one more time with Force=true
+    $UnzipRetryStatus = CommonLibrary\Expand-Zip -ZipPath $TempToolPath `
+                                            -OutputDirectory $InstallDirectory `
+                                            -Force:$True `
+                                            -Verbose:$Verbose
+    if ($UnzipRetryStatus -Eq $False)
+    {
+      Write-Error "Last attempt of unzip failed as well"
+      # Clean up partial zips and extracts
+      if (Test-Path $TempToolPath) {
+        Remove-Item $TempToolPath -Force
+      }
+      if (Test-Path $InstallDirectory) {
+        Remove-Item $InstallDirectory -Force -Recurse
+      }
+      return $False
+    }
   }
+
   return $True
 }
 

external/linker/eng/common/native/install-cmake-test.sh

@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+
+source="${BASH_SOURCE[0]}"
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+. $scriptroot/common-library.sh
+
+base_uri=
+install_path=
+version=
+clean=false
+force=false
+download_retries=5
+retry_wait_time_seconds=30
+
+while (($# > 0)); do
+  lowerI="$(echo $1 | awk '{print tolower($0)}')"
+  case $lowerI in
+    --baseuri)
+      base_uri=$2
+      shift 2
+      ;;
+    --installpath)
+      install_path=$2
+      shift 2
+      ;;
+    --version)
+      version=$2
+      shift 2
+      ;;
+    --clean)
+      clean=true
+      shift 1
+      ;;
+    --force)
+      force=true
+      shift 1
+      ;;
+    --downloadretries)
+      download_retries=$2
+      shift 2
+      ;;
+    --retrywaittimeseconds)
+      retry_wait_time_seconds=$2
+      shift 2
+      ;;
+    --help)
+      echo "Common settings:"
+      echo "  --baseuri <value>        Base file directory or Url wrom which to acquire tool archives"
+      echo "  --installpath <value>    Base directory to install native tool to"
+      echo "  --clean                  Don't install the tool, just clean up the current install of the tool"
+      echo "  --force                  Force install of tools even if they previously exist"
+      echo "  --help                   Print help and exit"
+      echo ""
+      echo "Advanced settings:"
+      echo "  --downloadretries        Total number of retry attempts"
+      echo "  --retrywaittimeseconds   Wait time between retry attempts in seconds"
+      echo ""
+      exit 0
+      ;;
+  esac
+done
+
+tool_name="cmake-test"
+tool_os=$(GetCurrentOS)
+tool_folder=$(echo $tool_os | awk '{print tolower($0)}')
+tool_arch="x86_64"
+tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch"
+tool_install_directory="$install_path/$tool_name/$version"
+tool_file_path="$tool_install_directory/$tool_name_moniker/bin/$tool_name"
+shim_path="$install_path/$tool_name.sh"
+uri="${base_uri}/$tool_folder/$tool_name/$tool_name_moniker.tar.gz"
+
+# Clean up tool and installers
+if [[ $clean = true ]]; then
+  echo "Cleaning $tool_install_directory"
+  if [[ -d $tool_install_directory ]]; then
+    rm -rf $tool_install_directory
+  fi
+
+  echo "Cleaning $shim_path"
+  if [[ -f $shim_path ]]; then
+    rm -rf $shim_path
+  fi
+
+  tool_temp_path=$(GetTempPathFileName $uri)
+  echo "Cleaning $tool_temp_path"
+  if [[ -f $tool_temp_path ]]; then
+    rm -rf $tool_temp_path
+  fi
+
+  exit 0
+fi
+
+# Install tool
+if [[ -f $tool_file_path ]] && [[ $force = false ]]; then
+  echo "$tool_name ($version) already exists, skipping install"
+  exit 0
+fi
+
+DownloadAndExtract $uri $tool_install_directory $force $download_retries $retry_wait_time_seconds
+
+if [[ $? != 0 ]]; then
+  echo "Installation failed" >&2
+  exit 1
+fi
+
+# Generate Shim
+# Always rewrite shims so that we are referencing the expected version
+NewScriptShim $shim_path $tool_file_path true
+
+if [[ $? != 0 ]]; then
+  echo "Shim generation failed" >&2
+  exit 1
+fi
+
+exit 0

external/linker/eng/common/native/install-cmake.sh

@@ -69,7 +69,7 @@ tool_name_moniker="$tool_name-$version-$tool_os-$tool_arch"
 tool_install_directory="$install_path/$tool_name/$version"
 tool_file_path="$tool_install_directory/$tool_name_moniker/bin/$tool_name"
 shim_path="$install_path/$tool_name.sh"
-uri="${base_uri}/$tool_folder/cmake/$tool_name_moniker.tar.gz"
+uri="${base_uri}/$tool_folder/$tool_name/$tool_name_moniker.tar.gz"
 
 # Clean up tool and installers
 if [[ $clean = true ]]; then

external/linker/eng/common/performance/perfhelixpublish.proj

@@ -0,0 +1,77 @@
+<Project Sdk="Microsoft.DotNet.Helix.Sdk" DefaultTargets="Test">
+
+  <PropertyGroup Condition="'$(AGENT_OS)' == 'Windows_NT'">
+    <WorkItemCommand>%HELIX_CORRELATION_PAYLOAD%\performance\scripts\benchmarks_ci.py --csproj %HELIX_CORRELATION_PAYLOAD%\performance\$(TargetCsproj)</WorkItemCommand>
+    <CliArguments>--dotnet-versions %DOTNET_VERSION% --cli-source-info args --cli-branch %PERFLAB_BRANCH% --cli-commit-sha %PERFLAB_HASH% --cli-repository https://github.com/%PERFLAB_REPO% --cli-source-timestamp %PERFLAB_BUILDTIMESTAMP%</CliArguments>
+    <Python>py -3</Python>
+    <CoreRun>%HELIX_CORRELATION_PAYLOAD%\Core_Root\CoreRun.exe</CoreRun>
+    <HelixPreCommands>$(HelixPreCommands);call %HELIX_CORRELATION_PAYLOAD%\performance\tools\machine-setup.cmd</HelixPreCommands>
+    <ArtifactsDirectory>%HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts</ArtifactsDirectory>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(AGENT_OS)' != 'Windows_NT' and '$(RunFromPerfRepo)' == 'false'">
+    <BaseDirectory>$HELIX_CORRELATION_PAYLOAD</BaseDirectory>
+    <PerformanceDirectory>$(BaseDirectory)/performance</PerformanceDirectory>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(AGENT_OS)' != 'Windows_NT' and '$(RunFromPerfRepo)' == 'true'">
+    <BaseDirectory>$HELIX_WORKITEM_PAYLOAD</BaseDirectory>
+    <PerformanceDirectory>$(BaseDirectory)</PerformanceDirectory>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(AGENT_OS)' != 'Windows_NT'">
+    <WorkItemCommand>$(PerformanceDirectory)/scripts/benchmarks_ci.py --csproj $(PerformanceDirectory)/$(TargetCsproj)</WorkItemCommand>
+    <CliArguments>--dotnet-versions $DOTNET_VERSION --cli-source-info args --cli-branch $PERFLAB_BRANCH --cli-commit-sha $PERFLAB_HASH --cli-repository https://github.com/$PERFLAB_REPO --cli-source-timestamp $PERFLAB_BUILDTIMESTAMP</CliArguments>
+    <Python>python3</Python>
+    <CoreRun>$(BaseDirectory)/Core_Root/corerun</CoreRun>
+    <HelixPreCommands>$(HelixPreCommands);chmod +x $(PerformanceDirectory)/tools/machine-setup.sh;. $(PerformanceDirectory)/tools/machine-setup.sh</HelixPreCommands>
+    <ArtifactsDirectory>$(BaseDirectory)/artifacts/BenchmarkDotNet.Artifacts</ArtifactsDirectory>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(UseCoreRun)' == 'true'">
+    <CoreRunArgument>--corerun $(CoreRun)</CoreRunArgument>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(WorkItemCommand)' != ''">
+    <WorkItemCommand>$(Python) $(WorkItemCommand) --incremental no --architecture $(Architecture) -f $(_Framework) $(PerfLabArguments)</WorkItemCommand>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(_Framework)' != 'net461'">
+    <WorkItemCommand>$(WorkItemCommand) $(CliArguments)</WorkItemCommand>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <HelixCorrelationPayload Include="$(CorrelationPayloadDirectory)">
+      <PayloadDirectory>%(Identity)</PayloadDirectory>
+    </HelixCorrelationPayload>
+  </ItemGroup>
+
+  <PropertyGroup>
+    <PartitionCount>5</PartitionCount>
+  </PropertyGroup>
+  <ItemGroup>
+    <Partition Include="$(BuildConfig).Partition0" Index="0" />
+    <Partition Include="$(BuildConfig).Partition1" Index="1" />
+    <Partition Include="$(BuildConfig).Partition2" Index="2" />
+    <Partition Include="$(BuildConfig).Partition3" Index="3" />
+    <Partition Include="$(BuildConfig).Partition4" Index="4" />
+  </ItemGroup>
+
+  <!-- 
+    Partition the Microbenchmarks project, but nothing else
+  -->
+  <ItemGroup Condition="$(TargetCsproj.Contains('MicroBenchmarks.csproj'))">
+    <HelixWorkItem Include="@(Partition)">
+      <PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
+      <Command>$(WorkItemCommand) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --artifacts $(ArtifactsDirectory) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)"</Command>
+      <Timeout>4:00</Timeout>
+    </HelixWorkItem>
+  </ItemGroup>
+  <ItemGroup Condition="!$(TargetCsproj.Contains('MicroBenchmarks.csproj'))">
+    <HelixWorkItem Include="$(BuildConfig).WorkItem">
+      <PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
+      <Command>$(WorkItemCommand) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --artifacts $(ArtifactsDirectory)"</Command>
+      <Timeout>4:00</Timeout>
+    </HelixWorkItem>
+  </ItemGroup>
+</Project>

external/linker/eng/common/performance/performance-setup.ps1

@@ -0,0 +1,91 @@
+Param(
+    [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY,
+    [string] $CoreRootDirectory,
+    [string] $Architecture="x64",
+    [string] $Framework="netcoreapp3.0",
+    [string] $CompilationMode="Tiered",
+    [string] $Repository=$env:BUILD_REPOSITORY_NAME,
+    [string] $Branch=$env:BUILD_SOURCEBRANCH,
+    [string] $CommitSha=$env:BUILD_SOURCEVERSION,
+    [string] $BuildNumber=$env:BUILD_BUILDNUMBER,
+    [string] $RunCategories="coreclr corefx",
+    [string] $Csproj="src\benchmarks\micro\MicroBenchmarks.csproj",
+    [string] $Kind="micro",
+    [switch] $Internal,
+    [string] $Configurations="CompilationMode=$CompilationMode"
+)
+
+$RunFromPerformanceRepo = ($Repository -eq "dotnet/performance")
+$UseCoreRun = ($CoreRootDirectory -ne [string]::Empty)
+
+$PayloadDirectory = (Join-Path $SourceDirectory "Payload")
+$PerformanceDirectory = (Join-Path $PayloadDirectory "performance")
+$WorkItemDirectory = (Join-Path $SourceDirectory "workitem")
+$ExtraBenchmarkDotNetArguments = "--iterationCount 1 --warmupCount 0 --invocationCount 1 --unrollFactor 1 --strategy ColdStart --stopOnFirstError true"
+$Creator = $env:BUILD_DEFINITIONNAME
+$PerfLabArguments = ""
+$HelixSourcePrefix = "pr"
+
+$Queue = "Windows.10.Amd64.ClientRS4.DevEx.15.8.Open"
+
+if ($Framework.StartsWith("netcoreapp")) {
+    $Queue = "Windows.10.Amd64.ClientRS4.Open"
+}
+
+if ($Internal) {
+    $Queue = "Windows.10.Amd64.ClientRS5.Perf"
+    $PerfLabArguments = "--upload-to-perflab-container"
+    $ExtraBenchmarkDotNetArguments = ""
+    $Creator = ""
+    $HelixSourcePrefix = "official"
+}
+
+$CommonSetupArguments="--frameworks $Framework --queue $Queue --build-number $BuildNumber --build-configs $Configurations"
+$SetupArguments = "--repository https://github.com/$Repository --branch $Branch --get-perf-hash --commit-sha $CommitSha $CommonSetupArguments"
+
+if ($RunFromPerformanceRepo) {
+    $SetupArguments = "--perf-hash $CommitSha $CommonSetupArguments"
+    
+    robocopy $SourceDirectory $PerformanceDirectory /E /XD $PayloadDirectory $SourceDirectory\artifacts $SourceDirectory\.git
+}
+else {
+    git clone --branch master --depth 1 --quiet https://github.com/dotnet/performance $PerformanceDirectory
+}
+
+if ($UseCoreRun) {
+    $NewCoreRoot = (Join-Path $PayloadDirectory "Core_Root")
+    Move-Item -Path $CoreRootDirectory -Destination $NewCoreRoot
+}
+
+$DocsDir = (Join-Path $PerformanceDirectory "docs")
+robocopy $DocsDir $WorkItemDirectory
+
+# Set variables that we will need to have in future steps
+$ci = $true
+
+. "$PSScriptRoot\..\pipeline-logging-functions.ps1"
+
+# Directories
+Write-PipelineSetVariable -Name 'PayloadDirectory' -Value "$PayloadDirectory" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'PerformanceDirectory' -Value "$PerformanceDirectory" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'WorkItemDirectory' -Value "$WorkItemDirectory" -IsMultiJobVariable $false
+
+# Script Arguments
+Write-PipelineSetVariable -Name 'Python' -Value "py -3" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'ExtraBenchmarkDotNetArguments' -Value "$ExtraBenchmarkDotNetArguments" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'SetupArguments' -Value "$SetupArguments" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'PerfLabArguments' -Value "$PerfLabArguments" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'BDNCategories' -Value "$RunCategories" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'TargetCsproj' -Value "$Csproj" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'Kind' -Value "$Kind" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'Architecture' -Value "$Architecture" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'UseCoreRun' -Value "$UseCoreRun" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'RunFromPerfRepo' -Value "$RunFromPerformanceRepo" -IsMultiJobVariable $false
+
+# Helix Arguments
+Write-PipelineSetVariable -Name 'Creator' -Value "$Creator" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'Queue' -Value "$Queue" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'HelixSourcePrefix' -Value "$HelixSourcePrefix" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name '_BuildConfig' -Value "$Architecture.$Kind.$Framework" -IsMultiJobVariable $false
+
+exit 0

external/linker/eng/common/performance/performance-setup.sh

@@ -0,0 +1,176 @@
+#!/usr/bin/env bash
+
+source_directory=$BUILD_SOURCESDIRECTORY
+core_root_directory=
+architecture=x64
+framework=netcoreapp3.0
+compilation_mode=tiered
+repository=$BUILD_REPOSITORY_NAME
+branch=$BUILD_SOURCEBRANCH
+commit_sha=$BUILD_SOURCEVERSION
+build_number=$BUILD_BUILDNUMBER
+internal=false
+kind="micro"
+run_categories="coreclr corefx"
+csproj="src\benchmarks\micro\MicroBenchmarks.csproj"
+configurations=
+run_from_perf_repo=false
+use_core_run=true
+
+while (($# > 0)); do
+  lowerI="$(echo $1 | awk '{print tolower($0)}')"
+  case $lowerI in
+    --sourcedirectory)
+      source_directory=$2
+      shift 2
+      ;;
+    --corerootdirectory)
+      core_root_directory=$2
+      shift 2
+      ;;
+    --architecture)
+      architecture=$2
+      shift 2
+      ;;
+    --framework)
+      framework=$2
+      shift 2
+      ;;
+    --compilationmode)
+      compilation_mode=$2
+      shift 2
+      ;;
+    --repository)
+      repository=$2
+      shift 2
+      ;;
+    --branch)
+      branch=$2
+      shift 2
+      ;;
+    --commitsha)
+      commit_sha=$2
+      shift 2
+      ;;
+    --buildnumber)
+      build_number=$2
+      shift 2
+      ;;
+    --kind)
+      kind=$2
+      shift 2
+      ;;
+    --runcategories)
+      run_categories=$2
+      shift 2
+      ;;
+    --csproj)
+      csproj=$2
+      shift 2
+      ;;
+    --internal)
+      internal=true
+      shift 1
+      ;;
+    --configurations)
+      configurations=$2
+      shift 2
+      ;;
+    --help)
+      echo "Common settings:"
+      echo "  --corerootdirectory <value>    Directory where Core_Root exists, if running perf testing with --corerun"
+      echo "  --architecture <value>         Architecture of the testing being run"
+      echo "  --configurations <value>       List of key=value pairs that will be passed to perf testing infrastructure."
+      echo "                                 ex: --configurations \"CompilationMode=Tiered OptimzationLevel=PGO\""
+      echo "  --help                         Print help and exit"
+      echo ""
+      echo "Advanced settings:"
+      echo "  --framework <value>            The framework to run, if not running in master"
+      echo "  --compliationmode <value>      The compilation mode if not passing --configurations"
+      echo "  --sourcedirectory <value>      The directory of the sources. Defaults to env:BUILD_SOURCESDIRECTORY"
+      echo "  --repository <value>           The name of the repository in the <owner>/<repository name> format. Defaults to env:BUILD_REPOSITORY_NAME"
+      echo "  --branch <value>               The name of the branch. Defaults to env:BUILD_SOURCEBRANCH"
+      echo "  --commitsha <value>            The commit sha1 to run against. Defaults to env:BUILD_SOURCEVERSION"
+      echo "  --buildnumber <value>          The build number currently running. Defaults to env:BUILD_BUILDNUMBER"
+      echo "  --csproj                       The relative path to the benchmark csproj whose tests should be run. Defaults to src\benchmarks\micro\MicroBenchmarks.csproj"
+      echo "  --kind <value>                 Related to csproj. The kind of benchmarks that should be run. Defaults to micro"
+      echo "  --runcategories <value>        Related to csproj. Categories of benchmarks to run. Defaults to \"coreclr corefx\""
+      echo "  --internal                     If the benchmarks are running as an official job."
+      echo ""
+      exit 0
+      ;;
+  esac
+done
+
+if [[ "$repository" == "dotnet/performance" ]]; then
+    run_from_perf_repo=true
+fi
+
+if [ -z "$configurations" ]; then
+    configurations="CompliationMode=$compilation_mode"
+fi
+
+if [ -z "$core_root_directory" ]; then
+    use_core_run=false
+fi
+
+payload_directory=$source_directory/Payload
+performance_directory=$payload_directory/performance
+workitem_directory=$source_directory/workitem
+extra_benchmark_dotnet_arguments="--iterationCount 1 --warmupCount 0 --invocationCount 1 --unrollFactor 1 --strategy ColdStart --stopOnFirstError true"
+perflab_arguments=
+queue=Ubuntu.1804.Amd64.Open
+creator=$BUILD_DEFINITIONNAME
+helix_source_prefix="pr"
+
+if [[ "$internal" == true ]]; then
+    perflab_arguments="--upload-to-perflab-container"
+    helix_source_prefix="official"
+    creator=
+    extra_benchmark_dotnet_arguments=
+    
+    if [[ "$architecture" = "arm64" ]]; then
+        queue=Ubuntu.1804.Arm64.Perf
+    else
+        queue=Ubuntu.1804.Amd64.Perf
+    fi
+fi
+
+common_setup_arguments="--frameworks $framework --queue $queue --build-number $build_number --build-configs $configurations"
+setup_arguments="--repository https://github.com/$repository --branch $branch --get-perf-hash --commit-sha $commit_sha $common_setup_arguments"
+
+if [[ "$run_from_perf_repo" = true ]]; then
+    payload_directory=
+    workitem_directory=$source_directory
+    performance_directory=$workitem_directory
+    setup_arguments="--perf-hash $commit_sha $common_setup_arguments"
+else
+    git clone --branch master --depth 1 --quiet https://github.com/dotnet/performance $performance_directory
+    
+    docs_directory=$performance_directory/docs
+    mv $docs_directory $workitem_directory
+fi
+
+if [[ "$use_core_run" = true ]]; then
+    new_core_root=$payload_directory/Core_Root
+    mv $core_root_directory $new_core_root
+fi
+
+# Make sure all of our variables are available for future steps
+echo "##vso[task.setvariable variable=UseCoreRun]$use_core_run"
+echo "##vso[task.setvariable variable=Architecture]$architecture"
+echo "##vso[task.setvariable variable=PayloadDirectory]$payload_directory"
+echo "##vso[task.setvariable variable=PerformanceDirectory]$performance_directory"
+echo "##vso[task.setvariable variable=WorkItemDirectory]$workitem_directory"
+echo "##vso[task.setvariable variable=Queue]$queue"
+echo "##vso[task.setvariable variable=SetupArguments]$setup_arguments"
+echo "##vso[task.setvariable variable=Python]python3"
+echo "##vso[task.setvariable variable=PerfLabArguments]$perflab_arguments"
+echo "##vso[task.setvariable variable=ExtraBenchmarkDotNetArguments]$extra_benchmark_dotnet_arguments"
+echo "##vso[task.setvariable variable=BDNCategories]$run_categories"
+echo "##vso[task.setvariable variable=TargetCsproj]$csproj"
+echo "##vso[task.setvariable variable=RunFromPerfRepo]$run_from_perf_repo"
+echo "##vso[task.setvariable variable=Creator]$creator"
+echo "##vso[task.setvariable variable=HelixSourcePrefix]$helix_source_prefix"
+echo "##vso[task.setvariable variable=Kind]$kind"
+echo "##vso[task.setvariable variable=_BuildConfig]$architecture.$kind.$framework"

external/linker/eng/common/pipeline-logging-functions.ps1

@@ -77,13 +77,14 @@ function Write-PipelineTaskError {
       [string]$Name,
       [string]$Value,
       [switch]$Secret,
-      [switch]$AsOutput)
-  
+      [switch]$AsOutput,
+      [bool]$IsMultiJobVariable=$true)
+
       if($ci) {
         Write-LoggingCommand -Area 'task' -Event 'setvariable' -Data $Value -Properties @{
           'variable' = $Name
           'isSecret' = $Secret
-          'isOutput' = 'true'
+          'isOutput' = $IsMultiJobVariable
         } -AsOutput:$AsOutput
       }
   }

external/linker/eng/common/pipeline-logging-functions.sh

@@ -39,11 +39,11 @@ function Write-PipelineTaskError {
     return
   fi
 
-  message_type="error"
-  sourcepath=''
-  linenumber=''
-  columnnumber=''
-  error_code=''
+  local message_type="error"
+  local sourcepath=''
+  local linenumber=''
+  local columnnumber=''
+  local error_code=''
 
   while [[ $# -gt 0 ]]; do
     opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
@@ -76,7 +76,7 @@ function Write-PipelineTaskError {
     shift
   done
 
-  message="##vso[task.logissue"
+  local message="##vso[task.logissue"
 
   message="$message type=$message_type"
 
@@ -100,3 +100,73 @@ function Write-PipelineTaskError {
   echo "$message"
 }
 
+function Write-PipelineSetVariable {
+  if [[ "$ci" != true ]]; then
+    return
+  fi
+
+  local name=''
+  local value=''
+  local secret=false
+  local as_output=false
+  local is_multi_job_variable=true
+
+  while [[ $# -gt 0 ]]; do
+    opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
+    case "$opt" in
+      -name|-n)
+        name=$2
+        shift
+        ;;
+      -value|-v)
+        value=$2
+        shift
+        ;;
+      -secret|-s)
+        secret=true
+        ;;
+      -as_output|-a)
+        as_output=true
+        ;;
+      -is_multi_job_variable|-i)
+        is_multi_job_variable=$2
+        shift
+        ;;
+    esac
+    shift
+  done
+
+  value=${value/;/%3B}
+  value=${value/\\r/%0D}
+  value=${value/\\n/%0A}
+  value=${value/]/%5D}
+
+  local message="##vso[task.setvariable variable=$name;isSecret=$secret;isOutput=$is_multi_job_variable]$value"
+
+  if [[ "$as_output" == true ]]; then
+    $message
+  else
+    echo "$message"
+  fi
+}
+
+function Write-PipelinePrependPath {
+  local prepend_path=''
+
+  while [[ $# -gt 0 ]]; do
+    opt="$(echo "${1/#--/-}" | awk '{print tolower($0)}')"
+    case "$opt" in
+      -path|-p)
+        prepend_path=$2
+        shift
+        ;;
+    esac
+    shift
+  done
+
+  export PATH="$prepend_path:$PATH"
+
+  if [[ "$ci" == true ]]; then
+    echo "##vso[task.prependpath]$prepend_path"
+  fi
+}