Imported Upstream version 6.10.0.49

Former-commit-id: 1d6753294b2993e1fbf92de9366bb9544db4189b

external/linker/.editorconfig

@@ -2,4 +2,12 @@
 root = true
 
 [*.cs]
-indent_style = tab
+indent_style = tab
+csharp_new_line_before_open_brace = types,methods
+csharp_new_line_before_else = false
+csharp_new_line_before_catch = false
+csharp_new_line_before_finally = false
+csharp_space_after_keywords_in_control_flow_statements = true
+csharp_space_between_method_declaration_name_and_open_parenthesis = true
+csharp_space_between_method_call_name_and_opening_parenthesis = true
+csharp_space_before_open_square_brackets = true

external/linker/README.md

@@ -45,7 +45,7 @@ A link xml file can be used to explicitly preserve assemblies, types, and member
     <type fullname="Assembly1.B"/>
     <!--Preserve all fields on a type-->
     <type fullname="Assembly1.C" preserve="fields"/>
-    <!--Preserve all fields on a type-->
+    <!--Preserve all methods on a type-->
     <type fullname="Assembly1.D" preserve="methods"/>
     <!--Preserve the type only-->
     <type fullname="Assembly1.E" preserve="nothing"/>

external/linker/eng/Version.Details.xml

@@ -3,9 +3,9 @@
   <ProductDependencies>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="5.0.0-beta.19502.11">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="5.0.0-beta.19601.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>da75c6547c64a91eb8a9cecf8c503331c2cc394a</Sha>
+      <Sha>c0b56ff3569e3c7475070486c40543ea4c6f6dc7</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

external/linker/eng/common/CheckSymbols.ps1

@@ -5,11 +5,12 @@ param(
 )
 
 Add-Type -AssemblyName System.IO.Compression.FileSystem
+. $PSScriptRoot\pipeline-logging-functions.ps1
 
 function FirstMatchingSymbolDescriptionOrDefault {
   param( 
     [string] $FullPath,                  # Full path to the module that has to be checked
-    [string] $TargetServerParam,         # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols
+    [string] $TargetServerParameter,         # Parameter to pass to `Symbol Tool` indicating the server to lookup for symbols
     [string] $SymbolsPath
   )
 
@@ -21,36 +22,36 @@ function FirstMatchingSymbolDescriptionOrDefault {
   # checking and which type of file was uploaded.
 
   # The file itself is returned
-  $SymbolPath = $SymbolsPath + "\" + $FileName
+  $SymbolPath = $SymbolsPath + '\' + $FileName
 
   # PDB file for the module
-  $PdbPath = $SymbolPath.Replace($Extension, ".pdb")
+  $PdbPath = $SymbolPath.Replace($Extension, '.pdb')
 
   # PDB file for R2R module (created by crossgen)
-  $NGenPdb = $SymbolPath.Replace($Extension, ".ni.pdb")
+  $NGenPdb = $SymbolPath.Replace($Extension, '.ni.pdb')
 
   # DBG file for a .so library
-  $SODbg = $SymbolPath.Replace($Extension, ".so.dbg")
+  $SODbg = $SymbolPath.Replace($Extension, '.so.dbg')
 
   # DWARF file for a .dylib
-  $DylibDwarf = $SymbolPath.Replace($Extension, ".dylib.dwarf")
+  $DylibDwarf = $SymbolPath.Replace($Extension, '.dylib.dwarf')
  
-  .\dotnet-symbol.exe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null
+  .\dotnet-symbol.exe --symbols --modules --windows-pdbs $TargetServerParameter $FullPath -o $SymbolsPath | Out-Null
 
   if (Test-Path $PdbPath) {
-    return "PDB"
+    return 'PDB'
   }
   elseif (Test-Path $NGenPdb) {
-    return "NGen PDB"
+    return 'NGen PDB'
   }
   elseif (Test-Path $SODbg) {
-    return "DBG for SO"
+    return 'DBG for SO'
   }  
   elseif (Test-Path $DylibDwarf) {
-    return "Dwarf for Dylib"
+    return 'Dwarf for Dylib'
   }  
   elseif (Test-Path $SymbolPath) {
-    return "Module"
+    return 'Module'
   }
   else {
     return $null
@@ -68,7 +69,7 @@ function CountMissingSymbols {
   }
   
   # Extensions for which we'll look for symbols
-  $RelevantExtensions = @(".dll", ".exe", ".so", ".dylib")
+  $RelevantExtensions = @('.dll', '.exe', '.so', '.dylib')
 
   # How many files are missing symbol information
   $MissingSymbols = 0
@@ -76,7 +77,7 @@ function CountMissingSymbols {
   $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath)
   $PackageGuid = New-Guid
   $ExtractPath = Join-Path -Path $ExtractPath -ChildPath $PackageGuid
-  $SymbolsPath = Join-Path -Path $ExtractPath -ChildPath "Symbols"
+  $SymbolsPath = Join-Path -Path $ExtractPath -ChildPath 'Symbols'
   
   [System.IO.Compression.ZipFile]::ExtractToDirectory($PackagePath, $ExtractPath)
 
@@ -86,31 +87,31 @@ function CountMissingSymbols {
   Get-ChildItem -Recurse $ExtractPath |
     Where-Object {$RelevantExtensions -contains $_.Extension} |
     ForEach-Object {
-      if ($_.FullName -Match "\\ref\\") {
+      if ($_.FullName -Match '\\ref\\') {
         Write-Host "`t Ignoring reference assembly file" $_.FullName
         return
       }
 
-      $SymbolsOnMSDL = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--microsoft-symbol-server" $SymbolsPath
-      $SymbolsOnSymWeb = FirstMatchingSymbolDescriptionOrDefault $_.FullName "--internal-server" $SymbolsPath
+      $SymbolsOnMSDL = FirstMatchingSymbolDescriptionOrDefault -FullPath $_.FullName  -TargetServerParameter '--microsoft-symbol-server' -SymbolsPath $SymbolsPath
+      $SymbolsOnSymWeb = FirstMatchingSymbolDescriptionOrDefault -FullPath $_.FullName -TargetServerParameter '--internal-server' -SymbolsPath $SymbolsPath
 
       Write-Host -NoNewLine "`t Checking file" $_.FullName "... "
   
       if ($SymbolsOnMSDL -ne $null -and $SymbolsOnSymWeb -ne $null) {
-        Write-Host "Symbols found on MSDL (" $SymbolsOnMSDL ") and SymWeb (" $SymbolsOnSymWeb ")"
+        Write-Host "Symbols found on MSDL (${$SymbolsOnMSDL}) and SymWeb (${$SymbolsOnSymWeb})"
       }
       else {
         $MissingSymbols++
 
         if ($SymbolsOnMSDL -eq $null -and $SymbolsOnSymWeb -eq $null) {
-          Write-Host "No symbols found on MSDL or SymWeb!"
+          Write-Host 'No symbols found on MSDL or SymWeb!'
         }
         else {
           if ($SymbolsOnMSDL -eq $null) {
-            Write-Host "No symbols found on MSDL!"
+            Write-Host 'No symbols found on MSDL!'
           }
           else {
-            Write-Host "No symbols found on SymWeb!"
+            Write-Host 'No symbols found on SymWeb!'
           }
         }
       }
@@ -129,26 +130,26 @@ function CheckSymbolsAvailable {
   Get-ChildItem "$InputPath\*.nupkg" |
     ForEach-Object {
       $FileName = $_.Name
-	  
+
       # These packages from Arcade-Services include some native libraries that
       # our current symbol uploader can't handle. Below is a workaround until
       # we get issue: https://github.com/dotnet/arcade/issues/2457 sorted.
-      if ($FileName -Match "Microsoft\.DotNet\.Darc\.") {
+      if ($FileName -Match 'Microsoft\.DotNet\.Darc\.') {
         Write-Host "Ignoring Arcade-services file: $FileName"
         Write-Host
         return
       }
-      elseif ($FileName -Match "Microsoft\.DotNet\.Maestro\.Tasks\.") {
+      elseif ($FileName -Match 'Microsoft\.DotNet\.Maestro\.Tasks\.') {
         Write-Host "Ignoring Arcade-services file: $FileName"
         Write-Host
         return
       }
-	  
+
       Write-Host "Validating $FileName "
       $Status = CountMissingSymbols "$InputPath\$FileName"
   
       if ($Status -ne 0) {
-        Write-Error "Missing symbols for $Status modules in the package $FileName"
+        Write-PipelineTelemetryError -Category 'CheckSymbols' -Message "Missing symbols for $Status modules in the package $FileName"
       }
 
       Write-Host

external/linker/eng/common/PublishToSymbolServers.proj

@@ -37,6 +37,8 @@
     <PropertyGroup>
       <DotNetSymbolExpirationInDays Condition="'$(DotNetSymbolExpirationInDays)' == ''">3650</DotNetSymbolExpirationInDays>
       <PublishToSymbolServer>true</PublishToSymbolServer>
+      <PublishToSymWeb Condition="'$(PublishToSymWeb)' == ''">true</PublishToSymWeb>
+      <PublishToMSDL Condition="'$(PublishToMSDL)' == ''">true</PublishToMSDL>
       <PublishToSymbolServer Condition="'@(FilesToPublishToSymbolServer)' == '' and '@(PackagesToPublishToSymbolServer)' == ''">false</PublishToSymbolServer>
     </PropertyGroup>
 
@@ -56,7 +58,7 @@
                     DryRun="false"
                     ConvertPortablePdbsToWindowsPdbs="false"
                     PdbConversionTreatAsWarning=""
-                    Condition="$(PublishToSymbolServer)"/>
+                    Condition="$(PublishToSymbolServer) and $(PublishToMSDL)"/>
 
     <!-- 
       Symbol Uploader: SymWeb 
@@ -73,7 +75,7 @@
                     DryRun="false"
                     ConvertPortablePdbsToWindowsPdbs="false"
                     PdbConversionTreatAsWarning=""
-                    Condition="$(PublishToSymbolServer)"/>
+                    Condition="$(PublishToSymbolServer) and $(PublishToSymWeb)"/>
   </Target>
 
   <ItemGroup>

external/linker/eng/common/SetupNugetSources.ps1

@@ -0,0 +1,134 @@
+# This file is a temporary workaround for internal builds to be able to restore from private AzDO feeds.
+# This file should be removed as part of this issue: https://github.com/dotnet/arcade/issues/4080
+#
+# What the script does is iterate over all package sources in the pointed NuGet.config and add a credential entry
+# under <packageSourceCredentials> for each Maestro managed private feed. Two additional credential 
+# entries are also added for the two private static internal feeds: dotnet3-internal and dotnet3-internal-transport.
+#
+# This script needs to be called in every job that will restore packages and which the base repo has
+# private AzDO feeds in the NuGet.config.
+#
+# See example YAML call for this script below. Note the use of the variable `$(dn-bot-dnceng-artifact-feeds-rw)`
+# from the AzureDevOps-Artifact-Feeds-Pats variable group.
+#
+#  - task: PowerShell@2
+#    displayName: Setup Private Feeds Credentials
+#    condition: eq(variables['Agent.OS'], 'Windows_NT')
+#    inputs:
+#      filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
+#      arguments: -ConfigFile ${Env:BUILD_SOURCESDIRECTORY}/NuGet.config -Password $Env:Token
+#    env:
+#      Token: $(dn-bot-dnceng-artifact-feeds-rw)
+
+[CmdletBinding()]
+param (
+    [Parameter(Mandatory = $true)][string]$ConfigFile,
+    [Parameter(Mandatory = $true)][string]$Password
+)
+
+$ErrorActionPreference = "Stop"
+Set-StrictMode -Version 2.0
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+
+. $PSScriptRoot\tools.ps1
+
+# Add source entry to PackageSources
+function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $Password) {
+    $packageSource = $sources.SelectSingleNode("add[@key='$SourceName']")
+    
+    if ($packageSource -eq $null)
+    {
+        $packageSource = $doc.CreateElement("add")
+        $packageSource.SetAttribute("key", $SourceName)
+        $packageSource.SetAttribute("value", $SourceEndPoint)
+        $sources.AppendChild($packageSource) | Out-Null
+    }
+    else {
+        Write-Host "Package source $SourceName already present."
+    }
+    
+    AddCredential -Creds $creds -Source $SourceName -Username $Username -Password $Password
+}
+
+# Add a credential node for the specified source
+function AddCredential($creds, $source, $username, $password) {
+    # Looks for credential configuration for the given SourceName. Create it if none is found.
+    $sourceElement = $creds.SelectSingleNode($Source)
+    if ($sourceElement -eq $null)
+    {
+        $sourceElement = $doc.CreateElement($Source)
+        $creds.AppendChild($sourceElement) | Out-Null
+    }
+
+    # Add the <Username> node to the credential if none is found.
+    $usernameElement = $sourceElement.SelectSingleNode("add[@key='Username']")
+    if ($usernameElement -eq $null)
+    {
+        $usernameElement = $doc.CreateElement("add")
+        $usernameElement.SetAttribute("key", "Username")
+        $sourceElement.AppendChild($usernameElement) | Out-Null
+    }
+    $usernameElement.SetAttribute("value", $Username)
+
+    # Add the <ClearTextPassword> to the credential if none is found.
+    # Add it as a clear text because there is no support for encrypted ones in non-windows .Net SDKs.
+    #   -> https://github.com/NuGet/Home/issues/5526
+    $passwordElement = $sourceElement.SelectSingleNode("add[@key='ClearTextPassword']")
+    if ($passwordElement -eq $null)
+    {
+        $passwordElement = $doc.CreateElement("add")
+        $passwordElement.SetAttribute("key", "ClearTextPassword")
+        $sourceElement.AppendChild($passwordElement) | Out-Null
+    }
+    $passwordElement.SetAttribute("value", $Password)
+}
+
+function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Password) {
+    $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]")
+
+    Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds."
+    
+    ForEach ($PackageSource in $maestroPrivateSources) {
+        Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key
+        AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -Password $Password
+    }
+}
+
+try {
+    if (!(Test-Path $ConfigFile -PathType Leaf)) {
+    Write-PipelineTelemetryError -Category 'Build' -Message "Couldn't find the file NuGet config file: $ConfigFile"
+    ExitWithExitCode 1
+    }
+
+    # Load NuGet.config
+    $doc = New-Object System.Xml.XmlDocument
+    $filename = (Get-Item $ConfigFile).FullName
+    $doc.Load($filename)
+
+    # Get reference to <PackageSources> or create one if none exist already
+    $sources = $doc.DocumentElement.SelectSingleNode("packageSources")
+    if ($sources -eq $null) {
+        $sources = $doc.CreateElement("packageSources")
+        $doc.DocumentElement.AppendChild($sources) | Out-Null
+    }
+
+    # Looks for a <PackageSourceCredentials> node. Create it if none is found.
+    $creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials")
+    if ($creds -eq $null) {
+        $creds = $doc.CreateElement("packageSourceCredentials")
+        $doc.DocumentElement.AppendChild($creds) | Out-Null
+    }
+
+    # Insert credential nodes for Maestro's private feeds
+    InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Password $Password
+
+    AddPackageSource -Sources $sources -SourceName "dotnet3-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password
+    AddPackageSource -Sources $sources -SourceName "dotnet3-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal-transport/nuget/v2" -Creds $creds -Username "dn-bot" -Password $Password
+
+    $doc.Save($filename)
+}
+catch {
+    Write-Host $_.ScriptStackTrace
+    Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_
+    ExitWithExitCode 1
+}

external/linker/eng/common/SetupNugetSources.sh

@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+
+# This file is a temporary workaround for internal builds to be able to restore from private AzDO feeds.
+# This file should be removed as part of this issue: https://github.com/dotnet/arcade/issues/4080
+#
+# What the script does is iterate over all package sources in the pointed NuGet.config and add a credential entry
+# under <packageSourceCredentials> for each Maestro's managed private feed. Two additional credential 
+# entries are also added for the two private static internal feeds: dotnet3-internal and dotnet3-internal-transport.
+#
+# This script needs to be called in every job that will restore packages and which the base repo has
+# private AzDO feeds in the NuGet.config.
+#
+# See example YAML call for this script below. Note the use of the variable `$(dn-bot-dnceng-artifact-feeds-rw)`
+# from the AzureDevOps-Artifact-Feeds-Pats variable group.
+#
+#  - task: Bash@3
+#    displayName: Setup Private Feeds Credentials
+#    inputs:
+#      filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
+#      arguments: $BUILD_SOURCESDIRECTORY/NuGet.config $Token
+#    condition: ne(variables['Agent.OS'], 'Windows_NT')
+#    env:
+#      Token: $(dn-bot-dnceng-artifact-feeds-rw)
+
+ConfigFile=$1
+CredToken=$2
+NL='\n'
+TB='    '
+
+source="${BASH_SOURCE[0]}"
+
+# resolve $source until the file is no longer a symlink
+while [[ -h "$source" ]]; do
+  scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+  source="$(readlink "$source")"
+  # if $source was a relative symlink, we need to resolve it relative to the path where the
+  # symlink file was located
+  [[ $source != /* ]] && source="$scriptroot/$source"
+done
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+. "$scriptroot/tools.sh"
+
+if [ ! -f "$ConfigFile" ]; then
+    Write-PipelineTelemetryError -Category 'Build' -Message "Couldn't find the file NuGet config file: $ConfigFile"
+    ExitWithExitCode 1
+fi
+
+if [[ `uname -s` == "Darwin" ]]; then
+    NL=$'\\\n'
+    TB=''
+fi
+
+# Ensure there is a <packageSources>...</packageSources> section.
+grep -i "<packageSources>" $ConfigFile 
+if [ "$?" != "0" ]; then
+    echo "Adding <packageSources>...</packageSources> section."
+    ConfigNodeHeader="<configuration>"
+    PackageSourcesTemplate="${TB}<packageSources>${NL}${TB}</packageSources>"
+
+    sed -i.bak "s|$ConfigNodeHeader|$ConfigNodeHeader${NL}$PackageSourcesTemplate|" NuGet.config
+fi
+
+# Ensure there is a <packageSourceCredentials>...</packageSourceCredentials> section. 
+grep -i "<packageSourceCredentials>" $ConfigFile 
+if [ "$?" != "0" ]; then
+    echo "Adding <packageSourceCredentials>...</packageSourceCredentials> section."
+
+    PackageSourcesNodeFooter="</packageSources>"
+    PackageSourceCredentialsTemplate="${TB}<packageSourceCredentials>${NL}${TB}</packageSourceCredentials>"
+
+    sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourcesNodeFooter${NL}$PackageSourceCredentialsTemplate|" NuGet.config
+fi
+
+# Ensure dotnet3-internal and dotnet3-internal-transport is in the packageSources
+grep -i "<add key=\"dotnet3-internal\">" $ConfigFile 
+if [ "$?" != "0" ]; then
+    echo "Adding dotnet3-internal to the packageSources."
+
+    PackageSourcesNodeFooter="</packageSources>"
+    PackageSourceTemplate="${TB}<add key=\"dotnet3-internal\" value=\"https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal/nuget/v2\" />"
+
+    sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" NuGet.config
+fi
+
+# Ensure dotnet3-internal and dotnet3-internal-transport is in the packageSources
+grep -i "<add key=\"dotnet3-internal-transport\">" $ConfigFile 
+if [ "$?" != "0" ]; then
+    echo "Adding dotnet3-internal-transport to the packageSources."
+
+    PackageSourcesNodeFooter="</packageSources>"
+    PackageSourceTemplate="${TB}<add key=\"dotnet3-internal-transport\" value=\"https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3-internal-transport/nuget/v2\" />"
+
+    sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" NuGet.config
+fi
+
+# I want things split line by line
+PrevIFS=$IFS
+IFS=$'\n'
+PackageSources=$(grep -oh '"darc-int-[^"]*"' $ConfigFile | tr -d '"')
+IFS=$PrevIFS
+
+PackageSources+=('dotnet3-internal')
+PackageSources+=('dotnet3-internal-transport')
+
+for FeedName in ${PackageSources[@]} ; do
+    # Check if there is no existing credential for this FeedName
+    grep -i "<$FeedName>" $ConfigFile 
+    if [ "$?" != "0" ]; then
+        echo "Adding credentials for $FeedName."
+
+        PackageSourceCredentialsNodeFooter="</packageSourceCredentials>"
+        NewCredential="${TB}${TB}<$FeedName>${NL}<add key=\"Username\" value=\"dn-bot\" />${NL}<add key=\"ClearTextPassword\" value=\"$CredToken\" />${NL}</$FeedName>"
+
+        sed -i.bak "s|$PackageSourceCredentialsNodeFooter|$NewCredential${NL}$PackageSourceCredentialsNodeFooter|" NuGet.config
+    fi
+done

external/linker/eng/common/SourceLinkValidation.ps1

@@ -6,140 +6,15 @@ param(
   [Parameter(Mandatory=$true)][string] $GHCommit             # GitHub commit SHA used to build the packages
 )
 
-# Cache/HashMap (File -> Exist flag) used to consult whether a file exist 
-# in the repository at a specific commit point. This is populated by inserting
-# all files present in the repo at a specific commit point.
-$global:RepoFiles = @{}
-
-$ValidatePackage = {
-  param( 
-    [string] $PackagePath                                 # Full path to a Symbols.NuGet package
-  )
-
-  # Ensure input file exist
-  if (!(Test-Path $PackagePath)) {
-    throw "Input file does not exist: $PackagePath"
-  }
-
-  # Extensions for which we'll look for SourceLink information
-  # For now we'll only care about Portable & Embedded PDBs
-  $RelevantExtensions = @(".dll", ".exe", ".pdb")
- 
-  Write-Host -NoNewLine "Validating" ([System.IO.Path]::GetFileName($PackagePath)) "... "
-
-  $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath)
-  $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId
-  $FailedFiles = 0
-
-  Add-Type -AssemblyName System.IO.Compression.FileSystem
-
-  [System.IO.Directory]::CreateDirectory($ExtractPath);
-
-  $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath)
-
-  $zip.Entries | 
-    Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} |
-      ForEach-Object {
-        $FileName = $_.FullName
-        $Extension = [System.IO.Path]::GetExtension($_.Name)
-        $FakeName = -Join((New-Guid), $Extension)
-        $TargetFile = Join-Path -Path $ExtractPath -ChildPath $FakeName 
-
-        # We ignore resource DLLs
-        if ($FileName.EndsWith(".resources.dll")) {
-          return
-        }
-
-        [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true)
-
-        $ValidateFile = {
-          param( 
-            [string] $FullPath,                                # Full path to the module that has to be checked
-            [string] $RealPath,
-            [ref] $FailedFiles
-          )
-
-          # Makes easier to reference `sourcelink cli`
-          Push-Location $using:SourceLinkToolPath
-
-          $SourceLinkInfos = .\sourcelink.exe print-urls $FullPath | Out-String
-
-          if ($LASTEXITCODE -eq 0 -and -not ([string]::IsNullOrEmpty($SourceLinkInfos))) {
-            $NumFailedLinks = 0
-
-            # We only care about Http addresses
-            $Matches = (Select-String '(http[s]?)(:\/\/)([^\s,]+)' -Input $SourceLinkInfos -AllMatches).Matches
-
-            if ($Matches.Count -ne 0) {
-              $Matches.Value |
-                ForEach-Object {
-                  $Link = $_
-                  $CommitUrl = -Join("https://raw.githubusercontent.com/", $using:GHRepoName, "/", $using:GHCommit, "/")
-                  $FilePath = $Link.Replace($CommitUrl, "")
-                  $Status = 200
-                  $Cache = $using:RepoFiles
-
-                  if ( !($Cache.ContainsKey($FilePath)) ) {
-                    try {
-                      $Uri = $Link -as [System.URI]
-                    
-                      # Only GitHub links are valid
-                      if ($Uri.AbsoluteURI -ne $null -and $Uri.Host -match "github") {
-                        $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode
-                      }
-                      else {
-                        $Status = 0
-                      }
-                    }
-                    catch {
-                      $Status = 0
-                    }
-                  }
-
-                  if ($Status -ne 200) {
-                    if ($NumFailedLinks -eq 0) {
-                      if ($FailedFiles.Value -eq 0) {
-                        Write-Host
-                      }
-
-                      Write-Host "`tFile $RealPath has broken links:"
-                    }
-
-                    Write-Host "`t`tFailed to retrieve $Link"
-
-                    $NumFailedLinks++
-                  }
-                }
-            }
-
-            if ($NumFailedLinks -ne 0) {
-              $FailedFiles.value++
-              $global:LASTEXITCODE = 1
-            }
-          }
-
-          Pop-Location
-        }
-      
-        &$ValidateFile $TargetFile $FileName ([ref]$FailedFiles)
-      }
-
-  $zip.Dispose()
-
-  if ($FailedFiles -eq 0) {
-    Write-Host "Passed."
-  }
-}
-
 function ValidateSourceLinkLinks {
   if (!($GHRepoName -Match "^[^\s\/]+/[^\s\/]+$")) {
-    Write-Host "GHRepoName should be in the format <org>/<repo>"
+    Write-PipelineTelemetryError -Category "Build" -Message "GHRepoName should be in the format <org>/<repo>"
     $global:LASTEXITCODE = 1
     return
   }
 
   if (!($GHCommit -Match "^[0-9a-fA-F]{40}$")) {
-    Write-Host "GHCommit should be a 40 chars hexadecimal string"
+    Write-PipelineTelemetryError -Category "Build" -Message "GHCommit should be a 40 chars hexadecimal string"
     $global:LASTEXITCODE = 1
     return
   }
@@ -160,7 +35,7 @@ function ValidateSourceLinkLinks {
     }
   }
   catch {
-    Write-Host "Problems downloading the list of files from the repo. Url used: $RepoTreeURL"
+    Write-PipelineTelemetryError -Category "Build" -Message "Problems downloading the list of files from the repo. Url used: $RepoTreeURL"
     $global:LASTEXITCODE = 1
     return
   }
@@ -181,4 +56,140 @@ function ValidateSourceLinkLinks {
   }
 }
 
-Measure-Command { ValidateSourceLinkLinks }
+try {
+  . $PSScriptRoot\pipeline-logging-functions.ps1
+
+  # Cache/HashMap (File -> Exist flag) used to consult whether a file exist 
+  # in the repository at a specific commit point. This is populated by inserting
+  # all files present in the repo at a specific commit point.
+  $global:RepoFiles = @{}
+
+  $ValidatePackage = {
+    param( 
+      [string] $PackagePath                                 # Full path to a Symbols.NuGet package
+    )
+
+    # Ensure input file exist
+    if (!(Test-Path $PackagePath)) {
+      throw "Input file does not exist: $PackagePath"
+    }
+
+    # Extensions for which we'll look for SourceLink information
+    # For now we'll only care about Portable & Embedded PDBs
+    $RelevantExtensions = @(".dll", ".exe", ".pdb")
+  
+    Write-Host -NoNewLine "Validating" ([System.IO.Path]::GetFileName($PackagePath)) "... "
+
+    $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath)
+    $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId
+    $FailedFiles = 0
+
+    Add-Type -AssemblyName System.IO.Compression.FileSystem
+
+    [System.IO.Directory]::CreateDirectory($ExtractPath);
+
+    $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath)
+
+    $zip.Entries | 
+      Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} |
+        ForEach-Object {
+          $FileName = $_.FullName
+          $Extension = [System.IO.Path]::GetExtension($_.Name)
+          $FakeName = -Join((New-Guid), $Extension)
+          $TargetFile = Join-Path -Path $ExtractPath -ChildPath $FakeName 
+
+          # We ignore resource DLLs
+          if ($FileName.EndsWith(".resources.dll")) {
+            return
+          }
+
+          [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true)
+
+          $ValidateFile = {
+            param( 
+              [string] $FullPath,                                # Full path to the module that has to be checked
+              [string] $RealPath,
+              [ref] $FailedFiles
+            )
+
+            # Makes easier to reference `sourcelink cli`
+            Push-Location $using:SourceLinkToolPath
+
+            $SourceLinkInfos = .\sourcelink.exe print-urls $FullPath | Out-String
+
+            if ($LASTEXITCODE -eq 0 -and -not ([string]::IsNullOrEmpty($SourceLinkInfos))) {
+              $NumFailedLinks = 0
+
+              # We only care about Http addresses
+              $Matches = (Select-String '(http[s]?)(:\/\/)([^\s,]+)' -Input $SourceLinkInfos -AllMatches).Matches
+
+              if ($Matches.Count -ne 0) {
+                $Matches.Value |
+                  ForEach-Object {
+                    $Link = $_
+                    $CommitUrl = -Join("https://raw.githubusercontent.com/", $using:GHRepoName, "/", $using:GHCommit, "/")
+                    $FilePath = $Link.Replace($CommitUrl, "")
+                    $Status = 200
+                    $Cache = $using:RepoFiles
+
+                    if ( !($Cache.ContainsKey($FilePath)) ) {
+                      try {
+                        $Uri = $Link -as [System.URI]
+                      
+                        # Only GitHub links are valid
+                        if ($Uri.AbsoluteURI -ne $null -and $Uri.Host -match "github") {
+                          $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode
+                        }
+                        else {
+                          $Status = 0
+                        }
+                      }
+                      catch {
+                        $Status = 0
+                      }
+                    }
+
+                    if ($Status -ne 200) {
+                      if ($NumFailedLinks -eq 0) {
+                        if ($FailedFiles.Value -eq 0) {
+                          Write-Host
+                        }
+
+                        Write-Host "`tFile $RealPath has broken links:"
+                      }
+
+                      Write-Host "`t`tFailed to retrieve $Link"
+
+                      $NumFailedLinks++
+                    }
+                  }
+              }
+
+              if ($NumFailedLinks -ne 0) {
+                $FailedFiles.value++
+                $global:LASTEXITCODE = 1
+              }
+            }
+
+            Pop-Location
+          }
+        
+          &$ValidateFile $TargetFile $FileName ([ref]$FailedFiles)
+        }
+
+    $zip.Dispose()
+
+    if ($FailedFiles -eq 0) {
+      Write-Host "Passed."
+    }
+  }
+
+  Measure-Command { ValidateSourceLinkLinks }
+}
+catch {
+  Write-Host $_.ScriptStackTrace
+  Write-PipelineTelemetryError -Category 'SourceLink' -Message $_
+  ExitWithExitCode 1
+}
+
+

external/linker/eng/common/build.ps1

@@ -18,6 +18,7 @@ Param(
   [switch] $sign,
   [switch] $pack,
   [switch] $publish,
+  [switch] $clean,
   [switch][Alias('bl')]$binaryLog,
   [switch] $ci,
   [switch] $prepareMachine,
@@ -25,41 +26,40 @@ Param(
   [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties
 )
 
-. $PSScriptRoot\tools.ps1
-
 function Print-Usage() {
-    Write-Host "Common settings:"
-    Write-Host "  -configuration <value>  Build configuration: 'Debug' or 'Release' (short: -c)"
-    Write-Host "  -platform <value>       Platform configuration: 'x86', 'x64' or any valid Platform value to pass to msbuild"
-    Write-Host "  -verbosity <value>      Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)"
-    Write-Host "  -binaryLog              Output binary log (short: -bl)"
-    Write-Host "  -help                   Print help and exit"
-    Write-Host ""
+  Write-Host "Common settings:"
+  Write-Host "  -configuration <value>  Build configuration: 'Debug' or 'Release' (short: -c)"
+  Write-Host "  -platform <value>       Platform configuration: 'x86', 'x64' or any valid Platform value to pass to msbuild"
+  Write-Host "  -verbosity <value>      Msbuild verbosity: q[uiet], m[inimal], n[ormal], d[etailed], and diag[nostic] (short: -v)"
+  Write-Host "  -binaryLog              Output binary log (short: -bl)"
+  Write-Host "  -help                   Print help and exit"
+  Write-Host ""
 
-    Write-Host "Actions:"
-    Write-Host "  -restore                Restore dependencies (short: -r)"
-    Write-Host "  -build                  Build solution (short: -b)"
-    Write-Host "  -rebuild                Rebuild solution"
-    Write-Host "  -deploy                 Deploy built VSIXes"
-    Write-Host "  -deployDeps             Deploy dependencies (e.g. VSIXes for integration tests)"
-    Write-Host "  -test                   Run all unit tests in the solution (short: -t)"
-    Write-Host "  -integrationTest        Run all integration tests in the solution"
-    Write-Host "  -performanceTest        Run all performance tests in the solution"
-    Write-Host "  -pack                   Package build outputs into NuGet packages and Willow components"
-    Write-Host "  -sign                   Sign build outputs"
-    Write-Host "  -publish                Publish artifacts (e.g. symbols)"
-    Write-Host ""
+  Write-Host "Actions:"
+  Write-Host "  -restore                Restore dependencies (short: -r)"
+  Write-Host "  -build                  Build solution (short: -b)"
+  Write-Host "  -rebuild                Rebuild solution"
+  Write-Host "  -deploy                 Deploy built VSIXes"
+  Write-Host "  -deployDeps             Deploy dependencies (e.g. VSIXes for integration tests)"
+  Write-Host "  -test                   Run all unit tests in the solution (short: -t)"
+  Write-Host "  -integrationTest        Run all integration tests in the solution"
+  Write-Host "  -performanceTest        Run all performance tests in the solution"
+  Write-Host "  -pack                   Package build outputs into NuGet packages and Willow components"
+  Write-Host "  -sign                   Sign build outputs"
+  Write-Host "  -publish                Publish artifacts (e.g. symbols)"
+  Write-Host "  -clean                  Clean the solution"
+  Write-Host ""
 
-    Write-Host "Advanced settings:"
-    Write-Host "  -projects <value>       Semi-colon delimited list of sln/proj's to build. Globbing is supported (*.sln)"
-    Write-Host "  -ci                     Set when running on CI server"
-    Write-Host "  -prepareMachine         Prepare machine for CI run, clean up processes after build"
-    Write-Host "  -warnAsError <value>    Sets warnaserror msbuild parameter ('true' or 'false')"
-    Write-Host "  -msbuildEngine <value>  Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)."
-    Write-Host ""
+  Write-Host "Advanced settings:"
+  Write-Host "  -projects <value>       Semi-colon delimited list of sln/proj's to build. Globbing is supported (*.sln)"
+  Write-Host "  -ci                     Set when running on CI server"
+  Write-Host "  -prepareMachine         Prepare machine for CI run, clean up processes after build"
+  Write-Host "  -warnAsError <value>    Sets warnaserror msbuild parameter ('true' or 'false')"
+  Write-Host "  -msbuildEngine <value>  Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)."
+  Write-Host ""
 
-    Write-Host "Command line arguments not listed above are passed thru to msbuild."
-    Write-Host "The above arguments can be shortened as much as to be unambiguous (e.g. -co for configuration, -t for test, etc.)."
+  Write-Host "Command line arguments not listed above are passed thru to msbuild."
+  Write-Host "The above arguments can be shortened as much as to be unambiguous (e.g. -co for configuration, -t for test, etc.)."
 }
 
 function InitializeCustomToolset {
@@ -67,7 +67,7 @@ function InitializeCustomToolset {
     return
   }
 
-  $script = Join-Path $EngRoot "restore-toolset.ps1"
+  $script = Join-Path $EngRoot 'restore-toolset.ps1'
 
   if (Test-Path $script) {
     . $script
@@ -78,8 +78,8 @@ function Build {
   $toolsetBuildProj = InitializeToolset
   InitializeCustomToolset
 
-  $bl = if ($binaryLog) { "/bl:" + (Join-Path $LogDir "Build.binlog") } else { "" }
-  $platformArg = if ($platform) { "/p:Platform=$platform" } else { "" }
+  $bl = if ($binaryLog) { '/bl:' + (Join-Path $LogDir 'Build.binlog') } else { '' }
+  $platformArg = if ($platform) { "/p:Platform=$platform" } else { '' }
 
   if ($projects) {
     # Re-assign properties to a new variable because PowerShell doesn't let us append properties directly for unclear reasons.
@@ -113,7 +113,22 @@ function Build {
 }
 
 try {
-  if ($help -or (($null -ne $properties) -and ($properties.Contains("/help") -or $properties.Contains("/?")))) {
+  . $PSScriptRoot\tools.ps1
+  
+  if ($clean) {
+    if (Test-Path $ArtifactsDir) {
+      Remove-Item -Recurse -Force $ArtifactsDir
+      Write-Host 'Artifacts directory deleted.'
+    }
+    exit 0
+  }
+  
+  if ((Test-Path variable:LastExitCode) -And ($LastExitCode -ne 0)) {
+    Write-PipelineTelemetryError -Category 'InitializeToolset' -Message 'Eng/common/tools.ps1 returned a non-zero exit code.'
+    ExitWithExitCode $LastExitCode
+  }
+  
+  if ($help -or (($null -ne $properties) -and ($properties.Contains('/help') -or $properties.Contains('/?')))) {
     Print-Usage
     exit 0
   }
@@ -123,14 +138,7 @@ try {
     $nodeReuse = $false
   }
 
-  # Import custom tools configuration, if present in the repo.
-  # Note: Import in global scope so that the script set top-level variables without qualification.
-  $configureToolsetScript = Join-Path $EngRoot "configure-toolset.ps1"
-  if (Test-Path $configureToolsetScript) {
-    . $configureToolsetScript
-  }
-
-  if (($restore) -and ($null -eq $env:DisableNativeToolsetInstalls)) {
+  if ($restore) {
     InitializeNativeTools
   }
 
@@ -138,7 +146,7 @@ try {
 }
 catch {
   Write-Host $_.ScriptStackTrace
-  Write-PipelineTelemetryError -Category "InitializeToolset" -Message $_
+  Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_
   ExitWithExitCode 1
 }
 

external/linker/eng/common/build.sh

@@ -26,6 +26,7 @@ usage()
   echo "  --pack                     Package build outputs into NuGet packages and Willow components"
   echo "  --sign                     Sign build outputs"
   echo "  --publish                  Publish artifacts (e.g. symbols)"
+  echo "  --clean                    Clean the solution"
   echo ""
 
   echo "Advanced settings:"
@@ -62,6 +63,7 @@ publish=false
 sign=false
 public=false
 ci=false
+clean=false
 
 warn_as_error=true
 node_reuse=true
@@ -82,6 +84,9 @@ while [[ $# > 0 ]]; do
       usage
       exit 0
       ;;
+    -clean)
+      clean=true
+      ;;
     -configuration|-c)
       configuration=$2
       shift
@@ -196,20 +201,15 @@ function Build {
   ExitWithExitCode 0
 }
 
-# Import custom tools configuration, if present in the repo.
-configure_toolset_script="$eng_root/configure-toolset.sh"
-if [[ -a "$configure_toolset_script" ]]; then
-  . "$configure_toolset_script"
+if [[ "$clean" == true ]]; then
+  if [ -d "$artifacts_dir" ]; then
+    rm -rf $artifacts_dir
+    echo "Artifacts directory deleted."
+  fi
+  exit 0
 fi
 
-# TODO: https://github.com/dotnet/arcade/issues/1468
-# Temporary workaround to avoid breaking change.
-# Remove once repos are updated.
-if [[ -n "${useInstalledDotNetCli:-}" ]]; then
-  use_installed_dotnet_cli="$useInstalledDotNetCli"
-fi
-
-if [[ "$restore" == true && -z ${DisableNativeToolsetInstalls:-} ]]; then
+if [[ "$restore" == true ]]; then
   InitializeNativeTools
 fi
 

external/linker/eng/common/cross/build-rootfs.sh

@@ -193,7 +193,7 @@ fi
 
 if [[ "$__LinuxCodeName" == "alpine" ]]; then
     __ApkToolsVersion=2.9.1
-    __AlpineVersion=3.7
+    __AlpineVersion=3.9
     __ApkToolsDir=$(mktemp -d)
     wget https://github.com/alpinelinux/apk-tools/releases/download/v$__ApkToolsVersion/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -P $__ApkToolsDir
     tar -xf $__ApkToolsDir/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -C $__ApkToolsDir

external/linker/eng/common/cross/toolchain.cmake

@@ -31,6 +31,10 @@ else()
   message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only armel, arm, arm64 and x86 are supported!")
 endif()
 
+if(DEFINED ENV{TOOLCHAIN})
+  set(TOOLCHAIN $ENV{TOOLCHAIN})
+endif()
+
 # Specify include paths
 if(TARGET_ARCH_NAME STREQUAL "armel")
   if(DEFINED TIZEN_TOOLCHAIN)
@@ -39,48 +43,25 @@ if(TARGET_ARCH_NAME STREQUAL "armel")
   endif()
 endif()
 
-# add_compile_param - adds only new options without duplicates.
-# arg0 - list with result options, arg1 - list with new options.
-# arg2 - optional argument, quick summary string for optional using CACHE FORCE mode.
-macro(add_compile_param)
-  if(NOT ${ARGC} MATCHES "^(2|3)$")
-    message(FATAL_ERROR "Wrong using add_compile_param! Two or three parameters must be given! See add_compile_param description.")
-  endif()
-  foreach(OPTION ${ARGV1})
-    if(NOT ${ARGV0} MATCHES "${OPTION}($| )")
-      set(${ARGV0} "${${ARGV0}} ${OPTION}")
-      if(${ARGC} EQUAL "3") # CACHE FORCE mode
-        set(${ARGV0} "${${ARGV0}}" CACHE STRING "${ARGV2}" FORCE)
-      endif()
-    endif()
-  endforeach()
-endmacro()
+set(CMAKE_SYSROOT "${CROSS_ROOTFS}")
+set(CMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/usr")
+set(CMAKE_CXX_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/usr")
+set(CMAKE_ASM_COMPILER_EXTERNAL_TOOLCHAIN "${CROSS_ROOTFS}/usr")
 
 # Specify link flags
-add_compile_param(CROSS_LINK_FLAGS "--sysroot=${CROSS_ROOTFS}")
-add_compile_param(CROSS_LINK_FLAGS "--gcc-toolchain=${CROSS_ROOTFS}/usr")
-add_compile_param(CROSS_LINK_FLAGS "--target=${TOOLCHAIN}")
-add_compile_param(CROSS_LINK_FLAGS "-fuse-ld=gold")
 
 if(TARGET_ARCH_NAME STREQUAL "armel")
   if(DEFINED TIZEN_TOOLCHAIN) # For Tizen only
-    add_compile_param(CROSS_LINK_FLAGS "-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}")
-    add_compile_param(CROSS_LINK_FLAGS "-L${CROSS_ROOTFS}/lib")
-    add_compile_param(CROSS_LINK_FLAGS "-L${CROSS_ROOTFS}/usr/lib")
-    add_compile_param(CROSS_LINK_FLAGS "-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}")
+    add_link_options("-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}")
+    add_link_options("-L${CROSS_ROOTFS}/lib")
+    add_link_options("-L${CROSS_ROOTFS}/usr/lib")
+    add_link_options("-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}")
   endif()
 elseif(TARGET_ARCH_NAME STREQUAL "x86")
-  add_compile_param(CROSS_LINK_FLAGS "-m32")
+  add_link_options(-m32)
 endif()
 
-add_compile_param(CMAKE_EXE_LINKER_FLAGS "${CROSS_LINK_FLAGS}" "TOOLCHAIN_EXE_LINKER_FLAGS")
-add_compile_param(CMAKE_SHARED_LINKER_FLAGS "${CROSS_LINK_FLAGS}" "TOOLCHAIN_EXE_LINKER_FLAGS")
-add_compile_param(CMAKE_MODULE_LINKER_FLAGS "${CROSS_LINK_FLAGS}" "TOOLCHAIN_EXE_LINKER_FLAGS")
-
 # Specify compile options
-add_compile_options("--sysroot=${CROSS_ROOTFS}")
-add_compile_options("--target=${TOOLCHAIN}")
-add_compile_options("--gcc-toolchain=${CROSS_ROOTFS}/usr")
 
 if(TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64)$")
   set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN})
@@ -103,7 +84,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "x86")
   add_compile_options(-Wno-error=unused-command-line-argument)
 endif()
 
-# Set LLDB include and library paths
+# Set LLDB include and library paths for builds that need lldb.
 if(TARGET_ARCH_NAME MATCHES "^(arm|armel|x86)$")
   if(TARGET_ARCH_NAME STREQUAL "x86")
     set(LLVM_CROSS_DIR "$ENV{LLVM_CROSS_HOME}")
@@ -131,7 +112,7 @@ if(TARGET_ARCH_NAME MATCHES "^(arm|armel|x86)$")
   endif()
 endif()
 
-set(CMAKE_FIND_ROOT_PATH "${CROSS_ROOTFS}")
+
 set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
 set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
 set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)

external/linker/eng/common/darc-init.ps1

@@ -1,13 +1,14 @@
 param (
     $darcVersion = $null,
-    $versionEndpoint = "https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16",
-    $verbosity = "m"
+    $versionEndpoint = 'https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16',
+    $verbosity = 'minimal',
+    $toolpath = $null
 )
 
 . $PSScriptRoot\tools.ps1
 
 function InstallDarcCli ($darcVersion) {
-  $darcCliPackageName = "microsoft.dotnet.darc"
+  $darcCliPackageName = 'microsoft.dotnet.darc'
 
   $dotnetRoot = InitializeDotNetCli -install:$true
   $dotnet = "$dotnetRoot\dotnet.exe"
@@ -26,8 +27,19 @@ function InstallDarcCli ($darcVersion) {
   $arcadeServicesSource = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
 
   Write-Host "Installing Darc CLI version $darcVersion..."
-  Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed."
-  & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g
+  Write-Host 'You may need to restart your command window if this is the first dotnet tool you have installed.'
+  if (-not $toolpath) {
+    & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g
+  }else {
+    & "$dotnet" tool install $darcCliPackageName --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath"
+  }
 }
 
-InstallDarcCli $darcVersion
+try {
+  InstallDarcCli $darcVersion
+}
+catch {
+  Write-Host $_.ScriptStackTrace
+  Write-PipelineTelemetryError -Category 'Darc' -Message $_
+  ExitWithExitCode 1
+}

external/linker/eng/common/darc-init.sh

@@ -2,8 +2,8 @@
 
 source="${BASH_SOURCE[0]}"
 darcVersion=''
-versionEndpoint="https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16"
-verbosity=m
+versionEndpoint='https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16'
+verbosity='minimal'
 
 while [[ $# > 0 ]]; do
   opt="$(echo "$1" | awk '{print tolower($0)}')"
@@ -20,6 +20,10 @@ while [[ $# > 0 ]]; do
       verbosity=$2
       shift
       ;;
+    --toolpath)
+      toolpath=$2
+      shift
+      ;;
     *)
       echo "Invalid argument: $1"
       usage
@@ -52,17 +56,27 @@ function InstallDarcCli {
   InitializeDotNetCli
   local dotnet_root=$_InitializeDotNetCli
 
-  local uninstall_command=`$dotnet_root/dotnet tool uninstall $darc_cli_package_name -g`
-  local tool_list=$($dotnet_root/dotnet tool list -g)
-  if [[ $tool_list = *$darc_cli_package_name* ]]; then
-    echo $($dotnet_root/dotnet tool uninstall $darc_cli_package_name -g)
+  if [ -z "$toolpath" ]; then
+    local tool_list=$($dotnet_root/dotnet tool list -g)
+    if [[ $tool_list = *$darc_cli_package_name* ]]; then
+      echo $($dotnet_root/dotnet tool uninstall $darc_cli_package_name -g)
+    fi
+  else
+    local tool_list=$($dotnet_root/dotnet tool list --tool-path "$toolpath")
+    if [[ $tool_list = *$darc_cli_package_name* ]]; then
+      echo $($dotnet_root/dotnet tool uninstall $darc_cli_package_name --tool-path "$toolpath")
+    fi
   fi
 
   local arcadeServicesSource="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json"
 
   echo "Installing Darc CLI version $darcVersion..."
   echo "You may need to restart your command shell if this is the first dotnet tool you have installed."
-  echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g)
+  if [ -z "$toolpath" ]; then
+    echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity -g)
+  else
+    echo $($dotnet_root/dotnet tool install $darc_cli_package_name --version $darcVersion --add-source "$arcadeServicesSource" -v $verbosity --tool-path "$toolpath")
+  fi
 }
 
 InstallDarcCli

external/linker/eng/common/dotnet-install.ps1

@@ -1,26 +1,27 @@
 [CmdletBinding(PositionalBinding=$false)]
 Param(
-  [string] $verbosity = "minimal",
-  [string] $architecture = "",
-  [string] $version = "Latest",
-  [string] $runtime = "dotnet"
+  [string] $verbosity = 'minimal',
+  [string] $architecture = '',
+  [string] $version = 'Latest',
+  [string] $runtime = 'dotnet',
+  [string] $RuntimeSourceFeed = '',
+  [string] $RuntimeSourceFeedKey = ''
 )
 
 . $PSScriptRoot\tools.ps1
 
-$dotnetRoot = Join-Path $RepoRoot ".dotnet"
+$dotnetRoot = Join-Path $RepoRoot '.dotnet'
 
 $installdir = $dotnetRoot
 try {
-    if ($architecture -and $architecture.Trim() -eq "x86") {
-        $installdir = Join-Path $installdir "x86"
+    if ($architecture -and $architecture.Trim() -eq 'x86') {
+        $installdir = Join-Path $installdir 'x86'
     }
-   InstallDotNet $installdir $version $architecture $runtime $true
-} 
+    InstallDotNet $installdir $version $architecture $runtime $true -RuntimeSourceFeed $RuntimeSourceFeed -RuntimeSourceFeedKey $RuntimeSourceFeedKey
+}
 catch {
-  Write-Host $_
-  Write-Host $_.Exception
   Write-Host $_.ScriptStackTrace
+  Write-PipelineTelemetryError -Category 'InitializeToolset' -Message $_
   ExitWithExitCode 1
 }
 

external/linker/eng/common/dotnet-install.sh

@@ -11,9 +11,13 @@ while [[ -h "$source" ]]; do
 done
 scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
 
+. "$scriptroot/tools.sh"
+
 version='Latest'
 architecture=''
 runtime='dotnet'
+runtimeSourceFeed=''
+runtimeSourceFeedKey=''
 while [[ $# > 0 ]]; do
   opt="$(echo "$1" | awk '{print tolower($0)}')"
   case "$opt" in
@@ -29,20 +33,56 @@ while [[ $# > 0 ]]; do
       shift
       runtime="$1"
       ;;
+    -runtimesourcefeed)
+      shift
+      runtimeSourceFeed="$1"
+      ;;
+    -runtimesourcefeedkey)
+      shift
+      runtimeSourceFeedKey="$1"
+      ;;
     *)
-      echo "Invalid argument: $1"
-      usage
+      Write-PipelineTelemetryError -Category 'Build' -Message "Invalid argument: $1"
       exit 1
       ;;
   esac
   shift
 done
 
-. "$scriptroot/tools.sh"
+# Use uname to determine what the CPU is.
+cpuname=$(uname -p)
+# Some Linux platforms report unknown for platform, but the arch for machine.
+if [[ "$cpuname" == "unknown" ]]; then
+  cpuname=$(uname -m)
+fi
+
+case $cpuname in
+  aarch64)
+    buildarch=arm64
+    ;;
+  amd64|x86_64)
+    buildarch=x64
+    ;;
+  armv7l)
+    buildarch=arm
+    ;;
+  i686)
+    buildarch=x86
+    ;;
+  *)
+    echo "Unknown CPU $cpuname detected, treating it as x64"
+    buildarch=x64
+    ;;
+esac
+
 dotnetRoot="$repo_root/.dotnet"
-InstallDotNet $dotnetRoot $version "$architecture" $runtime true || {
+if [[ $architecture != "" ]] && [[ $architecture != $buildarch ]]; then
+  dotnetRoot="$dotnetRoot/$architecture"
+fi
+
+InstallDotNet $dotnetRoot $version "$architecture" $runtime true $runtimeSourceFeed $runtimeSourceFeedKey || {
   local exit_code=$?
-  echo "dotnet-install.sh failed (exit code '$exit_code')." >&2
+  Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "dotnet-install.sh failed (exit code '$exit_code')." >&2
   ExitWithExitCode $exit_code
 }
 

external/linker/eng/common/enable-cross-org-publishing.ps1

@@ -2,5 +2,7 @@ param(
   [string] $token
 )
 
-Write-Host "##vso[task.setvariable variable=VSS_NUGET_ACCESSTOKEN]$token"
-Write-Host "##vso[task.setvariable variable=VSS_NUGET_URI_PREFIXES]https://dnceng.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/dnceng/;https://devdiv.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/devdiv/"
+. $PSScriptRoot\pipeline-logging-functions.ps1
+
+Write-PipelineSetVariable -Name 'VSS_NUGET_ACCESSTOKEN' -Value $token
+Write-PipelineSetVariable -Name 'VSS_NUGET_URI_PREFIXES' -Value 'https://dnceng.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/dnceng/;https://devdiv.pkgs.visualstudio.com/;https://pkgs.dev.azure.com/devdiv/'

external/linker/eng/common/generate-graph-files.ps1

@@ -9,33 +9,33 @@ Param(
                                                         # toolset dependencies see https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#toolset-vs-product-dependencies
 )
 
-$ErrorActionPreference = "Stop"
-. $PSScriptRoot\tools.ps1
-
-Import-Module -Name (Join-Path $PSScriptRoot "native\CommonLibrary.psm1")
-
 function CheckExitCode ([string]$stage)
 {
   $exitCode = $LASTEXITCODE
   if ($exitCode  -ne 0) {
-    Write-Host "Something failed in stage: '$stage'. Check for errors above. Exiting now..."
+    Write-PipelineTelemetryError -Category 'Arcade' -Message "Something failed in stage: '$stage'. Check for errors above. Exiting now..."
     ExitWithExitCode $exitCode
   }
 }
 
 try {
+  $ErrorActionPreference = 'Stop'
+  . $PSScriptRoot\tools.ps1
+  
+  Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1')
+
   Push-Location $PSScriptRoot
 
-  Write-Host "Installing darc..."
+  Write-Host 'Installing darc...'
   . .\darc-init.ps1 -darcVersion $darcVersion
-  CheckExitCode "Running darc-init"
+  CheckExitCode 'Running darc-init'
 
-  $engCommonBaseDir = Join-Path $PSScriptRoot "native\"
+  $engCommonBaseDir = Join-Path $PSScriptRoot 'native\'
   $graphvizInstallDir = CommonLibrary\Get-NativeInstallDirectory
-  $nativeToolBaseUri = "https://netcorenativeassets.blob.core.windows.net/resource-packages/external"
-  $installBin = Join-Path $graphvizInstallDir "bin"
+  $nativeToolBaseUri = 'https://netcorenativeassets.blob.core.windows.net/resource-packages/external'
+  $installBin = Join-Path $graphvizInstallDir 'bin'
 
-  Write-Host "Installing dot..."
+  Write-Host 'Installing dot...'
   .\native\install-tool.ps1 -ToolName graphviz -InstallPath $installBin -BaseUri $nativeToolBaseUri -CommonLibraryDirectory $engCommonBaseDir -Version $graphvizVersion -Verbose
 
   $darcExe = "$env:USERPROFILE\.dotnet\tools"
@@ -51,37 +51,36 @@ try {
   $graphVizImageFilePath = "$outputFolder\graph.png"
   $normalGraphFilePath = "$outputFolder\graph-full.txt"
   $flatGraphFilePath = "$outputFolder\graph-flat.txt"
-  $baseOptions = @( "--github-pat", "$gitHubPat", "--azdev-pat", "$azdoPat", "--password", "$barToken" )
+  $baseOptions = @( '--github-pat', "$gitHubPat", '--azdev-pat', "$azdoPat", '--password', "$barToken" )
 
   if ($includeToolset) {
-    Write-Host "Toolsets will be included in the graph..."
-    $baseOptions += @( "--include-toolset" )
+    Write-Host 'Toolsets will be included in the graph...'
+    $baseOptions += @( '--include-toolset' )
   }
 
-  Write-Host "Generating standard dependency graph..."
+  Write-Host 'Generating standard dependency graph...'
   & "$darcExe" get-dependency-graph @baseOptions --output-file $normalGraphFilePath
-  CheckExitCode "Generating normal dependency graph"
+  CheckExitCode 'Generating normal dependency graph'
 
-  Write-Host "Generating flat dependency graph and graphviz file..."
+  Write-Host 'Generating flat dependency graph and graphviz file...'
   & "$darcExe" get-dependency-graph @baseOptions --flat --coherency --graphviz $graphVizFilePath --output-file $flatGraphFilePath
-  CheckExitCode "Generating flat and graphviz dependency graph"
+  CheckExitCode 'Generating flat and graphviz dependency graph'
 
   Write-Host "Generating graph image $graphVizFilePath"
   $dotFilePath = Join-Path $installBin "graphviz\$graphvizVersion\release\bin\dot.exe"
   & "$dotFilePath" -Tpng -o"$graphVizImageFilePath" "$graphVizFilePath"
-  CheckExitCode "Generating graphviz image"
+  CheckExitCode 'Generating graphviz image'
 
   Write-Host "'$graphVizFilePath', '$flatGraphFilePath', '$normalGraphFilePath' and '$graphVizImageFilePath' created!"
 }
 catch {
   if (!$includeToolset) {
-    Write-Host "This might be a toolset repo which includes only toolset dependencies. " -NoNewline -ForegroundColor Yellow
-    Write-Host "Since -includeToolset is not set there is no graph to create. Include -includeToolset and try again..." -ForegroundColor Yellow
+    Write-Host 'This might be a toolset repo which includes only toolset dependencies. ' -NoNewline -ForegroundColor Yellow
+    Write-Host 'Since -includeToolset is not set there is no graph to create. Include -includeToolset and try again...' -ForegroundColor Yellow
   }
-  Write-Host $_
-  Write-Host $_.Exception
   Write-Host $_.ScriptStackTrace
+  Write-PipelineTelemetryError -Category 'Arcade' -Message $_
   ExitWithExitCode 1
 } finally {
-    Pop-Location
+  Pop-Location
 }

external/linker/eng/common/init-tools-native.ps1

@@ -35,7 +35,7 @@ File path to global.json file
 #>
 [CmdletBinding(PositionalBinding=$false)]
 Param (
-  [string] $BaseUri = "https://netcorenativeassets.blob.core.windows.net/resource-packages/external",
+  [string] $BaseUri = 'https://netcorenativeassets.blob.core.windows.net/resource-packages/external',
   [string] $InstallDirectory,
   [switch] $Clean = $False,
   [switch] $Force = $False,
@@ -45,26 +45,27 @@ Param (
 )
 
 if (!$GlobalJsonFile) {
-  $GlobalJsonFile = Join-Path (Get-Item $PSScriptRoot).Parent.Parent.FullName "global.json"
+  $GlobalJsonFile = Join-Path (Get-Item $PSScriptRoot).Parent.Parent.FullName 'global.json'
 }
 
 Set-StrictMode -version 2.0
-$ErrorActionPreference="Stop"
+$ErrorActionPreference='Stop'
 
-Import-Module -Name (Join-Path $PSScriptRoot "native\CommonLibrary.psm1")
+. $PSScriptRoot\pipeline-logging-functions.ps1
+Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1')
 
 try {
   # Define verbose switch if undefined
-  $Verbose = $VerbosePreference -Eq "Continue"
+  $Verbose = $VerbosePreference -Eq 'Continue'
 
-  $EngCommonBaseDir = Join-Path $PSScriptRoot "native\"
+  $EngCommonBaseDir = Join-Path $PSScriptRoot 'native\'
   $NativeBaseDir = $InstallDirectory
   if (!$NativeBaseDir) {
     $NativeBaseDir = CommonLibrary\Get-NativeInstallDirectory
   }
   $Env:CommonLibrary_NativeInstallDir = $NativeBaseDir
-  $InstallBin = Join-Path $NativeBaseDir "bin"
-  $InstallerPath = Join-Path $EngCommonBaseDir "install-tool.ps1"
+  $InstallBin = Join-Path $NativeBaseDir 'bin'
+  $InstallerPath = Join-Path $EngCommonBaseDir 'install-tool.ps1'
 
   # Process tools list
   Write-Host "Processing $GlobalJsonFile"
@@ -74,7 +75,7 @@ try {
   }
   $NativeTools = Get-Content($GlobalJsonFile) -Raw |
                     ConvertFrom-Json |
-                    Select-Object -Expand "native-tools" -ErrorAction SilentlyContinue
+                    Select-Object -Expand 'native-tools' -ErrorAction SilentlyContinue
   if ($NativeTools) {
     $NativeTools.PSObject.Properties | ForEach-Object {
       $ToolName = $_.Name
@@ -112,18 +113,19 @@ try {
             }
             $toolInstallationFailure = $true
         } else {
-            Write-Error $errMsg
+            Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message $errMsg
             exit 1
         }
       }
     }
 
     if ((Get-Variable 'toolInstallationFailure' -ErrorAction 'SilentlyContinue') -and $toolInstallationFailure) {
+        Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message 'Native tools bootstrap failed'
         exit 1
     }
   }
   else {
-    Write-Host "No native tools defined in global.json"
+    Write-Host 'No native tools defined in global.json'
     exit 0
   }
 
@@ -131,18 +133,18 @@ try {
     exit 0
   }
   if (Test-Path $InstallBin) {
-    Write-Host "Native tools are available from" (Convert-Path -Path $InstallBin)
+    Write-Host 'Native tools are available from ' (Convert-Path -Path $InstallBin)
     Write-Host "##vso[task.prependpath]$(Convert-Path -Path $InstallBin)"
     return $InstallBin
   }
   else {
-    Write-Error "Native tools install directory does not exist, installation failed"
+    Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message 'Native tools install directory does not exist, installation failed'
     exit 1
   }
   exit 0
 }
 catch {
-  Write-Host $_
-  Write-Host $_.Exception
-  exit 1
+  Write-Host $_.ScriptStackTrace
+  Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message $_
+  ExitWithExitCode 1
 }

external/linker/eng/common/init-tools-native.sh

@@ -12,6 +12,7 @@ retry_wait_time_seconds=30
 global_json_file="$(dirname "$(dirname "${scriptroot}")")/global.json"
 declare -A native_assets
 
+. $scriptroot/pipeline-logging-functions.sh
 . $scriptroot/native/common-library.sh
 
 while (($# > 0)); do
@@ -120,7 +121,7 @@ else
     $installer_command
 
     if [[ $? != 0 ]]; then
-      echo "Execution Failed" >&2
+      Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Execution Failed"
       exit 1
     fi
   done
@@ -134,7 +135,7 @@ if [[ -d $install_bin ]]; then
   echo "Native tools are available from $install_bin"
   echo "##vso[task.prependpath]$install_bin"
 else
-  echo "Native tools install directory does not exist, installation failed" >&2
+  Write-PipelineTelemetryError -category 'NativeToolsBootstrap' "Native tools install directory does not exist, installation failed"
   exit 1
 fi