Imported Upstream version 5.20.0.180

Former-commit-id: ff953ca879339fe1e1211f7220f563e1342e66cb

external/corert/tests/CoreFX/runtest/CoreFXTestHarness/Test.csproj

@@ -0,0 +1,74 @@
+<Project DefaultTargets="LinkNative">
+    <Import Project="Sdk.props" Sdk="Microsoft.NET.Sdk" />
+
+    <Import Project="$(TestRootDir)\dependencies.props" />
+
+    <!-- Override Default MSBuild properties-->
+    <PropertyGroup>
+        <TargetName>$(ExecutableName)</TargetName>
+        <TargetExt>.exe</TargetExt>
+        <OutputType>Exe</OutputType>
+    </PropertyGroup>
+
+    <PropertyGroup>
+        <ToolsDir>$(FrameworkLibPath)\..\tools\</ToolsDir>
+        <TargetFramework>netcoreapp2.1</TargetFramework>
+        <!-- Don't warn if some dependencies were rolled forward -->
+        <NoWarn>$(NoWarn);NU1603</NoWarn>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <RdXmlFile Include="default.rd.xml" />
+        <AppContextSwitchOverrides Include="Switch.System.Enum.RelaxedGetValues" />
+    </ItemGroup>
+    <ItemGroup>
+        <PackageReference Include="xunit.abstractions">
+            <Version>$(XunitAbstractionsVersion)</Version>
+        </PackageReference>
+        <PackageReference Include="xunit.assert">
+            <Version>$(XunitPackageVersion)</Version>
+        </PackageReference>
+        <PackageReference Include="xunit.extensibility.core">
+            <Version>$(XunitPackageVersion)</Version>
+        </PackageReference>
+        <PackageReference Include="xunit.core">
+            <Version>$(XunitPackageVersion)</Version>
+        </PackageReference>
+        <PackageReference Include="xunit.runner.utility">
+            <Version>$(XunitRunnerUtilityVersion)</Version>
+        </PackageReference>
+        <PackageReference Include="xunit.extensibility.execution">
+            <Version>$(XunitPackageVersion)</Version>
+        </PackageReference>
+        <PackageReference Include="microsoft.xunit.netcore.extensions">
+            <Version>$(XunitNetcoreExtensionsVersion)</Version>
+        </PackageReference>
+        <PackageReference Include="CoreFx.Private.TestUtilities">
+            <Version>$(CoreFxTestUtilitiesVersion)</Version>
+        </PackageReference>
+    </ItemGroup>
+    <ItemGroup>
+        <!-- Some tests consist of multiple assemblies - make sure ILC sees them -->
+        <IlcCompileInput Include="$(MSBuildProjectDirectory)\*.dll" />
+    </ItemGroup>
+    <Import Project="Sdk.targets" Sdk="Microsoft.NET.Sdk" />
+
+    <!-- Set OutputPath after the SDK targets have been imported-->
+    <PropertyGroup>
+        <OutputPath>$(MSBuildProjectDirectory)\</OutputPath>
+        <IntermediateOutputPath>$(MSBuildProjectDirectory)\</IntermediateOutputPath>
+    </PropertyGroup>
+    
+    <!-- Switch RuntimeIdentifier according to currently running OSGroup -->
+    <PropertyGroup>
+        <RuntimeIdentifier Condition="'$(OSGroup)' == 'Windows_NT'">win-x64</RuntimeIdentifier>
+        <RuntimeIdentifier Condition="'$(OSGroup)' == 'Linux'">linux-x64</RuntimeIdentifier>
+        <RuntimeIdentifier Condition="'$(OSGroup)' == 'OSX'">osx-x64</RuntimeIdentifier>
+    </PropertyGroup>
+
+    <!-- Import CoreRT build targets -->
+    <Import Project="$(IlcPath)\build\Microsoft.NETCore.Native.targets" />
+    
+    <!-- Since tests are already compiled, override Compile target to prevent CSC running -->
+    <Target Name="Compile" />
+</Project>

external/corert/tests/CoreFX/runtest/CoreFXTestHarness/default.rd.xml

@@ -0,0 +1,41 @@
+<Directives 
+    xmlns="http://schemas.microsoft.com/netfx/2013/01/metadata">
+    <Application>
+        <Assembly Name="*Application*" Dynamic="Required All" />
+        <Assembly Name="xunit.abstractions" Dynamic="Required All" />
+        <Assembly Name="System.Private.CoreLib">
+            <Type Name="System.Runtime.CompilerServices.CompilationRelaxationsAttribute" Dynamic="Required All" />
+            <Type Name="System.Runtime.CompilerServices.RuntimeCompatibilityAttribute" Dynamic="Required All" />
+            <Type Name="System.Runtime.CompilerServices.ExtensionAttribute" Dynamic="Required All" />
+            <Type Name="System.Runtime.CompilerServices.IntrinsicAttribute" Dynamic="Required All" />
+            <Type Name="System.Diagnostics.DebuggableAttribute" Dynamic="Required All" />
+            <Type Name="System.Reflection.AssemblyInformationalVersionAttribute" Dynamic="Required All" />
+            <Type Name="System.Reflection.AssemblyTitleAttribute" Dynamic="Required All" />
+            <Type Name="System.Reflection.AssemblyFileVersionAttribute" Dynamic="Required All" />
+            <Type Name="System.Reflection.AssemblyDescriptionAttribute" Dynamic="Required All" />
+            <Type Name="System.Reflection.AssemblyCompanyAttribute" Dynamic="Required All" />
+            <Type Name="System.Reflection.AssemblyCopyrightAttribute" Dynamic="Required All" />
+            <Type Name="System.Reflection.AssemblyProductAttribute" Dynamic="Required All" />
+            <Type Name="System.Reflection.AssemblyDefaultAliasAttribute" Dynamic="Required All" />
+            <Type Name="System.Reflection.AssemblyMetadataAttribute" Dynamic="Required All" />
+            <Type Name="System.AttributeUsageAttribute" Dynamic="Required All" />
+            <Type Name="System.AttributeUsageAttribute[]" Dynamic="Required All" />
+            <Type Name="System.Runtime.InteropServices.ComVisibleAttribute" Dynamic="Required All" />
+            <Type Name="System.CLSCompliantAttribute" Dynamic="Required All" />
+        </Assembly>
+        <Assembly Name="mscorlib">
+            <Type Name="System.IO.File" Dynamic="Required All"/>
+            <Type Name="System.Threading.ExecutionContext" Dynamic="Required All"/>
+            <Type Name="System.Threading.ContextCallback" Dynamic="Required All"/>
+        </Assembly>
+        <Assembly Name="xunit.core" Dynamic="Required All" />
+        <Assembly Name="xunit.runner.utility.dotnet" Dynamic="Required All" />
+        <Assembly Name="xunit.execution.dotnet" Dynamic="Required All" />
+        <Assembly Name="xunit.abstractions" Dynamic="Required All" />
+        <Assembly Name="xunit.core" Dynamic="Required All">
+            <Type Name="Xunit.Sdk.BeforeAfterTestAttribute[]" Dynamic="Required All" />
+        </Assembly>
+        <Assembly Name="Xunit.NetCore.Extensions" Dynamic="Required All" />
+        <Assembly Name="System.Linq.Expressions" Dynamic="Required All" />
+    </Application>
+</Directives>

external/corert/tests/CoreFX/runtest/dir.props

@@ -0,0 +1,33 @@
+<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  
+  <!-- Don't import the project's parent dir.props - all other projects in the CoreRT repo are set up for the old project format -->
+
+  <!--
+    $(OS) is set to Unix/Windows_NT. This comes from an environment variable on Windows and MSBuild on Unix.
+  -->
+  <PropertyGroup>
+    <OsEnvironment Condition="'$(OsEnvironment)'==''">$(OS)</OsEnvironment>
+  </PropertyGroup>
+  
+  <!-- Common repo directories -->
+  <PropertyGroup>
+    <CopyNuGetImplementations Condition="'$(CopyNuGetImplementations)'==''">false</CopyNuGetImplementations>
+    <ProjectDir>$(MSBuildThisFileDirectory)\</ProjectDir>
+    <SourceDir>$(ProjectDir)src\</SourceDir>
+    <PackagesDir>$(ProjectDir)..\..\..\packages\</PackagesDir>
+    <ToolsDir Condition="'$(ToolsDir)'==''">$(ProjectDir)..\Tools\</ToolsDir>
+    <DotnetCliPath Condition="'$(DotnetCliPath)'==''">$(ToolsDir)dotnetcli/</DotnetCliPath>
+    <SkipImportILTargets>true</SkipImportILTargets>
+  </PropertyGroup>
+
+  <!-- Provides properties for dependency versions and configures dependency verification/auto-upgrade. -->
+  <Import Project="$(MSBuildThisFileDirectory)\..\dependencies.props" />
+
+  <!-- list of nuget package sources passed to dnu --> 
+  <ItemGroup> 
+    <!-- Need to escape double forward slash (%2F) or MSBuild will normalize to one slash on Unix. -->
+    <DotnetSourceList Include="https:%2F%2Fdotnet.myget.org/F/dotnet-core/api/v3/index.json" />
+    <DotnetSourceList Include="https:%2F%2Fapi.nuget.org/v3/index.json" /> 
+  </ItemGroup> 
+  
+</Project>

external/corert/tests/CoreFX/runtest/runtest.cmd

@@ -0,0 +1,107 @@
+@if not defined __echo @echo off
+setlocal EnableDelayedExpansion
+
+:: Set the default arguments
+set __BuildArch=x64
+set __BuildType=Debug
+set __BuildOS=Windows_NT
+
+:: Default to highest Visual Studio version available
+set __VSVersion=vs2017
+set __VSProductVersion=150
+
+:: Define a prefix for most output progress messages that come from this script. That makes
+:: it easier to see where these are coming from. Note that there is a trailing space here.
+set __MsgPrefix=RUNTEST: 
+
+set __ProjectDir=%~dp0
+:: remove trailing slash
+if %__ProjectDir:~-1%==\ set "__ProjectDir=%__ProjectDir:~0,-1%"
+set "__ProjectFilesDir=%__ProjectDir%"
+set "__RootBinDir=%__ProjectDir%\..\bin"
+set "__LogsDir=%__RootBinDir%\Logs"
+
+:Arg_Loop
+if "%1" == "" goto ArgsDone
+
+if /i "%1" == "/?"    goto Usage
+if /i "%1" == "-?"    goto Usage
+if /i "%1" == "/h"    goto Usage
+if /i "%1" == "-h"    goto Usage
+if /i "%1" == "/help" goto Usage
+if /i "%1" == "-help" goto Usage
+
+if /i "%1" == "x64"                   (set __BuildArch=x64&set __MSBuildBuildArch=x64&shift&goto Arg_Loop)
+if /i "%1" == "x86"                   (set __BuildArch=x86&set __MSBuildBuildArch=x86&shift&goto Arg_Loop)
+
+if /i "%1" == "debug"                 (set __BuildType=Debug&shift&goto Arg_Loop)
+if /i "%1" == "release"               (set __BuildType=Release&shift&goto Arg_Loop)
+
+if /i "%1" == "LogsDir"         (set __LogsDir=%2&shift&shift&goto Arg_Loop) 
+
+if /i not "%1" == "msbuildargs" goto SkipMsbuildArgs
+
+set CORE_ROOT=%1
+echo %__MsgPrefix%CORE_ROOT is initially set to: "%CORE_ROOT%"
+shift 
+:ArgsDone
+
+set "__TestWorkingDir=%CoreRT_TestRoot\CoreFX%"
+
+if not defined XunitTestBinBase set  XunitTestBinBase=%__TestWorkingDir%
+
+if not exist %__LogsDir% md %__LogsDir%
+
+:: Check presence of VS
+if not defined VS%__VSProductVersion%COMNTOOLS goto NoVS
+
+set __VSToolsRoot=!VS%__VSProductVersion%COMNTOOLS!
+if %__VSToolsRoot:~-1%==\ set "__VSToolsRoot=%__VSToolsRoot:~0,-1%"
+
+:: Set the environment for the  build- VS cmd prompt
+echo %__MsgPrefix%Using environment: "%__VSToolsRoot%\VsDevCmd.bat"
+call                                 "%__VSToolsRoot%\VsDevCmd.bat"
+
+if not defined VSINSTALLDIR (
+    echo %__MsgPrefix%Error: runtest.cmd should be run from a Visual Studio Command Prompt.  Please see https://github.com/dotnet/coreclr/blob/master/Documentation/project-docs/developer-guide.md for build instructions.
+    exit /b 1
+)
+
+if not defined FXCustomTestLauncher (
+    echo The CoreFX test launcher is not defined 
+    exit /b 1
+)
+
+set SAVED_ERROR_LEVEL=0
+:: Iterate through unzipped CoreFX tests 
+for /D %%i in ("%XunitTestBinBase%\*" ) do (
+    set TestFolderName=%%i
+    set TestFileName=%%~nxi
+
+    echo %FXCustomTestLauncher% !TestFolderName! !TestFileName!
+    call %FXCustomTestLauncher% !TestFolderName! !TestFileName!
+    set TestExitCode=!errorlevel!
+    if !TestExitCode! neq 0 (
+        echo Test !TestFileName! failed with !TestExitCode!
+        set SAVED_ERROR_LEVEL=!TestExitCode!
+    )
+
+)
+
+exit /b !SAVED_ERROR_LEVEL!
+
+:Usage
+echo.
+echo Usage:
+echo   %0 BuildArch BuildType
+echo where:
+echo.
+echo./? -? /h -h /help -help: view this message.
+echo BuildArch- Optional parameter - x64 or x86 ^(default: x64^).
+echo BuildType- Optional parameter - Debug, Release, or Checked ^(default: Debug^).
+exit /b 1
+
+:NoVS
+echo Visual Studio 2017 ^(Community is free^) is a prerequisite to build this repository.
+echo See: https://github.com/dotnet/coreclr/blob/master/Documentation/project-docs/developer-guide.md#prerequisites
+exit /b 1

external/corert/tests/CoreFX/runtest/runtest.sh

@@ -0,0 +1,80 @@
+# Exit code constants
+readonly EXIT_CODE_SUCCESS=0       # Script ran normally.
+readonly EXIT_CODE_EXCEPTION=1     # Script exited because something exceptional happened (e.g. bad arguments, Ctrl-C interrupt).
+readonly EXIT_CODE_TEST_FAILURE=2  # Script completed successfully, but one or more tests failed.
+
+function print_usage {
+    echo ''
+    echo 'CoreFX test runner script.'
+    echo ''
+    echo 'Typical command line:'
+    echo ''
+    echo 'corefx/tests/runtest.sh'
+    echo '    --testRootDir="tests_downloaded/CoreFX"'
+    echo '    --testLauncher="tests/CoreFX/corerun"'    
+    echo ''
+    echo 'Required arguments:'
+    echo '  --testRootDir=<path>             : Root directory of the CoreFX test build '
+    echo '  --testLauncher=<path>            : Path to the test launcher script'
+}
+
+function run_tests_in_directory {
+    local savedErroLevel=$EXIT_CODE_SUCCESS
+    local testDir=$1
+    for testSubDir in ${testDir}/* ; do
+      # Build and run each test
+      echo Building ${testSubDir}
+      ${FXCustomTestLauncher} ${testSubDir} ${__LogDir}
+      if [ $? != 0 ]; 
+      then 
+        savedErroLevel=$EXIT_CODE_TEST_FAILURE
+      fi 
+    done
+    return $savedErroLevel
+    
+}
+
+
+
+# Argument variables
+testRootDir=
+__LogDir=
+FXCustomTestLauncher=
+
+for i in "$@"
+do
+    case $i in
+        --testRootDir=*)
+            testRootDir=${i#*=}
+            ;;
+        --logdir=*)
+            __LogDir=${i#*=}
+            ;;
+        --testLauncher=*)
+            FXCustomTestLauncher=${i#*=}
+            ;;
+        *)
+            echo "Unknown switch: $i"
+            print_usage
+            exit $EXIT_CODE_SUCCESS
+            ;;
+    esac
+done
+
+if [ -z "$testRootDir" ]; then
+    echo "--testRootDir is required."
+    print_usage
+    exit $EXIT_CODE_EXCEPTION
+fi
+if [ ! -d "$testRootDir" ]; then
+    echo "Directory specified by --testRootDir does not exist: $testRootDir"
+    exit $EXIT_CODE_EXCEPTION
+fi
+
+if [ -z "$__LogDir" ]; then
+    __LogDir=$testRootDir
+fi
+
+run_tests_in_directory ${testRootDir}
+
+exit 

external/corert/tests/CoreFX/runtest/src/TestUtils/TestFileSetup/CoreFX.TestUtils.TestFileSetup.csproj

@@ -0,0 +1,18 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <Import Project="..\..\..\dir.props" />
+  <PropertyGroup>
+    <TargetFramework>netcoreapp2.1</TargetFramework>
+    <OutputType>Exe</OutputType>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="Newtonsoft.Json">
+      <Version>$(NewtonsoftJsonVersion)</Version>
+    </PackageReference>
+    <PackageReference Include="Newtonsoft.Json.Schema">
+      <Version>$(NewtonsoftJsonSchemaVersion)</Version>
+    </PackageReference>
+    <PackageReference Include="System.CommandLine">
+      <Version>$(SystemCommandLineVersion)</Version>
+    </PackageReference>
+  </ItemGroup>
+</Project>

external/corert/tests/CoreFX/runtest/src/TestUtils/TestFileSetup/RSPGenerator.cs

@@ -0,0 +1,69 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+
+namespace CoreFX.TestUtils.TestFileSetup
+{
+    public class RSPGenerator
+    {
+        public void GenerateRSPFile(XUnitTestAssembly testDefinition, string outputPath)
+        {
+            if (!Directory.Exists(outputPath))
+            {
+                Directory.CreateDirectory(outputPath);
+            }
+            string rspFilePath = Path.Combine(outputPath, testDefinition.Name + ".rsp");
+
+            if (File.Exists(rspFilePath))
+                File.Delete(rspFilePath);
+
+
+            using (StreamWriter sr = File.CreateText(rspFilePath))
+            {
+                if (testDefinition.Exclusions == null)
+                    return;
+
+                // Method exclusions
+                if (testDefinition.Exclusions.Methods != null)
+                {
+                    foreach (Exclusion exclusion in testDefinition.Exclusions.Methods)
+                    {
+                        if (String.IsNullOrWhiteSpace(exclusion.Name))
+                            continue;
+                        sr.Write("-skipmethod ");
+                        sr.Write(exclusion.Name);
+                        sr.WriteLine();
+                    }
+                }
+
+                // Class exclusions
+                if (testDefinition.Exclusions.Classes != null)
+                {
+                    foreach (Exclusion exclusion in testDefinition.Exclusions.Classes)
+                    {
+                        if (String.IsNullOrWhiteSpace(exclusion.Name))
+                            continue;
+                        sr.Write("-skipclass ");
+                        sr.Write(exclusion.Name);
+                        sr.WriteLine();
+                    }
+
+                }
+
+                // Namespace exclusions
+                if (testDefinition.Exclusions.Namespaces != null)
+                {
+                    foreach (Exclusion exclusion in testDefinition.Exclusions.Namespaces)
+                    {
+                        if (String.IsNullOrWhiteSpace(exclusion.Name))
+                            continue;
+                        sr.Write("-skipnamespace ");
+                        sr.Write(exclusion.Name);
+                        sr.WriteLine();
+                    }
+                }
+            }
+        }
+    }
+}

external/corert/tests/CoreFX/runtest/src/TestUtils/TestFileSetup/TestFileSetup.cs

@@ -0,0 +1,329 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.CommandLine;
+using System.Diagnostics;
+using System.IO;
+using System.IO.Compression;
+using System.Net.Http;
+using System.Text;
+using System.Threading.Tasks;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Schema;
+using Newtonsoft.Json.Schema.Generation;
+
+namespace CoreFX.TestUtils.TestFileSetup
+{
+    /// <summary>
+    /// Defines the set of flags that represent exit codes
+    /// </summary>
+    [Flags]
+    public enum ExitCode : int
+    {
+        Success = 0,
+        HttpError = 1,
+        IOError = 2,
+        JsonSchemaValidationError = 3,
+        UnknownError = 10
+
+    }
+
+    /// <summary>
+    /// This helper class is used to fetch CoreFX tests from a specified URL, unarchive them and create a flat directory structure
+    /// through which to iterate.
+    /// </summary>
+    public static class TestFileSetup
+    {
+        private static HttpClient httpClient;
+        private static bool cleanTestBuild = false;
+
+        private static string outputDir;
+        private static string testUrl;
+        private static string testListPath;
+
+        public static void Main(string[] args)
+        {
+            ExitCode exitCode = ExitCode.UnknownError;
+            ArgumentSyntax argSyntax = ParseCommandLine(args);
+
+            try
+            {
+                if (!Directory.Exists(outputDir))
+                    Directory.CreateDirectory(outputDir);
+
+                if (cleanTestBuild)
+                {
+                    CleanBuild(outputDir);
+                }
+
+                // Map test names to their definitions
+                Dictionary<string, XUnitTestAssembly> testAssemblyDefinitions = DeserializeTestJson(testListPath);
+
+                SetupTests(testUrl, outputDir, testAssemblyDefinitions).Wait();
+                exitCode = ExitCode.Success;
+            }
+
+            catch (AggregateException e)
+            {
+                e.Handle(innerExc =>
+                {
+
+                    if (innerExc is HttpRequestException)
+                    {
+                        exitCode = ExitCode.HttpError;
+                        Console.WriteLine("Error downloading tests from: " + testUrl);
+                        Console.WriteLine(innerExc.Message);
+                        return true;
+                    }
+                    else if (innerExc is IOException)
+                    {
+                        exitCode = ExitCode.IOError;
+                        Console.WriteLine(innerExc.Message);
+                        return true;
+                    }
+                    else if (innerExc is JSchemaValidationException || innerExc is JsonSerializationException)
+                    {
+                        exitCode = ExitCode.JsonSchemaValidationError;
+                        Console.WriteLine("Error validating test list: ");
+                        Console.WriteLine(innerExc.Message);
+                        return true;
+                    }
+                    return false;
+                });
+            }
+
+            Environment.Exit((int)exitCode);
+        }
+
+        private static ArgumentSyntax ParseCommandLine(string[] args)
+        {
+            ArgumentSyntax argSyntax = ArgumentSyntax.Parse(args, syntax =>
+            {
+                syntax.DefineOption("out|outDir|outputDirectory", ref outputDir, "Directory where tests are downloaded");
+                syntax.DefineOption("testUrl", ref testUrl, "URL, pointing to the list of tests");
+                syntax.DefineOption("testListJsonPath", ref testListPath, "JSON-formatted list of test assembly names to download");
+                syntax.DefineOption("clean|cleanOutputDir", ref cleanTestBuild, "Clean test assembly output directory");
+            });
+
+            return argSyntax;
+        }
+
+        private static Dictionary<string, XUnitTestAssembly> DeserializeTestJson(string testDefinitionFilePath)
+        {
+            JSchemaGenerator jsonGenerator = new JSchemaGenerator();
+
+            JSchema testDefinitionSchema = jsonGenerator.Generate(typeof(IList<XUnitTestAssembly>));
+            IList<XUnitTestAssembly> testAssemblies = new List<XUnitTestAssembly>();
+
+            IList<string> validationMessages = new List<string>();
+
+            using (var sr = new StreamReader(testDefinitionFilePath))
+            using (var jsonReader = new JsonTextReader(sr))
+            using (var jsonValidationReader = new JSchemaValidatingReader(jsonReader))
+            {
+                // Create schema validator
+                jsonValidationReader.Schema = testDefinitionSchema;
+                jsonValidationReader.ValidationEventHandler += (o, a) => validationMessages.Add(a.Message);
+
+                // Deserialize json test assembly definitions
+                JsonSerializer serializer = new JsonSerializer();
+                try
+                {
+                    testAssemblies = serializer.Deserialize<List<XUnitTestAssembly>>(jsonValidationReader);
+                }
+                catch (JsonSerializationException ex)
+                {
+                    throw new AggregateException(ex);
+                }
+            }
+
+            // TODO - ABORT AND WARN
+            if (validationMessages.Count != 0)
+            {
+                StringBuilder aggregateExceptionMessage = new StringBuilder();
+                foreach (string validationMessage in validationMessages)
+                {
+                    aggregateExceptionMessage.Append("JSON Validation Error: ");
+                    aggregateExceptionMessage.Append(validationMessage);
+                    aggregateExceptionMessage.AppendLine();
+                }
+
+                throw new AggregateException(new JSchemaValidationException(aggregateExceptionMessage.ToString()));
+
+            }
+
+            var nameToTestAssemblyDef = new Dictionary<string, XUnitTestAssembly>();
+
+            // Map test names to their definitions
+            foreach (XUnitTestAssembly assembly in testAssemblies)
+            {
+                nameToTestAssemblyDef.Add(assembly.Name, assembly);
+            }
+
+            return nameToTestAssemblyDef;
+        }
+
+        private static async Task SetupTests(string jsonUrl, string destinationDirectory, Dictionary<string, XUnitTestAssembly> testDefinitions = null, bool runAllTests = false)
+        {
+            Debug.Assert(Directory.Exists(destinationDirectory));
+            Debug.Assert(runAllTests || testDefinitions != null);
+
+            string tempDirPath = Path.Combine(destinationDirectory, "temp");
+            if (!Directory.Exists(tempDirPath))
+            {
+                Directory.CreateDirectory(tempDirPath);
+            }
+            Dictionary<string, XUnitTestAssembly> testPayloads = await GetTestUrls(jsonUrl, testDefinitions, runAllTests);
+
+            if (testPayloads == null)
+            {
+                return;
+            }
+
+            await GetTestArchives(testPayloads, tempDirPath);
+            ExpandArchivesInDirectory(tempDirPath, destinationDirectory);
+
+            RSPGenerator rspGenerator = new RSPGenerator();
+            foreach (XUnitTestAssembly assembly in testDefinitions.Values)
+            {
+                rspGenerator.GenerateRSPFile(assembly, Path.Combine(destinationDirectory, assembly.Name));
+            }
+
+            Directory.Delete(tempDirPath);
+        }
+
+        private static async Task<Dictionary<string, XUnitTestAssembly>> GetTestUrls(string jsonUrl, Dictionary<string, XUnitTestAssembly> testDefinitions = null, bool runAllTests = false)
+        {
+            if (httpClient is null)
+            {
+                httpClient = new HttpClient();
+            }
+
+            Debug.Assert(runAllTests || testDefinitions != null);
+            // Set up the json stream reader
+            using (var responseStream = await httpClient.GetStreamAsync(jsonUrl))
+            using (var streamReader = new StreamReader(responseStream))
+            using (var jsonReader = new JsonTextReader(streamReader))
+            {
+                // Manual parsing - we only need to key-value pairs from each object and this avoids deserializing all of the work items into objects
+                string markedTestName = string.Empty;
+                string currentPropertyName = string.Empty;
+
+                while (jsonReader.Read())
+                {
+                    if (jsonReader.Value != null)
+                    {
+                        switch (jsonReader.TokenType)
+                        {
+                            case JsonToken.PropertyName:
+                                currentPropertyName = jsonReader.Value.ToString();
+                                break;
+                            case JsonToken.String:
+                                if (currentPropertyName.Equals("WorkItemId"))
+                                {
+                                    string currentTestName = jsonReader.Value.ToString();
+
+                                    if (runAllTests || testDefinitions.ContainsKey(currentTestName))
+                                    {
+                                        markedTestName = currentTestName;
+                                    }
+                                }
+                                else if (currentPropertyName.Equals("PayloadUri") && markedTestName != string.Empty)
+                                {
+                                    if (!testDefinitions.ContainsKey(markedTestName))
+                                    {
+                                        testDefinitions[markedTestName] = new XUnitTestAssembly() { Name = markedTestName };
+                                    }
+                                    testDefinitions[markedTestName].Url = jsonReader.Value.ToString();
+                                    markedTestName = string.Empty;
+                                }
+                                break;
+                        }
+                    }
+                }
+
+            }
+            return testDefinitions;
+        }
+
+        private static async Task GetTestArchives(Dictionary<string, XUnitTestAssembly> testPayloads, string downloadDir)
+        {
+            if (httpClient is null)
+            {
+                httpClient = new HttpClient();
+            }
+
+            foreach (string testName in testPayloads.Keys)
+            {
+                string payloadUri = testPayloads[testName].Url;
+
+                if (!Uri.IsWellFormedUriString(payloadUri, UriKind.Absolute))
+                    continue;
+
+                using (var response = await httpClient.GetStreamAsync(payloadUri))
+                {
+                    if (response.CanRead)
+                    {
+                        // Create the test setup directory if it doesn't exist
+                        if (!Directory.Exists(downloadDir))
+                        {
+                            Directory.CreateDirectory(downloadDir);
+                        }
+
+                        // CoreFX test archives are output as .zip regardless of platform
+                        string archivePath = Path.Combine(downloadDir, testName + ".zip");
+
+                        // Copy to a temp folder 
+                        using (FileStream file = new FileStream(archivePath, FileMode.Create))
+                        {
+                            await response.CopyToAsync(file);
+                        }
+
+                    }
+                }
+            }
+        }
+
+        private static void ExpandArchivesInDirectory(string archiveDirectory, string destinationDirectory, bool cleanup = true)
+        {
+            Debug.Assert(Directory.Exists(archiveDirectory));
+            Debug.Assert(Directory.Exists(destinationDirectory));
+
+            string[] archives = Directory.GetFiles(archiveDirectory, "*.zip", SearchOption.TopDirectoryOnly);
+
+            foreach (string archivePath in archives)
+            {
+                string destinationDirName = Path.Combine(destinationDirectory, Path.GetFileNameWithoutExtension(archivePath));
+
+                ZipFile.ExtractToDirectory(archivePath, destinationDirName);
+
+
+                // Delete archives
+                if (cleanup)
+                {
+                    File.Delete(archivePath);
+                }
+            }
+        }
+
+        private static void CleanBuild(string directoryToClean)
+        {
+            Debug.Assert(Directory.Exists(directoryToClean));
+            DirectoryInfo dirInfo = new DirectoryInfo(directoryToClean);
+
+            foreach (FileInfo file in dirInfo.EnumerateFiles())
+            {
+                file.Delete();
+            }
+
+            foreach (DirectoryInfo dir in dirInfo.EnumerateDirectories())
+            {
+                dir.Delete(true);
+            }
+        }
+
+    }
+}

external/corert/tests/CoreFX/runtest/src/TestUtils/TestFileSetup/XUnitTestAssembly.cs

@@ -0,0 +1,47 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Newtonsoft.Json;
+
+namespace CoreFX.TestUtils.TestFileSetup
+{
+    public class XUnitTestAssembly
+    {
+        [JsonRequired]
+        [JsonProperty("name")]
+        public string Name;
+
+        [JsonRequired]
+        [JsonProperty("exclusions")]
+        public Exclusions Exclusions;
+
+        // Used to assign a test url or to override it via the json file definition
+        [JsonIgnore]
+        [JsonProperty(Required = Required.Default)]
+        public string Url;
+
+    }
+
+    public class Exclusions
+    {
+        [JsonProperty("namespaces")]
+        public Exclusion[] Namespaces;
+
+        [JsonProperty("classes")]
+        public Exclusion[] Classes;
+
+        [JsonProperty("methods")]
+        public Exclusion[] Methods;
+    }
+
+    public class Exclusion
+    {
+        [JsonRequired]
+        [JsonProperty("name", Required = Required.DisallowNull)]
+        public string Name;
+
+        [JsonRequired]
+        [JsonProperty("reason", Required = Required.DisallowNull)]
+        public string Reason;
+    }
+}

external/corert/tests/CoreFX/runtest/src/TestUtils/XUnit/CoreFX.TestUtils.XUnit.csproj

@@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <Import Project="..\..\..\dir.props" />
+  <PropertyGroup>
+    <TargetFramework>netcoreapp2.1</TargetFramework>
+    <OutputType>Exe</OutputType>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="Newtonsoft.Json">
+      <Version>$(NewtonsoftJsonVersion)</Version>
+    </PackageReference>
+    <PackageReference Include="System.CommandLine">
+      <Version>$(SystemCommandLineVersion)</Version>
+    </PackageReference>
+  </ItemGroup>
+</Project>

external/corert/tests/CoreFX/runtest/src/TestUtils/XUnit/ResultFormatter.cs

@@ -0,0 +1,132 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+/// <summary>
+/// This helper class reads xunit xml log files and prints the tallied test results to the console
+/// </summary>
+using System;
+using System.Collections.Generic;
+using System.CommandLine;
+using System.Diagnostics;
+using System.IO;
+using System.IO.Compression;
+using System.Net.Http;
+using System.Reflection;
+using System.Threading.Tasks;
+using System.Xml;
+using Newtonsoft.Json;
+
+namespace CoreFX.TestUtils.XUnit
+{
+    public static class ResultFormatter
+    {
+        private static string logDir;
+        private static string logPattern;
+
+        public static void Main(string[] args)
+        {
+            ArgumentSyntax syntax = ParseCommandLine(args);
+            IEnumerable<string> logFiles = DiscoverLogs(logDir, logPattern);
+            PrintTotals(logFiles);
+        }
+
+        private static void PrintTotals(IEnumerable<string> logFiles)
+        {
+            int total = 0;
+            int passed = 0;
+            int failed = 0;
+            int skipped = 0;
+            ulong timeElapsed = 0;
+            bool printTotals = false;
+
+            foreach (string logFile in logFiles)
+            {
+                try
+                {
+
+                    // XMLReader escapes the character sequence \\.. as just a single backslash \ - Is this intended behavior? 
+                    using (XmlReader reader = XmlReader.Create(logFile.Replace(@"\\..", @"\..")))
+                    {
+                        reader.MoveToContent();
+                        reader.ReadToDescendant("collection");
+                        do
+                        {
+                            // Get total tests in current element
+                            string totalAttr = reader.GetAttribute("total");
+                            int currentTotal;
+                            Int32.TryParse(totalAttr, out currentTotal);
+
+                            // Get passed tests 
+                            string passedAttr = reader.GetAttribute("passed");
+                            int currentPassed;
+                            Int32.TryParse(passedAttr, out currentPassed);
+
+                            // Get failed tests
+                            string failedAttr = reader.GetAttribute("failed");
+                            int currentFailed;
+                            Int32.TryParse(failedAttr, out currentFailed);
+
+                            // Get skipped tests
+                            string skippedAttr = reader.GetAttribute("skipped");
+                            int currentSkipped;
+                            Int32.TryParse(skippedAttr, out currentSkipped);
+
+                            // Get time elapsed
+                            string timeAttr = reader.GetAttribute("time");
+                            ulong currentTime;
+                            UInt64.TryParse(timeAttr, out currentTime);
+
+                            // Update running total only once current element has been parsed
+                            total += currentTotal;
+                            passed += currentPassed;
+                            failed += currentFailed;
+                            skipped += currentSkipped;
+                            timeElapsed += currentTime;
+
+                        } while (reader.ReadToNextSibling("collection"));
+
+                        // If we've fully parsed a log, print totals
+                        printTotals = true;
+                    }
+                }
+                catch (XmlException exc)
+                {
+                    Console.WriteLine("Malformed Log: {0} ", logFile);
+                    Console.WriteLine("Reason: {0} ", exc.Message);
+                    continue;
+                }
+            }
+
+            if (printTotals)
+            {
+                Console.WriteLine("=== CoreFX TEST EXECUTION SUMMARY ===: ");
+                Console.WriteLine(String.Format("Total: {0}, Passed: {1}, Failed: {2}, Skipped: {3}", total, passed, failed, timeElapsed));
+                Console.WriteLine("Detailed logs written to: " + logDir);
+            }
+        }
+
+        private static ArgumentSyntax ParseCommandLine(string[] args)
+        {
+
+            ArgumentSyntax argSyntax = ArgumentSyntax.Parse(args, syntax =>
+            {
+                syntax.DefineOption("log|logDirectory|logDir", ref logDir, "Path to directory of xml test results");
+                syntax.DefineOption("pattern|p", ref logPattern, "Pattern of XUnit log filenames for which to search");
+            });
+
+            return argSyntax;
+        }
+
+        public static IEnumerable<string> DiscoverLogs(string logDirectory, string logPattern)
+        {
+            Debug.Assert(Directory.Exists(logDirectory));
+            Console.WriteLine(logDirectory);
+            var logFiles = Directory.EnumerateFiles(logDirectory, logPattern, SearchOption.AllDirectories);
+
+            return logFiles;
+        }
+
+
+    }
+}