1860 lines
50 KiB
C#
1860 lines
50 KiB
C#
//
|
|
// compiler-tester.cs
|
|
//
|
|
// Author:
|
|
// Marek Safar (marek.safar@gmail.com)
|
|
//
|
|
// Copyright (C) 2008, 2009 Novell, Inc (http://www.novell.com)
|
|
// Copyright (C) 2012 Xamarin Inc (http://www.xamarin.com)
|
|
//
|
|
// Permission is hereby granted, free of charge, to any person obtaining
|
|
// a copy of this software and associated documentation files (the
|
|
// "Software"), to deal in the Software without restriction, including
|
|
// without limitation the rights to use, copy, modify, merge, publish,
|
|
// distribute, sublicense, and/or sell copies of the Software, and to
|
|
// permit persons to whom the Software is furnished to do so, subject to
|
|
// the following conditions:
|
|
//
|
|
// The above copyright notice and this permission notice shall be
|
|
// included in all copies or substantial portions of the Software.
|
|
//
|
|
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
|
// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
|
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
|
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
//
|
|
|
|
using System;
|
|
using System.IO;
|
|
using System.Diagnostics;
|
|
using System.Reflection;
|
|
using System.Text;
|
|
using System.Collections;
|
|
using System.Xml;
|
|
using System.Collections.Generic;
|
|
using Mono.CompilerServices.SymbolWriter;
|
|
using System.Globalization;
|
|
|
|
namespace TestRunner {
|
|
|
|
interface ITester
|
|
{
|
|
string Output { get; }
|
|
bool Invoke (string[] args);
|
|
bool IsWarning (int warningNumber);
|
|
}
|
|
|
|
class ReflectionTester: ITester {
|
|
MethodInfo ep;
|
|
object[] method_arg;
|
|
StringWriter output;
|
|
int[] all_warnings;
|
|
|
|
public ReflectionTester (Assembly a)
|
|
{
|
|
Type t = a.GetType ("Mono.CSharp.CompilerCallableEntryPoint");
|
|
|
|
if (t == null)
|
|
Console.Error.WriteLine ("null, huh?");
|
|
|
|
ep = t.GetMethod ("InvokeCompiler",
|
|
BindingFlags.Static | BindingFlags.Public);
|
|
if (ep == null)
|
|
throw new MissingMethodException ("static InvokeCompiler");
|
|
method_arg = new object [2];
|
|
|
|
PropertyInfo pi = t.GetProperty ("AllWarningNumbers");
|
|
all_warnings = (int[])pi.GetValue (null, null);
|
|
Array.Sort (all_warnings);
|
|
}
|
|
|
|
public string Output {
|
|
get {
|
|
return output.GetStringBuilder ().ToString ();
|
|
}
|
|
}
|
|
|
|
public bool Invoke(string[] args)
|
|
{
|
|
output = new StringWriter ();
|
|
method_arg [0] = args;
|
|
method_arg [1] = output;
|
|
return (bool)ep.Invoke (null, method_arg);
|
|
}
|
|
|
|
public bool IsWarning (int warningNumber)
|
|
{
|
|
return Array.BinarySearch (all_warnings, warningNumber) >= 0;
|
|
}
|
|
}
|
|
|
|
#if !MOBILE
|
|
class ProcessTester: ITester
|
|
{
|
|
ProcessStartInfo pi;
|
|
string output;
|
|
|
|
public ProcessTester (string p_path)
|
|
{
|
|
pi = new ProcessStartInfo ();
|
|
pi.FileName = p_path;
|
|
pi.CreateNoWindow = true;
|
|
pi.WindowStyle = ProcessWindowStyle.Hidden;
|
|
pi.RedirectStandardOutput = true;
|
|
pi.RedirectStandardError = true;
|
|
pi.UseShellExecute = false;
|
|
}
|
|
|
|
public string Output {
|
|
get {
|
|
return output;
|
|
}
|
|
}
|
|
|
|
public bool Invoke(string[] args)
|
|
{
|
|
StringBuilder sb = new StringBuilder ("/nologo ");
|
|
foreach (string s in args) {
|
|
sb.Append (s);
|
|
sb.Append (" ");
|
|
}
|
|
pi.Arguments = sb.ToString ();
|
|
Process p = Process.Start (pi);
|
|
output = p.StandardError.ReadToEnd ();
|
|
if (output.Length == 0)
|
|
output = p.StandardOutput.ReadToEnd ();
|
|
p.WaitForExit ();
|
|
return p.ExitCode == 0;
|
|
}
|
|
|
|
public bool IsWarning (int warningNumber)
|
|
{
|
|
throw new NotImplementedException ();
|
|
}
|
|
}
|
|
#endif
|
|
|
|
class TestCase : MarshalByRefObject
|
|
{
|
|
public readonly string FileName;
|
|
public readonly string[] CompilerOptions;
|
|
public readonly string[] Dependencies;
|
|
|
|
public TestCase (string filename, string[] options, string[] deps)
|
|
{
|
|
this.FileName = filename;
|
|
this.CompilerOptions = options;
|
|
this.Dependencies = deps;
|
|
}
|
|
}
|
|
|
|
class NUnitChecker : PositiveChecker
|
|
{
|
|
class TestCaseEntry
|
|
{
|
|
string name;
|
|
string referenceFile;
|
|
string executedMethod;
|
|
bool has_return;
|
|
|
|
public TestCaseEntry (string name, string referenceFile, MethodInfo executedMethod)
|
|
{
|
|
this.name = name.Replace ('-', '_');
|
|
this.referenceFile = referenceFile;
|
|
this.executedMethod = ConvertMethodInfoToText (executedMethod, out has_return);
|
|
}
|
|
|
|
public string Name
|
|
{
|
|
get
|
|
{
|
|
return name;
|
|
}
|
|
}
|
|
|
|
public string ReferenceFile {
|
|
get {
|
|
return referenceFile;
|
|
}
|
|
}
|
|
|
|
static string ConvertMethodInfoToText (MethodInfo mi, out bool hasReturn)
|
|
{
|
|
hasReturn = mi.ReturnType != typeof (void);
|
|
string declaring = mi.DeclaringType.FullName.Replace ('+', '.');
|
|
var param = mi.GetParameters ();
|
|
if (param.Length == 0)
|
|
return declaring + "." + mi.Name + " ()";
|
|
|
|
return declaring + "." + mi.Name + " (new string[0])";
|
|
}
|
|
|
|
public string GetTestFixture ()
|
|
{
|
|
var call = name + "::" + executedMethod;
|
|
if (!has_return)
|
|
return call;
|
|
|
|
return string.Format ("Assert.AreEqual (0, {0})", call);
|
|
}
|
|
}
|
|
|
|
List<TestCaseEntry> entries = new List<TestCaseEntry> ();
|
|
|
|
public NUnitChecker (ITester tester)
|
|
: base (tester, null)
|
|
{
|
|
}
|
|
|
|
public override void CleanUp ()
|
|
{
|
|
base.CleanUp ();
|
|
|
|
StringBuilder aliases = new StringBuilder ();
|
|
var src_dir = Path.Combine ("projects", "MonoTouch");
|
|
string src_file = Path.Combine (src_dir, "tests.cs");
|
|
|
|
using (var file = new StreamWriter (src_file, false)) {
|
|
foreach (var e in entries) {
|
|
file.WriteLine ("extern alias {0};", e.Name);
|
|
aliases.AppendFormat (" <Reference Include=\"{0}\">", Path.GetFileNameWithoutExtension (e.ReferenceFile));
|
|
aliases.Append (Environment.NewLine);
|
|
aliases.AppendFormat (" <Aliases>{0}</Aliases>", e.Name);
|
|
aliases.Append (Environment.NewLine);
|
|
aliases.AppendFormat (" <HintPath>..\\..\\{0}</HintPath>", Path.GetFileName (e.ReferenceFile));
|
|
aliases.Append (Environment.NewLine);
|
|
aliases.AppendLine (" </Reference>");
|
|
}
|
|
|
|
file.WriteLine ();
|
|
file.WriteLine ("using NUnit.Framework;");
|
|
file.WriteLine ();
|
|
file.WriteLine ("[TestFixture]");
|
|
file.WriteLine ("public class Tests {");
|
|
|
|
foreach (var e in entries) {
|
|
file.WriteLine ("\t[Test]");
|
|
file.WriteLine ("\tpublic void TestFile_{0} ()", e.Name);
|
|
file.WriteLine ("\t{");
|
|
file.WriteLine ("\t\t{0};", e.GetTestFixture ());
|
|
file.WriteLine ("\t}");
|
|
file.WriteLine ();
|
|
}
|
|
|
|
file.WriteLine ("}");
|
|
}
|
|
|
|
var input = File.ReadAllText (Path.Combine (src_dir, "MonoTouch.csproj.template"));
|
|
input = input.Replace ("@GENERATED_REFERENCES", aliases.ToString ());
|
|
input = input.Replace ("@TEST_SOURCEFILE", Path.GetFileName (src_file));
|
|
|
|
File.WriteAllText (Path.Combine (src_dir, "MonoTouch.csproj"), input);
|
|
return;
|
|
}
|
|
|
|
protected override bool ExecuteTestFile (TestCase test, string binaryFileName)
|
|
{
|
|
Assembly assembly = Assembly.LoadFile (binaryFileName);
|
|
var ep = assembly.EntryPoint;
|
|
if (!ep.IsPublic) {
|
|
HandleFailure (test.FileName, TestResult.LoadError, "Entry method is private");
|
|
return false;
|
|
}
|
|
|
|
if (ep.DeclaringType.IsNestedPrivate || ep.DeclaringType.IsNestedFamily) {
|
|
HandleFailure (test.FileName, TestResult.LoadError, "Entry method in hidden nested type");
|
|
return false;
|
|
}
|
|
|
|
entries.Add (new TestCaseEntry (Path.GetFileNameWithoutExtension (test.FileName), binaryFileName, ep));
|
|
HandleFailure (test.FileName, TestResult.Success, null);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
class PositiveTestCase : TestCase
|
|
{
|
|
public class VerificationData : MarshalByRefObject
|
|
{
|
|
public class MethodData : MarshalByRefObject
|
|
{
|
|
public MethodData (MethodBase mi, int il_size)
|
|
{
|
|
this.Type = mi.DeclaringType.ToString ();
|
|
this.MethodName = mi.ToString ();
|
|
this.MethodAttributes = (int) mi.Attributes;
|
|
this.ILSize = il_size;
|
|
}
|
|
|
|
public MethodData (string type_name, string method_name, int method_attributes, int il_size)
|
|
{
|
|
this.Type = type_name;
|
|
this.MethodName = method_name;
|
|
this.MethodAttributes = method_attributes;
|
|
this.ILSize = il_size;
|
|
}
|
|
|
|
public string Type;
|
|
public string MethodName;
|
|
public int ILSize;
|
|
public bool Checked;
|
|
public int MethodAttributes;
|
|
}
|
|
|
|
ArrayList methods;
|
|
public bool IsNewSet;
|
|
|
|
public VerificationData (string test_file)
|
|
{
|
|
this.test_file = test_file;
|
|
}
|
|
|
|
string test_file;
|
|
|
|
public static VerificationData FromFile (string name, XmlReader r)
|
|
{
|
|
VerificationData tc = new VerificationData (name);
|
|
ArrayList methods = new ArrayList ();
|
|
r.Read ();
|
|
while (r.ReadToNextSibling ("type")) {
|
|
string type_name = r ["name"];
|
|
r.Read ();
|
|
while (r.ReadToNextSibling ("method")) {
|
|
string m_name = r ["name"];
|
|
int method_attrs = int.Parse (r["attrs"]);
|
|
|
|
r.ReadToDescendant ("size");
|
|
int il_size = r.ReadElementContentAsInt ();
|
|
methods.Add (new MethodData (type_name, m_name, method_attrs, il_size));
|
|
r.Read ();
|
|
}
|
|
r.Read ();
|
|
}
|
|
|
|
tc.methods = methods;
|
|
return tc;
|
|
}
|
|
|
|
public void WriteCodeInfoTo (XmlWriter w)
|
|
{
|
|
w.WriteStartElement ("test");
|
|
w.WriteAttributeString ("name", test_file);
|
|
|
|
string type = null;
|
|
foreach (MethodData data in methods) {
|
|
if (!data.Checked)
|
|
continue;
|
|
|
|
if (type != data.Type) {
|
|
if (type != null)
|
|
w.WriteEndElement ();
|
|
|
|
type = data.Type;
|
|
w.WriteStartElement ("type");
|
|
w.WriteAttributeString ("name", type);
|
|
}
|
|
|
|
w.WriteStartElement ("method");
|
|
w.WriteAttributeString ("name", data.MethodName);
|
|
int v = data.MethodAttributes;
|
|
w.WriteAttributeString ("attrs", v.ToString ());
|
|
w.WriteStartElement ("size");
|
|
w.WriteValue (data.ILSize);
|
|
w.WriteEndElement ();
|
|
w.WriteEndElement ();
|
|
}
|
|
|
|
if (type != null)
|
|
w.WriteEndElement ();
|
|
|
|
w.WriteEndElement ();
|
|
}
|
|
|
|
public MethodData FindMethodData (string method_name, string declaring_type)
|
|
{
|
|
if (methods == null)
|
|
return null;
|
|
|
|
foreach (MethodData md in methods) {
|
|
if (md.MethodName == method_name && md.Type == declaring_type)
|
|
return md;
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
public void AddNewMethod (MethodBase mb, int il_size)
|
|
{
|
|
if (methods == null)
|
|
methods = new ArrayList ();
|
|
|
|
MethodData md = new MethodData (mb, il_size);
|
|
md.Checked = true;
|
|
methods.Add (md);
|
|
}
|
|
}
|
|
|
|
VerificationData verif_data;
|
|
|
|
public PositiveTestCase (string filename, string [] options, string [] deps)
|
|
: base (filename, options, deps)
|
|
{
|
|
}
|
|
|
|
public void CreateNewTest ()
|
|
{
|
|
verif_data = new VerificationData (FileName);
|
|
verif_data.IsNewSet = true;
|
|
}
|
|
|
|
public VerificationData VerificationProvider {
|
|
set {
|
|
verif_data = value;
|
|
}
|
|
get {
|
|
return verif_data;
|
|
}
|
|
}
|
|
}
|
|
|
|
class Checker: MarshalByRefObject, IDisposable
|
|
{
|
|
protected ITester tester;
|
|
protected int total;
|
|
protected int ignored;
|
|
protected int syntax_errors;
|
|
string issue_file;
|
|
StreamWriter log_file;
|
|
StreamWriter result_xml;
|
|
protected string[] extra_compiler_options;
|
|
protected string reference_dir;
|
|
// protected string[] compiler_options;
|
|
// protected string[] dependencies;
|
|
|
|
protected ArrayList tests = new ArrayList ();
|
|
protected Hashtable test_hash = new Hashtable ();
|
|
protected Dictionary<string, StringBuilder> file_log_lines = new Dictionary<string, StringBuilder> ();
|
|
protected ArrayList succeeded = new ArrayList ();
|
|
protected ArrayList regression = new ArrayList ();
|
|
protected ArrayList know_issues = new ArrayList ();
|
|
protected ArrayList ignore_list = new ArrayList ();
|
|
protected ArrayList no_error_list = new ArrayList ();
|
|
ArrayList skip = new ArrayList ();
|
|
|
|
protected bool verbose;
|
|
protected bool safe_execution;
|
|
|
|
int total_known_issues;
|
|
|
|
protected Checker (ITester tester)
|
|
{
|
|
this.tester = tester;
|
|
}
|
|
|
|
public string IssueFile {
|
|
set {
|
|
this.issue_file = value;
|
|
ReadWrongErrors (issue_file);
|
|
}
|
|
}
|
|
|
|
public string LogFile {
|
|
set {
|
|
this.log_file = new StreamWriter (value, false);
|
|
}
|
|
}
|
|
|
|
public string ResultXml {
|
|
set {
|
|
this.result_xml = new StreamWriter (value, false);
|
|
}
|
|
}
|
|
|
|
public string Name {
|
|
get;
|
|
set;
|
|
}
|
|
|
|
public bool Verbose {
|
|
set {
|
|
verbose = value;
|
|
}
|
|
}
|
|
|
|
public bool SafeExecution {
|
|
set {
|
|
safe_execution = value;
|
|
}
|
|
}
|
|
|
|
public string[] ExtraCompilerOptions {
|
|
set {
|
|
extra_compiler_options = value;
|
|
}
|
|
}
|
|
|
|
public string ReferenceDirectory {
|
|
set {
|
|
reference_dir = value;
|
|
}
|
|
}
|
|
|
|
protected virtual bool GetExtraOptions (string file, out string[] compiler_options,
|
|
out string[] dependencies)
|
|
{
|
|
int row = 0;
|
|
compiler_options = null;
|
|
dependencies = null;
|
|
try {
|
|
using (StreamReader sr = new StreamReader (file)) {
|
|
String line;
|
|
while (row++ < 3 && (line = sr.ReadLine()) != null) {
|
|
if (!AnalyzeTestFile (file, ref row, line, ref compiler_options,
|
|
ref dependencies))
|
|
return false;
|
|
}
|
|
}
|
|
} catch {
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
protected virtual bool AnalyzeTestFile (string file, ref int row, string line,
|
|
ref string[] compiler_options,
|
|
ref string[] dependencies)
|
|
{
|
|
const string options = "// Compiler options:";
|
|
const string depends = "// Dependencies:";
|
|
|
|
if (row == 1) {
|
|
compiler_options = null;
|
|
dependencies = null;
|
|
}
|
|
|
|
int index = line.IndexOf (options);
|
|
if (index != -1) {
|
|
compiler_options = line.Substring (index + options.Length).Trim().Split (' ');
|
|
for (int i = 0; i < compiler_options.Length; i++)
|
|
compiler_options[i] = compiler_options[i].TrimStart ().Replace ("$REF_DIR", reference_dir);
|
|
}
|
|
index = line.IndexOf (depends);
|
|
if (index != -1) {
|
|
dependencies = line.Substring (index + depends.Length).Trim().Split (' ');
|
|
for (int i = 0; i < dependencies.Length; i++)
|
|
dependencies[i] = dependencies[i].TrimStart ();
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
public bool Do (string filename)
|
|
{
|
|
if (test_hash.Contains (filename))
|
|
return true;
|
|
|
|
file_log_lines.Add (filename, new StringBuilder ());
|
|
|
|
if (verbose)
|
|
Log (filename + "...\t");
|
|
|
|
if (skip.Contains (filename)) {
|
|
return false;
|
|
}
|
|
|
|
if (ignore_list.Contains (filename)) {
|
|
++ignored;
|
|
LogFileLine (filename, "NOT TESTED");
|
|
return false;
|
|
}
|
|
|
|
string[] compiler_options, dependencies;
|
|
if (!GetExtraOptions (filename, out compiler_options, out dependencies)) {
|
|
LogFileLine (filename, "ERROR");
|
|
return false;
|
|
}
|
|
|
|
if (extra_compiler_options != null) {
|
|
if (compiler_options == null)
|
|
compiler_options = extra_compiler_options;
|
|
else {
|
|
string[] new_options = new string [compiler_options.Length + extra_compiler_options.Length];
|
|
extra_compiler_options.CopyTo (new_options, 0);
|
|
compiler_options.CopyTo (new_options, extra_compiler_options.Length);
|
|
compiler_options = new_options;
|
|
}
|
|
}
|
|
|
|
TestCase test = CreateTestCase (filename, compiler_options, dependencies);
|
|
test_hash.Add (filename, test);
|
|
|
|
++total;
|
|
if (dependencies != null) {
|
|
foreach (string dependency in dependencies) {
|
|
if (!Do (dependency)) {
|
|
LogFileLine (filename, "DEPENDENCY FAILED");
|
|
return false;
|
|
}
|
|
}
|
|
}
|
|
|
|
tests.Add (test);
|
|
|
|
return Check (test);
|
|
}
|
|
|
|
protected virtual bool Check (TestCase test)
|
|
{
|
|
string[] test_args;
|
|
|
|
if (test.CompilerOptions != null) {
|
|
test_args = new string[1 + test.CompilerOptions.Length];
|
|
test.CompilerOptions.CopyTo (test_args, 0);
|
|
} else {
|
|
test_args = new string[1];
|
|
}
|
|
test_args[test_args.Length - 1] = test_args[0];
|
|
test_args[0] = test.FileName;
|
|
|
|
return tester.Invoke (test_args);
|
|
}
|
|
|
|
protected virtual TestCase CreateTestCase (string filename, string [] options, string [] deps)
|
|
{
|
|
return new TestCase (filename, options, deps);
|
|
}
|
|
|
|
void ReadWrongErrors (string file)
|
|
{
|
|
const string ignored = "IGNORE";
|
|
const string no_error = "NO ERROR";
|
|
const string skip_tag = "SKIP";
|
|
|
|
using (StreamReader sr = new StreamReader (file)) {
|
|
string line;
|
|
while ((line = sr.ReadLine()) != null) {
|
|
if (line.StartsWith ("#"))
|
|
continue;
|
|
|
|
ArrayList active_cont = know_issues;
|
|
|
|
if (line.IndexOf (ignored) > 0)
|
|
active_cont = ignore_list;
|
|
else if (line.IndexOf (no_error) > 0)
|
|
active_cont = no_error_list;
|
|
else if (line.Contains (skip_tag))
|
|
active_cont = skip;
|
|
|
|
string file_name = line.Split (' ')[0];
|
|
if (file_name.Length == 0)
|
|
continue;
|
|
|
|
active_cont.Add (file_name);
|
|
}
|
|
}
|
|
total_known_issues = know_issues.Count;
|
|
}
|
|
|
|
protected virtual void PrintSummary ()
|
|
{
|
|
LogLine ("Done" + Environment.NewLine);
|
|
float rate = 0;
|
|
if (total > 0)
|
|
rate = (float) (succeeded.Count) / (float)total;
|
|
LogLine ("{0} test cases passed ({1:0.##%})", succeeded.Count, rate);
|
|
|
|
if (syntax_errors > 0)
|
|
LogLine ("{0} test(s) ignored because of wrong syntax !", syntax_errors);
|
|
|
|
if (ignored > 0)
|
|
LogLine ("{0} test(s) ignored", ignored);
|
|
|
|
if (total_known_issues - know_issues.Count > 0)
|
|
LogLine ("{0} known issue(s)", total_known_issues - know_issues.Count);
|
|
|
|
know_issues.AddRange (no_error_list);
|
|
if (know_issues.Count > 0) {
|
|
LogLine ("");
|
|
LogLine (issue_file + " contains {0} already fixed issues. Please remove", know_issues.Count);
|
|
foreach (string s in know_issues)
|
|
LogLine (s);
|
|
}
|
|
if (regression.Count > 0) {
|
|
LogLine ("");
|
|
LogLine ("The latest changes caused regression in {0} file(s)", regression.Count);
|
|
foreach (string s in regression)
|
|
LogLine (s);
|
|
}
|
|
}
|
|
|
|
protected virtual void OutputResultXml ()
|
|
{
|
|
if (result_xml == null)
|
|
return;
|
|
|
|
var xmlWriter = new XmlTextWriter (result_xml);
|
|
xmlWriter.Formatting = Formatting.Indented;
|
|
|
|
xmlWriter.WriteStartDocument ();
|
|
|
|
xmlWriter.WriteStartElement ("assemblies");
|
|
|
|
xmlWriter.WriteStartElement ("assembly");
|
|
|
|
xmlWriter.WriteAttributeString ("name", Name);
|
|
xmlWriter.WriteAttributeString ("environment", $"compiler-tester-version: {Assembly.GetExecutingAssembly ().GetName ()}, clr-version: {Environment.Version}, os-version: {Environment.OSVersion}, platform: {Environment.OSVersion.Platform}, cwd: {Environment.CurrentDirectory}, machine-name: {Environment.MachineName}, user: {Environment.UserName}, user-domain: {Environment.UserDomainName}");
|
|
xmlWriter.WriteAttributeString ("test-framework", "compiler-tester");
|
|
xmlWriter.WriteAttributeString ("run-date", XmlConvert.ToString (DateTime.Now, "yyyy-MM-dd"));
|
|
xmlWriter.WriteAttributeString ("run-time", XmlConvert.ToString (DateTime.Now, "HH:mm:ss"));
|
|
|
|
xmlWriter.WriteAttributeString ("total", (succeeded.Count + regression.Count).ToString ()); // ignore known issues and ignored tests for now, we care mostly about failures
|
|
xmlWriter.WriteAttributeString ("errors", 0.ToString ());
|
|
xmlWriter.WriteAttributeString ("failed", (regression.Count).ToString ());
|
|
xmlWriter.WriteAttributeString ("skipped", 0.ToString ());
|
|
|
|
xmlWriter.WriteAttributeString ("passed", succeeded.Count.ToString ());
|
|
|
|
xmlWriter.WriteStartElement ("collection");
|
|
xmlWriter.WriteAttributeString ("name", "tests");
|
|
|
|
foreach (var t in succeeded) {
|
|
xmlWriter.WriteStartElement ("test");
|
|
xmlWriter.WriteAttributeString ("name", Name + ".tests." + t);
|
|
xmlWriter.WriteAttributeString ("type", Name + ".tests");
|
|
xmlWriter.WriteAttributeString ("method", t.ToString ());
|
|
xmlWriter.WriteAttributeString ("result", "Pass");
|
|
xmlWriter.WriteAttributeString ("time", "0");
|
|
xmlWriter.WriteEndElement (); // test element
|
|
}
|
|
|
|
foreach (var t in regression) {
|
|
xmlWriter.WriteStartElement ("test");
|
|
xmlWriter.WriteAttributeString ("name", Name + ".tests." + t);
|
|
xmlWriter.WriteAttributeString ("type", Name + ".tests");
|
|
xmlWriter.WriteAttributeString ("method", t.ToString ());
|
|
xmlWriter.WriteAttributeString ("result", "Fail");
|
|
xmlWriter.WriteAttributeString ("time", "0");
|
|
|
|
xmlWriter.WriteStartElement ("failure");
|
|
xmlWriter.WriteAttributeString ("exception-type", "CompilerTesterException");
|
|
xmlWriter.WriteStartElement ("message");
|
|
xmlWriter.WriteCData (file_log_lines[(string)t].ToString ());
|
|
xmlWriter.WriteEndElement (); // message element
|
|
xmlWriter.WriteEndElement(); // failure element
|
|
|
|
xmlWriter.WriteEndElement(); // test element
|
|
}
|
|
|
|
xmlWriter.WriteEndElement (); // collection
|
|
xmlWriter.WriteEndElement (); // assembly
|
|
xmlWriter.WriteEndElement (); // assemblies
|
|
xmlWriter.WriteEndDocument ();
|
|
xmlWriter.Flush ();
|
|
xmlWriter.Close ();
|
|
}
|
|
|
|
public int ResultCode
|
|
{
|
|
get {
|
|
return regression.Count == 0 ? 0 : 1;
|
|
}
|
|
}
|
|
|
|
protected void Log (string msg, params object [] rest)
|
|
{
|
|
Console.Write (msg, rest);
|
|
if (log_file != null)
|
|
log_file.Write (msg, rest);
|
|
}
|
|
|
|
protected void LogLine (string msg)
|
|
{
|
|
Console.WriteLine (msg);
|
|
if (log_file != null)
|
|
log_file.WriteLine (msg);
|
|
}
|
|
|
|
protected void LogLine (string msg, params object [] rest)
|
|
{
|
|
Console.WriteLine (msg, rest);
|
|
if (log_file != null)
|
|
log_file.WriteLine (msg, rest);
|
|
}
|
|
|
|
protected void LogLineForFile (string file, string msg)
|
|
{
|
|
file_log_lines[file].AppendLine (msg);
|
|
LogLine (msg);
|
|
}
|
|
|
|
protected void LogLineForFile (string file, string msg, params object [] rest)
|
|
{
|
|
file_log_lines[file].AppendLine (String.Format (msg, rest));
|
|
LogLine (msg, rest);
|
|
}
|
|
|
|
public void LogFileLine (string file, string msg)
|
|
{
|
|
string s = verbose ? msg : file + "...\t" + msg;
|
|
|
|
Console.WriteLine (s);
|
|
file_log_lines[file].AppendLine (s);
|
|
if (log_file != null)
|
|
log_file.WriteLine (s);
|
|
}
|
|
|
|
#region IDisposable Members
|
|
|
|
public void Dispose()
|
|
{
|
|
if (log_file != null)
|
|
log_file.Close ();
|
|
}
|
|
|
|
#endregion
|
|
|
|
public virtual void Initialize ()
|
|
{
|
|
}
|
|
|
|
public virtual void CleanUp ()
|
|
{
|
|
PrintSummary ();
|
|
OutputResultXml ();
|
|
}
|
|
}
|
|
|
|
class PositiveChecker: Checker
|
|
{
|
|
readonly string files_folder;
|
|
readonly static object[] default_args = new object[1] { new string[] {} };
|
|
string doc_output;
|
|
string verif_file;
|
|
bool update_verif_file;
|
|
Hashtable verif_data;
|
|
|
|
#if !MOBILE
|
|
ProcessStartInfo pi;
|
|
#endif
|
|
readonly string mono;
|
|
|
|
public enum TestResult {
|
|
CompileError,
|
|
ExecError,
|
|
LoadError,
|
|
XmlError,
|
|
Success,
|
|
ILError,
|
|
DebugError,
|
|
MethodAttributesError
|
|
}
|
|
|
|
public PositiveChecker (ITester tester, string verif_file):
|
|
base (tester)
|
|
{
|
|
files_folder = Directory.GetCurrentDirectory ();
|
|
this.verif_file = verif_file;
|
|
|
|
#if !MOBILE
|
|
pi = new ProcessStartInfo ();
|
|
pi.CreateNoWindow = true;
|
|
pi.WindowStyle = ProcessWindowStyle.Hidden;
|
|
pi.RedirectStandardOutput = true;
|
|
pi.RedirectStandardError = true;
|
|
pi.UseShellExecute = false;
|
|
|
|
mono = Environment.GetEnvironmentVariable ("MONO_RUNTIME");
|
|
if (mono != null) {
|
|
pi.FileName = mono;
|
|
}
|
|
#endif
|
|
}
|
|
|
|
public bool UpdateVerificationDataFile {
|
|
set {
|
|
update_verif_file = value;
|
|
}
|
|
get {
|
|
return update_verif_file;
|
|
}
|
|
}
|
|
|
|
protected override bool GetExtraOptions(string file, out string[] compiler_options,
|
|
out string[] dependencies) {
|
|
if (!base.GetExtraOptions (file, out compiler_options, out dependencies))
|
|
return false;
|
|
|
|
doc_output = null;
|
|
if (compiler_options == null)
|
|
return true;
|
|
|
|
foreach (string one_opt in compiler_options) {
|
|
if (one_opt.StartsWith ("-doc:")) {
|
|
doc_output = one_opt.Split (':', '/')[1];
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
class DomainTester : MarshalByRefObject
|
|
{
|
|
public bool CheckILSize (PositiveTestCase test, PositiveChecker checker, string file)
|
|
{
|
|
Assembly assembly = Assembly.LoadFile (file);
|
|
|
|
bool success = true;
|
|
Type[] types = assembly.GetTypes ();
|
|
foreach (Type t in types) {
|
|
|
|
// Skip interfaces
|
|
if (!t.IsClass && !t.IsValueType)
|
|
continue;
|
|
|
|
if (test.VerificationProvider == null) {
|
|
if (!checker.UpdateVerificationDataFile)
|
|
checker.LogFileLine (test.FileName, "Missing IL verification data");
|
|
test.CreateNewTest ();
|
|
}
|
|
|
|
foreach (MemberInfo m in t.GetMembers (BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Static | BindingFlags.Instance | BindingFlags.DeclaredOnly)) {
|
|
MethodBase mi = m as MethodBase;
|
|
if (mi == null)
|
|
continue;
|
|
|
|
if ((mi.Attributes & (MethodAttributes.PinvokeImpl)) != 0)
|
|
continue;
|
|
|
|
success &= CompareIL (mi, test, checker);
|
|
}
|
|
}
|
|
|
|
return success;
|
|
}
|
|
|
|
bool CompareIL (MethodBase mi, PositiveTestCase test, PositiveChecker checker)
|
|
{
|
|
string m_name = mi.ToString ();
|
|
string decl_type = mi.DeclaringType.ToString ();
|
|
PositiveTestCase.VerificationData data_provider = test.VerificationProvider;
|
|
|
|
PositiveTestCase.VerificationData.MethodData md = data_provider.FindMethodData (m_name, decl_type);
|
|
if (md == null) {
|
|
data_provider.AddNewMethod (mi, GetILSize (mi));
|
|
if (!data_provider.IsNewSet) {
|
|
checker.HandleFailure (test.FileName, PositiveChecker.TestResult.ILError, decl_type + ": " + m_name + " (new method?)");
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
if (md.Checked) {
|
|
checker.HandleFailure (test.FileName, PositiveChecker.TestResult.ILError, decl_type + ": " + m_name + " has a duplicate");
|
|
return false;
|
|
}
|
|
|
|
md.Checked = true;
|
|
|
|
if (md.MethodAttributes != (int) mi.Attributes) {
|
|
checker.HandleFailure (test.FileName, PositiveChecker.TestResult.MethodAttributesError,
|
|
string.Format ("{0} ({1} -> {2})", decl_type + ": " + m_name, (MethodAttributes) md.MethodAttributes, mi.Attributes));
|
|
}
|
|
|
|
md.MethodAttributes = (int) mi.Attributes;
|
|
|
|
int il_size = GetILSize (mi);
|
|
if (md.ILSize == il_size)
|
|
return true;
|
|
|
|
if (md.ILSize > il_size) {
|
|
checker.LogFileLine (test.FileName, string.Format ("{0} (code size reduction {1} -> {2})", decl_type + ": " + m_name, md.ILSize, il_size));
|
|
md.ILSize = il_size;
|
|
return true;
|
|
}
|
|
|
|
checker.HandleFailure (test.FileName, PositiveChecker.TestResult.ILError,
|
|
string.Format ("{0} (code size {1} -> {2})", decl_type + ": " + m_name, md.ILSize, il_size));
|
|
|
|
md.ILSize = il_size;
|
|
|
|
return false;
|
|
}
|
|
|
|
static int GetILSize (MethodBase mi)
|
|
{
|
|
MethodBody body = mi.GetMethodBody ();
|
|
if (body != null)
|
|
return body.GetILAsByteArray ().Length;
|
|
|
|
return 0;
|
|
}
|
|
|
|
bool ExecuteFile (MethodInfo entry_point, string filename)
|
|
{
|
|
TextWriter stdout = Console.Out;
|
|
TextWriter stderr = Console.Error;
|
|
Console.SetOut (TextWriter.Null);
|
|
Console.SetError (TextWriter.Null);
|
|
ParameterInfo[] pi = entry_point.GetParameters ();
|
|
object[] args = pi.Length == 0 ? null : default_args;
|
|
|
|
object result = null;
|
|
try {
|
|
try {
|
|
result = entry_point.Invoke (null, args);
|
|
} finally {
|
|
Console.SetOut (stdout);
|
|
Console.SetError (stderr);
|
|
}
|
|
} catch (Exception e) {
|
|
throw new ApplicationException (e.ToString ());
|
|
}
|
|
|
|
if (result is int && (int) result != 0)
|
|
throw new ApplicationException ("Wrong return code: " + result.ToString ());
|
|
|
|
return true;
|
|
}
|
|
|
|
public bool Test (string file)
|
|
{
|
|
Assembly assembly = Assembly.LoadFile (file);
|
|
return ExecuteFile (assembly.EntryPoint, file);
|
|
}
|
|
}
|
|
|
|
protected override bool Check(TestCase test)
|
|
{
|
|
string filename = test.FileName;
|
|
try {
|
|
if (!base.Check (test)) {
|
|
HandleFailure (filename, TestResult.CompileError, tester.Output);
|
|
return false;
|
|
}
|
|
}
|
|
catch (Exception e) {
|
|
if (e.InnerException != null)
|
|
e = e.InnerException;
|
|
|
|
HandleFailure (filename, TestResult.CompileError, e.ToString ());
|
|
return false;
|
|
}
|
|
|
|
// Test setup
|
|
if (filename.EndsWith ("-lib.cs") || filename.EndsWith ("-mod.cs")) {
|
|
if (verbose)
|
|
LogFileLine (filename, "OK");
|
|
--total;
|
|
return true;
|
|
}
|
|
|
|
string file = Path.Combine (files_folder, Path.GetFileNameWithoutExtension (filename) + ".exe");
|
|
|
|
// Enable .dll only tests (no execution required)
|
|
if (!File.Exists(file)) {
|
|
HandleFailure (filename, TestResult.Success, null);
|
|
return true;
|
|
}
|
|
|
|
return ExecuteTestFile (test, file);
|
|
}
|
|
|
|
protected virtual bool ExecuteTestFile (TestCase test, string binaryFileName)
|
|
{
|
|
string filename = test.FileName;
|
|
|
|
AppDomain domain = null;
|
|
#if !MOBILE
|
|
if (safe_execution) {
|
|
// Create a new AppDomain, with the current directory as the base.
|
|
AppDomainSetup setupInfo = new AppDomainSetup ();
|
|
setupInfo.ApplicationBase = AppDomain.CurrentDomain.BaseDirectory;
|
|
setupInfo.LoaderOptimization = LoaderOptimization.SingleDomain;
|
|
domain = AppDomain.CreateDomain (Path.GetFileNameWithoutExtension (binaryFileName), null, setupInfo);
|
|
}
|
|
#endif
|
|
try {
|
|
DomainTester tester;
|
|
try {
|
|
#if !MOBILE
|
|
if (domain != null)
|
|
tester = (DomainTester) domain.CreateInstanceAndUnwrap (typeof (PositiveChecker).Assembly.FullName, typeof (DomainTester).FullName);
|
|
else
|
|
#endif
|
|
tester = new DomainTester ();
|
|
|
|
if (!tester.Test (binaryFileName))
|
|
return false;
|
|
|
|
} catch (ApplicationException e) {
|
|
HandleFailure (filename, TestResult.ExecError, e.Message);
|
|
return false;
|
|
} catch (Exception e) {
|
|
HandleFailure (filename, TestResult.LoadError, e.ToString ());
|
|
return false;
|
|
}
|
|
|
|
if (doc_output != null) {
|
|
string ref_file = filename.Replace (".cs", "-ref.xml");
|
|
try {
|
|
#if !MOBILE
|
|
new XmlComparer ("doc").Compare (ref_file, doc_output);
|
|
#endif
|
|
} catch (Exception e) {
|
|
HandleFailure (filename, TestResult.XmlError, e.Message);
|
|
return false;
|
|
}
|
|
} else {
|
|
if (verif_file != null) {
|
|
PositiveTestCase pt = (PositiveTestCase) test;
|
|
pt.VerificationProvider = (PositiveTestCase.VerificationData) verif_data[filename];
|
|
|
|
if (!tester.CheckILSize (pt, this, binaryFileName))
|
|
return false;
|
|
}
|
|
|
|
if (filename.StartsWith ("test-debug", StringComparison.OrdinalIgnoreCase)) {
|
|
var mdb_file_name = binaryFileName + ".mdb";
|
|
MonoSymbolFile mdb_file = MonoSymbolFile.ReadSymbolFile (mdb_file_name);
|
|
var mdb_xml_file = mdb_file_name + ".xml";
|
|
ConvertSymbolFileToXml (mdb_file, mdb_xml_file);
|
|
|
|
var ref_file = filename.Replace(".cs", "-ref.xml");
|
|
try {
|
|
new XmlComparer ("symbols").Compare (ref_file, mdb_xml_file);
|
|
} catch (Exception e) {
|
|
HandleFailure (filename, TestResult.DebugError, e.Message);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
}
|
|
} finally {
|
|
if (domain != null)
|
|
AppDomain.Unload (domain);
|
|
}
|
|
|
|
HandleFailure (filename, TestResult.Success, null);
|
|
return true;
|
|
}
|
|
|
|
static void ConvertSymbolFileToXml (MonoSymbolFile symbolFile, string xmlFile)
|
|
{
|
|
using (XmlTextWriter writer = new XmlTextWriter (xmlFile, Encoding.UTF8)) {
|
|
writer.Formatting = Formatting.Indented;
|
|
|
|
writer.WriteStartDocument ();
|
|
|
|
writer.WriteStartElement ("symbols");
|
|
|
|
writer.WriteStartElement ("files");
|
|
foreach (var file in symbolFile.Sources) {
|
|
writer.WriteStartElement ("file");
|
|
writer.WriteAttributeString ("id", file.Index.ToString ());
|
|
writer.WriteAttributeString ("name", Path.GetFileName (file.FileName));
|
|
var checksum = file.Checksum;
|
|
if (checksum != null)
|
|
writer.WriteAttributeString ("checksum", ChecksumToString (checksum));
|
|
|
|
writer.WriteEndElement ();
|
|
}
|
|
writer.WriteEndElement ();
|
|
|
|
writer.WriteStartElement ("methods");
|
|
foreach (var method in symbolFile.Methods) {
|
|
writer.WriteStartElement ("method");
|
|
writer.WriteAttributeString ("token", IntToHex (method.Token));
|
|
|
|
var il_entries = method.GetLineNumberTable ();
|
|
writer.WriteStartElement ("sequencepoints");
|
|
foreach (var entry in il_entries.LineNumbers) {
|
|
writer.WriteStartElement ("entry");
|
|
writer.WriteAttributeString ("il", IntToHex (entry.Offset));
|
|
writer.WriteAttributeString ("row", entry.Row.ToString ());
|
|
writer.WriteAttributeString ("col", entry.Column.ToString ());
|
|
writer.WriteAttributeString ("file_ref", entry.File.ToString ());
|
|
writer.WriteAttributeString ("hidden", BoolToString (entry.IsHidden));
|
|
writer.WriteEndElement ();
|
|
}
|
|
writer.WriteEndElement ();
|
|
|
|
writer.WriteStartElement ("locals");
|
|
foreach (var local in method.GetLocals ()) {
|
|
writer.WriteStartElement ("entry");
|
|
writer.WriteAttributeString ("name", local.Name);
|
|
writer.WriteAttributeString ("il_index", local.Index.ToString ());
|
|
writer.WriteAttributeString ("scope_ref", local.BlockIndex.ToString ());
|
|
writer.WriteEndElement ();
|
|
}
|
|
writer.WriteEndElement ();
|
|
|
|
writer.WriteStartElement ("scopes");
|
|
foreach (var scope in method.GetCodeBlocks ()) {
|
|
writer.WriteStartElement ("entry");
|
|
writer.WriteAttributeString ("index", scope.Index.ToString ());
|
|
writer.WriteAttributeString ("start", IntToHex (scope.StartOffset));
|
|
writer.WriteAttributeString ("end", IntToHex (scope.EndOffset));
|
|
writer.WriteEndElement ();
|
|
}
|
|
writer.WriteEndElement ();
|
|
|
|
writer.WriteEndElement ();
|
|
}
|
|
writer.WriteEndElement ();
|
|
|
|
writer.WriteEndElement ();
|
|
writer.WriteEndDocument ();
|
|
}
|
|
}
|
|
|
|
static string ChecksumToString (byte[] checksum)
|
|
{
|
|
var sb = new StringBuilder (checksum.Length * 2);
|
|
for (int i = 0; i < checksum.Length; i++) {
|
|
sb.Append ("0123456789abcdef"[checksum[i] >> 4]);
|
|
sb.Append ("0123456789abcdef"[checksum[i] & 0x0F]);
|
|
}
|
|
|
|
return sb.ToString ();
|
|
}
|
|
|
|
static string IntToHex (int value)
|
|
{
|
|
return "0x" + value.ToString ("x", CultureInfo.InvariantCulture);
|
|
}
|
|
|
|
static string BoolToString (bool value)
|
|
{
|
|
return value ? "true" : "false";
|
|
}
|
|
|
|
protected override TestCase CreateTestCase (string filename, string [] options, string [] deps)
|
|
{
|
|
return new PositiveTestCase (filename, options, deps);
|
|
}
|
|
|
|
public void HandleFailure (string file, TestResult status, string extra)
|
|
{
|
|
switch (status) {
|
|
case TestResult.Success:
|
|
succeeded.Add (file);
|
|
if (know_issues.Contains (file)) {
|
|
LogFileLine (file, "FIXED ISSUE");
|
|
return;
|
|
}
|
|
if (verbose)
|
|
LogFileLine (file, "OK");
|
|
return;
|
|
|
|
case TestResult.CompileError:
|
|
if (know_issues.Contains (file)) {
|
|
LogFileLine (file, "KNOWN ISSUE (Compilation error)");
|
|
know_issues.Remove (file);
|
|
return;
|
|
}
|
|
LogFileLine (file, "REGRESSION (SUCCESS -> COMPILATION ERROR)");
|
|
break;
|
|
|
|
case TestResult.ExecError:
|
|
if (know_issues.Contains (file)) {
|
|
LogFileLine (file, "KNOWN ISSUE (Execution error)");
|
|
know_issues.Remove (file);
|
|
return;
|
|
}
|
|
LogFileLine (file, "REGRESSION (SUCCESS -> EXECUTION ERROR)");
|
|
break;
|
|
|
|
case TestResult.XmlError:
|
|
if (know_issues.Contains (file)) {
|
|
LogFileLine (file, "KNOWN ISSUE (Xml comparision error)");
|
|
know_issues.Remove (file);
|
|
return;
|
|
}
|
|
LogFileLine (file, "REGRESSION (SUCCESS -> DOCUMENTATION ERROR)");
|
|
break;
|
|
|
|
case TestResult.LoadError:
|
|
if (extra != null)
|
|
extra = ": " + extra;
|
|
|
|
LogFileLine (file, "REGRESSION (SUCCESS -> LOAD ERROR)" + extra);
|
|
extra = null;
|
|
break;
|
|
|
|
case TestResult.MethodAttributesError:
|
|
case TestResult.ILError:
|
|
if (!update_verif_file) {
|
|
LogFileLine (file, "IL REGRESSION: " + extra);
|
|
}
|
|
extra = null;
|
|
break;
|
|
|
|
case TestResult.DebugError:
|
|
LogFileLine (file, "REGRESSION (SUCCESS -> SYMBOL FILE ERROR)");
|
|
break;
|
|
}
|
|
|
|
if (extra != null)
|
|
LogLineForFile (file, "{0}", extra);
|
|
|
|
if (!regression.Contains (file))
|
|
regression.Add (file);
|
|
}
|
|
|
|
public override void Initialize ()
|
|
{
|
|
if (verif_file != null) {
|
|
LoadVerificationData (verif_file);
|
|
}
|
|
|
|
base.Initialize ();
|
|
}
|
|
|
|
public override void CleanUp ()
|
|
{
|
|
base.CleanUp ();
|
|
|
|
if (update_verif_file) {
|
|
UpdateVerificationData (verif_file);
|
|
}
|
|
}
|
|
|
|
void LoadVerificationData (string file)
|
|
{
|
|
verif_data = new Hashtable ();
|
|
|
|
if (!File.Exists (file)) {
|
|
LogLine ("Writing verification data to `{0}' ...", file);
|
|
return;
|
|
}
|
|
|
|
LogLine ("Loading verification data from `{0}' ...", file);
|
|
|
|
using (XmlReader r = XmlReader.Create (file)) {
|
|
r.ReadStartElement ("tests");
|
|
|
|
while (r.Read ()) {
|
|
if (r.Name != "test")
|
|
continue;
|
|
|
|
string name = r.GetAttribute ("name");
|
|
PositiveTestCase.VerificationData tc = PositiveTestCase.VerificationData.FromFile (name, r);
|
|
verif_data.Add (name, tc);
|
|
}
|
|
}
|
|
}
|
|
|
|
void UpdateVerificationData (string file)
|
|
{
|
|
LogLine ("Updating verification data `{0}' ...", file);
|
|
|
|
XmlWriterSettings s = new XmlWriterSettings ();
|
|
s.Indent = true;
|
|
using (XmlWriter w = XmlWriter.Create (new StreamWriter (file, false, Encoding.UTF8), s)) {
|
|
w.WriteStartDocument ();
|
|
w.WriteComment ("This file contains expected IL and metadata produced by compiler for each test");
|
|
w.WriteStartElement ("tests");
|
|
foreach (PositiveTestCase tc in tests) {
|
|
if (tc.VerificationProvider != null)
|
|
tc.VerificationProvider.WriteCodeInfoTo (w);
|
|
}
|
|
w.WriteEndElement ();
|
|
}
|
|
}
|
|
}
|
|
|
|
class NegativeChecker: Checker
|
|
{
|
|
string expected_message;
|
|
string error_message;
|
|
bool check_msg;
|
|
bool check_error_line;
|
|
bool is_warning;
|
|
IDictionary wrong_warning;
|
|
|
|
protected enum CompilerError {
|
|
Expected,
|
|
Wrong,
|
|
Missing,
|
|
WrongMessage,
|
|
MissingLocation,
|
|
Duplicate
|
|
}
|
|
|
|
public NegativeChecker (ITester tester, bool check_msg):
|
|
base (tester)
|
|
{
|
|
this.check_msg = check_msg;
|
|
wrong_warning = new Hashtable ();
|
|
}
|
|
|
|
protected override bool AnalyzeTestFile (string file, ref int row, string line,
|
|
ref string[] compiler_options,
|
|
ref string[] dependencies)
|
|
{
|
|
if (row == 1) {
|
|
expected_message = null;
|
|
|
|
int index = line.IndexOf (':');
|
|
if (index == -1 || index > 15) {
|
|
LogFileLine (file, "IGNORING: Wrong test file syntax (missing error mesage text)");
|
|
++syntax_errors;
|
|
base.AnalyzeTestFile (file, ref row, line, ref compiler_options,
|
|
ref dependencies);
|
|
return false;
|
|
}
|
|
|
|
expected_message = line.Substring (index + 1).Trim ();
|
|
}
|
|
|
|
if (row == 2) {
|
|
string filtered = line.Replace(" ", "");
|
|
|
|
// Some error tests require to have different error text for different runtimes.
|
|
if (filtered.StartsWith ("//GMCS")) {
|
|
row = 1;
|
|
return AnalyzeTestFile(file, ref row, line, ref compiler_options, ref dependencies);
|
|
}
|
|
|
|
check_error_line = !filtered.StartsWith ("//Line:0");
|
|
|
|
if (!filtered.StartsWith ("//Line:")) {
|
|
LogFileLine (file, "IGNORING: Wrong test syntax (following line after an error messsage must have `// Line: xx' syntax");
|
|
++syntax_errors;
|
|
return false;
|
|
}
|
|
}
|
|
|
|
if (!base.AnalyzeTestFile (file, ref row, line, ref compiler_options, ref dependencies))
|
|
return false;
|
|
|
|
is_warning = false;
|
|
if (compiler_options != null) {
|
|
foreach (string s in compiler_options) {
|
|
if (s.StartsWith ("-warnaserror") || s.StartsWith ("/warnaserror"))
|
|
is_warning = true;
|
|
}
|
|
}
|
|
return true;
|
|
}
|
|
|
|
|
|
protected override bool Check (TestCase test)
|
|
{
|
|
string filename = test.FileName;
|
|
|
|
int start_char = 0;
|
|
while (Char.IsLetter (filename, start_char))
|
|
++start_char;
|
|
|
|
int end_char = filename.IndexOfAny (new char [] { '-', '.' } );
|
|
string expected = filename.Substring (start_char, end_char - start_char);
|
|
|
|
try {
|
|
if (base.Check (test)) {
|
|
HandleFailure (filename, CompilerError.Missing);
|
|
return false;
|
|
}
|
|
}
|
|
catch (Exception e) {
|
|
HandleFailure (filename, CompilerError.Missing);
|
|
if (e.InnerException != null)
|
|
e = e.InnerException;
|
|
|
|
Log (e.ToString ());
|
|
return false;
|
|
}
|
|
|
|
int err_id = int.Parse (expected, System.Globalization.CultureInfo.InvariantCulture);
|
|
if (tester.IsWarning (err_id)) {
|
|
if (!is_warning)
|
|
wrong_warning [err_id] = true;
|
|
} else {
|
|
if (is_warning)
|
|
wrong_warning [err_id] = false;
|
|
}
|
|
|
|
CompilerError result_code = GetCompilerError (expected, tester.Output);
|
|
if (HandleFailure (filename, result_code)) {
|
|
succeeded.Add (filename);
|
|
return true;
|
|
}
|
|
|
|
if (result_code == CompilerError.Wrong)
|
|
LogLineForFile (filename, tester.Output);
|
|
|
|
return false;
|
|
}
|
|
|
|
CompilerError GetCompilerError (string expected, string buffer)
|
|
{
|
|
const string error_prefix = "CS";
|
|
const string ignored_error = "error CS5001";
|
|
string tested_text = "error " + error_prefix + expected;
|
|
StringReader sr = new StringReader (buffer);
|
|
string line = sr.ReadLine ();
|
|
ArrayList ld = new ArrayList ();
|
|
CompilerError result = CompilerError.Missing;
|
|
while (line != null) {
|
|
if (ld.Contains (line) && result == CompilerError.Expected) {
|
|
if (line.IndexOf ("Location of the symbol related to previous") == -1)
|
|
return CompilerError.Duplicate;
|
|
}
|
|
ld.Add (line);
|
|
|
|
if (result != CompilerError.Expected) {
|
|
if (line.IndexOf (tested_text) != -1) {
|
|
if (check_msg) {
|
|
int first = line.IndexOf (':');
|
|
int second = line.IndexOf (':', first + 1);
|
|
if (line.IndexOf ("Warning as Error: ", first, StringComparison.Ordinal) > 0) {
|
|
if (check_error_line) {
|
|
second = line.IndexOf (':', second + 1);
|
|
}
|
|
} else if (second == -1 || !check_error_line) {
|
|
second = first;
|
|
}
|
|
|
|
string msg = line.Substring (second + 1).TrimEnd ('.').Trim ();
|
|
if (msg != expected_message && !TryToMatchErrorMessage (msg, expected_message)) {
|
|
error_message = msg;
|
|
return CompilerError.WrongMessage;
|
|
}
|
|
|
|
if (check_error_line && line.IndexOf (".cs(") == -1)
|
|
return CompilerError.MissingLocation;
|
|
}
|
|
result = CompilerError.Expected;
|
|
} else if (line.IndexOf (error_prefix) != -1 &&
|
|
line.IndexOf (ignored_error) == -1)
|
|
result = CompilerError.Wrong;
|
|
}
|
|
|
|
line = sr.ReadLine ();
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
static bool TryToMatchErrorMessage (string actual, string expected)
|
|
{
|
|
actual = actual.Replace ("\\", "/");
|
|
var path_mask_start = expected.IndexOf ("*PATH*", StringComparison.Ordinal);
|
|
if (path_mask_start > 0 && actual.Length > path_mask_start) {
|
|
var parts = expected.Split (new [] { "*PATH*" }, StringSplitOptions.None);
|
|
foreach (var part in parts) {
|
|
if (!actual.Contains (part))
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
bool HandleFailure (string file, CompilerError status)
|
|
{
|
|
switch (status) {
|
|
case CompilerError.Expected:
|
|
if (know_issues.Contains (file) || no_error_list.Contains (file)) {
|
|
LogFileLine (file, "FIXED ISSUE");
|
|
return true;
|
|
}
|
|
|
|
if (verbose)
|
|
LogFileLine (file, "OK");
|
|
return true;
|
|
|
|
case CompilerError.Wrong:
|
|
if (know_issues.Contains (file)) {
|
|
LogFileLine (file, "KNOWN ISSUE (Wrong error reported)");
|
|
know_issues.Remove (file);
|
|
return false;
|
|
}
|
|
if (no_error_list.Contains (file)) {
|
|
LogFileLine (file, "REGRESSION (NO ERROR -> WRONG ERROR CODE)");
|
|
no_error_list.Remove (file);
|
|
}
|
|
else {
|
|
LogFileLine (file, "REGRESSION (CORRECT ERROR -> WRONG ERROR CODE)");
|
|
}
|
|
break;
|
|
|
|
case CompilerError.WrongMessage:
|
|
if (know_issues.Contains (file)) {
|
|
LogFileLine (file, "KNOWN ISSUE (Wrong error message reported)");
|
|
know_issues.Remove (file);
|
|
return false;
|
|
}
|
|
if (no_error_list.Contains (file)) {
|
|
LogFileLine (file, "REGRESSION (NO ERROR -> WRONG ERROR MESSAGE)");
|
|
no_error_list.Remove (file);
|
|
}
|
|
else {
|
|
LogFileLine (file, "REGRESSION (CORRECT ERROR -> WRONG ERROR MESSAGE)");
|
|
LogLineForFile (file, "Exp: {0}", expected_message);
|
|
LogLineForFile (file, "Was: {0}", error_message);
|
|
}
|
|
break;
|
|
|
|
case CompilerError.Missing:
|
|
if (no_error_list.Contains (file)) {
|
|
LogFileLine (file, "KNOWN ISSUE (No error reported)");
|
|
no_error_list.Remove (file);
|
|
return false;
|
|
}
|
|
|
|
if (know_issues.Contains (file)) {
|
|
LogFileLine (file, "REGRESSION (WRONG ERROR -> NO ERROR)");
|
|
know_issues.Remove (file);
|
|
}
|
|
else {
|
|
LogFileLine (file, "REGRESSION (CORRECT ERROR -> NO ERROR)");
|
|
}
|
|
|
|
break;
|
|
|
|
case CompilerError.MissingLocation:
|
|
if (know_issues.Contains (file)) {
|
|
LogFileLine (file, "KNOWN ISSUE (Missing error location)");
|
|
know_issues.Remove (file);
|
|
return false;
|
|
}
|
|
if (no_error_list.Contains (file)) {
|
|
LogFileLine (file, "REGRESSION (NO ERROR -> MISSING ERROR LOCATION)");
|
|
no_error_list.Remove (file);
|
|
}
|
|
else {
|
|
LogFileLine (file, "REGRESSION (CORRECT ERROR -> MISSING ERROR LOCATION)");
|
|
}
|
|
break;
|
|
|
|
case CompilerError.Duplicate:
|
|
// Will become an error soon
|
|
LogFileLine (file, "WARNING: EXACTLY SAME ERROR HAS BEEN ISSUED MULTIPLE TIMES");
|
|
return true;
|
|
}
|
|
|
|
regression.Add (file);
|
|
return false;
|
|
}
|
|
|
|
protected override void PrintSummary()
|
|
{
|
|
base.PrintSummary ();
|
|
|
|
if (wrong_warning.Count > 0) {
|
|
LogLine ("");
|
|
LogLine ("List of incorectly defined warnings (they should be either defined in the compiler as a warning or a test-case has redundant `warnaserror' option)");
|
|
LogLine ("");
|
|
foreach (DictionaryEntry de in wrong_warning)
|
|
LogLine ("CS{0:0000} : {1}", de.Key, (bool)de.Value ? "incorrect warning definition" : "missing warning definition");
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
class Tester {
|
|
|
|
static int Main(string[] args)
|
|
{
|
|
string temp;
|
|
|
|
if (GetOption ("help", args, false, out temp)) {
|
|
Usage ();
|
|
return 0;
|
|
}
|
|
|
|
string compiler;
|
|
if (!GetOption ("compiler", args, true, out compiler)) {
|
|
Usage ();
|
|
return 1;
|
|
}
|
|
|
|
compiler = Path.GetFullPath (compiler);
|
|
|
|
ITester tester;
|
|
try {
|
|
Console.WriteLine ("Loading " + compiler + " ...");
|
|
tester = new ReflectionTester (Assembly.LoadFile (compiler));
|
|
} catch (Exception) {
|
|
Console.Error.WriteLine ("Switching to command line mode (compiler entry point was not found)");
|
|
if (!File.Exists (compiler)) {
|
|
Console.Error.WriteLine ("ERROR: Tested compiler was not found");
|
|
return 1;
|
|
}
|
|
tester = new ProcessTester (compiler);
|
|
}
|
|
|
|
string mode;
|
|
if (!GetOption ("mode", args, true, out mode)) {
|
|
Usage ();
|
|
return 1;
|
|
}
|
|
|
|
Checker checker;
|
|
bool positive;
|
|
switch (mode) {
|
|
case "neg":
|
|
checker = new NegativeChecker (tester, true);
|
|
positive = false;
|
|
break;
|
|
case "pos":
|
|
string iltest;
|
|
GetOption ("il", args, false, out iltest);
|
|
checker = new PositiveChecker (tester, iltest);
|
|
positive = true;
|
|
|
|
if (iltest != null && GetOption ("update-il", args, false, out temp)) {
|
|
((PositiveChecker) checker).UpdateVerificationDataFile = true;
|
|
}
|
|
|
|
break;
|
|
case "nunit":
|
|
positive = true;
|
|
checker = new NUnitChecker (tester);
|
|
break;
|
|
default:
|
|
Console.Error.WriteLine ("Invalid -mode argument");
|
|
return 1;
|
|
}
|
|
|
|
|
|
if (GetOption ("issues", args, true, out temp))
|
|
checker.IssueFile = temp;
|
|
if (GetOption ("log", args, true, out temp))
|
|
checker.LogFile = temp;
|
|
if (GetOption ("resultXml", args, true, out temp))
|
|
checker.ResultXml = temp;
|
|
if (GetOption ("verbose", args, false, out temp))
|
|
checker.Verbose = true;
|
|
if (GetOption ("safe-execution", args, false, out temp))
|
|
checker.SafeExecution = true;
|
|
if (GetOption ("reference-dir", args, true, out temp))
|
|
checker.ReferenceDirectory = temp;
|
|
if (GetOption ("compiler-options", args, true, out temp)) {
|
|
string[] extra = temp.Split (' ');
|
|
checker.ExtraCompilerOptions = extra;
|
|
}
|
|
|
|
string test_pattern;
|
|
if (!GetOption ("files", args, true, out test_pattern)) {
|
|
Usage ();
|
|
return 1;
|
|
}
|
|
|
|
var files = new List<FileInfo> ();
|
|
var test_directory = new DirectoryInfo (".");
|
|
switch (test_pattern) {
|
|
case "v1":
|
|
files.AddRange (test_directory.EnumerateFiles (positive ? "test*.cs" : "cs*.cs"));
|
|
break;
|
|
case "v2":
|
|
files.AddRange (test_directory.EnumerateFiles (positive ? "gtest*.cs" : "gcs*.cs"));
|
|
goto case "v1";
|
|
case "v4":
|
|
files.AddRange (test_directory.EnumerateFiles (positive ? "dtest*.cs" : "dcs*.cs"));
|
|
goto case "v2";
|
|
default:
|
|
files.AddRange (test_directory.EnumerateFiles (test_pattern));
|
|
break;
|
|
}
|
|
|
|
if (files.Count == 0) {
|
|
Console.Error.WriteLine ("No files matching `{0}' found", test_pattern);
|
|
return 2;
|
|
}
|
|
|
|
checker.Name = test_directory.Name;
|
|
checker.Initialize ();
|
|
|
|
files.Sort ((a, b) => {
|
|
if (a.Name.EndsWith ("-lib.cs", StringComparison.Ordinal)) {
|
|
if (!b.Name.EndsWith ("-lib.cs", StringComparison.Ordinal))
|
|
return -1;
|
|
} else if (b.Name.EndsWith ("-lib.cs", StringComparison.Ordinal)) {
|
|
if (!a.Name.EndsWith ("-lib.cs", StringComparison.Ordinal))
|
|
return 1;
|
|
}
|
|
|
|
if (a.Name.EndsWith ("-mod.cs", StringComparison.Ordinal)) {
|
|
if (!b.Name.EndsWith ("-mod.cs", StringComparison.Ordinal))
|
|
return -1;
|
|
} else if (b.Name.EndsWith ("-mod.cs", StringComparison.Ordinal)) {
|
|
if (!a.Name.EndsWith ("-mod.cs", StringComparison.Ordinal))
|
|
return 1;
|
|
}
|
|
|
|
return a.Name.CompareTo (b.Name);
|
|
});
|
|
|
|
foreach (FileInfo s in files) {
|
|
string filename = s.Name;
|
|
if (Char.IsUpper (filename, 0)) { // Windows hack
|
|
continue;
|
|
}
|
|
|
|
if (filename.EndsWith ("-p2.cs"))
|
|
continue;
|
|
|
|
checker.Do (filename);
|
|
}
|
|
|
|
checker.CleanUp ();
|
|
|
|
checker.Dispose ();
|
|
|
|
return checker.ResultCode;
|
|
}
|
|
|
|
static bool GetOption (string opt, string[] args, bool req_arg, out string value)
|
|
{
|
|
opt = "-" + opt;
|
|
foreach (string a in args) {
|
|
if (a.StartsWith (opt)) {
|
|
int sep = a.IndexOf (':');
|
|
if (sep > 0) {
|
|
value = a.Substring (sep + 1);
|
|
} else {
|
|
value = null;
|
|
if (req_arg) {
|
|
Console.Error.WriteLine ("Missing argument in option " + opt);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
return true;
|
|
}
|
|
}
|
|
|
|
value = null;
|
|
return false;
|
|
}
|
|
|
|
static void Usage ()
|
|
{
|
|
Console.WriteLine (
|
|
"Mono compiler tester, (C) 2009 Novell, Inc.\n" +
|
|
"compiler-tester -mode:[pos|neg] -compiler:FILE -files:file-list [options]\n" +
|
|
" \n" +
|
|
" -compiler:FILE The file which will be used to compiler tests\n" +
|
|
" -compiler-options:OPTIONS Add global compiler options\n" +
|
|
" -reference-dir:DIRECTORY Use this directory for $REF_DIR variable in tests\n" +
|
|
" -il:IL-FILE XML file with expected IL details for each test\n" +
|
|
" -issues:FILE The list of expected failures\n" +
|
|
" -log:FILE Writes any output also to the file\n" +
|
|
" -resultXml:FILE Writes test result data in xUnit.net v2 format to the file\n" +
|
|
" -help Lists all options\n" +
|
|
" -mode:[pos|neg] Specifies compiler test mode\n" +
|
|
" -safe-execution Runs compiled executables in separate app-domain\n" +
|
|
" -update-il Updates IL-FILE to match compiler output\n" +
|
|
" -verbose Prints more details during testing\n"
|
|
);
|
|
}
|
|
}
|
|
}
|