UnrealEngineUWP/Engine/Source/Programs/AutomationTool/Gauntlet/Framework/Devices/Gauntlet.DevicePool.cs

// Copyright 1998-2019 Epic Games, Inc. All Rights Reserved.

using System;
using System.Collections.Generic;
using System.IO;
using AutomationTool;
using AutomationTool.DeviceReservation;
using UnrealBuildTool;
using System.Threading;
using System.Text.RegularExpressions;
using System.Linq;
using Newtonsoft.Json;

namespace Gauntlet
{

	/// <summary>
	/// Device performance specification
	/// </summary>
	public enum EPerfSpec
	{
		Unspecified,
		Minimum,
		Recommended,
		High
	};

	/// <summary>
	/// Information that defines a device
	/// </summary>
	public class DeviceDefinition
	{
		public string Name;

		public string Address;

		public string DeviceData;

		// legacy - remove!
		public UnrealTargetPlatform Type;

		public UnrealTargetPlatform Platform;

		public EPerfSpec PerfSpec;

		public string Available;
		
		public bool RemoveOnShutdown;
		
		public override string ToString()
		{
			return string.Format("{0} @ {1}. Platform={2}", Name, Address, Platform);
		}
	}

	/// <summary>
	/// Device target constraint, can be expanded for specifying installed RAM, OS version, etc
	/// </summary>
	public class UnrealTargetConstraint : IEquatable<UnrealTargetConstraint>
	{
		public UnrealTargetPlatform Platform;
		public EPerfSpec PerfSpec;

		public UnrealTargetConstraint(UnrealTargetPlatform Platform, EPerfSpec PerfSpec = EPerfSpec.Unspecified)
		{
			this.Platform = Platform;
			this.PerfSpec = PerfSpec;
		}

		/// <summary>
		/// Tests whether the constraint is identity, ie. unconstrained
		/// </summary>
		public bool IsIdentity()
		{
			return PerfSpec == EPerfSpec.Unspecified;
		}


		/// <summary>
		/// Check whether device satisfies the constraint
		/// </summary>
		public bool Check(ITargetDevice Device)
		{
			return Platform == Device.Platform && (IsIdentity() || this == DevicePool.Instance.GetConstraint(Device));
		}


		public bool Equals(UnrealTargetConstraint Other)
		{
			if (ReferenceEquals(Other, null))
			{
				throw new AutomationException("Comparing null target constraint");
			}

			if (ReferenceEquals(this, Other)) return true;
			return Other.Platform == Platform && Other.PerfSpec == PerfSpec;
		}

		public override bool Equals(object Obj)
		{
			if (ReferenceEquals(Obj, null))
			{
				throw new AutomationException("Comparing null target constraint");
			}

			if (ReferenceEquals(this, Obj)) return true;
			if (Obj.GetType() != typeof(UnrealTargetConstraint)) return false;
			return Equals((UnrealTargetConstraint)Obj);
		}

		public static bool operator ==(UnrealTargetConstraint C1, UnrealTargetConstraint C2)
		{
			if (ReferenceEquals(C1, null) || ReferenceEquals(C2, null))
			{
				throw new AutomationException("Comparing null target constraint");
			}

			return C1.Equals(C2);
		}

		public static bool operator !=(UnrealTargetConstraint C1, UnrealTargetConstraint C2)
		{
			return !(C1 == C2);
		}

		public override string ToString()
		{
			return string.Format("{0}:{1}", Platform, PerfSpec);
		}

		public override int GetHashCode()
		{
			return ToString().GetHashCode();
		}
	}

	/// <summary>
	/// Device marked as having a problem
	/// </summary>
	internal struct ProblemDevice
	{
		public ProblemDevice(string Name, UnrealTargetPlatform Platform)
		{
			this.Name = Name;
			this.Platform = Platform;
		}

		public string Name;
		public UnrealTargetPlatform Platform;
	}

	/// <summary>
	/// Singleton class that's responsible for providing a list of devices and reserving them. Code should call
	/// EnumerateDevices to build a list of desired devices, which must then be reserved by calling ReserveDevices.
	/// Once done ReleaseDevices should be called.
	/// 
	/// These reservations exist at the process level and we rely on the device implementation to provide arbitrage 
	/// between difference processes and machines
	/// 
	/// </summary>
	public class DevicePool : IDisposable
	{

		/// <summary>
		/// Object used for locking access to internal data
		/// </summary>
		private Object LockObject = new Object();

		/// <summary>
		/// List of all provisioned devices that are available for reservations
		/// </summary>
		List<ITargetDevice> AvailableDevices = new List<ITargetDevice>();

		/// <summary>
		/// List of all devices that have been reserved
		/// </summary>
		List<ITargetDevice> ReservedDevices = new List<ITargetDevice>();

		/// <summary>
		/// Device reservation service URL 
		/// </summary>
		public string DeviceURL;

		/// <summary>
		/// Device to reservation lookup
		/// </summary>
		Dictionary<ITargetDevice, DeviceReservationAutoRenew> ServiceReservations = new Dictionary<ITargetDevice, DeviceReservationAutoRenew>();


		/// <summary>
		/// Target device info, private for reservation use
		/// </summary>
		Dictionary<ITargetDevice, DeviceDefinition> ServiceDeviceInfo = new Dictionary<ITargetDevice, DeviceDefinition>();

		/// <summary>
		/// List of device definitions that can be provisioned on demand
		/// </summary>
		List<DeviceDefinition> UnprovisionedDevices = new List<DeviceDefinition>();

		/// <summary>
		/// List of definitions that failed to provision
		/// </summary>
		List<DeviceDefinition> FailedProvisions = new List<DeviceDefinition>();

		/// <summary>
		/// Local connection state for any device. We use this to avoid disconnecting devices that the
		/// user may already be connected to when running tests
		/// </summary>
		Dictionary<ITargetDevice, bool> InitialConnectState = new Dictionary<ITargetDevice, bool>();

		/// <summary>
		/// Device constraints for performance profiles, etc
		/// </summary>
		Dictionary<ITargetDevice, UnrealTargetConstraint> Constraints = new Dictionary<ITargetDevice, UnrealTargetConstraint>();

		/// <summary>
		/// Local tempdir...
		/// </summary>
		string LocalTempDir;


		bool UniqueLocalTemps;

		/// <summary>
		/// Static instance
		/// </summary>
		private static DevicePool _Instance;

		/// <summary>
		/// The maximum number of problem devices to report to device backend
		/// This mitigates issues on builders, such as hung local processes, incorrectly reporting problem devices
		/// </summary>
		private int MaxDeviceErrorReports = 3;

		/// <summary>
		/// Protected constructor - code should use DevicePool.Instance
		/// </summary>
		protected DevicePool()
		{
			if (_Instance == null)
			{
				_Instance = this;
			}

			// create two local devices by default?
			AddLocalDevices(2);
		}

		~DevicePool()
		{
			Dispose(false);
		}

		/// <summary>
		/// Access to our singleton
		/// </summary>
		public static DevicePool Instance
		{
			get
			{
				if (_Instance == null)
				{
					new DevicePool();
				}
				return _Instance;
			}
		}

		/// <summary>
		/// Shutdown the pool and release all devices.
		/// </summary>
		public static void Shutdown()
		{
			if (_Instance != null)
			{
				_Instance.Dispose();
				_Instance = null;
			}
		}

		public void Dispose()
		{
			Dispose(true);
			GC.SuppressFinalize(this);
		}

		/// <summary>
		/// Perform actual dispose behavior
		/// </summary>
		/// <param name="disposing"></param>
		protected virtual void Dispose(bool disposing)
		{
			lock (LockObject)
			{
				if (disposing)
				{
					// warn if more than a third of the devices in our pool had issues.
					int TotalDevices = AvailableDevices.Count + UnprovisionedDevices.Count;

					if (TotalDevices > 0)
					{
						float Failures = FailedProvisions.Count / TotalDevices;

						if (Failures >= 0.33)
						{
							FailedProvisions.ForEach(D => Log.Warning("Device {0} could not be added", D));
						}
					}

					// release any outstanding reservations
					ReleaseReservations();

					// warn if anything was left registered
					ReservedDevices.ForEach(D =>
					{
						Log.Warning("Device {0} was not unregistered prior to DevicePool.Dispose!. Forcibly disposing.", D.Name);
						D.Dispose();
					});

					ReleaseDevices(ReservedDevices);

					// Cleanup our available devices
					foreach (ITargetDevice Device in AvailableDevices)
					{
						if (Device.IsConnected && InitialConnectState[Device] == false)
						{
							Device.Disconnect();
						}

						Device.Dispose();
					}

					AvailableDevices.Clear();
					UnprovisionedDevices.Clear();
					ReservedDevices.Clear();
				}
			}
		}

		/// <summary>
		/// Explicitly release all device reservations
		/// </summary>
		private void ReleaseReservations()
		{

			ServiceDeviceInfo.Clear();

			foreach (ITargetDevice Device in ServiceReservations.Keys)
			{
				AvailableDevices.Remove(Device);
				ReservedDevices.Remove(Device);
				Device.Dispose();
			}

			foreach (DeviceReservationAutoRenew reservation in ServiceReservations.Values.Distinct())
			{
				reservation.Dispose();
			}

			ServiceReservations.Clear();

		}

		/// <summary>
		/// Release all devices in the provided list from our reserved list
		/// </summary>
		/// <param name="DeviceList"></param>
		public void ReleaseDevices(IEnumerable<ITargetDevice> DeviceList)
		{
			lock (LockObject)
			{
				// remove all these devices from our reserved list
				ReservedDevices = ReservedDevices.Where(D => DeviceList.Contains(D) == false).ToList();

				// disconnect them
				List<ITargetDevice> KeepList = new List<ITargetDevice>();
				foreach (ITargetDevice Device in DeviceList)
				{
					if (Device.IsConnected && InitialConnectState[Device] == false)
					{
						Device.Disconnect();
					}

					// if device is from reservation service, unregister it
					DeviceReservationAutoRenew Reservation;
					if (ServiceReservations.TryGetValue(Device, out Reservation))
					{
						bool DisposeReservation = ServiceReservations.Count(Entry => Entry.Value == Reservation) == 1;

						// remove and dispose of device
						// @todo: add support for reservation modification on server (partial device release)
						ServiceReservations.Remove(Device);
						ServiceDeviceInfo.Remove(Device);
						AvailableDevices.Remove(Device);

						InitialConnectState.Remove(Device);
						Constraints.Remove(Device);

						Device.Dispose();

						// if last device in reservation, dispose the reservation
						if (DisposeReservation)
						{
							Reservation.Dispose();
						}
					}
					else
					{
						KeepList.Add(Device);
					}

				}

				// Put all devices back into the available list at the front - if another device is requested we want to
				// try and use one we just put a build on
				KeepList.ForEach(D => { AvailableDevices.Remove(D); AvailableDevices.Insert(0, D); });
			}
		}

		/// <summary>
		/// Report target device issue to service with given error message
		/// </summary>
		public void ReportDeviceError(ITargetDevice Device, string ErrorMessage)
		{
			DeviceDefinition Def = null;
			if (!ServiceDeviceInfo.TryGetValue(Device, out Def))
			{
				return;
			}

			ReportDeviceError(Def.Name, ErrorMessage);
		}

		/// <summary>
		/// Reporting device issue using the device name provided by service
		/// </summary>
		private void ReportDeviceError(string ServiceDeviceName, string ErrorMessage)
		{
			if (MaxDeviceErrorReports == 0)
			{
				Log.Warning("Maximum device errors reported to backend, {0} : {1} ignored", ServiceDeviceName, ErrorMessage);
				return;
			}

			MaxDeviceErrorReports--;

			Reservation.ReportDeviceError(DeviceURL, ServiceDeviceName, ErrorMessage);
		}

		/// <summary>
		/// Reserve all devices in the provided list. Once reserved a device will not be seen by any code that
		/// calls EnumerateDevices
		/// </summary>
		/// <param name="DeviceList"></param>
		/// <returns></returns>
		public bool ReserveDevices(IEnumerable<ITargetDevice> DeviceList)
		{
			lock (LockObject)
			{
				// can reserve if not reserved...
				if (ReservedDevices.Intersect(DeviceList).Count() > 0)
				{
					return false;
				}

				// remove these devices from the available list
				AvailableDevices = AvailableDevices.Where(D => DeviceList.Contains(D) == false).ToList();

				ReservedDevices.AddRange(DeviceList);
			}
			return true;
		}

		public void SetLocalOptions(string InLocalTemp, bool InUniqueTemps, string InDeviceURL = "")
		{
			LocalTempDir = InLocalTemp;
			UniqueLocalTemps = InUniqueTemps;
			DeviceURL = InDeviceURL;
		}

		public void AddLocalDevices(int Count)
		{
			// todo - default per platform?

			UnrealTargetPlatform LocalPlat = UnrealTargetPlatform.Win64;

			int NumDevices = GetAvailableDeviceCount(new UnrealTargetConstraint(LocalPlat));

			// add local PCs (todo - some max number?)
			for (int i = NumDevices; i < Count + NumDevices; i++)
			{
				DeviceDefinition Def = new DeviceDefinition();
				Def.Name = string.Format("LocalDevice{0}", i);
				Def.Platform = BuildHostPlatform.Current.Platform;
				UnprovisionedDevices.Add(Def);
			}
		}

		/// <summary>
		/// Reserve devices from service
		/// </summary>
		public bool ReserveDevicesFromService(string DeviceURL, Dictionary<UnrealTargetConstraint, int> DeviceTypes)
		{
			if (String.IsNullOrEmpty(DeviceURL))
			{
				return false;
			}

			Dictionary<UnrealTargetPlatform, string> DeviceMap = new Dictionary<UnrealTargetPlatform, string>()
			{
				// todo: add other platforms and externalize this mapping
				{ UnrealTargetPlatform.PS4 , "PS4-DevKit" },
				{ UnrealTargetPlatform.XboxOne , "XboxOne-DevKit" },
				{ UnrealTargetPlatform.Android , "Android" },
				{ UnrealTargetPlatform.Switch , "Switch" }
		};

			List<string> Devices = new List<string>();

			// convert devices to request list
			foreach (KeyValuePair<UnrealTargetConstraint, int> Entry in DeviceTypes)
			{				
                if (!DeviceMap.ContainsKey(Entry.Key.Platform))
				{
					// if an unsupported device, we can't reserve it
					Log.Error("Unable to reserve service device of type: {0}", Entry.Key);
					return false;
				}

				for (int i = 0; i < Entry.Value; i++)
				{
					// @todo: if any additional reservation requirements, encode constraint into json
					Devices.Add(DeviceMap[Entry.Key.Platform] + ":" + Entry.Key.PerfSpec);
				}				
			}

			// reserve devices
			Uri ReservationServerUri;
			if (Uri.TryCreate(DeviceURL, UriKind.Absolute, out ReservationServerUri))
			{
				DeviceReservationAutoRenew DeviceReservation = null;

				try
				{
					DeviceReservation = new DeviceReservationAutoRenew(DeviceURL, 0, Devices.ToArray());
				}
				catch (Exception Ex)
				{
					Log.Info("Unable to make device registration: {0}", Ex.Message);
					return false;
				}

				if (DeviceReservation == null || DeviceReservation.Devices.Count != Devices.Count())
				{
					return false;
				}

				// Add target devices from reservation
				List<ITargetDevice> ReservedDevices = new List<ITargetDevice>();
				foreach (var Device in DeviceReservation.Devices)
				{
					DeviceDefinition Def = new DeviceDefinition();
					Def.Address = Device.IPOrHostName;
					Def.Name = Device.Name;
					Def.Platform = DeviceMap.FirstOrDefault(Entry => Entry.Value == Device.Type).Key;
					Def.DeviceData = Device.DeviceData;

					if (!String.IsNullOrEmpty(Device.PerfSpec) && !Enum.TryParse<EPerfSpec>(Device.PerfSpec, true, out Def.PerfSpec))
					{
						throw new AutomationException("Unable to convert perfspec '{0}' into an EPerfSpec", Device.PerfSpec);
					}

					ITargetDevice TargetDevice = CreateAndRegisterDeviceFromDefinition(Def);

					// If a device from service can't be added, fail reservation and cleanup devices
					// @todo: device problem reporting, requesting additional devices
					if (TargetDevice == null)
					{

						ReportDeviceError(Device.Name, "CreateDeviceError");

						// If some devices from reservation have been created, release them which will also dispose of reservation
						if (ReservedDevices.Count > 0)
						{
							ReleaseDevices(ReservedDevices);
						}
						else
						{
							// otherwise, no devices have been creation so just cancel this reservation
							DeviceReservation.Dispose();
						}

						Log.Info("Unable to make device registration: device registration failed for {0}:{1}", Def.Platform, Def.Name);
						return false;
					}
					else
					{						
						ReservedDevices.Add(TargetDevice);
					}

					ServiceDeviceInfo[TargetDevice] = Def;
					ServiceReservations[TargetDevice] = DeviceReservation;
				}

				Log.Info("Successfully reserved service devices");
				Devices.ForEach(Device => Log.Verbose("    Device: {0}", Device));

				return true;
			}

			Log.Error("Unable to reserve service devices:");
			Devices.ForEach(Device => Log.Error("    Device: {0}", Device));
			return false;
		}

		/// <summary>
		/// Created a list of device definitions from the passed in reference. Needs work....
		/// </summary>
		/// <param name="InputReference"></param>
		/// <param name="InLocalTempDir"></param>
		/// <param name="ObeyConstraints"></param>
		public void AddDevices(UnrealTargetPlatform DefaultPlatform, string InputReference, bool ObeyConstraints=true)
		{
			lock (LockObject)
			{
				List<ITargetDevice> NewDevices = new List<ITargetDevice>();

				int SlashIndex = InputReference.IndexOf("\\") >= 0 ? InputReference.IndexOf("\\") : InputReference.IndexOf("/");

				bool PossibleFileName = InputReference.IndexOfAny(Path.GetInvalidPathChars()) < 0 &&
							(InputReference.IndexOf(":") == -1 || (InputReference.IndexOf(":") == SlashIndex - 1));
				// Did they specify a file?
				if (PossibleFileName && File.Exists(InputReference))
				{
					Gauntlet.Log.Info("Adding devices from {0}", InputReference);
					List<DeviceDefinition> DeviceDefinitions = JsonConvert.DeserializeObject<List<DeviceDefinition>>(File.ReadAllText(InputReference));

					foreach (DeviceDefinition Def in DeviceDefinitions)
					{
						Gauntlet.Log.Info("Adding {0}", Def);

						// use Legacy field if it exists
						if (Def.Platform == UnrealTargetPlatform.Unknown)
						{
							Def.Platform = Def.Type;
						}

						// check for an availability constraint
						if (string.IsNullOrEmpty(Def.Available) == false && ObeyConstraints)
						{
							// check whether disabled
							if (String.Compare(Def.Available, "disabled", true) == 0)
							{
								Gauntlet.Log.Info("Skipping {0} due to being disabled", Def.Name);
								continue;
							}

							// availability is specified as a range, e.g 21:00-09:00.
							Match M = Regex.Match(Def.Available, @"(\d{1,2}:\d\d)\s*-\s*(\d{1,2}:\d\d)");

							if (M.Success)
							{
								DateTime From, To;

								if (DateTime.TryParse(M.Groups[1].Value, out From) && DateTime.TryParse(M.Groups[2].Value, out To))
								{
									// these are just times so when parsed will have todays date. If the To time is less than
									// From (22:00-06:00) time it spans midnight so move it to the next day
									if (To < From)
									{
										To = To.AddDays(1);
									}

									// if From is in the future (e.g. it's 01:00 and range is 22:00-08:00) we may be in the previous days window, 
									// so move them both back a day
									if (From > DateTime.Now)
									{
										From = From.AddDays(-1);
										To = To.AddDays(-1);
									}

									if (DateTime.Now < From || DateTime.Now > To)
									{
										Gauntlet.Log.Info("Skipping {0} due to availability constraint {1}", Def.Name, Def.Available);
										continue;
									}
								}
								else
								{
									Gauntlet.Log.Warning("Failed to parse availability {0} for {1}", Def.Available, Def.Name);
								}
							}
						}

						Def.RemoveOnShutdown = true;

						if (Def.Platform == UnrealTargetPlatform.Unknown)
						{
							Def.Platform = DefaultPlatform;
						}

						UnprovisionedDevices.Add(Def);
					}

					// randomize devices so if there's a bad device st the start so we don't always hit it (or we do if its later)
					UnprovisionedDevices = UnprovisionedDevices.OrderBy(D => Guid.NewGuid()).ToList();
				}
				else
				{
					if (string.IsNullOrEmpty(InputReference) == false)
					{
						string[] DevicesList = InputReference.Split(',');

						foreach (string DeviceRef in DevicesList)
						{							

							// check for <platform>:<address>:<port>. We pass the last two combined to the
							// device constructor
							Match M = Regex.Match(DeviceRef, @"(.+?):(.+)");

							UnrealTargetPlatform DevicePlatform = DefaultPlatform;
							string DeviceAddress = DeviceRef;

							// when using device service, skip adding local non-desktop devices to pool
							bool IsDesktop = UnrealBuildTool.Utils.GetPlatformsInClass(UnrealPlatformClass.Desktop).Contains(DevicePlatform);							
							if (!IsDesktop && DeviceRef.Equals("default", StringComparison.OrdinalIgnoreCase) && !String.IsNullOrEmpty(DeviceURL))
							{
								continue;
							}

							if (M.Success)
							{
								if (Enum.TryParse(M.Groups[1].ToString(), true, out DevicePlatform) == false)
								{
									throw new AutomationException("platform {0} is not a recognized device type", M.Groups[1].ToString());
								}
								DeviceAddress = M.Groups[2].ToString();
							}

							Log.Info("Added device {0}:{1} to pool", DevicePlatform, DeviceAddress);
							DeviceDefinition Def = new DeviceDefinition();
							Def.Address = DeviceAddress;
							Def.Name = DeviceAddress;
							Def.Platform = DevicePlatform;
							UnprovisionedDevices.Add(Def);
						}
					}
				}
			}
		}

		/// <summary>
		/// Adds the list of devices to our internal availability list
		/// </summary>
		/// <param name="InDevices"></param>
		public void RegisterDevices(IEnumerable<ITargetDevice> InDevices)
		{
			lock (LockObject)
			{
				AvailableDevices = AvailableDevices.Union(InDevices).ToList();
			}
		}

		/// <summary>
		/// Registers the provided device for availability
		/// </summary>
		/// <param name="Device"></param>
		public void RegisterDevice(ITargetDevice Device, UnrealTargetConstraint Constraint = null)
		{
			lock (LockObject)
			{
				if (AvailableDevices.Contains(Device))
				{
					throw new Exception("Device already registered!");
				}

				InitialConnectState[Device] = Device.IsConnected;
				Constraints[Device] = Constraint ?? new UnrealTargetConstraint(Device.Platform);

				AvailableDevices.Add(Device);

				if (Log.IsVerbose)
				{
					Device.RunOptions = Device.RunOptions & ~CommandUtils.ERunOptions.NoLoggingOfRunCommand;
				}
			}
		}

		/// <summary>
		/// Created and registered a device from the provided definition 
		/// </summary>
		/// <param name="Def"></param>
		/// <returns></returns>
		protected ITargetDevice CreateAndRegisterDeviceFromDefinition(DeviceDefinition Def)
		{
			ITargetDevice NewDevice = null;

			IDeviceFactory Factory = Gauntlet.Utils.InterfaceHelpers.FindImplementations<IDeviceFactory>()
					.Where(F => F.CanSupportPlatform(Def.Platform))
					.FirstOrDefault();

			if (Factory == null)
			{
				throw new AutomationException("No IDeviceFactory implementation that supports {0}", Def.Platform);
			}

			try
			{
				bool IsDesktop = UnrealBuildTool.Utils.GetPlatformsInClass(UnrealPlatformClass.Desktop).Contains(Def.Platform);

				if (IsDesktop)
				{
					string ClientTempDir = Path.Combine(LocalTempDir, "PCTemp");
					int DeviceCount = AvailableDevices.Union(ReservedDevices).Where(D => D.Platform == Def.Platform).Count();

					NewDevice = Factory.CreateDevice(Def.Name, ClientTempDir);
				}
				else
				{
					NewDevice = Factory.CreateDevice(Def.Address, Def.DeviceData);
				}
					
				lock (LockObject)
				{
					if (NewDevice != null)
					{
						RegisterDevice(NewDevice, new UnrealTargetConstraint(NewDevice.Platform, Def.PerfSpec));
					}
				}
			}
			catch (Exception Ex)
			{
				Log.Info("Failed to create device {0}. {1}", Def.ToString(), Ex.Message);
			}

			return NewDevice;
		}

		public UnrealTargetConstraint GetConstraint(ITargetDevice Device)
		{

			if (!Constraints.ContainsKey(Device))
			{
				throw new AutomationException("Device pool has no contstaint for {0} (device was likely released)", Device);
			}

			return Constraints[Device];
		}

		/// <summary>
		/// Returns the number of available devices of the provided type. This includes unprovisioned devices but not reserved ones.
		/// Note: unprovisioned devices are currently only returned when device is not constrained
		/// </summary>
		public int GetAvailableDeviceCount(UnrealTargetConstraint Constraint, Func<ITargetDevice, bool> Validate = null)
		{
			lock (LockObject)
			{
				return AvailableDevices.Where(D => Validate == null ? Constraint.Check(D) : Validate(D)).Count() +
					UnprovisionedDevices.Where(D => D.Platform == Constraint.Platform && Constraint.IsIdentity()).Count();
			}
		}

		/// <summary>
		/// Returns the number of available devices of the provided type. This includes unprovisioned devices but not reserved ones
		/// Note: unprovisioned devices are currently only returned when device is not constrained
		/// </summary>
		public int GetTotalDeviceCount(UnrealTargetConstraint Constraint, Func<ITargetDevice, bool> Validate = null)
		{
			lock (LockObject)
			{
				return AvailableDevices.Union(ReservedDevices).Where(D => Validate == null ? Constraint.Check(D) : Validate(D)).Count() +
					UnprovisionedDevices.Where(D => D.Platform == Constraint.Platform && Constraint.IsIdentity()).Count();
			}
		}

		/// <summary>
		///	Checks whether device pool can accommodate requirements, optionally add service devices to meet demand
		/// </summary>
		internal bool CheckAvailableDevices(Dictionary<UnrealTargetConstraint, int> RequiredDevices, List<ProblemDevice> ProblemDevices = null, bool UseServiceDevices = true)
		{
			Dictionary<UnrealTargetConstraint, int> AvailableDeviceTypes = new Dictionary<UnrealTargetConstraint, int>();
			Dictionary<UnrealTargetConstraint, int> TotalDeviceTypes = new Dictionary<UnrealTargetConstraint, int>();

			// Do these "how many available" checks every time because the DevicePool provisions on demand so while it may think it has N machines, 
			// some of them may fail to be provisioned and we could end up with none!

			// See how many of these types are in device pool (mostly to supply informative info if we can't meet these)

			foreach (var PlatformRequirement in RequiredDevices)
			{
				UnrealTargetConstraint Constraint = PlatformRequirement.Key;

				Func<ITargetDevice, bool> Validate = (ITargetDevice Device) =>
				{

					if (!Constraint.Check(Device))
					{
						return false;
					}

					if (ProblemDevices == null)
					{
						return true;
					}

					foreach (ProblemDevice PDevice in ProblemDevices)
					{
						if (PDevice.Platform == Device.Platform && PDevice.Name == Device.Name)
						{
							return false;
						}
					}

					return true;
				};

				AvailableDeviceTypes[Constraint] = DevicePool.Instance.GetAvailableDeviceCount(Constraint, Validate);
				TotalDeviceTypes[Constraint] = DevicePool.Instance.GetTotalDeviceCount(Constraint, Validate);


				Log.Verbose("{0}: {1} devices required. Total:{2}, Available:{3}",
					Constraint, PlatformRequirement.Value,
					TotalDeviceTypes[PlatformRequirement.Key], AvailableDeviceTypes[PlatformRequirement.Key]);
			}

			// get a list of any platforms where we don't have enough
			var TooFewTotalDevices = RequiredDevices.Where(KP => TotalDeviceTypes[KP.Key] < RequiredDevices[KP.Key]).Select(KP => KP.Key);
			var TooFewCurrentDevices = RequiredDevices.Where(KP => AvailableDeviceTypes[KP.Key] < RequiredDevices[KP.Key]).Select(KP => KP.Key);

			// Request devices from the service if we need them
			if (UseServiceDevices && !String.IsNullOrEmpty(DeviceURL) && (TooFewTotalDevices.Count() > 0 || TooFewCurrentDevices.Count() > 0))
			{
				var Devices = TooFewTotalDevices.Concat(TooFewCurrentDevices);

				Dictionary<UnrealTargetConstraint, int> DeviceCounts = new Dictionary<UnrealTargetConstraint, int>();

				Devices.ToList().ForEach(Platform => DeviceCounts[Platform] = RequiredDevices[Platform]);

				Log.Info("Requesting devices from service at {0}", DeviceURL);

				// Acquire necessary devices from service
				if (!ReserveDevicesFromService(DeviceURL, DeviceCounts))
				{
					return false;
				}
			}
			else
			{
				// if we can't ever run then throw an exception
				if (TooFewTotalDevices.Count() > 0)
				{
					var MissingDeviceStrings = TooFewTotalDevices.Select(D => string.Format("Not enough devices of type {0} exist for test. ({1} required, {2} available)", D, RequiredDevices[D], AvailableDeviceTypes[D]));
					Log.Error("{0}", string.Join("\n", MissingDeviceStrings));
					throw new AutomationException("Not enough devices available");
				}

				// if we can't  run now then return false
				if (TooFewCurrentDevices.Count() > 0)
				{
					var MissingDeviceStrings = TooFewCurrentDevices.Select(D => string.Format("Not enough devices of type {0} available for test. ({1} required, {2} available)", D, RequiredDevices[D], AvailableDeviceTypes[D]));
					Log.Verbose("{0}", string.Join("\n", MissingDeviceStrings));
					return false;
				}
			}

			return true;

		}

		/// <summary>
		/// Run the provided function across all our devices until it returns false. Devices are provisioned on demand (e.g turned from info into an ITargetDevice)
		/// </summary>		
		public void EnumerateDevices(UnrealTargetPlatform Platform, Func<ITargetDevice, bool> Predicate)
		{
			EnumerateDevices(new UnrealTargetConstraint(Platform), Predicate);
		}

		public void EnumerateDevices(UnrealTargetConstraint Constraint, Func<ITargetDevice, bool> Predicate)
		{
			lock (LockObject)
			{
				List<ITargetDevice> Selection = new List<ITargetDevice>();

				// randomize the order of all devices that are of this platform
				var MatchingProvisionedDevices = AvailableDevices.Where(D => Constraint.Check(D)).ToList();
				var MatchingUnprovisionedDevices = UnprovisionedDevices.Where(D => D.Platform == Constraint.Platform && Constraint.IsIdentity()).ToList();

				bool OutOfDevices = false;
				bool ContinuePredicate = true;

				do
				{
					// Go through all our provisioned devices to see if these fulfill the predicates
					// requirements

					ITargetDevice NextDevice = MatchingProvisionedDevices.FirstOrDefault();

					while (NextDevice != null && ContinuePredicate)
					{
						Log.Verbose("Checking {0} against predicate", NextDevice.Name);
						MatchingProvisionedDevices.Remove(NextDevice);
						ContinuePredicate = Predicate(NextDevice);

						NextDevice = MatchingProvisionedDevices.FirstOrDefault();
					}

					if (ContinuePredicate)
					{
						// add more devices if possible
						OutOfDevices = MatchingUnprovisionedDevices.Count() == 0;

						DeviceDefinition NextDeviceDef = MatchingUnprovisionedDevices.FirstOrDefault();

						if (NextDeviceDef != null)
						{
							Log.Verbose("Provisioning device {0} for the pool", NextDeviceDef.Name);

							// try to create a device. This can fail, but if so we'll just end up back here
							// on the next iteration
							ITargetDevice NewDevice = CreateAndRegisterDeviceFromDefinition(NextDeviceDef);

							MatchingUnprovisionedDevices.Remove(NextDeviceDef);
							UnprovisionedDevices.Remove(NextDeviceDef);

							if (NewDevice != null)
							{
								MatchingProvisionedDevices.Add(NewDevice);
								Log.Verbose("Added device {0} to pool", NewDevice.Name);
							}
							else
							{
								Log.Info("Failed to provision {0}", NextDeviceDef.Name);
								// track this
								if (FailedProvisions.Contains(NextDeviceDef) == false)
								{
									FailedProvisions.Add(NextDeviceDef);
								}
							}
						}
						else
						{
							Log.Info("Pool ran out of devices of type {0}!", Constraint);
							OutOfDevices = true;
						}
					}
				} while (OutOfDevices == false && ContinuePredicate);
			}
		}
	}
}