Files
UnrealEngineUWP/Engine/Source/Programs/ImageValidator/ImageValidatorData.cs
Gil Gribb 28af311319 Copying //UE4/Dev-Rendering to //UE4/Dev-Main (Source: //UE4/Dev-Rendering @ 3006421)
#lockdown nick.penwarden

==========================
MAJOR FEATURES + CHANGES
==========================

Change 2998081 on 2016/06/02 by Rolando.Caloca

	DR - Update vulkan headers to 1.0.13.0

Change 2998087 on 2016/06/02 by Rolando.Caloca

	DR - Added r.DumpShaderDebugWorkerCommandLine to dump a batch file containing a command line for SCW's -directcompile mode

Change 2998092 on 2016/06/02 by Rolando.Caloca

	DR - Updated ThirdParty/glslang to 1.0.13.0

Change 2998113 on 2016/06/02 by Martin.Mittring

	Added -Deterministic and -BuildName=... as command line option for Screenshot verification

Change 2998115 on 2016/06/02 by Martin.Mittring

	optimied Tonemapper sharpen 25->17 instructions
	fixes: very bright HDR pixel appeared to not be antialiased
	no blonger blurs very bright pixels (we could bring that back if needed but counters the sharpen)
	moved one multipy into C++
	Added debug visualization (HLSL define)

Change 2998132 on 2016/06/02 by Rolando.Caloca

	DR - Remove auto from VulkanRHI wherever possible

Change 2998148 on 2016/06/02 by Rolando.Caloca

	DR - Move FVulkanPendingState out of device and into cmd list for more RHI thread fixes

Change 2998180 on 2016/06/02 by Rolando.Caloca

	DR - Temp fix for (bogus?) fence validation warning using VULKAN_REUSE_FENCES=0
	- Check we are not asking for a VkFormat out of bounds (ie only works with core formats)

Change 2998293 on 2016/06/02 by Rolando.Caloca

	DR - Add support for BC6H & BC7 in desktop Vulkan

Change 2998419 on 2016/06/02 by Brian.Karis

	Optimized CountBits

Change 2998480 on 2016/06/02 by Rolando.Caloca

	DR - Fix for RHI thread failing in an ensure on es31 (doesn't happen on bypass as the ensure is inside RHI cmd list)

Change 2998486 on 2016/06/02 by Martin.Mittring

	fixed ES2 shader compile

Change 2998527 on 2016/06/02 by Daniel.Wright

	Added cvar r.StencilForLODDither, which is compiled into shaders and forces a full prepass when enabled.  This allows r.EarlyZPass to remain changeable at runtime (when not forced to a value by features that are enabled).

Change 2998531 on 2016/06/02 by Daniel.Wright

	Fixed Pixel Normal Offset refraction mode with materials using world space normals

Change 2998568 on 2016/06/02 by Rolando.Caloca

	DR - Fix typo

Change 2998630 on 2016/06/02 by Rolando.Caloca

	DR - Relaxed glslang strictness to get Pos tCombineLUTs working
	- Reenabled Post CombineLUTs on Vulkan
	- Removed some extra glsl output from Vulkan backend
	- Fixed RHI thread uniform buffer assert

Change 2998639 on 2016/06/02 by Brian.Karis

	Fix for crash when removing instances while lighting is building

Change 2998640 on 2016/06/02 by Martin.Mittring

	added r.ResetViewState to allow for more deterministic rendering
	made SSR FrameRandom resetable by not using FrameNumber (Note: SSR on a view without state was flickering before, not it's stabele and doesn't flicker)

Change 2998790 on 2016/06/02 by Martin.Mittring

	minor optimization to SSR

Change 2999849 on 2016/06/03 by Martin.Mittring

	fixed issue with sort order

Change 3000340 on 2016/06/03 by Rolando.Caloca

	DR - Integrate change from 3000177
	Workaround for random crash shutting down D3D11/NV driver
	jira UE-18906

Change 3000422 on 2016/06/03 by Rolando.Caloca

	DR - Mirror change in Odin

Change 3000571 on 2016/06/03 by Rolando.Caloca

	DR - Vulkan SM4 prep

Change 3001652 on 2016/06/06 by Gil.Gribb

	Merging //UE4/Dev-Main@3001532 to Dev-Rendering (//UE4/Dev-Rendering)

Change 3001935 on 2016/06/06 by Gil.Gribb

	UE4 - Fixed botched merge

Change 3002390 on 2016/06/06 by Gil.Gribb

	UE4 - Added code to prevent flooding of rendering thread with heartbeats.

Change 3002442 on 2016/06/06 by Martin.Mittring

	Moved ImageInvalidator out of NotForLicensees - V1.0 works well enough

Change 3002460 on 2016/06/06 by Martin.Mittring

	fixed missing file in ImageValidator
	added .exe to extras

Change 3002514 on 2016/06/06 by Martin.Mittring

	added OpenSubDiv 3.0.2, not yet used

Change 3002536 on 2016/06/06 by Martin.Mittring

	fixed help text on cvar

Change 3002647 on 2016/06/06 by Martin.Mittring

	adding r.ResetViewState to RenderOutputValidation
	added r.Streaming.FramesForFullUpdate to r.DisplayInternals
	#code_review:Benjamin.Hyder

Change 3002945 on 2016/06/06 by Rolando.Caloca

	DR - Fix hlslcc issue with access to matrix elements:
	This would fail:
	M._m30 += f;
	- Fixed IRDump for matrix swizzled
	- Fix hlslcc_exe not linking

Change 3002979 on 2016/06/06 by John.Billon

	DDS unsupported format error
	#Jira UE-24529

Change 3002983 on 2016/06/06 by Martin.Mittring

	split FPixelShaderInOut in in and out

Change 3003011 on 2016/06/06 by Martin.Mittring

	updated OpenSubDiv to 3.0.2
	Fixed SubDivisonRendring by using OpenSubDiv

Change 3003264 on 2016/06/06 by Daniel.Wright

	Fixed bUseSingleSampleShadowFromStationaryLights on particles

Change 3003296 on 2016/06/06 by Daniel.Wright

	Renamed files Forward* to Mobile*

Change 3003350 on 2016/06/06 by Daniel.Wright

	Improved "Repaired Painted Vertex Colors" log message.  This is now done once on levels at load in the editor, instead of for every component (massive log spam).  Total load time caused by the fixup is reported.

Change 3003815 on 2016/06/07 by Chris.Bunner

	Updating to more recent D3DCompiler DLL, seeing up to 2x speed-up in testing.
	Removed compiler override cvar.
	#jira UE-28574

Change 3003827 on 2016/06/07 by Gil.Gribb

	UE4 - Increased stack sizes for thread pools.

Change 3003971 on 2016/06/07 by Martin.Mittring

	fixed compiler warnings

Change 3004028 on 2016/06/07 by Rolando.Caloca

	DR - Update glslang exe

Change 3004555 on 2016/06/07 by Rolando.Caloca

	DR - vk - Fix validation warning

Change 3004637 on 2016/06/07 by Martin.Mittring

	fixed compiler warning

Change 3004841 on 2016/06/07 by Daniel.Wright

	Fix for shadowed variable

Change 3005044 on 2016/06/07 by Daniel.Wright

	SubUV Animations can get opacity information from any channel of the source texture

Change 3005057 on 2016/06/07 by Daniel.Wright

	Renamed ForwardShading* to Mobile*

Change 3005135 on 2016/06/07 by Uriel.Doyon

	Moving wanted mip computation (according to budget) to async task.
	Update "stat streaming" to show visible mips and also loading progression.
	Fixed overly wanted mip issue cause be max range clamping.
	Optimized cost of the texture streamer on the gamethread.
	Character, Terrain and Forced Load are now loaded with priority in the AsyncIO
	Reduced streaming temp memory requirements on PS4.
	Async streaming task now runs in parallel to the incremental update.
	Fixed bug with bProcessEverything not working as expected in UpdateResourceStreaming.
	Fixed metrics with HiddenScale being applied in addition to overbudget limitation.
	Improved budget stability when using split load request (one for visible mips + one for hidden mips)
	Implemented a more agressive visibility test (used to be a seen in the last 5 sec, now closer to .5)
	Streaming stats are now within a single class named FTextureStreamingStats and updated in a single function.
	Cleanup of FStreamingTexture state update into a single function (UpdateDynamicData).
	Retention logic now drops texture based on the last render time (for non visible textures) to reduce looping effects.

Change 3005207 on 2016/06/07 by Uriel.Doyon

	Fixed warning

[CL 3006426 by Gil Gribb in Main branch]
2016-06-08 16:02:23 -04:00

480 lines
15 KiB
C#

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Drawing; // Color
using System.IO; // Directory.
using System.Diagnostics; // Debug.
using System.Drawing.Imaging; // ImageData
namespace ImageValidator
{
// per folder, later stored in xml file
public class ImageValidatorSettings
{
public string TestDir;
public string RefDir;
public uint Threshold;
public uint PixelCountToFail;
public ImageValidatorSettings()
{
Threshold = 16;
PixelCountToFail = 32;
}
public static string GetVersionString()
{
return "V1.0";
}
};
public struct ImageValidatorIntermediate
{
public struct PixelElement
{
// blue, green, red, alpha
public byte b, g, r, a;
public void SetErrorColor()
{
// opaque
a = 0xff;
r = 0xff;
g = 0;
b = 0;
}
public void SetErrorColor(float squaredError)
{
a = 0xff;
if (squaredError > 0.0f)
{
// no error is black, minor error is a noticeable color
squaredError += 800;
}
squaredError /= 8;
r = (byte)(Math.Min(0xff, squaredError));
g = (byte)(Math.Min(0xff, squaredError / 8));
b = (byte)(Math.Min(0xff, squaredError / 64));
}
public static float ComputeSquaredError(PixelElement Test, PixelElement Ref)
{
float R = (float)Test.r - (float)Ref.r;
float G = (float)Test.g - (float)Ref.g;
float B = (float)Test.b - (float)Ref.b;
return R * R + G * G + B * B;
}
public static uint ComputeAbsDiff(PixelElement Test, PixelElement Ref)
{
int R = Math.Abs((int)Test.r - (int)Ref.r);
int G = Math.Abs((int)Test.g - (int)Ref.g);
int B = Math.Abs((int)Test.b - (int)Ref.b);
return (uint)Math.Max(R, Math.Max(G, B));
}
};
public Bitmap imageTest;
public Bitmap imageDiff;
public Bitmap imageRef;
private Bitmap LoadBitmap(string filename)
{
Image tmpImage;
Bitmap Ret = null;
try
{
// http://stackoverflow.com/questions/788335/why-does-image-fromfile-keep-a-file-handle-open-sometimes
// load without keeping the file locked
using (var fs = new FileStream(filename, FileMode.Open, FileAccess.Read))
{
tmpImage = Image.FromStream(fs);
Ret = new Bitmap(tmpImage);
tmpImage.Dispose();
}
}
catch (System.IO.DirectoryNotFoundException)
{
// no need to handle this
}
catch (System.IO.FileNotFoundException)
{
// no need to handle this
}
return Ret;
}
public void Process(ImageValidatorSettings settings, ImageValidatorData.ImageEntry imageEntry)
{
if (imageTest != null) imageTest.Dispose();
if (imageRef != null) imageRef.Dispose();
imageTest = LoadBitmap(settings.TestDir + imageEntry.Name);
imageRef = LoadBitmap(settings.RefDir + imageEntry.Name);
imageEntry.testResult = ComputeDiff(settings.Threshold);
}
public static System.Drawing.Size GetImageSize(Bitmap bitmap)
{
GraphicsUnit Unit = GraphicsUnit.Pixel;
System.Drawing.Size Ret = new System.Drawing.Size(1, 1);
if (bitmap != null)
{
RectangleF bounds = bitmap.GetBounds(ref Unit);
Ret.Width = (int)bounds.Width;
Ret.Height = (int)bounds.Height;
}
return Ret;
}
public System.Drawing.Size GetImagesSize()
{
System.Drawing.Size sizeTest = GetImageSize(imageTest);
System.Drawing.Size sizeRef = GetImageSize(imageRef);
return new System.Drawing.Size(
Math.Max(sizeTest.Width, sizeRef.Width),
Math.Max(sizeTest.Height, sizeRef.Height)
);
}
public static Bitmap resizeImage(Image image)
{
// uint MaxSize = 64;
uint MaxSize = 128;
float Scale = MaxSize / (float)Math.Max(image.Width, image.Height);
int new_width = Math.Max(1, (int)(image.Width * Scale));
int new_height = Math.Max(1, (int)(image.Height * Scale));
// see http://base64image.org/
Bitmap new_image = new Bitmap(new_width, new_height);
Graphics g = Graphics.FromImage((Image)new_image);
//g.InterpolationMode = System.Drawing.Drawing2D.InterpolationMode.High; // creates alpha channel from border pixels - doesn't look good for the Report
g.InterpolationMode = System.Drawing.Drawing2D.InterpolationMode.Low;
g.DrawImage(image, 0, 0, new_width, new_height);
return new_image;
}
// compute imageDiff from imageTest and imageRef
public TestResult ComputeDiff(uint Threshold)
{
if (imageDiff != null) imageDiff.Dispose();
TestResult Ret = new TestResult();
if (imageTest == null)
{
Ret.ErrorText = "missing Test";
return Ret;
}
if (imageRef == null)
{
Ret.ErrorText = "missing Ref";
return Ret;
}
System.Drawing.Size sizeTest = GetImageSize(imageTest);
System.Drawing.Size sizeRef = GetImageSize(imageRef);
if (sizeTest != sizeRef)
{
Ret.ErrorText = "Size " +
sizeTest.Width.ToString() + "x" + sizeTest.Height.ToString() +
" != " +
sizeRef.Width.ToString() + "x" + sizeRef.Height.ToString();
return Ret;
}
System.Drawing.Size size = GetImagesSize();
// todo: exit if Size or format is not the same
Debug.Assert(imageTest.PixelFormat == PixelFormat.Format32bppArgb);
imageDiff = new Bitmap(size.Width, size.Height);
uint CountedErrorPixels = 0;
BitmapData dataTest = imageTest.LockBits(new Rectangle(0, 0, size.Width, size.Height), ImageLockMode.ReadOnly, imageTest.PixelFormat);
BitmapData dataDiff = imageDiff.LockBits(new Rectangle(0, 0, size.Width, size.Height), ImageLockMode.WriteOnly, imageTest.PixelFormat);
BitmapData dataRef = imageRef.LockBits(new Rectangle(0, 0, size.Width, size.Height), ImageLockMode.ReadOnly, imageTest.PixelFormat);
unsafe
{
for (int x = 0; x < size.Width; ++x)
{
for (int y = 0; y < size.Height; ++y)
{
PixelElement* valueTest = (PixelElement*)((byte*)dataTest.Scan0.ToPointer() + dataTest.Stride * y + x * sizeof(PixelElement));
PixelElement* valueDiff = (PixelElement*)((byte*)dataDiff.Scan0.ToPointer() + dataDiff.Stride * y + x * sizeof(PixelElement));
PixelElement* valueRef = (PixelElement*)((byte*)dataRef.Scan0.ToPointer() + dataRef.Stride * y + x * sizeof(PixelElement));
// float Diff = PixelElement.ComputeSquaredError(*valueDiff, *valueRef);
uint localError = PixelElement.ComputeAbsDiff(*valueTest, *valueRef);
// all pixels opaque
valueDiff->a = 0xff;
if (localError >= Threshold)
{
++CountedErrorPixels;
// valueDiff->SetErrorColor(PixelElement.ComputeSquaredError(*valueTest, *valueRef));
valueDiff->SetErrorColor();
// *valueDiff = *valueRef;
}
}
}
}
imageRef.UnlockBits(dataRef);
imageDiff.UnlockBits(dataDiff);
imageTest.UnlockBits(dataTest);
Ret.ErrorPixels = CountedErrorPixels;
// update Thumbnails
{
Ret.ThumbnailTest = resizeImage(imageTest);
Ret.ThumbnailDiff = resizeImage(imageDiff);
Ret.ThumbnailRef = resizeImage(imageRef);
}
return Ret;
}
};
public class TestResult
{
public static Color GetColor(bool bPassed)
{
return bPassed ? Color.FromArgb(0x88ff88) : Color.FromArgb(0xff8888);
}
public Color GetColor(ref ImageValidatorSettings settings)
{
return GetColor(IsPassed(ref settings));
}
public bool IsPassed(ref ImageValidatorSettings settings)
{
return ErrorText == null && ErrorPixels < settings.PixelCountToFail;
}
public string GetString(ref ImageValidatorSettings settings)
{
if (ErrorText != null)
{
return ErrorText;
}
else
{
string Ret = IsPassed(ref settings) ? "Passed" : "Failed";
Ret += " (" + ErrorPixels.ToString() + ")";
return Ret;
}
}
//
public string ErrorText;
// only used if ErrorText is null
public uint ErrorPixels;
public Bitmap ThumbnailTest;
public Bitmap ThumbnailDiff;
public Bitmap ThumbnailRef;
};
public struct ImageValidatorData
{
public struct ImageEntryColumnData
{
public string Platform;
public string Map;
public string Time;
public string Actor;
public ImageEntryColumnData(string Name)
{
Platform = Path.GetDirectoryName(Name);
if (Platform.StartsWith(@"\"))
{
Platform = Platform.Substring(1);
}
string FileNameWithoutExtension = Path.GetFileNameWithoutExtension(Name);
string[] keyValuePairs = FileNameWithoutExtension.Split(' ');
Map = "";
Time = "";
Actor = "";
foreach (string keyvalue in keyValuePairs)
{
string trimKeyValue = keyvalue;
if (trimKeyValue.EndsWith(")"))
{
trimKeyValue = trimKeyValue.Substring(0, trimKeyValue.Length - 1);
}
if (trimKeyValue.StartsWith("Map("))
{
Map = trimKeyValue.Substring(4);
}
if (trimKeyValue.StartsWith("Time("))
{
Time = trimKeyValue.Substring(5);
}
if (trimKeyValue.StartsWith("Actor("))
{
Actor = trimKeyValue.Substring(6);
}
}
// clernup some legacy nameing convention
if (Time.EndsWith("s"))
{
Time = Time.Substring(0, Time.Length - 1);
}
}
}
public class ImageEntry : IComparable
{
public ImageEntry()
{
bRefExists = false;
bTestExists = false;
}
// key, without front path
public string Name;
//
public bool bRefExists;
//
public bool bTestExists;
// can be null
public TestResult testResult;
public int CompareTo(object _rhs)
{
ImageEntry rhs = _rhs as ImageEntry;
if (rhs == null)
return 1;
ImageValidatorData.ImageEntryColumnData columnThis = new ImageValidatorData.ImageEntryColumnData(Name);
ImageValidatorData.ImageEntryColumnData columnRhs = new ImageValidatorData.ImageEntryColumnData(rhs.Name);
float timeThis = float.Parse(columnThis.Time);
float timeRhs = float.Parse(columnRhs.Time);
int x = timeThis.CompareTo(timeRhs);
if(x != 0)
return x;
return Name.CompareTo(rhs.Name);
}
};
// --------------------------------------------------------------------
public List<ImageEntry> imageEntries;
private void PopulatePartList(bool bRef, string path)
{
try
{
string[] files = Directory.GetFiles(path, "*.png", SearchOption.AllDirectories);
int pathlen = path.Length;
foreach (string it in files)
{
string substr = it.Substring(pathlen);
ImageEntry thisEntry = null;
foreach (ImageEntry entry in imageEntries)
{
if (entry.Name == substr)
{
thisEntry = entry;
break;
}
}
if (thisEntry == null)
{
thisEntry = new ImageEntry();
thisEntry.Name = substr;
imageEntries.Add(thisEntry);
}
if (bRef)
{
thisEntry.bRefExists = true;
}
else
{
thisEntry.bTestExists = true;
}
}
}
catch (System.ArgumentException)
{
}
catch (System.IO.DirectoryNotFoundException)
{
}
catch (System.IO.FileNotFoundException)
{
}
}
// @retur if a element was processed
public bool ProcessOneElement(ImageValidatorSettings settings)
{
foreach (ImageEntry entry in imageEntries)
{
if (entry.testResult == null)
{
ImageValidatorIntermediate intermediate = new ImageValidatorIntermediate();
intermediate.Process(settings, entry);
return true;
}
}
return false;
}
public void PopulateList(string TestDir, string RefDir)
{
imageEntries = new List<ImageEntry>();
PopulatePartList(false, TestDir);
PopulatePartList(true, RefDir);
imageEntries.Sort();
}
}
}