Moved StructUtils types (InstancedStruct, StructView, PropertyBag, etc.) to CoreUObject

StructUtilsTestSuite has been moved to Developper
StructUtilsEditor has been moved to Engine/Editor
NetSerialization for FInstancedStruct not using native serialization has been moved to the engine module since FRepLayout is not available in CoreUObject module.
Old plugins and modules are kept temporarily and will be remove in a different CL
Temporary header files added to facilitate transition to the new include path
#jira UE-216472
#rb Devin.Doucette, Francis.Hurteau

[CL 34509881 by yoan stamant in ue5-main branch]

Engine/Source/Runtime/CoreUObject/Private/StructUtils/SharedStruct.cpp

@@ -0,0 +1,55 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+#include "StructUtils/SharedStruct.h"
+#include "StructUtils/StructView.h"
+
+#include UE_INLINE_GENERATED_CPP_BY_NAME(SharedStruct)
+
+///////////////////////////////////////////////////////////////// FConstSharedStruct /////////////////////////////////////////////////////////////////
+
+bool FConstSharedStruct::Identical(const FConstSharedStruct* Other, uint32 PortFlags) const
+{
+	// Only empty or strictly identical is considered equal
+	return Other != nullptr
+		&& GetMemory() == Other->GetMemory()
+		&& GetScriptStruct() == Other->GetScriptStruct();
+}
+
+void FConstSharedStruct::AddStructReferencedObjects(class FReferenceCollector& Collector)
+{
+	if (auto* Struct = GetScriptStructPtr(); Struct && *Struct)
+	{
+		Collector.AddReferencedObjects(*Struct, const_cast<uint8*>(GetMemory()));
+	}
+}
+
+///////////////////////////////////////////////////////////////// FSharedStruct /////////////////////////////////////////////////////////////////
+
+FSharedStruct::FSharedStruct(const FConstStructView& InOther)
+{
+	InitializeAs(InOther.GetScriptStruct(), InOther.GetMemory());
+}
+
+FSharedStruct& FSharedStruct::operator=(const FConstStructView& InOther)
+{
+	if (*this != InOther)
+	{
+		InitializeAs(InOther.GetScriptStruct(), InOther.GetMemory());
+	}
+	return *this;
+}
+
+bool FSharedStruct::Identical(const FSharedStruct* Other, uint32 PortFlags) const
+{
+	// Only empty or strictly identical is considered equal
+	return Other != nullptr 
+		&& GetMemory() == Other->GetMemory() 
+		&& GetScriptStruct() == Other->GetScriptStruct();
+}
+
+void FSharedStruct::AddStructReferencedObjects(class FReferenceCollector& Collector)
+{
+	if (auto* Struct = GetScriptStructPtr(); Struct && *Struct)
+	{
+		Collector.AddReferencedObjects(*Struct, const_cast<uint8*>(GetMemory()));
+	}
+}

Engine/Source/Runtime/CoreUObject/Private/StructUtils/StructTypeBitSet.cpp

@@ -0,0 +1,235 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "StructUtils/StructTypeBitSet.h"
+#include "Serialization/Archive.h"
+
+
+DEFINE_LOG_CATEGORY_STATIC(LogStructUtils, Warning, All);
+
+namespace FBitSetSerializationDictionary
+{
+	TMap<uint32 /*CurrentStoredTypesHash*/, const FStructTracker* /*Tracker*/> TrackerMap;
+	TMap<uint32 /*NotUpToDateStoredTypesHash*/, TArray<int32> /*Mapping*/> BitMappings;
+
+	void RegisterHash(const uint32 SerializationHash, const FStructTracker& Tracker)
+	{
+		if (const FStructTracker** StoredTracer = TrackerMap.Find(SerializationHash))
+		{
+			CA_SUPPRESS(6269); // warning C6269: Possibly incorrect order of operations.
+			// disabling the warning since it's wrong. 
+			// adding:
+			//		const FStructTracker* StoredTracerPtr = *StoredTracer;
+			// and then using StoredTracerPtr clears out the warning.
+			ensureMsgf(((*StoredTracer) == nullptr) || ((*StoredTracer) == &Tracker)
+				, TEXT("Hash conflict when registering a FStructTracker instance"));
+		}
+		TrackerMap.Add(SerializationHash, &Tracker);
+	}
+
+	const FStructTracker** GetTracker(const uint32 SerializationHash)
+	{
+		return TrackerMap.Find(SerializationHash);
+	}
+
+	TArray<int32>& GetOrCreateBitMapping(const uint32 SerializationHash)
+	{
+		return BitMappings.FindOrAdd(SerializationHash);
+	}
+
+	void UnregisterTracker(const FStructTracker& Tracker)
+	{
+		for (auto It = TrackerMap.CreateIterator(); It; ++It)
+		{
+			if (It->Value == &Tracker)
+			{
+				// note that we're not bailing out once the entry is removes, since
+				// Trackers are being registered for all the hashes calculated as 
+				// the new types to track get added
+				It.RemoveCurrent();
+			}
+		}
+	}
+}
+
+
+//-----------------------------------------------------------------------------
+// FStructTracker
+//-----------------------------------------------------------------------------
+FStructTracker::FStructTracker(const FBaseStructGetter& InBaseStructGetter)
+	: BaseStructGetter(InBaseStructGetter)
+{
+}
+
+FStructTracker::~FStructTracker()
+{
+	FBitSetSerializationDictionary::UnregisterTracker(*this);
+}
+
+int32 FStructTracker::FindOrAddStructTypeIndex(const UStruct& InStructType)
+{
+	// Get existing index...
+	const uint32 Hash = PointerHash(&InStructType);
+	FSetElementId ElementId = StructTypeToIndexSet.FindIdByHash(Hash, Hash);
+
+	if (!ElementId.IsValidId())
+	{
+		checkSlow(InStructType.IsChildOf(BaseStructGetter()));
+
+		// .. or create a new one
+		ElementId = StructTypeToIndexSet.AddByHash(Hash, Hash);
+		checkSlow(ElementId.IsValidId());
+
+		const int32 NewIndex = ElementId.AsInteger();
+
+		checkSlow(StructTypesList.Num() == NewIndex);
+		StructTypesList.Add(&InStructType);
+
+		// first-time SerializationHash initialization
+		if (SerializationHash == 0)
+		{
+			ensure(StructTypesList.Num() == 1);
+			UStruct* BaseType = BaseStructGetter();
+			CA_ASSUME(BaseType);
+			SerializationHash = GetTypeHash(BaseType->GetFullName());
+		}
+		SerializationHash = HashCombine(SerializationHash, GetTypeHash(InStructType.GetFullName()));
+
+		// it's worth pointing out that we're registering a given tracker for all the hashes created along the way.
+		// This will help with loading bitsets from serialized data.
+		FBitSetSerializationDictionary::RegisterHash(SerializationHash, *this);
+
+#if WITH_STRUCTUTILS_DEBUG
+		DebugStructTypeNamesList.Add(InStructType.GetFName());
+		ensure(StructTypeToIndexSet.Num() == DebugStructTypeNamesList.Num());
+#endif // WITH_STRUCTUTILS_DEBUG
+	}
+
+	return ElementId.AsInteger();
+}
+
+void FStructTracker::Serialize(FArchive& Ar, FStructTypeBitSet::FBitSetContainer& StructTypesBitArray)
+{	
+	enum class EVersion : uint8
+	{
+		Initial,
+		Last,
+		Current = Last - 1
+	};
+
+	uint8 Version = uint8(EVersion::Current);
+	Ar << Version;
+
+	if (Ar.IsSaving())
+	{
+		Ar << SerializationHash;
+		
+		const int64 SizeOffset = Ar.Tell(); 
+		int32 SerialSize = 0;
+		Ar << SerialSize;
+
+		// Serialized memory
+		const int64 InitialDataOffset = Ar.Tell();
+
+		// store information on the base type, so that we can verify we're trying to load the right data later on
+		FTopLevelAssetPath BaseStructPath(BaseStructGetter());
+		Ar << BaseStructPath;
+
+		int32 StructTypesListNum = StructTypesList.Num();
+		Ar << StructTypesListNum;
+	
+		for (const TWeakObjectPtr<const UStruct>& StructType : StructTypesList)
+		{
+			FTopLevelAssetPath StructPath(StructType.Get());
+			Ar << StructPath;
+		}
+
+		const int64 FinalDataOffset = Ar.Tell();
+
+		// Size of the serialized memory
+		Ar.Seek(SizeOffset);	
+		SerialSize = (int32)(FinalDataOffset - InitialDataOffset);
+		Ar << SerialSize;
+		// switch back to the end position
+		Ar.Seek(FinalDataOffset);
+
+		// serializing the actual bits
+		Ar << StructTypesBitArray;
+	}
+	else if (Ar.IsLoading())
+	{
+		uint32 LoadedSerializationHash = 0;
+		Ar << LoadedSerializationHash;
+		
+		int32 SerialSize = 0;
+		Ar << SerialSize;
+
+		const FStructTracker** KnownTracker = FBitSetSerializationDictionary::GetTracker(LoadedSerializationHash);
+
+		if (KnownTracker && *KnownTracker == this)
+		{
+			// can skip the whole Tracker saved data
+			Ar.Seek(Ar.Tell() + SerialSize);
+			Ar << StructTypesBitArray;
+		}
+		else
+		{
+			FTopLevelAssetPath BaseStructPath;
+			Ar << BaseStructPath;
+			// first, verify that the base type of the tracker matches - otherwise we can accidentally storing 
+			// the wrong types creating the Mapping (via FindOrAddStructTypeIndex below)
+			UStruct* SerializedBaseStructType = FindObject<UStruct>(BaseStructPath);
+
+			if (SerializedBaseStructType == BaseStructGetter())
+			{
+				// create a translator 
+				TArray<int32>& BitMapping = FBitSetSerializationDictionary::GetOrCreateBitMapping(LoadedSerializationHash);
+				if (BitMapping.Num() == 0)
+				{
+					int32 StructTypesListNum = 0;
+					Ar << StructTypesListNum;
+
+					BitMapping.AddUninitialized(StructTypesListNum);
+
+					for (int32 Index = 0; Index < StructTypesListNum; ++Index)
+					{
+						FTopLevelAssetPath TypePath;
+						Ar << TypePath;
+
+						if (UStruct* StructType = FindObject<UStruct>(TypePath))
+						{
+							checkSlow(StructType->IsChildOf(BaseStructGetter()));
+							BitMapping[Index] = FindOrAddStructTypeIndex(*StructType);
+						}
+						else
+						{
+							BitMapping[Index] = INDEX_NONE;
+						}
+					}
+				}
+				else
+				{
+					Ar.Seek(Ar.Tell() + SerialSize);
+				}
+
+				// it's where we read in data saved with a different order
+				TBitArray<> TempStructTypesBitArray;
+				Ar << TempStructTypesBitArray;
+			
+				StructTypesBitArray.Init(false, StructTypeToIndexSet.Num());
+				for (TBitArray<>::FConstIterator It(TempStructTypesBitArray); It; ++It)
+				{
+					if (It.GetValue())
+					{
+						const int32 TranslatedIndex = BitMapping[It.GetIndex()];
+						StructTypesBitArray.AddAtIndex(TranslatedIndex);
+					}
+				}
+			}
+			else
+			{
+				UE_LOG(LogStructUtils, Error, TEXT("Trying to load mismatching BitSet data. Current base class: %s, read base class: %s")
+					, *GetNameSafe(BaseStructGetter()), *BaseStructPath.ToString());
+			}
+		}
+	}
+}

Engine/Source/Runtime/CoreUObject/Private/StructUtils/StructUtilsTypes.cpp

@@ -0,0 +1,195 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#include "StructUtils/StructUtilsTypes.h"
+#include "Hash/CityHash.h"
+#include "Serialization/ArchiveCrc32.h"
+#include "StructUtils/StructView.h"
+#include "StructUtils/SharedStruct.h"
+#include "StructUtils/UserDefinedStruct.h"
+
+
+namespace UE::StructUtils
+{
+	uint32 GetStructCrc32(const UScriptStruct& ScriptStruct, const uint8* StructMemory, const uint32 CRC /*= 0*/)
+	{
+		FArchiveCrc32 Ar(HashCombine(CRC, PointerHash(&ScriptStruct)));
+		if (StructMemory)
+		{
+			UScriptStruct& NonConstScriptStruct = const_cast<UScriptStruct&>(ScriptStruct);
+			NonConstScriptStruct.SerializeItem(Ar, const_cast<uint8*>(StructMemory), nullptr);
+		}
+		return Ar.GetCrc();
+	}
+
+	template<typename T>
+	uint32 GetStructCrc32Helper(const T& Struct, const uint32 CRC /*= 0*/)
+	{
+		if (const UScriptStruct* ScriptStruct = Struct.GetScriptStruct())
+		{
+			return GetStructCrc32(*ScriptStruct, Struct.GetMemory(), CRC);
+		}
+		return 0;
+	}
+
+	uint32 GetStructCrc32(const FStructView& StructView, const uint32 CRC /*= 0*/)
+	{
+		return GetStructCrc32Helper(StructView, CRC);
+	}
+
+	uint32 GetStructCrc32(const FConstStructView& StructView, const uint32 CRC /*= 0*/)
+	{
+		return GetStructCrc32Helper(StructView, CRC);
+	}
+
+	uint32 GetStructCrc32(const FSharedStruct& SharedView, const uint32 CRC /*= 0*/)
+	{
+		return GetStructCrc32Helper(SharedView, CRC);
+	}
+
+	uint32 GetStructCrc32(const FConstSharedStruct& SharedView, const uint32 CRC /*= 0*/)
+	{
+		return GetStructCrc32Helper(SharedView, CRC);
+	}
+
+	class FArchiveCityHash64 : public FArchiveUObject
+	{
+	public:
+		explicit FArchiveCityHash64(const uint64 InHash = 0)
+			: Hash(InHash)
+		{}
+
+		/**
+		 * @return The hash computed so far.
+		 */
+		uint64 GetHash() const { return Hash; }
+
+		//~ Begin FArchive Interface
+		virtual void Serialize(void* Data, int64 Num) override
+		{
+			const char* BytePointer = static_cast<const char*>(Data);
+			while (Num > 0)
+			{
+				const int32 BytesToHash = static_cast<int32>(FMath::Min(Num, static_cast<int64>(MAX_int32)));
+
+				Hash = CityHash64WithSeed(BytePointer, BytesToHash, Hash);
+
+				Num -= BytesToHash;
+				BytePointer += BytesToHash;
+			}
+		}
+
+		virtual FArchive& operator<<(class FName& Name) override
+		{
+			Hash = HashString(Name.ToString(), Hash);
+			return *this;
+		}
+
+		virtual FArchive& operator<<(class UObject*& Object) override
+		{
+			Hash = HashString(GetPathNameSafe(Object), Hash);
+			return *this;
+		}
+
+		virtual FString GetArchiveName() const override { return TEXT("FArchiveCityHash64"); }
+		//~ End FArchive Interface
+
+		FORCEINLINE static uint64 HashString(const FString& InString, const uint64 InHash = 0)
+		{
+			return CityHash64WithSeed(reinterpret_cast<const char*>(*InString), static_cast<uint32>(InString.GetAllocatedSize()), InHash);
+		}
+
+	private:
+		uint64 Hash;
+	};
+
+	uint64 GetStructHash64(const UScriptStruct& ScriptStruct, const uint8* StructMemory)
+	{
+		const uint64 BaseHash = FArchiveCityHash64::HashString(ScriptStruct.GetPathName());
+
+		if (ScriptStruct.GetCppStructOps()->HasGetTypeHash())
+		{
+			const uint32 StructHash = ScriptStruct.GetStructTypeHash(StructMemory);
+			return CityHash128to64(Uint128_64(BaseHash, StructHash));
+		}
+		else if (StructMemory)
+		{
+			FArchiveCityHash64 Ar(BaseHash);
+			UScriptStruct& NonConstScriptStruct = const_cast<UScriptStruct&>(ScriptStruct);
+			NonConstScriptStruct.SerializeItem(Ar, const_cast<uint8*>(StructMemory), nullptr);
+
+			return Ar.GetHash();
+		}
+		
+		return BaseHash;
+	}
+
+	template<typename T>
+	uint64 GetStructHash64Helper(const T& Struct)
+	{
+		if (const UScriptStruct* ScriptStruct = Struct.GetScriptStruct())
+		{
+			return GetStructHash64(*ScriptStruct, Struct.GetMemory());
+		}
+		return 0;
+	}
+
+	uint64 GetStructHash64(const FStructView& StructView)
+	{
+		return GetStructHash64Helper(StructView);
+	}
+
+	uint64 GetStructHash64(const FConstStructView& StructView)
+	{
+		return GetStructHash64Helper(StructView);
+	}
+
+	uint64 GetStructHash64(const FSharedStruct& SharedView)
+	{
+		return GetStructHash64Helper(SharedView);
+	}
+
+	uint64 GetStructHash64(const FConstSharedStruct& SharedView)
+	{
+		return GetStructHash64Helper(SharedView);
+	}
+
+	namespace Private
+	{
+#if WITH_EDITOR
+		const UUserDefinedStruct* GStructureToReinstantiate = nullptr;
+		UObject* GCurrentReinstantiationOuterObject = nullptr;
+
+		FStructureToReinstantiateScope::FStructureToReinstantiateScope(const UUserDefinedStruct* StructureToReinstantiate)
+		{
+			OldStructureToReinstantiate = GStructureToReinstantiate;
+			GStructureToReinstantiate = StructureToReinstantiate;
+		}
+	
+		FStructureToReinstantiateScope::~FStructureToReinstantiateScope()
+		{
+			GStructureToReinstantiate = OldStructureToReinstantiate;
+		}
+
+		FCurrentReinstantiationOuterObjectScope::FCurrentReinstantiationOuterObjectScope(UObject* CurrentReinstantiateOuterObject)
+		{
+			OldCurrentReinstantiateOuterObject = GCurrentReinstantiationOuterObject;
+			GCurrentReinstantiationOuterObject = CurrentReinstantiateOuterObject;
+		}
+
+		FCurrentReinstantiationOuterObjectScope::~FCurrentReinstantiationOuterObjectScope()
+		{
+			GCurrentReinstantiationOuterObject = OldCurrentReinstantiateOuterObject;
+		}
+
+		const UUserDefinedStruct* GetStructureToReinstantiate()
+		{
+			return GStructureToReinstantiate;
+		}
+	
+		UObject* GetCurrentReinstantiationOuterObject()
+		{
+			return GCurrentReinstantiationOuterObject;
+		}
+#endif // WITH_EDITOR
+	}
+}