// Copyright Epic Games, Inc. All Rights Reserved. using EpicGames.Core; using System; using System.Buffers.Binary; using System.Collections.Generic; using System.Linq; using System.Text; using System.Text.Json; namespace EpicGames.Serialization { /// /// Exception for /// public class CbWriterException : Exception { /// /// Constructor /// /// public CbWriterException(string Message) : base(Message) { } /// /// Constructor /// /// /// public CbWriterException(string Message, Exception? Ex) : base(Message, Ex) { } } /// /// Forward-only writer for compact binary objects /// public class CbWriter { /// /// Stores information about an object or array scope within the written buffer which requires a header to be inserted containing /// the size or number of elements when copied to an output buffer /// class Scope { public CbFieldType FieldType; public CbFieldType UniformFieldType; public int Offset; // Offset to insert the length/count public int Length; // Excludes the size of this field's headers, and child fields' headers. public int Count; public List? Children; public int SizeOfChildHeaders; // Sum of additional headers for child items, recursively. public Scope(CbFieldType FieldType, CbFieldType UniformFieldType, int Offset) { this.FieldType = FieldType; this.UniformFieldType = UniformFieldType; this.Offset = Offset; } } /// /// Chunk of written data. Chunks are allocated as needed and chained together with scope annotations to produce the output data. /// class Chunk { public int Offset; public int Length; public byte[] Data; public List Scopes = new List(); public Chunk(int Offset, int MaxLength) { this.Offset = Offset; this.Data = new byte[MaxLength]; } } const int DefaultChunkSize = 1024; List Chunks = new List(); Stack OpenScopes = new Stack(); Chunk CurrentChunk => Chunks[Chunks.Count - 1]; Scope CurrentScope => OpenScopes.Peek(); int CurrentOffset; /// /// Constructor /// public CbWriter() : this(DefaultChunkSize) { } /// /// Constructor /// /// Amount of data to reserve for output public CbWriter(int Reserve) { Chunks.Add(new Chunk(0, Reserve)); OpenScopes.Push(new Scope(CbFieldType.Array, CbFieldType.None, 0)); } /// /// Ensure that a block of contiguous memory of the given length is available in the output buffer /// /// /// The allocated memory Memory Allocate(int Length) { Chunk LastChunk = CurrentChunk; if (LastChunk.Length + Length > LastChunk.Data.Length) { int ChunkSize = Math.Max(Length, DefaultChunkSize); LastChunk = new Chunk(CurrentOffset, ChunkSize); Chunks.Add(LastChunk); } Memory Buffer = LastChunk.Data.AsMemory(LastChunk.Length, Length); LastChunk.Length += Length; CurrentOffset += Length; return Buffer; } /// /// Insert a new scope /// /// /// void PushScope(CbFieldType FieldType, CbFieldType UniformFieldType) { Scope NewScope = new Scope(FieldType, UniformFieldType, CurrentOffset); CurrentScope.Children ??= new List(); CurrentScope.Children.Add(NewScope); OpenScopes.Push(NewScope); CurrentChunk.Scopes.Add(NewScope); } /// /// Pop a scope from the current open list /// void PopScope() { Scope Scope = CurrentScope; Scope.Length = CurrentOffset - Scope.Offset; Scope.SizeOfChildHeaders = ComputeSizeOfChildHeaders(Scope); OpenScopes.Pop(); } /// /// Writes the header for an unnamed field /// /// void WriteField(CbFieldType Type) { Scope Scope = CurrentScope; if (!CbFieldUtils.IsArray(Scope.FieldType)) { throw new CbWriterException($"Anonymous fields are not allowed within fields of type {Scope.FieldType}"); } if (Scope.UniformFieldType == CbFieldType.None) { Allocate(1).Span[0] = (byte)Type; } else if (Scope.UniformFieldType != Type) { throw new CbWriterException($"Mismatched type for uniform array - expected {Scope.UniformFieldType}, not {Type}"); } Scope.Count++; } /// /// Writes the header for a named field /// /// /// void WriteField(CbFieldType Type, Utf8String Name) { Scope Scope = CurrentScope; if (!CbFieldUtils.IsObject(Scope.FieldType)) { throw new CbWriterException($"Named fields are not allowed within fields of type {Scope.FieldType}"); } int NameVarIntLength = VarInt.Measure(Name.Length); if (Scope.UniformFieldType == CbFieldType.None) { Span Buffer = Allocate(1 + NameVarIntLength + Name.Length).Span; Buffer[0] = (byte)(Type | CbFieldType.HasFieldName); WriteBinaryPayload(Buffer[1..], Name.Span); } else { if (Scope.UniformFieldType != Type) { throw new CbWriterException($"Mismatched type for uniform object - expected {Scope.UniformFieldType}, not {Type}"); } WriteBinaryPayload(Name.Span); } Scope.Count++; } /// /// Begin writing an object field /// public void BeginObject() { WriteField(CbFieldType.Object); PushScope(CbFieldType.Object, CbFieldType.None); } /// /// Begin writing an object field /// /// Name of the field public void BeginObject(Utf8String Name) { WriteField(CbFieldType.Object, Name); PushScope(CbFieldType.Object, CbFieldType.None); } /// /// End the current object /// public void EndObject() { PopScope(); } /// /// Begin writing an array field /// public void BeginArray() { WriteField(CbFieldType.Array); PushScope(CbFieldType.Array, CbFieldType.None); } /// /// Begin writing a named array field /// /// public void BeginArray(Utf8String Name) { WriteField(CbFieldType.Array, Name); PushScope(CbFieldType.Array, CbFieldType.None); } /// /// End the current array /// public void EndArray() { PopScope(); } /// /// Begin writing a uniform array field /// /// The field type for elements in the array public void BeginUniformArray(CbFieldType FieldType) { WriteField(CbFieldType.UniformArray); PushScope(CbFieldType.UniformArray, FieldType); Allocate(1).Span[0] = (byte)FieldType; } /// /// Begin writing a named uniform array field /// /// Name of the field /// The field type for elements in the array public void BeginUniformArray(Utf8String Name, CbFieldType FieldType) { WriteField(CbFieldType.UniformArray, Name); PushScope(CbFieldType.UniformArray, FieldType); Allocate(1).Span[0] = (byte)FieldType; } /// /// End the current array /// public void EndUniformArray() { PopScope(); } /// /// Write a null field /// public void WriteNullValue() { WriteField(CbFieldType.Null); } /// /// Write a named null field /// /// Name of the field public void WriteNull(Utf8String Name) { WriteField(CbFieldType.Null, Name); } /// /// Writes a boolean value /// /// public void WriteBoolValue(bool Value) { WriteField(Value? CbFieldType.BoolTrue : CbFieldType.BoolFalse); } /// /// Writes a boolean value /// /// Name of the field /// public void WriteBool(Utf8String Name, bool Value) { WriteField(Value ? CbFieldType.BoolTrue : CbFieldType.BoolFalse, Name); } /// /// Writes the payload for an integer /// /// Value to write void WriteIntegerPayload(ulong Value) { int Length = VarInt.Measure(Value); Span Buffer = Allocate(Length).Span; VarInt.Write(Buffer, Value); } /// /// Writes an unnamed integer field /// /// Value to be written public void WriteIntegerValue(long Value) { if (Value >= 0) { WriteField(CbFieldType.IntegerPositive); WriteIntegerPayload((ulong)Value); } else { WriteField(CbFieldType.IntegerNegative); WriteIntegerPayload((ulong)-Value); } } /// /// Writes an named integer field /// /// Name of the field /// Value to be written public void WriteInteger(Utf8String Name, long Value) { if (Value >= 0) { WriteField(CbFieldType.IntegerPositive, Name); WriteIntegerPayload((ulong)Value); } else { WriteField(CbFieldType.IntegerNegative, Name); WriteIntegerPayload((ulong)-Value); } } /// /// Writes an unnamed integer field /// /// Value to be written public void WriteIntegerValue(ulong Value) { WriteField(CbFieldType.IntegerPositive); WriteIntegerPayload(Value); } /// /// Writes a named integer field /// /// Name of the field /// Value to be written public void WriteInteger(Utf8String Name, ulong Value) { WriteField(CbFieldType.IntegerPositive, Name); WriteIntegerPayload(Value); } /// /// Writes the payload for a value /// /// The value to write void WriteDateTimePayload(DateTime DateTime) { Span Buffer = Allocate(sizeof(long)).Span; BinaryPrimitives.WriteInt64BigEndian(Buffer, DateTime.Ticks); } /// /// Writes an unnamed field /// /// Value to be written public void WriteDateTimeValue(DateTime Value) { WriteField(CbFieldType.DateTime); WriteDateTimePayload(Value); } /// /// Writes a named field /// /// Name of the field /// Value to be written public void WriteDateTime(Utf8String Name, DateTime Value) { WriteField(CbFieldType.DateTime, Name); WriteDateTimePayload(Value); } /// /// Writes the payload for a hash /// /// void WriteHashPayload(IoHash Hash) { Span Buffer = Allocate(IoHash.NumBytes).Span; Hash.Span.CopyTo(Buffer); } /// /// Writes an unnamed field /// /// public void WriteHashValue(IoHash Hash) { WriteField(CbFieldType.Hash); WriteHashPayload(Hash); } /// /// Writes a named field /// /// Name of the field /// Value to be written public void WriteHash(Utf8String Name, IoHash Value) { WriteField(CbFieldType.Hash, Name); WriteHashPayload(Value); } /// /// Writes an unnamed reference to a binary attachment /// /// Hash of the attachment public void WriteBinaryAttachmentValue(IoHash Hash) { WriteField(CbFieldType.BinaryAttachment); WriteHashPayload(Hash); } /// /// Writes a named reference to a binary attachment /// /// Name of the field /// Hash of the attachment public void WriteBinaryAttachment(Utf8String Name, IoHash Hash) { WriteField(CbFieldType.BinaryAttachment, Name); WriteHashPayload(Hash); } /// /// Writes the payload for an object to the buffer /// /// void WriteObjectPayload(CbObject Object) { CbField Field = Object.AsField(); Memory Buffer = Allocate(Field.Payload.Length); Field.Payload.CopyTo(Buffer); } /// /// Writes an object directly into the writer /// /// Object to write public void WriteObject(CbObject Object) { WriteField(CbFieldType.Object); WriteObjectPayload(Object); } /// /// Writes an object directly into the writer /// /// Name of the object /// Object to write public void WriteObject(Utf8String Name, CbObject Object) { WriteField(CbFieldType.Object, Name); WriteObjectPayload(Object); } /// /// Writes an unnamed reference to an object attachment /// /// Hash of the attachment public void WriteObjectAttachmentValue(IoHash Hash) { WriteField(CbFieldType.ObjectAttachment); WriteHashPayload(Hash); } /// /// Writes a named reference to an object attachment /// /// Name of the field /// Hash of the attachment public void WriteObjectAttachment(Utf8String Name, IoHash Hash) { WriteField(CbFieldType.ObjectAttachment, Name); WriteHashPayload(Hash); } /// /// Writes the payload for a binary value /// /// Output buffer /// Value to be written static void WriteBinaryPayload(Span Output, ReadOnlySpan Value) { int VarIntLength = VarInt.Write(Output, Value.Length); Output = Output[VarIntLength..]; Value.CopyTo(Output); CheckSize(Output, Value.Length); } /// /// Writes the payload for a binary value /// /// Value to be written void WriteBinaryPayload(ReadOnlySpan Value) { int ValueVarIntLength = VarInt.Measure(Value.Length); Span Buffer = Allocate(ValueVarIntLength + Value.Length).Span; WriteBinaryPayload(Buffer, Value); } /// /// Writes an unnamed string value /// /// Value to be written public void WriteStringValue(Utf8String Value) { WriteField(CbFieldType.String); WriteBinaryPayload(Value.Span); } /// /// Writes a named string value /// /// Name of the field /// Value to be written public void WriteString(Utf8String Name, Utf8String Value) { WriteField(CbFieldType.String, Name); WriteBinaryPayload(Value.Span); } /// /// Writes an unnamed binary value /// /// Value to be written public void WriteBinaryValue(ReadOnlySpan Value) { WriteField(CbFieldType.Binary); WriteBinaryPayload(Value); } /// /// Writes a named binary value /// /// Name of the field /// Value to be written public void WriteBinary(Utf8String Name, ReadOnlySpan Value) { WriteField(CbFieldType.Binary, Name); WriteBinaryPayload(Value); } /// /// Check that the given span is the required size /// /// /// static void CheckSize(Span Span, int ExpectedSize) { if (Span.Length != ExpectedSize) { throw new Exception("Size of buffer is not correct"); } } /// /// Gets the size of the serialized data /// /// public int GetSize() { if (OpenScopes.Count != 1) { throw new CbWriterException("Unfinished scope in writer"); } return CurrentOffset + ComputeSizeOfChildHeaders(CurrentScope); } /// /// Copy the data from this writer to a buffer /// /// public void CopyTo(Span Buffer) { int BufferOffset = 0; int SourceOffset = 0; foreach (Chunk Chunk in Chunks) { foreach (Scope Scope in Chunk.Scopes) { ReadOnlySpan SourceData = Chunk.Data.AsSpan(SourceOffset - Chunk.Offset, Scope.Offset - SourceOffset); SourceData.CopyTo(Buffer.Slice(BufferOffset)); BufferOffset += SourceData.Length; SourceOffset += SourceData.Length; BufferOffset += WriteScopeHeader(Buffer.Slice(BufferOffset), Scope); } ReadOnlySpan LastSourceData = Chunk.Data.AsSpan(SourceOffset - Chunk.Offset, (Chunk.Offset + Chunk.Length) - SourceOffset); LastSourceData.CopyTo(Buffer.Slice(BufferOffset)); BufferOffset += LastSourceData.Length; SourceOffset += LastSourceData.Length; } } /// /// Convert the data into a compact binary object /// /// public CbObject ToObject() { return new CbObject(ToByteArray()); } /// /// Convert the data into a flat array /// /// public byte[] ToByteArray() { byte[] Buffer = new byte[GetSize()]; CopyTo(Buffer); return Buffer; } /// /// Comptues the size of any child headers /// /// static int ComputeSizeOfChildHeaders(Scope Scope) { int SizeOfChildHeaders = 0; if (Scope.Children != null) { foreach (Scope ChildScope in Scope.Children) { switch (ChildScope.FieldType) { case CbFieldType.Object: case CbFieldType.UniformObject: SizeOfChildHeaders += ChildScope.SizeOfChildHeaders + VarInt.Measure(ChildScope.Length + ChildScope.SizeOfChildHeaders); break; case CbFieldType.Array: case CbFieldType.UniformArray: int ArrayCountLength = VarInt.Measure(ChildScope.Count); SizeOfChildHeaders += ChildScope.SizeOfChildHeaders + VarInt.Measure(ChildScope.Length + ChildScope.SizeOfChildHeaders + ArrayCountLength) + ArrayCountLength; break; default: throw new InvalidOperationException(); } } } return SizeOfChildHeaders; } /// /// Writes the header for a particular scope /// /// /// /// static int WriteScopeHeader(Span Span, Scope Scope) { switch (Scope.FieldType) { case CbFieldType.Object: case CbFieldType.UniformObject: return VarInt.Write(Span, Scope.Length + Scope.SizeOfChildHeaders); case CbFieldType.Array: case CbFieldType.UniformArray: int NumItemsLength = VarInt.Measure(Scope.Count); int Offset = VarInt.Write(Span, Scope.Length + Scope.SizeOfChildHeaders + NumItemsLength); return Offset + VarInt.Write(Span.Slice(Offset), Scope.Count); default: throw new InvalidOperationException(); } } } }