// Copyright Epic Games, Inc. All Rights Reserved. using EpicGames.Core; using System; using System.Buffers.Binary; using System.Collections.Generic; using System.IO; namespace EpicGames.Serialization { /// /// Exception for /// public class CbWriterException : Exception { /// /// Constructor /// /// public CbWriterException(string message) : base(message) { } /// /// Constructor /// /// /// public CbWriterException(string message, Exception? ex) : base(message, ex) { } } /// /// Forward-only writer for compact binary objects /// public class CbWriter { /// /// Stores information about an object or array scope within the written buffer which requires a header to be inserted containing /// the size or number of elements when copied to an output buffer /// class Scope { public CbFieldType _fieldType; public CbFieldType _uniformFieldType; public int _offset; // Offset to insert the length/count public int _length; // Excludes the size of this field's headers, and child fields' headers. public int _count; public List _children = new List(); public int _sizeOfChildHeaders; // Sum of additional headers for child items, recursively. public Scope(CbFieldType fieldType, CbFieldType uniformFieldType, int offset) { Reset(fieldType, uniformFieldType, offset); } public void Reset(CbFieldType fieldType, CbFieldType uniformFieldType, int offset) { _fieldType = fieldType; _uniformFieldType = uniformFieldType; _offset = offset; _length = 0; _count = 0; _children.Clear(); _sizeOfChildHeaders = 0; } } /// /// Chunk of written data. Chunks are allocated as needed and chained together with scope annotations to produce the output data. /// class Chunk { public int _offset; public int _length; public byte[] _data; public List _scopes = new List(); public Chunk(int offset, int maxLength) { _data = new byte[maxLength]; Reset(offset); } public void Reset(int offset) { _offset = offset; _length = 0; _scopes.Clear(); } } const int DefaultChunkSize = 1024; readonly List _chunks = new List(); readonly Stack _openScopes = new Stack(); Chunk CurrentChunk => _chunks[^1]; Scope CurrentScope => _openScopes.Peek(); int _currentOffset; readonly List _freeChunks = new List(); readonly List _freeScopes = new List(); /// /// Constructor /// public CbWriter() : this(DefaultChunkSize) { } /// /// Constructor /// /// Amount of data to reserve for output public CbWriter(int reserve) { _chunks.Add(new Chunk(0, reserve)); _openScopes.Push(new Scope(CbFieldType.Array, CbFieldType.None, 0)); } /// /// /// public void Clear() { foreach (Chunk chunk in _chunks) { FreeChunk(chunk); } _currentOffset = 0; _chunks.Clear(); _chunks.Add(AllocChunk(0, DefaultChunkSize)); _openScopes.Clear(); _openScopes.Push(AllocScope(CbFieldType.Array, CbFieldType.None, 0)); } /// /// Allocate a new chunk object /// /// Offset of the chunk /// Maximum length of the chunk /// New chunk object Chunk AllocChunk(int offset, int maxLength) { for(int idx = _freeChunks.Count - 1; idx >= 0; idx--) { Chunk chunk = _freeChunks[idx]; if (chunk._data.Length >= maxLength) { _freeChunks.RemoveAt(idx); chunk.Reset(offset); return chunk; } } return new Chunk(offset, maxLength); } /// /// Adds a chunk to the free list /// /// void FreeChunk(Chunk chunk) { // Add the scopes to the free list _freeScopes.AddRange(chunk._scopes); chunk._scopes.Clear(); // Insert it into the free list, sorted by descending size for (int idx = 0; ; idx++) { if (idx == _freeChunks.Count || chunk._data.Length >= _freeChunks[idx]._data.Length) { _freeChunks.Insert(idx, chunk); break; } } } /// /// Allocate a scope object /// /// /// /// /// Scope AllocScope(CbFieldType fieldType, CbFieldType uniformFieldType, int offset) { if (_freeScopes.Count > 0) { Scope scope = _freeScopes[^1]; scope.Reset(fieldType, uniformFieldType, offset); _freeScopes.RemoveAt(_freeScopes.Count - 1); return scope; } return new Scope(fieldType, uniformFieldType, offset); } /// /// Ensure that a block of contiguous memory of the given length is available in the output buffer /// /// /// The allocated memory Memory Allocate(int length) { Chunk lastChunk = CurrentChunk; if (lastChunk._length + length > lastChunk._data.Length) { int chunkSize = Math.Max(length, DefaultChunkSize); lastChunk = AllocChunk(_currentOffset, chunkSize); _chunks.Add(lastChunk); } Memory buffer = lastChunk._data.AsMemory(lastChunk._length, length); lastChunk._length += length; _currentOffset += length; return buffer; } /// /// Insert a new scope /// /// /// void PushScope(CbFieldType fieldType, CbFieldType uniformFieldType) { Scope newScope = AllocScope(fieldType, uniformFieldType, _currentOffset); CurrentScope._children.Add(newScope); _openScopes.Push(newScope); CurrentChunk._scopes.Add(newScope); } /// /// Pop a scope from the current open list /// void PopScope() { Scope scope = CurrentScope; scope._length = _currentOffset - scope._offset; scope._sizeOfChildHeaders = ComputeSizeOfChildHeaders(scope); _openScopes.Pop(); } /// /// Writes the header for an unnamed field /// /// void WriteFieldHeader(CbFieldType type) { Scope scope = CurrentScope; if (!CbFieldUtils.IsArray(scope._fieldType)) { throw new CbWriterException($"Anonymous fields are not allowed within fields of type {scope._fieldType}"); } if (scope._uniformFieldType == CbFieldType.None) { Allocate(1).Span[0] = (byte)type; } else if (scope._uniformFieldType != type) { throw new CbWriterException($"Mismatched type for uniform array - expected {scope._uniformFieldType}, not {type}"); } scope._count++; } /// /// Writes the header for a named field /// /// /// void WriteFieldHeader(CbFieldType type, Utf8String name) { Scope scope = CurrentScope; if (!CbFieldUtils.IsObject(scope._fieldType)) { throw new CbWriterException($"Named fields are not allowed within fields of type {scope._fieldType}"); } int nameVarIntLength = VarInt.Measure(name.Length); if (scope._uniformFieldType == CbFieldType.None) { Span buffer = Allocate(1 + nameVarIntLength + name.Length).Span; buffer[0] = (byte)(type | CbFieldType.HasFieldName); WriteBinaryPayload(buffer[1..], name.Span); } else { if (scope._uniformFieldType != type) { throw new CbWriterException($"Mismatched type for uniform object - expected {scope._uniformFieldType}, not {type}"); } WriteBinaryPayload(name.Span); } scope._count++; } /// /// Copies an entire field value to the output /// /// public void WriteFieldValue(CbField field) { WriteFieldHeader(field.GetType()); int size = (int)field.GetPayloadSize(); field.GetPayloadView().CopyTo(Allocate(size)); } /// /// Copies an entire field value to the output, using the name from the field /// /// public void WriteField(CbField field) => WriteField(field.GetName(), field); /// /// Copies an entire field value to the output /// /// public void WriteField(Utf8String name, CbField field) { WriteFieldHeader(field.GetType(), name); int size = (int)field.GetPayloadSize(); field.GetPayloadView().CopyTo(Allocate(size)); } /// /// Begin writing an object field /// public void BeginObject() { WriteFieldHeader(CbFieldType.Object); PushScope(CbFieldType.Object, CbFieldType.None); } /// /// Begin writing an object field /// /// Name of the field public void BeginObject(Utf8String name) { WriteFieldHeader(CbFieldType.Object, name); PushScope(CbFieldType.Object, CbFieldType.None); } /// /// End the current object /// public void EndObject() { PopScope(); } /// /// Begin writing an array field /// public void BeginArray() { WriteFieldHeader(CbFieldType.Array); PushScope(CbFieldType.Array, CbFieldType.None); } /// /// Begin writing a named array field /// /// public void BeginArray(Utf8String name) { WriteFieldHeader(CbFieldType.Array, name); PushScope(CbFieldType.Array, CbFieldType.None); } /// /// End the current array /// public void EndArray() { PopScope(); } /// /// Begin writing a uniform array field /// /// The field type for elements in the array public void BeginUniformArray(CbFieldType fieldType) { WriteFieldHeader(CbFieldType.UniformArray); PushScope(CbFieldType.UniformArray, fieldType); Allocate(1).Span[0] = (byte)fieldType; } /// /// Begin writing a named uniform array field /// /// Name of the field /// The field type for elements in the array public void BeginUniformArray(Utf8String name, CbFieldType fieldType) { WriteFieldHeader(CbFieldType.UniformArray, name); PushScope(CbFieldType.UniformArray, fieldType); Allocate(1).Span[0] = (byte)fieldType; } /// /// End the current array /// public void EndUniformArray() { PopScope(); } /// /// Write a null field /// public void WriteNullValue() { WriteFieldHeader(CbFieldType.Null); } /// /// Write a named null field /// /// Name of the field public void WriteNull(Utf8String name) { WriteFieldHeader(CbFieldType.Null, name); } /// /// Writes a boolean value /// /// public void WriteBoolValue(bool value) { WriteFieldHeader(value? CbFieldType.BoolTrue : CbFieldType.BoolFalse); } /// /// Writes a boolean value /// /// Name of the field /// public void WriteBool(Utf8String name, bool value) { WriteFieldHeader(value ? CbFieldType.BoolTrue : CbFieldType.BoolFalse, name); } /// /// Writes the payload for an integer /// /// Value to write void WriteIntegerPayload(ulong value) { int length = VarInt.Measure(value); Span buffer = Allocate(length).Span; VarInt.Write(buffer, value); } /// /// Writes an unnamed integer field /// /// Value to be written public void WriteIntegerValue(int value) { WriteIntegerValue((long)value); } /// /// Writes an unnamed integer field /// /// Value to be written public void WriteIntegerValue(long value) { if (value >= 0) { WriteFieldHeader(CbFieldType.IntegerPositive); WriteIntegerPayload((ulong)value); } else { WriteFieldHeader(CbFieldType.IntegerNegative); WriteIntegerPayload((ulong)-value); } } /// /// Writes an named integer field /// /// Name of the field /// Value to be written public void WriteInteger(Utf8String name, int value) { WriteInteger(name, (long)value); } /// /// Writes an named integer field /// /// Name of the field /// Value to be written public void WriteInteger(Utf8String name, long value) { if (value >= 0) { WriteFieldHeader(CbFieldType.IntegerPositive, name); WriteIntegerPayload((ulong)value); } else { WriteFieldHeader(CbFieldType.IntegerNegative, name); WriteIntegerPayload((ulong)-value); } } /// /// Writes an unnamed integer field /// /// Value to be written public void WriteIntegerValue(ulong value) { WriteFieldHeader(CbFieldType.IntegerPositive); WriteIntegerPayload(value); } /// /// Writes a named integer field /// /// Name of the field /// Value to be written public void WriteInteger(Utf8String name, ulong value) { WriteFieldHeader(CbFieldType.IntegerPositive, name); WriteIntegerPayload(value); } /// /// Writes an unnamed double field /// /// Value to be written public void WriteDoubleValue(double value) { WriteFieldHeader(CbFieldType.Float64); BinaryPrimitives.WriteInt64BigEndian(Allocate(sizeof(double)).Span, BitConverter.DoubleToInt64Bits(value)); } /// /// Writes a named double field /// /// Name of the field /// Value to be written public void WriteDouble(Utf8String name, double value) { WriteFieldHeader(CbFieldType.Float64, name); BinaryPrimitives.WriteInt64BigEndian(Allocate(sizeof(double)).Span, BitConverter.DoubleToInt64Bits(value)); } /// /// Writes the payload for a value /// /// The value to write void WriteDateTimePayload(DateTime dateTime) { Span buffer = Allocate(sizeof(long)).Span; BinaryPrimitives.WriteInt64BigEndian(buffer, dateTime.Ticks); } /// /// Writes an unnamed field /// /// Value to be written public void WriteDateTimeValue(DateTime value) { WriteFieldHeader(CbFieldType.DateTime); WriteDateTimePayload(value); } /// /// Writes a named field /// /// Name of the field /// Value to be written public void WriteDateTime(Utf8String name, DateTime value) { WriteFieldHeader(CbFieldType.DateTime, name); WriteDateTimePayload(value); } /// /// Writes the payload for a hash /// /// void WriteHashPayload(IoHash hash) { Span buffer = Allocate(IoHash.NumBytes).Span; hash.CopyTo(buffer); } /// /// Writes an unnamed field /// /// public void WriteHashValue(IoHash hash) { WriteFieldHeader(CbFieldType.Hash); WriteHashPayload(hash); } /// /// Writes a named field /// /// Name of the field /// Value to be written public void WriteHash(Utf8String name, IoHash value) { WriteFieldHeader(CbFieldType.Hash, name); WriteHashPayload(value); } /// /// Writes an unnamed reference to a binary attachment /// /// Hash of the attachment public void WriteBinaryAttachmentValue(IoHash hash) { WriteFieldHeader(CbFieldType.BinaryAttachment); WriteHashPayload(hash); } /// /// Writes a named reference to a binary attachment /// /// Name of the field /// Hash of the attachment public void WriteBinaryAttachment(Utf8String name, IoHash hash) { WriteFieldHeader(CbFieldType.BinaryAttachment, name); WriteHashPayload(hash); } /// /// Writes the payload for an object to the buffer /// /// void WriteObjectPayload(CbObject obj) { CbField field = obj.AsField(); Memory buffer = Allocate(field.Payload.Length); field.Payload.CopyTo(buffer); } /// /// Writes an object directly into the writer /// /// Object to write public void WriteObject(CbObject obj) { WriteFieldHeader(CbFieldType.Object); WriteObjectPayload(obj); } /// /// Writes an object directly into the writer /// /// Name of the object /// Object to write public void WriteObject(Utf8String name, CbObject obj) { WriteFieldHeader(CbFieldType.Object, name); WriteObjectPayload(obj); } /// /// Writes an unnamed reference to an object attachment /// /// Hash of the attachment public void WriteObjectAttachmentValue(IoHash hash) { WriteFieldHeader(CbFieldType.ObjectAttachment); WriteHashPayload(hash); } /// /// Writes a named reference to an object attachment /// /// Name of the field /// Hash of the attachment public void WriteObjectAttachment(Utf8String name, IoHash hash) { WriteFieldHeader(CbFieldType.ObjectAttachment, name); WriteHashPayload(hash); } /// /// Writes the payload for a binary value /// /// Output buffer /// Value to be written static void WriteBinaryPayload(Span output, ReadOnlySpan value) { int varIntLength = VarInt.Write(output, value.Length); output = output[varIntLength..]; value.CopyTo(output); CheckSize(output, value.Length); } /// /// Writes the payload for a binary value /// /// Value to be written void WriteBinaryPayload(ReadOnlySpan value) { int valueVarIntLength = VarInt.Measure(value.Length); Span buffer = Allocate(valueVarIntLength + value.Length).Span; WriteBinaryPayload(buffer, value); } /// /// Writes an unnamed string value /// /// Value to be written public void WriteStringValue(string value) => WriteUtf8StringValue(value); /// /// Writes a named string value /// /// Name of the field /// Value to be written public void WriteString(Utf8String name, string? value) { if(value != null) { WriteUtf8String(name, value); } } /// /// Writes an unnamed string value /// /// Value to be written public void WriteUtf8StringValue(Utf8String value) { WriteFieldHeader(CbFieldType.String); WriteBinaryPayload(value.Span); } /// /// Writes a named string value /// /// Name of the field /// Value to be written public void WriteUtf8String(Utf8String name, Utf8String value) { if (value.Length > 0) { WriteFieldHeader(CbFieldType.String, name); WriteBinaryPayload(value.Span); } } /// /// Writes an unnamed binary value /// /// Value to be written public void WriteBinarySpanValue(ReadOnlySpan value) { WriteFieldHeader(CbFieldType.Binary); WriteBinaryPayload(value); } /// /// Writes a named binary value /// /// Name of the field /// Value to be written public void WriteBinarySpan(Utf8String name, ReadOnlySpan value) { WriteFieldHeader(CbFieldType.Binary, name); WriteBinaryPayload(value); } /// /// Writes an unnamed binary value /// /// Value to be written public void WriteBinaryValue(ReadOnlyMemory value) { WriteBinarySpanValue(value.Span); } /// /// Writes a named binary value /// /// Name of the field /// Value to be written public void WriteBinary(Utf8String name, ReadOnlyMemory value) { WriteBinarySpan(name, value.Span); } /// /// Writes an unnamed binary value /// /// Value to be written public void WriteBinaryArrayValue(byte[] value) { WriteBinarySpanValue(value.AsSpan()); } /// /// Writes a named binary value /// /// Name of the field /// Value to be written public void WriteBinaryArray(Utf8String name, byte[] value) { WriteBinarySpan(name, value.AsSpan()); } /// /// Check that the given span is the required size /// /// /// static void CheckSize(Span span, int expectedSize) { if (span.Length != expectedSize) { throw new Exception("Size of buffer is not correct"); } } /// /// Computes the hash for this object /// /// Hash for the object public IoHash ComputeHash() { using (Blake3.Hasher hasher = Blake3.Hasher.New()) { foreach (ReadOnlyMemory segment in EnumerateSegments()) { hasher.Update(segment.Span); } return IoHash.FromBlake3(hasher); } } /// /// Gets the size of the serialized data /// /// public int GetSize() { if (_openScopes.Count != 1) { throw new CbWriterException("Unfinished scope in writer"); } return _currentOffset + ComputeSizeOfChildHeaders(CurrentScope); } /// /// Gets the contents of this writer as a stream /// /// New stream for the contents of this object public Stream AsStream() => new ReadStream(EnumerateSegments().GetEnumerator(), GetSize()); private IEnumerable> EnumerateSegments() { byte[] scopeHeader = new byte[64]; int sourceOffset = 0; foreach (Chunk chunk in _chunks) { foreach (Scope scope in chunk._scopes) { ReadOnlyMemory sourceData = chunk._data.AsMemory(sourceOffset - chunk._offset, scope._offset - sourceOffset); yield return sourceData; sourceOffset += sourceData.Length; int headerLength = WriteScopeHeader(scopeHeader, scope); yield return scopeHeader.AsMemory(0, headerLength); } ReadOnlyMemory lastSourceData = chunk._data.AsMemory(sourceOffset - chunk._offset, (chunk._offset + chunk._length) - sourceOffset); yield return lastSourceData; sourceOffset += lastSourceData.Length; } } /// /// Copy the data from this writer to a buffer /// /// public void CopyTo(Span buffer) { int bufferOffset = 0; int sourceOffset = 0; foreach (Chunk chunk in _chunks) { foreach (Scope scope in chunk._scopes) { ReadOnlySpan sourceData = chunk._data.AsSpan(sourceOffset - chunk._offset, scope._offset - sourceOffset); sourceData.CopyTo(buffer.Slice(bufferOffset)); bufferOffset += sourceData.Length; sourceOffset += sourceData.Length; bufferOffset += WriteScopeHeader(buffer.Slice(bufferOffset), scope); } ReadOnlySpan lastSourceData = chunk._data.AsSpan(sourceOffset - chunk._offset, (chunk._offset + chunk._length) - sourceOffset); lastSourceData.CopyTo(buffer.Slice(bufferOffset)); bufferOffset += lastSourceData.Length; sourceOffset += lastSourceData.Length; } } class ReadStream : Stream { readonly IEnumerator> _enumerator; ReadOnlyMemory _segment; long _positionInternal; public ReadStream(IEnumerator> enumerator, long length) { _enumerator = enumerator; Length = length; } /// public override bool CanRead => true; /// public override bool CanSeek => false; /// public override bool CanWrite => false; /// public override long Length { get; } /// public override long Position { get => _positionInternal; set => throw new NotSupportedException(); } /// public override void Flush() { } /// public override int Read(Span buffer) { int readLength = 0; while (readLength < buffer.Length) { while (_segment.Length == 0) { if (!_enumerator.MoveNext()) { return readLength; } _segment = _enumerator.Current; } int copyLength = Math.Min(_segment.Length, buffer.Length); _segment.Span.Slice(0, copyLength).CopyTo(buffer.Slice(readLength)); _segment = _segment.Slice(copyLength); _positionInternal += copyLength; readLength += copyLength; } return readLength; } /// public override int Read(byte[] buffer, int offset, int count) => Read(buffer.AsSpan(offset, count)); /// public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); /// public override void SetLength(long value) => throw new NotSupportedException(); /// public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); } /// /// Convert the data into a compact binary object /// /// public CbObject ToObject() { return new CbObject(ToByteArray()); } /// /// Convert the data into a flat array /// /// public byte[] ToByteArray() { byte[] buffer = new byte[GetSize()]; CopyTo(buffer); return buffer; } /// /// Comptues the size of any child headers /// /// static int ComputeSizeOfChildHeaders(Scope scope) { int sizeOfChildHeaders = 0; foreach (Scope childScope in scope._children) { switch (childScope._fieldType) { case CbFieldType.Object: case CbFieldType.UniformObject: sizeOfChildHeaders += childScope._sizeOfChildHeaders + VarInt.Measure(childScope._length + childScope._sizeOfChildHeaders); break; case CbFieldType.Array: case CbFieldType.UniformArray: int arrayCountLength = VarInt.Measure(childScope._count); sizeOfChildHeaders += childScope._sizeOfChildHeaders + VarInt.Measure(childScope._length + childScope._sizeOfChildHeaders + arrayCountLength) + arrayCountLength; break; default: throw new InvalidOperationException(); } } return sizeOfChildHeaders; } /// /// Writes the header for a particular scope /// /// /// /// static int WriteScopeHeader(Span span, Scope scope) { switch (scope._fieldType) { case CbFieldType.Object: case CbFieldType.UniformObject: return VarInt.Write(span, scope._length + scope._sizeOfChildHeaders); case CbFieldType.Array: case CbFieldType.UniformArray: int numItemsLength = VarInt.Measure(scope._count); int offset = VarInt.Write(span, scope._length + scope._sizeOfChildHeaders + numItemsLength); return offset + VarInt.Write(span.Slice(offset), scope._count); default: throw new InvalidOperationException(); } } } }