// Copyright Epic Games, Inc. All Rights Reserved.
using EpicGames.Core;
using System;
using System.Buffers.Binary;
using System.Collections.Generic;
using System.IO;
namespace EpicGames.Serialization
{
///
/// Exception for
///
public class CbWriterException : Exception
{
///
/// Constructor
///
///
public CbWriterException(string message)
: base(message)
{
}
///
/// Constructor
///
///
///
public CbWriterException(string message, Exception? ex)
: base(message, ex)
{
}
}
///
/// Forward-only writer for compact binary objects
///
public class CbWriter
{
///
/// Stores information about an object or array scope within the written buffer which requires a header to be inserted containing
/// the size or number of elements when copied to an output buffer
///
class Scope
{
public CbFieldType _fieldType;
public CbFieldType _uniformFieldType;
public int _offset; // Offset to insert the length/count
public int _length; // Excludes the size of this field's headers, and child fields' headers.
public int _count;
public List _children = new List();
public int _sizeOfChildHeaders; // Sum of additional headers for child items, recursively.
public Scope(CbFieldType fieldType, CbFieldType uniformFieldType, int offset)
{
Reset(fieldType, uniformFieldType, offset);
}
public void Reset(CbFieldType fieldType, CbFieldType uniformFieldType, int offset)
{
_fieldType = fieldType;
_uniformFieldType = uniformFieldType;
_offset = offset;
_length = 0;
_count = 0;
_children.Clear();
_sizeOfChildHeaders = 0;
}
}
///
/// Chunk of written data. Chunks are allocated as needed and chained together with scope annotations to produce the output data.
///
class Chunk
{
public int _offset;
public int _length;
public byte[] _data;
public List _scopes = new List();
public Chunk(int offset, int maxLength)
{
_data = new byte[maxLength];
Reset(offset);
}
public void Reset(int offset)
{
_offset = offset;
_length = 0;
_scopes.Clear();
}
}
const int DefaultChunkSize = 1024;
readonly List _chunks = new List();
readonly Stack _openScopes = new Stack();
Chunk CurrentChunk => _chunks[^1];
Scope CurrentScope => _openScopes.Peek();
int _currentOffset;
readonly List _freeChunks = new List();
readonly List _freeScopes = new List();
///
/// Constructor
///
public CbWriter()
: this(DefaultChunkSize)
{
}
///
/// Constructor
///
/// Amount of data to reserve for output
public CbWriter(int reserve)
{
_chunks.Add(new Chunk(0, reserve));
_openScopes.Push(new Scope(CbFieldType.Array, CbFieldType.None, 0));
}
///
///
///
public void Clear()
{
foreach (Chunk chunk in _chunks)
{
FreeChunk(chunk);
}
_currentOffset = 0;
_chunks.Clear();
_chunks.Add(AllocChunk(0, DefaultChunkSize));
_openScopes.Clear();
_openScopes.Push(AllocScope(CbFieldType.Array, CbFieldType.None, 0));
}
///
/// Allocate a new chunk object
///
/// Offset of the chunk
/// Maximum length of the chunk
/// New chunk object
Chunk AllocChunk(int offset, int maxLength)
{
for(int idx = _freeChunks.Count - 1; idx >= 0; idx--)
{
Chunk chunk = _freeChunks[idx];
if (chunk._data.Length >= maxLength)
{
_freeChunks.RemoveAt(idx);
chunk.Reset(offset);
return chunk;
}
}
return new Chunk(offset, maxLength);
}
///
/// Adds a chunk to the free list
///
///
void FreeChunk(Chunk chunk)
{
// Add the scopes to the free list
_freeScopes.AddRange(chunk._scopes);
chunk._scopes.Clear();
// Insert it into the free list, sorted by descending size
for (int idx = 0; ; idx++)
{
if (idx == _freeChunks.Count || chunk._data.Length >= _freeChunks[idx]._data.Length)
{
_freeChunks.Insert(idx, chunk);
break;
}
}
}
///
/// Allocate a scope object
///
///
///
///
///
Scope AllocScope(CbFieldType fieldType, CbFieldType uniformFieldType, int offset)
{
if (_freeScopes.Count > 0)
{
Scope scope = _freeScopes[^1];
scope.Reset(fieldType, uniformFieldType, offset);
_freeScopes.RemoveAt(_freeScopes.Count - 1);
return scope;
}
return new Scope(fieldType, uniformFieldType, offset);
}
///
/// Ensure that a block of contiguous memory of the given length is available in the output buffer
///
///
/// The allocated memory
Memory Allocate(int length)
{
Chunk lastChunk = CurrentChunk;
if (lastChunk._length + length > lastChunk._data.Length)
{
int chunkSize = Math.Max(length, DefaultChunkSize);
lastChunk = AllocChunk(_currentOffset, chunkSize);
_chunks.Add(lastChunk);
}
Memory buffer = lastChunk._data.AsMemory(lastChunk._length, length);
lastChunk._length += length;
_currentOffset += length;
return buffer;
}
///
/// Insert a new scope
///
///
///
void PushScope(CbFieldType fieldType, CbFieldType uniformFieldType)
{
Scope newScope = AllocScope(fieldType, uniformFieldType, _currentOffset);
CurrentScope._children.Add(newScope);
_openScopes.Push(newScope);
CurrentChunk._scopes.Add(newScope);
}
///
/// Pop a scope from the current open list
///
void PopScope()
{
Scope scope = CurrentScope;
scope._length = _currentOffset - scope._offset;
scope._sizeOfChildHeaders = ComputeSizeOfChildHeaders(scope);
_openScopes.Pop();
}
///
/// Writes the header for an unnamed field
///
///
void WriteFieldHeader(CbFieldType type)
{
Scope scope = CurrentScope;
if (!CbFieldUtils.IsArray(scope._fieldType))
{
throw new CbWriterException($"Anonymous fields are not allowed within fields of type {scope._fieldType}");
}
if (scope._uniformFieldType == CbFieldType.None)
{
Allocate(1).Span[0] = (byte)type;
}
else if (scope._uniformFieldType != type)
{
throw new CbWriterException($"Mismatched type for uniform array - expected {scope._uniformFieldType}, not {type}");
}
scope._count++;
}
///
/// Writes the header for a named field
///
///
///
void WriteFieldHeader(CbFieldType type, Utf8String name)
{
Scope scope = CurrentScope;
if (!CbFieldUtils.IsObject(scope._fieldType))
{
throw new CbWriterException($"Named fields are not allowed within fields of type {scope._fieldType}");
}
int nameVarIntLength = VarInt.Measure(name.Length);
if (scope._uniformFieldType == CbFieldType.None)
{
Span buffer = Allocate(1 + nameVarIntLength + name.Length).Span;
buffer[0] = (byte)(type | CbFieldType.HasFieldName);
WriteBinaryPayload(buffer[1..], name.Span);
}
else
{
if (scope._uniformFieldType != type)
{
throw new CbWriterException($"Mismatched type for uniform object - expected {scope._uniformFieldType}, not {type}");
}
WriteBinaryPayload(name.Span);
}
scope._count++;
}
///
/// Copies an entire field value to the output
///
///
public void WriteFieldValue(CbField field)
{
WriteFieldHeader(field.GetType());
int size = (int)field.GetPayloadSize();
field.GetPayloadView().CopyTo(Allocate(size));
}
///
/// Copies an entire field value to the output, using the name from the field
///
///
public void WriteField(CbField field) => WriteField(field.GetName(), field);
///
/// Copies an entire field value to the output
///
///
public void WriteField(Utf8String name, CbField field)
{
WriteFieldHeader(field.GetType(), name);
int size = (int)field.GetPayloadSize();
field.GetPayloadView().CopyTo(Allocate(size));
}
///
/// Begin writing an object field
///
public void BeginObject()
{
WriteFieldHeader(CbFieldType.Object);
PushScope(CbFieldType.Object, CbFieldType.None);
}
///
/// Begin writing an object field
///
/// Name of the field
public void BeginObject(Utf8String name)
{
WriteFieldHeader(CbFieldType.Object, name);
PushScope(CbFieldType.Object, CbFieldType.None);
}
///
/// End the current object
///
public void EndObject()
{
PopScope();
}
///
/// Begin writing an array field
///
public void BeginArray()
{
WriteFieldHeader(CbFieldType.Array);
PushScope(CbFieldType.Array, CbFieldType.None);
}
///
/// Begin writing a named array field
///
///
public void BeginArray(Utf8String name)
{
WriteFieldHeader(CbFieldType.Array, name);
PushScope(CbFieldType.Array, CbFieldType.None);
}
///
/// End the current array
///
public void EndArray()
{
PopScope();
}
///
/// Begin writing a uniform array field
///
/// The field type for elements in the array
public void BeginUniformArray(CbFieldType fieldType)
{
WriteFieldHeader(CbFieldType.UniformArray);
PushScope(CbFieldType.UniformArray, fieldType);
Allocate(1).Span[0] = (byte)fieldType;
}
///
/// Begin writing a named uniform array field
///
/// Name of the field
/// The field type for elements in the array
public void BeginUniformArray(Utf8String name, CbFieldType fieldType)
{
WriteFieldHeader(CbFieldType.UniformArray, name);
PushScope(CbFieldType.UniformArray, fieldType);
Allocate(1).Span[0] = (byte)fieldType;
}
///
/// End the current array
///
public void EndUniformArray()
{
PopScope();
}
///
/// Write a null field
///
public void WriteNullValue()
{
WriteFieldHeader(CbFieldType.Null);
}
///
/// Write a named null field
///
/// Name of the field
public void WriteNull(Utf8String name)
{
WriteFieldHeader(CbFieldType.Null, name);
}
///
/// Writes a boolean value
///
///
public void WriteBoolValue(bool value)
{
WriteFieldHeader(value? CbFieldType.BoolTrue : CbFieldType.BoolFalse);
}
///
/// Writes a boolean value
///
/// Name of the field
///
public void WriteBool(Utf8String name, bool value)
{
WriteFieldHeader(value ? CbFieldType.BoolTrue : CbFieldType.BoolFalse, name);
}
///
/// Writes the payload for an integer
///
/// Value to write
void WriteIntegerPayload(ulong value)
{
int length = VarInt.Measure(value);
Span buffer = Allocate(length).Span;
VarInt.Write(buffer, value);
}
///
/// Writes an unnamed integer field
///
/// Value to be written
public void WriteIntegerValue(int value)
{
WriteIntegerValue((long)value);
}
///
/// Writes an unnamed integer field
///
/// Value to be written
public void WriteIntegerValue(long value)
{
if (value >= 0)
{
WriteFieldHeader(CbFieldType.IntegerPositive);
WriteIntegerPayload((ulong)value);
}
else
{
WriteFieldHeader(CbFieldType.IntegerNegative);
WriteIntegerPayload((ulong)-value);
}
}
///
/// Writes an named integer field
///
/// Name of the field
/// Value to be written
public void WriteInteger(Utf8String name, int value)
{
WriteInteger(name, (long)value);
}
///
/// Writes an named integer field
///
/// Name of the field
/// Value to be written
public void WriteInteger(Utf8String name, long value)
{
if (value >= 0)
{
WriteFieldHeader(CbFieldType.IntegerPositive, name);
WriteIntegerPayload((ulong)value);
}
else
{
WriteFieldHeader(CbFieldType.IntegerNegative, name);
WriteIntegerPayload((ulong)-value);
}
}
///
/// Writes an unnamed integer field
///
/// Value to be written
public void WriteIntegerValue(ulong value)
{
WriteFieldHeader(CbFieldType.IntegerPositive);
WriteIntegerPayload(value);
}
///
/// Writes a named integer field
///
/// Name of the field
/// Value to be written
public void WriteInteger(Utf8String name, ulong value)
{
WriteFieldHeader(CbFieldType.IntegerPositive, name);
WriteIntegerPayload(value);
}
///
/// Writes an unnamed double field
///
/// Value to be written
public void WriteDoubleValue(double value)
{
WriteFieldHeader(CbFieldType.Float64);
BinaryPrimitives.WriteInt64BigEndian(Allocate(sizeof(double)).Span, BitConverter.DoubleToInt64Bits(value));
}
///
/// Writes a named double field
///
/// Name of the field
/// Value to be written
public void WriteDouble(Utf8String name, double value)
{
WriteFieldHeader(CbFieldType.Float64, name);
BinaryPrimitives.WriteInt64BigEndian(Allocate(sizeof(double)).Span, BitConverter.DoubleToInt64Bits(value));
}
///
/// Writes the payload for a value
///
/// The value to write
void WriteDateTimePayload(DateTime dateTime)
{
Span buffer = Allocate(sizeof(long)).Span;
BinaryPrimitives.WriteInt64BigEndian(buffer, dateTime.Ticks);
}
///
/// Writes an unnamed field
///
/// Value to be written
public void WriteDateTimeValue(DateTime value)
{
WriteFieldHeader(CbFieldType.DateTime);
WriteDateTimePayload(value);
}
///
/// Writes a named field
///
/// Name of the field
/// Value to be written
public void WriteDateTime(Utf8String name, DateTime value)
{
WriteFieldHeader(CbFieldType.DateTime, name);
WriteDateTimePayload(value);
}
///
/// Writes the payload for a hash
///
///
void WriteHashPayload(IoHash hash)
{
Span buffer = Allocate(IoHash.NumBytes).Span;
hash.CopyTo(buffer);
}
///
/// Writes an unnamed field
///
///
public void WriteHashValue(IoHash hash)
{
WriteFieldHeader(CbFieldType.Hash);
WriteHashPayload(hash);
}
///
/// Writes a named field
///
/// Name of the field
/// Value to be written
public void WriteHash(Utf8String name, IoHash value)
{
WriteFieldHeader(CbFieldType.Hash, name);
WriteHashPayload(value);
}
///
/// Writes an unnamed reference to a binary attachment
///
/// Hash of the attachment
public void WriteBinaryAttachmentValue(IoHash hash)
{
WriteFieldHeader(CbFieldType.BinaryAttachment);
WriteHashPayload(hash);
}
///
/// Writes a named reference to a binary attachment
///
/// Name of the field
/// Hash of the attachment
public void WriteBinaryAttachment(Utf8String name, IoHash hash)
{
WriteFieldHeader(CbFieldType.BinaryAttachment, name);
WriteHashPayload(hash);
}
///
/// Writes the payload for an object to the buffer
///
///
void WriteObjectPayload(CbObject obj)
{
CbField field = obj.AsField();
Memory buffer = Allocate(field.Payload.Length);
field.Payload.CopyTo(buffer);
}
///
/// Writes an object directly into the writer
///
/// Object to write
public void WriteObject(CbObject obj)
{
WriteFieldHeader(CbFieldType.Object);
WriteObjectPayload(obj);
}
///
/// Writes an object directly into the writer
///
/// Name of the object
/// Object to write
public void WriteObject(Utf8String name, CbObject obj)
{
WriteFieldHeader(CbFieldType.Object, name);
WriteObjectPayload(obj);
}
///
/// Writes an unnamed reference to an object attachment
///
/// Hash of the attachment
public void WriteObjectAttachmentValue(IoHash hash)
{
WriteFieldHeader(CbFieldType.ObjectAttachment);
WriteHashPayload(hash);
}
///
/// Writes a named reference to an object attachment
///
/// Name of the field
/// Hash of the attachment
public void WriteObjectAttachment(Utf8String name, IoHash hash)
{
WriteFieldHeader(CbFieldType.ObjectAttachment, name);
WriteHashPayload(hash);
}
///
/// Writes the payload for a binary value
///
/// Output buffer
/// Value to be written
static void WriteBinaryPayload(Span output, ReadOnlySpan value)
{
int varIntLength = VarInt.Write(output, value.Length);
output = output[varIntLength..];
value.CopyTo(output);
CheckSize(output, value.Length);
}
///
/// Writes the payload for a binary value
///
/// Value to be written
void WriteBinaryPayload(ReadOnlySpan value)
{
int valueVarIntLength = VarInt.Measure(value.Length);
Span buffer = Allocate(valueVarIntLength + value.Length).Span;
WriteBinaryPayload(buffer, value);
}
///
/// Writes an unnamed string value
///
/// Value to be written
public void WriteStringValue(string value) => WriteUtf8StringValue(value);
///
/// Writes a named string value
///
/// Name of the field
/// Value to be written
public void WriteString(Utf8String name, string? value)
{
if(value != null)
{
WriteUtf8String(name, value);
}
}
///
/// Writes an unnamed string value
///
/// Value to be written
public void WriteUtf8StringValue(Utf8String value)
{
WriteFieldHeader(CbFieldType.String);
WriteBinaryPayload(value.Span);
}
///
/// Writes a named string value
///
/// Name of the field
/// Value to be written
public void WriteUtf8String(Utf8String name, Utf8String value)
{
if (value.Length > 0)
{
WriteFieldHeader(CbFieldType.String, name);
WriteBinaryPayload(value.Span);
}
}
///
/// Writes an unnamed binary value
///
/// Value to be written
public void WriteBinarySpanValue(ReadOnlySpan value)
{
WriteFieldHeader(CbFieldType.Binary);
WriteBinaryPayload(value);
}
///
/// Writes a named binary value
///
/// Name of the field
/// Value to be written
public void WriteBinarySpan(Utf8String name, ReadOnlySpan value)
{
WriteFieldHeader(CbFieldType.Binary, name);
WriteBinaryPayload(value);
}
///
/// Writes an unnamed binary value
///
/// Value to be written
public void WriteBinaryValue(ReadOnlyMemory value)
{
WriteBinarySpanValue(value.Span);
}
///
/// Writes a named binary value
///
/// Name of the field
/// Value to be written
public void WriteBinary(Utf8String name, ReadOnlyMemory value)
{
WriteBinarySpan(name, value.Span);
}
///
/// Writes an unnamed binary value
///
/// Value to be written
public void WriteBinaryArrayValue(byte[] value)
{
WriteBinarySpanValue(value.AsSpan());
}
///
/// Writes a named binary value
///
/// Name of the field
/// Value to be written
public void WriteBinaryArray(Utf8String name, byte[] value)
{
WriteBinarySpan(name, value.AsSpan());
}
///
/// Check that the given span is the required size
///
///
///
static void CheckSize(Span span, int expectedSize)
{
if (span.Length != expectedSize)
{
throw new Exception("Size of buffer is not correct");
}
}
///
/// Computes the hash for this object
///
/// Hash for the object
public IoHash ComputeHash()
{
using (Blake3.Hasher hasher = Blake3.Hasher.New())
{
foreach (ReadOnlyMemory segment in EnumerateSegments())
{
hasher.Update(segment.Span);
}
return IoHash.FromBlake3(hasher);
}
}
///
/// Gets the size of the serialized data
///
///
public int GetSize()
{
if (_openScopes.Count != 1)
{
throw new CbWriterException("Unfinished scope in writer");
}
return _currentOffset + ComputeSizeOfChildHeaders(CurrentScope);
}
///
/// Gets the contents of this writer as a stream
///
/// New stream for the contents of this object
public Stream AsStream() => new ReadStream(EnumerateSegments().GetEnumerator(), GetSize());
private IEnumerable> EnumerateSegments()
{
byte[] scopeHeader = new byte[64];
int sourceOffset = 0;
foreach (Chunk chunk in _chunks)
{
foreach (Scope scope in chunk._scopes)
{
ReadOnlyMemory sourceData = chunk._data.AsMemory(sourceOffset - chunk._offset, scope._offset - sourceOffset);
yield return sourceData;
sourceOffset += sourceData.Length;
int headerLength = WriteScopeHeader(scopeHeader, scope);
yield return scopeHeader.AsMemory(0, headerLength);
}
ReadOnlyMemory lastSourceData = chunk._data.AsMemory(sourceOffset - chunk._offset, (chunk._offset + chunk._length) - sourceOffset);
yield return lastSourceData;
sourceOffset += lastSourceData.Length;
}
}
///
/// Copy the data from this writer to a buffer
///
///
public void CopyTo(Span buffer)
{
int bufferOffset = 0;
int sourceOffset = 0;
foreach (Chunk chunk in _chunks)
{
foreach (Scope scope in chunk._scopes)
{
ReadOnlySpan sourceData = chunk._data.AsSpan(sourceOffset - chunk._offset, scope._offset - sourceOffset);
sourceData.CopyTo(buffer.Slice(bufferOffset));
bufferOffset += sourceData.Length;
sourceOffset += sourceData.Length;
bufferOffset += WriteScopeHeader(buffer.Slice(bufferOffset), scope);
}
ReadOnlySpan lastSourceData = chunk._data.AsSpan(sourceOffset - chunk._offset, (chunk._offset + chunk._length) - sourceOffset);
lastSourceData.CopyTo(buffer.Slice(bufferOffset));
bufferOffset += lastSourceData.Length;
sourceOffset += lastSourceData.Length;
}
}
class ReadStream : Stream
{
readonly IEnumerator> _enumerator;
ReadOnlyMemory _segment;
long _positionInternal;
public ReadStream(IEnumerator> enumerator, long length)
{
_enumerator = enumerator;
Length = length;
}
///
public override bool CanRead => true;
///
public override bool CanSeek => false;
///
public override bool CanWrite => false;
///
public override long Length { get; }
///
public override long Position
{
get => _positionInternal;
set => throw new NotSupportedException();
}
///
public override void Flush() { }
///
public override int Read(Span buffer)
{
int readLength = 0;
while (readLength < buffer.Length)
{
while (_segment.Length == 0)
{
if (!_enumerator.MoveNext())
{
return readLength;
}
_segment = _enumerator.Current;
}
int copyLength = Math.Min(_segment.Length, buffer.Length);
_segment.Span.Slice(0, copyLength).CopyTo(buffer.Slice(readLength));
_segment = _segment.Slice(copyLength);
_positionInternal += copyLength;
readLength += copyLength;
}
return readLength;
}
///
public override int Read(byte[] buffer, int offset, int count) => Read(buffer.AsSpan(offset, count));
///
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
///
public override void SetLength(long value) => throw new NotSupportedException();
///
public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException();
}
///
/// Convert the data into a compact binary object
///
///
public CbObject ToObject()
{
return new CbObject(ToByteArray());
}
///
/// Convert the data into a flat array
///
///
public byte[] ToByteArray()
{
byte[] buffer = new byte[GetSize()];
CopyTo(buffer);
return buffer;
}
///
/// Comptues the size of any child headers
///
///
static int ComputeSizeOfChildHeaders(Scope scope)
{
int sizeOfChildHeaders = 0;
foreach (Scope childScope in scope._children)
{
switch (childScope._fieldType)
{
case CbFieldType.Object:
case CbFieldType.UniformObject:
sizeOfChildHeaders += childScope._sizeOfChildHeaders + VarInt.Measure(childScope._length + childScope._sizeOfChildHeaders);
break;
case CbFieldType.Array:
case CbFieldType.UniformArray:
int arrayCountLength = VarInt.Measure(childScope._count);
sizeOfChildHeaders += childScope._sizeOfChildHeaders + VarInt.Measure(childScope._length + childScope._sizeOfChildHeaders + arrayCountLength) + arrayCountLength;
break;
default:
throw new InvalidOperationException();
}
}
return sizeOfChildHeaders;
}
///
/// Writes the header for a particular scope
///
///
///
///
static int WriteScopeHeader(Span span, Scope scope)
{
switch (scope._fieldType)
{
case CbFieldType.Object:
case CbFieldType.UniformObject:
return VarInt.Write(span, scope._length + scope._sizeOfChildHeaders);
case CbFieldType.Array:
case CbFieldType.UniformArray:
int numItemsLength = VarInt.Measure(scope._count);
int offset = VarInt.Write(span, scope._length + scope._sizeOfChildHeaders + numItemsLength);
return offset + VarInt.Write(span.Slice(offset), scope._count);
default:
throw new InvalidOperationException();
}
}
}
}