Files
UnrealEngineUWP/Engine/Source/Programs/Shared/EpicGames.Core/ReadOnlySequenceBuilder.cs
Ben Marsh 35e9ebc4c1 Horde: API for manipulating tree structures in storage. Designed to abstract the logical tree structure away from storage/retrieval when allow working on larger-than-memory data sets.
* ITreeStore defines an API for reading/writing named tree structures.
* ITreeStore<T> allows reading/writing trees of nodes derived from the TreeNode base class, which can be specialized by clients. TreeNode tracks whether the node has been mutated, and allows recursing through the tree with strongly typed objects. Weak references are kept to deserialized TreeNode objects to ensure synchronization with the serialized tree objects.
* Default (and only current) implementation of ITreeStore is BundleStore, which packs nodes together in an efficient binary format and supports incrementally updating bundles. Nodes are hashed and deduplicated within bundles (and the known working set of the BundleStore instance), but packed Bundles are stored using a unique id returned by IBlobStore.
* DirectoryNode and FileNode nodes allow storing file trees in bundles, and implement content-aware chunking of binary data using a rolling BuzHash over an input stream.

#fyi Joakim.Lindqvist
#preflight 62acb0a5b47403e5aef09f74

[CL 20887473 by Ben Marsh in ue5-main branch]
2022-06-29 21:14:05 -04:00

117 lines
2.8 KiB
C#

// Copyright Epic Games, Inc. All Rights Reserved.
using System;
using System.Buffers;
using System.Collections.Generic;
using System.Text;
namespace EpicGames.Core
{
/// <summary>
/// Utility class to combine buffers into a <see cref="ReadOnlySequence{T}"/>
/// </summary>
/// <typeparam name="T">Element type</typeparam>
public class ReadOnlySequenceBuilder<T>
{
class Segment : ReadOnlySequenceSegment<T>
{
public Segment(long runningIndex, ReadOnlyMemory<T> memory)
{
RunningIndex = runningIndex;
Memory = memory;
}
public void SetNext(Segment next)
{
Next = next;
}
}
readonly List<ReadOnlyMemory<T>> _segments = new List<ReadOnlyMemory<T>>();
/// <summary>
/// Current length of the sequence
/// </summary>
public long Length { get; private set; }
/// <summary>
/// Create a sequence from a list of segments
/// </summary>
/// <param name="segments"></param>
/// <returns>Sequence for the list of segments</returns>
public static ReadOnlySequence<T> Create(IReadOnlyList<ReadOnlyMemory<T>> segments)
{
if (segments.Count == 0)
{
return ReadOnlySequence<T>.Empty;
}
Segment first = new Segment(0, segments[0]);
Segment last = first;
for (int idx = 1; idx < segments.Count; idx++)
{
Segment next = new Segment(last.RunningIndex + last.Memory.Length, segments[idx]);
last.SetNext(next);
last = next;
}
return new ReadOnlySequence<T>(first, 0, last, last.Memory.Length);
}
/// <summary>
/// Append a block of memory to the end of the sequence
/// </summary>
/// <param name="memory">Memory to append</param>
public void Append(ReadOnlyMemory<T> memory)
{
if (memory.Length > 0)
{
_segments.Add(memory);
Length += memory.Length;
}
}
/// <summary>
/// Append another sequence to the end of this one
/// </summary>
/// <param name="sequence">Sequence to append</param>
public void Append(ReadOnlySequence<T> sequence)
{
foreach (ReadOnlyMemory<T> segment in sequence)
{
Append(segment);
}
}
/// <summary>
/// Construct a sequence from the added blocks
/// </summary>
/// <returns>Sequence for the added memory blocks</returns>
public ReadOnlySequence<T> Construct() => Create(_segments);
}
/// <summary>
/// Extension methods for <see cref="ReadOnlySequence{T}"/>
/// </summary>
public static class ReadOnlySequenceExtensions
{
/// <summary>
/// Gets the data from a sequence as a contiguous block of memory
/// </summary>
/// <param name="sequence">Sequence to return</param>
/// <returns>Data for the blob</returns>
public static ReadOnlyMemory<T> AsSingleSegment<T>(this ReadOnlySequence<T> sequence)
{
if (sequence.IsSingleSegment)
{
return sequence.First;
}
else
{
return sequence.ToArray();
}
}
}
}