izzy/UnrealEngineUWP
0
0
Fork 0
mirror of https://github.com/izzy2lost/UnrealEngineUWP.git synced 2026-03-26 18:15:20 -07:00
Code Issues Packages Projects Releases Wiki Activity

Unreal Cloud DDC - Removed usage of Newtonsoft.Json for serilization, previously had some usages of System.Text.Json which we now use everywhere.

Browse Source 
This breaks field filtering on json objects because system.text.json does not yet support solving that but .NET 7 adds support for it.

#preflight none

[CL 25143384 by Joakim Lindqvist in ue5-main branch]
This commit is contained in:
Joakim Lindqvist
2023-04-21 05:50:36 -04:00
parent a44fb36fd6
commit eb573db4ea
28 changed files with 363 additions and 303 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
Engine/Source/Programs/UnrealCloudDDC/Jupiter.Common/BaseStartup.cs
View File

@@ -9,16 +9,18 @@ using System.IO;
using System.Net.Mime;
using System.Reflection;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using Amazon;
using EpicGames.AspNet;
using EpicGames.Horde.Storage;
using Jupiter.Common;
using Jupiter.Implementation;
using Microsoft.AspNetCore.Authentication;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Diagnostics.HealthChecks;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.HttpOverrides;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Configuration;
@@ -27,11 +29,8 @@ using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using Microsoft.Extensions.Primitives;
using Microsoft.Net.Http.Headers;
using Microsoft.OpenApi.Models;
using Newtonsoft.Json;
using Newtonsoft.Json.Serialization;
using Okta.AspNet.Abstractions;
using Okta.AspNetCore;
using OpenTelemetry.Logs;
@@ -94,14 +93,7 @@ namespace Jupiter
services.AddSingleton(typeof(INamespacePolicyResolver), typeof(NamespacePolicyResolver));
// this is the same as invoke MvcBuilder.AddJsonOptions but with a service provider passed so we can use DI in the options creation
// see https://stackoverflow.com/questions/53288633/net-core-api-custom-json-resolver-based-on-request-values
// inject our custom json options and then pass a DI context to customize the serialization
services.Configure<MvcNewtonsoftJsonOptions>(OnJsonOptions);
services.AddTransient<IConfigureOptions<MvcNewtonsoftJsonOptions>, MvcJsonOptionsWrapper>();
services.AddControllers()
.AddNewtonsoftJson()
.AddMvcOptions(options =>
{
options.InputFormatters.Add(new CbInputFormatter());
@@ -123,7 +115,7 @@ namespace Jupiter
return result;
};
});
}).AddJsonOptions(jsonOptions => ConfigureJsonOptions(jsonOptions.JsonSerializerOptions));
services.AddHttpContextAccessor();
@@ -340,6 +332,15 @@ namespace Jupiter
OnAddHealthChecks(services);
}
public static void ConfigureJsonOptions(JsonSerializerOptions options)
{
options.AllowTrailingCommas = true;
options.DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull;
options.PropertyNameCaseInsensitive = true;
options.Converters.Add(new JsonStringEnumConverter());
}
private Meter CreateMeter(IServiceProvider provider)
{
return new Meter("UnrealCloudDDC");
@@ -375,10 +376,6 @@ namespace Jupiter
protected abstract void OnAddAuthorization(AuthorizationOptions authorizationOptions, List<string> defaultSchemes);
protected virtual void OnJsonOptions(MvcNewtonsoftJsonOptions options)
{
}
protected virtual void OnAddControllers(MvcOptions options)
{
}
@@ -484,7 +481,7 @@ namespace Jupiter
}
}
public class MvcJsonOptionsWrapper : IConfigureOptions<MvcNewtonsoftJsonOptions>
/*public class MvcJsonOptionsWrapper : IConfigureOptions<MvcNewtonsoftJsonOptions>
{
readonly IServiceProvider ServiceProvider;
@@ -496,9 +493,9 @@ namespace Jupiter
{
options.SerializerSettings.ContractResolver = new FieldFilteringResolver(ServiceProvider);
}
}
}*/
public class FieldFilteringResolver : DefaultContractResolver
/*public class FieldFilteringResolver : DefaultContractResolver
{
private readonly IHttpContextAccessor _httpContextAccessor;
@@ -549,7 +546,7 @@ namespace Jupiter
return property;
}
}
}*/
public enum SchemeImplementations
{

55
Engine/Source/Programs/UnrealCloudDDC/Jupiter.Common/Implementation/BlobIdentifier.cs
View File

@@ -5,18 +5,18 @@ using System.ComponentModel;
using System.Globalization;
using System.IO;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading.Tasks;
using Blake3;
using EpicGames.Core;
using EpicGames.Horde.Storage;
using EpicGames.Serialization;
using Newtonsoft.Json;
using JsonWriter = Newtonsoft.Json.JsonWriter;
namespace Jupiter.Implementation
{
[JsonConverter(typeof(BlobIdentifierConverter))]
[TypeConverter(typeof(BlobIdentifierTypeConverter))]
[JsonConverter(typeof(BlobIdentifierJsonConverter))]
[CbConverter(typeof(BlobIdentifierCbConverter))]
public class BlobIdentifier : ContentHash, IEquatable<BlobIdentifier>
{
@@ -180,31 +180,44 @@ namespace Jupiter.Implementation
}
return base.ConvertFrom(context, culture, value);
}
}
public class BlobIdentifierConverter : JsonConverter<BlobIdentifier?>
{
public override void WriteJson(JsonWriter writer, BlobIdentifier? value, JsonSerializer serializer)
{
writer.WriteValue(value!.ToString());
}
public override BlobIdentifier? ReadJson(JsonReader reader, Type objectType, BlobIdentifier? existingValue, bool hasExistingValue, JsonSerializer serializer)
public override bool CanConvertTo(ITypeDescriptorContext? context, Type? destinationType)
{
if (reader.TokenType == JsonToken.Null)
if (destinationType == typeof(string))
{
return null;
return true;
}
return base.CanConvertTo(context, destinationType);
}
public override object? ConvertTo(ITypeDescriptorContext? context, CultureInfo? culture, object? value, Type destinationType)
{
if (destinationType == typeof(string))
{
BlobIdentifier? identifier = (BlobIdentifier?)value;
return identifier?.ToString();
}
return base.ConvertTo(context, culture, value, destinationType);
}
}
public class BlobIdentifierJsonConverter : JsonConverter<BlobIdentifier>
{
public override BlobIdentifier? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
string? str = reader.GetString();
if (str == null)
{
throw new InvalidDataException("Unable to parse blob identifier");
}
string? s = (string?)reader.Value;
return new BlobIdentifier(str);
}
if (s == null)
{
return null;
}
return new BlobIdentifier(s!);
public override void Write(Utf8JsonWriter writer, BlobIdentifier value, JsonSerializerOptions options)
{
writer.WriteStringValue(value.ToString());
}
}

9
Engine/Source/Programs/UnrealCloudDDC/Jupiter.Common/Implementation/ClientCredentialOAuthAuthenticator.cs
View File

@@ -3,6 +3,7 @@
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Net.Http.Json;
using System.Threading.Tasks;
namespace Jupiter.Implementation
@@ -75,7 +76,13 @@ namespace Jupiter.Implementation
{
throw new AuthenticationFailedException(s);
}
return (await response.Content.ReadAsAsync<ClientCredentialsResponse>(), s);
ClientCredentialsResponse? responseBody = await response.Content.ReadFromJsonAsync<ClientCredentialsResponse>();
if (responseBody == null)
{
throw new Exception("Unable to deserialize client credential response");
}
return (responseBody, s);
}
}

88
Engine/Source/Programs/UnrealCloudDDC/Jupiter.Common/Implementation/ContentHash.cs
View File

@@ -1,21 +1,26 @@
// Copyright Epic Games, Inc. All Rights Reserved.
using System;
using System.ComponentModel;
using System.Globalization;
using System.IO;
using System.Text.Json;
using System.Text.Json.Serialization;
using Blake3;
using EpicGames.Core;
using EpicGames.Serialization;
using Newtonsoft.Json;
using JsonWriter = Newtonsoft.Json.JsonWriter;
namespace Jupiter.Implementation
{
[JsonConverter(typeof(ContentHashConverter))]
[TypeConverter(typeof(ContentHashTypeConverter))]
[JsonConverter(typeof(ContentHashJsonConverter))]
[CbConverter(typeof(ContentHashCbConverter))]
public class ContentHash : IEquatable<ContentHash>, IEquatable<byte[]>
{
protected ByteArrayComparer Comparer { get; } = new ByteArrayComparer();
protected ByteArrayComparer Comparer { get; } = new ByteArrayComparer();
protected byte[] Identifier { get; init; }
public const int HashLength = 20;
public byte[] HashData => Identifier;
public ContentHash(byte[] identifier)
{
@@ -105,21 +110,6 @@ namespace Jupiter.Implementation
}
}
public class ContentHashConverter : JsonConverter<ContentHash>
{
public override void WriteJson(JsonWriter writer, ContentHash? value, JsonSerializer serializer)
{
writer.WriteValue(value!.ToString());
}
public override ContentHash ReadJson(JsonReader reader, Type objectType, ContentHash? existingValue, bool hasExistingValue, JsonSerializer serializer)
{
string? s = (string?)reader.Value;
return new ContentHash(s!);
}
}
public class ContentHashCbConverter : CbConverterBase<ContentHash>
{
public override ContentHash Read(CbField field) => new ContentHash(field.AsHash().ToByteArray());
@@ -130,4 +120,64 @@ namespace Jupiter.Implementation
/// <inheritdoc/>
public override void WriteNamed(CbWriter writer, Utf8String name, ContentHash value) => writer.WriteHash(name, new IoHash(value.HashData));
}
public class ContentHashJsonConverter : JsonConverter<ContentHash>
{
public override ContentHash Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
string? str = reader.GetString();
if (str == null)
{
throw new InvalidDataException("Unable to parse content hash");
}
return new ContentHash(str);
}
public override void Write(Utf8JsonWriter writer, ContentHash value, JsonSerializerOptions options)
{
writer.WriteStringValue(value.ToString());
}
}
public class ContentHashTypeConverter : TypeConverter
{
public override bool CanConvertFrom(ITypeDescriptorContext? context, Type sourceType)
{
if (sourceType == typeof(string))
{
return true;
}
return base.CanConvertFrom(context, sourceType);
}
public override object? ConvertFrom(ITypeDescriptorContext? context, CultureInfo? culture, object value)
{
if (value is string s)
{
return new ContentHash(s);
}
return base.ConvertFrom(context, culture, value);
}
public override bool CanConvertTo(ITypeDescriptorContext? context, Type? destinationType)
{
if (destinationType == typeof(string))
{
return true;
}
return base.CanConvertTo(context, destinationType);
}
public override object? ConvertTo(ITypeDescriptorContext? context, CultureInfo? culture, object? value, Type destinationType)
{
if (destinationType == typeof(string))
{
ContentHash? identifier = (ContentHash?)value;
return identifier?.ToString();
}
return base.ConvertTo(context, culture, value, destinationType);
}
}
}

29
Engine/Source/Programs/UnrealCloudDDC/Jupiter.Common/Implementation/ContentId.cs
View File

@@ -3,13 +3,11 @@
using System;
using System.ComponentModel;
using System.Globalization;
using System.Text.Json.Serialization;
using EpicGames.Core;
using Newtonsoft.Json;
using JsonWriter = Newtonsoft.Json.JsonWriter;
namespace Jupiter.Implementation
{
[JsonConverter(typeof(ContentIdConverter))]
[TypeConverter(typeof(ContentIdTypeConverter))]
public class ContentId : ContentHash, IEquatable<ContentId>
{
@@ -105,29 +103,4 @@ namespace Jupiter.Implementation
return base.ConvertFrom(context, culture, value);
}
}
public class ContentIdConverter : JsonConverter<ContentId?>
{
public override void WriteJson(JsonWriter writer, ContentId? value, JsonSerializer serializer)
{
writer.WriteValue(value!.ToString());
}
public override ContentId? ReadJson(JsonReader reader, Type objectType, ContentId? existingValue, bool hasExistingValue, JsonSerializer serializer)
{
if (reader.TokenType == JsonToken.Null)
{
return null;
}
string? s = (string?)reader.Value;
if (s == null)
{
return null;
}
return new ContentId(s!);
}
}
}

41
Engine/Source/Programs/UnrealCloudDDC/Jupiter.Common/Implementation/IoHashKey.cs
View File

@@ -3,16 +3,17 @@
using System;
using System.ComponentModel;
using System.Globalization;
using System.IO;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using EpicGames.Core;
using EpicGames.Serialization;
using Newtonsoft.Json;
using JsonWriter = Newtonsoft.Json.JsonWriter;
namespace Jupiter.Implementation
{
[JsonConverter(typeof(IoHashKeyJsonConverter))]
[TypeConverter(typeof(IoHashKeyTypeConverter))]
[JsonConverter(typeof(IoHashKeyJsonConverter))]
[CbConverter(typeof(IoHashKeyCbConverter))]
public readonly struct IoHashKey: IEquatable<IoHashKey>
@@ -85,21 +86,6 @@ namespace Jupiter.Implementation
}
}
public class IoHashKeyJsonConverter: JsonConverter<IoHashKey>
{
public override void WriteJson(JsonWriter writer, IoHashKey value, JsonSerializer serializer)
{
writer.WriteValue(value!.ToString());
}
public override IoHashKey ReadJson(JsonReader reader, Type objectType, IoHashKey existingValue, bool hasExistingValue, Newtonsoft.Json.JsonSerializer serializer)
{
string? s = (string?)reader.Value;
return new IoHashKey(s!);
}
}
public sealed class IoHashKeyTypeConverter : TypeConverter
{
/// <inheritdoc/>
@@ -124,6 +110,25 @@ namespace Jupiter.Implementation
}
}
public class IoHashKeyJsonConverter : JsonConverter<IoHashKey>
{
public override IoHashKey Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
string? str = reader.GetString();
if (str == null)
{
throw new InvalidDataException("Unable to parse io hash key");
}
return new IoHashKey(str);
}
public override void Write(Utf8JsonWriter writer, IoHashKey value, JsonSerializerOptions options)
{
writer.WriteStringValue(value.ToString());
}
}
sealed class IoHashKeyCbConverter : CbConverterBase<IoHashKey>
{
public override IoHashKey Read(CbField field) => new IoHashKey(field.AsString());

1
Engine/Source/Programs/UnrealCloudDDC/Jupiter.Common/Jupiter.Common.csproj
View File

@@ -29,7 +29,6 @@
<PackageReference Include="Microsoft.AspNet.WebApi.Client" Version="5.2.7" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="6.0.0" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="6.0.0" />
<PackageReference Include="Microsoft.Extensions.Diagnostics.HealthChecks" Version="6.0.0" />
<PackageReference Include="Microsoft.Extensions.Hosting" Version="6.0.0" />

5
Engine/Source/Programs/UnrealCloudDDC/Jupiter/Controllers/AdminController.cs
View File

@@ -10,7 +10,6 @@ using Jupiter.Implementation;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Configuration;
using Newtonsoft.Json;
namespace Jupiter.Controllers
{
@@ -128,9 +127,9 @@ namespace Jupiter.Controllers
return new JsonResult(new
{
Settings = settings
}, new JsonSerializerSettings
}, new System.Text.Json.JsonSerializerOptions
{
Formatting = Formatting.Indented
WriteIndented = true
})
{
StatusCode = (int)HttpStatusCode.OK

5
Engine/Source/Programs/UnrealCloudDDC/Jupiter/Controllers/AuthController.cs
View File

@@ -8,8 +8,6 @@ using Microsoft.AspNetCore.Authorization;
using EpicGames.Horde.Storage;
using Jupiter.Common;
using Microsoft.Extensions.Options;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
namespace Jupiter.Controllers
{
@@ -62,13 +60,12 @@ namespace Jupiter.Controllers
allowedActions.AddRange(acl.Resolve(User));
}
return Ok(new ActionsResult {Actions = allowedActions});
return Ok(new JsonResult(new {Actions = allowedActions}));
}
}
public class ActionsResult
{
[JsonProperty (ItemConverterType = typeof(StringEnumConverter))]
[System.Diagnostics.CodeAnalysis.SuppressMessage("Usage", "CA2227:Collection properties should be read only", Justification = "Used by serialization")]
public List<AclAction> Actions { get; set; } = new List<AclAction>();
}

1
Engine/Source/Programs/UnrealCloudDDC/Jupiter/Controllers/BlobController.cs
View File

@@ -8,6 +8,7 @@ using System.IO;
using System.Linq;
using System.Net;
using System.Net.Mime;
using System.Text.Json.Serialization;
using System.Threading.Tasks;
using EpicGames.AspNet;
using EpicGames.Horde.Storage;

10
Engine/Source/Programs/UnrealCloudDDC/Jupiter/Controllers/ReferencesController.cs
View File

@@ -10,6 +10,7 @@ using System.Linq;
using System.Net;
using System.Net.Mime;
using System.Text;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using EpicGames.AspNet;
@@ -27,7 +28,6 @@ using Microsoft.AspNetCore.Http.Extensions;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using OpenTelemetry.Trace;
using ContentHash = Jupiter.Implementation.ContentHash;
@@ -1291,9 +1291,9 @@ namespace Jupiter.Controllers
public class ExistCheckMultipleRefsResponse
{
public ExistCheckMultipleRefsResponse(List<(BucketId,IoHashKey)> missingNames)
public ExistCheckMultipleRefsResponse(List<(BucketId,IoHashKey)> missing)
{
Missing = missingNames.Select(pair =>
Missing = missing.Select(pair =>
{
(BucketId bucketId, IoHashKey ioHashKey) = pair;
return new MissingReference()
@@ -1305,9 +1305,9 @@ namespace Jupiter.Controllers
}
[JsonConstructor]
public ExistCheckMultipleRefsResponse(List<MissingReference> missingNames)
public ExistCheckMultipleRefsResponse(List<MissingReference> missing)
{
Missing = missingNames;
Missing = missing;
}
[CbField("missing")]

4
Engine/Source/Programs/UnrealCloudDDC/Jupiter/Controllers/ReplicationLogController.cs
View File

@@ -4,6 +4,7 @@ using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.Linq;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using EpicGames.Horde.Storage;
@@ -15,7 +16,6 @@ using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Options;
using Newtonsoft.Json;
namespace Jupiter.Controllers
{
@@ -115,7 +115,7 @@ namespace Jupiter.Controllers
{
Title = $"Log file is not available, use snapshot {snapshot.SnapshotBlob} instead",
Type = ProblemTypes.UseSnapshot,
Extensions = { { "SnapshotId", snapshot.SnapshotBlob } }
Extensions = { { "SnapshotId", snapshot.SnapshotBlob }, { "BlobNamespace", snapshot.BlobNamespace }, }
});
}

4
Engine/Source/Programs/UnrealCloudDDC/Jupiter/Controllers/StatusController.cs
View File

@@ -4,11 +4,11 @@ using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using System.Text.Json.Serialization;
using Jupiter.Implementation;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Options;
using Newtonsoft.Json;
namespace Jupiter.Controllers
{
@@ -98,7 +98,7 @@ namespace Jupiter.Controllers
Peers = clusterSettings.CurrentValue.Peers.Select(settings => new KnownPeer(settings, includeInternalEndpoints, peerStatusService)).ToList();
}
public string CurrentSite { get; set; }
public string CurrentSite { get; set; } = null!;
[System.Diagnostics.CodeAnalysis.SuppressMessage("Usage", "CA2227:Collection properties should be read only", Justification = "Used by serialization")]
public List<KnownPeer> Peers { get; set; } = new List<KnownPeer>();

7
Engine/Source/Programs/UnrealCloudDDC/Jupiter/Implementation/ContentId/CacheContentIdStore.cs
View File

@@ -3,6 +3,7 @@
using System;
using System.Net;
using System.Net.Http;
using System.Net.Http.Json;
using System.Threading.Tasks;
using EpicGames.Horde.Storage;
using Jupiter.Controllers;
@@ -28,8 +29,12 @@ namespace Jupiter.Implementation
}
response.EnsureSuccessStatusCode();
ResolvedContentIdResponse resolvedContentId = await response.Content.ReadAsAsync<ResolvedContentIdResponse>();
ResolvedContentIdResponse? resolvedContentId = await response.Content.ReadFromJsonAsync<ResolvedContentIdResponse>();
if (resolvedContentId == null)
{
throw new Exception("Unable to deserialize resolved content id response");
}
return resolvedContentId.Blobs;
}

27
Engine/Source/Programs/UnrealCloudDDC/Jupiter/Implementation/Objects/CacheReferencesStore.cs
View File

@@ -6,6 +6,7 @@ using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Net.Http.Json;
using System.Net.Mime;
using System.Threading.Tasks;
using EpicGames.AspNet;
@@ -116,8 +117,11 @@ namespace Jupiter.Implementation
response.EnsureSuccessStatusCode();
RefMetadataResponse metadataResponse = await response.Content.ReadAsAsync<RefMetadataResponse>();
RefMetadataResponse? metadataResponse = await response.Content.ReadFromJsonAsync<RefMetadataResponse>();
if (metadataResponse == null)
{
throw new Exception("Unable to deserialize metadata response");
}
return new ObjectRecord(metadataResponse.Ns, metadataResponse.Bucket, metadataResponse.Name, metadataResponse.LastAccess, metadataResponse.InlinePayload, metadataResponse.PayloadIdentifier, metadataResponse.IsFinalized);
}
@@ -133,7 +137,11 @@ namespace Jupiter.Implementation
HttpResponseMessage response = await HttpClient.SendAsync(putObjectRequest);
if (response.StatusCode == HttpStatusCode.BadRequest)
{
ProblemDetails? problem = await response.Content.ReadAsAsync<ProblemDetails>();
ProblemDetails? problem = await response.Content.ReadFromJsonAsync<ProblemDetails>();
if (problem == null)
{
throw new Exception("Unable to deserialize problem details when bad request was encountered");
}
throw new Exception("Upstream returned 400 (BadRequest) with error: " + problem.Title);
}
response.EnsureSuccessStatusCode();
@@ -150,12 +158,21 @@ namespace Jupiter.Implementation
if (response.StatusCode == HttpStatusCode.BadRequest)
{
ProblemDetails? problem = await response.Content.ReadAsAsync<ProblemDetails>();
ProblemDetails? problem = await response.Content.ReadFromJsonAsync<ProblemDetails>();
if (problem == null)
{
throw new Exception("Unable to deserialize problem details when bad request was encountered during finalize");
}
throw new Exception("Upstream returned 400 (BadRequest) with error: " + problem.Title);
}
response.EnsureSuccessStatusCode();
PutObjectResponse putObjectResponse = await response.Content.ReadAsAsync<PutObjectResponse>();
PutObjectResponse? putObjectResponse = await response.Content.ReadFromJsonAsync<PutObjectResponse>();
if (putObjectResponse == null)
{
throw new Exception("Unable to deserialize put object response when finalizing ref");
}
if (putObjectResponse.Needs.Length != 0)
{
// TODO: we assume here that all the objects that are missing are content ids, that may not be true, but as needs only contains a list with mixed hashes we can not detect the difference

25
Engine/Source/Programs/UnrealCloudDDC/Jupiter/Implementation/Replication/RefsReplicator.cs
View File

@@ -10,6 +10,7 @@ using System.Net;
using System.Net.Http;
using System.Runtime.CompilerServices;
using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using EpicGames.Horde.Storage;
@@ -18,7 +19,6 @@ using Jupiter.Implementation.TransactionLog;
using Jupiter.Common;
using Jupiter.Common.Implementation;
using Microsoft.AspNetCore.Mvc;
using Newtonsoft.Json;
using OpenTelemetry.Trace;
using Microsoft.Extensions.Logging;
@@ -238,7 +238,7 @@ namespace Jupiter.Implementation
continue;
}
(string eventBucket, Guid eventId, int countOfEventsReplicated) = await ReplicateFromSnapshot(ns, replicationToken, useSnapshotException.SnapshotBlob, INamespacePolicyResolver.JupiterInternalNamespace);
(string eventBucket, Guid eventId, int countOfEventsReplicated) = await ReplicateFromSnapshot(ns, replicationToken, useSnapshotException.SnapshotBlob, useSnapshotException.BlobNamespace);
countOfReplicationsDone += countOfEventsReplicated;
// resume from these new events instead
@@ -274,7 +274,7 @@ namespace Jupiter.Implementation
snapshotResponse.EnsureSuccessStatusCode();
string s = await snapshotResponse.Content.ReadAsStringAsync(cancellationToken);
ReplicationLogSnapshots? snapshots = JsonConvert.DeserializeObject<ReplicationLogSnapshots>(s);
ReplicationLogSnapshots? snapshots = JsonSerializer.Deserialize<ReplicationLogSnapshots>(s);
if (snapshots == null)
{
throw new NotImplementedException();
@@ -479,12 +479,13 @@ namespace Jupiter.Implementation
//if (await _blobService.Exists(ns, blob))
// return currentOffset;
using HttpRequestMessage referencesRequest = await BuildHttpRequest(HttpMethod.Get, new Uri($"api/v1/objects/{ns}/{objectToReplicate}/references", UriKind.Relative));
HttpResponseMessage? referencesResponse = null;
Exception? lastException = null;
const int RetryAttempts = 3;
for (int i = 0; i < RetryAttempts; i++)
{
using HttpRequestMessage referencesRequest = await BuildHttpRequest(HttpMethod.Get, new Uri($"api/v1/objects/{ns}/{objectToReplicate}/references", UriKind.Relative));
try
{
referencesResponse = await _httpClient.SendAsync(referencesRequest, cancellationToken);
@@ -523,7 +524,7 @@ namespace Jupiter.Implementation
}
referencesResponse.EnsureSuccessStatusCode();
ResolvedReferencesResult? refs = JsonConvert.DeserializeObject<ResolvedReferencesResult>(body);
ResolvedReferencesResult? refs = JsonSerializer.Deserialize<ResolvedReferencesResult>(body);
if (refs == null)
{
throw new Exception($"Unable to resolve references for object {objectToReplicate} in namespace {ns}");
@@ -608,7 +609,7 @@ namespace Jupiter.Implementation
string body = await response.Content.ReadAsStringAsync(cancellationToken);
if (response.StatusCode == HttpStatusCode.BadRequest)
{
ProblemDetails? problemDetails = JsonConvert.DeserializeObject<ProblemDetails>(body);
ProblemDetails? problemDetails = JsonSerializer.Deserialize<ProblemDetails>(body);
if (problemDetails == null)
{
throw new Exception($"Unknown bad request body when reading incremental replication log. Body: {body}");
@@ -616,7 +617,7 @@ namespace Jupiter.Implementation
if (problemDetails.Type == ProblemTypes.UseSnapshot)
{
ProblemDetailsWithSnapshots? problemDetailsWithSnapshots = JsonConvert.DeserializeObject<ProblemDetailsWithSnapshots>(body);
ProblemDetailsWithSnapshots? problemDetailsWithSnapshots = JsonSerializer.Deserialize<ProblemDetailsWithSnapshots>(body);
if (problemDetailsWithSnapshots == null)
{
@@ -624,12 +625,13 @@ namespace Jupiter.Implementation
}
BlobIdentifier snapshotBlob = problemDetailsWithSnapshots.SnapshotId;
throw new UseSnapshotException(snapshotBlob);
NamespaceId? blobNamespace = problemDetailsWithSnapshots.BlobNamespace;
throw new UseSnapshotException(snapshotBlob, blobNamespace!.Value);
}
}
response.EnsureSuccessStatusCode();
ReplicationLogEvents? e = JsonConvert.DeserializeObject<ReplicationLogEvents>(body);
ReplicationLogEvents? e = JsonSerializer.Deserialize<ReplicationLogEvents>(body);
if (e == null)
{
throw new Exception($"Unknown error when deserializing replication log events {ns} {lastBucket} {lastEvent}");
@@ -698,10 +700,12 @@ namespace Jupiter.Implementation
public class UseSnapshotException : Exception
{
public BlobIdentifier SnapshotBlob { get; }
public NamespaceId BlobNamespace { get; }
public UseSnapshotException(BlobIdentifier snapshotBlob)
public UseSnapshotException(BlobIdentifier snapshotBlob, NamespaceId blobNamespace)
{
SnapshotBlob = snapshotBlob;
BlobNamespace = blobNamespace;
}
}
@@ -727,5 +731,6 @@ namespace Jupiter.Implementation
public class ProblemDetailsWithSnapshots : ProblemDetails
{
public BlobIdentifier SnapshotId { get; set; } = null!;
public NamespaceId? BlobNamespace { get; set; } = null;
}
}

11
Engine/Source/Programs/UnrealCloudDDC/Jupiter/Implementation/TransactionLog/IReplicationLog.cs
View File

@@ -2,9 +2,10 @@
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;
using System.Threading.Tasks;
using EpicGames.Horde.Storage;
using Newtonsoft.Json;
using EpicGames.Serialization;
namespace Jupiter.Implementation
{
@@ -43,6 +44,7 @@ namespace Jupiter.Implementation
public class ReplicationLogEvent
{
[JsonConstructor]
public ReplicationLogEvent(NamespaceId @namespace, BucketId bucket, IoHashKey key, BlobIdentifier? blob, Guid eventId, string timeBucket, DateTime timestamp, OpType op)
{
Namespace = @namespace;
@@ -63,12 +65,19 @@ namespace Jupiter.Implementation
};
public NamespaceId Namespace { get; }
public BucketId Bucket { get; }
public IoHashKey Key { get; }
public OpType Op { get; }
public DateTime Timestamp { get; }
public string TimeBucket { get; }
public Guid EventId { get; }
public BlobIdentifier? Blob { get; }
}

7
Engine/Source/Programs/UnrealCloudDDC/Jupiter/Implementation/TransactionLog/ReplicationLogSnapshot.cs
View File

@@ -5,10 +5,10 @@ using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Text;
using System.Text.Json;
using EpicGames.Horde.Storage;
using JetBrains.Annotations;
using Jupiter.Common.Implementation;
using Newtonsoft.Json;
using OpenTelemetry.Trace;
namespace Jupiter.Implementation.TransactionLog
@@ -38,10 +38,7 @@ namespace Jupiter.Implementation.TransactionLog
{
// legacy json snapshot found
using GZipStream gzipStream = new GZipStream(stream, CompressionMode.Decompress);
using TextReader textReader = new StreamReader(gzipStream);
using JsonReader reader = new JsonTextReader(textReader);
JsonSerializer serializer = JsonSerializer.Create();
ReplicationLogSnapshotState? snapshotState = serializer.Deserialize<ReplicationLogSnapshotState>(reader);
ReplicationLogSnapshotState? snapshotState = JsonSerializer.Deserialize<ReplicationLogSnapshotState>(gzipStream);
if (snapshotState == null)
{
throw new NotImplementedException();

4
Engine/Source/Programs/UnrealCloudDDC/Jupiter/Tests/Functional/Blob/NginxRedirectTests.cs
View File

@@ -6,6 +6,7 @@ using System.IO;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Net.Http.Json;
using System.Net.Mime;
using System.Text;
using System.Threading.Tasks;
@@ -85,7 +86,8 @@ namespace Jupiter.FunctionalTests.Storage
requestContent.Headers.ContentLength = FileContents.Length;
HttpResponseMessage putResponse = await _httpClient!.PutAsync(new Uri($"api/v1/blobs/{TestNamespaceName}/{FileContentsHash}", UriKind.Relative), requestContent);
putResponse.EnsureSuccessStatusCode();
InsertResponse response = await putResponse.Content.ReadAsAsync<InsertResponse>();
InsertResponse? response = await putResponse.Content.ReadFromJsonAsync<InsertResponse>();
Assert.IsNotNull(response);
Assert.AreEqual(FileContentsHash, response.Identifier);
using HttpRequestMessage getObjectRequest = new HttpRequestMessage(HttpMethod.Get, new Uri($"api/v1/blobs/{TestNamespaceName}/{FileContentsHash}", UriKind.Relative));

10
Engine/Source/Programs/UnrealCloudDDC/Jupiter/Tests/Functional/Blob/PeerBlobTest.cs
View File

@@ -6,6 +6,7 @@ using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Net.Http.Json;
using System.Text;
using System.Threading.Tasks;
using Amazon.S3;
@@ -140,7 +141,8 @@ namespace Jupiter.FunctionalTests.Storage
requestContent.Headers.ContentLength = S3FileContent.Length;
HttpResponseMessage putResponse = await _httpClient!.PutAsync(new Uri($"api/v1/s/{TestNamespaceName}/{S3FileContentHash}", UriKind.Relative), requestContent);
putResponse.EnsureSuccessStatusCode();
InsertResponse response = await putResponse.Content.ReadAsAsync<InsertResponse>();
InsertResponse? response = await putResponse.Content.ReadFromJsonAsync<InsertResponse>();
Assert.IsNotNull(response);
Assert.AreEqual(S3FileContentHash, response.Identifier);
HttpResponseMessage getResponse = await _httpClient.GetAsync(new Uri($"api/v1/s/{TestNamespaceName}/{S3FileContentHash}?storageLayers=AmazonS3Store", UriKind.Relative));
@@ -161,7 +163,8 @@ namespace Jupiter.FunctionalTests.Storage
requestContent.Headers.ContentLength = S3FileContent.Length;
HttpResponseMessage putResponse = await _httpClient!.PutAsync(new Uri($"api/v1/s/{TestNamespaceName}/{S3FileContentHash}", UriKind.Relative), requestContent);
putResponse.EnsureSuccessStatusCode();
InsertResponse response = await putResponse.Content.ReadAsAsync<InsertResponse>();
InsertResponse? response = await putResponse.Content.ReadFromJsonAsync<InsertResponse>();
Assert.IsNotNull(response);
Assert.AreEqual(S3FileContentHash, response.Identifier);
HttpResponseMessage getResponse = await _httpClient.GetAsync(new Uri($"api/v1/s/{TestNamespaceName}/{S3FileContentHash}?storageLayers=ThisImplemenationDoesNotExist", UriKind.Relative));
@@ -177,7 +180,8 @@ namespace Jupiter.FunctionalTests.Storage
requestContent.Headers.ContentLength = payload.Length;
HttpResponseMessage putResponse = await _httpClient!.PutAsync(new Uri($"api/v1/s/{TestNamespaceName}/{OtherPeerContentHash}", UriKind.Relative), requestContent);
putResponse.EnsureSuccessStatusCode();
InsertResponse response = await putResponse.Content.ReadAsAsync<InsertResponse>();
InsertResponse? response = await putResponse.Content.ReadFromJsonAsync<InsertResponse>();
Assert.IsNotNull(response);
Assert.AreEqual(OtherPeerContentHash, response.Identifier);
HttpResponseMessage getResponse = await _httpClient.GetAsync(new Uri($"api/v1/s/{TestNamespaceName}/{OtherPeerContentHash}", UriKind.Relative));

Some files were not shown because too many files have changed in this diff Show More